[前][次][番号順一覧][スレッド一覧]

ruby-changes:51185

From: normal <ko1@a...>
Date: Thu, 10 May 2018 17:54:33 +0900 (JST)
Subject: [ruby-changes:51185] normal:r63392 (trunk): variable.c: fix multiple autoload with identical file (again)

normal	2018-05-10 17:54:26 +0900 (Thu, 10 May 2018)

  New Revision: 63392

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=63392

  Log:
    variable.c: fix multiple autoload with identical file (again)
    
    We need to ensure autoload declarations pointing to the same
    feature (aka "file") can wait on each other to avoid deadlock
    situations.
    
    So, reorganize autoload data structures to maintain a
    feature => autoload_data_i mapping, and have module constant
    tables point to the new autoload_const struct instead of
    directly to autoload_data_i.  This allows multiple
    autoload_const structs to refer to the SAME autoload_data_i
    struct, and with it, the on-stack autoload_state.waitq.
    
    The end result is different constants can share the same waitq
    (tied to the feature name), and not deadlock each other during
    loading.
    
    Thanks to Eugene Kenny for the bug report and reproducible test case.
    
    Reported-by: Eugene Kenny <elkenny@g...>
    
    * variable.c (autoload_featuremap): new global
      (struct autoload_const): new per-const struct
      (struct autoload_state): reference autoload_const instead of autoload_data_i
      (struct autoload_data_i): remove per-const
      (autoload_i_mark): delete from autoload_featuremap if unreferenced
      (autoload_c_mark): new dmark callback
      (autoload_c_free): new dfree callback
      (autoload_c_memsize): new memsize callback
      (autoload_const_type): new data type
      (get_autoload_data): set autoload_const as well
      (rb_autoload_str): use new data structures
      (autoload_delete): cleanup from autoload_featuremap
      (check_autoload_required): adjust for new internals
      (rb_autoloading_value): ditto
      (struct autoload_const_set_args): remove, redundant with autoload_const
      (const_tbl_update): adjust for new internals
      (autoload_const_set): ditto
      (autoload_require): ditto
      (autoload_reset): ditto
      (rb_autoload_load): ditto
      (rb_const_set): ditto
      (current_autoload_data): ditto
      (set_const_visibility): ditto
    * test/ruby/test_autoload.rb (test_autoload_same_file): new test
      (test_no_leak): new test
      [ruby-core:86935] [Bug #14742]

  Modified files:
    trunk/test/ruby/test_autoload.rb
    trunk/variable.c
Index: test/ruby/test_autoload.rb
===================================================================
--- test/ruby/test_autoload.rb	(revision 63391)
+++ test/ruby/test_autoload.rb	(revision 63392)
@@ -311,6 +311,42 @@ p Foo::Bar https://github.com/ruby/ruby/blob/trunk/test/ruby/test_autoload.rb#L311
     end
   end if Process.respond_to?(:fork)
 
+  def test_autoload_same_file
+    Dir.mktmpdir('autoload') do |tmpdir|
+      File.write("#{tmpdir}/b.rb", "#{<<~'begin;'}\n#{<<~'end;'}")
+      begin;
+        module Foo; end
+        module Bar; end
+      end;
+      3.times do # timing-dependent, needs a few times to hit [Bug #14742]
+        assert_separately(%W[-I #{tmpdir}], "#{<<-'begin;'}\n#{<<-'end;'}")
+        begin;
+          autoload :Foo, 'b'
+          autoload :Bar, 'b'
+          t1 = Thread.new do Foo end
+          t2 = Thread.new do Bar end
+          t1.join
+          t2.join
+          bug = '[ruby-core:86935] [Bug #14742]'
+          assert_instance_of Module, t1.value, bug
+          assert_instance_of Module, t2.value, bug
+        end;
+      end
+    end
+  end
+
+  def test_no_leak
+    assert_no_memory_leak([], '', <<~'end;', 'many autoloads', timeout: 30)
+      200000.times do |i|
+        m = Module.new
+        m.instance_eval do
+          autoload :Foo, 'x'
+          autoload :Bar, i.to_s
+        end
+      end
+    end;
+  end
+
   def add_autoload(path)
     (@autoload_paths ||= []) << path
     ::Object.class_eval {autoload(:AutoloadTest, path)}
Index: variable.c
===================================================================
--- variable.c	(revision 63391)
+++ variable.c	(revision 63392)
@@ -25,6 +25,7 @@ https://github.com/ruby/ruby/blob/trunk/variable.c#L25
 
 static struct rb_id_table *rb_global_tbl;
 static ID autoload, classpath, tmp_classpath, classid;
+static VALUE autoload_featuremap; /* feature => autoload_i */
 
 static void check_before_mod_set(VALUE, ID, VALUE, const char *);
 static void setup_const_entry(rb_const_entry_t *, VALUE, VALUE, rb_const_flag_t);
@@ -1842,31 +1843,53 @@ autoload_data(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L1843
     return (VALUE)val;
 }
 
+struct autoload_const {
+    struct list_node cnode; /* <=> autoload_data_i.constants */
+    VALUE mod;
+    VALUE ad; /* autoload_data_i */
+    VALUE value;
+    ID id;
+    int safe_level;
+    rb_const_flag_t flag;
+};
+
 /* always on stack, no need to mark */
 struct autoload_state {
-    struct autoload_data_i *ele;
-    VALUE mod;
+    struct autoload_const *ac;
     VALUE result;
-    ID id;
     VALUE thread;
     struct list_node waitq;
 };
 
 struct autoload_data_i {
     VALUE feature;
-    int safe_level;
-    rb_const_flag_t flag;
-    VALUE value;
     struct autoload_state *state; /* points to on-stack struct */
     rb_serial_t fork_gen;
+    struct list_head constants; /* <=> autoload_const.cnode */
 };
 
 static void
 autoload_i_mark(void *ptr)
 {
     struct autoload_data_i *p = ptr;
+
     rb_gc_mark(p->feature);
-    rb_gc_mark(p->value);
+
+    /* allow GC to free us if no modules refer to this via autoload_const.ad */
+    if (list_empty(&p->constants)) {
+        rb_hash_delete(autoload_featuremap, p->feature);
+    }
+}
+
+static void
+autoload_i_free(void *ptr)
+{
+    struct autoload_data_i *p = ptr;
+
+    /* we may leak some memory at VM shutdown time, no big deal */
+    if (list_empty(&p->constants)) {
+	xfree(p);
+    }
 }
 
 static size_t
@@ -1877,20 +1900,53 @@ autoload_i_memsize(const void *ptr) https://github.com/ruby/ruby/blob/trunk/variable.c#L1900
 
 static const rb_data_type_t autoload_data_i_type = {
     "autoload_i",
-    {autoload_i_mark, RUBY_TYPED_DEFAULT_FREE, autoload_i_memsize,},
+    {autoload_i_mark, autoload_i_free, autoload_i_memsize,},
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static void
+autoload_c_mark(void *ptr)
+{
+    struct autoload_const *ac = ptr;
+
+    rb_gc_mark(ac->mod);
+    rb_gc_mark(ac->ad);
+    rb_gc_mark(ac->value);
+}
+
+static void
+autoload_c_free(void *ptr)
+{
+    struct autoload_const *ac = ptr;
+    list_del(&ac->cnode);
+    xfree(ac);
+}
+
+static size_t
+autoload_c_memsize(const void *ptr)
+{
+    return sizeof(struct autoload_const);
+}
+
+static const rb_data_type_t autoload_const_type = {
+    "autoload_const",
+    {autoload_c_mark, autoload_c_free, autoload_c_memsize,},
     0, 0, RUBY_TYPED_FREE_IMMEDIATELY
 };
 
 static struct autoload_data_i *
-get_autoload_data(VALUE av)
+get_autoload_data(VALUE acv, struct autoload_const **acp)
 {
-    struct autoload_data_i *ele = rb_check_typeddata(av, &autoload_data_i_type);
+    struct autoload_const *ac = rb_check_typeddata(acv, &autoload_const_type);
+    struct autoload_data_i *ele;
 
+    ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
     /* do not reach across stack for ->state after forking: */
     if (ele && ele->state && ele->fork_gen != GET_VM()->fork_gen) {
         ele->state = 0;
         ele->fork_gen = 0;
     }
+    if (acp) *acp = ac;
     return ele;
 }
 
@@ -1940,17 +1996,42 @@ rb_autoload_str(VALUE mod, ID id, VALUE https://github.com/ruby/ruby/blob/trunk/variable.c#L1996
 	DATA_PTR(av) = tbl = st_init_numtable();
     }
 
-    ad = TypedData_Make_Struct(0, struct autoload_data_i, &autoload_data_i_type, ele);
     if (OBJ_TAINTED(file)) {
 	file = rb_str_dup(file);
 	FL_UNSET(file, FL_TAINT);
     }
-    ele->feature = rb_fstring(file);
-    ele->safe_level = rb_safe_level();
-    ele->value = Qundef;
-    ele->state = 0;
-    ele->flag = CONST_PUBLIC;
-    st_insert(tbl, (st_data_t)id, (st_data_t)ad);
+    file = rb_fstring(file);
+    if (!autoload_featuremap) {
+        autoload_featuremap = rb_hash_new_compare_by_id();
+        rb_obj_hide(autoload_featuremap);
+        rb_gc_register_mark_object(autoload_featuremap);
+    }
+    ad = rb_hash_aref(autoload_featuremap, file);
+    if (NIL_P(ad)) {
+        ad = TypedData_Make_Struct(0, struct autoload_data_i,
+                                    &autoload_data_i_type, ele);
+        ele->feature = file;
+        ele->state = 0;
+        list_head_init(&ele->constants);
+        rb_hash_aset(autoload_featuremap, file, ad);
+    }
+    else {
+        ele = rb_check_typeddata(ad, &autoload_data_i_type);
+    }
+    {
+        VALUE acv;
+        struct autoload_const *ac;
+        acv = TypedData_Make_Struct(0, struct autoload_const,
+                                    &autoload_const_type, ac);
+        ac->mod = mod;
+        ac->id = id;
+        ac->value = Qundef;
+        ac->safe_level = rb_safe_level();
+        ac->flag = CONST_PUBLIC;
+        ac->ad = ad;
+        list_add_tail(&ele->constants, &ac->cnode);
+        st_insert(tbl, (st_data_t)id, (st_data_t)acv);
+    }
 }
 
 static void
@@ -1960,8 +2041,22 @@ autoload_delete(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L2041
 
     if (st_lookup(RCLASS_IV_TBL(mod), (st_data_t)autoload, &val)) {
 	struct st_table *tbl = check_autoload_table((VALUE)val);
+	struct autoload_data_i *ele;
+	struct autoload_const *ac;
 
 	st_delete(tbl, &n, &load);
+	ele = get_autoload_data((VALUE)load, &ac);
+	VM_ASSERT(ele);
+	if (ele) {
+	    VM_ASSERT(!list_empty(&ele->constants));
+	}
+
+	/*
+	 * we must delete here to avoid "already initialized" warnings
+	 * with parallel autoload.  Using list_del_init here so list_del
+	 * works in autoload_c_free
+	 */
+	list_del_init(&ac->cnode);
 
 	if (tbl->num_entries == 0) {
 	    n = autoload;
@@ -1987,12 +2082,13 @@ reset_safe(VALUE safe) https://github.com/ruby/ruby/blob/trunk/variable.c#L2082
 static VALUE
 check_autoload_required(VALUE mod, ID id, const char **loadingpath)
 {
-    VALUE file, load;
+    VALUE file;
+    VALUE load = autoload_data(mod, id);
     struct autoload_data_i *ele;
     const char *loading;
     int safe;
 
-    if (!(load = autoload_data(mod, id)) || !(ele = get_autoload_data(load))) {
+    if (!load || !(ele = get_autoload_data(load, 0))) {
 	return 0;
     }
     file = ele->feature;
@@ -2027,19 +2123,21 @@ check_autoload_required(VALUE mod, ID id https://github.com/ruby/ruby/blob/trunk/variable.c#L2123
 MJIT_FUNC_EXPORTED int
 rb_autoloading_value(VALUE mod, ID id, VALUE* value, rb_const_flag_t *flag)
 {
-    VALUE load;
+    VALUE load = autoload_data(mod, id);
     struct autoload_data_i *ele;
+    struct autoload_const *ac;
 
-    if (!(load = autoload_data(mod, id)) || !(ele = get_autoload_data(load))) {
-	return 0;
+    if (!load || !(ele = get_autoload_data(load, &ac))) {
+        return 0;
     }
+
     if (ele->state && ele->state->thread == rb_thread_current()) {
-	if (ele->value != Qundef) {
+	if (ac->value != Qundef) {
 	    if (value) {
-		*value = ele->value;
+		*value = ac->value;
 	    }
 	    if (flag) {
-		*flag = ele->flag;
+		*flag = ac->flag;
 	    }
 	    return 1;
 	}
@@ -2058,23 +2156,16 @@ autoload_defined_p(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L2156
     return !rb_autoloading_value(mod, id, NULL, NULL);
 }
 
-struct autoload_const_set_args {
-    VALUE mod;
-    ID id;
-    VALUE value;
-    rb_const_flag_t flag;
-};
-
-static void const_tbl_update(struct autoload_const_set_args *);
+static void const_tbl_update(struct autoload_const *);
 
 static VALUE
 autoload_const_set(VALUE arg)
 {
-    struct autoload_const_set_args* args = (struct autoload_const_set_args *)arg;
-    VALUE klass = args->mod;
-    ID id = args->id;
-    check_before_mod_set(klass, id, args->value, "constant");
-    const_tbl_update(args);
+    struct autoload_const *ac = (struct autoload_const *)arg;
+    VALUE klass = ac->mod;
+    ID id = ac->id;
+    check_before_mod_set(klass, id, ac->value, "constant");
+    const_tbl_update(ac);
     return 0;			/* ignored */
 }
 
@@ -2082,10 +2173,13 @@ static VALUE https://github.com/ruby/ruby/blob/trunk/variable.c#L2173
 autoload_require(VALUE arg)
 {
     struct autoload_state *state = (struct autoload_state *)arg;
+    struct autoload_const *ac = state->ac;
+    struct autoload_data_i *ele;
 
+    ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
     /* this may release GVL and switch threads: */
     state->result = rb_funcall(rb_vm_top_self(), rb_intern("require"), 1,
-			       state->ele->feature);
+			       ele->feature);
 
     return state->result;
 }
@@ -2095,26 +2189,27 @@ autoload_reset(VALUE arg) https://github.com/ruby/ruby/blob/trunk/variable.c#L2189
 {
     struct autoload_state *state = (struct autoload_state *)arg;
     int need_wakeups = 0;
+    struct autoload_const *ac = state->ac;
+    struct autoload_data_i *ele;
 
-    if (state->ele->state == state) {
+    ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
+    if (ele->state == state) {
         need_wakeups = 1;
-        state->ele->state = 0;
-        state->ele->fork_gen = 0;
+        ele->state = 0;
+        ele->fork_gen = 0;
     }
 
     /* At the last, move a value defined in autoload to constant table */
-    if (RTEST(state->result) && state->ele->value != Qundef) {
-	int safe_backup;
-	struct autoload_const_set_args args;
-
-	args.mod = state->mod;
-	args.id = state->id;
-	args.value = state->ele->value;
-	args.flag = state->ele->flag;
-	safe_backup = rb_safe_level();
-	rb_set_safe_level_force(state->ele->safe_level);
-	rb_ensure(autoload_const_set, (VALUE)&args,
-	          reset_safe, (VALUE)safe_backup);
+    if (RTEST(state->result)) {
+        struct autoload_const *next;
+        int safe_backup = rb_safe_level();
+
+        list_for_each_safe(&ele->constants, ac, next, cnode) {
+            if (ac->value != Qundef) {
+                rb_ensure(autoload_const_set, (VALUE)ac,
+                          reset_safe, (VALUE)safe_backup);
+            }
+        }
     }
 
     /* wakeup any waiters we had */
@@ -2172,6 +2267,7 @@ rb_autoload_load(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L2267
     VALUE load, result;
     const char *loading = 0, *src;
     struct autoload_data_i *ele;
+    struct autoload_const *ac;
     struct autoload_state state;
 
     if (!autoload_defined_p(mod, id)) return Qfalse;
@@ -2181,13 +2277,10 @@ rb_autoload_load(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L2277
     if (src && loading && strcmp(src, loading) == 0) return Qfalse;
 
     /* set ele->state for a marker of autoloading thread */
-    if (!(ele = get_autoload_data(load))) {
+    if (!(ele = get_autoload_data(load, &ac))) {
 	return Qfalse;
     }
-
-    state.ele = ele;
-    state.mod = mod;
-    state.id = id;
+    state.ac = ac;
     state.thread = rb_thread_current();
     if (!ele->state) {
 	ele->state = &state;
@@ -2229,7 +2322,7 @@ rb_autoload_p(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L2322
     }
     load = check_autoload_required(mod, id, 0);
     if (!load) return Qnil;
-    return (ele = get_autoload_data(load)) ? ele->feature : Qnil;
+    return (ele = get_autoload_data(load, 0)) ? ele->feature : Qnil;
 }
 
 MJIT_FUNC_EXPORTED void
@@ -2610,12 +2703,12 @@ rb_const_set(VALUE klass, ID id, VALUE v https://github.com/ruby/ruby/blob/trunk/variable.c#L2703
 	setup_const_entry(ce, klass, val, CONST_PUBLIC);
     }
     else {
-	struct autoload_const_set_args args;
-	args.mod = klass;
-	args.id = id;
-	args.value = val;
-	args.flag = CONST_PUBLIC;
-	const_tbl_update(&args);
+	struct autoload_const ac;
+	ac.mod = klass;
+	ac.id = id;
+	ac.value = val;
+	ac.flag = CONST_PUBLIC;
+	const_tbl_update(&ac);
     }
     /*
      * Resolve and cache class name immediately to resolve ambiguity
@@ -2647,12 +2740,12 @@ rb_const_set(VALUE klass, ID id, VALUE v https://github.com/ruby/ruby/blob/trunk/variable.c#L2740
 }
 
 static struct autoload_data_i *
-current_autoload_data(VALUE mod, ID id)
+current_autoload_data(VALUE mod, ID id, struct autoload_const **acp)
 {
     struct autoload_data_i *ele;
     VALUE load = autoload_data(mod, id);
     if (!load) return 0;
-    ele = get_autoload_data(load);
+    ele = get_autoload_data(load, acp);
     if (!ele) return 0;
     /* for autoloading thread, keep the defined value to autoloading storage */
     if (ele->state && (ele->state->thread == rb_thread_current())) {
@@ -2662,25 +2755,25 @@ current_autoload_data(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L2755
 }
 
 static void
-const_tbl_update(struct autoload_const_set_args *args)
+const_tbl_update(struct autoload_const *ac)
 {
     VALUE value;
-    VALUE klass = args->mod;
-    VALUE val = args->value;
-    ID id = args->id;
+    VALUE klass = ac->mod;
+    VALUE val = ac->value;
+    ID id = ac->id;
     struct rb_id_table *tbl = RCLASS_CONST_TBL(klass);
-    rb_const_flag_t visibility = args->flag;
+    rb_const_flag_t visibility = ac->flag;
     rb_const_entry_t *ce;
 
     if (rb_id_table_lookup(tbl, id, &value)) {
 	ce = (rb_const_entry_t *)value;
 	if (ce->value == Qundef) {
-	    struct autoload_data_i *ele = current_autoload_data(klass, id);
+	    struct autoload_data_i *ele = current_autoload_data(klass, id, &ac);
 
 	    if (ele) {
 		rb_clear_constant_cache();
 
-		ele->value = val; /* autoload_i is non-WB-protected */
+		ac->value = val; /* autoload_i is non-WB-protected */
 		return;
 	    }
 	    /* otherwise, allow to override */
@@ -2753,6 +2846,7 @@ set_const_visibility(VALUE mod, int argc https://github.com/ruby/ruby/blob/trunk/variable.c#L2846
     }
 
     for (i = 0; i < argc; i++) {
+	struct autoload_const *ac;
 	VALUE val = argv[i];
 	id = rb_check_id(&val);
 	if (!id) {
@@ -2767,10 +2861,12 @@ set_const_visibility(VALUE mod, int argc https://github.com/ruby/ruby/blob/trunk/variable.c#L2861
 	    ce->flag &= ~mask;
 	    ce->flag |= flag;
 	    if (ce->value == Qundef) {
-		struct autoload_data_i *ele = current_autoload_data(mod, id);
+		struct autoload_data_i *ele;
+
+		ele = current_autoload_data(mod, id, &ac);
 		if (ele) {
-		    ele->flag &= ~mask;
-		    ele->flag |= flag;
+		    ac->flag &= ~mask;
+		    ac->flag |= flag;
 		}
 	    }
 	}

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/

[前][次][番号順一覧][スレッド一覧]