ruby-changes:71830
From: Samuel <ko1@a...>
Date: Sun, 15 May 2022 13:07:36 +0900 (JST)
Subject: [ruby-changes:71830] 32de6097b2 (master): Fix various autoload race conditions. (#5898)
https://git.ruby-lang.org/ruby.git/commit/?id=32de6097b2 From 32de6097b2b5d8394b3a1399e13d309444697954 Mon Sep 17 00:00:00 2001 From: Samuel Williams <samuel.williams@o...> Date: Sun, 15 May 2022 16:07:12 +1200 Subject: Fix various autoload race conditions. (#5898) * Add RUBY_VM_CRITICAL_SECTION for detecting unexpected context switch. * Prevent race between GC mark and autoload setup. * Protect race on autoload state. * Avoid potential race condition when allocating `autoload_featuremap`. * Add NEWS entry for autoload fixes. --- NEWS.md | 3 + thread.c | 4 ++ variable.c | 204 ++++++++++++++++++++++++++++++++++++++----------------------- vm.c | 4 ++ vm_core.h | 11 +++- 5 files changed, 149 insertions(+), 77 deletions(-) diff --git a/NEWS.md b/NEWS.md index a4b686789f..6499832cb6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -208,6 +208,8 @@ The following deprecated APIs are removed. https://github.com/ruby/ruby/blob/trunk/NEWS.md#L208 ## Implementation improvements +* Fixed several race conditions in `Kernel#autoload`. [[Bug #18782]] + ## JIT ### MJIT @@ -253,3 +255,4 @@ The following deprecated APIs are removed. https://github.com/ruby/ruby/blob/trunk/NEWS.md#L255 [Feature #18598]: https://bugs.ruby-lang.org/issues/18598 [Bug #18625]: https://bugs.ruby-lang.org/issues/18625 [Bug #18633]: https://bugs.ruby-lang.org/issues/18633 +[Bug #18782]: https://bugs.ruby-lang.org/issues/18782 diff --git a/thread.c b/thread.c index aeb1a6308d..7582849767 100644 --- a/thread.c +++ b/thread.c @@ -1562,6 +1562,10 @@ static inline int https://github.com/ruby/ruby/blob/trunk/thread.c#L1562 blocking_region_begin(rb_thread_t *th, struct rb_blocking_region_buffer *region, rb_unblock_function_t *ubf, void *arg, int fail_if_interrupted) { +#ifdef RUBY_VM_CRITICAL_SECTION + VM_ASSERT(rb_vm_critical_section_entered == 0); +#endif + region->prev_status = th->status; if (unblock_function_set(th, ubf, arg, fail_if_interrupted)) { th->blocking_region_buffer = region; diff --git a/variable.c b/variable.c index 9816784a82..15dc6b9fc0 100644 --- a/variable.c +++ b/variable.c @@ -46,7 +46,16 @@ typedef void rb_gvar_compact_t(void *var); https://github.com/ruby/ruby/blob/trunk/variable.c#L46 static struct rb_id_table *rb_global_tbl; static ID autoload, classpath, tmp_classpath; -static VALUE autoload_featuremap; /* feature => autoload_i */ + +// This hash table maps file paths to loadable features. We use this to track +// autoload state until it's no longer needed. +// feature (file path) => struct autoload_i +static VALUE autoload_featuremap; + +// This mutex is used to protect autoloading state. We use a global mutex which +// is held until a per-feature mutex can be created. This ensures there are no +// race conditions relating to autoload state. +static VALUE autoload_mutex; static void check_before_mod_set(VALUE, ID, VALUE, const char *); static void setup_const_entry(rb_const_entry_t *, VALUE, VALUE, rb_const_flag_t); @@ -72,6 +81,14 @@ Init_var_tables(void) https://github.com/ruby/ruby/blob/trunk/variable.c#L81 classpath = rb_intern_const("__classpath__"); /* __tmp_classpath__: temporary class path which contains anonymous names */ tmp_classpath = rb_intern_const("__tmp_classpath__"); + + autoload_mutex = rb_mutex_new(); + rb_obj_hide(autoload_mutex); + rb_gc_register_mark_object(autoload_mutex); + + autoload_featuremap = rb_ident_hash_new(); + rb_obj_hide(autoload_featuremap); + rb_gc_register_mark_object(autoload_featuremap); } static inline bool @@ -2141,22 +2158,6 @@ autoload_i_mark(void *ptr) https://github.com/ruby/ruby/blob/trunk/variable.c#L2158 struct autoload_data_i *p = ptr; rb_gc_mark_movable(p->feature); - - /* allow GC to free us if no modules refer to this via autoload_const.ad */ - if (ccan_list_empty(&p->constants)) { - rb_hash_delete(autoload_featuremap, p->feature); - } -} - -static void -autoload_i_free(void *ptr) -{ - struct autoload_data_i *p = ptr; - - /* we may leak some memory at VM shutdown time, no big deal */ - if (ccan_list_empty(&p->constants)) { - xfree(p); - } } static size_t @@ -2167,7 +2168,7 @@ autoload_i_memsize(const void *ptr) https://github.com/ruby/ruby/blob/trunk/variable.c#L2168 static const rb_data_type_t autoload_data_i_type = { "autoload_i", - {autoload_i_mark, autoload_i_free, autoload_i_memsize, autoload_i_compact}, + {autoload_i_mark, ruby_xfree, autoload_i_memsize, autoload_i_compact}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; @@ -2193,14 +2194,6 @@ autoload_c_mark(void *ptr) https://github.com/ruby/ruby/blob/trunk/variable.c#L2194 rb_gc_mark_movable(ac->file); } -static void -autoload_c_free(void *ptr) -{ - struct autoload_const *ac = ptr; - ccan_list_del(&ac->cnode); - xfree(ac); -} - static size_t autoload_c_memsize(const void *ptr) { @@ -2209,7 +2202,7 @@ autoload_c_memsize(const void *ptr) https://github.com/ruby/ruby/blob/trunk/variable.c#L2202 static const rb_data_type_t autoload_const_type = { "autoload_const", - {autoload_c_mark, autoload_c_free, autoload_c_memsize, autoload_c_compact,}, + {autoload_c_mark, ruby_xfree, autoload_c_memsize, autoload_c_compact,}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; @@ -2257,18 +2250,18 @@ rb_autoload_str(VALUE mod, ID id, VALUE file) https://github.com/ruby/ruby/blob/trunk/variable.c#L2250 Check_Type(file, T_STRING); if (!RSTRING_LEN(file)) { - rb_raise(rb_eArgError, "empty file name"); + rb_raise(rb_eArgError, "empty file name"); } ce = rb_const_lookup(mod, id); if (ce && ce->value != Qundef) { - return; + return; } const_set(mod, id, Qundef); tbl = RCLASS_IV_TBL(mod); if (tbl && st_lookup(tbl, (st_data_t)autoload, &av)) { - tbl = check_autoload_table((VALUE)av); + tbl = check_autoload_table((VALUE)av); } else { if (!tbl) tbl = RCLASS_IV_TBL(mod) = st_init_numtable(); @@ -2279,11 +2272,6 @@ rb_autoload_str(VALUE mod, ID id, VALUE file) https://github.com/ruby/ruby/blob/trunk/variable.c#L2272 } file = rb_fstring(file); - if (!autoload_featuremap) { - autoload_featuremap = rb_ident_hash_new(); - rb_obj_hide(autoload_featuremap); - rb_gc_register_mark_object(autoload_featuremap); - } ad = rb_hash_aref(autoload_featuremap, file); if (NIL_P(ad)) { ad = TypedData_Make_Struct(0, struct autoload_data_i, @@ -2406,9 +2394,11 @@ rb_autoloading_value(VALUE mod, ID id, VALUE* value, rb_const_flag_t *flag) https://github.com/ruby/ruby/blob/trunk/variable.c#L2394 if (value) { *value = ac->value; } + if (flag) { *flag = ac->flag; } + return TRUE; } @@ -2417,6 +2407,9 @@ autoload_by_current(struct autoload_data_i *ele) { https://github.com/ruby/ruby/blob/trunk/variable.c#L2407 return ele->state && ele->state->mutex != Qnil && rb_mutex_owned_p(ele->state->mutex); } +// If there is an autoloading constant and it has been set by the current +// execution context, return it. This allows threads which are loading code to +// refer to their own autoloaded constants. struct autoload_const * autoloading_const_entry(VALUE mod, ID id) { @@ -2424,10 +2417,13 @@ autoloading_const_entry(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L2417 struct autoload_data_i *ele; struct autoload_const *ac; + // Find the autoloading state: if (!load || !(ele = get_autoload_data(load, &ac))) { + // Couldn't be found: return 0; } + // Check if it's being loaded by the current thread/fiber: if (autoload_by_current(ele)) { if (ac->value != Qundef) { return ac; @@ -2442,13 +2438,17 @@ autoload_defined_p(VALUE mod, ID id) https://github.com/ruby/ruby/blob/trunk/variable.c#L2438 { rb_const_entry_t *ce = rb_const_lookup(mod, id); + // If there is no constant or the constant is not undefined (special marker for autoloading): if (!ce || ce->value != Qundef) { - return 0; + // We are not autoloading: + return 0; } + + // Otherwise check if there is an autoload in flight right now: return !rb_autoloading_value(mod, id, NULL, NULL); } -static void const_tbl_update(struct autoload_const *); +static void const_tbl_update(struct autoload_const *, int); static VALUE autoload_const_set(struct autoload_const *ac) @@ -2459,7 +2459,7 @@ autoload_const_set(struct autoload_const *ac) https://github.com/ruby/ruby/blob/trunk/variable.c#L2459 RB_VM_LOCK_ENTER(); { - const_tbl_update(ac); + const_tbl_update(ac, true); } RB_VM_LOCK_LEAVE(); @@ -2490,50 +2490,65 @@ autoload_reset(VALUE arg) https://github.com/ruby/ruby/blob/trunk/variable.c#L2490 ele = rb_check_typeddata(ac->ad, &autoload_data_i_type); VALUE mutex = state->mutex; + // If we are the main thread to execute... if (ele->state == state) { - ele->state = 0; - ele->fork_gen = 0; - } + // Prepare to update the state of the world: + rb_mutex_lock(autoload_mutex); - rb_mutex_unlock(mutex); + // At the last, move a value defined in autoload to constant table: + if (RTEST(state->result)) { + // Can help to test race conditions: + // rb_thread_schedule(); - /* At the last, move a value defined in autoload to constant table */ - if (RTEST(state->result)) { - struct autoload_const *next; + struct autoload_const *next; - ccan_list_for_each_safe(&ele->constants, ac, next, cnode) { - if (ac->value != Qundef) { - autoload_const_set(ac); + ccan_list_for_each_safe(&ele->constants, ac, next, cnode) { + if (ac->value != Qundef) { + autoload_const_set(ac); + } } } + + // Reset the auto (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/