ruby-changes:65082
From: Koichi <ko1@a...>
Date: Fri, 29 Jan 2021 16:22:34 +0900 (JST)
Subject: [ruby-changes:65082] 1ecda21366 (master): global call-cache cache table for rb_funcall*
https://git.ruby-lang.org/ruby.git/commit/?id=1ecda21366 From 1ecda213668644d656eb0d60654737482447dd92 Mon Sep 17 00:00:00 2001 From: Koichi Sasada <ko1@a...> Date: Thu, 21 Jan 2021 03:33:59 +0900 Subject: global call-cache cache table for rb_funcall* rb_funcall* (rb_funcall(), rb_funcallv(), ...) functions invokes Ruby's method with given receiver. Ruby 2.7 introduced inline method cache with static memory area. However, Ruby 3.0 reimplemented the method cache data structures and the inline cache was removed. Without inline cache, rb_funcall* searched methods everytime. Most of cases per-Class Method Cache (pCMC) will be helped but pCMC requires VM-wide locking and it hurts performance on multi-Ractor execution, especially all Ractors calls methods with rb_funcall*. This patch introduced Global Call-Cache Cache Table (gccct) for rb_funcall*. Call-Cache was introduced from Ruby 3.0 to manage method cache entry atomically and gccct enables method-caching without VM-wide locking. This table solves the performance issue on multi-ractor execution. [Bug #17497] Ruby-level method invocation does not use gccct because it has inline-method-cache and the table size is limited. Basically rb_funcall* is not used frequently, so 1023 entries can be enough. We will revisit the table size if it is not enough. --- debug_counter.h | 3 + vm.c | 12 +++ vm_callinfo.h | 12 --- vm_core.h | 5 + vm_eval.c | 288 ++++++++++++++++++++++++++++++++++++++++++-------------- vm_insnhelper.c | 11 ++- vm_method.c | 14 ++- 7 files changed, 255 insertions(+), 90 deletions(-) diff --git a/debug_counter.h b/debug_counter.h index 6dc66ef..3c20821 100644 --- a/debug_counter.h +++ b/debug_counter.h @@ -62,6 +62,9 @@ RB_DEBUG_COUNTER(ccs_not_found) // count for not found corresponding ccs on met https://github.com/ruby/ruby/blob/trunk/debug_counter.h#L62 // vm_eval.c RB_DEBUG_COUNTER(call0_public) RB_DEBUG_COUNTER(call0_other) +RB_DEBUG_COUNTER(gccct_hit) +RB_DEBUG_COUNTER(gccct_miss) +RB_DEBUG_COUNTER(gccct_null) // iseq RB_DEBUG_COUNTER(iseq_num) // number of total created iseq diff --git a/vm.c b/vm.c index 5beb6d0..2899433 100644 --- a/vm.c +++ b/vm.c @@ -2588,6 +2588,18 @@ rb_vm_mark(void *ptr) https://github.com/ruby/ruby/blob/trunk/vm.c#L2588 rb_gc_mark_values(RUBY_NSIG, vm->trap_list.cmd); rb_id_table_foreach_values(vm->negative_cme_table, vm_mark_negative_cme, NULL); + for (i=0; i<VM_GLOBAL_CC_CACHE_TABLE_SIZE; i++) { + const struct rb_callcache *cc = vm->global_cc_cache_table[i]; + + if (cc != NULL) { + if (!vm_cc_invalidated_p(cc)) { + rb_gc_mark((VALUE)cc); + } + else { + vm->global_cc_cache_table[i] = NULL; + } + } + } mjit_mark(); } diff --git a/vm_callinfo.h b/vm_callinfo.h index 7fc0a93..c0716d6 100644 --- a/vm_callinfo.h +++ b/vm_callinfo.h @@ -387,18 +387,6 @@ extern const struct rb_callcache *rb_vm_empty_cc(void); https://github.com/ruby/ruby/blob/trunk/vm_callinfo.h#L387 /* callcache: mutate */ static inline void -vm_cc_cme_set(const struct rb_callcache *cc, const struct rb_callable_method_entry_struct *cme) -{ - VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); - VM_ASSERT(cc != vm_cc_empty()); - VM_ASSERT(vm_cc_cme(cc) != NULL); - VM_ASSERT(vm_cc_cme(cc)->called_id == cme->called_id); - VM_ASSERT(!vm_cc_markable(cc)); // only used for vm_eval.c - - *((const struct rb_callable_method_entry_struct **)&cc->cme_) = cme; -} - -static inline void vm_cc_call_set(const struct rb_callcache *cc, vm_call_handler call) { VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); diff --git a/vm_core.h b/vm_core.h index 5f8d4ab..a7644eb 100644 --- a/vm_core.h +++ b/vm_core.h @@ -658,6 +658,11 @@ typedef struct rb_vm_struct { https://github.com/ruby/ruby/blob/trunk/vm_core.h#L658 struct rb_id_table *negative_cme_table; +#ifndef VM_GLOBAL_CC_CACHE_TABLE_SIZE +#define VM_GLOBAL_CC_CACHE_TABLE_SIZE 1023 +#endif + const struct rb_callcache *global_cc_cache_table[VM_GLOBAL_CC_CACHE_TABLE_SIZE]; // vm_eval.c + #if USE_VM_CLOCK uint32_t clock; #endif diff --git a/vm_eval.c b/vm_eval.c index 8eb1d5c..c75e7d7 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -48,7 +48,22 @@ rb_vm_call0(rb_execution_context_t *ec, VALUE recv, ID id, int argc, const VALUE https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L48 struct rb_calling_info calling = { .ci = &VM_CI_ON_STACK(id, kw_splat ? VM_CALL_KW_SPLAT : 0, argc, NULL), .cc = &VM_CC_ON_STACK(Qfalse, vm_call_general, { 0 }, cme), - .block_handler = VM_BLOCK_HANDLER_NONE, + .block_handler = vm_passed_block_handler(ec), + .recv = recv, + .argc = argc, + .kw_splat = kw_splat, + }; + + return vm_call0_body(ec, &calling, argv); +} + +static inline VALUE +vm_call0_cc(rb_execution_context_t *ec, VALUE recv, ID id, int argc, const VALUE *argv, const struct rb_callcache *cc, int kw_splat) +{ + struct rb_calling_info calling = { + .ci = &VM_CI_ON_STACK(id, kw_splat ? VM_CALL_KW_SPLAT : 0, argc, NULL), + .cc = cc, + .block_handler = vm_passed_block_handler(ec), .recv = recv, .argc = argc, .kw_splat = kw_splat, @@ -58,12 +73,37 @@ rb_vm_call0(rb_execution_context_t *ec, VALUE recv, ID id, int argc, const VALUE https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L73 } static VALUE +vm_call0_cme(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv, const rb_callable_method_entry_t *cme) +{ + calling->cc = &VM_CC_ON_STACK(Qfalse, vm_call_general, { 0 }, cme); + return vm_call0_body(ec, calling, argv); +} + +static VALUE +vm_call0_super(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv, VALUE klass, enum method_missing_reason ex) +{ + ID mid = vm_ci_mid(calling->ci); + klass = RCLASS_SUPER(klass); + + if (klass) { + const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid); + + if (cme) { + RUBY_VM_CHECK_INTS(ec); + return vm_call0_cme(ec, calling, argv, cme); + } + } + + vm_passed_block_handler_set(ec, calling->block_handler); + return method_missing(ec, calling->recv, mid, calling->argc, argv, ex, calling->kw_splat); +} + +static VALUE vm_call0_cfunc_with_frame(rb_execution_context_t* ec, struct rb_calling_info *calling, const VALUE *argv) { const struct rb_callinfo *ci = calling->ci; - const struct rb_callcache *cc = calling->cc; VALUE val; - const rb_callable_method_entry_t *me = vm_cc_cme(cc); + const rb_callable_method_entry_t *me = vm_cc_cme(calling->cc); const rb_method_cfunc_t *cfunc = UNALIGNED_MEMBER_PTR(me->def, body.cfunc); int len = cfunc->argc; VALUE recv = calling->recv; @@ -115,12 +155,8 @@ vm_call0_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L155 { const struct rb_callinfo *ci = calling->ci; const struct rb_callcache *cc = calling->cc; - VALUE ret; - calling->block_handler = vm_passed_block_handler(ec); - - again: switch (vm_cc_cme(cc)->def->type) { case VM_METHOD_TYPE_ISEQ: { @@ -169,38 +205,27 @@ vm_call0_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L205 ret = vm_call_bmethod_body(ec, calling, argv); goto success; case VM_METHOD_TYPE_ZSUPER: + { + VALUE klass = RCLASS_ORIGIN(vm_cc_cme(cc)->defined_class); + return vm_call0_super(ec, calling, argv, klass, MISSING_SUPER); + } case VM_METHOD_TYPE_REFINED: { - const rb_method_type_t type = vm_cc_cme(cc)->def->type; - VALUE super_class = vm_cc_cme(cc)->defined_class; - - if (type == VM_METHOD_TYPE_ZSUPER) { - super_class = RCLASS_ORIGIN(super_class); - } - else if (vm_cc_cme(cc)->def->body.refined.orig_me) { - vm_cc_cme_set(cc, refined_method_callable_without_refinement(vm_cc_cme(cc))); - goto again; - } + const rb_callable_method_entry_t *cme = vm_cc_cme(cc); - super_class = RCLASS_SUPER(super_class); - if (super_class) { - vm_cc_cme_set(cc, rb_callable_method_entry(super_class, vm_ci_mid(ci))); - if (vm_cc_cme(cc)) { - RUBY_VM_CHECK_INTS(ec); - goto again; - } + if (cme->def->body.refined.orig_me) { + const rb_callable_method_entry_t *orig_cme = refined_method_callable_without_refinement(cme); + return vm_call0_cme(ec, calling, argv, orig_cme); } - enum method_missing_reason ex = (type == VM_METHOD_TYPE_ZSUPER) ? MISSING_SUPER : 0; - ret = method_missing(ec, calling->recv, vm_ci_mid(ci), calling->argc, argv, ex, calling->kw_splat); - goto success; + VALUE klass = cme->defined_class; + return vm_call0_super(ec, calling, argv, klass, 0); } case VM_METHOD_TYPE_ALIAS: - vm_cc_cme_set(cc, aliased_callable_method_entry(vm_cc_cme(cc))); - goto again; + return vm_call0_cme(ec, calling, argv, aliased_callable_method_entry(vm_cc_cme(cc))); case VM_METHOD_TYPE_MISSING: { - vm_passed_block_handler_set(ec, calling->block_handler); + vm_passed_block_handler_set(ec, calling->block_handler); return method_missing(ec, calling->recv, vm_ci_mid(ci), calling->argc, argv, MISSING_NOENTRY, calling->kw_splat); } @@ -312,9 +337,106 @@ rb_check_stack_overflow(void) https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L337 if (ec) stack_check(ec); } +NORETURN(static void uncallable_object(VALUE recv, ID mid)); static inline const rb_callable_method_entry_t *rb_search_method_entry( (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/