ruby-changes:68804
From: Alan <ko1@a...>
Date: Thu, 21 Oct 2021 08:13:37 +0900 (JST)
Subject: [ruby-changes:68804] 57977ba30d (master): uJIT: Implement opt_getinlinecache
https://git.ruby-lang.org/ruby.git/commit/?id=57977ba30d From 57977ba30d35f6f9de3d2802d1894e1f0d23286d Mon Sep 17 00:00:00 2001 From: Alan Wu <XrXr@u...> Date: Thu, 25 Feb 2021 15:10:38 -0500 Subject: uJIT: Implement opt_getinlinecache * ujit: implement opt_getinlinecache Aggressively bet that writes to constants don't happen and invalidate all opt_getinlinecache blocks on any and all constant writes. Use alignment padding on block_t to track this assumption. No change to sizeof(block_t). * Fix compile warnings when not RUBY_DEBUG * Fix reversed condition * Switch to st_table to keep track of assumptions Co-authored-by: Aaron Patterson <aaron.patterson@g...> Co-authored-by: Maxime Chevalier-Boisvert <maximechevalierb@g...> --- common.mk | 1 + ractor.c | 2 + ujit.h | 1 + ujit_codegen.c | 57 ++++++++++++++++++++++++-- ujit_core.c | 9 ++++- ujit_core.h | 9 +++-- ujit_iface.c | 123 +++++++++++++++++++++++++++++++++++++++++---------------- ujit_iface.h | 7 ++++ 8 files changed, 165 insertions(+), 44 deletions(-) diff --git a/common.mk b/common.mk index abe03d487e..cdbbbf3c4f 100644 --- a/common.mk +++ b/common.mk @@ -10590,6 +10590,7 @@ ractor.$(OBJEXT): {$(VPATH)}thread.h https://github.com/ruby/ruby/blob/trunk/common.mk#L10590 ractor.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h ractor.$(OBJEXT): {$(VPATH)}thread_native.h ractor.$(OBJEXT): {$(VPATH)}transient_heap.h +ractor.$(OBJEXT): {$(VPATH)}ujit.h ractor.$(OBJEXT): {$(VPATH)}variable.h ractor.$(OBJEXT): {$(VPATH)}vm_core.h ractor.$(OBJEXT): {$(VPATH)}vm_debug.h diff --git a/ractor.c b/ractor.c index bfc61f99fe..fa78d16411 100644 --- a/ractor.c +++ b/ractor.c @@ -16,6 +16,7 @@ https://github.com/ruby/ruby/blob/trunk/ractor.c#L16 #include "variable.h" #include "gc.h" #include "transient_heap.h" +#include "ujit.h" VALUE rb_cRactor; @@ -1604,6 +1605,7 @@ ractor_create(rb_execution_context_t *ec, VALUE self, VALUE loc, VALUE name, VAL https://github.com/ruby/ruby/blob/trunk/ractor.c#L1605 r->verbose = cr->verbose; r->debug = cr->debug; + rb_ujit_before_ractor_spawn(); rb_thread_create_ractor(r, args, block); RB_GC_GUARD(rv); diff --git a/ujit.h b/ujit.h index 6957e9178f..f3c2bffae6 100644 --- a/ujit.h +++ b/ujit.h @@ -56,5 +56,6 @@ void rb_ujit_constant_state_changed(void); https://github.com/ruby/ruby/blob/trunk/ujit.h#L56 void rb_ujit_iseq_mark(const struct rb_iseq_constant_body *body); void rb_ujit_iseq_update_references(const struct rb_iseq_constant_body *body); void rb_ujit_iseq_free(const struct rb_iseq_constant_body *body); +void rb_ujit_before_ractor_spawn(void); #endif // #ifndef UJIT_H diff --git a/ujit_codegen.c b/ujit_codegen.c index 6ac6dc013c..ad48029a31 100644 --- a/ujit_codegen.c +++ b/ujit_codegen.c @@ -60,19 +60,20 @@ jit_get_arg(jitstate_t* jit, size_t arg_idx) https://github.com/ruby/ruby/blob/trunk/ujit_codegen.c#L60 return *(jit->pc + arg_idx + 1); } -// Load a pointer to a GC'd object into a register and keep track of the reference +// Load a VALUE into a register and keep track of the reference if it is on the GC heap. static void jit_mov_gc_ptr(jitstate_t* jit, codeblock_t* cb, x86opnd_t reg, VALUE ptr) { RUBY_ASSERT(reg.type == OPND_REG && reg.num_bits == 64); - RUBY_ASSERT(!SPECIAL_CONST_P(ptr)); mov(cb, reg, const_ptr_opnd((void*)ptr)); // The pointer immediate is encoded as the last part of the mov written out. uint32_t ptr_offset = cb->write_pos - sizeof(VALUE); - if (!rb_darray_append(&jit->block->gc_object_offsets, ptr_offset)) { - rb_bug("allocation failed"); + if (!SPECIAL_CONST_P(ptr)) { + if (!rb_darray_append(&jit->block->gc_object_offsets, ptr_offset)) { + rb_bug("allocation failed"); + } } } @@ -252,12 +253,14 @@ ujit_gen_block(ctx_t* ctx, block_t* block) https://github.com/ruby/ruby/blob/trunk/ujit_codegen.c#L253 break; } +#if RUBY_DEBUG // Accumulate stats about instructions executed if (rb_ujit_opts.gen_stats) { // Count instructions executed by the JIT mov(cb, REG0, const_ptr_opnd((void *)&rb_ujit_exec_insns_count)); add(cb, mem_opnd(64, REG0, 0), imm_opnd(1)); } +#endif //fprintf(stderr, "compiling %d: %s\n", insn_idx, insn_name(opcode)); //print_str(cb, insn_name(opcode)); @@ -1115,6 +1118,7 @@ gen_oswb_cfunc(jitstate_t* jit, ctx_t* ctx, struct rb_call_data * cd, const rb_c https://github.com/ruby/ruby/blob/trunk/ujit_codegen.c#L1118 // Pointer to the klass field of the receiver &(recv->klass) x86opnd_t klass_opnd = mem_opnd(64, REG0, offsetof(struct RBasic, klass)); + // FIXME: This leaks when st_insert raises NoMemoryError assume_method_lookup_stable(cd->cc, cme, jit->block); // Bail if receiver class is different from compile-time call cache class @@ -1570,6 +1574,48 @@ gen_leave(jitstate_t* jit, ctx_t* ctx) https://github.com/ruby/ruby/blob/trunk/ujit_codegen.c#L1574 return true; } +RUBY_EXTERN rb_serial_t ruby_vm_global_constant_state; +static bool +gen_opt_getinlinecache(jitstate_t *jit, ctx_t *ctx) +{ + VALUE jump_offset = jit_get_arg(jit, 0); + VALUE const_cache_as_value = jit_get_arg(jit, 1); + IC ic = (IC)const_cache_as_value; + + // See vm_ic_hit_p(). + struct iseq_inline_constant_cache_entry *ice = ic->entry; + if (!ice) return false; // cache not filled + if (ice->ic_serial != ruby_vm_global_constant_state) { + // Cache miss at compile time. + return false; + } + if (ice->ic_cref) { + // Only compile for caches that don't care about lexical scope. + return false; + } + + // Optimize for single ractor mode. + // FIXME: This leaks when st_insert raises NoMemoryError + if (!assume_single_ractor_mode(jit->block)) return false; + + // Invalidate output code on any and all constant writes + // FIXME: This leaks when st_insert raises NoMemoryError + if (!assume_stable_global_constant_state(jit->block)) return false; + + x86opnd_t stack_top = ctx_stack_push(ctx, T_NONE); + jit_mov_gc_ptr(jit, cb, REG0, ice->value); + mov(cb, stack_top, REG0); + + // Jump over the code for filling the cache + uint32_t jump_idx = jit_next_insn_idx(jit) + (int32_t)jump_offset; + gen_direct_jump( + ctx, + (blockid_t){ .iseq = jit->iseq, .idx = jump_idx } + ); + + return true; +} + void ujit_reg_op(int opcode, codegen_fn gen_fn, bool is_branch) { // Check that the op wasn't previously registered @@ -1620,6 +1666,9 @@ ujit_init_codegen(void) https://github.com/ruby/ruby/blob/trunk/ujit_codegen.c#L1666 ujit_reg_op(BIN(opt_and), gen_opt_and, false); ujit_reg_op(BIN(opt_minus), gen_opt_minus, false); ujit_reg_op(BIN(opt_plus), gen_opt_plus, false); + + // Map branch instruction opcodes to codegen functions + ujit_reg_op(BIN(opt_getinlinecache), gen_opt_getinlinecache, true); ujit_reg_op(BIN(branchif), gen_branchif, true); ujit_reg_op(BIN(branchunless), gen_branchunless, true); ujit_reg_op(BIN(jump), gen_jump, true); diff --git a/ujit_core.c b/ujit_core.c index 0e6492b4f8..fea49ce7f6 100644 --- a/ujit_core.c +++ b/ujit_core.c @@ -175,8 +175,10 @@ add_block_version(blockid_t blockid, block_t* block) https://github.com/ruby/ruby/blob/trunk/ujit_core.c#L175 rb_bug("allocation failed"); } +#if RUBY_DEBUG // First block compiled for this iseq rb_compiled_iseq_count++; +#endif } block_t *first_version = get_first_version(iseq, blockid.idx); @@ -199,7 +201,7 @@ add_block_version(blockid_t blockid, block_t* block) https://github.com/ruby/ruby/blob/trunk/ujit_core.c#L201 RB_OBJ_WRITTEN(iseq, Qundef, block->dependencies.cc); RB_OBJ_WRITTEN(iseq, Qundef, block->dependencies.cme); - // Run write barrier for all objects in generated code. + // Run write barriers for all objects in generated code. uint32_t *offset_element; rb_darray_foreach(block->gc_object_offsets, offset_idx, offset_element) { uint32_t offset_to_value = *offset_element; @@ -601,9 +603,12 @@ void https://github.com/ruby/ruby/blob/trunk/ujit_core.c#L603 ujit_free_block(block_t *block) { ujit_unlink_method_lookup_dependency(block); + ujit_block_assumptions_free(block); + rb_darray_free(block->incoming); - free(block); rb_darray_free(block->gc_object_offsets); + + free(block); } // Invalidate one specific block version diff --git a/ujit_core.h b/ujit_core.h index 64de5ad979..1fe58856fa 100644 --- a/ujit_core.h +++ b/ujit_core.h @@ -107,9 +107,6 @@ typedef struct ujit_block_version https://github.com/ruby/ruby/blob/trunk/ujit_core.h#L107 // Bytecode sequence (iseq, idx) this is a version of blockid_t blockid; - // Index one past the last instruction in the iseq - uint32_t end_idx; - // Context at the start of the block ctx_t ctx; @@ -120,6 +117,9 @@ typedef struct ujit_block_version https://github.com/ruby/ruby/blob/trunk/ujit_core.h#L117 // List of incoming branches indices int32_array_t incoming; + // Offsets for GC managed objects in the mainline code block + int32_array_t gc_object_offsets; + // Next block version for this blockid (singly-linked list) struct ujit_block_version *next; @@ -132,6 +132,9 @@ typedef struct ujit_block_version https://github.com/ruby/ruby/blob/trunk/ujit_core.h#L132 VALUE cme; VALUE iseq; } dependencies; + + // Index one past the last instruction in the iseq + uint32_t end_idx; } block_t; // Context object methods diff --git a/ujit_iface.c b/ujit_iface.c index 64f9fe9a81..f2eb657b3b 100644 --- a/ujit_iface.c +++ b/ujit_iface.c @@ -24,10 +24,12 @@ VALUE cUjitBlock; https://github.com/ruby/ruby/blob/trunk/ujit_iface.c#L24 VALUE cUjitDisasm; VALUE cUjitDisasmInsn; +# (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/