https://git.ruby-lang.org/ruby.git/commit/?id=c378c7a7cb

From c378c7a7cb937cd9fe5814f2838b1d6cd1d177b2 Mon Sep 17 00:00:00 2001
From: Alan Wu <XrXr@u...>
Date: Tue, 27 Oct 2020 18:49:17 -0400
Subject: MicroJIT: generate less code for CFUNCs

Added UJIT_CHECK_MODE. Set to 1 to double check method dispatch in
generated code.

It's surprising to me that we need to watch both cc and cme. There might
be opportunities to simplify there.
---
 gc.c           |   2 +-
 ujit_asm.c     |   2 +-
 ujit_asm.h     |   2 +-
 ujit_compile.c | 273 ++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 ujit_compile.h |   2 +
 vm_callinfo.h  |  11 +--
 vm_method.c    |  15 ++++
 7 files changed, 270 insertions(+), 37 deletions(-)

diff --git a/gc.c b/gc.c
index 4451218f71..879b6e4dc0 100644
--- a/gc.c
+++ b/gc.c
@@ -2907,7 +2907,7 @@ vm_ccs_free(struct rb_class_cc_entries *ccs, int alive, rb_objspace_t *objspace, https://github.com/ruby/ruby/blob/trunk/gc.c#L2907
                     asan_poison_object((VALUE)cc);
                 }
             }
-            vm_cc_invalidate(cc);
+            rb_vm_cc_invalidate(cc);
         }
         ruby_xfree(ccs->entries);
     }
diff --git a/ujit_asm.c b/ujit_asm.c
index 3ffc5503b0..4cb7bb7caa 100644
--- a/ujit_asm.c
+++ b/ujit_asm.c
@@ -73,7 +73,7 @@ x86opnd_t imm_opnd(int64_t imm) https://github.com/ruby/ruby/blob/trunk/ujit_asm.c#L73
     return opnd;
 }
 
-x86opnd_t const_ptr_opnd(void* ptr)
+x86opnd_t const_ptr_opnd(const void *ptr)
 {
     x86opnd_t opnd = {
         OPND_IMM,
diff --git a/ujit_asm.h b/ujit_asm.h
index cf8c72f30d..43b3665486 100644
--- a/ujit_asm.h
+++ b/ujit_asm.h
@@ -220,7 +220,7 @@ x86opnd_t mem_opnd(size_t num_bits, x86opnd_t base_reg, int32_t disp); https://github.com/ruby/ruby/blob/trunk/ujit_asm.h#L220
 x86opnd_t imm_opnd(int64_t val);
 
 // Constant pointer operand
-x86opnd_t const_ptr_opnd(void* ptr);
+x86opnd_t const_ptr_opnd(const void *ptr);
 
 // Struct member operand
 #define member_opnd(base_reg, struct_type, member_name) mem_opnd( \
diff --git a/ujit_compile.c b/ujit_compile.c
index 93c9303763..e78d328c3e 100644
--- a/ujit_compile.c
+++ b/ujit_compile.c
@@ -21,6 +21,14 @@ https://github.com/ruby/ruby/blob/trunk/ujit_compile.c#L21
 #define PLATFORM_SUPPORTED_P 1
 #endif
 
+#ifndef UJIT_CHECK_MODE
+#define UJIT_CHECK_MODE 0
+#endif
+
+#ifndef UJIT_DUMP_MODE
+#define UJIT_DUMP_MODE 0
+#endif
+
 bool rb_ujit_enabled;
 
 // Hash table of encoded instructions
@@ -35,7 +43,12 @@ typedef struct ctx_struct https://github.com/ruby/ruby/blob/trunk/ujit_compile.c#L43
     // Difference between the current stack pointer and actual stack top
     int32_t stack_diff;
 
+    // The iseq that owns the region that is compiling
     const rb_iseq_t *iseq;
+    // Index in the iseq to the opcode we are replacing
+    size_t replacement_idx;
+    // The start of output code
+    uint8_t *region_start;
 
 } ctx_t;
 
@@ -82,6 +95,137 @@ addr2insn_bookkeeping(void *code_ptr, int insn) https://github.com/ruby/ruby/blob/trunk/ujit_compile.c#L95
     }
 }
 
+// GC root for interacting with the GC
+struct ujit_root_struct {};
+
+// Map cme_or_cc => [[iseq, offset]]. An entry in the map means compiled code at iseq[offset]
+// is only valid when cme_or_cc is valid
+static st_table *method_lookup_dependency;
+
+struct compiled_region_array {
+    int32_t size;
+    int32_t capa;
+    struct compiled_region {
+        const rb_iseq_t *iseq;
+        size_t replacement_idx;
+        uint8_t *code;
+    }data[];
+};
+
+// Add an element to a region array, or allocate a new region array.
+static struct compiled_region_array *
+add_compiled_region(struct compiled_region_array *array, const rb_iseq_t *iseq, size_t replacement_idx, uint8_t *code)
+{
+    if (!array) {
+        // Allocate a brand new array with space for one
+        array = malloc(sizeof(*array) + sizeof(struct compiled_region));
+        if (!array) {
+            return NULL;
+        }
+        array->size = 0;
+        array->capa = 1;
+    }
+    if (array->size == INT32_MAX) {
+        return NULL;
+    }
+    // Check if the region is already present
+    for (int32_t i = 0; i < array->size; i++) {
+        if (array->data[i].iseq == iseq && array->data[i].replacement_idx == replacement_idx) {
+            return array;
+        }
+    }
+    if (array->size + 1 > array->capa) {
+        // Double the array's capacity.
+        int64_t double_capa = ((int64_t)array->capa) * 2;
+        int32_t new_capa = (int32_t)double_capa;
+        if (new_capa != double_capa) {
+            return NULL;
+        }
+        array = realloc(array, sizeof(*array) + new_capa * sizeof(struct compiled_region));
+        if (array == NULL) {
+            return NULL;
+        }
+        array->capa = new_capa;
+    }
+
+    int32_t size = array->size;
+    array->data[size].iseq = iseq;
+    array->data[size].replacement_idx = replacement_idx;
+    array->data[size].code = code;
+    array->size++;
+    return array;
+}
+
+static int
+add_lookup_dependency_i(st_data_t *key, st_data_t *value, st_data_t data, int existing)
+{
+    ctx_t *ctx = (ctx_t *)data;
+    struct compiled_region_array *regions = NULL;
+    if (existing) {
+        regions = (struct compiled_region_array *)*value;
+    }
+    regions = add_compiled_region(regions, ctx->iseq, ctx->replacement_idx, ctx->region_start);
+    if (!regions) {
+        rb_bug("ujit: failed to add method lookup dependency"); // TODO: we could bail out of compiling instead
+    }
+    *value = (st_data_t)regions;
+    return ST_CONTINUE;
+}
+
+// Store info to remember that the currently compiling region is only valid while cme and cc and valid.
+static void
+ujit_assume_method_lookup_stable(const struct rb_callcache *cc, const rb_callable_method_entry_t *cme,  ctx_t *ctx)
+{
+    st_update(method_lookup_dependency, (st_data_t)cme, add_lookup_dependency_i, (st_data_t)ctx);
+    st_update(method_lookup_dependency, (st_data_t)cc, add_lookup_dependency_i, (st_data_t)ctx);
+    // FIXME: This is a leak! When either the cme or the cc become invalid, the other also needs to go
+}
+
+static int
+ujit_root_mark_i(st_data_t k, st_data_t v, st_data_t ignore)
+{
+    // FIXME: This leaks everything that end up in the dependency table!
+    // One way to deal with this is with weak references...
+    rb_gc_mark((VALUE)k);
+    struct compiled_region_array *regions = (void *)v;
+    for (int32_t i = 0; i < regions->size; i++) {
+        rb_gc_mark((VALUE)regions->data[i].iseq);
+    }
+
+    return ST_CONTINUE;
+}
+
+// GC callback during mark phase
+static void
+ujit_root_mark(void *ptr)
+{
+    if (method_lookup_dependency) {
+        st_foreach(method_lookup_dependency, ujit_root_mark_i, 0);
+    }
+}
+
+static void
+ujit_root_free(void *ptr)
+{
+    // Do nothing. The root lives as long as the process.
+}
+
+static size_t
+ujit_root_memsize(const void *ptr)
+{
+    // Count off-gc-heap allocation size of the dependency table
+    return st_memsize(method_lookup_dependency); // TODO: more accurate accounting
+}
+
+// Custom type for interacting with the GC
+// TODO: compaction support
+// TODO: make this write barrier protected
+static const rb_data_type_t ujit_root_type = {
+    "ujit_root",
+    {ujit_root_mark, ujit_root_free, ujit_root_memsize, },
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
 static int
 opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc)
 {
@@ -247,6 +391,8 @@ ujit_compile_insn(const rb_iseq_t *iseq, unsigned int insn_idx, unsigned int* ne https://github.com/ruby/ruby/blob/trunk/ujit_compile.c#L391
     ctx.pc = NULL;
     ctx.stack_diff = 0;
     ctx.iseq = iseq;
+    ctx.region_start = code_ptr;
+    ctx.replacement_idx = insn_idx;
 
     // For each instruction to compile
     size_t num_instrs;
@@ -483,6 +629,27 @@ gen_opt_minus(codeblock_t* cb, codeblock_t* ocb, ctx_t* ctx) https://github.com/ruby/ruby/blob/trunk/ujit_compile.c#L629
     return true;
 }
 
+// Verify that calling with cd on receiver goes to callee
+static void
+check_cfunc_dispatch(VALUE receiver, struct rb_call_data *cd, void *callee, rb_callable_method_entry_t *compile_time_cme)
+{
+    if (METHOD_ENTRY_INVALIDATED(compile_time_cme)) {
+        rb_bug("ujit: output code uses invalidated cme %p", (void *)compile_time_cme);
+    }
+
+    bool callee_correct = false;
+    const rb_callable_method_entry_t *cme = rb_callable_method_entry(CLASS_OF(receiver), vm_ci_mid(cd->ci));
+    if (cme->def->type == VM_METHOD_TYPE_CFUNC) {
+        const rb_method_cfunc_t *cfunc = UNALIGNED_MEMBER_PTR(cme->def, body.cfunc);
+        if ((void *)cfunc->func == callee) {
+            callee_correct = true;
+        }
+    }
+    if (!callee_correct) {
+        rb_bug("ujit: output code calls wrong method cd->cc->klass: %p", (void *)cd->cc->klass);
+    }
+}
+
 MJIT_FUNC_EXPORTED VALUE rb_hash_has_key(VALUE hash, VALUE key);
 
 bool
@@ -524,21 +691,24 @@ gen_opt_send_without_block(codeblock_t* cb, codeblock_t* ocb, ctx_t* ctx) https://github.com/ruby/ruby/blob/trunk/ujit_compile.c#L691
     }
 
     // Don't JIT if the inline cache is not set
-    if (cd->cc == vm_cc_empty())
-    {
-        //printf("call cache is empty\n");
+    if (!cd->cc || !cd->cc->klass) {
         return false;
     }
 
-    const rb_callable_method_entry_t *me = vm_cc_cme(cd->cc);
+    const rb_callable_method_entry_t *cme = vm_cc_cme(cd->cc);
+
+    // Don't JIT if the method entry is out of date
+    if (METHOD_ENTRY_INVALIDATED(cme)) {
+        return false;
+    }
 
     // Don't JIT if this is not a C call
-    if (me->def->type != VM_METHOD_TYPE_CFUNC)
+    if (cme->def->type != VM_METHOD_TYPE_CFUNC)
     {
         return false;
     }
 
-    const rb_method_cfunc_t *cfunc = UNALIGNED_MEMBER_PTR(me->def, body.cfunc);
+    const rb_method_cfunc_t *cfunc = UNALIGNED_MEMBER_PTR(cme->def, body.cfunc);
 
     // Don't JIT if the argument count doesn't match
     if (cfunc->argc < 0 || cfunc->argc != argc)
@@ -586,24 +756,14 @@ gen_opt_send_without_block(codeblock_t* cb, codeb (... truncated)

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/