ruby-changes:72298
From: Aaron <ko1@a...>
Date: Fri, 24 Jun 2022 06:01:59 +0900 (JST)
Subject: [ruby-changes:72298] e23540e566 (master): Speed up ISeq by marking via bitmaps and IC rearranging
https://git.ruby-lang.org/ruby.git/commit/?id=e23540e566 From e23540e5666664e23f2adecdc2cc591f3ff6fe2f Mon Sep 17 00:00:00 2001 From: Aaron Patterson <tenderlove@r...> Date: Fri, 17 Jun 2022 15:28:14 -0700 Subject: Speed up ISeq by marking via bitmaps and IC rearranging This commit adds a bitfield to the iseq body that stores offsets inside the iseq buffer that contain values we need to mark. We can use this bitfield to mark objects instead of disassembling the instructions. This commit also groups inline storage entries and adds a counter for each entry. This allows us to iterate and mark each entry without disassembling instructions Since we have a bitfield and grouped inline caches, we can mark all VALUE objects associated with instructions without actually disassembling the instructions at mark time. [Feature #18875] [ruby-core:109042] --- compile.c | 118 +++++++++++++++++++++++++++++++++++++++++++++------------ iseq.c | 70 +++++++++++++++++++++++++++++----- iseq.h | 6 +++ mjit_compile.c | 6 +-- vm_core.h | 11 +++++- 5 files changed, 172 insertions(+), 39 deletions(-) diff --git a/compile.c b/compile.c index b820d9be8c..06c5557672 100644 --- a/compile.c +++ b/compile.c @@ -2069,7 +2069,7 @@ get_ivar_ic_value(rb_iseq_t *iseq,ID id) https://github.com/ruby/ruby/blob/trunk/compile.c#L2069 tbl = rb_id_table_create(1); ISEQ_COMPILE_DATA(iseq)->ivar_cache_table = tbl; } - val = INT2FIX(ISEQ_BODY(iseq)->is_size++); + val = INT2FIX(ISEQ_BODY(iseq)->ivc_size++); rb_id_table_insert(tbl,id,val); return val; } @@ -2327,14 +2327,23 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) https://github.com/ruby/ruby/blob/trunk/compile.c#L2327 generated_iseq = ALLOC_N(VALUE, code_index); insns_info = ALLOC_N(struct iseq_insn_info_entry, insn_num); positions = ALLOC_N(unsigned int, insn_num); - body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, body->is_size); + body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, ISEQ_IS_SIZE(body)); body->call_data = ZALLOC_N(struct rb_call_data, body->ci_size); ISEQ_COMPILE_DATA(iseq)->ci_index = 0; + // Calculate the bitmask buffer size. + // Round the generated_iseq size up to the nearest multiple + // of the number if bits in an unsigned long. + + // Allocate enough room for the bitmask list + iseq_bits_t * mark_offset_bits = ZALLOC_N(iseq_bits_t, ISEQ_MBITS_BUFLEN(code_index)); + list = FIRST_ELEMENT(anchor); insns_info_index = code_index = sp = 0; while (list) { + unsigned int ic_index = 0; + switch (list->type) { case ISEQ_ELEMENT_INSN: { @@ -2375,6 +2384,7 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) https://github.com/ruby/ruby/blob/trunk/compile.c#L2384 rb_hash_rehash(map); freeze_hide_obj(map); generated_iseq[code_index + 1 + j] = map; + ISEQ_MBITS_SET(mark_offset_bits, code_index + 1 + j); RB_OBJ_WRITTEN(iseq, Qundef, map); FL_SET(iseqv, ISEQ_MARKABLE_ISEQ); break; @@ -2383,30 +2393,34 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) https://github.com/ruby/ruby/blob/trunk/compile.c#L2393 case TS_NUM: /* ulong */ generated_iseq[code_index + 1 + j] = FIX2INT(operands[j]); break; - case TS_VALUE: /* VALUE */ case TS_ISEQ: /* iseq */ + case TS_VALUE: /* VALUE */ { VALUE v = operands[j]; generated_iseq[code_index + 1 + j] = v; /* to mark ruby object */ if (!SPECIAL_CONST_P(v)) { RB_OBJ_WRITTEN(iseq, Qundef, v); + ISEQ_MBITS_SET(mark_offset_bits, code_index + 1 + j); FL_SET(iseqv, ISEQ_MARKABLE_ISEQ); } break; } - case TS_IC: /* inline cache */ - case TS_ISE: /* inline storage entry */ - case TS_ICVARC: /* inline cvar cache */ + /* [ TS_(ICVARC|IVC) ... | TS_ISE | TS_IC ] */ + case TS_IC: /* inline cache: constants */ + ic_index += body->ise_size; + case TS_ISE: /* inline storage entry: `once` insn */ + ic_index += body->ivc_size; + case TS_ICVARC: /* inline cvar cache */ case TS_IVC: /* inline ivar cache */ { - unsigned int ic_index = FIX2UINT(operands[j]); + ic_index += FIX2UINT(operands[j]); IC ic = (IC)&body->is_entries[ic_index]; - if (UNLIKELY(ic_index >= body->is_size)) { + if (UNLIKELY(ic_index >= ISEQ_IS_SIZE(body))) { BADINSN_DUMP(anchor, &iobj->link, 0); COMPILE_ERROR(iseq, iobj->insn_info.line_no, "iseq_set_sequence: ic_index overflow: index: %d, size: %d", - ic_index, body->is_size); + ic_index, ISEQ_IS_SIZE(body)); } generated_iseq[code_index + 1 + j] = (VALUE)ic; FL_SET(iseqv, ISEQ_MARKABLE_ISEQ); @@ -2491,6 +2505,7 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) https://github.com/ruby/ruby/blob/trunk/compile.c#L2505 xfree(generated_iseq); xfree(insns_info); xfree(positions); + xfree(mark_offset_bits); debug_list(anchor, list); COMPILE_ERROR(iseq, adjust->line_no, "iseq_set_sequence: adjust bug to %d %d < %d", @@ -2510,6 +2525,7 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) https://github.com/ruby/ruby/blob/trunk/compile.c#L2525 body->iseq_encoded = (void *)generated_iseq; body->iseq_size = code_index; body->stack_max = stack_max; + body->mark_offset_bits = mark_offset_bits; /* get rid of memory leak when REALLOC failed */ body->insns_info.body = insns_info; @@ -8843,7 +8859,7 @@ compile_colon2(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, https://github.com/ruby/ruby/blob/trunk/compile.c#L8859 if (rb_is_const_id(node->nd_mid)) { /* constant */ LABEL *lend = NEW_LABEL(line); - int ic_index = ISEQ_BODY(iseq)->is_size++; + int ic_index = ISEQ_BODY(iseq)->ic_size++; DECL_ANCHOR(pref); DECL_ANCHOR(body); @@ -8888,7 +8904,7 @@ compile_colon3(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const node, https://github.com/ruby/ruby/blob/trunk/compile.c#L8904 { const int line = nd_line(node); LABEL *lend = NEW_LABEL(line); - int ic_index = ISEQ_BODY(iseq)->is_size++; + int ic_index = ISEQ_BODY(iseq)->ic_size++; debugi("colon3#nd_mid", node->nd_mid); @@ -9407,7 +9423,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no https://github.com/ruby/ruby/blob/trunk/compile.c#L9423 if (ISEQ_COMPILE_DATA(iseq)->option->inline_const_cache) { LABEL *lend = NEW_LABEL(line); - int ic_index = body->is_size++; + int ic_index = body->ic_size++; ADD_INSN2(ret, node, opt_getinlinecache, lend, INT2FIX(ic_index)); ADD_INSN1(ret, node, putobject, Qtrue); @@ -9532,7 +9548,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no https://github.com/ruby/ruby/blob/trunk/compile.c#L9548 break; } case NODE_ONCE:{ - int ic_index = body->is_size++; + int ic_index = body->ise_size++; const rb_iseq_t *block_iseq; block_iseq = NEW_CHILD_ISEQ(node->nd_body, make_name_for_block(iseq), ISEQ_TYPE_PLAIN, line); @@ -9763,7 +9779,7 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const no https://github.com/ruby/ruby/blob/trunk/compile.c#L9779 /* compiled to: * ONCE{ rb_mRubyVMFrozenCore::core#set_postexe{ ... } } */ - int is_index = body->is_size++; + int is_index = body->ise_size++; struct rb_iseq_new_with_callback_callback_func *ifunc = rb_iseq_new_with_callback_new_callback(build_postexe_iseq, node->nd_body); const rb_iseq_t *once_iseq = @@ -10294,12 +10310,24 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, https://github.com/ruby/ruby/blob/trunk/compile.c#L10310 } break; case TS_ISE: + argv[j] = op; + if (NUM2UINT(op) >= ISEQ_BODY(iseq)->ise_size) { + ISEQ_BODY(iseq)->ise_size = NUM2INT(op) + 1; + } + FL_SET((VALUE)iseq, ISEQ_MARKABLE_ISEQ); + break; case TS_IC: + argv[j] = op; + if (NUM2UINT(op) >= ISEQ_BODY(iseq)->ic_size) { + ISEQ_BODY(iseq)->ic_size = NUM2INT(op) + 1; + } + FL_SET((VALUE)iseq, ISEQ_MARKABLE_ISEQ); + break; case TS_IVC: /* inline ivar cache */ case TS_ICVARC: /* inline cvar cache */ argv[j] = op; - if (NUM2UINT(op) >= ISEQ_BODY(iseq)->is_size) { - ISEQ_BODY(iseq)->is_size = NUM2INT(op) + 1; + if (NUM2UINT(op) >= ISEQ_BODY(iseq)->ivc_size) { + ISEQ_BODY(iseq)->ivc_size = NUM2INT(op) + 1; } FL_SET((VALUE)iseq, ISEQ_MARKABLE_ISEQ); break; @@ -11110,12 +11138,12 @@ ibf_dump_code(struct ibf_dump *dump, const rb_iseq_t *iseq) https://github.com/ruby/ruby/blob/trunk/compile.c#L11138 wv = (VALUE)ibf_dump_iseq(dump, (const rb_iseq_t *)op); break; case TS_IC: + case TS_ISE: case TS_IVC: case TS_ICVARC: - case TS_ISE: { unsigned int i; - for (i=0; i<body->is_size; i++) { + for (i=0; i<ISEQ_IS_SIZE(body); i+ (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/