ruby-changes:74248
From: Takashi <ko1@a...>
Date: Wed, 26 Oct 2022 01:24:31 +0900 (JST)
Subject: [ruby-changes:74248] b7644a2311 (master): YJIT: GC and recompile all code pages (#6406)
https://git.ruby-lang.org/ruby.git/commit/?id=b7644a2311 From b7644a231100b1e1b70af528f9629d2e39572087 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun <takashikkbn@g...> Date: Tue, 25 Oct 2022 09:07:10 -0700 Subject: YJIT: GC and recompile all code pages (#6406) when it fails to allocate a new page. Co-authored-by: Alan Wu <alansi.xingwu@s...> --- cont.c | 2 +- test/ruby/test_yjit.rb | 121 +++++++++++++++++++++++++++- yjit.c | 23 +++++- yjit.rb | 6 +- yjit/bindgen/src/main.rs | 4 + yjit/src/asm/mod.rs | 179 ++++++++++++++++++++++++++++++++++++++--- yjit/src/codegen.rs | 29 +++++++ yjit/src/core.rs | 56 +++++++++++-- yjit/src/cruby_bindings.inc.rs | 6 ++ yjit/src/options.rs | 2 +- yjit/src/stats.rs | 29 +++++-- yjit/src/virtualmem.rs | 29 +++++++ 12 files changed, 454 insertions(+), 32 deletions(-) diff --git a/cont.c b/cont.c index b3c84d82ac..577a30a57a 100644 --- a/cont.c +++ b/cont.c @@ -69,7 +69,7 @@ static VALUE rb_cFiberPool; https://github.com/ruby/ruby/blob/trunk/cont.c#L69 #define FIBER_POOL_ALLOCATION_FREE #endif -#define jit_cont_enabled mjit_enabled // To be used by YJIT later +#define jit_cont_enabled (mjit_enabled || rb_yjit_enabled_p()) enum context_type { CONTINUATION_CONTEXT = 0, diff --git a/test/ruby/test_yjit.rb b/test/ruby/test_yjit.rb index 6cafb21698..09b5989a06 100644 --- a/test/ruby/test_yjit.rb +++ b/test/ruby/test_yjit.rb @@ -825,12 +825,126 @@ class TestYJIT < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ruby/test_yjit.rb#L825 RUBY end + def test_code_gc + assert_compiles(code_gc_helpers + <<~'RUBY', exits: :any, result: :ok) + return :not_paged unless add_pages(100) # prepare freeable pages + code_gc # first code GC + return :not_compiled1 unless compiles { nil } # should be JITable again + + code_gc # second code GC + return :not_compiled2 unless compiles { nil } # should be JITable again + + code_gc_count = RubyVM::YJIT.runtime_stats[:code_gc_count] + return :"code_gc_#{code_gc_count}" if code_gc_count && code_gc_count != 2 + + :ok + RUBY + end + + def test_on_stack_code_gc_call + assert_compiles(code_gc_helpers + <<~'RUBY', exits: :any, result: :ok) + fiber = Fiber.new { + # Loop to call the same basic block again after Fiber.yield + while true + Fiber.yield(nil.to_i) + end + } + + return :not_paged1 unless add_pages(400) # go to a page without initial ocb code + return :broken_resume1 if fiber.resume != 0 # JIT the fiber + code_gc # first code GC, which should not free the fiber page + return :broken_resume2 if fiber.resume != 0 # The code should be still callable + + code_gc_count = RubyVM::YJIT.runtime_stats[:code_gc_count] + return :"code_gc_#{code_gc_count}" if code_gc_count && code_gc_count != 1 + + :ok + RUBY + end + + def test_on_stack_code_gc_twice + assert_compiles(code_gc_helpers + <<~'RUBY', exits: :any, result: :ok) + fiber = Fiber.new { + # Loop to call the same basic block again after Fiber.yield + while Fiber.yield(nil.to_i); end + } + + return :not_paged1 unless add_pages(400) # go to a page without initial ocb code + return :broken_resume1 if fiber.resume(true) != 0 # JIT the fiber + code_gc # first code GC, which should not free the fiber page + + return :not_paged2 unless add_pages(300) # add some stuff to be freed + # Not calling fiber.resume here to test the case that the YJIT payload loses some + # information at the previous code GC. The payload should still be there, and + # thus we could know the fiber ISEQ is still on stack on this second code GC. + code_gc # second code GC, which should still not free the fiber page + + return :not_paged3 unless add_pages(200) # attempt to overwrite the fiber page (it shouldn't) + return :broken_resume2 if fiber.resume(true) != 0 # The fiber code should be still fine + + return :broken_resume3 if fiber.resume(false) != nil # terminate the fiber + code_gc # third code GC, freeing a page that used to be on stack + + return :not_paged4 unless add_pages(100) # check everything still works + + code_gc_count = RubyVM::YJIT.runtime_stats[:code_gc_count] + return :"code_gc_#{code_gc_count}" if code_gc_count && code_gc_count != 3 + + :ok + RUBY + end + + def test_code_gc_with_many_iseqs + assert_compiles(code_gc_helpers + <<~'RUBY', exits: :any, result: :ok, mem_size: 1) + fiber = Fiber.new { + # Loop to call the same basic block again after Fiber.yield + while true + Fiber.yield(nil.to_i) + end + } + + return :not_paged1 unless add_pages(500) # use some pages + return :broken_resume1 if fiber.resume != 0 # leave an on-stack code as well + + add_pages(2000) # use a whole lot of pages to run out of 1MiB + return :broken_resume2 if fiber.resume != 0 # on-stack code should be callable + + code_gc_count = RubyVM::YJIT.runtime_stats[:code_gc_count] + return :"code_gc_#{code_gc_count}" if code_gc_count && code_gc_count == 0 + + :ok + RUBY + end + + private + + def code_gc_helpers + <<~'RUBY' + def compiles(&block) + failures = RubyVM::YJIT.runtime_stats[:compilation_failure] + block.call + failures == RubyVM::YJIT.runtime_stats[:compilation_failure] + end + + def add_pages(num_jits) + pages = RubyVM::YJIT.runtime_stats[:compiled_page_count] + num_jits.times { return false unless eval('compiles { nil.to_i }') } + pages.nil? || pages < RubyVM::YJIT.runtime_stats[:compiled_page_count] + end + + def code_gc + RubyVM::YJIT.simulate_oom! # bump write_pos + eval('proc { nil }.call') # trigger code GC + end + RUBY + end + def assert_no_exits(script) assert_compiles(script) end ANY = Object.new - def assert_compiles(test_script, insns: [], call_threshold: 1, stdout: nil, exits: {}, result: ANY, frozen_string_literal: nil) + def assert_compiles(test_script, insns: [], call_threshold: 1, stdout: nil, exits: {}, result: ANY, frozen_string_literal: nil, mem_size: nil) reset_stats = <<~RUBY RubyVM::YJIT.runtime_stats RubyVM::YJIT.reset_stats! @@ -864,7 +978,7 @@ class TestYJIT < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ruby/test_yjit.rb#L978 #{write_results} RUBY - status, out, err, stats = eval_with_jit(script, call_threshold: call_threshold) + status, out, err, stats = eval_with_jit(script, call_threshold:, mem_size:) assert status.success?, "exited with status #{status.to_i}, stderr:\n#{err}" @@ -918,12 +1032,13 @@ class TestYJIT < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ruby/test_yjit.rb#L1032 s.chars.map { |c| c.ascii_only? ? c : "\\u%x" % c.codepoints[0] }.join end - def eval_with_jit(script, call_threshold: 1, timeout: 1000) + def eval_with_jit(script, call_threshold: 1, timeout: 1000, mem_size: nil) args = [ "--disable-gems", "--yjit-call-threshold=#{call_threshold}", "--yjit-stats" ] + args << "--yjit-exec-mem-size=#{mem_size}" if mem_size args << "-e" << script_shell_encode(script) stats_r, stats_w = IO.pipe out, err, status = EnvUtil.invoke_ruby(args, diff --git a/yjit.c b/yjit.c index f6e64aad65..7e6fc9e3fb 100644 --- a/yjit.c +++ b/yjit.c @@ -27,6 +27,7 @@ https://github.com/ruby/ruby/blob/trunk/yjit.c#L27 #include "probes_helper.h" #include "iseq.h" #include "ruby/debug.h" +#include "internal/cont.h" // For mmapp(), sysconf() #ifndef _WIN32 @@ -65,10 +66,7 @@ STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM); https://github.com/ruby/ruby/blob/trunk/yjit.c#L66 bool rb_yjit_mark_writable(void *mem_block, uint32_t mem_size) { - if (mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE)) { - return false; - } - return true; + return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0; } void @@ -85,6 +83,20 @@ rb_yjit_mark_executable(void *mem_block, uint32_t mem_size) https://github.com/ruby/ruby/blob/trunk/yjit.c#L83 } } +// Free the specified memory block. +bool +rb_yjit_mark_unused(void *mem_block, uint32_t mem_size) +{ + // On Linux, you need to use madvise MADV_DONTNEED to free memory. + // We might not need to call this on macOS, but it's not really documented. + // We generally prefer to do the same thing on both to ease testing too. + madvise(mem_block, mem_size, MADV_DONTNEED); + + // On macOS, mprotect PROT_NONE seems to reduce RSS. + // We also call this on Linux to avoid executing unused pages. + return mprotect(mem_block, mem_size, PROT_NONE) == 0; +} + // `start` is inclusive and `end` is exclusive. void rb_yjit_icache_invalidate(void *start, void *end) @@ -387,6 +399,9 @@ rb_iseq_reset_jit_func(const rb_iseq_t *iseq) https://github.com/ruby/ruby/blob/trunk/yjit.c#L399 { RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq)); iseq->body->jit_func = NULL; + // Enable re-compiling this ISEQ. Event when it's invalidated for TracePoint, + // we'd like to re-compile ISEQs that haven't been converted to trace_* insns. + iseq->body->total_calls = 0; } // Get the PC for a given index in an iseq diff --git a/yjit.rb b/yjit.rb index b80861dbfb..2a0b3dc6c6 100644 --- a/yjit.rb +++ b/yjit.rb @@ -212,13 +212,17 @@ module RubyVM::YJIT https://github.com/ruby/ruby/blob/trunk/yjit.rb#L212 $stderr.puts "bindings_allocations: " + ("%10d" % stats[:binding_allocations]) $stderr.puts "bindings_set: " + ("%10d" % stats[:binding_set]) $stderr.puts "compilation_failure: " + ("%10d" % compilation_failure) if compilation_failure != 0 - $stderr.puts "compi (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/