[前][次][番号順一覧][スレッド一覧]

ruby-changes:57910

From: Takashi <ko1@a...>
Date: Thu, 26 Sep 2019 16:35:34 +0900 (JST)
Subject: [ruby-changes:57910] 4a4c502825 (master): Add special runner to benchmark mjit_exec

https://git.ruby-lang.org/ruby.git/commit/?id=4a4c502825

From 4a4c5028258e53f3395af29655a66bcef796fd73 Mon Sep 17 00:00:00 2001
From: Takashi Kokubun <takashikkbn@g...>
Date: Thu, 26 Sep 2019 12:57:43 +0900
Subject: Add special runner to benchmark mjit_exec

I wanted to dynamically generate benchmark cases to test various number
of methods. Thus I added a dedicated runner of benchmark-driver.

diff --git a/benchmark/lib/benchmark_driver/runner/mjit_exec.rb b/benchmark/lib/benchmark_driver/runner/mjit_exec.rb
new file mode 100644
index 0000000..7477fa1
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/mjit_exec.rb
@@ -0,0 +1,237 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/mjit_exec.rb#L1
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'erb'
+
+# A special runner dedicated for measuring mjit_exec overhead.
+class BenchmarkDriver::Runner::MjitExec
+  METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s')
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = ::BenchmarkDriver::Struct.new(
+    :name,        # @param [String] name - This is mandatory for all runner
+    :metrics,     # @param [Array<BenchmarkDriver::Metric>]
+    :num_methods, # @param [Integer] num_methods - The number of methods to be defined
+    :loop_count,  # @param [Integer] loop_count
+    :from_jit,    # @param [TrueClass,FalseClass] from_jit - Whether the mjit_exec() is from JIT or not
+    :to_jit,      # @param [TrueClass,FalseClass] to_jit - Whether the mjit_exec() is to JIT or not
+  )
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  class << JobParser = Module.new
+    # @param [Array,String] num_methods
+    # @param [Integer] loop_count
+    # @param [TrueClass,FalseClass] from_jit
+    # @param [TrueClass,FalseClass] to_jit
+    def parse(num_methods:, loop_count:, from_jit:, to_jit:)
+      if num_methods.is_a?(String)
+        num_methods = eval(num_methods)
+      end
+
+      num_methods.map do |num|
+        if num_methods.size > 1
+          suffix = "[#{'%4d' % num}]"
+        else
+          suffix = "_#{num}"
+        end
+        Job.new(
+          name: "mjit_exec_#{from_jit ? 'JT' : 'VM'}2#{to_jit ? 'JT' : 'VM'}#{suffix}",
+          metrics: [METRIC],
+          num_methods: num,
+          loop_count: loop_count,
+          from_jit: from_jit,
+          to_jit: to_jit,
+        )
+      end
+    end
+  end
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output] output
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
+    @config = config
+    @output = output
+    @contexts = contexts
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Runner::Peak::Job>] jobs
+  def run(jobs)
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          @contexts.each do |context|
+            result = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: true, rest_on_average: :average) do
+              run_benchmark(job, context: context)
+            end
+            value, duration = result.value
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
+              @output.report(values: { METRIC => value }, duration: duration, loop_count: job.loop_count)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Context] context
+  # @return [BenchmarkDriver::Metrics]
+  def run_benchmark(job, context:)
+    if job.from_jit
+      if job.to_jit
+        benchmark = BenchmarkJT2JT.new(num_methods: job.num_methods, loop_count: job.loop_count)
+      else
+        raise NotImplementedError, "JT2VM is not implemented yet"
+      end
+    else
+      if job.to_jit
+        benchmark = BenchmarkVM2JT.new(num_methods: job.num_methods, loop_count: job.loop_count)
+      else
+        benchmark = BenchmarkVM2VM.new(num_methods: job.num_methods, loop_count: job.loop_count)
+      end
+    end
+
+    duration = Tempfile.open(['benchmark_driver-result', '.txt']) do |f|
+      with_script(benchmark.render(result: f.path)) do |path|
+        opt = []
+        if context.executable.command.any? { |c| c.start_with?('--jit') }
+          opt << '--jit-min-calls=2'
+        end
+        IO.popen([*context.executable.command, '--disable-gems', *opt, path], &:read)
+        if $?.success?
+          Float(f.read)
+        else
+          BenchmarkDriver::Result::ERROR
+        end
+      end
+    end
+
+    [job.loop_count.to_f / duration, duration]
+  end
+
+  def with_script(script)
+    if @config.verbose >= 2
+      sep = '-' * 30
+      $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
+    end
+
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      f.puts script
+      f.close
+      return yield(f.path)
+    end
+  end
+
+  # @param [Integer] num_methods
+  # @param [Integer] loop_count
+  BenchmarkVM2VM = ::BenchmarkDriver::Struct.new(:num_methods, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(result:)
+      ERB.new(<<~EOS, trim_mode: '%').result(binding)
+        % num_methods.times do |i|
+        def a<%= i %>
+          nil
+        end
+        % end
+        RubyVM::MJIT.pause if RubyVM::MJIT.enabled?
+
+        def vm
+          t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          i = 0
+          while i < 10000
+        % (loop_count / 10000).times do |i|
+            a<%= i % num_methods %>
+        % end
+            i += 1
+          end
+        % (loop_count % 10000).times do |i|
+          a<%= i % num_methods %>
+        % end
+          Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
+        end
+
+        vm # warmup call cache
+        File.write(<%= result.dump %>, vm)
+      EOS
+    end
+  end
+  private_constant :BenchmarkVM2VM
+
+  # @param [Integer] num_methods
+  # @param [Integer] loop_count
+  BenchmarkVM2JT = ::BenchmarkDriver::Struct.new(:num_methods, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(result:)
+      ERB.new(<<~EOS, trim_mode: '%').result(binding)
+        % num_methods.times do |i|
+        def a<%= i %>
+          nil
+        end
+        a<%= i %>
+        a<%= i %> # --jit-min-calls=2
+        % end
+        RubyVM::MJIT.pause if RubyVM::MJIT.enabled?
+
+        def vm
+          t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          i = 0
+          while i < 10000
+        % (loop_count / 10000).times do |i|
+            a<%= i % num_methods %>
+        % end
+            i += 1
+          end
+        % (loop_count % 10000).times do |i|
+          a<%= i % num_methods %>
+        % end
+          Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
+        end
+
+        vm # warmup call cache
+        File.write(<%= result.dump %>, vm)
+      EOS
+    end
+  end
+  private_constant :BenchmarkVM2JT
+
+  # @param [Integer] num_methods
+  # @param [Integer] loop_count
+  BenchmarkJT2JT = ::BenchmarkDriver::Struct.new(:num_methods, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(result:)
+      ERB.new(<<~EOS, trim_mode: '%').result(binding)
+        % num_methods.times do |i|
+        def a<%= i %>
+          nil
+        end
+        % end
+
+        # You may need to:
+        #   * Increase `JIT_ISEQ_SIZE_THRESHOLD` to 10000000 in mjit.h
+        #   * Always return false in `inlinable_iseq_p()` of mjit_compile.c
+        def jit
+          t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          i = 0
+          while i < 10000
+        % (loop_count / 10000).times do |i|
+            a<%= i % num_methods %>
+        % end
+            i += 1
+          end
+        % (loop_count % 10000).times do |i|
+          a<%= i % num_methods %>
+        % end
+          Process.clock_gettime(Process::CLOCK_MONOTONIC) - t
+        end
+
+        jit
+        jit
+        RubyVM::MJIT.pause if RubyVM::MJIT.enabled?
+        File.write(<%= result.dump %>, jit)
+      EOS
+    end
+  end
+  private_constant :BenchmarkJT2JT
+end
diff --git a/benchmark/mjit_exec.yml b/benchmark/mjit_exec.yml
deleted file mode 100644
index d262188..0000000
--- a/benchmark/mjit_exec.yml
+++ /dev/null
@@ -1,46 +0,0 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/mjit_exec.rb#L0
-prelude: |
-  # to be used with: --disable-gems --jit-min-calls=2
-  def compile(call)
-    eval(<<~EOS)
-      #{call}; #{call}
-      if RubyVM::MJIT.enabled?
-        RubyVM::MJIT.pause(wait: true)
-      end
-    EOS
-  end
-benchmark:
-  - name: "mjit_exec_iseq_vme_jit    "
-    prelude: |
-      def jit() end
-      compile('jit')
-    script: jit
-  - name: mjit_exec_iseq_vme_jit_jit
-    prelude: |
-      def jit2() end
-      def jit() jit2() end
-      compile('jit')
-    script: jit
-  - name: mjit_exec_iseq_vme_jit_vme
-    prelude: |
-      def jit2() rescue; end
-      def jit() jit2() end
-      compile('jit')
-    script: jit
-  - name: "mjit_exec_send_vme_jit    "
-    prelude: |
-      def jit() end
-      compile('send(:jit)')
-    script: send(:jit)
-  - name: mjit_exec_send_vme_jit_jit
-    prelude: |
-      def jit2() end
-      def jit() send(:jit2) end
-      compile('send(:jit)')
-    script: send(:jit)
-  - name: mjit_exec_send_vme_jit_vme
-    prelude: |
-      def jit2() rescue; end
-      def jit() send(:jit2) end
-      compile('send(:jit)')
-    script: send(:jit)
-loop_count: 30000000
diff --git a/benchmark/mjit_exec_jt2jt.yml b/benchmark/mjit_exec_jt2jt.yml
new file mode 100644
index 0000000..78c31b3
--- /dev/null
+++ b/benchmark/mjit_exec_jt2jt (... truncated)

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/

[前][次][番号順一覧][スレッド一覧]