ruby-changes:51723
From: k0kubun <ko1@a...>
Date: Wed, 11 Jul 2018 00:01:42 +0900 (JST)
Subject: [ruby-changes:51723] k0kubun:r63933 (trunk): benchmark_driver/runner: add runners for metrics
k0kubun 2018-07-11 00:01:27 +0900 (Wed, 11 Jul 2018) New Revision: 63933 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=63933 Log: benchmark_driver/runner: add runners for metrics supported by legacy benchmark/driver.rb. benchmark/README.md: document them common.mk: update benchmark_driver to correct 0.0 output and to fix spacing format of `-o simple` and `-o markdown`. Added files: trunk/benchmark/lib/benchmark_driver/runner/cstime.rb trunk/benchmark/lib/benchmark_driver/runner/cutime.rb trunk/benchmark/lib/benchmark_driver/runner/stime.rb trunk/benchmark/lib/benchmark_driver/runner/total.rb trunk/benchmark/lib/benchmark_driver/runner/utime.rb Modified files: trunk/benchmark/README.md trunk/common.mk Index: benchmark/README.md =================================================================== --- benchmark/README.md (revision 63932) +++ benchmark/README.md (revision 63933) @@ -16,6 +16,9 @@ benchmark-driver benchmark/*.yml -e /pat https://github.com/ruby/ruby/blob/trunk/benchmark/README.md#L16 # Or compare Ruby versions managed by rbenv benchmark-driver benchmark/*.yml --rbenv '2.5.1;2.6.0-preview2 --jit' + +# You can collect many metrics in many ways +benchmark-driver benchmark/*.yml --runner memory --output markdown ``` See also: @@ -62,6 +65,6 @@ make benchmark ARGS=../benchmark/erb_ren https://github.com/ruby/ruby/blob/trunk/benchmark/README.md#L65 make benchmark OPTS="--help" # With `make benchmark`, some special runner plugins are available: -# -r peak, -r size +# -r peak, -r size, -r total, -r utime, -r stime, -r cutime, -r cstime make benchmark ITEM=vm2_bigarray OPTS="-r peak" ``` Index: benchmark/lib/benchmark_driver/runner/cutime.rb =================================================================== --- benchmark/lib/benchmark_driver/runner/cutime.rb (nonexistent) +++ benchmark/lib/benchmark_driver/runner/cutime.rb (revision 63933) @@ -0,0 +1,22 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/cutime.rb#L1 +require 'benchmark_driver/runner/total' + +class BenchmarkDriver::Runner::Cutime < BenchmarkDriver::Runner::Total + METRIC = BenchmarkDriver::Metric.new(name: 'cutime', unit: 's', larger_better: false) + + # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job" + Job = Class.new(BenchmarkDriver::DefaultJob) + # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse` + JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]) + + private + + # Overriding BenchmarkDriver::Runner::Total#metric + def metric + METRIC + end + + # Overriding BenchmarkDriver::Runner::Total#target + def target + :cutime + end +end Index: benchmark/lib/benchmark_driver/runner/stime.rb =================================================================== --- benchmark/lib/benchmark_driver/runner/stime.rb (nonexistent) +++ benchmark/lib/benchmark_driver/runner/stime.rb (revision 63933) @@ -0,0 +1,22 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/stime.rb#L1 +require 'benchmark_driver/runner/total' + +class BenchmarkDriver::Runner::Stime < BenchmarkDriver::Runner::Total + METRIC = BenchmarkDriver::Metric.new(name: 'stime', unit: 's', larger_better: false) + + # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job" + Job = Class.new(BenchmarkDriver::DefaultJob) + # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse` + JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]) + + private + + # Overriding BenchmarkDriver::Runner::Total#metric + def metric + METRIC + end + + # Overriding BenchmarkDriver::Runner::Total#target + def target + :stime + end +end Index: benchmark/lib/benchmark_driver/runner/total.rb =================================================================== --- benchmark/lib/benchmark_driver/runner/total.rb (nonexistent) +++ benchmark/lib/benchmark_driver/runner/total.rb (revision 63933) @@ -0,0 +1,137 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/total.rb#L1 +require 'benchmark_driver/struct' +require 'benchmark_driver/metric' +require 'benchmark_driver/default_job' +require 'benchmark_driver/default_job_parser' +require 'tempfile' + +class BenchmarkDriver::Runner::Total + METRIC = BenchmarkDriver::Metric.new(name: 'Total time', unit: 's', larger_better: false) + + # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job" + Job = Class.new(BenchmarkDriver::DefaultJob) + # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse` + JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]) + + # @param [BenchmarkDriver::Config::RunnerConfig] config + # @param [BenchmarkDriver::Output] output + # @param [BenchmarkDriver::Context] contexts + def initialize(config:, output:, contexts:) + @config = config + @output = output + @contexts = contexts + end + + # This method is dynamically called by `BenchmarkDriver::JobRunner.run` + # @param [Array<BenchmarkDriver::Runner::Total::Job>] jobs + def run(jobs) + if jobs.any? { |job| job.loop_count.nil? } + raise 'missing loop_count is not supported in Ruby repository' + end + + @output.with_benchmark do + jobs.each do |job| + @output.with_job(name: job.name) do + job.runnable_contexts(@contexts).each do |context| + duration = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do + run_benchmark(job, context: context) + end + @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do + @output.report(values: { metric => duration }, duration: duration, loop_count: job.loop_count) + end + end + end + end + end + end + + private + + # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil + # @param [BenchmarkDriver::Context] context + # @return [BenchmarkDriver::Metrics] + def run_benchmark(job, context:) + benchmark = BenchmarkScript.new( + preludes: [context.prelude, job.prelude], + script: job.script, + teardown: job.teardown, + loop_count: job.loop_count, + ) + + Tempfile.open(['benchmark_driver-', '.rb']) do |f| + with_script(benchmark.render(result: f.path, target: target)) do |path| + IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2 + if $?.success? + Float(f.read) + else + BenchmarkDriver::Result::ERROR + end + end + end + end + + # This method is overridden by some subclasses + def metric + METRIC + end + + # This method is overridden by some subclasses + def target + :total + end + + def with_script(script) + if @config.verbose >= 2 + sep = '-' * 30 + $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n" + end + + Tempfile.open(['benchmark_driver-', '.rb']) do |f| + f.puts script + f.close + return yield(f.path) + end + end + + # @param [String] prelude + # @param [String] script + # @param [String] teardown + # @param [Integer] loop_count + BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do + # @param [String] result - A file to write result + def render(result:, target:) + prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n") + <<-RUBY +#{prelude} + +require 'benchmark' +__bmdv_result = Benchmark.measure { + #{while_loop(script, loop_count)} +} + +#{teardown} + +File.write(#{result.dump}, __bmdv_result.#{target}) + RUBY + end + + private + + def while_loop(content, times) + if !times.is_a?(Integer) || times <= 0 + raise ArgumentError.new("Unexpected times: #{times.inspect}") + elsif times == 1 + return content + end + + # TODO: execute in batch + <<-RUBY +__bmdv_i = 0 +while __bmdv_i < #{times} + #{content} + __bmdv_i += 1 +end + RUBY + end + end + private_constant :BenchmarkScript +end Index: benchmark/lib/benchmark_driver/runner/utime.rb =================================================================== --- benchmark/lib/benchmark_driver/runner/utime.rb (nonexistent) +++ benchmark/lib/benchmark_driver/runner/utime.rb (revision 63933) @@ -0,0 +1,22 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/utime.rb#L1 +require 'benchmark_driver/runner/total' + +class BenchmarkDriver::Runner::Utime < BenchmarkDriver::Runner::Total + METRIC = BenchmarkDriver::Metric.new(name: 'utime', unit: 's', larger_better: false) + + # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job" + Job = Class.new(BenchmarkDriver::DefaultJob) + # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse` + JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]) + + private + + # Overriding BenchmarkDriver::Runner::Total#metric + def metric + METRIC + end + + # Overriding BenchmarkDriver::Runner::Total#target + def target + :utime + end +end Index: benchmark/lib/benchmark_driver/runner/cstime.rb =================================================================== --- benchmark/lib/benchmark_driver/runner/cstime.rb (nonexistent) +++ benchmark/lib/benchmark_driver/runner/cstime.rb (revision 63933) @@ -0,0 +1,22 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/cstime.rb#L1 +require 'benchmark_driver/runner/total' + +class BenchmarkDriver::Runner::Cstime < BenchmarkDriver::Runner::Total + METRIC = BenchmarkDriver::Metric.new(name: 'cstime', unit: 's', larger_better: false) + + # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job" + Job = Class.new(BenchmarkDriver::DefaultJob) + # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse` + JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]) + + private + + # Overriding BenchmarkDriver::Runner::Total#metric + def metric + METRIC + end + + # Overriding BenchmarkDriver::Runner::Total#target + def target + :cstime + end +end Index: common.mk =================================================================== --- common.mk (revision 63932) +++ common.mk (revision 63933) @@ -42,7 +42,7 @@ GEM_PATH = https://github.com/ruby/ruby/blob/trunk/common.mk#L42 GEM_VENDOR = BENCHMARK_DRIVER_GIT_URL = https://github.com/benchmark-driver/benchmark-driver -BENCHMARK_DRIVER_GIT_REF = v0.14.3 +BENCHMARK_DRIVER_GIT_REF = v0.14.5 SIMPLECOV_GIT_URL = git://github.com/colszowka/simplecov.git SIMPLECOV_GIT_REF = v0.15.0 SIMPLECOV_HTML_GIT_URL = git://github.com/colszowka/simplecov-html.git -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/