ruby-changes:51712
From: k0kubun <ko1@a...>
Date: Tue, 10 Jul 2018 21:14:10 +0900 (JST)
Subject: [ruby-changes:51712] k0kubun:r63924 (trunk): benchmark: resurrect peak / size metrics
k0kubun 2018-07-10 21:14:04 +0900 (Tue, 10 Jul 2018) New Revision: 63924 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=63924 Log: benchmark: resurrect peak / size metrics by adding runner plugins for them. benchmark/lib/benchmark_driver/runner/peak.rb: added peak runner plugin benchmark/lib/benchmark_driver/runner/size.rb: added size runner plugin common.mk: allow using them benchmark/memory_wrapper.rb: deleted in favor of those runner plugins benchmark/README.md: document them Added directories: trunk/benchmark/lib/ trunk/benchmark/lib/benchmark_driver/ trunk/benchmark/lib/benchmark_driver/runner/ Added files: trunk/benchmark/lib/benchmark_driver/runner/peak.rb trunk/benchmark/lib/benchmark_driver/runner/size.rb Removed files: trunk/benchmark/memory_wrapper.rb Modified files: trunk/benchmark/README.md trunk/common.mk Index: benchmark/memory_wrapper.rb =================================================================== --- benchmark/memory_wrapper.rb (revision 63923) +++ benchmark/memory_wrapper.rb (nonexistent) @@ -1,16 +0,0 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/memory_wrapper.rb#L0 - -write_file, target, script_file = ARGV - -load(script_file) -require_relative '../test/lib/memory_status' -open(write_file, 'wb'){|f| - ms = Memory::Status.new - case target.to_sym - when :peak - key = ms.respond_to?(:hwm) ? :hwm : :peak - when :size - key = ms.respond_to?(:rss) ? :rss : :size - end - - f.puts ms[key] -} Property changes on: benchmark/memory_wrapper.rb ___________________________________________________________________ Deleted: svn:eol-style ## -1 +0,0 ## -LF \ No newline at end of property Index: common.mk =================================================================== --- common.mk (revision 63923) +++ common.mk (revision 63924) @@ -42,7 +42,7 @@ GEM_PATH = https://github.com/ruby/ruby/blob/trunk/common.mk#L42 GEM_VENDOR = BENCHMARK_DRIVER_GIT_URL = https://github.com/benchmark-driver/benchmark-driver -BENCHMARK_DRIVER_GIT_REF = v0.14.0 +BENCHMARK_DRIVER_GIT_REF = v0.14.3 SIMPLECOV_GIT_URL = git://github.com/colszowka/simplecov.git SIMPLECOV_GIT_REF = v0.15.0 SIMPLECOV_HTML_GIT_URL = git://github.com/colszowka/simplecov-html.git @@ -1122,7 +1122,7 @@ OPTS = https://github.com/ruby/ruby/blob/trunk/common.mk#L1122 # $ make benchmark COMPARE_RUBY="ruby-trunk" OPTS="-e ruby-2.2.2" # This command compares trunk and built-ruby and 2.2.2 benchmark: miniruby$(EXEEXT) update-benchmark-driver PHONY - $(BASERUBY) -rrubygems $(srcdir)/benchmark/benchmark-driver/exe/benchmark-driver \ + $(BASERUBY) -rrubygems -I$(srcdir)/benchmark/lib $(srcdir)/benchmark/benchmark-driver/exe/benchmark-driver \ --executables="compare-ruby::$(COMPARE_RUBY) -I$(EXTOUT)/common --disable-gem" \ --executables="built-ruby::$(MINIRUBY) -r$(srcdir)/prelude --disable-gem" \ $(ARGS) $(OPTS) Index: benchmark/README.md =================================================================== --- benchmark/README.md (revision 63923) +++ benchmark/README.md (revision 63924) @@ -43,4 +43,8 @@ make benchmark ARGS=../benchmark/erb_ren https://github.com/ruby/ruby/blob/trunk/benchmark/README.md#L43 # You can specify any option via $OPTS make benchmark OPTS="--help" + +# With `make benchmark`, some special runner plugins are available: +# -r peak, -r size +make benchmark ITEM=vm2_bigarray OPTS="-r peak" ``` Index: benchmark/lib/benchmark_driver/runner/size.rb =================================================================== --- benchmark/lib/benchmark_driver/runner/size.rb (nonexistent) +++ benchmark/lib/benchmark_driver/runner/size.rb (revision 63924) @@ -0,0 +1,20 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/size.rb#L1 +require 'benchmark_driver/runner/peak' + +# Actually the same as BenchmarkDriver::Runner::Memory +class BenchmarkDriver::Runner::Size < BenchmarkDriver::Runner::Peak + METRIC = BenchmarkDriver::Metric.new( + name: 'Max resident set size', unit: 'bytes', larger_better: false, worse_word: 'larger', + ) + + # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job" + Job = Class.new(BenchmarkDriver::DefaultJob) + # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse` + JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]) + + private + + # Overriding BenchmarkDriver::Runner::Peak#target + def target + 'size' + end +end Index: benchmark/lib/benchmark_driver/runner/peak.rb =================================================================== --- benchmark/lib/benchmark_driver/runner/peak.rb (nonexistent) +++ benchmark/lib/benchmark_driver/runner/peak.rb (revision 63924) @@ -0,0 +1,146 @@ https://github.com/ruby/ruby/blob/trunk/benchmark/lib/benchmark_driver/runner/peak.rb#L1 +require 'benchmark_driver/struct' +require 'benchmark_driver/metric' +require 'benchmark_driver/default_job' +require 'benchmark_driver/default_job_parser' +require 'tempfile' + +class BenchmarkDriver::Runner::Peak + METRIC = BenchmarkDriver::Metric.new( + name: 'Peak memory usage', unit: 'bytes', larger_better: false, worse_word: 'larger', + ) + + # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job" + Job = Class.new(BenchmarkDriver::DefaultJob) + # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse` + JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]) + + # @param [BenchmarkDriver::Config::RunnerConfig] config + # @param [BenchmarkDriver::Output] output + # @param [BenchmarkDriver::Context] contexts + def initialize(config:, output:, contexts:) + @config = config + @output = output + @contexts = contexts + end + + # This method is dynamically called by `BenchmarkDriver::JobRunner.run` + # @param [Array<BenchmarkDriver::Runner::Peak::Job>] jobs + def run(jobs) + if jobs.any? { |job| job.loop_count.nil? } + jobs = jobs.map do |job| + job.loop_count ? job : Job.new(job.to_h.merge(loop_count: 1)) + end + end + + @output.with_benchmark do + jobs.each do |job| + @output.with_job(name: job.name) do + job.runnable_contexts(@contexts).each do |context| + value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do + run_benchmark(job, context: context) + end + @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do + @output.report(values: { METRIC => value }, loop_count: job.loop_count) + end + end + end + end + end + end + + private + + # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil + # @param [BenchmarkDriver::Context] context + # @return [BenchmarkDriver::Metrics] + def run_benchmark(job, context:) + benchmark = BenchmarkScript.new( + preludes: [context.prelude, job.prelude], + script: job.script, + teardown: job.teardown, + loop_count: job.loop_count, + ) + + memory_status = File.expand_path('../../../../test/lib/memory_status', __dir__) + Tempfile.open(['benchmark_driver-', '.rb']) do |f| + with_script(benchmark.render) do |path| + output = IO.popen([*context.executable.command, path, f.path, target, memory_status], &:read) + if $?.success? + Integer(f.read) + else + $stdout.print(output) + BenchmarkDriver::Result::ERROR + end + end + end + end + + # Overridden by BenchmarkDriver::Runner::Size + def target + 'peak' + end + + def with_script(script) + if @config.verbose >= 2 + sep = '-' * 30 + $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n" + end + + Tempfile.open(['benchmark_driver-', '.rb']) do |f| + f.puts script + f.close + return yield(f.path) + end + end + + # @param [String] prelude + # @param [String] script + # @param [String] teardown + # @param [Integer] loop_count + BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do + def render + prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n") + <<-RUBY +#{prelude} +#{while_loop(script, loop_count)} +#{teardown} + +result_file, target, memory_status = ARGV +require_relative memory_status + +ms = Memory::Status.new +case target.to_sym +when :peak + key = ms.respond_to?(:hwm) ? :hwm : :peak +when :size + key = ms.respond_to?(:rss) ? :rss : :size +else + raise('unexpected target: ' + target) +end + +File.write(result_file, ms[key]) + RUBY + end + + private + + def while_loop(content, times) + if !times.is_a?(Integer) || times <= 0 + raise ArgumentError.new("Unexpected times: #{times.inspect}") + end + + if times > 1 + <<-RUBY +__bmdv_i = 0 +while __bmdv_i < #{times} + #{content} + __bmdv_i += 1 +end + RUBY + else + content + end + end + end + private_constant :BenchmarkScript +end -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/