ruby-changes:68244
From: Nobuyoshi <ko1@a...>
Date: Mon, 4 Oct 2021 23:03:18 +0900 (JST)
Subject: [ruby-changes:68244] c4570acc86 (master): Refactor ordering of tests
https://git.ruby-lang.org/ruby.git/commit/?id=c4570acc86 From c4570acc86837fefa542a678dfdaba73cdd1fd03 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada <nobu@r...> Date: Sat, 18 Sep 2021 16:05:26 +0900 Subject: Refactor ordering of tests * Split the sorting types into classes. * Apply the same sorting to method sorting under the parallel test. --- tool/lib/test/unit.rb | 113 +++++++++++++++++++++++++------ tool/lib/test/unit/testcase.rb | 40 +---------- tool/test/testunit/test_minitest_unit.rb | 25 ++----- tool/test/testunit/test_sorting.rb | 57 ++++++++++++++++ 4 files changed, 157 insertions(+), 78 deletions(-) diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index bfa1964963..5fbb8db1a9 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -57,6 +57,84 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit.rb#L57 class PendedError < AssertionFailedError; end + module Order + class NoSort + def initialize(seed) + end + + def sort_by_name(list) + list + end + + alias sort_by_string sort_by_name + + def group(list) + # JIT first + jit, others = list.partition {|e| /test_jit/ =~ e} + jit + others + end + end + + class Alpha < NoSort + def sort_by_name(list) + list.sort_by(&:name) + end + + def sort_by_string(list) + list.sort + end + + end + + # shuffle test suites based on CRC32 of their names + Shuffle = Struct.new(:seed, :salt) do + def initialize(seed) + self.class::CRC_TBL ||= (0..255).map {|i| + (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } + }.freeze + + salt = [seed].pack("V").unpack1("H*") + super(seed, "\n#{salt}".freeze).freeze + end + + def sort_by_name(list) + list.sort_by {|e| randomize_key(e.name)} + end + + def sort_by_string(list) + list.sort_by {|e| randomize_key(e)} + end + + def group(list) + list + end + + private + + def crc32(str, crc32 = 0xffffffff) + crc_tbl = self.class::CRC_TBL + str.each_byte do |data| + crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) + end + crc32 + end + + def randomize_key(name) + crc32(salt, crc32(name)) ^ 0xffffffff + end + end + + Types = { + random: Shuffle, + alpha: Alpha, + sorted: Alpha, + nosort: NoSort, + } + Types.default_proc = proc {|_, order| + raise "Unknown test_order: #{order.inspect}" + } + end + module RunCount # :nodoc: all @@run_count = 0 @@ -103,13 +181,13 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit.rb#L181 order = options[:test_order] if seed = options[:seed] order ||= :random - srand(seed) - else - seed = options[:seed] = srand % 100_000 - srand(seed) + elsif order == :random + seed = options[:seed] = rand(0x10000) orig_args.unshift "--seed=#{seed}" end Test::Unit::TestCase.test_order = order if order + order = Test::Unit::TestCase.test_order + @order = Test::Unit::Order::Types[order].new(seed) @help = "\n" + orig_args.map { |s| " " + (s =~ /[\s|&<>$()]/ ? s.inspect : s) @@ -139,7 +217,8 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit.rb#L217 (options[:filter] ||= []) << a end - opts.on '--test-order=random|alpha|sorted|nosort', [:random, :alpha, :sorted, :nosort] do |a| + orders = Test::Unit::Order::Types.keys + opts.on "--test-order=#{orders.join('|')}", orders do |a| options[:test_order] = a end end @@ -545,16 +624,7 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit.rb#L624 # Require needed thing for parallel running require 'timeout' - @tasks = @files.dup # Array of filenames. - - case Test::Unit::TestCase.test_order - when :random - @tasks.shuffle! - else - # JIT first - ts = @tasks.group_by{|e| /test_jit/ =~ e ? 0 : 1} - @tasks = ts[0] + ts[1] if ts.size == 2 - end + @tasks = @order.group(@order.sort_by_string(@files)) # Array of filenames. @need_quit = false @dead_workers = [] # Array of dead workers. @@ -1302,6 +1372,8 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit.rb#L1372 suites = Test::Unit::TestCase.send "#{type}_suites" return if suites.empty? + suites = @order.sort_by_name(suites) + puts puts "# Running #{type}s:" puts @@ -1356,6 +1428,12 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit.rb#L1428 filter = options[:filter] all_test_methods = suite.send "#{type}_methods" + if filter + all_test_methods.select! {|method| + filter === method || filter === "#{suite}##{method}" + } + end + all_test_methods = @order.sort_by_name(all_test_methods) leakchecker = LeakChecker.new if ENV["LEAK_CHECKER_TRACE_OBJECT_ALLOCATION"] @@ -1363,10 +1441,7 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit.rb#L1441 trace = true end - assertions = all_test_methods.filter_map { |method| - if filter - next unless filter === method || filter === "#{suite}##{method}" - end + assertions = all_test_methods.map { |method| inst = suite.new method inst._assertions = 0 diff --git a/tool/lib/test/unit/testcase.rb b/tool/lib/test/unit/testcase.rb index 241421d6d9..4cc1aae3e4 100644 --- a/tool/lib/test/unit/testcase.rb +++ b/tool/lib/test/unit/testcase.rb @@ -159,7 +159,6 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit/testcase.rb#L159 start_time = Time.now result = "" - srand(runner.options[:seed]) begin @passed = nil @@ -267,46 +266,11 @@ module Test https://github.com/ruby/ruby/blob/trunk/tool/lib/test/unit/testcase.rb#L266 end def self.test_suites # :nodoc: - suites = @@test_suites.keys - - case self.test_order - when :random - # shuffle test suites based on CRC32 of their names - salt = "\n" + rand(1 << 32).to_s - crc_tbl = (0..255).map do |i| - (0..7).inject(i) {|c,| (c & 1 == 1) ? (0xEDB88320 ^ (c >> 1)) : (c >> 1) } - end - suites = suites.sort_by do |suite| - crc32 = 0xffffffff - "#{suite.name}#{salt}".each_byte do |data| - crc32 = crc_tbl[(crc32 ^ data) & 0xff] ^ (crc32 >> 8) - end - crc32 ^ 0xffffffff - end - when :nosort - suites - else - suites.sort_by { |ts| ts.name.to_s } - end + @@test_suites.keys end def self.test_methods # :nodoc: - methods = public_instance_methods(true).grep(/^test/).map { |m| m.to_s } - - case self.test_order - when :parallel - max = methods.size - ParallelEach.new methods.sort.sort_by { rand max } - when :random then - max = methods.size - methods.sort.sort_by { rand max } - when :alpha, :sorted then - methods.sort - when :nosort - methods - else - raise "Unknown test_order: #{self.test_order.inspect}" - end + public_instance_methods(true).grep(/^test/) end ## diff --git a/tool/test/testunit/test_minitest_unit.rb b/tool/test/testunit/test_minitest_unit.rb index 68e88f574e..5941392fa0 100644 --- a/tool/test/testunit/test_minitest_unit.rb +++ b/tool/test/testunit/test_minitest_unit.rb @@ -238,7 +238,7 @@ class TestMiniTestRunner < MetaMetaMetaTestCase https://github.com/ruby/ruby/blob/trunk/tool/test/testunit/test_minitest_unit.rb#L238 tc = Class.new(Test::Unit::TestCase) assert_equal 2, Test::Unit::TestCase.test_suites.size - assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites + assert_equal [tc, Test::Unit::TestCase], Test::Unit::TestCase.test_suites.sort_by {|ts| ts.name.to_s} end def assert_filtering name, expected, a = false @@ -1331,34 +1331,17 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/tool/test/testunit/test_minitest_unit.rb#L1331 end end - def test_test_methods_random + def test_test_methods @assertion_count = 0 sample_test_case = Class.new Test::Unit::TestCase do - def self.test_order; :random; end def test_test1; assert "does not matter" end def test_test2; assert "does not matter" end def test_test3; assert "does not matter" end - @test_order = [1, 0, 2] - def self.rand(n) @test_order.shift; end end - expected = %w(test_test2 test_test1 test_test3) - assert_equal expected, sample_test_case.test_methods - end - - def test_test_methods_sorted - @assertion_count = 0 - - sample_test_case = Class.new Test::Unit::TestCase do - def self.test_order; :sorted end - def test_test3; assert "does not matter" end - def test_test2; assert "does not matter" end - def test_test1; assert "does not matter" end - end - - expected = %w(test_test1 test_test2 test_test3) - assert_equal expected, sample_test_case.test_methods + expected = %i(test_test1 test_test2 test_test3) + assert_equal expected, sample_test_case.test_methods.sort end def assert_triggered expected, klass = Test::Unit::AssertionFailedError diff --git a/tool/test/testunit/test_s (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/