ruby-changes:73808
From: tompng <ko1@a...>
Date: Sat, 1 Oct 2022 04:17:27 +0900 (JST)
Subject: [ruby-changes:73808] 641310ce37 (master): [ruby/irb] Fix ripper_lex_without_warning duplicated heredoc token
https://git.ruby-lang.org/ruby.git/commit/?id=641310ce37 From 641310ce371bd220a5ddd5e54a86979f4055608b Mon Sep 17 00:00:00 2001 From: tompng <tomoyapenguin@g...> Date: Tue, 27 Sep 2022 13:14:42 +0900 Subject: [ruby/irb] Fix ripper_lex_without_warning duplicated heredoc token https://github.com/ruby/irb/commit/45b539af39 --- lib/irb/ruby-lex.rb | 12 ++++-------- test/irb/test_ruby_lex.rb | 13 +++++++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index 29862f5507..766e797fca 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -152,17 +152,13 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L152 lexer = Ripper::Lexer.new(inner_code, '-', line_no) if lexer.respond_to?(:scan) # Ruby 2.7+ tokens = [] - pos_to_index = {} lexer.scan.each do |t| next if t.pos.first == 0 - if pos_to_index.has_key?(t.pos) - index = pos_to_index[t.pos] - found_tk = tokens[index] - if ERROR_TOKENS.include?(found_tk.event) && !ERROR_TOKENS.include?(t.event) - tokens[index] = t - end + prev_tk = tokens.last + position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize + if position_overlapped + tokens[-1] = t if ERROR_TOKENS.include?(prev_tk.event) && !ERROR_TOKENS.include?(t.event) else - pos_to_index[t.pos] = tokens.size tokens << t end end diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb index 47435d675e..2c94a36a5d 100644 --- a/test/irb/test_ruby_lex.rb +++ b/test/irb/test_ruby_lex.rb @@ -619,5 +619,18 @@ module TestIRB https://github.com/ruby/ruby/blob/trunk/test/irb/test_ruby_lex.rb#L619 pos_to_index[t.pos] = i } end + + def test_unterminated_code + if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0') + pend 'This test needs Ripper::Lexer#scan to take broken tokens' + end + + ['do', '<<A'].each do |code| + tokens = RubyLex.ripper_lex_without_warning(code) + assert_equal(code, tokens.map(&:tok).join, "Cannot reconstruct code from tokens") + error_tokens = tokens.map(&:event).grep(/error/) + assert_empty(error_tokens, 'Error tokens must be ignored if there is corresponding non-error token') + end + end end end -- cgit v1.2.1 -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/