https://git.ruby-lang.org/ruby.git/commit/?id=b0e2b7a5ff

From b0e2b7a5ff0df14b3c8062c31ebb526a73a03763 Mon Sep 17 00:00:00 2001
From: Nobuyoshi Nakada <nobu@r...>
Date: Thu, 30 May 2019 21:49:08 +0900
Subject: Include stack elements left after errors


diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index d1d2575..696471a 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -103,7 +103,7 @@ class Ripper https://github.com/ruby/ruby/blob/trunk/ext/ripper/lib/ripper/lexer.rb#L103
 
     # parse the code and returns elements including errors.
     def scan
-      (parse() + errors).sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
+      (parse() + errors + @stack.flatten).uniq.sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
     end
 
     def parse
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 30b85ef..4b70915 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -113,4 +113,26 @@ class TestRipper::Lexer < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ripper/test_lexer.rb#L113
     assert_equal [[1,0],:on_cvar,"@@1",state(:EXPR_END)], Ripper.lex("@@1").last
     assert_equal [[1,1],:on_cvar,"@@1",state(:EXPR_ENDFN)], Ripper.lex(":@@1").last
   end
+
+  def test_token_aftr_error_heredoc
+    code = "<<A.upcase\n"
+    result = Ripper::Lexer.new(code).scan
+    message = proc {result.pretty_inspect}
+    expected = [
+      [[1, 0], :on_heredoc_beg, "<<A", state(:EXPR_BEG)],
+      [[1, 3], :on_period, ".", state(:EXPR_DOT)],
+      [[1, 4], :on_ident, "upcase", state(:EXPR_ARG)],
+      [[1, 10], :on_nl, "\n", state(:EXPR_BEG)],
+      [[1, 11], :compile_error, "", state(:EXPR_BEG), "can't find string \"A\" anywhere before EOF"],
+    ]
+    pos = 0
+    expected.each_with_index do |ex, i|
+      s = result[i]
+      assert_equal ex, s.to_a, message
+      assert_equal pos, s.pos[1], message
+      pos += s.tok.bytesize
+    end
+    assert_equal pos, code.bytesize
+    assert_equal expected.size, result.size
+  end
 end
-- 
cgit v0.10.2


--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/