https://git.ruby-lang.org/ruby.git/commit/?id=1800f3fa5c

From 1800f3fa5c29515113ecdcc5695e8a96f462b74c Mon Sep 17 00:00:00 2001
From: Nobuhiro IMAI <nov@y...>
Date: Thu, 19 Nov 2020 23:19:08 +0900
Subject: Ripper.{lex,tokenize} return full tokens even if syntax error

yet another implements [Feature #17276]

diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 052aa52..636a87f 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -31,6 +31,10 @@ class Ripper https://github.com/ruby/ruby/blob/trunk/ext/ripper/lib/ripper/lexer.rb#L31
       raise SyntaxError, r.errors.map(&:message).join(' ;')
     end
 
+    until (tokens = r.tokenize).empty?
+      ret.concat(tokens)
+    end
+
     ret
   end
 
@@ -65,6 +69,10 @@ class Ripper https://github.com/ruby/ruby/blob/trunk/ext/ripper/lib/ripper/lexer.rb#L69
       raise SyntaxError, r.errors.map(&:message).join(' ;')
     end
 
+    until (tokens = r.lex).empty?
+      ret.concat(tokens)
+    end
+
     ret
   end
 
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 542db2f..3d546c2 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -150,4 +150,12 @@ class TestRipper::Lexer < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ripper/test_lexer.rb#L150
     assert_raise(SyntaxError) { Ripper.tokenize('def req(true) end', raise_errors: true) }
     assert_raise(SyntaxError) { Ripper.tokenize('def req(true) end', raise_errors: true) }
   end
+
+  def test_tokenize_with_syntax_error
+    assert_equal "end", Ripper.tokenize("def req(true) end").last
+  end
+
+  def test_lex_with_syntax_error
+    assert_equal [[1, 14], :on_kw, "end", state(:EXPR_END)], Ripper.lex("def req(true) end").last
+  end
 end
-- 
cgit v0.10.2


--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/