https://git.ruby-lang.org/ruby.git/commit/?id=92ecf58b1e

From 92ecf58b1e8ed15f5ec2bf69e3871de4343f71e0 Mon Sep 17 00:00:00 2001
From: Nobuyoshi Nakada <nobu@r...>
Date: Thu, 30 May 2019 22:03:33 +0900
Subject: parse.y: adjust here-doc error token

* parse.y (here_document): adjust token to the here-doc identifier
  in compile_error when a here-document misses the closing
  identifier.

diff --git a/parse.y b/parse.y
index 5b33ef4..58b8714 100644
--- a/parse.y
+++ b/parse.y
@@ -7005,9 +7005,9 @@ heredoc_restore(struct parser_params *p, rb_strterm_heredoc_t *here) https://github.com/ruby/ruby/blob/trunk/parse.y#L7005
     p->lex.pbeg = RSTRING_PTR(line);
     p->lex.pend = p->lex.pbeg + RSTRING_LEN(line);
     p->lex.pcur = p->lex.pbeg + here->offset + here->length + here->quote;
+    p->lex.ptok = p->lex.pbeg + here->offset - here->quote;
     p->heredoc_end = p->ruby_sourceline;
     p->ruby_sourceline = (int)here->sourceline;
-    token_flush(p);
     if (p->eofp) p->lex.nextline = Qnil;
     p->eofp = 0;
 }
@@ -7238,8 +7238,6 @@ here_document(struct parser_params *p, rb_strterm_heredoc_t *here) https://github.com/ruby/ruby/blob/trunk/parse.y#L7238
 
     if ((c = nextc(p)) == -1) {
       error:
-	compile_error(p, "can't find string \"%.*s\" anywhere before EOF",
-		      (int)len, eos);
 #ifdef RIPPER
 	if (!has_delayed_token(p)) {
 	    dispatch_scan_event(p, tSTRING_CONTENT);
@@ -7264,8 +7262,10 @@ here_document(struct parser_params *p, rb_strterm_heredoc_t *here) https://github.com/ruby/ruby/blob/trunk/parse.y#L7262
 	}
 	lex_goto_eol(p);
 #endif
-      restore:
 	heredoc_restore(p, &p->lex.strterm->u.heredoc);
+	compile_error(p, "can't find string \"%.*s\" anywhere before EOF",
+		      (int)len, eos);
+	token_flush(p);
 	p->lex.strterm = 0;
 	SET_LEX_STATE(EXPR_END);
 	return tSTRING_END;
@@ -7283,7 +7283,9 @@ here_document(struct parser_params *p, rb_strterm_heredoc_t *here) https://github.com/ruby/ruby/blob/trunk/parse.y#L7283
     }
     else if (whole_match_p(p, eos, len, indent)) {
 	dispatch_heredoc_end(p);
+      restore:
 	heredoc_restore(p, &p->lex.strterm->u.heredoc);
+	token_flush(p);
 	p->lex.strterm = 0;
 	SET_LEX_STATE(EXPR_END);
 	return tSTRING_END;
@@ -7379,6 +7381,7 @@ here_document(struct parser_params *p, rb_strterm_heredoc_t *here) https://github.com/ruby/ruby/blob/trunk/parse.y#L7381
 			    yylval.val, str);
 #endif
     heredoc_restore(p, &p->lex.strterm->u.heredoc);
+    token_flush(p);
     p->lex.strterm = NEW_STRTERM(func | STR_FUNC_TERM, 0, 0);
     set_yylval_str(str);
 #ifndef RIPPER
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 4b70915..c8cc512 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -120,17 +120,21 @@ class TestRipper::Lexer < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ripper/test_lexer.rb#L120
     message = proc {result.pretty_inspect}
     expected = [
       [[1, 0], :on_heredoc_beg, "<<A", state(:EXPR_BEG)],
+      [[1, 2], :compile_error, "A", state(:EXPR_BEG), "can't find string \"A\" anywhere before EOF"],
       [[1, 3], :on_period, ".", state(:EXPR_DOT)],
       [[1, 4], :on_ident, "upcase", state(:EXPR_ARG)],
       [[1, 10], :on_nl, "\n", state(:EXPR_BEG)],
-      [[1, 11], :compile_error, "", state(:EXPR_BEG), "can't find string \"A\" anywhere before EOF"],
     ]
     pos = 0
     expected.each_with_index do |ex, i|
       s = result[i]
       assert_equal ex, s.to_a, message
-      assert_equal pos, s.pos[1], message
-      pos += s.tok.bytesize
+      if pos > s.pos[1]
+        assert_equal pos, s.pos[1] + s.tok.bytesize, message
+      else
+        assert_equal pos, s.pos[1], message
+        pos += s.tok.bytesize
+      end
     end
     assert_equal pos, code.bytesize
     assert_equal expected.size, result.size
-- 
cgit v0.10.2


--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/