yui-knk	2017-11-16 10:06:10 +0900 (Thu, 16 Nov 2017)

  New Revision: 60780

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=60780

  Log:
    parse.y: Preserve previous line and restore it when read '\n'
    
    * parse.y (parser_params): Add prevline to store previous line.
    * parse.y (yycompile0): Initialize prevline with 0.
    * parse.y (parser_nextline): Store previous line on prevline.
    * parse.y (parser_nextc): Check parser is on EOF or has nextline.
      Now parser_yylex does not always set lex_p as lex_pend, we should check
      EOF flag and nextline is set.
    * parse.y (parser_yylex): Restore previous line, set lex_p and tokp
      on '\n'. Before this commit, tokp is on the head of next line of '\n'
      and lex_p is on the tail of next line when next token is '\n'.
      By this behavior, in some case the last column of NODE_CALL (or NODE_QCALL) is
      set to the last column of next line. NODE_CALL can be generated
      via `primary_value call_op operation2 {} opt_paren_args` and opt_paren_args
      can be none. If none is generated with next token '\n', the last column of
      none is set to the last column of next line.
    
      e.g. :
    
      ```
      a.b
      cd.ef
      ```
    
      The location of NODE_CALL of first line is set to,
    
      * Before
    
      ```
      NODE_CALL (line: 1, first_lineno: 1, first_column: 0, last_lineno: 1, last_column: 6)
      ```
    
      * After
    
      ```
      NODE_CALL (line: 1, first_lineno: 1, first_column: 0, last_lineno: 1, last_column: 3)
      ```
    
    * parse.y (parser_mark): GC mark prevline.

  Modified files:
    trunk/parse.y
Index: parse.y
===================================================================
--- parse.y	(revision 60779)
+++ parse.y	(revision 60780)
@@ -196,6 +196,7 @@ struct parser_params { https://github.com/ruby/ruby/blob/trunk/parse.y#L196
 	rb_strterm_t *strterm;
 	VALUE (*gets)(struct parser_params*,VALUE);
 	VALUE input;
+	VALUE prevline;
 	VALUE lastline;
 	VALUE nextline;
 	const char *pbeg;
@@ -305,6 +306,7 @@ static int parser_yyerror(struct parser_ https://github.com/ruby/ruby/blob/trunk/parse.y#L306
 #define toksiz			(parser->toksiz)
 #define tokline 		(parser->tokline)
 #define lex_input		(parser->lex.input)
+#define lex_prevline		(parser->lex.prevline)
 #define lex_lastline		(parser->lex.lastline)
 #define lex_nextline		(parser->lex.nextline)
 #define lex_pbeg		(parser->lex.pbeg)
@@ -5558,7 +5560,7 @@ yycompile0(VALUE arg) https://github.com/ruby/ruby/blob/trunk/parse.y#L5560
 
     lex_strterm = 0;
     lex_p = lex_pbeg = lex_pend = 0;
-    lex_lastline = lex_nextline = 0;
+    lex_prevline = lex_lastline = lex_nextline = 0;
     if (parser->error_p) {
 	VALUE mesg = parser->error_buffer;
 	if (!mesg) {
@@ -5830,6 +5832,7 @@ parser_nextline(struct parser_params *pa https://github.com/ruby/ruby/blob/trunk/parse.y#L5832
     lex_pbeg = lex_p = RSTRING_PTR(v);
     lex_pend = lex_p + RSTRING_LEN(v);
     token_flush(parser);
+    lex_prevline = lex_lastline;
     lex_lastline = v;
     return 0;
 }
@@ -5854,7 +5857,7 @@ parser_nextc(struct parser_params *parse https://github.com/ruby/ruby/blob/trunk/parse.y#L5857
 {
     int c;
 
-    if (UNLIKELY(lex_p == lex_pend)) {
+    if (UNLIKELY((lex_p == lex_pend) || parser->eofp || lex_nextline)) {
 	if (parser_nextline(parser)) return -1;
     }
     c = (unsigned char)*lex_p++;
@@ -8300,8 +8303,14 @@ parser_yylex(struct parser_params *parse https://github.com/ruby/ruby/blob/trunk/parse.y#L8303
 		--ruby_sourceline;
 		lex_nextline = lex_lastline;
 	      case -1:		/* EOF no decrement*/
+#ifndef RIPPER
+		if (lex_prevline && !parser->eofp) lex_lastline = lex_prevline;
+		lex_pbeg = RSTRING_PTR(lex_lastline);
+		lex_pend = lex_p = lex_pbeg + RSTRING_LEN(lex_lastline);
+		pushback(1); /* always pushback */
+		parser->tokp = lex_p;
+#else
 		lex_goto_eol(parser);
-#ifdef RIPPER
 		if (c != -1) {
 		    parser->tokp = lex_p;
 		}
@@ -11447,6 +11456,7 @@ parser_mark(void *ptr) https://github.com/ruby/ruby/blob/trunk/parse.y#L11456
     struct parser_params *parser = (struct parser_params*)ptr;
 
     rb_gc_mark(lex_input);
+    rb_gc_mark(lex_prevline);
     rb_gc_mark(lex_lastline);
     rb_gc_mark(lex_nextline);
     rb_gc_mark(ruby_sourcefile_string);

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/