[前][次][番号順一覧][スレッド一覧]

ruby-changes:48767

From: nobu <ko1@a...>
Date: Thu, 23 Nov 2017 13:30:28 +0900 (JST)
Subject: [ruby-changes:48767] nobu:r60883 (trunk): ripper.y: fix word list events

nobu	2017-11-23 13:30:23 +0900 (Thu, 23 Nov 2017)

  New Revision: 60883

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=60883

  Log:
    ripper.y: fix word list events
    
    * parse.y (parser_skip_words_sep): QWORDS_BEG should not include
      the first separators in ripper.
    
    * parse.y (parser_parse_string): WORDS_SEP should not include
      the closing parentheses of a word list in ripper, should include
      spaces at beginning of lines.  [ruby-core:83864] [Bug #14126]

  Modified files:
    trunk/parse.y
    trunk/test/ripper/test_scanner_events.rb
Index: parse.y
===================================================================
--- parse.y	(revision 60882)
+++ parse.y	(revision 60883)
@@ -5819,6 +5819,26 @@ parser_str_new(const char *p, long n, rb https://github.com/ruby/ruby/blob/trunk/parse.y#L5819
 #define peekc() peekc_n(0)
 #define peekc_n(n) (lex_p+(n) < lex_pend ? (unsigned char)lex_p[n] : -1)
 
+#ifdef RIPPER
+static void
+parser_add_delayed_token(struct parser_params *parser, const char *tok, const char *end)
+{
+    if (tok < end) {
+	if (!has_delayed_token()) {
+	    parser->delayed = rb_str_buf_new(1024);
+	    rb_enc_associate(parser->delayed, current_enc);
+	    parser->delayed_line = ruby_sourceline;
+	    parser->delayed_col = (int)(tok - lex_pbeg);
+	}
+	rb_str_buf_cat(parser->delayed, tok, end - tok);
+	parser->tokp = end;
+    }
+}
+#define add_delayed_token(tok, end) parser_add_delayed_token(parser, (tok), (end))
+#else
+#define add_delayed_token(tok, end) ((void)(tok), (void)(end))
+#endif
+
 static int
 parser_nextline(struct parser_params *parser)
 {
@@ -5835,22 +5855,7 @@ parser_nextline(struct parser_params *pa https://github.com/ruby/ruby/blob/trunk/parse.y#L5855
 	}
 	parser->cr_seen = FALSE;
     }
-#ifdef RIPPER
-    if (parser->tokp < lex_pend) {
-	if (!has_delayed_token()) {
-	    parser->delayed = rb_str_buf_new(1024);
-	    rb_enc_associate(parser->delayed, current_enc);
-	    rb_str_buf_cat(parser->delayed,
-			   parser->tokp, lex_pend - parser->tokp);
-	    parser->delayed_line = ruby_sourceline;
-	    parser->delayed_col = (int)(parser->tokp - lex_pbeg);
-	}
-	else {
-	    rb_str_buf_cat(parser->delayed,
-			   parser->tokp, lex_pend - parser->tokp);
-	}
-    }
-#endif
+    add_delayed_token(parser->tokp, lex_pend);
     if (heredoc_end > 0) {
 	ruby_sourceline = heredoc_end;
 	heredoc_end = 0;
@@ -6597,6 +6602,9 @@ parser_parse_string(struct parser_params https://github.com/ruby/ruby/blob/trunk/parse.y#L6602
     VALUE lit;
 
     if (func & STR_FUNC_TERM) {
+#ifdef RIPPER
+	if (func & STR_FUNC_QWORDS) nextc(); /* delayed term */
+#endif
 	SET_LEX_STATE(EXPR_END|EXPR_ENDARG);
 	lex_strterm = 0;
 	return func & STR_FUNC_REGEXP ? tREGEXP_END : tSTRING_END;
@@ -6609,12 +6617,17 @@ parser_parse_string(struct parser_params https://github.com/ruby/ruby/blob/trunk/parse.y#L6617
     if (c == term && !quote->u0.nest) {
 	if (func & STR_FUNC_QWORDS) {
 	    quote->u1.func |= STR_FUNC_TERM;
+#ifdef RIPPER
+	    pushback(c); /* dispatch the term at tSTRING_END */
+#endif
+	    add_delayed_token(parser->tokp, lex_p);
 	    return ' ';
 	}
 	return parser_string_term(parser, func);
     }
     if (space) {
 	pushback(c);
+	add_delayed_token(parser->tokp, lex_p);
 	return ' ';
     }
     newtok();
@@ -7853,6 +7866,19 @@ parse_qmark(struct parser_params *parser https://github.com/ruby/ruby/blob/trunk/parse.y#L7866
     return tCHAR;
 }
 
+#ifndef RIPPER
+static void
+parser_skip_words_sep(struct parser_params *parser)
+{
+    int c;
+    do {c = nextc();} while (ISSPACE(c));
+    pushback(c);
+}
+#define skip_words_sep() parser_skip_words_sep(parser)
+#else
+#define skip_words_sep() ((void)0)
+#endif
+
 static enum yytokentype
 parse_percent(struct parser_params *parser, const int space_seen, const enum lex_state_e last_state)
 {
@@ -7897,26 +7923,22 @@ parse_percent(struct parser_params *pars https://github.com/ruby/ruby/blob/trunk/parse.y#L7923
 
 	  case 'W':
 	    lex_strterm = NEW_STRTERM(str_dword, term, paren);
-	    do {c = nextc();} while (ISSPACE(c));
-	    pushback(c);
+	    skip_words_sep();
 	    return tWORDS_BEG;
 
 	  case 'w':
 	    lex_strterm = NEW_STRTERM(str_sword, term, paren);
-	    do {c = nextc();} while (ISSPACE(c));
-	    pushback(c);
+	    skip_words_sep();
 	    return tQWORDS_BEG;
 
 	  case 'I':
 	    lex_strterm = NEW_STRTERM(str_dword, term, paren);
-	    do {c = nextc();} while (ISSPACE(c));
-	    pushback(c);
+	    skip_words_sep();
 	    return tSYMBOLS_BEG;
 
 	  case 'i':
 	    lex_strterm = NEW_STRTERM(str_sword, term, paren);
-	    do {c = nextc();} while (ISSPACE(c));
-	    pushback(c);
+	    skip_words_sep();
 	    return tQSYMBOLS_BEG;
 
 	  case 'x':
Index: test/ripper/test_scanner_events.rb
===================================================================
--- test/ripper/test_scanner_events.rb	(revision 60882)
+++ test/ripper/test_scanner_events.rb	(revision 60883)
@@ -637,8 +637,10 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/test/ripper/test_scanner_events.rb#L637
                  scan('words_beg', '%W()')
     assert_equal ['%W('],
                  scan('words_beg', '%W(w w w)')
-    assert_equal ['%W( '],
+    assert_equal ['%W('],
                  scan('words_beg', '%W( w w w )')
+    assert_equal ['%W('],
+                 scan('words_beg', "%W(\nw)")
   end
 
   def test_qwords_beg
@@ -648,8 +650,10 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/test/ripper/test_scanner_events.rb#L650
                  scan('qwords_beg', '%w()')
     assert_equal ['%w('],
                  scan('qwords_beg', '%w(w w w)')
-    assert_equal ['%w( '],
+    assert_equal ['%w('],
                  scan('qwords_beg', '%w( w w w )')
+    assert_equal ['%w('],
+                 scan('qwords_beg', "%w(\nw)")
   end
 
   def test_qsymbols_beg
@@ -659,8 +663,10 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/test/ripper/test_scanner_events.rb#L663
                  scan('qsymbols_beg', '%i()')
     assert_equal ['%i('],
                  scan('qsymbols_beg', '%i(w w w)')
-    assert_equal ['%i( '],
+    assert_equal ['%i('],
                  scan('qsymbols_beg', '%i( w w w )')
+    assert_equal ['%i('],
+                 scan('qsymbols_beg', "%i(\nw)")
   end
 
   def test_symbols_beg
@@ -670,22 +676,25 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/test/ripper/test_scanner_events.rb#L676
                  scan('symbols_beg', '%I()')
     assert_equal ['%I('],
                  scan('symbols_beg', '%I(w w w)')
-    assert_equal ['%I( '],
+    assert_equal ['%I('],
                  scan('symbols_beg', '%I( w w w )')
+    assert_equal ['%I('],
+                 scan('symbols_beg', "%I(\nw)")
   end
 
-  # FIXME: Close paren must not present (`words_end' scanner event?).
   def test_words_sep
     assert_equal [],
                  scan('words_sep', '')
-    assert_equal [')'],
+    assert_equal [],
                  scan('words_sep', '%w()')
-    assert_equal [' ', ' ', ')'],
+    assert_equal [' ', ' '],
                  scan('words_sep', '%w(w w w)')
-    assert_equal [' ', ' ', ' )'],
+    assert_equal [' ', ' ', ' ', ' '],
                  scan('words_sep', '%w( w w w )')
-    assert_equal ["\n", ' ', ' )'],
+    assert_equal [' ', "\n", ' ', ' '],
                  scan('words_sep', "%w( w\nw w )")
+    assert_equal ["\n\n", "\n ", ' ', ' '],
+                 scan('words_sep', "%w(\n\nw\n w w )")
   end
 
   def test_heredoc_beg

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/

[前][次][番号順一覧][スレッド一覧]