ruby-changes:48767
From: nobu <ko1@a...>
Date: Thu, 23 Nov 2017 13:30:28 +0900 (JST)
Subject: [ruby-changes:48767] nobu:r60883 (trunk): ripper.y: fix word list events
nobu 2017-11-23 13:30:23 +0900 (Thu, 23 Nov 2017) New Revision: 60883 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=60883 Log: ripper.y: fix word list events * parse.y (parser_skip_words_sep): QWORDS_BEG should not include the first separators in ripper. * parse.y (parser_parse_string): WORDS_SEP should not include the closing parentheses of a word list in ripper, should include spaces at beginning of lines. [ruby-core:83864] [Bug #14126] Modified files: trunk/parse.y trunk/test/ripper/test_scanner_events.rb Index: parse.y =================================================================== --- parse.y (revision 60882) +++ parse.y (revision 60883) @@ -5819,6 +5819,26 @@ parser_str_new(const char *p, long n, rb https://github.com/ruby/ruby/blob/trunk/parse.y#L5819 #define peekc() peekc_n(0) #define peekc_n(n) (lex_p+(n) < lex_pend ? (unsigned char)lex_p[n] : -1) +#ifdef RIPPER +static void +parser_add_delayed_token(struct parser_params *parser, const char *tok, const char *end) +{ + if (tok < end) { + if (!has_delayed_token()) { + parser->delayed = rb_str_buf_new(1024); + rb_enc_associate(parser->delayed, current_enc); + parser->delayed_line = ruby_sourceline; + parser->delayed_col = (int)(tok - lex_pbeg); + } + rb_str_buf_cat(parser->delayed, tok, end - tok); + parser->tokp = end; + } +} +#define add_delayed_token(tok, end) parser_add_delayed_token(parser, (tok), (end)) +#else +#define add_delayed_token(tok, end) ((void)(tok), (void)(end)) +#endif + static int parser_nextline(struct parser_params *parser) { @@ -5835,22 +5855,7 @@ parser_nextline(struct parser_params *pa https://github.com/ruby/ruby/blob/trunk/parse.y#L5855 } parser->cr_seen = FALSE; } -#ifdef RIPPER - if (parser->tokp < lex_pend) { - if (!has_delayed_token()) { - parser->delayed = rb_str_buf_new(1024); - rb_enc_associate(parser->delayed, current_enc); - rb_str_buf_cat(parser->delayed, - parser->tokp, lex_pend - parser->tokp); - parser->delayed_line = ruby_sourceline; - parser->delayed_col = (int)(parser->tokp - lex_pbeg); - } - else { - rb_str_buf_cat(parser->delayed, - parser->tokp, lex_pend - parser->tokp); - } - } -#endif + add_delayed_token(parser->tokp, lex_pend); if (heredoc_end > 0) { ruby_sourceline = heredoc_end; heredoc_end = 0; @@ -6597,6 +6602,9 @@ parser_parse_string(struct parser_params https://github.com/ruby/ruby/blob/trunk/parse.y#L6602 VALUE lit; if (func & STR_FUNC_TERM) { +#ifdef RIPPER + if (func & STR_FUNC_QWORDS) nextc(); /* delayed term */ +#endif SET_LEX_STATE(EXPR_END|EXPR_ENDARG); lex_strterm = 0; return func & STR_FUNC_REGEXP ? tREGEXP_END : tSTRING_END; @@ -6609,12 +6617,17 @@ parser_parse_string(struct parser_params https://github.com/ruby/ruby/blob/trunk/parse.y#L6617 if (c == term && !quote->u0.nest) { if (func & STR_FUNC_QWORDS) { quote->u1.func |= STR_FUNC_TERM; +#ifdef RIPPER + pushback(c); /* dispatch the term at tSTRING_END */ +#endif + add_delayed_token(parser->tokp, lex_p); return ' '; } return parser_string_term(parser, func); } if (space) { pushback(c); + add_delayed_token(parser->tokp, lex_p); return ' '; } newtok(); @@ -7853,6 +7866,19 @@ parse_qmark(struct parser_params *parser https://github.com/ruby/ruby/blob/trunk/parse.y#L7866 return tCHAR; } +#ifndef RIPPER +static void +parser_skip_words_sep(struct parser_params *parser) +{ + int c; + do {c = nextc();} while (ISSPACE(c)); + pushback(c); +} +#define skip_words_sep() parser_skip_words_sep(parser) +#else +#define skip_words_sep() ((void)0) +#endif + static enum yytokentype parse_percent(struct parser_params *parser, const int space_seen, const enum lex_state_e last_state) { @@ -7897,26 +7923,22 @@ parse_percent(struct parser_params *pars https://github.com/ruby/ruby/blob/trunk/parse.y#L7923 case 'W': lex_strterm = NEW_STRTERM(str_dword, term, paren); - do {c = nextc();} while (ISSPACE(c)); - pushback(c); + skip_words_sep(); return tWORDS_BEG; case 'w': lex_strterm = NEW_STRTERM(str_sword, term, paren); - do {c = nextc();} while (ISSPACE(c)); - pushback(c); + skip_words_sep(); return tQWORDS_BEG; case 'I': lex_strterm = NEW_STRTERM(str_dword, term, paren); - do {c = nextc();} while (ISSPACE(c)); - pushback(c); + skip_words_sep(); return tSYMBOLS_BEG; case 'i': lex_strterm = NEW_STRTERM(str_sword, term, paren); - do {c = nextc();} while (ISSPACE(c)); - pushback(c); + skip_words_sep(); return tQSYMBOLS_BEG; case 'x': Index: test/ripper/test_scanner_events.rb =================================================================== --- test/ripper/test_scanner_events.rb (revision 60882) +++ test/ripper/test_scanner_events.rb (revision 60883) @@ -637,8 +637,10 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/test/ripper/test_scanner_events.rb#L637 scan('words_beg', '%W()') assert_equal ['%W('], scan('words_beg', '%W(w w w)') - assert_equal ['%W( '], + assert_equal ['%W('], scan('words_beg', '%W( w w w )') + assert_equal ['%W('], + scan('words_beg', "%W(\nw)") end def test_qwords_beg @@ -648,8 +650,10 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/test/ripper/test_scanner_events.rb#L650 scan('qwords_beg', '%w()') assert_equal ['%w('], scan('qwords_beg', '%w(w w w)') - assert_equal ['%w( '], + assert_equal ['%w('], scan('qwords_beg', '%w( w w w )') + assert_equal ['%w('], + scan('qwords_beg', "%w(\nw)") end def test_qsymbols_beg @@ -659,8 +663,10 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/test/ripper/test_scanner_events.rb#L663 scan('qsymbols_beg', '%i()') assert_equal ['%i('], scan('qsymbols_beg', '%i(w w w)') - assert_equal ['%i( '], + assert_equal ['%i('], scan('qsymbols_beg', '%i( w w w )') + assert_equal ['%i('], + scan('qsymbols_beg', "%i(\nw)") end def test_symbols_beg @@ -670,22 +676,25 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/test/ripper/test_scanner_events.rb#L676 scan('symbols_beg', '%I()') assert_equal ['%I('], scan('symbols_beg', '%I(w w w)') - assert_equal ['%I( '], + assert_equal ['%I('], scan('symbols_beg', '%I( w w w )') + assert_equal ['%I('], + scan('symbols_beg', "%I(\nw)") end - # FIXME: Close paren must not present (`words_end' scanner event?). def test_words_sep assert_equal [], scan('words_sep', '') - assert_equal [')'], + assert_equal [], scan('words_sep', '%w()') - assert_equal [' ', ' ', ')'], + assert_equal [' ', ' '], scan('words_sep', '%w(w w w)') - assert_equal [' ', ' ', ' )'], + assert_equal [' ', ' ', ' ', ' '], scan('words_sep', '%w( w w w )') - assert_equal ["\n", ' ', ' )'], + assert_equal [' ', "\n", ' ', ' '], scan('words_sep', "%w( w\nw w )") + assert_equal ["\n\n", "\n ", ' ', ' '], + scan('words_sep', "%w(\n\nw\n w w )") end def test_heredoc_beg -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/