ruby-changes:14188
From: yugui <ko1@a...>
Date: Sat, 5 Dec 2009 18:41:08 +0900 (JST)
Subject: [ruby-changes:14188] Ruby:r26008 (ruby_1_9_1): merges r25402 from trunk into ruby_1_9_1. fixes the backport task #1921.
yugui 2009-12-05 18:39:18 +0900 (Sat, 05 Dec 2009) New Revision: 26008 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=26008 Log: merges r25402 from trunk into ruby_1_9_1. fixes the backport task #1921. -- * parse.y (parser_here_document): dispatch delayed heredoc contents. based on a patch from Andy Keep in [ruby-core:24855]. Modified files: branches/ruby_1_9_1/ChangeLog branches/ruby_1_9_1/parse.y branches/ruby_1_9_1/test/ripper/dummyparser.rb branches/ruby_1_9_1/test/ripper/test_parser_events.rb branches/ruby_1_9_1/test/ripper/test_scanner_events.rb branches/ruby_1_9_1/version.h Index: ruby_1_9_1/ChangeLog =================================================================== --- ruby_1_9_1/ChangeLog (revision 26007) +++ ruby_1_9_1/ChangeLog (revision 26008) @@ -1,3 +1,8 @@ +Tue Oct 20 14:50:51 2009 Nobuyoshi Nakada <nobu@r...> + + * parse.y (parser_here_document): dispatch delayed heredoc + contents. based on a patch from Andy Keep in [ruby-core:24855]. + Fri Jan 30 18:04:23 2009 NAKAMURA Usaku <usa@r...> * win32/win32.c (rb_w32_write): limit write size to 32KB if the file Index: ruby_1_9_1/parse.y =================================================================== --- ruby_1_9_1/parse.y (revision 26007) +++ ruby_1_9_1/parse.y (revision 26008) @@ -4785,24 +4785,45 @@ # define yylval_id() yylval.id #endif -#ifdef RIPPER +#ifndef RIPPER +#define ripper_flush(p) (void)(p) +#else #define ripper_flush(p) (p->tokp = p->parser_lex_p) #define yylval_rval *(TYPE(yylval.val) == T_NODE ? &yylval.node->nd_rval : &yylval.val) -static void -ripper_dispatch_scan_event(struct parser_params *parser, int t) +static int +ripper_has_scan_event(struct parser_params *parser) { - VALUE str; if (lex_p < parser->tokp) rb_raise(rb_eRuntimeError, "lex_p < tokp"); - if (lex_p == parser->tokp) return; - str = STR_NEW(parser->tokp, lex_p - parser->tokp); - yylval_rval = ripper_dispatch1(parser, ripper_token2eventid(t), str); + return lex_p > parser->tokp; +} + +static VALUE +ripper_scan_event_val(struct parser_params *parser, int t) +{ + VALUE str = STR_NEW(parser->tokp, lex_p - parser->tokp); + VALUE rval = ripper_dispatch1(parser, ripper_token2eventid(t), str); ripper_flush(parser); + return rval; } static void +ripper_dispatch_scan_event(struct parser_params *parser, int t) +{ + if (!ripper_has_scan_event(parser)) return; + yylval_rval = ripper_scan_event_val(parser, t); +} + +static void +ripper_dispatch_ignored_scan_event(struct parser_params *parser, int t) +{ + if (!ripper_has_scan_event(parser)) return; + (void)ripper_scan_event_val(parser, t); +} + +static void ripper_dispatch_delayed_token(struct parser_params *parser, int t) { int saved_line = ruby_sourceline; @@ -5289,9 +5310,7 @@ parser->line_count++; lex_pbeg = lex_p = RSTRING_PTR(v); lex_pend = lex_p + RSTRING_LEN(v); -#ifdef RIPPER ripper_flush(parser); -#endif lex_lastline = v; } } @@ -5946,9 +5965,7 @@ len, /* nd_nth */ lex_lastline); /* nd_orig */ nd_set_line(lex_strterm, ruby_sourceline); -#ifdef RIPPER ripper_flush(parser); -#endif return term == '`' ? tXSTRING_BEG : tSTRING_BEG; } @@ -5957,12 +5974,6 @@ { VALUE line; -#ifdef RIPPER - if (!NIL_P(parser->delayed)) - ripper_dispatch_delayed_token(parser, tSTRING_CONTENT); - lex_goto_eol(parser); - ripper_dispatch_scan_event(parser, tHEREDOC_END); -#endif line = here->nd_orig; lex_lastline = line; lex_pbeg = RSTRING_PTR(line); @@ -5972,9 +5983,7 @@ ruby_sourceline = nd_line(here); dispose_string(here->nd_lit); rb_gc_force_recycle((VALUE)here); -#ifdef RIPPER ripper_flush(parser); -#endif } static int @@ -6000,6 +6009,7 @@ const char *eos, *p, *pend; long len; VALUE str = 0; + rb_encoding *enc = parser->enc; eos = RSTRING_PTR(here->nd_lit); len = RSTRING_LEN(here->nd_lit) - 1; @@ -6008,6 +6018,20 @@ if ((c = nextc()) == -1) { error: compile_error(PARSER_ARG "can't find string \"%s\" anywhere before EOF", eos); +#ifdef RIPPER + if (NIL_P(parser->delayed)) { + ripper_dispatch_scan_event(parser, tSTRING_CONTENT); + } + else { + if (str || + ((len = lex_p - parser->tokp) > 0 && + (str = STR_NEW3(parser->tokp, len, enc, func), 1))) { + rb_str_append(parser->delayed, str); + } + ripper_dispatch_delayed_token(parser, tSTRING_CONTENT); + } + lex_goto_eol(parser); +#endif restore: heredoc_restore(lex_strterm); lex_strterm = 0; @@ -6047,7 +6071,6 @@ } else { /* int mb = ENC_CODERANGE_7BIT, *mbp = &mb;*/ - rb_encoding *enc = parser->enc; newtok(); if (c == '#') { switch (c = nextc()) { @@ -6076,6 +6099,12 @@ } while (!whole_match_p(eos, len, indent)); str = STR_NEW3(tok(), toklen(), enc, func); } +#ifdef RIPPER + if (!NIL_P(parser->delayed)) + ripper_dispatch_delayed_token(parser, tSTRING_CONTENT); + lex_goto_eol(parser); + ripper_dispatch_ignored_scan_event(parser, tHEREDOC_END); +#endif heredoc_restore(lex_strterm); lex_strterm = NEW_STRTERM(-1, 0, 0); set_yylval_str(str); Index: ruby_1_9_1/version.h =================================================================== --- ruby_1_9_1/version.h (revision 26007) +++ ruby_1_9_1/version.h (revision 26008) @@ -1,5 +1,5 @@ #define RUBY_VERSION "1.9.1" -#define RUBY_PATCHLEVEL 361 +#define RUBY_PATCHLEVEL 362 #define RUBY_VERSION_MAJOR 1 #define RUBY_VERSION_MINOR 9 #define RUBY_VERSION_TEENY 1 Index: ruby_1_9_1/test/ripper/test_parser_events.rb =================================================================== --- ruby_1_9_1/test/ripper/test_parser_events.rb (revision 26007) +++ ruby_1_9_1/test/ripper/test_parser_events.rb (revision 26008) @@ -202,6 +202,20 @@ assert_equal true, thru_bodystmt end + def test_heredoc + bug1921 = '[ruby-core:24855]' + thru_heredoc_beg = false + tree = parse("<<EOS\nheredoc\nEOS\n", :on_heredoc_beg) {thru_heredoc_beg = true} + assert_equal true, thru_heredoc_beg + assert_match(/string_content\(\),heredoc\n/, tree, bug1921) + heredoc = nil + parse("<<EOS\nheredoc1\nheredoc2\nEOS\n", :on_string_add) {|n, s| heredoc = s} + assert_equal("heredoc1\nheredoc2\n", heredoc, bug1921) + heredoc = nil + parse("<<-EOS\nheredoc1\nheredoc2\n\tEOS\n", :on_string_add) {|n, s| heredoc = s} + assert_equal("heredoc1\nheredoc2\n", heredoc, bug1921) + end + =begin def test_brace_block assert_equal true, $thru__brace_block Index: ruby_1_9_1/test/ripper/dummyparser.rb =================================================================== --- ruby_1_9_1/test/ripper/dummyparser.rb (revision 26007) +++ ruby_1_9_1/test/ripper/dummyparser.rb (revision 26008) @@ -58,7 +58,7 @@ class << self; self; end.class_eval do define_method(name) do |*a, &b| result = super(*a, &b) - yield + yield(*a) result end end Index: ruby_1_9_1/test/ripper/test_scanner_events.rb =================================================================== --- ruby_1_9_1/test/ripper/test_scanner_events.rb (revision 26007) +++ ruby_1_9_1/test/ripper/test_scanner_events.rb (revision 26008) @@ -628,9 +628,9 @@ scan('tstring_content', "<<EOS\nheredoc\nEOS") assert_equal ["heredoc\n"], scan('tstring_content', "<<EOS\nheredoc\nEOS\n") - assert_equal ["heredoc \n"], - scan('tstring_content', "<<EOS\nheredoc \nEOS \n") - assert_equal ["heredoc\n"], + assert_equal ["here\ndoc \nEOS \n"], + scan('tstring_content', "<<EOS\nhere\ndoc \nEOS \n") + assert_equal ["heredoc\n\tEOS \n"], scan('tstring_content', "<<-EOS\nheredoc\n\tEOS \n") end @@ -641,9 +641,9 @@ scan('heredoc_end', "<<EOS\nheredoc\nEOS") assert_equal ["EOS\n"], scan('heredoc_end', "<<EOS\nheredoc\nEOS\n") - assert_equal ["EOS \n"], + assert_equal [], scan('heredoc_end', "<<EOS\nheredoc\nEOS \n") - assert_equal ["\tEOS \n"], + assert_equal [], scan('heredoc_end', "<<-EOS\nheredoc\n\tEOS \n") end -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/