ruby-changes:26392
From: drbrain <ko1@a...>
Date: Tue, 18 Dec 2012 16:31:48 +0900 (JST)
Subject: [ruby-changes:26392] drbrain:r38443 (trunk): * lib/rdoc/ruby_lex.rb: Return a TkHEREDOC instead of a TkSTRING when
drbrain 2012-12-18 16:31:36 +0900 (Tue, 18 Dec 2012) New Revision: 38443 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=38443 Log: * lib/rdoc/ruby_lex.rb: Return a TkHEREDOC instead of a TkSTRING when the heredoc identifier is followed by a line-end. This allows proper display of some HEREDOCs in source view. * lib/rdoc/ruby_token.rb: Added TkHEREDOC * test/rdoc/test_rdoc_ruby_lex.rb: Test for above. Modified files: trunk/ChangeLog trunk/lib/rdoc/ruby_lex.rb trunk/lib/rdoc/ruby_token.rb trunk/test/rdoc/test_rdoc_ruby_lex.rb Index: ChangeLog =================================================================== --- ChangeLog (revision 38442) +++ ChangeLog (revision 38443) @@ -1,3 +1,11 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1 +Tue Dec 18 16:31:20 2012 Eric Hodel <drbrain@s...> + + * lib/rdoc/ruby_lex.rb: Return a TkHEREDOC instead of a TkSTRING when + the heredoc identifier is followed by a line-end. This allows proper + display of some HEREDOCs in source view. + * lib/rdoc/ruby_token.rb: Added TkHEREDOC + * test/rdoc/test_rdoc_ruby_lex.rb: Test for above. + Tue Dec 18 09:45:14 2012 CHIKANAGA Tomoyuki <nagachika@r...> * vm.c (rb_vm_make_jump_tag_but_local_jump): take care of the case Index: lib/rdoc/ruby_token.rb =================================================================== --- lib/rdoc/ruby_token.rb (revision 38442) +++ lib/rdoc/ruby_token.rb (revision 38443) @@ -331,6 +331,7 @@ module RDoc::RubyToken https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_token.rb#L331 [:TkINTEGER, TkVal], [:TkFLOAT, TkVal], [:TkSTRING, TkVal], + [:TkHEREDOC, TkVal], [:TkXSTRING, TkVal], [:TkREGEXP, TkVal], [:TkSYMBOL, TkVal], Index: lib/rdoc/ruby_lex.rb =================================================================== --- lib/rdoc/ruby_lex.rb (revision 38442) +++ lib/rdoc/ruby_lex.rb (revision 38443) @@ -982,12 +982,13 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L982 indent = true end if /['"`]/ =~ ch - lt = ch + user_quote = lt = ch quoted = "" while (c = getc) && c != lt quoted.concat c end else + user_quote = nil lt = '"' quoted = ch.dup while (c = getc) && c =~ /\w/ @@ -1007,8 +1008,17 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L1008 end end + output_heredoc = reserve.join =~ /\A\r?\n\z/ + + if output_heredoc then + doc = '<<' + doc << '-' if indent + doc << "#{user_quote}#{quoted}#{user_quote}\n" + else + doc = '"' + end + @here_header = false - doc = '"' while l = gets l = l.sub(/(:?\r)?\n\z/, "\n") if (indent ? l.strip : l.chomp) == quoted @@ -1016,7 +1026,12 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L1026 end doc << l end - doc << '"' + + if output_heredoc then + doc << l.chomp + else + doc << '"' + end @here_header = true @here_readed.concat reserve @@ -1024,9 +1039,10 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L1039 ungetc ch end + token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt] @ltype = ltback @lex_state = EXPR_END - Token(Ltype2Token[lt], doc) + Token(token_class, doc) end def identify_quotation Index: test/rdoc/test_rdoc_ruby_lex.rb =================================================================== --- test/rdoc/test_rdoc_ruby_lex.rb (revision 38442) +++ test/rdoc/test_rdoc_ruby_lex.rb (revision 38443) @@ -70,7 +70,8 @@ end https://github.com/ruby/ruby/blob/trunk/test/rdoc/test_rdoc_ruby_lex.rb#L70 @TK::TkIDENTIFIER.new( 4, 1, 4, 'x'), @TK::TkNL .new( 5, 1, 5, "\n"), @TK::TkSPACE .new( 6, 2, 0, ' '), - @TK::TkSTRING .new( 8, 2, 2, %Q{"Line 1\nLine 2\n"}), + @TK::TkHEREDOC .new( 8, 2, 2, + %Q{<<E\nLine 1\nLine 2\nE}), @TK::TkNL .new(27, 5, 28, "\n"), @TK::TkEND .new(28, 6, 0, 'end'), @TK::TkNL .new(31, 6, 28, "\n"), @@ -96,12 +97,56 @@ end https://github.com/ruby/ruby/blob/trunk/test/rdoc/test_rdoc_ruby_lex.rb#L97 assert_equal expected, tokens end + def test_class_tokenize_heredoc_CR_NL + tokens = RDoc::RubyLex.tokenize <<-RUBY, nil +string = <<-STRING\r +Line 1\r +Line 2\r + STRING\r + RUBY + + expected = [ + @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), + @TK::TkSPACE .new( 6, 1, 6, ' '), + @TK::TkASSIGN .new( 7, 1, 7, '='), + @TK::TkSPACE .new( 8, 1, 8, ' '), + @TK::TkHEREDOC .new( 9, 1, 9, + %Q{<<-STRING\nLine 1\nLine 2\n STRING}), + @TK::TkSPACE .new(44, 4, 45, "\r"), + @TK::TkNL .new(45, 4, 46, "\n"), + ] + + assert_equal expected, tokens + end + + def test_class_tokenize_heredoc_call + tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil +string = <<-STRING.chomp +Line 1 +Line 2 + STRING + RUBY + + expected = [ + @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'), + @TK::TkSPACE .new( 6, 1, 6, ' '), + @TK::TkASSIGN .new( 7, 1, 7, '='), + @TK::TkSPACE .new( 8, 1, 8, ' '), + @TK::TkSTRING .new( 9, 1, 9, %Q{"Line 1\nLine 2\n"}), + @TK::TkDOT .new(41, 4, 42, '.'), + @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'), + @TK::TkNL .new(47, 4, 48, "\n"), + ] + + assert_equal expected, tokens + end + def test_class_tokenize_heredoc_indent tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil string = <<-STRING Line 1 Line 2 -STRING + STRING RUBY expected = [ @@ -109,8 +154,9 @@ STRING https://github.com/ruby/ruby/blob/trunk/test/rdoc/test_rdoc_ruby_lex.rb#L154 @TK::TkSPACE .new( 6, 1, 6, ' '), @TK::TkASSIGN .new( 7, 1, 7, '='), @TK::TkSPACE .new( 8, 1, 8, ' '), - @TK::TkSTRING .new( 9, 1, 9, %Q{"Line 1\nLine 2\n"}), - @TK::TkNL .new(39, 4, 40, "\n"), + @TK::TkHEREDOC .new( 9, 1, 9, + %Q{<<-STRING\nLine 1\nLine 2\n STRING}), + @TK::TkNL .new(41, 4, 42, "\n"), ] assert_equal expected, tokens @@ -128,7 +174,7 @@ U https://github.com/ruby/ruby/blob/trunk/test/rdoc/test_rdoc_ruby_lex.rb#L174 @TK::TkSPACE .new( 1, 1, 1, ' '), @TK::TkIDENTIFIER.new( 2, 1, 2, 'b'), @TK::TkSPACE .new( 3, 1, 3, ' '), - @TK::TkSTRING .new( 4, 1, 4, %Q{"%N\n"}), + @TK::TkHEREDOC .new( 4, 1, 4, %Q{<<-U\n%N\nU}), @TK::TkNL .new(13, 3, 14, "\n"), ] -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/