nobu	2017-07-06 13:12:51 +0900 (Thu, 06 Jul 2017)

  New Revision: 59269

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=59269

  Log:
    lexer.rb: ignore dedented space
    
    * ext/ripper/lib/ripper/lexer.rb (on_heredoc_dedent): replace an
      empty string content because of dedentation with :on_ignored_sp.
      an empty token makes the sorted order unstable.

  Modified files:
    trunk/ext/ripper/lib/ripper/lexer.rb
    trunk/test/ripper/test_lexer.rb
Index: ext/ripper/lib/ripper/lexer.rb
===================================================================
--- ext/ripper/lib/ripper/lexer.rb	(revision 59268)
+++ ext/ripper/lib/ripper/lexer.rb	(revision 59269)
@@ -72,6 +72,11 @@ class Ripper https://github.com/ruby/ruby/blob/trunk/ext/ripper/lib/ripper/lexer.rb#L72
         if Elem === e and e.event == :on_tstring_content
           tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok
           if (n = dedent_string(e.tok, w)) > 0
+            if e.tok.empty?
+              e.tok = tok[0, n]
+              e.event = :on_ignored_sp
+              next
+            end
             ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])]
             e.pos[1] += n
           end
Index: test/ripper/test_lexer.rb
===================================================================
--- test/ripper/test_lexer.rb	(revision 59268)
+++ test/ripper/test_lexer.rb	(revision 59269)
@@ -32,4 +32,23 @@ class TestRipper::Lexer < Test::Unit::Te https://github.com/ruby/ruby/blob/trunk/test/ripper/test_lexer.rb#L32
     E
     assert_equal(str, Ripper.tokenize(str).join(""), bug)
   end
+
+  def test_embedded_expr_in_heredoc
+    src = <<~'E'
+    <<~B
+      #{1}
+    B
+    E
+    expect = %I[
+      on_heredoc_beg
+      on_nl
+      on_ignored_sp
+      on_embexpr_beg
+      on_int
+      on_embexpr_end
+      on_tstring_content
+      on_heredoc_end
+    ]
+    assert_equal expect, Ripper.lex(src).map {|e| e[1]}
+  end
 end

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/