nagachika	2017-07-01 20:51:13 +0900 (Sat, 01 Jul 2017)

  New Revision: 59247

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=59247

  Log:
    merge revision(s) 58545,58584: [Backport #13536]
    
    ripper/lexer.rb: nested indented heredoc
    
    * ext/ripper/lib/ripper/lexer.rb (on_heredoc_dedent): fix for
      nested indedented here documents, where `Elem`s are nested too.
      [ruby-core:80977] [Bug #13536]
    ripper/lexer.rb: nested indented heredoc
    
    * ext/ripper/lib/ripper/lexer.rb (on_heredoc_dedent): insert
      stripped leading spaces as `on_ignored_sp` elements, so that the
      original source can be reconsructed.
      [ruby-core:80977] [Bug #13536]

  Added files:
    branches/ruby_2_4/test/ripper/test_lexer.rb
  Modified directories:
    branches/ruby_2_4/
  Modified files:
    branches/ruby_2_4/ext/ripper/lib/ripper/lexer.rb
    branches/ruby_2_4/test/ripper/test_scanner_events.rb
    branches/ruby_2_4/version.h
Index: ruby_2_4/ext/ripper/lib/ripper/lexer.rb
===================================================================
--- ruby_2_4/ext/ripper/lib/ripper/lexer.rb	(revision 59246)
+++ ruby_2_4/ext/ripper/lib/ripper/lexer.rb	(revision 59247)
@@ -66,13 +66,20 @@ class Ripper https://github.com/ruby/ruby/blob/trunk/ruby_2_4/ext/ripper/lib/ripper/lexer.rb#L66
     private
 
     def on_heredoc_dedent(v, w)
-      @buf.last.each do |e|
-        if e.event == :on_tstring_content
+      ignored_sp = []
+      heredoc = @buf.last
+      heredoc.each_with_index do |e, i|
+        if Elem === e and e.event == :on_tstring_content
+          tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok
           if (n = dedent_string(e.tok, w)) > 0
+            ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])]
             e.pos[1] += n
           end
         end
       end
+      ignored_sp.reverse_each do |i, e|
+        heredoc[i, 0] = [e]
+      end
       v
     end
 
Index: ruby_2_4/test/ripper/test_scanner_events.rb
===================================================================
--- ruby_2_4/test/ripper/test_scanner_events.rb	(revision 59246)
+++ ruby_2_4/test/ripper/test_scanner_events.rb	(revision 59247)
@@ -105,6 +105,7 @@ class TestRipper::ScannerEvents < Test:: https://github.com/ruby/ruby/blob/trunk/ruby_2_4/test/ripper/test_scanner_events.rb#L105
                  Ripper.lex("1r\n2i\n3ri\n4.2r\n5.6ri")
      assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS"],
                    [[1, 6], :on_nl, "\n"],
+                   [[2, 0], :on_ignored_sp, "  "],
                    [[2, 2], :on_tstring_content, "heredoc\n"],
                    [[3, 0], :on_heredoc_end, "EOS"]
                  ],
Index: ruby_2_4/test/ripper/test_lexer.rb
===================================================================
--- ruby_2_4/test/ripper/test_lexer.rb	(nonexistent)
+++ ruby_2_4/test/ripper/test_lexer.rb	(revision 59247)
@@ -0,0 +1,35 @@ https://github.com/ruby/ruby/blob/trunk/ruby_2_4/test/ripper/test_lexer.rb#L1
+# frozen_string_literal: true
+begin
+  require_relative 'dummyparser'
+  require 'test/unit'
+  ripper_test = true
+  module TestRipper; end
+rescue LoadError
+end
+
+class TestRipper::Lexer < Test::Unit::TestCase
+  def test_nested_dedent_heredoc
+    bug = '[ruby-core:80977] [Bug #13536]'
+    str = <<~'E'
+    <<~"D"
+    #{
+    <<~"B"
+    this must be a valid ruby
+    B
+    }
+    D
+    E
+    assert_equal(str, Ripper.tokenize(str).join(""), bug)
+
+    str = <<~'E'
+    <<~"D"
+    #{
+    <<~"B"
+      this must be a valid ruby
+    B
+    }
+    D
+    E
+    assert_equal(str, Ripper.tokenize(str).join(""), bug)
+  end
+end

Property changes on: ruby_2_4/test/ripper/test_lexer.rb
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+LF
\ No newline at end of property
Index: ruby_2_4/version.h
===================================================================
--- ruby_2_4/version.h	(revision 59246)
+++ ruby_2_4/version.h	(revision 59247)
@@ -1,10 +1,10 @@ https://github.com/ruby/ruby/blob/trunk/ruby_2_4/version.h#L1
 #define RUBY_VERSION "2.4.2"
-#define RUBY_RELEASE_DATE "2017-05-10"
-#define RUBY_PATCHLEVEL 131
+#define RUBY_RELEASE_DATE "2017-07-01"
+#define RUBY_PATCHLEVEL 132
 
 #define RUBY_RELEASE_YEAR 2017
-#define RUBY_RELEASE_MONTH 5
-#define RUBY_RELEASE_DAY 10
+#define RUBY_RELEASE_MONTH 7
+#define RUBY_RELEASE_DAY 1
 
 #include "ruby/version.h"
 
Index: ruby_2_4
===================================================================
--- ruby_2_4	(revision 59246)
+++ ruby_2_4	(revision 59247)

Property changes on: ruby_2_4
___________________________________________________________________
Modified: svn:mergeinfo
## -0,0 +0,1 ##
   Merged /trunk:r58545,58584

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/