ruby-changes:47571
From: hsbt <ko1@a...>
Date: Tue, 29 Aug 2017 20:52:58 +0900 (JST)
Subject: [ruby-changes:47571] hsbt:r59686 (trunk): Merge rdoc-6.0.0.beta1.
hsbt 2017-08-29 20:52:50 +0900 (Tue, 29 Aug 2017) New Revision: 59686 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=59686 Log: Merge rdoc-6.0.0.beta1. This version fixed strange behavior of ruby code parser. We will list all of impromovement to Changelog when 6.0.0 releasing. Modified files: trunk/lib/rdoc/code_object.rb trunk/lib/rdoc/context.rb trunk/lib/rdoc/generator/template/darkfish/class.rhtml trunk/lib/rdoc/i18n.rb trunk/lib/rdoc/markdown.rb trunk/lib/rdoc/parser/c.rb trunk/lib/rdoc/parser/ruby.rb trunk/lib/rdoc/parser/ruby_tools.rb trunk/lib/rdoc/rd/block_parser.rb trunk/lib/rdoc/rd/inline_parser.rb trunk/lib/rdoc/rdoc.gemspec trunk/lib/rdoc/ri/driver.rb trunk/lib/rdoc/ruby_lex.rb trunk/lib/rdoc/ruby_token.rb trunk/lib/rdoc/test_case.rb trunk/lib/rdoc/token_stream.rb trunk/lib/rdoc.rb trunk/test/rdoc/test_rdoc_code_object.rb trunk/test/rdoc/test_rdoc_context.rb trunk/test/rdoc/test_rdoc_markup_to_html.rb trunk/test/rdoc/test_rdoc_parser_c.rb trunk/test/rdoc/test_rdoc_parser_ruby.rb trunk/test/rdoc/test_rdoc_ruby_lex.rb trunk/test/rdoc/test_rdoc_store.rb trunk/test/rdoc/xref_data.rb trunk/test/rdoc/xref_test_case.rb trunk/tool/sync_default_gems.rb Index: lib/rdoc/rdoc.gemspec =================================================================== --- lib/rdoc/rdoc.gemspec (revision 59685) +++ lib/rdoc/rdoc.gemspec (revision 59686) @@ -21,7 +21,7 @@ Gem::Specification.new do |s| https://github.com/ruby/ruby/blob/trunk/lib/rdoc/rdoc.gemspec#L21 RDoc produces HTML and command-line documentation for Ruby projects. RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentation from the command-line. DESCRIPTION - s.homepage = "https://rdoc.github.io/rdoc" + s.homepage = "https://ruby.github.io/rdoc" s.licenses = ["Ruby"] s.bindir = "exe" @@ -46,7 +46,7 @@ RDoc includes the +rdoc+ and +ri+ tools https://github.com/ruby/ruby/blob/trunk/lib/rdoc/rdoc.gemspec#L46 TODO.rdoc ] - s.required_ruby_version = Gem::Requirement.new(">= 1.9.3") + s.required_ruby_version = Gem::Requirement.new(">= 2.2.2") s.rubygems_version = "2.5.2" s.required_rubygems_version = Gem::Requirement.new(">= 2.2") Index: lib/rdoc/test_case.rb =================================================================== --- lib/rdoc/test_case.rb (revision 59685) +++ lib/rdoc/test_case.rb (revision 59686) @@ -6,7 +6,7 @@ rescue NoMethodError, Gem::LoadError https://github.com/ruby/ruby/blob/trunk/lib/rdoc/test_case.rb#L6 end require 'minitest/autorun' -require 'minitest/benchmark' if ENV['BENCHMARK'] +require 'minitest/benchmark' unless ENV['NOBENCHMARK'] require 'fileutils' require 'pp' Index: lib/rdoc/i18n.rb =================================================================== --- lib/rdoc/i18n.rb (revision 59685) +++ lib/rdoc/i18n.rb (revision 59686) @@ -1,6 +1,6 @@ https://github.com/ruby/ruby/blob/trunk/lib/rdoc/i18n.rb#L1 # frozen_string_literal: false ## -# This module provides i18n related features. +# This module provides i18n realated features. module RDoc::I18n Index: lib/rdoc/ruby_token.rb =================================================================== --- lib/rdoc/ruby_token.rb (revision 59685) +++ lib/rdoc/ruby_token.rb (revision 59686) @@ -26,7 +26,8 @@ module RDoc::RubyToken https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_token.rb#L26 Symbol = Integer end - def set_token_position(line, char) + def set_token_position(seek, line, char) + @prev_seek = seek @prev_line_no = line @prev_char_no = char end @@ -302,8 +303,8 @@ module RDoc::RubyToken https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_token.rb#L303 [:TkIN, TkKW, "in", :EXPR_BEG], [:TkDO, TkKW, "do", :EXPR_BEG], [:TkRETURN, TkKW, "return", :EXPR_MID], - [:TkYIELD, TkKW, "yield", :EXPR_END], - [:TkSUPER, TkKW, "super", :EXPR_END], + [:TkYIELD, TkKW, "yield", :EXPR_ARG], + [:TkSUPER, TkKW, "super", :EXPR_ARG], [:TkSELF, TkKW, "self", :EXPR_END], [:TkNIL, TkKW, "nil", :EXPR_END], [:TkTRUE, TkKW, "true", :EXPR_END], @@ -316,11 +317,12 @@ module RDoc::RubyToken https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_token.rb#L317 [:TkWHILE_MOD, TkKW], [:TkUNTIL_MOD, TkKW], [:TkALIAS, TkKW, "alias", :EXPR_FNAME], - [:TkDEFINED, TkKW, "defined?", :EXPR_END], + [:TkDEFINED, TkKW, "defined?", :EXPR_ARG], [:TklBEGIN, TkKW, "BEGIN", :EXPR_END], [:TklEND, TkKW, "END", :EXPR_END], [:Tk__LINE__, TkKW, "__LINE__", :EXPR_END], [:Tk__FILE__, TkKW, "__FILE__", :EXPR_END], + [:Tk__ENCODING__,TkKW, "__ENCODING__", :EXPR_END], [:TkIDENTIFIER, TkId], [:TkFID, TkId], @@ -328,9 +330,13 @@ module RDoc::RubyToken https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_token.rb#L330 [:TkCVAR, TkId], [:TkIVAR, TkId], [:TkCONSTANT, TkId], + [:TkHEREDOCBEG, TkId], + [:TkHEREDOCEND, TkId], [:TkINTEGER, TkVal], [:TkFLOAT, TkVal], + [:TkRATIONAL, TkVal], + [:TkIMAGINARY, TkVal], [:TkSTRING, TkVal], [:TkHEREDOC, TkVal], [:TkXSTRING, TkVal], @@ -353,6 +359,7 @@ module RDoc::RubyToken https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_token.rb#L359 [:TkNEQ, TkOp, "!="], [:TkGEQ, TkOp, ">="], [:TkLEQ, TkOp, "<="], + [:TkHASHROCKET, TkOp, "=>"], [:TkANDOP, TkOp, "&&"], [:TkOROP, TkOp, "||"], [:TkMATCH, TkOp, "=~"], @@ -367,6 +374,7 @@ module RDoc::RubyToken https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_token.rb#L374 [:TkCOLON3, TkOp, '::'], #[:OPASGN, TkOp], # +=, -= etc. # [:TkASSOC, TkOp, "=>"], + [:TkLAMBDA, TkOp, "->"], [:TkQUESTION, TkOp, "?"], #? [:TkCOLON, TkOp, ":"], #: @@ -394,6 +402,7 @@ module RDoc::RubyToken https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_token.rb#L402 [:TkASSIGN, Token, "="], [:TkDOT, Token, "."], + [:TkSAFENAV, Token, "&."], [:TkLPAREN, Token, "("], #(exp) [:TkLBRACK, Token, "["], #[arry] [:TkLBRACE, Token, "{"], #{hash} Index: lib/rdoc/context.rb =================================================================== --- lib/rdoc/context.rb (revision 59685) +++ lib/rdoc/context.rb (revision 59686) @@ -762,7 +762,7 @@ class RDoc::Context < RDoc::CodeObject https://github.com/ruby/ruby/blob/trunk/lib/rdoc/context.rb#L762 attributes.default = [] sort_sections.each do |section| - yield section, constants[section].sort, attributes[section].sort + yield section, constants[section].select(&:display?).sort, attributes[section].select(&:display?).sort end end Index: lib/rdoc/ruby_lex.rb =================================================================== --- lib/rdoc/ruby_lex.rb (revision 59685) +++ lib/rdoc/ruby_lex.rb (revision 59686) @@ -45,6 +45,7 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L45 attr_accessor :continue attr_accessor :lex_state + attr_accessor :first_in_method_statement attr_reader :reader class << self @@ -106,10 +107,15 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L107 @rests = [] @seek = 0 + @heredoc_queue = [] + @indent = 0 @indent_stack = [] @lex_state = :EXPR_BEG @space_seen = false + @escaped_nl = false + @first_in_method_statement = false + @after_question = false @continue = false @line = "" @@ -350,6 +356,7 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L356 begin tk = @OP.match(self) @space_seen = tk.kind_of?(TkSPACE) + @first_in_method_statement = false if !@space_seen && @first_in_method_statement rescue SyntaxError => e raise Error, "syntax error: #{e.message}" if @exception_on_syntax_error @@ -361,6 +368,28 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L368 if @readed_auto_clean_up get_readed end + + if TkSYMBEG === tk then + tk1 = token + set_token_position tk.seek, tk.line_no, tk.char_no + + case tk1 + when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then + if tk1.respond_to?(:name) then + tk = Token(TkSYMBOL, ":" + tk1.name) + else + tk = Token(TkSYMBOL, ":" + tk1.text) + end + else + tk = tk1 + end + elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then + tk1 = token + set_token_position tk.seek, tk.line_no, tk.char_no + tk = Token(tk1.class, tk.text + tk1.text) + end + @after_question = false if @after_question and !(TkQUESTION === tk) + # Tracer.off tk end @@ -380,7 +409,9 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L409 "r" => "/", "w" => "]", "W" => "]", - "s" => ":" + "s" => ":", + "i" => "]", + "I" => "]" } PERCENT_PAREN = { @@ -430,15 +461,18 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L461 proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do |op, io| @ltype = "=" - res = '' - nil until getc == "\n" + res = op + until (ch = getc) == "\n" do + res << ch + end + res << ch until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do (ch = getc) res << ch end - gets # consume =end + res << gets # consume =end @ltype = nil Token(TkRD_COMMENT, res) @@ -446,42 +480,90 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L480 @OP.def_rule("\n") do |op, io| print "\\n\n" if RDoc::RubyLex.debug? + unless @heredoc_queue.empty? + info = @heredoc_queue[0] + if !info[:started] # "\n" + info[:started] = true + ungetc "\n" + elsif info[:heredoc_end].nil? # heredoc body + tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent]) + info[:heredoc_end] = heredoc_end + ungetc "\n" + else # heredoc end + @heredoc_queue.shift + @lex_state = :EXPR_BEG + tk = Token(TkHEREDOCEND, info[:heredoc_end]) + if !@heredoc_queue.empty? + @heredoc_queue[0][:started] = true + ungetc "\n" + end + end + end + unless tk + case @lex_state + when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT + @continue = true + else + @continue = false + @lex_state = :EXPR_BEG unless @escaped_nl + until (@indent_stack.empty? || + [TkLPAREN, TkLBRACK, TkLBRACE, + TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last)) + @indent_stack.pop + end + end + @current_readed = @readed + @here_readed.clear + tk = Token(TkNL) + end + @escaped_nl = false + tk + end + + @OP.def_rules("=") do + |op, io| case @lex_state - when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT - @continue = true + when :EXPR_FNAME, :EXPR_DOT + @lex_state = :EXPR_ARG else - @continue = false @lex_state = :EXPR_BEG - until (@indent_stack.empty? || - [TkLPAREN, TkLBRACK, TkLBRACE, - TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last)) - @indent_stack.pop - end end - @current_readed = @readed - @here_readed.clear - Token(TkNL) + Token(op) end @OP.def_rules("*", "**", - "=", "==", "===", + "==", "===", "=~", "<=>", "<", "<=", - ">", ">=", ">>") do + ">", ">=", ">>", "=>") do |op, io| case @lex_state when :EXPR_FNAME, :EXPR_DOT + tk = Token(TkId, op) @lex_state = :EXPR_ARG else + tk = Token(op) @lex_state = :EXPR_BEG end + tk + end + + @OP.def_rules("->") do + |op, io| + @lex_state = :EXPR_ENDFN Token(op) end @OP.def_rules("!", "!=", "!~") do |op, io| - @lex_state = :EXPR_BEG - Token(op) + case @lex_state + when :EXPR_FNAME, :EXPR_DOT + @lex_state = :EXPR_ARG + Token(TkId, op) + else + @lex_state = :EXPR_BEG + Token(op) + end end @OP.def_rules("<<") do @@ -490,16 +572,17 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L572 if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS && (@lex_state != :EXPR_ARG || @space_seen) c = peek(0) - if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-") - tk = identify_here_document + if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~") + tk = identify_here_document(op) end end unless tk - tk = Token(op) case @lex_state when :EXPR_FNAME, :EXPR_DOT + tk = Token(TkId, op) @lex_state = :EXPR_ARG else + tk = Token(op) @lex_state = :EXPR_BEG end end @@ -513,9 +596,9 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L596 @OP.def_rules("`") do |op, io| - if @lex_state == :EXPR_FNAME - @lex_state = :EXPR_END - Token(op) + if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state + @lex_state = :EXPR_ARG + Token(TkId, op) else identify_string(op) end @@ -525,6 +608,7 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L608 |op, io| if @lex_state == :EXPR_END @lex_state = :EXPR_BEG + @after_question = true Token(TkQUESTION) else ch = getc @@ -534,17 +618,31 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L618 Token(TkQUESTION) else @lex_state = :EXPR_END + ch << getc if "\\" == ch Token(TkCHAR, "?#{ch}") end end end - @OP.def_rules("&", "&&", "|", "||") do + @OP.def_rules("&&", "||") do |op, io| @lex_state = :EXPR_BEG Token(op) end + @OP.def_rules("&", "|") do + |op, io| + case @lex_state + when :EXPR_FNAME, :EXPR_DOT + tk = Token(TkId, op) + @lex_state = :EXPR_ARG + else + tk = Token(op) + @lex_state = :EXPR_BEG + end + tk + end + @OP.def_rules("+=", "-=", "*=", "**=", "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io| @@ -556,19 +654,22 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L654 @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do |op, io| @lex_state = :EXPR_ARG - Token(op) + Token(TkId, op) end @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do |op, io| @lex_state = :EXPR_ARG - Token(op) + Token(TkId, op) end @OP.def_rules("+", "-") do |op, io| catch(:RET) do - if @lex_state == :EXPR_ARG + if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state + tk = Token(TkId, op) + @lex_state = :EXPR_ARG + elsif @lex_state == :EXPR_ARG if @space_seen and peek(0) =~ /[0-9]/ throw :RET, identify_number(op) else @@ -579,20 +680,21 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L680 else @lex_state = :EXPR_BEG end - Token(op) + tk = Token(op) unless tk + tk end end - @OP.def_rule(".") do + @OP.def_rules(".", "&.") do |op, io| @lex_state = :EXPR_BEG if peek(0) =~ /[0-9]/ ungetc identify_number else - # for "obj.if" etc. + # for "obj.if" or "obj&.if" etc. @lex_state = :EXPR_DOT - Token(TkDOT) + Token(op) end end @@ -639,7 +741,10 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L741 @OP.def_rule("/") do |op, io| - if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID + if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state + @lex_state = :EXPR_ARG + Token(TkId, op) + elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID || @first_in_method_statement identify_string(op) elsif peek(0) == '=' getc @@ -655,8 +760,15 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L760 @OP.def_rules("^") do |op, io| - @lex_state = :EXPR_BEG - Token("^") + case @lex_state + when :EXPR_FNAME, :EXPR_DOT + tk = Token(TkId, op) + @lex_state = :EXPR_ARG + else + tk = Token(op) + @lex_state = :EXPR_BEG + end + tk end # @OP.def_rules("^=") do @@ -683,8 +795,14 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L795 @OP.def_rule("~") do |op, io| - @lex_state = :EXPR_BEG - Token("~") + case @lex_state + when :EXPR_FNAME, :EXPR_DOT + @lex_state = :EXPR_ARG + Token(TkId, op) + else + @lex_state = :EXPR_BEG + Token(op) + end end @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do @@ -710,17 +828,18 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L828 @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do |op, io| @lex_state = :EXPR_ARG - Token("[]") + Token(TkId, op) end @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do |op, io| @lex_state = :EXPR_ARG - Token("[]=") + Token(TkId, op) end @OP.def_rule("[") do |op, io| + text = nil @indent += 1 if @lex_state == :EXPR_FNAME tk_c = TkfLBRACK @@ -729,13 +848,25 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L848 tk_c = TkLBRACK elsif @lex_state == :EXPR_ARG && @space_seen tk_c = TkLBRACK + elsif @lex_state == :EXPR_DOT + if peek(0) == "]" + tk_c = TkIDENTIFIER + getc + if peek(0) == "=" + text = "[]=" + else + text = "[]" + end + else + tk_c = TkOp + end else tk_c = TkfLBRACK end @lex_state = :EXPR_BEG end @indent_stack.push tk_c - Token(tk_c) + Token(tk_c, text) end @OP.def_rule("{") do @@ -753,23 +884,25 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L884 @OP.def_rule('\\') do |op, io| - if getc == "\n" + if peek(0) == "\n" @space_seen = true @continue = true - Token(TkSPACE) - else - ungetc - Token("\\") + @escaped_nl = true end + Token("\\") end @OP.def_rule('%') do |op, io| - if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID + if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state + @lex_state = :EXPR_ARG + Token(TkId, op) + elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID identify_quotation elsif peek(0) == '=' getc - Token(TkOPASGN, :%) + @lex_state = :EXPR_BEG + Token(TkOPASGN, '%') elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/ identify_quotation else @@ -871,7 +1004,7 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L1004 ungetc - if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "=" + if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/ token.concat getc end @@ -944,44 +1077,63 @@ class RDoc::RubyLex https://github.com/ruby/ruby/blob/trunk/lib/rdoc/ruby_lex.rb#L1077 @lex_state = :EXPR_END end end + if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/) + token.concat getc + token_c = TkSYMBOL + end return Token(token_c, token) end end if @lex_state == :EXPR_FNAME @lex_state = :EXPR_END - if peek(0) == '=' + if peek(0) == '=' and peek(1) != '>' token.concat getc end elsif @lex_state == :EXPR_BEG || @lex_state == : (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/