ruby-changes:64289
From: aycabta <ko1@a...>
Date: Sat, 19 Dec 2020 02:24:07 +0900 (JST)
Subject: [ruby-changes:64289] dc61affd67 (master): [ruby/reline] [ruby/irb] Call ripper only once when generating dynamic prompt
https://git.ruby-lang.org/ruby.git/commit/?id=dc61affd67 From dc61affd67a3e93c870bf6f79eefc0473630d6f1 Mon Sep 17 00:00:00 2001 From: aycabta <aycabta@g...> Date: Sat, 19 Dec 2020 01:52:14 +0900 Subject: [ruby/reline] [ruby/irb] Call ripper only once when generating dynamic prompt https://github.com/ruby/irb/commit/babb122a48 https://github.com/ruby/reline/commit/e6dbcb3b42 diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index cfb57e3..8507717 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -60,11 +60,19 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L60 @io.dynamic_prompt do |lines| lines << '' if lines.empty? result = [] - lines.each_index { |i| - c = lines[0..i].map{ |l| l + "\n" }.join - ltype, indent, continue, code_block_open = check_state(c) - result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + i) - } + tokens = ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join) + code = String.new + partial_tokens = [] + line_num_offset = 0 + tokens.each do |t| + code << t[2] + partial_tokens << t + if t[2].include?("\n") + ltype, indent, continue, code_block_open = check_state(code, partial_tokens) + result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset) + line_num_offset += 1 + end + end result end end @@ -88,12 +96,9 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L96 def ripper_lex_without_warning(code) verbose, $VERBOSE = $VERBOSE, nil - tokens = [] + tokens = nil self.class.compile_with_errors_suppressed(code) do |inner_code, line_no| - lexer = Ripper::Lexer.new(inner_code, '-', line_no) - until (ts = lexer.lex).empty? - tokens.concat(ts) - end + tokens = Ripper.lex(inner_code, '-', line_no) end $VERBOSE = verbose tokens @@ -124,12 +129,12 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L129 end end - def check_state(code) - @tokens = ripper_lex_without_warning(code) - ltype = process_literal_type - indent = process_nesting_level - continue = process_continue - code_block_open = check_code_block(code) + def check_state(code, tokens = nil) + tokens = ripper_lex_without_warning(code) unless tokens + ltype = process_literal_type(tokens) + indent = process_nesting_level(tokens) + continue = process_continue(tokens) + code_block_open = check_code_block(code, tokens) [ltype, indent, continue, code_block_open] end @@ -189,36 +194,36 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L194 code = @line + (line.nil? ? '' : line) code.gsub!(/\s*\z/, '').concat("\n") @tokens = ripper_lex_without_warning(code) - @continue = process_continue - @code_block_open = check_code_block(code) - @indent = process_nesting_level - @ltype = process_literal_type + @continue = process_continue(@tokens) + @code_block_open = check_code_block(code, @tokens) + @indent = process_nesting_level(@tokens) + @ltype = process_literal_type(@tokens) line end - def process_continue + def process_continue(tokens) # last token is always newline - if @tokens.size >= 2 and @tokens[-2][1] == :on_regexp_end + if tokens.size >= 2 and tokens[-2][1] == :on_regexp_end # end of regexp literal return false - elsif @tokens.size >= 2 and @tokens[-2][1] == :on_semicolon + elsif tokens.size >= 2 and tokens[-2][1] == :on_semicolon return false - elsif @tokens.size >= 2 and @tokens[-2][1] == :on_kw and ['begin', 'else', 'ensure'].include?(@tokens[-2][2]) + elsif tokens.size >= 2 and tokens[-2][1] == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2][2]) return false - elsif !@tokens.empty? and @tokens.last[2] == "\\\n" + elsif !tokens.empty? and tokens.last[2] == "\\\n" return true - elsif @tokens.size >= 1 and @tokens[-1][1] == :on_heredoc_end # "EOH\n" + elsif tokens.size >= 1 and tokens[-1][1] == :on_heredoc_end # "EOH\n" return false - elsif @tokens.size >= 2 and defined?(Ripper::EXPR_BEG) and @tokens[-2][3].anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) + elsif tokens.size >= 2 and defined?(Ripper::EXPR_BEG) and tokens[-2][3].anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) # end of literal except for regexp return true end false end - def check_code_block(code) - return true if @tokens.empty? - if @tokens.last[1] == :on_heredoc_beg + def check_code_block(code, tokens) + return true if tokens.empty? + if tokens.last[1] == :on_heredoc_beg return true end @@ -290,7 +295,7 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L295 end if defined?(Ripper::EXPR_BEG) - last_lex_state = @tokens.last[3] + last_lex_state = tokens.last[3] if last_lex_state.allbits?(Ripper::EXPR_BEG) return false elsif last_lex_state.allbits?(Ripper::EXPR_DOT) @@ -309,10 +314,10 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L314 false end - def process_nesting_level + def process_nesting_level(tokens) indent = 0 in_oneliner_def = nil - @tokens.each_with_index { |t, index| + tokens.each_with_index { |t, index| # detecting one-liner method definition if in_oneliner_def.nil? if t[3].allbits?(Ripper::EXPR_ENDFN) @@ -340,10 +345,10 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L345 when :on_rbracket, :on_rbrace, :on_rparen indent -= 1 when :on_kw - next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME) + next if index > 0 and tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME) case t[2] when 'do' - if index > 0 and @tokens[index - 1][3].anybits?(Ripper::EXPR_CMDARG | Ripper::EXPR_ENDFN | Ripper::EXPR_ARG) + if index > 0 and tokens[index - 1][3].anybits?(Ripper::EXPR_CMDARG | Ripper::EXPR_ENDFN | Ripper::EXPR_ARG) # method_with_block do; end indent += 1 else @@ -525,12 +530,12 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L530 corresponding_token_depth end - def check_string_literal + def check_string_literal(tokens) i = 0 start_token = [] end_type = [] - while i < @tokens.size - t = @tokens[i] + while i < tokens.size + t = tokens[i] case t[1] when :on_tstring_beg start_token << t @@ -540,7 +545,7 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L545 end_type << :on_regexp_end when :on_symbeg acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw} - if (i + 1) < @tokens.size and acceptable_single_tokens.all?{ |t| @tokens[i + 1][1] != t } + if (i + 1) < tokens.size and acceptable_single_tokens.all?{ |t| tokens[i + 1][1] != t } start_token << t end_type << :on_tstring_end end @@ -562,8 +567,8 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L567 start_token.last.nil? ? '' : start_token.last end - def process_literal_type - start_token = check_string_literal + def process_literal_type(tokens) + start_token = check_string_literal(tokens) case start_token[1] when :on_tstring_beg case start_token[2] -- cgit v0.10.2 -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/