ruby-changes:66741
From: Jeremy <ko1@a...>
Date: Sat, 10 Jul 2021 13:44:40 +0900 (JST)
Subject: [ruby-changes:66741] 289fd3c801 (master): [ruby/irb] Pass local variables from workspace binding to lexer
https://git.ruby-lang.org/ruby.git/commit/?id=289fd3c801 From 289fd3c801495b8188b8549b5a095cd479d048de Mon Sep 17 00:00:00 2001 From: Jeremy Evans <code@j...> Date: Fri, 9 Jul 2021 21:44:01 -0700 Subject: [ruby/irb] Pass local variables from workspace binding to lexer This fixes at least an issue where irb will incorrectly assume code opens a heredoc when it does not, such as this code: ```ruby s1 = 'testing' s2 = 'this' s2 <<s1 p s1 s1 ``` Ruby parses the `s2 <<s1` as `s2.<<(s1)`, not as a heredoc, because `s2` is a local variable in scope. irb was using ripper without letting ripper know that `s2` was a local variable, so ripper would lex it as a heredoc instead of a method call. Fix the situation by prepending a line at line 0 with all local variable definitions in scope whenever lexing. This fixes the heredoc issue, and potentially other issues that depend on whether an identifier is a local variable or not. Fixes [Bug #17530] https://github.com/ruby/irb/commit/4ed2187f76 --- lib/irb.rb | 2 +- lib/irb/ruby-lex.rb | 49 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/lib/irb.rb b/lib/irb.rb index 038d45f..661c550 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -524,7 +524,7 @@ module IRB https://github.com/ruby/ruby/blob/trunk/lib/irb.rb#L524 @context.io.prompt end - @scanner.set_input(@context.io) do + @scanner.set_input(@context.io, context: @context) do signal_status(:IN_INPUT) do if l = @context.io.gets print l if @context.verbose? diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index 3b3b9b3..300bec9 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -30,26 +30,31 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L30 @prompt = nil end - def self.compile_with_errors_suppressed(code) - line_no = 1 + def self.compile_with_errors_suppressed(code, line_no: 1) begin result = yield code, line_no rescue ArgumentError + # Ruby can issue an error for the code if there is an + # incomplete magic comment for encoding in it. Force an + # expression with a new line before the code in this + # case to prevent magic comment handling. To make sure + # line numbers in the lexed code remain the same, + # decrease the line number by one. code = ";\n#{code}" - line_no = 0 + line_no -= 1 result = yield code, line_no end result end # io functions - def set_input(io, p = nil, &block) + def set_input(io, p = nil, context: nil, &block) @io = io if @io.respond_to?(:check_termination) @io.check_termination do |code| if Reline::IOGate.in_pasting? lex = RubyLex.new - rest = lex.check_termination_in_prev_line(code) + rest = lex.check_termination_in_prev_line(code, context: context) if rest Reline.delete_text rest.bytes.reverse_each do |c| @@ -61,7 +66,7 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L66 end else code.gsub!(/\s*\z/, '').concat("\n") - ltype, indent, continue, code_block_open = check_state(code) + ltype, indent, continue, code_block_open = check_state(code, context: context) if ltype or indent > 0 or continue or code_block_open false else @@ -74,7 +79,7 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L79 @io.dynamic_prompt do |lines| lines << '' if lines.empty? result = [] - tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join) + tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: context) code = String.new partial_tokens = [] unprocessed_tokens = [] @@ -86,7 +91,7 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L91 t_str = t[2] t_str.each_line("\n") do |s| code << s << "\n" - ltype, indent, continue, code_block_open = check_state(code, partial_tokens) + ltype, indent, continue, code_block_open = check_state(code, partial_tokens, context: context) result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset) line_num_offset += 1 end @@ -96,7 +101,7 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L101 end end unless unprocessed_tokens.empty? - ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens) + ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens, context: context) result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset) end result @@ -129,15 +134,25 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L134 :on_param_error ] - def self.ripper_lex_without_warning(code) + def self.ripper_lex_without_warning(code, context: nil) verbose, $VERBOSE = $VERBOSE, nil + if context + lvars = context&.workspace&.binding&.local_variables + if lvars && !lvars.empty? + code = "#{lvars.join('=')}=nil\n#{code}" + line_no = 0 + else + line_no = 1 + end + end tokens = nil - compile_with_errors_suppressed(code) do |inner_code, line_no| + compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no| lexer = Ripper::Lexer.new(inner_code, '-', line_no) if lexer.respond_to?(:scan) # Ruby 2.7+ tokens = [] pos_to_index = {} lexer.scan.each do |t| + next if t.pos.first == 0 if pos_to_index.has_key?(t[0]) index = pos_to_index[t[0]] found_tk = tokens[index] @@ -182,7 +197,7 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L197 if @io.respond_to?(:auto_indent) and context.auto_indent_mode @io.auto_indent do |lines, line_index, byte_pointer, is_newline| if is_newline - @tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n")) + @tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n"), context: context) prev_spaces = find_prev_spaces(line_index) depth_difference = check_newline_depth_difference depth_difference = 0 if depth_difference < 0 @@ -191,7 +206,7 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L206 code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join last_line = lines[line_index]&.byteslice(0, byte_pointer) code += last_line if last_line - @tokens = self.class.ripper_lex_without_warning(code) + @tokens = self.class.ripper_lex_without_warning(code, context: context) corresponding_token_depth = check_corresponding_token_depth if corresponding_token_depth corresponding_token_depth @@ -203,8 +218,8 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L218 end end - def check_state(code, tokens = nil) - tokens = self.class.ripper_lex_without_warning(code) unless tokens + def check_state(code, tokens = nil, context: nil) + tokens = self.class.ripper_lex_without_warning(code, context: context) unless tokens ltype = process_literal_type(tokens) indent = process_nesting_level(tokens) continue = process_continue(tokens) @@ -754,8 +769,8 @@ class RubyLex https://github.com/ruby/ruby/blob/trunk/lib/irb/ruby-lex.rb#L769 end end - def check_termination_in_prev_line(code) - tokens = self.class.ripper_lex_without_warning(code) + def check_termination_in_prev_line(code, context: nil) + tokens = self.class.ripper_lex_without_warning(code, context: context) past_first_newline = false index = tokens.rindex do |t| # traverse first token before last line -- cgit v1.1 -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/