ruby-changes:49667
From: shyouhei <ko1@a...>
Date: Fri, 12 Jan 2018 17:38:17 +0900 (JST)
Subject: [ruby-changes:49667] shyouhei:r61784 (trunk): delete tool/instruction.rb (2nd try)
shyouhei 2018-01-12 17:38:09 +0900 (Fri, 12 Jan 2018) New Revision: 61784 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=61784 Log: delete tool/instruction.rb (2nd try) Previous commit changed insns.def format. Now is the time for its generators. In doing so I chose to modernize the system, not just patch. My attempt includes - extensive use of Onigumo regular expressions - split from one big file (instruction.rb) into separated MVC - partial view Also, let me take this opportunity to kill old unused features such as - stack caching - minsns / yasmdata which are never seriously used - yarvarch document generation (moved to doc/) - vast majority of unused arguments to insns2vm.rb This commit generates VM source codes that cleanly compile, and the generated binary passes tests. At least for me. Added directories: trunk/tool/ruby_vm/controllers/ trunk/tool/ruby_vm/helpers/ trunk/tool/ruby_vm/loaders/ trunk/tool/ruby_vm/models/ trunk/tool/ruby_vm/views/ Added files: trunk/doc/yarvarch.en trunk/doc/yarvarch.ja trunk/tool/ruby_vm/controllers/application_controller.rb trunk/tool/ruby_vm/helpers/c_escape.rb trunk/tool/ruby_vm/helpers/dumper.rb trunk/tool/ruby_vm/helpers/scanner.rb trunk/tool/ruby_vm/loaders/insns_def.rb trunk/tool/ruby_vm/loaders/opt_insn_unif_def.rb trunk/tool/ruby_vm/loaders/opt_operand_def.rb trunk/tool/ruby_vm/loaders/vm_opts_h.rb trunk/tool/ruby_vm/models/attribute.rb trunk/tool/ruby_vm/models/bare_instructions.rb trunk/tool/ruby_vm/models/c_expr.rb trunk/tool/ruby_vm/models/instructions.rb trunk/tool/ruby_vm/models/instructions_unifications.rb trunk/tool/ruby_vm/models/operands_unifications.rb trunk/tool/ruby_vm/models/trace_instructions.rb trunk/tool/ruby_vm/models/typemap.rb trunk/tool/ruby_vm/scripts/insns2vm.rb trunk/tool/ruby_vm/views/_attributes.erb trunk/tool/ruby_vm/views/_c_expr.erb trunk/tool/ruby_vm/views/_copyright.erb trunk/tool/ruby_vm/views/_insn_entry.erb trunk/tool/ruby_vm/views/_insn_len_info.erb trunk/tool/ruby_vm/views/_insn_name_info.erb trunk/tool/ruby_vm/views/_insn_operand_info.erb trunk/tool/ruby_vm/views/_insn_stack_increase.erb trunk/tool/ruby_vm/views/_insn_type_chars.erb trunk/tool/ruby_vm/views/_notice.erb trunk/tool/ruby_vm/views/_trace_instruction.erb trunk/tool/ruby_vm/views/insns.inc.erb trunk/tool/ruby_vm/views/insns_info.inc.erb trunk/tool/ruby_vm/views/opt_sc.inc.erb trunk/tool/ruby_vm/views/optinsn.inc.erb trunk/tool/ruby_vm/views/optunifs.inc.erb trunk/tool/ruby_vm/views/vm.inc.erb trunk/tool/ruby_vm/views/vmtc.inc.erb Removed files: trunk/template/insns.inc.tmpl trunk/template/insns_info.inc.tmpl trunk/template/minsns.inc.tmpl trunk/template/opt_sc.inc.tmpl trunk/template/optinsn.inc.tmpl trunk/template/optunifs.inc.tmpl trunk/template/vm.inc.tmpl trunk/template/vmtc.inc.tmpl trunk/template/yarvarch.en trunk/template/yarvarch.ja trunk/template/yasmdata.rb.tmpl trunk/tool/instruction.rb Modified files: trunk/Makefile.in trunk/common.mk trunk/compile.c trunk/tool/insns2vm.rb trunk/vm_exec.h trunk/win32/Makefile.sub Index: tool/instruction.rb =================================================================== --- tool/instruction.rb (revision 61783) +++ tool/instruction.rb (nonexistent) @@ -1,1250 +0,0 @@ https://github.com/ruby/ruby/blob/trunk/tool/instruction.rb#L0 -#!./miniruby -# -*- coding: us-ascii -*- -# -# This library is used by insns2vm.rb as part of autogenerating -# instruction files with .inc extensions like insns.inc and vm.inc. - -require 'erb' -$:.unshift(File.dirname(__FILE__)) -require 'vpath' - -class RubyVM - class Instruction - def initialize name, opes, pops, rets, comm, body, tvars, sp_inc, - orig = self, defopes = [], type = nil, - nsc = [], psc = [[], []] - - @name = name - @opes = opes # [[type, name], ...] - @pops = pops # [[type, name], ...] - @rets = rets # [[type, name], ...] - @comm = comm # {:c => category, :e => en desc, :j => ja desc} - @body = body # '...' - - @orig = orig - @defopes = defopes - @type = type - @tvars = tvars - - @nextsc = nsc - @pushsc = psc - @sc = [] - @unifs = [] - @optimized = [] - @is_sc = false - @sp_inc = sp_inc - @trace = trace - end - - def add_sc sci - @sc << sci - sci.set_sc - end - - attr_reader :name, :opes, :pops, :rets - attr_reader :body, :comm - attr_reader :nextsc, :pushsc - attr_reader :orig, :defopes, :type - attr_reader :sc - attr_reader :unifs, :optimized - attr_reader :is_sc - attr_reader :tvars - attr_reader :sp_inc - attr_accessor :trace - - def set_sc - @is_sc = true - end - - def add_unif insns - @unifs << insns - end - - def add_optimized insn - @optimized << insn - end - - def sp_increase_c_expr - if(pops.any?{|t, v| v == '...'} || - rets.any?{|t, v| v == '...'}) - # user definition - raise "no sp increase definition" if @sp_inc.nil? - ret = "int inc = 0;\n" - - @opes.each_with_index{|(t, v), i| - if (t == 'rb_num_t' && ((re = /\b#{v}\b/n) =~ @sp_inc)) || - (@defopes.any?{|t, val| re =~ val}) - ret << " int #{v} = FIX2INT(opes[#{i}]);\n" - elsif (t == 'CALL_INFO' && ((re = /\b#{v}\b/n) =~ @sp_inc)) - ret << " CALL_INFO #{v} = (CALL_INFO)(opes[#{i}]);\n" - end - } - - @defopes.each_with_index{|((t, var), val), i| - if t == 'rb_num_t' && val != '*' && /\b#{var}\b/ =~ @sp_inc - ret << " #{t} #{var} = #{val};\n" - end - } - - ret << " #{@sp_inc};\n" - ret << " return depth + inc;" - ret - else - "return depth + #{rets.size - pops.size};" - end - end - - def inspect - "#<Instruction:#{@name}>" - end - end - - class InstructionsLoader - def initialize opts = {} - @insns = [] - @insn_map = {} - - @vpath = opts[:VPATH] || File - @use_const = opts[:use_const] - @verbose = opts[:verbose] - @destdir = opts[:destdir] - - (@vm_opts = load_vm_opts).each {|k, v| - @vm_opts[k] = opts[k] if opts.key?(k) - } - - load_insns_def opts[:"insns.def"] || 'insns.def' - - load_opt_operand_def opts[:"opope.def"] || 'defs/opt_operand.def' - load_insn_unification_def opts[:"unif.def"] || 'defs/opt_insn_unif.def' - make_stackcaching_insns if vm_opt?('STACK_CACHING') - make_trace_insns - end - - attr_reader :vpath - attr_reader :destdir - - %w[use_const verbose].each do |attr| - attr_reader attr - alias_method "#{attr}?", attr - remove_method attr - end - - def [](s) - @insn_map[s.to_s] - end - - def each - @insns.each{|insn| - yield insn - } - end - - def size - @insns.size - end - - ### - private - - def vm_opt? name - @vm_opts[name] - end - - def load_vm_opts file = nil - file ||= 'vm_opts.h' - opts = {} - vpath.open(file) do |f| - f.grep(/^\#define\s+OPT_([A-Z_]+)\s+(\d+)/) do - opts[$1] = !$2.to_i.zero? - end - end - opts - end - - SKIP_COMMENT_PATTERN = Regexp.compile(Regexp.escape('/** ##skip')) - - include Enumerable - - def add_insn insn - @insns << insn - @insn_map[insn.name] = insn - end - - def make_insn name, opes, pops, rets, comm, body, sp_inc - add_insn Instruction.new(name, opes, pops, rets, comm, body, [], sp_inc) - end - - # str -> [[type, var], ...] - def parse_vars line - raise unless /\((.*?)\)/ =~ line - vars = $1.split(',') - vars.map!{|v| - if /\s*(\S+)\s+(\S+)\s*/ =~ v - type = $1 - var = $2 - elsif /\s*\.\.\.\s*/ =~ v - type = var = '...' - else - raise - end - [type, var] - } - vars - end - - def parse_comment comm - c = 'others' - j = '' - e = '' - comm.each_line{|line| - case line - when /@c (.+)/ - c = $1 - when /@e (.+)/ - e = $1 - when /@e\s*$/ - e = '' - when /@j (.+)$/ - j = $1 - when /@j\s*$/ - j = '' - end - } - { :c => c, - :e => e, - :j => j, - } - end - - def load_insns_def file - body = insn = opes = pops = rets = nil - comment = '' - - vpath.open(file) {|f| - f.instance_variable_set(:@line_no, 0) - class << f - def line_no - @line_no - end - def gets - @line_no += 1 - super - end - end - - while line = f.gets - line.chomp! - case line - - when SKIP_COMMENT_PATTERN - while line = f.gets.chomp - if /\s+\*\/$/ =~ line - break - end - end - - # collect instruction comment - when /^\/\*\*$/ - while line = f.gets - if /\s+\*\/\s*$/ =~ line - break - else - comment << line - end - end - - # start instruction body - when /^DEFINE_INSN$/ - insn = f.gets.chomp - opes = parse_vars(f.gets.chomp) - pops = parse_vars(f.gets.chomp).reverse - rets_str = f.gets.chomp - rets = parse_vars(rets_str).reverse - comment = parse_comment(comment) - insn_in = true - body = '' - - - when /^\/\/ attr rb_snum_t sp_inc = (.+)$/ - sp_inc = 'inc +=' + $1 - - when /^\{$/ - line_no = f.line_no - - # end instruction body - when /^\}/ - if insn_in - body.instance_variable_set(:@line_no, line_no) - body.instance_variable_set(:@file, f.path) - insn = make_insn(insn, opes, pops, rets, comment, body, sp_inc) - insn_in = false - comment = '' - end - - else - if insn_in - body << line + "\n" - end - end - end - } - end - - ## opt op - def load_opt_operand_def file - vpath.foreach(file) {|line| - line = line.gsub(/\#.*/, '').strip - next if line.length == 0 - break if /__END__/ =~ line - /(\S+)\s+(.+)/ =~ line - insn = $1 - opts = $2 - add_opt_operand insn, opts.split(/,/).map{|e| e.strip} - } if file - end - - def label_escape label - label.gsub(/\(/, '_O_'). - gsub(/\)/, '_C_'). - gsub(/\*/, '_WC_') - end - - def add_opt_operand insn_name, opts - insn = @insn_map[insn_name] - opes = insn.opes - - if opes.size != opts.size - raise "operand size mismatch for #{insn.name} (opes: #{opes.size}, opts: #{opts.size})" - end - - ninsn = insn.name + '_OP_' + opts.map{|e| label_escape(e)}.join('_') - nopes = [] - defv = [] - - opts.each_with_index{|e, i| - if e == '*' - nopes << opes[i] - end - defv << [opes[i], e] - } - - make_insn_operand_optimized(insn, ninsn, nopes, defv) - end - - def make_insn_operand_optimized orig_insn, name, opes, defopes - comm = orig_insn.comm.dup - comm[:c] = 'optimize' - add_insn insn = Instruction.new( - name, opes, orig_insn.pops, orig_insn.rets, comm, - orig_insn.body, orig_insn.tvars, orig_insn.sp_inc, - orig_insn, defopes) - orig_insn.add_optimized insn - end - - ## insn unif - def load_insn_unification_def file - vpath.foreach(file) {|line| - line = line.gsub(/\#.*/, '').strip - next if line.length == 0 - break if /__END__/ =~ line - make_unified_insns line.split.map{|e| - raise "unknown insn: #{e}" unless @insn_map[e] - @insn_map[e] - } - } if file - end - - def all_combination sets - ret = sets.shift.map{|e| [e]} - - sets.each{|set| - prev = ret - ret = [] - prev.each{|ary| - set.each{|e| - eary = ary.dup - eary << e - ret << eary - } - } - } - ret - end - - def make_unified_insns insns - if vm_opt?('UNIFY_ALL_COMBINATION') - insn_sets = insns.map{|insn| - [insn] + insn.optimized - } - - all_combination(insn_sets).each{|insns_set| - make_unified_insn_each insns_set - } - else - make_unified_insn_each insns - end - end - - def mk_private_val vals, i, redef - vals.dup.map{|v| - # v[0] : type - # v[1] : var name - - v = v.dup - if v[0] != '...' - redef[v[1]] = v[0] - v[1] = "#{v[1]}_#{i}" - end - v - } - end - - def mk_private_val2 vals, i, redef - vals.dup.map{|v| - # v[0][0] : type - # v[0][1] : var name - # v[1] : default val - - pv = v.dup - v = pv[0] = pv[0].dup - if v[0] != '...' - redef[v[1]] = v[0] - v[1] = "#{v[1]}_#{i}" - end - pv - } - end - - def make_unified_insn_each insns - names = [] - opes = [] - pops = [] - rets = [] - comm = { - :c => 'optimize', - :e => 'unified insn', - :j => 'unified insn', - } - body = '' - passed = [] - tvars = [] - defopes = [] - sp_inc = '' - - insns.each_with_index{|insn, i| - names << insn.name - redef_vars = {} - - e_opes = mk_private_val(insn.opes, i, redef_vars) - e_pops = mk_private_val(insn.pops, i, redef_vars) - e_rets = mk_private_val(insn.rets, i, redef_vars) - # ToDo: fix it - e_defs = mk_private_val2(insn.defopes, i, redef_vars) - - passed_vars = [] - while pvar = e_pops.pop - rvar = rets.pop - if rvar - raise "unsupported unif insn: #{insns.inspect}" if rvar[0] == '...' - passed_vars << [pvar, rvar] - tvars << rvar - else - e_pops.push pvar - break - end - end - - opes.concat e_opes - pops.concat e_pops - rets.concat e_rets - defopes.concat e_defs - sp_inc << "#{insn.sp_inc}" - - body << "{ /* unif: #{i} */\n" + - passed_vars.map{|rpvars| - pv = rpvars[0] - rv = rpvars[1] - "#define #{pv[1]} #{rv[1]}" - }.join("\n") + - "\n" + - redef_vars.map{|v, type| - "#{type} #{v} = #{v}_#{i};" - }.join("\n") + "\n" - if line = insn.body.instance_variable_get(:@line_no) - file = insn.body.instance_variable_get(:@file) - body << "#line #{line+1} \"#{file}\"\n" - body << insn.body - body << "\n#line __CURRENT_LINE__ \"__CURRENT_FILE__\"\n" - else - body << insn.body - end - body << redef_vars.keys.map{|v| - "#{v}_#{i} = #{v};" - }.join("\n") + - "\n" + - passed_vars.map{|rpvars| - "#undef #{rpvars[0][1]}" - }.join("\n") + - "\n}\n" - } - - tvars_ary = [] - tvars.each{|tvar| - unless opes.any?{|var| - var[1] == tvar[1] - } || defopes.any?{|pvar| - pvar[0][1] == tvar[1] - } - tvars_ary << tvar - end - } - add_insn insn = Instruction.new("UNIFIED_" + names.join('_'), - opes, pops, rets.reverse, comm, body, - tvars_ary, sp_inc) - insn.defopes.replace defopes - insns[0].add_unif [insn, insns] - end - - ## sc - SPECIAL_INSN_FOR_SC_AFTER = { - /\Asend/ => [:a], - /\Aend/ => [:a], - /\Ayield/ => [:a], - /\Aclassdef/ => [:a], - /\Amoduledef/ => [:a], - } - FROM_SC = [[], [:a], [:b], [:a, :b], [:b, :a]] - - def make_stackcaching_insns - pops = rets = nil - - @insns.dup.each{|insn| - opops = insn.pops - orets = insn.rets - oopes = insn.opes - ocomm = insn.comm - oname = insn.name - - after = SPECIAL_INSN_FOR_SC_AFTER.find {|k, v| k =~ oname} - - insns = [] - FROM_SC.each{|from| - name, pops, rets, pushs1, pushs2, nextsc = - *calc_stack(insn, from, after, opops, orets) - - make_insn_sc(insn, name, oopes, pops, rets, [pushs1, pushs2], nextsc) - } - } - end - - def make_trace_insns - @insns.dup.each{|insn| - body = <<-EOS - vm_trace(ec, GET_CFP(), GET_PC()); - DISPATCH_ORIGINAL_INSN(#{insn.name}); - EOS - - trace_insn = Instruction.new(name = "trace_#{insn.name}", - insn.opes, insn.pops, insn.rets, insn.comm, - body, insn.tvars, insn.sp_inc) - trace_insn.trace = true - add_insn trace_insn - } - end - - def make_insn_sc orig_insn, name, opes, pops, rets, pushs, nextsc - comm = orig_insn.comm.dup - comm[:c] = 'optimize(sc)' - - scinsn = Instruction.new( - name, opes, pops, rets, comm, - orig_insn.body, orig_insn.tvars, orig_insn.sp_inc, - orig_insn, orig_insn.defopes, :sc, nextsc, pushs) - - add_insn scinsn - orig_insn.add_sc scinsn - end - - def self.complement_name st - "#{st[0] ? st[0] : 'x'}#{st[1] ? st[1] : 'x'}" - end - - def add_stack_value st - len = st.length - if len == 0 - st[0] = :a - [nil, :a] - elsif len == 1 - if st[0] == :a - st[1] = :b - else - st[1] = :a - end - [nil, st[1]] - else - st[0], st[1] = st[1], st[0] - [st[1], st[1]] - end - end - - def calc_stack insn, ofrom, oafter, opops, orets - from = ofrom.dup - pops = opops.dup - rets = orets.dup - rest_scr = ofrom.dup - - pushs_before = [] - pushs= [] - - pops.each_with_index{|e, i| - if e[0] == '...' - pushs_before = from - from = [] - end - r = from.pop - break unless r - pops[i] = pops[i].dup << r - } - - if oafter - from = oafter - from.each_with_index{|r, i| - rets[i] = rets[i].dup << r if rets[i] - } - else - rets = rets.reverse - rets.each_with_index{|e, i| - break if e[0] == '...' - pushed, r = add_stack_value from - rets[i] = rets[i].dup << r - if pushed - if rest_scr.pop - pushs << pushed - end - - if i - 2 >= 0 - rets[i-2].pop - end - end - } - end - - if false #|| insn.name =~ /test3/ - p ofrom - p pops - p rets - p pushs_before - p pushs - p from - exit - end - - ret = ["#{insn.name}_SC_#{InstructionsLoader.complement_name(ofrom)}_#{complement_name(from)}", - pops, rets, pushs_before, pushs, from] - end - end - - class SourceCodeGenerator - def initialize insns - @insns = insns - end - - attr_reader :insns - - def generate - raise "should not reach here" - end - - def vpath - @insns.vpath - end - - def verbose? - @insns.verbose? - end - - def use_const? - @insns.use_const? - end - - def template(name) - ERB.new(vpath.read("template/#{name}"), nil, '%-') - end - - def build_string - @lines = [] - yield - @lines.join("\n") - end - - EMPTY_STRING = ''.freeze - - def commit str = EMPTY_STRING - @lines << str - end - - def comment str - @lines << str if verbose? - end - - def output_path(fn) - d = @insns.destdir - fn = File.join(d, fn) if d - fn - end - end - - ################################################################### - # vm.inc - class VmBodyGenerator < SourceCodeGenerator - # vm.inc - def generate - template('vm.inc.tmpl').result(binding) - end - - def generate_from_insnname insnname - make_insn_def @insns[insnname.to_s] - end - - ####### - private - - def make_header_prepare_stack insn - comment " /* prepare stack status */" - - push_ba = insn.pushsc - raise "unsupport" if push_ba[0].size > 0 && push_ba[1].size > 0 - - n = 0 - push_ba.each {|pushs| n += pushs.length} - commit " CHECK_VM_STACK_OVERFLOW_FOR_INSN(VM_REG_CFP, #{n});" if n > 0 - push_ba.each{|pushs| - pushs.each{|r| - commit " PUSH(SCREG(#{r}));" - } - } - end - - def make_header_operands insn - comment " /* declare and get from iseq */" - - vars = insn.opes - n = 0 - ops = [] - - vars.each_with_index{|(type, var), i| - if type == '...' - break - end - - # skip make operands when body has no reference to this operand - # TODO: really needed? - re = /\b#{var}\b/n - if re =~ insn.body or re =~ insn.sp_inc or insn.rets.any?{|t, v| re =~ v} or re =~ 'ic' or re =~ 'ci' or re =~ 'cc' - ops << " #{type} #{var} = (#{type})GET_OPERAND(#{i+1});" - end - - n += 1 - } - @opn = n - - # reverse or not? - # ops.join - commit ops.reverse - end - - def make_header_default_operands insn - vars = insn.defopes - - vars.each{|e| - next if e[1] == '*' - if use_const? - commit " const #{e[0][0]} #{e[0][1]} = #{e[1]};" - else - commit " #define #{e[0][1]} #{e[1]}" - end - } - end - - def make_footer_default_operands insn - comment " /* declare and initialize default opes */" - if use_const? - commit - else - vars = insn.defopes - - vars.each{|e| - next if e[1] == '*' - commit "#undef #{e[0][1]}" - } - end - end - - def make_header_stack_pops insn - comment " /* declare and pop from stack */" - - n = 0 - pops = [] - vars = insn.pops - vars.each_with_index{|iter, i| - type, var, r = *iter - if type == '...' - break - end - if r - pops << " #{type} #{var} = SCREG(#{r});" - else - pops << " #{type} #{var} = TOPN(#{n});" - n += 1 - end - } - @popn = n - - # reverse or not? - commit pops.reverse - end - - def make_header_temporary_vars insn - comment " / (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/