ruby-changes:50069
From: k0kubun <ko1@a...>
Date: Sun, 4 Feb 2018 14:49:26 +0900 (JST)
Subject: [ruby-changes:50069] k0kubun:r62187 (trunk): common.mk: install a single header file for JIT
k0kubun 2018-02-04 14:49:21 +0900 (Sun, 04 Feb 2018) New Revision: 62187 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=62187 Log: common.mk: install a single header file for JIT compilation which is created by transforming a preprocessed vm.c. This file will be used by JIT compiler's generated code which we are going to have from succeeding commits. Makefile.in: generate MJIT header for UNIX environments. win32/Makefile.sub: generate MJIT header for mswin environments. At initial merge, we're going to support only MinGW for Windows. So the header installed by this file won't be used for short term, but we'll add mswin support in a half year or so, for sure. tool/transform_mjit_header.rb: New. This script was originally written as minimize_mjit_header.rb by Vladimir N. Makarov <vmakarov@r...> for Feature 12589. Then I refactored a little so that it can conform CodeClimate CI which is currently set for Ruby's GitHub repository, and fixed some bugs and ported it to work on Windows. Also, as original minimize_mjit_header.rb takes too long time to run, this is modified to skip minimization step because having *static* unused definitions does not waste compilation time on -O2 since compiler can skip to compile unused static functions. So this does no longer "minimize" the header and is renamed. This header installation does NOT include a header to automatically export symbols used by MJIT. That's because original MJIT code was failing to export symbols in the import header in macOS environment. But I would like to have the functionality for maintainability in the future. I'll manually export things but it would be just an intemediate solution. Patch by: Vladimir N. Makarov <vmakarov@r...> Part of: Feature 12589 and 14235. Added files: trunk/tool/transform_mjit_header.rb Modified files: trunk/.gitignore trunk/Makefile.in trunk/common.mk trunk/win32/Makefile.sub Index: .gitignore =================================================================== --- .gitignore (revision 62186) +++ .gitignore (revision 62187) @@ -195,3 +195,6 @@ lcov*.info https://github.com/ruby/ruby/blob/trunk/.gitignore#L195 # /win32/ /win32/*.ico /win32/.time + +# MJIT +/rb_mjit_header.h Index: tool/transform_mjit_header.rb =================================================================== --- tool/transform_mjit_header.rb (nonexistent) +++ tool/transform_mjit_header.rb (revision 62187) @@ -0,0 +1,183 @@ https://github.com/ruby/ruby/blob/trunk/tool/transform_mjit_header.rb#L1 +# Copyright (C) 2017 Vladimir Makarov, <vmakarov@r...> +# This is a script to transform functions to static inline. +# Usage: transform_mjit_header.rb <c-compiler> <header file> <out> + +require 'fileutils' +require 'tempfile' + +module MJITHeader + ATTR_VALUE_REGEXP = /[^()]|\([^()]*\)/ + ATTR_REGEXP = /__attribute__\s*\(\((#{ATTR_VALUE_REGEXP})*\)\)/ + FUNC_HEADER_REGEXP = /\A(\s*#{ATTR_REGEXP})*[^\[{(]*\((#{ATTR_REGEXP}|[^()])*\)(\s*#{ATTR_REGEXP})*\s*/ + + # For MinGW's ras.h. Those macros have its name in its definition and can't be preprocessed multiple times. + RECURSIVE_MACROS = %w[ + RASCTRYINFO + RASIPADDR + ] + + IGNORED_FUNCTIONS = [ + 'rb_equal_opt', # Not used from VM and not compilable + ] + + # Return start..stop of last decl in CODE ending STOP + def self.find_decl(code, stop) + level = 0 + stop.downto(0) do |i| + if level == 0 && stop != i && decl_found?(code, i) + return decl_start(code, i)..stop + elsif code[i] == '}' + level += 1 + elsif code[i] == '{' + level -= 1 + end + end + 0..-1 + end + + def self.decl_found?(code, i) + i == 0 || code[i] == ';' || code[i] == '}' + end + + def self.decl_start(code, i) + if i == 0 && code[i] != ';' && code[i] != '}' + 0 + else + i + 1 + end + end + + # Given DECL return the name of it, nil if failed + def self.decl_name_of(decl) + ident_regex = /\w+/ + decl = decl.gsub(/^#.+$/, '') # remove macros + reduced_decl = decl.gsub(/#{ATTR_REGEXP}/, '') # remove attributes + su1_regex = /{[^{}]*}/ + su2_regex = /{([^{}]|su1_regex)*}/ + su3_regex = /{([^{}]|su2_regex)*}/ # 3 nested structs/unions is probably enough + reduced_decl.gsub!(/#{su3_regex}/, '') # remove strutcs/unions in the header + id_seq_regex = /\s*(#{ident_regex}(\s+|\s*[*]+\s*))*/ + # Process function header: + match = /\A#{id_seq_regex}(?<name>#{ident_regex})\s*\(/.match(reduced_decl) + return match[:name] if match + # Process non-function declaration: + reduced_decl.gsub!(/\s*=[^;]+(?=;)/, '') # remove initialization + match = /#{id_seq_regex}(?<name>#{ident_regex})/.match(reduced_decl); + return match[:name] if match + nil + end + + # Return true if CC with CFLAGS compiles successfully the current code. + # Use STAGE in the message in case of a compilation failure + def self.check_code!(code, cc, cflags, stage) + Tempfile.open(['', '.c']) do |f| + f.puts code + f.close + unless system("#{cc} #{cflags} #{f.path} 2>#{File::NULL}") + STDERR.puts "error in #{stage} header file:" + system("#{cc} #{cflags} #{f.path}") + exit 1 + end + end + end + + # Remove unpreprocessable macros + def self.remove_harmful_macros!(code) + code.gsub!(/^#define #{Regexp.union(RECURSIVE_MACROS)} .*$/, '') + end + + # -dD outputs those macros, and it produces redefinition warnings + def self.remove_default_macros!(code) + code.gsub!(/^#define __STDC_.+$/, '') + code.gsub!(/^#define assert\([^\)]+\) .+$/, '') + end + + # This makes easier to process code + def self.separate_macro_and_code(code) + code.lines.partition { |l| !l.start_with?('#') }.flatten.join('') + end + + def self.write(code, out) + FileUtils.mkdir_p(File.dirname(out)) + File.write("#{out}.new", code) + FileUtils.mv("#{out}.new", out) + end + + # Note that this checks runruby. This conservatively covers platform names. + def self.windows? + RUBY_PLATFORM =~ /mswin|mingw|msys/ + end +end + +if ARGV.size != 3 + STDERR.puts 'Usage: transform_mjit_header.rb <c-compiler> <header file> <out>' + exit 1 +end + +cc = ARGV[0] +code = File.read(ARGV[1]) # Current version of the header file. +outfile = ARGV[2] +if cc =~ /\Acl(\z| |\.exe)/ + cflags = '-DMJIT_HEADER -Zs' +else + cflags = '-S -DMJIT_HEADER -fsyntax-only -Werror=implicit-function-declaration -Werror=implicit-int -Wfatal-errors' +end + +if MJITHeader.windows? + MJITHeader.remove_harmful_macros!(code) +end +MJITHeader.remove_default_macros!(code) + +# Check initial file correctness +MJITHeader.check_code!(code, cc, cflags, 'initial') + +if MJITHeader.windows? # transformation is broken with Windows headers for now + STDERR.puts "\nSkipped transforming external functions to static on Windows." + MJITHeader.write(code, outfile) + exit 0 +end +STDERR.puts "\nTransforming external functions to static:" + +code = MJITHeader.separate_macro_and_code(code) # note: this does not work on MinGW +stop_pos = code.match(/^#/).begin(0) # See `separate_macro_and_code`. This ignores proprocessors. +extern_names = [] + +# This loop changes function declarations to static inline. +loop do + decl_range = MJITHeader.find_decl(code, stop_pos) + break if decl_range.end < 0 + + stop_pos = decl_range.begin - 1 + decl = code[decl_range] + decl_name = MJITHeader.decl_name_of(decl) + + if MJITHeader::IGNORED_FUNCTIONS.include?(decl_name) && /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl) + STDERR.puts "transform_mjit_header: changing definition of '#{decl_name}' to declaration" + code[decl_range] = decl.sub(/{.+}/m, ';') + elsif extern_names.include?(decl_name) && (decl =~ /#{MJITHeader::FUNC_HEADER_REGEXP};/) + decl.sub!(/(extern|static|inline) /, ' ') + unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. + STDERR.puts "transform_mjit_header: making declaration of '#{decl_name}' static inline" + end + + code[decl_range] = "static inline #{decl}" + elsif (match = /#{MJITHeader::FUNC_HEADER_REGEXP}{/.match(decl)) && (header = match[0]) !~ /static/ + extern_names << decl_name + decl[match.begin(0)...match.end(0)] = '' + + if decl =~ /static/ + STDERR.puts "warning: a static decl inside external definition of '#{decl_name}'" + end + + header.sub!(/(extern|inline) /, ' ') + unless decl_name =~ /\Aattr_\w+_\w+\z/ # skip too-many false-positive warnings in insns_info.inc. + STDERR.puts "transform_mjit_header: making external definition of '#{decl_name}' static inline" + end + code[decl_range] = "static inline #{header}#{decl}" + end +end + +# Check the final file correctness +MJITHeader.check_code!(code, cc, cflags, 'final') + +MJITHeader.write(code, outfile) Index: Makefile.in =================================================================== --- Makefile.in (revision 62186) +++ Makefile.in (revision 62187) @@ -54,6 +54,7 @@ DOCTARGETS = @RDOCTARGET@ @CAPITARGET@ https://github.com/ruby/ruby/blob/trunk/Makefile.in#L54 EXTOUT = @EXTOUT@ arch_hdrdir = $(EXTOUT)/include/$(arch) VPATH = $(arch_hdrdir)/ruby:$(hdrdir)/ruby:$(srcdir):$(srcdir)/missing +MJIT_MIN_HEADER = $(EXTOUT)/include/$(arch)/rb_mjit_min_header-$(RUBY_PROGRAM_VERSION).h empty = CC_VERSION = @CC_VERSION@ @@ -407,6 +408,15 @@ probes.@OBJEXT@: $(srcdir)/probes.d $(DT https://github.com/ruby/ruby/blob/trunk/Makefile.in#L408 $(Q) $(RM) $@ $(Q) $(DTRACE) -G -C $(INCFLAGS) -s $(srcdir)/probes.d -o $@ $(DTRACE_REBUILD_OBJS) +rb_mjit_header.h: PHONY probes.h + $(ECHO) building $@ + $(Q) $(CC) $(CFLAGS) $(XCFLAGS) $(CPPFLAGS) -DMJIT_HEADER $(srcdir)/vm.c $(COUTFLAG) $@.new -E -P -dD + $(Q) (cmp $@.new $@ && $(ECHO0) $@ unchanged && $(RM) $@.new) || $(MV) $@.new $@ + +$(MJIT_MIN_HEADER): rb_mjit_header.h $(srcdir)/tool/transform_mjit_header.rb + $(ECHO) building $@ + $(BASERUBY) $(srcdir)/tool/transform_mjit_header.rb "$(CC)" rb_mjit_header.h $@ + # DTrace static library hacks described here: # http://mail.opensolaris.org/pipermail/dtrace-discuss/2005-August/000207.html ruby-glommed.$(OBJEXT): Index: win32/Makefile.sub =================================================================== --- win32/Makefile.sub (revision 62186) +++ win32/Makefile.sub (revision 62187) @@ -1186,6 +1186,15 @@ probes.h: {$(VPATH)}probes.dmyh https://github.com/ruby/ruby/blob/trunk/win32/Makefile.sub#L1186 #include "$(*F).dmyh" <<KEEP +rb_mjit_header.h: PHONY probes.h + $(ECHO) building $@ + $(Q) $(CC) $(CFLAGS) $(XCFLAGS) $(CPPFLAGS) -DMJIT_HEADER $(srcdir)/vm.c -P + $(Q) (cmp vm.i $@ && $(ECHO0) $@ unchanged && $(RM) vm.i) || $(MV) vm.i $@ + +$(MJIT_MIN_HEADER): rb_mjit_header.h $(srcdir)/tool/transform_mjit_header.rb + $(ECHO) building $@ + $(BASERUBY) $(srcdir)/tool/transform_mjit_header.rb "$(CC)" rb_mjit_header.h $@ + INSNS = opt_sc.inc optinsn.inc optunifs.inc insns.inc insns_info.inc \ vmtc.inc vm.inc Index: common.mk =================================================================== --- common.mk (revision 62186) +++ common.mk (revision 62187) @@ -59,6 +59,7 @@ ENC_TRANS_D = $(TIMESTAMPDIR)/.enc-tra https://github.com/ruby/ruby/blob/trunk/common.mk#L59 RDOCOUT = $(EXTOUT)/rdoc HTMLOUT = $(EXTOUT)/html CAPIOUT = doc/capi +MJIT_MIN_HEADER = $(EXTOUT)/include/$(arch)/rb_mjit_min_header-$(RUBY_PROGRAM_VERSION).h INITOBJS = dmyext.$(OBJEXT) dmyenc.$(OBJEXT) NORMALMAINOBJ = main.$(OBJEXT) @@ -186,7 +187,7 @@ SHOWFLAGS = showflags https://github.com/ruby/ruby/blob/trunk/common.mk#L187 all: $(SHOWFLAGS) main docs -main: $(SHOWFLAGS) exts $(ENCSTATIC:static=lib)encs +main: $(SHOWFLAGS) exts $(ENCSTATIC:static=lib)encs $(MJIT_MIN_HEADER) @$(NULLCMD) .PHONY: showflags -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/