ruby-changes:30178
From: nobu <ko1@a...>
Date: Mon, 29 Jul 2013 17:00:46 +0900 (JST)
Subject: [ruby-changes:30178] nobu:r42230 (trunk): parse.y, vm_eval.c: file encoding in eval
nobu 2013-07-29 17:00:34 +0900 (Mon, 29 Jul 2013) New Revision: 42230 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=42230 Log: parse.y, vm_eval.c: file encoding in eval * parse.y (yycompile): store file name as String to keep the encoding. * parse.y (rb_parser_compile_string_path, rb_parser_compile_file_path): new functions to pass file name as a String. * parse.y (gettable_gen): return a copy of the original file name, not a copy in filesystem encoding. * vm_eval.c (eval_string_with_cref): use Qundef instead of "(eval)". Modified files: trunk/iseq.c trunk/node.h trunk/parse.y trunk/test/ruby/test_eval.rb trunk/vm_eval.c Index: iseq.c =================================================================== --- iseq.c (revision 42229) +++ iseq.c (revision 42230) @@ -574,18 +574,6 @@ rb_iseq_load(VALUE data, VALUE parent, V https://github.com/ruby/ruby/blob/trunk/iseq.c#L574 return iseq_load(rb_cISeq, data, parent, opt); } -static NODE * -parse_string(VALUE str, const char *file, int line) -{ - VALUE parser = rb_parser_new(); - NODE *node = rb_parser_compile_string(parser, file, str, line); - - if (!node) { - rb_exc_raise(GET_THREAD()->errinfo); /* TODO: check err */ - } - return node; -} - VALUE rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE absolute_path, VALUE line, rb_block_t *base_block, VALUE opt) { @@ -598,17 +586,25 @@ rb_iseq_compile_with_option(VALUE src, V https://github.com/ruby/ruby/blob/trunk/iseq.c#L586 TH_PUSH_TAG(th); if ((state = EXEC_TAG()) == 0) { + VALUE parser; int ln = NUM2INT(line); - const char *fn = StringValueCStr(file); NODE *node; rb_compile_option_t option; + StringValueCStr(file); make_compile_option(&option, opt); + parser = rb_parser_new(); + if (RB_TYPE_P((src), T_FILE)) - node = rb_compile_file(fn, src, ln); - else - node = parse_string(StringValue(src), fn, ln); + node = rb_parser_compile_file_path(parser, file, src, ln); + else { + node = rb_parser_compile_string_path(parser, file, src, ln); + + if (!node) { + rb_exc_raise(GET_THREAD()->errinfo); /* TODO: check err */ + } + } if (base_block && base_block->iseq) { iseqval = rb_iseq_new_with_opt(node, base_block->iseq->location.label, Index: vm_eval.c =================================================================== --- vm_eval.c (revision 42229) +++ vm_eval.c (revision 42230) @@ -1176,7 +1176,7 @@ rb_each(VALUE obj) https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1176 } static VALUE -eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char *volatile file, volatile int line) +eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, volatile VALUE file, volatile int line) { int state; VALUE result = Qundef; @@ -1188,7 +1188,7 @@ eval_string_with_cref(VALUE self, VALUE https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1188 volatile int mild_compile_error; if (file == 0) { - file = rb_sourcefile(); + file = rb_sourcefilename(); line = rb_sourceline(); } @@ -1200,16 +1200,17 @@ eval_string_with_cref(VALUE self, VALUE https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1200 rb_iseq_t *iseq; volatile VALUE iseqval; VALUE absolute_path = Qnil; + VALUE fname; if (scope != Qnil) { bind = Check_TypedStruct(scope, &ruby_binding_data_type); { envval = bind->env; - if (strcmp(file, "(eval)") != 0) { - absolute_path = rb_str_new_cstr(file); + if (file != Qundef) { + absolute_path = file; } - else if (bind->path != Qnil) { - file = RSTRING_PTR(bind->path); + else if (!NIL_P(bind->path)) { + file = bind->path; line = bind->first_lineno; absolute_path = rb_current_realfilepath(); } @@ -1231,10 +1232,14 @@ eval_string_with_cref(VALUE self, VALUE https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1232 } } + if ((fname = file) == Qundef) { + fname = rb_usascii_str_new_cstr("(eval)"); + } + /* make eval iseq */ th->parse_in_eval++; th->mild_compile_error++; - iseqval = rb_iseq_compile_with_option(src, rb_str_new2(file), absolute_path, INT2FIX(line), base_block, Qnil); + iseqval = rb_iseq_compile_with_option(src, fname, absolute_path, INT2FIX(line), base_block, Qnil); th->mild_compile_error--; th->parse_in_eval--; @@ -1262,7 +1267,7 @@ eval_string_with_cref(VALUE self, VALUE https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1267 if (state) { if (state == TAG_RAISE) { VALUE errinfo = th->errinfo; - if (strcmp(file, "(eval)") == 0) { + if (file == Qundef) { VALUE mesg, errat, bt2; ID id_mesg; @@ -1292,7 +1297,7 @@ eval_string_with_cref(VALUE self, VALUE https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1297 } static VALUE -eval_string(VALUE self, VALUE src, VALUE scope, const char *file, int line) +eval_string(VALUE self, VALUE src, VALUE scope, VALUE file, int line) { return eval_string_with_cref(self, src, scope, 0, file, line); } @@ -1319,7 +1324,7 @@ VALUE https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1324 rb_f_eval(int argc, VALUE *argv, VALUE self) { VALUE src, scope, vfile, vline; - const char *file = "(eval)"; + VALUE file = Qundef; int line = 1; rb_scan_args(argc, argv, "13", &src, &scope, &vfile, &vline); @@ -1341,7 +1346,7 @@ rb_f_eval(int argc, VALUE *argv, VALUE s https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1346 } if (!NIL_P(vfile)) - file = RSTRING_PTR(vfile); + file = vfile; return eval_string(self, src, scope, file, line); } @@ -1349,27 +1354,28 @@ rb_f_eval(int argc, VALUE *argv, VALUE s https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1354 VALUE ruby_eval_string_from_file(const char *str, const char *filename) { - return eval_string(rb_vm_top_self(), rb_str_new2(str), Qnil, filename, 1); + VALUE file = filename ? rb_str_new_cstr(filename) : 0; + return eval_string(rb_vm_top_self(), rb_str_new2(str), Qnil, file, 1); } struct eval_string_from_file_arg { - const char *str; - const char *filename; + VALUE str; + VALUE filename; }; static VALUE eval_string_from_file_helper(VALUE data) { const struct eval_string_from_file_arg *const arg = (struct eval_string_from_file_arg*)data; - return eval_string(rb_vm_top_self(), rb_str_new2(arg->str), Qnil, arg->filename, 1); + return eval_string(rb_vm_top_self(), arg->str, Qnil, arg->filename, 1); } VALUE ruby_eval_string_from_file_protect(const char *str, const char *filename, int *state) { struct eval_string_from_file_arg arg; - arg.str = str; - arg.filename = filename; + arg.str = rb_str_new_cstr(str); + arg.filename = filename ? rb_str_new_cstr(filename) : 0; return rb_protect(eval_string_from_file_helper, (VALUE)&arg, state); } @@ -1529,7 +1535,7 @@ rb_yield_refine_block(VALUE refinement, https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1535 /* string eval under the class/module context */ static VALUE -eval_under(VALUE under, VALUE self, VALUE src, const char *file, int line) +eval_under(VALUE under, VALUE self, VALUE src, VALUE file, int line) { NODE *cref = vm_cref_push(GET_THREAD(), under, NOEX_PUBLIC, NULL); @@ -1554,7 +1560,7 @@ specific_eval(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1560 return yield_under(klass, self, Qundef); } else { - const char *file = "(eval)"; + VALUE file = Qundef; int line = 1; rb_check_arity(argc, 1, 3); @@ -1567,7 +1573,8 @@ specific_eval(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/vm_eval.c#L1573 if (argc > 2) line = NUM2INT(argv[2]); if (argc > 1) { - file = StringValuePtr(argv[1]); + file = argv[1]; + if (!NIL_P(file)) StringValue(file); } return eval_under(klass, self, argv[0], file, line); } Index: parse.y =================================================================== --- parse.y (revision 42229) +++ parse.y (revision 42230) @@ -265,6 +265,7 @@ struct parser_params { https://github.com/ruby/ruby/blob/trunk/parse.y#L265 int has_shebang; char *parser_ruby_sourcefile; /* current source file */ int parser_ruby_sourceline; /* current line no. */ + VALUE parser_ruby_sourcefile_string; rb_encoding *enc; int parser_yydebug; @@ -281,7 +282,6 @@ struct parser_params { https://github.com/ruby/ruby/blob/trunk/parse.y#L282 token_info *parser_token_info; #else /* Ripper only */ - VALUE parser_ruby_sourcefile_string; const char *tokp; VALUE delayed; int delayed_line; @@ -338,6 +338,7 @@ static int parser_yyerror(struct parser_ https://github.com/ruby/ruby/blob/trunk/parse.y#L338 #define ruby__end__seen (parser->parser_ruby__end__seen) #define ruby_sourceline (parser->parser_ruby_sourceline) #define ruby_sourcefile (parser->parser_ruby_sourcefile) +#define ruby_sourcefile_string (parser->parser_ruby_sourcefile_string) #define current_enc (parser->enc) #define yydebug (parser->parser_yydebug) #ifdef RIPPER @@ -5272,14 +5273,13 @@ static void parser_prepare(struct parser https://github.com/ruby/ruby/blob/trunk/parse.y#L5273 #ifndef RIPPER static VALUE -debug_lines(const char *f) +debug_lines(VALUE fname) { ID script_lines; CONST_ID(script_lines, "SCRIPT_LINES__"); if (rb_const_defined_at(rb_cObject, script_lines)) { VALUE hash = rb_const_get_at(rb_cObject, script_lines); if (RB_TYPE_P(hash, T_HASH)) { - VALUE fname = rb_external_str_new_with_enc(f, strlen(f), rb_filesystem_encoding()); VALUE lines = rb_ary_new(); rb_hash_aset(hash, fname, lines); return lines; @@ -5289,11 +5289,10 @@ debug_lines(const char *f) https://github.com/ruby/ruby/blob/trunk/parse.y#L5289 } static VALUE -coverage(const char *f, int n) +coverage(VALUE fname, int n) { VALUE coverages = rb_get_coverages(); if (RTEST(coverages) && RBASIC(coverages)->klass == 0) { - VALUE fname = rb_external_str_new_with_enc(f, strlen(f), rb_filesystem_encoding()); VALUE lines = rb_ary_new2(n); int i; RBASIC_CLEAR_CLASS(lines); @@ -5319,7 +5318,7 @@ yycompile0(VALUE arg) https://github.com/ruby/ruby/blob/trunk/parse.y#L5318 struct parser_params *parser = (struct parser_params *)arg; if (!compile_for_eval && rb_safe_level() == 0) { - ruby_debug_lines = debug_lines(ruby_sourcefile); + ruby_debug_lines = debug_lines(ruby_sourcefile_string); if (ruby_debug_lines && ruby_sourceline > 0) { VALUE str = STR_NEW0(); n = ruby_sourceline; @@ -5329,7 +5328,7 @@ yycompile0(VALUE arg) https://github.com/ruby/ruby/blob/trunk/parse.y#L5328 } if (!e_option_supplied(parser)) { - ruby_coverage = coverage(ruby_sourcefile, ruby_sourceline); + ruby_coverage = coverage(ruby_sourcefile_string, ruby_sourceline); } } @@ -5372,9 +5371,10 @@ yycompile0(VALUE arg) https://github.com/ruby/ruby/blob/trunk/parse.y#L5371 } static NODE* -yycompile(struct parser_params *parser, const char *f, int line) +yycompile(struct parser_params *parser, VALUE fname, int line) { - ruby_sourcefile = ruby_strdup(f); + ruby_sourcefile_string = rb_str_new_frozen(fname); + ruby_sourcefile = RSTRING_PTR(fname); ruby_sourceline = line - 1; return (NODE *)rb_suppress_tracing(yycompile0, (VALUE)parser); } @@ -5434,7 +5434,7 @@ static rb_data_type_t parser_data_type; https://github.com/ruby/ruby/blob/trunk/parse.y#L5434 static const rb_data_type_t parser_data_type; static NODE* -parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line) +parser_compile_string(volatile VALUE vparser, VALUE fname, VALUE s, int line) { struct parser_params *parser; NODE *node; @@ -5446,7 +5446,7 @@ parser_compile_string(volatile VALUE vpa https://github.com/ruby/ruby/blob/trunk/parse.y#L5446 lex_pbeg = lex_p = lex_pend = 0; compile_for_eval = rb_parse_in_eval(); - node = yycompile(parser, f, line); + node = yycompile(parser, fname, line); RB_GC_GUARD(vparser); /* prohibit tail call optimization */ return node; @@ -5456,12 +5456,18 @@ NODE* https://github.com/ruby/ruby/blob/trunk/parse.y#L5456 rb_compile_string(const char *f, VALUE s, int line) { must_be_ascii_compatible(s); - return parser_compile_string(rb_parser_new(), f, s, line); + return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), s, line); } NODE* rb_parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line) { + return rb_parser_compile_string_path(vparser, rb_filesystem_str_new_cstr(f), s, line); +} + +NODE* +rb_parser_compile_string_path(volatile VALUE vparser, VALUE f, VALUE s, int line) +{ must_be_ascii_compatible(s); return parser_compile_string(vparser, f, s, line); } @@ -5470,14 +5476,14 @@ NODE* https://github.com/ruby/ruby/blob/trunk/parse.y#L5476 rb_compile_cstr(const char *f, const char *s, int len, int line) { VALUE str = rb_str_new(s, len); - return parser_compile_string(rb_parser_new(), f, str, line); + return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), str, line); } NODE* rb_parser_compile_cstr(volatile VALUE vparser, const char *f, const char *s, int len, int line) { VALUE str = rb_str_new(s, len); - return parser_compile_string(vparser, f, str, line); + return parser_compile_string(vparser, rb_filesystem_str_new_cstr(f), str, line); } static VALUE @@ -5497,6 +5503,12 @@ rb_compile_file(const char *f, VALUE fil https://github.com/ruby/ruby/blob/trunk/parse.y#L5503 NODE* rb_parser_compile_file(volatile VALUE vparser, const char *f, VALUE file, int start) { + return rb_parser_compile_file_path(vparser, rb_filesystem_str_new_cstr(f), file, start); +} + +NODE* +rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE file, int start) +{ struct parser_params *parser; NODE *node; @@ -5506,7 +5518,7 @@ rb_parser_compile_file(volatile VALUE vp https://github.com/ruby/ruby/blob/trunk/parse.y#L5518 lex_pbeg = lex_p = lex_pend = 0; compile_for_eval = rb_parse_in_eval(); - node = yycompile(parser, f, start); + node = yycompile(parser, fname, start); RB_GC_GUARD(vparser); /* prohibit tail call optimization */ return node; @@ -8480,8 +8492,7 @@ gettable_gen(struct parser_params *parse https://github.com/ruby/ruby/blob/trunk/parse.y#L8492 case keyword_false: return NEW_FALSE(); case keyword__FILE__: - return NEW_STR(rb_external_str_new_with_enc(ruby_sourcefile, strlen(ruby_sourcefile), - rb_filesystem_encoding())); + return NEW_STR(rb_str_dup(ruby_sourcefile_string)); case keyword__LINE__: return NEW_LIT(INT2FIX(tokline)); case keyword__ENCODING__: @@ -10694,13 +10705,13 @@ parser_initialize(struct parser_params * https://github.com/ruby/ruby/blob/trunk/parse.y#L10705 parser->parser_lvtbl = 0; parser->parser_ruby__end__seen = 0; parser->parser_ruby_sourcefile = 0; + parser->parser_ruby_sourcefile_string = Qnil; #ifndef RIPPER parser->is_ripper = 0; parser->parser_eval_tree_begin = 0; parser->parser_eval_tree = 0; #else parser->is_ripper = 1; - parser->parser_ruby_sourcefile_string = Qnil; parser->delayed = Qnil; parser->result = Qnil; @@ -10728,12 +10739,12 @@ parser_mark(void *ptr) https://github.com/ruby/ruby/blob/trunk/parse.y#L10739 rb_gc_mark(p->parser_lex_input); rb_gc_mark(p->parser_lex_lastline); rb_gc_mark(p->parser_lex_nextline); + rb_gc_mark(p->parser_ruby_sourcefile_string); #ifndef RIPPER rb_gc_mark((VALUE)p->parser_eval_tree_begin) ; rb_gc_mark((VALUE)p->parser_eval_tree) ; rb_gc_mark(p->debug_lines); #else - rb_gc_mark(p->parser_ruby_sourcefile_string); rb_gc_mark(p->delayed); rb_gc_mark(p->value); rb_gc_mark(p->result); @@ -10758,9 +10769,6 @@ parser_free(void *ptr) https://github.com/ruby/ruby/blob/trunk/parse.y#L10769 prev = local->prev; xfree(local); } -#ifndef RIPPER - xfree(p->parser_ruby_sourcefile); -#endif xfree(p); } @@ -10777,11 +10785,6 @@ parser_memsize(const void *ptr) https://github.com/ruby/ruby/blob/trunk/parse.y#L10785 size += sizeof(*local); if (local->vars) size += local->vars->capa * sizeof(ID); } -#ifndef RIPPER - if (p->parser_ruby_sourcefile) { - size += strlen(p->parser_ruby_sourcefile) + 1; - } -#endif return size; } Index: test/ruby/test_eval.rb =================================================================== --- test/ruby/test_eval.rb (revision 42229) +++ test/ruby/test_eval.rb (revision 42230) @@ -468,4 +468,9 @@ class TestEval < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ruby/test_eval.rb#L468 result = foo.instance_eval(&foo_pr) assert_equal(1, result, 'Bug #3786, Bug #3860, [ruby-core:32501]') end + + def test_file_encoding + fname = "\u{3042}".encode("euc-jp") + assert_equal(fname, eval("__FILE__", nil, fname, 1)) + end end Index: node.h =================================================================== --- node.h (revision 42229) +++ node.h (revision 42230) @@ -486,6 +486,8 @@ NODE *rb_parser_while_loop(VALUE, NODE * https://github.com/ruby/ruby/blob/trunk/node.h#L486 NODE *rb_parser_compile_cstr(volatile VALUE, const char*, const char*, int, int); NODE *rb_parser_compile_string(volatile VALUE, const char*, VALUE, int); NODE *rb_parser_compile_file(volatile VALUE, const char*, VALUE, int); +NODE *rb_parser_compile_string_path(volatile VALUE vparser, VALUE fname, VALUE src, int line); +NODE *rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE input, int line); NODE *rb_compile_cstr(const char*, const char*, int, int); NODE *rb_compile_string(const char*, VALUE, int); -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/