ruby-changes:48522
From: mame <ko1@a...>
Date: Sat, 4 Nov 2017 16:21:40 +0900 (JST)
Subject: [ruby-changes:48522] mame:r60637 (trunk): Revert "Revert "Replace NODE_STRTERM and NODE_HEREDOC with imemo_strterm""
mame 2017-11-04 16:21:36 +0900 (Sat, 04 Nov 2017) New Revision: 60637 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=60637 Log: Revert "Revert "Replace NODE_STRTERM and NODE_HEREDOC with imemo_strterm"" Retry r60634 Modified files: trunk/gc.c trunk/internal.h trunk/parse.y Index: internal.h =================================================================== --- internal.h (revision 60636) +++ internal.h (revision 60637) @@ -845,7 +845,8 @@ enum imemo_type { https://github.com/ruby/ruby/blob/trunk/internal.h#L845 imemo_ment = 6, imemo_iseq = 7, imemo_alloc = 8, - imemo_ast = 9 + imemo_ast = 9, + imemo_strterm = 10 }; #define IMEMO_MASK 0x0f @@ -941,6 +942,8 @@ typedef struct rb_imemo_alloc_struct { https://github.com/ruby/ruby/blob/trunk/internal.h#L942 rb_imemo_alloc_t *rb_imemo_alloc_new(VALUE, VALUE, VALUE, VALUE); +void rb_strterm_mark(VALUE obj); + /*! MEMO * * @see imemo_type Index: parse.y =================================================================== --- parse.y (revision 60636) +++ parse.y (revision 60637) @@ -178,6 +178,8 @@ typedef struct token_info { https://github.com/ruby/ruby/blob/trunk/parse.y#L178 struct token_info *next; } token_info; +typedef struct rb_strterm_struct rb_strterm_t; + /* Structure of Lexer Buffer: @@ -193,7 +195,7 @@ struct parser_params { https://github.com/ruby/ruby/blob/trunk/parse.y#L195 YYSTYPE *lval; struct { - NODE *strterm; + rb_strterm_t *strterm; VALUE (*gets)(struct parser_params*,VALUE); VALUE input; VALUE lastline; @@ -738,17 +740,41 @@ static int lvar_defined_gen(struct parse https://github.com/ruby/ruby/blob/trunk/parse.y#L740 #define RE_OPTION_MASK 0xff #define RE_OPTION_ARG_ENCODING_NONE 32 -#define NODE_STRTERM NODE_ZARRAY /* nothing to gc */ -#define NODE_HEREDOC NODE_ARRAY /* 1, 3 to gc */ -#define SIGN_EXTEND(x,n) (((1<<(n)-1)^((x)&~(~0<<(n))))-(1<<(n)-1)) -#define nd_func u1.id -#if SIZEOF_SHORT == 2 -#define nd_term(node) ((signed short)(node)->u2.id) -#else -#define nd_term(node) SIGN_EXTEND((node)->u2.id, CHAR_BIT*2) -#endif -#define nd_paren(node) (char)((node)->u2.id >> CHAR_BIT*2) -#define nd_nest u3.cnt +/* structs for managing terminator of string literal and heredocment */ +typedef struct rb_strterm_literal_struct { + long nest; + long func; /* STR_FUNC_* (e.g., STR_FUNC_ESCAPE and STR_FUNC_EXPAND) */ + long paren; /* '(' of `%q(...)` */ + long term; /* ')' of `%q(...)` */ +} rb_strterm_literal_t; + +typedef struct rb_strterm_heredoc_struct { + VALUE sourceline; + VALUE term; /* `"END"` of `<<"END"` */ + long lastidx; /* the column of `<<"END"` */ + VALUE lastline; /* the string of line that contains `<<"END"` */ +} rb_strterm_heredoc_t; + +#define STRTERM_HEREDOC IMEMO_FL_USER0 + +typedef struct rb_strterm_struct { + VALUE flags; + union { + rb_strterm_literal_t literal; + rb_strterm_heredoc_t heredoc; + } u; +} rb_strterm_t; + +void +rb_strterm_mark(VALUE obj) +{ + rb_strterm_t *strterm = (rb_strterm_t*)obj; + if (RBASIC(obj)->flags & STRTERM_HEREDOC) { + rb_strterm_heredoc_t *heredoc = &strterm->u.heredoc; + rb_gc_mark(heredoc->term); + rb_gc_mark(heredoc->lastline); + } +} #define TOKEN2ID(tok) ( \ tTOKEN_LOCAL_BEGIN<(tok)&&(tok)<tTOKEN_LOCAL_END ? TOKEN2LOCALID(tok) : \ @@ -947,6 +973,7 @@ static void token_info_pop_gen(struct pa https://github.com/ruby/ruby/blob/trunk/parse.y#L973 ID id; int num; const struct vtable *vars; + struct rb_strterm_struct *strterm; } %token <id> @@ -4214,13 +4241,14 @@ string_content : tSTRING_CONTENT https://github.com/ruby/ruby/blob/trunk/parse.y#L4241 } | tSTRING_DVAR { - $<node>$ = lex_strterm; + /* need to backup lex_stream so that a string literal `%&foo,#$&,bar&` can be parsed */ + $<strterm>$ = lex_strterm; lex_strterm = 0; SET_LEX_STATE(EXPR_BEG); } string_dvar { - lex_strterm = $<node>2; + lex_strterm = $<strterm>2; /*%%%*/ $$ = NEW_EVSTR($3); nd_set_lineno($$, @1.first_line); @@ -4237,7 +4265,8 @@ string_content : tSTRING_CONTENT https://github.com/ruby/ruby/blob/trunk/parse.y#L4265 CMDARG_SET(0); } { - $<node>$ = lex_strterm; + /* need to backup lex_stream so that a string literal `%!foo,#{ !0 },bar!` can be parsed */ + $<strterm>$ = lex_strterm; lex_strterm = 0; } { @@ -4256,7 +4285,7 @@ string_content : tSTRING_CONTENT https://github.com/ruby/ruby/blob/trunk/parse.y#L4285 { COND_SET($<val>1); CMDARG_SET($<val>2); - lex_strterm = $<node>3; + lex_strterm = $<strterm>3; SET_LEX_STATE($<num>4); brace_nest = $<num>5; heredoc_indent = $<num>6; @@ -5135,8 +5164,8 @@ none : /* none */ https://github.com/ruby/ruby/blob/trunk/parse.y#L5164 static int parser_regx_options(struct parser_params*); static int parser_tokadd_string(struct parser_params*,int,int,int,long*,rb_encoding**); static void parser_tokaddmbc(struct parser_params *parser, int c, rb_encoding *enc); -static enum yytokentype parser_parse_string(struct parser_params*,NODE*); -static enum yytokentype parser_here_document(struct parser_params*,NODE*); +static enum yytokentype parser_parse_string(struct parser_params*,rb_strterm_literal_t*); +static enum yytokentype parser_here_document(struct parser_params*,rb_strterm_heredoc_t*); # define nextc() parser_nextc(parser) @@ -6466,8 +6495,9 @@ parser_tokadd_string(struct parser_param https://github.com/ruby/ruby/blob/trunk/parse.y#L6495 return c; } +/* imemo_strterm for literal */ #define NEW_STRTERM(func, term, paren) \ - rb_node_newnode(NODE_STRTERM, (func), (term) | ((paren) << (CHAR_BIT * 2)), 0) + (rb_strterm_t*)rb_imemo_new(imemo_strterm, (VALUE)(func), (VALUE)(paren), (VALUE)(term), 0) #ifdef RIPPER static void @@ -6566,7 +6596,6 @@ parser_peek_variable_name(struct parser_ https://github.com/ruby/ruby/blob/trunk/parse.y#L6596 static inline enum yytokentype parser_string_term(struct parser_params *parser, int func) { - rb_discard_node(lex_strterm); lex_strterm = 0; if (func & STR_FUNC_REGEXP) { set_yylval_num(regx_options()); @@ -6584,11 +6613,11 @@ parser_string_term(struct parser_params https://github.com/ruby/ruby/blob/trunk/parse.y#L6613 } static enum yytokentype -parser_parse_string(struct parser_params *parser, NODE *quote) +parser_parse_string(struct parser_params *parser, rb_strterm_literal_t *quote) { - int func = (int)quote->nd_func; - int term = nd_term(quote); - int paren = nd_paren(quote); + int func = (int)quote->func; + int term = (int)quote->term; + int paren = (int)quote->paren; int c, space = 0; rb_encoding *enc = current_enc; VALUE lit; @@ -6603,9 +6632,9 @@ parser_parse_string(struct parser_params https://github.com/ruby/ruby/blob/trunk/parse.y#L6632 do {c = nextc();} while (ISSPACE(c)); space = 1; } - if (c == term && !quote->nd_nest) { + if (c == term && !quote->nest) { if (func & STR_FUNC_QWORDS) { - quote->nd_func |= STR_FUNC_TERM; + quote->func |= STR_FUNC_TERM; return ' '; } return parser_string_term(parser, func); @@ -6622,7 +6651,7 @@ parser_parse_string(struct parser_params https://github.com/ruby/ruby/blob/trunk/parse.y#L6651 c = nextc(); } pushback(c); - if (tokadd_string(func, term, paren, "e->nd_nest, + if (tokadd_string(func, term, paren, "e->nest, &enc) == -1) { if (parser->eofp) { #ifndef RIPPER @@ -6637,7 +6666,7 @@ parser_parse_string(struct parser_params https://github.com/ruby/ruby/blob/trunk/parse.y#L6666 else { unterminated_literal("unterminated string meets end of file"); } - quote->nd_func |= STR_FUNC_TERM; + quote->func |= STR_FUNC_TERM; } } @@ -6657,7 +6686,6 @@ parser_heredoc_identifier(struct parser_ https://github.com/ruby/ruby/blob/trunk/parse.y#L6686 long len; int newline = 0; int indent = 0; - VALUE lit; if (c == '-') { c = nextc(); @@ -6722,13 +6750,14 @@ parser_heredoc_identifier(struct parser_ https://github.com/ruby/ruby/blob/trunk/parse.y#L6750 dispatch_scan_event(tHEREDOC_BEG); len = lex_p - lex_pbeg; lex_goto_eol(parser); - add_mark_object(lit = STR_NEW(tok(), toklen())); - add_mark_object(lex_lastline); - lex_strterm = rb_node_newnode(NODE_HEREDOC, - lit, /* nd_lit */ - len, /* nd_nth */ - lex_lastline); /* nd_orig */ - parser_set_line(lex_strterm, ruby_sourceline); + + lex_strterm = (rb_strterm_t*)rb_imemo_new(imemo_strterm, + STR_NEW(tok(), toklen()), /* term */ + len, /* lastidx */ + lex_lastline, /* lastline */ + ruby_sourceline); + lex_strterm->flags |= STRTERM_HEREDOC; + token_flush(parser); heredoc_indent = indent; heredoc_line_indent = 0; @@ -6736,20 +6765,18 @@ parser_heredoc_identifier(struct parser_ https://github.com/ruby/ruby/blob/trunk/parse.y#L6765 } static void -parser_heredoc_restore(struct parser_params *parser, NODE *here) +parser_heredoc_restore(struct parser_params *parser, rb_strterm_heredoc_t *here) { VALUE line; lex_strterm = 0; - line = here->nd_orig; + line = here->lastline; lex_lastline = line; lex_pbeg = RSTRING_PTR(line); lex_pend = lex_pbeg + RSTRING_LEN(line); - lex_p = lex_pbeg + here->nd_nth; + lex_p = lex_pbeg + here->lastidx; heredoc_end = ruby_sourceline; - ruby_sourceline = nd_line(here); - dispose_string(parser, here->nd_lit); - rb_discard_node(here); + ruby_sourceline = here->sourceline; token_flush(parser); } @@ -6945,7 +6972,7 @@ ripper_dispatch_heredoc_end(struct parse https://github.com/ruby/ruby/blob/trunk/parse.y#L6972 #endif static enum yytokentype -parser_here_document(struct parser_params *parser, NODE *here) +parser_here_document(struct parser_params *parser, rb_strterm_heredoc_t *here) { int c, func, indent = 0; const char *eos, *p, *pend; @@ -6953,8 +6980,8 @@ parser_here_document(struct parser_param https://github.com/ruby/ruby/blob/trunk/parse.y#L6980 VALUE str = 0; rb_encoding *enc = current_enc; - eos = RSTRING_PTR(here->nd_lit); - len = RSTRING_LEN(here->nd_lit) - 1; + eos = RSTRING_PTR(here->term); + len = RSTRING_LEN(here->term) - 1; indent = (func = *eos++) & STR_FUNC_INDENT; if ((c = nextc()) == -1) { @@ -6985,13 +7012,13 @@ parser_here_document(struct parser_param https://github.com/ruby/ruby/blob/trunk/parse.y#L7012 lex_goto_eol(parser); #endif restore: - heredoc_restore(lex_strterm); + heredoc_restore(&lex_strterm->u.heredoc); lex_strterm = 0; return 0; } if (was_bol() && whole_match_p(eos, len, indent)) { dispatch_heredoc_end(); - heredoc_restore(lex_strterm); + heredoc_restore(&lex_strterm->u.heredoc); lex_strterm = 0; SET_LEX_STATE(EXPR_END); return tSTRING_END; @@ -7085,7 +7112,7 @@ parser_here_document(struct parser_param https://github.com/ruby/ruby/blob/trunk/parse.y#L7112 str = ripper_new_yylval(ripper_token2eventid(tSTRING_CONTENT), yylval.val, str); #endif - heredoc_restore(lex_strterm); + heredoc_restore(&lex_strterm->u.heredoc); lex_strterm = NEW_STRTERM(func | STR_FUNC_TERM, 0, 0); set_yylval_str(str); add_mark_object(str); @@ -8243,11 +8270,11 @@ parser_yylex(struct parser_params *parse https://github.com/ruby/ruby/blob/trunk/parse.y#L8270 int token_seen = parser->token_seen; if (lex_strterm) { - if (nd_type(lex_strterm) == NODE_HEREDOC) { - return here_document(lex_strterm); + if (lex_strterm->flags & STRTERM_HEREDOC) { + return here_document(&lex_strterm->u.heredoc); } else { - return parse_string(lex_strterm); + return parse_string(&lex_strterm->u.literal); } } cmd_state = command_start; @@ -11542,12 +11569,12 @@ parser_mark(void *ptr) https://github.com/ruby/ruby/blob/trunk/parse.y#L11569 rb_gc_mark(lex_lastline); rb_gc_mark(lex_nextline); rb_gc_mark(ruby_sourcefile_string); + rb_gc_mark((VALUE)lex_strterm); #ifndef RIPPER rb_gc_mark(ruby_debug_lines); rb_gc_mark(parser->compile_option); rb_gc_mark(parser->error_buffer); #else - rb_gc_mark((VALUE)lex_strterm); rb_gc_mark(parser->delayed); rb_gc_mark(parser->value); rb_gc_mark(parser->result); Index: gc.c =================================================================== --- gc.c (revision 60636) +++ gc.c (revision 60637) @@ -4546,6 +4546,9 @@ gc_mark_imemo(rb_objspace_t *objspace, V https://github.com/ruby/ruby/blob/trunk/gc.c#L4546 case imemo_ast: rb_ast_mark(&RANY(obj)->as.imemo.ast); return; + case imemo_strterm: + rb_strterm_mark(obj); + return; #if VM_CHECK_MODE > 0 default: VM_UNREACHABLE(gc_mark_imemo); -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/