ruby-changes:58333
From: Nobuyoshi <ko1@a...>
Date: Mon, 21 Oct 2019 11:50:44 +0900 (JST)
Subject: [ruby-changes:58333] 44da5a1c87 (master): Look up event ID offsets by token as index
https://git.ruby-lang.org/ruby.git/commit/?id=44da5a1c87 From 44da5a1c872879b05acab045263d0d4f906f04ba Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada <nobu@r...> Date: Mon, 21 Oct 2019 11:45:54 +0900 Subject: Look up event ID offsets by token as index diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c index fd59f1b..3a5f955 100644 --- a/ext/ripper/eventids2.c +++ b/ext/ripper/eventids2.c @@ -128,180 +128,176 @@ ripper_init_eventids2(void) https://github.com/ruby/ruby/blob/trunk/ext/ripper/eventids2.c#L128 STATIC_ASSERT(k__END___range, k__END__ < SHRT_MAX); STATIC_ASSERT(ripper_scanner_ids_size, sizeof(ripper_scanner_ids) < SHRT_MAX); -#define O(member) (int)offsetof(ripper_scanner_ids_t, ripper_id_##member) - -static const struct token_assoc { - unsigned short token; - unsigned short id_offset; -} token_to_eventid[] = { - {' ', O(words_sep)}, - {'!', O(op)}, - {'%', O(op)}, - {'&', O(op)}, - {'*', O(op)}, - {'+', O(op)}, - {'-', O(op)}, - {'/', O(op)}, - {'<', O(op)}, - {'=', O(op)}, - {'>', O(op)}, - {'?', O(op)}, - {'^', O(op)}, - {'|', O(op)}, - {'~', O(op)}, - {':', O(op)}, - {',', O(comma)}, - {'.', O(period)}, - {';', O(semicolon)}, - {'`', O(backtick)}, - {'\n', O(nl)}, - {keyword_alias, O(kw)}, - {keyword_and, O(kw)}, - {keyword_begin, O(kw)}, - {keyword_break, O(kw)}, - {keyword_case, O(kw)}, - {keyword_class, O(kw)}, - {keyword_def, O(kw)}, - {keyword_defined, O(kw)}, - {keyword_do, O(kw)}, - {keyword_do_block, O(kw)}, - {keyword_do_cond, O(kw)}, - {keyword_else, O(kw)}, - {keyword_elsif, O(kw)}, - {keyword_end, O(kw)}, - {keyword_ensure, O(kw)}, - {keyword_false, O(kw)}, - {keyword_for, O(kw)}, - {keyword_if, O(kw)}, - {modifier_if, O(kw)}, - {keyword_in, O(kw)}, - {keyword_module, O(kw)}, - {keyword_next, O(kw)}, - {keyword_nil, O(kw)}, - {keyword_not, O(kw)}, - {keyword_or, O(kw)}, - {keyword_redo, O(kw)}, - {keyword_rescue, O(kw)}, - {modifier_rescue, O(kw)}, - {keyword_retry, O(kw)}, - {keyword_return, O(kw)}, - {keyword_self, O(kw)}, - {keyword_super, O(kw)}, - {keyword_then, O(kw)}, - {keyword_true, O(kw)}, - {keyword_undef, O(kw)}, - {keyword_unless, O(kw)}, - {modifier_unless, O(kw)}, - {keyword_until, O(kw)}, - {modifier_until, O(kw)}, - {keyword_when, O(kw)}, - {keyword_while, O(kw)}, - {modifier_while, O(kw)}, - {keyword_yield, O(kw)}, - {keyword__FILE__, O(kw)}, - {keyword__LINE__, O(kw)}, - {keyword__ENCODING__, O(kw)}, - {keyword_BEGIN, O(kw)}, - {keyword_END, O(kw)}, - {keyword_do_LAMBDA, O(kw)}, - {tAMPER, O(op)}, - {tANDOP, O(op)}, - {tAREF, O(op)}, - {tASET, O(op)}, - {tASSOC, O(op)}, - {tBACK_REF, O(backref)}, - {tCHAR, O(CHAR)}, - {tCMP, O(op)}, - {tCOLON2, O(op)}, - {tCOLON3, O(op)}, - {tCONSTANT, O(const)}, - {tCVAR, O(cvar)}, - {tDOT2, O(op)}, - {tDOT3, O(op)}, - {tBDOT2, O(op)}, - {tBDOT3, O(op)}, - {tEQ, O(op)}, - {tEQQ, O(op)}, - {tFID, O(ident)}, - {tFLOAT, O(float)}, - {tGEQ, O(op)}, - {tGVAR, O(gvar)}, - {tIDENTIFIER, O(ident)}, - {tIMAGINARY, O(imaginary)}, - {tINTEGER, O(int)}, - {tIVAR, O(ivar)}, - {tLBRACE, O(lbrace)}, - {tLBRACE_ARG, O(lbrace)}, - {'{', O(lbrace)}, - {'}', O(rbrace)}, - {tLBRACK, O(lbracket)}, - {'[', O(lbracket)}, - {']', O(rbracket)}, - {tLEQ, O(op)}, - {tLPAREN, O(lparen)}, - {tLPAREN_ARG, O(lparen)}, - {'(', O(lparen)}, - {')', O(rparen)}, - {tLSHFT, O(op)}, - {tMATCH, O(op)}, - {tNEQ, O(op)}, - {tNMATCH, O(op)}, - {tNTH_REF, O(backref)}, - {tOP_ASGN, O(op)}, - {tOROP, O(op)}, - {tPOW, O(op)}, - {tQWORDS_BEG, O(qwords_beg)}, - {tQSYMBOLS_BEG, O(qsymbols_beg)}, - {tSYMBOLS_BEG, O(symbols_beg)}, - {tRATIONAL, O(rational)}, - {tREGEXP_BEG, O(regexp_beg)}, - {tREGEXP_END, O(regexp_end)}, - {tRPAREN, O(rparen)}, - {tRSHFT, O(op)}, - {tSTAR, O(op)}, - {tDSTAR, O(op)}, - {tANDDOT, O(op)}, - {tMETHREF, O(op)}, - {tSTRING_BEG, O(tstring_beg)}, - {tSTRING_CONTENT, O(tstring_content)}, - {tSTRING_DBEG, O(embexpr_beg)}, - {tSTRING_DEND, O(embexpr_end)}, - {tSTRING_DVAR, O(embvar)}, - {tSTRING_END, O(tstring_end)}, - {tSYMBEG, O(symbeg)}, - {tUMINUS, O(op)}, - {tUMINUS_NUM, O(op)}, - {tUPLUS, O(op)}, - {tWORDS_BEG, O(words_beg)}, - {tXSTRING_BEG, O(backtick)}, - {tLABEL, O(label)}, - {tLABEL_END, O(label_end)}, - {tLAMBDA, O(tlambda)}, - {tLAMBEG, O(tlambeg)}, - - /* ripper specific tokens */ - {tIGNORED_NL, O(ignored_nl)}, - {tCOMMENT, O(comment)}, - {tEMBDOC_BEG, O(embdoc_beg)}, - {tEMBDOC, O(embdoc)}, - {tEMBDOC_END, O(embdoc_end)}, - {tSP, O(sp)}, - {tHEREDOC_BEG, O(heredoc_beg)}, - {tHEREDOC_END, O(heredoc_end)}, - {k__END__, O(__end__)}, -}; static ID ripper_token2eventid(enum yytokentype tok) { - int i; +#define O(member) (int)offsetof(ripper_scanner_ids_t, ripper_id_##member)+1 + static const unsigned short offsets[] = { + [' '] = O(words_sep), + ['!'] = O(op), + ['%'] = O(op), + ['&'] = O(op), + ['*'] = O(op), + ['+'] = O(op), + ['-'] = O(op), + ['/'] = O(op), + ['<'] = O(op), + ['='] = O(op), + ['>'] = O(op), + ['?'] = O(op), + ['^'] = O(op), + ['|'] = O(op), + ['~'] = O(op), + [':'] = O(op), + [','] = O(comma), + ['.'] = O(period), + [';'] = O(semicolon), + ['`'] = O(backtick), + ['\n'] = O(nl), + [keyword_alias] = O(kw), + [keyword_and] = O(kw), + [keyword_begin] = O(kw), + [keyword_break] = O(kw), + [keyword_case] = O(kw), + [keyword_class] = O(kw), + [keyword_def] = O(kw), + [keyword_defined] = O(kw), + [keyword_do] = O(kw), + [keyword_do_block] = O(kw), + [keyword_do_cond] = O(kw), + [keyword_else] = O(kw), + [keyword_elsif] = O(kw), + [keyword_end] = O(kw), + [keyword_ensure] = O(kw), + [keyword_false] = O(kw), + [keyword_for] = O(kw), + [keyword_if] = O(kw), + [modifier_if] = O(kw), + [keyword_in] = O(kw), + [keyword_module] = O(kw), + [keyword_next] = O(kw), + [keyword_nil] = O(kw), + [keyword_not] = O(kw), + [keyword_or] = O(kw), + [keyword_redo] = O(kw), + [keyword_rescue] = O(kw), + [modifier_rescue] = O(kw), + [keyword_retry] = O(kw), + [keyword_return] = O(kw), + [keyword_self] = O(kw), + [keyword_super] = O(kw), + [keyword_then] = O(kw), + [keyword_true] = O(kw), + [keyword_undef] = O(kw), + [keyword_unless] = O(kw), + [modifier_unless] = O(kw), + [keyword_until] = O(kw), + [modifier_until] = O(kw), + [keyword_when] = O(kw), + [keyword_while] = O(kw), + [modifier_while] = O(kw), + [keyword_yield] = O(kw), + [keyword__FILE__] = O(kw), + [keyword__LINE__] = O(kw), + [keyword__ENCODING__] = O(kw), + [keyword_BEGIN] = O(kw), + [keyword_END] = O(kw), + [keyword_do_LAMBDA] = O(kw), + [tAMPER] = O(op), + [tANDOP] = O(op), + [tAREF] = O(op), + [tASET] = O(op), + [tASSOC] = O(op), + [tBACK_REF] = O(backref), + [tCHAR] = O(CHAR), + [tCMP] = O(op), + [tCOLON2] = O(op), + [tCOLON3] = O(op), + [tCONSTANT] = O(const), + [tCVAR] = O(cvar), + [tDOT2] = O(op), + [tDOT3] = O(op), + [tBDOT2] = O(op), + [tBDOT3] = O(op), + [tEQ] = O(op), + [tEQQ] = O(op), + [tFID] = O(ident), + [tFLOAT] = O(float), + [tGEQ] = O(op), + [tGVAR] = O(gvar), + [tIDENTIFIER] = O(ident), + [tIMAGINARY] = O(imaginary), + [tINTEGER] = O(int), + [tIVAR] = O(ivar), + [tLBRACE] = O(lbrace), + [tLBRACE_ARG] = O(lbrace), + ['{'] = O(lbrace), + ['}'] = O(rbrace), + [tLBRACK] = O(lbracket), + ['['] = O(lbracket), + [']'] = O(rbracket), + [tLEQ] = O(op), + [tLPAREN] = O(lparen), + [tLPAREN_ARG] = O(lparen), + ['('] = O(lparen), + [')'] = O(rparen), + [tLSHFT] = O(op), + [tMATCH] = O(op), + [tNEQ] = O(op), + [tNMATCH] = O(op), + [tNTH_REF] = O(backref), + [tOP_ASGN] = O(op), + [tOROP] = O(op), + [tPOW] = O(op), + [tQWORDS_BEG] = O(qwords_beg), + [tQSYMBOLS_BEG] = O(qsymbols_beg), + [tSYMBOLS_BEG] = O(symbols_beg), + [tRATIONAL] = O(rational), + [tREGEXP_BEG] = O(regexp_beg), + [tREGEXP_END] = O(regexp_end), + [tRPAREN] = O(rparen), + [tRSHFT] = O(op), + [tSTAR] = O(op), + [tDSTAR] = O(op), + [tANDDOT] = O(op), + [tMETHREF] = O(op), + [tSTRING_BEG] = O(tstring_beg), + [tSTRING_CONTENT] = O(tstring_content), + [tSTRING_DBEG] = O(embexpr_beg), + [tSTRING_DEND] = O(embexpr_end), + [tSTRING_DVAR] = O(embvar), + [tSTRING_END] = O(tstring_end), + [tSYMBEG] = O(symbeg), + [tUMINUS] = O(op), + [tUMINUS_NUM] = O(op), + [tUPLUS] = O(op), + [tWORDS_BEG] = O(words_beg), + [tXSTRING_BEG] = O(backtick), + [tLABEL] = O(label), + [tLABEL_END] = O(label_end), + [tLAMBDA] = O(tlambda), + [tLAMBEG] = O(tlambeg), + + /* ripper specific tokens */ + [tIGNORED_NL] = O(ignored_nl), + [tCOMMENT] = O(comment), + [tEMBDOC_BEG] = O(embdoc_beg), + [tEMBDOC] = O(embdoc), + [tEMBDOC_END] = O(embdoc_end), + [tSP] = O(sp), + [tHEREDOC_BEG] = O(heredoc_beg), + [tHEREDOC_END] = O(heredoc_end), + [k__END__] = O(__end__), + }; +#undef O - for (i = 0; i < numberof(token_to_eventid); i++) { - const struct token_assoc *const a = &token_to_eventid[i]; - if ((enum yytokentype)a->token == tok) - return *(const ID *)((const char *)&ripper_scanner_ids + a->id_offset); + int i = (int)tok; + if (i >= 0 && i < numberof(offsets) && (i = offsets[i]) > 0) { + return *(const ID *)((const char *)&ripper_scanner_ids-1+i); } - if (tok < 256) { + /* 128..256 are used as operator tokens */ + if (tok < 128) { return ripper_scanner_ids.ripper_id_CHAR; } rb_raise(rb_eRuntimeError, "[Ripper FATAL] unknown token %d", tok); -- cgit v0.10.2 (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/