ruby-changes:15118
From: akr <ko1@a...>
Date: Sun, 21 Mar 2010 12:40:03 +0900 (JST)
Subject: [ruby-changes:15118] Ruby:r26995 (trunk): * tool/transcode-tblgen.rb (transcode_compile_tree): make
akr 2010-03-21 12:38:58 +0900 (Sun, 21 Mar 2010) New Revision: 26995 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=26995 Log: * tool/transcode-tblgen.rb (transcode_compile_tree): make valid_encoding mandatory unless from_encoding is registered in ValidEncoding. (transcode_tbl_only): ditto. (transcode_tblgen): ditto. (ValidEncoding): new function. * enc/trans/escape.trans: specify valid_encoding. * enc/trans/emoji_sjis_docomo.trans: ditto. * enc/trans/emoji.trans: ditto. * enc/trans/emoji_iso2022_kddi.trans: ditto. * enc/trans/big5.trans: ditto. * enc/trans/emoji_sjis_softbank.trans: ditto. * enc/trans/emoji_sjis_kddi.trans: ditto. * enc/trans/chinese.trans: use ValidEncoding() instead of ValidEncoding[]. Modified files: trunk/ChangeLog trunk/enc/trans/big5.trans trunk/enc/trans/chinese.trans trunk/enc/trans/emoji.trans trunk/enc/trans/emoji_iso2022_kddi.trans trunk/enc/trans/emoji_sjis_docomo.trans trunk/enc/trans/emoji_sjis_kddi.trans trunk/enc/trans/emoji_sjis_softbank.trans trunk/enc/trans/escape.trans trunk/tool/transcode-tblgen.rb Index: ChangeLog =================================================================== --- ChangeLog (revision 26994) +++ ChangeLog (revision 26995) @@ -1,3 +1,29 @@ +Sun Mar 21 12:32:39 2010 Tanaka Akira <akr@f...> + + * tool/transcode-tblgen.rb (transcode_compile_tree): make + valid_encoding mandatory unless from_encoding is registered in + ValidEncoding. + (transcode_tbl_only): ditto. + (transcode_tblgen): ditto. + (ValidEncoding): new function. + + * enc/trans/escape.trans: specify valid_encoding. + + * enc/trans/emoji_sjis_docomo.trans: ditto. + + * enc/trans/emoji.trans: ditto. + + * enc/trans/emoji_iso2022_kddi.trans: ditto. + + * enc/trans/big5.trans: ditto. + + * enc/trans/emoji_sjis_softbank.trans: ditto. + + * enc/trans/emoji_sjis_kddi.trans: ditto. + + * enc/trans/chinese.trans: use ValidEncoding() instead of + ValidEncoding[]. + Sun Mar 21 09:43:01 2010 Nobuyoshi Nakada <nobu@r...> * io.c (IO_[CRW]BUF_CAPA_MIN): replaced magic numbers. Index: enc/trans/escape.trans =================================================================== --- enc/trans/escape.trans (revision 26994) +++ enc/trans/escape.trans (revision 26995) @@ -8,14 +8,14 @@ transcode_tblgen("", "amp_escape", [ ["{00-25,27-FF}", :nomap], ["26", hexstr("&")] - ]) + ], nil) transcode_tblgen("", "xml_text_escape", [ ["{00-25,27-3B,3D,3F-FF}", :nomap], ["26", hexstr("&")], ["3C", hexstr("<")], ["3E", hexstr(">")] - ]) + ], nil) transcode_tblgen("", "xml_attr_content_escape", [ ["{00-21,23-25,27-3B,3D,3F-FF}", :nomap], @@ -23,7 +23,7 @@ ["26", hexstr("&")], ["3C", hexstr("<")], ["3E", hexstr(">")] - ]) + ], nil) map_xml_attr_quote = {} map_xml_attr_quote["{00-FF}"] = :func_so Index: enc/trans/chinese.trans =================================================================== --- enc/trans/chinese.trans (revision 26994) +++ enc/trans/chinese.trans (revision 26995) @@ -1,8 +1,8 @@ #include "transcode_data.h" <% - gb2312_valid_byte_pattern = ValidEncoding['EUC-KR'] - gb12345_valid_byte_pattern = ValidEncoding['EUC-KR'] + gb2312_valid_byte_pattern = ValidEncoding('EUC-KR') + gb12345_valid_byte_pattern = ValidEncoding('EUC-KR') transcode_tblgen "GB2312", "UTF-8", [["{00-7f}", :nomap]] + Index: enc/trans/emoji_sjis_docomo.trans =================================================================== --- enc/trans/emoji_sjis_docomo.trans (revision 26994) +++ enc/trans/emoji_sjis_docomo.trans (revision 26995) @@ -9,7 +9,8 @@ CP932VDC:NEC_IBM/UCS CP932VDC:IBM/UCS CP932UDA/UCS - JISX0208:MS/UCS).join(",")) + JISX0208:MS/UCS).join(",")), + ValidEncoding('Shift_JIS') transcode_tblgen "UTF8-DOCOMO", "SJIS-DOCOMO", [["{00-7f}", :nomap]] + @@ -19,7 +20,8 @@ UCS/CP932VDC:NEC_IBM UCS/CP932VDC:IBM UCS/CP932UDA - UCS/JISX0208:MS).join(",")) + UCS/JISX0208:MS).join(",")), + ValidEncoding('UTF-8') %> <%= transcode_generated_code %> Index: enc/trans/emoji.trans =================================================================== --- enc/trans/emoji.trans (revision 26994) +++ enc/trans/emoji.trans (revision 26995) @@ -23,7 +23,7 @@ to_encoding = (to_company == "Unicode") ? "UTF-8" : "UTF8-"+to_company strict_mapping = EMOJI_EXCHANGE_TBL[from_encoding][to_encoding] - transcode_tblgen(from_encoding, to_encoding, strict_mapping + nomap_table) + transcode_tblgen(from_encoding, to_encoding, strict_mapping + nomap_table, nil) end end %> Index: enc/trans/emoji_iso2022_kddi.trans =================================================================== --- enc/trans/emoji_iso2022_kddi.trans (revision 26994) +++ enc/trans/emoji_iso2022_kddi.trans (revision 26995) @@ -32,7 +32,8 @@ CP932VDC:NEC_IBM/UCS CP932VDC:IBM/UCS CP932UDA/UCS - JISX0208:MS/UCS).join(",")) + JISX0208:MS/UCS).join(",")), + nil # mapping from UTF8-KDDI to stateless-ISO-2022-JP-KDDI transcode_tblgen "UTF8-KDDI", "stateless-ISO-2022-JP-KDDI", @@ -43,7 +44,8 @@ UCS/CP932VDC:NEC_IBM UCS/CP932VDC:IBM UCS/CP932UDA - UCS/JISX0208:MS).join(",")) + UCS/JISX0208:MS).join(",")), + ValidEncoding('UTF-8') %> <%= transcode_generated_code %> Index: enc/trans/big5.trans =================================================================== --- enc/trans/big5.trans (revision 26994) +++ enc/trans/big5.trans (revision 26995) @@ -8,10 +8,10 @@ transcode_tblgen "Big5", "UTF-8", [["{00-7f}", :nomap], *BIG5_TO_UCS_TBL] transcode_tblgen "UTF-8", "Big5", [["{00-7f}", :nomap], *BIG5_TO_UCS_TBL.map {|a,b| [b,a] }] - transcode_tblgen "Big5-HKSCS", "UTF-8", [["{00-7f}", :nomap], *BIG5_HKSCS_TO_UCS_TBL] + transcode_tblgen "Big5-HKSCS", "UTF-8", [["{00-7f}", :nomap], *BIG5_HKSCS_TO_UCS_TBL], ValidEncoding('Big5') transcode_tblgen "UTF-8", "Big5-HKSCS", [["{00-7f}", :nomap], *BIG5_HKSCS_TO_UCS_TBL.map {|a,b| [b,a] }] - transcode_tblgen "Big5-UAO", "UTF-8", [["{00-7f}", :nomap], *BIG5_UAO_TO_UCS_TBL] + transcode_tblgen "Big5-UAO", "UTF-8", [["{00-7f}", :nomap], *BIG5_UAO_TO_UCS_TBL], ValidEncoding('Big5') transcode_tblgen "UTF-8", "Big5-UAO", [["{00-7f}", :nomap], *BIG5_UAO_TO_UCS_TBL.map {|a,b| [b,a] }] %> @@ -22,4 +22,4 @@ Init_big5(void) { <%= transcode_register_code %> -} \ No newline at end of file +} Index: enc/trans/emoji_sjis_softbank.trans =================================================================== --- enc/trans/emoji_sjis_softbank.trans (revision 26994) +++ enc/trans/emoji_sjis_softbank.trans (revision 26995) @@ -9,7 +9,8 @@ CP932VDC:NEC_IBM/UCS CP932VDC:IBM/UCS CP932UDA/UCS - JISX0208:MS/UCS).join(",")) + JISX0208:MS/UCS).join(",")), + ValidEncoding('Shift_JIS') transcode_tblgen "UTF8-SoftBank", "SJIS-SoftBank", [["{00-7f}", :nomap]] + @@ -19,7 +20,8 @@ UCS/CP932VDC:NEC_IBM UCS/CP932VDC:IBM UCS/CP932UDA - UCS/JISX0208:MS).join(",")) + UCS/JISX0208:MS).join(",")), + ValidEncoding('UTF-8') %> <%= transcode_generated_code %> Index: enc/trans/emoji_sjis_kddi.trans =================================================================== --- enc/trans/emoji_sjis_kddi.trans (revision 26994) +++ enc/trans/emoji_sjis_kddi.trans (revision 26995) @@ -9,7 +9,8 @@ CP932VDC:NEC_IBM/UCS CP932VDC:IBM/UCS CP932UDA/UCS - JISX0208:MS/UCS).join(",")) + JISX0208:MS/UCS).join(",")), + ValidEncoding('Shift_JIS') transcode_tblgen "UTF8-KDDI", "SJIS-KDDI", [["{00-7f}", :nomap]] + @@ -20,7 +21,8 @@ UCS/CP932VDC:NEC_IBM UCS/CP932VDC:IBM UCS/CP932UDA - UCS/JISX0208:MS).join(",")) + UCS/JISX0208:MS).join(",")), + ValidEncoding('UTF-8') %> <%= transcode_generated_code %> Index: tool/transcode-tblgen.rb =================================================================== --- tool/transcode-tblgen.rb (revision 26994) +++ tool/transcode-tblgen.rb (revision 26995) @@ -758,13 +758,17 @@ r end -def transcode_compile_tree(name, from, map, valid_encoding=nil) +UnspecifiedValidEncoding = Object.new + +def transcode_compile_tree(name, from, map, valid_encoding) map = encode_utf8(map) h = {} map.each {|k, v| h[k] = v unless h[k] # use first mapping } - valid_encoding = ValidEncoding[from] if valid_encoding == nil + if valid_encoding.equal? UnspecifiedValidEncoding + valid_encoding = ValidEncoding.fetch(from) + end if valid_encoding am = ActionMap.merge2(h, {valid_encoding => :undef}) {|prefix, as1, as2| a1 = as1.empty? ? nil : ActionMap.unambiguous_action(as1) @@ -787,7 +791,7 @@ TRANSCODERS = [] TRANSCODE_GENERATED_TRANSCODER_CODE = '' -def transcode_tbl_only(from, to, map, valid_encoding=nil) +def transcode_tbl_only(from, to, map, valid_encoding=UnspecifiedValidEncoding) if VERBOSE_MODE if from.empty? || to.empty? STDERR.puts "converter for #{from.empty? ? to : from}" @@ -808,7 +812,7 @@ return map, tree_name, real_tree_name, max_input end -def transcode_tblgen(from, to, map, valid_encoding=nil) +def transcode_tblgen(from, to, map, valid_encoding=UnspecifiedValidEncoding) map, tree_name, real_tree_name, max_input = transcode_tbl_only(from, to, map, valid_encoding) transcoder_name = "rb_#{tree_name}" TRANSCODERS << transcoder_name @@ -912,6 +916,10 @@ {81-fe}{30-39}{81-fe}{30-39}', } +def ValidEncoding(enc) + ValidEncoding.fetch(enc) +end + def set_valid_byte_pattern(encoding, pattern_or_label) pattern = if ValidEncoding[pattern_or_label] -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/