ruby-changes:7699
From: akr <ko1@a...>
Date: Sun, 7 Sep 2008 23:49:23 +0900 (JST)
Subject: [ruby-changes:7699] Ruby:r19220 (trunk): * enc/trans/iso2022.trans: stateless-iso-2022-jp is defined to avoid
akr 2008-09-07 23:49:05 +0900 (Sun, 07 Sep 2008) New Revision: 19220 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=19220 Log: * enc/trans/iso2022.trans: stateless-iso-2022-jp is defined to avoid undefined conversion error between iso-2022-jp and the corresponding stateless encoding. * enc/emacs_mule.c: replicate emacs-mule as stateless-iso-2022-jp. Modified files: trunk/ChangeLog trunk/enc/emacs_mule.c trunk/enc/trans/iso2022.trans trunk/test/ruby/test_econv.rb Index: ChangeLog =================================================================== --- ChangeLog (revision 19219) +++ ChangeLog (revision 19220) @@ -1,3 +1,11 @@ +Sun Sep 7 23:46:36 2008 Tanaka Akira <akr@f...> + + * enc/trans/iso2022.trans: stateless-iso-2022-jp is defined to avoid + undefined conversion error between iso-2022-jp and the corresponding + stateless encoding. + + * enc/emacs_mule.c: replicate emacs-mule as stateless-iso-2022-jp. + Sun Sep 7 20:03:01 2008 Tanaka Akira <akr@f...> * enc/trans/escape.trans (hexstr): renamed from str1. Index: enc/trans/iso2022.trans =================================================================== --- enc/trans/iso2022.trans (revision 19219) +++ enc/trans/iso2022.trans (revision 19220) @@ -1,35 +1,55 @@ #include "transcode_data.h" <% - map = {} - map["1b2842"] = :func_so # designate US-ASCII to G0. "ESC ( B" - map["1b284a"] = :func_so # designate JIS X 0201 latin to G0. "ESC ( J" - map["1b2440"] = :func_so # designate JIS X 0208 1978 to G0. "ESC $ @" - map["1b2442"] = :func_so # designate JIS X 0208 1983 to G0. "ESC $ B" - map["{00-0d,10-1a,1c-7f}"] = :func_si + map = { + "1b2842" => :func_so, # designate US-ASCII to G0. "ESC ( B" + "1b284a" => :func_so, # designate JIS X 0201 latin to G0. "ESC ( J" + "1b2440" => :func_so, # designate JIS X 0208 1978 to G0. "ESC $ @" + "1b2442" => :func_so, # designate JIS X 0208 1983 to G0. "ESC $ B" + "{00-0d,10-1a,1c-7f}" => :func_si, + } + transcode_generate_node(ActionMap.parse(map), "iso2022jp_decoder") - map_jisx0208_rest = {} - map_jisx0208_rest["{21-7e}"] = :func_so + map_jisx0208_rest = { + "{21-7e}" => :func_so + } + transcode_generate_node(ActionMap.parse(map_jisx0208_rest), "iso2022jp_decoder_jisx0208_rest") - transcode_generate_node(ActionMap.parse(map), "iso2022jp_to_eucjp") - transcode_generate_node(ActionMap.parse(map_jisx0208_rest), "iso2022jp_to_eucjp_jisx0208_rest") + map = { + "{00-0d,10-1a,1c-7f}" => :func_so, + "90{a1-fe}{a1-fe}" => :func_so, + "92{a1-fe}{a1-fe}" => :func_so, + } + transcode_generate_node(ActionMap.parse(map), "iso2022jp_encoder") - map_eucjp = { + map = { + "{00-0d,10-1a,1c-7f}" => :nomap, + "90{a1-fe}{a1-fe}" => :func_so, + "92{a1-fe}{a1-fe}" => :func_so, + } + transcode_generate_node(ActionMap.parse(map), "stateless_iso2022jp_to_eucjp") + + map = { "{0e,0f,1b}" => :undef, - "{00-0d,10-1a,1c-7f}" => :func_so, + "{00-0d,10-1a,1c-7f}" => :nomap, "{a1-fe}{a1-fe}" => :func_so, "8e{a1-fe}" => :undef, "8f{a1-fe}{a1-fe}" => :undef, } + transcode_generate_node(ActionMap.parse(map), "eucjp_to_stateless_iso2022jp") - transcode_generate_node(ActionMap.parse(map_eucjp), "eucjp_to_iso2022jp") %> <%= transcode_generated_code %> #define G0_ASCII 0 -#define G0_JISX0208 1 +/* ignore JIS X 0201 latin */ +#define G0_JISX0208_1978 1 +#define G0_JISX0208_1983 2 +#define EMACS_MULE_LEADING_CODE_JISX0208_1978 0220 +#define EMACS_MULE_LEADING_CODE_JISX0208_1983 0222 + static int iso2022jp_init(void *statep) { @@ -39,19 +59,19 @@ } static VALUE -fun_si_iso2022jp_to_eucjp(void *statep, const unsigned char *s, size_t l) +fun_si_iso2022jp_decoder(void *statep, const unsigned char *s, size_t l) { unsigned char *sp = statep; if (*sp == G0_ASCII) return (VALUE)NOMAP; else if (0x21 <= s[0] && s[0] <= 0x7e) - return (VALUE)iso2022jp_to_eucjp_jisx0208_rest; + return (VALUE)iso2022jp_decoder_jisx0208_rest; else return (VALUE)INVALID; } static int -fun_so_iso2022jp_to_eucjp(void *statep, const unsigned char *s, size_t l, unsigned char* o) +fun_so_iso2022jp_decoder(void *statep, const unsigned char *s, size_t l, unsigned char* o) { unsigned char *sp = statep; if (s[0] == 0x1b) { @@ -66,75 +86,94 @@ else { switch (s[l-1]) { case '@': + *sp = G0_JISX0208_1978; + break; + case 'B': - *sp = G0_JISX0208; + *sp = G0_JISX0208_1983; break; } } return 0; } else { - o[0] = s[0] | 0x80; - o[1] = s[1] | 0x80; - return 2; + if (*sp == G0_JISX0208_1978) + o[0] = EMACS_MULE_LEADING_CODE_JISX0208_1978; + else + o[0] = EMACS_MULE_LEADING_CODE_JISX0208_1983; + o[1] = s[0] | 0x80; + o[2] = s[1] | 0x80; + return 3; } } static const rb_transcoder -rb_ISO_2022_JP_to_EUC_JP = { - "ISO-2022-JP", "EUC-JP", iso2022jp_to_eucjp, +rb_iso2022jp_decoder = { + "ISO-2022-JP", "stateless-iso-2022-jp", iso2022jp_decoder, TRANSCODE_TABLE_INFO, 1, /* input_unit_length */ 3, /* max_input */ 3, /* max_output */ stateful_decoder, /* stateful_type */ 1, iso2022jp_init, iso2022jp_init, /* state_size, state_init, state_fini */ - NULL, fun_si_iso2022jp_to_eucjp, NULL, fun_so_iso2022jp_to_eucjp + NULL, fun_si_iso2022jp_decoder, NULL, fun_so_iso2022jp_decoder }; static int -fun_so_eucjp_to_iso2022jp(void *statep, const unsigned char *s, size_t l, unsigned char *o) +fun_so_iso2022jp_encoder(void *statep, const unsigned char *s, size_t l, unsigned char *o) { unsigned char *sp = statep; unsigned char *output0 = o; + int newstate; - if (*sp != (l == 1 ? G0_ASCII : G0_JISX0208)) { - if (l == 1) { + if (l == 1) + newstate = G0_ASCII; + else if (s[0] == EMACS_MULE_LEADING_CODE_JISX0208_1978) + newstate = G0_JISX0208_1978; + else + newstate = G0_JISX0208_1983; + + if (*sp != newstate) { + if (newstate == G0_ASCII) { *o++ = 0x1b; *o++ = '('; *o++ = 'B'; - *sp = G0_ASCII; } + else if (newstate == G0_JISX0208_1978) { + *o++ = 0x1b; + *o++ = '$'; + *o++ = '@'; + } else { *o++ = 0x1b; *o++ = '$'; *o++ = 'B'; - *sp = G0_JISX0208; /* JIS X 0208 1983 */ } + *sp = newstate; } if (l == 1) { *o++ = s[0] & 0x7f; } else { - *o++ = s[0] & 0x7f; *o++ = s[1] & 0x7f; + *o++ = s[2] & 0x7f; } return o - output0; } static int -iso2022jp_reset_sequence_size(void *statep) +iso2022jp_encoder_reset_sequence_size(void *statep) { unsigned char *sp = statep; - if (*sp == G0_JISX0208) + if (*sp != G0_ASCII) return 3; return 0; } static int -finish_eucjp_to_iso2022jp(void *statep, unsigned char *o) +finish_iso2022jp_encoder(void *statep, unsigned char *o) { unsigned char *sp = statep; unsigned char *output0 = o; @@ -151,23 +190,66 @@ } static const rb_transcoder -rb_EUC_JP_to_ISO_2022_JP = { - "EUC-JP", "ISO-2022-JP", eucjp_to_iso2022jp, +rb_iso2022jp_encoder = { + "stateless-iso-2022-jp", "ISO-2022-JP", iso2022jp_encoder, TRANSCODE_TABLE_INFO, 1, /* input_unit_length */ 3, /* max_input */ 5, /* max_output */ stateful_encoder, /* stateful_type */ 1, iso2022jp_init, iso2022jp_init, /* state_size, state_init, state_fini */ - NULL, NULL, NULL, fun_so_eucjp_to_iso2022jp, - finish_eucjp_to_iso2022jp, - iso2022jp_reset_sequence_size, finish_eucjp_to_iso2022jp + NULL, NULL, NULL, fun_so_iso2022jp_encoder, + finish_iso2022jp_encoder, + iso2022jp_encoder_reset_sequence_size, finish_iso2022jp_encoder }; +static int +fun_so_stateless_iso2022jp_to_eucjp(void *statep, const unsigned char *s, size_t l, unsigned char *o) +{ + o[0] = s[1]; + o[1] = s[2]; + return 2; +} + +static const rb_transcoder +rb_stateless_iso2022jp_to_eucjp = { + "stateless-iso-2022-jp", "EUC-JP", stateless_iso2022jp_to_eucjp, + TRANSCODE_TABLE_INFO, + 1, /* input_unit_length */ + 3, /* max_input */ + 2, /* max_output */ + stateless_converter, /* stateful_type */ + 0, NULL, NULL, /* state_size, state_init, state_fini */ + NULL, NULL, NULL, fun_so_stateless_iso2022jp_to_eucjp, +}; + +static int +fun_so_eucjp_to_stateless_iso2022jp(void *statep, const unsigned char *s, size_t l, unsigned char *o) +{ + o[0] = EMACS_MULE_LEADING_CODE_JISX0208_1983; + o[1] = s[0]; + o[2] = s[1]; + return 3; +} + +static const rb_transcoder +rb_eucjp_to_stateless_iso2022jp = { + "EUC-JP", "stateless-iso-2022-jp", eucjp_to_stateless_iso2022jp, + TRANSCODE_TABLE_INFO, + 1, /* input_unit_length */ + 3, /* max_input */ + 3, /* max_output */ + stateless_converter, /* stateful_type */ + 0, NULL, NULL, /* state_size, state_init, state_fini */ + NULL, NULL, NULL, fun_so_eucjp_to_stateless_iso2022jp, +}; + void Init_iso2022(void) { - rb_register_transcoder(&rb_ISO_2022_JP_to_EUC_JP); - rb_register_transcoder(&rb_EUC_JP_to_ISO_2022_JP); + rb_register_transcoder(&rb_iso2022jp_decoder); + rb_register_transcoder(&rb_iso2022jp_encoder); + rb_register_transcoder(&rb_stateless_iso2022jp_to_eucjp); + rb_register_transcoder(&rb_eucjp_to_stateless_iso2022jp); } Index: enc/emacs_mule.c =================================================================== --- enc/emacs_mule.c (revision 19219) +++ enc/emacs_mule.c (revision 19220) @@ -338,3 +338,4 @@ 0 }; +ENC_REPLICATE("stateless-iso-2022-jp", "Emacs-Mule") Index: test/ruby/test_econv.rb =================================================================== --- test/ruby/test_econv.rb (revision 19219) +++ test/ruby/test_econv.rb (revision 19220) @@ -28,8 +28,8 @@ end def test_s_stateless_encoding - assert_equal(Encoding::EUC_JP, Encoding::Converter.stateless_encoding("ISO-2022-JP")) - assert_equal(Encoding::EUC_JP, Encoding::Converter.stateless_encoding(Encoding::ISO_2022_JP)) + assert_equal(Encoding::STATELESS_ISO_2022_JP, Encoding::Converter.stateless_encoding("ISO-2022-JP")) + assert_equal(Encoding::STATELESS_ISO_2022_JP, Encoding::Converter.stateless_encoding(Encoding::ISO_2022_JP)) assert_nil(Encoding::Converter.stateless_encoding("EUC-JP")) assert_nil(Encoding::Converter.stateless_encoding("UTF-8")) assert_nil(Encoding::Converter.stateless_encoding("UTF-16BE")) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/