ruby-changes:28338
From: naruse <ko1@a...>
Date: Sat, 20 Apr 2013 02:50:51 +0900 (JST)
Subject: [ruby-changes:28338] naruse:r40390 (trunk): * string.c (str_scrub): add ruby method String#scrub which verify and
naruse 2013-04-20 02:50:38 +0900 (Sat, 20 Apr 2013) New Revision: 40390 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=40390 Log: * string.c (str_scrub): add ruby method String#scrub which verify and fix invalid byte sequence. * string.c (str_compat_and_valid): check given string is compatible and valid with given encoding. * transcode.c (str_transcode0): If invalid: :replace is specified for String#encode, replace invalid byte sequence even if the destination encoding equals to the source encoding. Modified files: trunk/ChangeLog trunk/NEWS trunk/string.c trunk/test/ruby/test_m17n.rb trunk/transcode.c Index: ChangeLog =================================================================== --- ChangeLog (revision 40389) +++ ChangeLog (revision 40390) @@ -1,3 +1,15 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1 +Sat Apr 20 02:33:27 2013 NARUSE, Yui <naruse@r...> + + * string.c (str_scrub): add ruby method String#scrub which verify and + fix invalid byte sequence. [ruby-dev:45975] [Feature #6752] + + * string.c (str_compat_and_valid): check given string is compatible + and valid with given encoding. + + * transcode.c (str_transcode0): If invalid: :replace is specified for + String#encode, replace invalid byte sequence even if the destination + encoding equals to the source encoding. + Fri Apr 19 21:55:40 2013 Kouhei Sutou <kou@c...> * README.EXT.ja (Data_Wrap_Struct): Remove a description about Index: string.c =================================================================== --- string.c (revision 40389) +++ string.c (revision 40390) @@ -7741,6 +7741,272 @@ rb_str_ellipsize(VALUE str, long len) https://github.com/ruby/ruby/blob/trunk/string.c#L7741 return ret; } +static VALUE +str_compat_and_valid(VALUE str, rb_encoding *enc) +{ + int cr; + str = StringValue(str); + cr = rb_enc_str_coderange(str); + if (cr == ENC_CODERANGE_BROKEN) { + rb_raise(rb_eArgError, "replacement must be valid byte sequence '%+"PRIsVALUE"'", str); + } + else if (cr == ENC_CODERANGE_7BIT) { + rb_encoding *e = STR_ENC_GET(str); + if (!rb_enc_asciicompat(enc)) { + rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s", + rb_enc_name(enc), rb_enc_name(e)); + } + } + else { /* ENC_CODERANGE_VALID */ + rb_encoding *e = STR_ENC_GET(str); + if (enc != e) { + rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s", + rb_enc_name(enc), rb_enc_name(e)); + } + } + return str; +} + +/* + * call-seq: + * str.scrub -> new_str + * str.scrub(repl) -> new_str + * str.scrub{|bytes|} -> new_str + * + * If the string is invalid byte sequence then replace invalid bytes with given replacement + * character, else returns self. + * If block is given, replace invalid bytes with returned value of the block. + */ +VALUE +rb_str_scrub(int argc, VALUE *argv, VALUE str) +{ + int cr = ENC_CODERANGE(str); + rb_encoding *enc; + VALUE repl; + + if (cr == ENC_CODERANGE_7BIT || cr == ENC_CODERANGE_VALID) + return rb_str_dup(str); + + enc = STR_ENC_GET(str); + rb_scan_args(argc, argv, "01", &repl); + if (argc != 0) { + repl = str_compat_and_valid(repl, enc); + } + + if (rb_enc_dummy_p(enc)) { + return rb_str_dup(str); + } + + if (rb_enc_asciicompat(enc)) { + const char *p = RSTRING_PTR(str); + const char *e = RSTRING_END(str); + const char *p1 = p; + const char *rep; + long replen; + int rep7bit_p; + VALUE buf = rb_str_buf_new(RSTRING_LEN(str)); + if (rb_block_given_p()) { + rep = NULL; + } + else if (!NIL_P(repl)) { + rep = RSTRING_PTR(repl); + replen = RSTRING_LEN(repl); + rep7bit_p = (ENC_CODERANGE(repl) == ENC_CODERANGE_7BIT); + } + else if (enc == rb_utf8_encoding()) { + rep = "\xEF\xBF\xBD"; + replen = strlen(rep); + rep7bit_p = FALSE; + } + else { + rep = "?"; + replen = strlen(rep); + rep7bit_p = TRUE; + } + cr = ENC_CODERANGE_7BIT; + + p = search_nonascii(p, e); + if (!p) { + p = e; + } + while (p < e) { + int ret = rb_enc_precise_mbclen(p, e, enc); + if (MBCLEN_NEEDMORE_P(ret)) { + break; + } + else if (MBCLEN_CHARFOUND_P(ret)) { + cr = ENC_CODERANGE_VALID; + p += MBCLEN_CHARFOUND_LEN(ret); + } + else if (MBCLEN_INVALID_P(ret)) { + /* + * p1~p: valid ascii/multibyte chars + * p ~e: invalid bytes + unknown bytes + */ + long clen = rb_enc_mbmaxlen(enc); + if (p > p1) { + rb_str_buf_cat(buf, p1, p - p1); + } + + if (e - p < clen) clen = e - p; + if (clen <= 2) { + clen = 1; + } + else { + const char *q = p; + clen--; + for (; clen > 1; clen--) { + ret = rb_enc_precise_mbclen(q, q + clen, enc); + if (MBCLEN_NEEDMORE_P(ret)) break; + else if (MBCLEN_INVALID_P(ret)) continue; + else UNREACHABLE; + } + } + if (rep) { + rb_str_buf_cat(buf, rep, replen); + if (!rep7bit_p) cr = ENC_CODERANGE_VALID; + } + else { + repl = rb_yield(rb_enc_str_new(p1, clen, enc)); + repl = str_compat_and_valid(repl, enc); + rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl)); + if (ENC_CODERANGE(repl) == ENC_CODERANGE_VALID) + cr = ENC_CODERANGE_VALID; + } + p += clen; + p1 = p; + p = search_nonascii(p, e); + if (!p) { + p = e; + break; + } + } + else { + UNREACHABLE; + } + } + if (p1 < p) { + rb_str_buf_cat(buf, p1, p - p1); + } + if (p < e) { + if (rep) { + rb_str_buf_cat(buf, rep, replen); + if (!rep7bit_p) cr = ENC_CODERANGE_VALID; + } + else { + repl = rb_yield(rb_enc_str_new(p, e-p, enc)); + repl = str_compat_and_valid(repl, enc); + rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl)); + if (ENC_CODERANGE(repl) == ENC_CODERANGE_VALID) + cr = ENC_CODERANGE_VALID; + } + } + ENCODING_CODERANGE_SET(buf, rb_enc_to_index(enc), cr); + return buf; + } + else { + /* ASCII incompatible */ + const char *p = RSTRING_PTR(str); + const char *e = RSTRING_END(str); + const char *p1 = p; + VALUE buf = rb_str_buf_new(RSTRING_LEN(str)); + const char *rep; + long replen; + long mbminlen = rb_enc_mbminlen(enc); + static rb_encoding *utf16be; + static rb_encoding *utf16le; + static rb_encoding *utf32be; + static rb_encoding *utf32le; + if (!utf16be) { + utf16be = rb_enc_find("UTF-16BE"); + utf16le = rb_enc_find("UTF-16LE"); + utf32be = rb_enc_find("UTF-32BE"); + utf32le = rb_enc_find("UTF-32LE"); + } + if (!NIL_P(repl)) { + rep = RSTRING_PTR(repl); + replen = RSTRING_LEN(repl); + } + else if (enc == utf16be) { + rep = "\xFF\xFD"; + replen = strlen(rep); + } + else if (enc == utf16le) { + rep = "\xFD\xFF"; + replen = strlen(rep); + } + else if (enc == utf32be) { + rep = "\x00\x00\xFF\xFD"; + replen = strlen(rep); + } + else if (enc == utf32le) { + rep = "\xFD\xFF\x00\x00"; + replen = strlen(rep); + } + else { + rep = "?"; + replen = strlen(rep); + } + + while (p < e) { + int ret = rb_enc_precise_mbclen(p, e, enc); + if (MBCLEN_NEEDMORE_P(ret)) { + break; + } + else if (MBCLEN_CHARFOUND_P(ret)) { + p += MBCLEN_CHARFOUND_LEN(ret); + } + else if (MBCLEN_INVALID_P(ret)) { + const char *q = p; + long clen = rb_enc_mbmaxlen(enc); + if (p > p1) rb_str_buf_cat(buf, p1, p - p1); + + if (e - p < clen) clen = e - p; + if (clen <= mbminlen * 2) { + clen = mbminlen; + } + else { + clen -= mbminlen; + for (; clen > mbminlen; clen-=mbminlen) { + ret = rb_enc_precise_mbclen(q, q + clen, enc); + if (MBCLEN_NEEDMORE_P(ret)) break; + else if (MBCLEN_INVALID_P(ret)) continue; + else UNREACHABLE; + } + } + if (rep) { + rb_str_buf_cat(buf, rep, replen); + } + else { + repl = rb_yield(rb_enc_str_new(p, e-p, enc)); + repl = str_compat_and_valid(repl, enc); + rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl)); + } + p += clen; + p1 = p; + } + else { + UNREACHABLE; + } + } + if (p1 < p) { + rb_str_buf_cat(buf, p1, p - p1); + } + if (p < e) { + if (rep) { + rb_str_buf_cat(buf, rep, replen); + } + else { + repl = rb_yield(rb_enc_str_new(p, e-p, enc)); + repl = str_compat_and_valid(repl, enc); + rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl)); + } + } + ENCODING_CODERANGE_SET(buf, rb_enc_to_index(enc), ENC_CODERANGE_VALID); + return buf; + } +} + /********************************************************************** * Document-class: Symbol * @@ -8226,6 +8492,7 @@ Init_String(void) https://github.com/ruby/ruby/blob/trunk/string.c#L8492 rb_define_method(rb_cString, "getbyte", rb_str_getbyte, 1); rb_define_method(rb_cString, "setbyte", rb_str_setbyte, 2); rb_define_method(rb_cString, "byteslice", rb_str_byteslice, -1); + rb_define_method(rb_cString, "scrub", rb_str_scrub, -1); rb_define_method(rb_cString, "to_i", rb_str_to_i, -1); rb_define_method(rb_cString, "to_f", rb_str_to_f, 0); Index: NEWS =================================================================== --- NEWS (revision 40389) +++ NEWS (revision 40390) @@ -26,6 +26,14 @@ with all sufficient information, see the https://github.com/ruby/ruby/blob/trunk/NEWS#L26 * misc * Mutex#owned? is no longer experimental. +* String + * New methods: + * added String#scrub to verify and fix invalid byte sequence. + * extended methods: + * If invalid: :replace is specified for String#encode, replace + invalid byte sequence even if the destination encoding equals to + the source encoding. + * pack/unpack (Array/String) * Q! and q! directives for long long type if platform has the type. Index: test/ruby/test_m17n.rb =================================================================== --- test/ruby/test_m17n.rb (revision 40389) +++ test/ruby/test_m17n.rb (revision 40390) @@ -1489,4 +1489,38 @@ class TestM17N < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ruby/test_m17n.rb#L1489 s.untrust assert_equal(true, s.b.untrusted?) end + + def test_scrub + assert_equal("\uFFFD\uFFFD\uFFFD", u("\x80\x80\x80").scrub) + assert_equal("\uFFFDA", u("\xF4\x80\x80A").scrub) + + # exapmles in Unicode 6.1.0 D93b + assert_equal("\x41\uFFFD\uFFFD\x41\uFFFD\x41", + u("\x41\xC0\xAF\x41\xF4\x80\x80\x41").scrub) + assert_equal("\x41\uFFFD\uFFFD\uFFFD\x41", + u("\x41\xE0\x9F\x80\x41").scrub) + assert_equal("\u0061\uFFFD\uFFFD\uFFFD\u0062\uFFFD\u0063\uFFFD\uFFFD\u0064", + u("\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64").scrub) + assert_equal("abcdefghijklmnopqrstuvwxyz\u0061\uFFFD\uFFFD\uFFFD\u0062\uFFFD\u0063\uFFFD\uFFFD\u0064", + u("abcdefghijklmnopqrstuvwxyz\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64").scrub) + + assert_equal("\u3042\u3013", u("\xE3\x81\x82\xE3\x81").scrub("\u3013")) + assert_raise(Encoding::CompatibilityError){ u("\xE3\x81\x82\xE3\x81").scrub(e("\xA4\xA2")) } + assert_raise(TypeError){ u("\xE3\x81\x82\xE3\x81").scrub(1) } + assert_raise(ArgumentError){ u("\xE3\x81\x82\xE3\x81\x82\xE3\x81").scrub(u("\x81")) } + assert_equal(e("\xA4\xA2\xA2\xAE"), e("\xA4\xA2\xA4").scrub(e("\xA2\xAE"))) + + assert_equal("\u3042<e381>", u("\xE3\x81\x82\xE3\x81").scrub{|x|'<'+x.unpack('H*')[0]+'>'}) + assert_raise(Encoding::CompatibilityError){ u("\xE3\x81\x82\xE3\x81").scrub{e("\xA4\xA2")} } + assert_raise(TypeError){ u("\xE3\x81\x82\xE3\x81").scrub{1} } + assert_raise(ArgumentError){ u("\xE3\x81\x82\xE3\x81\x82\xE3\x81").scrub{u("\x81")} } + assert_equal(e("\xA4\xA2\xA2\xAE"), e("\xA4\xA2\xA4").scrub{e("\xA2\xAE")}) + + assert_equal("\uFFFD\u3042".encode("UTF-16BE"), + "\xD8\x00\x30\x42".force_encoding(Encoding::UTF_16BE). + scrub) + assert_equal("\uFFFD\u3042".encode("UTF-16LE"), + "\x00\xD8\x42\x30".force_encoding(Encoding::UTF_16LE). + scrub) + end end Index: transcode.c =================================================================== --- transcode.c (revision 40389) +++ transcode.c (revision 40390) @@ -2652,6 +2652,8 @@ str_transcode_enc_args(VALUE str, volati https://github.com/ruby/ruby/blob/trunk/transcode.c#L2652 return dencidx; } +VALUE rb_str_scrub(int argc, VALUE *argv, VALUE str); + static int str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts) { @@ -2686,6 +2688,17 @@ str_transcode0(int argc, VALUE *argv, VA https://github.com/ruby/ruby/blob/trunk/transcode.c#L2688 ECONV_XML_ATTR_CONTENT_DECORATOR| ECONV_XML_ATTR_QUOTE_DECORATOR)) == 0) { if (senc && senc == denc) { + if (ecflags & ECONV_INVALID_MASK) { + if (!NIL_P(ecopts)) { + VALUE rep = rb_hash_aref(ecopts, sym_replace); + dest = rb_str_scrub(1, &rep, str); + } + else { + dest = rb_str_scrub(0, NULL, str); + } + *self = dest; + return dencidx; + } return NIL_P(arg2) ? -1 : dencidx; } if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) { @@ -2815,10 +2828,6 @@ static VALUE encoded_dup(VALUE newstr, V https://github.com/ruby/ruby/blob/trunk/transcode.c#L2828 * in the source encoding. The last form by default does not raise * exceptions but uses replacement strings. * - * Please note that conversion from an encoding +enc+ to the - * same encoding +enc+ is a no-op, i.e. the receiver is returned without - * any changes, and no exceptions are raised, even if there are invalid bytes. - * * The +options+ Hash gives details for conversion and can have the following * keys: * -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/