[前][次][番号順一覧][スレッド一覧]

ruby-changes:28338

From: naruse <ko1@a...>
Date: Sat, 20 Apr 2013 02:50:51 +0900 (JST)
Subject: [ruby-changes:28338] naruse:r40390 (trunk): * string.c (str_scrub): add ruby method String#scrub which verify and

naruse	2013-04-20 02:50:38 +0900 (Sat, 20 Apr 2013)

  New Revision: 40390

  http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=40390

  Log:
    * string.c (str_scrub): add ruby method String#scrub which verify and
      fix invalid byte sequence.
    
    * string.c (str_compat_and_valid): check given string is compatible
      and valid with given encoding.
    
    * transcode.c (str_transcode0): If invalid: :replace is specified for
      String#encode, replace invalid byte sequence even if the destination
      encoding equals to the source encoding.

  Modified files:
    trunk/ChangeLog
    trunk/NEWS
    trunk/string.c
    trunk/test/ruby/test_m17n.rb
    trunk/transcode.c

Index: ChangeLog
===================================================================
--- ChangeLog	(revision 40389)
+++ ChangeLog	(revision 40390)
@@ -1,3 +1,15 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1
+Sat Apr 20 02:33:27 2013  NARUSE, Yui  <naruse@r...>
+
+	* string.c (str_scrub): add ruby method String#scrub which verify and
+	  fix invalid byte sequence. [ruby-dev:45975] [Feature #6752]
+
+	* string.c (str_compat_and_valid): check given string is compatible
+	  and valid with given encoding.
+
+	* transcode.c (str_transcode0): If invalid: :replace is specified for
+	  String#encode, replace invalid byte sequence even if the destination
+	  encoding equals to the source encoding.
+
 Fri Apr 19 21:55:40 2013  Kouhei Sutou  <kou@c...>
 
 	* README.EXT.ja (Data_Wrap_Struct): Remove a description about
Index: string.c
===================================================================
--- string.c	(revision 40389)
+++ string.c	(revision 40390)
@@ -7741,6 +7741,272 @@ rb_str_ellipsize(VALUE str, long len) https://github.com/ruby/ruby/blob/trunk/string.c#L7741
     return ret;
 }
 
+static VALUE
+str_compat_and_valid(VALUE str, rb_encoding *enc)
+{
+    int cr;
+    str = StringValue(str);
+    cr = rb_enc_str_coderange(str);
+    if (cr == ENC_CODERANGE_BROKEN) {
+	rb_raise(rb_eArgError, "replacement must be valid byte sequence '%+"PRIsVALUE"'", str);
+    }
+    else if (cr == ENC_CODERANGE_7BIT) {
+	rb_encoding *e = STR_ENC_GET(str);
+	if (!rb_enc_asciicompat(enc)) {
+	    rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s",
+		    rb_enc_name(enc), rb_enc_name(e));
+	}
+    }
+    else { /* ENC_CODERANGE_VALID */
+	rb_encoding *e = STR_ENC_GET(str);
+	if (enc != e) {
+	    rb_raise(rb_eEncCompatError, "incompatible character encodings: %s and %s",
+		    rb_enc_name(enc), rb_enc_name(e));
+	}
+    }
+    return str;
+}
+
+/*
+ *  call-seq:
+ *    str.scrub -> new_str
+ *    str.scrub(repl) -> new_str
+ *    str.scrub{|bytes|} -> new_str
+ *
+ *  If the string is invalid byte sequence then replace invalid bytes with given replacement
+ *  character, else returns self.
+ *  If block is given, replace invalid bytes with returned value of the block.
+ */
+VALUE
+rb_str_scrub(int argc, VALUE *argv, VALUE str)
+{
+    int cr = ENC_CODERANGE(str);
+    rb_encoding *enc;
+    VALUE repl;
+
+    if (cr == ENC_CODERANGE_7BIT || cr == ENC_CODERANGE_VALID)
+	return rb_str_dup(str);
+
+    enc = STR_ENC_GET(str);
+    rb_scan_args(argc, argv, "01", &repl);
+    if (argc != 0) {
+	repl = str_compat_and_valid(repl, enc);
+    }
+
+    if (rb_enc_dummy_p(enc)) {
+	return rb_str_dup(str);
+    }
+
+    if (rb_enc_asciicompat(enc)) {
+	const char *p = RSTRING_PTR(str);
+	const char *e = RSTRING_END(str);
+	const char *p1 = p;
+	const char *rep;
+	long replen;
+	int rep7bit_p;
+	VALUE buf = rb_str_buf_new(RSTRING_LEN(str));
+	if (rb_block_given_p()) {
+	    rep = NULL;
+	}
+	else if (!NIL_P(repl)) {
+	    rep = RSTRING_PTR(repl);
+	    replen = RSTRING_LEN(repl);
+	    rep7bit_p = (ENC_CODERANGE(repl) == ENC_CODERANGE_7BIT);
+	}
+	else if (enc == rb_utf8_encoding()) {
+	    rep = "\xEF\xBF\xBD";
+	    replen = strlen(rep);
+	    rep7bit_p = FALSE;
+	}
+	else {
+	    rep = "?";
+	    replen = strlen(rep);
+	    rep7bit_p = TRUE;
+	}
+	cr = ENC_CODERANGE_7BIT;
+
+	p = search_nonascii(p, e);
+	if (!p) {
+	    p = e;
+	}
+	while (p < e) {
+	    int ret = rb_enc_precise_mbclen(p, e, enc);
+	    if (MBCLEN_NEEDMORE_P(ret)) {
+		break;
+	    }
+	    else if (MBCLEN_CHARFOUND_P(ret)) {
+		cr = ENC_CODERANGE_VALID;
+		p += MBCLEN_CHARFOUND_LEN(ret);
+	    }
+	    else if (MBCLEN_INVALID_P(ret)) {
+		/*
+		 * p1~p: valid ascii/multibyte chars
+		 * p ~e: invalid bytes + unknown bytes
+		 */
+		long clen = rb_enc_mbmaxlen(enc);
+		if (p > p1) {
+		    rb_str_buf_cat(buf, p1, p - p1);
+		}
+
+		if (e - p < clen) clen = e - p;
+		if (clen <= 2) {
+		    clen = 1;
+		}
+		else {
+		    const char *q = p;
+		    clen--;
+		    for (; clen > 1; clen--) {
+			ret = rb_enc_precise_mbclen(q, q + clen, enc);
+			if (MBCLEN_NEEDMORE_P(ret)) break;
+			else if (MBCLEN_INVALID_P(ret)) continue;
+			else UNREACHABLE;
+		    }
+		}
+		if (rep) {
+		    rb_str_buf_cat(buf, rep, replen);
+		    if (!rep7bit_p) cr = ENC_CODERANGE_VALID;
+		}
+		else {
+		    repl = rb_yield(rb_enc_str_new(p1, clen, enc));
+		    repl = str_compat_and_valid(repl, enc);
+		    rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl));
+		    if (ENC_CODERANGE(repl) == ENC_CODERANGE_VALID)
+			cr = ENC_CODERANGE_VALID;
+		}
+		p += clen;
+		p1 = p;
+		p = search_nonascii(p, e);
+		if (!p) {
+		    p = e;
+		    break;
+		}
+	    }
+	    else {
+		UNREACHABLE;
+	    }
+	}
+	if (p1 < p) {
+	    rb_str_buf_cat(buf, p1, p - p1);
+	}
+	if (p < e) {
+	    if (rep) {
+		rb_str_buf_cat(buf, rep, replen);
+		if (!rep7bit_p) cr = ENC_CODERANGE_VALID;
+	    }
+	    else {
+		repl = rb_yield(rb_enc_str_new(p, e-p, enc));
+		repl = str_compat_and_valid(repl, enc);
+		rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl));
+		if (ENC_CODERANGE(repl) == ENC_CODERANGE_VALID)
+		    cr = ENC_CODERANGE_VALID;
+	    }
+	}
+	ENCODING_CODERANGE_SET(buf, rb_enc_to_index(enc), cr);
+	return buf;
+    }
+    else {
+	/* ASCII incompatible */
+	const char *p = RSTRING_PTR(str);
+	const char *e = RSTRING_END(str);
+	const char *p1 = p;
+	VALUE buf = rb_str_buf_new(RSTRING_LEN(str));
+	const char *rep;
+	long replen;
+	long mbminlen = rb_enc_mbminlen(enc);
+	static rb_encoding *utf16be;
+	static rb_encoding *utf16le;
+	static rb_encoding *utf32be;
+	static rb_encoding *utf32le;
+	if (!utf16be) {
+	    utf16be = rb_enc_find("UTF-16BE");
+	    utf16le = rb_enc_find("UTF-16LE");
+	    utf32be = rb_enc_find("UTF-32BE");
+	    utf32le = rb_enc_find("UTF-32LE");
+	}
+	if (!NIL_P(repl)) {
+	    rep = RSTRING_PTR(repl);
+	    replen = RSTRING_LEN(repl);
+	}
+	else if (enc == utf16be) {
+	    rep = "\xFF\xFD";
+	    replen = strlen(rep);
+	}
+	else if (enc == utf16le) {
+	    rep = "\xFD\xFF";
+	    replen = strlen(rep);
+	}
+	else if (enc == utf32be) {
+	    rep = "\x00\x00\xFF\xFD";
+	    replen = strlen(rep);
+	}
+	else if (enc == utf32le) {
+	    rep = "\xFD\xFF\x00\x00";
+	    replen = strlen(rep);
+	}
+	else {
+	    rep = "?";
+	    replen = strlen(rep);
+	}
+
+	while (p < e) {
+	    int ret = rb_enc_precise_mbclen(p, e, enc);
+	    if (MBCLEN_NEEDMORE_P(ret)) {
+		break;
+	    }
+	    else if (MBCLEN_CHARFOUND_P(ret)) {
+		p += MBCLEN_CHARFOUND_LEN(ret);
+	    }
+	    else if (MBCLEN_INVALID_P(ret)) {
+		const char *q = p;
+		long clen = rb_enc_mbmaxlen(enc);
+		if (p > p1) rb_str_buf_cat(buf, p1, p - p1);
+
+		if (e - p < clen) clen = e - p;
+		if (clen <= mbminlen * 2) {
+		    clen = mbminlen;
+		}
+		else {
+		    clen -= mbminlen;
+		    for (; clen > mbminlen; clen-=mbminlen) {
+			ret = rb_enc_precise_mbclen(q, q + clen, enc);
+			if (MBCLEN_NEEDMORE_P(ret)) break;
+			else if (MBCLEN_INVALID_P(ret)) continue;
+			else UNREACHABLE;
+		    }
+		}
+		if (rep) {
+		    rb_str_buf_cat(buf, rep, replen);
+		}
+		else {
+		    repl = rb_yield(rb_enc_str_new(p, e-p, enc));
+		    repl = str_compat_and_valid(repl, enc);
+		    rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl));
+		}
+		p += clen;
+		p1 = p;
+	    }
+	    else {
+		UNREACHABLE;
+	    }
+	}
+	if (p1 < p) {
+	    rb_str_buf_cat(buf, p1, p - p1);
+	}
+	if (p < e) {
+	    if (rep) {
+		rb_str_buf_cat(buf, rep, replen);
+	    }
+	    else {
+		repl = rb_yield(rb_enc_str_new(p, e-p, enc));
+		repl = str_compat_and_valid(repl, enc);
+		rb_str_buf_cat(buf, RSTRING_PTR(repl), RSTRING_LEN(repl));
+	    }
+	}
+	ENCODING_CODERANGE_SET(buf, rb_enc_to_index(enc), ENC_CODERANGE_VALID);
+	return buf;
+    }
+}
+
 /**********************************************************************
  * Document-class: Symbol
  *
@@ -8226,6 +8492,7 @@ Init_String(void) https://github.com/ruby/ruby/blob/trunk/string.c#L8492
     rb_define_method(rb_cString, "getbyte", rb_str_getbyte, 1);
     rb_define_method(rb_cString, "setbyte", rb_str_setbyte, 2);
     rb_define_method(rb_cString, "byteslice", rb_str_byteslice, -1);
+    rb_define_method(rb_cString, "scrub", rb_str_scrub, -1);
 
     rb_define_method(rb_cString, "to_i", rb_str_to_i, -1);
     rb_define_method(rb_cString, "to_f", rb_str_to_f, 0);
Index: NEWS
===================================================================
--- NEWS	(revision 40389)
+++ NEWS	(revision 40390)
@@ -26,6 +26,14 @@ with all sufficient information, see the https://github.com/ruby/ruby/blob/trunk/NEWS#L26
   * misc
     * Mutex#owned? is no longer experimental.
 
+* String
+  * New methods:
+    * added String#scrub to verify and fix invalid byte sequence.
+  * extended methods:
+    * If invalid: :replace is specified for String#encode, replace
+      invalid byte sequence even if the destination encoding equals to
+      the source encoding.
+
 * pack/unpack (Array/String)
   * Q! and q! directives for long long type if platform has the type.
 
Index: test/ruby/test_m17n.rb
===================================================================
--- test/ruby/test_m17n.rb	(revision 40389)
+++ test/ruby/test_m17n.rb	(revision 40390)
@@ -1489,4 +1489,38 @@ class TestM17N < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ruby/test_m17n.rb#L1489
     s.untrust
     assert_equal(true, s.b.untrusted?)
   end
+
+  def test_scrub
+    assert_equal("\uFFFD\uFFFD\uFFFD", u("\x80\x80\x80").scrub)
+    assert_equal("\uFFFDA", u("\xF4\x80\x80A").scrub)
+
+    # exapmles in Unicode 6.1.0 D93b
+    assert_equal("\x41\uFFFD\uFFFD\x41\uFFFD\x41",
+                 u("\x41\xC0\xAF\x41\xF4\x80\x80\x41").scrub)
+    assert_equal("\x41\uFFFD\uFFFD\uFFFD\x41",
+                 u("\x41\xE0\x9F\x80\x41").scrub)
+    assert_equal("\u0061\uFFFD\uFFFD\uFFFD\u0062\uFFFD\u0063\uFFFD\uFFFD\u0064",
+                 u("\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64").scrub)
+    assert_equal("abcdefghijklmnopqrstuvwxyz\u0061\uFFFD\uFFFD\uFFFD\u0062\uFFFD\u0063\uFFFD\uFFFD\u0064",
+                 u("abcdefghijklmnopqrstuvwxyz\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64").scrub)
+
+    assert_equal("\u3042\u3013", u("\xE3\x81\x82\xE3\x81").scrub("\u3013"))
+    assert_raise(Encoding::CompatibilityError){ u("\xE3\x81\x82\xE3\x81").scrub(e("\xA4\xA2")) }
+    assert_raise(TypeError){ u("\xE3\x81\x82\xE3\x81").scrub(1) }
+    assert_raise(ArgumentError){ u("\xE3\x81\x82\xE3\x81\x82\xE3\x81").scrub(u("\x81")) }
+    assert_equal(e("\xA4\xA2\xA2\xAE"), e("\xA4\xA2\xA4").scrub(e("\xA2\xAE")))
+
+    assert_equal("\u3042<e381>", u("\xE3\x81\x82\xE3\x81").scrub{|x|'<'+x.unpack('H*')[0]+'>'})
+    assert_raise(Encoding::CompatibilityError){ u("\xE3\x81\x82\xE3\x81").scrub{e("\xA4\xA2")} }
+    assert_raise(TypeError){ u("\xE3\x81\x82\xE3\x81").scrub{1} }
+    assert_raise(ArgumentError){ u("\xE3\x81\x82\xE3\x81\x82\xE3\x81").scrub{u("\x81")} }
+    assert_equal(e("\xA4\xA2\xA2\xAE"), e("\xA4\xA2\xA4").scrub{e("\xA2\xAE")})
+
+    assert_equal("\uFFFD\u3042".encode("UTF-16BE"),
+                 "\xD8\x00\x30\x42".force_encoding(Encoding::UTF_16BE).
+                 scrub)
+    assert_equal("\uFFFD\u3042".encode("UTF-16LE"),
+                 "\x00\xD8\x42\x30".force_encoding(Encoding::UTF_16LE).
+                 scrub)
+  end
 end
Index: transcode.c
===================================================================
--- transcode.c	(revision 40389)
+++ transcode.c	(revision 40390)
@@ -2652,6 +2652,8 @@ str_transcode_enc_args(VALUE str, volati https://github.com/ruby/ruby/blob/trunk/transcode.c#L2652
     return dencidx;
 }
 
+VALUE rb_str_scrub(int argc, VALUE *argv, VALUE str);
+
 static int
 str_transcode0(int argc, VALUE *argv, VALUE *self, int ecflags, VALUE ecopts)
 {
@@ -2686,6 +2688,17 @@ str_transcode0(int argc, VALUE *argv, VA https://github.com/ruby/ruby/blob/trunk/transcode.c#L2688
                     ECONV_XML_ATTR_CONTENT_DECORATOR|
                     ECONV_XML_ATTR_QUOTE_DECORATOR)) == 0) {
         if (senc && senc == denc) {
+	    if (ecflags & ECONV_INVALID_MASK) {
+		if (!NIL_P(ecopts)) {
+		    VALUE rep = rb_hash_aref(ecopts, sym_replace);
+		    dest = rb_str_scrub(1, &rep, str);
+		}
+		else {
+		    dest = rb_str_scrub(0, NULL, str);
+		}
+		*self = dest;
+		return dencidx;
+	    }
             return NIL_P(arg2) ? -1 : dencidx;
         }
         if (senc && denc && rb_enc_asciicompat(senc) && rb_enc_asciicompat(denc)) {
@@ -2815,10 +2828,6 @@ static VALUE encoded_dup(VALUE newstr, V https://github.com/ruby/ruby/blob/trunk/transcode.c#L2828
  *  in the source encoding. The last form by default does not raise
  *  exceptions but uses replacement strings.
  *
- *  Please note that conversion from an encoding +enc+ to the
- *  same encoding +enc+ is a no-op, i.e. the receiver is returned without
- *  any changes, and no exceptions are raised, even if there are invalid bytes.
- *
  *  The +options+ Hash gives details for conversion and can have the following
  *  keys:
  *

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/

[前][次][番号順一覧][スレッド一覧]