[前][次][番号順一覧][スレッド一覧]

ruby-changes:46911

From: duerst <ko1@a...>
Date: Tue, 6 Jun 2017 19:34:16 +0900 (JST)
Subject: [ruby-changes:46911] duerst:r59026 (trunk): add test_ill_formed_utf_8_replace to test recommended number of \uFFFD

duerst	2017-06-06 19:34:10 +0900 (Tue, 06 Jun 2017)

  New Revision: 59026

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=59026

  Log:
    add test_ill_formed_utf_8_replace to test recommended number of \uFFFD
    
    Add test method test_ill_formed_utf_8_replace to test/ruby/test_transcode.rb
    to check for the recommended number of \uFFFD replacement characters.
    This is the first part, using ill-formed prefixes, with suffixes up to
    the length of the original UTF-8 structure (including overlongs and
    the full 31-bit space.)
    
    For more details, see Unicode 9.0.0, Section 3.9, Best Practices for Using U+FFFD.

  Modified files:
    trunk/test/ruby/test_transcode.rb
Index: test/ruby/test_transcode.rb
===================================================================
--- test/ruby/test_transcode.rb	(revision 59025)
+++ test/ruby/test_transcode.rb	(revision 59026)
@@ -998,6 +998,80 @@ class TestTranscode < Test::Unit::TestCa https://github.com/ruby/ruby/blob/trunk/test/ruby/test_transcode.rb#L998
     check_both_ways("\u00A0", "\xFF", 'CP855') # non-breaking space
   end
 
+  def test_ill_formed_utf_8_replace
+    fffd1 = "\uFFFD".encode 'UTF-16BE'
+    fffd2 = "\uFFFD\uFFFD".encode 'UTF-16BE'
+    fffd3 = "\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
+    fffd4 = "\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
+    fffd5 = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
+    fffd6 = "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD".encode 'UTF-16BE'
+
+    assert_equal fffd2, "\xC0\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xC0\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xC1\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xC1\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xE0\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xE0\x9F".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xE0\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xE0\x9F\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xED\xA0".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xED\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xED\xA0\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xED\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF0\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF0\x8F".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF0\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF0\x8F\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF0\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF0\x8F\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF4\x90".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF4\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF4\x90\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF4\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF4\x90\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF4\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF5\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF7\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF5\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF7\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF5\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF7\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xF8".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFB".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xF8\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFB\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xF8\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFB\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xF8\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFB\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xF8\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFB\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFC".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFD".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFC\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFD\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFC\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFD\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFC\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFD\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFC\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFD\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd6, "\xFC\x80\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd6, "\xFD\xBF\xBF\xBF\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFE".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd1, "\xFF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFE\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd2, "\xFF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFE\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd3, "\xFF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFE\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd4, "\xFF\xBF\xBF\xBF".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFE\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd5, "\xFF\xBF\xBF\xBF\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd6, "\xFE\x80\x80\x80\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+    assert_equal fffd6, "\xFF\xBF\xBF\xBF\x80\x80".encode("utf-16be", "utf-8", invalid: :replace)
+  end
+
   def check_utf_16_both_ways(utf8, raw)
     copy = raw.dup
     0.step(copy.length-1, 2) { |i| copy[i+1], copy[i] = copy[i], copy[i+1] }

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/

[前][次][番号順一覧][スレッド一覧]