ruby-changes:28408
From: naruse <ko1@a...>
Date: Thu, 25 Apr 2013 14:27:50 +0900 (JST)
Subject: [ruby-changes:28408] naruse:r40460 (trunk): * lib/uri/common.rb (URI.decode_www_form): follow current URL Standard.
naruse 2013-04-25 14:27:41 +0900 (Thu, 25 Apr 2013) New Revision: 40460 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=40460 Log: * lib/uri/common.rb (URI.decode_www_form): follow current URL Standard. It gets encoding argument to specify the character encoding. It now allows loose percent encoded strings, but denies ;-separator. [ruby-core:53475] [Bug #8103] * lib/uri/common.rb (URI.decode_www_form): follow current URL Standard. It gets encoding argument to convert before percent encode. Now UTF-16 strings aren't converted to UTF-8 before percent encode by default. Modified files: trunk/ChangeLog trunk/NEWS trunk/lib/uri/common.rb trunk/test/uri/test_common.rb Index: ChangeLog =================================================================== --- ChangeLog (revision 40459) +++ ChangeLog (revision 40460) @@ -1,3 +1,15 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1 +Thu Apr 25 14:26:32 2013 NARUSE, Yui <naruse@r...> + + * lib/uri/common.rb (URI.decode_www_form): follow current URL Standard. + It gets encoding argument to specify the character encoding. + It now allows loose percent encoded strings, but denies ;-separator. + [ruby-core:53475] [Bug #8103] + + * lib/uri/common.rb (URI.decode_www_form): follow current URL Standard. + It gets encoding argument to convert before percent encode. + Now UTF-16 strings aren't converted to UTF-8 before percent encode + by default. + Wed Apr 25 14:26:00 2013 Charlie Somerville <charlie@c...> * benchmark/bm_hash_shift.rb: add benchmark for Hash#shift Index: lib/uri/common.rb =================================================================== --- lib/uri/common.rb (revision 40459) +++ lib/uri/common.rb (revision 40460) @@ -873,18 +873,21 @@ module URI https://github.com/ruby/ruby/blob/trunk/lib/uri/common.rb#L873 # This method doesn't convert *, -, ., 0-9, A-Z, _, a-z, but does convert SP # (ASCII space) to + and converts others to %XX. # + # If +enc+ is given, convert +str+ to the encoding before percent encoding. + # # This is an implementation of # http://www.w3.org/TR/html5/association-of-controls-and-forms.html#url-encoded-form-data # # See URI.decode_www_form_component, URI.encode_www_form - def self.encode_www_form_component(str) - str = str.to_s - if HTML5ASCIIINCOMPAT.include?(str.encoding) - str = str.encode(Encoding::UTF_8) - else - str = str.dup + def self.encode_www_form_component(str, enc=nil) + str = str.to_s.dup + if str.encoding != Encoding::ASCII_8BIT + if enc && enc != Encoding::ASCII_8BIT + str.encode!(Encoding::UTF_8, invalid: :replace, undef: :replace) + str.encode!(enc, fallback: ->(x){"&#{x.ord};"}) + end + str.force_encoding(Encoding::ASCII_8BIT) end - str.force_encoding(Encoding::ASCII_8BIT) str.gsub!(/[^*\-.0-9A-Z_a-z]/, TBLENCWWWCOMP_) str.force_encoding(Encoding::US_ASCII) end @@ -914,8 +917,7 @@ module URI https://github.com/ruby/ruby/blob/trunk/lib/uri/common.rb#L917 # This method doesn't handle files. When you send a file, use # multipart/form-data. # - # This is an implementation of - # http://www.w3.org/TR/html5/forms.html#url-encoded-form-data + # This refers http://url.spec.whatwg.org/#concept-urlencoded-serializer # # URI.encode_www_form([["q", "ruby"], ["lang", "en"]]) # #=> "q=ruby&lang=en" @@ -927,39 +929,33 @@ module URI https://github.com/ruby/ruby/blob/trunk/lib/uri/common.rb#L929 # #=> "q=ruby&q=perl&lang=en" # # See URI.encode_www_form_component, URI.decode_www_form - def self.encode_www_form(enum) + def self.encode_www_form(enum, enc=nil) enum.map do |k,v| if v.nil? - encode_www_form_component(k) + encode_www_form_component(k, enc) elsif v.respond_to?(:to_ary) v.to_ary.map do |w| - str = encode_www_form_component(k) + str = encode_www_form_component(k, enc) unless w.nil? str << '=' - str << encode_www_form_component(w) + str << encode_www_form_component(w, enc) end end.join('&') else - str = encode_www_form_component(k) + str = encode_www_form_component(k, enc) str << '=' - str << encode_www_form_component(v) + str << encode_www_form_component(v, enc) end end.join('&') end - WFKV_ = '(?:[^%#=;&]*(?:%\h\h[^%#=;&]*)*)' # :nodoc: - # Decode URL-encoded form data from given +str+. # # This decodes application/x-www-form-urlencoded data # and returns array of key-value array. - # This internally uses URI.decode_www_form_component. - # - # _charset_ hack is not supported now because the mapping from given charset - # to Ruby's encoding is not clear yet. - # see also http://www.w3.org/TR/html5/syntax.html#character-encodings-0 # - # This refers http://www.w3.org/TR/html5/forms.html#url-encoded-form-data + # This refers http://url.spec.whatwg.org/#concept-urlencoded-parser , + # so this supports only &-separator, don't support ;-separator. # # ary = URI.decode_www_form("a=1&a=2&b=3") # p ary #=> [['a', '1'], ['a', '2'], ['b', '3']] @@ -969,17 +965,263 @@ module URI https://github.com/ruby/ruby/blob/trunk/lib/uri/common.rb#L965 # p Hash[ary] # => {"a"=>"2", "b"=>"3"} # # See URI.decode_www_form_component, URI.encode_www_form - def self.decode_www_form(str, enc=Encoding::UTF_8) - return [] if str.empty? - unless /\A#{WFKV_}=#{WFKV_}(?:[;&]#{WFKV_}=#{WFKV_})*\z/o =~ str - raise ArgumentError, "invalid data of application/x-www-form-urlencoded (#{str})" - end + def self.decode_www_form(str, enc=Encoding::UTF_8, separator: '&', use__charset_: false, isindex: false) + raise ArgumentError, "the input of #{self.name}.#{__method__} must be ASCII only string" unless str.ascii_only? ary = [] - $&.scan(/([^=;&]+)=([^;&]*)/) do - ary << [decode_www_form_component($1, enc), decode_www_form_component($2, enc)] + return ary if str.empty? + enc = Encoding.find(enc) + str.b.each_line(separator) do |string| + string.chomp!(separator) + key, sep, val = string.partition('=') + if isindex + if sep.empty? + val = key + key = '' + end + isindex = false + end + + if use__charset_ + if key == '_charset_' + if e = get_encoding(val) + enc = e + use__charset_ = false + ary.each do |k, v| + v.force_encoding(enc) + k.force_encoding(enc) + end + end + end + end + + key.gsub!(/\+|%\h\h/, TBLDECWWWCOMP_) + if val + val.gsub!(/\+|%\h\h/, TBLDECWWWCOMP_) + else + val = '' + end + + val.force_encoding(enc) + key.force_encoding(enc) + ary << [key, val] end ary end + + private + # curl http://encoding.spec.whatwg.org/encodings.json|rb -rpp -rjson -e'H={};h={"shift_jis"=>"Windows-31J","euc-jp"=>"cp51932","iso-2022-jp"=>"cp50221","x-mac-cyrillic"=>"macCyrillic"};JSON($<.read).map{|x|x["encodings"]}.flatten.each{|x|Encoding.find(n=h.fetch(n=x["name"],n))rescue next;x["labels"].each{|y|H[y]=n}};pp H' + WEB_ENCODINGS_ = { + "unicode-1-1-utf-8"=>"utf-8", + "utf-8"=>"utf-8", + "utf8"=>"utf-8", + "866"=>"ibm866", + "cp866"=>"ibm866", + "csibm866"=>"ibm866", + "ibm866"=>"ibm866", + "csisolatin2"=>"iso-8859-2", + "iso-8859-2"=>"iso-8859-2", + "iso-ir-101"=>"iso-8859-2", + "iso8859-2"=>"iso-8859-2", + "iso88592"=>"iso-8859-2", + "iso_8859-2"=>"iso-8859-2", + "iso_8859-2:1987"=>"iso-8859-2", + "l2"=>"iso-8859-2", + "latin2"=>"iso-8859-2", + "csisolatin3"=>"iso-8859-3", + "iso-8859-3"=>"iso-8859-3", + "iso-ir-109"=>"iso-8859-3", + "iso8859-3"=>"iso-8859-3", + "iso88593"=>"iso-8859-3", + "iso_8859-3"=>"iso-8859-3", + "iso_8859-3:1988"=>"iso-8859-3", + "l3"=>"iso-8859-3", + "latin3"=>"iso-8859-3", + "csisolatin4"=>"iso-8859-4", + "iso-8859-4"=>"iso-8859-4", + "iso-ir-110"=>"iso-8859-4", + "iso8859-4"=>"iso-8859-4", + "iso88594"=>"iso-8859-4", + "iso_8859-4"=>"iso-8859-4", + "iso_8859-4:1988"=>"iso-8859-4", + "l4"=>"iso-8859-4", + "latin4"=>"iso-8859-4", + "csisolatincyrillic"=>"iso-8859-5", + "cyrillic"=>"iso-8859-5", + "iso-8859-5"=>"iso-8859-5", + "iso-ir-144"=>"iso-8859-5", + "iso8859-5"=>"iso-8859-5", + "iso88595"=>"iso-8859-5", + "iso_8859-5"=>"iso-8859-5", + "iso_8859-5:1988"=>"iso-8859-5", + "arabic"=>"iso-8859-6", + "asmo-708"=>"iso-8859-6", + "csiso88596e"=>"iso-8859-6", + "csiso88596i"=>"iso-8859-6", + "csisolatinarabic"=>"iso-8859-6", + "ecma-114"=>"iso-8859-6", + "iso-8859-6"=>"iso-8859-6", + "iso-8859-6-e"=>"iso-8859-6", + "iso-8859-6-i"=>"iso-8859-6", + "iso-ir-127"=>"iso-8859-6", + "iso8859-6"=>"iso-8859-6", + "iso88596"=>"iso-8859-6", + "iso_8859-6"=>"iso-8859-6", + "iso_8859-6:1987"=>"iso-8859-6", + "csisolatingreek"=>"iso-8859-7", + "ecma-118"=>"iso-8859-7", + "elot_928"=>"iso-8859-7", + "greek"=>"iso-8859-7", + "greek8"=>"iso-8859-7", + "iso-8859-7"=>"iso-8859-7", + "iso-ir-126"=>"iso-8859-7", + "iso8859-7"=>"iso-8859-7", + "iso88597"=>"iso-8859-7", + "iso_8859-7"=>"iso-8859-7", + "iso_8859-7:1987"=>"iso-8859-7", + "sun_eu_greek"=>"iso-8859-7", + "csiso88598e"=>"iso-8859-8", + "csisolatinhebrew"=>"iso-8859-8", + "hebrew"=>"iso-8859-8", + "iso-8859-8"=>"iso-8859-8", + "iso-8859-8-e"=>"iso-8859-8", + "iso-ir-138"=>"iso-8859-8", + "iso8859-8"=>"iso-8859-8", + "iso88598"=>"iso-8859-8", + "iso_8859-8"=>"iso-8859-8", + "iso_8859-8:1988"=>"iso-8859-8", + "visual"=>"iso-8859-8", + "csisolatin6"=>"iso-8859-10", + "iso-8859-10"=>"iso-8859-10", + "iso-ir-157"=>"iso-8859-10", + "iso8859-10"=>"iso-8859-10", + "iso885910"=>"iso-8859-10", + "l6"=>"iso-8859-10", + "latin6"=>"iso-8859-10", + "iso-8859-13"=>"iso-8859-13", + "iso8859-13"=>"iso-8859-13", + "iso885913"=>"iso-8859-13", + "iso-8859-14"=>"iso-8859-14", + "iso8859-14"=>"iso-8859-14", + "iso885914"=>"iso-8859-14", + "csisolatin9"=>"iso-8859-15", + "iso-8859-15"=>"iso-8859-15", + "iso8859-15"=>"iso-8859-15", + "iso885915"=>"iso-8859-15", + "iso_8859-15"=>"iso-8859-15", + "l9"=>"iso-8859-15", + "iso-8859-16"=>"iso-8859-16", + "cskoi8r"=>"koi8-r", + "koi"=>"koi8-r", + "koi8"=>"koi8-r", + "koi8-r"=>"koi8-r", + "koi8_r"=>"koi8-r", + "koi8-u"=>"koi8-u", + "dos-874"=>"windows-874", + "iso-8859-11"=>"windows-874", + "iso8859-11"=>"windows-874", + "iso885911"=>"windows-874", + "tis-620"=>"windows-874", + "windows-874"=>"windows-874", + "cp1250"=>"windows-1250", + "windows-1250"=>"windows-1250", + "x-cp1250"=>"windows-1250", + "cp1251"=>"windows-1251", + "windows-1251"=>"windows-1251", + "x-cp1251"=>"windows-1251", + "ansi_x3.4-1968"=>"windows-1252", + "ascii"=>"windows-1252", + "cp1252"=>"windows-1252", + "cp819"=>"windows-1252", + "csisolatin1"=>"windows-1252", + "ibm819"=>"windows-1252", + "iso-8859-1"=>"windows-1252", + "iso-ir-100"=>"windows-1252", + "iso8859-1"=>"windows-1252", + "iso88591"=>"windows-1252", + "iso_8859-1"=>"windows-1252", + "iso_8859-1:1987"=>"windows-1252", + "l1"=>"windows-1252", + "latin1"=>"windows-1252", + "us-ascii"=>"windows-1252", + "windows-1252"=>"windows-1252", + "x-cp1252"=>"windows-1252", + "cp1253"=>"windows-1253", + "windows-1253"=>"windows-1253", + "x-cp1253"=>"windows-1253", + "cp1254"=>"windows-1254", + "csisolatin5"=>"windows-1254", + "iso-8859-9"=>"windows-1254", + "iso-ir-148"=>"windows-1254", + "iso8859-9"=>"windows-1254", + "iso88599"=>"windows-1254", + "iso_8859-9"=>"windows-1254", + "iso_8859-9:1989"=>"windows-1254", + "l5"=>"windows-1254", + "latin5"=>"windows-1254", + "windows-1254"=>"windows-1254", + "x-cp1254"=>"windows-1254", + "cp1255"=>"windows-1255", + "windows-1255"=>"windows-1255", + "x-cp1255"=>"windows-1255", + "cp1256"=>"windows-1256", + "windows-1256"=>"windows-1256", + "x-cp1256"=>"windows-1256", + "cp1257"=>"windows-1257", + "windows-1257"=>"windows-1257", + "x-cp1257"=>"windows-1257", + "cp1258"=>"windows-1258", + "windows-1258"=>"windows-1258", + "x-cp1258"=>"windows-1258", + "x-mac-cyrillic"=>"macCyrillic", + "x-mac-ukrainian"=>"macCyrillic", + "chinese"=>"gbk", + "csgb2312"=>"gbk", + "csiso58gb231280"=>"gbk", + "gb2312"=>"gbk", + "gb_2312"=>"gbk", + "gb_2312-80"=>"gbk", + "gbk"=>"gbk", + "iso-ir-58"=>"gbk", + "x-gbk"=>"gbk", + "gb18030"=>"gb18030", + "big5"=>"big5", + "big5-hkscs"=>"big5", + "cn-big5"=>"big5", + "csbig5"=>"big5", + "x-x-big5"=>"big5", + "cseucpkdfmtjapanese"=>"cp51932", + "euc-jp"=>"cp51932", + "x-euc-jp"=>"cp51932", + "csiso2022jp"=>"cp50221", + "iso-2022-jp"=>"cp50221", + "csshiftjis"=>"Windows-31J", + "ms_kanji"=>"Windows-31J", + "shift-jis"=>"Windows-31J", + "shift_jis"=>"Windows-31J", + "sjis"=>"Windows-31J", + "windows-31j"=>"Windows-31J", + "x-sjis"=>"Windows-31J", + "cseuckr"=>"euc-kr", + "csksc56011987"=>"euc-kr", + "euc-kr"=>"euc-kr", + "iso-ir-149"=>"euc-kr", + "korean"=>"euc-kr", + "ks_c_5601-1987"=>"euc-kr", + "ks_c_5601-1989"=>"euc-kr", + "ksc5601"=>"euc-kr", + "ksc_5601"=>"euc-kr", + "windows-949"=>"euc-kr", + "utf-16be"=>"utf-16be", + "utf-16"=>"utf-16le", + "utf-16le"=>"utf-16le" + } # :nodoc: + + # :nodoc: + # return encoding or nil + # http://encoding.spec.whatwg.org/#concept-encoding-get + def self.get_encoding(label) + Encoding.find(WEB_ENCODINGS_[label.to_str.strip.downcase]) rescue nil + end end # module URI module Kernel Index: NEWS =================================================================== --- NEWS (revision 40459) +++ NEWS (revision 40460) @@ -75,4 +75,14 @@ with all sufficient information, see the https://github.com/ruby/ruby/blob/trunk/NEWS#L75 * Tempfile.create === Stdlib compatibility issues (excluding feature bug fixes) + +* URI + * incompatible changes: + * URI.decode_www_form follows current WHATWG URL Standard. + It gets encoding argument to specify the character encoding. + It now allows loose percent encoded strings, but denies ;-separator. + * URI.encode_www_form follows current WHATWG URL Standard. + It gets encoding argument to convert before percent encode. + UTF-16 strings aren't converted to UTF-8 before percent encode by default. + === C API updates Index: test/uri/test_common.rb =================================================================== --- test/uri/test_common.rb (revision 40459) +++ test/uri/test_common.rb (revision 40460) @@ -56,10 +56,32 @@ class TestCommon < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/uri/test_common.rb#L56 URI.encode_www_form_component("\x00 !\"\#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~")) assert_equal("%95A", URI.encode_www_form_component( "\x95\x41".force_encoding(Encoding::Shift_JIS))) - assert_equal("%E3%81%82", URI.encode_www_form_component( + assert_equal("0B", URI.encode_www_form_component( "\x30\x42".force_encoding(Encoding::UTF_16BE))) assert_equal("%1B%24B%24%22%1B%28B", URI.encode_www_form_component( "\e$B$\"\e(B".force_encoding(Encoding::ISO_2022_JP))) + + assert_equal("%E3%81%82", URI.encode_www_form_component( + "\u3042", Encoding::ASCII_8BIT)) + assert_equal("%82%A0", URI.encode_www_form_component( + "\u3042", Encoding::Windows_31J)) + assert_equal("%E3%81%82", URI.encode_www_form_component( + "\u3042", Encoding::UTF_8)) + + assert_equal("%82%A0", URI.encode_www_form_component( + "\u3042".encode("sjis"), Encoding::ASCII_8BIT)) + assert_equal("%A4%A2", URI.encode_www_form_component( + "\u3042".encode("sjis"), Encoding::EUC_JP)) + assert_equal("%E3%81%82", URI.encode_www_form_component( + "\u3042".encode("sjis"), Encoding::UTF_8)) + assert_equal("B0", URI.encode_www_form_component( + "\u3042".encode("sjis"), Encoding::UTF_16LE)) + + # invalid + assert_equal("%EF%BF%BD%EF%BF%BD", URI.encode_www_form_component( + "\xE3\x81\xFF", "utf-8")) + assert_equal("%E6%9F%8A%EF%BF%BD%EF%BF%BD", URI.encode_www_form_component( + "\x95\x41\xff\xff".force_encoding(Encoding::Shift_JIS), "utf-8")) end def test_decode_www_form_component @@ -82,6 +104,8 @@ class TestCommon < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/uri/test_common.rb#L104 assert_equal(expected, URI.encode_www_form(a: 1, :"\u3042" => "\u6F22")) assert_equal(expected, URI.encode_www_form([["a", "1"], ["\u3042", "\u6F22"]])) assert_equal(expected, URI.encode_www_form([[:a, 1], [:"\u3042", "\u6F22"]])) + assert_equal("a=1&%82%A0=%8A%BF", + URI.encode_www_form({"a" => "1", "\u3042" => "\u6F22"}, "sjis")) assert_equal('+a+=+1+', URI.encode_www_form([[' a ', ' 1 ']])) assert_equal('text=x%0Ay', URI.encode_www_form([['text', "x\u000Ay"]])) @@ -106,18 +130,32 @@ class TestCommon < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/uri/test_common.rb#L130 def test_decode_www_form assert_equal([%w[a 1], %w[a 2]], URI.decode_www_form("a=1&a=2")) + assert_equal([%w[a 1;a=2]], URI.decode_www_form("a=1;a=2")) + assert_equal([%w[a 1], ['', ''], %w[a 2]], URI.decode_www_form("a=1&&a=2")) + assert_raise(ArgumentError){URI.decode_www_form("\u3042")} assert_equal([%w[a 1], ["\u3042", "\u6F22"]], - URI.decode_www_form("a=1;%E3%81%82=%E6%BC%A2")) + URI.decode_www_form("a=1&%E3%81%82=%E6%BC%A2")) assert_equal([%w[?a 1], %w[a 2]], URI.decode_www_form("?a=1&a=2")) assert_equal([], URI.decode_www_form("")) - assert_raise(ArgumentError){URI.decode_www_form("%=1")} - assert_raise(ArgumentError){URI.decode_www_form("a=%")} - assert_raise(ArgumentError){URI.decode_www_form("a=1&%=2")} - assert_raise(ArgumentError){URI.decode_www_form("a=1&b=%")} - assert_raise(ArgumentError){URI.decode_www_form("a&b")} + assert_equal([%w[% 1]], URI.decode_www_form("%=1")) + assert_equal([%w[a %]], URI.decode_www_form("a=%")) + assert_equal([%w[a 1], %w[% 2]], URI.decode_www_form("a=1&%=2")) + assert_equal([%w[a 1], %w[b %]], URI.decode_www_form("a=1&b=%")) + assert_equal([['a', ''], ['b', '']], URI.decode_www_form("a&b")) bug4098 = '[ruby-core:33464]' - assert_raise(ArgumentError, bug4098){URI.decode_www_form("a=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&b")} + assert_equal([['a', 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'], ['b', '']], URI.decode_www_form("a=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA&b"), bug4098) + + assert_raise(ArgumentError){ URI.decode_www_form("a=1&%82%A0=%8A%BF", "x-sjis") } + assert_equal([["a", "1"], [s("\x82\xA0"), s("\x8a\xBF")]], + URI.decode_www_form("a=1&%82%A0=%8A%BF", "sjis")) + assert_equal([["a", "1"], [s("\x82\xA0"), s("\x8a\xBF")], %w[_charset_ sjis], [s("\x82\xA1"), s("\x8a\xC0")]], + URI.decode_www_form("a=1&%82%A0=%8A%BF&_charset_=sjis&%82%A1=%8A%C0", use__charset_: true)) + assert_equal([["", "isindex"], ["a", "1"]], + URI.decode_www_form("isindex&a=1", isindex: true)) end + + private + def s(str) str.force_encoding(Encoding::Windows_31J); end end -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/