ruby-changes:50867
From: ko1 <ko1@a...>
Date: Tue, 3 Apr 2018 19:22:49 +0900 (JST)
Subject: [ruby-changes:50867] ko1:r63074 (trunk): repatch r62966 and r62969.
ko1 2018-04-03 19:22:45 +0900 (Tue, 03 Apr 2018) New Revision: 63074 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=63074 Log: repatch r62966 and r62969. Modified files: trunk/lib/webrick/httpproxy.rb trunk/lib/webrick/httprequest.rb trunk/test/webrick/test_httpauth.rb trunk/test/webrick/test_httpproxy.rb trunk/test/webrick/test_httprequest.rb Index: lib/webrick/httpproxy.rb =================================================================== --- lib/webrick/httpproxy.rb (revision 63073) +++ lib/webrick/httpproxy.rb (revision 63074) @@ -211,21 +211,15 @@ module WEBrick https://github.com/ruby/ruby/blob/trunk/lib/webrick/httpproxy.rb#L211 end def do_GET(req, res) - perform_proxy_request(req, res) do |http, path, header| - http.get(path, header) - end + perform_proxy_request(req, res, Net::HTTP::Get) end def do_HEAD(req, res) - perform_proxy_request(req, res) do |http, path, header| - http.head(path, header) - end + perform_proxy_request(req, res, Net::HTTP::Head) end def do_POST(req, res) - perform_proxy_request(req, res) do |http, path, header| - http.post(path, req.body || "", header) - end + perform_proxy_request(req, res, Net::HTTP::Post, req.body_reader) end def do_OPTIONS(req, res) @@ -301,38 +295,56 @@ module WEBrick https://github.com/ruby/ruby/blob/trunk/lib/webrick/httpproxy.rb#L295 return FakeProxyURI end - def perform_proxy_request(req, res) + def perform_proxy_request(req, res, req_class, body_stream = nil) uri = req.request_uri path = uri.path.dup path << "?" << uri.query if uri.query header = setup_proxy_header(req, res) upstream = setup_upstream_proxy_authentication(req, res, header) - response = nil + body_tmp = [] http = Net::HTTP.new(uri.host, uri.port, upstream.host, upstream.port) - http.start do - if @config[:ProxyTimeout] - ################################## these issues are - http.open_timeout = 30 # secs # necessary (maybe because - http.read_timeout = 60 # secs # Ruby's bug, but why?) - ################################## + req_fib = Fiber.new do + http.start do + if @config[:ProxyTimeout] + ################################## these issues are + http.open_timeout = 30 # secs # necessary (maybe because + http.read_timeout = 60 # secs # Ruby's bug, but why?) + ################################## + end + if body_stream && req['transfer-encoding'] =~ /\bchunked\b/i + header['Transfer-Encoding'] = 'chunked' + end + http_req = req_class.new(path, header) + http_req.body_stream = body_stream if body_stream + http.request(http_req) do |response| + # Persistent connection requirements are mysterious for me. + # So I will close the connection in every response. + res['proxy-connection'] = "close" + res['connection'] = "close" + + # stream Net::HTTP::HTTPResponse to WEBrick::HTTPResponse + res.status = response.code.to_i + res.chunked = response.chunked? + choose_header(response, res) + set_cookie(response, res) + set_via(res) + response.read_body do |buf| + body_tmp << buf + Fiber.yield # wait for res.body Proc#call + end + end # http.request + end + end + req_fib.resume # read HTTP response headers and first chunk of the body + res.body = ->(socket) do + while buf = body_tmp.shift + socket.write(buf) + buf.clear + req_fib.resume # continue response.read_body end - response = yield(http, path, header) end - - # Persistent connection requirements are mysterious for me. - # So I will close the connection in every response. - res['proxy-connection'] = "close" - res['connection'] = "close" - - # Convert Net::HTTP::HTTPResponse to WEBrick::HTTPResponse - res.status = response.code.to_i - choose_header(response, res) - set_cookie(response, res) - set_via(res) - res.body = response.body end - # :stopdoc: end end Index: lib/webrick/httprequest.rb =================================================================== --- lib/webrick/httprequest.rb (revision 63073) +++ lib/webrick/httprequest.rb (revision 63074) @@ -258,6 +258,32 @@ module WEBrick https://github.com/ruby/ruby/blob/trunk/lib/webrick/httprequest.rb#L258 end ## + # Prepares the HTTPRequest object for use as the + # source for IO.copy_stream + + def body_reader + @body_tmp = [] + @body_rd = Fiber.new do + body do |buf| + @body_tmp << buf + Fiber.yield + end + end + @body_rd.resume # grab the first chunk and yield + self + end + + # for IO.copy_stream. Note: we may return a larger string than +size+ + # here; but IO.copy_stream does not care. + def readpartial(size, buf = ''.b) # :nodoc + res = @body_tmp.shift or raise EOFError, 'end of file reached' + buf.replace(res) + res.clear + @body_rd.resume # get more chunks + buf + end + + ## # Request query as a Hash def query Index: test/webrick/test_httprequest.rb =================================================================== --- test/webrick/test_httprequest.rb (revision 63073) +++ test/webrick/test_httprequest.rb (revision 63074) @@ -237,6 +237,7 @@ GET / https://github.com/ruby/ruby/blob/trunk/test/webrick/test_httprequest.rb#L237 def test_chunked crlf = "\x0d\x0a" + expect = File.read(__FILE__).freeze msg = <<-_end_of_message_ POST /path HTTP/1.1 Host: test.ruby-lang.org:8080 @@ -253,7 +254,14 @@ GET / https://github.com/ruby/ruby/blob/trunk/test/webrick/test_httprequest.rb#L254 msg << "0" << crlf req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) req.parse(StringIO.new(msg)) - assert_equal(File.read(__FILE__), req.body) + assert_equal(expect, req.body) + + # chunked req.body_reader + req = WEBrick::HTTPRequest.new(WEBrick::Config::HTTP) + req.parse(StringIO.new(msg)) + dst = StringIO.new + IO.copy_stream(req.body_reader, dst) + assert_equal(expect, dst.string) end def test_forwarded Index: test/webrick/test_httpauth.rb =================================================================== --- test/webrick/test_httpauth.rb (revision 63073) +++ test/webrick/test_httpauth.rb (revision 63074) @@ -230,7 +230,7 @@ class TestWEBrickHTTPAuth < Test::Unit:: https://github.com/ruby/ruby/blob/trunk/test/webrick/test_httpauth.rb#L230 log.reject! {|line| pat =~ line } } assert_equal([], log) - } + } TestWEBrick.start_httpserver({}, log_tester) {|server, addr, port, log| realm = "wb auth-int realm" path = "/digest_auth_int" Index: test/webrick/test_httpproxy.rb =================================================================== --- test/webrick/test_httpproxy.rb (revision 63073) +++ test/webrick/test_httpproxy.rb (revision 63074) @@ -118,6 +118,101 @@ class TestWEBrickHTTPProxy < Test::Unit: https://github.com/ruby/ruby/blob/trunk/test/webrick/test_httpproxy.rb#L118 } end + def test_big_bodies + require 'digest/md5' + rand_str = File.read(__FILE__) + rand_str.freeze + nr = 1024 ** 2 / rand_str.size # bigger works, too + exp = Digest::MD5.new + nr.times { exp.update(rand_str) } + exp = exp.hexdigest + TestWEBrick.start_httpserver do |o_server, o_addr, o_port, o_log| + o_server.mount_proc('/') do |req, res| + case req.request_method + when 'GET' + res['content-type'] = 'application/octet-stream' + if req.path == '/length' + res['content-length'] = (nr * rand_str.size).to_s + else + res.chunked = true + end + res.body = ->(socket) { nr.times { socket.write(rand_str) } } + when 'POST' + dig = Digest::MD5.new + req.body { |buf| dig.update(buf); buf.clear } + res['content-type'] = 'text/plain' + res['content-length'] = '32' + res.body = dig.hexdigest + end + end + + http = Net::HTTP.new(o_addr, o_port) + IO.pipe do |rd, wr| + headers = { + 'Content-Type' => 'application/octet-stream', + 'Transfer-Encoding' => 'chunked', + } + post = Net::HTTP::Post.new('/', headers) + th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } + post.body_stream = rd + http.request(post) do |res| + assert_equal 'text/plain', res['content-type'] + assert_equal 32, res.content_length + assert_equal exp, res.body + end + assert_nil th.value + end + + TestWEBrick.start_httpproxy do |p_server, p_addr, p_port, p_log| + http = Net::HTTP.new(o_addr, o_port, p_addr, p_port) + http.request_get('/length') do |res| + assert_equal(nr * rand_str.size, res.content_length) + dig = Digest::MD5.new + res.read_body { |buf| dig.update(buf); buf.clear } + assert_equal exp, dig.hexdigest + end + http.request_get('/') do |res| + assert_predicate res, :chunked? + dig = Digest::MD5.new + res.read_body { |buf| dig.update(buf); buf.clear } + assert_equal exp, dig.hexdigest + end + + IO.pipe do |rd, wr| + headers = { + 'Content-Type' => 'application/octet-stream', + 'Content-Length' => (nr * rand_str.size).to_s, + } + post = Net::HTTP::Post.new('/', headers) + th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } + post.body_stream = rd + http.request(post) do |res| + assert_equal 'text/plain', res['content-type'] + assert_equal 32, res.content_length + assert_equal exp, res.body + end + assert_nil th.value + end + + IO.pipe do |rd, wr| + headers = { + 'Content-Type' => 'application/octet-stream', + 'Transfer-Encoding' => 'chunked', + } + post = Net::HTTP::Post.new('/', headers) + th = Thread.new { nr.times { wr.write(rand_str) }; wr.close } + post.body_stream = rd + http.request(post) do |res| + assert_equal 'text/plain', res['content-type'] + assert_equal 32, res.content_length + assert_equal exp, res.body + end + assert_nil th.value + end + end + end + end + def make_certificate(key, cn) subject = OpenSSL::X509::Name.parse("/DC=org/DC=ruby-lang/CN=#{cn}") exts = [ -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/