ruby-changes:47585
From: nobu <ko1@a...>
Date: Thu, 31 Aug 2017 17:21:51 +0900 (JST)
Subject: [ruby-changes:47585] nobu:r59701 (trunk): io.c: shrink read buffer
nobu 2017-08-31 17:21:46 +0900 (Thu, 31 Aug 2017) New Revision: 59701 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=59701 Log: io.c: shrink read buffer * io.c (io_setstrbuf): return true if the buffer is newly created. * io.c (io_set_read_length): shrink the read buffer if it is a new object and is too large. [ruby-core:81370] [Bug #13597] Modified files: trunk/io.c trunk/test/-ext-/string/test_capacity.rb Index: test/-ext-/string/test_capacity.rb =================================================================== --- test/-ext-/string/test_capacity.rb (revision 59700) +++ test/-ext-/string/test_capacity.rb (revision 59701) @@ -29,4 +29,12 @@ class Test_StringCapacity < Test::Unit:: https://github.com/ruby/ruby/blob/trunk/test/-ext-/string/test_capacity.rb#L29 assert_equal("", String.new(capacity: -1000)) assert_equal(capa(String.new(capacity: -10000)), capa(String.new(capacity: -1000))) end + + def test_io_read + s = String.new(capacity: 1000) + open(__FILE__) {|f|f.read(1024*1024, s)} + assert_equal(1024*1024, capa(s)) + open(__FILE__) {|f|s = f.read(1024*1024)} + assert_operator(capa(s), :<=, s.bytesize+4096) + end end Index: io.c =================================================================== --- io.c (revision 59700) +++ io.c (revision 59701) @@ -2140,7 +2140,7 @@ io_bufread(char *ptr, long len, rb_io_t https://github.com/ruby/ruby/blob/trunk/io.c#L2140 return len - n; } -static void io_setstrbuf(VALUE *str, long len); +static int io_setstrbuf(VALUE *str, long len); struct bufread_arg { char *str_ptr; @@ -2359,33 +2359,45 @@ io_shift_cbuf(rb_io_t *fptr, int len, VA https://github.com/ruby/ruby/blob/trunk/io.c#L2359 return str; } -static void +static int io_setstrbuf(VALUE *str, long len) { #ifdef _WIN32 len = (len + 1) & ~1L; /* round up for wide char */ #endif if (NIL_P(*str)) { - *str = rb_str_new(0, 0); + *str = rb_str_new(0, len); + return TRUE; } else { VALUE s = StringValue(*str); long clen = RSTRING_LEN(s); if (clen >= len) { rb_str_modify(s); - return; + return FALSE; } len -= clen; } rb_str_modify_expand(*str, len); + return FALSE; } +#define MAX_REALLOC_GAP 4096 static void -io_set_read_length(VALUE str, long n) +io_shrink_read_string(VALUE str, long n) +{ + if (rb_str_capacity(str) - n > MAX_REALLOC_GAP) { + rb_str_resize(str, n); + } +} + +static void +io_set_read_length(VALUE str, long n, int shrinkable) { if (RSTRING_LEN(str) != n) { rb_str_modify(str); rb_str_set_len(str, n); + if (shrinkable) io_shrink_read_string(str, n); } } @@ -2397,11 +2409,12 @@ read_all(rb_io_t *fptr, long siz, VALUE https://github.com/ruby/ruby/blob/trunk/io.c#L2409 long pos; rb_encoding *enc; int cr; + int shrinkable; if (NEED_READCONV(fptr)) { int first = !NIL_P(str); SET_BINARY_MODE(fptr); - io_setstrbuf(&str,0); + shrinkable = io_setstrbuf(&str,0); make_readconv(fptr, 0); while (1) { VALUE v; @@ -2420,6 +2433,7 @@ read_all(rb_io_t *fptr, long siz, VALUE https://github.com/ruby/ruby/blob/trunk/io.c#L2433 if (v == MORE_CHAR_FINISHED) { clear_readconv(fptr); if (first) rb_str_set_len(str, first = 0); + if (shrinkable) io_shrink_read_string(str, RSTRING_LEN(str)); return io_enc_str(str, fptr); } } @@ -2433,7 +2447,7 @@ read_all(rb_io_t *fptr, long siz, VALUE https://github.com/ruby/ruby/blob/trunk/io.c#L2447 cr = 0; if (siz == 0) siz = BUFSIZ; - io_setstrbuf(&str,siz); + shrinkable = io_setstrbuf(&str, siz); for (;;) { READ_CHECK(fptr); n = io_fread(str, bytes, siz - bytes, fptr); @@ -2449,6 +2463,7 @@ read_all(rb_io_t *fptr, long siz, VALUE https://github.com/ruby/ruby/blob/trunk/io.c#L2463 siz += BUFSIZ; rb_str_modify_expand(str, BUFSIZ); } + if (shrinkable) io_shrink_read_string(str, RSTRING_LEN(str)); str = io_enc_str(str, fptr); ENC_CODERANGE_SET(str, cr); return str; @@ -2511,6 +2526,7 @@ io_getpartial(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/io.c#L2526 VALUE length, str; long n, len; struct read_internal_arg arg; + int shrinkable; rb_scan_args(argc, argv, "11", &length, &str); @@ -2518,7 +2534,7 @@ io_getpartial(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/io.c#L2534 rb_raise(rb_eArgError, "negative length %ld given", len); } - io_setstrbuf(&str,len); + shrinkable = io_setstrbuf(&str, len); OBJ_TAINT(str); GetOpenFile(io, fptr); @@ -2555,7 +2571,7 @@ io_getpartial(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/io.c#L2571 rb_syserr_fail_path(e, fptr->pathv); } } - io_set_read_length(str, n); + io_set_read_length(str, n, shrinkable); if (n == 0) return Qnil; @@ -2651,12 +2667,13 @@ io_read_nonblock(VALUE io, VALUE length, https://github.com/ruby/ruby/blob/trunk/io.c#L2667 rb_io_t *fptr; long n, len; struct read_internal_arg arg; + int shrinkable; if ((len = NUM2LONG(length)) < 0) { rb_raise(rb_eArgError, "negative length %ld given", len); } - io_setstrbuf(&str,len); + shrinkable = io_setstrbuf(&str, len); OBJ_TAINT(str); GetOpenFile(io, fptr); rb_io_check_byte_readable(fptr); @@ -2667,7 +2684,7 @@ io_read_nonblock(VALUE io, VALUE length, https://github.com/ruby/ruby/blob/trunk/io.c#L2684 n = read_buffered_data(RSTRING_PTR(str), len, fptr); if (n <= 0) { rb_io_set_nonblock(fptr); - io_setstrbuf(&str, len); + shrinkable |= io_setstrbuf(&str, len); arg.fd = fptr->fd; arg.str_ptr = RSTRING_PTR(str); arg.len = len; @@ -2683,7 +2700,7 @@ io_read_nonblock(VALUE io, VALUE length, https://github.com/ruby/ruby/blob/trunk/io.c#L2700 rb_syserr_fail_path(e, fptr->pathv); } } - io_set_read_length(str, n); + io_set_read_length(str, n, shrinkable); if (n == 0) { if (ex == Qfalse) return Qnil; @@ -2802,6 +2819,7 @@ io_read(int argc, VALUE *argv, VALUE io) https://github.com/ruby/ruby/blob/trunk/io.c#L2819 rb_io_t *fptr; long n, len; VALUE length, str; + int shrinkable; #if defined(RUBY_TEST_CRLF_ENVIRONMENT) || defined(_WIN32) int previous_mode; #endif @@ -2818,12 +2836,12 @@ io_read(int argc, VALUE *argv, VALUE io) https://github.com/ruby/ruby/blob/trunk/io.c#L2836 rb_raise(rb_eArgError, "negative length %ld given", len); } - io_setstrbuf(&str,len); + shrinkable = io_setstrbuf(&str,len); GetOpenFile(io, fptr); rb_io_check_byte_readable(fptr); if (len == 0) { - io_set_read_length(str, 0); + io_set_read_length(str, 0, shrinkable); return str; } @@ -2832,7 +2850,7 @@ io_read(int argc, VALUE *argv, VALUE io) https://github.com/ruby/ruby/blob/trunk/io.c#L2850 previous_mode = set_binary_mode_with_seek_cur(fptr); #endif n = io_fread(str, 0, len, fptr); - io_set_read_length(str, n); + io_set_read_length(str, n, shrinkable); #if defined(RUBY_TEST_CRLF_ENVIRONMENT) || defined(_WIN32) if (previous_mode == O_TEXT) { setmode(fptr->fd, O_TEXT); @@ -4787,11 +4805,12 @@ rb_io_sysread(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/io.c#L4805 rb_io_t *fptr; long n, ilen; struct read_internal_arg arg; + int shrinkable; rb_scan_args(argc, argv, "11", &len, &str); ilen = NUM2LONG(len); - io_setstrbuf(&str,ilen); + shrinkable = io_setstrbuf(&str, ilen); if (ilen == 0) return str; GetOpenFile(io, fptr); @@ -4823,7 +4842,7 @@ rb_io_sysread(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/io.c#L4842 if (n == -1) { rb_sys_fail_path(fptr->pathv); } - io_set_read_length(str, n); + io_set_read_length(str, n, shrinkable); if (n == 0 && ilen > 0) { rb_eof_error(); } @@ -4884,12 +4903,13 @@ rb_io_pread(int argc, VALUE *argv, VALUE https://github.com/ruby/ruby/blob/trunk/io.c#L4903 rb_io_t *fptr; ssize_t n; struct prdwr_internal_arg arg; + int shrinkable; rb_scan_args(argc, argv, "21", &len, &offset, &str); arg.count = NUM2SIZET(len); arg.offset = NUM2OFFT(offset); - io_setstrbuf(&str, (long)arg.count); + shrinkable = io_setstrbuf(&str, (long)arg.count); if (arg.count == 0) return str; arg.buf = RSTRING_PTR(str); @@ -4905,7 +4925,7 @@ rb_io_pread(int argc, VALUE *argv, VALUE https://github.com/ruby/ruby/blob/trunk/io.c#L4925 if (n == -1) { rb_sys_fail_path(fptr->pathv); } - io_set_read_length(str, n); + io_set_read_length(str, n, shrinkable); if (n == 0 && arg.count > 0) { rb_eof_error(); } -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/