[前][次][番号順一覧][スレッド一覧]

ruby-changes:46413

From: nobu <ko1@a...>
Date: Mon, 1 May 2017 11:01:42 +0900 (JST)
Subject: [ruby-changes:46413] nobu:r58526 (trunk): zlib.c: no buf_filled in zstream

nobu	2017-05-01 11:01:38 +0900 (Mon, 01 May 2017)

  New Revision: 58526

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=58526

  Log:
    zlib.c: no buf_filled in zstream
    
    * ext/zlib/zlib.c (zstream): manage capacity and size of `buf`
      instead of size and separated member `buf_filled`.  reported by
      Christian Jalio (jalio) at https://hackerone.com/reports/211958

  Modified files:
    trunk/ext/zlib/zlib.c
    trunk/test/zlib/test_zlib.rb
Index: ext/zlib/zlib.c
===================================================================
--- ext/zlib/zlib.c	(revision 58525)
+++ ext/zlib/zlib.c	(revision 58526)
@@ -528,7 +528,6 @@ rb_zlib_crc_table(VALUE obj) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L528
 struct zstream {
     unsigned long flags;
     VALUE buf;
-    long buf_filled;
     VALUE input;
     z_stream stream;
     const struct zstream_funcs {
@@ -551,6 +550,7 @@ struct zstream { https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L550
 #define ZSTREAM_IS_FINISHED(z) ((z)->flags & ZSTREAM_FLAG_FINISHED)
 #define ZSTREAM_IS_CLOSING(z)  ((z)->flags & ZSTREAM_FLAG_CLOSING)
 #define ZSTREAM_IS_GZFILE(z)   ((z)->flags & ZSTREAM_FLAG_GZFILE)
+#define ZSTREAM_BUF_FILLED(z)  (NIL_P((z)->buf) ? 0 : RSTRING_LEN((z)->buf))
 
 #define ZSTREAM_EXPAND_BUFFER_OK          0
 
@@ -600,7 +600,6 @@ zstream_init(struct zstream *z, const st https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L600
 {
     z->flags = 0;
     z->buf = Qnil;
-    z->buf_filled = 0;
     z->input = Qnil;
     z->stream.zalloc = zlib_mem_alloc;
     z->stream.zfree = zlib_mem_free;
@@ -625,11 +624,11 @@ zstream_expand_buffer(struct zstream *z) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L624
     }
 
     if (!ZSTREAM_IS_GZFILE(z) && rb_block_given_p()) {
-	if (z->buf_filled >= ZSTREAM_AVAIL_OUT_STEP_MAX) {
+	long buf_filled = ZSTREAM_BUF_FILLED(z);
+	if (buf_filled >= ZSTREAM_AVAIL_OUT_STEP_MAX) {
 	    int state = 0;
 	    VALUE self = (VALUE)z->stream.opaque;
 
-	    rb_str_resize(z->buf, z->buf_filled);
 	    rb_obj_reveal(z->buf, rb_cString);
 	    OBJ_INFECT(z->buf, self);
 
@@ -645,7 +644,7 @@ zstream_expand_buffer(struct zstream *z) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L644
 	}
 	else {
 	    zstream_expand_buffer_into(z,
-		    ZSTREAM_AVAIL_OUT_STEP_MAX - z->buf_filled);
+		    ZSTREAM_AVAIL_OUT_STEP_MAX - buf_filled);
 	}
     }
     else {
@@ -659,15 +658,14 @@ zstream_expand_buffer_into(struct zstrea https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L658
     if (NIL_P(z->buf)) {
 	/* I uses rb_str_new here not rb_str_buf_new because
 	   rb_str_buf_new makes a zero-length string. */
-	z->buf = rb_str_new(0, size);
-	z->buf_filled = 0;
+	z->buf = rb_str_buf_new(size);
 	z->stream.next_out = (Bytef*)RSTRING_PTR(z->buf);
 	z->stream.avail_out = MAX_UINT(size);
 	rb_obj_hide(z->buf);
     }
     else if (z->stream.avail_out != size) {
-	rb_str_resize(z->buf, z->buf_filled + size);
-	z->stream.next_out = (Bytef*)RSTRING_PTR(z->buf) + z->buf_filled;
+	rb_str_modify_expand(z->buf, size);
+	z->stream.next_out = (Bytef*)RSTRING_END(z->buf);
 	z->stream.avail_out = MAX_UINT(size);
     }
 }
@@ -686,32 +684,22 @@ zstream_expand_buffer_protect(void *ptr) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L684
 static int
 zstream_expand_buffer_non_stream(struct zstream *z)
 {
-    char * new_str;
-    long inc, len;
+    long inc, len = ZSTREAM_BUF_FILLED(z);
 
-    if (RSTRING_LEN(z->buf) - z->buf_filled >= ZSTREAM_AVAIL_OUT_STEP_MAX) {
+    if (rb_str_capacity(z->buf) - len >= ZSTREAM_AVAIL_OUT_STEP_MAX) {
 	z->stream.avail_out = ZSTREAM_AVAIL_OUT_STEP_MAX;
     }
     else {
-	inc = z->buf_filled / 2;
+	inc = len / 2;
 	if (inc < ZSTREAM_AVAIL_OUT_STEP_MIN) {
 	    inc = ZSTREAM_AVAIL_OUT_STEP_MIN;
 	}
 
-	len = z->buf_filled + inc;
-
-	new_str = ruby_xrealloc(RSTRING(z->buf)->as.heap.ptr, len + 1);
-
-	/* from rb_str_resize */
-	RSTRING(z->buf)->as.heap.ptr = new_str;
-	RSTRING(z->buf)->as.heap.ptr[len] = '\0'; /* sentinel */
-	RSTRING(z->buf)->as.heap.len =
-	    RSTRING(z->buf)->as.heap.aux.capa = len;
-
+	rb_str_modify_expand(z->buf, inc);
 	z->stream.avail_out = (inc < ZSTREAM_AVAIL_OUT_STEP_MAX) ?
 	    (int)inc : ZSTREAM_AVAIL_OUT_STEP_MAX;
     }
-    z->stream.next_out = (Bytef*)RSTRING_PTR(z->buf) + z->buf_filled;
+    z->stream.next_out = (Bytef*)RSTRING_END(z->buf);
 
     return ZSTREAM_EXPAND_BUFFER_OK;
 }
@@ -722,15 +710,14 @@ zstream_append_buffer(struct zstream *z, https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L710
     if (NIL_P(z->buf)) {
 	z->buf = rb_str_buf_new(len);
 	rb_str_buf_cat(z->buf, (const char*)src, len);
-	z->buf_filled = len;
 	z->stream.next_out = (Bytef*)RSTRING_PTR(z->buf);
 	z->stream.avail_out = 0;
 	rb_obj_hide(z->buf);
 	return;
     }
 
-    if (RSTRING_LEN(z->buf) < z->buf_filled + len) {
-	rb_str_resize(z->buf, z->buf_filled + len);
+    if ((long)rb_str_capacity(z->buf) < ZSTREAM_BUF_FILLED(z) + len) {
+	rb_str_modify_expand(z->buf, len);
 	z->stream.avail_out = 0;
     }
     else {
@@ -741,9 +728,8 @@ zstream_append_buffer(struct zstream *z, https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L728
 	    z->stream.avail_out = 0;
 	}
     }
-    memcpy(RSTRING_PTR(z->buf) + z->buf_filled, src, len);
-    z->buf_filled += len;
-    z->stream.next_out = (Bytef*)RSTRING_PTR(z->buf) + z->buf_filled;
+    rb_str_cat(z->buf, (const char *)src, len);
+    z->stream.next_out = (Bytef*)RSTRING_END(z->buf);
 }
 
 #define zstream_append_buffer2(z,v) \
@@ -766,14 +752,12 @@ zstream_detach_buffer(struct zstream *z) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L752
     }
     else {
 	dst = z->buf;
-	rb_str_resize(dst, z->buf_filled);
 	rb_obj_reveal(dst, rb_cString);
     }
 
     OBJ_INFECT(dst, self);
 
     z->buf = Qnil;
-    z->buf_filled = 0;
     z->stream.next_out = 0;
     z->stream.avail_out = 0;
 
@@ -789,18 +773,20 @@ static VALUE https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L773
 zstream_shift_buffer(struct zstream *z, long len)
 {
     VALUE dst;
-    long buflen;
+    char *bufptr;
+    long buflen = ZSTREAM_BUF_FILLED(z);
 
-    if (z->buf_filled <= len) {
+    if (buflen <= len) {
 	return zstream_detach_buffer(z);
     }
 
-    dst = rb_str_new(RSTRING_PTR(z->buf), len);
-    z->buf_filled -= len;
-    memmove(RSTRING_PTR(z->buf), RSTRING_PTR(z->buf) + len,
-	    z->buf_filled);
-    z->stream.next_out = (Bytef*)RSTRING_PTR(z->buf) + z->buf_filled;
-    buflen = RSTRING_LEN(z->buf) - z->buf_filled;
+    bufptr = RSTRING_PTR(z->buf);
+    dst = rb_str_new(bufptr, len);
+    buflen -= len;
+    memmove(bufptr, bufptr + len, buflen);
+    rb_str_set_len(z->buf, buflen);
+    z->stream.next_out = (Bytef*)RSTRING_END(z->buf);
+    buflen = (long)rb_str_capacity(z->buf) - ZSTREAM_BUF_FILLED(z);
     if (buflen > ZSTREAM_AVAIL_OUT_STEP_MAX) {
 	buflen = ZSTREAM_AVAIL_OUT_STEP_MAX;
     }
@@ -812,13 +798,17 @@ zstream_shift_buffer(struct zstream *z, https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L798
 static void
 zstream_buffer_ungets(struct zstream *z, const Bytef *b, unsigned long len)
 {
-    if (NIL_P(z->buf) || RSTRING_LEN(z->buf) - z->buf_filled == 0) {
+    char *bufptr;
+    long filled;
+
+    if (NIL_P(z->buf) || (long)rb_str_capacity(z->buf) <= ZSTREAM_BUF_FILLED(z)) {
 	zstream_expand_buffer_into(z, len);
     }
 
-    memmove(RSTRING_PTR(z->buf) + len, RSTRING_PTR(z->buf), z->buf_filled);
-    memmove(RSTRING_PTR(z->buf), b, len);
-    z->buf_filled+=len;
+    RSTRING_GETMEM(z->buf, bufptr, filled);
+    memmove(bufptr + len, bufptr, filled);
+    memmove(bufptr, b, len);
+    rb_str_set_len(z->buf, filled + len);
     if (z->stream.avail_out > 0) {
 	if (len > z->stream.avail_out) len = z->stream.avail_out;
 	z->stream.next_out+=len;
@@ -907,7 +897,6 @@ zstream_reset(struct zstream *z) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L897
     }
     z->flags = ZSTREAM_FLAG_READY;
     z->buf = Qnil;
-    z->buf_filled = 0;
     z->stream.next_out = 0;
     z->stream.avail_out = 0;
     zstream_reset_input(z);
@@ -948,7 +937,7 @@ zstream_run_func(void *ptr) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L937
     while (!args->interrupt) {
 	n = z->stream.avail_out;
 	err = z->func->run(&z->stream, flush);
-	z->buf_filled += n - z->stream.avail_out;
+	rb_str_set_len(z->buf, ZSTREAM_BUF_FILLED(z) + (n - z->stream.avail_out));
 
 	if (err == Z_STREAM_END) {
 	    z->flags &= ~ZSTREAM_FLAG_IN_STREAM;
@@ -1557,7 +1546,6 @@ rb_deflate_init_copy(VALUE self, VALUE o https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L1546
     }
     z1->input = NIL_P(z2->input) ? Qnil : rb_str_dup(z2->input);
     z1->buf   = NIL_P(z2->buf)   ? Qnil : rb_str_dup(z2->buf);
-    z1->buf_filled = z2->buf_filled;
     z1->flags = z2->flags;
 
     return self;
@@ -1741,23 +1729,26 @@ rb_deflate_params(VALUE obj, VALUE v_lev https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L1729
     int level, strategy;
     int err;
     uInt n;
+    long filled;
 
     level = ARG_LEVEL(v_level);
     strategy = ARG_STRATEGY(v_strategy);
 
     n = z->stream.avail_out;
     err = deflateParams(&z->stream, level, strategy);
-    z->buf_filled += n - z->stream.avail_out;
+    filled = n - z->stream.avail_out;
     while (err == Z_BUF_ERROR) {
 	rb_warning("deflateParams() returned Z_BUF_ERROR");
 	zstream_expand_buffer(z);
+	rb_str_set_len(z->buf, RSTRING_LEN(z->buf) + filled);
 	n = z->stream.avail_out;
 	err = deflateParams(&z->stream, level, strategy);
-	z->buf_filled += n - z->stream.avail_out;
+	filled = n - z->stream.avail_out;
     }
     if (err != Z_OK) {
 	raise_zlib_error(err, z->stream.msg);
     }
+    rb_str_set_len(z->buf, RSTRING_LEN(z->buf) + filled);
 
     return Qnil;
 }
@@ -2210,7 +2201,7 @@ struct gzfile { https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L2201
 #define GZFILE_FLAG_FOOTER_FINISHED  (ZSTREAM_FLAG_UNUSED << 2)
 
 #define GZFILE_IS_FINISHED(gz) \
-    (ZSTREAM_IS_FINISHED(&(gz)->z) && (gz)->z.buf_filled == 0)
+    (ZSTREAM_IS_FINISHED(&(gz)->z) && ZSTREAM_BUF_FILLED(&(gz)->z) == 0)
 
 #define GZFILE_READ_SIZE  2048
 
@@ -2336,7 +2327,7 @@ gzfile_write_raw(struct gzfile *gz) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L2327
 {
     VALUE str;
 
-    if (gz->z.buf_filled > 0) {
+    if (ZSTREAM_BUF_FILLED(&gz->z) > 0) {
 	str = zstream_detach_buffer(&gz->z);
 	OBJ_TAINT(str);  /* for safe */
 	rb_funcall(gz->io, id_write, 1, str);
@@ -2669,9 +2660,9 @@ gzfile_read_more(struct gzfile *gz) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L2660
 			Z_SYNC_FLUSH);
 	    RB_GC_GUARD(str);
 	}
-	if (gz->z.buf_filled > 0) break;
+	if (ZSTREAM_BUF_FILLED(&gz->z) > 0) break;
     }
-    return gz->z.buf_filled;
+    return ZSTREAM_BUF_FILLED(&gz->z);
 }
 
 static void
@@ -2712,7 +2703,7 @@ gzfile_fill(struct gzfile *gz, long len) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L2703
         rb_raise(rb_eArgError, "negative length %ld given", len);
     if (len == 0)
 	return 0;
-    while (!ZSTREAM_IS_FINISHED(&gz->z) && gz->z.buf_filled < len) {
+    while (!ZSTREAM_IS_FINISHED(&gz->z) && ZSTREAM_BUF_FILLED(&gz->z) < len) {
 	gzfile_read_more(gz);
     }
     if (GZFILE_IS_FINISHED(gz)) {
@@ -2721,7 +2712,7 @@ gzfile_fill(struct gzfile *gz, long len) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L2712
 	}
 	return -1;
     }
-    return len < gz->z.buf_filled ? len : gz->z.buf_filled;
+    return len < ZSTREAM_BUF_FILLED(&gz->z) ? len : ZSTREAM_BUF_FILLED(&gz->z);
 }
 
 static VALUE
@@ -2756,7 +2747,7 @@ gzfile_readpartial(struct gzfile *gz, lo https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L2747
             return outbuf;
         }
     }
-    while (!ZSTREAM_IS_FINISHED(&gz->z) && gz->z.buf_filled == 0) {
+    while (!ZSTREAM_IS_FINISHED(&gz->z) && ZSTREAM_BUF_FILLED(&gz->z) == 0) {
 	gzfile_read_more(gz);
     }
     if (GZFILE_IS_FINISHED(gz)) {
@@ -2810,7 +2801,7 @@ gzfile_getc(struct gzfile *gz) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L2801
     int len;
 
     len = rb_enc_mbmaxlen(gz->enc);
-    while (!ZSTREAM_IS_FINISHED(&gz->z) && gz->z.buf_filled < len) {
+    while (!ZSTREAM_IS_FINISHED(&gz->z) && ZSTREAM_BUF_FILLED(&gz->z) < len) {
 	gzfile_read_more(gz);
     }
     if (GZFILE_IS_FINISHED(gz)) {
@@ -2828,7 +2819,7 @@ gzfile_getc(struct gzfile *gz) https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L2819
 	    gz->cbuf = ALLOC_N(char, GZFILE_CBUF_CAPA);
 	}
         ss = sp = (const unsigned char*)RSTRING_PTR(gz->z.buf);
-        se = sp + gz->z.buf_filled;
+        se = sp + ZSTREAM_BUF_FILLED(&gz->z);
         ds = dp = (unsigned char *)gz->cbuf;
         de = (unsigned char *)ds + GZFILE_CBUF_CAPA;
         (void)rb_econv_convert(gz->ec, &sp, se, &dp, de, ECONV_PARTIAL_INPUT|ECONV_AFTER_OUTPUT);
@@ -3402,7 +3393,7 @@ static VALUE https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L3393
 rb_gzfile_total_out(VALUE obj)
 {
     struct gzfile *gz = get_gzfile(obj);
-    return rb_uint2inum(gz->z.stream.total_out - gz->z.buf_filled);
+    return rb_uint2inum(gz->z.stream.total_out - ZSTREAM_BUF_FILLED(&gz->z));
 }
 
 /*
@@ -3976,7 +3967,7 @@ gzreader_skip_linebreaks(struct gzfile * https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L3967
     char *p;
     int n;
 
-    while (gz->z.buf_filled == 0) {
+    while (ZSTREAM_BUF_FILLED(&gz->z) == 0) {
 	if (GZFILE_IS_FINISHED(gz)) return;
 	gzfile_read_more(gz);
     }
@@ -3984,10 +3975,10 @@ gzreader_skip_linebreaks(struct gzfile * https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L3975
     p = RSTRING_PTR(gz->z.buf);
 
     while (n++, *(p++) == '\n') {
-	if (n >= gz->z.buf_filled) {
+	if (n >= ZSTREAM_BUF_FILLED(&gz->z)) {
 	    str = zstream_detach_buffer(&gz->z);
 	    gzfile_calc_crc(gz, str);
-	    while (gz->z.buf_filled == 0) {
+	    while (ZSTREAM_BUF_FILLED(&gz->z) == 0) {
 		if (GZFILE_IS_FINISHED(gz)) return;
 		gzfile_read_more(gz);
 	    }
@@ -4011,7 +4002,7 @@ static long https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L4002
 gzreader_charboundary(struct gzfile *gz, long n)
 {
     char *s = RSTRING_PTR(gz->z.buf);
-    char *e = s + gz->z.buf_filled;
+    char *e = s + ZSTREAM_BUF_FILLED(&gz->z);
     char *p = rb_enc_left_char_head(s, s + n, e, gz->enc);
     long l = p - s;
     if (l < n) {
@@ -4106,9 +4097,9 @@ gzreader_gets(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L4097
 	gzreader_skip_linebreaks(gz);
     }
 
-    while (gz->z.buf_filled < rslen) {
+    while (ZSTREAM_BUF_FILLED(&gz->z) < rslen) {
 	if (ZSTREAM_IS_FINISHED(&gz->z)) {
-	    if (gz->z.buf_filled > 0) gz->lineno++;
+	    if (ZSTREAM_BUF_FILLED(&gz->z) > 0) gz->lineno++;
 	    return gzfile_read(gz, rslen);
 	}
 	gzfile_read_more(gz);
@@ -4118,13 +4109,13 @@ gzreader_gets(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L4109
     n = rslen;
     for (;;) {
 	long filled;
-	if (n > gz->z.buf_filled) {
+	if (n > ZSTREAM_BUF_FILLED(&gz->z)) {
 	    if (ZSTREAM_IS_FINISHED(&gz->z)) break;
 	    gzfile_read_more(gz);
 	    p = RSTRING_PTR(gz->z.buf) + n - rslen;
 	}
 	if (!rspara) rscheck(rsptr, rslen, rs);
-	filled = gz->z.buf_filled;
+	filled = ZSTREAM_BUF_FILLED(&gz->z);
 	if (limit > 0 && filled >= limit) {
 	    filled = limit;
 	}
@@ -4141,7 +4132,7 @@ gzreader_gets(int argc, VALUE *argv, VAL https://github.com/ruby/ruby/blob/trunk/ext/zlib/zlib.c#L4132
 	    p++, n++;
 	}
     }
-    if (maxlen > 1 && n == limit && (gz->z.buf_filled > n || !ZSTREAM_IS_FINISHED(&gz->z))) {
+    if (maxlen > 1 && n == limit && (ZSTREAM_BUF_FILLED(&gz->z) > n || !ZSTREAM_IS_FINISHED(&gz->z))) {
 	n = gzreader_charboundary(gz, n);
     }
 
Index: test/zlib/test_zlib.rb
===================================================================
--- test/zlib/test_zlib.rb	(revision 58525)
+++ test/zlib/test_zlib.rb	(revision 58526)
@@ -127,6 +127,16 @@ if defined? Zlib https://github.com/ruby/ruby/blob/trunk/test/zlib/test_zlib.rb#L127
       assert_equal("foobar", Zlib::Inflate.inflate(s))
     end
 
+    def test_expand_buffer;
+      z = Zlib::Deflate.new
+      src = "baz" * 1000
+      z.avail_out = 1
+      GC.stress = true
+      s = z.deflate(src, Zlib::FINISH)
+      GC.stress = false
+      assert_equal(src, Zlib::Inflate.inflate(s))
+    end
+
     def test_total
       z = Zlib::Deflate.new
       1000.times { z << "foo" }

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/

[前][次][番号順一覧][スレッド一覧]