nobu	2014-12-23 11:42:16 +0900 (Tue, 23 Dec 2014)

  New Revision: 48923

  http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=48923

  Log:
    ossl_cipher.c: workaround of OpenSSL API
    
    * ext/openssl/ossl_cipher.c (ossl_cipher_update_long): update huge
      data gradually not to exceed INT_MAX.  workaround of OpenSSL API
      limitation.  [ruby-core:67043] [Bug #10633]

  Modified files:
    trunk/ChangeLog
    trunk/ext/openssl/ossl_cipher.c
Index: ChangeLog
===================================================================
--- ChangeLog	(revision 48922)
+++ ChangeLog	(revision 48923)
@@ -1,3 +1,9 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1
+Tue Dec 23 11:42:14 2014  Nobuyoshi Nakada  <nobu@r...>
+
+	* ext/openssl/ossl_cipher.c (ossl_cipher_update_long): update huge
+	  data gradually not to exceed INT_MAX.  workaround of OpenSSL API
+	  limitation.  [ruby-core:67043] [Bug #10633]
+
 Mon Dec 22 21:30:16 2014  Masaki Suketa <masaki.suketa@n...>
 
 	* test/win32ole/test_win32ole_event.rb: some tests are
Index: ext/openssl/ossl_cipher.c
===================================================================
--- ext/openssl/ossl_cipher.c	(revision 48922)
+++ ext/openssl/ossl_cipher.c	(revision 48923)
@@ -346,6 +346,33 @@ ossl_cipher_pkcs5_keyivgen(int argc, VAL https://github.com/ruby/ruby/blob/trunk/ext/openssl/ossl_cipher.c#L346
     return Qnil;
 }
 
+static int
+ossl_cipher_update_long(EVP_CIPHER_CTX *ctx, unsigned char *out, long *out_len_ptr,
+			const unsigned char *in, long in_len)
+{
+    int out_part_len;
+    long out_len = 0;
+#define UPDATE_LENGTH_LIMIT INT_MAX
+
+#if SIZEOF_LONG > UPDATE_LENGTH_LIMIT
+    if (in_len > UPDATE_LENGTH_LIMIT) {
+	const int in_part_len = (UPDATE_LENGTH_LIMIT / 2 + 1) & ~1;
+	do {
+	    if (!EVP_CipherUpdate(ctx, out ? (out + out_len) : 0,
+				  &out_part_len, in, in_part_len))
+		return 0;
+	    out_len += out_part_len;
+	    in += in_part_len;
+	} while ((in_len -= in_part_len) > UPDATE_LENGTH_LIMIT);
+    }
+#endif
+    if (!EVP_CipherUpdate(ctx, out ? (out + out_len) : 0,
+			  &out_part_len, in, (int)in_len))
+	return 0;
+    if (out_len_ptr) *out_len_ptr = out_len += out_part_len;
+    return 1;
+}
+
 /*
  *  call-seq:
  *     cipher.update(data [, buffer]) -> string or buffer
@@ -364,17 +391,21 @@ ossl_cipher_update(int argc, VALUE *argv https://github.com/ruby/ruby/blob/trunk/ext/openssl/ossl_cipher.c#L391
 {
     EVP_CIPHER_CTX *ctx;
     unsigned char *in;
-    int in_len, out_len;
+    long in_len, out_len;
     VALUE data, str;
 
     rb_scan_args(argc, argv, "11", &data, &str);
 
     StringValue(data);
     in = (unsigned char *)RSTRING_PTR(data);
-    if ((in_len = RSTRING_LENINT(data)) == 0)
+    if ((in_len = RSTRING_LEN(data)) == 0)
         ossl_raise(rb_eArgError, "data must not be empty");
     GetCipher(self, ctx);
     out_len = in_len+EVP_CIPHER_CTX_block_size(ctx);
+    if (out_len <= 0) {
+	ossl_raise(rb_eRangeError,
+		   "data too big to make output buffer: %ld bytes", in_len);
+    }
 
     if (NIL_P(str)) {
         str = rb_str_new(0, out_len);
@@ -383,7 +414,7 @@ ossl_cipher_update(int argc, VALUE *argv https://github.com/ruby/ruby/blob/trunk/ext/openssl/ossl_cipher.c#L414
         rb_str_resize(str, out_len);
     }
 
-    if (!EVP_CipherUpdate(ctx, (unsigned char *)RSTRING_PTR(str), &out_len, in, in_len))
+    if (!ossl_cipher_update_long(ctx, (unsigned char *)RSTRING_PTR(str), &out_len, in, in_len))
 	ossl_raise(eCipherError, NULL);
     assert(out_len < RSTRING_LEN(str));
     rb_str_set_len(str, out_len);
@@ -523,17 +554,16 @@ ossl_cipher_set_auth_data(VALUE self, VA https://github.com/ruby/ruby/blob/trunk/ext/openssl/ossl_cipher.c#L554
 {
     EVP_CIPHER_CTX *ctx;
     unsigned char *in;
-    int in_len;
-    int out_len;
+    long in_len, out_len;
 
     StringValue(data);
 
     in = (unsigned char *) RSTRING_PTR(data);
-    in_len = RSTRING_LENINT(data);
+    in_len = RSTRING_LEN(data);
 
     GetCipher(self, ctx);
 
-    if (!EVP_CipherUpdate(ctx, NULL, &out_len, in, in_len))
+    if (!ossl_cipher_update_long(ctx, NULL, &out_len, in, in_len))
         ossl_raise(eCipherError, "couldn't set additional authenticated data");
 
     return data;

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/