nobu	2015-08-11 15:22:34 +0900 (Tue, 11 Aug 2015)

  New Revision: 51533

  http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=51533

  Log:
    ruby.h: check integer overflow
    
    * include/ruby/ruby.h (ALLOCV_N): check integer overflow, as well
      as ruby_xmalloc2.  pointed out by Paul <pawlkt AT gmail.com>.

  Modified files:
    trunk/ChangeLog
    trunk/gc.c
    trunk/include/ruby/ruby.h
Index: include/ruby/ruby.h
===================================================================
--- include/ruby/ruby.h	(revision 51532)
+++ include/ruby/ruby.h	(revision 51533)
@@ -1431,14 +1431,31 @@ rb_num2char_inline(VALUE x) https://github.com/ruby/ruby/blob/trunk/include/ruby/ruby.h#L1431
 
 void *rb_alloc_tmp_buffer(volatile VALUE *store, long len) RUBY_ATTR_ALLOC_SIZE((2));
 void rb_free_tmp_buffer(volatile VALUE *store);
+NORETURN(void ruby_malloc_size_overflow(size_t, size_t));
+static inline size_t
+ruby_xmalloc2_size(const size_t count, const size_t elsize)
+{
+    if (count > SIZE_MAX / elsize) {
+	ruby_malloc_size_overflow(count, elsize);
+    }
+    return count * elsize;
+}
 /* allocates _n_ bytes temporary buffer and stores VALUE including it
  * in _v_.  _n_ may be evaluated twice. */
 #ifdef C_ALLOCA
 # define ALLOCV(v, n) rb_alloc_tmp_buffer(&(v), (n))
+# define ALLOCV_N(type, v, n) \
+    ((type*)ALLOCV((v), ruby_xmalloc2_size((n), sizeof(type))))
 #else
-# define ALLOCV(v, n) ((n) < 1024 ? (RB_GC_GUARD(v) = 0, alloca(n)) : rb_alloc_tmp_buffer(&(v), (n)))
+# define ALLOCV_LIMIT 1024
+# define ALLOCV(v, n) ((n) < ALLOCV_LIMIT ? \
+		       (RB_GC_GUARD(v) = 0, alloca(n)) : \
+		       rb_alloc_tmp_buffer(&(v), (n)))
+# define ALLOCV_N(type, v, n) \
+    ((type*)(ruby_xmalloc2_size((n), sizeof(type)) < ALLOCV_LIMIT ? \
+	     (RB_GC_GUARD(v) = 0, alloca((n) * sizeof(type))) : \
+	     rb_alloc_tmp_buffer(&(v), (n) * sizeof(type))))
 #endif
-#define ALLOCV_N(type, v, n) ((type*)ALLOCV((v), sizeof(type)*(n)))
 #define ALLOCV_END(v) rb_free_tmp_buffer(&(v))
 
 #define MEMZERO(p,type,n) memset((p), 0, sizeof(type)*(n))
Index: ChangeLog
===================================================================
--- ChangeLog	(revision 51532)
+++ ChangeLog	(revision 51533)
@@ -1,3 +1,8 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1
+Tue Aug 11 15:22:31 2015  Nobuyoshi Nakada  <nobu@r...>
+
+	* include/ruby/ruby.h (ALLOCV_N): check integer overflow, as well
+	  as ruby_xmalloc2.  pointed out by Paul <pawlkt AT gmail.com>.
+
 Tue Aug 11 14:57:09 2015  Nobuyoshi Nakada  <nobu@r...>
 
 	* array.c (rb_ary_repeated_permutation): fix buffer size, ALLOCV_N
Index: gc.c
===================================================================
--- gc.c	(revision 51532)
+++ gc.c	(revision 51533)
@@ -7649,16 +7649,16 @@ ruby_xmalloc(size_t size) https://github.com/ruby/ruby/blob/trunk/gc.c#L7649
     return objspace_xmalloc(&rb_objspace, size);
 }
 
-static inline size_t
-xmalloc2_size(size_t n, size_t size)
+void
+ruby_malloc_size_overflow(size_t count, size_t elsize)
 {
-    size_t len = size * n;
-    if (n != 0 && size != len / n) {
-	rb_raise(rb_eArgError, "malloc: possible integer overflow");
-    }
-    return len;
+    rb_raise(rb_eArgError,
+	     "malloc: possible integer overflow (%"PRIdSIZE"*%"PRIdSIZE")",
+	     count, elsize);
 }
 
+#define xmalloc2_size ruby_xmalloc2_size
+
 void *
 ruby_xmalloc2(size_t n, size_t size)
 {

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/