https://git.ruby-lang.org/ruby.git/commit/?id=f364564e66

From f364564e66d1db1de8e80d669287386595c8bc46 Mon Sep 17 00:00:00 2001
From: Yusuke Endoh <mame@r...>
Date: Mon, 21 Oct 2019 21:22:53 +0900
Subject: bignum.c (estimate_initial_sqrt): prevent integer overflow

`Integer.sqrt(0xffff_ffff_ffff_ffff ** 2)` caused assertion failure
because of integer overflow.  [ruby-core:95453] [Bug #16269]

diff --git a/bignum.c b/bignum.c
index f379b10..0539232 100644
--- a/bignum.c
+++ b/bignum.c
@@ -6888,7 +6888,15 @@ estimate_initial_sqrt(VALUE *xp, const size_t xn, const BDIGIT *nds, size_t len) https://github.com/ruby/ruby/blob/trunk/bignum.c#L6888
     rshift /= 2;
     rshift += (2-(len&1))*BITSPERDIG/2;
     if (rshift >= 0) {
-	d <<= rshift;
+        if (nlz((BDIGIT)d) + rshift >= BITSPERDIG) {
+            /* (d << rshift) does cause overflow.
+             * example: Integer.sqrt(0xffff_ffff_ffff_ffff ** 2)
+             */
+            d = ~(BDIGIT_DBL)0;
+        }
+        else {
+            d <<= rshift;
+        }
     }
     BDIGITS_ZERO(xds, xn-2);
     bdigitdbl2bary(&xds[xn-2], 2, d);
diff --git a/test/ruby/test_integer.rb b/test/ruby/test_integer.rb
index 9a4f560..a111698 100644
--- a/test/ruby/test_integer.rb
+++ b/test/ruby/test_integer.rb
@@ -640,6 +640,9 @@ class TestInteger < Test::Unit::TestCase https://github.com/ruby/ruby/blob/trunk/test/ruby/test_integer.rb#L640
       failures << n  unless root*root <= n && (root+1)*(root+1) > n
     end
     assert_empty(failures, bug13440)
+
+    x = 0xffff_ffff_ffff_ffff
+    assert_equal(x, Integer.sqrt(x ** 2), "[ruby-core:95453]")
   end
 
   def test_fdiv
-- 
cgit v0.10.2


--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/