[前][次][番号順一覧][スレッド一覧]

ruby-changes:52557

From: hsbt <ko1@a...>
Date: Tue, 18 Sep 2018 17:37:22 +0900 (JST)
Subject: [ruby-changes:52557] hsbt:r64769 (trunk): Merge upstream revision of rubygems/rubygems.

hsbt	2018-09-18 17:37:18 +0900 (Tue, 18 Sep 2018)

  New Revision: 64769

  https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=64769

  Log:
    Merge upstream revision of rubygems/rubygems.
    
      This commits includes tiny bugfix and new features listed here:
        * Add --re-sign flag to cert command by bronzdoc: https://github.com/rubygems/rubygems/pull/2391
        * Download gems with threads. by indirect: https://github.com/rubygems/rubygems/pull/1898

  Modified files:
    trunk/lib/rubygems/commands/cert_command.rb
    trunk/lib/rubygems/config_file.rb
    trunk/lib/rubygems/installer.rb
    trunk/lib/rubygems/request_set.rb
    trunk/lib/rubygems/resolver/specification.rb
    trunk/lib/rubygems/security/signer.rb
    trunk/lib/rubygems/user_interaction.rb
    trunk/test/rubygems/test_gem_commands_cert_command.rb
    trunk/test/rubygems/test_gem_stream_ui.rb
Index: lib/rubygems/request_set.rb
===================================================================
--- lib/rubygems/request_set.rb	(revision 64768)
+++ lib/rubygems/request_set.rb	(revision 64769)
@@ -152,7 +152,34 @@ class Gem::RequestSet https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set.rb#L152
     @prerelease = options[:prerelease]
 
     requests = []
+    download_queue = Queue.new
 
+    # Create a thread-safe list of gems to download
+    sorted_requests.each do |req|
+      download_queue << req
+    end
+
+    # Create N threads in a pool, have them download all the gems
+    threads = Gem.configuration.concurrent_downloads.times.map do
+      # When a thread pops this item, it knows to stop running. The symbol
+      # is queued here so that there will be one symbol per thread.
+      download_queue << :stop
+
+      Thread.new do
+        # The pop method will block waiting for items, so the only way
+        # to stop a thread from running is to provide a final item that
+        # means the thread should stop.
+        while req = download_queue.pop
+          break if req == :stop
+          req.spec.download options unless req.installed?
+        end
+      end
+    end
+
+    # Wait for all the downloads to finish before continuing
+    threads.each(&:value)
+
+    # Install requested gems after they have been downloaded
     sorted_requests.each do |req|
       if req.installed? then
         req.spec.spec.build_extensions
Index: lib/rubygems/config_file.rb
===================================================================
--- lib/rubygems/config_file.rb	(revision 64768)
+++ lib/rubygems/config_file.rb	(revision 64769)
@@ -27,6 +27,7 @@ require 'rbconfig' https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L27
 # +:backtrace+:: See #backtrace
 # +:sources+:: Sets Gem::sources
 # +:verbose+:: See #verbose
+# +:concurrent_downloads+:: See #concurrent_downloads
 #
 # gemrc files may exist in various locations and are read and merged in
 # the following order:
@@ -43,6 +44,7 @@ class Gem::ConfigFile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L44
   DEFAULT_BULK_THRESHOLD = 1000
   DEFAULT_VERBOSITY = true
   DEFAULT_UPDATE_SOURCES = true
+  DEFAULT_CONCURRENT_DOWNLOADS = 8
 
   ##
   # For Ruby packagers to set configuration defaults.  Set in
@@ -105,6 +107,11 @@ class Gem::ConfigFile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L107
   attr_accessor :verbose
 
   ##
+  # Number of gem downloads that should be performed concurrently.
+
+  attr_accessor :concurrent_downloads
+
+  ##
   # True if we want to update the SourceInfoCache every time, false otherwise
 
   attr_accessor :update_sources
@@ -177,6 +184,7 @@ class Gem::ConfigFile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L184
     @bulk_threshold = DEFAULT_BULK_THRESHOLD
     @verbose = DEFAULT_VERBOSITY
     @update_sources = DEFAULT_UPDATE_SOURCES
+    @concurrent_downloads = DEFAULT_CONCURRENT_DOWNLOADS
 
     operating_system_config = Marshal.load Marshal.dump(OPERATING_SYSTEM_DEFAULTS)
     platform_config = Marshal.load Marshal.dump(PLATFORM_DEFAULTS)
@@ -200,6 +208,7 @@ class Gem::ConfigFile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L208
     @path                       = @hash[:gempath]                    if @hash.key? :gempath
     @update_sources             = @hash[:update_sources]             if @hash.key? :update_sources
     @verbose                    = @hash[:verbose]                    if @hash.key? :verbose
+    @concurrent_downloads       = @hash[:concurrent_downloads]       if @hash.key? :concurrent_downloads
     @disable_default_gem_server = @hash[:disable_default_gem_server] if @hash.key? :disable_default_gem_server
     @sources                    = @hash[:sources]                    if @hash.key? :sources
 
@@ -415,6 +424,9 @@ if you believe they were disclosed to a https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L424
     yaml_hash[:update_sources] = @hash.fetch(:update_sources, DEFAULT_UPDATE_SOURCES)
     yaml_hash[:verbose] = @hash.fetch(:verbose, DEFAULT_VERBOSITY)
 
+    yaml_hash[:concurrent_downloads] =
+      @hash.fetch(:concurrent_downloads, DEFAULT_CONCURRENT_DOWNLOADS)
+
     yaml_hash[:ssl_verify_mode] =
       @hash[:ssl_verify_mode] if @hash.key? :ssl_verify_mode
 
Index: lib/rubygems/commands/cert_command.rb
===================================================================
--- lib/rubygems/commands/cert_command.rb	(revision 64768)
+++ lib/rubygems/commands/cert_command.rb	(revision 64769)
@@ -14,15 +14,16 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L14
     super 'cert', 'Manage RubyGems certificates and signing settings',
           :add => [], :remove => [], :list => [], :build => [], :sign => []
 
-    OptionParser.accept OpenSSL::X509::Certificate do |certificate|
+    OptionParser.accept OpenSSL::X509::Certificate do |certificate_file|
       begin
-        OpenSSL::X509::Certificate.new File.read certificate
+        certificate = OpenSSL::X509::Certificate.new File.read certificate_file
       rescue Errno::ENOENT
-        raise OptionParser::InvalidArgument, "#{certificate}: does not exist"
+        raise OptionParser::InvalidArgument, "#{certificate_file}: does not exist"
       rescue OpenSSL::X509::CertificateError
         raise OptionParser::InvalidArgument,
-          "#{certificate}: invalid X509 certificate"
+          "#{certificate_file}: invalid X509 certificate"
       end
+      [certificate, certificate_file]
     end
 
     OptionParser.accept OpenSSL::PKey::RSA do |key_file|
@@ -42,7 +43,7 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L43
     end
 
     add_option('-a', '--add CERT', OpenSSL::X509::Certificate,
-               'Add a trusted certificate.') do |cert, options|
+               'Add a trusted certificate.') do |(cert, _), options|
       options[:add] << cert
     end
 
@@ -67,8 +68,9 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L68
     end
 
     add_option('-C', '--certificate CERT', OpenSSL::X509::Certificate,
-               'Signing certificate for --sign') do |cert, options|
+               'Signing certificate for --sign') do |(cert, cert_file), options|
       options[:issuer_cert] = cert
+      options[:issuer_cert_file] = cert_file
     end
 
     add_option('-K', '--private-key KEY', OpenSSL::PKey::RSA,
@@ -89,6 +91,11 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L91
                'Days before the certificate expires') do |days, options|
       options[:expiration_length_days] = days.to_i
     end
+
+    add_option('-R', '--re-sign',
+               'Re-signs the certificate from -C with the key from -K') do |resign, options|
+      options[:resign] = resign
+    end
   end
 
   def add_certificate certificate # :nodoc:
@@ -114,6 +121,14 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L121
       build email
     end
 
+    if options[:resign]
+      re_sign_cert(
+        options[:issuer_cert],
+        options[:issuer_cert_file],
+        options[:key]
+      )
+    end
+
     sign_certificates unless options[:sign].empty?
   end
 
@@ -290,6 +305,13 @@ For further reading on signing gems see https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L305
     end
   end
 
+  def re_sign_cert(cert, cert_path, private_key)
+    Gem::Security::Signer.re_sign_cert(cert, cert_path, private_key) do |expired_cert_path, new_expired_cert_path|
+      alert("Your certificate #{expired_cert_path} has been re-signed")
+      alert("Your expired certificate will be located at: #{new_expired_cert_path}")
+    end
+  end
+
   private
 
   def valid_email? email
Index: lib/rubygems/security/signer.rb
===================================================================
--- lib/rubygems/security/signer.rb	(revision 64768)
+++ lib/rubygems/security/signer.rb	(revision 64769)
@@ -30,6 +30,24 @@ class Gem::Security::Signer https://github.com/ruby/ruby/blob/trunk/lib/rubygems/security/signer.rb#L30
   attr_reader :digest_name # :nodoc:
 
   ##
+  # Attemps to re-sign an expired cert with a given private key
+  def self.re_sign_cert(expired_cert, expired_cert_path, private_key)
+    return unless expired_cert.not_after < Time.now
+
+    expiry = expired_cert.not_after.strftime('%Y%m%d%H%M%S')
+    expired_cert_file = "#{File.basename(expired_cert_path)}.expired.#{expiry}"
+    new_expired_cert_path = File.join(Gem.user_home, ".gem", expired_cert_file)
+
+    Gem::Security.write(expired_cert, new_expired_cert_path)
+
+    re_signed_cert = Gem::Security.re_sign(expired_cert, private_key)
+
+    Gem::Security.write(re_signed_cert, expired_cert_path)
+
+    yield(expired_cert_path, new_expired_cert_path) if block_given?
+  end
+
+  ##
   # Creates a new signer with an RSA +key+ or path to a key, and a certificate
   # +chain+ containing X509 certificates, encoding certificates or paths to
   # certificates.
Index: lib/rubygems/user_interaction.rb
===================================================================
--- lib/rubygems/user_interaction.rb	(revision 64768)
+++ lib/rubygems/user_interaction.rb	(revision 64769)
@@ -512,11 +512,10 @@ class Gem::StreamUI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/user_interaction.rb#L512
   # Return a download reporter object chosen from the current verbosity
 
   def download_reporter(*args)
-    case Gem.configuration.verbose
-    when nil, false
+    if [nil, false].include?(Gem.configuration.verbose) || !@outs.tty?
       SilentDownloadReporter.new(@outs, *args)
     else
-      VerboseDownloadReporter.new(@outs, *args)
+      ThreadedDownloadReporter.new(@outs, *args)
     end
   end
 
@@ -553,9 +552,11 @@ class Gem::StreamUI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/user_interaction.rb#L552
   end
 
   ##
-  # A progress reporter that prints out messages about the current progress.
+  # A progress reporter that behaves nicely with threaded downloading.
 
-  class VerboseDownloadReporter
+  class ThreadedDownloadReporter
+
+    MUTEX = Mutex.new
 
     ##
     # The current file name being displayed
@@ -563,71 +564,43 @@ class Gem::StreamUI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/user_interaction.rb#L564
     attr_reader :file_name
 
     ##
-    # The total bytes in the file
-
-    attr_reader :total_bytes
-
-    ##
-    # The current progress (0 to 100)
-
-    attr_reader :progress
-
-    ##
-    # Creates a new verbose download reporter that will display on
+    # Creates a new threaded download reporter that will display on
     # +out_stream+.  The other arguments are ignored.
 
     def initialize(out_stream, *args)
       @out = out_stream
-      @progress = 0
     end
 
     ##
-    # Tells the download reporter that the +file_name+ is being fetched and
-    # contains +total_bytes+.
+    # Tells the download reporter that the +file_name+ is being fetched.
+    # The other arguments are ignored.
 
-    def fetch(file_name, total_bytes)
-      @file_name = file_name
-      @total_bytes = total_bytes.to_i
-      @units = @total_bytes.zero? ? 'B' : '%'
-
-      update_display(false)
+    def fetch(file_name, *args)
+      if @file_name.nil?
+        @file_name = file_name
+        locked_puts "Fetching #{@file_name}"
+      end
     end
 
     ##
-    # Updates the verbose download reporter for the given number of +bytes+.
+    # Updates the threaded download reporter for the given number of +bytes+.
 
     def update(bytes)
-      new_progress = if @units == 'B' then
-                       bytes
-                     else
-                       ((bytes.to_f * 100) / total_bytes.to_f).ceil
-                     end
-
-      return if new_progress == @progress
-
-      @progress = new_progress
-      update_display
+      # Do nothing.
     end
 
     ##
     # Indicates the download is complete.
 
     def done
-      @progress = 100 if @units == '%'
-      update_display(true, true)
+      # Do nothing.
     end
 
     private
-
-    def update_display(show_progress = true, new_line = false) # :nodoc:
-      return unless @out.tty?
-
-      if show_progress then
-        @out.print "\rFetching: %s (%3d%s)" % [@file_name, @progress, @units]
-      else
-        @out.print "Fetching: %s" % @file_name
+    def locked_puts(message)
+      MUTEX.synchronize do
+        @out.puts message
       end
-      @out.puts if new_line
     end
   end
 end
Index: lib/rubygems/installer.rb
===================================================================
--- lib/rubygems/installer.rb	(revision 64768)
+++ lib/rubygems/installer.rb	(revision 64769)
@@ -771,30 +771,26 @@ TEXT https://github.com/ruby/ruby/blob/trunk/lib/rubygems/installer.rb#L771
   # return the stub script text used to launch the true Ruby script
 
   def windows_stub_script(bindir, bin_file_name)
-    rb_config = RbConfig::CONFIG
-    rb_topdir = RbConfig::TOPDIR || File.dirname(rb_config["bindir"])
-    # get ruby executable file name from RbConfig
-    ruby_exe = "#{rb_config['RUBY_INSTALL_NAME']}#{rb_config['EXEEXT']}"
-
-    if File.exist?(File.join bindir, ruby_exe)
+    rb_bindir = RbConfig::CONFIG["bindir"]
+    # All comparisons should be case insensitive
+    if bindir.downcase == rb_bindir.downcase
       # stub & ruby.exe withing same folder.  Portable
       <<-TEXT
 @ECHO OFF
 @"%~dp0ruby.exe" "%~dpn0" %*
       TEXT
-    elsif bindir.downcase.start_with? rb_topdir.downcase
-      # stub within ruby folder, but not standard bin.  Portable
+    elsif bindir.downcase.start_with?((RbConfig::TOPDIR || File.dirname(rb_bindir)).downcase)
+      # stub within ruby folder, but not standard bin.  Not portable
       require 'pathname'
       from = Pathname.new bindir
-      to   = Pathname.new "#{rb_topdir}/bin"
+      to   = Pathname.new rb_bindir
       rel  = to.relative_path_from from
       <<-TEXT
 @ECHO OFF
 @"%~dp0#{rel}/ruby.exe" "%~dpn0" %*
       TEXT
     else
-      # outside ruby folder, maybe -user-install or bundler.  Portable, but ruby
-      # is dependent on PATH
+      # outside ruby folder, maybe -user-install or bundler.  Portable
       <<-TEXT
 @ECHO OFF
 @ruby.exe "%~dpn0" %*
Index: lib/rubygems/resolver/specification.rb
===================================================================
--- lib/rubygems/resolver/specification.rb	(revision 64768)
+++ lib/rubygems/resolver/specification.rb	(revision 64769)
@@ -84,11 +84,7 @@ class Gem::Resolver::Specification https://github.com/ruby/ruby/blob/trunk/lib/rubygems/resolver/specification.rb#L84
   def install options = {}
     require 'rubygems/installer'
 
-    destination = options[:install_dir] || Gem.dir
-
-    Gem.ensure_gem_subdirectories destination
-
-    gem = source.download spec, destination
+    gem = download options
 
     installer = Gem::Installer.at gem, options
 
@@ -97,6 +93,14 @@ class Gem::Resolver::Specification https://github.com/ruby/ruby/blob/trunk/lib/rubygems/resolver/specification.rb#L93
     @spec = installer.install
   end
 
+  def download options
+    dir = options[:install_dir] || Gem.dir
+
+    Gem.ensure_gem_subdirectories dir
+
+    source.download spec, dir
+  end
+
   ##
   # Returns true if this specification is installable on this platform.
 
Index: test/rubygems/test_gem_stream_ui.rb
===================================================================
--- test/rubygems/test_gem_stream_ui.rb	(revision 64768)
+++ test/rubygems/test_gem_stream_ui.rb	(revision 64769)
@@ -156,14 +156,14 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L156
   def test_download_reporter_anything
     @cfg.verbose = 0
     reporter = @sui.download_reporter
-    assert_kind_of Gem::StreamUI::VerboseDownloadReporter, reporter
+    assert_kind_of Gem::StreamUI::ThreadedDownloadReporter, reporter
   end
 
-  def test_verbose_download_reporter
+  def test_threaded_download_reporter
     @cfg.verbose = true
     reporter = @sui.download_reporter
     reporter.fetch 'a.gem', 1024
-    assert_equal "Fetching: a.gem", @out.string
+    assert_equal "Fetching a.gem\n", @out.string
   end
 
   def test_verbose_download_reporter_progress
@@ -171,7 +171,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L171
     reporter = @sui.download_reporter
     reporter.fetch 'a.gem', 1024
     reporter.update 512
-    assert_equal "Fetching: a.gem\rFetching: a.gem ( 50%)", @out.string
+    assert_equal "Fetching a.gem\n", @out.string
   end
 
   def test_verbose_download_reporter_progress_once
@@ -180,7 +180,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L180
     reporter.fetch 'a.gem', 1024
     reporter.update 510
     reporter.update 512
-    assert_equal "Fetching: a.gem\rFetching: a.gem ( 50%)", @out.string
+    assert_equal "Fetching a.gem\n", @out.string
   end
 
   def test_verbose_download_reporter_progress_complete
@@ -189,7 +189,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L189
     reporter.fetch 'a.gem', 1024
     reporter.update 510
     reporter.done
-    assert_equal "Fetching: a.gem\rFetching: a.gem ( 50%)\rFetching: a.gem (100%)\n", @out.string
+    assert_equal "Fetching a.gem\n", @out.string
   end
 
   def test_verbose_download_reporter_progress_nil_length
@@ -198,7 +198,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L198
     reporter.fetch 'a.gem', nil
     reporter.update 1024
     reporter.done
-    assert_equal "Fetching: a.gem\rFetching: a.gem (1024B)\rFetching: a.gem (1024B)\n", @out.string
+    assert_equal "Fetching a.gem\n", @out.string
   end
 
   def test_verbose_download_reporter_progress_zero_length
@@ -207,7 +207,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L207
     reporter.fetch 'a.gem', 0
     reporter.update 1024
     reporter.done
-    assert_equal "Fetching: a.gem\rFetching: a.gem (1024B)\rFetching: a.gem (1024B)\n", @out.string
+    assert_equal "Fetching a.gem\n", @out.string
   end
 
   def test_verbose_download_reporter_no_tty
Index: test/rubygems/test_gem_commands_cert_command.rb
===================================================================
--- test/rubygems/test_gem_commands_cert_command.rb	(revision 64768)
+++ test/rubygems/test_gem_commands_cert_command.rb	(revision 64769)
@@ -9,14 +9,16 @@ end https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_commands_cert_command.rb#L9
 class TestGemCommandsCertCommand < Gem::TestCase
 
   ALTERNATE_CERT = load_cert 'alternate'
+  EXPIRED_PUBLIC_CERT = load_cert 'expired'
 
   ALTERNATE_KEY_FILE = key_path 'alternate'
   PRIVATE_KEY_FILE   = key_path 'private'
   PUBLIC_KEY_FILE    = key_path 'public'
 
-  ALTERNATE_CERT_FILE = cert_path 'alternate'
-  CHILD_CERT_FILE     = cert_path 'child'
-  PUBLIC_CERT_FILE    = cert_path 'public'
+  ALTERNATE_CERT_FILE      = cert_path 'alternate'
+  CHILD_CERT_FILE          = cert_path 'child'
+  PUBLIC_CERT_FILE         = cert_path 'public'
+  EXPIRED_PUBLIC_CERT_FILE = cert_path 'expired'
 
   def setup
     super
@@ -582,6 +584,37 @@ ERROR:  --private-key not specified and https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_commands_cert_command.rb#L584
     assert_equal expected, @ui.error
   end
 
+  def test_execute_re_sign
+    gem_path = File. (... truncated)

--
ML: ruby-changes@q...
Info: http://www.atdot.net/~ko1/quickml/

[前][次][番号順一覧][スレッド一覧]