ruby-changes:52557
From: hsbt <ko1@a...>
Date: Tue, 18 Sep 2018 17:37:22 +0900 (JST)
Subject: [ruby-changes:52557] hsbt:r64769 (trunk): Merge upstream revision of rubygems/rubygems.
hsbt 2018-09-18 17:37:18 +0900 (Tue, 18 Sep 2018) New Revision: 64769 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=64769 Log: Merge upstream revision of rubygems/rubygems. This commits includes tiny bugfix and new features listed here: * Add --re-sign flag to cert command by bronzdoc: https://github.com/rubygems/rubygems/pull/2391 * Download gems with threads. by indirect: https://github.com/rubygems/rubygems/pull/1898 Modified files: trunk/lib/rubygems/commands/cert_command.rb trunk/lib/rubygems/config_file.rb trunk/lib/rubygems/installer.rb trunk/lib/rubygems/request_set.rb trunk/lib/rubygems/resolver/specification.rb trunk/lib/rubygems/security/signer.rb trunk/lib/rubygems/user_interaction.rb trunk/test/rubygems/test_gem_commands_cert_command.rb trunk/test/rubygems/test_gem_stream_ui.rb Index: lib/rubygems/request_set.rb =================================================================== --- lib/rubygems/request_set.rb (revision 64768) +++ lib/rubygems/request_set.rb (revision 64769) @@ -152,7 +152,34 @@ class Gem::RequestSet https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set.rb#L152 @prerelease = options[:prerelease] requests = [] + download_queue = Queue.new + # Create a thread-safe list of gems to download + sorted_requests.each do |req| + download_queue << req + end + + # Create N threads in a pool, have them download all the gems + threads = Gem.configuration.concurrent_downloads.times.map do + # When a thread pops this item, it knows to stop running. The symbol + # is queued here so that there will be one symbol per thread. + download_queue << :stop + + Thread.new do + # The pop method will block waiting for items, so the only way + # to stop a thread from running is to provide a final item that + # means the thread should stop. + while req = download_queue.pop + break if req == :stop + req.spec.download options unless req.installed? + end + end + end + + # Wait for all the downloads to finish before continuing + threads.each(&:value) + + # Install requested gems after they have been downloaded sorted_requests.each do |req| if req.installed? then req.spec.spec.build_extensions Index: lib/rubygems/config_file.rb =================================================================== --- lib/rubygems/config_file.rb (revision 64768) +++ lib/rubygems/config_file.rb (revision 64769) @@ -27,6 +27,7 @@ require 'rbconfig' https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L27 # +:backtrace+:: See #backtrace # +:sources+:: Sets Gem::sources # +:verbose+:: See #verbose +# +:concurrent_downloads+:: See #concurrent_downloads # # gemrc files may exist in various locations and are read and merged in # the following order: @@ -43,6 +44,7 @@ class Gem::ConfigFile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L44 DEFAULT_BULK_THRESHOLD = 1000 DEFAULT_VERBOSITY = true DEFAULT_UPDATE_SOURCES = true + DEFAULT_CONCURRENT_DOWNLOADS = 8 ## # For Ruby packagers to set configuration defaults. Set in @@ -105,6 +107,11 @@ class Gem::ConfigFile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L107 attr_accessor :verbose ## + # Number of gem downloads that should be performed concurrently. + + attr_accessor :concurrent_downloads + + ## # True if we want to update the SourceInfoCache every time, false otherwise attr_accessor :update_sources @@ -177,6 +184,7 @@ class Gem::ConfigFile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L184 @bulk_threshold = DEFAULT_BULK_THRESHOLD @verbose = DEFAULT_VERBOSITY @update_sources = DEFAULT_UPDATE_SOURCES + @concurrent_downloads = DEFAULT_CONCURRENT_DOWNLOADS operating_system_config = Marshal.load Marshal.dump(OPERATING_SYSTEM_DEFAULTS) platform_config = Marshal.load Marshal.dump(PLATFORM_DEFAULTS) @@ -200,6 +208,7 @@ class Gem::ConfigFile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L208 @path = @hash[:gempath] if @hash.key? :gempath @update_sources = @hash[:update_sources] if @hash.key? :update_sources @verbose = @hash[:verbose] if @hash.key? :verbose + @concurrent_downloads = @hash[:concurrent_downloads] if @hash.key? :concurrent_downloads @disable_default_gem_server = @hash[:disable_default_gem_server] if @hash.key? :disable_default_gem_server @sources = @hash[:sources] if @hash.key? :sources @@ -415,6 +424,9 @@ if you believe they were disclosed to a https://github.com/ruby/ruby/blob/trunk/lib/rubygems/config_file.rb#L424 yaml_hash[:update_sources] = @hash.fetch(:update_sources, DEFAULT_UPDATE_SOURCES) yaml_hash[:verbose] = @hash.fetch(:verbose, DEFAULT_VERBOSITY) + yaml_hash[:concurrent_downloads] = + @hash.fetch(:concurrent_downloads, DEFAULT_CONCURRENT_DOWNLOADS) + yaml_hash[:ssl_verify_mode] = @hash[:ssl_verify_mode] if @hash.key? :ssl_verify_mode Index: lib/rubygems/commands/cert_command.rb =================================================================== --- lib/rubygems/commands/cert_command.rb (revision 64768) +++ lib/rubygems/commands/cert_command.rb (revision 64769) @@ -14,15 +14,16 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L14 super 'cert', 'Manage RubyGems certificates and signing settings', :add => [], :remove => [], :list => [], :build => [], :sign => [] - OptionParser.accept OpenSSL::X509::Certificate do |certificate| + OptionParser.accept OpenSSL::X509::Certificate do |certificate_file| begin - OpenSSL::X509::Certificate.new File.read certificate + certificate = OpenSSL::X509::Certificate.new File.read certificate_file rescue Errno::ENOENT - raise OptionParser::InvalidArgument, "#{certificate}: does not exist" + raise OptionParser::InvalidArgument, "#{certificate_file}: does not exist" rescue OpenSSL::X509::CertificateError raise OptionParser::InvalidArgument, - "#{certificate}: invalid X509 certificate" + "#{certificate_file}: invalid X509 certificate" end + [certificate, certificate_file] end OptionParser.accept OpenSSL::PKey::RSA do |key_file| @@ -42,7 +43,7 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L43 end add_option('-a', '--add CERT', OpenSSL::X509::Certificate, - 'Add a trusted certificate.') do |cert, options| + 'Add a trusted certificate.') do |(cert, _), options| options[:add] << cert end @@ -67,8 +68,9 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L68 end add_option('-C', '--certificate CERT', OpenSSL::X509::Certificate, - 'Signing certificate for --sign') do |cert, options| + 'Signing certificate for --sign') do |(cert, cert_file), options| options[:issuer_cert] = cert + options[:issuer_cert_file] = cert_file end add_option('-K', '--private-key KEY', OpenSSL::PKey::RSA, @@ -89,6 +91,11 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L91 'Days before the certificate expires') do |days, options| options[:expiration_length_days] = days.to_i end + + add_option('-R', '--re-sign', + 'Re-signs the certificate from -C with the key from -K') do |resign, options| + options[:resign] = resign + end end def add_certificate certificate # :nodoc: @@ -114,6 +121,14 @@ class Gem::Commands::CertCommand < Gem:: https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L121 build email end + if options[:resign] + re_sign_cert( + options[:issuer_cert], + options[:issuer_cert_file], + options[:key] + ) + end + sign_certificates unless options[:sign].empty? end @@ -290,6 +305,13 @@ For further reading on signing gems see https://github.com/ruby/ruby/blob/trunk/lib/rubygems/commands/cert_command.rb#L305 end end + def re_sign_cert(cert, cert_path, private_key) + Gem::Security::Signer.re_sign_cert(cert, cert_path, private_key) do |expired_cert_path, new_expired_cert_path| + alert("Your certificate #{expired_cert_path} has been re-signed") + alert("Your expired certificate will be located at: #{new_expired_cert_path}") + end + end + private def valid_email? email Index: lib/rubygems/security/signer.rb =================================================================== --- lib/rubygems/security/signer.rb (revision 64768) +++ lib/rubygems/security/signer.rb (revision 64769) @@ -30,6 +30,24 @@ class Gem::Security::Signer https://github.com/ruby/ruby/blob/trunk/lib/rubygems/security/signer.rb#L30 attr_reader :digest_name # :nodoc: ## + # Attemps to re-sign an expired cert with a given private key + def self.re_sign_cert(expired_cert, expired_cert_path, private_key) + return unless expired_cert.not_after < Time.now + + expiry = expired_cert.not_after.strftime('%Y%m%d%H%M%S') + expired_cert_file = "#{File.basename(expired_cert_path)}.expired.#{expiry}" + new_expired_cert_path = File.join(Gem.user_home, ".gem", expired_cert_file) + + Gem::Security.write(expired_cert, new_expired_cert_path) + + re_signed_cert = Gem::Security.re_sign(expired_cert, private_key) + + Gem::Security.write(re_signed_cert, expired_cert_path) + + yield(expired_cert_path, new_expired_cert_path) if block_given? + end + + ## # Creates a new signer with an RSA +key+ or path to a key, and a certificate # +chain+ containing X509 certificates, encoding certificates or paths to # certificates. Index: lib/rubygems/user_interaction.rb =================================================================== --- lib/rubygems/user_interaction.rb (revision 64768) +++ lib/rubygems/user_interaction.rb (revision 64769) @@ -512,11 +512,10 @@ class Gem::StreamUI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/user_interaction.rb#L512 # Return a download reporter object chosen from the current verbosity def download_reporter(*args) - case Gem.configuration.verbose - when nil, false + if [nil, false].include?(Gem.configuration.verbose) || !@outs.tty? SilentDownloadReporter.new(@outs, *args) else - VerboseDownloadReporter.new(@outs, *args) + ThreadedDownloadReporter.new(@outs, *args) end end @@ -553,9 +552,11 @@ class Gem::StreamUI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/user_interaction.rb#L552 end ## - # A progress reporter that prints out messages about the current progress. + # A progress reporter that behaves nicely with threaded downloading. - class VerboseDownloadReporter + class ThreadedDownloadReporter + + MUTEX = Mutex.new ## # The current file name being displayed @@ -563,71 +564,43 @@ class Gem::StreamUI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/user_interaction.rb#L564 attr_reader :file_name ## - # The total bytes in the file - - attr_reader :total_bytes - - ## - # The current progress (0 to 100) - - attr_reader :progress - - ## - # Creates a new verbose download reporter that will display on + # Creates a new threaded download reporter that will display on # +out_stream+. The other arguments are ignored. def initialize(out_stream, *args) @out = out_stream - @progress = 0 end ## - # Tells the download reporter that the +file_name+ is being fetched and - # contains +total_bytes+. + # Tells the download reporter that the +file_name+ is being fetched. + # The other arguments are ignored. - def fetch(file_name, total_bytes) - @file_name = file_name - @total_bytes = total_bytes.to_i - @units = @total_bytes.zero? ? 'B' : '%' - - update_display(false) + def fetch(file_name, *args) + if @file_name.nil? + @file_name = file_name + locked_puts "Fetching #{@file_name}" + end end ## - # Updates the verbose download reporter for the given number of +bytes+. + # Updates the threaded download reporter for the given number of +bytes+. def update(bytes) - new_progress = if @units == 'B' then - bytes - else - ((bytes.to_f * 100) / total_bytes.to_f).ceil - end - - return if new_progress == @progress - - @progress = new_progress - update_display + # Do nothing. end ## # Indicates the download is complete. def done - @progress = 100 if @units == '%' - update_display(true, true) + # Do nothing. end private - - def update_display(show_progress = true, new_line = false) # :nodoc: - return unless @out.tty? - - if show_progress then - @out.print "\rFetching: %s (%3d%s)" % [@file_name, @progress, @units] - else - @out.print "Fetching: %s" % @file_name + def locked_puts(message) + MUTEX.synchronize do + @out.puts message end - @out.puts if new_line end end end Index: lib/rubygems/installer.rb =================================================================== --- lib/rubygems/installer.rb (revision 64768) +++ lib/rubygems/installer.rb (revision 64769) @@ -771,30 +771,26 @@ TEXT https://github.com/ruby/ruby/blob/trunk/lib/rubygems/installer.rb#L771 # return the stub script text used to launch the true Ruby script def windows_stub_script(bindir, bin_file_name) - rb_config = RbConfig::CONFIG - rb_topdir = RbConfig::TOPDIR || File.dirname(rb_config["bindir"]) - # get ruby executable file name from RbConfig - ruby_exe = "#{rb_config['RUBY_INSTALL_NAME']}#{rb_config['EXEEXT']}" - - if File.exist?(File.join bindir, ruby_exe) + rb_bindir = RbConfig::CONFIG["bindir"] + # All comparisons should be case insensitive + if bindir.downcase == rb_bindir.downcase # stub & ruby.exe withing same folder. Portable <<-TEXT @ECHO OFF @"%~dp0ruby.exe" "%~dpn0" %* TEXT - elsif bindir.downcase.start_with? rb_topdir.downcase - # stub within ruby folder, but not standard bin. Portable + elsif bindir.downcase.start_with?((RbConfig::TOPDIR || File.dirname(rb_bindir)).downcase) + # stub within ruby folder, but not standard bin. Not portable require 'pathname' from = Pathname.new bindir - to = Pathname.new "#{rb_topdir}/bin" + to = Pathname.new rb_bindir rel = to.relative_path_from from <<-TEXT @ECHO OFF @"%~dp0#{rel}/ruby.exe" "%~dpn0" %* TEXT else - # outside ruby folder, maybe -user-install or bundler. Portable, but ruby - # is dependent on PATH + # outside ruby folder, maybe -user-install or bundler. Portable <<-TEXT @ECHO OFF @ruby.exe "%~dpn0" %* Index: lib/rubygems/resolver/specification.rb =================================================================== --- lib/rubygems/resolver/specification.rb (revision 64768) +++ lib/rubygems/resolver/specification.rb (revision 64769) @@ -84,11 +84,7 @@ class Gem::Resolver::Specification https://github.com/ruby/ruby/blob/trunk/lib/rubygems/resolver/specification.rb#L84 def install options = {} require 'rubygems/installer' - destination = options[:install_dir] || Gem.dir - - Gem.ensure_gem_subdirectories destination - - gem = source.download spec, destination + gem = download options installer = Gem::Installer.at gem, options @@ -97,6 +93,14 @@ class Gem::Resolver::Specification https://github.com/ruby/ruby/blob/trunk/lib/rubygems/resolver/specification.rb#L93 @spec = installer.install end + def download options + dir = options[:install_dir] || Gem.dir + + Gem.ensure_gem_subdirectories dir + + source.download spec, dir + end + ## # Returns true if this specification is installable on this platform. Index: test/rubygems/test_gem_stream_ui.rb =================================================================== --- test/rubygems/test_gem_stream_ui.rb (revision 64768) +++ test/rubygems/test_gem_stream_ui.rb (revision 64769) @@ -156,14 +156,14 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L156 def test_download_reporter_anything @cfg.verbose = 0 reporter = @sui.download_reporter - assert_kind_of Gem::StreamUI::VerboseDownloadReporter, reporter + assert_kind_of Gem::StreamUI::ThreadedDownloadReporter, reporter end - def test_verbose_download_reporter + def test_threaded_download_reporter @cfg.verbose = true reporter = @sui.download_reporter reporter.fetch 'a.gem', 1024 - assert_equal "Fetching: a.gem", @out.string + assert_equal "Fetching a.gem\n", @out.string end def test_verbose_download_reporter_progress @@ -171,7 +171,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L171 reporter = @sui.download_reporter reporter.fetch 'a.gem', 1024 reporter.update 512 - assert_equal "Fetching: a.gem\rFetching: a.gem ( 50%)", @out.string + assert_equal "Fetching a.gem\n", @out.string end def test_verbose_download_reporter_progress_once @@ -180,7 +180,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L180 reporter.fetch 'a.gem', 1024 reporter.update 510 reporter.update 512 - assert_equal "Fetching: a.gem\rFetching: a.gem ( 50%)", @out.string + assert_equal "Fetching a.gem\n", @out.string end def test_verbose_download_reporter_progress_complete @@ -189,7 +189,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L189 reporter.fetch 'a.gem', 1024 reporter.update 510 reporter.done - assert_equal "Fetching: a.gem\rFetching: a.gem ( 50%)\rFetching: a.gem (100%)\n", @out.string + assert_equal "Fetching a.gem\n", @out.string end def test_verbose_download_reporter_progress_nil_length @@ -198,7 +198,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L198 reporter.fetch 'a.gem', nil reporter.update 1024 reporter.done - assert_equal "Fetching: a.gem\rFetching: a.gem (1024B)\rFetching: a.gem (1024B)\n", @out.string + assert_equal "Fetching a.gem\n", @out.string end def test_verbose_download_reporter_progress_zero_length @@ -207,7 +207,7 @@ class TestGemStreamUI < Gem::TestCase https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_stream_ui.rb#L207 reporter.fetch 'a.gem', 0 reporter.update 1024 reporter.done - assert_equal "Fetching: a.gem\rFetching: a.gem (1024B)\rFetching: a.gem (1024B)\n", @out.string + assert_equal "Fetching a.gem\n", @out.string end def test_verbose_download_reporter_no_tty Index: test/rubygems/test_gem_commands_cert_command.rb =================================================================== --- test/rubygems/test_gem_commands_cert_command.rb (revision 64768) +++ test/rubygems/test_gem_commands_cert_command.rb (revision 64769) @@ -9,14 +9,16 @@ end https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_commands_cert_command.rb#L9 class TestGemCommandsCertCommand < Gem::TestCase ALTERNATE_CERT = load_cert 'alternate' + EXPIRED_PUBLIC_CERT = load_cert 'expired' ALTERNATE_KEY_FILE = key_path 'alternate' PRIVATE_KEY_FILE = key_path 'private' PUBLIC_KEY_FILE = key_path 'public' - ALTERNATE_CERT_FILE = cert_path 'alternate' - CHILD_CERT_FILE = cert_path 'child' - PUBLIC_CERT_FILE = cert_path 'public' + ALTERNATE_CERT_FILE = cert_path 'alternate' + CHILD_CERT_FILE = cert_path 'child' + PUBLIC_CERT_FILE = cert_path 'public' + EXPIRED_PUBLIC_CERT_FILE = cert_path 'expired' def setup super @@ -582,6 +584,37 @@ ERROR: --private-key not specified and https://github.com/ruby/ruby/blob/trunk/test/rubygems/test_gem_commands_cert_command.rb#L584 assert_equal expected, @ui.error end + def test_execute_re_sign + gem_path = File. (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/