ruby-changes:31975
From: drbrain <ko1@a...>
Date: Sun, 8 Dec 2013 10:22:52 +0900 (JST)
Subject: [ruby-changes:31975] drbrain:r44054 (trunk): * lib/rubygems: Update to RubyGems master 14749ce. This fixes bugs
drbrain 2013-12-08 10:22:39 +0900 (Sun, 08 Dec 2013) New Revision: 44054 http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=rev&revision=44054 Log: * lib/rubygems: Update to RubyGems master 14749ce. This fixes bugs handling of gem dependencies lockfiles (Gemfile.lock). * test/rubygems: ditto. Added files: trunk/lib/rubygems/resolver/lock_specification.rb trunk/lib/rubygems/resolver/stats.rb trunk/test/rubygems/test_gem_resolver_lock_specification.rb Modified files: trunk/ChangeLog trunk/lib/rubygems/request_set/gem_dependency_api.rb trunk/lib/rubygems/request_set/lockfile.rb trunk/lib/rubygems/request_set.rb trunk/lib/rubygems/requirement.rb trunk/lib/rubygems/resolver/git_set.rb trunk/lib/rubygems/resolver/lock_set.rb trunk/lib/rubygems/resolver/requirement_list.rb trunk/lib/rubygems/resolver.rb trunk/lib/rubygems/source/git.rb trunk/lib/rubygems/source/lock.rb trunk/lib/rubygems/source.rb trunk/lib/rubygems/test_case.rb trunk/lib/rubygems/test_utilities.rb trunk/test/rubygems/test_gem_request_set.rb trunk/test/rubygems/test_gem_request_set_gem_dependency_api.rb trunk/test/rubygems/test_gem_request_set_lockfile.rb trunk/test/rubygems/test_gem_requirement.rb trunk/test/rubygems/test_gem_resolver.rb trunk/test/rubygems/test_gem_resolver_git_set.rb trunk/test/rubygems/test_gem_resolver_git_specification.rb trunk/test/rubygems/test_gem_resolver_lock_set.rb trunk/test/rubygems/test_gem_resolver_requirement_list.rb trunk/test/rubygems/test_gem_source_lock.rb Index: ChangeLog =================================================================== --- ChangeLog (revision 44053) +++ ChangeLog (revision 44054) @@ -1,3 +1,10 @@ https://github.com/ruby/ruby/blob/trunk/ChangeLog#L1 +Sun Dec 8 10:21:36 2013 Eric Hodel <drbrain@s...> + + * lib/rubygems: Update to RubyGems master 14749ce. This fixes bugs + handling of gem dependencies lockfiles (Gemfile.lock). + + * test/rubygems: ditto. + Sun Dec 8 09:40:00 2013 Charlie Somerville <charliesome@r...> * array.c (rb_ary_or): use RHASH_TBL_RAW instead of RHASH_TBL Index: lib/rubygems/request_set.rb =================================================================== --- lib/rubygems/request_set.rb (revision 44053) +++ lib/rubygems/request_set.rb (revision 44054) @@ -158,6 +158,10 @@ class Gem::RequestSet https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set.rb#L158 specs.map { |s| s.full_name }.sort.each do |s| puts " #{s}" end + + if Gem.configuration.really_verbose + @resolver.stats.display + end else installed = install options, &block @@ -169,6 +173,8 @@ class Gem::RequestSet https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set.rb#L173 end def install_into dir, force = true, options = {} + gem_home, ENV['GEM_HOME'] = ENV['GEM_HOME'], dir + existing = force ? [] : specs_in(dir) existing.delete_if { |s| @always_install.include? s } @@ -195,6 +201,8 @@ class Gem::RequestSet https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set.rb#L201 end installed + ensure + ENV['GEM_HOME'] = gem_home end ## @@ -229,6 +237,8 @@ class Gem::RequestSet https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set.rb#L237 resolver.development = @development resolver.soft_missing = @soft_missing + @resolver = resolver + @requests = resolver.resolve end Index: lib/rubygems/source/git.rb =================================================================== --- lib/rubygems/source/git.rb (revision 44053) +++ lib/rubygems/source/git.rb (revision 44054) @@ -147,6 +147,16 @@ class Gem::Source::Git < Gem::Source https://github.com/ruby/ruby/blob/trunk/lib/rubygems/source/git.rb#L147 File.join base_dir, 'gems', "#{@name}-#{dir_shortref}" end + def pretty_print q # :nodoc: + q.group 2, '[Git: ', ']' do + q.breakable + q.text @repository + + q.breakable + q.text @reference + end + end + ## # The directory where the git gem's repository will be cached. Index: lib/rubygems/source/lock.rb =================================================================== --- lib/rubygems/source/lock.rb (revision 44053) +++ lib/rubygems/source/lock.rb (revision 44054) @@ -40,5 +40,9 @@ class Gem::Source::Lock < Gem::Source https://github.com/ruby/ruby/blob/trunk/lib/rubygems/source/lock.rb#L40 @wrapped.fetch_spec name_tuple end + def uri # :nodoc: + @wrapped.uri + end + end Index: lib/rubygems/requirement.rb =================================================================== --- lib/rubygems/requirement.rb (revision 44053) +++ lib/rubygems/requirement.rb (revision 44054) @@ -133,7 +133,15 @@ class Gem::Requirement https://github.com/ruby/ruby/blob/trunk/lib/rubygems/requirement.rb#L133 # Formats this requirement for use in a Gem::RequestSet::Lockfile. def for_lockfile # :nodoc: - " (#{to_s})" unless [DefaultRequirement] == @requirements + return if [DefaultRequirement] == @requirements + + list = requirements.sort_by { |_, version| + version + }.map { |op, version| + "#{op} #{version}" + }.uniq + + " (#{list.join ', '})" end ## @@ -147,6 +155,14 @@ class Gem::Requirement https://github.com/ruby/ruby/blob/trunk/lib/rubygems/requirement.rb#L155 end end + ## + # true if the requirement is for only an exact version + + def exact? + return false unless @requirements.size == 1 + @requirements[0][0] == "=" + end + def as_list # :nodoc: requirements.map { |op, version| "#{op} #{version}" }.sort end Index: lib/rubygems/source.rb =================================================================== --- lib/rubygems/source.rb (revision 44053) +++ lib/rubygems/source.rb (revision 44054) @@ -202,7 +202,10 @@ class Gem::Source https://github.com/ruby/ruby/blob/trunk/lib/rubygems/source.rb#L202 q.group 2, '[Remote:', ']' do q.breakable q.text @uri.to_s + if api = api_uri + q.breakable + q.text 'API URI: ' q.text api.to_s end end Index: lib/rubygems/request_set/gem_dependency_api.rb =================================================================== --- lib/rubygems/request_set/gem_dependency_api.rb (revision 44053) +++ lib/rubygems/request_set/gem_dependency_api.rb (revision 44054) @@ -221,13 +221,7 @@ class Gem::RequestSet::GemDependencyAPI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/gem_dependency_api.rb#L221 return unless (groups & @without_groups).empty? - unless source_set then - raise ArgumentError, - "duplicate source (default) for gem #{name}" if - @gem_sources.include? name - - @gem_sources[name] = :default - end + pin_gem_source name, :default unless source_set gem_requires name, options @@ -246,9 +240,7 @@ class Gem::RequestSet::GemDependencyAPI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/gem_dependency_api.rb#L240 return unless repository = options.delete(:git) - raise ArgumentError, - "duplicate source git: #{repository} for gem #{name}" if - @gem_sources.include? name + pin_gem_source name, :git, repository reference = nil reference ||= options.delete :ref @@ -260,8 +252,6 @@ class Gem::RequestSet::GemDependencyAPI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/gem_dependency_api.rb#L252 @git_set.add_git_gem name, repository, reference, submodules - @gem_sources[name] = repository - true end @@ -310,14 +300,10 @@ class Gem::RequestSet::GemDependencyAPI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/gem_dependency_api.rb#L300 def gem_path name, options # :nodoc: return unless directory = options.delete(:path) - raise ArgumentError, - "duplicate source path: #{directory} for gem #{name}" if - @gem_sources.include? name + pin_gem_source name, :path, directory @vendor_set.add_vendor_gem name, directory - @gem_sources[name] = directory - true end @@ -430,6 +416,28 @@ class Gem::RequestSet::GemDependencyAPI https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/gem_dependency_api.rb#L416 end ## + # Pins the gem +name+ to the given +source+. Adding a gem with the same + # name from a different +source+ will raise an exception. + + def pin_gem_source name, type = :default, source = nil + source_description = + case type + when :default then '(default)' + when :path then "path: #{source}" + when :git then "git: #{source}" + else '(unknown)' + end + + raise ArgumentError, + "duplicate source #{source_description} for gem #{name}" if + @gem_sources.fetch(name, source) != source + + @gem_sources[name] = source + end + + private :pin_gem_source + + ## # :category: Gem Dependencies DSL # # Block form for restricting gems to a particular platform. Index: lib/rubygems/request_set/lockfile.rb =================================================================== --- lib/rubygems/request_set/lockfile.rb (revision 44053) +++ lib/rubygems/request_set/lockfile.rb (revision 44054) @@ -1,3 +1,5 @@ https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L1 +require 'strscan' + ## # Parses a gem.deps.rb.lock file and constructs a LockSet containing the # dependencies found inside. If the lock file is missing no LockSet is @@ -29,11 +31,11 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L31 # Raises a ParseError with the given +message+ which was encountered at a # +line+ and +column+ while parsing. - def initialize message, line, column, path + def initialize message, column, line, path @line = line @column = column @path = path - super "#{message} (at #{line}:#{column})" + super "#{message} (at line #{line} column #{column})" end end @@ -62,30 +64,31 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L64 def add_DEPENDENCIES out # :nodoc: out << "DEPENDENCIES" - @set.dependencies.sort.map do |dependency| - source = @requests.find do |req| - req.name == dependency.name and - req.spec.class == Gem::Resolver::VendorSpecification - end - - source_dep = '!' if source + @requests.sort_by { |r| r.name }.each do |request| + spec = request.spec - requirement = dependency.requirement + if [Gem::Resolver::VendorSpecification, + Gem::Resolver::GitSpecification].include? spec.class then + out << " #{request.name}!" + else + requirement = request.request.dependency.requirement - out << " #{dependency.name}#{source_dep}#{requirement.for_lockfile}" + out << " #{request.name}#{requirement.for_lockfile}" + end end out << nil end def add_GEM out # :nodoc: - out << "GEM" + return if @spec_groups.empty? source_groups = @spec_groups.values.flatten.group_by do |request| request.spec.source.uri end - source_groups.map do |group, requests| + source_groups.sort_by { |group,| group.to_s }.map do |group, requests| + out << "GEM" out << " remote: #{group}" out << " specs:" @@ -100,6 +103,33 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L103 out << " #{dependency.name}#{requirement.for_lockfile}" end end + out << nil + end + end + + def add_GIT out + return unless git_requests = + @spec_groups.delete(Gem::Resolver::GitSpecification) + + by_repository_revision = git_requests.group_by do |request| + source = request.spec.source + [source.repository, source.rev_parse] + end + + out << "GIT" + by_repository_revision.each do |(repository, revision), requests| + out << " remote: #{repository}" + out << " revision: #{revision}" + out << " specs:" + + requests.sort_by { |request| request.name }.each do |request| + out << " #{request.name} (#{request.version})" + + dependencies = request.spec.dependencies.sort_by { |dep| dep.name } + dependencies.each do |dep| + out << " #{dep.name}#{dep.requirement.for_lockfile}" + end + end end out << nil @@ -148,27 +178,28 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L178 ## # Gets the next token for a Lockfile - def get expected_type = nil, expected_value = nil # :nodoc: + def get expected_types = nil, expected_value = nil # :nodoc: @current_token = @tokens.shift - type, value, line, column = @current_token + type, value, column, line = @current_token - if expected_type and expected_type != type then + if expected_types and not Array(expected_types).include? type then unget message = "unexpected token [#{type.inspect}, #{value.inspect}], " + - "expected #{expected_type.inspect}" + "expected #{expected_types.inspect}" - raise ParseError.new message, line, column, "#{@gem_deps_file}.lock" + raise ParseError.new message, column, line, "#{@gem_deps_file}.lock" end if expected_value and expected_value != value then unget message = "unexpected token [#{type.inspect}, #{value.inspect}], " + - "expected [#{expected_type.inspect}, #{expected_value.inspect}]" + "expected [#{expected_types.inspect}, " + + "#{expected_value.inspect}]" - raise ParseError.new message, line, column, "#{@gem_deps_file}.lock" + raise ParseError.new message, column, line, "#{@gem_deps_file}.lock" end @current_token @@ -187,6 +218,8 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L218 case data when 'DEPENDENCIES' then parse_DEPENDENCIES + when 'GIT' then + parse_GIT when 'GEM' then parse_GEM when 'PLATFORMS' then @@ -195,7 +228,7 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L228 type, = get until @tokens.empty? or peek.first == :section end else - raise "BUG: unhandled token #{type} (#{data.inspect}) at #{line}:#{column}" + raise "BUG: unhandled token #{type} (#{data.inspect}) at line #{line} column #{column}" end end end @@ -204,7 +237,37 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L237 while not @tokens.empty? and :text == peek.first do _, name, = get :text - @set.gem name + requirements = [] + + case peek[0] + when :bang then + get :bang + + git_spec = @set.sets.select { |set| + Gem::Resolver::GitSet === set + }.map { |set| + set.specs[name] + }.first + + requirements << git_spec.version + when :l_paren then + get :l_paren + + loop do + _, op, = get :requirement + _, version, = get :text + + requirements << "#{op} #{version}" + + break unless peek[0] == :comma + + get :comma + end + + get :r_paren + end + + @set.gem name, *requirements skip :newline end @@ -223,20 +286,76 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L286 skip :newline set = Gem::Resolver::LockSet.new source + last_spec = nil while not @tokens.empty? and :text == peek.first do - _, name, = get :text + _, name, column, = get :text case peek[0] - when :newline then # ignore + when :newline then + last_spec.add_dependency Gem::Dependency.new name if column == 6 when :l_paren then get :l_paren - _, version, = get :text + type, data, = get [:text, :requirement] + + if type == :text and column == 4 then + last_spec = set.add name, data, Gem::Platform::RUBY + else + dependency = parse_dependency name, data + + last_spec.add_dependency dependency + end get :r_paren + else + raise "BUG: unknown token #{peek}" + end - set.add name, version, Gem::Platform::RUBY + skip :newline + end + + @set.sets << set + end + + def parse_GIT # :nodoc: + get :entry, 'remote' + _, repository, = get :text + + skip :newline + + get :entry, 'revision' + _, revision, = get :text + + skip :newline + + get :entry, 'specs' + + skip :newline + + set = Gem::Resolver::GitSet.new + last_spec = nil + + while not @tokens.empty? and :text == peek.first do + _, name, column, = get :text + + case peek[0] + when :newline then + last_spec.add_dependency Gem::Dependency.new name if column == 6 + when :l_paren then + get :l_paren + + type, data, = get [:text, :requirement] + + if type == :text and column == 4 then + last_spec = set.add_git_spec name, data, repository, revision, true + else + dependency = parse_dependency name, data + + last_spec.spec.dependencies << dependency + end + + get :r_paren else raise "BUG: unknown token #{peek}" end @@ -258,10 +377,32 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L377 end ## + # Parses the requirements following the dependency +name+ and the +op+ for + # the first token of the requirements and returns a Gem::Dependency object. + + def parse_dependency name, op # :nodoc: + return Gem::Dependency.new name unless peek[0] == :text + + _, version, = get :text + + requirements = ["#{op} #{version}"] + + while peek[0] == :comma do + get :comma + _, op, = get :requirement + _, version, = get :text + + requirements << "#{op} #{version}" + end + + Gem::Dependency.new name, requirements + end + + ## # Peeks at the next token for Lockfile def peek # :nodoc: - @tokens.first + @tokens.first || :EOF end def skip type # :nodoc: @@ -284,6 +425,8 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L425 add_PATH out + add_GIT out + add_GEM out add_PLATFORMS out @@ -321,14 +464,13 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L464 until s.eos? do pos = s.pos - # leading whitespace is for the user's convenience - next if s.scan(/ +/) + pos = s.pos if leading_whitespace = s.scan(/ +/) if s.scan(/[<|=>]{7}/) then message = "your #{lock_file} contains merge conflict markers" - line, column = token_pos pos + column, line = token_pos pos - raise ParseError.new message, line, column, lock_file + raise ParseError.new message, column, line, lock_file end @tokens << @@ -339,7 +481,13 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L481 @line += 1 token when s.scan(/[A-Z]+/) then - [:section, s.matched, *token_pos(pos)] + if leading_whitespace then + text = s.matched + text += s.scan(/[^\s)]*/).to_s # in case of no match + [:text, text, *token_pos(pos)] + else + [:section, s.matched, *token_pos(pos)] + end when s.scan(/([a-z]+):\s/) then s.pos -= 1 # rewind for possible newline [:entry, s[1], *token_pos(pos)] @@ -347,7 +495,13 @@ class Gem::RequestSet::Lockfile https://github.com/ruby/ruby/blob/trunk/lib/rubygems/request_set/lockfile.rb#L495 [:l_paren, nil, *token_pos(pos)] when s.scan(/\)/) then [:r_paren, nil, *token_pos(pos)] - when s.scan(/[^\s)]*/) then + when s.scan(/<=|>=|=|~>|<|>|!=/) then + [:requirement, s.matched, *token_pos(pos)] + when s.scan(/,/) then + [:comma, nil, *token_pos(pos)] + when s.scan(/!/) then + [:bang, nil, *token_pos(pos)] + when s.scan(/[^\s),!]*/) then [:text, s.matched, *token_pos(pos)] else raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}" Index: lib/rubygems/resolver/lock_specification.rb =================================================================== --- lib/rubygems/resolver/lock_specification.rb (revision 0) +++ lib/rubygems/resolver/lock_specification.rb (revision 44054) @@ -0,0 +1,58 @@ https://github.com/ruby/ruby/blob/trunk/lib/rubygems/resolver/lock_specification.rb#L1 +## +# The LockSpecification comes from a lockfile (Gem::RequestSet::Lockfile). +# +# A LockSpecification's dependency information is pre-filled from the +# lockfile. + +class Gem::Resolver::LockSpecification < Gem::Resolver::Specification + + def initialize set, name, version, (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/