ruby-changes:54298
From: kou <ko1@a...>
Date: Sun, 23 Dec 2018 16:00:42 +0900 (JST)
Subject: [ruby-changes:54298] kou:r66507 (trunk): Import CSV 3.0.2
kou 2018-12-23 16:00:35 +0900 (Sun, 23 Dec 2018) New Revision: 66507 https://svn.ruby-lang.org/cgi-bin/viewvc.cgi?view=revision&revision=66507 Log: Import CSV 3.0.2 This includes performance improvement especially writing. Writing is about 2 times faster. Added files: trunk/lib/csv/fields_converter.rb trunk/lib/csv/match_p.rb trunk/lib/csv/parser.rb trunk/lib/csv/writer.rb Modified files: trunk/NEWS trunk/lib/csv/csv.gemspec trunk/lib/csv/table.rb trunk/lib/csv/version.rb trunk/lib/csv.rb trunk/test/csv/test_csv_parsing.rb trunk/test/csv/test_data_converters.rb trunk/test/csv/test_features.rb trunk/test/csv/test_interface.rb trunk/test/csv/test_table.rb Index: lib/csv.rb =================================================================== --- lib/csv.rb (revision 66506) +++ lib/csv.rb (revision 66507) @@ -93,36 +93,22 @@ require "forwardable" https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L93 require "English" require "date" require "stringio" -require_relative "csv/table" -require_relative "csv/row" -# This provides String#match? and Regexp#match? for Ruby 2.3. -unless String.method_defined?(:match?) - class CSV - module MatchP - refine String do - def match?(pattern) - self =~ pattern - end - end - - refine Regexp do - def match?(string) - self =~ string - end - end - end - end +require_relative "csv/fields_converter" +require_relative "csv/match_p" +require_relative "csv/parser" +require_relative "csv/row" +require_relative "csv/table" +require_relative "csv/writer" - using CSV::MatchP -end +using CSV::MatchP if CSV.const_defined?(:MatchP) # # This class provides a complete interface to CSV files and data. It offers # tools to enable you to read and write to and from Strings or IO objects, as # needed. # -# The most generic interface of a class is: +# The most generic interface of the library is: # # csv = CSV.new(string_or_io, **options) # @@ -204,18 +190,18 @@ end https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L190 # # Headers are part of data # data = CSV.parse(<<~ROWS, headers: true) # Name,Department,Salary -# Bob,Engeneering,1000 +# Bob,Engineering,1000 # Jane,Sales,2000 # John,Management,5000 # ROWS # # data.class #=> CSV::Table -# data.first #=> #<CSV::Row "Name":"Bob" "Department":"Engeneering" "Salary":"1000"> -# data.first.to_h #=> {"Name"=>"Bob", "Department"=>"Engeneering", "Salary"=>"1000"} +# data.first #=> #<CSV::Row "Name":"Bob" "Department":"Engineering" "Salary":"1000"> +# data.first.to_h #=> {"Name"=>"Bob", "Department"=>"Engineering", "Salary"=>"1000"} # # # Headers provided by developer # data = CSV.parse('Bob,Engeneering,1000', headers: %i[name department salary]) -# data.first #=> #<CSV::Row name:"Bob" department:"Engeneering" salary:"1000"> +# data.first #=> #<CSV::Row name:"Bob" department:"Engineering" salary:"1000"> # # === Typed data reading # @@ -902,10 +888,24 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L888 # Options cannot be overridden in the instance methods for performance reasons, # so be sure to set what you want here. # - def initialize(data, col_sep: ",", row_sep: :auto, quote_char: '"', field_size_limit: nil, - converters: nil, unconverted_fields: nil, headers: false, return_headers: false, - write_headers: nil, header_converters: nil, skip_blanks: false, force_quotes: false, - skip_lines: nil, liberal_parsing: false, internal_encoding: nil, external_encoding: nil, encoding: nil, + def initialize(data, + col_sep: ",", + row_sep: :auto, + quote_char: '"', + field_size_limit: nil, + converters: nil, + unconverted_fields: nil, + headers: false, + return_headers: false, + write_headers: nil, + header_converters: nil, + skip_blanks: false, + force_quotes: false, + skip_lines: nil, + liberal_parsing: false, + internal_encoding: nil, + external_encoding: nil, + encoding: nil, nil_value: nil, empty_value: "") raise ArgumentError.new("Cannot parse nil as CSV") if data.nil? @@ -913,64 +913,79 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L913 # create the IO object we will read from @io = data.is_a?(String) ? StringIO.new(data) : data @encoding = determine_encoding(encoding, internal_encoding) - # - # prepare for building safe regular expressions in the target encoding, - # if we can transcode the needed characters - # - @re_esc = "\\".encode(@encoding).freeze rescue "" - @re_chars = /#{%"[-\\]\\[\\.^$?*+{}()|# \r\n\t\f\v]".encode(@encoding)}/ - @unconverted_fields = unconverted_fields - - # Stores header row settings and loads header converters, if needed. - @use_headers = headers - @return_headers = return_headers - @write_headers = write_headers - - # headers must be delayed until shift(), in case they need a row of content - @headers = nil - - @nil_value = nil_value - @empty_value = empty_value - @empty_value_is_empty_string = (empty_value == "") - - init_separators(col_sep, row_sep, quote_char, force_quotes) - init_parsers(skip_blanks, field_size_limit, liberal_parsing) - init_converters(converters, :@converters, :convert) - init_converters(header_converters, :@header_converters, :header_convert) - init_comments(skip_lines) - - @force_encoding = !!encoding - - # track our own lineno since IO gets confused about line-ends is CSV fields - @lineno = 0 - - # make sure headers have been assigned - if header_row? and [Array, String].include? @use_headers.class and @write_headers - parse_headers # won't read data for Array or String - self << @headers - end + + @base_fields_converter_options = { + nil_value: nil_value, + empty_value: empty_value, + } + @initial_converters = converters + @initial_header_converters = header_converters + + @parser_options = { + column_separator: col_sep, + row_separator: row_sep, + quote_character: quote_char, + field_size_limit: field_size_limit, + unconverted_fields: unconverted_fields, + headers: headers, + return_headers: return_headers, + skip_blanks: skip_blanks, + skip_lines: skip_lines, + liberal_parsing: liberal_parsing, + encoding: @encoding, + nil_value: nil_value, + empty_value: empty_value, + } + @parser = nil + + @writer_options = { + encoding: @encoding, + force_encoding: (not encoding.nil?), + force_quotes: force_quotes, + headers: headers, + write_headers: write_headers, + column_separator: col_sep, + row_separator: row_sep, + quote_character: quote_char, + } + + @writer = nil + writer if @writer_options[:write_headers] end # # The encoded <tt>:col_sep</tt> used in parsing and writing. See CSV::new # for details. # - attr_reader :col_sep + def col_sep + parser.column_separator + end + # # The encoded <tt>:row_sep</tt> used in parsing and writing. See CSV::new # for details. # - attr_reader :row_sep + def row_sep + parser.row_separator + end + # # The encoded <tt>:quote_char</tt> used in parsing and writing. See CSV::new # for details. # - attr_reader :quote_char + def quote_char + parser.quote_character + end + # The limit for field size, if any. See CSV::new for details. - attr_reader :field_size_limit + def field_size_limit + parser.field_size_limit + end # The regex marking a line as a comment. See CSV::new for details - attr_reader :skip_lines + def skip_lines + parser.skip_lines + end # # Returns the current list of converters in effect. See CSV::new for details. @@ -978,7 +993,7 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L993 # as is. # def converters - @converters.map do |converter| + fields_converter.map do |converter| name = Converters.rassoc(converter) name ? name.first : converter end @@ -987,42 +1002,68 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1002 # Returns +true+ if unconverted_fields() to parsed results. See CSV::new # for details. # - def unconverted_fields?() @unconverted_fields end + def unconverted_fields? + parser.unconverted_fields? + end + # # Returns +nil+ if headers will not be used, +true+ if they will but have not # yet been read, or the actual headers after they have been read. See # CSV::new for details. # def headers - @headers || true if @use_headers + if @writer + @writer.headers + else + parsed_headers = parser.headers + return parsed_headers if parsed_headers + raw_headers = @parser_options[:headers] + raw_headers = nil if raw_headers == false + raw_headers + end end # # Returns +true+ if headers will be returned as a row of results. # See CSV::new for details. # - def return_headers?() @return_headers end + def return_headers? + parser.return_headers? + end + # Returns +true+ if headers are written in output. See CSV::new for details. - def write_headers?() @write_headers end + def write_headers? + @writer_options[:write_headers] + end + # # Returns the current list of converters in effect for headers. See CSV::new # for details. Built-in converters will be returned by name, while others # will be returned as is. # def header_converters - @header_converters.map do |converter| + header_fields_converter.map do |converter| name = HeaderConverters.rassoc(converter) name ? name.first : converter end end + # # Returns +true+ blank lines are skipped by the parser. See CSV::new # for details. # - def skip_blanks?() @skip_blanks end + def skip_blanks? + parser.skip_blanks? + end + # Returns +true+ if all output fields are quoted. See CSV::new for details. - def force_quotes?() @force_quotes end + def force_quotes? + @writer_options[:force_quotes] + end + # Returns +true+ if illegal input is handled. See CSV::new for details. - def liberal_parsing?() @liberal_parsing end + def liberal_parsing? + parser.liberal_parsing? + end # # The Encoding CSV is parsing or writing in. This will be the Encoding you @@ -1031,10 +1072,23 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1072 attr_reader :encoding # - # The line number of the last row read from this file. Fields with nested + # The line number of the last row read from this file. Fields with nested # line-end characters will not affect this count. # - attr_reader :lineno, :line + def lineno + if @writer + @writer.lineno + else + parser.lineno + end + end + + # + # The last row read from this file. + # + def line + parser.line + end ### IO and StringIO Delegation ### @@ -1048,9 +1102,9 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1102 # Rewinds the underlying IO object and resets CSV's lineno() counter. def rewind - @headers = nil - @lineno = 0 - + @parser = nil + @parser_enumerator = nil + @writer.rewind if @writer @io.rewind end @@ -1064,34 +1118,8 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1118 # The data source must be open for writing. # def <<(row) - # make sure headers have been assigned - if header_row? and [Array, String].include? @use_headers.class and !@write_headers - parse_headers # won't read data for Array or String - end - - # handle CSV::Row objects and Hashes - row = case row - when self.class::Row then row.fields - when Hash then @headers.map { |header| row[header] } - else row - end - - @headers = row if header_row? - @lineno += 1 - - output = row.map(&@quote).join(@col_sep) + @row_sep # quote and separate - if @io.is_a?(StringIO) and - output.encoding != (encoding = raw_encoding) - if @force_encoding - output = output.encode(encoding) - elsif (compatible_encoding = Encoding.compatible?(@io.string, output)) - @io.set_encoding(compatible_encoding) - @io.seek(0, IO::SEEK_END) - end - end - @io << output - - self # for chaining + writer << row + self end alias_method :add_row, :<< alias_method :puts, :<< @@ -1112,7 +1140,7 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1140 # converted field or the field itself. # def convert(name = nil, &converter) - add_converter(:@converters, self.class::Converters, name, &converter) + fields_converter.add_converter(name, &converter) end # @@ -1127,10 +1155,7 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1155 # effect. # def header_convert(name = nil, &converter) - add_converter( :@header_converters, - self.class::HeaderConverters, - name, - &converter ) + header_fields_converter.add_converter(name, &converter) end include Enumerable @@ -1142,14 +1167,8 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1167 # # The data source must be open for reading. # - def each - if block_given? - while row = shift - yield row - end - else - to_enum - end + def each(&block) + parser.parse(&block) end # @@ -1159,8 +1178,9 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1178 # def read rows = to_a - if @use_headers - Table.new(rows) + headers = parser.headers + if headers + Table.new(rows, headers: headers) else rows end @@ -1169,7 +1189,7 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1189 # Returns +true+ if the next row read will be a header row. def header_row? - @use_headers and @headers.nil? + parser.header_row? end # @@ -1180,171 +1200,11 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1200 # The data source must be open for reading. # def shift - ######################################################################### - ### This method is purposefully kept a bit long as simple conditional ### - ### checks are faster than numerous (expensive) method calls. ### - ######################################################################### - - # handle headers not based on document content - if header_row? and @return_headers and - [Array, String].include? @use_headers.class - if @unconverted_fields - return add_unconverted_fields(parse_headers, Array.new) - else - return parse_headers - end - end - - # - # it can take multiple calls to <tt>@io.gets()</tt> to get a full line, - # because of \r and/or \n characters embedded in quoted fields - # - in_extended_col = false - csv = Array.new - - loop do - # add another read to the line - unless parse = @io.gets(@row_sep) - return nil - end - - if in_extended_col - @line.concat(parse) - else - @line = parse.clone - end - - begin - parse.sub!(@parsers[:line_end], "") - rescue ArgumentError - unless parse.valid_encoding? - message = "Invalid byte sequence in #{parse.encoding}" - raise MalformedCSVError.new(message, lineno + 1) - end - raise - end - - if csv.empty? - # - # I believe a blank line should be an <tt>Array.new</tt>, not Ruby 1.8 - # CSV's <tt>[nil]</tt> - # - if parse.empty? - @lineno += 1 - if @skip_blanks - next - elsif @unconverted_fields - return add_unconverted_fields(Array.new, Array.new) - elsif @use_headers - return self.class::Row.new(@headers, Array.new) - else - return Array.new - end - end - end - - next if @skip_lines and @skip_lines.match parse - - parts = parse.split(@col_sep_split_separator, -1) - if parts.empty? - if in_extended_col - csv[-1] << @col_sep # will be replaced with a @row_sep after the parts.each loop - else - csv << nil - end - end - - # This loop is the hot path of csv parsing. Some things may be non-dry - # for a reason. Make sure to benchmark when refactoring. - parts.each do |part| - if in_extended_col - # If we are continuing a previous column - if part.end_with?(@quote_char) && part.count(@quote_char) % 2 != 0 - # extended column ends - csv.last << part[0..-2] - if csv.last.match?(@parsers[:stray_quote]) - raise MalformedCSVError.new("Missing or stray quote", - lineno + 1) - end - csv.last.gsub!(@double_quote_char, @quote_char) - in_extended_col = false - else - csv.last << part << @col_sep - end - elsif part.start_with?(@quote_char) - # If we are starting a new quoted column - if part.count(@quote_char) % 2 != 0 - # start an extended column - csv << (part[1..-1] << @col_sep) - in_extended_col = true - elsif part.end_with?(@quote_char) - # regular quoted column - csv << part[1..-2] - if csv.last.match?(@parsers[:stray_quote]) - raise MalformedCSVError.new("Missing or stray quote", - lineno + 1) - end - csv.last.gsub!(@double_quote_char, @quote_char) - elsif @liberal_parsing - csv << part - else - raise MalformedCSVError.new("Missing or stray quote", - lineno + 1) - end - elsif part.match?(@parsers[:quote_or_nl]) - # Unquoted field with bad characters. - if part.match?(@parsers[:nl_or_lf]) - message = "Unquoted fields do not allow \\r or \\n" - raise MalformedCSVError.new(message, lineno + 1) - else - if @liberal_parsing - csv << part - else - raise MalformedCSVError.new("Illegal quoting", lineno + 1) - end - end - else - # Regular ole unquoted field. - csv << (part.empty? ? nil : part) - end - end - - # Replace tacked on @col_sep with @row_sep if we are still in an extended - # column. - csv[-1][-1] = @row_sep if in_extended_col - - if in_extended_col - # if we're at eof?(), a quoted field wasn't closed... - if @io.eof? - raise MalformedCSVError.new("Unclosed quoted field", - lineno + 1) - elsif @field_size_limit and csv.last.size >= @field_size_limit - raise MalformedCSVError.new("Field size exceeded", - lineno + 1) - end - # otherwise, we need to loop and pull some more data to complete the row - else - @lineno += 1 - - # save fields unconverted fields, if needed... - unconverted = csv.dup if @unconverted_fields - - if @use_headers - # parse out header rows and handle CSV::Row conversions... - csv = parse_headers(csv) - else - # convert fields, if needed... - csv = convert_fields(csv) - end - - # inject unconverted fields and accessor, if requested... - if @unconverted_fields and not csv.respond_to? :unconverted_fields - add_unconverted_fields(csv, unconverted) - end - - # return the results - break csv - end + @parser_enumerator ||= parser.parse + begin + @parser_enumerator.next + rescue StopIteration + nil end end alias_method :gets, :shift @@ -1369,15 +1229,19 @@ class CSV https://github.com/ruby/ruby/blob/trunk/lib/csv.rb#L1229 # s (... truncated) -- ML: ruby-changes@q... Info: http://www.atdot.net/~ko1/quickml/