34718.patch

Go MAEDA, 2022-01-02 09:09

Download (4.27 KB)

View differences:

app/models/import.rb
65 65

  
66 66
  def set_default_settings(options={})
67 67
    separator = lu(user, :general_csv_separator)
68
    encoding = lu(user, :general_csv_encoding)
68 69
    if file_exists?
69 70
      begin
70 71
        content = File.read(filepath, 256)
72

  
71 73
        separator = [',', ';'].sort_by {|sep| content.count(sep)}.last
74

  
75
        guessed_encoding = Redmine::CodesetUtil.guess_encoding(file_content)
76
        encoding =
77
          (guessed_encoding && (
78
            Setting::ENCODINGS.detect {|e| e.casecmp?(guessed_encoding)} ||
79
            Setting::ENCODINGS.detect {|e| Encoding.find(e) == Encoding.find(guessed_encoding)}
80
          )) || lu(user, :general_csv_encoding)
72 81
      rescue => e
73 82
      end
74 83
    end
75 84
    wrapper = '"'
76
    encoding = lu(user, :general_csv_encoding)
77 85

  
78 86
    date_format = lu(user, "date.formats.default", :default => "foo")
79 87
    date_format = DATE_FORMATS.first unless DATE_FORMATS.include?(date_format)
lib/redmine/codeset_util.rb
75 75
        str = self.replace_invalid_utf8(str)
76 76
      end
77 77
    end
78

  
79
    def self.guess_encoding(str)
80
      return if str.nil?
81

  
82
      str = str.dup
83
      encodings = Setting.repositories_encodings.split(',').collect(&:strip)
84
      encodings = encodings.presence || ['UTF-8']
85

  
86
      encodings.each do |encoding|
87
        begin
88
          str.force_encoding(encoding)
89
        rescue Encoding::ConverterNotFoundError
90
          # ignore if the encoding name is invalid
91
        end
92
        return encoding if str.valid_encoding?
93
      end
94
      nil
95
    end
78 96
  end
79 97
end
test/unit/issue_import_test.rb
411 411

  
412 412
    assert_empty import.mapping
413 413
  end
414

  
415
  def test_set_default_settings_should_guess_encoding
416
    import = generate_import('import_iso8859-1.csv')
417
    with_settings :repositories_encodings => 'UTF-8,ISO-8859-1' do
418
      import.set_default_settings
419
      guessed_encoding = import.settings['encoding']
420
      assert_equal 'ISO-8859-1', guessed_encoding
421
    end
422
    with_settings :repositories_encodings => 'UTF-8,iso8859-1' do
423
      import.set_default_settings
424
      guessed_encoding = import.settings['encoding']
425
      assert_equal 'ISO-8859-1', guessed_encoding
426
      assert_includes Setting::ENCODINGS, guessed_encoding
427
    end
428
  end
429

  
430
  def test_set_default_settings_should_use_general_csv_encoding_when_cannnot_guess_encoding
431
    import = generate_import('import_iso8859-1.csv')
432
    user = User.generate!(:language => 'ja')
433
    import.user = user
434
    with_settings :repositories_encodings => 'UTF-8' do
435
      import.set_default_settings
436
      guessed_encoding = import.settings['encoding']
437
      assert_equal 'CP932', lu(user, :general_csv_encoding)
438
      assert_equal 'CP932', guessed_encoding
439
    end
440
  end
414 441
end
test/unit/lib/redmine/codeset_util_test.rb
101 101
    assert_equal "UTF-8", s2.encoding.to_s
102 102
    assert_equal 'こんにち?', s2
103 103
  end
104

  
105
  test_guess_encoding_should_return_guessed_encoding do
106
    str = '日本語'.encode('Windows-31J').b
107
    with_settings :repositories_encodings => 'UTF-8,Windows-31J' do
108
      assert_equal 'Windows-31J', Redmine::CodesetUtil.guess_encoding(str)
109
    end
110
    with_settings :repositories_encodings => 'UTF-8,csWindows31J' do
111
      assert_equal 'csWindows31J', Redmine::CodesetUtil.guess_encoding(str)
112
    end
113
  end
114

  
115
  test guess_encoding_should_return_nil_if_cannot_guess_encoding do
116
    str = '日本語'.encode('Windows-31J').b
117
    with_settings :repositories_encodings => 'UTF-8,EUC-JP' do
118
      assert_nil Redmine::CodesetUtil.guess_encoding(str)
119
    end
120
  end
104 121
end