0003-Mulit-pass-CSV-import.patch

Gregor Schmidt, 2016-05-02 14:52

Download (10.6 KB)

View differences:

app/models/import.rb
139 139
  end
140 140

  
141 141
  # Imports items and returns the position of the last processed item
142
  def run(options={})
142
  def run(options = {}, current_pass = completed_passes + 1)
143
    if current_pass > required_passes
144
      # Abort recursion
145

  
146
      update_attribute :finished, true
147
      remove_file
148

  
149
      return total_items
150
    end
151

  
143 152
    max_items = options[:max_items]
144 153
    max_time = options[:max_time]
145
    current = 0
146 154
    imported = 0
147
    resume_after = items.maximum(:position) || 0
148 155
    interrupted = false
149 156
    started_on = Time.now
150 157

  
151
    read_items do |row, position|
152
      if (max_items && imported >= max_items) || (max_time && Time.now >= started_on + max_time)
158
    current = run_pass(options, current_pass) do |position|
159
      if (max_items && imported >= max_items) || (max_time && Time.now - started_on >= max_time)
160
        # interrupt import
153 161
        interrupted = true
154
        break
162
        false
163
      else
164
        # continue importing item
165
        imported += 1
166
        true
155 167
      end
156
      if position > resume_after
157
        item = items.build
158
        item.position = position
159

  
160
        if object = build_object(row)
161
          if object.save
162
            item.obj_id = object.id
163
          else
164
            item.message = object.errors.full_messages.join("\n")
165
          end
166
        end
168
    end
167 169

  
168
        item.save!
169
        imported += 1
170
    if !interrupted
171
      update_attribute(:total_items, current) if total_items.nil?
172

  
173
      if max_items
174
        options = options.merge(:max_items => max_items - imported)
170 175
      end
171
      current = position
176
      if max_time
177
        options = options.merge(:max_time => max_time - (Time.now - started_on))
178
      end
179

  
180
      run(options, current_pass + 1)
181
    else
182
      current
172 183
    end
184
  end
185

  
186
  def run_pass(options={}, current_pass)
187
    resume_after = items.where(:completed_passes => current_pass).maximum(:position) || 0
173 188

  
174
    if imported == 0 || interrupted == false
175
      if total_items.nil?
176
        update_attribute :total_items, current
189
    current = 0
190

  
191
    read_items do |row, position|
192
      next unless position > resume_after
193

  
194
      break unless yield(position)
195

  
196
      current = position
197

  
198
      item = items.where(:position => position).first_or_initialize
199

  
200
      if object = build_object(row, item, current_pass)
201
        if object.save
202
          item.obj_id = object.id
203
        else
204
          item.message = object.errors.full_messages.join("\n")
205
        end
177 206
      end
178
      update_attribute :finished, true
179
      remove_file
207

  
208
      item.completed_passes = current_pass
209
      item.save!
180 210
    end
181 211

  
182 212
    current
......
190 220
    items.where("obj_id IS NOT NULL")
191 221
  end
192 222

  
223
  # Should be overridden in sub class to implement multi-pass import
224
  def required_passes
225
    1
226
  end
227

  
228
  def completed_passes
229
    if total_items.present? && items.count == total_items
230
      items.minimum(:completed_passes)
231
    else
232
      0
233
    end
234
  end
235

  
193 236
  private
194 237

  
195 238
  def read_rows
......
223 266

  
224 267
  # Builds a record for the given row and returns it
225 268
  # To be implemented by subclasses
226
  def build_object(row)
269
  def build_object(row, item, pass)
270
    raise NotImplementedError, "Subclass responsibility"
227 271
  end
228 272

  
229 273
  # Generates a filename used to store the import file
app/models/import_item.rb
19 19
  belongs_to :import
20 20

  
21 21
  validates_presence_of :import_id, :position
22

  
23
  validates_numericality_of :completed_passes, :only_integer => true,
24
                                               :greater_than_or_equal_to => 0
22 25
end
app/models/issue_import.rb
57 57
      mapping['create_versions'] == '1'
58 58
  end
59 59

  
60
  def required_passes
61
    if mapping['parent_issue_id']
62
      2
63
    else
64
      1
65
    end
66
  end
67

  
60 68
  private
61 69

  
62
  def build_object(row)
63
    issue = Issue.new
64
    issue.author = user
70
  def build_object(row, item, pass)
71
    if (pass == 1)
72
      issue = Issue.new
73
      issue.author = user
74
      build_issue(row, issue)
75
    else
76
      issue = Issue.find(item.obj_id)
77
      build_relations(row, issue)
78
    end
65 79
    issue.notify = false
66 80

  
81
    issue
82
  end
83

  
84

  
85
  def build_issue(row, issue)
67 86
    attributes = {
68 87
      'project_id' => mapping['project_id'],
69 88
      'tracker_id' => mapping['tracker_id'],
......
110 129
        attributes['is_private'] = '1'
111 130
      end
112 131
    end
113
    if parent_issue_id = row_value(row, 'parent_issue_id')
114
      if parent_issue_id =~ /\A(#)?(\d+)\z/
115
        parent_issue_id = $2
116
        if $1
117
          attributes['parent_issue_id'] = parent_issue_id
118
        elsif issue_id = items.where(:position => parent_issue_id).first.try(:obj_id)
119
          attributes['parent_issue_id'] = issue_id
120
        end
121
      else
122
        attributes['parent_issue_id'] = parent_issue_id
123
      end
124
    end
125 132
    if start_date = row_date(row, 'start_date')
126 133
      attributes['start_date'] = start_date
127 134
    end
......
151 158
    issue.send :safe_attributes=, attributes, user
152 159
    issue
153 160
  end
161

  
162
  def build_relations(row, issue)
163
    attributes = {}
164

  
165
    if parent_issue_id = row_value(row, 'parent_issue_id')
166
      if parent_issue_id =~ /\A(#)?(\d+)\z/
167
        parent_issue_id = $2
168
        if $1
169
          attributes['parent_issue_id'] = parent_issue_id
170
        elsif issue_id = items.where(:position => parent_issue_id).first.try(:obj_id)
171
          attributes['parent_issue_id'] = issue_id
172
        end
173
      else
174
        attributes['parent_issue_id'] = parent_issue_id
175
      end
176
    end
177

  
178
    issue.send :safe_attributes=, attributes, user
179
    issue
180
  end
154 181
end
db/migrate/20160426125940_add_completed_passes_to_import_items.rb
1
class AddCompletedPassesToImportItems < ActiveRecord::Migration
2
  def self.up
3
    add_column :import_items, :completed_passes, :integer, :default => 0, :null => false
4
  end
5

  
6
  def self.donw
7
    remove_column :imports_item, :completed_passes
8
  end
9
end
test/unit/issue_import_test.rb
130 130
    import.run
131 131
    assert !File.exists?(file_path)
132 132
  end
133

  
134
  def test_multi_step_run
135
    # max_items < total_items
136
    import = generate_import_with_mapping
137

  
138
    assert_difference 'Issue.count', 5 do
139
      assert_equal 2, import.run(:max_items => 2)
140
      assert_not import.finished
141
      assert_equal 4, import.run(:max_items => 2)
142
      assert_not import.finished
143
      assert_equal 5, import.run(:max_items => 2)
144
      assert import.finished
145
    end
146

  
147

  
148
    # max_items > total_items
149
    import = generate_import_with_mapping
150

  
151
    assert_difference 'Issue.count', 5 do
152
      assert_equal 5, import.run(:max_items => 6)
153
      assert import.finished
154
    end
155

  
156

  
157
    # max_items == total_items
158
    import = generate_import_with_mapping
159

  
160
    assert_difference 'Issue.count', 5 do
161
      assert_equal 5, import.run(:max_items => 5)
162
      assert import.finished
163
    end
164
  end
165

  
166
  def test_multi_step_multi_pass_run
167
    # max_items < total_items
168
    import = generate_import_with_mapping
169
    import.mapping.merge!('parent_issue_id' => '5')
170

  
171
    assert_difference 'Issue.count', 5 do
172
      assert_equal 2, import.run(:max_items => 2)
173
      assert_not import.finished
174

  
175
      assert_equal 4, import.run(:max_items => 2)
176
      assert_not import.finished
177

  
178
      assert_equal 1, import.run(:max_items => 2)
179
      assert_not import.finished
180

  
181
      assert_equal 3, import.run(:max_items => 2)
182
      assert_not import.finished
183

  
184
      assert_equal 5, import.run(:max_items => 2)
185
      assert import.finished
186
    end
187

  
188

  
189
    # max_items > total_items
190
    import = generate_import_with_mapping
191
    import.mapping.merge!('parent_issue_id' => '5')
192

  
193
    assert_difference 'Issue.count', 5 do
194
      assert_equal 1, import.run(:max_items => 6)
195
      assert_not import.finished
196

  
197
      assert_equal 5, import.run(:max_items => 6)
198
      assert import.finished
199
    end
200

  
201

  
202
    # max_items == total_items
203
    import = generate_import_with_mapping
204
    import.mapping.merge!('parent_issue_id' => '5')
205

  
206
    assert_difference 'Issue.count', 5 do
207
      assert_equal 0, import.run(:max_items => 5)
208
      assert_not import.finished
209

  
210
      assert_equal 5, import.run(:max_items => 5)
211
      assert import.finished
212
    end
213
  end
214

  
215
  def test_required_passes
216
    # Imports w/o relation mappings need just a single pass
217
    import = generate_import_with_mapping
218

  
219
    assert_equal 1, import.required_passes
220
    import.run
221
    assert_equal 1, import.completed_passes
222

  
223
    # Imports w/ references to other rows need 2 passes
224
    import = generate_import_with_mapping
225
    import.mapping.merge!('parent_issue_id' => '5')
226

  
227
    assert_equal 2, import.required_passes
228
    import.run
229
    assert_equal 2, import.completed_passes
230
  end
133 231
end
134
-