Feature #22701 » 0003-Mulit-pass-CSV-import.patch
| app/models/import.rb | ||
|---|---|---|
| 139 | 139 |
end |
| 140 | 140 | |
| 141 | 141 |
# Imports items and returns the position of the last processed item |
| 142 |
def run(options={})
|
|
| 142 |
def run(options = {}, current_pass = completed_passes + 1)
|
|
| 143 |
if current_pass > required_passes |
|
| 144 |
# Abort recursion |
|
| 145 | ||
| 146 |
update_attribute :finished, true |
|
| 147 |
remove_file |
|
| 148 | ||
| 149 |
return total_items |
|
| 150 |
end |
|
| 151 | ||
| 143 | 152 |
max_items = options[:max_items] |
| 144 | 153 |
max_time = options[:max_time] |
| 145 |
current = 0 |
|
| 146 | 154 |
imported = 0 |
| 147 |
resume_after = items.maximum(:position) || 0 |
|
| 148 | 155 |
interrupted = false |
| 149 | 156 |
started_on = Time.now |
| 150 | 157 | |
| 151 |
read_items do |row, position| |
|
| 152 |
if (max_items && imported >= max_items) || (max_time && Time.now >= started_on + max_time) |
|
| 158 |
current = run_pass(options, current_pass) do |position| |
|
| 159 |
if (max_items && imported >= max_items) || (max_time && Time.now - started_on >= max_time) |
|
| 160 |
# interrupt import |
|
| 153 | 161 |
interrupted = true |
| 154 |
break |
|
| 162 |
false |
|
| 163 |
else |
|
| 164 |
# continue importing item |
|
| 165 |
imported += 1 |
|
| 166 |
true |
|
| 155 | 167 |
end |
| 156 |
if position > resume_after |
|
| 157 |
item = items.build |
|
| 158 |
item.position = position |
|
| 159 | ||
| 160 |
if object = build_object(row) |
|
| 161 |
if object.save |
|
| 162 |
item.obj_id = object.id |
|
| 163 |
else |
|
| 164 |
item.message = object.errors.full_messages.join("\n")
|
|
| 165 |
end |
|
| 166 |
end |
|
| 168 |
end |
|
| 167 | 169 | |
| 168 |
item.save! |
|
| 169 |
imported += 1 |
|
| 170 |
if !interrupted |
|
| 171 |
update_attribute(:total_items, current) if total_items.nil? |
|
| 172 | ||
| 173 |
if max_items |
|
| 174 |
options = options.merge(:max_items => max_items - imported) |
|
| 170 | 175 |
end |
| 171 |
current = position |
|
| 176 |
if max_time |
|
| 177 |
options = options.merge(:max_time => max_time - (Time.now - started_on)) |
|
| 178 |
end |
|
| 179 | ||
| 180 |
run(options, current_pass + 1) |
|
| 181 |
else |
|
| 182 |
current |
|
| 172 | 183 |
end |
| 184 |
end |
|
| 185 | ||
| 186 |
def run_pass(options={}, current_pass)
|
|
| 187 |
resume_after = items.where(:completed_passes => current_pass).maximum(:position) || 0 |
|
| 173 | 188 | |
| 174 |
if imported == 0 || interrupted == false |
|
| 175 |
if total_items.nil? |
|
| 176 |
update_attribute :total_items, current |
|
| 189 |
current = 0 |
|
| 190 | ||
| 191 |
read_items do |row, position| |
|
| 192 |
next unless position > resume_after |
|
| 193 | ||
| 194 |
break unless yield(position) |
|
| 195 | ||
| 196 |
current = position |
|
| 197 | ||
| 198 |
item = items.where(:position => position).first_or_initialize |
|
| 199 | ||
| 200 |
if object = build_object(row, item, current_pass) |
|
| 201 |
if object.save |
|
| 202 |
item.obj_id = object.id |
|
| 203 |
else |
|
| 204 |
item.message = object.errors.full_messages.join("\n")
|
|
| 205 |
end |
|
| 177 | 206 |
end |
| 178 |
update_attribute :finished, true |
|
| 179 |
remove_file |
|
| 207 | ||
| 208 |
item.completed_passes = current_pass |
|
| 209 |
item.save! |
|
| 180 | 210 |
end |
| 181 | 211 | |
| 182 | 212 |
current |
| ... | ... | |
| 190 | 220 |
items.where("obj_id IS NOT NULL")
|
| 191 | 221 |
end |
| 192 | 222 | |
| 223 |
# Should be overridden in sub class to implement multi-pass import |
|
| 224 |
def required_passes |
|
| 225 |
1 |
|
| 226 |
end |
|
| 227 | ||
| 228 |
def completed_passes |
|
| 229 |
if total_items.present? && items.count == total_items |
|
| 230 |
items.minimum(:completed_passes) |
|
| 231 |
else |
|
| 232 |
0 |
|
| 233 |
end |
|
| 234 |
end |
|
| 235 | ||
| 193 | 236 |
private |
| 194 | 237 | |
| 195 | 238 |
def read_rows |
| ... | ... | |
| 223 | 266 | |
| 224 | 267 |
# Builds a record for the given row and returns it |
| 225 | 268 |
# To be implemented by subclasses |
| 226 |
def build_object(row) |
|
| 269 |
def build_object(row, item, pass) |
|
| 270 |
raise NotImplementedError, "Subclass responsibility" |
|
| 227 | 271 |
end |
| 228 | 272 | |
| 229 | 273 |
# Generates a filename used to store the import file |
| app/models/import_item.rb | ||
|---|---|---|
| 19 | 19 |
belongs_to :import |
| 20 | 20 | |
| 21 | 21 |
validates_presence_of :import_id, :position |
| 22 | ||
| 23 |
validates_numericality_of :completed_passes, :only_integer => true, |
|
| 24 |
:greater_than_or_equal_to => 0 |
|
| 22 | 25 |
end |
| app/models/issue_import.rb | ||
|---|---|---|
| 57 | 57 |
mapping['create_versions'] == '1' |
| 58 | 58 |
end |
| 59 | 59 | |
| 60 |
def required_passes |
|
| 61 |
if mapping['parent_issue_id'] |
|
| 62 |
2 |
|
| 63 |
else |
|
| 64 |
1 |
|
| 65 |
end |
|
| 66 |
end |
|
| 67 | ||
| 60 | 68 |
private |
| 61 | 69 | |
| 62 |
def build_object(row) |
|
| 63 |
issue = Issue.new |
|
| 64 |
issue.author = user |
|
| 70 |
def build_object(row, item, pass) |
|
| 71 |
if (pass == 1) |
|
| 72 |
issue = Issue.new |
|
| 73 |
issue.author = user |
|
| 74 |
build_issue(row, issue) |
|
| 75 |
else |
|
| 76 |
issue = Issue.find(item.obj_id) |
|
| 77 |
build_relations(row, issue) |
|
| 78 |
end |
|
| 65 | 79 |
issue.notify = false |
| 66 | 80 | |
| 81 |
issue |
|
| 82 |
end |
|
| 83 | ||
| 84 | ||
| 85 |
def build_issue(row, issue) |
|
| 67 | 86 |
attributes = {
|
| 68 | 87 |
'project_id' => mapping['project_id'], |
| 69 | 88 |
'tracker_id' => mapping['tracker_id'], |
| ... | ... | |
| 110 | 129 |
attributes['is_private'] = '1' |
| 111 | 130 |
end |
| 112 | 131 |
end |
| 113 |
if parent_issue_id = row_value(row, 'parent_issue_id') |
|
| 114 |
if parent_issue_id =~ /\A(#)?(\d+)\z/ |
|
| 115 |
parent_issue_id = $2 |
|
| 116 |
if $1 |
|
| 117 |
attributes['parent_issue_id'] = parent_issue_id |
|
| 118 |
elsif issue_id = items.where(:position => parent_issue_id).first.try(:obj_id) |
|
| 119 |
attributes['parent_issue_id'] = issue_id |
|
| 120 |
end |
|
| 121 |
else |
|
| 122 |
attributes['parent_issue_id'] = parent_issue_id |
|
| 123 |
end |
|
| 124 |
end |
|
| 125 | 132 |
if start_date = row_date(row, 'start_date') |
| 126 | 133 |
attributes['start_date'] = start_date |
| 127 | 134 |
end |
| ... | ... | |
| 151 | 158 |
issue.send :safe_attributes=, attributes, user |
| 152 | 159 |
issue |
| 153 | 160 |
end |
| 161 | ||
| 162 |
def build_relations(row, issue) |
|
| 163 |
attributes = {}
|
|
| 164 | ||
| 165 |
if parent_issue_id = row_value(row, 'parent_issue_id') |
|
| 166 |
if parent_issue_id =~ /\A(#)?(\d+)\z/ |
|
| 167 |
parent_issue_id = $2 |
|
| 168 |
if $1 |
|
| 169 |
attributes['parent_issue_id'] = parent_issue_id |
|
| 170 |
elsif issue_id = items.where(:position => parent_issue_id).first.try(:obj_id) |
|
| 171 |
attributes['parent_issue_id'] = issue_id |
|
| 172 |
end |
|
| 173 |
else |
|
| 174 |
attributes['parent_issue_id'] = parent_issue_id |
|
| 175 |
end |
|
| 176 |
end |
|
| 177 | ||
| 178 |
issue.send :safe_attributes=, attributes, user |
|
| 179 |
issue |
|
| 180 |
end |
|
| 154 | 181 |
end |
| db/migrate/20160426125940_add_completed_passes_to_import_items.rb | ||
|---|---|---|
| 1 |
class AddCompletedPassesToImportItems < ActiveRecord::Migration |
|
| 2 |
def self.up |
|
| 3 |
add_column :import_items, :completed_passes, :integer, :default => 0, :null => false |
|
| 4 |
end |
|
| 5 | ||
| 6 |
def self.donw |
|
| 7 |
remove_column :imports_item, :completed_passes |
|
| 8 |
end |
|
| 9 |
end |
|
| test/unit/issue_import_test.rb | ||
|---|---|---|
| 130 | 130 |
import.run |
| 131 | 131 |
assert !File.exists?(file_path) |
| 132 | 132 |
end |
| 133 | ||
| 134 |
def test_multi_step_run |
|
| 135 |
# max_items < total_items |
|
| 136 |
import = generate_import_with_mapping |
|
| 137 | ||
| 138 |
assert_difference 'Issue.count', 5 do |
|
| 139 |
assert_equal 2, import.run(:max_items => 2) |
|
| 140 |
assert_not import.finished |
|
| 141 |
assert_equal 4, import.run(:max_items => 2) |
|
| 142 |
assert_not import.finished |
|
| 143 |
assert_equal 5, import.run(:max_items => 2) |
|
| 144 |
assert import.finished |
|
| 145 |
end |
|
| 146 | ||
| 147 | ||
| 148 |
# max_items > total_items |
|
| 149 |
import = generate_import_with_mapping |
|
| 150 | ||
| 151 |
assert_difference 'Issue.count', 5 do |
|
| 152 |
assert_equal 5, import.run(:max_items => 6) |
|
| 153 |
assert import.finished |
|
| 154 |
end |
|
| 155 | ||
| 156 | ||
| 157 |
# max_items == total_items |
|
| 158 |
import = generate_import_with_mapping |
|
| 159 | ||
| 160 |
assert_difference 'Issue.count', 5 do |
|
| 161 |
assert_equal 5, import.run(:max_items => 5) |
|
| 162 |
assert import.finished |
|
| 163 |
end |
|
| 164 |
end |
|
| 165 | ||
| 166 |
def test_multi_step_multi_pass_run |
|
| 167 |
# max_items < total_items |
|
| 168 |
import = generate_import_with_mapping |
|
| 169 |
import.mapping.merge!('parent_issue_id' => '5')
|
|
| 170 | ||
| 171 |
assert_difference 'Issue.count', 5 do |
|
| 172 |
assert_equal 2, import.run(:max_items => 2) |
|
| 173 |
assert_not import.finished |
|
| 174 | ||
| 175 |
assert_equal 4, import.run(:max_items => 2) |
|
| 176 |
assert_not import.finished |
|
| 177 | ||
| 178 |
assert_equal 1, import.run(:max_items => 2) |
|
| 179 |
assert_not import.finished |
|
| 180 | ||
| 181 |
assert_equal 3, import.run(:max_items => 2) |
|
| 182 |
assert_not import.finished |
|
| 183 | ||
| 184 |
assert_equal 5, import.run(:max_items => 2) |
|
| 185 |
assert import.finished |
|
| 186 |
end |
|
| 187 | ||
| 188 | ||
| 189 |
# max_items > total_items |
|
| 190 |
import = generate_import_with_mapping |
|
| 191 |
import.mapping.merge!('parent_issue_id' => '5')
|
|
| 192 | ||
| 193 |
assert_difference 'Issue.count', 5 do |
|
| 194 |
assert_equal 1, import.run(:max_items => 6) |
|
| 195 |
assert_not import.finished |
|
| 196 | ||
| 197 |
assert_equal 5, import.run(:max_items => 6) |
|
| 198 |
assert import.finished |
|
| 199 |
end |
|
| 200 | ||
| 201 | ||
| 202 |
# max_items == total_items |
|
| 203 |
import = generate_import_with_mapping |
|
| 204 |
import.mapping.merge!('parent_issue_id' => '5')
|
|
| 205 | ||
| 206 |
assert_difference 'Issue.count', 5 do |
|
| 207 |
assert_equal 0, import.run(:max_items => 5) |
|
| 208 |
assert_not import.finished |
|
| 209 | ||
| 210 |
assert_equal 5, import.run(:max_items => 5) |
|
| 211 |
assert import.finished |
|
| 212 |
end |
|
| 213 |
end |
|
| 214 | ||
| 215 |
def test_required_passes |
|
| 216 |
# Imports w/o relation mappings need just a single pass |
|
| 217 |
import = generate_import_with_mapping |
|
| 218 | ||
| 219 |
assert_equal 1, import.required_passes |
|
| 220 |
import.run |
|
| 221 |
assert_equal 1, import.completed_passes |
|
| 222 | ||
| 223 |
# Imports w/ references to other rows need 2 passes |
|
| 224 |
import = generate_import_with_mapping |
|
| 225 |
import.mapping.merge!('parent_issue_id' => '5')
|
|
| 226 | ||
| 227 |
assert_equal 2, import.required_passes |
|
| 228 |
import.run |
|
| 229 |
assert_equal 2, import.completed_passes |
|
| 230 |
end |
|
| 133 | 231 |
end |
- « Previous
- 1
- 2
- 3
- Next »