Project

General

Profile

RE: migrate_from_mantis Mysql::Error ยป migrate_from_mantis.rake

john lin, 2012-02-03 08:39

 
1
# redMine - project management software
2
# Copyright (C) 2006-2007  Jean-Philippe Lang
3
#
4
# This program is free software; you can redistribute it and/or
5
# modify it under the terms of the GNU General Public License
6
# as published by the Free Software Foundation; either version 2
7
# of the License, or (at your option) any later version.
8
# 
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
# GNU General Public License for more details.
13
# 
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
17

    
18
desc 'Mantis migration script'
19

    
20
require 'active_record'
21
require 'iconv'
22
require 'pp'
23
require 'uri'
24

    
25
namespace :redmine do
26
task :migrate_from_mantis => :environment do
27
  
28
  module MantisMigrate
29
   
30
      DEFAULT_STATUS = IssueStatus.default
31
      assigned_status = IssueStatus.find_by_position(2)
32
      resolved_status = IssueStatus.find_by_position(3)
33
      feedback_status = IssueStatus.find_by_position(4)
34
      closed_status = IssueStatus.find :first, :conditions => { :is_closed => true }
35
      STATUS_MAPPING = {10 => DEFAULT_STATUS,  # new
36
                        20 => feedback_status, # feedback
37
                        30 => DEFAULT_STATUS,  # acknowledged
38
                        40 => DEFAULT_STATUS,  # confirmed
39
                        50 => assigned_status, # assigned
40
                        80 => resolved_status, # resolved
41
                        90 => closed_status    # closed
42
                        }
43
                        
44
      priorities = IssuePriority.all
45
      DEFAULT_PRIORITY = priorities[2]
46
      PRIORITY_MAPPING = {10 => priorities[1], # none
47
                          20 => priorities[1], # low
48
                          30 => priorities[2], # normal
49
                          40 => priorities[3], # high
50
                          50 => priorities[4], # urgent
51
                          60 => priorities[5]  # immediate
52
                          }
53
    
54
      TRACKER_BUG = Tracker.find_by_position(1)
55
      TRACKER_FEATURE = Tracker.find_by_position(2)
56
      
57
      roles = Role.find(:all, :conditions => {:builtin => 0}, :order => 'position ASC')
58
      manager_role = roles[0]
59
      developer_role = roles[1]
60
      DEFAULT_ROLE = roles.last
61
      ROLE_MAPPING = {10 => DEFAULT_ROLE,   # viewer
62
                      25 => DEFAULT_ROLE,   # reporter
63
                      40 => DEFAULT_ROLE,   # updater
64
                      55 => developer_role, # developer
65
                      70 => manager_role,   # manager
66
                      90 => manager_role    # administrator
67
                      }
68
      
69
      CUSTOM_FIELD_TYPE_MAPPING = {0 => 'string', # String
70
                                   1 => 'int',    # Numeric
71
                                   2 => 'int',    # Float
72
                                   3 => 'list',   # Enumeration
73
                                   4 => 'string', # Email
74
                                   5 => 'bool',   # Checkbox
75
                                   6 => 'list',   # List
76
                                   7 => 'list',   # Multiselection list
77
                                   8 => 'date',   # Date
78
                                   }
79
                                   
80
      RELATION_TYPE_MAPPING = {1 => IssueRelation::TYPE_RELATES,    # related to
81
                               2 => IssueRelation::TYPE_RELATES,    # parent of
82
                               3 => IssueRelation::TYPE_RELATES,    # child of
83
                               0 => IssueRelation::TYPE_DUPLICATES, # duplicate of
84
                               4 => IssueRelation::TYPE_DUPLICATES  # has duplicate
85
                               }
86
                                                                   
87
    class MantisUser < ActiveRecord::Base
88
      set_table_name :mantis_user_table
89
      
90
      def firstname
91
        @firstname = realname.blank? ? username : realname.split.first[0..29]
92
        @firstname
93
      end
94
      
95
      def lastname
96
        @lastname = realname.blank? ? '-' : realname.split[1..-1].join(' ')[0..29]
97
        @lastname = '-' if @lastname.blank?
98
        @lastname
99
      end
100
      
101
      def email
102
        if read_attribute(:email).match(/^([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})$/i) &&
103
             !User.find_by_mail(read_attribute(:email))
104
          @email = read_attribute(:email)
105
        else
106
          @email = "#{username}@foo.bar"
107
        end
108
      end
109
      
110
      def username
111
        read_attribute(:username)[0..29].gsub(/[^a-zA-Z0-9_\-@\.]/, '-')
112
      end
113
    end
114
    
115
    class MantisProject < ActiveRecord::Base
116
      set_table_name :mantis_project_table
117
      has_many :versions, :class_name => "MantisVersion", :foreign_key => :project_id
118
      has_many :categories, :class_name => "MantisCategory", :foreign_key => :project_id
119
      has_many :news, :class_name => "MantisNews", :foreign_key => :project_id
120
      has_many :members, :class_name => "MantisProjectUser", :foreign_key => :project_id
121
      
122
      def identifier
123
        attr_name = read_attribute(:name)
124
	if not attr_name.match /^[^a-z0-9\-]+$/
125
          attr_name = attr_name.gsub(/[^a-z0-9\-]+/, '-')
126
        else
127
          # when name is written in multi-byte languages
128
          attr_name = "prj" << attr_name.hash.to_s
129
        end
130
        attr_name.slice(0, Project::IDENTIFIER_MAX_LENGTH)
131
      end
132
    end
133
    
134
    class MantisVersion < ActiveRecord::Base
135
      set_table_name :mantis_project_version_table
136
      
137
      def version
138
        read_attribute(:version)[0..29]
139
      end
140
      
141
      def description
142
        read_attribute(:description)[0..254]
143
      end
144
    end
145
    
146
    class MantisCategory < ActiveRecord::Base
147
      set_table_name :mantis_category_table
148
    end
149
    
150
    class MantisProjectUser < ActiveRecord::Base
151
      set_table_name :mantis_project_user_list_table
152
    end
153
    
154
    class MantisBug < ActiveRecord::Base
155
      set_table_name :mantis_bug_table
156
      belongs_to :bug_text, :class_name => "MantisBugText", :foreign_key => :bug_text_id
157
      belongs_to :category, :class_name => "MantisCategory", :foreign_key => :category_id
158
      has_many :bug_notes, :class_name => "MantisBugNote", :foreign_key => :bug_id
159
      has_many :bug_files, :class_name => "MantisBugFile", :foreign_key => :bug_id
160
      has_many :bug_monitors, :class_name => "MantisBugMonitor", :foreign_key => :bug_id
161

    
162
      def created_on
163
        Time.at read_attribute(:date_submitted)
164
      end
165

    
166
      def updated_on
167
        Time.at read_attribute(:last_updated)
168
      end
169
    end
170
    
171
    class MantisBugText < ActiveRecord::Base
172
      set_table_name :mantis_bug_text_table
173
      
174
      # Adds Mantis steps_to_reproduce and additional_information fields
175
      # to description if any
176
      def full_description
177
        full_description = description
178
        full_description += "\n\n*Steps to reproduce:*\n\n#{steps_to_reproduce}" unless steps_to_reproduce.blank?
179
        full_description += "\n\n*Additional information:*\n\n#{additional_information}" unless additional_information.blank?
180
        full_description
181
      end
182
    end
183
    
184
    class MantisBugNote < ActiveRecord::Base
185
      set_table_name :mantis_bugnote_table
186
      belongs_to :bug, :class_name => "MantisBug", :foreign_key => :bug_id
187
      belongs_to :bug_note_text, :class_name => "MantisBugNoteText", :foreign_key => :bugnote_text_id
188

    
189
      def created_on
190
        Time.at read_attribute(:date_submitted)
191
      end
192
    end
193
    
194
    class MantisBugNoteText < ActiveRecord::Base
195
      set_table_name :mantis_bugnote_text_table
196
    end
197
    
198
    class MantisBugFile < ActiveRecord::Base
199
      set_table_name :mantis_bug_file_table
200
      
201
      def size
202
        filesize
203
      end
204
      
205
      def original_filename
206
        MantisMigrate.encode(filename)
207
      end
208
      
209
      def content_type
210
        file_type
211
      end
212

    
213
      def created_on
214
        Time.at read_attribute(:date_added)
215
      end
216
      
217
      def read(*args)
218
      	if @read_finished
219
      		nil
220
      	else
221
      		@read_finished = true
222
      		content
223
      	end
224
      end
225
    end
226
    
227
    class MantisBugRelationship < ActiveRecord::Base
228
      set_table_name :mantis_bug_relationship_table
229
    end
230
    
231
    class MantisBugMonitor < ActiveRecord::Base
232
      set_table_name :mantis_bug_monitor_table
233
    end
234
    
235
    class MantisNews < ActiveRecord::Base
236
      set_table_name :mantis_news_table
237

    
238
      def created_on
239
        Time.at read_attribute(:date_posted)
240
      end
241
    end
242
    
243
    class MantisCustomField < ActiveRecord::Base
244
      set_table_name :mantis_custom_field_table
245
      set_inheritance_column :none  
246
      has_many :values, :class_name => "MantisCustomFieldString", :foreign_key => :field_id
247
      has_many :projects, :class_name => "MantisCustomFieldProject", :foreign_key => :field_id
248
      
249
      def format
250
        read_attribute :type
251
      end
252
      
253
      def name
254
        read_attribute(:name)[0..29]
255
      end
256
    end
257
    
258
    class MantisCustomFieldProject < ActiveRecord::Base
259
      set_table_name :mantis_custom_field_project_table  
260
    end
261
    
262
    class MantisCustomFieldString < ActiveRecord::Base
263
      set_table_name :mantis_custom_field_string_table  
264
    end
265
  
266
  
267
    def self.migrate
268
          
269
      # Users
270
      puts "Migrating users\n"
271
      #User.delete_all "login <> 'admin'"
272
      users_map = {}
273
      users_migrated = 0
274
      MantisUser.find(:all).each do |user|
275
        print "  search user [login = '#{user.username}'] ..."
276
        found = User.find :first, :conditions => ["login = ?", user.username]
277
        if found
278
          u = found
279
          puts " #{found.login} is found"
280
        else
281
          u = User.new :firstname => encode(user.firstname), 
282
                       :lastname => encode(user.lastname),
283
                       :mail => user.email,
284
                       :last_login_on => user.last_visit
285
          u.login = user.username
286
          u.password = 'mantis'
287
          u.status = User::STATUS_LOCKED if user.enabled != 1
288
          u.admin = true if user.access_level == 90
289
          next unless u.save!
290
          puts " #{user.username} is added"
291
        end
292
        users_migrated += 1
293
        users_map[user.id] = u.id
294
      end
295
      puts "Migrating users ... done"
296
    
297
      # Projects
298
      puts "Migrating projects"
299
      #Project.destroy_all
300
      projects_map = {}
301
      versions_map = {}
302
      categories_map = {}
303
      MantisProject.find(:all).each do |project|
304
        print "  search project [identifier = '#{project.identifier}'] ..."
305
        found = Project.find :first, :conditions => ["identifier = ?", project.identifier]
306
        if found
307
          p = found
308
          puts " #{found.identifier} is found\n"
309
          projects_map[project.id] = p.id
310
        else
311
          p = Project.new :name => encode(project.name), 
312
                          :description => encode(project.description)
313
          p.identifier = project.identifier
314
          if not p.save
315
            puts " Failure:#{project.name}(#{p.identifier}) is not added"
316
            next
317
	  end
318
          puts " #{project.name} is added"
319
          projects_map[project.id] = p.id
320
          p.enabled_module_names = ['issue_tracking', 'news', 'wiki']
321
	  p.trackers.clear
322
          print "."
323
          p.trackers << TRACKER_BUG
324
          print "."
325
          p.trackers << TRACKER_FEATURE
326
          puts ".success"
327
        end
328
        
329
        # Project members
330
        project.members.each do |member|
331
          print "   search member for #{project.name} [user_id = users_map[#{member.user_id}]] ..."
332
          found = Member.find :first, :conditions => ["user_id = ? AND project_id = ?", users_map[member.user_id], p.id]
333
          if found
334
            m = found
335
            m.roles = [ROLE_MAPPING[member.access_level] || DEFAULT_ROLE]
336
            m.save
337
            puts " #{found.user.login} is found"
338
          else
339
            m = Member.new :user => User.find_by_id(users_map[member.user_id]),
340
                           :roles => [ROLE_MAPPING[member.access_level] || DEFAULT_ROLE]
341
            m.project = p
342
            m.save
343
            puts " #{m.user.login} is added"
344
          end
345
        end
346
        
347
        # Project versions
348
        project.versions.each do |version|
349
          print "   search version for #{project.name} [name = '#{version.version}'] ..."
350
          found = Member.find :first, ["name = ? AND project_id = ?", encode(version.version), p.id]
351
          if found
352
            v = found
353
            puts " #{found.name} is found"
354
          else
355
            v = Version.new :name => encode(version.version),
356
                            :description => encode(version.description),
357
                            :effective_date => (version.date_order ? version.date_order.to_date : nil)
358
            v.project = p
359
            v.save
360
            puts " #{v.name} is added"
361
          end
362
          versions_map[version.id] = v.id
363
        end
364
        
365
        # Project categories
366
        project.categories.each do |category|
367
          cname = category.name[0,30]
368
          print "   search category for #{project.name} [name = '#{cname}'] ..."
369
          found = IssueCategory.find :first, ["name = ? AND project_id = ?", cname, p.id]
370
          if found
371
            g = found
372
            puts "  #{found.name} is found"
373
          else
374
            g = IssueCategory.new :name => cname
375
            g.project = p
376
            g.save
377
            puts " #{category.name} is added"
378
          end
379
          categories_map[category.id] = g.id
380
        end
381
      end
382
      puts "Migrating projects ... done"
383
    
384
      # Bugs
385
      print "Migrating bugs"
386
      #Issue.destroy_all
387
      issues_map = {}
388
      keep_bug_ids = (Issue.count == 0)
389
      MantisBug.find_each(:batch_size => 200) do |bug|
390
        next unless projects_map[bug.project_id] && users_map[bug.reporter_id]
391
        i = Issue.new :project_id => projects_map[bug.project_id], 
392
                      :subject => encode(bug.summary),
393
                      :description => encode(bug.bug_text.full_description),
394
                      :priority => PRIORITY_MAPPING[bug.priority] || DEFAULT_PRIORITY,
395
                      :created_on => bug.created_on,
396
                      :updated_on => bug.updated_on
397
        i.author = User.find_by_id(users_map[bug.reporter_id])
398
        i.category = IssueCategory.find_by_id(categories_map[bug.category_id]) unless bug.category.blank?
399
        i.fixed_version = Version.find_by_project_id_and_name(i.project_id, bug.fixed_in_version) unless bug.fixed_in_version.blank?
400
        i.status = STATUS_MAPPING[bug.status] || DEFAULT_STATUS
401
        i.tracker = (bug.severity == 10 ? TRACKER_FEATURE : TRACKER_BUG)
402
        i.id = bug.id if keep_bug_ids
403
        next unless i.save
404
        issues_map[bug.id] = i.id
405
        print '.'
406
      STDOUT.flush
407

    
408
        # Assignee
409
        # Redmine checks that the assignee is a project member
410
        if (bug.handler_id && users_map[bug.handler_id])
411
          i.assigned_to = User.find_by_id(users_map[bug.handler_id])
412
          i.save_with_validation(false)
413
        end        
414
        
415
        # Bug notes
416
        bug.bug_notes.each do |note|
417
          next unless users_map[note.reporter_id]
418
          n = Journal.new :notes => encode(note.bug_note_text.note),
419
                          :created_on => note.created_on
420
          n.user = User.find_by_id(users_map[note.reporter_id])
421
          n.journalized = i
422
          n.save
423
        end
424
        
425
        # Bug files
426
        bug.bug_files.each do |file|
427
          a = Attachment.new :created_on => file.created_on
428
          a.file = file
429
          a.author = User.find :first
430
          a.container = i
431
          a.save
432
        end
433
        
434
        # Bug monitors
435
        bug.bug_monitors.each do |monitor|
436
          next unless users_map[monitor.user_id]
437
          i.add_watcher(User.find_by_id(users_map[monitor.user_id]))
438
        end
439
      end
440
      
441
      # update issue id sequence if needed (postgresql)
442
      Issue.connection.reset_pk_sequence!(Issue.table_name) if Issue.connection.respond_to?('reset_pk_sequence!')
443
      puts "done"
444
      
445
      # Bug relationships
446
      print "Migrating bug relations"
447
      MantisBugRelationship.find(:all).each do |relation|
448
        next unless issues_map[relation.source_bug_id] && issues_map[relation.destination_bug_id]
449
        r = IssueRelation.new :relation_type => RELATION_TYPE_MAPPING[relation.relationship_type]
450
        r.issue_from = Issue.find_by_id(issues_map[relation.source_bug_id])
451
        r.issue_to = Issue.find_by_id(issues_map[relation.destination_bug_id])
452
        pp r unless r.save
453
        print '.'
454
        STDOUT.flush
455
      end
456
      puts "done"
457
      
458
      # News
459
      print "Migrating news"
460
      News.destroy_all
461
      MantisNews.find(:all, :conditions => 'project_id > 0').each do |news|
462
        next unless projects_map[news.project_id]
463
        n = News.new :project_id => projects_map[news.project_id],
464
                     :title => encode(news.headline[0..59]),
465
                     :description => encode(news.body),
466
                     :created_on => news.created_on
467
        n.author = User.find_by_id(users_map[news.poster_id])
468
        n.save
469
        print '.'
470
        STDOUT.flush
471
      end
472
      puts
473
      
474
      # Custom fields
475
      print "Migrating custom fields"
476
      IssueCustomField.destroy_all
477
      MantisCustomField.find(:all).each do |field|
478
        f = IssueCustomField.new :name => field.name[0..29],
479
                                 :field_format => CUSTOM_FIELD_TYPE_MAPPING[field.format],
480
                                 :min_length => field.length_min,
481
                                 :max_length => field.length_max,
482
                                 :regexp => field.valid_regexp,
483
                                 :possible_values => field.possible_values.split('|'),
484
                                 :is_required => field.require_report?
485
        next unless f.save
486
        print '.'
487
        STDOUT.flush
488
        # Trackers association
489
        f.trackers = Tracker.find :all
490
        
491
        # Projects association
492
        field.projects.each do |project|
493
          f.projects << Project.find_by_id(projects_map[project.project_id]) if projects_map[project.project_id]
494
        end
495
        
496
        # Values
497
        field.values.each do |value|
498
          v = CustomValue.new :custom_field_id => f.id,
499
                              :value => value.value
500
          v.customized = Issue.find_by_id(issues_map[value.bug_id]) if issues_map[value.bug_id]
501
          v.save
502
        end unless f.new_record?
503
      end
504
      puts
505
    
506
      puts
507
      puts "Users:           #{users_migrated}/#{MantisUser.count}"
508
      puts "Projects:        #{Project.count}/#{MantisProject.count}"
509
      puts "Memberships:     #{Member.count}/#{MantisProjectUser.count}"
510
      puts "Versions:        #{Version.count}/#{MantisVersion.count}"
511
      puts "Categories:      #{IssueCategory.count}/#{MantisCategory.count}"
512
      puts "Bugs:            #{Issue.count}/#{MantisBug.count}"
513
      puts "Bug notes:       #{Journal.count}/#{MantisBugNote.count}"
514
      puts "Bug files:       #{Attachment.count}/#{MantisBugFile.count}"
515
      puts "Bug relations:   #{IssueRelation.count}/#{MantisBugRelationship.count}"
516
      puts "Bug monitors:    #{Watcher.count}/#{MantisBugMonitor.count}"
517
      puts "News:            #{News.count}/#{MantisNews.count}"
518
      puts "Custom fields:   #{IssueCustomField.count}/#{MantisCustomField.count}"
519
    end
520
  
521
    def self.encoding(charset)
522
      @ic = Iconv.new('UTF-8', charset)
523
    rescue Iconv::InvalidEncoding
524
      return false      
525
    end
526
    
527
    def self.establish_connection(params)
528
      constants.each do |const|
529
        klass = const_get(const)
530
        next unless klass.respond_to? 'establish_connection'
531
        klass.establish_connection params
532
      end
533
    end
534
    
535
    def self.encode(text)
536
      @ic.iconv text
537
    rescue
538
      text
539
    end
540
  end
541
  
542
  puts
543
  if Redmine::DefaultData::Loader.no_data?
544
    puts "Redmine configuration need to be loaded before importing data."
545
    puts "Please, run this first:"
546
    puts
547
    puts "  rake redmine:load_default_data RAILS_ENV=\"#{ENV['RAILS_ENV']}\""
548
    exit
549
  end
550
  
551
  puts "WARNING: Your Redmine data will be deleted during this process."
552
  print "Are you sure you want to continue ? [y/N] "
553
  STDOUT.flush
554
  break unless STDIN.gets.match(/^y$/i)
555
  
556
  # Default Mantis database settings
557
  db_params = {:adapter => 'mysql', 
558
               :database => 'bugtracker', 
559
               :host => 'localhost', 
560
               :username => 'root', 
561
               :password => '' }
562

    
563
  puts				
564
  puts "Please enter settings for your Mantis database"  
565
  [:adapter, :host, :database, :username, :password].each do |param|
566
    print "#{param} [#{db_params[param]}]: "
567
    value = STDIN.gets.chomp!
568
    db_params[param] = value unless value.blank?
569
  end
570
    
571
  while true
572
    print "encoding [UTF-8]: "
573
    STDOUT.flush
574
    encoding = STDIN.gets.chomp!
575
    encoding = 'UTF-8' if encoding.blank?
576
    break if MantisMigrate.encoding encoding
577
    puts "Invalid encoding!"
578
  end
579
  puts
580
  
581
  # Make sure bugs can refer bugs in other projects
582
  Setting.cross_project_issue_relations = 1 if Setting.respond_to? 'cross_project_issue_relations'
583
  
584
  # Turn off email notifications
585
  Setting.notified_events = []
586
  
587
  MantisMigrate.establish_connection db_params
588
  MantisMigrate.migrate
589
end
590
end
    (1-1/1)