Index: lib/tasks/migrate_from_trac.rake =================================================================== --- lib/tasks/migrate_from_trac.rake (revision 3597) +++ lib/tasks/migrate_from_trac.rake (working copy) @@ -19,6 +19,13 @@ require 'iconv' require 'pp' +require 'redmine/scm/adapters/abstract_adapter' +require 'redmine/scm/adapters/subversion_adapter' +require 'rexml/document' +require 'uri' + +require 'tempfile' + namespace :redmine do desc 'Trac migration script' task :migrate_from_trac => :environment do @@ -192,7 +199,8 @@ def time; Time.at(read_attribute(:time)) end end - TRAC_WIKI_PAGES = %w(InterMapTxt InterTrac InterWiki RecentChanges SandBox TracAccessibility TracAdmin TracBackup TracBrowser TracCgi TracChangeset \ + TRAC_WIKI_PAGES = %w(InterMapTxt TracInstallPlatforms TracMultipleProjects InterTrac InterWiki RecentChanges SandBox \ + TracAccessibility TracAdmin TracBackup TracBrowser TracCgi TracChangeset \ TracEnvironment TracFastCgi TracGuide TracImport TracIni TracInstall TracInterfaceCustomization \ TracLinks TracLogging TracModPython TracNotification TracPermissions TracPlugins TracQuery \ TracReports TracRevisionLog TracRoadmap TracRss TracSearch TracStandalone TracSupport TracSyntaxColoring TracTickets \ @@ -271,96 +279,7 @@ # Basic wiki syntax conversion def self.convert_wiki_text(text) - # Titles - text = text.gsub(/^(\=+)\s(.+)\s(\=+)/) {|s| "\nh#{$1.length}. #{$2}\n"} - # External Links - text = text.gsub(/\[(http[^\s]+)\s+([^\]]+)\]/) {|s| "\"#{$2}\":#{$1}"} - # Ticket links: - # [ticket:234 Text],[ticket:234 This is a test] - text = text.gsub(/\[ticket\:([^\ ]+)\ (.+?)\]/, '"\2":/issues/show/\1') - # ticket:1234 - # #1 is working cause Redmine uses the same syntax. - text = text.gsub(/ticket\:([^\ ]+)/, '#\1') - # Milestone links: - # [milestone:"0.1.0 Mercury" Milestone 0.1.0 (Mercury)] - # The text "Milestone 0.1.0 (Mercury)" is not converted, - # cause Redmine's wiki does not support this. - text = text.gsub(/\[milestone\:\"([^\"]+)\"\ (.+?)\]/, 'version:"\1"') - # [milestone:"0.1.0 Mercury"] - text = text.gsub(/\[milestone\:\"([^\"]+)\"\]/, 'version:"\1"') - text = text.gsub(/milestone\:\"([^\"]+)\"/, 'version:"\1"') - # milestone:0.1.0 - text = text.gsub(/\[milestone\:([^\ ]+)\]/, 'version:\1') - text = text.gsub(/milestone\:([^\ ]+)/, 'version:\1') - # Internal Links - text = text.gsub(/\[\[BR\]\]/, "\n") # This has to go before the rules below - text = text.gsub(/\[\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"} - text = text.gsub(/\[wiki:\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"} - text = text.gsub(/\[wiki:\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"} - text = text.gsub(/\[wiki:([^\s\]]+)\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"} - text = text.gsub(/\[wiki:([^\s\]]+)\s(.*)\]/) {|s| "[[#{$1.delete(',./?;|:')}|#{$2.delete(',./?;|:')}]]"} - - # Links to pages UsingJustWikiCaps - text = text.gsub(/([^!]|^)(^| )([A-Z][a-z]+[A-Z][a-zA-Z]+)/, '\\1\\2[[\3]]') - # Normalize things that were supposed to not be links - # like !NotALink - text = text.gsub(/(^| )!([A-Z][A-Za-z]+)/, '\1\2') - # Revisions links - text = text.gsub(/\[(\d+)\]/, 'r\1') - # Ticket number re-writing - text = text.gsub(/#(\d+)/) do |s| - if $1.length < 10 -# TICKET_MAP[$1.to_i] ||= $1 - "\##{TICKET_MAP[$1.to_i] || $1}" - else - s - end - end - # We would like to convert the Code highlighting too - # This will go into the next line. - shebang_line = false - # Reguar expression for start of code - pre_re = /\{\{\{/ - # Code hightlighing... - shebang_re = /^\#\!([a-z]+)/ - # Regular expression for end of code - pre_end_re = /\}\}\}/ - - # Go through the whole text..extract it line by line - text = text.gsub(/^(.*)$/) do |line| - m_pre = pre_re.match(line) - if m_pre - line = '
'
- else
- m_sl = shebang_re.match(line)
- if m_sl
- shebang_line = true
- line = ''
- end
- m_pre_end = pre_end_re.match(line)
- if m_pre_end
- line = '
'
- if shebang_line
- line = '' + line
- end
- end
- end
- line
- end
-
- # Highlighting
- text = text.gsub(/'''''([^\s])/, '_*\1')
- text = text.gsub(/([^\s])'''''/, '\1*_')
- text = text.gsub(/'''/, '*')
- text = text.gsub(/''/, '_')
- text = text.gsub(/__/, '+')
- text = text.gsub(/~~/, '-')
- text = text.gsub(/`/, '@')
- text = text.gsub(/,,/, '~')
- # Lists
- text = text.gsub(/^([ ]+)\* /) {|s| '*' * $1.length + " "}
-
- text
+ convert_wiki_text_mapping(text, TICKET_MAP)
end
def self.migrate
@@ -400,6 +319,7 @@
# Milestones
print "Migrating milestones"
version_map = {}
+ milestone_wiki = Array.new
TracMilestone.find(:all).each do |milestone|
print '.'
STDOUT.flush
@@ -419,14 +339,16 @@
next unless v.save
version_map[milestone.name] = v
+ milestone_wiki.push(milestone.name);
migrated_milestones += 1
end
puts
-
+
# Custom fields
# TODO: read trac.ini instead
print "Migrating custom fields"
custom_field_map = {}
+
TracTicketCustom.find_by_sql("SELECT DISTINCT name FROM #{TracTicketCustom.table_name}").each do |field|
print '.'
STDOUT.flush
@@ -456,6 +378,16 @@
r.save!
custom_field_map['resolution'] = r
+ # Trac ticket id as a Redmine custom field
+ tid = IssueCustomField.find(:first, :conditions => { :name => "TracID" })
+ tid = IssueCustomField.new(:name => 'TracID',
+ :field_format => 'string',
+ :is_filter => true) if tid.nil?
+ tid.trackers = Tracker.find(:all)
+ tid.projects << @target_project
+ tid.save!
+ custom_field_map['tracid'] = tid
+
# Tickets
print "Migrating tickets"
TracTicket.find_each(:batch_size => 200) do |ticket|
@@ -463,7 +395,7 @@
STDOUT.flush
i = Issue.new :project => @target_project,
:subject => encode(ticket.summary[0, limit_for(Issue, 'subject')]),
- :description => convert_wiki_text(encode(ticket.description)),
+ :description => encode(ticket.description),
:priority => PRIORITY_MAPPING[ticket.priority] || DEFAULT_PRIORITY,
:created_on => ticket.time
i.author = find_or_create_user(ticket.reporter)
@@ -488,7 +420,7 @@
resolution_change = changeset.select {|change| change.field == 'resolution'}.first
comment_change = changeset.select {|change| change.field == 'comment'}.first
- n = Journal.new :notes => (comment_change ? convert_wiki_text(encode(comment_change.newvalue)) : ''),
+ n = Journal.new :notes => (comment_change ? encode(comment_change.newvalue) : ''),
:created_on => time
n.user = find_or_create_user(changeset.first.author)
n.journalized = i
@@ -534,6 +466,9 @@
if custom_field_map['resolution'] && !ticket.resolution.blank?
custom_values[custom_field_map['resolution'].id] = ticket.resolution
end
+ if custom_field_map['tracid']
+ custom_values[custom_field_map['tracid'].id] = ticket.id
+ end
i.custom_field_values = custom_values
i.save_custom_field_values
end
@@ -576,14 +511,67 @@
end
end
+ end
+ puts
+
+ # Now load each wiki page and transform its content into textile format
+ print "Fixing ticket identifiers"
+ puts
+
+ print "...in Wiki pages"
wiki.reload
wiki.pages.each do |page|
+ print '.'
page.content.text = convert_wiki_text(page.content.text)
Time.fake(page.content.updated_on) { page.content.save }
end
- end
- puts
+ puts
+ print "...in Issue descriptions"
+ TICKET_MAP.each do |newId|
+
+ next if newId.nil?
+
+ print '.'
+ issue = findIssue(newId)
+ next if issue.nil?
+
+ issue.description = convert_wiki_text(issue.description)
+ issue.save
+ end
+ puts
+
+ print "...in Issue journal descriptions"
+ TICKET_MAP.each do |newId|
+ next if newId.nil?
+
+ print '.'
+ issue = findIssue(newId)
+ next if issue.nil?
+
+ issue.journals.find(:all).each do |journal|
+ print '.'
+ journal.notes = convert_wiki_text(journal.notes)
+ journal.save
+ end
+
+ end
+ puts
+
+ print "...in Milestone descriptions"
+
+
+ # Now load each page and transform its content into textile format
+ milestone_wiki.each do |name|
+ p = wiki.find_page(name)
+ next if p.nil?
+
+ print '.'
+ p.content.text = convert_wiki_text(p.content.text)
+ p.content.save
+ end
+ puts
+
puts
puts "Components: #{migrated_components}/#{TracComponent.count}"
puts "Milestones: #{migrated_milestones}/#{TracMilestone.count}"
@@ -593,7 +581,18 @@
puts "Wiki edits: #{migrated_wiki_edits}/#{wiki_edit_count}"
puts "Wiki files: #{migrated_wiki_attachments}/" + TracAttachment.count(:conditions => {:type => 'wiki'}).to_s
end
+
+ def self.findIssue(id)
+
+ return Issue.find(id)
+ rescue ActiveRecord::RecordNotFound
+ puts
+ print "[#{id}] not found"
+
+ nil
+ end
+
def self.limit_for(klass, attribute)
klass.columns_hash[attribute.to_s].limit
end
@@ -746,7 +745,7 @@
DEFAULT_PORTS = {'mysql' => 3306, 'postgresql' => 5432}
prompt('Trac directory') {|directory| TracMigrate.set_trac_directory directory.strip}
- prompt('Trac database adapter (sqlite, sqlite3, mysql, postgresql)', :default => 'sqlite') {|adapter| TracMigrate.set_trac_adapter adapter}
+ prompt('Trac database adapter (sqlite, sqlite3, mysql, postgresql)', :default => 'sqlite3') {|adapter| TracMigrate.set_trac_adapter adapter}
unless %w(sqlite sqlite3).include?(TracMigrate.trac_adapter)
prompt('Trac database host', :default => 'localhost') {|host| TracMigrate.set_trac_db_host host}
prompt('Trac database port', :default => DEFAULT_PORTS[TracMigrate.trac_adapter]) {|port| TracMigrate.set_trac_db_port port}
@@ -764,5 +763,298 @@
TracMigrate.migrate
end
+
+
+ desc 'Subversion migration script'
+ task :migrate_svn_commits => :environment do
+
+ module SvnMigrate
+ TICKET_MAP = []
+
+ class Commit
+ attr_accessor :revision, :message
+
+ def initialize(attributes={})
+ self.message = attributes[:message] || ""
+ self.revision = attributes[:revision]
+ end
+ end
+
+ class SvnExtendedAdapter < Redmine::Scm::Adapters::SubversionAdapter
+
+
+
+ def set_message(path=nil, revision=nil, msg=nil)
+ path ||= ''
+
+ Tempfile.open('msg') do |tempfile|
+
+ # This is a weird thing. We need to cleanup cr/lf so we have uniform line separators
+ tempfile.print msg.gsub(/\r\n/,'\n')
+ tempfile.flush
+
+ filePath = tempfile.path.gsub(File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR)
+
+ cmd = "#{SVN_BIN} propset svn:log --quiet --revprop -r #{revision} -F \"#{filePath}\" "
+ cmd << credentials_string
+ cmd << ' ' + target(URI.escape(path))
+
+ shellout(cmd) do |io|
+ begin
+ loop do
+ line = io.readline
+ puts line
+ end
+ rescue EOFError
+ end
+ end
+
+ raise if $? && $?.exitstatus != 0
+
+ end
+
+ end
+
+ def messages(path=nil)
+ path ||= ''
+
+ commits = Array.new
+
+ cmd = "#{SVN_BIN} log --xml -r 1:HEAD"
+ cmd << credentials_string
+ cmd << ' ' + target(URI.escape(path))
+
+ shellout(cmd) do |io|
+ begin
+ doc = REXML::Document.new(io)
+ doc.elements.each("log/logentry") do |logentry|
+
+ commits << Commit.new(
+ {
+ :revision => logentry.attributes['revision'].to_i,
+ :message => logentry.elements['msg'].text
+ })
+ end
+ rescue => e
+ puts"Error !!!"
+ puts e
+ end
+ end
+ return nil if $? && $?.exitstatus != 0
+ commits
+ end
+
+ end
+
+ def self.migrate
+
+ project = Project.find(@@redmine_project)
+ if !project
+ puts "Could not find project identifier '#{@@redmine_project}'"
+ raise
+ end
+
+ tid = IssueCustomField.find(:first, :conditions => { :name => "TracID" })
+ if !tid
+ puts "Could not find issue custom field 'TracID'"
+ raise
+ end
+
+ Issue.find( :all, :conditions => { :project_id => project }).each do |issue|
+ val = nil
+ issue.custom_values.each do |value|
+ if value.custom_field.id == tid.id
+ val = value
+ break
+ end
+ end
+
+ TICKET_MAP[val.value.to_i] = issue.id if !val.nil?
+ end
+
+ svn = self.scm
+ msgs = svn.messages(@svn_url)
+ msgs.each do |commit|
+
+ newText = convert_wiki_text(commit.message)
+
+ if newText != commit.message
+ puts "Updating message #{commit.revision}"
+ scm.set_message(@svn_url, commit.revision, newText)
+ end
+ end
+
+
+ end
+
+ # Basic wiki syntax conversion
+ def self.convert_wiki_text(text)
+ convert_wiki_text_mapping(text, TICKET_MAP )
+ end
+
+ def self.set_svn_url(url)
+ @@svn_url = url
+ end
+
+ def self.set_svn_username(username)
+ @@svn_username = username
+ end
+
+ def self.set_svn_password(password)
+ @@svn_password = password
+ end
+
+ def self.set_redmine_project_identifier(identifier)
+ @@redmine_project = identifier
+ end
+
+ def self.scm
+ @scm ||= SvnExtendedAdapter.new @@svn_url, @@svn_url, @@svn_username, @@svn_password, 0, "", nil
+ @scm
+ end
+ end
+
+ def prompt(text, options = {}, &block)
+ default = options[:default] || ''
+ while true
+ print "#{text} [#{default}]: "
+ value = STDIN.gets.chomp!
+ value = default if value.blank?
+ break if yield value
+ end
+ end
+
+ puts
+ if Redmine::DefaultData::Loader.no_data?
+ puts "Redmine configuration need to be loaded before importing data."
+ puts "Please, run this first:"
+ puts
+ puts " rake redmine:load_default_data RAILS_ENV=\"#{ENV['RAILS_ENV']}\""
+ exit
+ end
+
+ puts "WARNING: all commit messages with references to trac pages will be modified"
+ print "Are you sure you want to continue ? [y/N] "
+ break unless STDIN.gets.match(/^y$/i)
+ puts
+
+ prompt('Subversion repository url') {|repository| SvnMigrate.set_svn_url repository.strip}
+ prompt('Subversion repository username') {|username| SvnMigrate.set_svn_username username}
+ prompt('Subversion repository password') {|password| SvnMigrate.set_svn_password password}
+ prompt('Redmine project identifier') {|identifier| SvnMigrate.set_redmine_project_identifier identifier}
+ puts
+
+ SvnMigrate.migrate
+
+ end
+
+
+ # Basic wiki syntax conversion
+ def convert_wiki_text_mapping(text, ticket_map = [])
+ # Images
+ text = text.gsub(/\[\[Image\(([a-zA-Z]+\:(\d+\:)?)?([^\)^,]+)[^\)]*\)\]\]/, '!\3!')
+ # Titles
+ text = text.gsub(/^(\=+)\s(.+)\s(\=+)/) {|s| "\nh#{$1.length}. #{$2}\n"}
+ # External Links
+ text = text.gsub(/\[(http[^\s]+)\s+([^\]]+)\]/) {|s| "\"#{$2}\":#{$1}"}
+ # Ticket links:
+ # [ticket:234 Text],[ticket:234 This is a test]
+ text = text.gsub(/\[ticket\:([^\ ]+)\ (.+?)\]/, '"\2":/issues/show/\1')
+ # ticket:1234
+ # #1 is working cause Redmine uses the same syntax.
+ text = text.gsub(/ticket\:([^\ ]+)/, '#\1')
+
+ # Source links:
+ # [source:/trunk/readme.txt Readme File]
+ # The text "Readme File" is not converted,
+ # cause Redmine's wiki does not support this.
+ text = text.gsub(/\[source\:\"([^\"]+)\"\ (.+?)\]/, 'source:"\1"')
+ # [source:/trunk/readme.txt]
+ text = text.gsub(/\[source\:\"([^\"]+)\"\]/, 'source:"\1"')
+ text = text.gsub(/source\:\"([^\"]+)\"/, 'source:"\1"')
+
+ # Milestone links:
+ # [milestone:"0.1.0 Mercury" Milestone 0.1.0 (Mercury)]
+ # The text "Milestone 0.1.0 (Mercury)" is not converted,
+ # cause Redmine's wiki does not support this.
+ text = text.gsub(/\[milestone\:\"([^\"]+)\"\ (.+?)\]/, 'version:"\1"')
+ # [milestone:"0.1.0 Mercury"]
+ text = text.gsub(/\[milestone\:\"([^\"]+)\"\]/, 'version:"\1"')
+ text = text.gsub(/milestone\:\"([^\"]+)\"/, 'version:"\1"')
+ # milestone:0.1.0
+ text = text.gsub(/\[milestone\:([^\ ]+)\]/, 'version:\1')
+ text = text.gsub(/milestone\:([^\ ]+)/, 'version:\1')
+ # Internal Links
+ text = text.gsub(/\[\[BR\]\]/, "\n") # This has to go before the rules below
+ text = text.gsub(/\[\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
+ text = text.gsub(/\[wiki:\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
+ text = text.gsub(/\[wiki:\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
+ text = text.gsub(/\[wiki:([^\s\]]+)\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
+ text = text.gsub(/\[wiki:([^\s\]]+)\s(.*)\]/) {|s| "[[#{$1.delete(',./?;|:')}|#{$2.delete(',./?;|:')}]]"}
+
+ # Links to pages UsingJustWikiCaps
+ text = text.gsub(/([^!]|^)(^| )([A-Z][a-z]+[A-Z][a-zA-Z]+)/, '\\1\\2[[\3]]')
+ # Normalize things that were supposed to not be links
+ # like !NotALink
+ text = text.gsub(/(^| )!([A-Z][A-Za-z]+)/, '\1\2')
+
+ # Revisions links
+ text = text.gsub(/\[(\d+)\]/, 'r\1')
+ # Ticket number re-writing
+ text = text.gsub(/#(\d+)/) do |s|
+ if $1.length < 10
+ ticket_map[$1.to_i] ||= $1
+ "\##{ticket_map[$1.to_i] || $1}"
+ else
+ s
+ end
+ end
+ # We would like to convert the Code highlighting too
+ # This will go into the next line.
+ shebang_line = false
+ # Reguar expression for start of code
+ pre_re = /\{\{\{/
+ # Code hightlighing...
+ shebang_re = /^\#\!([a-z]+)/
+ # Regular expression for end of code
+ pre_end_re = /\}\}\}/
+
+ # Go through the whole text..extract it line by line
+ text = text.gsub(/^(.*)$/) do |line|
+ m_pre = pre_re.match(line)
+ if m_pre
+ line = ''
+ else
+ m_sl = shebang_re.match(line)
+ if m_sl
+ shebang_line = true
+ line = ''
+ end
+ m_pre_end = pre_end_re.match(line)
+ if m_pre_end
+ line = '
'
+ if shebang_line
+ line = '' + line
+ end
+ end
+ end
+ line
+ end
+
+ # Highlighting
+ text = text.gsub(/'''''([^\s])/, '_*\1')
+ text = text.gsub(/([^\s])'''''/, '\1*_')
+ text = text.gsub(/'''/, '*')
+ text = text.gsub(/''/, '_')
+ text = text.gsub(/__/, '+')
+ text = text.gsub(/~~/, '-')
+ text = text.gsub(/`/, '@')
+ text = text.gsub(/,,/, '~')
+ # Lists
+ text = text.gsub(/^([ ]+)\* /) {|s| '*' * $1.length + " "}
+ text = text.gsub(/^([ ]+)[0-9]+\. /) {|s| '#' * $1.length + " "}
+
+ text
+ end
end