Index: lib/tasks/migrate_from_trac.rake
===================================================================
--- lib/tasks/migrate_from_trac.rake (revision 3603)
+++ lib/tasks/migrate_from_trac.rake (working copy)
@@ -19,6 +19,13 @@
require 'iconv'
require 'pp'
+require 'redmine/scm/adapters/abstract_adapter'
+require 'redmine/scm/adapters/subversion_adapter'
+require 'rexml/document'
+require 'uri'
+
+require 'tempfile'
+
namespace :redmine do
desc 'Trac migration script'
task :migrate_from_trac => :environment do
@@ -192,14 +199,14 @@
def time; Time.at(read_attribute(:time)) end
end
- TRAC_WIKI_PAGES = %w(InterMapTxt InterTrac InterWiki RecentChanges SandBox TracAccessibility TracAdmin TracBackup TracBrowser TracCgi TracChangeset \
+ TRAC_WIKI_PAGES = %w(InterMapTxt InterTrac InterWiki RecentChanges SandBox TracAccessibility TracAdmin TracBackup \
+ TracBrowser TracCgi TracChangeset TracInstallPlatforms TracMultipleProjects TracModWSGI \
TracEnvironment TracFastCgi TracGuide TracImport TracIni TracInstall TracInterfaceCustomization \
TracLinks TracLogging TracModPython TracNotification TracPermissions TracPlugins TracQuery \
TracReports TracRevisionLog TracRoadmap TracRss TracSearch TracStandalone TracSupport TracSyntaxColoring TracTickets \
TracTicketsCustomFields TracTimeline TracUnicode TracUpgrade TracWiki WikiDeletePage WikiFormatting \
WikiHtml WikiMacros WikiNewPage WikiPageNames WikiProcessors WikiRestructuredText WikiRestructuredTextLinks \
CamelCase TitleIndex)
-
class TracWikiPage < ActiveRecord::Base
set_table_name :wiki
set_primary_key :name
@@ -241,7 +248,7 @@
if name_attr = TracSessionAttribute.find_by_sid_and_name(username, 'name')
name = name_attr.value
end
- name =~ (/(.*)(\s+\w+)?/)
+ name =~ (/(.+?)(?:[\ \t]+(.+)?|[\ \t]+|)$/)
fn = $1.strip
ln = ($2 || '-').strip
@@ -271,96 +278,7 @@
# Basic wiki syntax conversion
def self.convert_wiki_text(text)
- # Titles
- text = text.gsub(/^(\=+)\s(.+)\s(\=+)/) {|s| "\nh#{$1.length}. #{$2}\n"}
- # External Links
- text = text.gsub(/\[(http[^\s]+)\s+([^\]]+)\]/) {|s| "\"#{$2}\":#{$1}"}
- # Ticket links:
- # [ticket:234 Text],[ticket:234 This is a test]
- text = text.gsub(/\[ticket\:([^\ ]+)\ (.+?)\]/, '"\2":/issues/show/\1')
- # ticket:1234
- # #1 is working cause Redmine uses the same syntax.
- text = text.gsub(/ticket\:([^\ ]+)/, '#\1')
- # Milestone links:
- # [milestone:"0.1.0 Mercury" Milestone 0.1.0 (Mercury)]
- # The text "Milestone 0.1.0 (Mercury)" is not converted,
- # cause Redmine's wiki does not support this.
- text = text.gsub(/\[milestone\:\"([^\"]+)\"\ (.+?)\]/, 'version:"\1"')
- # [milestone:"0.1.0 Mercury"]
- text = text.gsub(/\[milestone\:\"([^\"]+)\"\]/, 'version:"\1"')
- text = text.gsub(/milestone\:\"([^\"]+)\"/, 'version:"\1"')
- # milestone:0.1.0
- text = text.gsub(/\[milestone\:([^\ ]+)\]/, 'version:\1')
- text = text.gsub(/milestone\:([^\ ]+)/, 'version:\1')
- # Internal Links
- text = text.gsub(/\[\[BR\]\]/, "\n") # This has to go before the rules below
- text = text.gsub(/\[\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
- text = text.gsub(/\[wiki:\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
- text = text.gsub(/\[wiki:\"(.+)\".*\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
- text = text.gsub(/\[wiki:([^\s\]]+)\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
- text = text.gsub(/\[wiki:([^\s\]]+)\s(.*)\]/) {|s| "[[#{$1.delete(',./?;|:')}|#{$2.delete(',./?;|:')}]]"}
-
- # Links to pages UsingJustWikiCaps
- text = text.gsub(/([^!]|^)(^| )([A-Z][a-z]+[A-Z][a-zA-Z]+)/, '\\1\\2[[\3]]')
- # Normalize things that were supposed to not be links
- # like !NotALink
- text = text.gsub(/(^| )!([A-Z][A-Za-z]+)/, '\1\2')
- # Revisions links
- text = text.gsub(/\[(\d+)\]/, 'r\1')
- # Ticket number re-writing
- text = text.gsub(/#(\d+)/) do |s|
- if $1.length < 10
-# TICKET_MAP[$1.to_i] ||= $1
- "\##{TICKET_MAP[$1.to_i] || $1}"
- else
- s
- end
- end
- # We would like to convert the Code highlighting too
- # This will go into the next line.
- shebang_line = false
- # Reguar expression for start of code
- pre_re = /\{\{\{/
- # Code hightlighing...
- shebang_re = /^\#\!([a-z]+)/
- # Regular expression for end of code
- pre_end_re = /\}\}\}/
-
- # Go through the whole text..extract it line by line
- text = text.gsub(/^(.*)$/) do |line|
- m_pre = pre_re.match(line)
- if m_pre
- line = ''
- else
- m_sl = shebang_re.match(line)
- if m_sl
- shebang_line = true
- line = ''
- end
- m_pre_end = pre_end_re.match(line)
- if m_pre_end
- line = '
'
- if shebang_line
- line = '' + line
- end
- end
- end
- line
- end
-
- # Highlighting
- text = text.gsub(/'''''([^\s])/, '_*\1')
- text = text.gsub(/([^\s])'''''/, '\1*_')
- text = text.gsub(/'''/, '*')
- text = text.gsub(/''/, '_')
- text = text.gsub(/__/, '+')
- text = text.gsub(/~~/, '-')
- text = text.gsub(/`/, '@')
- text = text.gsub(/,,/, '~')
- # Lists
- text = text.gsub(/^([ ]+)\* /) {|s| '*' * $1.length + " "}
-
- text
+ convert_wiki_text_mapping(text, TICKET_MAP)
end
def self.migrate
@@ -400,6 +318,7 @@
# Milestones
print "Migrating milestones"
version_map = {}
+ milestone_wiki = Array.new
TracMilestone.find(:all).each do |milestone|
print '.'
STDOUT.flush
@@ -419,6 +338,7 @@
next unless v.save
version_map[milestone.name] = v
+ milestone_wiki.push(milestone.name);
migrated_milestones += 1
end
puts
@@ -456,6 +376,16 @@
r.save!
custom_field_map['resolution'] = r
+ # Trac ticket id as a Redmine custom field
+ tid = IssueCustomField.find(:first, :conditions => { :name => "TracID" })
+ tid = IssueCustomField.new(:name => 'TracID',
+ :field_format => 'string',
+ :is_filter => true) if tid.nil?
+ tid.trackers = Tracker.find(:all)
+ tid.projects << @target_project
+ tid.save!
+ custom_field_map['tracid'] = tid
+
# Tickets
print "Migrating tickets"
TracTicket.find_each(:batch_size => 200) do |ticket|
@@ -463,7 +393,7 @@
STDOUT.flush
i = Issue.new :project => @target_project,
:subject => encode(ticket.summary[0, limit_for(Issue, 'subject')]),
- :description => convert_wiki_text(encode(ticket.description)),
+ :description => encode(ticket.description),
:priority => PRIORITY_MAPPING[ticket.priority] || DEFAULT_PRIORITY,
:created_on => ticket.time
i.author = find_or_create_user(ticket.reporter)
@@ -488,7 +418,7 @@
resolution_change = changeset.select {|change| change.field == 'resolution'}.first
comment_change = changeset.select {|change| change.field == 'comment'}.first
- n = Journal.new :notes => (comment_change ? convert_wiki_text(encode(comment_change.newvalue)) : ''),
+ n = Journal.new :notes => (comment_change ? encode(comment_change.newvalue) : ''),
:created_on => time
n.user = find_or_create_user(changeset.first.author)
n.journalized = i
@@ -534,6 +464,9 @@
if custom_field_map['resolution'] && !ticket.resolution.blank?
custom_values[custom_field_map['resolution'].id] = ticket.resolution
end
+ if custom_field_map['tracid']
+ custom_values[custom_field_map['tracid'].id] = ticket.id
+ end
i.custom_field_values = custom_values
i.save_custom_field_values
end
@@ -576,14 +509,64 @@
end
end
+ end
+ puts
+
+ # Now load each wiki page and transform its content into textile format
+ print "Transform texts to textile format:"
+ puts
+
+ print " in Wiki pages"
wiki.reload
wiki.pages.each do |page|
+ print '.'
page.content.text = convert_wiki_text(page.content.text)
Time.fake(page.content.updated_on) { page.content.save }
end
- end
- puts
+ puts
+ print " in Issue descriptions"
+ TICKET_MAP.each do |newId|
+
+ next if newId.nil?
+
+ print '.'
+ issue = findIssue(newId)
+ next if issue.nil?
+
+ issue.description = convert_wiki_text(issue.description)
+ issue.save
+ end
+ puts
+
+ print " in Issue journal descriptions"
+ TICKET_MAP.each do |newId|
+ next if newId.nil?
+
+ print '.'
+ issue = findIssue(newId)
+ next if issue.nil?
+
+ issue.journals.find(:all).each do |journal|
+ print '.'
+ journal.notes = convert_wiki_text(journal.notes)
+ journal.save
+ end
+
+ end
+ puts
+
+ print " in Milestone descriptions"
+ milestone_wiki.each do |name|
+ p = wiki.find_page(name)
+ next if p.nil?
+
+ print '.'
+ p.content.text = convert_wiki_text(p.content.text)
+ p.content.save
+ end
+ puts
+
puts
puts "Components: #{migrated_components}/#{TracComponent.count}"
puts "Milestones: #{migrated_milestones}/#{TracMilestone.count}"
@@ -593,7 +576,18 @@
puts "Wiki edits: #{migrated_wiki_edits}/#{wiki_edit_count}"
puts "Wiki files: #{migrated_wiki_attachments}/" + TracAttachment.count(:conditions => {:type => 'wiki'}).to_s
end
+
+ def self.findIssue(id)
+
+ return Issue.find(id)
+ rescue ActiveRecord::RecordNotFound
+ puts
+ print "[#{id}] not found"
+
+ nil
+ end
+
def self.limit_for(klass, attribute)
klass.columns_hash[attribute.to_s].limit
end
@@ -746,7 +740,7 @@
DEFAULT_PORTS = {'mysql' => 3306, 'postgresql' => 5432}
prompt('Trac directory') {|directory| TracMigrate.set_trac_directory directory.strip}
- prompt('Trac database adapter (sqlite, sqlite3, mysql, postgresql)', :default => 'sqlite') {|adapter| TracMigrate.set_trac_adapter adapter}
+ prompt('Trac database adapter (sqlite, sqlite3, mysql, postgresql)', :default => 'sqlite3') {|adapter| TracMigrate.set_trac_adapter adapter}
unless %w(sqlite sqlite3).include?(TracMigrate.trac_adapter)
prompt('Trac database host', :default => 'localhost') {|host| TracMigrate.set_trac_db_host host}
prompt('Trac database port', :default => DEFAULT_PORTS[TracMigrate.trac_adapter]) {|port| TracMigrate.set_trac_db_port port}
@@ -764,5 +758,327 @@
TracMigrate.migrate
end
+
+
+ desc 'Subversion migration script'
+ task :migrate_from_trac_svn => :environment do
+
+ module SvnMigrate
+ TICKET_MAP = []
+
+ class Commit
+ attr_accessor :revision, :message
+
+ def initialize(attributes={})
+ self.message = attributes[:message] || ""
+ self.revision = attributes[:revision]
+ end
+ end
+
+ class SvnExtendedAdapter < Redmine::Scm::Adapters::SubversionAdapter
+
+
+
+ def set_message(path=nil, revision=nil, msg=nil)
+ path ||= ''
+
+ Tempfile.open('msg') do |tempfile|
+
+ # This is a weird thing. We need to cleanup cr/lf so we have uniform line separators
+ tempfile.print msg.gsub(/\r\n/,'\n')
+ tempfile.flush
+
+ filePath = tempfile.path.gsub(File::SEPARATOR, File::ALT_SEPARATOR || File::SEPARATOR)
+
+ cmd = "#{SVN_BIN} propset svn:log --quiet --revprop -r #{revision} -F \"#{filePath}\" "
+ cmd << credentials_string
+ cmd << ' ' + target(URI.escape(path))
+
+ shellout(cmd) do |io|
+ begin
+ loop do
+ line = io.readline
+ puts line
+ end
+ rescue EOFError
+ end
+ end
+
+ raise if $? && $?.exitstatus != 0
+
+ end
+
+ end
+
+ def messages(path=nil)
+ path ||= ''
+
+ commits = Array.new
+
+ cmd = "#{SVN_BIN} log --xml -r 1:HEAD"
+ cmd << credentials_string
+ cmd << ' ' + target(URI.escape(path))
+
+ shellout(cmd) do |io|
+ begin
+ doc = REXML::Document.new(io)
+ doc.elements.each("log/logentry") do |logentry|
+
+ commits << Commit.new(
+ {
+ :revision => logentry.attributes['revision'].to_i,
+ :message => logentry.elements['msg'].text
+ })
+ end
+ rescue => e
+ puts"Error !!!"
+ puts e
+ end
+ end
+ return nil if $? && $?.exitstatus != 0
+ commits
+ end
+
+ end
+
+ def self.migrate
+
+ project = Project.find(@@redmine_project)
+ if !project
+ puts "Could not find project identifier '#{@@redmine_project}'"
+ raise
+ end
+
+ tid = IssueCustomField.find(:first, :conditions => { :name => "TracID" })
+ if !tid
+ puts "Could not find issue custom field 'TracID'"
+ raise
+ end
+
+ Issue.find( :all, :conditions => { :project_id => project }).each do |issue|
+ val = nil
+ issue.custom_values.each do |value|
+ if value.custom_field.id == tid.id
+ val = value
+ break
+ end
+ end
+
+ TICKET_MAP[val.value.to_i] = issue.id if !val.nil?
+ end
+
+ svn = self.scm
+ msgs = svn.messages(@svn_url)
+ msgs.each do |commit|
+
+ newText = convert_wiki_text(commit.message)
+
+ if newText != commit.message
+ puts "Updating message #{commit.revision}"
+ scm.set_message(@svn_url, commit.revision, newText)
+ end
+ end
+
+
+ end
+
+ # Basic wiki syntax conversion
+ def self.convert_wiki_text(text)
+ convert_wiki_text_mapping(text, TICKET_MAP )
+ end
+
+ def self.set_svn_url(url)
+ @@svn_url = url
+ end
+
+ def self.set_svn_username(username)
+ @@svn_username = username
+ end
+
+ def self.set_svn_password(password)
+ @@svn_password = password
+ end
+
+ def self.set_redmine_project_identifier(identifier)
+ @@redmine_project = identifier
+ end
+
+ def self.scm
+ @scm ||= SvnExtendedAdapter.new @@svn_url, @@svn_url, @@svn_username, @@svn_password, 0, "", nil
+ @scm
+ end
+ end
+
+ def prompt(text, options = {}, &block)
+ default = options[:default] || ''
+ while true
+ print "#{text} [#{default}]: "
+ value = STDIN.gets.chomp!
+ value = default if value.blank?
+ break if yield value
+ end
+ end
+
+ puts
+ if Redmine::DefaultData::Loader.no_data?
+ puts "Redmine configuration need to be loaded before importing data."
+ puts "Please, run this first:"
+ puts
+ puts " rake redmine:load_default_data RAILS_ENV=\"#{ENV['RAILS_ENV']}\""
+ exit
+ end
+
+ puts "WARNING: all commit messages with references to trac pages will be modified"
+ print "Are you sure you want to continue ? [y/N] "
+ break unless STDIN.gets.match(/^y$/i)
+ puts
+
+ prompt('Subversion repository url') {|repository| SvnMigrate.set_svn_url repository.strip}
+ prompt('Subversion repository username') {|username| SvnMigrate.set_svn_username username}
+ prompt('Subversion repository password') {|password| SvnMigrate.set_svn_password password}
+ prompt('Redmine project identifier') {|identifier| SvnMigrate.set_redmine_project_identifier identifier}
+ puts
+
+ SvnMigrate.migrate
+
+ end
+
+
+ # Basic wiki syntax conversion
+ def convert_wiki_text_mapping(text, ticket_map = [])
+ # New line
+ text = text.gsub(/\[\[BR\]\]/, "\n") # This has to go before the rules below
+ # Titles (only h1. to h6., and remove #...)
+ text = text.gsub(/(?:^|^\ +)(\={1,6})\ (.+)\ (?:\1)(?:\ *(\ \#.*))?/) {|s| "\nh#{$1.length}. #{$2}#{$3}\n"}
+
+ # External Links:
+ # [http://example.com/]
+ text = text.gsub(/\[((?:https?|s?ftp)\:\S+)\]/, '\1')
+ # [http://example.com/ Example],[http://example.com/ "Example"]
+ text = text.gsub(/\[((?:https?|s?ftp)\:\S+)[\ \t]+([\"']?)(.+?)\2\]/, '"\3":\1')
+ # [mailto:some@example.com],[mailto:"some@example.com"]
+ text = text.gsub(/\[mailto\:([\"']?)(.+?)\1\]/, '\2')
+
+ # Ticket links:
+ # [ticket:234 Text],[ticket:234 This is a test],[ticket:234 "This is a test"]
+ text = text.gsub(/\[ticket\:([^\ ]+)[\ \t]+([\"']?)(.+?)\2\]/, '"\3":/issues/show/\1')
+ # ticket:1234
+ # #1 - working cause Redmine uses the same syntax.
+ text = text.gsub(/ticket\:([^\ ]+?)/, '#\1')
+
+ # Source links:
+ # [source:/trunk/readme.txt Readme File],[source:"/trunk/readme.txt" Readme File],
+ # [source:/trunk/readme.txt],[source:"/trunk/readme.txt"]
+ # The text "Readme File" is not converted,
+ # cause Redmine's wiki does not support this.
+ text = text.gsub(/\[source\:([\"']?)([^\"']+?)\1(?:\ +(.+?))?\]/, 'source:"\2"')
+ # source:"/trunk/readme.txt"
+ # source:/trunk/readme.txt - working cause Redmine uses the same syntax.
+ text = text.gsub(/source\:([\"'])([^\"']+?)\1/, 'source:"\2"')
+
+ # Milestone links:
+ # [milestone:"0.1.0 Mercury" Milestone 0.1.0 (Mercury)],
+ # [milestone:"0.1.0 Mercury"],milestone:"0.1.0 Mercury"
+ # The text "Milestone 0.1.0 (Mercury)" is not converted,
+ # cause Redmine's wiki does not support this.
+ text = text.gsub(/\[milestone\:([\"'])([^\"']+?)\1(?:\ +(.+?))?\]/, 'version:"\2"')
+ text = text.gsub(/milestone\:([\"'])([^\"']+?)\1/, 'version:"\2"')
+ # [milestone:0.1.0],milestone:0.1.0
+ text = text.gsub(/\[milestone\:([^\ ]+?)\]/, 'version:\1')
+ text = text.gsub(/milestone\:([^\ ]+?)/, 'version:\1')
+
+ # Internal Links:
+ # ["Some Link"]
+ text = text.gsub(/\[([\"'])(.+?)\1\]/) {|s| "[[#{$2.delete(',./?;|:')}]]"}
+ # [wiki:"Some Link" "Link description"],[wiki:"Some Link" Link description]
+ text = text.gsub(/\[wiki\:([\"'])([^\]\"']+?)\1[\ \t]+([\"']?)(.+?)\3\]/) {|s| "[[#{$2.delete(',./?;|:')}|#{$4}]]"}
+ # [wiki:"Some Link"]
+ text = text.gsub(/\[wiki\:([\"'])([^\]\"']+?)\1\]/) {|s| "[[#{$2.delete(',./?;|:')}]]"}
+ # [wiki:SomeLink]
+ text = text.gsub(/\[wiki\:([^\s\]]+?)\]/) {|s| "[[#{$1.delete(',./?;|:')}]]"}
+ # [wiki:SomeLink Link description],[wiki:SomeLink "Link description"]
+ text = text.gsub(/\[wiki\:([^\s\]\"']+?)[\ \t]+([\"']?)(.+?)\2\]/) {|s| "[[#{$1.delete(',./?;|:')}|#{$3}]]"}
+
+ # Links to pages UsingJustWikiCaps (not work for unicode)
+ text = text.gsub(/([^!]|^)(^| )([A-Z][a-z]+[A-Z][a-zA-Z]+)/, '\\1\\2[[\3]]')
+ # Normalize things that were supposed to not be links
+ # like !NotALink
+ text = text.gsub(/(^| )!([A-Z][A-Za-z]+)/, '\1\2')
+
+ # Revisions links
+ text = text.gsub(/\[(\d+)\]/, 'r\1')
+ # Ticket number re-writing
+ text = text.gsub(/#(\d+)/) do |s|
+ if $1.length < 10
+# ticket_map[$1.to_i] ||= $1
+ "\##{ticket_map[$1.to_i] || $1}"
+ else
+ s
+ end
+ end
+
+ # Before convert Code highlighting, need processing inline code
+ # {{{hello world}}}
+ text = text.gsub(/\{\{\{(.+?)\}\}\}/, '@\1@')
+
+ # We would like to convert the Code highlighting too
+ # This will go into the next line.
+ shebang_line = false
+ # Reguar expression for start of code
+ pre_re = /\{\{\{/
+ # Code hightlighing...
+ shebang_re = /^\#\!([a-z]+)/
+ # Regular expression for end of code
+ pre_end_re = /\}\}\}/
+
+ # Go through the whole text..extract it line by line
+ text = text.gsub(/^(.*)$/) do |line|
+ m_pre = pre_re.match(line)
+ if m_pre
+ line = ''
+ else
+ m_sl = shebang_re.match(line)
+ if m_sl
+ shebang_line = true
+ line = ''
+ end
+ m_pre_end = pre_end_re.match(line)
+ if m_pre_end
+ line = '
'
+ if shebang_line
+ line = '' + line
+ end
+ end
+ end
+ line
+ end
+
+ # Highlighting
+ text = text.gsub(/'''''([^\s])/, '_*\1')
+ text = text.gsub(/([^\s])'''''/, '\1*_')
+ text = text.gsub(/'''/, '*')
+ text = text.gsub(/''/, '_')
+ text = text.gsub(/__/, '+')
+ text = text.gsub(/~~/, '-')
+ text = text.gsub(/`/, '@')
+ text = text.gsub(/,,/, '~')
+ # Tables
+ text = text.gsub(/\|\|/, '|')
+ # Lists:
+ # bullet
+ text = text.gsub(/^(\ +)\* /) {|s| '*' * $1.length + " "}
+ # numbered
+ text = text.gsub(/^(\ +)\d+\. /) {|s| '#' * $1.length + " "}
+ # Images (work for only attached in current page [[Image(picture.gif)]])
+ # need rules for: * [[Image(wiki:WikiFormatting:picture.gif)]] (referring to attachment on another page)
+ # * [[Image(ticket:1:picture.gif)]] (file attached to a ticket)
+ # * [[Image(htdocs:picture.gif)]] (referring to a file inside project htdocs)
+ # * [[Image(source:/trunk/trac/htdocs/trac_logo_mini.png)]] (a file in repository)
+ text = text.gsub(/\[\[image\((.+?)(?:,.+?)?\)\]\]/i, '!\1!')
+ # TOC
+ text = text.gsub(/\[\[TOC(?:\((.*?)\))?\]\]/m) {|s| "{{>toc}}\n"}
+
+ text
+ end
end