Project

General

Profile

Feature #1224 » migrate_from_mediawiki.rake

Carl Nygard, 2008-06-20 20:28

 
# redMine - project management software
# Copyright (C) 2006-2007 Jean-Philippe Lang
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

require 'active_record'
require 'iconv'
require 'pp'

namespace :redmine do
desc 'MediaWiki migration script'
task :migrate_from_mediawiki => :environment do
module MWMigrate

TRACKER_BUG = Tracker.find_by_position(1)
TRACKER_FEATURE = Tracker.find_by_position(2)
TRACKER_SUPPORT = Tracker.find_by_position(3)

class MWText < ActiveRecord::Base
set_table_name :text
set_primary_key :old_id
end

class MWRev < ActiveRecord::Base
set_table_name :revision
set_primary_key :rev_id
belongs_to :page, :class_name => "MWPage", :foreign_key => :rev_page
belongs_to :text, :class_name => "MWText", :foreign_key => :rev_text_id
end

class MWPage < ActiveRecord::Base
set_table_name :page
set_primary_key :page_id
has_many :revisions, :class_name => "MWRev", :foreign_key => :rev_page, :order => "rev_timestamp DESC"
end

def self.find_or_create_user(email, project_member = false)
u = User.find_by_mail(email)
if !u
u = User.find_by_mail(@@mw_default_user)
end
if(!u)
# Create a new user if not found
mail = email[0,limit_for(User, 'mail')]
mail = "#{mail}@fortna.com" unless mail.include?("@")
name = email[0,email.index("@")];
u = User.new :firstname => name[0,limit_for(User, 'firstname')].gsub(/[^\w\s\'\-]/i, '-'),
:lastname => '-',
:mail => mail.gsub(/[^-@a-z0-9\.]/i, '-')
u.login = email[0,limit_for(User, 'login')].gsub(/[^a-z0-9_\-@\.]/i, '-')
u.password = 'bugzilla'
u.admin = false
# finally, a default user is used if the new user is not valid
puts "Created User: "+ u.to_yaml
u = User.find(:first) unless u.save
else
# puts "Found User: " + u.to_yaml
end
# Make sure he is a member of the project
## if project_member && !u.member_of?(@target_project)
## role = ROLE_MAPPING['developer']
## Member.create(:user => u, :project => @target_project, :role => role)
## u.reload
## end
u
end
# Basic wiki syntax conversion
def self.convert_wiki_text(text)
# Titles
text = text.gsub(/^(\=+)\s*([^=]+)\s*\=+\s*$/) {|s| "\nh#{$1.length}. #{$2}\n"}

# Internal links
text = text.gsub(/\[\[(.*)\s+\|(.*)\]\]/) {|s| "[[#{$1}|#{$2}]]"}

# External Links
text = text.gsub(/\[(http[^\s]+)\s+([^\]]+)\]/) {|s| "\"#{$2}\":#{$1}"}
text = text.gsub(/\[(http[^\s]+)\]/) {|s| "#{$1}"}

# Highlighting
text = text.gsub(/'''''([^\s])/, '_*\1')
text = text.gsub(/([^\s])'''''/, '\1*_')
text = text.gsub(/'''([^\s])/, '*\1')
text = text.gsub(/([^\s])'''/, '\1*')
text = text.gsub(/''([^\s])/, '_*\1')
text = text.gsub(/([^\s])''/, '\1*_')

# code
text = text.gsub(/((^ [^\n]*\n)+)/m) { |s| "<pre>\n#{$1}</pre>\n" }
# text = text.gsub(/(^\n^ .*?$)/m) { |s| "<pre><code>#{$1}" }
# text = text.gsub(/(^ .*?\n)\n/m) { |s| "#{$1}</pre></code>\n" }

# Tables
# Half-assed attempt
# First strip off the table formatting
text = text.gsub(/^\![^\|]*/, '')
text = text.gsub(/^\{\|[^\|]*$/, '{|')

# Now congeal the rows
while( text.gsub!(/(\|-.*)\n(\|\w.*)$/m, '\1\2'))
end

# Now congeal the headers
while( text.gsub!(/(\{\|.*)\n(\|\w.*)$/m, '\1\2'))
end

# format the headers properly
while( text.gsub!(/(\{\|.*)\|([^_].*)$/, '\1|_. \2'))
end

# get rid of leading '{|'
text = text.gsub(/^\{\|(.*)$/) { |s| "table(stdtbl)\n#{$1}|" }

# get rid of leading '|-'
text = text.gsub(/^\|-(.*)$/, '\1|')

# get rid of trailing '|}'
text = text.gsub(/^\|\}.*$/, '')

# Internal Links
text = text.gsub(/\[\[Image:([^\s]+)\]\]/) { |s| "!#{$1}!" }

# Wiki page separator ':'
while( text.gsub!(/(\[\[\s*\w+):(\w+)/, '\1_\2') )
end

text
end
def self.migrate
establish_connection

# Quick database test
pages = MWPage.count
migrated_wiki_edits = 0

puts "No wiki defined" unless @target_project.wiki
wiki = @target_project.wiki ||
Wiki.new(:project => @target_project,
:start_page => @target_project.name)

# Wiki
puts "Migrating #{mw_page_title}, 1 of #{pages} pages"
pages = MWPage.find(:all,
:conditions => ["page_title = ?", mw_page_title])

if((pages.size > 0) && (@@mw_whole_namespace == "y" || @@mw_whole_namespace == "Y"))
pages = MWPage.find(:all,
:conditions => ["page_namespace = ?", pages[0].page_namespace])
end

pages.each do |page|
print "Translate #{page.page_title} (y/N)? "
next unless STDIN.gets.match(/^[yY]$/i)

STDOUT.flush
new_title = page.page_title.gsub(/:/, "_")
p = wiki.find_or_new_page(new_title)
p.content = WikiContent.new(:page => p) if p.new_record?
p.content.text = convert_wiki_text(page.revisions[0].text.old_text)
p.content.author = User.find_by_mail(@@mw_default_user)
p.content.comments = page.revisions[0].rev_comment
puts "Record: " + p.content.to_s
puts " Text: " + p.content.text
print "Save translated page (y/N)? "
next unless STDIN.gets.match(/^[yY]$/i)

p.new_record? ? p.save : p.content.save
migrated_wiki_edits += 1 unless p.content.new_record?
end
puts
puts "Wiki edits: #{migrated_wiki_edits}/#{MWPage.count}"
end
def self.limit_for(klass, attribute)
klass.columns_hash[attribute.to_s].limit
end
def self.encoding(charset)
@ic = Iconv.new('UTF-8', charset)
rescue Iconv::InvalidEncoding
puts "Invalid encoding!"
return false
end
def self.set_mw_directory(path)
@@bz_directory = path
raise "This directory doesn't exist!" unless File.directory?(path)
@@bz_directory
rescue Exception => e
puts e
return false
end

def self.mw_directory
@@mw_directory
end

def self.set_mw_adapter(adapter)
return false if adapter.blank?
raise "Unknown adapter: #{adapter}!" unless %w(sqlite sqlite3 mysql postgresql).include?(adapter)
# If adapter is sqlite or sqlite3, make sure that mw.db exists
raise "#{mw_db_path} doesn't exist!" if %w(sqlite sqlite3).include?(adapter) && !File.exist?(mw_db_path)
@@mw_adapter = adapter
rescue Exception => e
puts e
return false
end
def self.set_mw_db_host(host)
return nil if host.blank?
@@mw_db_host = host
end

def self.set_mw_db_port(port)
return nil if port.to_i == 0
@@mw_db_port = port.to_i
end
def self.set_mw_db_socket(sock)
@@mw_db_socket = sock
end
def self.set_mw_db_name(name)
return nil if name.blank?
@@mw_db_name = name
end

def self.set_mw_db_username(username)
@@mw_db_username = username
end
def self.set_mw_db_password(password)
@@mw_db_password = password
end
def self.set_mw_default_user(username)
@@mw_default_user = username
end

def self.set_mw_page_title(name)
@@mw_page_title = name
end

def self.set_mw_whole_namespace(flag)
@@mw_whole_namespace = flag
end

mattr_reader :mw_directory, :mw_adapter, :mw_db_host, :mw_db_port, :mw_db_name, :mw_db_username, :mw_db_password, :mw_db_socket, :mw_page_title, :mw_whole_namespace
def self.mw_db_path; "#{mw_directory}/db/wiki.db" end
def self.mw_attachments_directory; "#{mw_directory}/attachments" end

def self.target_project_identifier(identifier)
project = Project.find_by_identifier(identifier)
if !project
# create the target project
project = Project.new :name => identifier.humanize,
:description => identifier.humanize
project.identifier = identifier
puts "Created Project: "+ project.to_s
puts "Unable to create a project with identifier '#{identifier}'!" unless project.save
# enable issues and wiki for the created project
project.enabled_module_names = ['issue_tracking', 'wiki']
project.trackers << TRACKER_BUG
project.trackers << TRACKER_FEATURE
project.trackers << TRACKER_SUPPORT
else
puts "Found Project: " + project.to_yaml
end
@target_project = project.new_record? ? nil : project
end
def self.connection_params
if %w(sqlite sqlite3).include?(mw_adapter)
{:adapter => mw_adapter,
:database => mw_db_path}
else
{:adapter => mw_adapter,
:database => mw_db_name,
:host => mw_db_host,
:port => mw_db_port,
:socket => mw_db_socket,
:username => mw_db_username,
:password => mw_db_password}
end
end
def self.establish_connection
constants.each do |const|
klass = const_get(const)
next unless klass.respond_to? 'establish_connection'
klass.establish_connection connection_params
end
end
private
def self.encode(text)
@ic.iconv text
rescue
text
end
end
puts
puts "WARNING: a new project will be added to Redmine during this process."
print "Are you sure you want to continue ? [y/N] "
break unless STDIN.gets.match(/^[yY]$/i)
puts

def prompt(text, options = {}, &block)
default = options[:default] || ''
while true
print "#{text} [#{default}]: "
value = STDIN.gets.chomp!
value = default if value.blank?
break if yield value
end
end
DEFAULT_PORTS = {'mysql' => 3306, 'postgresl' => 5432}
DEFAULT_SOCKETS = {'mysql' => '/var/lib/mysql/mysql.sock'}
prompt('MW directory',:default => '/var/www/html/mediawiki-1.8.2') {|directory| MWMigrate.set_mw_directory directory}
prompt('MW database adapter (sqlite, sqlite3, mysql, postgresql)', :default => 'mysql') {|adapter| MWMigrate.set_mw_adapter adapter}
unless %w(sqlite sqlite3).include?(MWMigrate.mw_adapter)
prompt('MW database host', :default => 'localhost') {|host| MWMigrate.set_mw_db_host host}
prompt('MW database port', :default => DEFAULT_PORTS[MWMigrate.mw_adapter]) {|port| MWMigrate.set_mw_db_port port}
prompt('MW database socket', :default => DEFAULT_SOCKETS[MWMigrate.mw_adapter]) {|sock| MWMigrate.set_mw_db_socket sock}
prompt('MW database name', :default => 'wikidb') {|name| MWMigrate.set_mw_db_name name}
prompt('MW database username', :default => 'wiki') {|username| MWMigrate.set_mw_db_username username}
prompt('MW database password', :default => 'wikidb') {|password| MWMigrate.set_mw_db_password password}
end
prompt('MW database encoding', :default => 'UTF-8') {|encoding| MWMigrate.encoding encoding}
prompt('Target project identifier', :default => 'CommCore') {|identifier| MWMigrate.target_project_identifier identifier}
prompt('MW Page Title', :default => 'CommCore:Devel:XMLDB') {|identifier| MWMigrate.set_mw_page_title identifier}
prompt('Page Author', :default => 'carlnygard@fortna.com') {|identifier| MWMigrate.set_mw_default_user identifier}
prompt('Whole namespace (Y/n)?', :default => 'y') {|flag| MWMigrate.set_mw_whole_namespace flag}
puts
MWMigrate.migrate
end
end
(2-2/2)