Feature #306 » 0002-implements-fulltext-extraction-for-attachments.patch
| app/controllers/admin_controller.rb | ||
|---|---|---|
|
[:text_file_repository_writable, File.writable?(Attachment.storage_path)],
|
||
|
["#{l :text_plugin_assets_writable} (./public/plugin_assets)", File.writable?(Redmine::Plugin.public_directory)],
|
||
|
[:text_rmagick_available, Object.const_defined?(:Magick)],
|
||
|
[:text_convert_available, Redmine::Thumbnail.convert_available?]
|
||
|
[:text_convert_available, Redmine::Thumbnail.convert_available?],
|
||
|
[:text_pdftotext_available, Redmine::TextExtractor::PdfHandler.available?],
|
||
|
[:text_unrtf_available, Redmine::TextExtractor::RtfHandler.available?],
|
||
|
[:text_catdoc_available, Redmine::TextExtractor::DocHandler.available?],
|
||
|
[:text_xls2csv_available, Redmine::TextExtractor::XlsHandler.available?],
|
||
|
[:text_catppt_available, Redmine::TextExtractor::PptHandler.available?]
|
||
|
]
|
||
|
end
|
||
|
end
|
||
| config/configuration.yml.example | ||
|---|---|---|
|
# allowed values: :memory, :file, :memcache
|
||
|
#openid_authentication_store: :memory
|
||
|
# Text extraction helper programs.
|
||
|
#
|
||
|
# commands should write the resulting plain text to STDOUT. Use __FILE__ as
|
||
|
# placeholder for the file path. The values below are the defaults.
|
||
|
text_extractors:
|
||
|
# apt install poppler-utils
|
||
|
# pdftotext:
|
||
|
# - /usr/bin/pdftotext
|
||
|
# - -enc
|
||
|
# - UTF-8
|
||
|
# - __FILE__
|
||
|
# - '-'
|
||
|
# apt install unrtf
|
||
|
# unrtf:
|
||
|
# - /usr/bin/unrtf
|
||
|
# - --text
|
||
|
# - __FILE__
|
||
|
# apt install catdoc
|
||
|
# catdoc:
|
||
|
# - /usr/bin/catdoc
|
||
|
# - -dutf-8
|
||
|
# - __FILE__
|
||
|
# xls2csv:
|
||
|
# - /usr/bin/xls2csv
|
||
|
# - -dutf-8
|
||
|
# - __FILE__
|
||
|
# catppt:
|
||
|
# - /usr/bin/catppt
|
||
|
# - -dutf-8
|
||
|
# - __FILE__
|
||
|
# specific configuration options for production environment
|
||
|
# that overrides the default ones
|
||
|
production:
|
||
| config/locales/en.yml | ||
|---|---|---|
|
text_plugin_assets_writable: Plugin assets directory writable
|
||
|
text_rmagick_available: RMagick available (optional)
|
||
|
text_convert_available: ImageMagick convert available (optional)
|
||
|
text_pdftotext_available: Pdftotext available (optional)
|
||
|
text_unrtf_available: Unrtf available (optional)
|
||
|
text_catdoc_available: Catdoc available (optional)
|
||
|
text_xls2csv_available: Xls2csv available (optional)
|
||
|
text_catppt_available: Catppt available (optional)
|
||
|
text_destroy_time_entries_question: "%{hours} hours were reported on the issues you are about to delete. What do you want to do?"
|
||
|
text_destroy_time_entries: Delete reported hours
|
||
|
text_assign_time_entries_to_project: Assign reported hours to the project
|
||
| lib/redmine/text_extractor.rb | ||
|---|---|---|
|
module Redmine
|
||
|
class TextExtractor
|
||
|
MAX_FULLTEXT_LENGTH = 4.megabytes
|
||
|
TEXT_EXTRACTORS = Redmine::Configuration['text_extractors'] || {}
|
||
|
def initialize(attachment)
|
||
|
@attachment = attachment
|
||
|
end
|
||
|
# returns the extracted fulltext or nil if no matching handler was found
|
||
|
# for the file type.
|
||
|
def text
|
||
|
if handler = find_handler and text = handler.text(@attachment)
|
||
|
text.gsub! /\s+/m, ' '
|
||
|
text.strip!
|
||
|
text.mb_chars.compose.limit(MAX_FULLTEXT_LENGTH).to_s
|
||
|
end
|
||
|
rescue Exception => e
|
||
|
Rails.logger.error "error in fulltext extraction: #{e}"
|
||
|
raise e unless e.is_a? StandardError # re-raise Signals / SyntaxErrors etc
|
||
|
end
|
||
|
private
|
||
|
def find_handler
|
||
|
@@file_handlers.detect{|h| h.accept? @attachment }
|
||
|
end
|
||
|
class FileHandler
|
||
|
def accept?(attachment)
|
||
|
if @content_type
|
||
|
attachment.content_type == @content_type
|
||
|
elsif @content_types
|
||
|
@content_types.include? attachment.content_type
|
||
|
else
|
||
|
false
|
||
|
end
|
||
|
end
|
||
|
end
|
||
|
class ExternalCommandHandler < FileHandler
|
||
|
include Redmine::Utils::Shell
|
||
|
FILE_PLACEHOLDER = '__FILE__'.freeze
|
||
|
def text(attachment)
|
||
|
cmd = @command.dup
|
||
|
cmd[cmd.index(FILE_PLACEHOLDER)] = attachment.diskfile
|
||
|
shellout(cmd){ |io| io.read }.to_s
|
||
|
end
|
||
|
def accept?(attachment)
|
||
|
super and available?
|
||
|
end
|
||
|
def available?
|
||
|
@command.present? and File.executable?(@command[0])
|
||
|
end
|
||
|
def self.available?
|
||
|
new.available?
|
||
|
end
|
||
|
end
|
||
|
class PdfHandler < ExternalCommandHandler
|
||
|
DEFAULT = [
|
||
|
'/usr/bin/pdftotext', '-enc', 'UTF-8', '__FILE__', '-'
|
||
|
].freeze
|
||
|
def initialize
|
||
|
@content_type = 'application/pdf'
|
||
|
@command = TEXT_EXTRACTORS['pdftotext'] || DEFAULT
|
||
|
end
|
||
|
end
|
||
|
class RtfHandler < ExternalCommandHandler
|
||
|
DEFAULT = [
|
||
|
'/usr/bin/unrtf', '--text', '__FILE__'
|
||
|
].freeze
|
||
|
def initialize
|
||
|
@content_type = 'application/rtf'
|
||
|
@command = TEXT_EXTRACTORS['unrtf'] || DEFAULT
|
||
|
end
|
||
|
end
|
||
|
# Handler base class for XML based (MS / Open / Libre) office documents.
|
||
|
class ZippedXmlHandler < FileHandler
|
||
|
class SaxDocument < Nokogiri::XML::SAX::Document
|
||
|
attr_reader :text
|
||
|
def initialize(text_element, text_namespace)
|
||
|
@element = text_element
|
||
|
@namespace_uri = text_namespace
|
||
|
@text = ''.dup
|
||
|
@is_text = false
|
||
|
end
|
||
|
# Handle each element, expecting the name and any attributes
|
||
|
def start_element_namespace(name, attrs = [], prefix = nil, uri = nil, ns = [])
|
||
|
if name == @element and uri == @namespace_uri
|
||
|
@is_text = true
|
||
|
end
|
||
|
end
|
||
|
# Any characters between the start and end element expected as a string
|
||
|
def characters(string)
|
||
|
@text << string if @is_text
|
||
|
end
|
||
|
# Given the name of an element once its closing tag is reached
|
||
|
def end_element_namespace(name, prefix = nil, uri = nil)
|
||
|
if name == @element and uri == @namespace_uri
|
||
|
@text << ' '
|
||
|
@is_text = false
|
||
|
end
|
||
|
end
|
||
|
end
|
||
|
def text(attachment)
|
||
|
Zip::File.open(attachment.diskfile) do |zip_file|
|
||
|
zip_file.each do |entry|
|
||
|
if entry.name == @file
|
||
|
return xml_to_text entry.get_input_stream
|
||
|
end
|
||
|
end
|
||
|
end
|
||
|
end
|
||
|
private
|
||
|
def xml_to_text(io)
|
||
|
sax_doc = SaxDocument.new @element, @namespace_uri
|
||
|
Nokogiri::XML::SAX::Parser.new(sax_doc).parse(io)
|
||
|
sax_doc.text
|
||
|
end
|
||
|
end
|
||
|
# Base class for extractors for MS Office formats
|
||
|
class OfficeDocumentHandler < ZippedXmlHandler
|
||
|
def initialize
|
||
|
super
|
||
|
@element = 't'
|
||
|
end
|
||
|
end
|
||
|
class DocxHandler < OfficeDocumentHandler
|
||
|
def initialize
|
||
|
super
|
||
|
@content_type = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||
|
@file = 'word/document.xml'
|
||
|
@namespace_uri = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'
|
||
|
end
|
||
|
end
|
||
|
class XlsxHandler < OfficeDocumentHandler
|
||
|
def initialize
|
||
|
super
|
||
|
@content_type = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
||
|
@file = 'xl/sharedStrings.xml'
|
||
|
@namespace_uri = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'
|
||
|
end
|
||
|
end
|
||
|
class PptxHandler < OfficeDocumentHandler
|
||
|
CONTENT_TYPES = [
|
||
|
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||
|
'application/vnd.openxmlformats-officedocument.presentationml.slideshow',
|
||
|
'application/vnd.ms-powerpoint.template.macroEnabled.12',
|
||
|
]
|
||
|
def initialize
|
||
|
super
|
||
|
@content_types = CONTENT_TYPES
|
||
|
@namespace_uri = 'http://schemas.openxmlformats.org/drawingml/2006/main'
|
||
|
end
|
||
|
def text(attachment)
|
||
|
slides = []
|
||
|
Zip::File.open(attachment.diskfile) do |zip_file|
|
||
|
zip_file.each do |entry|
|
||
|
if entry.name =~ /slide(\d+)\.xml/
|
||
|
slides << [$1, xml_to_text(entry.get_input_stream)]
|
||
|
end
|
||
|
end
|
||
|
end
|
||
|
slides.sort!{|a, b| a.first <=> b.first}
|
||
|
slides.map(&:last).join ' '
|
||
|
end
|
||
|
end
|
||
|
# Extractor for Open / Libre Office formats
|
||
|
class OpendocumentHandler < ZippedXmlHandler
|
||
|
CONTENT_TYPES = [
|
||
|
'application/vnd.oasis.opendocument.presentation',
|
||
|
'application/vnd.oasis.opendocument.presentation-template',
|
||
|
'application/vnd.oasis.opendocument.text',
|
||
|
'application/vnd.oasis.opendocument.text-template',
|
||
|
'application/vnd.oasis.opendocument.spreadsheet',
|
||
|
'application/vnd.oasis.opendocument.spreadsheet-template'
|
||
|
]
|
||
|
def initialize
|
||
|
super
|
||
|
@file = 'content.xml'
|
||
|
@content_types = CONTENT_TYPES
|
||
|
@element = 'p'
|
||
|
@namespace_uri = 'urn:oasis:names:tc:opendocument:xmlns:text:1.0'
|
||
|
end
|
||
|
end
|
||
|
class DocHandler < ExternalCommandHandler
|
||
|
CONTENT_TYPES = [
|
||
|
'application/vnd.ms-word',
|
||
|
'application/msword',
|
||
|
]
|
||
|
DEFAULT = [
|
||
|
'/usr/bin/catdoc', '-dutf-8', '__FILE__'
|
||
|
]
|
||
|
def initialize
|
||
|
@content_types = CONTENT_TYPES
|
||
|
@command = TEXT_EXTRACTORS['catdoc'] || DEFAULT
|
||
|
end
|
||
|
end
|
||
|
class XlsHandler < ExternalCommandHandler
|
||
|
CONTENT_TYPES = [
|
||
|
'application/vnd.ms-excel',
|
||
|
'application/excel',
|
||
|
]
|
||
|
DEFAULT = [
|
||
|
'/usr/bin/xls2csv', '-dutf-8', '__FILE__'
|
||
|
]
|
||
|
def initialize
|
||
|
@content_types = CONTENT_TYPES
|
||
|
@command = TEXT_EXTRACTORS['xls2csv'] || DEFAULT
|
||
|
end
|
||
|
def text(*_)
|
||
|
if str = super
|
||
|
str.delete('"').gsub /,+/, ' '
|
||
|
end
|
||
|
end
|
||
|
end
|
||
|
class PptHandler < ExternalCommandHandler
|
||
|
CONTENT_TYPES = [
|
||
|
'application/vnd.ms-powerpoint',
|
||
|
'application/powerpoint',
|
||
|
]
|
||
|
DEFAULT = [
|
||
|
'/usr/bin/catppt', '-dutf-8', '__FILE__'
|
||
|
]
|
||
|
def initialize
|
||
|
@content_types = CONTENT_TYPES
|
||
|
@command = TEXT_EXTRACTORS['catppt'] || DEFAULT
|
||
|
end
|
||
|
end
|
||
|
class PlaintextHandler < FileHandler
|
||
|
CONTENT_TYPES = %w(text/csv text/plain)
|
||
|
def initialize
|
||
|
@content_types = CONTENT_TYPES
|
||
|
end
|
||
|
def text(attachment)
|
||
|
Redmine::CodesetUtil.to_utf8 IO.read(attachment.diskfile), 'UTF-8'
|
||
|
end
|
||
|
end
|
||
|
# the handler chain. List most specific handlers first and more general
|
||
|
# (fallback) handlers later.
|
||
|
@@file_handlers = [
|
||
|
PdfHandler,
|
||
|
OpendocumentHandler,
|
||
|
DocxHandler, XlsxHandler, PptxHandler,
|
||
|
DocHandler, XlsHandler, PptHandler,
|
||
|
RtfHandler,
|
||
|
PlaintextHandler,
|
||
|
].map(&:new)
|
||
|
end
|
||
|
end
|
||