Feature #306 » 0001-implements-fulltext-extraction-for-attachments.patch
| Gemfile | ||
|---|---|---|
|
gem 'tzinfo-data', platforms: [:mingw, :x64_mingw, :mswin]
|
||
|
gem "rbpdf", "~> 1.19.6"
|
||
|
gem 'plaintext'
|
||
|
# Optional gem for LDAP authentication
|
||
|
group :ldap do
|
||
|
gem "net-ldap", "~> 0.16.0"
|
||
| config/configuration.yml.example | ||
|---|---|---|
|
# allowed values: :memory, :file, :memcache
|
||
|
#openid_authentication_store: :memory
|
||
|
# Text extraction helper programs.
|
||
|
#
|
||
|
# commands should write the resulting plain text to STDOUT. Use __FILE__ as
|
||
|
# placeholder for the file path. The values below are the defaults.
|
||
|
#
|
||
|
# To disable a certain extractor without having to remove it from your
|
||
|
# system, set it's command to a non-existant binary, i.e:
|
||
|
#
|
||
|
# pdftotext:
|
||
|
# - /usr/bin/pdftotext_disabled
|
||
|
#
|
||
|
text_extractors:
|
||
|
# apt install poppler-utils
|
||
|
# pdftotext:
|
||
|
# - /usr/bin/pdftotext
|
||
|
# - -enc
|
||
|
# - UTF-8
|
||
|
# - __FILE__
|
||
|
# - '-'
|
||
|
# apt install unrtf
|
||
|
# unrtf:
|
||
|
# - /usr/bin/unrtf
|
||
|
# - --text
|
||
|
# - __FILE__
|
||
|
# apt install catdoc
|
||
|
# catdoc:
|
||
|
# - /usr/bin/catdoc
|
||
|
# - -dutf-8
|
||
|
# - __FILE__
|
||
|
# xls2csv:
|
||
|
# - /usr/bin/xls2csv
|
||
|
# - -dutf-8
|
||
|
# - __FILE__
|
||
|
# catppt:
|
||
|
# - /usr/bin/catppt
|
||
|
# - -dutf-8
|
||
|
# - __FILE__
|
||
|
# apt-get install tesseract-ocr
|
||
|
# tesseract:
|
||
|
# - /usr/bin/tesseract
|
||
|
# - -dutf-8
|
||
|
# - __FILE__
|
||
|
# specific configuration options for production environment
|
||
|
# that overrides the default ones
|
||
|
production:
|
||
| lib/redmine/configuration.rb | ||
|---|---|---|
|
end
|
||
|
end
|
||
|
if text_extractors = @config['text_extractors']
|
||
|
Plaintext::Configuration.load YAML.dump text_extractors
|
||
|
end
|
||
|
check_regular_expressions
|
||
|
@config
|
||
|
end
|
||
| lib/redmine/text_extractor.rb | ||
|---|---|---|
|
module Redmine
|
||
|
class TextExtractor
|
||
|
def initialize(attachment)
|
||
|
@attachment = attachment
|
||
|
end
|
||
|
# returns the extracted fulltext or nil if no matching handler was found
|
||
|
# for the file type.
|
||
|
def text
|
||
|
Plaintext::Resolver.new(@attachment.diskfile,
|
||
|
@attachment.content_type).text
|
||
|
rescue Exception => e
|
||
|
Rails.logger.error "error in fulltext extraction: #{e}"
|
||
|
raise e unless e.is_a? StandardError # re-raise Signals / SyntaxErrors etc
|
||
|
end
|
||
|
end
|
||
|
end
|
||
| test/unit/lib/redmine/text_extractor_test.rb | ||
|---|---|---|
|
require_relative '../../../test_helper'
|
||
|
class Redmine::TextExtractorTest < ActiveSupport::TestCase
|
||
|
fixtures :projects, :users, :attachments
|
||
|
setup do
|
||
|
@project = Project.find_by_identifier 'ecookbook'
|
||
|
set_fixtures_attachments_directory
|
||
|
@dlopper = User.find_by_login 'dlopper'
|
||
|
end
|
||
|
def attachment_for(filename, content_type = nil)
|
||
|
Attachment.new(container: @project,
|
||
|
file: uploaded_test_file(filename, content_type),
|
||
|
filename: filename,
|
||
|
author: @dlopper).tap do |a|
|
||
|
a.content_type = content_type if content_type
|
||
|
a.save!
|
||
|
end
|
||
|
end
|
||
|
test "should extract text from text file" do
|
||
|
a = attachment_for "testfile.txt"
|
||
|
te = Redmine::TextExtractor.new a
|
||
|
assert text = te.text
|
||
|
assert_match /this is a text file for upload tests with multiple lines/, text
|
||
|
end
|
||
|
test "should extract text from csv" do
|
||
|
a = attachment_for "import_dates.csv"
|
||
|
te = Redmine::TextExtractor.new a
|
||
|
assert text = te.text
|
||
|
assert_match /Invalid start date/, text
|
||
|
end
|
||
|
end
|
||