Project

General

Profile

Feature #306 » 0001-implements-fulltext-extraction-for-attachments.patch

[new] adds text extractor using the plaintext gem - Jens Krämer, 2018-10-26 07:35

View differences:

Gemfile
gem 'tzinfo-data', platforms: [:mingw, :x64_mingw, :mswin]
gem "rbpdf", "~> 1.19.6"
gem 'plaintext'
# Optional gem for LDAP authentication
group :ldap do
gem "net-ldap", "~> 0.16.0"
config/configuration.yml.example
# allowed values: :memory, :file, :memcache
#openid_authentication_store: :memory
# Text extraction helper programs.
#
# commands should write the resulting plain text to STDOUT. Use __FILE__ as
# placeholder for the file path. The values below are the defaults.
#
# To disable a certain extractor without having to remove it from your
# system, set it's command to a non-existant binary, i.e:
#
# pdftotext:
# - /usr/bin/pdftotext_disabled
#
text_extractors:
# apt install poppler-utils
# pdftotext:
# - /usr/bin/pdftotext
# - -enc
# - UTF-8
# - __FILE__
# - '-'
# apt install unrtf
# unrtf:
# - /usr/bin/unrtf
# - --text
# - __FILE__
# apt install catdoc
# catdoc:
# - /usr/bin/catdoc
# - -dutf-8
# - __FILE__
# xls2csv:
# - /usr/bin/xls2csv
# - -dutf-8
# - __FILE__
# catppt:
# - /usr/bin/catppt
# - -dutf-8
# - __FILE__
# apt-get install tesseract-ocr
# tesseract:
# - /usr/bin/tesseract
# - -dutf-8
# - __FILE__
# specific configuration options for production environment
# that overrides the default ones
production:
lib/redmine/configuration.rb
end
end
if text_extractors = @config['text_extractors']
Plaintext::Configuration.load YAML.dump text_extractors
end
check_regular_expressions
@config
end
lib/redmine/text_extractor.rb
module Redmine
class TextExtractor
def initialize(attachment)
@attachment = attachment
end
# returns the extracted fulltext or nil if no matching handler was found
# for the file type.
def text
Plaintext::Resolver.new(@attachment.diskfile,
@attachment.content_type).text
rescue Exception => e
Rails.logger.error "error in fulltext extraction: #{e}"
raise e unless e.is_a? StandardError # re-raise Signals / SyntaxErrors etc
end
end
end
test/unit/lib/redmine/text_extractor_test.rb
require_relative '../../../test_helper'
class Redmine::TextExtractorTest < ActiveSupport::TestCase
fixtures :projects, :users, :attachments
setup do
@project = Project.find_by_identifier 'ecookbook'
set_fixtures_attachments_directory
@dlopper = User.find_by_login 'dlopper'
end
def attachment_for(filename, content_type = nil)
Attachment.new(container: @project,
file: uploaded_test_file(filename, content_type),
filename: filename,
author: @dlopper).tap do |a|
a.content_type = content_type if content_type
a.save!
end
end
test "should extract text from text file" do
a = attachment_for "testfile.txt"
te = Redmine::TextExtractor.new a
assert text = te.text
assert_match /this is a text file for upload tests with multiple lines/, text
end
test "should extract text from csv" do
a = attachment_for "import_dates.csv"
te = Redmine::TextExtractor.new a
assert text = te.text
assert_match /Invalid start date/, text
end
end
(7-7/11)