Redmine

gem 'plaintext'

  # Text extraction helper programs.

  #

  # commands should write the resulting plain text to STDOUT. Use __FILE__ as

  # placeholder for the file path. The values below are the defaults.

  #

  # To disable a certain extractor without having to remove it from your

  # system, set it's command to a non-existant binary, i.e:

  #

  #   pdftotext:

  #     - /usr/bin/pdftotext_disabled

  #

  text_extractors:

    # apt install poppler-utils

    # pdftotext:

    #   - /usr/bin/pdftotext

    #   - -enc

    #   - UTF-8

    #   - __FILE__

    #   - '-'

    # apt install unrtf

    # unrtf:

    #   - /usr/bin/unrtf

    #   - --text

    #   - __FILE__

    # apt install catdoc

    # catdoc:

    #   - /usr/bin/catdoc

    #   - -dutf-8

    #   - __FILE__

    # xls2csv:

    #   - /usr/bin/xls2csv

    #   - -dutf-8

    #   - __FILE__

    # catppt:

    #   - /usr/bin/catppt

    #   - -dutf-8

    #   - __FILE__

    # apt-get install tesseract-ocr

    # tesseract:

    #   - /usr/bin/tesseract

    #   - -dutf-8

    #   - __FILE__

        if text_extractors = @config['text_extractors']

          Plaintext::Configuration.load YAML.dump text_extractors

        end

module Redmine

  class TextExtractor

    def initialize(attachment)

      @attachment = attachment

    end

    # returns the extracted fulltext or nil if no matching handler was found

    # for the file type.

    def text

      Plaintext::Resolver.new(@attachment.diskfile,

                              @attachment.content_type).text

    rescue Exception => e

      Rails.logger.error "error in fulltext extraction: #{e}"

      raise e unless e.is_a? StandardError # re-raise Signals / SyntaxErrors etc

    end

  end

end

require_relative '../../../test_helper'

class Redmine::TextExtractorTest < ActiveSupport::TestCase

  fixtures :projects, :users, :attachments

  setup do

    @project = Project.find_by_identifier 'ecookbook'

    set_fixtures_attachments_directory

    @dlopper = User.find_by_login 'dlopper'

  end

  def attachment_for(filename, content_type = nil)

    Attachment.new(container: @project,

                   file: uploaded_test_file(filename, content_type),

                   filename: filename,

                   author: @dlopper).tap do |a|

      a.content_type = content_type if content_type

      a.save!

    end

  end

  test "should extract text from text file" do

    a = attachment_for "testfile.txt"

    te = Redmine::TextExtractor.new a

    assert text = te.text

    assert_match /this is a text file for upload tests with multiple lines/, text

  end

  test "should extract text from csv" do

    a = attachment_for "import_dates.csv"

    te = Redmine::TextExtractor.new a

    assert text = te.text

    assert_match /Invalid start date/, text

  end

end