Redmine

          handle_issues_export

          return

          handle_issues_export

          return

  # exports with less than this number of issues are done inline (not handed

  # over to the configured AJ backend at all)

  PERFORM_NOW_LIMIT = 1000

  # Issues CSV / PDF export via ActiveJob

  # See Redmine::Exports::BackgroundExport for how this works.

  def handle_issues_export

    limit = Setting.issues_export_limit.to_i

    issue_count = @query.issue_count

    export = Redmine::Export::BackgroundExport.new(

      @query,

      worker_class: Redmine::Export::IssueExport,

      perform_now: issue_count <= PERFORM_NOW_LIMIT,

      params: {

        format: params[:format],

        options: {

          project_id: @project.try(:identifier),

          limit: limit,

          csv: { encoding: params[:encoding] }.merge(params[:csv]||{})

        }

      }

    )

    if export.wait

      if attachment = export.result

        send_file attachment.diskfile,

                  filename: Redmine::Export::IssueExport.filename(params[:format]),

                  type: attachment.content_type,

                  disposition: 'attachment'

        return

      else

        flash[:error] = l(:error_issue_export_failed)

      end

    else

      flash[:notice] = l(:notice_issue_export_scheduled)

    end

    params[:format] = nil

    parameters = params.permit!.to_h.except(:action, :controller, :csv, :project_id, :format, :encoding)

    if @project

      redirect_to project_issues_url(@project, parameters)

    else

      redirect_to issues_url(parameters)

    end

  end

# frozen_string_literal: true

# Background job for creating CSV/PDF exports.

#

class ExportJob < ActiveJob::Base

  include Redmine::I18n

  def perform(filename, query_hash, user_id, created_at,

              worker_class_name, params, notify_if_finished_after = nil)

    User.current = User.anonymous

    if user = User.active.find_by_id(user_id)

      User.current = user

      set_language_if_valid user.language || Setting.default_language

    end

    @notify_if_finished_after = notify_if_finished_after.to_i

    worker_class = worker_class_name.constantize

    export = worker_class.new(query_hash, params)

    if data = export.generate_data

      file = DataFile.new data, filename, export.content_type

      @attachment = Attachment.new(file: file, author: User.current)

      @attachment.skip_filesize_validation!

      unless @attachment.save

        @error = @attachment.errors.full_messages.join("\n").presence

      end

    else

      @error = I18n.t(:error_export_failed)

    end

    if notify?

      if @attachment && @error.blank?

        # the controller has given up waiting now, so there is no need for the

        # unique filename anymore. Let's change it to something nice. This

        # allows us to have URLs like attachments/download/100/issues.pdf

        # instead of attachments/download/100/<uuid>.pdf

        @attachment.update_column :filename, export.filename

      end

      Mailer.export_notification(

        User.current, Time.at(created_at), @attachment, @error

      ).deliver

    end

  end

  private

  def notify?

    if @notify_if_finished_after > 0

      # 1 second margin to avoid lost exports. This may lead to

      # a notification being sent out even if the file was delivered

      # directly but this is the lesser evil.

      Time.at(@notify_if_finished_after) - 1.second < Time.now

    else

      false

    end

  end

  # StringIO plus filename and content_type.

  # The result of #generate_data should be of this kind

  class DataFile < StringIO

    attr_reader :original_filename, :content_type

    def initialize(string, filename, content_type)

      super string

      @original_filename = filename

      @content_type = content_type

    end

  end

end

    return if @skip_filesize_validation

  def skip_filesize_validation!

    @skip_filesize_validation = true

  end

  def export_notification(user, date, attachment, error)

    set_language_if_valid(user.language)

    @date = date

    @error = error

    if attachment

      @filename = attachment.filename

      @url = download_named_attachment_url(attachment, @filename)

    else

      @url = issues_url

    end

    redmine_headers 'Sender' => user.login, 'Url' => @url

    mail to: user,

         subject: l(error.blank? ?

                    :mail_subject_export_finished :

                    :mail_subject_export_failed)

  end

<% if @error.present? %>

  <p><%= l :mail_body_export_failed, date: format_time(@date) %></p>

  <p><%= @error %></p>

<% else %>

  <p><%= l :mail_body_export_finished, date: format_time(@date) %></p>

  <p><%= link_to @filename, @url %></p>

<% end %>

<% if @error.present? %>

<%= l :mail_body_export_failed, date: format_time(@date) %>

<%= @error %>

<% else %>

<%= l :mail_body_export_finished, date: format_time(@date) %>

<%= @url %>

<% end %>

  notice_issue_export_scheduled: "Exporting your issues takes longer than usual andwill be continued in the background. You will be notified once the process is completed."

  error_issue_export_failed: The issue export failed.

  mail_subject_export_failed: Export failed

  mail_body_export_failed: The export that was started by you on %{date} has failed.

  mail_subject_export_finished: Export finished

  mail_body_export_finished: "The export that was started by you on %{date} is available for download now:"

# frozen_string_literal: true

require 'timeout'

require 'securerandom'

module Redmine

  module Export

    # This class models an export that may or may not take too long to wait for

    # in a web request.

    #

    # The initializer creates an export that will result in an attachment with

    # a random filename. Use the wait instance methods to wait at most the

    # specified number of seconds for completion of the job. If this returns

    # true, the generated file is available as an Attachment object through

    # #result. Otherwise, the export is taking longer and the user will be

    # notified by email.

    #

    # This makes most sense with a background job runner like DelayedJob

    # configured for ActiveJob. Without this, the job will be run immediately

    # and everything will be like before - #initialize (which launches the

    # background job) will block until the export is done, and wait will return

    # immediately true since the attachments exists when calling code gets

    # around to call it.

    #

    # Jobs can be forced to run inline via the perform_now: argument. This is

    # used in IssuesController to always run small exports immediately instead

    # of putting them in a queue where they potentially have to wait

    # unnecessarily long for large exports to complete.

    class BackgroundExport

      def initialize(query, wait_for: 30.seconds,

                     user: User.current,

                     perform_now: false,

                     worker_class: Redmine::Export::IssueExport,

                     params: {})

        @wait_for = wait_for

        @filename = SecureRandom.uuid

        @user = user

        ExportJob.send(

          perform_now ? :perform_now : :perform_later,

          @filename,

          serialize_query(query), user.id, Time.now.to_i,

          worker_class.name, params,

          wait_for.from_now.to_i

        )

      end

      def serialize_query(query)

        query.as_params.tap do |params|

          params[:c].map!(&:to_s) if params[:c]

        end

      end

      # returns true if the export finished in time

      def wait

        Timeout.timeout(@wait_for) {

          sleep 1 while not finished?

          true

        }

      rescue Timeout::Error

        false

      end

      def result

        scope.first

      end

      private

      def finished?

        Attachment.uncached { scope.any? }

      end

      def scope

        Attachment.where(author: @user, filename: @filename)

      end

    end

  end

end

# frozen_string_literal: true

module Redmine

  module Export

    class IssueExport

      include IssuesHelper

      include QueriesHelper

      include Redmine::Export::PDF::IssuesPdfHelper

      def initialize(query_hash, params)

        @format = params[:format]

        unless %w(csv pdf).include? @format

          raise "Cannot handle export format: #{params[:format]}"

        end

        if options = params[:options]

          @project_id    = options[:project_id]

          @limit         = options[:limit]

          @csv_options   = options[:csv]

        end

        @project = Project.find @project_id if @project_id

        if id = query_hash[:query_id]

          @query = IssueQuery.find id

        else

          @query = IssueQuery.new name: "_"

          @query.build_from_params(query_hash)

        end

        @query.project = @project

      end

      # expected by QueriesHelper#query_to_csv

      def params

        @csv_options || {}

      end

      def generate_data

        issues = @query.lazy_issues limit: @limit

        case @format

        when "csv"

          query_to_csv(issues, @query, @csv_options)

        when "pdf"

          issues_to_pdf(issues, @project, @query)

        end

      end

      def content_type

        self.class.content_type @format

      end

      def filename

        self.class.filename @format

      end

      # format potentially is params[:format] which may be something

      # unexpected, which is why we dont just use "issues#{format}".

      def self.filename(format)

        case format

        when "csv"

          "issues.csv"

        when "pdf"

          "issues.pdf"

        end

      end

      def self.content_type(format)

        case format

        when "csv"

          "text/csv; header=present"

        when "pdf"

          "application/pdf"

        end

      end

    end

  end

end

require_relative '../test_helper'

class DummyWorker

  def initialize(hsh, params)

    @data = hsh[:data] + params[:more_data]

  end

  def generate_data

    @data

  end

  def content_type

    'text/plain'

  end

  def filename

    'dummy.txt'

  end

end

class ExportJobTest < ActiveJob::TestCase

  fixtures :users

  setup do

    set_tmp_attachments_directory

  end

  test 'should call worker class and generate attachment' do

    assert_difference 'Attachment.count' do

      ExportJob.perform_now(

        'random-filename', {data: 'some test'}, 1, Time.now.to_i,

        'DummyWorker', {more_data: ' data'}

      )

    end

    assert a = Attachment.find_by_filename('random-filename')

    assert_equal 1, a.author_id

    assert_equal 'text/plain', a.content_type

    assert_equal "some test data", IO.read(a.diskfile)

  end

end

require_relative '../../../../test_helper'

class BackgroundExportTest < ActiveJob::TestCase

  fixtures :projects, :enabled_modules, :users, :members,

           :member_roles, :roles, :trackers, :issue_statuses,

           :issue_categories, :enumerations, :issues,

           :watchers, :custom_fields, :custom_values, :versions,

           :queries,

           :projects_trackers,

           :custom_fields_trackers,

           :workflows

  setup do

    @query = IssueQuery.new name: '_'

    @project = Project.find 1

    User.current = @user = User.find 1

  end

  test 'should enqueue export job' do

    assert_enqueued_with(job: ExportJob) do

      Redmine::Export::BackgroundExport.new @query,

                           params: {format: 'csv', options:{limit: 10000}}

    end

  end

  test 'wait should wait and return false if not finished' do

    e = Redmine::Export::BackgroundExport.new @query,

                             wait_for: 1.second,

                             params: {format: 'csv', options:{limit: 10000}}

    t = Time.now

    refute e.wait

    assert Time.now - t > 1.second

  end

  test 'wait should return true if finished' do

    e = Redmine::Export::BackgroundExport.new @query,

                             params: {format: 'csv', options:{limit: 10000}}

    class << e

      def finished?; true end

    end

    Timeout.timeout(1) { assert e.wait }

  end

end

require_relative '../../../../test_helper'

class IssueExportTest < ActiveSupport::TestCase

  fixtures :projects, :enabled_modules, :users, :members,

           :member_roles, :roles, :trackers, :issue_statuses,

           :issue_categories, :enumerations, :issues,

           :watchers, :custom_fields, :custom_values, :versions,

           :queries,

           :projects_trackers,

           :custom_fields_trackers,

           :workflows

  setup do

    @query = IssueQuery.new name: '_'

    @project = Project.find 1

    User.current = @user = User.find 1

  end

  test 'should generate issues pdf' do

    export = Redmine::Export::IssueExport.new @query.as_params,

                             format: 'pdf',

                             options: { limit: 10000 }

    assert data = export.generate_data

    assert data.length > 0

    assert_equal 'issues.pdf', export.filename

  end

  test 'should generate issues csv with new query' do

    export = Redmine::Export::IssueExport.new @query.as_params,

                             format: 'csv',

                             options: { limit: 10000 }

    assert data = export.generate_data

    assert data.length > 0

    assert_equal 'issues.csv', export.filename

    assert s = data.lines

    s.shift # headers

    assert_equal Issue.visible.open.count, s.size

  end

  test 'should honor filter and project' do

    @query.add_filter("issue_id", '><', ['2','3'])

    export = Redmine::Export::IssueExport.new @query.as_params,

                             format: 'csv',

                             options: { limit: 10000, project_id: 1 }

    assert data = export.generate_data

    assert s = data.lines

    s.shift # headers

    assert_equal (@project.issues.visible.open.count - 2), s.size

  end

  test 'should honor limit' do

    export = Redmine::Export::IssueExport.new @query.as_params,

                             format: 'csv',

                             options: { limit: 2 }

    assert data = export.generate_data

    assert s = data.lines

    assert_equal 3, s.size

  end

  test 'should honor sorting' do

    @query.sort_criteria = [['id', 'desc']]

    export = Redmine::Export::IssueExport.new @query.as_params, format: 'csv'

    assert data = export.generate_data

    assert s = data.lines

    s.shift # headers

    ids = s.map{|l| l.split(',').first.to_i}

    assert_equal ids, ids.sort{|a, b| b <=> a}

    @query.sort_criteria = [['id', 'asc']]

    export = Redmine::Export::IssueExport.new @query.as_params, format: 'csv'

    assert data = export.generate_data

    assert s = data.lines

    s.shift # headers

    assert s.many?

    ids = s.map{|l| l.split(',').first.to_i}

    assert_equal ids, ids.sort{|a, b| a <=> b}

  end

end