Project

General

Profile

Feature #44726 ยป 0001-Stream-ZIP-downloads-of-attachments-and-wiki-pages-u.patch

Go MAEDA, 2026-11-08 10:48

View differences:

Gemfile
15 15
# Loaded on demand by lib/redmine/export/pdf/itcpdf.rb, keep `require: false`
16 16
gem 'rbpdf', '~> 1.21.4', require: false
17 17
gem 'addressable'
18
gem 'rubyzip', '~> 3.7.0'
18
gem 'zip_kit', '~> 6.4.1'
19 19
gem 'propshaft', '~> 1.3.0'
20 20
gem 'rack', '>= 3.1.3'
21 21
gem "stimulus-rails", "~> 1.3"
......
124 124
  # for testing oauth provider capabilities
125 125
  gem 'oauth2'
126 126
  gem 'webrick'
127
  # for reading ZIP files generated by zip_kit
128
  gem 'rubyzip'
127 129
end
128 130

  
129 131
local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local")
app/controllers/application_controller.rb
773 773
    name
774 774
  end
775 775

  
776
  # Streams a ZIP file built in the given block, which receives a
777
  # Redmine::Export::ZipArchive
778
  def send_zip_stream(file_name)
779
    user = User.current
780
    # Let reverse proxies such as nginx buffer the response so that
781
    # the application process is released without waiting for slow clients
782
    response.headers['x-accel-buffering'] = 'yes'
783
    zip_kit_stream(:filename => filename_for_content_disposition(file_name)) do |streamer|
784
      yield Redmine::Export::ZipArchive.new(streamer, user)
785
    end
786
  end
787

  
776 788
  def api_request?
777 789
    %w(xml json).include? params[:format]
778 790
  end
app/controllers/attachments_controller.rb
144 144
  end
145 145

  
146 146
  def download_all
147
    zip_data = Attachment.archive_attachments(@attachments)
148
    if zip_data
149
      file_name = "#{@container.class.to_s.downcase}-#{@container.id}-attachments.zip"
150
      send_data(
151
        zip_data,
152
        :type => Redmine::MimeType.of(file_name),
153
        :filename => file_name
154
      )
155
    else
147
    if @attachments.blank?
156 148
      render_404
149
      return
150
    end
151

  
152
    send_zip_stream("#{@container.class.to_s.downcase}-#{@container.id}-attachments.zip") do |zip|
153
      @attachments.each do |attachment|
154
        zip.add_file(attachment.filename, attachment.created_on) do |sink|
155
          File.open(attachment.diskfile, 'rb') {|file| IO.copy_stream(file, sink)}
156
        end
157
      end
157 158
    end
158 159
  end
159 160

  
app/controllers/wiki_controller.rb
17 17
# along with this program; if not, write to the Free Software
18 18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
19 19

  
20
require 'zip'
21

  
22 20
# The WikiController follows the Rails REST controller pattern but with
23 21
# a few differences
24 22
#
......
311 309

  
312 310
  # Export wiki to a single pdf or html file
313 311
  def export
314
    @pages = @wiki.pages.
315
                      includes([:content, {:attachments => :author}]).
316
                      to_a
317 312
    respond_to do |format|
318 313
      format.html do
314
        load_pages_for_export
319 315
        export = render_to_string :action => 'export_multiple', :layout => false
320 316
        send_data(export, :type => 'text/html', :filename => "wiki.html")
321 317
      end
322 318
      format.pdf do
319
        load_pages_for_export
323 320
        send_file_headers! :type => 'application/pdf', :filename => "#{@project.identifier}.pdf"
324 321
      end
325 322
      format.zip do
326
        file_name = "#{@project.identifier}-wiki.zip"
327
        send_data(
328
          wiki_pages_to_zip(@pages),
329
          :type => Redmine::MimeType.of(file_name),
330
          :filename => filename_for_content_disposition(file_name)
331
        )
323
        send_zip_stream("#{@project.identifier}-wiki.zip") do |zip|
324
          @wiki.pages.with_updated_on.each do |page|
325
            text = WikiContent.where(:page_id => page.id).pick(:text).to_s
326
            zip.add_file("#{archived_wiki_page_title(page)}.txt", page.updated_on) do |sink|
327
              IO.copy_stream(StringIO.new(text), sink)
328
            end
329
          end
330
        end
332 331
      end
333 332
    end
334 333
  end
......
411 410
                to_a
412 411
  end
413 412

  
414
  def wiki_pages_to_zip(pages)
415
    Zip.unicode_names = true
416
    archived_file_names = []
417
    buffer = Zip::OutputStream.write_buffer do |zos|
418
      pages.each do |page|
419
        filename = archived_wiki_page_filename(page, archived_file_names)
420
        entry = Zip::Entry.new('', filename)
421
        if page.updated_on.present?
422
          local_time = User.current.convert_time_to_user_timezone(page.updated_on)
423
          # DOS timestamp stores user's displayed local time
424
          entry.time = Zip::DOSTime.new(
425
            local_time.year, local_time.month, local_time.day,
426
            local_time.hour, local_time.min, local_time.sec
427
          )
428
          # UT extra field stores time in UTC
429
          entry.extra[:universaltime].mtime = local_time.utc
430
        end
431
        zos.put_next_entry(entry)
432
        zos << page.content.text.to_s
433
      end
434
    end
435
    buffer.string
436
  ensure
437
    buffer&.close
413
  def load_pages_for_export
414
    @pages = @wiki.pages.
415
                includes([:content, {:attachments => :author}]).
416
                to_a
438 417
  end
439 418

  
440
  def archived_wiki_page_filename(page, archived_file_names)
441
    extension = '.txt'
419
  def archived_wiki_page_title(page)
442 420
    # Keep this character set aligned with Attachment#sanitize_filename.
443 421
    # Unlike attachments, do not drop path-like components from wiki titles.
444
    sanitized_title = page.title.tr('\\', '_').gsub(/[\/?%*:|"'<>\n\r\x00]+/, '_')
445
    filename = "#{sanitized_title}#{extension}"
446
    dup_count = 0
447

  
448
    while archived_file_names.include?(filename)
449
      dup_count += 1
450
      filename = "#{sanitized_title}(#{dup_count})#{extension}"
451
    end
452

  
453
    archived_file_names << filename
454
    filename
422
    page.title.tr('\\', '_').gsub(/[\/?%*:|"'<>\n\r\x00]+/, '_')
455 423
  end
456 424
end
app/models/attachment.rb
19 19

  
20 20
require "digest"
21 21
require "fileutils"
22
require "zip"
23 22

  
24 23
class Attachment < ApplicationRecord
25 24
  include Redmine::SafeAttributes
......
452 451
    Attachment.where("created_on < ? AND (container_type IS NULL OR container_type = '')", Time.now - age).destroy_all
453 452
  end
454 453

  
455
  def self.archive_attachments(attachments)
456
    attachments = attachments.select(&:readable?)
457
    return nil if attachments.blank?
458

  
459
    Zip.unicode_names = true
460
    archived_file_names = []
461
    buffer = Zip::OutputStream.write_buffer do |zos|
462
      attachments.each do |attachment|
463
        filename = attachment.filename
464
        # rename the file if a file with the same name already exists
465
        dup_count = 0
466
        while archived_file_names.include?(filename)
467
          dup_count += 1
468
          extname = File.extname(attachment.filename)
469
          basename = File.basename(attachment.filename, extname)
470
          filename = "#{basename}(#{dup_count})#{extname}"
471
        end
472
        zos.put_next_entry(filename)
473
        zos << File.binread(attachment.diskfile)
474
        archived_file_names << filename
475
      end
476
    end
477
    buffer.string
478
  ensure
479
    buffer&.close
480
  end
481

  
482 454
  # Moves an existing attachment to its target directory
483 455
  def move_to_target_directory!
484 456
    return unless !new_record? && readable?
lib/redmine/export/zip_archive.rb
1
# frozen_string_literal: true
2

  
3
# Redmine - project management software
4
# Copyright (C) 2006-  Jean-Philippe Lang
5
#
6
# This program is free software; you can redistribute it and/or
7
# modify it under the terms of the GNU General Public License
8
# as published by the Free Software Foundation; either version 2
9
# of the License, or (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
19

  
20
module Redmine
21
  module Export
22
    class ZipArchive
23
      def initialize(streamer, user)
24
        @streamer = streamer
25
        @user = user
26
        @entry_names = Set.new
27
      end
28

  
29
      def add_file(name, modification_time, &)
30
        entry_name = unique_entry_name(name)
31
        @entry_names << entry_name
32
        @streamer.write_file(entry_name, :modification_time => zip_time(modification_time), &)
33
      end
34

  
35
      private
36

  
37
      def unique_entry_name(name)
38
        entry_name = name
39
        extname = File.extname(name)
40
        basename = File.basename(name, extname)
41
        dup_count = 0
42
        while @entry_names.include?(entry_name)
43
          dup_count += 1
44
          entry_name = "#{basename}(#{dup_count})#{extname}"
45
        end
46
        entry_name
47
      end
48

  
49
      # DateTime is used because zip_kit calls #utc on the given time,
50
      # which converts a Time object in place.
51
      def zip_time(time)
52
        @user.convert_time_to_user_timezone(time).to_datetime
53
      end
54
    end
55
  end
56
end
test/functional/attachments_controller_test.rb
793 793
    assert_response :ok
794 794
    assert_equal response.headers['Content-Type'], 'application/zip'
795 795
    assert_match /issue-2-attachments.zip/, response.headers['Content-Disposition']
796
    assert_not_includes Dir.entries(Rails.root.join('tmp')), /attachments_zip/
796
    assert_equal 'yes', response.headers['X-Accel-Buffering']
797
    zip_data = +""
798
    response.body.each {|chunk| zip_data << chunk}
799
    Zip::File.open_buffer(zip_data) do |zip|
800
      assert_equal ['source.rb'], zip.entries.map(&:name)
801
      assert_equal File.binread(Attachment.find(4).diskfile), zip.read('source.rb')
802
    end
797 803
  end
798 804

  
799 805
  def test_download_all_with_invalid_container
test/functional/wiki_controller_test.rb
1389 1389

  
1390 1390
  def zip_entries_from_response
1391 1391
    entries = {}
1392
    zip_data = +''
1393
    @response.body.each {|chunk| zip_data << chunk}
1392 1394

  
1393
    Zip::InputStream.open(StringIO.new(@response.body)) do |io|
1394
      while (entry = io.get_next_entry)
1395
    Zip::File.open_buffer(zip_data) do |zip|
1396
      zip.each do |entry|
1395 1397
        entries[entry.name] = {
1396
          :content => io.read,
1398
          :content => entry.get_input_stream.read,
1397 1399
          :dos_time => entry.instance_variable_get(:@time),
1398 1400
          :utc_time => entry.extra[:universaltime]&.mtime
1399 1401
        }
test/unit/attachment_test.rb
397 397
    end
398 398
  end
399 399

  
400
  def test_archive_attachments
401
    attachment = Attachment.create!(:file => uploaded_test_file("testfile.txt", ""), :author_id => 1)
402
    zip_data = Attachment.archive_attachments([attachment])
403
    file_names = []
404
    Zip::InputStream.open(StringIO.new(zip_data)) do |io|
405
      while (entry = io.get_next_entry)
406
        file_names << entry.name
407
      end
408
    end
409
    assert_equal ['testfile.txt'], file_names
410
  end
411

  
412
  def test_archive_attachments_without_attachments
413
    zip_data = Attachment.archive_attachments([])
414
    assert_nil zip_data
415
  end
416

  
417
  def test_archive_attachments_should_rename_duplicate_file_names
418
    attachment1 = Attachment.create!(:file => uploaded_test_file("testfile.txt", ""), :author_id => 1)
419
    attachment2 = Attachment.create!(:file => uploaded_test_file("testfile.txt", ""), :author_id => 1)
420
    zip_data = Attachment.archive_attachments([attachment1, attachment2])
421
    file_names = []
422
    Zip::InputStream.open(StringIO.new(zip_data)) do |io|
423
      while (entry = io.get_next_entry)
424
        file_names << entry.name
425
      end
426
    end
427
    assert_equal ['testfile.txt', 'testfile(1).txt'], file_names
428
  end
429

  
430 400
  def test_move_from_root_to_target_directory_should_move_root_files
431 401
    a = Attachment.find(20)
432 402
    assert a.disk_directory.blank?
test/unit/lib/redmine/export/zip_archive_test.rb
1
# frozen_string_literal: true
2

  
3
# Redmine - project management software
4
# Copyright (C) 2006-  Jean-Philippe Lang
5
#
6
# This program is free software; you can redistribute it and/or
7
# modify it under the terms of the GNU General Public License
8
# as published by the Free Software Foundation; either version 2
9
# of the License, or (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
# GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
19

  
20
require_relative '../../../../test_helper'
21

  
22
class ZipArchiveTest < ActiveSupport::TestCase
23
  # Uses a server time zone other than UTC so that a local time converted
24
  # to UTC can be detected
25
  def setup
26
    @original_tz = ENV['TZ']
27
    ENV['TZ'] = 'HST10'
28
  end
29

  
30
  def teardown
31
    ENV['TZ'] = @original_tz
32
  end
33

  
34
  def test_add_file_should_rename_duplicate_names
35
    entries = write_zip(User.find(2)) do |zip|
36
      3.times { zip.add_file('foo.txt', Time.now) {|sink| sink << 'foo'} }
37
    end
38

  
39
    assert_equal ['foo.txt', 'foo(1).txt', 'foo(2).txt'], entries.map(&:name)
40
  end
41

  
42
  def test_add_file_should_store_modification_time_in_user_time_zone
43
    user = User.find(2)
44
    user.pref.update!(:time_zone => 'Tokyo')
45
    entries = write_zip(user) do |zip|
46
      zip.add_file('foo.txt', Time.utc(2026, 1, 1, 18, 4, 6)) {|sink| sink << 'foo'}
47
    end
48

  
49
    assert_equal [2026, 1, 2, 3, 4, 6], dos_time_components(entries.first)
50
    assert_equal Time.utc(2026, 1, 1, 18, 4, 6), entries.first.extra[:universaltime].mtime.utc
51
  end
52

  
53
  def test_add_file_should_store_modification_time_in_server_local_time_without_user_time_zone
54
    user = User.find(2)
55
    user.pref.update!(:time_zone => '')
56
    entries = write_zip(user) do |zip|
57
      zip.add_file('foo.txt', Time.utc(2026, 1, 1, 18, 4, 6)) {|sink| sink << 'foo'}
58
    end
59

  
60
    # 18:04:06 UTC in the server time zone (HST, UTC-10)
61
    assert_equal [2026, 1, 1, 8, 4, 6], dos_time_components(entries.first)
62
    assert_equal Time.utc(2026, 1, 1, 18, 4, 6), entries.first.extra[:universaltime].mtime.utc
63
  end
64

  
65
  private
66

  
67
  def write_zip(user)
68
    buffer = StringIO.new(+''.b)
69
    ZipKit::Streamer.open(buffer) do |streamer|
70
      yield Redmine::Export::ZipArchive.new(streamer, user)
71
    end
72
    Zip::File.open_buffer(buffer.string).entries
73
  end
74

  
75
  def dos_time_components(entry)
76
    dos_time = entry.instance_variable_get(:@time)
77
    [dos_time.year, dos_time.month, dos_time.day, dos_time.hour, dos_time.min, dos_time.sec]
78
  end
79
end
    (1-1/1)