Feature #44726 ยป 0001-Stream-ZIP-downloads-of-attachments-and-wiki-pages-u.patch
| Gemfile | ||
|---|---|---|
| 15 | 15 |
# Loaded on demand by lib/redmine/export/pdf/itcpdf.rb, keep `require: false` |
| 16 | 16 |
gem 'rbpdf', '~> 1.21.4', require: false |
| 17 | 17 |
gem 'addressable' |
| 18 |
gem 'rubyzip', '~> 3.7.0'
|
|
| 18 |
gem 'zip_kit', '~> 6.4.1'
|
|
| 19 | 19 |
gem 'propshaft', '~> 1.3.0' |
| 20 | 20 |
gem 'rack', '>= 3.1.3' |
| 21 | 21 |
gem "stimulus-rails", "~> 1.3" |
| ... | ... | |
| 124 | 124 |
# for testing oauth provider capabilities |
| 125 | 125 |
gem 'oauth2' |
| 126 | 126 |
gem 'webrick' |
| 127 |
# for reading ZIP files generated by zip_kit |
|
| 128 |
gem 'rubyzip' |
|
| 127 | 129 |
end |
| 128 | 130 | |
| 129 | 131 |
local_gemfile = File.join(File.dirname(__FILE__), "Gemfile.local") |
| app/controllers/application_controller.rb | ||
|---|---|---|
| 773 | 773 |
name |
| 774 | 774 |
end |
| 775 | 775 | |
| 776 |
# Streams a ZIP file built in the given block, which receives a |
|
| 777 |
# Redmine::Export::ZipArchive |
|
| 778 |
def send_zip_stream(file_name) |
|
| 779 |
user = User.current |
|
| 780 |
# Let reverse proxies such as nginx buffer the response so that |
|
| 781 |
# the application process is released without waiting for slow clients |
|
| 782 |
response.headers['x-accel-buffering'] = 'yes' |
|
| 783 |
zip_kit_stream(:filename => filename_for_content_disposition(file_name)) do |streamer| |
|
| 784 |
yield Redmine::Export::ZipArchive.new(streamer, user) |
|
| 785 |
end |
|
| 786 |
end |
|
| 787 | ||
| 776 | 788 |
def api_request? |
| 777 | 789 |
%w(xml json).include? params[:format] |
| 778 | 790 |
end |
| app/controllers/attachments_controller.rb | ||
|---|---|---|
| 144 | 144 |
end |
| 145 | 145 | |
| 146 | 146 |
def download_all |
| 147 |
zip_data = Attachment.archive_attachments(@attachments) |
|
| 148 |
if zip_data |
|
| 149 |
file_name = "#{@container.class.to_s.downcase}-#{@container.id}-attachments.zip"
|
|
| 150 |
send_data( |
|
| 151 |
zip_data, |
|
| 152 |
:type => Redmine::MimeType.of(file_name), |
|
| 153 |
:filename => file_name |
|
| 154 |
) |
|
| 155 |
else |
|
| 147 |
if @attachments.blank? |
|
| 156 | 148 |
render_404 |
| 149 |
return |
|
| 150 |
end |
|
| 151 | ||
| 152 |
send_zip_stream("#{@container.class.to_s.downcase}-#{@container.id}-attachments.zip") do |zip|
|
|
| 153 |
@attachments.each do |attachment| |
|
| 154 |
zip.add_file(attachment.filename, attachment.created_on) do |sink| |
|
| 155 |
File.open(attachment.diskfile, 'rb') {|file| IO.copy_stream(file, sink)}
|
|
| 156 |
end |
|
| 157 |
end |
|
| 157 | 158 |
end |
| 158 | 159 |
end |
| 159 | 160 | |
| app/controllers/wiki_controller.rb | ||
|---|---|---|
| 17 | 17 |
# along with this program; if not, write to the Free Software |
| 18 | 18 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| 19 | 19 | |
| 20 |
require 'zip' |
|
| 21 | ||
| 22 | 20 |
# The WikiController follows the Rails REST controller pattern but with |
| 23 | 21 |
# a few differences |
| 24 | 22 |
# |
| ... | ... | |
| 311 | 309 | |
| 312 | 310 |
# Export wiki to a single pdf or html file |
| 313 | 311 |
def export |
| 314 |
@pages = @wiki.pages. |
|
| 315 |
includes([:content, {:attachments => :author}]).
|
|
| 316 |
to_a |
|
| 317 | 312 |
respond_to do |format| |
| 318 | 313 |
format.html do |
| 314 |
load_pages_for_export |
|
| 319 | 315 |
export = render_to_string :action => 'export_multiple', :layout => false |
| 320 | 316 |
send_data(export, :type => 'text/html', :filename => "wiki.html") |
| 321 | 317 |
end |
| 322 | 318 |
format.pdf do |
| 319 |
load_pages_for_export |
|
| 323 | 320 |
send_file_headers! :type => 'application/pdf', :filename => "#{@project.identifier}.pdf"
|
| 324 | 321 |
end |
| 325 | 322 |
format.zip do |
| 326 |
file_name = "#{@project.identifier}-wiki.zip"
|
|
| 327 |
send_data( |
|
| 328 |
wiki_pages_to_zip(@pages), |
|
| 329 |
:type => Redmine::MimeType.of(file_name), |
|
| 330 |
:filename => filename_for_content_disposition(file_name) |
|
| 331 |
) |
|
| 323 |
send_zip_stream("#{@project.identifier}-wiki.zip") do |zip|
|
|
| 324 |
@wiki.pages.with_updated_on.each do |page| |
|
| 325 |
text = WikiContent.where(:page_id => page.id).pick(:text).to_s |
|
| 326 |
zip.add_file("#{archived_wiki_page_title(page)}.txt", page.updated_on) do |sink|
|
|
| 327 |
IO.copy_stream(StringIO.new(text), sink) |
|
| 328 |
end |
|
| 329 |
end |
|
| 330 |
end |
|
| 332 | 331 |
end |
| 333 | 332 |
end |
| 334 | 333 |
end |
| ... | ... | |
| 411 | 410 |
to_a |
| 412 | 411 |
end |
| 413 | 412 | |
| 414 |
def wiki_pages_to_zip(pages) |
|
| 415 |
Zip.unicode_names = true |
|
| 416 |
archived_file_names = [] |
|
| 417 |
buffer = Zip::OutputStream.write_buffer do |zos| |
|
| 418 |
pages.each do |page| |
|
| 419 |
filename = archived_wiki_page_filename(page, archived_file_names) |
|
| 420 |
entry = Zip::Entry.new('', filename)
|
|
| 421 |
if page.updated_on.present? |
|
| 422 |
local_time = User.current.convert_time_to_user_timezone(page.updated_on) |
|
| 423 |
# DOS timestamp stores user's displayed local time |
|
| 424 |
entry.time = Zip::DOSTime.new( |
|
| 425 |
local_time.year, local_time.month, local_time.day, |
|
| 426 |
local_time.hour, local_time.min, local_time.sec |
|
| 427 |
) |
|
| 428 |
# UT extra field stores time in UTC |
|
| 429 |
entry.extra[:universaltime].mtime = local_time.utc |
|
| 430 |
end |
|
| 431 |
zos.put_next_entry(entry) |
|
| 432 |
zos << page.content.text.to_s |
|
| 433 |
end |
|
| 434 |
end |
|
| 435 |
buffer.string |
|
| 436 |
ensure |
|
| 437 |
buffer&.close |
|
| 413 |
def load_pages_for_export |
|
| 414 |
@pages = @wiki.pages. |
|
| 415 |
includes([:content, {:attachments => :author}]).
|
|
| 416 |
to_a |
|
| 438 | 417 |
end |
| 439 | 418 | |
| 440 |
def archived_wiki_page_filename(page, archived_file_names) |
|
| 441 |
extension = '.txt' |
|
| 419 |
def archived_wiki_page_title(page) |
|
| 442 | 420 |
# Keep this character set aligned with Attachment#sanitize_filename. |
| 443 | 421 |
# Unlike attachments, do not drop path-like components from wiki titles. |
| 444 |
sanitized_title = page.title.tr('\\', '_').gsub(/[\/?%*:|"'<>\n\r\x00]+/, '_')
|
|
| 445 |
filename = "#{sanitized_title}#{extension}"
|
|
| 446 |
dup_count = 0 |
|
| 447 | ||
| 448 |
while archived_file_names.include?(filename) |
|
| 449 |
dup_count += 1 |
|
| 450 |
filename = "#{sanitized_title}(#{dup_count})#{extension}"
|
|
| 451 |
end |
|
| 452 | ||
| 453 |
archived_file_names << filename |
|
| 454 |
filename |
|
| 422 |
page.title.tr('\\', '_').gsub(/[\/?%*:|"'<>\n\r\x00]+/, '_')
|
|
| 455 | 423 |
end |
| 456 | 424 |
end |
| app/models/attachment.rb | ||
|---|---|---|
| 19 | 19 | |
| 20 | 20 |
require "digest" |
| 21 | 21 |
require "fileutils" |
| 22 |
require "zip" |
|
| 23 | 22 | |
| 24 | 23 |
class Attachment < ApplicationRecord |
| 25 | 24 |
include Redmine::SafeAttributes |
| ... | ... | |
| 452 | 451 |
Attachment.where("created_on < ? AND (container_type IS NULL OR container_type = '')", Time.now - age).destroy_all
|
| 453 | 452 |
end |
| 454 | 453 | |
| 455 |
def self.archive_attachments(attachments) |
|
| 456 |
attachments = attachments.select(&:readable?) |
|
| 457 |
return nil if attachments.blank? |
|
| 458 | ||
| 459 |
Zip.unicode_names = true |
|
| 460 |
archived_file_names = [] |
|
| 461 |
buffer = Zip::OutputStream.write_buffer do |zos| |
|
| 462 |
attachments.each do |attachment| |
|
| 463 |
filename = attachment.filename |
|
| 464 |
# rename the file if a file with the same name already exists |
|
| 465 |
dup_count = 0 |
|
| 466 |
while archived_file_names.include?(filename) |
|
| 467 |
dup_count += 1 |
|
| 468 |
extname = File.extname(attachment.filename) |
|
| 469 |
basename = File.basename(attachment.filename, extname) |
|
| 470 |
filename = "#{basename}(#{dup_count})#{extname}"
|
|
| 471 |
end |
|
| 472 |
zos.put_next_entry(filename) |
|
| 473 |
zos << File.binread(attachment.diskfile) |
|
| 474 |
archived_file_names << filename |
|
| 475 |
end |
|
| 476 |
end |
|
| 477 |
buffer.string |
|
| 478 |
ensure |
|
| 479 |
buffer&.close |
|
| 480 |
end |
|
| 481 | ||
| 482 | 454 |
# Moves an existing attachment to its target directory |
| 483 | 455 |
def move_to_target_directory! |
| 484 | 456 |
return unless !new_record? && readable? |
| lib/redmine/export/zip_archive.rb | ||
|---|---|---|
| 1 |
# frozen_string_literal: true |
|
| 2 | ||
| 3 |
# Redmine - project management software |
|
| 4 |
# Copyright (C) 2006- Jean-Philippe Lang |
|
| 5 |
# |
|
| 6 |
# This program is free software; you can redistribute it and/or |
|
| 7 |
# modify it under the terms of the GNU General Public License |
|
| 8 |
# as published by the Free Software Foundation; either version 2 |
|
| 9 |
# of the License, or (at your option) any later version. |
|
| 10 |
# |
|
| 11 |
# This program is distributed in the hope that it will be useful, |
|
| 12 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
| 13 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
| 14 |
# GNU General Public License for more details. |
|
| 15 |
# |
|
| 16 |
# You should have received a copy of the GNU General Public License |
|
| 17 |
# along with this program; if not, write to the Free Software |
|
| 18 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
|
| 19 | ||
| 20 |
module Redmine |
|
| 21 |
module Export |
|
| 22 |
class ZipArchive |
|
| 23 |
def initialize(streamer, user) |
|
| 24 |
@streamer = streamer |
|
| 25 |
@user = user |
|
| 26 |
@entry_names = Set.new |
|
| 27 |
end |
|
| 28 | ||
| 29 |
def add_file(name, modification_time, &) |
|
| 30 |
entry_name = unique_entry_name(name) |
|
| 31 |
@entry_names << entry_name |
|
| 32 |
@streamer.write_file(entry_name, :modification_time => zip_time(modification_time), &) |
|
| 33 |
end |
|
| 34 | ||
| 35 |
private |
|
| 36 | ||
| 37 |
def unique_entry_name(name) |
|
| 38 |
entry_name = name |
|
| 39 |
extname = File.extname(name) |
|
| 40 |
basename = File.basename(name, extname) |
|
| 41 |
dup_count = 0 |
|
| 42 |
while @entry_names.include?(entry_name) |
|
| 43 |
dup_count += 1 |
|
| 44 |
entry_name = "#{basename}(#{dup_count})#{extname}"
|
|
| 45 |
end |
|
| 46 |
entry_name |
|
| 47 |
end |
|
| 48 | ||
| 49 |
# DateTime is used because zip_kit calls #utc on the given time, |
|
| 50 |
# which converts a Time object in place. |
|
| 51 |
def zip_time(time) |
|
| 52 |
@user.convert_time_to_user_timezone(time).to_datetime |
|
| 53 |
end |
|
| 54 |
end |
|
| 55 |
end |
|
| 56 |
end |
|
| test/functional/attachments_controller_test.rb | ||
|---|---|---|
| 793 | 793 |
assert_response :ok |
| 794 | 794 |
assert_equal response.headers['Content-Type'], 'application/zip' |
| 795 | 795 |
assert_match /issue-2-attachments.zip/, response.headers['Content-Disposition'] |
| 796 |
assert_not_includes Dir.entries(Rails.root.join('tmp')), /attachments_zip/
|
|
| 796 |
assert_equal 'yes', response.headers['X-Accel-Buffering'] |
|
| 797 |
zip_data = +"" |
|
| 798 |
response.body.each {|chunk| zip_data << chunk}
|
|
| 799 |
Zip::File.open_buffer(zip_data) do |zip| |
|
| 800 |
assert_equal ['source.rb'], zip.entries.map(&:name) |
|
| 801 |
assert_equal File.binread(Attachment.find(4).diskfile), zip.read('source.rb')
|
|
| 802 |
end |
|
| 797 | 803 |
end |
| 798 | 804 | |
| 799 | 805 |
def test_download_all_with_invalid_container |
| test/functional/wiki_controller_test.rb | ||
|---|---|---|
| 1389 | 1389 | |
| 1390 | 1390 |
def zip_entries_from_response |
| 1391 | 1391 |
entries = {}
|
| 1392 |
zip_data = +'' |
|
| 1393 |
@response.body.each {|chunk| zip_data << chunk}
|
|
| 1392 | 1394 | |
| 1393 |
Zip::InputStream.open(StringIO.new(@response.body)) do |io|
|
|
| 1394 |
while (entry = io.get_next_entry)
|
|
| 1395 |
Zip::File.open_buffer(zip_data) do |zip|
|
|
| 1396 |
zip.each do |entry|
|
|
| 1395 | 1397 |
entries[entry.name] = {
|
| 1396 |
:content => io.read,
|
|
| 1398 |
:content => entry.get_input_stream.read,
|
|
| 1397 | 1399 |
:dos_time => entry.instance_variable_get(:@time), |
| 1398 | 1400 |
:utc_time => entry.extra[:universaltime]&.mtime |
| 1399 | 1401 |
} |
| test/unit/attachment_test.rb | ||
|---|---|---|
| 397 | 397 |
end |
| 398 | 398 |
end |
| 399 | 399 | |
| 400 |
def test_archive_attachments |
|
| 401 |
attachment = Attachment.create!(:file => uploaded_test_file("testfile.txt", ""), :author_id => 1)
|
|
| 402 |
zip_data = Attachment.archive_attachments([attachment]) |
|
| 403 |
file_names = [] |
|
| 404 |
Zip::InputStream.open(StringIO.new(zip_data)) do |io| |
|
| 405 |
while (entry = io.get_next_entry) |
|
| 406 |
file_names << entry.name |
|
| 407 |
end |
|
| 408 |
end |
|
| 409 |
assert_equal ['testfile.txt'], file_names |
|
| 410 |
end |
|
| 411 | ||
| 412 |
def test_archive_attachments_without_attachments |
|
| 413 |
zip_data = Attachment.archive_attachments([]) |
|
| 414 |
assert_nil zip_data |
|
| 415 |
end |
|
| 416 | ||
| 417 |
def test_archive_attachments_should_rename_duplicate_file_names |
|
| 418 |
attachment1 = Attachment.create!(:file => uploaded_test_file("testfile.txt", ""), :author_id => 1)
|
|
| 419 |
attachment2 = Attachment.create!(:file => uploaded_test_file("testfile.txt", ""), :author_id => 1)
|
|
| 420 |
zip_data = Attachment.archive_attachments([attachment1, attachment2]) |
|
| 421 |
file_names = [] |
|
| 422 |
Zip::InputStream.open(StringIO.new(zip_data)) do |io| |
|
| 423 |
while (entry = io.get_next_entry) |
|
| 424 |
file_names << entry.name |
|
| 425 |
end |
|
| 426 |
end |
|
| 427 |
assert_equal ['testfile.txt', 'testfile(1).txt'], file_names |
|
| 428 |
end |
|
| 429 | ||
| 430 | 400 |
def test_move_from_root_to_target_directory_should_move_root_files |
| 431 | 401 |
a = Attachment.find(20) |
| 432 | 402 |
assert a.disk_directory.blank? |
| test/unit/lib/redmine/export/zip_archive_test.rb | ||
|---|---|---|
| 1 |
# frozen_string_literal: true |
|
| 2 | ||
| 3 |
# Redmine - project management software |
|
| 4 |
# Copyright (C) 2006- Jean-Philippe Lang |
|
| 5 |
# |
|
| 6 |
# This program is free software; you can redistribute it and/or |
|
| 7 |
# modify it under the terms of the GNU General Public License |
|
| 8 |
# as published by the Free Software Foundation; either version 2 |
|
| 9 |
# of the License, or (at your option) any later version. |
|
| 10 |
# |
|
| 11 |
# This program is distributed in the hope that it will be useful, |
|
| 12 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
| 13 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
| 14 |
# GNU General Public License for more details. |
|
| 15 |
# |
|
| 16 |
# You should have received a copy of the GNU General Public License |
|
| 17 |
# along with this program; if not, write to the Free Software |
|
| 18 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
|
| 19 | ||
| 20 |
require_relative '../../../../test_helper' |
|
| 21 | ||
| 22 |
class ZipArchiveTest < ActiveSupport::TestCase |
|
| 23 |
# Uses a server time zone other than UTC so that a local time converted |
|
| 24 |
# to UTC can be detected |
|
| 25 |
def setup |
|
| 26 |
@original_tz = ENV['TZ'] |
|
| 27 |
ENV['TZ'] = 'HST10' |
|
| 28 |
end |
|
| 29 | ||
| 30 |
def teardown |
|
| 31 |
ENV['TZ'] = @original_tz |
|
| 32 |
end |
|
| 33 | ||
| 34 |
def test_add_file_should_rename_duplicate_names |
|
| 35 |
entries = write_zip(User.find(2)) do |zip| |
|
| 36 |
3.times { zip.add_file('foo.txt', Time.now) {|sink| sink << 'foo'} }
|
|
| 37 |
end |
|
| 38 | ||
| 39 |
assert_equal ['foo.txt', 'foo(1).txt', 'foo(2).txt'], entries.map(&:name) |
|
| 40 |
end |
|
| 41 | ||
| 42 |
def test_add_file_should_store_modification_time_in_user_time_zone |
|
| 43 |
user = User.find(2) |
|
| 44 |
user.pref.update!(:time_zone => 'Tokyo') |
|
| 45 |
entries = write_zip(user) do |zip| |
|
| 46 |
zip.add_file('foo.txt', Time.utc(2026, 1, 1, 18, 4, 6)) {|sink| sink << 'foo'}
|
|
| 47 |
end |
|
| 48 | ||
| 49 |
assert_equal [2026, 1, 2, 3, 4, 6], dos_time_components(entries.first) |
|
| 50 |
assert_equal Time.utc(2026, 1, 1, 18, 4, 6), entries.first.extra[:universaltime].mtime.utc |
|
| 51 |
end |
|
| 52 | ||
| 53 |
def test_add_file_should_store_modification_time_in_server_local_time_without_user_time_zone |
|
| 54 |
user = User.find(2) |
|
| 55 |
user.pref.update!(:time_zone => '') |
|
| 56 |
entries = write_zip(user) do |zip| |
|
| 57 |
zip.add_file('foo.txt', Time.utc(2026, 1, 1, 18, 4, 6)) {|sink| sink << 'foo'}
|
|
| 58 |
end |
|
| 59 | ||
| 60 |
# 18:04:06 UTC in the server time zone (HST, UTC-10) |
|
| 61 |
assert_equal [2026, 1, 1, 8, 4, 6], dos_time_components(entries.first) |
|
| 62 |
assert_equal Time.utc(2026, 1, 1, 18, 4, 6), entries.first.extra[:universaltime].mtime.utc |
|
| 63 |
end |
|
| 64 | ||
| 65 |
private |
|
| 66 | ||
| 67 |
def write_zip(user) |
|
| 68 |
buffer = StringIO.new(+''.b) |
|
| 69 |
ZipKit::Streamer.open(buffer) do |streamer| |
|
| 70 |
yield Redmine::Export::ZipArchive.new(streamer, user) |
|
| 71 |
end |
|
| 72 |
Zip::File.open_buffer(buffer.string).entries |
|
| 73 |
end |
|
| 74 | ||
| 75 |
def dos_time_components(entry) |
|
| 76 |
dos_time = entry.instance_variable_get(:@time) |
|
| 77 |
[dos_time.year, dos_time.month, dos_time.day, dos_time.hour, dos_time.min, dos_time.sec] |
|
| 78 |
end |
|
| 79 |
end |
|