From 25ed9d81292962d4c6399d018a2b079cb09778f8 Mon Sep 17 00:00:00 2001 From: MAEDA Go Date: Thu, 27 Sep 2018 18:42:33 +0900 Subject: [PATCH] Switch syntax highlighter to Rouge from CodeRay. --- Gemfile | 2 +- lib/redmine/syntax_highlighting.rb | 59 +++++--- lib/redmine/wiki_formatting/textile/formatter.rb | 1 + public/javascripts/jstoolbar/jstoolbar.js | 6 +- public/stylesheets/application.css | 167 +++++++++------------ public/stylesheets/rtl.css | 3 - test/helpers/application_helper_test.rb | 24 ++- .../redmine/syntax_highlighting/coderay_test.rb | 37 ----- .../wiki_formatting/markdown_formatter_test.rb | 2 +- .../wiki_formatting/textile_formatter_test.rb | 4 +- 10 files changed, 140 insertions(+), 165 deletions(-) delete mode 100644 test/unit/lib/redmine/syntax_highlighting/coderay_test.rb diff --git a/Gemfile b/Gemfile index 69a86c716..89ea33a56 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,7 @@ if Gem::Version.new(Bundler::VERSION) < Gem::Version.new('1.5.0') end gem "rails", "5.2.1" -gem "coderay", "~> 1.1.1" +gem "rouge", "~> 3.2.1" gem "request_store", "1.0.5" gem "mini_mime", "~> 1.0.1" gem "actionpack-xml_parser" diff --git a/lib/redmine/syntax_highlighting.rb b/lib/redmine/syntax_highlighting.rb index 49d9f4110..e2d47f277 100644 --- a/lib/redmine/syntax_highlighting.rb +++ b/lib/redmine/syntax_highlighting.rb @@ -52,42 +52,65 @@ module Redmine end end - module CodeRay - require 'coderay' + module Rouge + require 'rouge' - def self.retrieve_supported_languages - ::CodeRay::Scanners.list + - # Add CodeRay scanner aliases - ::CodeRay::Scanners.plugin_hash.keys.map(&:to_sym) - - # Remove internal CodeRay scanners - %w(debug default raydebug scanner).map(&:to_sym) - end - private_class_method :retrieve_supported_languages + # Customized formatter based on Rouge::Formatters::HTMLLinewise + # Syntax highlighting is completed within each line. + class CustomHTMLLinewise < ::Rouge::Formatter + def initialize(formatter) + @formatter = formatter + end - SUPPORTED_LANGUAGES = retrieve_supported_languages + def stream(tokens, &b) + token_lines(tokens) do |line| + line.each do |tok, val| + yield @formatter.span(tok, val) + end + yield "\n" + end + end + end class << self # Highlights +text+ as the content of +filename+ # Should not return line numbers nor outer pre tag def highlight_by_filename(text, filename) - language = ::CodeRay::FileType[filename] - language ? ::CodeRay.scan(text, language).html(:break_lines => true) : ERB::Util.h(text) + lexer =::Rouge::Lexer.guess_by_filename(filename) + formatter = ::Rouge::Formatters::HTML.new + ::Rouge.highlight(text, lexer, CustomHTMLLinewise.new(formatter)) end # Highlights +text+ using +language+ syntax # Should not return outer pre tag def highlight_by_language(text, language) - ::CodeRay.scan(text, language).html(:wrap => :span) + lexer = + find_lexer(language.to_s.downcase) || ::Rouge::Lexers::PlainText + ::Rouge.highlight(text, lexer, ::Rouge::Formatters::HTML) end def language_supported?(language) - SUPPORTED_LANGUAGES.include?(language.to_s.downcase.to_sym) - rescue - false + find_lexer(language.to_s.downcase) ? true : false + end + + private + # Alias names used by CodeRay and not supported by Rouge + LANG_ALIASES = { + 'delphi' => 'pascal', + 'cplusplus' => 'cpp', + 'ecmascript' => 'javascript', + 'ecma_script' => 'javascript', + 'java_script' => 'javascript', + 'xhtml' => 'html' + } + + def find_lexer(language) + ::Rouge::Lexer.find(language) || + ::Rouge::Lexer.find(LANG_ALIASES[language]) end end end end - SyntaxHighlighting.highlighter = 'CodeRay' + SyntaxHighlighting.highlighter = 'Rouge' end diff --git a/lib/redmine/wiki_formatting/textile/formatter.rb b/lib/redmine/wiki_formatting/textile/formatter.rb index b0b65bbbc..89e6105a8 100644 --- a/lib/redmine/wiki_formatting/textile/formatter.rb +++ b/lib/redmine/wiki_formatting/textile/formatter.rb @@ -125,6 +125,7 @@ module Redmine language = $1 text = $2 if Redmine::SyntaxHighlighting.language_supported?(language) + text.gsub!(/x%x%/, '&') content = "" + Redmine::SyntaxHighlighting.highlight_by_language(text, language) else diff --git a/public/javascripts/jstoolbar/jstoolbar.js b/public/javascripts/jstoolbar/jstoolbar.js index 608541414..41e12f830 100644 --- a/public/javascripts/jstoolbar/jstoolbar.js +++ b/public/javascripts/jstoolbar/jstoolbar.js @@ -453,10 +453,10 @@ jsToolBar.prototype.resizeDragStop = function(event) { /* Code highlighting menu */ jsToolBar.prototype.precodeMenu = function(fn){ - var codeRayLanguages = ["c", "clojure", "cpp", "css", "delphi", "diff", "erb", "go", "groovy", "haml", "html", "java", "javascript", "json", "lua", "php", "python", "ruby", "sass", "sql", "taskpaper", "text", "xml", "yaml"]; + var hlLanguages = ["c", "clojure", "cpp", "css", "diff", "erb", "go", "groovy", "haml", "html", "java", "javascript", "json", "lua", "pascal", "php", "python", "ruby", "sass", "sql", "text", "xml", "yaml"]; var menu = $(""); - for (var i = 0; i < codeRayLanguages.length; i++) { - $("
  • ").text(codeRayLanguages[i]).appendTo(menu).mousedown(function(){ + for (var i = 0; i < hlLanguages.length; i++) { + $("
  • ").text(hlLanguages[i]).appendTo(menu).mousedown(function(){ fn($(this).text()); }); } diff --git a/public/stylesheets/application.css b/public/stylesheets/application.css index e98350679..50e03624a 100644 --- a/public/stylesheets/application.css +++ b/public/stylesheets/application.css @@ -1407,102 +1407,77 @@ img.filecontent.image {background-image: url(../images/transparent.png);} .ui-datepicker-title select {width:70px !important; margin-top:-2px !important; margin-right:4px !important;} -/************* CodeRay styles *************/ -.syntaxhl div {display: inline;} -.syntaxhl .code pre { overflow: auto } - -.syntaxhl .annotation { color:#007 } -.syntaxhl .attribute-name { color:#b48 } -.syntaxhl .attribute-value { color:#700 } -.syntaxhl .binary { color:#549 } -.syntaxhl .binary .char { color:#325 } -.syntaxhl .binary .delimiter { color:#325 } -.syntaxhl .char { color:#D20 } -.syntaxhl .char .content { color:#D20 } -.syntaxhl .char .delimiter { color:#710 } -.syntaxhl .class { color:#B06; font-weight:bold } -.syntaxhl .class-variable { color:#369 } -.syntaxhl .color { color:#0A0 } -.syntaxhl .comment { color:#777 } -.syntaxhl .comment .char { color:#444 } -.syntaxhl .comment .delimiter { color:#444 } -.syntaxhl .constant { color:#036; font-weight:bold } -.syntaxhl .decorator { color:#B0B } -.syntaxhl .definition { color:#099; font-weight:bold } -.syntaxhl .delimiter { color:black } -.syntaxhl .directive { color:#088; font-weight:bold } -.syntaxhl .docstring { color:#D42; } -.syntaxhl .doctype { color:#34b } -.syntaxhl .done { text-decoration: line-through; color: gray } -.syntaxhl .entity { color:#800; font-weight:bold } -.syntaxhl .error { color:#F00; background-color:#FAA } -.syntaxhl .escape { color:#666 } -.syntaxhl .exception { color:#C00; font-weight:bold } -.syntaxhl .float { color:#60E } -.syntaxhl .function { color:#06B; font-weight:bold } -.syntaxhl .function .delimiter { color:#059 } -.syntaxhl .function .content { color:#037 } -.syntaxhl .global-variable { color:#d70 } -.syntaxhl .hex { color:#02b } -.syntaxhl .id { color:#33D; font-weight:bold } -.syntaxhl .include { color:#B44; font-weight:bold } -.syntaxhl .inline { background-color: hsla(0,0%,0%,0.07); color: black } -.syntaxhl .inline-delimiter { font-weight: bold; color: #666 } -.syntaxhl .instance-variable { color:#33B } -.syntaxhl .integer { color:#00D } -.syntaxhl .imaginary { color:#f00 } -.syntaxhl .important { color:#D00 } -.syntaxhl .key { color: #606 } -.syntaxhl .key .char { color: #60f } -.syntaxhl .key .delimiter { color: #404 } -.syntaxhl .keyword { color:#080; font-weight:bold } -.syntaxhl .label { color:#970; font-weight:bold } -.syntaxhl .local-variable { color:#950 } -.syntaxhl .map .content { color:#808 } -.syntaxhl .map .delimiter { color:#40A} -.syntaxhl .map { background-color:hsla(200,100%,50%,0.06); } -.syntaxhl .namespace { color:#707; font-weight:bold } -.syntaxhl .octal { color:#40E } -.syntaxhl .operator { } -.syntaxhl .predefined { color:#369; font-weight:bold } -.syntaxhl .predefined-constant { color:#069 } -.syntaxhl .predefined-type { color:#0a8; font-weight:bold } -.syntaxhl .preprocessor { color:#579 } -.syntaxhl .pseudo-class { color:#00C; font-weight:bold } -.syntaxhl .regexp { background-color:hsla(300,100%,50%,0.06); } -.syntaxhl .regexp .content { color:#808 } -.syntaxhl .regexp .delimiter { color:#404 } -.syntaxhl .regexp .modifier { color:#C2C } -.syntaxhl .reserved { color:#080; font-weight:bold } -.syntaxhl .shell { background-color:hsla(120,100%,50%,0.06); } -.syntaxhl .shell .content { color:#2B2 } -.syntaxhl .shell .delimiter { color:#161 } -.syntaxhl .string { background-color:hsla(0,100%,50%,0.05); } -.syntaxhl .string .char { color: #b0b } -.syntaxhl .string .content { color: #D20 } -.syntaxhl .string .delimiter { color: #710 } -.syntaxhl .string .modifier { color: #E40 } -.syntaxhl .symbol { color:#A60 } -.syntaxhl .symbol .content { color:#A60 } -.syntaxhl .symbol .delimiter { color:#740 } -.syntaxhl .tag { color:#070; font-weight:bold } -.syntaxhl .type { color:#339; font-weight:bold } -.syntaxhl .value { color: #088 } -.syntaxhl .variable { color:#037 } - -.syntaxhl .insert { background: hsla(120,100%,50%,0.12) } -.syntaxhl .delete { background: hsla(0,100%,50%,0.12) } -.syntaxhl .change { color: #bbf; background: #007 } -.syntaxhl .head { color: #f8f; background: #505 } -.syntaxhl .head .filename { color: white; } - -.syntaxhl .delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; } -.syntaxhl .insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; } - -.syntaxhl .insert .insert { color: #0c0; background:transparent; font-weight:bold } -.syntaxhl .delete .delete { color: #c00; background:transparent; font-weight:bold } -.syntaxhl .change .change { color: #88f } -.syntaxhl .head .head { color: #f4f } +/************* Rouge styles *************/ +/* generated by: pygmentize -f html -a .syntaxhl -S colorful */ +.syntaxhl .hll { background-color: #ffffcc } +.syntaxhl { background: #fafafa; } +.syntaxhl .c { color: #888888 } /* Comment */ +.syntaxhl .err { color: #FF0000; background-color: #FFAAAA } /* Error */ +.syntaxhl .k { color: #008800; font-weight: bold } /* Keyword */ +.syntaxhl .o { color: #333333 } /* Operator */ +.syntaxhl .ch { color: #888888 } /* Comment.Hashbang */ +.syntaxhl .cm { color: #888888 } /* Comment.Multiline */ +.syntaxhl .cp { color: #557799 } /* Comment.Preproc */ +.syntaxhl .cpf { color: #888888 } /* Comment.PreprocFile */ +.syntaxhl .c1 { color: #888888 } /* Comment.Single */ +.syntaxhl .cs { color: #cc0000; font-weight: bold } /* Comment.Special */ +.syntaxhl .gd { color: #A00000 } /* Generic.Deleted */ +.syntaxhl .ge { font-style: italic } /* Generic.Emph */ +.syntaxhl .gr { color: #FF0000 } /* Generic.Error */ +.syntaxhl .gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.syntaxhl .gi { color: #00A000 } /* Generic.Inserted */ +.syntaxhl .go { color: #888888 } /* Generic.Output */ +.syntaxhl .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ +.syntaxhl .gs { font-weight: bold } /* Generic.Strong */ +.syntaxhl .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.syntaxhl .gt { color: #0044DD } /* Generic.Traceback */ +.syntaxhl .kc { color: #008800; font-weight: bold } /* Keyword.Constant */ +.syntaxhl .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */ +.syntaxhl .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */ +.syntaxhl .kp { color: #003388; font-weight: bold } /* Keyword.Pseudo */ +.syntaxhl .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */ +.syntaxhl .kt { color: #333399; font-weight: bold } /* Keyword.Type */ +.syntaxhl .m { color: #6600EE; font-weight: bold } /* Literal.Number */ +.syntaxhl .s { background-color: #fff0f0 } /* Literal.String */ +.syntaxhl .na { color: #0000CC } /* Name.Attribute */ +.syntaxhl .nb { color: #007020 } /* Name.Builtin */ +.syntaxhl .nc { color: #BB0066; font-weight: bold } /* Name.Class */ +.syntaxhl .no { color: #003366; font-weight: bold } /* Name.Constant */ +.syntaxhl .nd { color: #555555; font-weight: bold } /* Name.Decorator */ +.syntaxhl .ni { color: #880000; font-weight: bold } /* Name.Entity */ +.syntaxhl .ne { color: #FF0000; font-weight: bold } /* Name.Exception */ +.syntaxhl .nf { color: #0066BB; font-weight: bold } /* Name.Function */ +.syntaxhl .nl { color: #997700; font-weight: bold } /* Name.Label */ +.syntaxhl .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ +.syntaxhl .nt { color: #007700 } /* Name.Tag */ +.syntaxhl .nv { color: #996633 } /* Name.Variable */ +.syntaxhl .ow { color: #000000; font-weight: bold } /* Operator.Word */ +.syntaxhl .w { color: #bbbbbb } /* Text.Whitespace */ +.syntaxhl .mb { color: #6600EE; font-weight: bold } /* Literal.Number.Bin */ +.syntaxhl .mf { color: #6600EE; font-weight: bold } /* Literal.Number.Float */ +.syntaxhl .mh { color: #005588; font-weight: bold } /* Literal.Number.Hex */ +.syntaxhl .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */ +.syntaxhl .mo { color: #4400EE; font-weight: bold } /* Literal.Number.Oct */ +.syntaxhl .sa { background-color: #fff0f0 } /* Literal.String.Affix */ +.syntaxhl .sb { background-color: #fff0f0 } /* Literal.String.Backtick */ +.syntaxhl .sc { color: #0044DD } /* Literal.String.Char */ +.syntaxhl .dl { background-color: #fff0f0 } /* Literal.String.Delimiter */ +.syntaxhl .sd { color: #DD4422 } /* Literal.String.Doc */ +.syntaxhl .s2 { background-color: #fff0f0 } /* Literal.String.Double */ +.syntaxhl .se { color: #666666; font-weight: bold; background-color: #fff0f0 } /* Literal.String.Escape */ +.syntaxhl .sh { background-color: #fff0f0 } /* Literal.String.Heredoc */ +.syntaxhl .si { background-color: #eeeeee } /* Literal.String.Interpol */ +.syntaxhl .sx { color: #DD2200; background-color: #fff0f0 } /* Literal.String.Other */ +.syntaxhl .sr { color: #000000; background-color: #fff0ff } /* Literal.String.Regex */ +.syntaxhl .s1 { background-color: #fff0f0 } /* Literal.String.Single */ +.syntaxhl .ss { color: #AA6600 } /* Literal.String.Symbol */ +.syntaxhl .bp { color: #007020 } /* Name.Builtin.Pseudo */ +.syntaxhl .fm { color: #0066BB; font-weight: bold } /* Name.Function.Magic */ +.syntaxhl .vc { color: #336699 } /* Name.Variable.Class */ +.syntaxhl .vg { color: #dd7700; font-weight: bold } /* Name.Variable.Global */ +.syntaxhl .vi { color: #3333BB } /* Name.Variable.Instance */ +.syntaxhl .vm { color: #996633 } /* Name.Variable.Magic */ +.syntaxhl .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ /***** Media print specific styles *****/ @media print { diff --git a/public/stylesheets/rtl.css b/public/stylesheets/rtl.css index 4063e21ce..ec3ea20ba 100644 --- a/public/stylesheets/rtl.css +++ b/public/stylesheets/rtl.css @@ -373,9 +373,6 @@ td.username img.gravatar {margin:0 0 0 0.5em; } /* Custom JQuery styles */ .ui-datepicker-title select {margin-left:4px !important; margin-right:0 !important;} -/************* CodeRay styles *************/ -.syntaxhl .line-numbers {margin:0px 0px 0px 5px;} - /***** Media print specific styles *****/ @media print { } diff --git a/test/helpers/application_helper_test.rb b/test/helpers/application_helper_test.rb index d4990d39f..5561e2f40 100644 --- a/test/helpers/application_helper_test.rb +++ b/test/helpers/application_helper_test.rb @@ -1094,19 +1094,35 @@ EXPECTED def test_syntax_highlight raw = <<-RAW -
    
-# Some ruby code here
+
    
+/* Hello */
+document.write("Hello World!");
 
    
 RAW
 
     expected = <<-EXPECTED
-
    # Some ruby code here
-
    
+
    /* Hello */document.write(\"Hello World!\");
    
 EXPECTED
 
     assert_equal expected.gsub(%r{[\r\n\t]}, ''), textilizable(raw).gsub(%r{[\r\n\t]}, '')
   end
 
+  def test_syntax_highlight_ampersand_in_textile
+    raw = <<-RAW
+
    
+x = a & b
+
    
+RAW
+
+    expected = <<-EXPECTED
+
    x = a & b
    
+EXPECTED
+
+    with_settings :text_formatting => 'textile' do
+      assert_equal expected.gsub(%r{[\r\n\t]}, ''), textilizable(raw).gsub(%r{[\r\n\t]}, '')
+    end
+  end
+
   def test_to_path_param
     assert_equal 'test1/test2', to_path_param('test1/test2')
     assert_equal 'test1/test2', to_path_param('/test1/test2/')
diff --git a/test/unit/lib/redmine/syntax_highlighting/coderay_test.rb b/test/unit/lib/redmine/syntax_highlighting/coderay_test.rb
deleted file mode 100644
index 1a2714b16..000000000
--- a/test/unit/lib/redmine/syntax_highlighting/coderay_test.rb
+++ /dev/null
@@ -1,37 +0,0 @@
-# Redmine - project management software
-# Copyright (C) 2006-2017  Jean-Philippe Lang
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-
-require File.expand_path('../../../../../test_helper', __FILE__)
-
-class Redmine::SyntaxHighlighting::CodeRayTest < ActiveSupport::TestCase
-  def test_retrieve_supported_languages_should_return_array_of_symbols
-    assert_kind_of Array, Redmine::SyntaxHighlighting::CodeRay.send(:retrieve_supported_languages)
-    assert_kind_of Symbol, Redmine::SyntaxHighlighting::CodeRay.send(:retrieve_supported_languages).first
-  end
-
-  def test_retrieve_supported_languages_should_return_array_of_symbols_holding_languages
-    assert_includes Redmine::SyntaxHighlighting::CodeRay.send(:retrieve_supported_languages), :ruby
-  end
-
-  def test_retrieve_supported_languages_should_return_array_of_symbols_holding_languages_aliases
-    assert_includes Redmine::SyntaxHighlighting::CodeRay.send(:retrieve_supported_languages), :javascript
-  end
-
-  def test_retrieve_supported_languages_should_return_array_of_symbols_not_holding_internal_languages
-    refute_includes Redmine::SyntaxHighlighting::CodeRay.send(:retrieve_supported_languages), :default
-  end
-end
diff --git a/test/unit/lib/redmine/wiki_formatting/markdown_formatter_test.rb b/test/unit/lib/redmine/wiki_formatting/markdown_formatter_test.rb
index 07d17efa7..cfe8188f6 100644
--- a/test/unit/lib/redmine/wiki_formatting/markdown_formatter_test.rb
+++ b/test/unit/lib/redmine/wiki_formatting/markdown_formatter_test.rb
@@ -66,7 +66,7 @@ end
 ~~~
 STR
     assert_select_in @formatter.new(text).to_html, 'pre code.ruby.syntaxhl' do
-      assert_select 'span.keyword', :text => 'def'
+      assert_select 'span.k', :text => 'def'
     end
   end
 
diff --git a/test/unit/lib/redmine/wiki_formatting/textile_formatter_test.rb b/test/unit/lib/redmine/wiki_formatting/textile_formatter_test.rb
index 712bc5000..cc31a4c80 100644
--- a/test/unit/lib/redmine/wiki_formatting/textile_formatter_test.rb
+++ b/test/unit/lib/redmine/wiki_formatting/textile_formatter_test.rb
@@ -547,9 +547,9 @@ STR
 
   def test_should_allow_valid_language_class_attribute_on_code_tags
     # language name is double-quoted
-    assert_html_output({"test" => "test"}, false)
+    assert_html_output({"test" => "test"}, false)
     # language name is single-quoted
-    assert_html_output({"test" => "test"}, false)
+    assert_html_output({"test" => "test"}, false)
   end
 
   def test_should_not_allow_valid_language_class_attribute_on_non_code_offtags
-- 
2.14.3 (Apple Git-98)