Optimize-Git-operations-for-new-branches.patch

Modifies the changeset processing logic for Git to skip already processed revisions - Jeremy Bopp, 2011-12-08 17:06

Download (15.2 KB)

View differences:

app/models/repository/git.rb
79 79
    scm.tags
80 80
  end
81 81

  
82
  def head_revisions
83
    scm.head_revisions
84
  end
85

  
82 86
  def default_branch
83 87
    scm.default_branch
84 88
  rescue Exception => e
......
129 133
  # The repository can still be fully reloaded by calling #clear_changesets
130 134
  # before fetching changesets (eg. for offline resync)
131 135
  def fetch_changesets
132
    scm_brs = branches
133
    return if scm_brs.nil? || scm_brs.empty?
136
    new_head_scmids = head_revisions
137
    return if new_head_scmids.nil? || new_head_scmids.empty?
134 138
    h1 = extra_info || {}
135 139
    h  = h1.dup
136
    h["branches"]       ||= {}
137 140
    h["db_consistent"]  ||= {}
138 141
    if changesets.count == 0
139 142
      h["db_consistent"]["ordering"] = 1
......
144 147
      merge_extra_info(h)
145 148
      self.save
146 149
    end
147
    scm_brs.each do |br1|
148
      br = br1.to_s
149
      from_scmid = nil
150
      from_scmid = h["branches"][br]["last_scmid"] if h["branches"][br]
151
      h["branches"][br] ||= {}
152
      scm.revisions('', from_scmid, br, {:reverse => true}) do |rev|
150
    h["head_scmids"] ||= []
151
    transaction do
152
      scm.revisions('',
153
                    h["head_scmids"], new_head_scmids,
154
                    {:reverse => true}) do |rev|
153 155
        db_rev = find_changeset_by_name(rev.revision)
154
        transaction do
155
          if db_rev.nil?
156
            db_saved_rev = save_revision(rev)
157
            parents = {}
158
            parents[db_saved_rev] = rev.parents unless rev.parents.nil?
159
            parents.each do |ch, chparents|
160
              ch.parents = chparents.collect{|rp| find_changeset_by_name(rp)}.compact
161
            end
156
        if db_rev.nil?
157
          db_saved_rev = save_revision(rev)
158
          unless rev.parents.nil?
159
            db_saved_rev.parents =
160
              rev.parents.collect{|rp| find_changeset_by_name(rp)}.compact
162 161
          end
163
          h["branches"][br]["last_scmid"] = rev.scmid
164
          merge_extra_info(h)
165
          self.save
166 162
        end
167 163
      end
164
      # Update saved head revisions.
165
      h["head_scmids"] = new_head_scmids.dup
166
      merge_extra_info(h)
167
      self.save
168 168
    end
169 169
  end
170 170

  
db/migrate/20111129232941_reformat_git_extra_info.rb
1
class ReformatGitExtraInfo < ActiveRecord::Migration
2
  def self.up
3
    Repository.find(:all, :conditions => ['type = ?', 'Git']).each do |repo|
4
      repo.extra_info['head_scmids'] =
5
        repo.extra_info['branches'].map{|br, h| h['last_scmid']}
6
      repo.extra_info.delete('branches')
7
      repo.write_attribute(:extra_info, repo.extra_info)
8
      repo.save
9
    end
10
  end
11

  
12
  def self.down
13
    raise ActiveRecord::IrreversibleMigration
14
  end
15
end
lib/redmine/scm/adapters/git_adapter.rb
102 102
          nil
103 103
        end
104 104

  
105
        def head_revisions
106
          return @head_revisions if @head_revisions
107
          cmd_args = %w|show-ref --heads|
108
          @head_revisions = scm_cmd(*cmd_args) do |io|
109
            io.map{|line| line.match('(.*?)\s+refs/heads/.*')[1]}.sort!
110
          end
111
        rescue ScmCommandAborted
112
          nil
113
        end
114

  
105 115
        def default_branch
106 116
          bras = self.branches
107 117
          return nil if bras.nil?
......
187 197
          nil
188 198
        end
189 199

  
190
        def revisions(path, identifier_from, identifier_to, options={})
200
        def revisions(path, excludes, includes, options={})
201
          # Ensure that includes and excludes are lists (possibly empty).
202
          includes = [includes].compact.flatten
203
          excludes = [excludes].compact.flatten
204

  
191 205
          revs = Revisions.new
192 206
          cmd_args = %w|log --no-color --encoding=UTF-8 --raw --date=iso --pretty=fuller --parents|
193 207
          cmd_args << "--reverse" if options[:reverse]
194 208
          cmd_args << "--all" if options[:all]
195 209
          cmd_args << "-n" << "#{options[:limit].to_i}" if options[:limit]
196
          from_to = ""
197
          from_to << "#{identifier_from}.." if identifier_from
198
          from_to << "#{identifier_to}" if identifier_to
199
          cmd_args << from_to if !from_to.empty?
200 210
          cmd_args << "--since=#{options[:since].strftime("%Y-%m-%d %H:%M:%S")}" if options[:since]
211
          cmd_args.push(*includes) unless includes.empty?
212
          cmd_args.push("--not", *excludes) unless excludes.empty?
201 213
          cmd_args << "--" << scm_iconv(@path_encoding, 'UTF-8', path) if path && !path.empty?
202 214

  
203 215
          scm_cmd *cmd_args do |io|
......
284 296
          end
285 297
          revs
286 298
        rescue ScmCommandAborted => e
287
          logger.error("git log #{from_to.to_s} error: #{e.message}")
299
          logger.error("git log error: #{e.message}")
288 300
          revs
289 301
        end
290 302

  
test/unit/lib/redmine/scm/adapters/git_adapter_test.rb
87 87
            ], @adapter.tags
88 88
      end
89 89

  
90
      def test_head_revisions
91
        assert_equal  [
92
              "1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127",
93
              "67e7792ce20ccae2e4bb73eed09bb397819c8834",
94
              "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
95
              "fba357b886984ee71185ad2065e65fc0417d9b92"
96
            ], @adapter.head_revisions
97
      end
98

  
90 99
      def test_getting_all_revisions
91 100
        assert_equal 21, @adapter.revisions('',nil,nil,:all => true).length
92 101
      end
......
95 104
        assert_equal 1, @adapter.revisions('','899a15d^','899a15d').length
96 105
      end
97 106

  
107
      def test_getting_multiple_named_revisions
108
        revs = @adapter.revisions('',nil,['4f26664'])
109
        assert_equal 13, revs.length
110
        rev_ids = revs.map{|rev| rev.identifier}
111
        assert rev_ids.include?("4f26664364207fa8b1af9f8722647ab2d4ac5d43"),
112
             "Expected #{rev_ids.inspect} to include \"4f26664364207fa8b1af9f8722647ab2d4ac5d43\""
113

  
114
        revs = @adapter.revisions('',nil,['4f26664', '57ca437'])
115
        assert_equal 15, revs.length
116
        rev_ids = revs.map{|rev| rev.identifier}
117
        assert rev_ids.include?("4f26664364207fa8b1af9f8722647ab2d4ac5d43"),
118
             "Expected #{rev_ids.inspect} to include \"4f26664364207fa8b1af9f8722647ab2d4ac5d43\""
119
        assert rev_ids.include?("57ca437c0acbbcb749821fdf3726a1367056d364"),
120
             "Expected #{rev_ids.inspect} to include \"57ca437c0acbbcb749821fdf3726a1367056d364\""
121
      end
122

  
123
      def test_excluding_multiple_revisions
124
        revs = @adapter.revisions('',['32ae898'],'4f26664')
125
        assert_equal 2, revs.length
126
        rev_ids = revs.map{|rev| rev.identifier}
127
        assert rev_ids.include?("4f26664364207fa8b1af9f8722647ab2d4ac5d43"),
128
             "Expected #{rev_ids.inspect} to include \"4f26664364207fa8b1af9f8722647ab2d4ac5d43\""
129
        assert rev_ids.include?("deff712f05a90d96edbd70facc47d944be5897e3"),
130
             "Expected #{rev_ids.inspect} to include \"deff712f05a90d96edbd70facc47d944be5897e3\""
131

  
132
        revs = @adapter.revisions('',['713f494', '4a07fe3'],'4f26664')
133
        assert_equal 5, revs.length
134
        rev_ids = revs.map{|rev| rev.identifier}
135
        assert rev_ids.include?("4f26664364207fa8b1af9f8722647ab2d4ac5d43"),
136
             "Expected #{rev_ids.inspect} to include \"4f26664364207fa8b1af9f8722647ab2d4ac5d43\""
137
        assert rev_ids.include?("deff712f05a90d96edbd70facc47d944be5897e3"),
138
             "Expected #{rev_ids.inspect} to include \"deff712f05a90d96edbd70facc47d944be5897e3\""
139
        assert rev_ids.include?("7e61ac704deecde634b51e59daa8110435dcb3da"),
140
             "Expected #{rev_ids.inspect} to include \"7e61ac704deecde634b51e59daa8110435dcb3da\""
141
      end
142

  
98 143
      def test_revisions_reverse
99 144
        revs1 = @adapter.revisions('',nil,nil,{:all => true, :reverse => true })
100 145
        assert_equal 21, revs1.length
test/unit/repository_git_test.rb
87 87
      assert_equal "README", change.path
88 88
      assert_equal "A", change.action
89 89

  
90
      assert_equal 4, @repository.extra_info["branches"].size
90
      assert_equal 4, @repository.extra_info["head_scmids"].size
91 91
    end
92 92

  
93 93
    def test_fetch_changesets_incremental
......
96 96
      @project.reload
97 97
      assert_equal NUM_REV, @repository.changesets.count
98 98
      assert_equal 33, @repository.changes.count
99
      extra_info_db = @repository.extra_info["branches"]
99
      extra_info_db = @repository.extra_info["head_scmids"]
100 100
      assert_equal 4, extra_info_db.size
101
      assert_equal "1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127",
102
                    extra_info_db["latin-1-path-encoding"]["last_scmid"]
103
      assert_equal "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
104
                    extra_info_db["master"]["last_scmid"]
101
      assert extra_info_db.include?("1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127"),
102
             "Expected #{extra_info_db.inspect} to include \"1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127\""
103
      assert extra_info_db.include?("83ca5fd546063a3c7dc2e568ba3355661a9e2b2c"),
104
             "Expected #{extra_info_db.inspect} to include \"83ca5fd546063a3c7dc2e568ba3355661a9e2b2c\""
105 105

  
106 106
      del_revs = [
107 107
          "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
......
118 118
      cs1 = @repository.changesets
119 119
      assert_equal 15, cs1.count
120 120
      h = @repository.extra_info.dup
121
      h["branches"]["master"]["last_scmid"] =
122
            "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"
121
      h["head_scmids"].delete("83ca5fd546063a3c7dc2e568ba3355661a9e2b2c")
122
      h["head_scmids"] << "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"
123 123
      @repository.merge_extra_info(h)
124 124
      @repository.save
125 125
      @project.reload
126
      extra_info_db_1 = @repository.extra_info["branches"]
127
      assert_equal "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8",
128
                    extra_info_db_1["master"]["last_scmid"]
126
      extra_info_db_1 = @repository.extra_info["head_scmids"]
127
      assert extra_info_db_1.include?("4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"),
128
             "Expected #{extra_info_db_1.inspect} to include \"4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8\""
129 129

  
130 130
      @repository.fetch_changesets
131 131
      @project.reload
......
136 136
      assert_equal 0, @repository.changesets.count
137 137
      @repository.fetch_changesets
138 138
      @project.reload
139
      assert_equal NUM_REV, @repository.changesets.count
140
      extra_info_db = @repository.extra_info["branches"]
139
      assert_equal 21, @repository.changesets.count
140
      assert_equal 33, @repository.changes.count
141
      extra_info_db = @repository.extra_info["head_scmids"]
141 142
      assert_equal 4, extra_info_db.size
142
      assert_equal "1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127",
143
                    extra_info_db["latin-1-path-encoding"]["last_scmid"]
144
      assert_equal "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
145
                    extra_info_db["master"]["last_scmid"]
143
      assert extra_info_db.include?("1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127"),
144
             "Expected #{extra_info_db.inspect} to include \"1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127\""
145
      assert extra_info_db.include?("83ca5fd546063a3c7dc2e568ba3355661a9e2b2c"),
146
             "Expected #{extra_info_db.inspect} to include \"83ca5fd546063a3c7dc2e568ba3355661a9e2b2c\""
146 147

  
147 148
      del_revs = [
148 149
          "83ca5fd546063a3c7dc2e568ba3355661a9e2b2c",
......
159 160
      cs1 = @repository.changesets
160 161
      assert_equal 15, cs1.count
161 162
      h = @repository.extra_info.dup
162
      h["branches"]["master"]["last_scmid"] =
163
            "abcd1234efgh"
163
      h["head_scmids"].delete("83ca5fd546063a3c7dc2e568ba3355661a9e2b2c")
164
      h["head_scmids"] << "abcd1234efgh"
164 165
      @repository.merge_extra_info(h)
165 166
      @repository.save
166 167
      @project.reload
167
      extra_info_db_1 = @repository.extra_info["branches"]
168
      assert_equal "abcd1234efgh",
169
                    extra_info_db_1["master"]["last_scmid"]
168
      extra_info_db_1 = @repository.extra_info["head_scmids"]
169
      assert extra_info_db_1.include?("abcd1234efgh"),
170
             "Expected #{extra_info_db_1.inspect} to include \"abcd1234efgh\""
170 171

  
171 172
      @repository.fetch_changesets
172 173
      @project.reload
......
230 231
      assert_equal 15, cs1.count
231 232
      assert_equal 0, @repository.extra_info["db_consistent"]["ordering"]
232 233
      h = @repository.extra_info.dup
233
      h["branches"]["master"]["last_scmid"] =
234
            "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"
234
      h["head_scmids"].delete("83ca5fd546063a3c7dc2e568ba3355661a9e2b2c")
235
      h["head_scmids"] << "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"
235 236
      @repository.merge_extra_info(h)
236 237
      @repository.save
237 238
      @project.reload
238
      extra_info_db_1 = @repository.extra_info["branches"]
239
      assert_equal "4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8",
240
                    extra_info_db_1["master"]["last_scmid"]
239
      extra_info_db_1 = @repository.extra_info["head_scmids"]
240
      assert extra_info_db_1.include?("4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8"),
241
             "Expected #{extra_info_db_1.inspect} to include \"4a07fe31bffcf2888791f3e6cbc9c4545cefe3e8\""
241 242

  
242 243
      @repository.fetch_changesets
243 244
      assert_equal NUM_REV, @repository.changesets.count
......
486 487
      assert_equal 0, @repository.changesets.count
487 488
      @repository.fetch_changesets
488 489
      @project.reload
489
      assert_equal NUM_REV, @repository.changesets.count
490
      %w|67e7792ce20ccae2e4bb73eed09bb397819c8834 67e7792ce20cca|.each do |r1|
490
      %w|1ca7f5ed374f3cb31a93ae5215c2e25cc6ec5127 1ca7f5ed374f3c|.each do |r1|
491 491
        changeset = @repository.find_changeset_by_name(r1)
492 492
        assert_nil changeset.next
493 493
      end
494
-