Improve performance of cache filling

James Moger

2013-06-24 3c4ce1d366641158927cee8b4e47191ed681b524

Improve performance of cache filling

 src/main/java/com/gitblit/GitBlit.java

@@ -102,6 +102,7 @@
import com.gitblit.models.GitClientApplication;
import com.gitblit.models.Metric;
import com.gitblit.models.ProjectModel;
import com.gitblit.models.RefModel;
import com.gitblit.models.RegistrantAccessPermission;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.RepositoryUrl;
@@ -3402,9 +3403,8 @@
        configureJGit();
        configureFanout();
        configureGitDaemon();
		
        CommitCache.instance().setCacheDays(settings.getInteger(Keys.web.activityCacheDays, 14));
		
        configureCommitCache();

        ContainerUtils.CVE_2007_0450.test();
    }
    
@@ -3514,6 +3514,42 @@
        }
    }
    
    protected void configureCommitCache() {
        int daysToCache = settings.getInteger(Keys.web.activityCacheDays, 14);
        if (daysToCache <= 0) {
            logger.info("commit cache disabled");
        } else {
            long start = System.nanoTime();
            long repoCount = 0;
            long commitCount = 0;
            logger.info(MessageFormat.format("preparing {0} day commit cache. please wait...", daysToCache));
            CommitCache.instance().setCacheDays(daysToCache);
            Date cutoff = CommitCache.instance().getCutoffDate();
            for (String repositoryName : getRepositoryList()) {
                RepositoryModel model = getRepositoryModel(repositoryName);
                if (model.hasCommits && model.lastChange.after(cutoff)) {
                    repoCount++;
                    Repository repository = getRepository(repositoryName);
                    for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) {
                        if (!ref.getDate().after(cutoff)) {
                            // branch not recently updated
                            continue;
                        }
                        List<?> commits = CommitCache.instance().getCommits(repositoryName, repository, ref.getName());
                        if (commits.size() > 0) {
                            logger.info(MessageFormat.format("  cached {0} commits for {1}:{2}",
                                    commits.size(), repositoryName, ref.getName()));
                            commitCount += commits.size();
                        }
                    }
                    repository.close();
                }
            }
            logger.info(MessageFormat.format("built {0} day commit cache of {1} commits across {2} repositories in {3} msecs",
                    daysToCache, commitCount, repoCount, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
        }
    }
	
    protected final Logger getLogger() {
        return logger;
    }

 src/main/java/com/gitblit/utils/ActivityUtils.java

@@ -105,6 +105,10 @@
                if (StringUtils.isEmpty(objectId)) {

                    for (RefModel local : JGitUtils.getLocalBranches(

                            repository, true, -1)) {

                        if (!local.getDate().after(thresholdDate)) {

                            // branch not recently updated

                            continue;

                        }

                        branches.add(local.getName());

                    }

                } else {


 src/main/java/com/gitblit/utils/CommitCache.java

@@ -67,7 +67,7 @@
     * 
     * @return
     */
    protected Date getCacheCutoffDate() {
    public Date getCutoffDate() {
        final Calendar cal = Calendar.getInstance();
        cal.setTimeInMillis(System.currentTimeMillis());
        cal.set(Calendar.HOUR_OF_DAY, 0);
@@ -118,7 +118,7 @@
     * @return a list of commits
     */
    public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch) {
        return getCommits(repositoryName, repository, branch, getCacheCutoffDate());
        return getCommits(repositoryName, repository, branch, getCutoffDate());
    }
    
    /**
@@ -134,7 +134,7 @@
     */
    public List<RepositoryCommit> getCommits(String repositoryName, Repository repository, String branch, Date sinceDate) {
        long start = System.nanoTime();
        Date cacheCutoffDate = getCacheCutoffDate();
        Date cacheCutoffDate = getCutoffDate();
        List<RepositoryCommit> list;
        if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) {
            // request fits within the cache window

 src/main/java/com/gitblit/utils/RefLogUtils.java

@@ -548,6 +548,10 @@
        Map<String, DailyLogEntry> dailydigests = new HashMap<String, DailyLogEntry>();
        String linearParent = null;
        for (RefModel local : JGitUtils.getLocalBranches(repository, true, -1)) {
            if (!local.getDate().after(minimumDate)) {
                // branch not recently updated
                continue;
            }
            String branch = local.getName();
            List<RepositoryCommit> commits = CommitCache.instance().getCommits(repositoryName, repository,  branch, minimumDate);
            for (RepositoryCommit commit : commits) {

 src/main/java/com/gitblit/wicket/pages/DashboardPage.java

@@ -16,9 +16,7 @@
package com.gitblit.wicket.pages;



import java.io.Serializable;

import java.text.DateFormat;

import java.text.MessageFormat;

import java.text.SimpleDateFormat;

import java.util.ArrayList;

import java.util.Calendar;

import java.util.Collection;

@@ -55,7 +53,6 @@
import com.gitblit.wicket.charting.GoogleChart;

import com.gitblit.wicket.charting.GoogleCharts;

import com.gitblit.wicket.charting.GooglePieChart;

import com.gitblit.wicket.ng.NgController;

import com.gitblit.wicket.panels.DigestsPanel;

import com.gitblit.wicket.panels.LinkPanel;



@@ -83,10 +80,15 @@
        // create daily commit digest feed

        List<DailyLogEntry> digests = new ArrayList<DailyLogEntry>();

        for (RepositoryModel model : repositories) {

            Repository repository = GitBlit.self().getRepository(model.name);

            List<DailyLogEntry> entries = RefLogUtils.getDailyLogByRef(model.name, repository, minimumDate, timezone);

            digests.addAll(entries);

            repository.close();

            if (model.isCollectingGarbage) {

                continue;

            }

            if (model.hasCommits && model.lastChange.after(minimumDate)) {

                Repository repository = GitBlit.self().getRepository(model.name);

                List<DailyLogEntry> entries = RefLogUtils.getDailyLogByRef(model.name, repository, minimumDate, timezone);

                digests.addAll(entries);

                repository.close();

            }

        }

        

        Fragment activityFragment = new Fragment("activity", "activityFragment", this);