From b32fa68832412374a1a905525a4e395d35b4d1ae Mon Sep 17 00:00:00 2001 From: James Moger <james.moger@gitblit.com> Date: Wed, 20 Apr 2016 10:06:09 -0400 Subject: [PATCH] Use getInteger not getFilesize, fixes #1049 --- src/main/java/com/gitblit/manager/RepositoryManager.java | 238 ++++++++++++++++++++++++++++++++++++++++++++++------------- 1 files changed, 184 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/gitblit/manager/RepositoryManager.java b/src/main/java/com/gitblit/manager/RepositoryManager.java index c141f08..e9bf5b8 100644 --- a/src/main/java/com/gitblit/manager/RepositoryManager.java +++ b/src/main/java/com/gitblit/manager/RepositoryManager.java @@ -21,6 +21,7 @@ import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.Charset; import java.text.MessageFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -52,6 +53,7 @@ import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.RawParseUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,6 +68,7 @@ import com.gitblit.GitBlitException; import com.gitblit.IStoredSettings; import com.gitblit.Keys; +import com.gitblit.extensions.RepositoryLifeCycleListener; import com.gitblit.models.ForkModel; import com.gitblit.models.Metric; import com.gitblit.models.RefModel; @@ -88,6 +91,8 @@ import com.gitblit.utils.ObjectCache; import com.gitblit.utils.StringUtils; import com.gitblit.utils.TimeUtils; +import com.google.inject.Inject; +import com.google.inject.Singleton; /** * Repository manager creates, updates, deletes and caches git repositories. It @@ -96,6 +101,7 @@ * @author James Moger * */ +@Singleton public class RepositoryManager implements IRepositoryManager { private final Logger logger = LoggerFactory.getLogger(getClass()); @@ -114,9 +120,11 @@ private final IRuntimeManager runtimeManager; + private final IPluginManager pluginManager; + private final IUserManager userManager; - private final File repositoriesFolder; + private File repositoriesFolder; private LuceneService luceneExecutor; @@ -124,18 +132,21 @@ private MirrorService mirrorExecutor; + @Inject public RepositoryManager( IRuntimeManager runtimeManager, + IPluginManager pluginManager, IUserManager userManager) { this.settings = runtimeManager.getSettings(); this.runtimeManager = runtimeManager; + this.pluginManager = pluginManager; this.userManager = userManager; - this.repositoriesFolder = runtimeManager.getFileOrFolder(Keys.git.repositoriesFolder, "${baseFolder}/git"); } @Override public RepositoryManager start() { + repositoriesFolder = runtimeManager.getFileOrFolder(Keys.git.repositoriesFolder, "${baseFolder}/git"); logger.info("Repositories folder : {}", repositoriesFolder.getAbsolutePath()); // initialize utilities @@ -417,11 +428,12 @@ @Override public void addToCachedRepositoryList(RepositoryModel model) { if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) { - repositoryListCache.put(model.name.toLowerCase(), model); + String key = getRepositoryKey(model.name); + repositoryListCache.put(key, model); // update the fork origin repository with this repository clone if (!StringUtils.isEmpty(model.originRepository)) { - String originKey = model.originRepository.toLowerCase(); + String originKey = getRepositoryKey(model.originRepository); if (repositoryListCache.containsKey(originKey)) { RepositoryModel origin = repositoryListCache.get(originKey); origin.addFork(model.name); @@ -440,7 +452,8 @@ if (StringUtils.isEmpty(name)) { return null; } - return repositoryListCache.remove(name.toLowerCase()); + String key = getRepositoryKey(name); + return repositoryListCache.remove(key); } /** @@ -464,6 +477,8 @@ public void resetRepositoryCache(String repositoryName) { removeFromCachedRepositoryList(repositoryName); clearRepositoryMetadataCache(repositoryName); + // force a reload of the repository data (ticket-82, issue-433) + getRepositoryModel(repositoryName); } /** @@ -547,7 +562,7 @@ // rebuild fork networks for (RepositoryModel model : repositoryListCache.values()) { if (!StringUtils.isEmpty(model.originRepository)) { - String originKey = model.originRepository.toLowerCase(); + String originKey = getRepositoryKey(model.originRepository); if (repositoryListCache.containsKey(originKey)) { RepositoryModel origin = repositoryListCache.get(originKey); origin.addFork(model.name); @@ -583,15 +598,13 @@ /** * Returns the JGit repository for the specified name. * - * @param repositoryName + * @param name * @param logError * @return repository or null */ @Override - public Repository getRepository(String repositoryName, boolean logError) { - // Decode url-encoded repository name (issue-278) - // http://stackoverflow.com/questions/17183110 - repositoryName = repositoryName.replace("%7E", "~").replace("%7e", "~"); + public Repository getRepository(String name, boolean logError) { + String repositoryName = fixRepositoryName(name); if (isCollectingGarbage(repositoryName)) { logger.warn(MessageFormat.format("Rejecting request for {0}, busy collecting garbage!", repositoryName)); @@ -613,6 +626,27 @@ } } return r; + } + + /** + * Returns the list of all repository models. + * + * @return list of all repository models + */ + @Override + public List<RepositoryModel> getRepositoryModels() { + long methodStart = System.currentTimeMillis(); + List<String> list = getRepositoryList(); + List<RepositoryModel> repositories = new ArrayList<RepositoryModel>(); + for (String repo : list) { + RepositoryModel model = getRepositoryModel(repo); + if (model != null) { + repositories.add(model); + } + } + long duration = System.currentTimeMillis() - methodStart; + logger.info(MessageFormat.format("{0} repository models loaded in {1} msecs", duration)); + return repositories; } /** @@ -673,16 +707,15 @@ * Returns the repository model for the specified repository. This method * does not consider user access permissions. * - * @param repositoryName + * @param name * @return repository model or null */ @Override - public RepositoryModel getRepositoryModel(String repositoryName) { - // Decode url-encoded repository name (issue-278) - // http://stackoverflow.com/questions/17183110 - repositoryName = repositoryName.replace("%7E", "~").replace("%7e", "~"); + public RepositoryModel getRepositoryModel(String name) { + String repositoryName = fixRepositoryName(name); - if (!repositoryListCache.containsKey(repositoryName)) { + String repositoryKey = getRepositoryKey(repositoryName); + if (!repositoryListCache.containsKey(repositoryKey)) { RepositoryModel model = loadRepositoryModel(repositoryName); if (model == null) { return null; @@ -692,9 +725,9 @@ } // cached model - RepositoryModel model = repositoryListCache.get(repositoryName.toLowerCase()); + RepositoryModel model = repositoryListCache.get(repositoryKey); - if (gcExecutor.isCollectingGarbage(model.name)) { + if (isCollectingGarbage(model.name)) { // Gitblit is busy collecting garbage, use our cached model RepositoryModel rm = DeepCopier.copy(model); rm.isCollectingGarbage = true; @@ -747,6 +780,52 @@ } } return count; + } + + /** + * Replaces illegal character patterns in a repository name. + * + * @param repositoryName + * @return a corrected name + */ + private String fixRepositoryName(String repositoryName) { + if (StringUtils.isEmpty(repositoryName)) { + return repositoryName; + } + + // Decode url-encoded repository name (issue-278) + // http://stackoverflow.com/questions/17183110 + String name = repositoryName.replace("%7E", "~").replace("%7e", "~"); + name = name.replace("%2F", "/").replace("%2f", "/"); + + if (name.charAt(name.length() - 1) == '/') { + name = name.substring(0, name.length() - 1); + } + + // strip duplicate-slashes from requests for repositoryName (ticket-117, issue-454) + // specify first char as slash so we strip leading slashes + char lastChar = '/'; + StringBuilder sb = new StringBuilder(); + for (char c : name.toCharArray()) { + if (c == '/' && lastChar == c) { + continue; + } + sb.append(c); + lastChar = c; + } + + return sb.toString(); + } + + /** + * Returns the cache key for the repository name. + * + * @param repositoryName + * @return the cache key for the repository + */ + private String getRepositoryKey(String repositoryName) { + String name = fixRepositoryName(repositoryName); + return StringUtils.stripDotGit(name).toLowerCase(); } /** @@ -924,7 +1003,8 @@ if (!caseSensitiveCheck && settings.getBoolean(Keys.git.cacheRepositoryList, true)) { // if we are caching use the cache to determine availability // otherwise we end up adding a phantom repository to the cache - return repositoryListCache.containsKey(repositoryName.toLowerCase()); + String key = getRepositoryKey(repositoryName); + return repositoryListCache.containsKey(key); } Repository r = getRepository(repositoryName, false); if (r == null) { @@ -962,7 +1042,7 @@ } String userProject = ModelUtils.getPersonalPath(username); if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) { - String originKey = origin.toLowerCase(); + String originKey = getRepositoryKey(origin); String userPath = userProject + "/"; // collect all origin nodes in fork network @@ -979,7 +1059,7 @@ } if (originModel.originRepository != null) { - String ooKey = originModel.originRepository.toLowerCase(); + String ooKey = getRepositoryKey(originModel.originRepository); roots.add(ooKey); originModel = repositoryListCache.get(ooKey); } else { @@ -992,7 +1072,8 @@ if (repository.startsWith(userPath)) { RepositoryModel model = repositoryListCache.get(repository); if (!StringUtils.isEmpty(model.originRepository)) { - if (roots.contains(model.originRepository.toLowerCase())) { + String ooKey = getRepositoryKey(model.originRepository); + if (roots.contains(ooKey)) { // user has a fork in this graph return model.name; } @@ -1030,9 +1111,18 @@ public ForkModel getForkNetwork(String repository) { if (settings.getBoolean(Keys.git.cacheRepositoryList, true)) { // find the root, cached - RepositoryModel model = repositoryListCache.get(repository.toLowerCase()); + String key = getRepositoryKey(repository); + RepositoryModel model = repositoryListCache.get(key); + if (model == null) { + return null; + } + while (model.originRepository != null) { - model = repositoryListCache.get(model.originRepository.toLowerCase()); + String originKey = getRepositoryKey(model.originRepository); + model = repositoryListCache.get(originKey); + if (model == null) { + return null; + } } ForkModel root = getForkModelFromCache(model.name); return root; @@ -1048,7 +1138,8 @@ } private ForkModel getForkModelFromCache(String repository) { - RepositoryModel model = repositoryListCache.get(repository.toLowerCase()); + String key = getRepositoryKey(repository); + RepositoryModel model = repositoryListCache.get(key); if (model == null) { return null; } @@ -1263,7 +1354,7 @@ } /** - * Creates/updates the repository model keyed by reopsitoryName. Saves all + * Creates/updates the repository model keyed by repositoryName. Saves all * repository settings in .git/config. This method allows for renaming * repositories and will update user access permissions accordingly. * @@ -1279,7 +1370,7 @@ @Override public void updateRepositoryModel(String repositoryName, RepositoryModel repository, boolean isCreate) throws GitBlitException { - if (gcExecutor.isCollectingGarbage(repositoryName)) { + if (isCollectingGarbage(repositoryName)) { throw new GitBlitException(MessageFormat.format("sorry, Gitblit is busy collecting garbage in {0}", repositoryName)); } @@ -1291,6 +1382,7 @@ repository.name = repository.name.substring(projectPath.length() + 1); } } + boolean isRename = false; if (isCreate) { // ensure created repository name ends with .git if (!repository.name.toLowerCase().endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT_EXT)) { @@ -1307,7 +1399,8 @@ r = JGitUtils.createRepository(repositoriesFolder, repository.name, shared); } else { // rename repository - if (!repositoryName.equalsIgnoreCase(repository.name)) { + isRename = !repositoryName.equalsIgnoreCase(repository.name); + if (isRename) { if (!repository.name.toLowerCase().endsWith( org.eclipse.jgit.lib.Constants.DOT_GIT_EXT)) { repository.name += org.eclipse.jgit.lib.Constants.DOT_GIT_EXT; @@ -1363,7 +1456,8 @@ // update this repository's origin's fork list if (!StringUtils.isEmpty(repository.originRepository)) { - RepositoryModel origin = repositoryListCache.get(repository.originRepository.toLowerCase()); + String originKey = getRepositoryKey(repository.originRepository); + RepositoryModel origin = repositoryListCache.get(originKey); if (origin != null && !ArrayUtils.isEmpty(origin.forks)) { origin.forks.remove(repositoryName); origin.forks.add(repository.name); @@ -1417,6 +1511,24 @@ removeFromCachedRepositoryList(repositoryName); // model will actually be replaced on next load because config is stale addToCachedRepositoryList(repository); + + if (isCreate && pluginManager != null) { + for (RepositoryLifeCycleListener listener : pluginManager.getExtensions(RepositoryLifeCycleListener.class)) { + try { + listener.onCreation(repository); + } catch (Throwable t) { + logger.error(String.format("failed to call plugin onCreation %s", repositoryName), t); + } + } + } else if (isRename && pluginManager != null) { + for (RepositoryLifeCycleListener listener : pluginManager.getExtensions(RepositoryLifeCycleListener.class)) { + try { + listener.onRename(repositoryName, repository); + } catch (Throwable t) { + logger.error(String.format("failed to call plugin onRename %s", repositoryName), t); + } + } + } } /** @@ -1585,6 +1697,16 @@ FileUtils.delete(folder, FileUtils.RECURSIVE | FileUtils.RETRY); if (userManager.deleteRepositoryRole(repositoryName)) { logger.info(MessageFormat.format("Repository \"{0}\" deleted", repositoryName)); + + if (pluginManager != null) { + for (RepositoryLifeCycleListener listener : pluginManager.getExtensions(RepositoryLifeCycleListener.class)) { + try { + listener.onDeletion(repository); + } catch (Throwable t) { + logger.error(String.format("failed to call plugin onDeletion %s", repositoryName), t); + } + } + } return true; } } @@ -1745,9 +1867,10 @@ protected void configureLuceneIndexing() { luceneExecutor = new LuceneService(settings, this); - int period = 2; - scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, period, TimeUnit.MINUTES); - logger.info("Lucene will process indexed branches every {} minutes.", period); + String frequency = settings.getString(Keys.web.luceneFrequency, "2 mins"); + int mins = TimeUtils.convertFrequencyToMinutes(frequency, 2); + scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, mins, TimeUnit.MINUTES); + logger.info("Lucene will process indexed branches every {} minutes.", mins); } protected void configureGarbageCollector() { @@ -1782,10 +1905,7 @@ protected void configureMirrorExecutor() { mirrorExecutor = new MirrorService(settings, this); if (mirrorExecutor.isReady()) { - int mins = TimeUtils.convertFrequencyToMinutes(settings.getString(Keys.git.mirrorPeriod, "30 mins")); - if (mins < 5) { - mins = 5; - } + int mins = TimeUtils.convertFrequencyToMinutes(settings.getString(Keys.git.mirrorPeriod, "30 mins"), 5); int delay = 1; scheduledExecutor.scheduleAtFixedRate(mirrorExecutor, delay, mins, TimeUnit.MINUTES); logger.info("Mirror service will fetch updates every {} minutes.", mins); @@ -1802,8 +1922,7 @@ cfg.setPackedGitWindowSize(settings.getFilesize(Keys.git.packedGitWindowSize, cfg.getPackedGitWindowSize())); cfg.setPackedGitLimit(settings.getFilesize(Keys.git.packedGitLimit, cfg.getPackedGitLimit())); cfg.setDeltaBaseCacheLimit(settings.getFilesize(Keys.git.deltaBaseCacheLimit, cfg.getDeltaBaseCacheLimit())); - cfg.setPackedGitOpenFiles(settings.getFilesize(Keys.git.packedGitOpenFiles, cfg.getPackedGitOpenFiles())); - cfg.setStreamFileThreshold(settings.getFilesize(Keys.git.streamFileThreshold, cfg.getStreamFileThreshold())); + cfg.setPackedGitOpenFiles(settings.getInteger(Keys.git.packedGitOpenFiles, cfg.getPackedGitOpenFiles())); cfg.setPackedGitMMAP(settings.getBoolean(Keys.git.packedGitMmap, cfg.isPackedGitMMAP())); try { @@ -1812,10 +1931,23 @@ logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.packedGitLimit, cfg.getPackedGitLimit())); logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.deltaBaseCacheLimit, cfg.getDeltaBaseCacheLimit())); logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.packedGitOpenFiles, cfg.getPackedGitOpenFiles())); - logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.streamFileThreshold, cfg.getStreamFileThreshold())); logger.debug(MessageFormat.format("{0} = {1}", Keys.git.packedGitMmap, cfg.isPackedGitMMAP())); } catch (IllegalArgumentException e) { logger.error("Failed to configure JGit parameters!", e); + } + + try { + // issue-486/ticket-151: UTF-9 & UTF-18 + // issue-560/ticket-237: 'UTF8' + Field field = RawParseUtils.class.getDeclaredField("encodingAliases"); + field.setAccessible(true); + Map<String, Charset> encodingAliases = (Map<String, Charset>) field.get(null); + encodingAliases.put("'utf8'", RawParseUtils.UTF8_CHARSET); + encodingAliases.put("utf-9", RawParseUtils.UTF8_CHARSET); + encodingAliases.put("utf-18", RawParseUtils.UTF8_CHARSET); + logger.info("Alias 'UTF8', UTF-9 & UTF-18 encodings as UTF-8 in JGit"); + } catch (Throwable t) { + logger.error("Failed to inject UTF-9 & UTF-18 encoding aliases into JGit", t); } } @@ -1856,21 +1988,19 @@ } protected void confirmWriteAccess() { - if (runtimeManager.isServingRepositories()) { - try { - if (!getRepositoriesFolder().exists()) { - getRepositoriesFolder().mkdirs(); - } - File file = File.createTempFile(".test-", ".txt", getRepositoriesFolder()); - file.delete(); - } catch (Exception e) { - logger.error(""); - logger.error(Constants.BORDER2); - logger.error("Please check filesystem permissions!"); - logger.error("FAILED TO WRITE TO REPOSITORIES FOLDER!!", e); - logger.error(Constants.BORDER2); - logger.error(""); + try { + if (!getRepositoriesFolder().exists()) { + getRepositoriesFolder().mkdirs(); } + File file = File.createTempFile(".test-", ".txt", getRepositoriesFolder()); + file.delete(); + } catch (Exception e) { + logger.error(""); + logger.error(Constants.BORDER2); + logger.error("Please check filesystem permissions!"); + logger.error("FAILED TO WRITE TO REPOSITORIES FOLDER!!", e); + logger.error(Constants.BORDER2); + logger.error(""); } } } -- Gitblit v1.9.1