build.xml
@@ -451,6 +451,7 @@ <resource file="${project.resources.dir}/commit_merge_16x16.png" /> <resource file="${project.resources.dir}/commit_divide_16x16.png" /> <resource file="${project.resources.dir}/star_16x16.png" /> <resource file="${project.resources.dir}/mirror_16x16.png" /> <resource file="${project.resources.dir}/blank.png" /> <resource file="${project.src.dir}/log4j.properties" /> <resource> releases.moxie
@@ -16,6 +16,7 @@ - Fix error on generating activity page when there is no activity - Fix raw page content type of binaries when running behind a reverse proxy changes: - Gitblit now rejects pushes to mirror repositories (issue-5) - Personal repository prefix (~) is now configurable (issue-265) - Reversed line links in blob view (issue-309) - Dashboard and Activity pages now obey the web.generateActivityGraph setting (issue-310) @@ -24,11 +25,12 @@ - Change the WAR baseFolder context parameter to a JNDI env-entry to improve enterprise deployments - Removed internal Gitblit ref exclusions in the upload pack - Removed "show readme" setting in favor of automatic detection - Support plain text "readme" files - Support plain text, markdown, confluence, mediawiki, textile, tracwiki, or twiki "readme" files - Determine best commit id (e.g. "master") for the tree and docs pages and use that in links - By default GO will now bind to all interfaces for both http and https connectors. This simplifies setup for first-time users. - Removed docs indicator on the repositories page additions: - Added an optional MirrorExecutor which will periodically fetch ref updates from source repositories for mirrors (issue-5). Repositories must be manually cloned using native git and "--mirror". - Added branch graph image servlet based on EGit's branch graph renderer (issue-194) - Added option to render Markdown commit messages (issue-203) - Added setting to control creating a repository as --shared on Unix servers (issue-263) @@ -47,7 +49,9 @@ settings: - { name: 'git.createRepositoriesShared', defaultValue: 'false' } - { name: 'git.allowAnonymousPushes', defaultValue: 'false' } - { name: 'git.enableMirroring', defaultValue: 'false' } - { name: 'git.defaultAccessRestriction', defaultValue: 'PUSH' } - { name: 'git.mirrorPeriod', defaultValue: '30 mins' } - { name: 'web.commitMessageRenderer', defaultValue: 'plain' } - { name: 'web.showBranchGraph', defaultValue: 'true' } - { name: 'server.redirectToHttpsPort', defaultValue: 'true' } src/main/distrib/data/gitblit.properties
@@ -276,6 +276,34 @@ # SINCE 1.2.0 git.defaultGarbageCollectionPeriod = 7 # Gitblit can automatically fetch ref updates for a properly configured mirror # repository. # # Requirements: # 1. you must manually clone the repository using native git # git clone --mirror git://somewhere.com/myrepo.git # 2. the "origin" remote must be the mirror source # 3. the "origin" repository must be accessible without authentication OR the # credentials must be embedded in the origin url (not recommended) # # Notes: # 1. "origin" SSH urls are untested and not likely to work # 2. mirrors cloned while Gitblit is running are likely to require clearing the # gitblit cache (link on the repositories page of an administrator account) # 3. Gitblit will automatically repair any invalid fetch refspecs with a "//" # sequence. # # SINCE 1.4.0 # RESTART REQUIRED git.enableMirroring = false # Specify the period between update checks for mirrored repositories. # The shortest period you may specify between mirror update checks is 5 mins. # # SINCE 1.4.0 # RESTART REQUIRED git.mirrorPeriod = 30 mins # Number of bytes of a pack file to load into memory in a single read operation. # This is the "page size" of the JGit buffer cache, used for all pack access # operations. All disk IO occurs as single window reads. Setting this too large src/main/java/com/gitblit/GitBlit.java
@@ -164,7 +164,7 @@ private final Logger logger = LoggerFactory.getLogger(GitBlit.class); private final ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(5); private final ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(10); private final List<FederationModel> federationRegistrations = Collections .synchronizedList(new ArrayList<FederationModel>()); @@ -206,6 +206,8 @@ private LuceneExecutor luceneExecutor; private GCExecutor gcExecutor; private MirrorExecutor mirrorExecutor; private TimeZone timezone; @@ -2035,6 +2037,7 @@ model.origin = config.getString("remote", "origin", "url"); if (model.origin != null) { model.origin = model.origin.replace('\\', '/'); model.isMirror = config.getBoolean("remote", "origin", "mirror", false); } model.preReceiveScripts = new ArrayList<String>(Arrays.asList(config.getStringList( Constants.CONFIG_GITBLIT, null, "preReceiveScript"))); @@ -3505,6 +3508,7 @@ mailExecutor = new MailExecutor(settings); luceneExecutor = new LuceneExecutor(settings, repositoriesFolder); gcExecutor = new GCExecutor(settings); mirrorExecutor = new MirrorExecutor(settings); // initialize utilities String prefix = settings.getString(Keys.git.userRepositoryPrefix, "~"); @@ -3544,6 +3548,7 @@ configureMailExecutor(); configureLuceneIndexing(); configureGarbageCollector(); configureMirrorExecutor(); if (startFederation) { configureFederation(); } @@ -3592,6 +3597,19 @@ } logger.info(MessageFormat.format("Next scheculed GC scan is in {0}", when)); scheduledExecutor.scheduleAtFixedRate(gcExecutor, delay, 60*24, TimeUnit.MINUTES); } } protected void configureMirrorExecutor() { if (mirrorExecutor.isReady()) { int mins = TimeUtils.convertFrequencyToMinutes(settings.getString(Keys.git.mirrorPeriod, "30 mins")); if (mins < 5) { mins = 5; } int delay = 1; scheduledExecutor.scheduleAtFixedRate(mirrorExecutor, delay, mins, TimeUnit.MINUTES); logger.info("Mirror executor is scheduled to fetch updates every {} minutes.", mins); logger.info("Next scheduled mirror fetch is in {} minutes", delay); } } @@ -3864,6 +3882,7 @@ scheduledExecutor.shutdownNow(); luceneExecutor.close(); gcExecutor.close(); mirrorExecutor.close(); if (fanoutService != null) { fanoutService.stop(); } src/main/java/com/gitblit/MirrorExecutor.java
New file @@ -0,0 +1,169 @@ /* * Copyright 2013 gitblit.com. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.gitblit; import java.text.MessageFormat; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.TrackingRefUpdate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.gitblit.models.RepositoryModel; import com.gitblit.models.UserModel; import com.gitblit.utils.JGitUtils; /** * The Mirror executor handles periodic fetching of mirrored repositories. * * @author James Moger * */ public class MirrorExecutor implements Runnable { private final Logger logger = LoggerFactory.getLogger(MirrorExecutor.class); private final Set<String> repairAttempted = Collections.synchronizedSet(new HashSet<String>()); private final IStoredSettings settings; private AtomicBoolean running = new AtomicBoolean(false); private AtomicBoolean forceClose = new AtomicBoolean(false); private final UserModel gitblitUser; public MirrorExecutor(IStoredSettings settings) { this.settings = settings; this.gitblitUser = new UserModel("gitblit"); this.gitblitUser.displayName = "Gitblit"; } public boolean isReady() { return settings.getBoolean(Keys.git.enableMirroring, false); } public boolean isRunning() { return running.get(); } public void close() { forceClose.set(true); } @Override public void run() { if (!isReady()) { return; } running.set(true); for (String repositoryName : GitBlit.self().getRepositoryList()) { if (forceClose.get()) { break; } if (GitBlit.self().isCollectingGarbage(repositoryName)) { logger.debug("mirror is skipping {} garbagecollection", repositoryName); continue; } RepositoryModel model = null; Repository repository = null; try { model = GitBlit.self().getRepositoryModel(repositoryName); if (!model.isMirror && !model.isBare) { // repository must be a valid bare git mirror logger.debug("mirror is skipping {} !mirror !bare", repositoryName); continue; } repository = GitBlit.self().getRepository(repositoryName); if (repository == null) { logger.warn(MessageFormat.format("MirrorExecutor is missing repository {0}?!?", repositoryName)); continue; } // automatically repair (some) invalid fetch ref specs if (!repairAttempted.contains(repositoryName)) { repairAttempted.add(repositoryName); JGitUtils.repairFetchSpecs(repository); } // find the first mirror remote - there should only be one StoredConfig rc = repository.getConfig(); RemoteConfig mirror = null; List<RemoteConfig> configs = RemoteConfig.getAllRemoteConfigs(rc); for (RemoteConfig config : configs) { if (config.isMirror()) { mirror = config; break; } } if (mirror == null) { // repository does not have a mirror remote logger.debug("mirror is skipping {} no mirror remote found", repositoryName); continue; } logger.debug("checking {} remote {} for ref updates", repositoryName, mirror.getName()); final boolean testing = false; Git git = new Git(repository); FetchResult result = git.fetch().setRemote(mirror.getName()).setDryRun(testing).call(); Collection<TrackingRefUpdate> refUpdates = result.getTrackingRefUpdates(); if (refUpdates.size() > 0) { for (TrackingRefUpdate ru : refUpdates) { StringBuilder sb = new StringBuilder(); sb.append("updated mirror "); sb.append(repositoryName); sb.append(" "); sb.append(ru.getRemoteName()); sb.append(" -> "); sb.append(ru.getLocalName()); if (ru.getResult() == Result.FORCED) { sb.append(" (forced)"); } sb.append(" "); sb.append(ru.getOldObjectId() == null ? "" : ru.getOldObjectId().abbreviate(7).name()); sb.append(".."); sb.append(ru.getNewObjectId() == null ? "" : ru.getNewObjectId().abbreviate(7).name()); logger.info(sb.toString()); } } } catch (Exception e) { logger.error("Error updating mirror " + repositoryName, e); } finally { // cleanup if (repository != null) { repository.close(); } } } running.set(false); } } src/main/java/com/gitblit/client/IndicatorsRenderer.java
@@ -56,6 +56,8 @@ private final ImageIcon sparkleshareIcon; private final ImageIcon mirrorIcon; public IndicatorsRenderer() { super(new FlowLayout(FlowLayout.RIGHT, 1, 0)); blankIcon = new ImageIcon(getClass().getResource("/blank.png")); @@ -67,6 +69,7 @@ federatedIcon = new ImageIcon(getClass().getResource("/federated_16x16.png")); forkIcon = new ImageIcon(getClass().getResource("/commit_divide_16x16.png")); sparkleshareIcon = new ImageIcon(getClass().getResource("/star_16x16.png")); mirrorIcon = new ImageIcon(getClass().getResource("/mirror_16x16.png")); } @Override @@ -85,6 +88,11 @@ tooltip.append(Translation.get("gb.isSparkleshared")).append("<br/>"); add(icon); } if (model.isMirror) { JLabel icon = new JLabel(mirrorIcon); tooltip.append(Translation.get("gb.isMirror")).append("<br/>"); add(icon); } if (model.isFork()) { JLabel icon = new JLabel(forkIcon); tooltip.append(Translation.get("gb.isFork")).append("<br/>"); src/main/java/com/gitblit/git/GitblitReceivePack.java
@@ -120,6 +120,14 @@ @Override public void onPreReceive(ReceivePack rp, Collection<ReceiveCommand> commands) { if (repository.isMirror) { // repository is a mirror for (ReceiveCommand cmd : commands) { sendRejection(cmd, "Gitblit does not allow pushes to \"{0}\" because it is a mirror!", repository.name); } return; } if (repository.isFrozen) { // repository is frozen/readonly for (ReceiveCommand cmd : commands) { src/main/java/com/gitblit/models/RepositoryModel.java
@@ -65,6 +65,7 @@ public boolean skipSummaryMetrics; public String frequency; public boolean isBare; public boolean isMirror; public String origin; public String HEAD; public List<String> availableRefs; src/main/java/com/gitblit/models/TeamModel.java
@@ -206,7 +206,7 @@ // determine maximum permission for the repository final AccessPermission maxPermission = (repository.isFrozen || !repository.isBare) ? (repository.isFrozen || !repository.isBare || repository.isMirror) ? AccessPermission.CLONE : AccessPermission.REWIND; if (AccessRestrictionType.NONE.equals(repository.accessRestriction)) { src/main/java/com/gitblit/models/UserModel.java
@@ -292,7 +292,7 @@ // determine maximum permission for the repository final AccessPermission maxPermission = (repository.isFrozen || !repository.isBare) ? (repository.isFrozen || !repository.isBare || repository.isMirror) ? AccessPermission.CLONE : AccessPermission.REWIND; if (AccessRestrictionType.NONE.equals(repository.accessRestriction)) { src/main/java/com/gitblit/utils/JGitUtils.java
@@ -2096,4 +2096,54 @@ } return StringUtils.decodeString(content); } /** * Automatic repair of (some) invalid refspecs. These are the result of a * bug in JGit cloning where a double forward-slash was injected. :( * * @param repository * @return true, if the refspecs were repaired */ public static boolean repairFetchSpecs(Repository repository) { StoredConfig rc = repository.getConfig(); // auto-repair broken fetch ref specs for (String name : rc.getSubsections("remote")) { int invalidSpecs = 0; int repairedSpecs = 0; List<String> specs = new ArrayList<String>(); for (String spec : rc.getStringList("remote", name, "fetch")) { try { RefSpec rs = new RefSpec(spec); // valid spec specs.add(spec); } catch (IllegalArgumentException e) { // invalid spec invalidSpecs++; if (spec.contains("//")) { // auto-repair this known spec bug spec = spec.replace("//", "/"); specs.add(spec); repairedSpecs++; } } } if (invalidSpecs == repairedSpecs && repairedSpecs > 0) { // the fetch specs were automatically repaired rc.setStringList("remote", name, "fetch", specs); try { rc.save(); rc.load(); LOGGER.debug("repaired {} invalid fetch refspecs for {}", repairedSpecs, repository.getDirectory()); return true; } catch (Exception e) { LOGGER.error(null, e); } } else if (invalidSpecs > 0) { LOGGER.error("mirror executor found {} invalid fetch refspecs for {}", invalidSpecs, repository.getDirectory()); } } return false; } } src/main/java/com/gitblit/wicket/GitBlitWebApp.properties
@@ -505,3 +505,6 @@ gb.commitMessageRenderer = commit message renderer gb.diffStat = {0} insertions & {1} deletions gb.home = home gb.isMirror = this repository is a mirror gb.mirrorOf = mirror of {0} gb.mirrorWarning = this repository is a mirror and can not receive pushes src/main/java/com/gitblit/wicket/pages/RepositoryPage.html
@@ -62,6 +62,10 @@ <wicket:fragment wicket:id="originFragment"> <p class="originRepository"><wicket:message key="gb.forkedFrom">[forked from]</wicket:message> <span wicket:id="originRepository">[origin repository]</span></p> </wicket:fragment> <wicket:fragment wicket:id="mirrorFragment"> <p class="originRepository"><span wicket:id="originRepository">[origin repository]</span></p> </wicket:fragment> </wicket:extend> </body> src/main/java/com/gitblit/wicket/pages/RepositoryPage.java
@@ -260,7 +260,14 @@ // indicate origin repository RepositoryModel model = getRepositoryModel(); if (StringUtils.isEmpty(model.originRepository)) { if (model.isMirror) { Fragment mirrorFrag = new Fragment("originRepository", "mirrorFragment", this); Label lbl = new Label("originRepository", MessageFormat.format(getString("gb.mirrorOf"), "<b>" + model.origin + "</b>")); mirrorFrag.add(lbl.setEscapeModelStrings(false)); add(mirrorFrag); } else { add(new Label("originRepository").setVisible(false)); } } else { RepositoryModel origin = GitBlit.self().getRepositoryModel(model.originRepository); if (origin == null) { src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.html
@@ -39,6 +39,7 @@ <span wicket:id="repositoryLinks"></span> <div> <img class="inlineIcon" wicket:id="sparkleshareIcon" /> <img class="inlineIcon" wicket:id="mirrorIcon" /> <img class="inlineIcon" wicket:id="frozenIcon" /> <img class="inlineIcon" wicket:id="federatedIcon" />
@@ -87,6 +87,12 @@ add(WicketUtils.newClearPixel("sparkleshareIcon").setVisible(false)); } if (entry.isMirror) { add(WicketUtils.newImage("mirrorIcon", "mirror_16x16.png", localizer.getString("gb.isMirror", parent))); } else { add(WicketUtils.newClearPixel("mirrorIcon").setVisible(false)); } if (entry.isFrozen) { add(WicketUtils.newImage("frozenIcon", "cold_16x16.png", localizer.getString("gb.isFrozen", parent))); } else { src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.html
@@ -89,7 +89,7 @@ <td class="left" style="padding-left:3px;" ><b><span class="repositorySwatch" wicket:id="repositorySwatch"></span></b> <span style="padding-left:3px;" wicket:id="repositoryName">[repository name]</span></td> <td class="hidden-phone"><span class="list" wicket:id="repositoryDescription">[repository description]</span></td> <td class="hidden-tablet hidden-phone author"><span wicket:id="repositoryOwner">[repository owner]</span></td> <td class="hidden-phone" style="text-align: right;padding-right:10px;"><img class="inlineIcon" wicket:id="sparkleshareIcon" /><img class="inlineIcon" wicket:id="forkIcon" /><img class="inlineIcon" wicket:id="frozenIcon" /><img class="inlineIcon" wicket:id="federatedIcon" /><img class="inlineIcon" wicket:id="accessRestrictionIcon" /></td> <td class="hidden-phone" style="text-align: right;padding-right:10px;"><img class="inlineIcon" wicket:id="sparkleshareIcon" /><img class="inlineIcon" wicket:id="mirrorIcon" /><img class="inlineIcon" wicket:id="forkIcon" /><img class="inlineIcon" wicket:id="frozenIcon" /><img class="inlineIcon" wicket:id="federatedIcon" /><img class="inlineIcon" wicket:id="accessRestrictionIcon" /></td> <td><span wicket:id="repositoryLastChange">[last change]</span></td> <td class="hidden-phone" style="text-align: right;padding-right:15px;"><span style="font-size:0.8em;" wicket:id="repositorySize">[repository size]</span></td> <td class="rightAlign"> src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.java
@@ -243,6 +243,13 @@ row.add(WicketUtils.newClearPixel("sparkleshareIcon").setVisible(false)); } if (entry.isMirror) { row.add(WicketUtils.newImage("mirrorIcon", "mirror_16x16.png", getString("gb.isMirror"))); } else { row.add(WicketUtils.newClearPixel("mirrorIcon").setVisible(false)); } if (entry.isFork()) { row.add(WicketUtils.newImage("forkIcon", "commit_divide_16x16.png", getString("gb.isFork"))); src/main/resources/mirror_16x16.png
@@ -19,6 +19,7 @@ - Optional feature to allow users to create personal repositories - Optional feature to fork a repository to a personal repository - Optional feature to create a repository on push - Optional feature to automatically fetch ref updates for repository mirrors - *Experimental* built-in Garbage Collection - Ability to federate with one or more other Gitblit instances - RSS/JSON RPC interface src/test/java/com/gitblit/tests/PermissionsTest.java
@@ -2878,4 +2878,22 @@ assertEquals("user has wrong permission!", AccessPermission.CLONE, user.getRepositoryPermission(repo).permission); assertEquals("team has wrong permission!", AccessPermission.CLONE, team.getRepositoryPermission(repo).permission); } @Test public void testIsMirror() throws Exception { RepositoryModel repo = new RepositoryModel("somerepo.git", null, null, new Date()); repo.authorizationControl = AuthorizationControl.NAMED; repo.accessRestriction = AccessRestrictionType.NONE; UserModel user = new UserModel("test"); TeamModel team = new TeamModel("team"); assertEquals("user has wrong permission!", AccessPermission.REWIND, user.getRepositoryPermission(repo).permission); assertEquals("team has wrong permission!", AccessPermission.REWIND, team.getRepositoryPermission(repo).permission); // set repo to be a mirror, pushes prohibited repo.isMirror = true; assertEquals("user has wrong permission!", AccessPermission.CLONE, user.getRepositoryPermission(repo).permission); assertEquals("team has wrong permission!", AccessPermission.CLONE, team.getRepositoryPermission(repo).permission); } }
