/*
|
* Copyright 2011 gitblit.com.
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*/
|
package com.gitblit;
|
|
import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;
|
|
import java.io.File;
|
import java.io.FileOutputStream;
|
import java.io.IOException;
|
import java.net.InetAddress;
|
import java.text.MessageFormat;
|
import java.util.ArrayList;
|
import java.util.Arrays;
|
import java.util.Collection;
|
import java.util.Date;
|
import java.util.HashMap;
|
import java.util.HashSet;
|
import java.util.List;
|
import java.util.Map;
|
import java.util.Properties;
|
import java.util.Set;
|
import java.util.concurrent.TimeUnit;
|
|
import org.eclipse.jgit.lib.Repository;
|
import org.eclipse.jgit.lib.StoredConfig;
|
import org.eclipse.jgit.revwalk.RevCommit;
|
import org.eclipse.jgit.transport.CredentialsProvider;
|
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
|
import org.slf4j.Logger;
|
import org.slf4j.LoggerFactory;
|
|
import com.gitblit.Constants.AccessPermission;
|
import com.gitblit.Constants.FederationPullStatus;
|
import com.gitblit.Constants.FederationStrategy;
|
import com.gitblit.GitBlitException.ForbiddenException;
|
import com.gitblit.models.FederationModel;
|
import com.gitblit.models.RefModel;
|
import com.gitblit.models.RepositoryModel;
|
import com.gitblit.models.TeamModel;
|
import com.gitblit.models.UserModel;
|
import com.gitblit.utils.ArrayUtils;
|
import com.gitblit.utils.FederationUtils;
|
import com.gitblit.utils.FileUtils;
|
import com.gitblit.utils.JGitUtils;
|
import com.gitblit.utils.JGitUtils.CloneResult;
|
import com.gitblit.utils.StringUtils;
|
import com.gitblit.utils.TimeUtils;
|
|
/**
|
* FederationPullExecutor pulls repository updates and, optionally, user
|
* accounts and server settings from registered Gitblit instances.
|
*/
|
public class FederationPullExecutor implements Runnable {
|
|
private final Logger logger = LoggerFactory.getLogger(FederationPullExecutor.class);
|
|
private final List<FederationModel> registrations;
|
|
private final boolean isDaemon;
|
|
/**
|
* Constructor for specifying a single federation registration. This
|
* constructor is used to schedule the next pull execution.
|
*
|
* @param registration
|
*/
|
private FederationPullExecutor(FederationModel registration) {
|
this(Arrays.asList(registration), true);
|
}
|
|
/**
|
* Constructor to specify a group of federation registrations. This is
|
* normally used at startup to pull and then schedule the next update based
|
* on each registrations frequency setting.
|
*
|
* @param registrations
|
* @param isDaemon
|
* if true, registrations are rescheduled in perpetuity. if
|
* false, the federation pull operation is executed once.
|
*/
|
public FederationPullExecutor(List<FederationModel> registrations, boolean isDaemon) {
|
this.registrations = registrations;
|
this.isDaemon = isDaemon;
|
}
|
|
/**
|
* Run method for this pull executor.
|
*/
|
@Override
|
public void run() {
|
for (FederationModel registration : registrations) {
|
FederationPullStatus was = registration.getLowestStatus();
|
try {
|
Date now = new Date(System.currentTimeMillis());
|
pull(registration);
|
sendStatusAcknowledgment(registration);
|
registration.lastPull = now;
|
FederationPullStatus is = registration.getLowestStatus();
|
if (is.ordinal() < was.ordinal()) {
|
// the status for this registration has downgraded
|
logger.warn("Federation pull status of {0} is now {1}", registration.name,
|
is.name());
|
if (registration.notifyOnError) {
|
String message = "Federation pull of " + registration.name + " @ "
|
+ registration.url + " is now at " + is.name();
|
GitBlit.self()
|
.sendMailToAdministrators(
|
"Pull Status of " + registration.name + " is " + is.name(),
|
message);
|
}
|
}
|
} catch (Throwable t) {
|
logger.error(MessageFormat.format(
|
"Failed to pull from federated gitblit ({0} @ {1})", registration.name,
|
registration.url), t);
|
} finally {
|
if (isDaemon) {
|
schedule(registration);
|
}
|
}
|
}
|
}
|
|
/**
|
* Mirrors a repository and, optionally, the server's users, and/or
|
* configuration settings from a origin Gitblit instance.
|
*
|
* @param registration
|
* @throws Exception
|
*/
|
private void pull(FederationModel registration) throws Exception {
|
Map<String, RepositoryModel> repositories = FederationUtils.getRepositories(registration,
|
true);
|
String registrationFolder = registration.folder.toLowerCase().trim();
|
// confirm valid characters in server alias
|
Character c = StringUtils.findInvalidCharacter(registrationFolder);
|
if (c != null) {
|
logger.error(MessageFormat
|
.format("Illegal character ''{0}'' in folder name ''{1}'' of federation registration {2}!",
|
c, registrationFolder, registration.name));
|
return;
|
}
|
File repositoriesFolder = GitBlit.getRepositoriesFolder();
|
File registrationFolderFile = new File(repositoriesFolder, registrationFolder);
|
registrationFolderFile.mkdirs();
|
|
// Clone/Pull the repository
|
for (Map.Entry<String, RepositoryModel> entry : repositories.entrySet()) {
|
String cloneUrl = entry.getKey();
|
RepositoryModel repository = entry.getValue();
|
if (!repository.hasCommits) {
|
logger.warn(MessageFormat.format(
|
"Skipping federated repository {0} from {1} @ {2}. Repository is EMPTY.",
|
repository.name, registration.name, registration.url));
|
registration.updateStatus(repository, FederationPullStatus.SKIPPED);
|
continue;
|
}
|
|
// Determine local repository name
|
String repositoryName;
|
if (StringUtils.isEmpty(registrationFolder)) {
|
repositoryName = repository.name;
|
} else {
|
repositoryName = registrationFolder + "/" + repository.name;
|
}
|
|
if (registration.bare) {
|
// bare repository, ensure .git suffix
|
if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
|
repositoryName += DOT_GIT_EXT;
|
}
|
} else {
|
// normal repository, strip .git suffix
|
if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
|
repositoryName = repositoryName.substring(0,
|
repositoryName.indexOf(DOT_GIT_EXT));
|
}
|
}
|
|
// confirm that the origin of any pre-existing repository matches
|
// the clone url
|
String fetchHead = null;
|
Repository existingRepository = GitBlit.self().getRepository(repositoryName);
|
|
if (existingRepository == null && GitBlit.self().isCollectingGarbage(repositoryName)) {
|
logger.warn(MessageFormat.format("Skipping local repository {0}, busy collecting garbage", repositoryName));
|
continue;
|
}
|
|
if (existingRepository != null) {
|
StoredConfig config = existingRepository.getConfig();
|
config.load();
|
String origin = config.getString("remote", "origin", "url");
|
RevCommit commit = JGitUtils.getCommit(existingRepository,
|
org.eclipse.jgit.lib.Constants.FETCH_HEAD);
|
if (commit != null) {
|
fetchHead = commit.getName();
|
}
|
existingRepository.close();
|
if (!origin.startsWith(registration.url)) {
|
logger.warn(MessageFormat
|
.format("Skipping federated repository {0} from {1} @ {2}. Origin does not match, consider EXCLUDING.",
|
repository.name, registration.name, registration.url));
|
registration.updateStatus(repository, FederationPullStatus.SKIPPED);
|
continue;
|
}
|
}
|
|
// clone/pull this repository
|
CredentialsProvider credentials = new UsernamePasswordCredentialsProvider(
|
Constants.FEDERATION_USER, registration.token);
|
logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",
|
repository.name, registration.name, registration.url));
|
|
CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,
|
cloneUrl, registration.bare, credentials);
|
Repository r = GitBlit.self().getRepository(repositoryName);
|
RepositoryModel rm = GitBlit.self().getRepositoryModel(repositoryName);
|
repository.isFrozen = registration.mirror;
|
if (result.createdRepository) {
|
// default local settings
|
repository.federationStrategy = FederationStrategy.EXCLUDE;
|
repository.isFrozen = registration.mirror;
|
repository.showRemoteBranches = !registration.mirror;
|
logger.info(MessageFormat.format(" cloning {0}", repository.name));
|
registration.updateStatus(repository, FederationPullStatus.MIRRORED);
|
} else {
|
// fetch and update
|
boolean fetched = false;
|
RevCommit commit = JGitUtils.getCommit(r, org.eclipse.jgit.lib.Constants.FETCH_HEAD);
|
String newFetchHead = commit.getName();
|
fetched = fetchHead == null || !fetchHead.equals(newFetchHead);
|
|
if (registration.mirror) {
|
// mirror
|
if (fetched) {
|
// update local branches to match the remote tracking branches
|
for (RefModel ref : JGitUtils.getRemoteBranches(r, false, -1)) {
|
if (ref.displayName.startsWith("origin/")) {
|
String branch = org.eclipse.jgit.lib.Constants.R_HEADS
|
+ ref.displayName.substring(ref.displayName.indexOf('/') + 1);
|
String hash = ref.getReferencedObjectId().getName();
|
|
JGitUtils.setBranchRef(r, branch, hash);
|
logger.info(MessageFormat.format(" resetting {0} of {1} to {2}", branch,
|
repository.name, hash));
|
}
|
}
|
|
String newHead;
|
if (StringUtils.isEmpty(repository.HEAD)) {
|
newHead = newFetchHead;
|
} else {
|
newHead = repository.HEAD;
|
}
|
JGitUtils.setHEADtoRef(r, newHead);
|
logger.info(MessageFormat.format(" resetting HEAD of {0} to {1}",
|
repository.name, newHead));
|
registration.updateStatus(repository, FederationPullStatus.MIRRORED);
|
} else {
|
// indicate no commits pulled
|
registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
|
}
|
} else {
|
// non-mirror
|
if (fetched) {
|
// indicate commits pulled to origin/master
|
registration.updateStatus(repository, FederationPullStatus.PULLED);
|
} else {
|
// indicate no commits pulled
|
registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
|
}
|
}
|
|
// preserve local settings
|
repository.isFrozen = rm.isFrozen;
|
repository.federationStrategy = rm.federationStrategy;
|
|
// merge federation sets
|
Set<String> federationSets = new HashSet<String>();
|
if (rm.federationSets != null) {
|
federationSets.addAll(rm.federationSets);
|
}
|
if (repository.federationSets != null) {
|
federationSets.addAll(repository.federationSets);
|
}
|
repository.federationSets = new ArrayList<String>(federationSets);
|
|
// merge indexed branches
|
Set<String> indexedBranches = new HashSet<String>();
|
if (rm.indexedBranches != null) {
|
indexedBranches.addAll(rm.indexedBranches);
|
}
|
if (repository.indexedBranches != null) {
|
indexedBranches.addAll(repository.indexedBranches);
|
}
|
repository.indexedBranches = new ArrayList<String>(indexedBranches);
|
|
}
|
// only repositories that are actually _cloned_ from the origin
|
// Gitblit repository are marked as federated. If the origin
|
// is from somewhere else, these repositories are not considered
|
// "federated" repositories.
|
repository.isFederated = cloneUrl.startsWith(registration.url);
|
|
GitBlit.self().updateConfiguration(r, repository);
|
r.close();
|
}
|
|
IUserService userService = null;
|
|
try {
|
// Pull USERS
|
// TeamModels are automatically pulled because they are contained
|
// within the UserModel. The UserService creates unknown teams
|
// and updates existing teams.
|
Collection<UserModel> users = FederationUtils.getUsers(registration);
|
if (users != null && users.size() > 0) {
|
File realmFile = new File(registrationFolderFile, registration.name + "_users.conf");
|
realmFile.delete();
|
userService = new ConfigUserService(realmFile);
|
for (UserModel user : users) {
|
userService.updateUserModel(user.username, user);
|
|
// merge the origin permissions and origin accounts into
|
// the user accounts of this Gitblit instance
|
if (registration.mergeAccounts) {
|
// reparent all repository permissions if the local
|
// repositories are stored within subfolders
|
if (!StringUtils.isEmpty(registrationFolder)) {
|
if (user.permissions != null) {
|
// pulling from >= 1.2 version
|
Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
|
user.permissions.clear();
|
for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
|
user.setRepositoryPermission(registrationFolder + "/" + entry.getKey(), entry.getValue());
|
}
|
} else {
|
// pulling from <= 1.1 version
|
List<String> permissions = new ArrayList<String>(user.repositories);
|
user.repositories.clear();
|
for (String permission : permissions) {
|
user.addRepositoryPermission(registrationFolder + "/" + permission);
|
}
|
}
|
}
|
|
// insert new user or update local user
|
UserModel localUser = GitBlit.self().getUserModel(user.username);
|
if (localUser == null) {
|
// create new local user
|
GitBlit.self().updateUserModel(user.username, user, true);
|
} else {
|
// update repository permissions of local user
|
if (user.permissions != null) {
|
// pulling from >= 1.2 version
|
Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
|
for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
|
localUser.setRepositoryPermission(entry.getKey(), entry.getValue());
|
}
|
} else {
|
// pulling from <= 1.1 version
|
for (String repository : user.repositories) {
|
localUser.addRepositoryPermission(repository);
|
}
|
}
|
localUser.password = user.password;
|
localUser.canAdmin = user.canAdmin;
|
GitBlit.self().updateUserModel(localUser.username, localUser, false);
|
}
|
|
for (String teamname : GitBlit.self().getAllTeamnames()) {
|
TeamModel team = GitBlit.self().getTeamModel(teamname);
|
if (user.isTeamMember(teamname) && !team.hasUser(user.username)) {
|
// new team member
|
team.addUser(user.username);
|
GitBlit.self().updateTeamModel(teamname, team, false);
|
} else if (!user.isTeamMember(teamname) && team.hasUser(user.username)) {
|
// remove team member
|
team.removeUser(user.username);
|
GitBlit.self().updateTeamModel(teamname, team, false);
|
}
|
|
// update team repositories
|
TeamModel remoteTeam = user.getTeam(teamname);
|
if (remoteTeam != null) {
|
if (remoteTeam.permissions != null) {
|
// pulling from >= 1.2
|
for (Map.Entry<String, AccessPermission> entry : remoteTeam.permissions.entrySet()){
|
team.setRepositoryPermission(entry.getKey(), entry.getValue());
|
}
|
GitBlit.self().updateTeamModel(teamname, team, false);
|
} else if(!ArrayUtils.isEmpty(remoteTeam.repositories)) {
|
// pulling from <= 1.1
|
team.addRepositoryPermissions(remoteTeam.repositories);
|
GitBlit.self().updateTeamModel(teamname, team, false);
|
}
|
}
|
}
|
}
|
}
|
}
|
} catch (ForbiddenException e) {
|
// ignore forbidden exceptions
|
} catch (IOException e) {
|
logger.warn(MessageFormat.format(
|
"Failed to retrieve USERS from federated gitblit ({0} @ {1})",
|
registration.name, registration.url), e);
|
}
|
|
try {
|
// Pull TEAMS
|
// We explicitly pull these even though they are embedded in
|
// UserModels because it is possible to use teams to specify
|
// mailing lists or push scripts without specifying users.
|
if (userService != null) {
|
Collection<TeamModel> teams = FederationUtils.getTeams(registration);
|
if (teams != null && teams.size() > 0) {
|
for (TeamModel team : teams) {
|
userService.updateTeamModel(team);
|
}
|
}
|
}
|
} catch (ForbiddenException e) {
|
// ignore forbidden exceptions
|
} catch (IOException e) {
|
logger.warn(MessageFormat.format(
|
"Failed to retrieve TEAMS from federated gitblit ({0} @ {1})",
|
registration.name, registration.url), e);
|
}
|
|
try {
|
// Pull SETTINGS
|
Map<String, String> settings = FederationUtils.getSettings(registration);
|
if (settings != null && settings.size() > 0) {
|
Properties properties = new Properties();
|
properties.putAll(settings);
|
FileOutputStream os = new FileOutputStream(new File(registrationFolderFile,
|
registration.name + "_" + Constants.PROPERTIES_FILE));
|
properties.store(os, null);
|
os.close();
|
}
|
} catch (ForbiddenException e) {
|
// ignore forbidden exceptions
|
} catch (IOException e) {
|
logger.warn(MessageFormat.format(
|
"Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
|
registration.name, registration.url), e);
|
}
|
|
try {
|
// Pull SCRIPTS
|
Map<String, String> scripts = FederationUtils.getScripts(registration);
|
if (scripts != null && scripts.size() > 0) {
|
for (Map.Entry<String, String> script : scripts.entrySet()) {
|
String scriptName = script.getKey();
|
if (scriptName.endsWith(".groovy")) {
|
scriptName = scriptName.substring(0, scriptName.indexOf(".groovy"));
|
}
|
File file = new File(registrationFolderFile, registration.name + "_"
|
+ scriptName + ".groovy");
|
FileUtils.writeContent(file, script.getValue());
|
}
|
}
|
} catch (ForbiddenException e) {
|
// ignore forbidden exceptions
|
} catch (IOException e) {
|
logger.warn(MessageFormat.format(
|
"Failed to retrieve SCRIPTS from federated gitblit ({0} @ {1})",
|
registration.name, registration.url), e);
|
}
|
}
|
|
/**
|
* Sends a status acknowledgment to the origin Gitblit instance. This
|
* includes the results of the federated pull.
|
*
|
* @param registration
|
* @throws Exception
|
*/
|
private void sendStatusAcknowledgment(FederationModel registration) throws Exception {
|
if (!registration.sendStatus) {
|
// skip status acknowledgment
|
return;
|
}
|
InetAddress addr = InetAddress.getLocalHost();
|
String federationName = GitBlit.getString(Keys.federation.name, null);
|
if (StringUtils.isEmpty(federationName)) {
|
federationName = addr.getHostName();
|
}
|
FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
|
logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
|
}
|
|
/**
|
* Schedules the next check of the federated Gitblit instance.
|
*
|
* @param registration
|
*/
|
private void schedule(FederationModel registration) {
|
// schedule the next pull
|
int mins = TimeUtils.convertFrequencyToMinutes(registration.frequency);
|
registration.nextPull = new Date(System.currentTimeMillis() + (mins * 60 * 1000L));
|
GitBlit.self().executor()
|
.schedule(new FederationPullExecutor(registration), mins, TimeUnit.MINUTES);
|
logger.info(MessageFormat.format(
|
"Next pull of {0} @ {1} scheduled for {2,date,yyyy-MM-dd HH:mm}",
|
registration.name, registration.url, registration.nextPull));
|
}
|
}
|
