githubFork/gitblit.git - Solinfo Gitblit

James Moger

2011-10-02 f762b160efd5cafd919a6fd7f9587f578eceb454

commit \| author \| age
831469	1	/*
JM	2	* Copyright 2011 gitblit.com.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16	package com.gitblit;
	17
f6740d	18	import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;
JM	19
831469	20	import java.io.File;
JM	21	import java.io.FileOutputStream;
31abc2	22	import java.io.IOException;
831469	23	import java.net.InetAddress;
JM	24	import java.text.MessageFormat;
	25	import java.util.ArrayList;
	26	import java.util.Arrays;
	27	import java.util.Collection;
	28	import java.util.Date;
f6740d	29	import java.util.HashSet;
831469	30	import java.util.List;
JM	31	import java.util.Map;
	32	import java.util.Properties;
f6740d	33	import java.util.Set;
831469	34	import java.util.concurrent.TimeUnit;
JM	35
2548a7	36	import org.eclipse.jgit.lib.Ref;
831469	37	import org.eclipse.jgit.lib.Repository;
JM	38	import org.eclipse.jgit.lib.StoredConfig;
f6740d	39	import org.eclipse.jgit.revwalk.RevCommit;
831469	40	import org.eclipse.jgit.transport.CredentialsProvider;
JM	41	import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
	42	import org.slf4j.Logger;
	43	import org.slf4j.LoggerFactory;
	44
	45	import com.gitblit.Constants.FederationPullStatus;
	46	import com.gitblit.Constants.FederationStrategy;
31abc2	47	import com.gitblit.GitBlitException.ForbiddenException;
831469	48	import com.gitblit.models.FederationModel;
JM	49	import com.gitblit.models.RepositoryModel;
	50	import com.gitblit.models.UserModel;
	51	import com.gitblit.utils.FederationUtils;
	52	import com.gitblit.utils.JGitUtils;
	53	import com.gitblit.utils.JGitUtils.CloneResult;
	54	import com.gitblit.utils.StringUtils;
2548a7	55	import com.gitblit.utils.TimeUtils;
831469	56
JM	57	/**
	58	* FederationPullExecutor pulls repository updates and, optionally, user
	59	* accounts and server settings from registered Gitblit instances.
	60	*/
	61	public class FederationPullExecutor implements Runnable {
	62
	63	private final Logger logger = LoggerFactory.getLogger(FederationPullExecutor.class);
	64
	65	private final List<FederationModel> registrations;
	66
f6740d	67	private final boolean isDaemon;
JM	68
831469	69	/**
JM	70	* Constructor for specifying a single federation registration. This
	71	* constructor is used to schedule the next pull execution.
	72	*
	73	* @param registration
	74	*/
	75	private FederationPullExecutor(FederationModel registration) {
f6740d	76	this(Arrays.asList(registration), true);
831469	77	}
JM	78
	79	/**
	80	* Constructor to specify a group of federation registrations. This is
	81	* normally used at startup to pull and then schedule the next update based
	82	* on each registrations frequency setting.
	83	*
	84	* @param registrations
f6740d	85	* @param isDaemon
31abc2	86	* if true, registrations are rescheduled in perpetuity. if
JM	87	* false, the federation pull operation is executed once.
831469	88	*/
f6740d	89	public FederationPullExecutor(List<FederationModel> registrations, boolean isDaemon) {
831469	90	this.registrations = registrations;
f6740d	91	this.isDaemon = isDaemon;
831469	92	}
JM	93
	94	/**
	95	* Run method for this pull executor.
	96	*/
	97	@Override
	98	public void run() {
	99	for (FederationModel registration : registrations) {
	100	FederationPullStatus was = registration.getLowestStatus();
	101	try {
	102	Date now = new Date(System.currentTimeMillis());
	103	pull(registration);
	104	sendStatusAcknowledgment(registration);
	105	registration.lastPull = now;
	106	FederationPullStatus is = registration.getLowestStatus();
	107	if (is.ordinal() < was.ordinal()) {
	108	// the status for this registration has downgraded
	109	logger.warn("Federation pull status of {0} is now {1}", registration.name,
	110	is.name());
	111	if (registration.notifyOnError) {
	112	String message = "Federation pull of " + registration.name + " @ "
	113	+ registration.url + " is now at " + is.name();
	114	GitBlit.self()
	115	.notifyAdministrators(
	116	"Pull Status of " + registration.name + " is " + is.name(),
	117	message);
	118	}
	119	}
	120	} catch (Throwable t) {
	121	logger.error(MessageFormat.format(
	122	"Failed to pull from federated gitblit ({0} @ {1})", registration.name,
	123	registration.url), t);
	124	} finally {
f6740d	125	if (isDaemon) {
JM	126	schedule(registration);
	127	}
831469	128	}
JM	129	}
	130	}
	131
	132	/**
	133	* Mirrors a repository and, optionally, the server's users, and/or
2548a7	134	* configuration settings from a origin Gitblit instance.
831469	135	*
JM	136	* @param registration
	137	* @throws Exception
	138	*/
	139	private void pull(FederationModel registration) throws Exception {
	140	Map<String, RepositoryModel> repositories = FederationUtils.getRepositories(registration,
	141	true);
	142	String registrationFolder = registration.folder.toLowerCase().trim();
	143	// confirm valid characters in server alias
	144	Character c = StringUtils.findInvalidCharacter(registrationFolder);
	145	if (c != null) {
	146	logger.error(MessageFormat
	147	.format("Illegal character ''{0}'' in folder name ''{1}'' of federation registration {2}!",
	148	c, registrationFolder, registration.name));
	149	return;
	150	}
	151	File repositoriesFolder = new File(GitBlit.getString(Keys.git.repositoriesFolder, "git"));
	152	File registrationFolderFile = new File(repositoriesFolder, registrationFolder);
	153	registrationFolderFile.mkdirs();
	154
	155	// Clone/Pull the repository
	156	for (Map.Entry<String, RepositoryModel> entry : repositories.entrySet()) {
	157	String cloneUrl = entry.getKey();
	158	RepositoryModel repository = entry.getValue();
	159	if (!repository.hasCommits) {
	160	logger.warn(MessageFormat.format(
	161	"Skipping federated repository {0} from {1} @ {2}. Repository is EMPTY.",
	162	repository.name, registration.name, registration.url));
	163	registration.updateStatus(repository, FederationPullStatus.SKIPPED);
	164	continue;
	165	}
	166
f6740d	167	// Determine local repository name
831469	168	String repositoryName;
JM	169	if (StringUtils.isEmpty(registrationFolder)) {
	170	repositoryName = repository.name;
	171	} else {
	172	repositoryName = registrationFolder + "/" + repository.name;
f6740d	173	}
31abc2	174
f6740d	175	if (registration.bare) {
JM	176	// bare repository, ensure .git suffix
	177	if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
	178	repositoryName += DOT_GIT_EXT;
	179	}
	180	} else {
	181	// normal repository, strip .git suffix
	182	if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
31abc2	183	repositoryName = repositoryName.substring(0,
JM	184	repositoryName.indexOf(DOT_GIT_EXT));
f6740d	185	}
831469	186	}
JM	187
	188	// confirm that the origin of any pre-existing repository matches
	189	// the clone url
2548a7	190	String fetchHead = null;
831469	191	Repository existingRepository = GitBlit.self().getRepository(repositoryName);
JM	192	if (existingRepository != null) {
	193	StoredConfig config = existingRepository.getConfig();
	194	config.load();
	195	String origin = config.getString("remote", "origin", "url");
31abc2	196	RevCommit commit = JGitUtils.getCommit(existingRepository,
JM	197	"refs/remotes/origin/master");
f6740d	198	if (commit != null) {
JM	199	fetchHead = commit.getName();
	200	}
831469	201	existingRepository.close();
JM	202	if (!origin.startsWith(registration.url)) {
	203	logger.warn(MessageFormat
	204	.format("Skipping federated repository {0} from {1} @ {2}. Origin does not match, consider EXCLUDING.",
	205	repository.name, registration.name, registration.url));
	206	registration.updateStatus(repository, FederationPullStatus.SKIPPED);
	207	continue;
	208	}
	209	}
	210
	211	// clone/pull this repository
	212	CredentialsProvider credentials = new UsernamePasswordCredentialsProvider(
	213	Constants.FEDERATION_USER, registration.token);
	214	logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",
	215	repository.name, registration.name, registration.url));
31abc2	216
831469	217	CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,
d7fb20	218	cloneUrl, registration.bare, credentials);
831469	219	Repository r = GitBlit.self().getRepository(repositoryName);
JM	220	RepositoryModel rm = GitBlit.self().getRepositoryModel(repositoryName);
2548a7	221	repository.isFrozen = registration.mirror;
831469	222	if (result.createdRepository) {
JM	223	// default local settings
	224	repository.federationStrategy = FederationStrategy.EXCLUDE;
2548a7	225	repository.isFrozen = registration.mirror;
JM	226	repository.showRemoteBranches = !registration.mirror;
	227	logger.info(MessageFormat.format(" cloning {0}", repository.name));
	228	registration.updateStatus(repository, FederationPullStatus.MIRRORED);
831469	229	} else {
2548a7	230	// fetch and update
JM	231	boolean fetched = false;
f6740d	232	RevCommit commit = JGitUtils.getCommit(r, "refs/remotes/origin/master");
JM	233	String origin = commit.getName();
	234	fetched = fetchHead == null \|\| !fetchHead.equals(origin);
2548a7	235
JM	236	if (registration.mirror) {
	237	// mirror
	238	if (fetched) {
	239	// reset the local HEAD to origin/master
	240	Ref ref = JGitUtils.resetHEAD(r, "origin/master");
	241	logger.info(MessageFormat.format(" resetting HEAD of {0} to {1}",
	242	repository.name, ref.getObjectId().getName()));
	243	registration.updateStatus(repository, FederationPullStatus.MIRRORED);
	244	} else {
	245	// indicate no commits pulled
	246	registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
	247	}
	248	} else {
	249	// non-mirror
	250	if (fetched) {
	251	// indicate commits pulled to origin/master
	252	registration.updateStatus(repository, FederationPullStatus.PULLED);
	253	} else {
	254	// indicate no commits pulled
	255	registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
	256	}
	257	}
	258
831469	259	// preserve local settings
JM	260	repository.isFrozen = rm.isFrozen;
	261	repository.federationStrategy = rm.federationStrategy;
31abc2	262
f6740d	263	// merge federation sets
JM	264	Set<String> federationSets = new HashSet<String>();
	265	if (rm.federationSets != null) {
	266	federationSets.addAll(rm.federationSets);
	267	}
	268	if (repository.federationSets != null) {
	269	federationSets.addAll(repository.federationSets);
	270	}
	271	repository.federationSets = new ArrayList<String>(federationSets);
831469	272	}
2548a7	273	// only repositories that are actually _cloned_ from the origin
831469	274	// Gitblit repository are marked as federated. If the origin
JM	275	// is from somewhere else, these repositories are not considered
	276	// "federated" repositories.
	277	repository.isFederated = cloneUrl.startsWith(registration.url);
	278
	279	GitBlit.self().updateConfiguration(r, repository);
	280	r.close();
	281	}
	282
	283	try {
	284	// Pull USERS
	285	Collection<UserModel> users = FederationUtils.getUsers(registration);
	286	if (users != null && users.size() > 0) {
	287	File realmFile = new File(registrationFolderFile, registration.name
	288	+ "_users.properties");
	289	realmFile.delete();
	290	FileUserService userService = new FileUserService(realmFile);
	291	for (UserModel user : users) {
	292	userService.updateUserModel(user.username, user);
	293
2548a7	294	// merge the origin permissions and origin accounts into
831469	295	// the user accounts of this Gitblit instance
JM	296	if (registration.mergeAccounts) {
	297	// reparent all repository permissions if the local
	298	// repositories are stored within subfolders
	299	if (!StringUtils.isEmpty(registrationFolder)) {
	300	List<String> permissions = new ArrayList<String>(user.repositories);
	301	user.repositories.clear();
	302	for (String permission : permissions) {
	303	user.addRepository(registrationFolder + "/" + permission);
	304	}
	305	}
	306
	307	// insert new user or update local user
	308	UserModel localUser = GitBlit.self().getUserModel(user.username);
	309	if (localUser == null) {
	310	// create new local user
	311	GitBlit.self().updateUserModel(user.username, user, true);
	312	} else {
	313	// update repository permissions of local user
	314	for (String repository : user.repositories) {
	315	localUser.addRepository(repository);
	316	}
	317	localUser.password = user.password;
	318	localUser.canAdmin = user.canAdmin;
	319	GitBlit.self().updateUserModel(localUser.username, localUser, false);
	320	}
	321	}
	322	}
	323	}
31abc2	324	} catch (ForbiddenException e) {
JM	325	// ignore forbidden exceptions
	326	} catch (IOException e) {
	327	logger.warn(MessageFormat.format(
	328	"Failed to retrieve USERS from federated gitblit ({0} @ {1})",
	329	registration.name, registration.url), e);
831469	330	}
JM	331
	332	try {
	333	// Pull SETTINGS
	334	Map<String, String> settings = FederationUtils.getSettings(registration);
	335	if (settings != null && settings.size() > 0) {
	336	Properties properties = new Properties();
	337	properties.putAll(settings);
	338	FileOutputStream os = new FileOutputStream(new File(registrationFolderFile,
	339	registration.name + "_" + Constants.PROPERTIES_FILE));
	340	properties.store(os, null);
	341	os.close();
	342	}
31abc2	343	} catch (ForbiddenException e) {
JM	344	// ignore forbidden exceptions
	345	} catch (IOException e) {
	346	logger.warn(MessageFormat.format(
	347	"Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
	348	registration.name, registration.url), e);
831469	349	}
JM	350	}
	351
	352	/**
2548a7	353	* Sends a status acknowledgment to the origin Gitblit instance. This
831469	354	* includes the results of the federated pull.
JM	355	*
	356	* @param registration
	357	* @throws Exception
	358	*/
	359	private void sendStatusAcknowledgment(FederationModel registration) throws Exception {
	360	if (!registration.sendStatus) {
	361	// skip status acknowledgment
	362	return;
	363	}
	364	InetAddress addr = InetAddress.getLocalHost();
	365	String federationName = GitBlit.getString(Keys.federation.name, null);
	366	if (StringUtils.isEmpty(federationName)) {
	367	federationName = addr.getHostName();
	368	}
	369	FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
2548a7	370	logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
831469	371	}
JM	372
	373	/**
	374	* Schedules the next check of the federated Gitblit instance.
	375	*
	376	* @param registration
	377	*/
	378	private void schedule(FederationModel registration) {
	379	// schedule the next pull
	380	int mins = TimeUtils.convertFrequencyToMinutes(registration.frequency);
	381	registration.nextPull = new Date(System.currentTimeMillis() + (mins * 60 * 1000L));
	382	GitBlit.self().executor()
	383	.schedule(new FederationPullExecutor(registration), mins, TimeUnit.MINUTES);
	384	logger.info(MessageFormat.format(
	385	"Next pull of {0} @ {1} scheduled for {2,date,yyyy-MM-dd HH:mm}",
	386	registration.name, registration.url, registration.nextPull));
	387	}
	388	}