githubFork/gitblit.git - Solinfo Gitblit

Kensuke Matsuzaki

2013-04-15 ed566a162c780faa0d6c84fac8fbbd3738586445

commit \| author \| age
831469	1	/*
JM	2	* Copyright 2011 gitblit.com.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16	package com.gitblit;
	17
f6740d	18	import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;
JM	19
831469	20	import java.io.File;
JM	21	import java.io.FileOutputStream;
31abc2	22	import java.io.IOException;
831469	23	import java.net.InetAddress;
JM	24	import java.text.MessageFormat;
	25	import java.util.ArrayList;
	26	import java.util.Arrays;
	27	import java.util.Collection;
	28	import java.util.Date;
20714a	29	import java.util.HashMap;
f6740d	30	import java.util.HashSet;
831469	31	import java.util.List;
JM	32	import java.util.Map;
	33	import java.util.Properties;
f6740d	34	import java.util.Set;
831469	35	import java.util.concurrent.TimeUnit;
JM	36
	37	import org.eclipse.jgit.lib.Repository;
	38	import org.eclipse.jgit.lib.StoredConfig;
f6740d	39	import org.eclipse.jgit.revwalk.RevCommit;
831469	40	import org.eclipse.jgit.transport.CredentialsProvider;
JM	41	import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
	42	import org.slf4j.Logger;
	43	import org.slf4j.LoggerFactory;
	44
20714a	45	import com.gitblit.Constants.AccessPermission;
831469	46	import com.gitblit.Constants.FederationPullStatus;
JM	47	import com.gitblit.Constants.FederationStrategy;
31abc2	48	import com.gitblit.GitBlitException.ForbiddenException;
831469	49	import com.gitblit.models.FederationModel;
87db82	50	import com.gitblit.models.RefModel;
831469	51	import com.gitblit.models.RepositoryModel;
997c16	52	import com.gitblit.models.TeamModel;
831469	53	import com.gitblit.models.UserModel;
0db5c4	54	import com.gitblit.utils.ArrayUtils;
831469	55	import com.gitblit.utils.FederationUtils;
df162c	56	import com.gitblit.utils.FileUtils;
831469	57	import com.gitblit.utils.JGitUtils;
JM	58	import com.gitblit.utils.JGitUtils.CloneResult;
	59	import com.gitblit.utils.StringUtils;
2548a7	60	import com.gitblit.utils.TimeUtils;
831469	61
JM	62	/**
	63	* FederationPullExecutor pulls repository updates and, optionally, user
	64	* accounts and server settings from registered Gitblit instances.
	65	*/
	66	public class FederationPullExecutor implements Runnable {
	67
	68	private final Logger logger = LoggerFactory.getLogger(FederationPullExecutor.class);
	69
	70	private final List<FederationModel> registrations;
	71
f6740d	72	private final boolean isDaemon;
JM	73
831469	74	/**
JM	75	* Constructor for specifying a single federation registration. This
	76	* constructor is used to schedule the next pull execution.
	77	*
	78	* @param registration
	79	*/
	80	private FederationPullExecutor(FederationModel registration) {
f6740d	81	this(Arrays.asList(registration), true);
831469	82	}
JM	83
	84	/**
	85	* Constructor to specify a group of federation registrations. This is
	86	* normally used at startup to pull and then schedule the next update based
	87	* on each registrations frequency setting.
	88	*
	89	* @param registrations
f6740d	90	* @param isDaemon
31abc2	91	* if true, registrations are rescheduled in perpetuity. if
JM	92	* false, the federation pull operation is executed once.
831469	93	*/
f6740d	94	public FederationPullExecutor(List<FederationModel> registrations, boolean isDaemon) {
831469	95	this.registrations = registrations;
f6740d	96	this.isDaemon = isDaemon;
831469	97	}
JM	98
	99	/**
	100	* Run method for this pull executor.
	101	*/
	102	@Override
	103	public void run() {
	104	for (FederationModel registration : registrations) {
	105	FederationPullStatus was = registration.getLowestStatus();
	106	try {
	107	Date now = new Date(System.currentTimeMillis());
	108	pull(registration);
	109	sendStatusAcknowledgment(registration);
	110	registration.lastPull = now;
	111	FederationPullStatus is = registration.getLowestStatus();
	112	if (is.ordinal() < was.ordinal()) {
	113	// the status for this registration has downgraded
	114	logger.warn("Federation pull status of {0} is now {1}", registration.name,
	115	is.name());
	116	if (registration.notifyOnError) {
	117	String message = "Federation pull of " + registration.name + " @ "
	118	+ registration.url + " is now at " + is.name();
	119	GitBlit.self()
eb96ea	120	.sendMailToAdministrators(
831469	121	"Pull Status of " + registration.name + " is " + is.name(),
JM	122	message);
	123	}
	124	}
	125	} catch (Throwable t) {
	126	logger.error(MessageFormat.format(
	127	"Failed to pull from federated gitblit ({0} @ {1})", registration.name,
	128	registration.url), t);
	129	} finally {
f6740d	130	if (isDaemon) {
JM	131	schedule(registration);
	132	}
831469	133	}
JM	134	}
	135	}
	136
	137	/**
	138	* Mirrors a repository and, optionally, the server's users, and/or
2548a7	139	* configuration settings from a origin Gitblit instance.
831469	140	*
JM	141	* @param registration
	142	* @throws Exception
	143	*/
	144	private void pull(FederationModel registration) throws Exception {
	145	Map<String, RepositoryModel> repositories = FederationUtils.getRepositories(registration,
	146	true);
	147	String registrationFolder = registration.folder.toLowerCase().trim();
	148	// confirm valid characters in server alias
	149	Character c = StringUtils.findInvalidCharacter(registrationFolder);
	150	if (c != null) {
	151	logger.error(MessageFormat
	152	.format("Illegal character ''{0}'' in folder name ''{1}'' of federation registration {2}!",
	153	c, registrationFolder, registration.name));
	154	return;
	155	}
e5c14e	156	File repositoriesFolder = GitBlit.getRepositoriesFolder();
831469	157	File registrationFolderFile = new File(repositoriesFolder, registrationFolder);
JM	158	registrationFolderFile.mkdirs();
	159
	160	// Clone/Pull the repository
	161	for (Map.Entry<String, RepositoryModel> entry : repositories.entrySet()) {
	162	String cloneUrl = entry.getKey();
	163	RepositoryModel repository = entry.getValue();
	164	if (!repository.hasCommits) {
	165	logger.warn(MessageFormat.format(
	166	"Skipping federated repository {0} from {1} @ {2}. Repository is EMPTY.",
	167	repository.name, registration.name, registration.url));
	168	registration.updateStatus(repository, FederationPullStatus.SKIPPED);
	169	continue;
	170	}
	171
f6740d	172	// Determine local repository name
831469	173	String repositoryName;
JM	174	if (StringUtils.isEmpty(registrationFolder)) {
	175	repositoryName = repository.name;
	176	} else {
	177	repositoryName = registrationFolder + "/" + repository.name;
f6740d	178	}
31abc2	179
f6740d	180	if (registration.bare) {
JM	181	// bare repository, ensure .git suffix
	182	if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
	183	repositoryName += DOT_GIT_EXT;
	184	}
	185	} else {
	186	// normal repository, strip .git suffix
	187	if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
31abc2	188	repositoryName = repositoryName.substring(0,
JM	189	repositoryName.indexOf(DOT_GIT_EXT));
f6740d	190	}
831469	191	}
e92c6d	192
831469	193	// confirm that the origin of any pre-existing repository matches
JM	194	// the clone url
2548a7	195	String fetchHead = null;
831469	196	Repository existingRepository = GitBlit.self().getRepository(repositoryName);
e92c6d	197
JM	198	if (existingRepository == null && GitBlit.self().isCollectingGarbage(repositoryName)) {
	199	logger.warn(MessageFormat.format("Skipping local repository {0}, busy collecting garbage", repositoryName));
	200	continue;
	201	}
	202
831469	203	if (existingRepository != null) {
JM	204	StoredConfig config = existingRepository.getConfig();
	205	config.load();
	206	String origin = config.getString("remote", "origin", "url");
31abc2	207	RevCommit commit = JGitUtils.getCommit(existingRepository,
87db82	208	org.eclipse.jgit.lib.Constants.FETCH_HEAD);
f6740d	209	if (commit != null) {
JM	210	fetchHead = commit.getName();
	211	}
831469	212	existingRepository.close();
JM	213	if (!origin.startsWith(registration.url)) {
	214	logger.warn(MessageFormat
	215	.format("Skipping federated repository {0} from {1} @ {2}. Origin does not match, consider EXCLUDING.",
	216	repository.name, registration.name, registration.url));
	217	registration.updateStatus(repository, FederationPullStatus.SKIPPED);
	218	continue;
	219	}
	220	}
	221
	222	// clone/pull this repository
	223	CredentialsProvider credentials = new UsernamePasswordCredentialsProvider(
	224	Constants.FEDERATION_USER, registration.token);
	225	logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",
	226	repository.name, registration.name, registration.url));
31abc2	227
831469	228	CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,
d7fb20	229	cloneUrl, registration.bare, credentials);
831469	230	Repository r = GitBlit.self().getRepository(repositoryName);
JM	231	RepositoryModel rm = GitBlit.self().getRepositoryModel(repositoryName);
2548a7	232	repository.isFrozen = registration.mirror;
831469	233	if (result.createdRepository) {
JM	234	// default local settings
	235	repository.federationStrategy = FederationStrategy.EXCLUDE;
2548a7	236	repository.isFrozen = registration.mirror;
JM	237	repository.showRemoteBranches = !registration.mirror;
	238	logger.info(MessageFormat.format(" cloning {0}", repository.name));
	239	registration.updateStatus(repository, FederationPullStatus.MIRRORED);
831469	240	} else {
2548a7	241	// fetch and update
JM	242	boolean fetched = false;
87db82	243	RevCommit commit = JGitUtils.getCommit(r, org.eclipse.jgit.lib.Constants.FETCH_HEAD);
JM	244	String newFetchHead = commit.getName();
	245	fetched = fetchHead == null \|\| !fetchHead.equals(newFetchHead);
2548a7	246
JM	247	if (registration.mirror) {
	248	// mirror
	249	if (fetched) {
f96d32	250	// update local branches to match the remote tracking branches
JM	251	for (RefModel ref : JGitUtils.getRemoteBranches(r, false, -1)) {
	252	if (ref.displayName.startsWith("origin/")) {
	253	String branch = org.eclipse.jgit.lib.Constants.R_HEADS
	254	+ ref.displayName.substring(ref.displayName.indexOf('/') + 1);
	255	String hash = ref.getReferencedObjectId().getName();
	256
	257	JGitUtils.setBranchRef(r, branch, hash);
	258	logger.info(MessageFormat.format(" resetting {0} of {1} to {2}", branch,
	259	repository.name, hash));
87db82	260	}
JM	261	}
f96d32	262
JM	263	String newHead;
	264	if (StringUtils.isEmpty(repository.HEAD)) {
	265	newHead = newFetchHead;
	266	} else {
	267	newHead = repository.HEAD;
	268	}
	269	JGitUtils.setHEADtoRef(r, newHead);
2548a7	270	logger.info(MessageFormat.format(" resetting HEAD of {0} to {1}",
f96d32	271	repository.name, newHead));
2548a7	272	registration.updateStatus(repository, FederationPullStatus.MIRRORED);
JM	273	} else {
	274	// indicate no commits pulled
	275	registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
	276	}
	277	} else {
	278	// non-mirror
	279	if (fetched) {
	280	// indicate commits pulled to origin/master
	281	registration.updateStatus(repository, FederationPullStatus.PULLED);
	282	} else {
	283	// indicate no commits pulled
	284	registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
	285	}
	286	}
	287
831469	288	// preserve local settings
JM	289	repository.isFrozen = rm.isFrozen;
	290	repository.federationStrategy = rm.federationStrategy;
31abc2	291
f6740d	292	// merge federation sets
JM	293	Set<String> federationSets = new HashSet<String>();
	294	if (rm.federationSets != null) {
	295	federationSets.addAll(rm.federationSets);
	296	}
	297	if (repository.federationSets != null) {
	298	federationSets.addAll(repository.federationSets);
	299	}
	300	repository.federationSets = new ArrayList<String>(federationSets);
5b29c5	301
JM	302	// merge indexed branches
	303	Set<String> indexedBranches = new HashSet<String>();
	304	if (rm.indexedBranches != null) {
	305	indexedBranches.addAll(rm.indexedBranches);
	306	}
	307	if (repository.indexedBranches != null) {
	308	indexedBranches.addAll(repository.indexedBranches);
	309	}
	310	repository.indexedBranches = new ArrayList<String>(indexedBranches);
	311
831469	312	}
2548a7	313	// only repositories that are actually _cloned_ from the origin
831469	314	// Gitblit repository are marked as federated. If the origin
JM	315	// is from somewhere else, these repositories are not considered
	316	// "federated" repositories.
	317	repository.isFederated = cloneUrl.startsWith(registration.url);
	318
	319	GitBlit.self().updateConfiguration(r, repository);
	320	r.close();
	321	}
	322
df162c	323	IUserService userService = null;
JM	324
831469	325	try {
JM	326	// Pull USERS
997c16	327	// TeamModels are automatically pulled because they are contained
JM	328	// within the UserModel. The UserService creates unknown teams
	329	// and updates existing teams.
831469	330	Collection<UserModel> users = FederationUtils.getUsers(registration);
JM	331	if (users != null && users.size() > 0) {
997c16	332	File realmFile = new File(registrationFolderFile, registration.name + "_users.conf");
831469	333	realmFile.delete();
df162c	334	userService = new ConfigUserService(realmFile);
831469	335	for (UserModel user : users) {
JM	336	userService.updateUserModel(user.username, user);
	337
2548a7	338	// merge the origin permissions and origin accounts into
831469	339	// the user accounts of this Gitblit instance
JM	340	if (registration.mergeAccounts) {
	341	// reparent all repository permissions if the local
	342	// repositories are stored within subfolders
	343	if (!StringUtils.isEmpty(registrationFolder)) {
822dfe	344	if (user.permissions != null) {
20714a	345	// pulling from >= 1.2 version
JM	346	Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
	347	user.permissions.clear();
	348	for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
	349	user.setRepositoryPermission(registrationFolder + "/" + entry.getKey(), entry.getValue());
	350	}
	351	} else {
	352	// pulling from <= 1.1 version
	353	List<String> permissions = new ArrayList<String>(user.repositories);
	354	user.repositories.clear();
	355	for (String permission : permissions) {
	356	user.addRepositoryPermission(registrationFolder + "/" + permission);
	357	}
831469	358	}
JM	359	}
	360
	361	// insert new user or update local user
	362	UserModel localUser = GitBlit.self().getUserModel(user.username);
	363	if (localUser == null) {
	364	// create new local user
	365	GitBlit.self().updateUserModel(user.username, user, true);
	366	} else {
	367	// update repository permissions of local user
822dfe	368	if (user.permissions != null) {
20714a	369	// pulling from >= 1.2 version
JM	370	Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
	371	for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
	372	localUser.setRepositoryPermission(entry.getKey(), entry.getValue());
	373	}
	374	} else {
	375	// pulling from <= 1.1 version
	376	for (String repository : user.repositories) {
	377	localUser.addRepositoryPermission(repository);
	378	}
831469	379	}
JM	380	localUser.password = user.password;
	381	localUser.canAdmin = user.canAdmin;
	382	GitBlit.self().updateUserModel(localUser.username, localUser, false);
	383	}
997c16	384
JM	385	for (String teamname : GitBlit.self().getAllTeamnames()) {
	386	TeamModel team = GitBlit.self().getTeamModel(teamname);
	387	if (user.isTeamMember(teamname) && !team.hasUser(user.username)) {
	388	// new team member
	389	team.addUser(user.username);
	390	GitBlit.self().updateTeamModel(teamname, team, false);
	391	} else if (!user.isTeamMember(teamname) && team.hasUser(user.username)) {
	392	// remove team member
	393	team.removeUser(user.username);
	394	GitBlit.self().updateTeamModel(teamname, team, false);
	395	}
	396
	397	// update team repositories
	398	TeamModel remoteTeam = user.getTeam(teamname);
20714a	399	if (remoteTeam != null) {
JM	400	if (remoteTeam.permissions != null) {
	401	// pulling from >= 1.2
	402	for (Map.Entry<String, AccessPermission> entry : remoteTeam.permissions.entrySet()){
	403	team.setRepositoryPermission(entry.getKey(), entry.getValue());
	404	}
	405	GitBlit.self().updateTeamModel(teamname, team, false);
	406	} else if(!ArrayUtils.isEmpty(remoteTeam.repositories)) {
	407	// pulling from <= 1.1
	408	team.addRepositoryPermissions(remoteTeam.repositories);
997c16	409	GitBlit.self().updateTeamModel(teamname, team, false);
JM	410	}
	411	}
	412	}
831469	413	}
JM	414	}
	415	}
31abc2	416	} catch (ForbiddenException e) {
JM	417	// ignore forbidden exceptions
	418	} catch (IOException e) {
	419	logger.warn(MessageFormat.format(
	420	"Failed to retrieve USERS from federated gitblit ({0} @ {1})",
	421	registration.name, registration.url), e);
831469	422	}
JM	423
	424	try {
df162c	425	// Pull TEAMS
JM	426	// We explicitly pull these even though they are embedded in
	427	// UserModels because it is possible to use teams to specify
	428	// mailing lists or push scripts without specifying users.
	429	if (userService != null) {
	430	Collection<TeamModel> teams = FederationUtils.getTeams(registration);
	431	if (teams != null && teams.size() > 0) {
	432	for (TeamModel team : teams) {
	433	userService.updateTeamModel(team);
	434	}
	435	}
	436	}
	437	} catch (ForbiddenException e) {
	438	// ignore forbidden exceptions
	439	} catch (IOException e) {
	440	logger.warn(MessageFormat.format(
	441	"Failed to retrieve TEAMS from federated gitblit ({0} @ {1})",
	442	registration.name, registration.url), e);
	443	}
	444
	445	try {
831469	446	// Pull SETTINGS
JM	447	Map<String, String> settings = FederationUtils.getSettings(registration);
	448	if (settings != null && settings.size() > 0) {
	449	Properties properties = new Properties();
	450	properties.putAll(settings);
	451	FileOutputStream os = new FileOutputStream(new File(registrationFolderFile,
	452	registration.name + "_" + Constants.PROPERTIES_FILE));
	453	properties.store(os, null);
	454	os.close();
	455	}
31abc2	456	} catch (ForbiddenException e) {
JM	457	// ignore forbidden exceptions
	458	} catch (IOException e) {
	459	logger.warn(MessageFormat.format(
	460	"Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
	461	registration.name, registration.url), e);
831469	462	}
0db5c4	463
df162c	464	try {
JM	465	// Pull SCRIPTS
	466	Map<String, String> scripts = FederationUtils.getScripts(registration);
	467	if (scripts != null && scripts.size() > 0) {
	468	for (Map.Entry<String, String> script : scripts.entrySet()) {
	469	String scriptName = script.getKey();
	470	if (scriptName.endsWith(".groovy")) {
0db5c4	471	scriptName = scriptName.substring(0, scriptName.indexOf(".groovy"));
df162c	472	}
0db5c4	473	File file = new File(registrationFolderFile, registration.name + "_"
JM	474	+ scriptName + ".groovy");
df162c	475	FileUtils.writeContent(file, script.getValue());
JM	476	}
	477	}
	478	} catch (ForbiddenException e) {
	479	// ignore forbidden exceptions
	480	} catch (IOException e) {
	481	logger.warn(MessageFormat.format(
	482	"Failed to retrieve SCRIPTS from federated gitblit ({0} @ {1})",
	483	registration.name, registration.url), e);
	484	}
831469	485	}
JM	486
	487	/**
2548a7	488	* Sends a status acknowledgment to the origin Gitblit instance. This
831469	489	* includes the results of the federated pull.
JM	490	*
	491	* @param registration
	492	* @throws Exception
	493	*/
	494	private void sendStatusAcknowledgment(FederationModel registration) throws Exception {
	495	if (!registration.sendStatus) {
	496	// skip status acknowledgment
	497	return;
	498	}
	499	InetAddress addr = InetAddress.getLocalHost();
	500	String federationName = GitBlit.getString(Keys.federation.name, null);
	501	if (StringUtils.isEmpty(federationName)) {
	502	federationName = addr.getHostName();
	503	}
	504	FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
2548a7	505	logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
831469	506	}
JM	507
	508	/**
	509	* Schedules the next check of the federated Gitblit instance.
	510	*
	511	* @param registration
	512	*/
	513	private void schedule(FederationModel registration) {
	514	// schedule the next pull
	515	int mins = TimeUtils.convertFrequencyToMinutes(registration.frequency);
	516	registration.nextPull = new Date(System.currentTimeMillis() + (mins * 60 * 1000L));
	517	GitBlit.self().executor()
	518	.schedule(new FederationPullExecutor(registration), mins, TimeUnit.MINUTES);
	519	logger.info(MessageFormat.format(
	520	"Next pull of {0} @ {1} scheduled for {2,date,yyyy-MM-dd HH:mm}",
	521	registration.name, registration.url, registration.nextPull));
	522	}
	523	}