James Moger
2012-09-10 fabe060d3a435f116128851f828e35c2af5fde67
commit | author | age
831469 1 /*
JM 2  * Copyright 2011 gitblit.com.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.gitblit;
17
f6740d 18 import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;
JM 19
831469 20 import java.io.File;
JM 21 import java.io.FileOutputStream;
31abc2 22 import java.io.IOException;
831469 23 import java.net.InetAddress;
JM 24 import java.text.MessageFormat;
25 import java.util.ArrayList;
26 import java.util.Arrays;
27 import java.util.Collection;
28 import java.util.Date;
f6740d 29 import java.util.HashSet;
831469 30 import java.util.List;
JM 31 import java.util.Map;
32 import java.util.Properties;
f6740d 33 import java.util.Set;
831469 34 import java.util.concurrent.TimeUnit;
JM 35
36 import org.eclipse.jgit.lib.Repository;
37 import org.eclipse.jgit.lib.StoredConfig;
f6740d 38 import org.eclipse.jgit.revwalk.RevCommit;
831469 39 import org.eclipse.jgit.transport.CredentialsProvider;
JM 40 import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
41 import org.slf4j.Logger;
42 import org.slf4j.LoggerFactory;
43
44 import com.gitblit.Constants.FederationPullStatus;
45 import com.gitblit.Constants.FederationStrategy;
31abc2 46 import com.gitblit.GitBlitException.ForbiddenException;
831469 47 import com.gitblit.models.FederationModel;
87db82 48 import com.gitblit.models.RefModel;
831469 49 import com.gitblit.models.RepositoryModel;
997c16 50 import com.gitblit.models.TeamModel;
831469 51 import com.gitblit.models.UserModel;
0db5c4 52 import com.gitblit.utils.ArrayUtils;
831469 53 import com.gitblit.utils.FederationUtils;
df162c 54 import com.gitblit.utils.FileUtils;
831469 55 import com.gitblit.utils.JGitUtils;
JM 56 import com.gitblit.utils.JGitUtils.CloneResult;
57 import com.gitblit.utils.StringUtils;
2548a7 58 import com.gitblit.utils.TimeUtils;
831469 59
JM 60 /**
61  * FederationPullExecutor pulls repository updates and, optionally, user
62  * accounts and server settings from registered Gitblit instances.
63  */
64 public class FederationPullExecutor implements Runnable {
65
66     private final Logger logger = LoggerFactory.getLogger(FederationPullExecutor.class);
67
68     private final List<FederationModel> registrations;
69
f6740d 70     private final boolean isDaemon;
JM 71
831469 72     /**
JM 73      * Constructor for specifying a single federation registration. This
74      * constructor is used to schedule the next pull execution.
75      * 
76      * @param registration
77      */
78     private FederationPullExecutor(FederationModel registration) {
f6740d 79         this(Arrays.asList(registration), true);
831469 80     }
JM 81
82     /**
83      * Constructor to specify a group of federation registrations. This is
84      * normally used at startup to pull and then schedule the next update based
85      * on each registrations frequency setting.
86      * 
87      * @param registrations
f6740d 88      * @param isDaemon
31abc2 89      *            if true, registrations are rescheduled in perpetuity. if
JM 90      *            false, the federation pull operation is executed once.
831469 91      */
f6740d 92     public FederationPullExecutor(List<FederationModel> registrations, boolean isDaemon) {
831469 93         this.registrations = registrations;
f6740d 94         this.isDaemon = isDaemon;
831469 95     }
JM 96
97     /**
98      * Run method for this pull executor.
99      */
100     @Override
101     public void run() {
102         for (FederationModel registration : registrations) {
103             FederationPullStatus was = registration.getLowestStatus();
104             try {
105                 Date now = new Date(System.currentTimeMillis());
106                 pull(registration);
107                 sendStatusAcknowledgment(registration);
108                 registration.lastPull = now;
109                 FederationPullStatus is = registration.getLowestStatus();
110                 if (is.ordinal() < was.ordinal()) {
111                     // the status for this registration has downgraded
112                     logger.warn("Federation pull status of {0} is now {1}", registration.name,
113                             is.name());
114                     if (registration.notifyOnError) {
115                         String message = "Federation pull of " + registration.name + " @ "
116                                 + registration.url + " is now at " + is.name();
117                         GitBlit.self()
eb96ea 118                                 .sendMailToAdministrators(
831469 119                                         "Pull Status of " + registration.name + " is " + is.name(),
JM 120                                         message);
121                     }
122                 }
123             } catch (Throwable t) {
124                 logger.error(MessageFormat.format(
125                         "Failed to pull from federated gitblit ({0} @ {1})", registration.name,
126                         registration.url), t);
127             } finally {
f6740d 128                 if (isDaemon) {
JM 129                     schedule(registration);
130                 }
831469 131             }
JM 132         }
133     }
134
135     /**
136      * Mirrors a repository and, optionally, the server's users, and/or
2548a7 137      * configuration settings from a origin Gitblit instance.
831469 138      * 
JM 139      * @param registration
140      * @throws Exception
141      */
142     private void pull(FederationModel registration) throws Exception {
143         Map<String, RepositoryModel> repositories = FederationUtils.getRepositories(registration,
144                 true);
145         String registrationFolder = registration.folder.toLowerCase().trim();
146         // confirm valid characters in server alias
147         Character c = StringUtils.findInvalidCharacter(registrationFolder);
148         if (c != null) {
149             logger.error(MessageFormat
150                     .format("Illegal character ''{0}'' in folder name ''{1}'' of federation registration {2}!",
151                             c, registrationFolder, registration.name));
152             return;
153         }
154         File repositoriesFolder = new File(GitBlit.getString(Keys.git.repositoriesFolder, "git"));
155         File registrationFolderFile = new File(repositoriesFolder, registrationFolder);
156         registrationFolderFile.mkdirs();
157
158         // Clone/Pull the repository
159         for (Map.Entry<String, RepositoryModel> entry : repositories.entrySet()) {
160             String cloneUrl = entry.getKey();
161             RepositoryModel repository = entry.getValue();
162             if (!repository.hasCommits) {
163                 logger.warn(MessageFormat.format(
164                         "Skipping federated repository {0} from {1} @ {2}. Repository is EMPTY.",
165                         repository.name, registration.name, registration.url));
166                 registration.updateStatus(repository, FederationPullStatus.SKIPPED);
167                 continue;
168             }
169
f6740d 170             // Determine local repository name
831469 171             String repositoryName;
JM 172             if (StringUtils.isEmpty(registrationFolder)) {
173                 repositoryName = repository.name;
174             } else {
175                 repositoryName = registrationFolder + "/" + repository.name;
f6740d 176             }
31abc2 177
f6740d 178             if (registration.bare) {
JM 179                 // bare repository, ensure .git suffix
180                 if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
181                     repositoryName += DOT_GIT_EXT;
182                 }
183             } else {
184                 // normal repository, strip .git suffix
185                 if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
31abc2 186                     repositoryName = repositoryName.substring(0,
JM 187                             repositoryName.indexOf(DOT_GIT_EXT));
f6740d 188                 }
831469 189             }
JM 190
191             // confirm that the origin of any pre-existing repository matches
192             // the clone url
2548a7 193             String fetchHead = null;
831469 194             Repository existingRepository = GitBlit.self().getRepository(repositoryName);
JM 195             if (existingRepository != null) {
196                 StoredConfig config = existingRepository.getConfig();
197                 config.load();
198                 String origin = config.getString("remote", "origin", "url");
31abc2 199                 RevCommit commit = JGitUtils.getCommit(existingRepository,
87db82 200                         org.eclipse.jgit.lib.Constants.FETCH_HEAD);
f6740d 201                 if (commit != null) {
JM 202                     fetchHead = commit.getName();
203                 }
831469 204                 existingRepository.close();
JM 205                 if (!origin.startsWith(registration.url)) {
206                     logger.warn(MessageFormat
207                             .format("Skipping federated repository {0} from {1} @ {2}. Origin does not match, consider EXCLUDING.",
208                                     repository.name, registration.name, registration.url));
209                     registration.updateStatus(repository, FederationPullStatus.SKIPPED);
210                     continue;
211                 }
212             }
213
214             // clone/pull this repository
215             CredentialsProvider credentials = new UsernamePasswordCredentialsProvider(
216                     Constants.FEDERATION_USER, registration.token);
217             logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",
218                     repository.name, registration.name, registration.url));
31abc2 219
831469 220             CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,
d7fb20 221                     cloneUrl, registration.bare, credentials);
831469 222             Repository r = GitBlit.self().getRepository(repositoryName);
JM 223             RepositoryModel rm = GitBlit.self().getRepositoryModel(repositoryName);
2548a7 224             repository.isFrozen = registration.mirror;
831469 225             if (result.createdRepository) {
JM 226                 // default local settings
227                 repository.federationStrategy = FederationStrategy.EXCLUDE;
2548a7 228                 repository.isFrozen = registration.mirror;
JM 229                 repository.showRemoteBranches = !registration.mirror;
230                 logger.info(MessageFormat.format("     cloning {0}", repository.name));
231                 registration.updateStatus(repository, FederationPullStatus.MIRRORED);
831469 232             } else {
2548a7 233                 // fetch and update
JM 234                 boolean fetched = false;
87db82 235                 RevCommit commit = JGitUtils.getCommit(r, org.eclipse.jgit.lib.Constants.FETCH_HEAD);
JM 236                 String newFetchHead = commit.getName();
237                 fetched = fetchHead == null || !fetchHead.equals(newFetchHead);
2548a7 238
JM 239                 if (registration.mirror) {
240                     // mirror
241                     if (fetched) {
f96d32 242                         // update local branches to match the remote tracking branches
JM 243                         for (RefModel ref : JGitUtils.getRemoteBranches(r, false, -1)) {
244                             if (ref.displayName.startsWith("origin/")) {
245                                 String branch = org.eclipse.jgit.lib.Constants.R_HEADS
246                                         + ref.displayName.substring(ref.displayName.indexOf('/') + 1);
247                                 String hash = ref.getReferencedObjectId().getName();
248                                 
249                                 JGitUtils.setBranchRef(r, branch, hash);
250                                 logger.info(MessageFormat.format("     resetting {0} of {1} to {2}", branch,
251                                         repository.name, hash));
87db82 252                             }
JM 253                         }
f96d32 254                         
JM 255                         String newHead;
256                         if (StringUtils.isEmpty(repository.HEAD)) {
257                             newHead = newFetchHead;
258                         } else {
259                             newHead = repository.HEAD;
260                         }
261                         JGitUtils.setHEADtoRef(r, newHead);
2548a7 262                         logger.info(MessageFormat.format("     resetting HEAD of {0} to {1}",
f96d32 263                                 repository.name, newHead));
2548a7 264                         registration.updateStatus(repository, FederationPullStatus.MIRRORED);
JM 265                     } else {
266                         // indicate no commits pulled
267                         registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
268                     }
269                 } else {
270                     // non-mirror
271                     if (fetched) {
272                         // indicate commits pulled to origin/master
273                         registration.updateStatus(repository, FederationPullStatus.PULLED);
274                     } else {
275                         // indicate no commits pulled
276                         registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
277                     }
278                 }
279
831469 280                 // preserve local settings
JM 281                 repository.isFrozen = rm.isFrozen;
282                 repository.federationStrategy = rm.federationStrategy;
31abc2 283
f6740d 284                 // merge federation sets
JM 285                 Set<String> federationSets = new HashSet<String>();
286                 if (rm.federationSets != null) {
287                     federationSets.addAll(rm.federationSets);
288                 }
289                 if (repository.federationSets != null) {
290                     federationSets.addAll(repository.federationSets);
291                 }
292                 repository.federationSets = new ArrayList<String>(federationSets);
5b29c5 293                 
JM 294                 // merge indexed branches
295                 Set<String> indexedBranches = new HashSet<String>();
296                 if (rm.indexedBranches != null) {
297                     indexedBranches.addAll(rm.indexedBranches);
298                 }
299                 if (repository.indexedBranches != null) {
300                     indexedBranches.addAll(repository.indexedBranches);
301                 }
302                 repository.indexedBranches = new ArrayList<String>(indexedBranches);
303
831469 304             }
2548a7 305             // only repositories that are actually _cloned_ from the origin
831469 306             // Gitblit repository are marked as federated. If the origin
JM 307             // is from somewhere else, these repositories are not considered
308             // "federated" repositories.
309             repository.isFederated = cloneUrl.startsWith(registration.url);
310
311             GitBlit.self().updateConfiguration(r, repository);
312             r.close();
313         }
314
df162c 315         IUserService userService = null;
JM 316
831469 317         try {
JM 318             // Pull USERS
997c16 319             // TeamModels are automatically pulled because they are contained
JM 320             // within the UserModel. The UserService creates unknown teams
321             // and updates existing teams.
831469 322             Collection<UserModel> users = FederationUtils.getUsers(registration);
JM 323             if (users != null && users.size() > 0) {
997c16 324                 File realmFile = new File(registrationFolderFile, registration.name + "_users.conf");
831469 325                 realmFile.delete();
df162c 326                 userService = new ConfigUserService(realmFile);
831469 327                 for (UserModel user : users) {
JM 328                     userService.updateUserModel(user.username, user);
329
2548a7 330                     // merge the origin permissions and origin accounts into
831469 331                     // the user accounts of this Gitblit instance
JM 332                     if (registration.mergeAccounts) {
333                         // reparent all repository permissions if the local
334                         // repositories are stored within subfolders
335                         if (!StringUtils.isEmpty(registrationFolder)) {
336                             List<String> permissions = new ArrayList<String>(user.repositories);
337                             user.repositories.clear();
338                             for (String permission : permissions) {
339                                 user.addRepository(registrationFolder + "/" + permission);
340                             }
341                         }
342
343                         // insert new user or update local user
344                         UserModel localUser = GitBlit.self().getUserModel(user.username);
345                         if (localUser == null) {
346                             // create new local user
347                             GitBlit.self().updateUserModel(user.username, user, true);
348                         } else {
349                             // update repository permissions of local user
350                             for (String repository : user.repositories) {
351                                 localUser.addRepository(repository);
352                             }
353                             localUser.password = user.password;
354                             localUser.canAdmin = user.canAdmin;
355                             GitBlit.self().updateUserModel(localUser.username, localUser, false);
356                         }
997c16 357
JM 358                         for (String teamname : GitBlit.self().getAllTeamnames()) {
359                             TeamModel team = GitBlit.self().getTeamModel(teamname);
360                             if (user.isTeamMember(teamname) && !team.hasUser(user.username)) {
361                                 // new team member
362                                 team.addUser(user.username);
363                                 GitBlit.self().updateTeamModel(teamname, team, false);
364                             } else if (!user.isTeamMember(teamname) && team.hasUser(user.username)) {
365                                 // remove team member
366                                 team.removeUser(user.username);
367                                 GitBlit.self().updateTeamModel(teamname, team, false);
368                             }
369
370                             // update team repositories
371                             TeamModel remoteTeam = user.getTeam(teamname);
0db5c4 372                             if (remoteTeam != null && !ArrayUtils.isEmpty(remoteTeam.repositories)) {
997c16 373                                 int before = team.repositories.size();
JM 374                                 team.addRepositories(remoteTeam.repositories);
375                                 int after = team.repositories.size();
376                                 if (after > before) {
377                                     // repository count changed, update
378                                     GitBlit.self().updateTeamModel(teamname, team, false);
379                                 }
380                             }
381                         }
831469 382                     }
JM 383                 }
384             }
31abc2 385         } catch (ForbiddenException e) {
JM 386             // ignore forbidden exceptions
387         } catch (IOException e) {
388             logger.warn(MessageFormat.format(
389                     "Failed to retrieve USERS from federated gitblit ({0} @ {1})",
390                     registration.name, registration.url), e);
831469 391         }
JM 392
393         try {
df162c 394             // Pull TEAMS
JM 395             // We explicitly pull these even though they are embedded in
396             // UserModels because it is possible to use teams to specify
397             // mailing lists or push scripts without specifying users.
398             if (userService != null) {
399                 Collection<TeamModel> teams = FederationUtils.getTeams(registration);
400                 if (teams != null && teams.size() > 0) {
401                     for (TeamModel team : teams) {
402                         userService.updateTeamModel(team);
403                     }
404                 }
405             }
406         } catch (ForbiddenException e) {
407             // ignore forbidden exceptions
408         } catch (IOException e) {
409             logger.warn(MessageFormat.format(
410                     "Failed to retrieve TEAMS from federated gitblit ({0} @ {1})",
411                     registration.name, registration.url), e);
412         }
413
414         try {
831469 415             // Pull SETTINGS
JM 416             Map<String, String> settings = FederationUtils.getSettings(registration);
417             if (settings != null && settings.size() > 0) {
418                 Properties properties = new Properties();
419                 properties.putAll(settings);
420                 FileOutputStream os = new FileOutputStream(new File(registrationFolderFile,
421                         registration.name + "_" + Constants.PROPERTIES_FILE));
422                 properties.store(os, null);
423                 os.close();
424             }
31abc2 425         } catch (ForbiddenException e) {
JM 426             // ignore forbidden exceptions
427         } catch (IOException e) {
428             logger.warn(MessageFormat.format(
429                     "Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
430                     registration.name, registration.url), e);
831469 431         }
0db5c4 432
df162c 433         try {
JM 434             // Pull SCRIPTS
435             Map<String, String> scripts = FederationUtils.getScripts(registration);
436             if (scripts != null && scripts.size() > 0) {
437                 for (Map.Entry<String, String> script : scripts.entrySet()) {
438                     String scriptName = script.getKey();
439                     if (scriptName.endsWith(".groovy")) {
0db5c4 440                         scriptName = scriptName.substring(0, scriptName.indexOf(".groovy"));
df162c 441                     }
0db5c4 442                     File file = new File(registrationFolderFile, registration.name + "_"
JM 443                             + scriptName + ".groovy");
df162c 444                     FileUtils.writeContent(file, script.getValue());
JM 445                 }
446             }
447         } catch (ForbiddenException e) {
448             // ignore forbidden exceptions
449         } catch (IOException e) {
450             logger.warn(MessageFormat.format(
451                     "Failed to retrieve SCRIPTS from federated gitblit ({0} @ {1})",
452                     registration.name, registration.url), e);
453         }
831469 454     }
JM 455
456     /**
2548a7 457      * Sends a status acknowledgment to the origin Gitblit instance. This
831469 458      * includes the results of the federated pull.
JM 459      * 
460      * @param registration
461      * @throws Exception
462      */
463     private void sendStatusAcknowledgment(FederationModel registration) throws Exception {
464         if (!registration.sendStatus) {
465             // skip status acknowledgment
466             return;
467         }
468         InetAddress addr = InetAddress.getLocalHost();
469         String federationName = GitBlit.getString(Keys.federation.name, null);
470         if (StringUtils.isEmpty(federationName)) {
471             federationName = addr.getHostName();
472         }
473         FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
2548a7 474         logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
831469 475     }
JM 476
477     /**
478      * Schedules the next check of the federated Gitblit instance.
479      * 
480      * @param registration
481      */
482     private void schedule(FederationModel registration) {
483         // schedule the next pull
484         int mins = TimeUtils.convertFrequencyToMinutes(registration.frequency);
485         registration.nextPull = new Date(System.currentTimeMillis() + (mins * 60 * 1000L));
486         GitBlit.self().executor()
487                 .schedule(new FederationPullExecutor(registration), mins, TimeUnit.MINUTES);
488         logger.info(MessageFormat.format(
489                 "Next pull of {0} @ {1} scheduled for {2,date,yyyy-MM-dd HH:mm}",
490                 registration.name, registration.url, registration.nextPull));
491     }
492 }