Gitblit devel
view | history | commit | commitdiff
James Moger
2016-01-25 252dc07d7f85cc344b5919bb7c6166ef84b2102e
commit | author | age
59b817 1 /*
JM 2  * Copyright 2012 gitblit.com.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.gitblit.utils;
17
efdb2b 18 import java.io.ByteArrayOutputStream;
46f33f 19 import java.io.EOFException;
PM 20 import java.io.File;
21 import java.io.FileInputStream;
59b817 22 import java.io.IOException;
JM 23 import java.io.OutputStream;
24 import java.text.MessageFormat;
25 import java.util.ArrayList;
26 import java.util.List;
27
28 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
29 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
1d9ac5 30 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
JM 31 import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
59b817 32 import org.apache.commons.compress.compressors.CompressorException;
JM 33 import org.apache.commons.compress.compressors.CompressorStreamFactory;
46f33f 34 import org.apache.commons.io.IOUtils;
59b817 35 import org.eclipse.jgit.lib.Constants;
JM 36 import org.eclipse.jgit.lib.FileMode;
1d9ac5 37 import org.eclipse.jgit.lib.MutableObjectId;
46f33f 38 import org.eclipse.jgit.lib.ObjectId;
59b817 39 import org.eclipse.jgit.lib.ObjectLoader;
1d9ac5 40 import org.eclipse.jgit.lib.ObjectReader;
59b817 41 import org.eclipse.jgit.lib.Repository;
JM 42 import org.eclipse.jgit.revwalk.RevCommit;
43 import org.eclipse.jgit.revwalk.RevWalk;
44 import org.eclipse.jgit.treewalk.TreeWalk;
45 import org.eclipse.jgit.treewalk.filter.PathFilter;
46 import org.slf4j.Logger;
47 import org.slf4j.LoggerFactory;
46f33f 48
PM 49 import com.gitblit.GitBlit;
50 import com.gitblit.manager.IFilestoreManager;
51 import com.gitblit.models.FilestoreModel;
52 import com.gitblit.models.FilestoreModel.Status;
59b817 53
JM 54 /**
55  * Collection of static methods for retrieving information from a repository.
699e71 56  *
59b817 57  * @author James Moger
699e71 58  *
59b817 59  */
JM 60 public class CompressionUtils {
61
62     static final Logger LOGGER = LoggerFactory.getLogger(CompressionUtils.class);
63
64     /**
65      * Log an error message and exception.
699e71 66      *
59b817 67      * @param t
JM 68      * @param repository
69      *            if repository is not null it MUST be the {0} parameter in the
70      *            pattern.
71      * @param pattern
72      * @param objects
73      */
74     private static void error(Throwable t, Repository repository, String pattern, Object... objects) {
75         List<Object> parameters = new ArrayList<Object>();
76         if (objects != null && objects.length > 0) {
77             for (Object o : objects) {
78                 parameters.add(o);
79             }
80         }
81         if (repository != null) {
82             parameters.add(0, repository.getDirectory().getAbsolutePath());
83         }
84         LOGGER.error(MessageFormat.format(pattern, parameters.toArray()), t);
85     }
86
87     /**
88      * Zips the contents of the tree at the (optionally) specified revision and
89      * the (optionally) specified basepath to the supplied outputstream.
699e71 90      *
59b817 91      * @param repository
JM 92      * @param basePath
93      *            if unspecified, entire repository is assumed.
94      * @param objectId
95      *            if unspecified, HEAD is assumed.
96      * @param os
97      * @return true if repository was successfully zipped to supplied output
98      *         stream
99      */
46f33f 100     public static boolean zip(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
59b817 101             OutputStream os) {
JM 102         RevCommit commit = JGitUtils.getCommit(repository, objectId);
103         if (commit == null) {
104             return false;
105         }
106         boolean success = false;
107         RevWalk rw = new RevWalk(repository);
108         TreeWalk tw = new TreeWalk(repository);
109         try {
1d9ac5 110             tw.reset();
59b817 111             tw.addTree(commit.getTree());
1d9ac5 112             ZipArchiveOutputStream zos = new ZipArchiveOutputStream(os);
59b817 113             zos.setComment("Generated by Gitblit");
JM 114             if (!StringUtils.isEmpty(basePath)) {
115                 PathFilter f = PathFilter.create(basePath);
116                 tw.setFilter(f);
117             }
118             tw.setRecursive(true);
1d9ac5 119             MutableObjectId id = new MutableObjectId();
JM 120             ObjectReader reader = tw.getObjectReader();
121             long modified = commit.getAuthorIdent().getWhen().getTime();
59b817 122             while (tw.next()) {
1d9ac5 123                 FileMode mode = tw.getFileMode(0);
JM 124                 if (mode == FileMode.GITLINK || mode == FileMode.TREE) {
59b817 125                     continue;
JM 126                 }
1d9ac5 127                 tw.getObjectId(id, 0);
46f33f 128                 
PM 129                 ObjectLoader loader = repository.open(id);
130                 
1d9ac5 131                 ZipArchiveEntry entry = new ZipArchiveEntry(tw.getPathString());
46f33f 132
PM 133                 FilestoreModel filestoreItem = null;
134                 
135                 if (JGitUtils.isPossibleFilestoreItem(loader.getSize())) {
136                     filestoreItem = JGitUtils.getFilestoreItem(tw.getObjectReader().open(id));
137                 }
138
139                 final long size = (filestoreItem == null) ? loader.getSize() : filestoreItem.getSize();  
140
141                 entry.setSize(size);
59b817 142                 entry.setComment(commit.getName());
1d9ac5 143                 entry.setUnixMode(mode.getBits());
JM 144                 entry.setTime(modified);
145                 zos.putArchiveEntry(entry);
46f33f 146                 
PM 147                 if (filestoreItem == null) {
148                     //Copy repository stored file
149                     loader.copyTo(zos);
150                 } else {
151                     //Copy filestore file
152                     try (FileInputStream streamIn = new FileInputStream(filestoreManager.getStoragePath(filestoreItem.oid))) {
153                         IOUtils.copyLarge(streamIn, zos);
154                     } catch (Throwable e) {
155                         LOGGER.error(MessageFormat.format("Failed to archive filestore item {0}", filestoreItem.oid), e);
59b817 156
46f33f 157                         //Handle as per other errors 
PM 158                         throw e; 
159                     }
160                 }
161
1d9ac5 162                 zos.closeArchiveEntry();
59b817 163             }
JM 164             zos.finish();
165             success = true;
166         } catch (IOException e) {
167             error(e, repository, "{0} failed to zip files from commit {1}", commit.getName());
168         } finally {
a1cee6 169             tw.close();
59b817 170             rw.dispose();
JM 171         }
172         return success;
173     }
699e71 174
59b817 175     /**
JM 176      * tar the contents of the tree at the (optionally) specified revision and
177      * the (optionally) specified basepath to the supplied outputstream.
699e71 178      *
59b817 179      * @param repository
JM 180      * @param basePath
181      *            if unspecified, entire repository is assumed.
182      * @param objectId
183      *            if unspecified, HEAD is assumed.
184      * @param os
185      * @return true if repository was successfully zipped to supplied output
186      *         stream
187      */
46f33f 188     public static boolean tar(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
59b817 189             OutputStream os) {
46f33f 190         return tar(null, repository, filestoreManager, basePath, objectId, os);
59b817 191     }
699e71 192
59b817 193     /**
JM 194      * tar.gz the contents of the tree at the (optionally) specified revision and
195      * the (optionally) specified basepath to the supplied outputstream.
699e71 196      *
59b817 197      * @param repository
JM 198      * @param basePath
199      *            if unspecified, entire repository is assumed.
200      * @param objectId
201      *            if unspecified, HEAD is assumed.
202      * @param os
203      * @return true if repository was successfully zipped to supplied output
204      *         stream
205      */
46f33f 206     public static boolean gz(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
59b817 207             OutputStream os) {
46f33f 208         return tar(CompressorStreamFactory.GZIP, repository, filestoreManager, basePath, objectId, os);
59b817 209     }
699e71 210
59b817 211     /**
JM 212      * tar.xz the contents of the tree at the (optionally) specified revision and
213      * the (optionally) specified basepath to the supplied outputstream.
699e71 214      *
59b817 215      * @param repository
JM 216      * @param basePath
217      *            if unspecified, entire repository is assumed.
218      * @param objectId
219      *            if unspecified, HEAD is assumed.
220      * @param os
221      * @return true if repository was successfully zipped to supplied output
222      *         stream
223      */
46f33f 224     public static boolean xz(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
59b817 225             OutputStream os) {
46f33f 226         return tar(CompressorStreamFactory.XZ, repository, filestoreManager, basePath, objectId, os);
59b817 227     }
699e71 228
59b817 229     /**
JM 230      * tar.bzip2 the contents of the tree at the (optionally) specified revision and
231      * the (optionally) specified basepath to the supplied outputstream.
699e71 232      *
59b817 233      * @param repository
JM 234      * @param basePath
235      *            if unspecified, entire repository is assumed.
236      * @param objectId
237      *            if unspecified, HEAD is assumed.
238      * @param os
239      * @return true if repository was successfully zipped to supplied output
240      *         stream
241      */
46f33f 242     public static boolean bzip2(Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
59b817 243             OutputStream os) {
699e71 244
46f33f 245         return tar(CompressorStreamFactory.BZIP2, repository, filestoreManager, basePath, objectId, os);
59b817 246     }
699e71 247
59b817 248     /**
JM 249      * Compresses/archives the contents of the tree at the (optionally)
250      * specified revision and the (optionally) specified basepath to the
251      * supplied outputstream.
699e71 252      *
59b817 253      * @param algorithm
JM 254      *            compression algorithm for tar (optional)
255      * @param repository
256      * @param basePath
257      *            if unspecified, entire repository is assumed.
258      * @param objectId
259      *            if unspecified, HEAD is assumed.
260      * @param os
261      * @return true if repository was successfully zipped to supplied output
262      *         stream
263      */
46f33f 264     private static boolean tar(String algorithm, Repository repository, IFilestoreManager filestoreManager, String basePath, String objectId,
59b817 265             OutputStream os) {
JM 266         RevCommit commit = JGitUtils.getCommit(repository, objectId);
267         if (commit == null) {
268             return false;
269         }
699e71 270
59b817 271         OutputStream cos = os;
JM 272         if (!StringUtils.isEmpty(algorithm)) {
273             try {
274                 cos = new CompressorStreamFactory().createCompressorOutputStream(algorithm, os);
275             } catch (CompressorException e1) {
276                 error(e1, repository, "{0} failed to open {1} stream", algorithm);
277             }
278         }
279         boolean success = false;
280         RevWalk rw = new RevWalk(repository);
281         TreeWalk tw = new TreeWalk(repository);
282         try {
1d9ac5 283             tw.reset();
59b817 284             tw.addTree(commit.getTree());
JM 285             TarArchiveOutputStream tos = new TarArchiveOutputStream(cos);
286             tos.setAddPaxHeadersForNonAsciiNames(true);
287             tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
288             if (!StringUtils.isEmpty(basePath)) {
289                 PathFilter f = PathFilter.create(basePath);
290                 tw.setFilter(f);
291             }
292             tw.setRecursive(true);
1d9ac5 293             MutableObjectId id = new MutableObjectId();
JM 294             long modified = commit.getAuthorIdent().getWhen().getTime();
59b817 295             while (tw.next()) {
JM 296                 FileMode mode = tw.getFileMode(0);
1d9ac5 297                 if (mode == FileMode.GITLINK || mode == FileMode.TREE) {
59b817 298                     continue;
JM 299                 }
46f33f 300                 
1d9ac5 301                 tw.getObjectId(id, 0);
699e71 302
8ee931 303                 ObjectLoader loader = repository.open(id);
JM 304                 if (FileMode.SYMLINK == mode) {
305                     TarArchiveEntry entry = new TarArchiveEntry(tw.getPathString(),TarArchiveEntry.LF_SYMLINK);
306                     ByteArrayOutputStream bos = new ByteArrayOutputStream();
307                     loader.copyTo(bos);
308                     entry.setLinkName(bos.toString());
309                     entry.setModTime(modified);
310                     tos.putArchiveEntry(entry);
311                     tos.closeArchiveEntry();
312                 } else {
313                     TarArchiveEntry entry = new TarArchiveEntry(tw.getPathString());
314                     entry.setMode(mode.getBits());
315                     entry.setModTime(modified);
46f33f 316
PM 317                     FilestoreModel filestoreItem = null;
318                     
319                     if (JGitUtils.isPossibleFilestoreItem(loader.getSize())) {
320                         filestoreItem = JGitUtils.getFilestoreItem(tw.getObjectReader().open(id));
321                     }
322
323                     final long size = (filestoreItem == null) ? loader.getSize() : filestoreItem.getSize();  
324
325                     entry.setSize(size);
699e71 326                     tos.putArchiveEntry(entry);
46f33f 327                     
PM 328                     if (filestoreItem == null) {
329                         //Copy repository stored file
330                         loader.copyTo(tos);
331                     } else {
332                         //Copy filestore file
333                         try (FileInputStream streamIn = new FileInputStream(filestoreManager.getStoragePath(filestoreItem.oid))) {
334
335                             IOUtils.copyLarge(streamIn, tos);
336                         } catch (Throwable e) {
337                             LOGGER.error(MessageFormat.format("Failed to archive filestore item {0}", filestoreItem.oid), e);
338
339                             //Handle as per other errors 
340                             throw e; 
341                         }
342                     }
343                     
8ee931 344                     tos.closeArchiveEntry();
JM 345                 }
59b817 346             }
JM 347             tos.finish();
348             tos.close();
349             cos.close();
350             success = true;
351         } catch (IOException e) {
352             error(e, repository, "{0} failed to {1} stream files from commit {2}", algorithm, commit.getName());
353         } finally {
a1cee6 354             tw.close();
59b817 355             rw.dispose();
JM 356         }
357         return success;
358     }
359 }