From a3a18a0ebfeb65777ad5bd065e26fa9c00e8100c Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gmail.com>
Date: Sat, 10 Oct 2015 08:13:12 -0400
Subject: [PATCH] Merge pull request #921 from paulsputer/git-lfs-support
---
src/main/java/com/gitblit/wicket/pages/FilestorePage.html | 37
src/main/java/com/gitblit/Constants.java | 4
src/test/java/com/gitblit/tests/GitblitUnitTest.java | 5
src/main/java/com/gitblit/wicket/GitBlitWebApp.java | 16
src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java | 54 +
src/main/java/com/gitblit/wicket/pages/RootPage.java | 1
src/main/java/com/gitblit/guice/WebModule.java | 6
src/test/java/com/gitblit/tests/GitBlitSuite.java | 2
src/main/java/com/gitblit/manager/IGitblit.java | 3
src/main/java/com/gitblit/models/FilestoreModel.java | 159 +++
src/main/java/com/gitblit/servlet/FilestoreServlet.java | 493 ++++++++++
src/main/java/com/gitblit/manager/GitblitManager.java | 73 +
src/main/java/com/gitblit/manager/IFilestoreManager.java | 54 +
src/main/resources/gitblit.css | 16
src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java | 25
src/main/java/com/gitblit/servlet/RawFilter.java | 8
src/main/java/com/gitblit/wicket/FilestoreUI.java | 62 +
src/test/java/com/gitblit/tests/FilestoreManagerTest.java | 547 ++++++++++++
src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html | 69 +
src/main/java/com/gitblit/servlet/GitblitContext.java | 2
src/main/java/com/gitblit/utils/JsonUtils.java | 21
src/main/java/com/gitblit/GitBlit.java | 7
src/main/java/com/gitblit/wicket/GitBlitWebApp.properties | 8
src/test/java/com/gitblit/tests/FilestoreServletTest.java | 355 +++++++
src/main/java/com/gitblit/servlet/GitFilter.java | 78 +
src/main/java/com/gitblit/wicket/GitblitWicketApp.java | 3
src/main/java/com/gitblit/wicket/pages/FilestorePage.java | 114 ++
src/main/java/com/gitblit/FederationClient.java | 2
src/main/java/com/gitblit/manager/FilestoreManager.java | 439 +++++++++
src/main/distrib/data/defaults.properties | 14
src/main/java/com/gitblit/servlet/DownloadZipFilter.java | 8
src/main/java/com/gitblit/guice/CoreModule.java | 3
32 files changed, 2,651 insertions(+), 37 deletions(-)
diff --git a/src/main/distrib/data/defaults.properties b/src/main/distrib/data/defaults.properties
index 4606f5f..ce6267a 100644
--- a/src/main/distrib/data/defaults.properties
+++ b/src/main/distrib/data/defaults.properties
@@ -2030,3 +2030,17 @@
# SINCE 0.5.0
# RESTART REQUIRED
server.shutdownPort = 8081
+
+#
+# Gitblit Filestore Settings
+#
+# The location to save the filestore blobs
+#
+# SINCE 1.7.0
+filestore.storageFolder = ${baseFolder}/lfs
+
+# Maximum allowable upload size
+# The default value, -1, disables upload limits.
+# Common unit suffixes of k, m, or g are supported.
+# SINCE 1.7.0
+filestore.maxUploadSize = -1
diff --git a/src/main/java/com/gitblit/Constants.java b/src/main/java/com/gitblit/Constants.java
index 787d726..4aa8c0c 100644
--- a/src/main/java/com/gitblit/Constants.java
+++ b/src/main/java/com/gitblit/Constants.java
@@ -60,6 +60,8 @@
public static final String R_PATH = "/r/";
public static final String GIT_PATH = "/git/";
+
+ public static final String REGEX_SHA256 = "[a-fA-F0-9]{64}";
public static final String ZIP_PATH = "/zip/";
@@ -140,6 +142,8 @@
public static final String ATTRIB_AUTHTYPE = NAME + ":authentication-type";
public static final String ATTRIB_AUTHUSER = NAME + ":authenticated-user";
+
+ public static final String R_LFS = "info/lfs/";
public static String getVersion() {
String v = Constants.class.getPackage().getImplementationVersion();
diff --git a/src/main/java/com/gitblit/FederationClient.java b/src/main/java/com/gitblit/FederationClient.java
index 487080e..64ff017 100644
--- a/src/main/java/com/gitblit/FederationClient.java
+++ b/src/main/java/com/gitblit/FederationClient.java
@@ -100,7 +100,7 @@
UserManager users = new UserManager(runtime, null).start();
RepositoryManager repositories = new RepositoryManager(runtime, null, users).start();
FederationManager federation = new FederationManager(runtime, notifications, repositories).start();
- IGitblit gitblit = new GitblitManager(null, null, runtime, null, notifications, users, null, repositories, null, federation);
+ IGitblit gitblit = new GitblitManager(null, null, runtime, null, notifications, users, null, repositories, null, federation, null);
FederationPullService puller = new FederationPullService(gitblit, federation.getFederationRegistrations()) {
@Override
diff --git a/src/main/java/com/gitblit/GitBlit.java b/src/main/java/com/gitblit/GitBlit.java
index 68a91bb..4e25d5c 100644
--- a/src/main/java/com/gitblit/GitBlit.java
+++ b/src/main/java/com/gitblit/GitBlit.java
@@ -18,6 +18,7 @@
import com.gitblit.manager.GitblitManager;
import com.gitblit.manager.IAuthenticationManager;
import com.gitblit.manager.IFederationManager;
+import com.gitblit.manager.IFilestoreManager;
import com.gitblit.manager.INotificationManager;
import com.gitblit.manager.IPluginManager;
import com.gitblit.manager.IProjectManager;
@@ -52,7 +53,8 @@
IAuthenticationManager authenticationManager,
IRepositoryManager repositoryManager,
IProjectManager projectManager,
- IFederationManager federationManager) {
+ IFederationManager federationManager,
+ IFilestoreManager filestoreManager) {
super(
publicKeyManagerProvider,
@@ -64,6 +66,7 @@
authenticationManager,
repositoryManager,
projectManager,
- federationManager);
+ federationManager,
+ filestoreManager);
}
}
diff --git a/src/main/java/com/gitblit/guice/CoreModule.java b/src/main/java/com/gitblit/guice/CoreModule.java
index a942b2e..e2d1439 100644
--- a/src/main/java/com/gitblit/guice/CoreModule.java
+++ b/src/main/java/com/gitblit/guice/CoreModule.java
@@ -20,8 +20,10 @@
import com.gitblit.IStoredSettings;
import com.gitblit.manager.AuthenticationManager;
import com.gitblit.manager.FederationManager;
+import com.gitblit.manager.FilestoreManager;
import com.gitblit.manager.IAuthenticationManager;
import com.gitblit.manager.IFederationManager;
+import com.gitblit.manager.IFilestoreManager;
import com.gitblit.manager.IGitblit;
import com.gitblit.manager.INotificationManager;
import com.gitblit.manager.IPluginManager;
@@ -72,6 +74,7 @@
bind(IRepositoryManager.class).to(RepositoryManager.class);
bind(IProjectManager.class).to(ProjectManager.class);
bind(IFederationManager.class).to(FederationManager.class);
+ bind(IFilestoreManager.class).to(FilestoreManager.class);
// the monolithic manager
bind(IGitblit.class).to(GitBlit.class);
diff --git a/src/main/java/com/gitblit/guice/WebModule.java b/src/main/java/com/gitblit/guice/WebModule.java
index a406270..7c83e45 100644
--- a/src/main/java/com/gitblit/guice/WebModule.java
+++ b/src/main/java/com/gitblit/guice/WebModule.java
@@ -26,6 +26,7 @@
import com.gitblit.servlet.DownloadZipServlet;
import com.gitblit.servlet.EnforceAuthenticationFilter;
import com.gitblit.servlet.FederationServlet;
+import com.gitblit.servlet.FilestoreServlet;
import com.gitblit.servlet.GitFilter;
import com.gitblit.servlet.GitServlet;
import com.gitblit.servlet.LogoServlet;
@@ -62,12 +63,14 @@
bind(AvatarGenerator.class).toProvider(AvatarGeneratorProvider.class);
// servlets
+ serveRegex(FilestoreServlet.REGEX_PATH).with(FilestoreServlet.class);
serve(fuzzy(Constants.R_PATH), fuzzy(Constants.GIT_PATH)).with(GitServlet.class);
serve(fuzzy(Constants.RAW_PATH)).with(RawServlet.class);
serve(fuzzy(Constants.PAGES)).with(PagesServlet.class);
serve(fuzzy(Constants.RPC_PATH)).with(RpcServlet.class);
serve(fuzzy(Constants.ZIP_PATH)).with(DownloadZipServlet.class);
serve(fuzzy(Constants.SYNDICATION_PATH)).with(SyndicationServlet.class);
+
serve(fuzzy(Constants.FEDERATION_PATH)).with(FederationServlet.class);
serve(fuzzy(Constants.SPARKLESHARE_INVITE_PATH)).with(SparkleShareInviteServlet.class);
@@ -98,7 +101,8 @@
filter(fuzzy(Constants.RPC_PATH)).through(RpcFilter.class);
filter(fuzzy(Constants.ZIP_PATH)).through(DownloadZipFilter.class);
filter(fuzzy(Constants.SYNDICATION_PATH)).through(SyndicationFilter.class);
-
+
+
// Wicket
String toIgnore = Joiner.on(",").join(Constants.R_PATH, Constants.GIT_PATH, Constants.RAW_PATH,
Constants.PAGES, Constants.RPC_PATH, Constants.ZIP_PATH, Constants.SYNDICATION_PATH,
diff --git a/src/main/java/com/gitblit/manager/FilestoreManager.java b/src/main/java/com/gitblit/manager/FilestoreManager.java
new file mode 100644
index 0000000..33672e4
--- /dev/null
+++ b/src/main/java/com/gitblit/manager/FilestoreManager.java
@@ -0,0 +1,439 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.manager;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.lang.reflect.Type;
+import java.nio.file.Files;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.bouncycastle.util.io.StreamOverflowException;
+import org.eclipse.jetty.io.EofException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.gitblit.IStoredSettings;
+import com.gitblit.Keys;
+import com.gitblit.models.FilestoreModel.Status;
+import com.gitblit.models.FilestoreModel;
+import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.UserModel;
+import com.gitblit.utils.ArrayUtils;
+import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter;
+import com.google.gson.ExclusionStrategy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * FilestoreManager handles files uploaded via:
+ * + git-lfs
+ * + ticket attachment (TBD)
+ *
+ * Files are stored using their SHA256 hash (as per git-lfs)
+ * If the same file is uploaded through different repositories no additional space is used
+ * Access is controlled through the current repository permissions.
+ *
+ * TODO: Identify what and how the actual BLOBs should work with federation
+ *
+ * @author Paul Martin
+ *
+ */
+@Singleton
+public class FilestoreManager implements IFilestoreManager {
+
+ private final Logger logger = LoggerFactory.getLogger(getClass());
+
+ private final IRuntimeManager runtimeManager;
+
+ private final IStoredSettings settings;
+
+ public static final int UNDEFINED_SIZE = -1;
+
+ private static final String METAFILE = "filestore.json";
+
+ private static final String METAFILE_TMP = "filestore.json.tmp";
+
+ protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType();
+
+ private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>();
+
+
+ @Inject
+ FilestoreManager(
+ IRuntimeManager runtimeManager) {
+ this.runtimeManager = runtimeManager;
+ this.settings = runtimeManager.getSettings();
+ }
+
+ @Override
+ public IManager start() {
+
+ //Try to load any existing metadata
+ File metadata = new File(getStorageFolder(), METAFILE);
+
+ if (metadata.exists()) {
+ Collection<FilestoreModel> items = null;
+
+ Gson gson = gson();
+ try (FileReader file = new FileReader(metadata)) {
+ items = gson.fromJson(file, METAFILE_TYPE);
+ file.close();
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) {
+ FilestoreModel model = itr.next();
+ fileCache.put(model.oid, model);
+ }
+
+ logger.info("Loaded {} items from filestore metadata file", fileCache.size());
+ }
+ else
+ {
+ logger.info("No filestore metadata file found");
+ }
+
+ return this;
+ }
+
+ @Override
+ public IManager stop() {
+ return this;
+ }
+
+
+ @Override
+ public boolean isValidOid(String oid) {
+ //NOTE: Assuming SHA256 support only as per git-lfs
+ return Pattern.matches("[a-fA-F0-9]{64}", oid);
+ }
+
+ @Override
+ public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) {
+
+ //Handle access control
+ if (!user.canPush(repo)) {
+ if (user == UserModel.ANONYMOUS) {
+ return Status.AuthenticationRequired;
+ } else {
+ return Status.Error_Unauthorized;
+ }
+ }
+
+ //Handle object details
+ if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; }
+
+ if (fileCache.containsKey(oid)) {
+ FilestoreModel item = fileCache.get(oid);
+
+ if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) {
+ return Status.Error_Size_Mismatch;
+ }
+
+ item.addRepository(repo.name);
+
+ if (item.isInErrorState()) {
+ item.reset(user, size);
+ }
+ } else {
+
+ if (size < 0) {return Status.Error_Invalid_Size; }
+ if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; }
+
+ FilestoreModel model = new FilestoreModel(oid, size, user, repo.name);
+ fileCache.put(oid, model);
+ saveFilestoreModel(model);
+ }
+
+ return fileCache.get(oid).getStatus();
+ }
+
+ @Override
+ public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) {
+
+ //Access control and object logic
+ Status state = addObject(oid, size, user, repo);
+
+ if (state != Status.Upload_Pending) {
+ return state;
+ }
+
+ FilestoreModel model = fileCache.get(oid);
+
+ if (!model.actionUpload(user)) {
+ return Status.Upload_In_Progress;
+ } else {
+ long actualSize = 0;
+ File file = getStoragePath(oid);
+
+ try {
+ file.getParentFile().mkdirs();
+ file.createNewFile();
+
+ try (FileOutputStream streamOut = new FileOutputStream(file)) {
+
+ actualSize = IOUtils.copyLarge(streamIn, streamOut);
+
+ streamOut.flush();
+ streamOut.close();
+
+ if (model.getSize() != actualSize) {
+ model.setStatus(Status.Error_Size_Mismatch, user);
+
+ logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}",
+ oid, model.getSize(), actualSize));
+ } else {
+ String actualOid = "";
+
+ try (FileInputStream fileForHash = new FileInputStream(file)) {
+ actualOid = DigestUtils.sha256Hex(fileForHash);
+ fileForHash.close();
+ }
+
+ if (oid.equalsIgnoreCase(actualOid)) {
+ model.setStatus(Status.Available, user);
+ } else {
+ model.setStatus(Status.Error_Hash_Mismatch, user);
+
+ logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid));
+ }
+ }
+ }
+ } catch (Exception e) {
+
+ model.setStatus(Status.Error_Unknown, user);
+ logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e);
+ } finally {
+ saveFilestoreModel(model);
+ }
+
+ if (model.isInErrorState()) {
+ file.delete();
+ model.removeRepository(repo.name);
+ }
+ }
+
+ return model.getStatus();
+ }
+
+ private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) {
+
+ //Access Control
+ if (!user.canView(repo)) {
+ if (user == UserModel.ANONYMOUS) {
+ return Status.AuthenticationRequired;
+ } else {
+ return Status.Error_Unauthorized;
+ }
+ }
+
+ //Object Logic
+ if (!isValidOid(oid)) {
+ return Status.Error_Invalid_Oid;
+ }
+
+ if (!fileCache.containsKey(oid)) {
+ return Status.Unavailable;
+ }
+
+ FilestoreModel item = fileCache.get(oid);
+
+ if (item.getStatus() == Status.Available) {
+ return Status.Available;
+ }
+
+ return Status.Unavailable;
+ }
+
+ @Override
+ public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) {
+
+ if (canGetObject(oid, user, repo) == Status.Available) {
+ return fileCache.get(oid);
+ }
+
+ return null;
+ }
+
+ @Override
+ public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) {
+
+ //Access control and object logic
+ Status status = canGetObject(oid, user, repo);
+
+ if (status != Status.Available) {
+ return status;
+ }
+
+ FilestoreModel item = fileCache.get(oid);
+
+ if (streamOut != null) {
+ try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) {
+
+ IOUtils.copyLarge(streamIn, streamOut);
+
+ streamOut.flush();
+ streamIn.close();
+ } catch (EofException e) {
+ logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e);
+ return Status.Error_Unexpected_Stream_End;
+ } catch (Exception e) {
+ logger.error(MessageFormat.format("Failed to download blob {0}", oid), e);
+ return Status.Error_Unknown;
+ }
+ }
+
+ return item.getStatus();
+ }
+
+ @Override
+ public List<FilestoreModel> getAllObjects() {
+ return new ArrayList<FilestoreModel>(fileCache.values());
+ }
+
+ @Override
+ public File getStorageFolder() {
+ return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs");
+ }
+
+ @Override
+ public File getStoragePath(String oid) {
+ return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2)));
+ }
+
+ @Override
+ public long getMaxUploadSize() {
+ return settings.getLong(Keys.filestore.maxUploadSize, -1);
+ }
+
+ @Override
+ public long getFilestoreUsedByteCount() {
+ Iterator<FilestoreModel> iterator = fileCache.values().iterator();
+ long total = 0;
+
+ while (iterator.hasNext()) {
+
+ FilestoreModel item = iterator.next();
+ if (item.getStatus() == Status.Available) {
+ total += item.getSize();
+ }
+ }
+
+ return total;
+ }
+
+ @Override
+ public long getFilestoreAvailableByteCount() {
+
+ try {
+ return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace();
+ } catch (IOException e) {
+ logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e));
+ }
+
+ return UNDEFINED_SIZE;
+ };
+
+ private synchronized void saveFilestoreModel(FilestoreModel model) {
+
+ File metaFile = new File(getStorageFolder(), METAFILE);
+ File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP);
+ boolean isNewFile = false;
+
+ try {
+ if (!metaFile.exists()) {
+ metaFile.getParentFile().mkdirs();
+ metaFile.createNewFile();
+ isNewFile = true;
+ }
+ FileUtils.copyFile(metaFile, metaFileTmp);
+
+ } catch (IOException e) {
+ logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
+ }
+
+ try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) {
+
+ if (isNewFile) {
+ fs.writeBytes("[");
+ } else {
+ fs.seek(fs.length() - 1);
+ fs.writeBytes(",");
+ }
+
+ fs.writeBytes(gson().toJson(model));
+ fs.writeBytes("]");
+
+ fs.close();
+
+ } catch (IOException e) {
+ logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e);
+ }
+
+ try {
+ if (metaFileTmp.exists()) {
+ FileUtils.copyFile(metaFileTmp, metaFile);
+
+ metaFileTmp.delete();
+ } else {
+ logger.error("Writing filestore model to file {0}", METAFILE);
+ }
+ }
+ catch (IOException e) {
+ logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
+ }
+ }
+
+ /*
+ * Intended for testing purposes only
+ */
+ public void clearFilestoreCache() {
+ fileCache.clear();
+ }
+
+ private static Gson gson(ExclusionStrategy... strategies) {
+ GsonBuilder builder = new GsonBuilder();
+ builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter());
+ if (!ArrayUtils.isEmpty(strategies)) {
+ builder.setExclusionStrategies(strategies);
+ }
+ return builder.create();
+ }
+
+}
diff --git a/src/main/java/com/gitblit/manager/GitblitManager.java b/src/main/java/com/gitblit/manager/GitblitManager.java
index 8d25b3f..a34e29d 100644
--- a/src/main/java/com/gitblit/manager/GitblitManager.java
+++ b/src/main/java/com/gitblit/manager/GitblitManager.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.OutputStream;
import java.lang.reflect.Type;
import java.text.MessageFormat;
import java.util.ArrayList;
@@ -58,6 +59,7 @@
import com.gitblit.models.FederationModel;
import com.gitblit.models.FederationProposal;
import com.gitblit.models.FederationSet;
+import com.gitblit.models.FilestoreModel;
import com.gitblit.models.ForkModel;
import com.gitblit.models.GitClientApplication;
import com.gitblit.models.Mailing;
@@ -132,6 +134,8 @@
protected final IFederationManager federationManager;
+ protected final IFilestoreManager filestoreManager;
+
@Inject
public GitblitManager(
Provider<IPublicKeyManager> publicKeyManagerProvider,
@@ -143,7 +147,8 @@
IAuthenticationManager authenticationManager,
IRepositoryManager repositoryManager,
IProjectManager projectManager,
- IFederationManager federationManager) {
+ IFederationManager federationManager,
+ IFilestoreManager filestoreManager) {
this.publicKeyManagerProvider = publicKeyManagerProvider;
this.ticketServiceProvider = ticketServiceProvider;
@@ -157,6 +162,7 @@
this.repositoryManager = repositoryManager;
this.projectManager = projectManager;
this.federationManager = federationManager;
+ this.filestoreManager = filestoreManager;
}
@Override
@@ -1239,6 +1245,70 @@
}
/*
+ * FILE STORAGE MANAGER
+ */
+
+ @Override
+ public boolean isValidOid(String oid) {
+ return filestoreManager.isValidOid(oid);
+ }
+
+ @Override
+ public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) {
+ return filestoreManager.addObject(oid, size, user, repo);
+ }
+
+ @Override
+ public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) {
+ return filestoreManager.getObject(oid, user, repo);
+ };
+
+ @Override
+ public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn ) {
+ return filestoreManager.uploadBlob(oid, size, user, repo, streamIn);
+ }
+
+ @Override
+ public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut ) {
+ return filestoreManager.downloadBlob(oid, user, repo, streamOut);
+ }
+
+ @Override
+ public List<FilestoreModel> getAllObjects() {
+ return filestoreManager.getAllObjects();
+ }
+
+ @Override
+ public File getStorageFolder() {
+ return filestoreManager.getStorageFolder();
+ }
+
+ @Override
+ public File getStoragePath(String oid) {
+ return filestoreManager.getStoragePath(oid);
+ }
+
+ @Override
+ public long getMaxUploadSize() {
+ return filestoreManager.getMaxUploadSize();
+ };
+
+ @Override
+ public void clearFilestoreCache() {
+ filestoreManager.clearFilestoreCache();
+ };
+
+ @Override
+ public long getFilestoreUsedByteCount() {
+ return filestoreManager.getFilestoreUsedByteCount();
+ };
+
+ @Override
+ public long getFilestoreAvailableByteCount() {
+ return filestoreManager.getFilestoreAvailableByteCount();
+ };
+
+ /*
* PLUGIN MANAGER
*/
@@ -1341,4 +1411,5 @@
public PluginRelease lookupRelease(String pluginId, String version) {
return pluginManager.lookupRelease(pluginId, version);
}
+
}
diff --git a/src/main/java/com/gitblit/manager/IFilestoreManager.java b/src/main/java/com/gitblit/manager/IFilestoreManager.java
new file mode 100644
index 0000000..0720650
--- /dev/null
+++ b/src/main/java/com/gitblit/manager/IFilestoreManager.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.manager;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+
+import com.gitblit.models.FilestoreModel;
+import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.UserModel;
+
+
+public interface IFilestoreManager extends IManager {
+
+ boolean isValidOid(String oid);
+
+ FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo);
+
+ FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo);
+
+ FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn );
+
+ FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut );
+
+ List<FilestoreModel> getAllObjects();
+
+ File getStorageFolder();
+
+ File getStoragePath(String oid);
+
+ long getMaxUploadSize();
+
+ void clearFilestoreCache();
+
+ long getFilestoreUsedByteCount();
+
+ long getFilestoreAvailableByteCount();
+
+}
diff --git a/src/main/java/com/gitblit/manager/IGitblit.java b/src/main/java/com/gitblit/manager/IGitblit.java
index 6c5b374..489de62 100644
--- a/src/main/java/com/gitblit/manager/IGitblit.java
+++ b/src/main/java/com/gitblit/manager/IGitblit.java
@@ -33,7 +33,8 @@
IAuthenticationManager,
IRepositoryManager,
IProjectManager,
- IFederationManager {
+ IFederationManager,
+ IFilestoreManager {
/**
* Creates a complete user object.
diff --git a/src/main/java/com/gitblit/models/FilestoreModel.java b/src/main/java/com/gitblit/models/FilestoreModel.java
new file mode 100644
index 0000000..ff7b210
--- /dev/null
+++ b/src/main/java/com/gitblit/models/FilestoreModel.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.models;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * A FilestoreModel represents a file stored outside a repository but referenced by the repository using a unique objectID
+ *
+ * @author Paul Martin
+ *
+ */
+public class FilestoreModel implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public final String oid;
+
+ private Long size;
+ private Status status;
+
+ //Audit
+ private String stateChangedBy;
+ private Date stateChangedOn;
+
+ //Access Control
+ private List<String> repositories;
+
+ public FilestoreModel(String id, long expectedSize, UserModel user, String repo) {
+ oid = id;
+ size = expectedSize;
+ status = Status.Upload_Pending;
+ stateChangedBy = user.getName();
+ stateChangedOn = new Date();
+ repositories = new ArrayList<String>();
+ repositories.add(repo);
+ }
+
+ public synchronized long getSize() {
+ return size;
+ }
+
+ public synchronized Status getStatus() {
+ return status;
+ }
+
+ public synchronized String getChangedBy() {
+ return stateChangedBy;
+ }
+
+ public synchronized Date getChangedOn() {
+ return stateChangedOn;
+ }
+
+ public synchronized void setStatus(Status status, UserModel user) {
+ this.status = status;
+ stateChangedBy = user.getName();
+ stateChangedOn = new Date();
+ }
+
+ public synchronized void reset(UserModel user, long size) {
+ status = Status.Upload_Pending;
+ stateChangedBy = user.getName();
+ stateChangedOn = new Date();
+ this.size = size;
+ }
+
+ /*
+ * Handles possible race condition with concurrent connections
+ * @return true if action can proceed, false otherwise
+ */
+ public synchronized boolean actionUpload(UserModel user) {
+ if (status == Status.Upload_Pending) {
+ status = Status.Upload_In_Progress;
+ stateChangedBy = user.getName();
+ stateChangedOn = new Date();
+ return true;
+ }
+
+ return false;
+ }
+
+ public synchronized boolean isInErrorState() {
+ return (this.status.value < 0);
+ }
+
+ public synchronized void addRepository(String repo) {
+ if (!repositories.contains(repo)) {
+ repositories.add(repo);
+ }
+ }
+
+ public synchronized void removeRepository(String repo) {
+ repositories.remove(repo);
+ }
+
+ public static enum Status {
+
+ Deleted(-30),
+ AuthenticationRequired(-20),
+
+ Error_Unknown(-8),
+ Error_Unexpected_Stream_End(-7),
+ Error_Invalid_Oid(-6),
+ Error_Invalid_Size(-5),
+ Error_Hash_Mismatch(-4),
+ Error_Size_Mismatch(-3),
+ Error_Exceeds_Size_Limit(-2),
+ Error_Unauthorized(-1),
+ //Negative values provide additional information and may be treated as 0 when not required
+ Unavailable(0),
+ Upload_Pending(1),
+ Upload_In_Progress(2),
+ Available(3);
+
+ final int value;
+
+ Status(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return name().toLowerCase().replace('_', ' ');
+ }
+
+ public static Status fromState(int state) {
+ for (Status s : values()) {
+ if (s.getValue() == state) {
+ return s;
+ }
+ }
+ throw new NoSuchElementException(String.valueOf(state));
+ }
+ }
+
+}
+
diff --git a/src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java b/src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java
index ee4a91a..bfbc089 100644
--- a/src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java
+++ b/src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java
@@ -17,6 +17,8 @@
import java.io.IOException;
import java.text.MessageFormat;
+import java.util.Collections;
+import java.util.Iterator;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
@@ -84,16 +86,17 @@
*
* @return true if the filter allows repository creation
*/
- protected abstract boolean isCreationAllowed();
+ protected abstract boolean isCreationAllowed(String action);
/**
* Determine if the action may be executed on the repository.
*
* @param repository
* @param action
+ * @param method
* @return true if the action may be performed
*/
- protected abstract boolean isActionAllowed(RepositoryModel repository, String action);
+ protected abstract boolean isActionAllowed(RepositoryModel repository, String action, String method);
/**
* Determine if the repository requires authentication.
@@ -102,7 +105,7 @@
* @param action
* @return true if authentication required
*/
- protected abstract boolean requiresAuthentication(RepositoryModel repository, String action);
+ protected abstract boolean requiresAuthentication(RepositoryModel repository, String action, String method);
/**
* Determine if the user can access the repository and perform the specified
@@ -126,7 +129,26 @@
protected RepositoryModel createRepository(UserModel user, String repository, String action) {
return null;
}
-
+
+ /**
+ * Allows authentication header to be altered based on the action requested
+ * Default is WWW-Authenticate
+ * @param action
+ * @return authentication type header
+ */
+ protected String getAuthenticationHeader(String action) {
+ return "WWW-Authenticate";
+ }
+
+ /**
+ * Allows request headers to be used as part of filtering
+ * @param request
+ * @return true (default) if headers are valid, false otherwise
+ */
+ protected boolean hasValidRequestHeader(String action, HttpServletRequest request) {
+ return true;
+ }
+
/**
* doFilter does the actual work of preprocessing the request to ensure that
* the user may proceed.
@@ -163,13 +185,14 @@
// Load the repository model
RepositoryModel model = repositoryManager.getRepositoryModel(repository);
if (model == null) {
- if (isCreationAllowed()) {
+ if (isCreationAllowed(urlRequestType)) {
if (user == null) {
// challenge client to provide credentials for creation. send 401.
if (runtimeManager.isDebugMode()) {
logger.info(MessageFormat.format("ARF: CREATE CHALLENGE {0}", fullUrl));
}
- httpResponse.setHeader("WWW-Authenticate", CHALLENGE);
+
+ httpResponse.setHeader(getAuthenticationHeader(urlRequestType), CHALLENGE);
httpResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED);
return;
} else {
@@ -188,7 +211,7 @@
}
// Confirm that the action may be executed on the repository
- if (!isActionAllowed(model, urlRequestType)) {
+ if (!isActionAllowed(model, urlRequestType, httpRequest.getMethod())) {
logger.info(MessageFormat.format("ARF: action {0} on {1} forbidden ({2})",
urlRequestType, model, HttpServletResponse.SC_FORBIDDEN));
httpResponse.sendError(HttpServletResponse.SC_FORBIDDEN);
@@ -210,13 +233,13 @@
}
// BASIC authentication challenge and response processing
- if (!StringUtils.isEmpty(urlRequestType) && requiresAuthentication(model, urlRequestType)) {
+ if (!StringUtils.isEmpty(urlRequestType) && requiresAuthentication(model, urlRequestType, httpRequest.getMethod())) {
if (user == null) {
// challenge client to provide credentials. send 401.
if (runtimeManager.isDebugMode()) {
logger.info(MessageFormat.format("ARF: CHALLENGE {0}", fullUrl));
}
- httpResponse.setHeader("WWW-Authenticate", CHALLENGE);
+ httpResponse.setHeader(getAuthenticationHeader(urlRequestType), CHALLENGE);
httpResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED);
return;
} else {
@@ -248,4 +271,17 @@
// pass processing to the restricted servlet.
chain.doFilter(authenticatedRequest, httpResponse);
}
+
+ public static boolean hasContentInRequestHeader(HttpServletRequest request, String headerName, String content)
+ {
+ Iterator<String> headerItr = Collections.list(request.getHeaders(headerName)).iterator();
+
+ while (headerItr.hasNext()) {
+ if (headerItr.next().contains(content)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
}
\ No newline at end of file
diff --git a/src/main/java/com/gitblit/servlet/DownloadZipFilter.java b/src/main/java/com/gitblit/servlet/DownloadZipFilter.java
index de47148..146f6d4 100644
--- a/src/main/java/com/gitblit/servlet/DownloadZipFilter.java
+++ b/src/main/java/com/gitblit/servlet/DownloadZipFilter.java
@@ -81,7 +81,7 @@
* @return true if the filter allows repository creation
*/
@Override
- protected boolean isCreationAllowed() {
+ protected boolean isCreationAllowed(String action) {
return false;
}
@@ -90,10 +90,11 @@
*
* @param repository
* @param action
+ * @param method
* @return true if the action may be performed
*/
@Override
- protected boolean isActionAllowed(RepositoryModel repository, String action) {
+ protected boolean isActionAllowed(RepositoryModel repository, String action, String method) {
return true;
}
@@ -102,10 +103,11 @@
*
* @param repository
* @param action
+ * @param method
* @return true if authentication required
*/
@Override
- protected boolean requiresAuthentication(RepositoryModel repository, String action) {
+ protected boolean requiresAuthentication(RepositoryModel repository, String action, String method) {
return repository.accessRestriction.atLeast(AccessRestrictionType.VIEW);
}
diff --git a/src/main/java/com/gitblit/servlet/FilestoreServlet.java b/src/main/java/com/gitblit/servlet/FilestoreServlet.java
new file mode 100644
index 0000000..1975148
--- /dev/null
+++ b/src/main/java/com/gitblit/servlet/FilestoreServlet.java
@@ -0,0 +1,493 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.servlet;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.Serializable;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.gitblit.Constants;
+import com.gitblit.IStoredSettings;
+import com.gitblit.models.FilestoreModel;
+import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.FilestoreModel.Status;
+import com.gitblit.manager.FilestoreManager;
+import com.gitblit.manager.IGitblit;
+import com.gitblit.models.UserModel;
+import com.gitblit.utils.JsonUtils;
+
+
+/**
+ * Handles large file storage as per the Git LFS v1 Batch API
+ *
+ * Further details can be found at https://github.com/github/git-lfs
+ *
+ * @author Paul Martin
+ */
+@Singleton
+public class FilestoreServlet extends HttpServlet {
+
+ private static final long serialVersionUID = 1L;
+ public static final int PROTOCOL_VERSION = 1;
+
+ public static final String GIT_LFS_META_MIME = "application/vnd.git-lfs+json";
+
+ public static final String REGEX_PATH = "^(.*?)/(r|git)/(.*?)/info/lfs/objects/(batch|" + Constants.REGEX_SHA256 + ")";
+ public static final int REGEX_GROUP_BASE_URI = 1;
+ public static final int REGEX_GROUP_PREFIX = 2;
+ public static final int REGEX_GROUP_REPOSITORY = 3;
+ public static final int REGEX_GROUP_ENDPOINT = 4;
+
+ protected final Logger logger;
+
+ private static IGitblit gitblit;
+
+ @Inject
+ public FilestoreServlet(IStoredSettings settings, IGitblit gitblit) {
+
+ super();
+ logger = LoggerFactory.getLogger(getClass());
+
+ FilestoreServlet.gitblit = gitblit;
+ }
+
+
+ /**
+ * Handles batch upload request (metadata)
+ *
+ * @param request
+ * @param response
+ * @throws javax.servlet.ServletException
+ * @throws java.io.IOException
+ */
+ @Override
+ protected void doPost(HttpServletRequest request,
+ HttpServletResponse response) throws ServletException ,IOException {
+
+ UrlInfo info = getInfoFromRequest(request);
+ if (info == null) {
+ sendError(response, HttpServletResponse.SC_NOT_FOUND);
+ return;
+ }
+
+ //Post is for batch operations so no oid should be defined
+ if (info.oid != null) {
+ sendError(response, HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+
+ IGitLFS.Batch batch = deserialize(request, response, IGitLFS.Batch.class);
+
+ if (batch == null) {
+ sendError(response, HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+
+ UserModel user = getUserOrAnonymous(request);
+
+ IGitLFS.BatchResponse batchResponse = new IGitLFS.BatchResponse();
+
+ if (batch.operation.equalsIgnoreCase("upload")) {
+ for (IGitLFS.Request item : batch.objects) {
+
+ Status state = gitblit.addObject(item.oid, item.size, user, info.repository);
+
+ batchResponse.objects.add(getResponseForUpload(info.baseUrl, item.oid, item.size, user.getName(), info.repository.name, state));
+ }
+ } else if (batch.operation.equalsIgnoreCase("download")) {
+ for (IGitLFS.Request item : batch.objects) {
+
+ Status state = gitblit.downloadBlob(item.oid, user, info.repository, null);
+ batchResponse.objects.add(getResponseForDownload(info.baseUrl, item.oid, item.size, user.getName(), info.repository.name, state));
+ }
+ } else {
+ sendError(response, HttpServletResponse.SC_NOT_IMPLEMENTED);
+ return;
+ }
+
+ response.setStatus(HttpServletResponse.SC_OK);
+ serialize(response, batchResponse);
+ }
+
+ /**
+ * Handles the actual upload (BLOB)
+ *
+ * @param request
+ * @param response
+ * @throws javax.servlet.ServletException
+ * @throws java.io.IOException
+ */
+ @Override
+ protected void doPut(HttpServletRequest request,
+ HttpServletResponse response) throws ServletException ,IOException {
+
+ UrlInfo info = getInfoFromRequest(request);
+
+ if (info == null) {
+ sendError(response, HttpServletResponse.SC_NOT_FOUND);
+ return;
+ }
+
+ //Put is a singular operation so must have oid
+ if (info.oid == null) {
+ sendError(response, HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+
+ UserModel user = getUserOrAnonymous(request);
+ long size = FilestoreManager.UNDEFINED_SIZE;
+
+
+
+ FilestoreModel.Status status = gitblit.uploadBlob(info.oid, size, user, info.repository, request.getInputStream());
+ IGitLFS.Response responseObject = getResponseForUpload(info.baseUrl, info.oid, size, user.getName(), info.repository.name, status);
+
+ logger.info(MessageFormat.format("FILESTORE-AUDIT {0}:{4} {1} {2}@{3}",
+ "PUT", info.oid, user.getName(), info.repository.name, status.toString() ));
+
+ if (responseObject.error == null) {
+ response.setStatus(responseObject.successCode);
+ } else {
+ serialize(response, responseObject.error);
+ }
+ };
+
+ /**
+ * Handles a download
+ * Treated as hypermedia request if accept header contains Git-LFS MIME
+ * otherwise treated as a download of the blob
+ * @param request
+ * @param response
+ * @throws javax.servlet.ServletException
+ * @throws java.io.IOException
+ */
+ @Override
+ protected void doGet(HttpServletRequest request,
+ HttpServletResponse response) throws ServletException ,IOException {
+
+ UrlInfo info = getInfoFromRequest(request);
+
+ if (info == null || info.oid == null) {
+ sendError(response, HttpServletResponse.SC_NOT_FOUND);
+ return;
+ }
+
+ UserModel user = getUserOrAnonymous(request);
+
+ FilestoreModel model = gitblit.getObject(info.oid, user, info.repository);
+ long size = FilestoreManager.UNDEFINED_SIZE;
+
+ boolean isMetaRequest = AccessRestrictionFilter.hasContentInRequestHeader(request, "Accept", GIT_LFS_META_MIME);
+ FilestoreModel.Status status = Status.Unavailable;
+
+ if (model != null) {
+ size = model.getSize();
+ status = model.getStatus();
+ }
+
+ if (!isMetaRequest) {
+ status = gitblit.downloadBlob(info.oid, user, info.repository, response.getOutputStream());
+
+ logger.info(MessageFormat.format("FILESTORE-AUDIT {0}:{4} {1} {2}@{3}",
+ "GET", info.oid, user.getName(), info.repository.name, status.toString() ));
+ }
+
+ if (status == Status.Error_Unexpected_Stream_End) {
+ return;
+ }
+
+ IGitLFS.Response responseObject = getResponseForDownload(info.baseUrl,
+ info.oid, size, user.getName(), info.repository.name, status);
+
+ if (responseObject.error == null) {
+ response.setStatus(responseObject.successCode);
+
+ if (isMetaRequest) {
+ serialize(response, responseObject);
+ }
+ } else {
+ response.setStatus(responseObject.error.code);
+ serialize(response, responseObject.error);
+ }
+ };
+
+ private void sendError(HttpServletResponse response, int code) throws IOException {
+
+ String msg = "";
+
+ switch (code)
+ {
+ case HttpServletResponse.SC_NOT_FOUND: msg = "Not Found"; break;
+ case HttpServletResponse.SC_NOT_IMPLEMENTED: msg = "Not Implemented"; break;
+ case HttpServletResponse.SC_BAD_REQUEST: msg = "Malformed Git-LFS request"; break;
+
+ default: msg = "Unknown Error";
+ }
+
+ response.setStatus(code);
+ serialize(response, new IGitLFS.ObjectError(code, msg));
+ }
+
+ @SuppressWarnings("incomplete-switch")
+ private IGitLFS.Response getResponseForUpload(String baseUrl, String oid, long size, String user, String repo, FilestoreModel.Status state) {
+
+ switch (state) {
+ case AuthenticationRequired:
+ return new IGitLFS.Response(oid, size, 401, MessageFormat.format("Authentication required to write to repository {0}", repo));
+ case Error_Unauthorized:
+ return new IGitLFS.Response(oid, size, 403, MessageFormat.format("User {0}, does not have write permissions to repository {1}", user, repo));
+ case Error_Exceeds_Size_Limit:
+ return new IGitLFS.Response(oid, size, 509, MessageFormat.format("Object is larger than allowed limit of {1}", gitblit.getMaxUploadSize()));
+ case Error_Hash_Mismatch:
+ return new IGitLFS.Response(oid, size, 422, "Hash mismatch");
+ case Error_Invalid_Oid:
+ return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid oid", oid));
+ case Error_Invalid_Size:
+ return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid size", size));
+ case Error_Size_Mismatch:
+ return new IGitLFS.Response(oid, size, 422, "Object size mismatch");
+ case Deleted:
+ return new IGitLFS.Response(oid, size, 410, "Object was deleted : ".concat("TBD Reason") );
+ case Upload_In_Progress:
+ return new IGitLFS.Response(oid, size, 503, "File currently being uploaded by another user");
+ case Unavailable:
+ return new IGitLFS.Response(oid, size, 404, MessageFormat.format("Repository {0}, does not exist for user {1}", repo, user));
+ case Upload_Pending:
+ return new IGitLFS.Response(oid, size, 202, "upload", getObjectUri(baseUrl, repo, oid) );
+ case Available:
+ return new IGitLFS.Response(oid, size, 200, "upload", getObjectUri(baseUrl, repo, oid) );
+ }
+
+ return new IGitLFS.Response(oid, size, 500, "Unknown Error");
+ }
+
+ @SuppressWarnings("incomplete-switch")
+ private IGitLFS.Response getResponseForDownload(String baseUrl, String oid, long size, String user, String repo, FilestoreModel.Status state) {
+
+ switch (state) {
+ case Error_Unauthorized:
+ return new IGitLFS.Response(oid, size, 403, MessageFormat.format("User {0}, does not have read permissions to repository {1}", user, repo));
+ case Error_Invalid_Oid:
+ return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid oid", oid));
+ case Error_Unknown:
+ return new IGitLFS.Response(oid, size, 500, "Unknown Error");
+ case Deleted:
+ return new IGitLFS.Response(oid, size, 410, "Object was deleted : ".concat("TBD Reason") );
+ case Available:
+ return new IGitLFS.Response(oid, size, 200, "download", getObjectUri(baseUrl, repo, oid) );
+ }
+
+ return new IGitLFS.Response(oid, size, 404, "Object not available");
+ }
+
+
+ private String getObjectUri(String baseUrl, String repo, String oid) {
+ return baseUrl + "/" + repo + "/" + Constants.R_LFS + "objects/" + oid;
+ }
+
+
+ protected void serialize(HttpServletResponse response, Object o) throws IOException {
+ if (o != null) {
+ // Send JSON response
+ String json = JsonUtils.toJsonString(o);
+ response.setCharacterEncoding(Constants.ENCODING);
+ response.setContentType(GIT_LFS_META_MIME);
+ response.getWriter().append(json);
+ }
+ }
+
+ protected <X> X deserialize(HttpServletRequest request, HttpServletResponse response,
+ Class<X> clazz) {
+
+ String json = "";
+ try {
+
+ json = readJson(request, response);
+
+ return JsonUtils.fromJsonString(json.toString(), clazz);
+
+ } catch (Exception e) {
+ //Intentional silent fail
+ }
+
+ return null;
+ }
+
+ private String readJson(HttpServletRequest request, HttpServletResponse response)
+ throws IOException {
+ BufferedReader reader = request.getReader();
+ StringBuilder json = new StringBuilder();
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ json.append(line);
+ }
+ reader.close();
+
+ if (json.length() == 0) {
+ logger.error(MessageFormat.format("Failed to receive json data from {0}",
+ request.getRemoteAddr()));
+ response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ return null;
+ }
+ return json.toString();
+ }
+
+ private UserModel getUserOrAnonymous(HttpServletRequest r) {
+ UserModel user = (UserModel) r.getUserPrincipal();
+ if (user != null) { return user; }
+ return UserModel.ANONYMOUS;
+ }
+
+ private static class UrlInfo {
+ public RepositoryModel repository;
+ public String oid;
+ public String baseUrl;
+
+ public UrlInfo(RepositoryModel repo, String oid, String baseUrl) {
+ this.repository = repo;
+ this.oid = oid;
+ this.baseUrl = baseUrl;
+ }
+ }
+
+ public static UrlInfo getInfoFromRequest(HttpServletRequest httpRequest) {
+
+ String url = httpRequest.getRequestURL().toString();
+ Pattern p = Pattern.compile(REGEX_PATH);
+ Matcher m = p.matcher(url);
+
+
+ if (m.find()) {
+ RepositoryModel repo = gitblit.getRepositoryModel(m.group(REGEX_GROUP_REPOSITORY));
+ String baseUrl = m.group(REGEX_GROUP_BASE_URI) + "/" + m.group(REGEX_GROUP_PREFIX);
+
+ if (m.group(REGEX_GROUP_ENDPOINT).equals("batch")) {
+ return new UrlInfo(repo, null, baseUrl);
+ } else {
+ return new UrlInfo(repo, m.group(REGEX_GROUP_ENDPOINT), baseUrl);
+ }
+ }
+
+ return null;
+ }
+
+
+ public interface IGitLFS {
+
+ @SuppressWarnings("serial")
+ public class Request implements Serializable
+ {
+ public String oid;
+ public long size;
+ }
+
+
+ @SuppressWarnings("serial")
+ public class Batch implements Serializable
+ {
+ public String operation;
+ public List<Request> objects;
+ }
+
+
+ @SuppressWarnings("serial")
+ public class Response implements Serializable
+ {
+ public String oid;
+ public long size;
+ public Map<String, HyperMediaLink> actions;
+ public ObjectError error;
+ public transient int successCode;
+
+ public Response(String id, long itemSize, int errorCode, String errorText) {
+ oid = id;
+ size = itemSize;
+ actions = null;
+ successCode = 0;
+ error = new ObjectError(errorCode, errorText);
+ }
+
+ public Response(String id, long itemSize, int actionCode, String action, String uri) {
+ oid = id;
+ size = itemSize;
+ error = null;
+ successCode = actionCode;
+ actions = new HashMap<String, HyperMediaLink>();
+ actions.put(action, new HyperMediaLink(action, uri));
+ }
+
+ }
+
+ @SuppressWarnings("serial")
+ public class BatchResponse implements Serializable {
+ public List<Response> objects;
+
+ public BatchResponse() {
+ objects = new ArrayList<Response>();
+ }
+ }
+
+
+ @SuppressWarnings("serial")
+ public class ObjectError implements Serializable
+ {
+ public String message;
+ public int code;
+ public String documentation_url;
+ public Integer request_id;
+
+ public ObjectError(int errorCode, String errorText) {
+ code = errorCode;
+ message = errorText;
+ request_id = null;
+ }
+ }
+
+ @SuppressWarnings("serial")
+ public class HyperMediaLink implements Serializable
+ {
+ public String href;
+ public transient String header;
+ //public Date expires_at;
+
+ public HyperMediaLink(String action, String uri) {
+ header = action;
+ href = uri;
+ }
+ }
+ }
+
+
+
+}
diff --git a/src/main/java/com/gitblit/servlet/GitFilter.java b/src/main/java/com/gitblit/servlet/GitFilter.java
index b29fdb6..27408f0 100644
--- a/src/main/java/com/gitblit/servlet/GitFilter.java
+++ b/src/main/java/com/gitblit/servlet/GitFilter.java
@@ -19,6 +19,7 @@
import com.google.inject.Inject;
import com.google.inject.Singleton;
+
import javax.servlet.http.HttpServletRequest;
import com.gitblit.Constants.AccessRestrictionType;
@@ -48,9 +49,11 @@
protected static final String gitReceivePack = "/git-receive-pack";
protected static final String gitUploadPack = "/git-upload-pack";
-
+
+ protected static final String gitLfs = "/info/lfs";
+
protected static final String[] suffixes = { gitReceivePack, gitUploadPack, "/info/refs", "/HEAD",
- "/objects" };
+ "/objects", gitLfs };
private IStoredSettings settings;
@@ -116,6 +119,8 @@
return gitReceivePack;
} else if (suffix.contains("?service=git-upload-pack")) {
return gitUploadPack;
+ } else if (suffix.startsWith(gitLfs)) {
+ return gitLfs;
} else {
return gitUploadPack;
}
@@ -144,7 +149,13 @@
* @return true if the server allows repository creation on-push
*/
@Override
- protected boolean isCreationAllowed() {
+ protected boolean isCreationAllowed(String action) {
+
+ //Repository must already exist before large files can be deposited
+ if (action.equals(gitLfs)) {
+ return false;
+ }
+
return settings.getBoolean(Keys.git.allowCreateOnPush, true);
}
@@ -156,9 +167,15 @@
* @return true if the action may be performed
*/
@Override
- protected boolean isActionAllowed(RepositoryModel repository, String action) {
+ protected boolean isActionAllowed(RepositoryModel repository, String action, String method) {
// the log here has been moved into ReceiveHook to provide clients with
// error messages
+ if (gitLfs.equals(action)) {
+ if (!method.matches("GET|POST|PUT|HEAD")) {
+ return false;
+ }
+ }
+
return true;
}
@@ -172,16 +189,25 @@
*
* @param repository
* @param action
+ * @param method
* @return true if authentication required
*/
@Override
- protected boolean requiresAuthentication(RepositoryModel repository, String action) {
+ protected boolean requiresAuthentication(RepositoryModel repository, String action, String method) {
if (gitUploadPack.equals(action)) {
// send to client
return repository.accessRestriction.atLeast(AccessRestrictionType.CLONE);
} else if (gitReceivePack.equals(action)) {
// receive from client
return repository.accessRestriction.atLeast(AccessRestrictionType.PUSH);
+ } else if (gitLfs.equals(action)) {
+
+ if (method.matches("GET|HEAD")) {
+ return repository.accessRestriction.atLeast(AccessRestrictionType.CLONE);
+ } else {
+ //NOTE: Treat POST as PUT as as without reading message type cannot determine
+ return repository.accessRestriction.atLeast(AccessRestrictionType.PUSH);
+ }
}
return false;
}
@@ -230,6 +256,12 @@
@Override
protected RepositoryModel createRepository(UserModel user, String repository, String action) {
boolean isPush = !StringUtils.isEmpty(action) && gitReceivePack.equals(action);
+
+ if (action.equals(gitLfs)) {
+ //Repository must already exist for any filestore actions
+ return null;
+ }
+
if (isPush) {
if (user.canCreate(repository)) {
// user is pushing to a new repository
@@ -281,4 +313,40 @@
// repository could not be created or action was not a push
return null;
}
+
+ /**
+ * Git lfs action uses an alternative authentication header,
+ *
+ * @param action
+ * @return
+ */
+ @Override
+ protected String getAuthenticationHeader(String action) {
+
+ if (action.equals(gitLfs)) {
+ return "LFS-Authenticate";
+ }
+
+ return super.getAuthenticationHeader(action);
+ }
+
+ /**
+ * Interrogates the request headers based on the action
+ * @param action
+ * @param request
+ * @return
+ */
+ @Override
+ protected boolean hasValidRequestHeader(String action,
+ HttpServletRequest request) {
+
+ if (action.equals(gitLfs) && request.getMethod().equals("POST")) {
+ if ( !hasContentInRequestHeader(request, "Accept", FilestoreServlet.GIT_LFS_META_MIME)
+ || !hasContentInRequestHeader(request, "Content-Type", FilestoreServlet.GIT_LFS_META_MIME)) {
+ return false;
+ }
+ }
+
+ return super.hasValidRequestHeader(action, request);
+ }
}
diff --git a/src/main/java/com/gitblit/servlet/GitblitContext.java b/src/main/java/com/gitblit/servlet/GitblitContext.java
index 077624c..fb8f6b9 100644
--- a/src/main/java/com/gitblit/servlet/GitblitContext.java
+++ b/src/main/java/com/gitblit/servlet/GitblitContext.java
@@ -44,6 +44,7 @@
import com.gitblit.guice.WebModule;
import com.gitblit.manager.IAuthenticationManager;
import com.gitblit.manager.IFederationManager;
+import com.gitblit.manager.IFilestoreManager;
import com.gitblit.manager.IGitblit;
import com.gitblit.manager.IManager;
import com.gitblit.manager.INotificationManager;
@@ -204,6 +205,7 @@
startManager(injector, ITicketService.class);
startManager(injector, IGitblit.class);
startManager(injector, IServicesManager.class);
+ startManager(injector, IFilestoreManager.class);
// start the plugin manager last so that plugins can depend on
// deterministic access to all other managers in their start() methods
diff --git a/src/main/java/com/gitblit/servlet/RawFilter.java b/src/main/java/com/gitblit/servlet/RawFilter.java
index fe4af04..8913a19 100644
--- a/src/main/java/com/gitblit/servlet/RawFilter.java
+++ b/src/main/java/com/gitblit/servlet/RawFilter.java
@@ -98,7 +98,7 @@
* @return true if the filter allows repository creation
*/
@Override
- protected boolean isCreationAllowed() {
+ protected boolean isCreationAllowed(String action) {
return false;
}
@@ -107,10 +107,11 @@
*
* @param repository
* @param action
+ * @param method
* @return true if the action may be performed
*/
@Override
- protected boolean isActionAllowed(RepositoryModel repository, String action) {
+ protected boolean isActionAllowed(RepositoryModel repository, String action, String method) {
return true;
}
@@ -119,10 +120,11 @@
*
* @param repository
* @param action
+ * @param method
* @return true if authentication required
*/
@Override
- protected boolean requiresAuthentication(RepositoryModel repository, String action) {
+ protected boolean requiresAuthentication(RepositoryModel repository, String action, String method) {
return repository.accessRestriction.atLeast(AccessRestrictionType.VIEW);
}
diff --git a/src/main/java/com/gitblit/utils/JsonUtils.java b/src/main/java/com/gitblit/utils/JsonUtils.java
index be7148c..f389776 100644
--- a/src/main/java/com/gitblit/utils/JsonUtils.java
+++ b/src/main/java/com/gitblit/utils/JsonUtils.java
@@ -46,6 +46,7 @@
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
@@ -79,23 +80,29 @@
/**
* Convert a json string to an object of the specified type.
- *
+ *
* @param json
* @param clazz
- * @return an object
+ * @return the deserialized object
+ * @throws JsonParseException
+ * @throws JsonSyntaxException
*/
- public static <X> X fromJsonString(String json, Class<X> clazz) {
+ public static <X> X fromJsonString(String json, Class<X> clazz) throws JsonParseException,
+ JsonSyntaxException {
return gson().fromJson(json, clazz);
}
/**
* Convert a json string to an object of the specified type.
- *
+ *
* @param json
- * @param clazz
- * @return an object
+ * @param type
+ * @return the deserialized object
+ * @throws JsonParseException
+ * @throws JsonSyntaxException
*/
- public static <X> X fromJsonString(String json, Type type) {
+ public static <X> X fromJsonString(String json, Type type) throws JsonParseException,
+ JsonSyntaxException {
return gson().fromJson(json, type);
}
diff --git a/src/main/java/com/gitblit/wicket/FilestoreUI.java b/src/main/java/com/gitblit/wicket/FilestoreUI.java
new file mode 100644
index 0000000..8837ba1
--- /dev/null
+++ b/src/main/java/com/gitblit/wicket/FilestoreUI.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.wicket;
+
+import org.apache.wicket.markup.html.basic.Label;
+
+import com.gitblit.models.FilestoreModel;
+import com.gitblit.models.FilestoreModel.Status;
+
+/**
+ * Common filestore ui methods and classes.
+ *
+ * @author Paul Martin
+ *
+ */
+public class FilestoreUI {
+
+ public static Label getStatusIcon(String wicketId, FilestoreModel item) {
+ return getStatusIcon(wicketId, item.getStatus());
+ }
+
+ public static Label getStatusIcon(String wicketId, Status status) {
+ Label label = new Label(wicketId);
+
+ switch (status) {
+ case Upload_Pending:
+ WicketUtils.setCssClass(label, "fa fa-spinner fa-fw file-negative");
+ break;
+ case Upload_In_Progress:
+ WicketUtils.setCssClass(label, "fa fa-spinner fa-spin fa-fw file-positive");
+ break;
+ case Available:
+ WicketUtils.setCssClass(label, "fa fa-check fa-fw file-positive");
+ break;
+ case Deleted:
+ WicketUtils.setCssClass(label, "fa fa-ban fa-fw file-negative");
+ break;
+ case Unavailable:
+ WicketUtils.setCssClass(label, "fa fa-times fa-fw file-negative");
+ break;
+ default:
+ WicketUtils.setCssClass(label, "fa fa-exclamation-triangle fa-fw file-negative");
+ }
+ WicketUtils.setHtmlTooltip(label, status.toString());
+
+ return label;
+ }
+
+}
diff --git a/src/main/java/com/gitblit/wicket/GitBlitWebApp.java b/src/main/java/com/gitblit/wicket/GitBlitWebApp.java
index 359040b..296c254 100644
--- a/src/main/java/com/gitblit/wicket/GitBlitWebApp.java
+++ b/src/main/java/com/gitblit/wicket/GitBlitWebApp.java
@@ -37,6 +37,7 @@
import com.gitblit.extensions.GitblitWicketPlugin;
import com.gitblit.manager.IAuthenticationManager;
import com.gitblit.manager.IFederationManager;
+import com.gitblit.manager.IFilestoreManager;
import com.gitblit.manager.IGitblit;
import com.gitblit.manager.INotificationManager;
import com.gitblit.manager.IPluginManager;
@@ -63,6 +64,7 @@
import com.gitblit.wicket.pages.EditTicketPage;
import com.gitblit.wicket.pages.ExportTicketPage;
import com.gitblit.wicket.pages.FederationRegistrationPage;
+import com.gitblit.wicket.pages.FilestorePage;
import com.gitblit.wicket.pages.ForkPage;
import com.gitblit.wicket.pages.ForksPage;
import com.gitblit.wicket.pages.GitSearchPage;
@@ -131,6 +133,8 @@
private final IGitblit gitblit;
private final IServicesManager services;
+
+ private final IFilestoreManager filestoreManager;
@Inject
public GitBlitWebApp(
@@ -145,7 +149,8 @@
IProjectManager projectManager,
IFederationManager federationManager,
IGitblit gitblit,
- IServicesManager services) {
+ IServicesManager services,
+ IFilestoreManager filestoreManager) {
super();
this.publicKeyManagerProvider = publicKeyManagerProvider;
@@ -162,6 +167,7 @@
this.federationManager = federationManager;
this.gitblit = gitblit;
this.services = services;
+ this.filestoreManager = filestoreManager;
}
@Override
@@ -238,6 +244,9 @@
mount("/user", UserPage.class, "user");
mount("/forks", ForksPage.class, "r");
mount("/fork", ForkPage.class, "r");
+
+ // filestore URL
+ mount("/filestore", FilestorePage.class);
// allow started Wicket plugins to initialize
for (PluginWrapper pluginWrapper : pluginManager.getPlugins()) {
@@ -476,4 +485,9 @@
public static GitBlitWebApp get() {
return (GitBlitWebApp) WebApplication.get();
}
+
+ @Override
+ public IFilestoreManager filestore() {
+ return filestoreManager;
+ }
}
diff --git a/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties b/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties
index d802754..36c416e 100644
--- a/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties
+++ b/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties
@@ -764,4 +764,10 @@
gb.deleteRepositoryDescription = Deleted repositories will be unrecoverable.
gb.show_whitespace = show whitespace
gb.ignore_whitespace = ignore whitespace
-gb.allRepositories = All Repositories
\ No newline at end of file
+gb.allRepositories = All Repositories
+gb.oid = object id
+gb.filestore = filestore
+gb.filestoreStats = Filestore contains {0} files with a total size of {1}. ({2} remaining)
+gb.statusChangedOn = status changed on
+gb.statusChangedBy = status changed by
+gb.filestoreHelp = How to use the Filestore?
\ No newline at end of file
diff --git a/src/main/java/com/gitblit/wicket/GitblitWicketApp.java b/src/main/java/com/gitblit/wicket/GitblitWicketApp.java
index 3041c5d..fefa0f4 100644
--- a/src/main/java/com/gitblit/wicket/GitblitWicketApp.java
+++ b/src/main/java/com/gitblit/wicket/GitblitWicketApp.java
@@ -8,6 +8,7 @@
import com.gitblit.IStoredSettings;
import com.gitblit.manager.IAuthenticationManager;
import com.gitblit.manager.IFederationManager;
+import com.gitblit.manager.IFilestoreManager;
import com.gitblit.manager.IGitblit;
import com.gitblit.manager.INotificationManager;
import com.gitblit.manager.IPluginManager;
@@ -74,5 +75,7 @@
public abstract ITicketService tickets();
public abstract TimeZone getTimezone();
+
+ public abstract IFilestoreManager filestore();
}
\ No newline at end of file
diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.html b/src/main/java/com/gitblit/wicket/pages/FilestorePage.html
new file mode 100644
index 0000000..e373e70
--- /dev/null
+++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.html
@@ -0,0 +1,37 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml"
+ xmlns:wicket="http://wicket.apache.org/dtds.data/wicket-xhtml1.3-strict.dtd"
+ xml:lang="en"
+ lang="en">
+
+<body>
+<wicket:extend>
+<div class="container">
+
+ <div class="markdown" style="padding: 10px 0px 5px 0px;">
+ <span wicket:id="repositoriesMessage">[repositories message]</span>
+ <span style="float:right"><a href="#" wicket:id="filestoreHelp"><span wicket:id="helpMessage">[help message]</span></a></span>
+ </div>
+
+ <table class="repositories">
+ <tr>
+ <th><wicket:message key="gb.status">[Object status]</wicket:message></th>
+ <th><wicket:message key="gb.statusChangedOn">[changedOn]</wicket:message></th>
+ <th><wicket:message key="gb.statusChangedBy">[changedBy]</wicket:message></th>
+ <th><wicket:message key="gb.oid">[Object ID]</wicket:message></th>
+ <th><wicket:message key="gb.size">[file size]</wicket:message></th>
+ </tr>
+ <tbody>
+ <tr wicket:id="fileRow">
+ <td><center><span class="list" wicket:id="status">[Object state]</span></center></td>
+ <td><span class="list" wicket:id="on">[changedOn]</span></td>
+ <td><span class="list" wicket:id="by">[changedBy]</span></td>
+ <td class="sha256"><span class="list" wicket:id="oid">[Object ID]</span></td>
+ <td><span class="list" wicket:id="size">[file size]</span></td>
+ </tr>
+ </tbody>
+ </table>
+</div>
+</wicket:extend>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java
new file mode 100644
index 0000000..5f103ed
--- /dev/null
+++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.wicket.pages;
+
+import java.text.DateFormat;
+import java.text.MessageFormat;
+import java.text.SimpleDateFormat;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.wicket.Component;
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.link.BookmarkablePageLink;
+import org.apache.wicket.markup.repeater.Item;
+import org.apache.wicket.markup.repeater.data.DataView;
+import org.apache.wicket.markup.repeater.data.ListDataProvider;
+
+import com.gitblit.Constants;
+import com.gitblit.Keys;
+import com.gitblit.models.FilestoreModel;
+import com.gitblit.models.UserModel;
+import com.gitblit.wicket.FilestoreUI;
+import com.gitblit.wicket.GitBlitWebSession;
+import com.gitblit.wicket.WicketUtils;
+
+/**
+ * Page to display the current status of the filestore.
+ * Certain errors also displayed to aid in fault finding
+ *
+ * @author Paul Martin
+ *
+ *
+ */
+public class FilestorePage extends RootPage {
+
+ public FilestorePage() {
+ super();
+ setupPage("", "");
+ // check to see if we should display a login message
+ boolean authenticateView = app().settings().getBoolean(Keys.web.authenticateViewPages, true);
+ if (authenticateView && !GitBlitWebSession.get().isLoggedIn()) {
+ String messageSource = app().settings().getString(Keys.web.loginMessage, "gitblit");
+ return;
+ }
+
+ final List<FilestoreModel> files = app().filestore().getAllObjects();
+ final long nBytesUsed = app().filestore().getFilestoreUsedByteCount();
+ final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount();
+
+ // Load the markdown welcome message
+ String messageSource = app().settings().getString(Keys.web.repositoriesMessage, "gitblit");
+ String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(),
+ FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) );
+
+ Component repositoriesMessage = new Label("repositoriesMessage", message)
+ .setEscapeModelStrings(false).setVisible(message.length() > 0);
+
+ add(repositoriesMessage);
+
+ BookmarkablePageLink<Void> helpLink = new BookmarkablePageLink<Void>("filestoreHelp", FilestoreUsage.class);
+ helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp")));
+ add(helpLink);
+
+
+ DataView<FilestoreModel> filesView = new DataView<FilestoreModel>("fileRow",
+ new ListDataProvider<FilestoreModel>(files)) {
+ private static final long serialVersionUID = 1L;
+ private int counter;
+
+ @Override
+ protected void onBeforeRender() {
+ super.onBeforeRender();
+ counter = 0;
+ }
+
+ @Override
+ public void populateItem(final Item<FilestoreModel> item) {
+ final FilestoreModel entry = item.getModelObject();
+
+ DateFormat dateFormater = new SimpleDateFormat(Constants.ISO8601);
+
+ UserModel user = app().users().getUserModel(entry.getChangedBy());
+ user = user == null ? UserModel.ANONYMOUS : user;
+
+ Label icon = FilestoreUI.getStatusIcon("status", entry);
+ item.add(icon);
+ item.add(new Label("on", dateFormater.format(entry.getChangedOn())));
+ item.add(new Label("by", user.getDisplayName()));
+
+ item.add(new Label("oid", entry.oid));
+ item.add(new Label("size", FileUtils.byteCountToDisplaySize(entry.getSize())));
+
+ WicketUtils.setAlternatingBackground(item, counter);
+ counter++;
+ }
+
+ };
+
+ add(filesView);
+ }
+}
diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html
new file mode 100644
index 0000000..e9bff47
--- /dev/null
+++ b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html
@@ -0,0 +1,69 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml"
+ xmlns:wicket="http://wicket.apache.org/dtds.data/wicket-xhtml1.3-strict.dtd"
+ xml:lang="en"
+ lang="en">
+
+<body>
+<wicket:extend>
+<div class="container">
+<div class="markdown">
+<div class="row">
+<div class="span10 offset1">
+
+ <div class="alert alert-danger">
+ <h3><center>Using the Filestore</center></h3>
+ <p>
+ <strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs init</code> to register the hooks globally.</strong><br/>
+ <i>This version of GitBlit has been verified with Git-LFS client version 0.6.0 which requires Git v1.8.2 or higher.</i>
+ </p>
+ </div>
+
+ <h3>Clone</h3>
+ <p>
+ Just <code>git clone</code> as usual, no further action is required as GitBlit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/>
+ <i>If the repository uses a 3rd party Git-LFS server you will need to <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md#the-server">manually configure the correct endpoints</a></i>.
+ </p>
+
+ <h3>Add</h3>
+ <p>After configuring the file types or paths to be tracked using <code>git lfs track "*.bin"</code> just add files as usual with <code>git add</code> command.<br/>
+ <i>Tracked files can also be configured manually using the <code>.gitattributes</code> file</i>.</p>
+
+ <h3>Remove</h3>
+ <p>When you remove a Git-LFS tracked file only the pointer file will be removed from your repository.<br/>
+ <i>All files remain on the server to allow previous versions to be checked out.</i>
+ </p>
+
+ <h3>Learn more...</h3>
+ <p><a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">See the current Git-LFS specification for further details</a>.</p>
+ <br />
+
+ <div class="alert alert-warn">
+ <h3><center>Limitations & Warnings</center></h3>
+ <p>GitBlit currently provides a server-only implementation of the opensource Git-LFS API, <a href="https://github.com/github/git-lfs/wiki/Implementations">other implementations</a> are available.<br/>
+ However, until <a href="https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333">JGit provides Git-LFS client capabilities</a> some GitBlit features may not be fully supported when using the filestore.
+ Notably:
+ <ul>
+ <li>Mirroring a repository that uses Git-LFS - Only the pointer files, not the large files, are mirrored.</li>
+ <li>Federation - Only the pointer files, not the large files, are transfered.</li>
+ </ul>
+ </p>
+ </div>
+
+ <div class="alert alert-info">
+ <h3><center>GitBlit Configuration</center></h3>
+ <p>GitBlit provides the following configuration items when using the filestore:
+ <h4>filestore.storageFolder</h4>
+ <p>Defines the path on the server where filestore objects are to be saved. This defaults to <code>${baseFolder}/lfs</code></p>
+ <h4>filestore.maxUploadSize</h4>
+ <p>Defines the maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. This defaults to <code>-1</code> indicating no limits.</p>
+ </p>
+ </div>
+
+</div>
+</div>
+</div>
+</div>
+</wicket:extend>
+</body>
+</html>
diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java
new file mode 100644
index 0000000..9bd8e55
--- /dev/null
+++ b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.wicket.pages;
+
+public class FilestoreUsage extends RootSubPage {
+
+ public FilestoreUsage() {
+ super();
+ setupPage("", "");
+ }
+
+}
diff --git a/src/main/java/com/gitblit/wicket/pages/RootPage.java b/src/main/java/com/gitblit/wicket/pages/RootPage.java
index 79a4fc6..93d44fc 100644
--- a/src/main/java/com/gitblit/wicket/pages/RootPage.java
+++ b/src/main/java/com/gitblit/wicket/pages/RootPage.java
@@ -191,6 +191,7 @@
}
navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class,
getRootPageParameters()));
+ navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters()));
navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters()));
if (allowLucene) {
navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class));
diff --git a/src/main/resources/gitblit.css b/src/main/resources/gitblit.css
index c032905..0cc8fd0 100644
--- a/src/main/resources/gitblit.css
+++ b/src/main/resources/gitblit.css
@@ -1729,6 +1729,12 @@
}
}
+td.sha256 {
+ max-width: 20em;
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
table.comments td {
padding: 4px;
line-height: 17px;
@@ -1922,7 +1928,7 @@
white-space: nowrap;
}
-span.sha1, span.sha1 a, span.sha1 a span, .commit_message, span.shortsha1, td.sha1 {
+span.sha1, span.sha1 a, span.sha1 a span, .commit_message, span.shortsha1, td.sha1, td.sha256 {
font-family: consolas, monospace;
font-size: 13px;
}
@@ -2340,3 +2346,11 @@
.priority-low {
color:#0072B2;
}
+
+.file-positive {
+ color:#009E73;
+}
+
+.file-negative {
+ color:#D51900;
+}
diff --git a/src/test/java/com/gitblit/tests/FilestoreManagerTest.java b/src/test/java/com/gitblit/tests/FilestoreManagerTest.java
new file mode 100644
index 0000000..c76e9dd
--- /dev/null
+++ b/src/test/java/com/gitblit/tests/FilestoreManagerTest.java
@@ -0,0 +1,547 @@
+package com.gitblit.tests;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.util.Date;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.gitblit.Constants.AccessPermission;
+import com.gitblit.Constants.AccessRestrictionType;
+import com.gitblit.Constants.AuthorizationControl;
+import com.gitblit.Keys;
+import com.gitblit.models.FilestoreModel.Status;
+import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.UserModel;
+import com.gitblit.utils.FileUtils;
+
+
+/**
+ * Test of the filestore manager and confirming filesystem updated
+ *
+ * @author Paul Martin
+ *
+ */
+public class FilestoreManagerTest extends GitblitUnitTest {
+
+ private static final AtomicBoolean started = new AtomicBoolean(false);
+
+ private static final BlobInfo blob_zero = new BlobInfo(0);
+ private static final BlobInfo blob_512KB = new BlobInfo(512*FileUtils.KB);
+ private static final BlobInfo blob_6MB = new BlobInfo(6*FileUtils.MB);
+
+ private static int download_limit_default = -1;
+ private static int download_limit_test = 5*FileUtils.MB;
+
+ private static final String invalid_hash_empty = "";
+ private static final String invalid_hash_major = "INVALID_HASH";
+ private static final String invalid_hash_regex_attack = blob_512KB.hash.replace('a', '*');
+ private static final String invalid_hash_one_long = blob_512KB.hash.concat("a");
+ private static final String invalid_hash_one_short = blob_512KB.hash.substring(1);
+
+
+
+ @BeforeClass
+ public static void startGitblit() throws Exception {
+ started.set(GitBlitSuite.startGitblit());
+ }
+
+ @AfterClass
+ public static void stopGitblit() throws Exception {
+ if (started.get()) {
+ GitBlitSuite.stopGitblit();
+ }
+ }
+
+
+
+ @Test
+ public void testAdminAccess() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ RepositoryModel r = new RepositoryModel("myrepo.git", null, null, new Date());
+ ByteArrayOutputStream streamOut = new ByteArrayOutputStream();
+
+ UserModel u = new UserModel("admin");
+ u.canAdmin = true;
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_default);
+
+ //Invalid hash tests
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_major, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_regex_attack, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_long, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_short, u, r, streamOut));
+
+ // Download prior to upload
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Bad input is rejected with no upload taking place
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_empty, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_major, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_regex_attack, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_long, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_short, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Invalid_Size, filestore().uploadBlob(blob_512KB.hash, -1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, 0, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Hash_Mismatch, filestore().uploadBlob(blob_zero.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ //Confirm no upload with bad input
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Good input will accept the upload
+ assertEquals(Status.Available, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ assertArrayEquals(blob_512KB.blob, streamOut.toByteArray());
+
+ //Subsequent failed uploads do not affect file
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ assertArrayEquals(blob_512KB.blob, streamOut.toByteArray());
+
+ //Zero length upload is valid
+ assertEquals(Status.Available, filestore().uploadBlob(blob_zero.hash, blob_zero.length, u, r, new ByteArrayInputStream(blob_zero.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_zero.hash, u, r, streamOut));
+ assertArrayEquals(blob_zero.blob, streamOut.toByteArray());
+
+
+ //Pre-informed upload identifies identical errors as immediate upload
+ assertEquals(Status.Upload_Pending, filestore().addObject(blob_6MB.hash, blob_6MB.length, u, r));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_empty, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_major, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_regex_attack, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_long, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_short, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, 0, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length-1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length+1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ //Good input will accept the upload
+ assertEquals(Status.Available, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+ assertArrayEquals(blob_6MB.blob, streamOut.toByteArray());
+
+ //Confirm the relevant files exist
+ assertTrue("Admin did not save zero length file!", filestore().getStoragePath(blob_zero.hash).exists());
+ assertTrue("Admin did not save 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists());
+ assertTrue("Admin did not save 6MB file!", filestore().getStoragePath(blob_6MB.hash).exists());
+
+ //Clear the files and cache to test upload limit property
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_test);
+
+ assertEquals(Status.Available, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ assertArrayEquals(blob_512KB.blob, streamOut.toByteArray());
+
+ assertEquals(Status.Error_Exceeds_Size_Limit, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ assertTrue("Admin did not save 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists());
+ assertFalse("Admin saved 6MB file despite (over filesize limit)!", filestore().getStoragePath(blob_6MB.hash).exists());
+
+ }
+
+ @Test
+ public void testAuthenticatedAccess() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ RepositoryModel r = new RepositoryModel("myrepo.git", null, null, new Date());
+ r.authorizationControl = AuthorizationControl.AUTHENTICATED;
+ r.accessRestriction = AccessRestrictionType.VIEW;
+
+ ByteArrayOutputStream streamOut = new ByteArrayOutputStream();
+
+ UserModel u = new UserModel("test");
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_default);
+
+ //Invalid hash tests
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_major, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_regex_attack, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_long, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_short, u, r, streamOut));
+
+ // Download prior to upload
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Bad input is rejected with no upload taking place
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_empty, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_major, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_regex_attack, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_long, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_short, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Invalid_Size, filestore().uploadBlob(blob_512KB.hash, -1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, 0, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Hash_Mismatch, filestore().uploadBlob(blob_zero.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ //Confirm no upload with bad input
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Good input will accept the upload
+ assertEquals(Status.Available, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ assertArrayEquals(blob_512KB.blob, streamOut.toByteArray());
+
+ //Subsequent failed uploads do not affect file
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ assertArrayEquals(blob_512KB.blob, streamOut.toByteArray());
+
+ //Zero length upload is valid
+ assertEquals(Status.Available, filestore().uploadBlob(blob_zero.hash, blob_zero.length, u, r, new ByteArrayInputStream(blob_zero.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_zero.hash, u, r, streamOut));
+ assertArrayEquals(blob_zero.blob, streamOut.toByteArray());
+
+
+ //Pre-informed upload identifies identical errors as immediate upload
+ assertEquals(Status.Upload_Pending, filestore().addObject(blob_6MB.hash, blob_6MB.length, u, r));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_empty, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_major, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_regex_attack, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_long, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_short, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, 0, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length-1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length+1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ //Good input will accept the upload
+ assertEquals(Status.Available, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+ assertArrayEquals(blob_6MB.blob, streamOut.toByteArray());
+
+ //Confirm the relevant files exist
+ assertTrue("Authenticated user did not save zero length file!", filestore().getStoragePath(blob_zero.hash).exists());
+ assertTrue("Authenticated user did not save 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists());
+ assertTrue("Authenticated user did not save 6MB file!", filestore().getStoragePath(blob_6MB.hash).exists());
+
+ //Clear the files and cache to test upload limit property
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_test);
+
+ assertEquals(Status.Available, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ assertArrayEquals(blob_512KB.blob, streamOut.toByteArray());
+
+ assertEquals(Status.Error_Exceeds_Size_Limit, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ assertTrue("Authenticated user did not save 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists());
+ assertFalse("Authenticated user saved 6MB file (over filesize limit)!", filestore().getStoragePath(blob_6MB.hash).exists());
+
+ }
+
+ @Test
+ public void testAnonymousAccess() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ RepositoryModel r = new RepositoryModel("myrepo.git", null, null, new Date());
+ r.authorizationControl = AuthorizationControl.NAMED;
+ r.accessRestriction = AccessRestrictionType.CLONE;
+
+ ByteArrayOutputStream streamOut = new ByteArrayOutputStream();
+
+ UserModel u = UserModel.ANONYMOUS;
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_default);
+
+ //Invalid hash tests
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_major, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_regex_attack, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_long, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_short, u, r, streamOut));
+
+ // Download prior to upload
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Bad input is rejected with no upload taking place
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_empty, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_major, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_regex_attack, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_one_long, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_one_short, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, -1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, 0, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_zero.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ //Confirm no upload with bad input
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Good input will accept the upload
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Subsequent failed uploads do not affect file
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Zero length upload is valid
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_zero.hash, blob_zero.length, u, r, new ByteArrayInputStream(blob_zero.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_zero.hash, u, r, streamOut));
+
+
+ //Pre-informed upload identifies identical errors as immediate upload
+ assertEquals(Status.AuthenticationRequired, filestore().addObject(blob_6MB.hash, blob_6MB.length, u, r));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_empty, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_major, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_regex_attack, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_one_long, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_one_short, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, -1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, 0, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length-1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length+1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ //Good input will accept the upload
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ //Confirm the relevant files do not exist
+ assertFalse("Anonymous user saved zero length file!", filestore().getStoragePath(blob_zero.hash).exists());
+ assertFalse("Anonymous user 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists());
+ assertFalse("Anonymous user 6MB file!", filestore().getStoragePath(blob_6MB.hash).exists());
+
+ //Clear the files and cache to test upload limit property
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_test);
+
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ assertFalse("Anonymous user saved 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists());
+ assertFalse("Anonymous user saved 6MB file (over filesize limit)!", filestore().getStoragePath(blob_6MB.hash).exists());
+
+ }
+
+ @Test
+ public void testUnauthorizedAccess() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ RepositoryModel r = new RepositoryModel("myrepo.git", null, null, new Date());
+ r.authorizationControl = AuthorizationControl.NAMED;
+ r.accessRestriction = AccessRestrictionType.VIEW;
+
+ ByteArrayOutputStream streamOut = new ByteArrayOutputStream();
+
+ UserModel u = new UserModel("test");
+ u.setRepositoryPermission(r.name, AccessPermission.CLONE);
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_default);
+
+ //Invalid hash tests
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_major, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_regex_attack, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_long, u, r, streamOut));
+ assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_short, u, r, streamOut));
+
+ // Download prior to upload
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Bad input is rejected with no upload taking place
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_major, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_regex_attack, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_one_long, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_empty, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_one_short, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, -1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, 0, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_zero.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ //Confirm no upload with bad input
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Good input will accept the upload
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Subsequent failed uploads do not affect file
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ //Zero length upload is valid
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_zero.hash, blob_zero.length, u, r, new ByteArrayInputStream(blob_zero.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_zero.hash, u, r, streamOut));
+
+
+ //Pre-informed upload identifies identical errors as immediate upload
+ assertEquals(Status.Error_Unauthorized, filestore().addObject(blob_6MB.hash, blob_6MB.length, u, r));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_empty, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_major, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_regex_attack, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_one_long, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_one_short, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, -1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, 0, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length-1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length+1, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ //Good input will accept the upload
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ //Confirm the relevant files exist
+ assertFalse("Unauthorized user saved zero length file!", filestore().getStoragePath(blob_zero.hash).exists());
+ assertFalse("Unauthorized user saved 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists());
+ assertFalse("Unauthorized user saved 6MB file!", filestore().getStoragePath(blob_6MB.hash).exists());
+
+ //Clear the files and cache to test upload limit property
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_test);
+
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut));
+
+ assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob)));
+ streamOut.reset();
+ assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut));
+
+ assertFalse("Unauthorized user saved 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists());
+ assertFalse("Unauthorized user saved 6MB file (over filesize limit)!", filestore().getStoragePath(blob_6MB.hash).exists());
+
+ }
+
+}
+
+/*
+ * Test helper structure to create blobs of a given size
+ */
+final class BlobInfo {
+ public byte[] blob;
+ public String hash;
+ public int length;
+
+ public BlobInfo(int nBytes) {
+ blob = new byte[nBytes];
+ new java.util.Random().nextBytes(blob);
+ hash = DigestUtils.sha256Hex(blob);
+ length = nBytes;
+ }
+}
\ No newline at end of file
diff --git a/src/test/java/com/gitblit/tests/FilestoreServletTest.java b/src/test/java/com/gitblit/tests/FilestoreServletTest.java
new file mode 100644
index 0000000..4e4b056
--- /dev/null
+++ b/src/test/java/com/gitblit/tests/FilestoreServletTest.java
@@ -0,0 +1,355 @@
+package com.gitblit.tests;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.gitblit.Keys;
+import com.gitblit.manager.FilestoreManager;
+import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.UserModel;
+import com.gitblit.models.FilestoreModel.Status;
+import com.gitblit.servlet.FilestoreServlet;
+import com.gitblit.utils.FileUtils;
+
+public class FilestoreServletTest extends GitblitUnitTest {
+
+ private static final AtomicBoolean started = new AtomicBoolean(false);
+
+ private static final String SHA256_EG = "9a712c5d4037503a2d5ee1d07ad191eb99d051e84cbb020c171a5ae19bbe3cbd";
+
+ private static final String repoName = "helloworld.git";
+
+ private static final String repoLfs = "/r/" + repoName + "/info/lfs/objects/";
+
+ @BeforeClass
+ public static void startGitblit() throws Exception {
+ started.set(GitBlitSuite.startGitblit());
+ }
+
+ @AfterClass
+ public static void stopGitblit() throws Exception {
+ if (started.get()) {
+ GitBlitSuite.stopGitblit();
+ }
+ }
+
+
+ @Test
+ public void testRegexGroups() throws Exception {
+
+ Pattern p = Pattern.compile(FilestoreServlet.REGEX_PATH);
+
+ String basicUrl = "https://localhost:8080/r/test.git/info/lfs/objects/";
+ String batchUrl = basicUrl + "batch";
+ String oidUrl = basicUrl + SHA256_EG;
+
+ Matcher m = p.matcher(batchUrl);
+ assertTrue(m.find());
+ assertEquals("https://localhost:8080", m.group(FilestoreServlet.REGEX_GROUP_BASE_URI));
+ assertEquals("r", m.group(FilestoreServlet.REGEX_GROUP_PREFIX));
+ assertEquals("test.git", m.group(FilestoreServlet.REGEX_GROUP_REPOSITORY));
+ assertEquals("batch", m.group(FilestoreServlet.REGEX_GROUP_ENDPOINT));
+
+ m = p.matcher(oidUrl);
+ assertTrue(m.find());
+ assertEquals("https://localhost:8080", m.group(FilestoreServlet.REGEX_GROUP_BASE_URI));
+ assertEquals("r", m.group(FilestoreServlet.REGEX_GROUP_PREFIX));
+ assertEquals("test.git", m.group(FilestoreServlet.REGEX_GROUP_REPOSITORY));
+ assertEquals(SHA256_EG, m.group(FilestoreServlet.REGEX_GROUP_ENDPOINT));
+ }
+
+ @Test
+ public void testRegexGroupsNestedRepo() throws Exception {
+
+ Pattern p = Pattern.compile(FilestoreServlet.REGEX_PATH);
+
+ String basicUrl = "https://localhost:8080/r/nested/test.git/info/lfs/objects/";
+ String batchUrl = basicUrl + "batch";
+ String oidUrl = basicUrl + SHA256_EG;
+
+ Matcher m = p.matcher(batchUrl);
+ assertTrue(m.find());
+ assertEquals("https://localhost:8080", m.group(FilestoreServlet.REGEX_GROUP_BASE_URI));
+ assertEquals("r", m.group(FilestoreServlet.REGEX_GROUP_PREFIX));
+ assertEquals("nested/test.git", m.group(FilestoreServlet.REGEX_GROUP_REPOSITORY));
+ assertEquals("batch", m.group(FilestoreServlet.REGEX_GROUP_ENDPOINT));
+
+ m = p.matcher(oidUrl);
+ assertTrue(m.find());
+ assertEquals("https://localhost:8080", m.group(FilestoreServlet.REGEX_GROUP_BASE_URI));
+ assertEquals("r", m.group(FilestoreServlet.REGEX_GROUP_PREFIX));
+ assertEquals("nested/test.git", m.group(FilestoreServlet.REGEX_GROUP_REPOSITORY));
+ assertEquals(SHA256_EG, m.group(FilestoreServlet.REGEX_GROUP_ENDPOINT));
+ }
+
+ @Test
+ public void testDownload() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ RepositoryModel r = gitblit().getRepositoryModel(repoName);
+
+ UserModel u = new UserModel("admin");
+ u.canAdmin = true;
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE);
+
+ final BlobInfo blob = new BlobInfo(512*FileUtils.KB);
+
+ //Emulate a pre-existing Git-LFS repository by using using internal pre-tested methods
+ assertEquals(Status.Available, filestore().uploadBlob(blob.hash, blob.length, u, r, new ByteArrayInputStream(blob.blob)));
+
+ final String downloadURL = GitBlitSuite.url + repoLfs + blob.hash;
+
+ HttpClient client = HttpClientBuilder.create().build();
+ HttpGet request = new HttpGet(downloadURL);
+
+ // add request header
+ request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME);
+ HttpResponse response = client.execute(request);
+
+ assertEquals(200, response.getStatusLine().getStatusCode());
+
+ String content = IOUtils.toString(response.getEntity().getContent(), "UTF-8");
+
+ String expectedContent = String.format("{%s:%s,%s:%d,%s:{%s:{%s:%s}}}",
+ "\"oid\"", "\"" + blob.hash + "\"",
+ "\"size\"", blob.length,
+ "\"actions\"",
+ "\"download\"",
+ "\"href\"", "\"" + downloadURL + "\"");
+
+ assertEquals(expectedContent, content);
+
+
+ //Now try the binary download
+ request.removeHeaders(HttpHeaders.ACCEPT);
+ response = client.execute(request);
+
+ assertEquals(200, response.getStatusLine().getStatusCode());
+
+ byte[] dlData = IOUtils.toByteArray(response.getEntity().getContent());
+
+ assertArrayEquals(blob.blob, dlData);
+
+ }
+
+ @Test
+ public void testDownloadMultiple() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ RepositoryModel r = gitblit().getRepositoryModel(repoName);
+
+ UserModel u = new UserModel("admin");
+ u.canAdmin = true;
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE);
+
+ final BlobInfo blob = new BlobInfo(512*FileUtils.KB);
+
+ //Emulate a pre-existing Git-LFS repository by using using internal pre-tested methods
+ assertEquals(Status.Available, filestore().uploadBlob(blob.hash, blob.length, u, r, new ByteArrayInputStream(blob.blob)));
+
+ final String batchURL = GitBlitSuite.url + repoLfs + "batch";
+ final String downloadURL = GitBlitSuite.url + repoLfs + blob.hash;
+
+ HttpClient client = HttpClientBuilder.create().build();
+ HttpPost request = new HttpPost(batchURL);
+
+ // add request header
+ request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME);
+ request.addHeader(HttpHeaders.CONTENT_ENCODING, FilestoreServlet.GIT_LFS_META_MIME);
+
+ String content = String.format("{%s:%s,%s:[{%s:%s,%s:%d},{%s:%s,%s:%d}]}",
+ "\"operation\"", "\"download\"",
+ "\"objects\"",
+ "\"oid\"", "\"" + blob.hash + "\"",
+ "\"size\"", blob.length,
+ "\"oid\"", "\"" + SHA256_EG + "\"",
+ "\"size\"", 0);
+
+ HttpEntity entity = new ByteArrayEntity(content.getBytes("UTF-8"));
+ request.setEntity(entity);
+
+ HttpResponse response = client.execute(request);
+
+ String responseMessage = IOUtils.toString(response.getEntity().getContent(), "UTF-8");
+ assertEquals(200, response.getStatusLine().getStatusCode());
+
+ String expectedContent = String.format("{%s:[{%s:%s,%s:%d,%s:{%s:{%s:%s}}},{%s:%s,%s:%d,%s:{%s:%s,%s:%d}}]}",
+ "\"objects\"",
+ "\"oid\"", "\"" + blob.hash + "\"",
+ "\"size\"", blob.length,
+ "\"actions\"",
+ "\"download\"",
+ "\"href\"", "\"" + downloadURL + "\"",
+ "\"oid\"", "\"" + SHA256_EG + "\"",
+ "\"size\"", 0,
+ "\"error\"",
+ "\"message\"", "\"Object not available\"",
+ "\"code\"", 404
+ );
+
+ assertEquals(expectedContent, responseMessage);
+ }
+
+ @Test
+ public void testDownloadUnavailable() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE);
+
+ final BlobInfo blob = new BlobInfo(512*FileUtils.KB);
+
+ final String downloadURL = GitBlitSuite.url + repoLfs + blob.hash;
+
+ HttpClient client = HttpClientBuilder.create().build();
+ HttpGet request = new HttpGet(downloadURL);
+
+ // add request header
+ request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME);
+ HttpResponse response = client.execute(request);
+
+ assertEquals(404, response.getStatusLine().getStatusCode());
+
+ String content = IOUtils.toString(response.getEntity().getContent(), "UTF-8");
+
+ String expectedError = String.format("{%s:%s,%s:%d}",
+ "\"message\"", "\"Object not available\"",
+ "\"code\"", 404);
+
+ assertEquals(expectedError, content);
+ }
+
+ @Test
+ public void testUpload() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ RepositoryModel r = gitblit().getRepositoryModel(repoName);
+
+ UserModel u = new UserModel("admin");
+ u.canAdmin = true;
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE);
+
+ final BlobInfo blob = new BlobInfo(512*FileUtils.KB);
+
+ final String expectedUploadURL = GitBlitSuite.url + repoLfs + blob.hash;
+ final String initialUploadURL = GitBlitSuite.url + repoLfs + "batch";
+
+ HttpClient client = HttpClientBuilder.create().build();
+ HttpPost request = new HttpPost(initialUploadURL);
+
+ // add request header
+ request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME);
+ request.addHeader(HttpHeaders.CONTENT_ENCODING, FilestoreServlet.GIT_LFS_META_MIME);
+
+ String content = String.format("{%s:%s,%s:[{%s:%s,%s:%d}]}",
+ "\"operation\"", "\"upload\"",
+ "\"objects\"",
+ "\"oid\"", "\"" + blob.hash + "\"",
+ "\"size\"", blob.length);
+
+ HttpEntity entity = new ByteArrayEntity(content.getBytes("UTF-8"));
+ request.setEntity(entity);
+
+ HttpResponse response = client.execute(request);
+ String responseMessage = IOUtils.toString(response.getEntity().getContent(), "UTF-8");
+ assertEquals(200, response.getStatusLine().getStatusCode());
+
+ String expectedContent = String.format("{%s:[{%s:%s,%s:%d,%s:{%s:{%s:%s}}}]}",
+ "\"objects\"",
+ "\"oid\"", "\"" + blob.hash + "\"",
+ "\"size\"", blob.length,
+ "\"actions\"",
+ "\"upload\"",
+ "\"href\"", "\"" + expectedUploadURL + "\"");
+
+ assertEquals(expectedContent, responseMessage);
+
+
+ //Now try to upload the binary download
+ HttpPut putRequest = new HttpPut(expectedUploadURL);
+ putRequest.setEntity(new ByteArrayEntity(blob.blob));
+ response = client.execute(putRequest);
+
+ responseMessage = IOUtils.toString(response.getEntity().getContent(), "UTF-8");
+
+ assertEquals(200, response.getStatusLine().getStatusCode());
+
+ //Confirm behind the scenes that it is available
+ ByteArrayOutputStream savedBlob = new ByteArrayOutputStream();
+ assertEquals(Status.Available, filestore().downloadBlob(blob.hash, u, r, savedBlob));
+ assertArrayEquals(blob.blob, savedBlob.toByteArray());
+ }
+
+ @Test
+ public void testMalformedUpload() throws Exception {
+
+ FileUtils.delete(filestore().getStorageFolder());
+ filestore().clearFilestoreCache();
+
+ //No upload limit
+ settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE);
+
+ final BlobInfo blob = new BlobInfo(512*FileUtils.KB);
+
+ final String initialUploadURL = GitBlitSuite.url + repoLfs + "batch";
+
+ HttpClient client = HttpClientBuilder.create().build();
+ HttpPost request = new HttpPost(initialUploadURL);
+
+ // add request header
+ request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME);
+ request.addHeader(HttpHeaders.CONTENT_ENCODING, FilestoreServlet.GIT_LFS_META_MIME);
+
+ //Malformed JSON, comma instead of colon and unquoted strings
+ String content = String.format("{%s:%s,%s:[{%s:%s,%s,%d}]}",
+ "operation", "upload",
+ "objects",
+ "oid", blob.hash,
+ "size", blob.length);
+
+ HttpEntity entity = new ByteArrayEntity(content.getBytes("UTF-8"));
+ request.setEntity(entity);
+
+ HttpResponse response = client.execute(request);
+ String responseMessage = IOUtils.toString(response.getEntity().getContent(), "UTF-8");
+ assertEquals(400, response.getStatusLine().getStatusCode());
+
+ String expectedError = String.format("{%s:%s,%s:%d}",
+ "\"message\"", "\"Malformed Git-LFS request\"",
+ "\"code\"", 400);
+
+ assertEquals(expectedError, responseMessage);
+ }
+
+}
diff --git a/src/test/java/com/gitblit/tests/GitBlitSuite.java b/src/test/java/com/gitblit/tests/GitBlitSuite.java
index af20a48..b01c82c 100644
--- a/src/test/java/com/gitblit/tests/GitBlitSuite.java
+++ b/src/test/java/com/gitblit/tests/GitBlitSuite.java
@@ -66,7 +66,7 @@
ModelUtilsTest.class, JnaUtilsTest.class, LdapSyncServiceTest.class, FileTicketServiceTest.class,
BranchTicketServiceTest.class, RedisTicketServiceTest.class, AuthenticationManagerTest.class,
SshKeysDispatcherTest.class, UITicketTest.class, PathUtilsTest.class, SshKerberosAuthenticationTest.class,
- GravatarTest.class })
+ GravatarTest.class, FilestoreManagerTest.class, FilestoreServletTest.class })
public class GitBlitSuite {
public static final File BASEFOLDER = new File("data");
diff --git a/src/test/java/com/gitblit/tests/GitblitUnitTest.java b/src/test/java/com/gitblit/tests/GitblitUnitTest.java
index 9dceaaf..58bc60e 100644
--- a/src/test/java/com/gitblit/tests/GitblitUnitTest.java
+++ b/src/test/java/com/gitblit/tests/GitblitUnitTest.java
@@ -18,6 +18,7 @@
import com.gitblit.IStoredSettings;
import com.gitblit.manager.IAuthenticationManager;
import com.gitblit.manager.IFederationManager;
+import com.gitblit.manager.IFilestoreManager;
import com.gitblit.manager.IGitblit;
import com.gitblit.manager.INotificationManager;
import com.gitblit.manager.IProjectManager;
@@ -64,4 +65,8 @@
public static IGitblit gitblit() {
return GitblitContext.getManager(IGitblit.class);
}
+
+ public static IFilestoreManager filestore() {
+ return GitblitContext.getManager(IFilestoreManager.class);
+ }
}
--
Gitblit v1.9.1