/*
|
* Copyright 2015 gitblit.com.
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*/
|
package com.gitblit.servlet;
|
|
import java.io.BufferedReader;
|
import java.io.IOException;
|
import java.io.Serializable;
|
import java.text.MessageFormat;
|
import java.util.ArrayList;
|
import java.util.HashMap;
|
import java.util.List;
|
import java.util.Map;
|
import java.util.regex.Matcher;
|
import java.util.regex.Pattern;
|
|
import com.google.inject.Inject;
|
import com.google.inject.Singleton;
|
|
import javax.servlet.ServletException;
|
import javax.servlet.http.HttpServlet;
|
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletResponse;
|
|
import org.slf4j.Logger;
|
import org.slf4j.LoggerFactory;
|
|
import com.gitblit.Constants;
|
import com.gitblit.IStoredSettings;
|
import com.gitblit.models.FilestoreModel;
|
import com.gitblit.models.RepositoryModel;
|
import com.gitblit.models.FilestoreModel.Status;
|
import com.gitblit.manager.FilestoreManager;
|
import com.gitblit.manager.IGitblit;
|
import com.gitblit.models.UserModel;
|
import com.gitblit.utils.JsonUtils;
|
|
|
/**
|
* Handles large file storage as per the Git LFS v1 Batch API
|
*
|
* Further details can be found at https://github.com/github/git-lfs
|
*
|
* @author Paul Martin
|
*/
|
@Singleton
|
public class FilestoreServlet extends HttpServlet {
|
|
private static final long serialVersionUID = 1L;
|
public static final int PROTOCOL_VERSION = 1;
|
|
public static final String GIT_LFS_META_MIME = "application/vnd.git-lfs+json";
|
|
public static final String REGEX_PATH = "^(.*?)/(r|git)/(.*?)/info/lfs/objects/(batch|" + Constants.REGEX_SHA256 + ")";
|
public static final int REGEX_GROUP_BASE_URI = 1;
|
public static final int REGEX_GROUP_PREFIX = 2;
|
public static final int REGEX_GROUP_REPOSITORY = 3;
|
public static final int REGEX_GROUP_ENDPOINT = 4;
|
|
protected final Logger logger;
|
|
private static IGitblit gitblit;
|
|
@Inject
|
public FilestoreServlet(IStoredSettings settings, IGitblit gitblit) {
|
|
super();
|
logger = LoggerFactory.getLogger(getClass());
|
|
FilestoreServlet.gitblit = gitblit;
|
}
|
|
|
/**
|
* Handles batch upload request (metadata)
|
*
|
* @param request
|
* @param response
|
* @throws javax.servlet.ServletException
|
* @throws java.io.IOException
|
*/
|
@Override
|
protected void doPost(HttpServletRequest request,
|
HttpServletResponse response) throws ServletException ,IOException {
|
|
UrlInfo info = getInfoFromRequest(request);
|
if (info == null) {
|
sendError(response, HttpServletResponse.SC_NOT_FOUND);
|
return;
|
}
|
|
//Post is for batch operations so no oid should be defined
|
if (info.oid != null) {
|
sendError(response, HttpServletResponse.SC_BAD_REQUEST);
|
return;
|
}
|
|
IGitLFS.Batch batch = deserialize(request, response, IGitLFS.Batch.class);
|
|
if (batch == null) {
|
sendError(response, HttpServletResponse.SC_BAD_REQUEST);
|
return;
|
}
|
|
UserModel user = getUserOrAnonymous(request);
|
|
IGitLFS.BatchResponse batchResponse = new IGitLFS.BatchResponse();
|
|
if (batch.operation.equalsIgnoreCase("upload")) {
|
for (IGitLFS.Request item : batch.objects) {
|
|
Status state = gitblit.addObject(item.oid, item.size, user, info.repository);
|
|
batchResponse.objects.add(getResponseForUpload(info.baseUrl, item.oid, item.size, user.getName(), info.repository.name, state));
|
}
|
} else if (batch.operation.equalsIgnoreCase("download")) {
|
for (IGitLFS.Request item : batch.objects) {
|
|
Status state = gitblit.downloadBlob(item.oid, user, info.repository, null);
|
batchResponse.objects.add(getResponseForDownload(info.baseUrl, item.oid, item.size, user.getName(), info.repository.name, state));
|
}
|
} else {
|
sendError(response, HttpServletResponse.SC_NOT_IMPLEMENTED);
|
return;
|
}
|
|
response.setStatus(HttpServletResponse.SC_OK);
|
serialize(response, batchResponse);
|
}
|
|
/**
|
* Handles the actual upload (BLOB)
|
*
|
* @param request
|
* @param response
|
* @throws javax.servlet.ServletException
|
* @throws java.io.IOException
|
*/
|
@Override
|
protected void doPut(HttpServletRequest request,
|
HttpServletResponse response) throws ServletException ,IOException {
|
|
UrlInfo info = getInfoFromRequest(request);
|
|
if (info == null) {
|
sendError(response, HttpServletResponse.SC_NOT_FOUND);
|
return;
|
}
|
|
//Put is a singular operation so must have oid
|
if (info.oid == null) {
|
sendError(response, HttpServletResponse.SC_BAD_REQUEST);
|
return;
|
}
|
|
UserModel user = getUserOrAnonymous(request);
|
long size = FilestoreManager.UNDEFINED_SIZE;
|
|
|
|
FilestoreModel.Status status = gitblit.uploadBlob(info.oid, size, user, info.repository, request.getInputStream());
|
IGitLFS.Response responseObject = getResponseForUpload(info.baseUrl, info.oid, size, user.getName(), info.repository.name, status);
|
|
logger.info(MessageFormat.format("FILESTORE-AUDIT {0}:{4} {1} {2}@{3}",
|
"PUT", info.oid, user.getName(), info.repository.name, status.toString() ));
|
|
if (responseObject.error == null) {
|
response.setStatus(responseObject.successCode);
|
} else {
|
serialize(response, responseObject.error);
|
}
|
};
|
|
/**
|
* Handles a download
|
* Treated as hypermedia request if accept header contains Git-LFS MIME
|
* otherwise treated as a download of the blob
|
* @param request
|
* @param response
|
* @throws javax.servlet.ServletException
|
* @throws java.io.IOException
|
*/
|
@Override
|
protected void doGet(HttpServletRequest request,
|
HttpServletResponse response) throws ServletException ,IOException {
|
|
UrlInfo info = getInfoFromRequest(request);
|
|
if (info == null || info.oid == null) {
|
sendError(response, HttpServletResponse.SC_NOT_FOUND);
|
return;
|
}
|
|
UserModel user = getUserOrAnonymous(request);
|
|
FilestoreModel model = gitblit.getObject(info.oid, user, info.repository);
|
long size = FilestoreManager.UNDEFINED_SIZE;
|
|
boolean isMetaRequest = AccessRestrictionFilter.hasContentInRequestHeader(request, "Accept", GIT_LFS_META_MIME);
|
FilestoreModel.Status status = Status.Unavailable;
|
|
if (model != null) {
|
size = model.getSize();
|
status = model.getStatus();
|
}
|
|
if (!isMetaRequest) {
|
status = gitblit.downloadBlob(info.oid, user, info.repository, response.getOutputStream());
|
|
logger.info(MessageFormat.format("FILESTORE-AUDIT {0}:{4} {1} {2}@{3}",
|
"GET", info.oid, user.getName(), info.repository.name, status.toString() ));
|
}
|
|
if (status == Status.Error_Unexpected_Stream_End) {
|
return;
|
}
|
|
IGitLFS.Response responseObject = getResponseForDownload(info.baseUrl,
|
info.oid, size, user.getName(), info.repository.name, status);
|
|
if (responseObject.error == null) {
|
response.setStatus(responseObject.successCode);
|
|
if (isMetaRequest) {
|
serialize(response, responseObject);
|
}
|
} else {
|
response.setStatus(responseObject.error.code);
|
serialize(response, responseObject.error);
|
}
|
};
|
|
private void sendError(HttpServletResponse response, int code) throws IOException {
|
|
String msg = "";
|
|
switch (code)
|
{
|
case HttpServletResponse.SC_NOT_FOUND: msg = "Not Found"; break;
|
case HttpServletResponse.SC_NOT_IMPLEMENTED: msg = "Not Implemented"; break;
|
case HttpServletResponse.SC_BAD_REQUEST: msg = "Malformed Git-LFS request"; break;
|
|
default: msg = "Unknown Error";
|
}
|
|
response.setStatus(code);
|
serialize(response, new IGitLFS.ObjectError(code, msg));
|
}
|
|
@SuppressWarnings("incomplete-switch")
|
private IGitLFS.Response getResponseForUpload(String baseUrl, String oid, long size, String user, String repo, FilestoreModel.Status state) {
|
|
switch (state) {
|
case AuthenticationRequired:
|
return new IGitLFS.Response(oid, size, 401, MessageFormat.format("Authentication required to write to repository {0}", repo));
|
case Error_Unauthorized:
|
return new IGitLFS.Response(oid, size, 403, MessageFormat.format("User {0}, does not have write permissions to repository {1}", user, repo));
|
case Error_Exceeds_Size_Limit:
|
return new IGitLFS.Response(oid, size, 509, MessageFormat.format("Object is larger than allowed limit of {1}", gitblit.getMaxUploadSize()));
|
case Error_Hash_Mismatch:
|
return new IGitLFS.Response(oid, size, 422, "Hash mismatch");
|
case Error_Invalid_Oid:
|
return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid oid", oid));
|
case Error_Invalid_Size:
|
return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid size", size));
|
case Error_Size_Mismatch:
|
return new IGitLFS.Response(oid, size, 422, "Object size mismatch");
|
case Deleted:
|
return new IGitLFS.Response(oid, size, 410, "Object was deleted : ".concat("TBD Reason") );
|
case Upload_In_Progress:
|
return new IGitLFS.Response(oid, size, 503, "File currently being uploaded by another user");
|
case Unavailable:
|
return new IGitLFS.Response(oid, size, 404, MessageFormat.format("Repository {0}, does not exist for user {1}", repo, user));
|
case Upload_Pending:
|
return new IGitLFS.Response(oid, size, 202, "upload", getObjectUri(baseUrl, repo, oid) );
|
case Available:
|
return new IGitLFS.Response(oid, size, 200, "upload", getObjectUri(baseUrl, repo, oid) );
|
}
|
|
return new IGitLFS.Response(oid, size, 500, "Unknown Error");
|
}
|
|
@SuppressWarnings("incomplete-switch")
|
private IGitLFS.Response getResponseForDownload(String baseUrl, String oid, long size, String user, String repo, FilestoreModel.Status state) {
|
|
switch (state) {
|
case Error_Unauthorized:
|
return new IGitLFS.Response(oid, size, 403, MessageFormat.format("User {0}, does not have read permissions to repository {1}", user, repo));
|
case Error_Invalid_Oid:
|
return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid oid", oid));
|
case Error_Unknown:
|
return new IGitLFS.Response(oid, size, 500, "Unknown Error");
|
case Deleted:
|
return new IGitLFS.Response(oid, size, 410, "Object was deleted : ".concat("TBD Reason") );
|
case Available:
|
return new IGitLFS.Response(oid, size, 200, "download", getObjectUri(baseUrl, repo, oid) );
|
}
|
|
return new IGitLFS.Response(oid, size, 404, "Object not available");
|
}
|
|
|
private String getObjectUri(String baseUrl, String repo, String oid) {
|
return baseUrl + "/" + repo + "/" + Constants.R_LFS + "objects/" + oid;
|
}
|
|
|
protected void serialize(HttpServletResponse response, Object o) throws IOException {
|
if (o != null) {
|
// Send JSON response
|
String json = JsonUtils.toJsonString(o);
|
response.setCharacterEncoding(Constants.ENCODING);
|
response.setContentType(GIT_LFS_META_MIME);
|
response.getWriter().append(json);
|
}
|
}
|
|
protected <X> X deserialize(HttpServletRequest request, HttpServletResponse response,
|
Class<X> clazz) {
|
|
String json = "";
|
try {
|
|
json = readJson(request, response);
|
|
return JsonUtils.fromJsonString(json.toString(), clazz);
|
|
} catch (Exception e) {
|
//Intentional silent fail
|
}
|
|
return null;
|
}
|
|
private String readJson(HttpServletRequest request, HttpServletResponse response)
|
throws IOException {
|
BufferedReader reader = request.getReader();
|
StringBuilder json = new StringBuilder();
|
String line = null;
|
while ((line = reader.readLine()) != null) {
|
json.append(line);
|
}
|
reader.close();
|
|
if (json.length() == 0) {
|
logger.error(MessageFormat.format("Failed to receive json data from {0}",
|
request.getRemoteAddr()));
|
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
|
return null;
|
}
|
return json.toString();
|
}
|
|
private UserModel getUserOrAnonymous(HttpServletRequest r) {
|
UserModel user = (UserModel) r.getUserPrincipal();
|
if (user != null) { return user; }
|
return UserModel.ANONYMOUS;
|
}
|
|
private static class UrlInfo {
|
public RepositoryModel repository;
|
public String oid;
|
public String baseUrl;
|
|
public UrlInfo(RepositoryModel repo, String oid, String baseUrl) {
|
this.repository = repo;
|
this.oid = oid;
|
this.baseUrl = baseUrl;
|
}
|
}
|
|
public static UrlInfo getInfoFromRequest(HttpServletRequest httpRequest) {
|
|
String url = httpRequest.getRequestURL().toString();
|
Pattern p = Pattern.compile(REGEX_PATH);
|
Matcher m = p.matcher(url);
|
|
|
if (m.find()) {
|
RepositoryModel repo = gitblit.getRepositoryModel(m.group(REGEX_GROUP_REPOSITORY));
|
String baseUrl = m.group(REGEX_GROUP_BASE_URI) + "/" + m.group(REGEX_GROUP_PREFIX);
|
|
if (m.group(REGEX_GROUP_ENDPOINT).equals("batch")) {
|
return new UrlInfo(repo, null, baseUrl);
|
} else {
|
return new UrlInfo(repo, m.group(REGEX_GROUP_ENDPOINT), baseUrl);
|
}
|
}
|
|
return null;
|
}
|
|
|
public interface IGitLFS {
|
|
@SuppressWarnings("serial")
|
public class Request implements Serializable
|
{
|
public String oid;
|
public long size;
|
}
|
|
|
@SuppressWarnings("serial")
|
public class Batch implements Serializable
|
{
|
public String operation;
|
public List<Request> objects;
|
}
|
|
|
@SuppressWarnings("serial")
|
public class Response implements Serializable
|
{
|
public String oid;
|
public long size;
|
public Map<String, HyperMediaLink> actions;
|
public ObjectError error;
|
public transient int successCode;
|
|
public Response(String id, long itemSize, int errorCode, String errorText) {
|
oid = id;
|
size = itemSize;
|
actions = null;
|
successCode = 0;
|
error = new ObjectError(errorCode, errorText);
|
}
|
|
public Response(String id, long itemSize, int actionCode, String action, String uri) {
|
oid = id;
|
size = itemSize;
|
error = null;
|
successCode = actionCode;
|
actions = new HashMap<String, HyperMediaLink>();
|
actions.put(action, new HyperMediaLink(action, uri));
|
}
|
|
}
|
|
@SuppressWarnings("serial")
|
public class BatchResponse implements Serializable {
|
public List<Response> objects;
|
|
public BatchResponse() {
|
objects = new ArrayList<Response>();
|
}
|
}
|
|
|
@SuppressWarnings("serial")
|
public class ObjectError implements Serializable
|
{
|
public String message;
|
public int code;
|
public String documentation_url;
|
public Integer request_id;
|
|
public ObjectError(int errorCode, String errorText) {
|
code = errorCode;
|
message = errorText;
|
request_id = null;
|
}
|
}
|
|
@SuppressWarnings("serial")
|
public class HyperMediaLink implements Serializable
|
{
|
public String href;
|
public transient String header;
|
//public Date expires_at;
|
|
public HyperMediaLink(String action, String uri) {
|
header = action;
|
href = uri;
|
}
|
}
|
}
|
|
|
|
}
|
