From f75535759570bbc4784ee8324b0d1b8dfb01766f Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gmail.com>
Date: Tue, 08 Dec 2015 07:58:21 -0500
Subject: [PATCH] Merge pull request #973 from paulsputer/967-Filestore-Help
---
src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html | 34 ++--------
src/main/java/com/gitblit/wicket/pages/FilestorePage.java | 14 +++-
src/main/java/com/gitblit/models/FilestoreModel.java | 9 +++
src/main/java/com/gitblit/manager/FilestoreManager.java | 31 +++++++++-
src/main/java/com/gitblit/manager/GitblitManager.java | 4
src/main/java/com/gitblit/manager/IFilestoreManager.java | 2
HOME.md | 2
build.xml | 1
src/main/java/com/gitblit/wicket/pages/RootPage.java | 6 +-
src/site/setup_filestore.mkd | 61 ++++++++++++++++++++
10 files changed, 126 insertions(+), 38 deletions(-)
diff --git a/HOME.md b/HOME.md
index 7c5449d..89527ee 100644
--- a/HOME.md
+++ b/HOME.md
@@ -33,6 +33,8 @@
[[src/site/setup_viewer.mkd]]
[[src/site/administration.mkd]]
[[src/site/setup_scaling.mkd]]
+[[src/site/setup_filestore.mkd]]
+
### Gitblit Tickets
diff --git a/build.xml b/build.xml
index c7dc499..7205bf8 100644
--- a/build.xml
+++ b/build.xml
@@ -514,6 +514,7 @@
<page name="mirrors" src="setup_mirrors.mkd" />
<page name="scaling" src="setup_scaling.mkd" />
<page name="fail2ban" src="setup_fail2ban.mkd" />
+ <page name="filestore (Git LFS)" src="setup_filestore.mkd" />
<divider />
<page name="Gitblit as a viewer" src="setup_viewer.mkd" />
</menu>
diff --git a/src/main/java/com/gitblit/manager/FilestoreManager.java b/src/main/java/com/gitblit/manager/FilestoreManager.java
index fe65e21..1110855 100644
--- a/src/main/java/com/gitblit/manager/FilestoreManager.java
+++ b/src/main/java/com/gitblit/manager/FilestoreManager.java
@@ -77,6 +77,8 @@
private final Logger logger = LoggerFactory.getLogger(getClass());
private final IRuntimeManager runtimeManager;
+
+ private final IRepositoryManager repositoryManager;
private final IStoredSettings settings;
@@ -93,8 +95,10 @@
@Inject
FilestoreManager(
- IRuntimeManager runtimeManager) {
+ IRuntimeManager runtimeManager,
+ IRepositoryManager repositoryManager) {
this.runtimeManager = runtimeManager;
+ this.repositoryManager = repositoryManager;
this.settings = runtimeManager.getSettings();
}
@@ -324,8 +328,29 @@
}
@Override
- public List<FilestoreModel> getAllObjects() {
- return new ArrayList<FilestoreModel>(fileCache.values());
+ public List<FilestoreModel> getAllObjects(UserModel user) {
+
+ final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user);
+ List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size());
+
+ for (RepositoryModel repository : viewableRepositories) {
+ viewableRepositoryNames.add(repository.name);
+ }
+
+ if (viewableRepositoryNames.size() == 0) {
+ return null;
+ }
+
+ final Collection<FilestoreModel> allFiles = fileCache.values();
+ List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size());
+
+ for (FilestoreModel file : allFiles) {
+ if (file.isInRepositoryList(viewableRepositoryNames)) {
+ userViewableFiles.add(file);
+ }
+ }
+
+ return userViewableFiles;
}
@Override
diff --git a/src/main/java/com/gitblit/manager/GitblitManager.java b/src/main/java/com/gitblit/manager/GitblitManager.java
index 4a385fc..85d5c19 100644
--- a/src/main/java/com/gitblit/manager/GitblitManager.java
+++ b/src/main/java/com/gitblit/manager/GitblitManager.java
@@ -1274,8 +1274,8 @@
}
@Override
- public List<FilestoreModel> getAllObjects() {
- return filestoreManager.getAllObjects();
+ public List<FilestoreModel> getAllObjects(UserModel user) {
+ return filestoreManager.getAllObjects(user);
}
@Override
diff --git a/src/main/java/com/gitblit/manager/IFilestoreManager.java b/src/main/java/com/gitblit/manager/IFilestoreManager.java
index 0720650..454331a 100644
--- a/src/main/java/com/gitblit/manager/IFilestoreManager.java
+++ b/src/main/java/com/gitblit/manager/IFilestoreManager.java
@@ -37,7 +37,7 @@
FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut );
- List<FilestoreModel> getAllObjects();
+ List<FilestoreModel> getAllObjects(UserModel user);
File getStorageFolder();
diff --git a/src/main/java/com/gitblit/models/FilestoreModel.java b/src/main/java/com/gitblit/models/FilestoreModel.java
index ff7b210..4144df6 100644
--- a/src/main/java/com/gitblit/models/FilestoreModel.java
+++ b/src/main/java/com/gitblit/models/FilestoreModel.java
@@ -111,6 +111,15 @@
repositories.remove(repo);
}
+ public synchronized boolean isInRepositoryList(List<String> repoList) {
+ for (String name : repositories) {
+ if (repoList.contains(name)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public static enum Status {
Deleted(-30),
diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java
index 97d5f25..be0181b 100644
--- a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java
+++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java
@@ -18,6 +18,7 @@
import java.text.DateFormat;
import java.text.MessageFormat;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;
@@ -31,9 +32,12 @@
import com.gitblit.Constants;
import com.gitblit.models.FilestoreModel;
import com.gitblit.models.UserModel;
+import com.gitblit.wicket.CacheControl;
import com.gitblit.wicket.FilestoreUI;
+import com.gitblit.wicket.GitBlitWebSession;
import com.gitblit.wicket.RequiresAdminRole;
import com.gitblit.wicket.WicketUtils;
+import com.gitblit.wicket.CacheControl.LastModified;
/**
* Page to display the current status of the filestore.
@@ -41,17 +45,22 @@
*
* @author Paul Martin
*/
-@RequiresAdminRole
+@CacheControl(LastModified.ACTIVITY)
public class FilestorePage extends RootPage {
public FilestorePage() {
super();
setupPage("", "");
- final List<FilestoreModel> files = app().filestore().getAllObjects();
+ final UserModel user = (GitBlitWebSession.get().getUser() == null) ? UserModel.ANONYMOUS : GitBlitWebSession.get().getUser();
final long nBytesUsed = app().filestore().getFilestoreUsedByteCount();
final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount();
+ List<FilestoreModel> files = app().filestore().getAllObjects(user);
+ if (files == null) {
+ files = new ArrayList<FilestoreModel>();
+ }
+
String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(),
FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) );
@@ -63,7 +72,6 @@
BookmarkablePageLink<Void> helpLink = new BookmarkablePageLink<Void>("filestoreHelp", FilestoreUsage.class);
helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp")));
add(helpLink);
-
DataView<FilestoreModel> filesView = new DataView<FilestoreModel>("fileRow",
new ListDataProvider<FilestoreModel>(files)) {
diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html
index e9bff47..89e98ca 100644
--- a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html
+++ b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html
@@ -12,16 +12,19 @@
<div class="span10 offset1">
<div class="alert alert-danger">
- <h3><center>Using the Filestore</center></h3>
+ <h3><center>Using the filestore</center></h3>
<p>
- <strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs init</code> to register the hooks globally.</strong><br/>
- <i>This version of GitBlit has been verified with Git-LFS client version 0.6.0 which requires Git v1.8.2 or higher.</i>
+ <strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs install</code></strong><br/>
+ <p>
+ If using password authentication it is recommended that you configure the <a href="https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage">git credential storage</a> to avoid Git-LFS asking for your password on each file<br/>
+ On Windows for example: <code>git config --global credential.helper wincred</code>
+ </p>
</p>
</div>
<h3>Clone</h3>
<p>
- Just <code>git clone</code> as usual, no further action is required as GitBlit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/>
+ Just <code>git clone</code> as usual, no further action is required as Gitblit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/>
<i>If the repository uses a 3rd party Git-LFS server you will need to <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md#the-server">manually configure the correct endpoints</a></i>.
</p>
@@ -38,28 +41,7 @@
<p><a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">See the current Git-LFS specification for further details</a>.</p>
<br />
- <div class="alert alert-warn">
- <h3><center>Limitations & Warnings</center></h3>
- <p>GitBlit currently provides a server-only implementation of the opensource Git-LFS API, <a href="https://github.com/github/git-lfs/wiki/Implementations">other implementations</a> are available.<br/>
- However, until <a href="https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333">JGit provides Git-LFS client capabilities</a> some GitBlit features may not be fully supported when using the filestore.
- Notably:
- <ul>
- <li>Mirroring a repository that uses Git-LFS - Only the pointer files, not the large files, are mirrored.</li>
- <li>Federation - Only the pointer files, not the large files, are transfered.</li>
- </ul>
- </p>
- </div>
-
- <div class="alert alert-info">
- <h3><center>GitBlit Configuration</center></h3>
- <p>GitBlit provides the following configuration items when using the filestore:
- <h4>filestore.storageFolder</h4>
- <p>Defines the path on the server where filestore objects are to be saved. This defaults to <code>${baseFolder}/lfs</code></p>
- <h4>filestore.maxUploadSize</h4>
- <p>Defines the maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. This defaults to <code>-1</code> indicating no limits.</p>
- </p>
- </div>
-
+
</div>
</div>
</div>
diff --git a/src/main/java/com/gitblit/wicket/pages/RootPage.java b/src/main/java/com/gitblit/wicket/pages/RootPage.java
index 6ed5a35..12779ca 100644
--- a/src/main/java/com/gitblit/wicket/pages/RootPage.java
+++ b/src/main/java/com/gitblit/wicket/pages/RootPage.java
@@ -197,9 +197,9 @@
}
navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class,
getRootPageParameters()));
- if (user.canAdmin()) {
- navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters()));
- }
+
+ navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters()));
+
navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters()));
if (allowLucene) {
navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class));
diff --git a/src/site/setup_filestore.mkd b/src/site/setup_filestore.mkd
new file mode 100644
index 0000000..9ffc9c6
--- /dev/null
+++ b/src/site/setup_filestore.mkd
@@ -0,0 +1,61 @@
+## Configure Git Large File Storage
+
+Gitblit provides a filestore that supports the [Git Large File Storage (LFS) API](https://git-lfs.github.com/).
+
+### Server Configuration
+
+Gitblit is configured to work straight away. However you may want to update the following in `gitblit.properties`:
+
+<table class="table">
+<thead>
+<tr><th>parameter</th><th>value</th><th>description</th></tr>
+</thead>
+<tbody>
+<tr>
+ <th>filestore.storageFolder</th><td>${baseFolder}/lfs</td>
+ <td>The path on the server where filestore objects are to be saved.</td>
+</tr>
+<tr>
+ <th>filestore.maxUploadSize</th><td>-1</td>
+ <td>The maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. The default of -1 indicates no limits.</td>
+</tr>
+</tbody>
+</table>
+
+
+### Limitations
+
+Gitblit currently provides a server-only implementation of the opensource Git LFS API.
+
+1. Files in the filestore are not currently searchable by Lucene.
+2. Mirroring a repository that uses Git LFS will only mirror the pointer files, not the large files.
+3. Federation - Only the pointer files, not the large files, are transfered.
+
+Items 2 & 3 are pending [JGit Git LFS client capabilities](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333).
+
+
+### How does it work?
+
+1. Files that should be handled by Git LFS are defined in the `.gitattributes` file.
+2. Git LFS installs a pre-commit hook when installed `git lfs install`.
+3. When a commit is made the pre-commit hook replaces the defined Git LFS files with a pointer file containing metadata about the file so that it can be found later.
+4. When a commit is pushed, the changeset is sent to the git repository and the large files are sent to the filestore.
+
+For further details check out the [Git LFS specification](https://github.com/github/git-lfs/blob/master/docs/spec.md).
+
+### Convert/Migrate existing repository
+
+It is possible to migrate an existing repository containing large files to one that leverages the filestore. However, commit hash history will be altered.
+
+The following command may be run on a local repository:
+
+ git filter-branch --prune-empty --tree-filter '
+ git lfs track "*.docx" "*.pdf" > /dev/null
+ git add .gitattributes
+ git ls-files | xargs -d "\n" git check-attr filter | grep "filter: lfs" | sed -r "s/(.*): filter: lfs/\1/" | xargs -d "\n" -r bash -c "git rm -f --cached \"\$@\"; git add \"\$@\"" bash \
+ ' --tag-name-filter cat -- --all
+
+
+### Further Considerations
+
+While [other Git LFS implementations are available](https://github.com/github/git-lfs/wiki/Implementations) as there is no current [JGit LFS client capability](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333), Gitblit will be unable to access them.
\ No newline at end of file
--
Gitblit v1.9.1