commit | author | age
|
bd0e83
|
1 |
/* |
PM |
2 |
* Copyright 2015 gitblit.com. |
|
3 |
* |
|
4 |
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
5 |
* you may not use this file except in compliance with the License. |
|
6 |
* You may obtain a copy of the License at |
|
7 |
* |
|
8 |
* http://www.apache.org/licenses/LICENSE-2.0 |
|
9 |
* |
|
10 |
* Unless required by applicable law or agreed to in writing, software |
|
11 |
* distributed under the License is distributed on an "AS IS" BASIS, |
|
12 |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
13 |
* See the License for the specific language governing permissions and |
|
14 |
* limitations under the License. |
|
15 |
*/ |
|
16 |
package com.gitblit.manager; |
|
17 |
|
28cf62
|
18 |
import java.io.EOFException; |
bd0e83
|
19 |
import java.io.File; |
PM |
20 |
import java.io.FileInputStream; |
|
21 |
import java.io.FileOutputStream; |
|
22 |
import java.io.FileReader; |
|
23 |
import java.io.IOException; |
|
24 |
import java.io.InputStream; |
|
25 |
import java.io.OutputStream; |
|
26 |
import java.io.RandomAccessFile; |
|
27 |
import java.lang.reflect.Type; |
|
28 |
import java.nio.file.Files; |
|
29 |
import java.text.MessageFormat; |
|
30 |
import java.util.ArrayList; |
|
31 |
import java.util.Collection; |
|
32 |
import java.util.Date; |
|
33 |
import java.util.Iterator; |
|
34 |
import java.util.List; |
|
35 |
import java.util.Map; |
|
36 |
import java.util.concurrent.ConcurrentHashMap; |
|
37 |
import java.util.regex.Pattern; |
|
38 |
|
|
39 |
import org.apache.commons.codec.digest.DigestUtils; |
|
40 |
import org.apache.commons.io.FileUtils; |
|
41 |
import org.apache.commons.io.IOUtils; |
|
42 |
import org.slf4j.Logger; |
|
43 |
import org.slf4j.LoggerFactory; |
|
44 |
|
|
45 |
import com.gitblit.IStoredSettings; |
|
46 |
import com.gitblit.Keys; |
|
47 |
import com.gitblit.models.FilestoreModel; |
28cf62
|
48 |
import com.gitblit.models.FilestoreModel.Status; |
bd0e83
|
49 |
import com.gitblit.models.RepositoryModel; |
PM |
50 |
import com.gitblit.models.UserModel; |
|
51 |
import com.gitblit.utils.ArrayUtils; |
|
52 |
import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter; |
|
53 |
import com.google.gson.ExclusionStrategy; |
|
54 |
import com.google.gson.Gson; |
|
55 |
import com.google.gson.GsonBuilder; |
|
56 |
import com.google.gson.reflect.TypeToken; |
|
57 |
import com.google.inject.Inject; |
|
58 |
import com.google.inject.Singleton; |
|
59 |
|
|
60 |
/** |
|
61 |
* FilestoreManager handles files uploaded via: |
|
62 |
* + git-lfs |
|
63 |
* + ticket attachment (TBD) |
28cf62
|
64 |
* |
bd0e83
|
65 |
* Files are stored using their SHA256 hash (as per git-lfs) |
PM |
66 |
* If the same file is uploaded through different repositories no additional space is used |
|
67 |
* Access is controlled through the current repository permissions. |
|
68 |
* |
|
69 |
* TODO: Identify what and how the actual BLOBs should work with federation |
|
70 |
* |
|
71 |
* @author Paul Martin |
|
72 |
* |
|
73 |
*/ |
|
74 |
@Singleton |
|
75 |
public class FilestoreManager implements IFilestoreManager { |
|
76 |
|
|
77 |
private final Logger logger = LoggerFactory.getLogger(getClass()); |
28cf62
|
78 |
|
bd0e83
|
79 |
private final IRuntimeManager runtimeManager; |
697905
|
80 |
|
bd0e83
|
81 |
private final IStoredSettings settings; |
28cf62
|
82 |
|
bd0e83
|
83 |
public static final int UNDEFINED_SIZE = -1; |
28cf62
|
84 |
|
bd0e83
|
85 |
private static final String METAFILE = "filestore.json"; |
28cf62
|
86 |
|
bd0e83
|
87 |
private static final String METAFILE_TMP = "filestore.json.tmp"; |
28cf62
|
88 |
|
bd0e83
|
89 |
protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType(); |
28cf62
|
90 |
|
bd0e83
|
91 |
private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>(); |
28cf62
|
92 |
|
JM |
93 |
|
bd0e83
|
94 |
@Inject |
eecaad
|
95 |
public FilestoreManager(IRuntimeManager runtimeManager) { |
bd0e83
|
96 |
this.runtimeManager = runtimeManager; |
PM |
97 |
this.settings = runtimeManager.getSettings(); |
|
98 |
} |
28cf62
|
99 |
|
bd0e83
|
100 |
@Override |
PM |
101 |
public IManager start() { |
|
102 |
|
e41e8f
|
103 |
// Try to load any existing metadata |
JM |
104 |
File dir = getStorageFolder(); |
|
105 |
dir.mkdirs(); |
|
106 |
File metadata = new File(dir, METAFILE); |
28cf62
|
107 |
|
bd0e83
|
108 |
if (metadata.exists()) { |
PM |
109 |
Collection<FilestoreModel> items = null; |
28cf62
|
110 |
|
bd0e83
|
111 |
Gson gson = gson(); |
PM |
112 |
try (FileReader file = new FileReader(metadata)) { |
|
113 |
items = gson.fromJson(file, METAFILE_TYPE); |
|
114 |
file.close(); |
28cf62
|
115 |
|
bd0e83
|
116 |
} catch (IOException e) { |
PM |
117 |
e.printStackTrace(); |
|
118 |
} |
28cf62
|
119 |
|
bd0e83
|
120 |
for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) { |
PM |
121 |
FilestoreModel model = itr.next(); |
|
122 |
fileCache.put(model.oid, model); |
|
123 |
} |
|
124 |
|
|
125 |
logger.info("Loaded {} items from filestore metadata file", fileCache.size()); |
|
126 |
} |
|
127 |
else |
|
128 |
{ |
|
129 |
logger.info("No filestore metadata file found"); |
|
130 |
} |
28cf62
|
131 |
|
bd0e83
|
132 |
return this; |
PM |
133 |
} |
|
134 |
|
|
135 |
@Override |
|
136 |
public IManager stop() { |
|
137 |
return this; |
|
138 |
} |
|
139 |
|
|
140 |
|
|
141 |
@Override |
|
142 |
public boolean isValidOid(String oid) { |
|
143 |
//NOTE: Assuming SHA256 support only as per git-lfs |
|
144 |
return Pattern.matches("[a-fA-F0-9]{64}", oid); |
|
145 |
} |
28cf62
|
146 |
|
bd0e83
|
147 |
@Override |
PM |
148 |
public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) { |
|
149 |
|
|
150 |
//Handle access control |
|
151 |
if (!user.canPush(repo)) { |
|
152 |
if (user == UserModel.ANONYMOUS) { |
|
153 |
return Status.AuthenticationRequired; |
|
154 |
} else { |
|
155 |
return Status.Error_Unauthorized; |
|
156 |
} |
|
157 |
} |
28cf62
|
158 |
|
bd0e83
|
159 |
//Handle object details |
PM |
160 |
if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; } |
28cf62
|
161 |
|
bd0e83
|
162 |
if (fileCache.containsKey(oid)) { |
PM |
163 |
FilestoreModel item = fileCache.get(oid); |
28cf62
|
164 |
|
bd0e83
|
165 |
if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) { |
PM |
166 |
return Status.Error_Size_Mismatch; |
|
167 |
} |
28cf62
|
168 |
|
bd0e83
|
169 |
item.addRepository(repo.name); |
28cf62
|
170 |
|
bd0e83
|
171 |
if (item.isInErrorState()) { |
PM |
172 |
item.reset(user, size); |
|
173 |
} |
|
174 |
} else { |
28cf62
|
175 |
|
bd0e83
|
176 |
if (size < 0) {return Status.Error_Invalid_Size; } |
PM |
177 |
if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; } |
28cf62
|
178 |
|
JM |
179 |
FilestoreModel model = new FilestoreModel(oid, size, user, repo.name); |
bd0e83
|
180 |
fileCache.put(oid, model); |
PM |
181 |
saveFilestoreModel(model); |
|
182 |
} |
28cf62
|
183 |
|
bd0e83
|
184 |
return fileCache.get(oid).getStatus(); |
PM |
185 |
} |
|
186 |
|
|
187 |
@Override |
|
188 |
public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) { |
|
189 |
|
|
190 |
//Access control and object logic |
28cf62
|
191 |
Status state = addObject(oid, size, user, repo); |
JM |
192 |
|
|
193 |
if (state != Status.Upload_Pending) { |
bd0e83
|
194 |
return state; |
PM |
195 |
} |
28cf62
|
196 |
|
bd0e83
|
197 |
FilestoreModel model = fileCache.get(oid); |
28cf62
|
198 |
|
bd0e83
|
199 |
if (!model.actionUpload(user)) { |
PM |
200 |
return Status.Upload_In_Progress; |
|
201 |
} else { |
|
202 |
long actualSize = 0; |
|
203 |
File file = getStoragePath(oid); |
|
204 |
|
|
205 |
try { |
|
206 |
file.getParentFile().mkdirs(); |
|
207 |
file.createNewFile(); |
28cf62
|
208 |
|
bd0e83
|
209 |
try (FileOutputStream streamOut = new FileOutputStream(file)) { |
28cf62
|
210 |
|
bd0e83
|
211 |
actualSize = IOUtils.copyLarge(streamIn, streamOut); |
28cf62
|
212 |
|
bd0e83
|
213 |
streamOut.flush(); |
PM |
214 |
streamOut.close(); |
28cf62
|
215 |
|
bd0e83
|
216 |
if (model.getSize() != actualSize) { |
PM |
217 |
model.setStatus(Status.Error_Size_Mismatch, user); |
28cf62
|
218 |
|
JM |
219 |
logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}", |
bd0e83
|
220 |
oid, model.getSize(), actualSize)); |
PM |
221 |
} else { |
|
222 |
String actualOid = ""; |
28cf62
|
223 |
|
bd0e83
|
224 |
try (FileInputStream fileForHash = new FileInputStream(file)) { |
PM |
225 |
actualOid = DigestUtils.sha256Hex(fileForHash); |
|
226 |
fileForHash.close(); |
|
227 |
} |
28cf62
|
228 |
|
bd0e83
|
229 |
if (oid.equalsIgnoreCase(actualOid)) { |
PM |
230 |
model.setStatus(Status.Available, user); |
|
231 |
} else { |
|
232 |
model.setStatus(Status.Error_Hash_Mismatch, user); |
28cf62
|
233 |
|
bd0e83
|
234 |
logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid)); |
PM |
235 |
} |
|
236 |
} |
|
237 |
} |
|
238 |
} catch (Exception e) { |
28cf62
|
239 |
|
bd0e83
|
240 |
model.setStatus(Status.Error_Unknown, user); |
PM |
241 |
logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e); |
|
242 |
} finally { |
|
243 |
saveFilestoreModel(model); |
|
244 |
} |
28cf62
|
245 |
|
bd0e83
|
246 |
if (model.isInErrorState()) { |
PM |
247 |
file.delete(); |
|
248 |
model.removeRepository(repo.name); |
|
249 |
} |
|
250 |
} |
28cf62
|
251 |
|
bd0e83
|
252 |
return model.getStatus(); |
PM |
253 |
} |
28cf62
|
254 |
|
bd0e83
|
255 |
private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) { |
28cf62
|
256 |
|
bd0e83
|
257 |
//Access Control |
PM |
258 |
if (!user.canView(repo)) { |
|
259 |
if (user == UserModel.ANONYMOUS) { |
|
260 |
return Status.AuthenticationRequired; |
|
261 |
} else { |
|
262 |
return Status.Error_Unauthorized; |
|
263 |
} |
|
264 |
} |
|
265 |
|
|
266 |
//Object Logic |
28cf62
|
267 |
if (!isValidOid(oid)) { |
bd0e83
|
268 |
return Status.Error_Invalid_Oid; |
PM |
269 |
} |
28cf62
|
270 |
|
JM |
271 |
if (!fileCache.containsKey(oid)) { |
bd0e83
|
272 |
return Status.Unavailable; |
PM |
273 |
} |
28cf62
|
274 |
|
bd0e83
|
275 |
FilestoreModel item = fileCache.get(oid); |
28cf62
|
276 |
|
bd0e83
|
277 |
if (item.getStatus() == Status.Available) { |
PM |
278 |
return Status.Available; |
|
279 |
} |
28cf62
|
280 |
|
bd0e83
|
281 |
return Status.Unavailable; |
PM |
282 |
} |
28cf62
|
283 |
|
bd0e83
|
284 |
@Override |
PM |
285 |
public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) { |
28cf62
|
286 |
|
bd0e83
|
287 |
if (canGetObject(oid, user, repo) == Status.Available) { |
PM |
288 |
return fileCache.get(oid); |
|
289 |
} |
28cf62
|
290 |
|
bd0e83
|
291 |
return null; |
PM |
292 |
} |
|
293 |
|
|
294 |
@Override |
|
295 |
public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) { |
28cf62
|
296 |
|
bd0e83
|
297 |
//Access control and object logic |
PM |
298 |
Status status = canGetObject(oid, user, repo); |
28cf62
|
299 |
|
JM |
300 |
if (status != Status.Available) { |
bd0e83
|
301 |
return status; |
PM |
302 |
} |
28cf62
|
303 |
|
bd0e83
|
304 |
FilestoreModel item = fileCache.get(oid); |
28cf62
|
305 |
|
bd0e83
|
306 |
if (streamOut != null) { |
PM |
307 |
try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) { |
28cf62
|
308 |
|
bd0e83
|
309 |
IOUtils.copyLarge(streamIn, streamOut); |
28cf62
|
310 |
|
bd0e83
|
311 |
streamOut.flush(); |
PM |
312 |
streamIn.close(); |
28cf62
|
313 |
} catch (EOFException e) { |
bd0e83
|
314 |
logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e); |
PM |
315 |
return Status.Error_Unexpected_Stream_End; |
|
316 |
} catch (Exception e) { |
|
317 |
logger.error(MessageFormat.format("Failed to download blob {0}", oid), e); |
|
318 |
return Status.Error_Unknown; |
|
319 |
} |
|
320 |
} |
28cf62
|
321 |
|
bd0e83
|
322 |
return item.getStatus(); |
PM |
323 |
} |
|
324 |
|
|
325 |
@Override |
eecaad
|
326 |
public List<FilestoreModel> getAllObjects(List<RepositoryModel> viewableRepositories) { |
697905
|
327 |
|
PM |
328 |
List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size()); |
|
329 |
|
|
330 |
for (RepositoryModel repository : viewableRepositories) { |
|
331 |
viewableRepositoryNames.add(repository.name); |
|
332 |
} |
|
333 |
|
|
334 |
if (viewableRepositoryNames.size() == 0) { |
|
335 |
return null; |
|
336 |
} |
|
337 |
|
|
338 |
final Collection<FilestoreModel> allFiles = fileCache.values(); |
|
339 |
List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size()); |
|
340 |
|
|
341 |
for (FilestoreModel file : allFiles) { |
|
342 |
if (file.isInRepositoryList(viewableRepositoryNames)) { |
|
343 |
userViewableFiles.add(file); |
|
344 |
} |
|
345 |
} |
|
346 |
|
|
347 |
return userViewableFiles; |
bd0e83
|
348 |
} |
PM |
349 |
|
|
350 |
@Override |
|
351 |
public File getStorageFolder() { |
|
352 |
return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs"); |
|
353 |
} |
28cf62
|
354 |
|
bd0e83
|
355 |
@Override |
PM |
356 |
public File getStoragePath(String oid) { |
|
357 |
return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2))); |
|
358 |
} |
|
359 |
|
|
360 |
@Override |
|
361 |
public long getMaxUploadSize() { |
|
362 |
return settings.getLong(Keys.filestore.maxUploadSize, -1); |
|
363 |
} |
28cf62
|
364 |
|
bd0e83
|
365 |
@Override |
PM |
366 |
public long getFilestoreUsedByteCount() { |
|
367 |
Iterator<FilestoreModel> iterator = fileCache.values().iterator(); |
|
368 |
long total = 0; |
28cf62
|
369 |
|
bd0e83
|
370 |
while (iterator.hasNext()) { |
28cf62
|
371 |
|
bd0e83
|
372 |
FilestoreModel item = iterator.next(); |
PM |
373 |
if (item.getStatus() == Status.Available) { |
|
374 |
total += item.getSize(); |
|
375 |
} |
|
376 |
} |
28cf62
|
377 |
|
bd0e83
|
378 |
return total; |
PM |
379 |
} |
28cf62
|
380 |
|
bd0e83
|
381 |
@Override |
PM |
382 |
public long getFilestoreAvailableByteCount() { |
28cf62
|
383 |
|
bd0e83
|
384 |
try { |
PM |
385 |
return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace(); |
|
386 |
} catch (IOException e) { |
|
387 |
logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e)); |
|
388 |
} |
28cf62
|
389 |
|
bd0e83
|
390 |
return UNDEFINED_SIZE; |
PM |
391 |
}; |
28cf62
|
392 |
|
bd0e83
|
393 |
private synchronized void saveFilestoreModel(FilestoreModel model) { |
28cf62
|
394 |
|
bd0e83
|
395 |
File metaFile = new File(getStorageFolder(), METAFILE); |
PM |
396 |
File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP); |
|
397 |
boolean isNewFile = false; |
28cf62
|
398 |
|
bd0e83
|
399 |
try { |
PM |
400 |
if (!metaFile.exists()) { |
|
401 |
metaFile.getParentFile().mkdirs(); |
|
402 |
metaFile.createNewFile(); |
|
403 |
isNewFile = true; |
|
404 |
} |
|
405 |
FileUtils.copyFile(metaFile, metaFileTmp); |
28cf62
|
406 |
|
bd0e83
|
407 |
} catch (IOException e) { |
PM |
408 |
logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); |
|
409 |
} |
28cf62
|
410 |
|
bd0e83
|
411 |
try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) { |
28cf62
|
412 |
|
bd0e83
|
413 |
if (isNewFile) { |
PM |
414 |
fs.writeBytes("["); |
|
415 |
} else { |
|
416 |
fs.seek(fs.length() - 1); |
|
417 |
fs.writeBytes(","); |
|
418 |
} |
28cf62
|
419 |
|
bd0e83
|
420 |
fs.writeBytes(gson().toJson(model)); |
PM |
421 |
fs.writeBytes("]"); |
28cf62
|
422 |
|
bd0e83
|
423 |
fs.close(); |
28cf62
|
424 |
|
bd0e83
|
425 |
} catch (IOException e) { |
PM |
426 |
logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e); |
|
427 |
} |
28cf62
|
428 |
|
bd0e83
|
429 |
try { |
PM |
430 |
if (metaFileTmp.exists()) { |
|
431 |
FileUtils.copyFile(metaFileTmp, metaFile); |
28cf62
|
432 |
|
bd0e83
|
433 |
metaFileTmp.delete(); |
PM |
434 |
} else { |
|
435 |
logger.error("Writing filestore model to file {0}", METAFILE); |
|
436 |
} |
|
437 |
} |
|
438 |
catch (IOException e) { |
|
439 |
logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); |
|
440 |
} |
|
441 |
} |
28cf62
|
442 |
|
bd0e83
|
443 |
/* |
PM |
444 |
* Intended for testing purposes only |
|
445 |
*/ |
28cf62
|
446 |
@Override |
bd0e83
|
447 |
public void clearFilestoreCache() { |
PM |
448 |
fileCache.clear(); |
|
449 |
} |
28cf62
|
450 |
|
bd0e83
|
451 |
private static Gson gson(ExclusionStrategy... strategies) { |
PM |
452 |
GsonBuilder builder = new GsonBuilder(); |
|
453 |
builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter()); |
|
454 |
if (!ArrayUtils.isEmpty(strategies)) { |
|
455 |
builder.setExclusionStrategies(strategies); |
|
456 |
} |
|
457 |
return builder.create(); |
|
458 |
} |
28cf62
|
459 |
|
bd0e83
|
460 |
} |