commit | author | age
|
bd0e83
|
1 |
/* |
PM |
2 |
* Copyright 2015 gitblit.com. |
|
3 |
* |
|
4 |
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
5 |
* you may not use this file except in compliance with the License. |
|
6 |
* You may obtain a copy of the License at |
|
7 |
* |
|
8 |
* http://www.apache.org/licenses/LICENSE-2.0 |
|
9 |
* |
|
10 |
* Unless required by applicable law or agreed to in writing, software |
|
11 |
* distributed under the License is distributed on an "AS IS" BASIS, |
|
12 |
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
13 |
* See the License for the specific language governing permissions and |
|
14 |
* limitations under the License. |
|
15 |
*/ |
|
16 |
package com.gitblit.manager; |
|
17 |
|
|
18 |
import java.io.File; |
|
19 |
import java.io.FileInputStream; |
|
20 |
import java.io.FileOutputStream; |
|
21 |
import java.io.FileReader; |
|
22 |
import java.io.IOException; |
|
23 |
import java.io.InputStream; |
|
24 |
import java.io.OutputStream; |
|
25 |
import java.io.RandomAccessFile; |
|
26 |
import java.lang.reflect.Type; |
|
27 |
import java.nio.file.Files; |
|
28 |
import java.text.MessageFormat; |
|
29 |
import java.util.ArrayList; |
|
30 |
import java.util.Collection; |
|
31 |
import java.util.Date; |
|
32 |
import java.util.Iterator; |
|
33 |
import java.util.List; |
|
34 |
import java.util.Map; |
|
35 |
import java.util.concurrent.ConcurrentHashMap; |
|
36 |
import java.util.regex.Pattern; |
|
37 |
|
|
38 |
import org.apache.commons.codec.digest.DigestUtils; |
|
39 |
import org.apache.commons.io.FileUtils; |
|
40 |
import org.apache.commons.io.IOUtils; |
|
41 |
import org.bouncycastle.util.io.StreamOverflowException; |
|
42 |
import org.eclipse.jetty.io.EofException; |
|
43 |
import org.slf4j.Logger; |
|
44 |
import org.slf4j.LoggerFactory; |
|
45 |
|
|
46 |
import com.gitblit.IStoredSettings; |
|
47 |
import com.gitblit.Keys; |
|
48 |
import com.gitblit.models.FilestoreModel.Status; |
|
49 |
import com.gitblit.models.FilestoreModel; |
|
50 |
import com.gitblit.models.RepositoryModel; |
|
51 |
import com.gitblit.models.UserModel; |
|
52 |
import com.gitblit.utils.ArrayUtils; |
|
53 |
import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter; |
|
54 |
import com.google.gson.ExclusionStrategy; |
|
55 |
import com.google.gson.Gson; |
|
56 |
import com.google.gson.GsonBuilder; |
|
57 |
import com.google.gson.reflect.TypeToken; |
|
58 |
import com.google.inject.Inject; |
|
59 |
import com.google.inject.Singleton; |
|
60 |
|
|
61 |
/** |
|
62 |
* FilestoreManager handles files uploaded via: |
|
63 |
* + git-lfs |
|
64 |
* + ticket attachment (TBD) |
|
65 |
* |
|
66 |
* Files are stored using their SHA256 hash (as per git-lfs) |
|
67 |
* If the same file is uploaded through different repositories no additional space is used |
|
68 |
* Access is controlled through the current repository permissions. |
|
69 |
* |
|
70 |
* TODO: Identify what and how the actual BLOBs should work with federation |
|
71 |
* |
|
72 |
* @author Paul Martin |
|
73 |
* |
|
74 |
*/ |
|
75 |
@Singleton |
|
76 |
public class FilestoreManager implements IFilestoreManager { |
|
77 |
|
|
78 |
private final Logger logger = LoggerFactory.getLogger(getClass()); |
|
79 |
|
|
80 |
private final IRuntimeManager runtimeManager; |
|
81 |
|
|
82 |
private final IStoredSettings settings; |
|
83 |
|
|
84 |
public static final int UNDEFINED_SIZE = -1; |
|
85 |
|
|
86 |
private static final String METAFILE = "filestore.json"; |
|
87 |
|
|
88 |
private static final String METAFILE_TMP = "filestore.json.tmp"; |
|
89 |
|
|
90 |
protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType(); |
|
91 |
|
|
92 |
private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>(); |
|
93 |
|
|
94 |
|
|
95 |
@Inject |
|
96 |
FilestoreManager( |
|
97 |
IRuntimeManager runtimeManager) { |
|
98 |
this.runtimeManager = runtimeManager; |
|
99 |
this.settings = runtimeManager.getSettings(); |
|
100 |
} |
|
101 |
|
|
102 |
@Override |
|
103 |
public IManager start() { |
|
104 |
|
e41e8f
|
105 |
// Try to load any existing metadata |
JM |
106 |
File dir = getStorageFolder(); |
|
107 |
dir.mkdirs(); |
|
108 |
File metadata = new File(dir, METAFILE); |
bd0e83
|
109 |
|
PM |
110 |
if (metadata.exists()) { |
|
111 |
Collection<FilestoreModel> items = null; |
|
112 |
|
|
113 |
Gson gson = gson(); |
|
114 |
try (FileReader file = new FileReader(metadata)) { |
|
115 |
items = gson.fromJson(file, METAFILE_TYPE); |
|
116 |
file.close(); |
|
117 |
|
|
118 |
} catch (IOException e) { |
|
119 |
e.printStackTrace(); |
|
120 |
} |
|
121 |
|
|
122 |
for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) { |
|
123 |
FilestoreModel model = itr.next(); |
|
124 |
fileCache.put(model.oid, model); |
|
125 |
} |
|
126 |
|
|
127 |
logger.info("Loaded {} items from filestore metadata file", fileCache.size()); |
|
128 |
} |
|
129 |
else |
|
130 |
{ |
|
131 |
logger.info("No filestore metadata file found"); |
|
132 |
} |
|
133 |
|
|
134 |
return this; |
|
135 |
} |
|
136 |
|
|
137 |
@Override |
|
138 |
public IManager stop() { |
|
139 |
return this; |
|
140 |
} |
|
141 |
|
|
142 |
|
|
143 |
@Override |
|
144 |
public boolean isValidOid(String oid) { |
|
145 |
//NOTE: Assuming SHA256 support only as per git-lfs |
|
146 |
return Pattern.matches("[a-fA-F0-9]{64}", oid); |
|
147 |
} |
|
148 |
|
|
149 |
@Override |
|
150 |
public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) { |
|
151 |
|
|
152 |
//Handle access control |
|
153 |
if (!user.canPush(repo)) { |
|
154 |
if (user == UserModel.ANONYMOUS) { |
|
155 |
return Status.AuthenticationRequired; |
|
156 |
} else { |
|
157 |
return Status.Error_Unauthorized; |
|
158 |
} |
|
159 |
} |
|
160 |
|
|
161 |
//Handle object details |
|
162 |
if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; } |
|
163 |
|
|
164 |
if (fileCache.containsKey(oid)) { |
|
165 |
FilestoreModel item = fileCache.get(oid); |
|
166 |
|
|
167 |
if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) { |
|
168 |
return Status.Error_Size_Mismatch; |
|
169 |
} |
|
170 |
|
|
171 |
item.addRepository(repo.name); |
|
172 |
|
|
173 |
if (item.isInErrorState()) { |
|
174 |
item.reset(user, size); |
|
175 |
} |
|
176 |
} else { |
|
177 |
|
|
178 |
if (size < 0) {return Status.Error_Invalid_Size; } |
|
179 |
if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; } |
|
180 |
|
|
181 |
FilestoreModel model = new FilestoreModel(oid, size, user, repo.name); |
|
182 |
fileCache.put(oid, model); |
|
183 |
saveFilestoreModel(model); |
|
184 |
} |
|
185 |
|
|
186 |
return fileCache.get(oid).getStatus(); |
|
187 |
} |
|
188 |
|
|
189 |
@Override |
|
190 |
public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) { |
|
191 |
|
|
192 |
//Access control and object logic |
|
193 |
Status state = addObject(oid, size, user, repo); |
|
194 |
|
|
195 |
if (state != Status.Upload_Pending) { |
|
196 |
return state; |
|
197 |
} |
|
198 |
|
|
199 |
FilestoreModel model = fileCache.get(oid); |
|
200 |
|
|
201 |
if (!model.actionUpload(user)) { |
|
202 |
return Status.Upload_In_Progress; |
|
203 |
} else { |
|
204 |
long actualSize = 0; |
|
205 |
File file = getStoragePath(oid); |
|
206 |
|
|
207 |
try { |
|
208 |
file.getParentFile().mkdirs(); |
|
209 |
file.createNewFile(); |
|
210 |
|
|
211 |
try (FileOutputStream streamOut = new FileOutputStream(file)) { |
|
212 |
|
|
213 |
actualSize = IOUtils.copyLarge(streamIn, streamOut); |
|
214 |
|
|
215 |
streamOut.flush(); |
|
216 |
streamOut.close(); |
|
217 |
|
|
218 |
if (model.getSize() != actualSize) { |
|
219 |
model.setStatus(Status.Error_Size_Mismatch, user); |
|
220 |
|
|
221 |
logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}", |
|
222 |
oid, model.getSize(), actualSize)); |
|
223 |
} else { |
|
224 |
String actualOid = ""; |
|
225 |
|
|
226 |
try (FileInputStream fileForHash = new FileInputStream(file)) { |
|
227 |
actualOid = DigestUtils.sha256Hex(fileForHash); |
|
228 |
fileForHash.close(); |
|
229 |
} |
|
230 |
|
|
231 |
if (oid.equalsIgnoreCase(actualOid)) { |
|
232 |
model.setStatus(Status.Available, user); |
|
233 |
} else { |
|
234 |
model.setStatus(Status.Error_Hash_Mismatch, user); |
|
235 |
|
|
236 |
logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid)); |
|
237 |
} |
|
238 |
} |
|
239 |
} |
|
240 |
} catch (Exception e) { |
|
241 |
|
|
242 |
model.setStatus(Status.Error_Unknown, user); |
|
243 |
logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e); |
|
244 |
} finally { |
|
245 |
saveFilestoreModel(model); |
|
246 |
} |
|
247 |
|
|
248 |
if (model.isInErrorState()) { |
|
249 |
file.delete(); |
|
250 |
model.removeRepository(repo.name); |
|
251 |
} |
|
252 |
} |
|
253 |
|
|
254 |
return model.getStatus(); |
|
255 |
} |
|
256 |
|
|
257 |
private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) { |
|
258 |
|
|
259 |
//Access Control |
|
260 |
if (!user.canView(repo)) { |
|
261 |
if (user == UserModel.ANONYMOUS) { |
|
262 |
return Status.AuthenticationRequired; |
|
263 |
} else { |
|
264 |
return Status.Error_Unauthorized; |
|
265 |
} |
|
266 |
} |
|
267 |
|
|
268 |
//Object Logic |
|
269 |
if (!isValidOid(oid)) { |
|
270 |
return Status.Error_Invalid_Oid; |
|
271 |
} |
|
272 |
|
|
273 |
if (!fileCache.containsKey(oid)) { |
|
274 |
return Status.Unavailable; |
|
275 |
} |
|
276 |
|
|
277 |
FilestoreModel item = fileCache.get(oid); |
|
278 |
|
|
279 |
if (item.getStatus() == Status.Available) { |
|
280 |
return Status.Available; |
|
281 |
} |
|
282 |
|
|
283 |
return Status.Unavailable; |
|
284 |
} |
|
285 |
|
|
286 |
@Override |
|
287 |
public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) { |
|
288 |
|
|
289 |
if (canGetObject(oid, user, repo) == Status.Available) { |
|
290 |
return fileCache.get(oid); |
|
291 |
} |
|
292 |
|
|
293 |
return null; |
|
294 |
} |
|
295 |
|
|
296 |
@Override |
|
297 |
public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) { |
|
298 |
|
|
299 |
//Access control and object logic |
|
300 |
Status status = canGetObject(oid, user, repo); |
|
301 |
|
|
302 |
if (status != Status.Available) { |
|
303 |
return status; |
|
304 |
} |
|
305 |
|
|
306 |
FilestoreModel item = fileCache.get(oid); |
|
307 |
|
|
308 |
if (streamOut != null) { |
|
309 |
try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) { |
|
310 |
|
|
311 |
IOUtils.copyLarge(streamIn, streamOut); |
|
312 |
|
|
313 |
streamOut.flush(); |
|
314 |
streamIn.close(); |
|
315 |
} catch (EofException e) { |
|
316 |
logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e); |
|
317 |
return Status.Error_Unexpected_Stream_End; |
|
318 |
} catch (Exception e) { |
|
319 |
logger.error(MessageFormat.format("Failed to download blob {0}", oid), e); |
|
320 |
return Status.Error_Unknown; |
|
321 |
} |
|
322 |
} |
|
323 |
|
|
324 |
return item.getStatus(); |
|
325 |
} |
|
326 |
|
|
327 |
@Override |
|
328 |
public List<FilestoreModel> getAllObjects() { |
|
329 |
return new ArrayList<FilestoreModel>(fileCache.values()); |
|
330 |
} |
|
331 |
|
|
332 |
@Override |
|
333 |
public File getStorageFolder() { |
|
334 |
return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs"); |
|
335 |
} |
|
336 |
|
|
337 |
@Override |
|
338 |
public File getStoragePath(String oid) { |
|
339 |
return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2))); |
|
340 |
} |
|
341 |
|
|
342 |
@Override |
|
343 |
public long getMaxUploadSize() { |
|
344 |
return settings.getLong(Keys.filestore.maxUploadSize, -1); |
|
345 |
} |
|
346 |
|
|
347 |
@Override |
|
348 |
public long getFilestoreUsedByteCount() { |
|
349 |
Iterator<FilestoreModel> iterator = fileCache.values().iterator(); |
|
350 |
long total = 0; |
|
351 |
|
|
352 |
while (iterator.hasNext()) { |
|
353 |
|
|
354 |
FilestoreModel item = iterator.next(); |
|
355 |
if (item.getStatus() == Status.Available) { |
|
356 |
total += item.getSize(); |
|
357 |
} |
|
358 |
} |
|
359 |
|
|
360 |
return total; |
|
361 |
} |
|
362 |
|
|
363 |
@Override |
|
364 |
public long getFilestoreAvailableByteCount() { |
|
365 |
|
|
366 |
try { |
|
367 |
return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace(); |
|
368 |
} catch (IOException e) { |
|
369 |
logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e)); |
|
370 |
} |
|
371 |
|
|
372 |
return UNDEFINED_SIZE; |
|
373 |
}; |
|
374 |
|
|
375 |
private synchronized void saveFilestoreModel(FilestoreModel model) { |
|
376 |
|
|
377 |
File metaFile = new File(getStorageFolder(), METAFILE); |
|
378 |
File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP); |
|
379 |
boolean isNewFile = false; |
|
380 |
|
|
381 |
try { |
|
382 |
if (!metaFile.exists()) { |
|
383 |
metaFile.getParentFile().mkdirs(); |
|
384 |
metaFile.createNewFile(); |
|
385 |
isNewFile = true; |
|
386 |
} |
|
387 |
FileUtils.copyFile(metaFile, metaFileTmp); |
|
388 |
|
|
389 |
} catch (IOException e) { |
|
390 |
logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); |
|
391 |
} |
|
392 |
|
|
393 |
try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) { |
|
394 |
|
|
395 |
if (isNewFile) { |
|
396 |
fs.writeBytes("["); |
|
397 |
} else { |
|
398 |
fs.seek(fs.length() - 1); |
|
399 |
fs.writeBytes(","); |
|
400 |
} |
|
401 |
|
|
402 |
fs.writeBytes(gson().toJson(model)); |
|
403 |
fs.writeBytes("]"); |
|
404 |
|
|
405 |
fs.close(); |
|
406 |
|
|
407 |
} catch (IOException e) { |
|
408 |
logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e); |
|
409 |
} |
|
410 |
|
|
411 |
try { |
|
412 |
if (metaFileTmp.exists()) { |
|
413 |
FileUtils.copyFile(metaFileTmp, metaFile); |
|
414 |
|
|
415 |
metaFileTmp.delete(); |
|
416 |
} else { |
|
417 |
logger.error("Writing filestore model to file {0}", METAFILE); |
|
418 |
} |
|
419 |
} |
|
420 |
catch (IOException e) { |
|
421 |
logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); |
|
422 |
} |
|
423 |
} |
|
424 |
|
|
425 |
/* |
|
426 |
* Intended for testing purposes only |
|
427 |
*/ |
|
428 |
public void clearFilestoreCache() { |
|
429 |
fileCache.clear(); |
|
430 |
} |
|
431 |
|
|
432 |
private static Gson gson(ExclusionStrategy... strategies) { |
|
433 |
GsonBuilder builder = new GsonBuilder(); |
|
434 |
builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter()); |
|
435 |
if (!ArrayUtils.isEmpty(strategies)) { |
|
436 |
builder.setExclusionStrategies(strategies); |
|
437 |
} |
|
438 |
return builder.create(); |
|
439 |
} |
|
440 |
|
|
441 |
} |