| | |
| | |
|
| | | import java.io.ByteArrayOutputStream;
|
| | | import java.io.File;
|
| | | import java.io.FileInputStream;
|
| | | import java.io.IOException;
|
| | | import java.io.InputStream;
|
| | | import java.text.MessageFormat;
|
| | |
| | | import org.apache.lucene.store.Directory;
|
| | | import org.apache.lucene.store.FSDirectory;
|
| | | import org.apache.lucene.util.Version;
|
| | | import org.apache.tika.metadata.Metadata;
|
| | | import org.apache.tika.parser.AutoDetectParser;
|
| | | import org.apache.tika.parser.ParseContext;
|
| | | import org.apache.tika.parser.pdf.PDFParser;
|
| | | import org.apache.tika.sax.BodyContentHandler;
|
| | | import org.eclipse.jgit.diff.DiffEntry.ChangeType;
|
| | | import org.eclipse.jgit.lib.Constants;
|
| | | import org.eclipse.jgit.lib.FileMode;
|
| | |
| | | import org.slf4j.LoggerFactory;
|
| | |
|
| | | import com.gitblit.Constants.SearchObjectType;
|
| | | import com.gitblit.GitBlit;
|
| | | import com.gitblit.IStoredSettings;
|
| | | import com.gitblit.Keys;
|
| | | import com.gitblit.manager.FilestoreManager;
|
| | | import com.gitblit.manager.IFilestoreManager;
|
| | | import com.gitblit.manager.IRepositoryManager;
|
| | | import com.gitblit.models.PathModel.PathChangeModel;
|
| | | import com.gitblit.models.RefModel;
|
| | |
| | | public class LuceneService implements Runnable {
|
| | |
|
| | |
|
| | | private static final int INDEX_VERSION = 5;
|
| | | private static final int INDEX_VERSION = 6;
|
| | |
|
| | | private static final String FIELD_OBJECT_TYPE = "type";
|
| | | private static final String FIELD_PATH = "path";
|
| | |
| | | private static final String CONF_ALIAS = "aliases";
|
| | | private static final String CONF_BRANCH = "branches";
|
| | |
|
| | | private static final Version LUCENE_VERSION = Version.LUCENE_46;
|
| | | private static final Version LUCENE_VERSION = Version.LUCENE_4_10_0;
|
| | |
|
| | | private final Logger logger = LoggerFactory.getLogger(LuceneService.class);
|
| | |
|
| | | private final IStoredSettings storedSettings;
|
| | | private final IRepositoryManager repositoryManager;
|
| | | private final IFilestoreManager filestoreManager;
|
| | | |
| | | private final File repositoriesFolder;
|
| | |
|
| | | private final Map<String, IndexSearcher> searchers = new ConcurrentHashMap<String, IndexSearcher>();
|
| | |
| | |
|
| | | public LuceneService(
|
| | | IStoredSettings settings,
|
| | | IRepositoryManager repositoryManager) {
|
| | | IRepositoryManager repositoryManager, |
| | | IFilestoreManager filestoreManager) {
|
| | |
|
| | | this.storedSettings = settings;
|
| | | this.repositoryManager = repositoryManager;
|
| | | this.filestoreManager = filestoreManager;
|
| | | this.repositoriesFolder = repositoryManager.getRepositoriesFolder();
|
| | | String exts = luceneIgnoreExtensions;
|
| | | if (settings != null) {
|
| | |
| | | * Synchronously indexes a repository. This may build a complete index of a
|
| | | * repository or it may update an existing index.
|
| | | *
|
| | | * @param name
|
| | | * @param displayName
|
| | | * the name of the repository
|
| | | * @param repository
|
| | | * the repository object
|
| | |
| | | // skip non-annotated tags
|
| | | continue;
|
| | | }
|
| | | if (!tags.containsKey(tag.getObjectId())) {
|
| | | if (!tags.containsKey(tag.getReferencedObjectId().getName())) {
|
| | | tags.put(tag.getReferencedObjectId().getName(), new ArrayList<String>());
|
| | | }
|
| | | tags.get(tag.getReferencedObjectId().getName()).add(tag.displayName);
|
| | |
| | | && branch.equals(defaultBranch)) {
|
| | | // indexing "default" branch
|
| | | indexBranch = true;
|
| | | } else if (branch.getName().startsWith(com.gitblit.Constants.R_GITBLIT)) {
|
| | | // skip Gitblit internal branches
|
| | | } else if (branch.getName().startsWith(com.gitblit.Constants.R_META)) {
|
| | | // skip internal meta branches
|
| | | indexBranch = false;
|
| | | } else {
|
| | | // normal explicit branch check
|
| | |
| | | if (!paths.containsKey(path)) {
|
| | | continue;
|
| | | }
|
| | |
|
| | | //TODO: Figure out filestore oid the path - bit more involved than updating the index
|
| | | |
| | | // remove path from set
|
| | | ObjectId blobId = paths.remove(path);
|
| | | result.blobCount++;
|
| | |
| | | }
|
| | |
|
| | | // finished
|
| | | reader.release();
|
| | | reader.close();
|
| | |
|
| | | // commit all changes and reset the searcher
|
| | | config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION);
|
| | |
| | | }
|
| | |
|
| | | if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {
|
| | | String str = "";
|
| | | // read the blob content
|
| | | String str = JGitUtils.getStringContent(repository, commit.getTree(),
|
| | | if (path.isFilestoreItem()) {
|
| | | //Get file from filestore
|
| | | BodyContentHandler handler = new BodyContentHandler();
|
| | | Metadata metadata = new Metadata();
|
| | | PDFParser parser = new PDFParser();
|
| | | |
| | | ParseContext parseContext = new ParseContext();
|
| | | File lfsFile = filestoreManager.getStoragePath(path.getFilestoreOid());
|
| | | FileInputStream inputstream = new FileInputStream(lfsFile);
|
| | | parser.parse(inputstream, handler, metadata, parseContext);
|
| | | str = handler.toString();
|
| | | } else {
|
| | | str = JGitUtils.getStringContent(repository, commit.getTree(),
|
| | | path.path, encodings);
|
| | | }
|
| | | |
| | | if (str != null) {
|
| | | doc.add(new Field(FIELD_CONTENT, str, TextField.TYPE_STORED));
|
| | | writer.addDocument(doc);
|
| | |
| | | // skip non-annotated tags
|
| | | continue;
|
| | | }
|
| | | if (!tags.containsKey(tag.getObjectId())) {
|
| | | if (!tags.containsKey(tag.getObjectId().getName())) {
|
| | | tags.put(tag.getReferencedObjectId().getName(), new ArrayList<String>());
|
| | | }
|
| | | tags.get(tag.getReferencedObjectId().getName()).add(tag.displayName);
|
| | |
| | | && branch.equals(defaultBranch)) {
|
| | | // indexing "default" branch
|
| | | indexBranch = true;
|
| | | } else if (branch.getName().startsWith(com.gitblit.Constants.R_GITBLIT)) {
|
| | | // ignore internal Gitblit branches
|
| | | } else if (branch.getName().startsWith(com.gitblit.Constants.R_META)) {
|
| | | // ignore internal meta branches
|
| | | indexBranch = false;
|
| | | } else {
|
| | | // normal explicit branch check
|
| | |
| | | content = "";
|
| | | }
|
| | |
|
| | | int tabLength = storedSettings.getInteger(Keys.web.tabLength, 4);
|
| | | int fragmentLength = SearchObjectType.commit == result.type ? 512 : 150;
|
| | |
|
| | | QueryScorer scorer = new QueryScorer(query, "content");
|
| | |
| | | if (fragment.length() > fragmentLength) {
|
| | | fragment = fragment.substring(0, fragmentLength) + "...";
|
| | | }
|
| | | return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true) + "</pre>";
|
| | | return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true, tabLength) + "</pre>";
|
| | | }
|
| | |
|
| | | // make sure we have unique fragments
|