From 55c385e96e6594ec1ac3b5cd41ccd2df6048b696 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gmail.com>
Date: Tue, 15 Sep 2015 07:42:11 -0400
Subject: [PATCH] Merge pull request #915 from lucamilanesio/lucene-5.2.1
---
src/main/java/com/gitblit/service/LuceneService.java | 126 +++++++++++++++++++-----------------------
1 files changed, 57 insertions(+), 69 deletions(-)
diff --git a/src/main/java/com/gitblit/service/LuceneService.java b/src/main/java/com/gitblit/service/LuceneService.java
index 97fe9e1..59b1ff2 100644
--- a/src/main/java/com/gitblit/service/LuceneService.java
+++ b/src/main/java/com/gitblit/service/LuceneService.java
@@ -21,7 +21,6 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.lang.reflect.Method;
import java.text.MessageFormat;
import java.text.ParseException;
import java.util.ArrayList;
@@ -42,15 +41,16 @@
import org.apache.lucene.document.DateTools.Resolution;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
@@ -105,7 +105,7 @@
public class LuceneService implements Runnable {
- private static final int INDEX_VERSION = 5;
+ private static final int INDEX_VERSION = 6;
private static final String FIELD_OBJECT_TYPE = "type";
private static final String FIELD_PATH = "path";
@@ -125,7 +125,7 @@
private static final String CONF_ALIAS = "aliases";
private static final String CONF_BRANCH = "branches";
- private static final Version LUCENE_VERSION = Version.LUCENE_35;
+ private static final Version LUCENE_VERSION = Version.LUCENE_4_10_0;
private final Logger logger = LoggerFactory.getLogger(LuceneService.class);
@@ -194,7 +194,7 @@
* Synchronously indexes a repository. This may build a complete index of a
* repository or it may update an existing index.
*
- * @param name
+ * @param displayName
* the name of the repository
* @param repository
* the repository object
@@ -267,7 +267,7 @@
// close all writers
for (String writer : writers.keySet()) {
try {
- writers.get(writer).close(true);
+ writers.get(writer).close();
} catch (Throwable t) {
logger.error("Failed to close Lucene writer for " + writer, t);
}
@@ -437,7 +437,7 @@
// skip non-annotated tags
continue;
}
- if (!tags.containsKey(tag.getObjectId())) {
+ if (!tags.containsKey(tag.getReferencedObjectId().getName())) {
tags.put(tag.getReferencedObjectId().getName(), new ArrayList<String>());
}
tags.get(tag.getReferencedObjectId().getName()).add(tag.displayName);
@@ -476,8 +476,8 @@
&& branch.equals(defaultBranch)) {
// indexing "default" branch
indexBranch = true;
- } else if (branch.getName().startsWith(com.gitblit.Constants.R_GITBLIT)) {
- // skip Gitblit internal branches
+ } else if (branch.getName().startsWith(com.gitblit.Constants.R_META)) {
+ // skip internal meta branches
indexBranch = false;
} else {
// normal explicit branch check
@@ -552,13 +552,13 @@
Resolution.MINUTE);
Document doc = new Document();
- doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.blob.name(), Store.YES, Index.NOT_ANALYZED_NO_NORMS));
- doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_PATH, path, Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_DATE, blobDate, Store.YES, Index.NO));
- doc.add(new Field(FIELD_AUTHOR, blobAuthor, Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_COMMITTER, blobCommitter, Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.blob.name(), StringField.TYPE_STORED));
+ doc.add(new Field(FIELD_BRANCH, branchName, TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_COMMIT, commit.getName(), TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_PATH, path, TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_DATE, blobDate, StringField.TYPE_STORED));
+ doc.add(new Field(FIELD_AUTHOR, blobAuthor, TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_COMMITTER, blobCommitter, TextField.TYPE_STORED));
// determine extension to compare to the extension
// blacklist
@@ -579,7 +579,7 @@
in.close();
byte[] content = os.toByteArray();
String str = StringUtils.decodeString(content, encodings);
- doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_CONTENT, str, TextField.TYPE_STORED));
os.reset();
}
@@ -593,7 +593,7 @@
// index the tip commit object
if (indexedCommits.add(tipId)) {
Document doc = createDocument(tip, tags.get(tipId));
- doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_BRANCH, branchName, TextField.TYPE_STORED));
writer.addDocument(doc);
result.commitCount += 1;
result.branchCount += 1;
@@ -607,7 +607,7 @@
String hash = rev.getId().getName();
if (indexedCommits.add(hash)) {
Document doc = createDocument(rev, tags.get(hash));
- doc.add(new Field(FIELD_BRANCH, branchName, Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_BRANCH, branchName, TextField.TYPE_STORED));
writer.addDocument(doc);
result.commitCount += 1;
}
@@ -615,7 +615,7 @@
}
// finished
- reader.release();
+ reader.close();
// commit all changes and reset the searcher
config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION);
@@ -660,14 +660,13 @@
if (!ChangeType.DELETE.equals(path.changeType)) {
result.blobCount++;
Document doc = new Document();
- doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.blob.name(), Store.YES,
- Index.NOT_ANALYZED));
- doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_PATH, path.path, Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_DATE, revDate, Store.YES, Index.NO));
- doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.blob.name(), StringField.TYPE_STORED));
+ doc.add(new Field(FIELD_BRANCH, branch, TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_COMMIT, commit.getName(), TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_PATH, path.path, TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_DATE, revDate, StringField.TYPE_STORED));
+ doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), TextField.TYPE_STORED));
// determine extension to compare to the extension
// blacklist
@@ -682,7 +681,7 @@
String str = JGitUtils.getStringContent(repository, commit.getTree(),
path.path, encodings);
if (str != null) {
- doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_CONTENT, str, TextField.TYPE_STORED));
writer.addDocument(doc);
}
}
@@ -700,7 +699,7 @@
// create and write the Lucene document
Document doc = createDocument(commit, commitTags);
- doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_BRANCH, branch, TextField.TYPE_STORED));
result.commitCount++;
result.success = index(repositoryName, doc);
} catch (Exception e) {
@@ -723,8 +722,8 @@
String q = MessageFormat.format(pattern, SearchObjectType.blob.name(), branch, path);
BooleanQuery query = new BooleanQuery();
- StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
- QueryParser qp = new QueryParser(LUCENE_VERSION, FIELD_SUMMARY, analyzer);
+ StandardAnalyzer analyzer = new StandardAnalyzer();
+ QueryParser qp = new QueryParser(FIELD_SUMMARY, analyzer);
query.add(qp.parse(q), Occur.MUST);
IndexWriter writer = getIndexWriter(repositoryName);
@@ -761,7 +760,7 @@
// skip non-annotated tags
continue;
}
- if (!tags.containsKey(tag.getObjectId())) {
+ if (!tags.containsKey(tag.getObjectId().getName())) {
tags.put(tag.getReferencedObjectId().getName(), new ArrayList<String>());
}
tags.get(tag.getReferencedObjectId().getName()).add(tag.displayName);
@@ -808,8 +807,8 @@
&& branch.equals(defaultBranch)) {
// indexing "default" branch
indexBranch = true;
- } else if (branch.getName().startsWith(com.gitblit.Constants.R_GITBLIT)) {
- // ignore internal Gitblit branches
+ } else if (branch.getName().startsWith(com.gitblit.Constants.R_META)) {
+ // ignore internal meta branches
indexBranch = false;
} else {
// normal explicit branch check
@@ -880,17 +879,16 @@
*/
private Document createDocument(RevCommit commit, List<String> tags) {
Document doc = new Document();
- doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.commit.name(), Store.YES,
- Index.NOT_ANALYZED));
- doc.add(new Field(FIELD_COMMIT, commit.getName(), Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_OBJECT_TYPE, SearchObjectType.commit.name(), StringField.TYPE_STORED));
+ doc.add(new Field(FIELD_COMMIT, commit.getName(), TextField.TYPE_STORED));
doc.add(new Field(FIELD_DATE, DateTools.timeToString(commit.getCommitTime() * 1000L,
- Resolution.MINUTE), Store.YES, Index.NO));
- doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), Store.YES, Index.ANALYZED));
- doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), Store.YES, Index.ANALYZED));
+ Resolution.MINUTE), StringField.TYPE_STORED));
+ doc.add(new Field(FIELD_AUTHOR, getAuthor(commit), TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_COMMITTER, getCommitter(commit), TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_SUMMARY, commit.getShortMessage(), TextField.TYPE_STORED));
+ doc.add(new Field(FIELD_CONTENT, commit.getFullMessage(), TextField.TYPE_STORED));
if (!ArrayUtils.isEmpty(tags)) {
- doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), Store.YES, Index.ANALYZED));
+ doc.add(new Field(FIELD_TAG, StringUtils.flattenStrings(tags), TextField.TYPE_STORED));
}
return doc;
}
@@ -952,7 +950,7 @@
IndexSearcher searcher = searchers.get(repository);
if (searcher == null) {
IndexWriter writer = getIndexWriter(repository);
- searcher = new IndexSearcher(IndexReader.open(writer, true));
+ searcher = new IndexSearcher(DirectoryReader.open(writer, true));
searchers.put(repository, searcher);
}
return searcher;
@@ -970,14 +968,14 @@
IndexWriter indexWriter = writers.get(repository);
File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED);
File indexFolder = new File(repositoryFolder, LUCENE_DIR);
- Directory directory = FSDirectory.open(indexFolder);
+ Directory directory = FSDirectory.open(indexFolder.toPath());
if (indexWriter == null) {
if (!indexFolder.exists()) {
indexFolder.mkdirs();
}
- StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
- IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, analyzer);
+ StandardAnalyzer analyzer = new StandardAnalyzer();
+ IndexWriterConfig config = new IndexWriterConfig(analyzer);
config.setOpenMode(OpenMode.CREATE_OR_APPEND);
indexWriter = new IndexWriter(directory, config);
writers.put(repository, indexWriter);
@@ -1030,16 +1028,16 @@
return null;
}
Set<SearchResult> results = new LinkedHashSet<SearchResult>();
- StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
+ StandardAnalyzer analyzer = new StandardAnalyzer();
try {
// default search checks summary and content
BooleanQuery query = new BooleanQuery();
QueryParser qp;
- qp = new QueryParser(LUCENE_VERSION, FIELD_SUMMARY, analyzer);
+ qp = new QueryParser(FIELD_SUMMARY, analyzer);
qp.setAllowLeadingWildcard(true);
query.add(qp.parse(text), Occur.SHOULD);
- qp = new QueryParser(LUCENE_VERSION, FIELD_CONTENT, analyzer);
+ qp = new QueryParser(FIELD_CONTENT, analyzer);
qp.setAllowLeadingWildcard(true);
query.add(qp.parse(text), Occur.SHOULD);
@@ -1062,7 +1060,7 @@
Query rewrittenQuery = searcher.rewrite(query);
logger.debug(rewrittenQuery.toString());
- TopScoreDocCollector collector = TopScoreDocCollector.create(5000, true);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(5000);
searcher.search(rewrittenQuery, collector);
int offset = Math.max(0, (page - 1) * pageSize);
ScoreDoc[] hits = collector.topDocs(offset, pageSize).scoreDocs;
@@ -1106,6 +1104,7 @@
content = "";
}
+ int tabLength = storedSettings.getInteger(Keys.web.tabLength, 4);
int fragmentLength = SearchObjectType.commit == result.type ? 512 : 150;
QueryScorer scorer = new QueryScorer(query, "content");
@@ -1128,7 +1127,7 @@
if (fragment.length() > fragmentLength) {
fragment = fragment.substring(0, fragmentLength) + "...";
}
- return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true) + "</pre>";
+ return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true, tabLength) + "</pre>";
}
// make sure we have unique fragments
@@ -1226,25 +1225,14 @@
*/
private class MultiSourceReader extends MultiReader {
- final Method method;
-
- MultiSourceReader(IndexReader[] subReaders) {
- super(subReaders);
- Method m = null;
- try {
- m = MultiReader.class.getDeclaredMethod("readerIndex", int.class);
- m.setAccessible(true);
- } catch (Exception e) {
- logger.error("Error getting readerIndex method", e);
- }
- method = m;
+ MultiSourceReader(IndexReader [] readers) throws IOException {
+ super(readers, false);
}
int getSourceIndex(int docId) {
int index = -1;
try {
- Object o = method.invoke(this, docId);
- index = (Integer) o;
+ index = super.readerIndex(docId);
} catch (Exception e) {
logger.error("Error getting source index", e);
}
--
Gitblit v1.9.1