/*
|
* Copyright 2014 gitblit.com.
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*/
|
package com.gitblit.tickets;
|
|
import java.io.File;
|
import java.io.IOException;
|
import java.text.MessageFormat;
|
import java.text.ParseException;
|
import java.util.ArrayList;
|
import java.util.Collections;
|
import java.util.Date;
|
import java.util.LinkedHashSet;
|
import java.util.List;
|
import java.util.Set;
|
|
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
import org.apache.lucene.document.Document;
|
import org.apache.lucene.document.Field.Store;
|
import org.apache.lucene.document.IntField;
|
import org.apache.lucene.document.LongField;
|
import org.apache.lucene.document.TextField;
|
import org.apache.lucene.index.DirectoryReader;
|
import org.apache.lucene.index.IndexWriter;
|
import org.apache.lucene.index.IndexWriterConfig;
|
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
|
import org.apache.lucene.queryparser.classic.QueryParser;
|
import org.apache.lucene.search.BooleanClause.Occur;
|
import org.apache.lucene.search.BooleanQuery;
|
import org.apache.lucene.search.IndexSearcher;
|
import org.apache.lucene.search.Query;
|
import org.apache.lucene.search.ScoreDoc;
|
import org.apache.lucene.search.Sort;
|
import org.apache.lucene.search.SortField;
|
import org.apache.lucene.search.SortField.Type;
|
import org.apache.lucene.search.TopFieldDocs;
|
import org.apache.lucene.search.TopScoreDocCollector;
|
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.FSDirectory;
|
import org.apache.lucene.util.Version;
|
import org.slf4j.Logger;
|
import org.slf4j.LoggerFactory;
|
|
import com.gitblit.Keys;
|
import com.gitblit.manager.IRuntimeManager;
|
import com.gitblit.models.RepositoryModel;
|
import com.gitblit.models.TicketModel;
|
import com.gitblit.models.TicketModel.Attachment;
|
import com.gitblit.models.TicketModel.Patchset;
|
import com.gitblit.models.TicketModel.Status;
|
import com.gitblit.utils.FileUtils;
|
import com.gitblit.utils.StringUtils;
|
|
/**
|
* Indexes tickets in a Lucene database.
|
*
|
* @author James Moger
|
*
|
*/
|
public class TicketIndexer {
|
|
/**
|
* Fields in the Lucene index
|
*/
|
public static enum Lucene {
|
|
rid(Type.STRING),
|
did(Type.STRING),
|
project(Type.STRING),
|
repository(Type.STRING),
|
number(Type.LONG),
|
title(Type.STRING),
|
body(Type.STRING),
|
topic(Type.STRING),
|
created(Type.LONG),
|
createdby(Type.STRING),
|
updated(Type.LONG),
|
updatedby(Type.STRING),
|
responsible(Type.STRING),
|
milestone(Type.STRING),
|
status(Type.STRING),
|
type(Type.STRING),
|
labels(Type.STRING),
|
participants(Type.STRING),
|
watchedby(Type.STRING),
|
mentions(Type.STRING),
|
attachments(Type.INT),
|
content(Type.STRING),
|
patchset(Type.STRING),
|
comments(Type.INT),
|
mergesha(Type.STRING),
|
mergeto(Type.STRING),
|
patchsets(Type.INT),
|
votes(Type.INT),
|
//NOTE: Indexing on the underlying value to allow flexibility on naming
|
priority(Type.INT),
|
severity(Type.INT);
|
|
final Type fieldType;
|
|
Lucene(Type fieldType) {
|
this.fieldType = fieldType;
|
}
|
|
public String colon() {
|
return name() + ":";
|
}
|
|
public String matches(String value) {
|
if (StringUtils.isEmpty(value)) {
|
return "";
|
}
|
boolean not = value.charAt(0) == '!';
|
if (not) {
|
return "!" + name() + ":" + escape(value.substring(1));
|
}
|
return name() + ":" + escape(value);
|
}
|
|
public String doesNotMatch(String value) {
|
if (StringUtils.isEmpty(value)) {
|
return "";
|
}
|
return "NOT " + name() + ":" + escape(value);
|
}
|
|
public String isNotNull() {
|
return matches("[* TO *]");
|
}
|
|
public SortField asSortField(boolean descending) {
|
return new SortField(name(), fieldType, descending);
|
}
|
|
private String escape(String value) {
|
if (value.charAt(0) != '"') {
|
for (char c : value.toCharArray()) {
|
if (!Character.isLetterOrDigit(c)) {
|
return "\"" + value + "\"";
|
}
|
}
|
}
|
return value;
|
}
|
|
public static Lucene fromString(String value) {
|
for (Lucene field : values()) {
|
if (field.name().equalsIgnoreCase(value)) {
|
return field;
|
}
|
}
|
return created;
|
}
|
}
|
|
private final Logger log = LoggerFactory.getLogger(getClass());
|
|
private final Version luceneVersion = Version.LUCENE_46;
|
|
private final File luceneDir;
|
|
private IndexWriter writer;
|
|
private IndexSearcher searcher;
|
|
public TicketIndexer(IRuntimeManager runtimeManager) {
|
this.luceneDir = runtimeManager.getFileOrFolder(Keys.tickets.indexFolder, "${baseFolder}/tickets/lucene");
|
}
|
|
/**
|
* Close all writers and searchers used by the ticket indexer.
|
*/
|
public void close() {
|
closeSearcher();
|
closeWriter();
|
}
|
|
/**
|
* Deletes the entire ticket index for all repositories.
|
*/
|
public void deleteAll() {
|
close();
|
FileUtils.delete(luceneDir);
|
}
|
|
/**
|
* Deletes all tickets for the the repository from the index.
|
*/
|
public boolean deleteAll(RepositoryModel repository) {
|
try {
|
IndexWriter writer = getWriter();
|
StandardAnalyzer analyzer = new StandardAnalyzer(luceneVersion);
|
QueryParser qp = new QueryParser(luceneVersion, Lucene.rid.name(), analyzer);
|
BooleanQuery query = new BooleanQuery();
|
query.add(qp.parse(repository.getRID()), Occur.MUST);
|
|
int numDocsBefore = writer.numDocs();
|
writer.deleteDocuments(query);
|
writer.commit();
|
closeSearcher();
|
int numDocsAfter = writer.numDocs();
|
if (numDocsBefore == numDocsAfter) {
|
log.debug(MessageFormat.format("no records found to delete in {0}", repository));
|
return false;
|
} else {
|
log.debug(MessageFormat.format("deleted {0} records in {1}", numDocsBefore - numDocsAfter, repository));
|
return true;
|
}
|
} catch (Exception e) {
|
log.error("error", e);
|
}
|
return false;
|
}
|
|
/**
|
* Bulk Add/Update tickets in the Lucene index
|
*
|
* @param tickets
|
*/
|
public void index(List<TicketModel> tickets) {
|
try {
|
IndexWriter writer = getWriter();
|
for (TicketModel ticket : tickets) {
|
Document doc = ticketToDoc(ticket);
|
writer.addDocument(doc);
|
}
|
writer.commit();
|
closeSearcher();
|
} catch (Exception e) {
|
log.error("error", e);
|
}
|
}
|
|
/**
|
* Add/Update a ticket in the Lucene index
|
*
|
* @param ticket
|
*/
|
public void index(TicketModel ticket) {
|
try {
|
IndexWriter writer = getWriter();
|
delete(ticket.repository, ticket.number, writer);
|
Document doc = ticketToDoc(ticket);
|
writer.addDocument(doc);
|
writer.commit();
|
closeSearcher();
|
} catch (Exception e) {
|
log.error("error", e);
|
}
|
}
|
|
/**
|
* Delete a ticket from the Lucene index.
|
*
|
* @param ticket
|
* @throws Exception
|
* @return true, if deleted, false if no record was deleted
|
*/
|
public boolean delete(TicketModel ticket) {
|
try {
|
IndexWriter writer = getWriter();
|
return delete(ticket.repository, ticket.number, writer);
|
} catch (Exception e) {
|
log.error("Failed to delete ticket " + ticket.number, e);
|
}
|
return false;
|
}
|
|
/**
|
* Delete a ticket from the Lucene index.
|
*
|
* @param repository
|
* @param ticketId
|
* @throws Exception
|
* @return true, if deleted, false if no record was deleted
|
*/
|
private boolean delete(String repository, long ticketId, IndexWriter writer) throws Exception {
|
StandardAnalyzer analyzer = new StandardAnalyzer(luceneVersion);
|
QueryParser qp = new QueryParser(luceneVersion, Lucene.did.name(), analyzer);
|
BooleanQuery query = new BooleanQuery();
|
query.add(qp.parse(StringUtils.getSHA1(repository + ticketId)), Occur.MUST);
|
|
int numDocsBefore = writer.numDocs();
|
writer.deleteDocuments(query);
|
writer.commit();
|
closeSearcher();
|
int numDocsAfter = writer.numDocs();
|
if (numDocsBefore == numDocsAfter) {
|
log.debug(MessageFormat.format("no records found to delete in {0}", repository));
|
return false;
|
} else {
|
log.debug(MessageFormat.format("deleted {0} records in {1}", numDocsBefore - numDocsAfter, repository));
|
return true;
|
}
|
}
|
|
/**
|
* Returns true if the repository has tickets in the index.
|
*
|
* @param repository
|
* @return true if there are indexed tickets
|
*/
|
public boolean hasTickets(RepositoryModel repository) {
|
return !queryFor(Lucene.rid.matches(repository.getRID()), 1, 0, null, true).isEmpty();
|
}
|
|
/**
|
* Search for tickets matching the query. The returned tickets are
|
* shadows of the real ticket, but suitable for a results list.
|
*
|
* @param repository
|
* @param text
|
* @param page
|
* @param pageSize
|
* @return search results
|
*/
|
public List<QueryResult> searchFor(RepositoryModel repository, String text, int page, int pageSize) {
|
if (StringUtils.isEmpty(text)) {
|
return Collections.emptyList();
|
}
|
Set<QueryResult> results = new LinkedHashSet<QueryResult>();
|
StandardAnalyzer analyzer = new StandardAnalyzer(luceneVersion);
|
try {
|
// search the title, description and content
|
BooleanQuery query = new BooleanQuery();
|
QueryParser qp;
|
|
qp = new QueryParser(luceneVersion, Lucene.title.name(), analyzer);
|
qp.setAllowLeadingWildcard(true);
|
query.add(qp.parse(text), Occur.SHOULD);
|
|
qp = new QueryParser(luceneVersion, Lucene.body.name(), analyzer);
|
qp.setAllowLeadingWildcard(true);
|
query.add(qp.parse(text), Occur.SHOULD);
|
|
qp = new QueryParser(luceneVersion, Lucene.content.name(), analyzer);
|
qp.setAllowLeadingWildcard(true);
|
query.add(qp.parse(text), Occur.SHOULD);
|
|
IndexSearcher searcher = getSearcher();
|
Query rewrittenQuery = searcher.rewrite(query);
|
|
log.debug(rewrittenQuery.toString());
|
|
TopScoreDocCollector collector = TopScoreDocCollector.create(5000, true);
|
searcher.search(rewrittenQuery, collector);
|
int offset = Math.max(0, (page - 1) * pageSize);
|
ScoreDoc[] hits = collector.topDocs(offset, pageSize).scoreDocs;
|
for (int i = 0; i < hits.length; i++) {
|
int docId = hits[i].doc;
|
Document doc = searcher.doc(docId);
|
QueryResult result = docToQueryResult(doc);
|
if (repository != null) {
|
if (!result.repository.equalsIgnoreCase(repository.name)) {
|
continue;
|
}
|
}
|
results.add(result);
|
}
|
} catch (Exception e) {
|
log.error(MessageFormat.format("Exception while searching for {0}", text), e);
|
}
|
return new ArrayList<QueryResult>(results);
|
}
|
|
/**
|
* Search for tickets matching the query. The returned tickets are
|
* shadows of the real ticket, but suitable for a results list.
|
*
|
* @param text
|
* @param page
|
* @param pageSize
|
* @param sortBy
|
* @param desc
|
* @return
|
*/
|
public List<QueryResult> queryFor(String queryText, int page, int pageSize, String sortBy, boolean desc) {
|
if (StringUtils.isEmpty(queryText)) {
|
return Collections.emptyList();
|
}
|
|
Set<QueryResult> results = new LinkedHashSet<QueryResult>();
|
StandardAnalyzer analyzer = new StandardAnalyzer(luceneVersion);
|
try {
|
QueryParser qp = new QueryParser(luceneVersion, Lucene.content.name(), analyzer);
|
Query query = qp.parse(queryText);
|
|
IndexSearcher searcher = getSearcher();
|
Query rewrittenQuery = searcher.rewrite(query);
|
|
log.debug(rewrittenQuery.toString());
|
|
Sort sort;
|
if (sortBy == null) {
|
sort = new Sort(Lucene.created.asSortField(desc));
|
} else {
|
sort = new Sort(Lucene.fromString(sortBy).asSortField(desc));
|
}
|
int maxSize = 5000;
|
TopFieldDocs docs = searcher.search(rewrittenQuery, null, maxSize, sort, false, false);
|
int size = (pageSize <= 0) ? maxSize : pageSize;
|
int offset = Math.max(0, (page - 1) * size);
|
ScoreDoc[] hits = subset(docs.scoreDocs, offset, size);
|
for (int i = 0; i < hits.length; i++) {
|
int docId = hits[i].doc;
|
Document doc = searcher.doc(docId);
|
QueryResult result = docToQueryResult(doc);
|
result.docId = docId;
|
result.totalResults = docs.totalHits;
|
results.add(result);
|
}
|
} catch (Exception e) {
|
log.error(MessageFormat.format("Exception while searching for {0}", queryText), e);
|
}
|
return new ArrayList<QueryResult>(results);
|
}
|
|
private ScoreDoc [] subset(ScoreDoc [] docs, int offset, int size) {
|
if (docs.length >= (offset + size)) {
|
ScoreDoc [] set = new ScoreDoc[size];
|
System.arraycopy(docs, offset, set, 0, set.length);
|
return set;
|
} else if (docs.length >= offset) {
|
ScoreDoc [] set = new ScoreDoc[docs.length - offset];
|
System.arraycopy(docs, offset, set, 0, set.length);
|
return set;
|
} else {
|
return new ScoreDoc[0];
|
}
|
}
|
|
private IndexWriter getWriter() throws IOException {
|
if (writer == null) {
|
Directory directory = FSDirectory.open(luceneDir);
|
|
if (!luceneDir.exists()) {
|
luceneDir.mkdirs();
|
}
|
|
StandardAnalyzer analyzer = new StandardAnalyzer(luceneVersion);
|
IndexWriterConfig config = new IndexWriterConfig(luceneVersion, analyzer);
|
config.setOpenMode(OpenMode.CREATE_OR_APPEND);
|
writer = new IndexWriter(directory, config);
|
}
|
return writer;
|
}
|
|
private synchronized void closeWriter() {
|
try {
|
if (writer != null) {
|
writer.close();
|
}
|
} catch (Exception e) {
|
log.error("failed to close writer!", e);
|
} finally {
|
writer = null;
|
}
|
}
|
|
private IndexSearcher getSearcher() throws IOException {
|
if (searcher == null) {
|
searcher = new IndexSearcher(DirectoryReader.open(getWriter(), true));
|
}
|
return searcher;
|
}
|
|
private synchronized void closeSearcher() {
|
try {
|
if (searcher != null) {
|
searcher.getIndexReader().close();
|
}
|
} catch (Exception e) {
|
log.error("failed to close searcher!", e);
|
} finally {
|
searcher = null;
|
}
|
}
|
|
/**
|
* Creates a Lucene document from a ticket.
|
*
|
* @param ticket
|
* @return a Lucene document
|
*/
|
private Document ticketToDoc(TicketModel ticket) {
|
Document doc = new Document();
|
// repository and document ids for Lucene querying
|
toDocField(doc, Lucene.rid, StringUtils.getSHA1(ticket.repository));
|
toDocField(doc, Lucene.did, StringUtils.getSHA1(ticket.repository + ticket.number));
|
|
toDocField(doc, Lucene.project, ticket.project);
|
toDocField(doc, Lucene.repository, ticket.repository);
|
toDocField(doc, Lucene.number, ticket.number);
|
toDocField(doc, Lucene.title, ticket.title);
|
toDocField(doc, Lucene.body, ticket.body);
|
toDocField(doc, Lucene.created, ticket.created);
|
toDocField(doc, Lucene.createdby, ticket.createdBy);
|
toDocField(doc, Lucene.updated, ticket.updated);
|
toDocField(doc, Lucene.updatedby, ticket.updatedBy);
|
toDocField(doc, Lucene.responsible, ticket.responsible);
|
toDocField(doc, Lucene.milestone, ticket.milestone);
|
toDocField(doc, Lucene.topic, ticket.topic);
|
toDocField(doc, Lucene.status, ticket.status.name());
|
toDocField(doc, Lucene.comments, ticket.getComments().size());
|
toDocField(doc, Lucene.type, ticket.type == null ? null : ticket.type.name());
|
toDocField(doc, Lucene.mergesha, ticket.mergeSha);
|
toDocField(doc, Lucene.mergeto, ticket.mergeTo);
|
toDocField(doc, Lucene.labels, StringUtils.flattenStrings(ticket.getLabels(), ";").toLowerCase());
|
toDocField(doc, Lucene.participants, StringUtils.flattenStrings(ticket.getParticipants(), ";").toLowerCase());
|
toDocField(doc, Lucene.watchedby, StringUtils.flattenStrings(ticket.getWatchers(), ";").toLowerCase());
|
toDocField(doc, Lucene.mentions, StringUtils.flattenStrings(ticket.getMentions(), ";").toLowerCase());
|
toDocField(doc, Lucene.votes, ticket.getVoters().size());
|
toDocField(doc, Lucene.priority, ticket.priority.getValue());
|
toDocField(doc, Lucene.severity, ticket.severity.getValue());
|
|
List<String> attachments = new ArrayList<String>();
|
for (Attachment attachment : ticket.getAttachments()) {
|
attachments.add(attachment.name.toLowerCase());
|
}
|
toDocField(doc, Lucene.attachments, StringUtils.flattenStrings(attachments, ";"));
|
|
List<Patchset> patches = ticket.getPatchsets();
|
if (!patches.isEmpty()) {
|
toDocField(doc, Lucene.patchsets, patches.size());
|
Patchset patchset = patches.get(patches.size() - 1);
|
String flat =
|
patchset.number + ":" +
|
patchset.rev + ":" +
|
patchset.tip + ":" +
|
patchset.base + ":" +
|
patchset.commits;
|
doc.add(new org.apache.lucene.document.Field(Lucene.patchset.name(), flat, TextField.TYPE_STORED));
|
}
|
|
doc.add(new TextField(Lucene.content.name(), ticket.toIndexableString(), Store.NO));
|
|
return doc;
|
}
|
|
private void toDocField(Document doc, Lucene lucene, Date value) {
|
if (value == null) {
|
return;
|
}
|
doc.add(new LongField(lucene.name(), value.getTime(), Store.YES));
|
}
|
|
private void toDocField(Document doc, Lucene lucene, long value) {
|
doc.add(new LongField(lucene.name(), value, Store.YES));
|
}
|
|
private void toDocField(Document doc, Lucene lucene, int value) {
|
doc.add(new IntField(lucene.name(), value, Store.YES));
|
}
|
|
private void toDocField(Document doc, Lucene lucene, String value) {
|
if (StringUtils.isEmpty(value)) {
|
return;
|
}
|
doc.add(new org.apache.lucene.document.Field(lucene.name(), value, TextField.TYPE_STORED));
|
}
|
|
/**
|
* Creates a query result from the Lucene document. This result is
|
* not a high-fidelity representation of the real ticket, but it is
|
* suitable for display in a table of search results.
|
*
|
* @param doc
|
* @return a query result
|
* @throws ParseException
|
*/
|
private QueryResult docToQueryResult(Document doc) throws ParseException {
|
QueryResult result = new QueryResult();
|
result.project = unpackString(doc, Lucene.project);
|
result.repository = unpackString(doc, Lucene.repository);
|
result.number = unpackLong(doc, Lucene.number);
|
result.createdBy = unpackString(doc, Lucene.createdby);
|
result.createdAt = unpackDate(doc, Lucene.created);
|
result.updatedBy = unpackString(doc, Lucene.updatedby);
|
result.updatedAt = unpackDate(doc, Lucene.updated);
|
result.title = unpackString(doc, Lucene.title);
|
result.body = unpackString(doc, Lucene.body);
|
result.status = Status.fromObject(unpackString(doc, Lucene.status), Status.New);
|
result.responsible = unpackString(doc, Lucene.responsible);
|
result.milestone = unpackString(doc, Lucene.milestone);
|
result.topic = unpackString(doc, Lucene.topic);
|
result.type = TicketModel.Type.fromObject(unpackString(doc, Lucene.type), TicketModel.Type.defaultType);
|
result.mergeSha = unpackString(doc, Lucene.mergesha);
|
result.mergeTo = unpackString(doc, Lucene.mergeto);
|
result.commentsCount = unpackInt(doc, Lucene.comments);
|
result.votesCount = unpackInt(doc, Lucene.votes);
|
result.attachments = unpackStrings(doc, Lucene.attachments);
|
result.labels = unpackStrings(doc, Lucene.labels);
|
result.participants = unpackStrings(doc, Lucene.participants);
|
result.watchedby = unpackStrings(doc, Lucene.watchedby);
|
result.mentions = unpackStrings(doc, Lucene.mentions);
|
result.priority = TicketModel.Priority.fromObject(unpackInt(doc, Lucene.priority), TicketModel.Priority.defaultPriority);
|
result.severity = TicketModel.Severity.fromObject(unpackInt(doc, Lucene.severity), TicketModel.Severity.defaultSeverity);
|
|
if (!StringUtils.isEmpty(doc.get(Lucene.patchset.name()))) {
|
// unpack most recent patchset
|
String [] values = doc.get(Lucene.patchset.name()).split(":", 5);
|
|
Patchset patchset = new Patchset();
|
patchset.number = Integer.parseInt(values[0]);
|
patchset.rev = Integer.parseInt(values[1]);
|
patchset.tip = values[2];
|
patchset.base = values[3];
|
patchset.commits = Integer.parseInt(values[4]);
|
|
result.patchset = patchset;
|
}
|
|
return result;
|
}
|
|
private String unpackString(Document doc, Lucene lucene) {
|
return doc.get(lucene.name());
|
}
|
|
private List<String> unpackStrings(Document doc, Lucene lucene) {
|
if (!StringUtils.isEmpty(doc.get(lucene.name()))) {
|
return StringUtils.getStringsFromValue(doc.get(lucene.name()), ";");
|
}
|
return null;
|
}
|
|
private Date unpackDate(Document doc, Lucene lucene) {
|
String val = doc.get(lucene.name());
|
if (!StringUtils.isEmpty(val)) {
|
long time = Long.parseLong(val);
|
Date date = new Date(time);
|
return date;
|
}
|
return null;
|
}
|
|
private long unpackLong(Document doc, Lucene lucene) {
|
String val = doc.get(lucene.name());
|
if (StringUtils.isEmpty(val)) {
|
return 0;
|
}
|
long l = Long.parseLong(val);
|
return l;
|
}
|
|
private int unpackInt(Document doc, Lucene lucene) {
|
String val = doc.get(lucene.name());
|
if (StringUtils.isEmpty(val)) {
|
return 0;
|
}
|
int i = Integer.parseInt(val);
|
return i;
|
}
|
}
|
