James Moger
2012-10-22 eba89539a29deba954035056437279088c3e047b
src/com/gitblit/LuceneExecutor.java
@@ -69,10 +69,12 @@
import org.apache.lucene.util.Version;
import org.eclipse.jgit.diff.DiffEntry.ChangeType;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.RepositoryCache.FileKey;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
@@ -104,7 +106,7 @@
public class LuceneExecutor implements Runnable {
   
      
   private static final int INDEX_VERSION = 2;
   private static final int INDEX_VERSION = 5;
   private static final String FIELD_OBJECT_TYPE = "type";
   private static final String FIELD_ISSUE = "issue";
@@ -284,7 +286,7 @@
         close(repositoryName);
         // delete the index folder
         File repositoryFolder = new File(repositoriesFolder, repositoryName);
         File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repositoryName), FS.DETECTED);
         File luceneIndex = new File(repositoryFolder, LUCENE_DIR);
         if (luceneIndex.exists()) {
            org.eclipse.jgit.util.FileUtils.delete(luceneIndex,
@@ -300,7 +302,6 @@
         throw new RuntimeException(e);
      }
   }
   
   /**
    * Returns the author for the commit, if this information is available.
@@ -412,7 +413,8 @@
      if (!deleteIndex(model.name)) {
         return result;
      }
      try {
      try {
         String [] encodings = storedSettings.getStrings(Keys.web.blobEncodings).toArray(new String[0]);
         FileBasedConfig config = getConfig(repository);
         Set<String> indexedCommits = new TreeSet<String>();
         IndexWriter writer = getIndexWriter(model.name);
@@ -447,7 +449,7 @@
         ObjectId defaultBranchId = JGitUtils.getDefaultBranch(repository);
         for (RefModel branch :  branches) {
            if (branch.getObjectId().equals(defaultBranchId)) {
               defaultBranch = branch;
               defaultBranch = branch;
               break;
            }
         }
@@ -457,8 +459,22 @@
         // walk through each branch
         for (RefModel branch : branches) {
            boolean indexBranch = false;
            if (model.indexedBranches.contains(com.gitblit.Constants.DEFAULT_BRANCH)
                  && branch.equals(defaultBranch)) {
               // indexing "default" branch
               indexBranch = true;
            } else if (IssueUtils.GB_ISSUES.equals(branch)) {
               // skip the GB_ISSUES branch because it is indexed later
               // note: this is different than updateIndex
               indexBranch = false;
            } else {
               // normal explicit branch check
               indexBranch = model.indexedBranches.contains(branch.getName());
            }
            // if this branch is not specifically indexed then skip
            if (!model.indexedBranches.contains(branch.getName())) {
            if (!indexBranch) {
               continue;
            }
@@ -478,7 +494,10 @@
            
            Map<String, ObjectId> paths = new TreeMap<String, ObjectId>();
            while (treeWalk.next()) {
               paths.put(treeWalk.getPathString(), treeWalk.getObjectId(0));
               // ensure path is not in a submodule
               if (treeWalk.getFileMode(0) != FileMode.GITLINK) {
                  paths.put(treeWalk.getPathString(), treeWalk.getObjectId(0));
               }
            }            
            ByteArrayOutputStream os = new ByteArrayOutputStream();
@@ -548,7 +567,7 @@
                     }
                     in.close();
                     byte[] content = os.toByteArray();
                     String str = new String(content, Constants.CHARACTER_ENCODING);
                     String str = StringUtils.decodeString(content, encodings);
                     doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
                     os.reset();
                  }                     
@@ -627,11 +646,15 @@
         String branch, RevCommit commit) {
      IndexResult result = new IndexResult();
      try {
         String [] encodings = storedSettings.getStrings(Keys.web.blobEncodings).toArray(new String[0]);
         List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit);
         String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L,
               Resolution.MINUTE);
         IndexWriter writer = getIndexWriter(repositoryName);
         for (PathChangeModel path : changedPaths) {
            if (path.isSubmodule()) {
               continue;
            }
            // delete the indexed blob
            deleteBlob(repositoryName, branch, path.name);
@@ -659,9 +682,11 @@
               if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {
                  // read the blob content
                  String str = JGitUtils.getStringContent(repository, commit.getTree(),
                        path.path);
                  doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
                  writer.addDocument(doc);
                        path.path, encodings);
                  if (str != null) {
                     doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
                     writer.addDocument(doc);
                  }
               }
            }
         }
@@ -669,7 +694,7 @@
         
         // get any annotated commit tags
         List<String> commitTags = new ArrayList<String>();
         for (RefModel ref : JGitUtils.getTags(repository, true, -1)) {
         for (RefModel ref : JGitUtils.getTags(repository, false, -1)) {
            if (ref.isAnnotatedTag() && ref.getReferencedObjectId().equals(commit.getId())) {
               commitTags.add(ref.displayName);
            }
@@ -712,8 +737,9 @@
    * @param repositoryName
    * @param issueId
    * @throws Exception
    * @return true, if deleted, false if no record was deleted
    */
   private void deleteIssue(String repositoryName, String issueId) throws Exception {
   private boolean deleteIssue(String repositoryName, String issueId) throws Exception {
      BooleanQuery query = new BooleanQuery();
      Term objectTerm = new Term(FIELD_OBJECT_TYPE, SearchObjectType.issue.name());
      query.add(new TermQuery(objectTerm), Occur.MUST);
@@ -721,8 +747,17 @@
      query.add(new TermQuery(issueidTerm), Occur.MUST);
      
      IndexWriter writer = getIndexWriter(repositoryName);
      int numDocsBefore = writer.numDocs();
      writer.deleteDocuments(query);
      writer.commit();
      int numDocsAfter = writer.numDocs();
      if (numDocsBefore == numDocsAfter) {
         logger.debug(MessageFormat.format("no records found to delete {0}", query.toString()));
         return false;
      } else {
         logger.debug(MessageFormat.format("deleted {0} records with {1}", numDocsBefore - numDocsAfter, query.toString()));
         return true;
      }
   }
   
   /**
@@ -732,19 +767,29 @@
    * @param branch
    * @param path
    * @throws Exception
    * @return true, if deleted, false if no record was deleted
    */
   private void deleteBlob(String repositoryName, String branch, String path) throws Exception {
      BooleanQuery query = new BooleanQuery();
      Term objectTerm = new Term(FIELD_OBJECT_TYPE, SearchObjectType.blob.name());
      query.add(new TermQuery(objectTerm), Occur.MUST);
      Term branchTerm = new Term(FIELD_BRANCH, branch);
      query.add(new TermQuery(branchTerm), Occur.MUST);
      Term pathTerm = new Term(FIELD_PATH, path);
      query.add(new TermQuery(pathTerm), Occur.MUST);
   public boolean deleteBlob(String repositoryName, String branch, String path) throws Exception {
      String pattern = MessageFormat.format("{0}:'{'0} AND {1}:\"'{'1'}'\" AND {2}:\"'{'2'}'\"", FIELD_OBJECT_TYPE, FIELD_BRANCH, FIELD_PATH);
      String q = MessageFormat.format(pattern, SearchObjectType.blob.name(), branch, path);
      
      BooleanQuery query = new BooleanQuery();
      StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);
      QueryParser qp = new QueryParser(LUCENE_VERSION, FIELD_SUMMARY, analyzer);
      query.add(qp.parse(q), Occur.MUST);
      IndexWriter writer = getIndexWriter(repositoryName);
      writer.deleteDocuments(query);
      int numDocsBefore = writer.numDocs();
      writer.deleteDocuments(query);
      writer.commit();
      int numDocsAfter = writer.numDocs();
      if (numDocsBefore == numDocsAfter) {
         logger.debug(MessageFormat.format("no records found to delete {0}", query.toString()));
         return false;
      } else {
         logger.debug(MessageFormat.format("deleted {0} records with {1}", numDocsBefore - numDocsAfter, query.toString()));
         return true;
      }
   }
   /**
@@ -782,20 +827,55 @@
            deletedBranches.add(branch);
         }
         // walk through each branches
         // get the local branches
         List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);
         // sort them by most recently updated
         Collections.sort(branches, new Comparator<RefModel>() {
            @Override
            public int compare(RefModel ref1, RefModel ref2) {
               return ref2.getDate().compareTo(ref1.getDate());
            }
         });
         // reorder default branch to first position
         RefModel defaultBranch = null;
         ObjectId defaultBranchId = JGitUtils.getDefaultBranch(repository);
         for (RefModel branch :  branches) {
            if (branch.getObjectId().equals(defaultBranchId)) {
               defaultBranch = branch;
               break;
            }
         }
         branches.remove(defaultBranch);
         branches.add(0, defaultBranch);
         // walk through each branches
         for (RefModel branch : branches) {
            String branchName = branch.getName();
            // determine if we should skip this branch
            if (!IssueUtils.GB_ISSUES.equals(branch)
                  && !model.indexedBranches.contains(branch.getName())) {
            boolean indexBranch = false;
            if (model.indexedBranches.contains(com.gitblit.Constants.DEFAULT_BRANCH)
                  && branch.equals(defaultBranch)) {
               // indexing "default" branch
               indexBranch = true;
            } else if (IssueUtils.GB_ISSUES.equals(branch)) {
               // update issues modified on the GB_ISSUES branch
               // note: this is different than reindex
               indexBranch = true;
            } else {
               // normal explicit branch check
               indexBranch = model.indexedBranches.contains(branch.getName());
            }
            // if this branch is not specifically indexed then skip
            if (!indexBranch) {
               continue;
            }
            
            // remove this branch from the deletedBranches set
            deletedBranches.remove(branchName);
            // determine last commit
            String keyName = getBranchKey(branchName);
            String lastCommit = config.getString(CONF_BRANCH, null, keyName);
@@ -830,7 +910,9 @@
                  IssueModel issue = IssueUtils.getIssue(repository, issueId);
                  if (issue == null) {
                     // issue was deleted, remove from index
                     deleteIssue(model.name, issueId);
                     if (!deleteIssue(model.name, issueId)) {
                        logger.error(MessageFormat.format("Failed to delete issue {0} from Lucene index!", issueId));
                     }
                  } else {
                     // issue was updated
                     index(model.name, issue);
@@ -993,8 +1075,8 @@
    * @throws IOException
    */
   private IndexWriter getIndexWriter(String repository) throws IOException {
      IndexWriter indexWriter = writers.get(repository);
      File repositoryFolder = new File(repositoriesFolder, repository);
      IndexWriter indexWriter = writers.get(repository);
      File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED);
      File indexFolder = new File(repositoryFolder, LUCENE_DIR);
      Directory directory = FSDirectory.open(indexFolder);      
@@ -1068,7 +1150,7 @@
         qp = new QueryParser(LUCENE_VERSION, FIELD_CONTENT, analyzer);
         qp.setAllowLeadingWildcard(true);
         query.add(qp.parse(text), Occur.SHOULD);
         IndexSearcher searcher;
         if (repositories.length == 1) {
            // single repository search
@@ -1084,7 +1166,10 @@
            MultiSourceReader reader = new MultiSourceReader(rdrs);
            searcher = new IndexSearcher(reader);
         }
         Query rewrittenQuery = searcher.rewrite(query);
         logger.debug(rewrittenQuery.toString());
         TopScoreDocCollector collector = TopScoreDocCollector.create(5000, true);
         searcher.search(rewrittenQuery, collector);
         int offset = Math.max(0, (page - 1) * pageSize);
@@ -1154,7 +1239,13 @@
         return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true) + "</pre>";
      }
      
      int contentPos = 0;
      // make sure we have unique fragments
      Set<String> uniqueFragments = new LinkedHashSet<String>();
      for (String fragment : fragments) {
         uniqueFragments.add(fragment);
      }
      fragments = uniqueFragments.toArray(new String[uniqueFragments.size()]);
      StringBuilder sb = new StringBuilder();
      for (int i = 0, len = fragments.length; i < len; i++) {
         String fragment = fragments[i];
@@ -1164,7 +1255,7 @@
         String raw = fragment.replace(termTag, "").replace(termTagEnd, "");
         // determine position of the raw fragment in the content
         int pos = content.indexOf(raw, contentPos);
         int pos = content.indexOf(raw);
            
         // restore complete first line of fragment
         int c = pos;
@@ -1192,8 +1283,6 @@
            }
            tag = MessageFormat.format("<pre class=\"prettyprint linenums:{0,number,0}{1}\">", line, lang);
                        
            // update offset into content
            contentPos = pos + raw.length() + 1;
         }
         
         sb.append(tag);