Proof of concept #1026

Paul Martin

2016-04-16 eecaad8b8e2c447429c31a01d49260ddd6b4ee03

 src/main/java/com/gitblit/service/LuceneService.java

@@ -19,6 +19,7 @@


import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStream;

import java.text.MessageFormat;

@@ -66,6 +67,11 @@
import org.apache.lucene.store.Directory;

import org.apache.lucene.store.FSDirectory;

import org.apache.lucene.util.Version;

import org.apache.tika.metadata.Metadata;

import org.apache.tika.parser.AutoDetectParser;

import org.apache.tika.parser.ParseContext;

import org.apache.tika.parser.pdf.PDFParser;

import org.apache.tika.sax.BodyContentHandler;

import org.eclipse.jgit.diff.DiffEntry.ChangeType;

import org.eclipse.jgit.lib.Constants;

import org.eclipse.jgit.lib.FileMode;

@@ -85,8 +91,11 @@
import org.slf4j.LoggerFactory;



import com.gitblit.Constants.SearchObjectType;

import com.gitblit.GitBlit;

import com.gitblit.IStoredSettings;

import com.gitblit.Keys;

import com.gitblit.manager.FilestoreManager;

import com.gitblit.manager.IFilestoreManager;

import com.gitblit.manager.IRepositoryManager;

import com.gitblit.models.PathModel.PathChangeModel;

import com.gitblit.models.RefModel;

@@ -131,6 +140,8 @@


   private final IStoredSettings storedSettings;

   private final IRepositoryManager repositoryManager;

   private final IFilestoreManager filestoreManager;

	
   private final File repositoriesFolder;



   private final Map<String, IndexSearcher> searchers = new ConcurrentHashMap<String, IndexSearcher>();

@@ -141,10 +152,12 @@


   public LuceneService(

         IStoredSettings settings,

         IRepositoryManager repositoryManager) {

         IRepositoryManager repositoryManager, 
         IFilestoreManager filestoreManager) {



      this.storedSettings = settings;

      this.repositoryManager = repositoryManager;

      this.filestoreManager = filestoreManager;

      this.repositoriesFolder = repositoryManager.getRepositoriesFolder();

      String exts = luceneIgnoreExtensions;

      if (settings != null) {

@@ -267,7 +280,7 @@
      // close all writers

      for (String writer : writers.keySet()) {

         try {

            writers.get(writer).close();

            writers.get(writer).close(true);

         } catch (Throwable t) {

            logger.error("Failed to close Lucene writer for " + writer, t);

         }

@@ -540,7 +553,8 @@
                  if (!paths.containsKey(path)) {

                     continue;

                  }



//TODO: Figure out filestore oid the path - bit more involved than updating the index

						
                  // remove path from set

                  ObjectId blobId = paths.remove(path);

                  result.blobCount++;

@@ -677,9 +691,24 @@
               }



               if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {

                  String str = "";

                  // read the blob content

                  String str = JGitUtils.getStringContent(repository, commit.getTree(),

                  if (path.isFilestoreItem()) {

                     //Get file from filestore

                     BodyContentHandler handler = new BodyContentHandler();

                           Metadata metadata = new Metadata();

                           PDFParser parser = new PDFParser();

	                        
                           ParseContext parseContext = new ParseContext();

                           File lfsFile = filestoreManager.getStoragePath(path.getFilestoreOid());

                           FileInputStream inputstream = new FileInputStream(lfsFile);

                           parser.parse(inputstream, handler, metadata, parseContext);

                     str = handler.toString();

                  } else {

                     str = JGitUtils.getStringContent(repository, commit.getTree(),

                        path.path, encodings);

                  }

						
                  if (str != null) {

                     doc.add(new Field(FIELD_CONTENT, str, TextField.TYPE_STORED));

                     writer.addDocument(doc);

@@ -722,8 +751,8 @@
      String q = MessageFormat.format(pattern, SearchObjectType.blob.name(), branch, path);



      BooleanQuery query = new BooleanQuery();

      StandardAnalyzer analyzer = new StandardAnalyzer();

      QueryParser qp = new QueryParser(FIELD_SUMMARY, analyzer);

      StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);

      QueryParser qp = new QueryParser(LUCENE_VERSION, FIELD_SUMMARY, analyzer);

      query.add(qp.parse(q), Occur.MUST);



      IndexWriter writer = getIndexWriter(repositoryName);

@@ -968,14 +997,14 @@
      IndexWriter indexWriter = writers.get(repository);

      File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED);

      File indexFolder = new File(repositoryFolder, LUCENE_DIR);

      Directory directory = FSDirectory.open(indexFolder.toPath());

      Directory directory = FSDirectory.open(indexFolder);



      if (indexWriter == null) {

         if (!indexFolder.exists()) {

            indexFolder.mkdirs();

         }

         StandardAnalyzer analyzer = new StandardAnalyzer();

         IndexWriterConfig config = new IndexWriterConfig(analyzer);

         StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);

         IndexWriterConfig config = new IndexWriterConfig(LUCENE_VERSION, analyzer);

         config.setOpenMode(OpenMode.CREATE_OR_APPEND);

         indexWriter = new IndexWriter(directory, config);

         writers.put(repository, indexWriter);

@@ -1028,16 +1057,16 @@
         return null;

      }

      Set<SearchResult> results = new LinkedHashSet<SearchResult>();

      StandardAnalyzer analyzer = new StandardAnalyzer();

      StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION);

      try {

         // default search checks summary and content

         BooleanQuery query = new BooleanQuery();

         QueryParser qp;

         qp = new QueryParser(FIELD_SUMMARY, analyzer);

         qp = new QueryParser(LUCENE_VERSION, FIELD_SUMMARY, analyzer);

         qp.setAllowLeadingWildcard(true);

         query.add(qp.parse(text), Occur.SHOULD);



         qp = new QueryParser(FIELD_CONTENT, analyzer);

         qp = new QueryParser(LUCENE_VERSION, FIELD_CONTENT, analyzer);

         qp.setAllowLeadingWildcard(true);

         query.add(qp.parse(text), Occur.SHOULD);



@@ -1060,7 +1089,7 @@
         Query rewrittenQuery = searcher.rewrite(query);

         logger.debug(rewrittenQuery.toString());



         TopScoreDocCollector collector = TopScoreDocCollector.create(5000);

         TopScoreDocCollector collector = TopScoreDocCollector.create(5000, true);

         searcher.search(rewrittenQuery, collector);

         int offset = Math.max(0, (page - 1) * pageSize);

         ScoreDoc[] hits = collector.topDocs(offset, pageSize).scoreDocs;

@@ -1225,7 +1254,7 @@
    */

   private class MultiSourceReader extends MultiReader {



      MultiSourceReader(IndexReader [] readers) throws IOException {

      MultiSourceReader(IndexReader [] readers) {

         super(readers, false);

      }