From 644bdd5a59a5ed5fbf93a0765f92608b0530c16a Mon Sep 17 00:00:00 2001 From: James Moger <james.moger@gitblit.com> Date: Wed, 31 Oct 2012 23:16:24 -0400 Subject: [PATCH] Improve transparency of permissions by indicating permission source --- src/com/gitblit/LuceneExecutor.java | 373 ++++++++++++++++++++++++++++++++++------------------ 1 files changed, 243 insertions(+), 130 deletions(-) diff --git a/src/com/gitblit/LuceneExecutor.java b/src/com/gitblit/LuceneExecutor.java index 220967b..0e4baae 100644 --- a/src/com/gitblit/LuceneExecutor.java +++ b/src/com/gitblit/LuceneExecutor.java @@ -25,7 +25,6 @@ import java.text.MessageFormat; import java.text.ParseException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -70,10 +69,12 @@ import org.apache.lucene.util.Version; import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.RepositoryCache.FileKey; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.revwalk.RevWalk; @@ -105,7 +106,7 @@ public class LuceneExecutor implements Runnable { - private static final int INDEX_VERSION = 2; + private static final int INDEX_VERSION = 5; private static final String FIELD_OBJECT_TYPE = "type"; private static final String FIELD_ISSUE = "issue"; @@ -138,40 +139,49 @@ private final Map<String, IndexSearcher> searchers = new ConcurrentHashMap<String, IndexSearcher>(); private final Map<String, IndexWriter> writers = new ConcurrentHashMap<String, IndexWriter>(); - private final Set<String> excludedExtensions = new TreeSet<String>(Arrays.asList("7z", "arc", - "arj", "bin", "bmp", "dll", "doc", "docx", "exe", "gif", "gz", "jar", "jpg", "lib", - "lzh", "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", "xlsx", "zip")); - + private final String luceneIgnoreExtensions = "7z arc arj bin bmp dll doc docx exe gif gz jar jpg lib lzh odg odf odt pdf ppt png so swf xcf xls xlsx zip"; + private Set<String> excludedExtensions; + public LuceneExecutor(IStoredSettings settings, File repositoriesFolder) { this.storedSettings = settings; this.repositoriesFolder = repositoriesFolder; + String exts = luceneIgnoreExtensions; + if (settings != null) { + exts = settings.getString(Keys.web.luceneIgnoreExtensions, exts); + } + excludedExtensions = new TreeSet<String>(StringUtils.getStringsFromValue(exts)); } /** - * Indicates if the Lucene executor can index repositories. - * - * @return true if the Lucene executor is ready to index repositories - */ - public boolean isReady() { - return storedSettings.getBoolean(Keys.lucene.enable, false); - } - - /** - * Run is executed by the gitblit executor service at whatever frequency - * is specified in the settings. Because this is called by an executor - * service, calls will queue - i.e. there can never be concurrent execution - * of repository index updates. + * Run is executed by the Gitblit executor service. Because this is called + * by an executor service, calls will queue - i.e. there can never be + * concurrent execution of repository index updates. */ @Override public void run() { - if (!isReady()) { + if (!storedSettings.getBoolean(Keys.web.allowLuceneIndexing, true)) { + // Lucene indexing is disabled return; } + // reload the excluded extensions + String exts = storedSettings.getString(Keys.web.luceneIgnoreExtensions, luceneIgnoreExtensions); + excludedExtensions = new TreeSet<String>(StringUtils.getStringsFromValue(exts)); + if (GitBlit.self().isCollectingGarbage()) { + // busy collecting garbage, try again later + return; + } + for (String repositoryName: GitBlit.self().getRepositoryList()) { RepositoryModel model = GitBlit.self().getRepositoryModel(repositoryName); if (model.hasCommits && !ArrayUtils.isEmpty(model.indexedBranches)) { Repository repository = GitBlit.self().getRepository(model.name); + if (repository == null) { + if (GitBlit.self().isCollectingGarbage(model.name)) { + logger.info(MessageFormat.format("Skipping Lucene index of {0}, busy garbage collecting", repositoryName)); + } + continue; + } index(model, repository); repository.close(); System.gc(); @@ -188,7 +198,7 @@ * @param repository * the repository object */ - protected void index(RepositoryModel model, Repository repository) { + private void index(RepositoryModel model, Repository repository) { try { if (shouldReindex(repository)) { // (re)build the entire index @@ -228,7 +238,16 @@ * * @param repositoryName */ - public void close(String repositoryName) { + public synchronized void close(String repositoryName) { + try { + IndexSearcher searcher = searchers.remove(repositoryName); + if (searcher != null) { + searcher.getIndexReader().close(); + } + } catch (Exception e) { + logger.error("Failed to close index searcher for " + repositoryName, e); + } + try { IndexWriter writer = writers.remove(repositoryName); if (writer != null) { @@ -236,23 +255,14 @@ } } catch (Exception e) { logger.error("Failed to close index writer for " + repositoryName, e); - } - - try { - IndexSearcher searcher = searchers.remove(repositoryName); - if (searcher != null) { - searcher.close(); - } - } catch (Exception e) { - logger.error("Failed to close index searcher for " + repositoryName, e); - } + } } /** * Close all Lucene indexers. * */ - public void close() { + public synchronized void close() { // close all writers for (String writer : writers.keySet()) { try { @@ -266,7 +276,7 @@ // close all searchers for (String searcher : searchers.keySet()) { try { - searchers.get(searcher).close(); + searchers.get(searcher).getIndexReader().close(); } catch (Throwable t) { logger.error("Failed to close Lucene searcher for " + searcher, t); } @@ -283,20 +293,11 @@ */ public boolean deleteIndex(String repositoryName) { try { - // remove the repository index writer from the cache and close it - IndexWriter writer = writers.remove(repositoryName); - if (writer != null) { - writer.close(); - writer = null; - } - // remove the repository index searcher from the cache and close it - IndexSearcher searcher = searchers.remove(repositoryName); - if (searcher != null) { - searcher.close(); - searcher = null; - } + // close any open writer/searcher + close(repositoryName); + // delete the index folder - File repositoryFolder = new File(repositoriesFolder, repositoryName); + File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repositoryName), FS.DETECTED); File luceneIndex = new File(repositoryFolder, LUCENE_DIR); if (luceneIndex.exists()) { org.eclipse.jgit.util.FileUtils.delete(luceneIndex, @@ -312,7 +313,6 @@ throw new RuntimeException(e); } } - /** * Returns the author for the commit, if this information is available. @@ -358,7 +358,7 @@ * @return tree * @throws IOException */ - protected RevTree getTree(final RevWalk walk, final RevCommit commit) + private RevTree getTree(final RevWalk walk, final RevCommit commit) throws IOException { final RevTree tree = commit.getTree(); if (tree != null) { @@ -398,7 +398,7 @@ * @param repository * @return true of the on-disk index format is different than INDEX_VERSION */ - protected boolean shouldReindex(Repository repository) { + private boolean shouldReindex(Repository repository) { try { FileBasedConfig config = getConfig(repository); config.load(); @@ -420,11 +420,12 @@ * @return IndexResult */ public IndexResult reindex(RepositoryModel model, Repository repository) { - IndexResult result = new IndexResult(); + IndexResult result = new IndexResult(); if (!deleteIndex(model.name)) { return result; } - try { + try { + String [] encodings = storedSettings.getStrings(Keys.web.blobEncodings).toArray(new String[0]); FileBasedConfig config = getConfig(repository); Set<String> indexedCommits = new TreeSet<String>(); IndexWriter writer = getIndexWriter(model.name); @@ -459,7 +460,7 @@ ObjectId defaultBranchId = JGitUtils.getDefaultBranch(repository); for (RefModel branch : branches) { if (branch.getObjectId().equals(defaultBranchId)) { - defaultBranch = branch; + defaultBranch = branch; break; } } @@ -469,8 +470,22 @@ // walk through each branch for (RefModel branch : branches) { + boolean indexBranch = false; + if (model.indexedBranches.contains(com.gitblit.Constants.DEFAULT_BRANCH) + && branch.equals(defaultBranch)) { + // indexing "default" branch + indexBranch = true; + } else if (IssueUtils.GB_ISSUES.equals(branch)) { + // skip the GB_ISSUES branch because it is indexed later + // note: this is different than updateIndex + indexBranch = false; + } else { + // normal explicit branch check + indexBranch = model.indexedBranches.contains(branch.getName()); + } + // if this branch is not specifically indexed then skip - if (!model.indexedBranches.contains(branch.getName())) { + if (!indexBranch) { continue; } @@ -490,7 +505,10 @@ Map<String, ObjectId> paths = new TreeMap<String, ObjectId>(); while (treeWalk.next()) { - paths.put(treeWalk.getPathString(), treeWalk.getObjectId(0)); + // ensure path is not in a submodule + if (treeWalk.getFileMode(0) != FileMode.GITLINK) { + paths.put(treeWalk.getPathString(), treeWalk.getObjectId(0)); + } } ByteArrayOutputStream os = new ByteArrayOutputStream(); @@ -553,14 +571,14 @@ // index the blob content if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) { ObjectLoader ldr = repository.open(blobId, Constants.OBJ_BLOB); - InputStream in = ldr.openStream(); + InputStream in = ldr.openStream(); int n; while ((n = in.read(tmp)) > 0) { os.write(tmp, 0, n); } in.close(); byte[] content = os.toByteArray(); - String str = new String(content, Constants.CHARACTER_ENCODING); + String str = StringUtils.decodeString(content, encodings); doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED)); os.reset(); } @@ -615,8 +633,8 @@ // commit all changes and reset the searcher config.setInt(CONF_INDEX, null, CONF_VERSION, INDEX_VERSION); config.save(); - resetIndexSearcher(model.name); writer.commit(); + resetIndexSearcher(model.name); result.success(); } catch (Exception e) { logger.error("Exception while reindexing " + model.name, e); @@ -639,13 +657,17 @@ String branch, RevCommit commit) { IndexResult result = new IndexResult(); try { + String [] encodings = storedSettings.getStrings(Keys.web.blobEncodings).toArray(new String[0]); List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit); String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L, Resolution.MINUTE); IndexWriter writer = getIndexWriter(repositoryName); for (PathChangeModel path : changedPaths) { + if (path.isSubmodule()) { + continue; + } // delete the indexed blob - deleteBlob(repositoryName, branch, path.path); + deleteBlob(repositoryName, branch, path.name); // re-index the blob if (!ChangeType.DELETE.equals(path.changeType)) { @@ -671,15 +693,26 @@ if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) { // read the blob content String str = JGitUtils.getStringContent(repository, commit.getTree(), - path.path); - doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED)); - writer.addDocument(doc); + path.path, encodings); + if (str != null) { + doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED)); + writer.addDocument(doc); + } } } } writer.commit(); - - Document doc = createDocument(commit, null); + + // get any annotated commit tags + List<String> commitTags = new ArrayList<String>(); + for (RefModel ref : JGitUtils.getTags(repository, false, -1)) { + if (ref.isAnnotatedTag() && ref.getReferencedObjectId().equals(commit.getId())) { + commitTags.add(ref.displayName); + } + } + + // create and write the Lucene document + Document doc = createDocument(commit, commitTags); doc.add(new Field(FIELD_BRANCH, branch, Store.YES, Index.ANALYZED)); result.commitCount++; result.success = index(repositoryName, doc); @@ -715,8 +748,9 @@ * @param repositoryName * @param issueId * @throws Exception + * @return true, if deleted, false if no record was deleted */ - private void deleteIssue(String repositoryName, String issueId) throws Exception { + private boolean deleteIssue(String repositoryName, String issueId) throws Exception { BooleanQuery query = new BooleanQuery(); Term objectTerm = new Term(FIELD_OBJECT_TYPE, SearchObjectType.issue.name()); query.add(new TermQuery(objectTerm), Occur.MUST); @@ -724,8 +758,17 @@ query.add(new TermQuery(issueidTerm), Occur.MUST); IndexWriter writer = getIndexWriter(repositoryName); + int numDocsBefore = writer.numDocs(); writer.deleteDocuments(query); writer.commit(); + int numDocsAfter = writer.numDocs(); + if (numDocsBefore == numDocsAfter) { + logger.debug(MessageFormat.format("no records found to delete {0}", query.toString())); + return false; + } else { + logger.debug(MessageFormat.format("deleted {0} records with {1}", numDocsBefore - numDocsAfter, query.toString())); + return true; + } } /** @@ -735,19 +778,29 @@ * @param branch * @param path * @throws Exception + * @return true, if deleted, false if no record was deleted */ - private void deleteBlob(String repositoryName, String branch, String path) throws Exception { - BooleanQuery query = new BooleanQuery(); - Term objectTerm = new Term(FIELD_OBJECT_TYPE, SearchObjectType.blob.name()); - query.add(new TermQuery(objectTerm), Occur.MUST); - Term branchTerm = new Term(FIELD_BRANCH, branch); - query.add(new TermQuery(branchTerm), Occur.MUST); - Term pathTerm = new Term(FIELD_PATH, path); - query.add(new TermQuery(pathTerm), Occur.MUST); + public boolean deleteBlob(String repositoryName, String branch, String path) throws Exception { + String pattern = MessageFormat.format("{0}:'{'0} AND {1}:\"'{'1'}'\" AND {2}:\"'{'2'}'\"", FIELD_OBJECT_TYPE, FIELD_BRANCH, FIELD_PATH); + String q = MessageFormat.format(pattern, SearchObjectType.blob.name(), branch, path); + BooleanQuery query = new BooleanQuery(); + StandardAnalyzer analyzer = new StandardAnalyzer(LUCENE_VERSION); + QueryParser qp = new QueryParser(LUCENE_VERSION, FIELD_SUMMARY, analyzer); + query.add(qp.parse(q), Occur.MUST); + IndexWriter writer = getIndexWriter(repositoryName); - writer.deleteDocuments(query); + int numDocsBefore = writer.numDocs(); + writer.deleteDocuments(query); writer.commit(); + int numDocsAfter = writer.numDocs(); + if (numDocsBefore == numDocsAfter) { + logger.debug(MessageFormat.format("no records found to delete {0}", query.toString())); + return false; + } else { + logger.debug(MessageFormat.format("deleted {0} records with {1}", numDocsBefore - numDocsAfter, query.toString())); + return true; + } } /** @@ -757,7 +810,7 @@ * @param repository * @return IndexResult */ - protected IndexResult updateIndex(RepositoryModel model, Repository repository) { + private IndexResult updateIndex(RepositoryModel model, Repository repository) { IndexResult result = new IndexResult(); try { FileBasedConfig config = getConfig(repository); @@ -785,20 +838,55 @@ deletedBranches.add(branch); } - // walk through each branches + // get the local branches List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1); + + // sort them by most recently updated + Collections.sort(branches, new Comparator<RefModel>() { + @Override + public int compare(RefModel ref1, RefModel ref2) { + return ref2.getDate().compareTo(ref1.getDate()); + } + }); + + // reorder default branch to first position + RefModel defaultBranch = null; + ObjectId defaultBranchId = JGitUtils.getDefaultBranch(repository); + for (RefModel branch : branches) { + if (branch.getObjectId().equals(defaultBranchId)) { + defaultBranch = branch; + break; + } + } + branches.remove(defaultBranch); + branches.add(0, defaultBranch); + + // walk through each branches for (RefModel branch : branches) { String branchName = branch.getName(); - // determine if we should skip this branch - if (!IssueUtils.GB_ISSUES.equals(branch) - && !model.indexedBranches.contains(branch.getName())) { + boolean indexBranch = false; + if (model.indexedBranches.contains(com.gitblit.Constants.DEFAULT_BRANCH) + && branch.equals(defaultBranch)) { + // indexing "default" branch + indexBranch = true; + } else if (IssueUtils.GB_ISSUES.equals(branch)) { + // update issues modified on the GB_ISSUES branch + // note: this is different than reindex + indexBranch = true; + } else { + // normal explicit branch check + indexBranch = model.indexedBranches.contains(branch.getName()); + } + + // if this branch is not specifically indexed then skip + if (!indexBranch) { continue; } // remove this branch from the deletedBranches set deletedBranches.remove(branchName); - + // determine last commit String keyName = getBranchKey(branchName); String lastCommit = config.getString(CONF_BRANCH, null, keyName); @@ -833,7 +921,9 @@ IssueModel issue = IssueUtils.getIssue(repository, issueId); if (issue == null) { // issue was deleted, remove from index - deleteIssue(model.name, issueId); + if (!deleteIssue(model.name, issueId)) { + logger.error(MessageFormat.format("Failed to delete issue {0} from Lucene index!", issueId)); + } } else { // issue was updated index(model.name, issue); @@ -931,8 +1021,8 @@ try { IndexWriter writer = getIndexWriter(repositoryName); writer.addDocument(doc); - resetIndexSearcher(repositoryName); writer.commit(); + resetIndexSearcher(repositoryName); return true; } catch (Exception e) { logger.error(MessageFormat.format("Exception while incrementally updating {0} Lucene index", repositoryName), e); @@ -966,7 +1056,7 @@ private synchronized void resetIndexSearcher(String repository) throws IOException { IndexSearcher searcher = searchers.remove(repository); if (searcher != null) { - searcher.close(); + searcher.getIndexReader().close(); } } @@ -996,8 +1086,8 @@ * @throws IOException */ private IndexWriter getIndexWriter(String repository) throws IOException { - IndexWriter indexWriter = writers.get(repository); - File repositoryFolder = new File(repositoriesFolder, repository); + IndexWriter indexWriter = writers.get(repository); + File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED); File indexFolder = new File(repositoryFolder, LUCENE_DIR); Directory directory = FSDirectory.open(indexFolder); @@ -1071,7 +1161,7 @@ qp = new QueryParser(LUCENE_VERSION, FIELD_CONTENT, analyzer); qp.setAllowLeadingWildcard(true); query.add(qp.parse(text), Occur.SHOULD); - + IndexSearcher searcher; if (repositories.length == 1) { // single repository search @@ -1087,7 +1177,10 @@ MultiSourceReader reader = new MultiSourceReader(rdrs); searcher = new IndexSearcher(reader); } + Query rewrittenQuery = searcher.rewrite(query); + logger.debug(rewrittenQuery.toString()); + TopScoreDocCollector collector = TopScoreDocCollector.create(5000, true); searcher.search(rewrittenQuery, collector); int offset = Math.max(0, (page - 1) * pageSize); @@ -1128,42 +1221,86 @@ */ private String getHighlightedFragment(Analyzer analyzer, Query query, String content, SearchResult result) throws IOException, InvalidTokenOffsetsException { - content = content == null ? "":StringUtils.escapeForHtml(content, false); - + if (content == null) { + content = ""; + } + + int fragmentLength = SearchObjectType.commit == result.type ? 512 : 150; + QueryScorer scorer = new QueryScorer(query, "content"); - Fragmenter fragmenter; - - // TODO improve the fragmenter - hopefully on line breaks - if (SearchObjectType.commit == result.type) { - fragmenter = new SimpleSpanFragmenter(scorer, 1024); - } else { - fragmenter = new SimpleSpanFragmenter(scorer, 150); - } + Fragmenter fragmenter = new SimpleSpanFragmenter(scorer, fragmentLength); // use an artificial delimiter for the token - String termTag = "<!--["; - String termTagEnd = "]-->"; + String termTag = "!!--["; + String termTagEnd = "]--!!"; SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(termTag, termTagEnd); Highlighter highlighter = new Highlighter(formatter, scorer); highlighter.setTextFragmenter(fragmenter); - - String [] fragments = highlighter.getBestFragments(analyzer, "content", content, 5); + + String [] fragments = highlighter.getBestFragments(analyzer, "content", content, 3); if (ArrayUtils.isEmpty(fragments)) { if (SearchObjectType.blob == result.type) { return ""; } - return "<pre class=\"text\">" + content + "</pre>"; + // clip commit message + String fragment = content; + if (fragment.length() > fragmentLength) { + fragment = fragment.substring(0, fragmentLength) + "..."; + } + return "<pre class=\"text\">" + StringUtils.escapeForHtml(fragment, true) + "</pre>"; } + + // make sure we have unique fragments + Set<String> uniqueFragments = new LinkedHashSet<String>(); + for (String fragment : fragments) { + uniqueFragments.add(fragment); + } + fragments = uniqueFragments.toArray(new String[uniqueFragments.size()]); + StringBuilder sb = new StringBuilder(); for (int i = 0, len = fragments.length; i < len; i++) { String fragment = fragments[i]; - + String tag = "<pre class=\"text\">"; + // resurrect the raw fragment from removing the artificial delimiters - String raw = fragment.replace(termTag, "").replace(termTagEnd, ""); - sb.append(getPreTag(result, raw, content)); + String raw = fragment.replace(termTag, "").replace(termTagEnd, ""); + + // determine position of the raw fragment in the content + int pos = content.indexOf(raw); + + // restore complete first line of fragment + int c = pos; + while (c > 0) { + c--; + if (content.charAt(c) == '\n') { + break; + } + } + if (c > 0) { + // inject leading chunk of first fragment line + fragment = content.substring(c + 1, pos) + fragment; + } + + if (SearchObjectType.blob == result.type) { + // count lines as offset into the content for this fragment + int line = Math.max(1, StringUtils.countLines(content.substring(0, pos))); + + // create fragment tag with line number and language + String lang = ""; + String ext = StringUtils.getFileExtension(result.path).toLowerCase(); + if (!StringUtils.isEmpty(ext)) { + // maintain leading space! + lang = " lang-" + ext; + } + tag = MessageFormat.format("<pre class=\"prettyprint linenums:{0,number,0}{1}\">", line, lang); + + } + sb.append(tag); + // replace the artificial delimiter with html tags - String html = fragment.replace(termTag, "<span class=\"highlight\">").replace(termTagEnd, "</span>"); + String html = StringUtils.escapeForHtml(fragment, false); + html = html.replace(termTag, "<span class=\"highlight\">").replace(termTagEnd, "</span>"); sb.append(html); sb.append("</pre>"); if (i < len - 1) { @@ -1171,31 +1308,7 @@ } } return sb.toString(); - } - - /** - * Returns the appropriate tag for a fragment. Commit messages are visually - * differentiated from blob fragments. - * - * @param result - * @param fragment - * @param content - * @return an html tag appropriate for the fragment - */ - private String getPreTag(SearchResult result, String fragment, String content) { - String pre = "<pre class=\"text\">"; - if (SearchObjectType.blob == result.type) { - int line = StringUtils.countLines(content.substring(0, content.indexOf(fragment))); - int lastDot = result.path.lastIndexOf('.'); - if (lastDot > -1) { - String ext = result.path.substring(lastDot + 1).toLowerCase(); - pre = MessageFormat.format("<pre class=\"prettyprint linenums:{0,number,0} lang-{1}\">", line, ext); - } else { - pre = MessageFormat.format("<pre class=\"prettyprint linenums:{0,number,0}\">", line); - } - } - return pre; - } + } /** * Simple class to track the results of an index update. -- Gitblit v1.9.1