| | |
| | | import java.text.MessageFormat;
|
| | | import java.text.ParseException;
|
| | | import java.util.ArrayList;
|
| | | import java.util.Arrays;
|
| | | import java.util.Collections;
|
| | | import java.util.Comparator;
|
| | | import java.util.HashMap;
|
| | |
| | | import org.eclipse.jgit.lib.ObjectLoader;
|
| | | import org.eclipse.jgit.lib.ObjectReader;
|
| | | import org.eclipse.jgit.lib.Repository;
|
| | | import org.eclipse.jgit.lib.RepositoryCache.FileKey;
|
| | | import org.eclipse.jgit.revwalk.RevCommit;
|
| | | import org.eclipse.jgit.revwalk.RevTree;
|
| | | import org.eclipse.jgit.revwalk.RevWalk;
|
| | |
| | | public class LuceneExecutor implements Runnable {
|
| | |
|
| | |
|
| | | private static final int INDEX_VERSION = 2;
|
| | | private static final int INDEX_VERSION = 3;
|
| | |
|
| | | private static final String FIELD_OBJECT_TYPE = "type";
|
| | | private static final String FIELD_ISSUE = "issue";
|
| | |
| | | private final Map<String, IndexSearcher> searchers = new ConcurrentHashMap<String, IndexSearcher>();
|
| | | private final Map<String, IndexWriter> writers = new ConcurrentHashMap<String, IndexWriter>();
|
| | |
|
| | | private final Set<String> excludedExtensions = new TreeSet<String>(Arrays.asList("7z", "arc",
|
| | | "arj", "bin", "bmp", "dll", "doc", "docx", "exe", "gif", "gz", "jar", "jpg", "lib",
|
| | | "lzh", "odg", "pdf", "ppt", "png", "so", "swf", "xcf", "xls", "xlsx", "zip"));
|
| | |
|
| | | private final String luceneIgnoreExtensions = "7z arc arj bin bmp dll doc docx exe gif gz jar jpg lib lzh odg odf odt pdf ppt png so swf xcf xls xlsx zip";
|
| | | private Set<String> excludedExtensions;
|
| | | |
| | | public LuceneExecutor(IStoredSettings settings, File repositoriesFolder) {
|
| | | this.storedSettings = settings;
|
| | | this.repositoriesFolder = repositoriesFolder;
|
| | | String exts = luceneIgnoreExtensions;
|
| | | if (settings != null) {
|
| | | exts = settings.getString(Keys.web.luceneIgnoreExtensions, exts);
|
| | | }
|
| | | excludedExtensions = new TreeSet<String>(StringUtils.getStringsFromValue(exts));
|
| | | }
|
| | |
|
| | | /**
|
| | |
| | | */
|
| | | @Override
|
| | | public void run() {
|
| | | if (!storedSettings.getBoolean(Keys.web.allowLuceneIndexing, true)) {
|
| | | // Lucene indexing is disabled
|
| | | return;
|
| | | }
|
| | | // reload the excluded extensions
|
| | | String exts = storedSettings.getString(Keys.web.luceneIgnoreExtensions, luceneIgnoreExtensions);
|
| | | excludedExtensions = new TreeSet<String>(StringUtils.getStringsFromValue(exts));
|
| | |
|
| | | for (String repositoryName: GitBlit.self().getRepositoryList()) {
|
| | | RepositoryModel model = GitBlit.self().getRepositoryModel(repositoryName);
|
| | | if (model.hasCommits && !ArrayUtils.isEmpty(model.indexedBranches)) {
|
| | |
| | | * @param repository
|
| | | * the repository object
|
| | | */
|
| | | protected void index(RepositoryModel model, Repository repository) {
|
| | | private void index(RepositoryModel model, Repository repository) {
|
| | | try {
|
| | | if (shouldReindex(repository)) {
|
| | | // (re)build the entire index
|
| | |
| | | * @return tree
|
| | | * @throws IOException
|
| | | */
|
| | | protected RevTree getTree(final RevWalk walk, final RevCommit commit)
|
| | | private RevTree getTree(final RevWalk walk, final RevCommit commit)
|
| | | throws IOException {
|
| | | final RevTree tree = commit.getTree();
|
| | | if (tree != null) {
|
| | |
| | | * @param repository
|
| | | * @return true of the on-disk index format is different than INDEX_VERSION
|
| | | */
|
| | | protected boolean shouldReindex(Repository repository) {
|
| | | private boolean shouldReindex(Repository repository) {
|
| | | try {
|
| | | FileBasedConfig config = getConfig(repository);
|
| | | config.load();
|
| | |
| | | ObjectId defaultBranchId = JGitUtils.getDefaultBranch(repository);
|
| | | for (RefModel branch : branches) {
|
| | | if (branch.getObjectId().equals(defaultBranchId)) {
|
| | | defaultBranch = branch; |
| | | defaultBranch = branch;
|
| | | break;
|
| | | }
|
| | | }
|
| | |
| | | // walk through each branch
|
| | | for (RefModel branch : branches) {
|
| | |
|
| | | boolean indexBranch = false;
|
| | | if (model.indexedBranches.contains(com.gitblit.Constants.DEFAULT_BRANCH)
|
| | | && branch.equals(defaultBranch)) {
|
| | | // indexing "default" branch
|
| | | indexBranch = true;
|
| | | } else if (IssueUtils.GB_ISSUES.equals(branch)) {
|
| | | // skip the GB_ISSUES branch because it is indexed later
|
| | | // note: this is different than updateIndex
|
| | | indexBranch = false;
|
| | | } else {
|
| | | // normal explicit branch check
|
| | | indexBranch = model.indexedBranches.contains(branch.getName());
|
| | | }
|
| | | |
| | | // if this branch is not specifically indexed then skip
|
| | | if (!model.indexedBranches.contains(branch.getName())) {
|
| | | if (!indexBranch) {
|
| | | continue;
|
| | | }
|
| | |
|
| | |
| | | // index the blob content
|
| | | if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {
|
| | | ObjectLoader ldr = repository.open(blobId, Constants.OBJ_BLOB);
|
| | | InputStream in = ldr.openStream(); |
| | | InputStream in = ldr.openStream(); |
| | | int n;
|
| | | while ((n = in.read(tmp)) > 0) {
|
| | | os.write(tmp, 0, n);
|
| | |
| | | String branch, RevCommit commit) {
|
| | | IndexResult result = new IndexResult();
|
| | | try {
|
| | | String [] encodings = storedSettings.getStrings(Keys.web.blobEncodings).toArray(new String[0]);
|
| | | List<PathChangeModel> changedPaths = JGitUtils.getFilesInCommit(repository, commit);
|
| | | String revDate = DateTools.timeToString(commit.getCommitTime() * 1000L,
|
| | | Resolution.MINUTE);
|
| | |
| | | if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) {
|
| | | // read the blob content
|
| | | String str = JGitUtils.getStringContent(repository, commit.getTree(),
|
| | | path.path);
|
| | | path.path, encodings);
|
| | | doc.add(new Field(FIELD_CONTENT, str, Store.YES, Index.ANALYZED));
|
| | | writer.addDocument(doc);
|
| | | }
|
| | |
| | |
|
| | | // get any annotated commit tags
|
| | | List<String> commitTags = new ArrayList<String>();
|
| | | for (RefModel ref : JGitUtils.getTags(repository, true, -1)) {
|
| | | for (RefModel ref : JGitUtils.getTags(repository, false, -1)) {
|
| | | if (ref.isAnnotatedTag() && ref.getReferencedObjectId().equals(commit.getId())) {
|
| | | commitTags.add(ref.displayName);
|
| | | }
|
| | |
| | | * @param repository
|
| | | * @return IndexResult
|
| | | */
|
| | | protected IndexResult updateIndex(RepositoryModel model, Repository repository) {
|
| | | private IndexResult updateIndex(RepositoryModel model, Repository repository) {
|
| | | IndexResult result = new IndexResult();
|
| | | try {
|
| | | FileBasedConfig config = getConfig(repository);
|
| | |
| | | deletedBranches.add(branch);
|
| | | }
|
| | |
|
| | | // walk through each branches
|
| | | // get the local branches
|
| | | List<RefModel> branches = JGitUtils.getLocalBranches(repository, true, -1);
|
| | | |
| | | // sort them by most recently updated
|
| | | Collections.sort(branches, new Comparator<RefModel>() {
|
| | | @Override
|
| | | public int compare(RefModel ref1, RefModel ref2) {
|
| | | return ref2.getDate().compareTo(ref1.getDate());
|
| | | }
|
| | | });
|
| | | |
| | | // reorder default branch to first position
|
| | | RefModel defaultBranch = null;
|
| | | ObjectId defaultBranchId = JGitUtils.getDefaultBranch(repository);
|
| | | for (RefModel branch : branches) {
|
| | | if (branch.getObjectId().equals(defaultBranchId)) {
|
| | | defaultBranch = branch;
|
| | | break;
|
| | | }
|
| | | }
|
| | | branches.remove(defaultBranch);
|
| | | branches.add(0, defaultBranch);
|
| | | |
| | | // walk through each branches
|
| | | for (RefModel branch : branches) {
|
| | | String branchName = branch.getName();
|
| | |
|
| | | // determine if we should skip this branch
|
| | | if (!IssueUtils.GB_ISSUES.equals(branch)
|
| | | && !model.indexedBranches.contains(branch.getName())) {
|
| | | boolean indexBranch = false;
|
| | | if (model.indexedBranches.contains(com.gitblit.Constants.DEFAULT_BRANCH)
|
| | | && branch.equals(defaultBranch)) {
|
| | | // indexing "default" branch
|
| | | indexBranch = true;
|
| | | } else if (IssueUtils.GB_ISSUES.equals(branch)) {
|
| | | // update issues modified on the GB_ISSUES branch
|
| | | // note: this is different than reindex
|
| | | indexBranch = true;
|
| | | } else {
|
| | | // normal explicit branch check
|
| | | indexBranch = model.indexedBranches.contains(branch.getName());
|
| | | }
|
| | | |
| | | // if this branch is not specifically indexed then skip
|
| | | if (!indexBranch) {
|
| | | continue;
|
| | | }
|
| | |
|
| | | // remove this branch from the deletedBranches set
|
| | | deletedBranches.remove(branchName);
|
| | |
|
| | | |
| | | // determine last commit
|
| | | String keyName = getBranchKey(branchName);
|
| | | String lastCommit = config.getString(CONF_BRANCH, null, keyName);
|
| | |
| | | * @throws IOException
|
| | | */
|
| | | private IndexWriter getIndexWriter(String repository) throws IOException {
|
| | | IndexWriter indexWriter = writers.get(repository); |
| | | File repositoryFolder = new File(repositoriesFolder, repository);
|
| | | IndexWriter indexWriter = writers.get(repository); |
| | | File repositoryFolder = FileKey.resolve(new File(repositoriesFolder, repository), FS.DETECTED);
|
| | | File indexFolder = new File(repositoryFolder, LUCENE_DIR);
|
| | | Directory directory = FSDirectory.open(indexFolder);
|
| | |
|
| | |
| | | Fragmenter fragmenter = new SimpleSpanFragmenter(scorer, fragmentLength);
|
| | |
|
| | | // use an artificial delimiter for the token
|
| | | String termTag = "<!--[";
|
| | | String termTagEnd = "]-->";
|
| | | String termTag = "!!--[";
|
| | | String termTagEnd = "]--!!";
|
| | | SimpleHTMLFormatter formatter = new SimpleHTMLFormatter(termTag, termTagEnd);
|
| | | Highlighter highlighter = new Highlighter(formatter, scorer);
|
| | | highlighter.setTextFragmenter(fragmenter);
|
| | |
| | |
|
| | | if (SearchObjectType.blob == result.type) {
|
| | | // count lines as offset into the content for this fragment
|
| | | int line = StringUtils.countLines(content.substring(0, pos));
|
| | | int line = Math.max(1, StringUtils.countLines(content.substring(0, pos)));
|
| | |
|
| | | // create fragment tag with line number and language
|
| | | String lang = "";
|
| | |
| | | sb.append(tag);
|
| | |
|
| | | // replace the artificial delimiter with html tags
|
| | | String html = fragment.replace(termTag, "<span class=\"highlight\">").replace(termTagEnd, "</span>");
|
| | | String html = StringUtils.escapeForHtml(fragment, false);
|
| | | html = html.replace(termTag, "<span class=\"highlight\">").replace(termTagEnd, "</span>");
|
| | | sb.append(html);
|
| | | sb.append("</pre>");
|
| | | if (i < len - 1) {
|