From eecaad8b8e2c447429c31a01d49260ddd6b4ee03 Mon Sep 17 00:00:00 2001 From: Paul Martin Date: Sat, 16 Apr 2016 22:35:32 +0100 Subject: [PATCH] Proof of concept #1026 + Update tika + Pull in Tika parsers + Remove chicken and egg scenario for FilestoreManager vs RepositoryManager --- build.moxie | 3 +- .../java/com/gitblit/FederationClient.java | 2 +- src/main/java/com/gitblit/MigrateTickets.java | 2 +- src/main/java/com/gitblit/ReindexTickets.java | 2 +- .../com/gitblit/manager/FilestoreManager.java | 10 ++---- .../com/gitblit/manager/GitblitManager.java | 4 +-- .../gitblit/manager/IFilestoreManager.java | 2 +- .../gitblit/manager/RepositoryManager.java | 8 +++-- .../com/gitblit/service/LuceneService.java | 35 +++++++++++++++++-- .../com/gitblit/utils/SyndicationUtils.java | 26 +++++++------- .../gitblit/wicket/pages/FilestorePage.java | 2 +- .../tests/BranchTicketServiceTest.java | 5 ++- .../gitblit/tests/FileTicketServiceTest.java | 3 +- .../com/gitblit/tests/LuceneExecutorTest.java | 6 ++-- .../gitblit/tests/RedisTicketServiceTest.java | 2 +- .../java/com/gitblit/tests/UITicketTest.java | 2 +- 16 files changed, 74 insertions(+), 40 deletions(-) diff --git a/build.moxie b/build.moxie index 8eb2bbfd2..022d574d7 100644 --- a/build.moxie +++ b/build.moxie @@ -178,7 +178,8 @@ dependencies: - compile 'commons-codec:commons-codec:1.7' :war - compile 'redis.clients:jedis:2.6.2' :war - compile 'ro.fortsoft.pf4j:pf4j:0.9.0' :war -- compile 'org.apache.tika:tika-core:1.5' :war +- compile 'org.apache.tika:tika-core:1.12' :war +- compile 'org.apache.tika:tika-parsers:1.12' :war - compile 'org.jsoup:jsoup:1.7.3' :war - test 'junit' # Dependencies for Selenium web page testing diff --git a/src/main/java/com/gitblit/FederationClient.java b/src/main/java/com/gitblit/FederationClient.java index 64ff01726..9b714dc56 100644 --- a/src/main/java/com/gitblit/FederationClient.java +++ b/src/main/java/com/gitblit/FederationClient.java @@ -98,7 +98,7 @@ public static void main(String[] args) { RuntimeManager runtime = new RuntimeManager(settings, xssFilter, baseFolder).start(); NoopNotificationManager notifications = new NoopNotificationManager().start(); UserManager users = new UserManager(runtime, null).start(); - RepositoryManager repositories = new RepositoryManager(runtime, null, users).start(); + RepositoryManager repositories = new RepositoryManager(runtime, null, users, null).start(); FederationManager federation = new FederationManager(runtime, notifications, repositories).start(); IGitblit gitblit = new GitblitManager(null, null, runtime, null, notifications, users, null, repositories, null, federation, null); diff --git a/src/main/java/com/gitblit/MigrateTickets.java b/src/main/java/com/gitblit/MigrateTickets.java index b08228efa..035b010e6 100644 --- a/src/main/java/com/gitblit/MigrateTickets.java +++ b/src/main/java/com/gitblit/MigrateTickets.java @@ -138,7 +138,7 @@ protected void migrate(File baseFolder, IStoredSettings settings, String outputS XssFilter xssFilter = new AllowXssFilter(); IRuntimeManager runtimeManager = new RuntimeManager(settings, xssFilter, baseFolder).start(); - IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, null, null).start(); + IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, null, null, null).start(); String inputServiceName = settings.getString(Keys.tickets.service, BranchTicketService.class.getSimpleName()); if (StringUtils.isEmpty(inputServiceName)) { diff --git a/src/main/java/com/gitblit/ReindexTickets.java b/src/main/java/com/gitblit/ReindexTickets.java index 858436afb..10b53e54d 100644 --- a/src/main/java/com/gitblit/ReindexTickets.java +++ b/src/main/java/com/gitblit/ReindexTickets.java @@ -130,7 +130,7 @@ protected void reindex(File baseFolder, IStoredSettings settings) { XssFilter xssFilter = new AllowXssFilter(); IRuntimeManager runtimeManager = new RuntimeManager(settings, xssFilter, baseFolder).start(); - IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, null, null).start(); + IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, null, null, null).start(); String serviceName = settings.getString(Keys.tickets.service, BranchTicketService.class.getSimpleName()); if (StringUtils.isEmpty(serviceName)) { diff --git a/src/main/java/com/gitblit/manager/FilestoreManager.java b/src/main/java/com/gitblit/manager/FilestoreManager.java index 111085573..6ed7c9299 100644 --- a/src/main/java/com/gitblit/manager/FilestoreManager.java +++ b/src/main/java/com/gitblit/manager/FilestoreManager.java @@ -78,8 +78,6 @@ public class FilestoreManager implements IFilestoreManager { private final IRuntimeManager runtimeManager; - private final IRepositoryManager repositoryManager; - private final IStoredSettings settings; public static final int UNDEFINED_SIZE = -1; @@ -94,11 +92,8 @@ public class FilestoreManager implements IFilestoreManager { @Inject - FilestoreManager( - IRuntimeManager runtimeManager, - IRepositoryManager repositoryManager) { + public FilestoreManager(IRuntimeManager runtimeManager) { this.runtimeManager = runtimeManager; - this.repositoryManager = repositoryManager; this.settings = runtimeManager.getSettings(); } @@ -328,9 +323,8 @@ public FilestoreModel.Status downloadBlob(String oid, UserModel user, Repository } @Override - public List getAllObjects(UserModel user) { + public List getAllObjects(List viewableRepositories) { - final List viewableRepositories = repositoryManager.getRepositoryModels(user); List viewableRepositoryNames = new ArrayList(viewableRepositories.size()); for (RepositoryModel repository : viewableRepositories) { diff --git a/src/main/java/com/gitblit/manager/GitblitManager.java b/src/main/java/com/gitblit/manager/GitblitManager.java index 85d5c19fa..bda7a1cb3 100644 --- a/src/main/java/com/gitblit/manager/GitblitManager.java +++ b/src/main/java/com/gitblit/manager/GitblitManager.java @@ -1274,8 +1274,8 @@ public FilestoreModel.Status downloadBlob(String oid, UserModel user, Repository } @Override - public List getAllObjects(UserModel user) { - return filestoreManager.getAllObjects(user); + public List getAllObjects(List viewableRepositories) { + return filestoreManager.getAllObjects(viewableRepositories); } @Override diff --git a/src/main/java/com/gitblit/manager/IFilestoreManager.java b/src/main/java/com/gitblit/manager/IFilestoreManager.java index 454331a3b..7c3f3c6aa 100644 --- a/src/main/java/com/gitblit/manager/IFilestoreManager.java +++ b/src/main/java/com/gitblit/manager/IFilestoreManager.java @@ -37,7 +37,7 @@ public interface IFilestoreManager extends IManager { FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut ); - List getAllObjects(UserModel user); + List getAllObjects(List viewableRepositories); File getStorageFolder(); diff --git a/src/main/java/com/gitblit/manager/RepositoryManager.java b/src/main/java/com/gitblit/manager/RepositoryManager.java index e2e4de686..8d1a6a71e 100644 --- a/src/main/java/com/gitblit/manager/RepositoryManager.java +++ b/src/main/java/com/gitblit/manager/RepositoryManager.java @@ -123,6 +123,8 @@ public class RepositoryManager implements IRepositoryManager { private final IPluginManager pluginManager; private final IUserManager userManager; + + private final IFilestoreManager filestoreManager; private File repositoriesFolder; @@ -136,12 +138,14 @@ public class RepositoryManager implements IRepositoryManager { public RepositoryManager( IRuntimeManager runtimeManager, IPluginManager pluginManager, - IUserManager userManager) { + IUserManager userManager, + IFilestoreManager filestoreManager) { this.settings = runtimeManager.getSettings(); this.runtimeManager = runtimeManager; this.pluginManager = pluginManager; this.userManager = userManager; + this.filestoreManager = filestoreManager; } @Override @@ -1866,7 +1870,7 @@ public List search(String query, int page, int pageSize, List searchers = new ConcurrentHashMap(); @@ -141,10 +152,12 @@ public class LuceneService implements Runnable { public LuceneService( IStoredSettings settings, - IRepositoryManager repositoryManager) { + IRepositoryManager repositoryManager, + IFilestoreManager filestoreManager) { this.storedSettings = settings; this.repositoryManager = repositoryManager; + this.filestoreManager = filestoreManager; this.repositoriesFolder = repositoryManager.getRepositoriesFolder(); String exts = luceneIgnoreExtensions; if (settings != null) { @@ -540,7 +553,8 @@ public int compare(RefModel ref1, RefModel ref2) { if (!paths.containsKey(path)) { continue; } - +//TODO: Figure out filestore oid the path - bit more involved than updating the index + // remove path from set ObjectId blobId = paths.remove(path); result.blobCount++; @@ -677,9 +691,24 @@ private IndexResult index(String repositoryName, Repository repository, } if (StringUtils.isEmpty(ext) || !excludedExtensions.contains(ext)) { + String str = ""; // read the blob content - String str = JGitUtils.getStringContent(repository, commit.getTree(), + if (path.isFilestoreItem()) { + //Get file from filestore + BodyContentHandler handler = new BodyContentHandler(); + Metadata metadata = new Metadata(); + PDFParser parser = new PDFParser(); + + ParseContext parseContext = new ParseContext(); + File lfsFile = filestoreManager.getStoragePath(path.getFilestoreOid()); + FileInputStream inputstream = new FileInputStream(lfsFile); + parser.parse(inputstream, handler, metadata, parseContext); + str = handler.toString(); + } else { + str = JGitUtils.getStringContent(repository, commit.getTree(), path.path, encodings); + } + if (str != null) { doc.add(new Field(FIELD_CONTENT, str, TextField.TYPE_STORED)); writer.addDocument(doc); diff --git a/src/main/java/com/gitblit/utils/SyndicationUtils.java b/src/main/java/com/gitblit/utils/SyndicationUtils.java index 7afd03839..dfbca3522 100644 --- a/src/main/java/com/gitblit/utils/SyndicationUtils.java +++ b/src/main/java/com/gitblit/utils/SyndicationUtils.java @@ -28,19 +28,19 @@ import com.gitblit.Constants.FeedObjectType; import com.gitblit.GitBlitException; import com.gitblit.models.FeedEntryModel; -import com.sun.syndication.feed.synd.SyndCategory; -import com.sun.syndication.feed.synd.SyndCategoryImpl; -import com.sun.syndication.feed.synd.SyndContent; -import com.sun.syndication.feed.synd.SyndContentImpl; -import com.sun.syndication.feed.synd.SyndEntry; -import com.sun.syndication.feed.synd.SyndEntryImpl; -import com.sun.syndication.feed.synd.SyndFeed; -import com.sun.syndication.feed.synd.SyndFeedImpl; -import com.sun.syndication.feed.synd.SyndImageImpl; -import com.sun.syndication.io.FeedException; -import com.sun.syndication.io.SyndFeedInput; -import com.sun.syndication.io.SyndFeedOutput; -import com.sun.syndication.io.XmlReader; +import com.rometools.rome.feed.synd.SyndCategory; +import com.rometools.rome.feed.synd.SyndCategoryImpl; +import com.rometools.rome.feed.synd.SyndContent; +import com.rometools.rome.feed.synd.SyndContentImpl; +import com.rometools.rome.feed.synd.SyndEntry; +import com.rometools.rome.feed.synd.SyndEntryImpl; +import com.rometools.rome.feed.synd.SyndFeed; +import com.rometools.rome.feed.synd.SyndFeedImpl; +import com.rometools.rome.feed.synd.SyndImageImpl; +import com.rometools.rome.io.FeedException; +import com.rometools.rome.io.SyndFeedInput; +import com.rometools.rome.io.SyndFeedOutput; +import com.rometools.rome.io.XmlReader; /** * Utility class for RSS feeds. diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java index 7130f6c22..cf9ee2514 100644 --- a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java +++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java @@ -67,7 +67,7 @@ public FilestorePage(PageParameters params) { final UserModel user = (GitBlitWebSession.get().getUser() == null) ? UserModel.ANONYMOUS : GitBlitWebSession.get().getUser(); final long nBytesUsed = app().filestore().getFilestoreUsedByteCount(); final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount(); - List files = app().filestore().getAllObjects(user); + List files = app().filestore().getAllObjects(app().repositories().getRepositoryModels(user)); if (files == null) { files = new ArrayList(); diff --git a/src/test/java/com/gitblit/tests/BranchTicketServiceTest.java b/src/test/java/com/gitblit/tests/BranchTicketServiceTest.java index 0a5de1961..248ee16e1 100644 --- a/src/test/java/com/gitblit/tests/BranchTicketServiceTest.java +++ b/src/test/java/com/gitblit/tests/BranchTicketServiceTest.java @@ -16,6 +16,8 @@ package com.gitblit.tests; import com.gitblit.IStoredSettings; +import com.gitblit.manager.FilestoreManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IPluginManager; import com.gitblit.manager.IRepositoryManager; @@ -57,7 +59,8 @@ protected ITicketService getService(boolean deleteAll) throws Exception { IPluginManager pluginManager = new PluginManager(runtimeManager).start(); INotificationManager notificationManager = new NotificationManager(settings).start(); IUserManager userManager = new UserManager(runtimeManager, pluginManager).start(); - IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, pluginManager, userManager).start(); + IRepositoryManager repositoryManager = new RepositoryManager( + runtimeManager, pluginManager, userManager, null).start(); BranchTicketService service = new BranchTicketService( runtimeManager, diff --git a/src/test/java/com/gitblit/tests/FileTicketServiceTest.java b/src/test/java/com/gitblit/tests/FileTicketServiceTest.java index 1fb2eed96..38523843b 100644 --- a/src/test/java/com/gitblit/tests/FileTicketServiceTest.java +++ b/src/test/java/com/gitblit/tests/FileTicketServiceTest.java @@ -56,7 +56,8 @@ protected ITicketService getService(boolean deleteAll) throws Exception { IPluginManager pluginManager = new PluginManager(runtimeManager).start(); INotificationManager notificationManager = new NotificationManager(settings).start(); IUserManager userManager = new UserManager(runtimeManager, pluginManager).start(); - IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, pluginManager, userManager).start(); + IRepositoryManager repositoryManager = new RepositoryManager( + runtimeManager, pluginManager, userManager, null).start(); FileTicketService service = new FileTicketService( runtimeManager, diff --git a/src/test/java/com/gitblit/tests/LuceneExecutorTest.java b/src/test/java/com/gitblit/tests/LuceneExecutorTest.java index a8358b99f..b9ecbbbdd 100644 --- a/src/test/java/com/gitblit/tests/LuceneExecutorTest.java +++ b/src/test/java/com/gitblit/tests/LuceneExecutorTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import com.gitblit.Keys; +import com.gitblit.manager.FilestoreManager; import com.gitblit.manager.RepositoryManager; import com.gitblit.manager.RuntimeManager; import com.gitblit.manager.UserManager; @@ -53,8 +54,9 @@ private LuceneService newLuceneExecutor() { XssFilter xssFilter = new AllowXssFilter(); RuntimeManager runtime = new RuntimeManager(settings, xssFilter, GitBlitSuite.BASEFOLDER).start(); UserManager users = new UserManager(runtime, null).start(); - RepositoryManager repos = new RepositoryManager(runtime, null, users); - return new LuceneService(settings, repos); + RepositoryManager repos = new RepositoryManager(runtime, null, users, null); + //TODO: May need filestore + return new LuceneService(settings, repos, null); } private RepositoryModel newRepositoryModel(Repository repository) { diff --git a/src/test/java/com/gitblit/tests/RedisTicketServiceTest.java b/src/test/java/com/gitblit/tests/RedisTicketServiceTest.java index 48011ade2..6bfab43d0 100644 --- a/src/test/java/com/gitblit/tests/RedisTicketServiceTest.java +++ b/src/test/java/com/gitblit/tests/RedisTicketServiceTest.java @@ -64,7 +64,7 @@ protected ITicketService getService(boolean deleteAll) throws Exception { IPluginManager pluginManager = new PluginManager(runtimeManager).start(); INotificationManager notificationManager = new NotificationManager(settings).start(); IUserManager userManager = new UserManager(runtimeManager, pluginManager).start(); - IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, pluginManager, userManager).start(); + IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, pluginManager, userManager, null).start(); RedisTicketService service = new RedisTicketService( runtimeManager, diff --git a/src/test/java/com/gitblit/tests/UITicketTest.java b/src/test/java/com/gitblit/tests/UITicketTest.java index 54aa1e1e7..cb61b3e8a 100644 --- a/src/test/java/com/gitblit/tests/UITicketTest.java +++ b/src/test/java/com/gitblit/tests/UITicketTest.java @@ -81,7 +81,7 @@ protected ITicketService getService(boolean deleteAll) throws Exception { IPluginManager pluginManager = new PluginManager(runtimeManager).start(); INotificationManager notificationManager = new NotificationManager(settings).start(); IUserManager userManager = new UserManager(runtimeManager, pluginManager).start(); - IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, pluginManager, userManager).start(); + IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, pluginManager, userManager, null).start(); BranchTicketService service = new BranchTicketService( runtimeManager,