diff --git a/src/pim/agent/collectionindexer.cpp b/src/pim/agent/collectionindexer.cpp index 91e08c25..93d57cf1 100644 --- a/src/pim/agent/collectionindexer.cpp +++ b/src/pim/agent/collectionindexer.cpp @@ -1,160 +1,157 @@ /* * Copyright 2014 Christian Mollekopf * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as * published by the Free Software Foundation; either version 2 of * the License or (at your option) version 3 or any later version * accepted by the membership of KDE e.V. (or its successor approved * by the membership of KDE e.V.), which shall act as a proxy * defined in Section 14 of version 3 of the license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . * */ +#include + #include "collectionindexer.h" #include #include -#include -#include -#include -#include -#include #include #include #include CollectionIndexer::CollectionIndexer(const QString &path) { Akonadi::AttributeFactory::registerAttribute(); m_db = new Xapian::WritableDatabase(path.toUtf8().constData(), Xapian::DB_CREATE_OR_OPEN); } CollectionIndexer::~CollectionIndexer() { commit(); delete m_db; } static QByteArray getPath(const Akonadi::Collection& collection) { QStringList pathParts; pathParts << collection.displayName(); Akonadi::Collection col = collection; while (col.parentCollection().isValid() && (col.parentCollection() != Akonadi::Collection::root())) { col = col.parentCollection(); pathParts.prepend(col.displayName()); } return "/" + pathParts.join(QLatin1String("/")).toUtf8(); } void CollectionIndexer::index(const Akonadi::Collection& collection) { if (!m_db) { return; } kDebug() << "Indexing " << collection.id() << collection.displayName() << collection.name(); try { Xapian::Document doc; Xapian::TermGenerator gen; gen.set_document(doc); gen.set_database(*m_db); gen.index_text_without_positions(collection.displayName().toUtf8().constData()); gen.index_text_without_positions(collection.displayName().toUtf8().constData(), 1, "N"); //We index with positions so we can do phrase searches (required for exact matches) { const QByteArray path = getPath(collection); gen.index_text(path.constData(), 1, "P"); const QByteArray term = "A" + path; doc.add_term(term.constData()); } Akonadi::Entity::Id colId = collection.parentCollection().id(); const QByteArray term = 'C' + QByteArray::number(colId); doc.add_boolean_term(term.constData()); QByteArray ns; if (CollectionIdentificationAttribute *folderAttribute = collection.attribute()) { if (!folderAttribute->collectionNamespace().isEmpty()) { ns = folderAttribute->collectionNamespace(); } if (!folderAttribute->identifier().isEmpty()) { const QByteArray term = "ID" + folderAttribute->identifier(); doc.add_boolean_term(term.constData()); } } { //We must add the term also with an empty namespace, so we can search for that as well const QByteArray term = "NS" + ns; doc.add_boolean_term(term.constData()); } Q_FOREACH(const QString &mt, collection.contentMimeTypes()) { const QByteArray term = "M" + mt.toUtf8(); doc.add_boolean_term(term.constData()); } m_db->replace_document(collection.id(), doc); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer:" << e.get_msg().c_str(); } } void CollectionIndexer::change(const Akonadi::Collection& col) { index(col); } void CollectionIndexer::remove(const Akonadi::Collection& col) { if (!m_db) { return; } //Remove collection try { m_db->delete_document(col.id()); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer:" << e.get_msg().c_str(); } //Remove subcollections try { Xapian::Query query('C'+ QString::number(col.id()).toStdString()); Xapian::Enquire enquire(*m_db); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, m_db->get_doccount()); Xapian::MSetIterator end = mset.end(); for (Xapian::MSetIterator it = mset.begin(); it != end; ++it) { const qint64 id = *it; remove(Akonadi::Collection(id)); } } catch (const Xapian::DocNotFoundError&) { return; } } void CollectionIndexer::move(const Akonadi::Collection& collection, const Akonadi::Collection& from, const Akonadi::Collection& to) { index(collection); } void CollectionIndexer::commit() { if (m_db) { m_db->commit(); } } diff --git a/src/pim/agent/index.cpp b/src/pim/agent/index.cpp index 7ca30d02..e4bcea5d 100644 --- a/src/pim/agent/index.cpp +++ b/src/pim/agent/index.cpp @@ -1,469 +1,467 @@ /* * Copyright 2014 Christian Mollekopf * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as * published by the Free Software Foundation; either version 2 of * the License or (at your option) version 3 or any later version * accepted by the membership of KDE e.V. (or its successor approved * by the membership of KDE e.V.), which shall act as a proxy * defined in Section 14 of version 3 of the license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . * */ +#include + #include "index.h" #include "emailindexer.h" #include "contactindexer.h" #include "akonotesindexer.h" #include "calendarindexer.h" #include #include #include #include -#include -#include -#include -#include Index::Index(QObject* parent) : QObject(parent), m_collectionIndexer(0) { m_commitTimer.setInterval(1000); m_commitTimer.setSingleShot(true); connect(&m_commitTimer, SIGNAL(timeout()), this, SLOT(commit())); } Index::~Index() { delete m_collectionIndexer; m_collectionIndexer = 0; qDeleteAll(m_indexer.values().toSet()); m_indexer.clear(); } static void removeDir(const QString& dirName) { QDir dir(dirName); if (dir.exists(dirName)) { Q_FOREACH(const QFileInfo &info, dir.entryInfoList(QDir::NoDotAndDotDot | QDir::System | QDir::Hidden | QDir::AllDirs | QDir::Files, QDir::DirsFirst)) { if (info.isDir()) { removeDir(info.absoluteFilePath()); } else { QFile::remove(info.absoluteFilePath()); } } dir.rmdir(dirName); } } void Index::removeDatabase() { delete m_collectionIndexer; m_collectionIndexer = 0; qDeleteAll(m_indexer.values().toSet()); m_indexer.clear(); kDebug() << "Removing database"; removeDir(emailIndexingPath()); removeDir(contactIndexingPath()); removeDir(emailContactsIndexingPath()); removeDir(akonotesIndexingPath()); removeDir(calendarIndexingPath()); removeDir(collectionIndexingPath()); } AbstractIndexer* Index::indexerForItem(const Akonadi::Item& item) const { return m_indexer.value(item.mimeType()); } QList Index::indexersForMimetypes(const QStringList& mimeTypes) const { QList indexers; Q_FOREACH (const QString& mimeType, mimeTypes) { AbstractIndexer *i = m_indexer.value(mimeType); if (i) { indexers.append(i); } } return indexers; } bool Index::haveIndexerForMimeTypes(const QStringList &mimeTypes) { return !indexersForMimetypes(mimeTypes).isEmpty(); } void Index::index(const Akonadi::Item& item) { AbstractIndexer *indexer = indexerForItem(item); if (!indexer) { return; } try { indexer->index(item); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } void Index::move(const Akonadi::Item::List& items, const Akonadi::Collection& from, const Akonadi::Collection& to) { //We always get items of the same type AbstractIndexer *indexer = indexerForItem(items.first()); if (!indexer) { return; } Q_FOREACH (const Akonadi::Item& item, items) { try { indexer->move(item.id(), from.id(), to.id()); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } void Index::updateFlags(const Akonadi::Item::List& items, const QSet& addedFlags, const QSet& removedFlags) { //We always get items of the same type AbstractIndexer *indexer = indexerForItem(items.first()); if (!indexer) { return; } Q_FOREACH (const Akonadi::Item& item, items) { try { indexer->updateFlags(item, addedFlags, removedFlags); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } void Index::remove(const QSet< Akonadi::Entity::Id >& ids, const QStringList& mimeTypes) { const QList indexers = indexersForMimetypes(mimeTypes); Q_FOREACH (const Akonadi::Item::Id& id, ids) { Q_FOREACH (AbstractIndexer *indexer, indexers) { try { indexer->remove(Akonadi::Item(id)); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } } void Index::remove(const Akonadi::Item::List& items) { AbstractIndexer *indexer = indexerForItem(items.first()); if (!indexer) { return; } Q_FOREACH (const Akonadi::Item& item, items) { try { indexer->remove(item); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } void Index::index(const Akonadi::Collection& collection) { if (m_collectionIndexer) { m_collectionIndexer->index(collection); m_collectionIndexer->commit(); } kDebug() << "indexed " << collection.id(); } void Index::change(const Akonadi::Collection& col) { if (m_collectionIndexer) { m_collectionIndexer->change(col); m_collectionIndexer->commit(); } } void Index::remove(const Akonadi::Collection& col) { //Remove items Q_FOREACH (AbstractIndexer *indexer, indexersForMimetypes(col.contentMimeTypes())) { try { indexer->remove(col); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } if (m_collectionIndexer) { m_collectionIndexer->remove(col); m_collectionIndexer->commit(); } } void Index::move(const Akonadi::Collection& collection, const Akonadi::Collection& from, const Akonadi::Collection& to) { if (m_collectionIndexer) { m_collectionIndexer->move(collection, from, to); m_collectionIndexer->commit(); } } void Index::addIndexer(AbstractIndexer* indexer) { Q_FOREACH (const QString& mimeType, indexer->mimeTypes()) { m_indexer.insert(mimeType, indexer); } } bool Index::createIndexers() { bool reindex = false; AbstractIndexer *indexer = 0; for (int i = 0; i < 2; i++){ try { QDir().mkpath(emailIndexingPath()); QDir().mkpath(emailContactsIndexingPath()); indexer = new EmailIndexer(emailIndexingPath(), emailContactsIndexingPath()); addIndexer(indexer); break; } catch (const Xapian::DatabaseError &e) { QFile::remove(emailIndexingPath() + "flintlock"); QFile::remove(emailContactsIndexingPath() + "flintlock"); reindex = true; delete indexer; kError() << "Failed to create email indexer:" << QString::fromStdString(e.get_msg()); } catch (...) { QFile::remove(emailIndexingPath() + "flintlock"); QFile::remove(emailContactsIndexingPath() + "flintlock"); reindex = true; delete indexer; kError() << "Random exception, but we do not want to crash"; } } for (int i = 0; i < 2; i++){ try { QDir().mkpath(contactIndexingPath()); indexer = new ContactIndexer(contactIndexingPath()); addIndexer(indexer); break; } catch (const Xapian::DatabaseError &e) { QFile::remove(contactIndexingPath() + "flintlock"); reindex = true; delete indexer; kError() << "Failed to create contact indexer:" << QString::fromStdString(e.get_msg()); } catch (...) { QFile::remove(contactIndexingPath() + "flintlock"); reindex = true; delete indexer; kError() << "Random exception, but we do not want to crash"; } } for (int i = 0; i < 2; i++){ try { QDir().mkpath(akonotesIndexingPath()); indexer = new AkonotesIndexer(akonotesIndexingPath()); addIndexer(indexer); break; } catch (const Xapian::DatabaseError &e) { QFile::remove(akonotesIndexingPath() + "flintlock"); reindex = true; delete indexer; kError() << "Failed to create akonotes indexer:" << QString::fromStdString(e.get_msg()); } catch (...) { QFile::remove(akonotesIndexingPath() + "flintlock"); reindex = true; delete indexer; kError() << "Random exception, but we do not want to crash"; } } for (int i = 0; i < 2; i++){ try { QDir().mkpath(calendarIndexingPath()); indexer = new CalendarIndexer(calendarIndexingPath()); addIndexer(indexer); break; } catch (const Xapian::DatabaseError &e) { QFile::remove(calendarIndexingPath() + "flintlock"); reindex = true; delete indexer; kError() << "Failed to create collection indexer:" << QString::fromStdString(e.get_msg()); } catch (...) { QFile::remove(calendarIndexingPath() + "flintlock"); reindex = true; delete indexer; kError() << "Random exception, but we do not want to crash"; } } for (int i = 0; i < 2; i++){ try { QDir().mkpath(collectionIndexingPath()); m_collectionIndexer = new CollectionIndexer(collectionIndexingPath()); break; } catch (const Xapian::DatabaseError &e) { QFile::remove(collectionIndexingPath() + "flintlock"); reindex = true; delete m_collectionIndexer; m_collectionIndexer = 0; kError() << "Failed to create akonotes indexer:" << QString::fromStdString(e.get_msg()); } catch (...) { QFile::remove(collectionIndexingPath() + "flintlock"); reindex = true; delete m_collectionIndexer; m_collectionIndexer = 0; kError() << "Random exception, but we do not want to crash"; } } if (reindex) { Q_EMIT reindexAll(); } return !m_indexer.isEmpty(); } void Index::scheduleCommit() { if (!m_commitTimer.isActive()) { m_commitTimer.start(); } } void Index::commit() { m_commitTimer.stop(); Q_FOREACH (AbstractIndexer *indexer, m_indexer) { try { indexer->commit(); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } void Index::findIndexedInDatabase(QSet &indexed, Akonadi::Entity::Id collectionId, const QString& dbPath) { Xapian::Database db; try { db = Xapian::Database(QFile::encodeName(dbPath).constData()); } catch (const Xapian::DatabaseError& e) { kError() << "Failed to open database" << dbPath << ":" << QString::fromStdString(e.get_msg()); return; } const std::string term = QString::fromLatin1("C%1").arg(collectionId).toStdString(); Xapian::Query query(term); Xapian::Enquire enquire(db); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, UINT_MAX); Xapian::MSetIterator it = mset.begin(); for (;it != mset.end(); it++) { indexed << *it; } } void Index::findIndexed(QSet& indexed, Akonadi::Entity::Id collectionId) { findIndexedInDatabase(indexed, collectionId, emailIndexingPath()); findIndexedInDatabase(indexed, collectionId, contactIndexingPath()); findIndexedInDatabase(indexed, collectionId, akonotesIndexingPath()); findIndexedInDatabase(indexed, collectionId, calendarIndexingPath()); } qlonglong Index::indexedItems(const qlonglong id) { const std::string term = QString::fromLatin1("C%1").arg(id).toStdString(); return indexedItemsInDatabase(term, emailIndexingPath()) + indexedItemsInDatabase(term, contactIndexingPath()) + indexedItemsInDatabase(term, akonotesIndexingPath()) + indexedItemsInDatabase(term, calendarIndexingPath()); } qlonglong Index::indexedItemsInDatabase(const std::string& term, const QString& dbPath) const { Xapian::Database db; try { db = Xapian::Database(QFile::encodeName(dbPath).constData()); } catch (const Xapian::DatabaseError& e) { kError() << "Failed to open database" << dbPath << ":" << QString::fromStdString(e.get_msg()); return 0; } return db.get_termfreq(term); } void Index::setOverrideDbPrefixPath(const QString& path) { m_overridePrefixPath = path; } QString Index::dbPath(const QString& dbName) const { if (!m_overridePrefixPath.isEmpty()) { return QString::fromLatin1("%1/%2/").arg(m_overridePrefixPath, dbName); } QString basePath = QLatin1String("baloo"); if (Akonadi::ServerManager::hasInstanceIdentifier()) { basePath = QString::fromLatin1("baloo/instances/%1").arg(Akonadi::ServerManager::instanceIdentifier()); } return KGlobal::dirs()->localxdgdatadir() + QString::fromLatin1("%1/%2/").arg(basePath, dbName); } QString Index::emailIndexingPath() const { return dbPath(QLatin1String("email")); } QString Index::contactIndexingPath() const { return dbPath(QLatin1String("contacts")); } QString Index::emailContactsIndexingPath() const { return dbPath(QLatin1String("emailContacts")); } QString Index::akonotesIndexingPath() const { return dbPath(QLatin1String("notes")); } QString Index::calendarIndexingPath() const { return dbPath(QLatin1String("calendars")); } QString Index::collectionIndexingPath() const { return dbPath(QLatin1String("collections")); }