diff --git a/src/pim/agent/agent.cpp b/src/pim/agent/agent.cpp index 3ab76d9c..47ddccdf 100644 --- a/src/pim/agent/agent.cpp +++ b/src/pim/agent/agent.cpp @@ -1,258 +1,264 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013 Vishesh Handa * Copyright (C) 2014 Christian Mollekopf * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "agent.h" #include "contactindexer.h" #include "emailindexer.h" #include "akonotesindexer.h" #include "calendarindexer.h" #include "balooindexeradaptor.h" #include "collectionupdatejob.h" #include "src/file/priority.h" #include #include #include #include #include #include #include #include #include #include #include #define INDEXING_AGENT_VERSION 4 BalooIndexingAgent::BalooIndexingAgent(const QString& id) : AgentBase(id), m_scheduler(m_index, QSharedPointer(new JobFactory)) { lowerIOPriority(); lowerSchedulingPriority(); lowerPriority(); KConfig config("baloorc"); KConfigGroup group = config.group("Akonadi"); const int agentIndexingVersion = group.readEntry("agentIndexingVersion", 0); if (agentIndexingVersionsetAllMonitored(true); changeRecorder()->itemFetchScope().setCacheOnly(true); changeRecorder()->itemFetchScope().setAncestorRetrieval(Akonadi::ItemFetchScope::Parent); changeRecorder()->itemFetchScope().setFetchRemoteIdentification(false); changeRecorder()->itemFetchScope().setFetchModificationTime(false); changeRecorder()->collectionFetchScope().setAncestorRetrieval(Akonadi::CollectionFetchScope::All); changeRecorder()->collectionFetchScope().ancestorFetchScope().fetchAttribute(); changeRecorder()->collectionFetchScope().setListFilter(Akonadi::CollectionFetchScope::Index); changeRecorder()->setChangeRecordingEnabled(false); changeRecorder()->fetchCollection(true); changeRecorder()->setExclusive(true); new BalooIndexerAdaptor(this); // Cleanup agentsrc after migration to 4.13 Akonadi::AgentManager* agentManager = Akonadi::AgentManager::self(); const Akonadi::AgentInstance::List allAgents = agentManager->instances(); const QStringList oldFeeders = QStringList() << "akonadi_nepomuk_feeder"; // Cannot use agentManager->instance(oldInstanceName) here, it wouldn't find broken instances. Q_FOREACH( const Akonadi::AgentInstance& inst, allAgents ) { if ( oldFeeders.contains( inst.identifier() ) ) { kDebug() << "Removing old nepomuk feeder" << inst.identifier(); agentManager->removeInstance( inst ); } } } BalooIndexingAgent::~BalooIndexingAgent() { } void BalooIndexingAgent::reindexAll() { kDebug() << "Reindexing everything"; m_scheduler.abort(); m_index.removeDatabase(); m_index.createIndexers(); QTimer::singleShot(0, &m_scheduler, SLOT(scheduleCompleteSync())); } +void BalooIndexingAgent::completeSync() +{ + QTimer::singleShot(0, &m_scheduler, SLOT(scheduleCompleteSync())); +} + void BalooIndexingAgent::reindexCollection(const qlonglong id) { kDebug() << "Reindexing collection " << id; m_scheduler.scheduleCollection(Akonadi::Collection(id), true); } qlonglong BalooIndexingAgent::indexedItems(const qlonglong id) { return m_index.indexedItems(id); } void BalooIndexingAgent::itemAdded(const Akonadi::Item& item, const Akonadi::Collection& collection) { Q_UNUSED(collection); m_scheduler.addItem(item); } void BalooIndexingAgent::itemChanged(const Akonadi::Item& item, const QSet& partIdentifiers) { // We don't index certain parts so we don't care when they change QSet pi = partIdentifiers; QMutableSetIterator it(pi); while (it.hasNext()) { it.next(); if (!it.value().startsWith("PLD:")) it.remove(); } if (pi.isEmpty()) { return; } m_scheduler.addItem(item); } void BalooIndexingAgent::itemsFlagsChanged(const Akonadi::Item::List& items, const QSet& addedFlags, const QSet& removedFlags) { // Akonadi always sends batch of items of the same type m_index.updateFlags(items, addedFlags, removedFlags); m_index.scheduleCommit(); } void BalooIndexingAgent::itemsRemoved(const Akonadi::Item::List& items) { m_index.remove(items); m_index.scheduleCommit(); } void BalooIndexingAgent::itemsMoved(const Akonadi::Item::List& items, const Akonadi::Collection& sourceCollection, const Akonadi::Collection& destinationCollection) { m_index.move(items, sourceCollection, destinationCollection); m_index.scheduleCommit(); } void BalooIndexingAgent::collectionAdded(const Akonadi::Collection& collection, const Akonadi::Collection& parent) { m_index.index(collection); m_index.scheduleCommit(); } void BalooIndexingAgent::collectionChanged(const Akonadi::Collection& collection, const QSet& changedAttributes) { QSet changes = changedAttributes; changes.remove("collectionquota"); changes.remove("timestamp"); changes.remove("imapquota"); if (changes.isEmpty()) { return; } if (changes.contains("ENTITYDISPLAY")) { //If the name changed we have to reindex all subcollections CollectionUpdateJob *job = new CollectionUpdateJob(m_index, collection, this); job->start(); } else { m_index.index(collection); m_index.scheduleCommit(); } } void BalooIndexingAgent::collectionRemoved(const Akonadi::Collection& collection) { m_index.remove(collection); m_index.scheduleCommit(); } void BalooIndexingAgent::collectionMoved(const Akonadi::Collection &collection, const Akonadi::Collection &collectionSource, const Akonadi::Collection &collectionDestination) { m_index.remove(collection); CollectionUpdateJob *job = new CollectionUpdateJob(m_index, collection, this); job->start(); } void BalooIndexingAgent::cleanup() { // Remove all the databases Akonadi::AgentBase::cleanup(); } void BalooIndexingAgent::onAbortRequested() { KConfig config(QLatin1String("baloorc")); KConfigGroup group = config.group("Akonadi"); group.writeEntry("aborted", true); group.sync(); m_scheduler.abort(); } void BalooIndexingAgent::onOnlineChanged(bool online) { // Ignore everything when offline changeRecorder()->setAllMonitored(online); // Index items that might have changed while we were offline if (online) { //We only reindex if this is not a regular start KConfig config(QLatin1String("baloorc")); KConfigGroup group = config.group("Akonadi"); const bool aborted = group.readEntry("aborted", false); if (aborted) { group.writeEntry("aborted", false); group.sync(); m_scheduler.scheduleCompleteSync(); } } else { // Abort ongoing indexing when switched to offline onAbortRequested(); } } AKONADI_AGENT_MAIN(BalooIndexingAgent) diff --git a/src/pim/agent/agent.h b/src/pim/agent/agent.h index 6274e7cd..f44d85bd 100644 --- a/src/pim/agent/agent.h +++ b/src/pim/agent/agent.h @@ -1,83 +1,84 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2012 Vishesh Handa * Copyright (C) 2014 Christian Mollekopf * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #ifndef AGENT_H #define AGENT_H #include #include #include #include #include #include "scheduler.h" #include "index.h" class KJob; class AbstractIndexer; namespace Akonadi { class ItemFetchJob; } class BalooIndexingAgent : public Akonadi::AgentBase, public Akonadi::AgentBase::ObserverV3 { Q_OBJECT public: BalooIndexingAgent(const QString& id); ~BalooIndexingAgent(); void reindexAll(); void reindexCollection(const qlonglong id); qlonglong indexedItems(const qlonglong id); virtual void itemAdded(const Akonadi::Item& item, const Akonadi::Collection& collection); virtual void itemChanged(const Akonadi::Item& item, const QSet& partIdentifiers); virtual void itemsFlagsChanged(const Akonadi::Item::List& items, const QSet& addedFlags, const QSet& removedFlags); virtual void itemsRemoved(const Akonadi::Item::List& items); virtual void itemsMoved(const Akonadi::Item::List& items, const Akonadi::Collection& sourceCollection, const Akonadi::Collection& destinationCollection); virtual void collectionAdded(const Akonadi::Collection& collection, const Akonadi::Collection& parent); virtual void collectionChanged(const Akonadi::Collection& collection, const QSet& changedAttributes); virtual void collectionRemoved(const Akonadi::Collection& collection); virtual void collectionMoved(const Akonadi::Collection &collection, const Akonadi::Collection &collectionSource, const Akonadi::Collection &collectionDestination); // Remove the entire db virtual void cleanup(); private Q_SLOTS: void onAbortRequested(); void onOnlineChanged(bool online); + void completeSync(); private: Index m_index; Scheduler m_scheduler; }; #endif // AGENT_H diff --git a/src/pim/agent/akonotesindexer.cpp b/src/pim/agent/akonotesindexer.cpp index 293e4c8e..1630c4f7 100644 --- a/src/pim/agent/akonotesindexer.cpp +++ b/src/pim/agent/akonotesindexer.cpp @@ -1,201 +1,190 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2014 Laurent Montel * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "akonotesindexer.h" #include AkonotesIndexer::AkonotesIndexer(const QString& path) : AbstractIndexer(), m_db(0), m_termGen( 0 ) { - try { - m_db = new Xapian::WritableDatabase(path.toUtf8().constData(), Xapian::DB_CREATE_OR_OPEN); - } - catch (const Xapian::DatabaseCorruptError& err) { - kError() << "Database Corrupted - What did you do?"; - kError() << err.get_error_string(); - m_db = 0; - } - catch (const Xapian::Error &e) { - kError() << QString::fromStdString(e.get_type()) << QString::fromStdString(e.get_description()); - m_db = 0; - } + m_db = new Xapian::WritableDatabase(path.toUtf8().constData(), Xapian::DB_CREATE_OR_OPEN); } AkonotesIndexer::~AkonotesIndexer() { if (m_db) { m_db->commit(); delete m_db; } } QStringList AkonotesIndexer::mimeTypes() const { return QStringList() << QString::fromLatin1( "text/x-vnd.akonadi.note" ); } void AkonotesIndexer::index(const Akonadi::Item &item) { if (!m_db) return; KMime::Message::Ptr msg; try { msg = item.payload(); } catch (const Akonadi::PayloadException&) { return; } m_doc = new Xapian::Document(); m_termGen = new Xapian::TermGenerator(); m_termGen->set_document(*m_doc); m_termGen->set_database(*m_db); process(msg); Akonadi::Entity::Id colId = item.parentCollection().id(); QByteArray term = 'C' + QByteArray::number(colId); m_doc->add_boolean_term(term.data()); m_db->replace_document(item.id(), *m_doc); delete m_doc; delete m_termGen; m_doc = 0; m_termGen = 0; } void AkonotesIndexer::process(const KMime::Message::Ptr &msg) { // // Process Headers // (Give the subject a higher priority) KMime::Headers::Subject* subject = msg->subject(false); if (subject) { std::string str(subject->asUnicodeString().toUtf8().constData()); kDebug() << "Indexing" << str.c_str(); m_termGen->index_text_without_positions(str, 1, "SU"); m_termGen->index_text_without_positions(str, 100); m_doc->set_data(str); } KMime::Content* mainBody = msg->mainBodyPart("text/plain"); if (mainBody) { const std::string text(mainBody->decodedText().toUtf8().constData()); m_termGen->index_text_without_positions(text); m_termGen->index_text_without_positions(text, 1, "BO"); } else { processPart(msg.get(), 0); } } void AkonotesIndexer::processPart(KMime::Content* content, KMime::Content* mainContent) { if (content == mainContent) { return; } KMime::Headers::ContentType* type = content->contentType(false); if (type) { if (type->isMultipart()) { if (type->isSubtype("encrypted")) return; Q_FOREACH (KMime::Content* c, content->contents()) { processPart(c, mainContent); } } // Only get HTML content, if no plain text content if (!mainContent && type->isHTMLText()) { QTextDocument doc; doc.setHtml(content->decodedText()); const std::string text(doc.toPlainText().toUtf8().constData()); m_termGen->index_text_without_positions(text); } } } void AkonotesIndexer::commit() { if (m_db) m_db->commit(); } void AkonotesIndexer::remove(const Akonadi::Item &item) { if (!m_db) return; try { m_db->delete_document(item.id()); } catch (const Xapian::DocNotFoundError&) { return; } } void AkonotesIndexer::remove(const Akonadi::Collection& collection) { if (!m_db) return; try { Xapian::Query query('C'+ QString::number(collection.id()).toStdString()); Xapian::Enquire enquire(*m_db); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, m_db->get_doccount()); Xapian::MSetIterator end(mset.end()); for (Xapian::MSetIterator it = mset.begin(); it != end; ++it) { const qint64 id = *it; remove(Akonadi::Item(id)); } } catch (const Xapian::DocNotFoundError&) { return; } } void AkonotesIndexer::move(const Akonadi::Item::Id& itemId, const Akonadi::Entity::Id& from, const Akonadi::Entity::Id& to) { if (!m_db) return; Xapian::Document doc; try { doc = m_db->get_document(itemId); } catch (const Xapian::DocNotFoundError&) { return; } const QByteArray ft = 'C' + QByteArray::number(from); const QByteArray tt = 'C' + QByteArray::number(to); doc.remove_term(ft.data()); doc.add_boolean_term(tt.data()); m_db->replace_document(doc.get_docid(), doc); } diff --git a/src/pim/agent/calendarindexer.cpp b/src/pim/agent/calendarindexer.cpp index fea0918c..bb567458 100644 --- a/src/pim/agent/calendarindexer.cpp +++ b/src/pim/agent/calendarindexer.cpp @@ -1,173 +1,162 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2014 Laurent Montel * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "calendarindexer.h" #include "xapiandocument.h" #include #include #include CalendarIndexer::CalendarIndexer(const QString& path) : AbstractIndexer(), m_db( 0 ), m_termGen( 0 ) { - try { - m_db = new Baloo::XapianDatabase(path, true); - } - catch (const Xapian::DatabaseCorruptError& err) { - kError() << "Database Corrupted - What did you do?"; - kError() << err.get_error_string(); - m_db = 0; - } - catch (const Xapian::Error &e) { - kError() << QString::fromStdString(e.get_type()) << QString::fromStdString(e.get_description()); - m_db = 0; - } + m_db = new Baloo::XapianDatabase(path, true); } CalendarIndexer::~CalendarIndexer() { if (m_db) { m_db->commit(); delete m_db; } } QStringList CalendarIndexer::mimeTypes() const { return QStringList() << QString::fromLatin1("application/x-vnd.akonadi.calendar.event") << QString::fromLatin1("application/x-vnd.akonadi.calendar.todo") << QString::fromLatin1("application/x-vnd.akonadi.calendar.journal") << QString::fromLatin1("application/x-vnd.akonadi.calendar.freebusy"); } void CalendarIndexer::index(const Akonadi::Item &item) { if ( item.hasPayload() ) { indexEventItem( item, item.payload() ); } else if ( item.hasPayload() ) { indexJournalItem( item, item.payload() ); } else if ( item.hasPayload() ) { indexTodoItem( item, item.payload() ); } else { return; } } void CalendarIndexer::commit() { if (m_db) m_db->commit(); } void CalendarIndexer::remove(const Akonadi::Item &item) { if (!m_db) return; try { m_db->deleteDocument(item.id()); } catch (const Xapian::DocNotFoundError&) { return; } } void CalendarIndexer::remove(const Akonadi::Collection& collection) { if (!m_db) return; try { Xapian::Query query('C'+ QString::number(collection.id()).toStdString()); Xapian::Enquire enquire(*(m_db->db())); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, m_db->db()->get_doccount()); Xapian::MSetIterator end(mset.end()); for (Xapian::MSetIterator it = mset.begin(); it != end; ++it) { const qint64 id = *it; remove(Akonadi::Item(id)); } } catch (const Xapian::DocNotFoundError&) { return; } } void CalendarIndexer::move(const Akonadi::Item::Id& itemId, const Akonadi::Entity::Id& from, const Akonadi::Entity::Id& to) { if (!m_db) return; Xapian::Document doc; try { doc = m_db->db()->get_document(itemId); } catch (const Xapian::DocNotFoundError&) { return; } const QByteArray ft = 'C' + QByteArray::number(from); const QByteArray tt = 'C' + QByteArray::number(to); doc.remove_term(ft.data()); doc.add_boolean_term(tt.data()); m_db->replaceDocument(doc.get_docid(), doc); } void CalendarIndexer::indexEventItem(const Akonadi::Item &item, const KCalCore::Event::Ptr &event) { Baloo::XapianDocument doc; doc.indexText(event->organizer()->email(), "O"); doc.indexText(event->summary(), "S"); doc.indexText(event->location(), "L"); KCalCore::Attendee::List attendees = event->attendees(); KCalCore::Attendee::List::ConstIterator it; for ( it = attendees.constBegin(); it != attendees.constEnd(); ++it ) { doc.addBoolTerm((*it)->email()+QString::number((*it)->status()), "PS"); } // Parent collection Q_ASSERT_X(item.parentCollection().isValid(), "Baloo::CalenderIndexer::index", "Item does not have a valid parent collection"); const Akonadi::Entity::Id colId = item.parentCollection().id(); doc.addBoolTerm(colId, "C"); m_db->replaceDocument(item.id(), doc); } void CalendarIndexer::indexJournalItem(const Akonadi::Item &item, const KCalCore::Journal::Ptr &journal) { //TODO } void CalendarIndexer::indexTodoItem(const Akonadi::Item &item, const KCalCore::Todo::Ptr &todo) { //TODO } void CalendarIndexer::updateIncidenceItem( const KCalCore::Incidence::Ptr &calInc ) { //TODO } diff --git a/src/pim/agent/contactindexer.cpp b/src/pim/agent/contactindexer.cpp index b6a9e205..95f6ab2e 100644 --- a/src/pim/agent/contactindexer.cpp +++ b/src/pim/agent/contactindexer.cpp @@ -1,205 +1,194 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "contactindexer.h" #include "xapiandocument.h" #include #include #include ContactIndexer::ContactIndexer(const QString& path): AbstractIndexer(), m_db( 0 ) { - try { - m_db = new Baloo::XapianDatabase(path, true); - } - catch (const Xapian::DatabaseCorruptError& err) { - kError() << "Database Corrupted - What did you do?"; - kError() << err.get_error_string(); - m_db = 0; - } - catch (const Xapian::Error &e) { - kError() << QString::fromStdString(e.get_type()) << QString::fromStdString(e.get_description()); - m_db = 0; - } + m_db = new Baloo::XapianDatabase(path, true); } ContactIndexer::~ContactIndexer() { if (m_db) { m_db->commit(); delete m_db; } } QStringList ContactIndexer::mimeTypes() const { return QStringList() << KABC::Addressee::mimeType() << KABC::ContactGroup::mimeType(); } bool ContactIndexer::indexContact(const Akonadi::Item& item) { if (!m_db) return false; KABC::Addressee addresse; try { addresse = item.payload(); } catch (const Akonadi::PayloadException&) { return false; } Baloo::XapianDocument doc; QString name; if (!addresse.formattedName().isEmpty()) { name = addresse.formattedName(); } else if (!addresse.assembledName().isEmpty()) { name = addresse.assembledName(); } else { name = addresse.name(); } kDebug() << "Indexing" << name << addresse.nickName(); doc.indexText(name); doc.indexText(addresse.nickName()); doc.indexText(addresse.uid()); doc.indexText(name, "NA"); doc.indexText(addresse.nickName(), "NI"); Q_FOREACH (const QString& email, addresse.emails()) { doc.addTerm(email); doc.indexText(email); } // Parent collection Q_ASSERT_X(item.parentCollection().isValid(), "Baloo::ContactIndexer::index", "Item does not have a valid parent collection"); const Akonadi::Entity::Id colId = item.parentCollection().id(); doc.addBoolTerm(colId, "C"); if (addresse.birthday().isValid()) { const QString julianDay = QString::number(addresse.birthday().date().toJulianDay()); doc.addValue(0, julianDay); } //TODO index anniversary ? m_db->replaceDocument(item.id(), doc); return true; } void ContactIndexer::indexContactGroup(const Akonadi::Item& item) { if (!m_db) return; KABC::ContactGroup group; try { group = item.payload(); } catch (const Akonadi::PayloadException&) { return; } Baloo::XapianDocument doc; const QString name = group.name(); doc.indexText(name); doc.indexText(name, "NA"); // Parent collection Q_ASSERT_X(item.parentCollection().isValid(), "Baloo::ContactIndexer::index", "Item does not have a valid parent collection"); const Akonadi::Entity::Id colId = item.parentCollection().id(); doc.addBoolTerm(colId, "C"); m_db->replaceDocument(item.id(), doc); } void ContactIndexer::index(const Akonadi::Item& item) { if (!indexContact(item)) { indexContactGroup(item); } } void ContactIndexer::remove(const Akonadi::Item& item) { if (m_db) m_db->deleteDocument(item.id()); } void ContactIndexer::remove(const Akonadi::Collection& collection) { if (!m_db) return; try { Xapian::Database* db = m_db->db(); Xapian::Query query('C'+ QString::number(collection.id()).toStdString()); Xapian::Enquire enquire(*db); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, db->get_doccount()); Xapian::MSetIterator end(mset.end()); for (Xapian::MSetIterator it = mset.begin(); it != end; ++it) { const qint64 id = *it; remove(Akonadi::Item(id)); } } catch (const Xapian::DocNotFoundError&) { return; } } void ContactIndexer::commit() { if (m_db) m_db->commit(); } void ContactIndexer::move(const Akonadi::Item::Id& itemId, const Akonadi::Entity::Id& from, const Akonadi::Entity::Id& to) { if (!m_db) return; Baloo::XapianDocument doc; try { doc = m_db->document(itemId); } catch (const Xapian::DocNotFoundError&) { return; } const QByteArray ft = 'C' + QByteArray::number(from); const QByteArray tt = 'C' + QByteArray::number(to); doc.removeTermStartsWith(ft.data()); doc.addBoolTerm(tt.data()); m_db->replaceDocument(doc.doc().get_docid(), doc); } diff --git a/src/pim/agent/emailindexer.cpp b/src/pim/agent/emailindexer.cpp index ea4ce356..673d59dc 100644 --- a/src/pim/agent/emailindexer.cpp +++ b/src/pim/agent/emailindexer.cpp @@ -1,427 +1,404 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "emailindexer.h" #include #include #include #include EmailIndexer::EmailIndexer(const QString& path, const QString& contactDbPath): AbstractIndexer(), m_doc( 0 ), m_termGen( 0 ), m_contactDb( 0 ) { - try { m_db = new Xapian::WritableDatabase(path.toUtf8().constData(), Xapian::DB_CREATE_OR_OPEN); - } - catch (const Xapian::DatabaseCorruptError& err) { - kError() << "Database Corrupted - What did you do?"; - kError() << err.get_error_string(); - m_db = 0; - } - catch (const Xapian::Error &e) { - kError() << QString::fromStdString(e.get_type()) << QString::fromStdString(e.get_description()); - m_db = 0; - } - - try { m_contactDb = new Xapian::WritableDatabase(contactDbPath.toUtf8().constData(), Xapian::DB_CREATE_OR_OPEN); - } - catch (const Xapian::DatabaseCorruptError& err) { - kError() << "Database Corrupted - What did you do?"; - kError() << err.get_error_string(); - m_contactDb = 0; - } - catch (const Xapian::Error &e) { - kError() << QString::fromStdString(e.get_type()) << QString::fromStdString(e.get_description()); - m_contactDb = 0; - } } EmailIndexer::~EmailIndexer() { if (m_db) { m_db->commit(); delete m_db; } if (m_contactDb) { m_contactDb->commit(); delete m_contactDb; } } QStringList EmailIndexer::mimeTypes() const { return QStringList() << KMime::Message::mimeType(); } void EmailIndexer::index(const Akonadi::Item& item) { if (!m_db) return; Akonadi::MessageStatus status; status.setStatusFromFlags(item.flags()); if (status.isSpam()) return; KMime::Message::Ptr msg; try { msg = item.payload(); } catch (const Akonadi::PayloadException&) { return; } m_doc = new Xapian::Document(); m_termGen = new Xapian::TermGenerator(); m_termGen->set_document(*m_doc); m_termGen->set_database(*m_db); processMessageStatus(status); process(msg); // Size m_doc->add_value(1, QString::number(item.size()).toStdString()); // Parent collection Q_ASSERT_X(item.parentCollection().isValid(), "Baloo::EmailIndexer::index", "Item does not have a valid parent collection"); Akonadi::Entity::Id colId = item.parentCollection().id(); QByteArray term = 'C' + QByteArray::number(colId); m_doc->add_boolean_term(term.data()); m_db->replace_document(item.id(), *m_doc); delete m_doc; delete m_termGen; m_doc = 0; m_termGen = 0; } void EmailIndexer::insert(const QByteArray& key, KMime::Headers::Base* unstructured) { if (unstructured) { m_termGen->index_text_without_positions(unstructured->asUnicodeString().toUtf8().constData(), 1, key.data()); } } void EmailIndexer::insert(const QByteArray& key, KMime::Headers::Generics::MailboxList* mlist) { if (mlist) insert(key, mlist->mailboxes()); } void EmailIndexer::insert(const QByteArray& key, KMime::Headers::Generics::AddressList* alist) { if (alist) insert(key, alist->mailboxes()); } namespace { // Does some extra stuff such as lower casing the email, removing all quotes // and removing extra spaces // TODO: Move this into KMime? // TODO: If name is all upper/lower then try to captialize it? QString prettyAddress(const KMime::Types::Mailbox& mbox) { const QString name = mbox.name().simplified(); const QByteArray email = mbox.address().simplified().toLower(); return KPIMUtils::normalizedAddress(name, QString::fromUtf8(email)); } } // Add once with a prefix and once without void EmailIndexer::insert(const QByteArray& key, const KMime::Types::Mailbox::List& list) { if (!m_contactDb) return; Q_FOREACH (const KMime::Types::Mailbox& mbox, list) { std::string name(mbox.name().toUtf8().constData()); m_termGen->index_text_without_positions(name, 1, key.data()); m_termGen->index_text_without_positions(name, 1); m_termGen->index_text_without_positions(mbox.address().data(), 1, key.data()); m_termGen->index_text_without_positions(mbox.address().data(), 1); m_doc->add_term((key + mbox.address()).data()); m_doc->add_term(mbox.address().data()); // // Add emails for email auto-completion // const QString pa = prettyAddress(mbox); int id = qHash(pa); try { Xapian::Document doc = m_contactDb->get_document(id); continue; } catch (const Xapian::DocNotFoundError&) { Xapian::Document doc; std::string pretty(pa.toUtf8().constData()); doc.set_data(pretty); Xapian::TermGenerator termGen; termGen.set_document(doc); termGen.index_text(pretty); doc.add_term(mbox.address().data()); m_contactDb->replace_document(id, doc); } } } // FIXME: Only index properties that are actually searched! void EmailIndexer::process(const KMime::Message::Ptr& msg) { // // Process Headers // (Give the subject a higher priority) KMime::Headers::Subject* subject = msg->subject(false); if (subject) { std::string str(subject->asUnicodeString().toUtf8().constData()); kDebug() << "Indexing" << str.c_str(); m_termGen->index_text_without_positions(str, 1, "SU"); m_termGen->index_text_without_positions(str, 100); m_doc->set_data(str); } KMime::Headers::Date* date = msg->date(false); if (date) { const QString str = QString::number(date->dateTime().toTime_t()); m_doc->add_value(0, str.toStdString()); const QString julianDay = QString::number(date->dateTime().date().toJulianDay()); m_doc->add_value(2, julianDay.toStdString()); } insert("F", msg->from(false)); insert("T", msg->to(false)); insert("CC", msg->cc(false)); insert("BC", msg->bcc(false)); insert("O", msg->organization(false)); insert("RT", msg->replyTo(false)); insert("RF", msg->headerByType("Resent-From")); insert("LI", msg->headerByType("List-Id")); insert("XL", msg->headerByType("X-Loop")); insert("XML", msg->headerByType("X-Mailing-List")); insert("XSF", msg->headerByType("X-Spam-Flag")); // // Process Plain Text Content // //Index all headers m_termGen->index_text_without_positions(std::string(msg->head().constData()), 1, "HE"); KMime::Content* mainBody = msg->mainBodyPart("text/plain"); if (mainBody) { const std::string text(mainBody->decodedText().toUtf8().constData()); m_termGen->index_text_without_positions(text); m_termGen->index_text_without_positions(text, 1, "BO"); } else { processPart(msg.get(), 0); } } void EmailIndexer::processPart(KMime::Content* content, KMime::Content* mainContent) { if (content == mainContent) { return; } KMime::Headers::ContentType* type = content->contentType(false); if (type) { if (type->isMultipart()) { if (type->isSubtype("encrypted")) return; Q_FOREACH (KMime::Content* c, content->contents()) { processPart(c, mainContent); } } // Only get HTML content, if no plain text content if (!mainContent && type->isHTMLText()) { QTextDocument doc; doc.setHtml(content->decodedText()); const std::string text(doc.toPlainText().toUtf8().constData()); m_termGen->index_text_without_positions(text); } } // FIXME: Handle attachments? } void EmailIndexer::processMessageStatus(const Akonadi::MessageStatus& status) { insertBool('R', status.isRead()); insertBool('A', status.hasAttachment()); insertBool('I', status.isImportant()); insertBool('W', status.isWatched()); insertBool('T', status.isToAct()); insertBool('D', status.isDeleted()); insertBool('S', status.isSpam()); insertBool('E', status.isReplied()); insertBool('G', status.isIgnored()); insertBool('F', status.isForwarded()); insertBool('N', status.isSent()); insertBool('Q', status.isQueued()); insertBool('H', status.isHam()); insertBool('C', status.isEncrypted()); insertBool('V', status.hasInvitation()); } void EmailIndexer::insertBool(char key, bool value) { QByteArray term("B"); if (value) { term.append(key); } else { term.append('N'); term.append(key); } m_doc->add_boolean_term(term.data()); } void EmailIndexer::toggleFlag(Xapian::Document& doc, const char* remove, const char* add) { try { doc.remove_term(remove); } catch (const Xapian::InvalidArgumentError &e) { // The previous flag state was not indexed, continue } doc.add_term(add); } void EmailIndexer::updateFlags(const Akonadi::Item& item, const QSet& added, const QSet& removed) { if (!m_db) return; Xapian::Document doc; try { doc = m_db->get_document(item.id()); } catch (const Xapian::DocNotFoundError&) { return; } Q_FOREACH (const QByteArray& flag, removed) { if (flag == Akonadi::MessageFlags::Seen) { toggleFlag(doc, "BR", "BNR"); } else if (flag == Akonadi::MessageFlags::Flagged) { toggleFlag(doc, "BI", "BNI"); } else if (flag == Akonadi::MessageFlags::Watched) { toggleFlag(doc, "BW", "BNW"); } } Q_FOREACH (const QByteArray& flag, added) { if (flag == Akonadi::MessageFlags::Seen) { toggleFlag(doc, "BNR", "BR"); } else if (flag == Akonadi::MessageFlags::Flagged) { toggleFlag(doc, "BNI", "BI"); } else if (flag == Akonadi::MessageFlags::Watched) { toggleFlag(doc, "BNW", "BW"); } } m_db->replace_document(doc.get_docid(), doc); } void EmailIndexer::remove(const Akonadi::Item& item) { if (!m_db) return; try { m_db->delete_document(item.id()); //TODO remove contacts from contact db? } catch (const Xapian::DocNotFoundError&) { return; } } void EmailIndexer::remove(const Akonadi::Collection& collection) { if (!m_db) return; try { Xapian::Query query('C'+ QString::number(collection.id()).toStdString()); Xapian::Enquire enquire(*m_db); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, m_db->get_doccount()); Xapian::MSetIterator end = mset.end(); for (Xapian::MSetIterator it = mset.begin(); it != end; ++it) { const qint64 id = *it; remove(Akonadi::Item(id)); } } catch (const Xapian::DocNotFoundError&) { return; } } void EmailIndexer::move(const Akonadi::Item::Id& itemId, const Akonadi::Entity::Id& from, const Akonadi::Entity::Id& to) { if (!m_db) return; Xapian::Document doc; try { doc = m_db->get_document(itemId); } catch (const Xapian::DocNotFoundError&) { return; } const QByteArray ft = 'C' + QByteArray::number(from); const QByteArray tt = 'C' + QByteArray::number(to); doc.remove_term(ft.data()); doc.add_boolean_term(tt.data()); m_db->replace_document(doc.get_docid(), doc); } void EmailIndexer::commit() { if (m_db) m_db->commit(); if (m_contactDb) m_contactDb->commit(); } diff --git a/src/pim/agent/index.cpp b/src/pim/agent/index.cpp index f8334e3e..7ca30d02 100644 --- a/src/pim/agent/index.cpp +++ b/src/pim/agent/index.cpp @@ -1,426 +1,469 @@ /* * Copyright 2014 Christian Mollekopf * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as * published by the Free Software Foundation; either version 2 of * the License or (at your option) version 3 or any later version * accepted by the membership of KDE e.V. (or its successor approved * by the membership of KDE e.V.), which shall act as a proxy * defined in Section 14 of version 3 of the license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . * */ #include "index.h" #include "emailindexer.h" #include "contactindexer.h" #include "akonotesindexer.h" #include "calendarindexer.h" #include #include #include #include #include #include #include #include Index::Index(QObject* parent) : QObject(parent), m_collectionIndexer(0) { m_commitTimer.setInterval(1000); m_commitTimer.setSingleShot(true); connect(&m_commitTimer, SIGNAL(timeout()), this, SLOT(commit())); } Index::~Index() { delete m_collectionIndexer; m_collectionIndexer = 0; qDeleteAll(m_indexer.values().toSet()); m_indexer.clear(); } static void removeDir(const QString& dirName) { QDir dir(dirName); if (dir.exists(dirName)) { Q_FOREACH(const QFileInfo &info, dir.entryInfoList(QDir::NoDotAndDotDot | QDir::System | QDir::Hidden | QDir::AllDirs | QDir::Files, QDir::DirsFirst)) { if (info.isDir()) { removeDir(info.absoluteFilePath()); } else { QFile::remove(info.absoluteFilePath()); } } dir.rmdir(dirName); } } void Index::removeDatabase() { delete m_collectionIndexer; m_collectionIndexer = 0; qDeleteAll(m_indexer.values().toSet()); m_indexer.clear(); kDebug() << "Removing database"; removeDir(emailIndexingPath()); removeDir(contactIndexingPath()); removeDir(emailContactsIndexingPath()); removeDir(akonotesIndexingPath()); removeDir(calendarIndexingPath()); removeDir(collectionIndexingPath()); } AbstractIndexer* Index::indexerForItem(const Akonadi::Item& item) const { return m_indexer.value(item.mimeType()); } QList Index::indexersForMimetypes(const QStringList& mimeTypes) const { QList indexers; Q_FOREACH (const QString& mimeType, mimeTypes) { AbstractIndexer *i = m_indexer.value(mimeType); if (i) { indexers.append(i); } } return indexers; } bool Index::haveIndexerForMimeTypes(const QStringList &mimeTypes) { return !indexersForMimetypes(mimeTypes).isEmpty(); } void Index::index(const Akonadi::Item& item) { AbstractIndexer *indexer = indexerForItem(item); if (!indexer) { return; } try { indexer->index(item); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } void Index::move(const Akonadi::Item::List& items, const Akonadi::Collection& from, const Akonadi::Collection& to) { //We always get items of the same type AbstractIndexer *indexer = indexerForItem(items.first()); if (!indexer) { return; } Q_FOREACH (const Akonadi::Item& item, items) { try { indexer->move(item.id(), from.id(), to.id()); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } void Index::updateFlags(const Akonadi::Item::List& items, const QSet& addedFlags, const QSet& removedFlags) { //We always get items of the same type AbstractIndexer *indexer = indexerForItem(items.first()); if (!indexer) { return; } Q_FOREACH (const Akonadi::Item& item, items) { try { indexer->updateFlags(item, addedFlags, removedFlags); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } void Index::remove(const QSet< Akonadi::Entity::Id >& ids, const QStringList& mimeTypes) { const QList indexers = indexersForMimetypes(mimeTypes); Q_FOREACH (const Akonadi::Item::Id& id, ids) { Q_FOREACH (AbstractIndexer *indexer, indexers) { try { indexer->remove(Akonadi::Item(id)); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } } void Index::remove(const Akonadi::Item::List& items) { AbstractIndexer *indexer = indexerForItem(items.first()); if (!indexer) { return; } Q_FOREACH (const Akonadi::Item& item, items) { try { indexer->remove(item); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } void Index::index(const Akonadi::Collection& collection) { if (m_collectionIndexer) { m_collectionIndexer->index(collection); m_collectionIndexer->commit(); } kDebug() << "indexed " << collection.id(); } void Index::change(const Akonadi::Collection& col) { if (m_collectionIndexer) { m_collectionIndexer->change(col); m_collectionIndexer->commit(); } } void Index::remove(const Akonadi::Collection& col) { //Remove items Q_FOREACH (AbstractIndexer *indexer, indexersForMimetypes(col.contentMimeTypes())) { try { indexer->remove(col); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } if (m_collectionIndexer) { m_collectionIndexer->remove(col); m_collectionIndexer->commit(); } } void Index::move(const Akonadi::Collection& collection, const Akonadi::Collection& from, const Akonadi::Collection& to) { if (m_collectionIndexer) { m_collectionIndexer->move(collection, from, to); m_collectionIndexer->commit(); } } void Index::addIndexer(AbstractIndexer* indexer) { Q_FOREACH (const QString& mimeType, indexer->mimeTypes()) { m_indexer.insert(mimeType, indexer); } } bool Index::createIndexers() { + bool reindex = false; AbstractIndexer *indexer = 0; - try { - QDir().mkpath(emailIndexingPath()); - QDir().mkpath(emailContactsIndexingPath()); - indexer = new EmailIndexer(emailIndexingPath(), emailContactsIndexingPath()); - addIndexer(indexer); - } - catch (const Xapian::DatabaseError &e) { - delete indexer; - kError() << "Failed to create email indexer:" << QString::fromStdString(e.get_msg()); - } - catch (...) { - delete indexer; - kError() << "Random exception, but we do not want to crash"; - } - try { - QDir().mkpath(contactIndexingPath()); - indexer = new ContactIndexer(contactIndexingPath()); - addIndexer(indexer); - } - catch (const Xapian::DatabaseError &e) { - delete indexer; - kError() << "Failed to create contact indexer:" << QString::fromStdString(e.get_msg()); - } - catch (...) { - delete indexer; - kError() << "Random exception, but we do not want to crash"; + for (int i = 0; i < 2; i++){ + try { + QDir().mkpath(emailIndexingPath()); + QDir().mkpath(emailContactsIndexingPath()); + indexer = new EmailIndexer(emailIndexingPath(), emailContactsIndexingPath()); + addIndexer(indexer); + break; + } + catch (const Xapian::DatabaseError &e) { + QFile::remove(emailIndexingPath() + "flintlock"); + QFile::remove(emailContactsIndexingPath() + "flintlock"); + reindex = true; + delete indexer; + kError() << "Failed to create email indexer:" << QString::fromStdString(e.get_msg()); + } + catch (...) { + QFile::remove(emailIndexingPath() + "flintlock"); + QFile::remove(emailContactsIndexingPath() + "flintlock"); + reindex = true; + delete indexer; + kError() << "Random exception, but we do not want to crash"; + } } - try { - QDir().mkpath(akonotesIndexingPath()); - indexer = new AkonotesIndexer(akonotesIndexingPath()); - addIndexer(indexer); - } - catch (const Xapian::DatabaseError &e) { - delete indexer; - kError() << "Failed to create akonotes indexer:" << QString::fromStdString(e.get_msg()); - } - catch (...) { - delete indexer; - kError() << "Random exception, but we do not want to crash"; + for (int i = 0; i < 2; i++){ + try { + QDir().mkpath(contactIndexingPath()); + indexer = new ContactIndexer(contactIndexingPath()); + addIndexer(indexer); + break; + } + catch (const Xapian::DatabaseError &e) { + QFile::remove(contactIndexingPath() + "flintlock"); + reindex = true; + delete indexer; + kError() << "Failed to create contact indexer:" << QString::fromStdString(e.get_msg()); + } + catch (...) { + QFile::remove(contactIndexingPath() + "flintlock"); + reindex = true; + delete indexer; + kError() << "Random exception, but we do not want to crash"; + } } - try { - QDir().mkpath(calendarIndexingPath()); - indexer = new CalendarIndexer(calendarIndexingPath()); - addIndexer(indexer); - } - catch (const Xapian::DatabaseError &e) { - delete indexer; - kError() << "Failed to create collection indexer:" << QString::fromStdString(e.get_msg()); - } - catch (...) { - delete indexer; - kError() << "Random exception, but we do not want to crash"; + for (int i = 0; i < 2; i++){ + try { + QDir().mkpath(akonotesIndexingPath()); + indexer = new AkonotesIndexer(akonotesIndexingPath()); + addIndexer(indexer); + break; + } + catch (const Xapian::DatabaseError &e) { + QFile::remove(akonotesIndexingPath() + "flintlock"); + reindex = true; + delete indexer; + kError() << "Failed to create akonotes indexer:" << QString::fromStdString(e.get_msg()); + } + catch (...) { + QFile::remove(akonotesIndexingPath() + "flintlock"); + reindex = true; + delete indexer; + kError() << "Random exception, but we do not want to crash"; + } } - try { - QDir().mkpath(collectionIndexingPath()); - m_collectionIndexer = new CollectionIndexer(collectionIndexingPath()); + for (int i = 0; i < 2; i++){ + try { + QDir().mkpath(calendarIndexingPath()); + indexer = new CalendarIndexer(calendarIndexingPath()); + addIndexer(indexer); + break; + } + catch (const Xapian::DatabaseError &e) { + QFile::remove(calendarIndexingPath() + "flintlock"); + reindex = true; + delete indexer; + kError() << "Failed to create collection indexer:" << QString::fromStdString(e.get_msg()); + } + catch (...) { + QFile::remove(calendarIndexingPath() + "flintlock"); + reindex = true; + delete indexer; + kError() << "Random exception, but we do not want to crash"; + } } - catch (const Xapian::DatabaseError &e) { - delete m_collectionIndexer; - m_collectionIndexer = 0; - kError() << "Failed to create akonotes indexer:" << QString::fromStdString(e.get_msg()); + + for (int i = 0; i < 2; i++){ + try { + QDir().mkpath(collectionIndexingPath()); + m_collectionIndexer = new CollectionIndexer(collectionIndexingPath()); + break; + } + catch (const Xapian::DatabaseError &e) { + QFile::remove(collectionIndexingPath() + "flintlock"); + reindex = true; + delete m_collectionIndexer; + m_collectionIndexer = 0; + kError() << "Failed to create akonotes indexer:" << QString::fromStdString(e.get_msg()); + } + catch (...) { + QFile::remove(collectionIndexingPath() + "flintlock"); + reindex = true; + delete m_collectionIndexer; + m_collectionIndexer = 0; + kError() << "Random exception, but we do not want to crash"; + } } - catch (...) { - delete m_collectionIndexer; - m_collectionIndexer = 0; - kError() << "Random exception, but we do not want to crash"; + + if (reindex) { + Q_EMIT reindexAll(); } return !m_indexer.isEmpty(); } void Index::scheduleCommit() { if (!m_commitTimer.isActive()) { m_commitTimer.start(); } } void Index::commit() { m_commitTimer.stop(); Q_FOREACH (AbstractIndexer *indexer, m_indexer) { try { indexer->commit(); } catch (const Xapian::Error &e) { kWarning() << "Xapian error in indexer" << indexer << ":" << e.get_msg().c_str(); } } } void Index::findIndexedInDatabase(QSet &indexed, Akonadi::Entity::Id collectionId, const QString& dbPath) { Xapian::Database db; try { db = Xapian::Database(QFile::encodeName(dbPath).constData()); } catch (const Xapian::DatabaseError& e) { kError() << "Failed to open database" << dbPath << ":" << QString::fromStdString(e.get_msg()); return; } const std::string term = QString::fromLatin1("C%1").arg(collectionId).toStdString(); Xapian::Query query(term); Xapian::Enquire enquire(db); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, UINT_MAX); Xapian::MSetIterator it = mset.begin(); for (;it != mset.end(); it++) { indexed << *it; } } void Index::findIndexed(QSet& indexed, Akonadi::Entity::Id collectionId) { findIndexedInDatabase(indexed, collectionId, emailIndexingPath()); findIndexedInDatabase(indexed, collectionId, contactIndexingPath()); findIndexedInDatabase(indexed, collectionId, akonotesIndexingPath()); findIndexedInDatabase(indexed, collectionId, calendarIndexingPath()); } qlonglong Index::indexedItems(const qlonglong id) { const std::string term = QString::fromLatin1("C%1").arg(id).toStdString(); return indexedItemsInDatabase(term, emailIndexingPath()) + indexedItemsInDatabase(term, contactIndexingPath()) + indexedItemsInDatabase(term, akonotesIndexingPath()) + indexedItemsInDatabase(term, calendarIndexingPath()); } qlonglong Index::indexedItemsInDatabase(const std::string& term, const QString& dbPath) const { Xapian::Database db; try { db = Xapian::Database(QFile::encodeName(dbPath).constData()); } catch (const Xapian::DatabaseError& e) { kError() << "Failed to open database" << dbPath << ":" << QString::fromStdString(e.get_msg()); return 0; } return db.get_termfreq(term); } void Index::setOverrideDbPrefixPath(const QString& path) { m_overridePrefixPath = path; } QString Index::dbPath(const QString& dbName) const { if (!m_overridePrefixPath.isEmpty()) { return QString::fromLatin1("%1/%2/").arg(m_overridePrefixPath, dbName); } QString basePath = QLatin1String("baloo"); if (Akonadi::ServerManager::hasInstanceIdentifier()) { basePath = QString::fromLatin1("baloo/instances/%1").arg(Akonadi::ServerManager::instanceIdentifier()); } return KGlobal::dirs()->localxdgdatadir() + QString::fromLatin1("%1/%2/").arg(basePath, dbName); } QString Index::emailIndexingPath() const { return dbPath(QLatin1String("email")); } QString Index::contactIndexingPath() const { return dbPath(QLatin1String("contacts")); } QString Index::emailContactsIndexingPath() const { return dbPath(QLatin1String("emailContacts")); } QString Index::akonotesIndexingPath() const { return dbPath(QLatin1String("notes")); } QString Index::calendarIndexingPath() const { return dbPath(QLatin1String("calendars")); } QString Index::collectionIndexingPath() const { return dbPath(QLatin1String("collections")); } diff --git a/src/pim/agent/index.h b/src/pim/agent/index.h index a944025b..d8de4052 100644 --- a/src/pim/agent/index.h +++ b/src/pim/agent/index.h @@ -1,94 +1,97 @@ /* * Copyright 2014 Christian Mollekopf * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as * published by the Free Software Foundation; either version 2 of * the License or (at your option) version 3 or any later version * accepted by the membership of KDE e.V. (or its successor approved * by the membership of KDE e.V.), which shall act as a proxy * defined in Section 14 of version 3 of the license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . * */ #ifndef INDEX_H #define INDEX_H #include #include #include #include #include "abstractindexer.h" #include "collectionindexer.h" /** * Maintains the variuous indexers and databases */ class Index : public QObject { Q_OBJECT public: explicit Index(QObject* parent = 0); virtual ~Index(); virtual void removeDatabase(); virtual bool createIndexers(); virtual void index(const Akonadi::Item& item); virtual void move(const Akonadi::Item::List& items, const Akonadi::Collection& from, const Akonadi::Collection& to); virtual void updateFlags(const Akonadi::Item::List& items, const QSet& addedFlags, const QSet& removed); virtual void remove(const QSet& ids, const QStringList& mimeTypes); virtual void remove(const Akonadi::Item::List& items); virtual void index(const Akonadi::Collection& collection); virtual void change(const Akonadi::Collection& collection); virtual void remove(const Akonadi::Collection& col); virtual void move(const Akonadi::Collection& collection, const Akonadi::Collection& from, const Akonadi::Collection& to); virtual bool haveIndexerForMimeTypes(const QStringList &); virtual qlonglong indexedItems(const qlonglong id); virtual void findIndexed(QSet& indexed, Akonadi::Collection::Id); virtual void scheduleCommit(); /// For testing void setOverrideDbPrefixPath(const QString& path); +Q_SIGNALS: + void reindexAll(); + public Q_SLOTS: virtual void commit(); private: void addIndexer(AbstractIndexer* indexer); AbstractIndexer* indexerForItem(const Akonadi::Item& item) const; QList indexersForMimetypes(const QStringList& mimeTypes) const; virtual qlonglong indexedItemsInDatabase(const std::string& term, const QString& dbPath) const; virtual void findIndexedInDatabase(QSet& indexed, Akonadi::Entity::Id collectionId, const QString& dbPath); QString dbPath(const QString& dbName) const; QString emailIndexingPath() const; QString contactIndexingPath() const; QString emailContactsIndexingPath() const; QString akonotesIndexingPath() const; QString calendarIndexingPath() const; QString collectionIndexingPath() const; QString m_overridePrefixPath; QHash m_indexer; QTimer m_commitTimer; CollectionIndexer *m_collectionIndexer; }; #endif diff --git a/src/xapian/xapiandatabase.cpp b/src/xapian/xapiandatabase.cpp index f5657eac..1985c935 100644 --- a/src/xapian/xapiandatabase.cpp +++ b/src/xapian/xapiandatabase.cpp @@ -1,202 +1,206 @@ /* * Copyright (C) 2014 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "xapiandatabase.h" #include "xapiandocument.h" #include #include #include +#include #ifdef __GNUC__ #include #endif #include using namespace Baloo; XapianDatabase::XapianDatabase(const QString& path, bool writeOnly) : m_db(0) , m_writeOnly(writeOnly) { QDir().mkpath(path); m_path = path.toUtf8().constData(); if (!writeOnly) { try { createWritableDb(); m_db = new Xapian::Database(m_path); } catch (const Xapian::DatabaseError& err) { kError() << "Serious Error: " << err.get_error_string(); kError() << err.get_msg().c_str() << err.get_context().c_str() << err.get_description().c_str(); } // Possible errors - DatabaseLock error // Corrupt and InvalidID error } else { m_wDb = createWritableDb(); } } XapianDatabase::~XapianDatabase() { delete m_db; } void XapianDatabase::replaceDocument(uint id, const XapianDocument& doc) { replaceDocument(id, doc.doc()); } void XapianDatabase::replaceDocument(uint id, const Xapian::Document& doc) { if (m_writeOnly) { try { m_wDb.replace_document(id, doc); } catch (const Xapian::Error&) { } return; } m_docsToAdd << qMakePair(id, doc); } void XapianDatabase::deleteDocument(uint id) { if (m_writeOnly) { try { m_wDb.delete_document(id); } catch (const Xapian::Error&) { } return; } m_docsToRemove << id; } void XapianDatabase::commit() { if (m_writeOnly) { try { m_wDb.commit(); } catch (const Xapian::Error& err) { kError() << err.get_error_string(); } return; } if (m_docsToAdd.isEmpty() && m_docsToRemove.isEmpty()) { return; } Xapian::WritableDatabase wdb = createWritableDb(); kDebug() << "Adding:" << m_docsToAdd.size() << "docs"; Q_FOREACH (const DocIdPair& doc, m_docsToAdd) { try { wdb.replace_document(doc.first, doc.second); } catch (const Xapian::Error&) { } } kDebug() << "Removing:" << m_docsToRemove.size() << "docs"; Q_FOREACH (Xapian::docid id, m_docsToRemove) { try { wdb.delete_document(id); } catch (const Xapian::Error&) { } } try { wdb.commit(); m_db->reopen(); } catch (const Xapian::Error& err) { kError() << err.get_error_string(); } kDebug() << "Xapian Committed"; m_docsToAdd.clear(); m_docsToRemove.clear(); #ifdef __GNUC__ malloc_trim(0); #endif } XapianDocument XapianDatabase::document(uint id) { try { Xapian::Document xdoc; if (m_writeOnly) { xdoc = m_wDb.get_document(id); } else { xdoc = m_db->get_document(id); } return XapianDocument(xdoc); } catch (const Xapian::DatabaseModifiedError&) { m_db->reopen(); return document(id); } catch (const Xapian::Error&) { return XapianDocument(); } } Xapian::WritableDatabase XapianDatabase::createWritableDb() { // We need to keep sleeping for a required amount, until we reach // a threshold. That's when we decide to abort? - for (int i = 1; i <= 20; i++) { + for (int i = 1; i <= 5; ++i) { try { Xapian::WritableDatabase wdb(m_path, Xapian::DB_CREATE_OR_OPEN); return wdb; } - catch (const Xapian::DatabaseLockError&) { - sleep(i * 50 ); + catch (const Xapian::DatabaseLockError &error) { + qWarning() << "Database lock error: " << error.get_error_string(); + sleep(1); } - catch (const Xapian::DatabaseModifiedError&) { - sleep(i * 50 ); + catch (const Xapian::DatabaseModifiedError &error) { + qWarning() << "Database modified error: " << error.get_error_string(); + sleep(1); } catch (const Xapian::DatabaseCreateError& err) { kDebug() << err.get_error_string(); return Xapian::WritableDatabase(); } catch (const Xapian::DatabaseCorruptError& err) { kError() << "Database Corrupted - What did you do?"; kError() << err.get_error_string(); return Xapian::WritableDatabase(); } catch (...) { kError() << "Bananana Error"; return Xapian::WritableDatabase(); } } kError() << "Could not obtain lock for Xapian Database. This is bad"; + throw Xapian::DatabaseError("Failed to obtain lock."); return Xapian::WritableDatabase(); }