Page MenuHomePhorge
Files F117755781

No OneTemporary
Actions

Authored By
Unknown
Size
11 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/src/file/fileindexingjob.cpp b/src/file/fileindexingjob.cpp
index 1ed82125..d20dfe46 100644
--- a/src/file/fileindexingjob.cpp
+++ b/src/file/fileindexingjob.cpp
@@ -1,145 +1,155 @@
/*
This file is part of the Nepomuk KDE project.
Copyright (C) 2010-2011 Sebastian Trueg <trueg@kde.org>
Copyright (C) 2012-2014 Vishesh Handa <me@vhanda.in>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) version 3, or any
later version accepted by the membership of KDE e.V. (or its
successor approved by the membership of KDE e.V.), which shall
act as a proxy defined in Section 6 of version 3 of the license.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library. If not, see <http://www.gnu.org/licenses/>.
*/
#include "fileindexingjob.h"
#include "util.h"
#include "fileindexerconfig.h"
#include "database.h"
#include <qjson/serializer.h>
#include <KDebug>
#include <KStandardDirs>
#include <QtCore/QFileInfo>
#include <QtCore/QTimer>
using namespace Baloo;
FileIndexingJob::FileIndexingJob(const QVector<uint>& files, QObject* parent)
: KJob(parent)
, m_files(files)
, m_process(0)
{
// setup the timer used to kill the indexer process if it seems to get stuck
m_processTimer = new QTimer(this);
m_processTimer->setSingleShot(true);
connect(m_processTimer, SIGNAL(timeout()),
this, SLOT(slotProcessTimerTimeout()));
}
void FileIndexingJob::start()
{
if (m_files.isEmpty()) {
emitResult();
return;
}
m_args = m_files;
m_files.clear();
start(m_args);
}
void FileIndexingJob::start(const QVector<uint>& files)
{
// setup the external process which does the actual indexing
const QString exe = KStandardDirs::findExe(QLatin1String("baloo_file_extractor"));
// Just in case
if (m_process) {
m_process->disconnect();
m_process->kill();
delete m_process;
m_process = 0;
}
m_process = new QProcess(this);
QStringList args;
Q_FOREACH (const uint& file, files)
args << QString::number(file);
+
+ if (!m_customDbPath.isEmpty()) {
+ args << "--db" << m_customDbPath;
+ }
kDebug() << args;
connect(m_process, SIGNAL(finished(int,QProcess::ExitStatus)),
this, SLOT(slotIndexedFile(int,QProcess::ExitStatus)));
m_process->setProcessChannelMode(QProcess::SeparateChannels);
m_process->start(exe, args);
// start the timer which will kill the process if it does not terminate after 5 minutes
m_processTimer->start(5 * 60 * 1000);
}
void FileIndexingJob::slotIndexedFile(int, QProcess::ExitStatus exitStatus)
{
// stop the timer since there is no need to kill the process anymore
m_processTimer->stop();
m_process->deleteLater();
m_process = 0;
if (exitStatus == QProcess::NormalExit) {
if (m_files.isEmpty()) {
emitResult();
return;
}
}
else {
if (m_args.size() == 1) {
uint doc = m_args.first();
kError() << "Indexer crashed while indexing" << doc;
kError() << "Blacklisting this file";
Q_EMIT indexingFailed(doc);
if (m_files.isEmpty()) {
emitResult();
return;
}
}
else {
m_files = m_args;
}
}
// Split the number of files into half
if (m_files.size() == 1) {
m_args = m_files;
m_files.clear();
start(m_args);
}
else {
int mid = m_files.size()/2;
m_args = m_files.mid(mid);
m_files.resize(mid);
start(m_args);
}
}
void FileIndexingJob::slotProcessTimerTimeout()
{
// Emulate a crash so that we narrow down the file which is taking too long
slotIndexedFile(1, QProcess::CrashExit);
}
+void FileIndexingJob::setCustomDbPath(const QString& path)
+{
+ m_customDbPath = path;
+}
+
+
#include "fileindexingjob.moc"
diff --git a/src/file/fileindexingjob.h b/src/file/fileindexingjob.h
index 666667b1..1c2c3218 100644
--- a/src/file/fileindexingjob.h
+++ b/src/file/fileindexingjob.h
@@ -1,67 +1,75 @@
/*
This file is part of the Nepomuk KDE project.
Copyright (C) 2010 Sebastian Trueg <trueg@kde.org>
Copyrigth (C) 2013 Vishesh Handa <me@vhanda.in>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) version 3, or any
later version accepted by the membership of KDE e.V. (or its
successor approved by the membership of KDE e.V.), which shall
act as a proxy defined in Section 6 of version 3 of the license.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _BALOO_INDEXING_JOB_H_
#define _BALOO_INDEXING_JOB_H_
#include <KJob>
#include <QProcess>
#include <QVector>
#include "filemapping.h"
class QTimer;
namespace Baloo
{
class FileIndexingJob : public KJob
{
Q_OBJECT
public:
FileIndexingJob(const QVector<uint>& files, QObject* parent = 0);
+ /**
+ * Set a custom path which should be sent to the baloo_file_extractor
+ * to use for the database. This is useful when debugging.
+ */
+ void setCustomDbPath(const QString& path);
+
virtual void start();
Q_SIGNALS:
/**
* This signal is emitted when the indexing fails on a particular document
*/
void indexingFailed(uint document);
private Q_SLOTS:
void slotIndexedFile(int exitCode, QProcess::ExitStatus exitStatus);
void slotProcessTimerTimeout();
private:
void start(const QVector<uint>& files);
QVector<uint> m_files;
QVector<uint> m_args;
QProcess* m_process;
QTimer* m_processTimer;
+
+ QString m_customDbPath;
};
}
#endif
diff --git a/src/file/tests/CMakeLists.txt b/src/file/tests/CMakeLists.txt
index 145adc2d..75a7afbe 100644
--- a/src/file/tests/CMakeLists.txt
+++ b/src/file/tests/CMakeLists.txt
@@ -1,40 +1,65 @@
include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/..
)
kde4_add_executable(indexerconfigtest
indexerconfigtest.cpp
../fileindexerconfig.cpp
../regexpcache.cpp
../fileexcludefilters.cpp)
target_link_libraries(indexerconfigtest
${QT_QTCORE_LIBRARY}
${KDE4_KDECORE_LIBS}
${KDE4_KDEUI_LIBS}
baloocore
)
kde4_add_executable(basicindexingqueuetest
basicindexingqueuetest.cpp
../fileindexerconfig.cpp
../regexpcache.cpp
../fileexcludefilters.cpp
../basicindexingqueue.cpp
../indexingqueue.cpp
../basicindexingjob.cpp
../database.cpp
../lib/filemapping.cpp
../commitqueue.cpp
)
target_link_libraries(basicindexingqueuetest
${QT_QTCORE_LIBRARY}
${QT_QTSQL_LIBRARY}
${KDE4_KDECORE_LIBS}
${KDE4_KIO_LIBS}
${XAPIAN_LIBRARIES}
${KFILEMETADATA_LIBRARY}
baloocore
balooxapian
)
+
+kde4_add_executable(fileindexingqueuetest
+ fileindexingqueuetest.cpp
+ ../fileindexingjob.cpp
+ ../fileindexerconfig.cpp
+ ../regexpcache.cpp
+ ../fileexcludefilters.cpp
+ ../basicindexingqueue.cpp
+ ../indexingqueue.cpp
+ ../basicindexingjob.cpp
+ ../database.cpp
+ ../lib/filemapping.cpp
+ ../commitqueue.cpp
+)
+
+target_link_libraries(fileindexingqueuetest
+ ${QT_QTCORE_LIBRARY}
+ ${QT_QTSQL_LIBRARY}
+ ${KDE4_KDECORE_LIBS}
+ ${KDE4_KIO_LIBS}
+ ${XAPIAN_LIBRARIES}
+ ${KFILEMETADATA_LIBRARY}
+ baloocore
+ balooxapian
+)
diff --git a/src/file/tests/fileindexingqueuetest.cpp b/src/file/tests/fileindexingqueuetest.cpp
new file mode 100644
index 00000000..67ff9ab6
--- /dev/null
+++ b/src/file/tests/fileindexingqueuetest.cpp
@@ -0,0 +1,79 @@
+/*
+ * This file is part of the KDE Baloo Project
+ * Copyright (C) 2014 Vishesh Handa <me@vhanda.in>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) version 3, or any
+ * later version accepted by the membership of KDE e.V. (or its
+ * successor approved by the membership of KDE e.V.), which shall
+ * act as a proxy defined in Section 6 of version 3 of the license.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <KDebug>
+#include <KTempDir>
+#include <QTime>
+#include <QCoreApplication>
+
+#include "../basicindexingqueue.h"
+#include "../fileindexingjob.h"
+#include "../commitqueue.h"
+#include "../database.h"
+#include "../fileindexerconfig.h"
+#include "../lib/filemapping.h"
+
+int main(int argc, char** argv)
+{
+ KTempDir tempDir;
+
+ Database db;
+ db.setPath(tempDir.name());
+ db.init();
+
+ Baloo::FileIndexerConfig config;
+ QCoreApplication app(argc, argv);
+
+ Baloo::BasicIndexingQueue basicIQ(&db, &config);
+ QObject::connect(&basicIQ, SIGNAL(finishedIndexing()), &app, SLOT(quit()));
+
+ Baloo::CommitQueue commitQueue(&db);
+ QObject::connect(&basicIQ, SIGNAL(newDocument(uint,Xapian::Document)),
+ &commitQueue, SLOT(add(uint,Xapian::Document)));
+
+ basicIQ.enqueue(Baloo::FileMapping(QDir::homePath()));
+ app.exec();
+
+ commitQueue.commit();
+
+ // Now the file indexing
+ Xapian::Database* xdb = db.xapianDatabase()->db();
+ Xapian::Enquire enquire(*xdb);
+ enquire.set_query(Xapian::Query("Z1"));
+
+ Xapian::MSet mset = enquire.get_mset(0, 50000);
+ Xapian::MSetIterator it = mset.begin();
+
+ QTime timer;
+ timer.start();
+ for (; it != mset.end(); it++) {
+ QVector<uint> files;
+ files << *it;
+
+ Baloo::FileIndexingJob* job = new Baloo::FileIndexingJob(files);
+ job->setCustomDbPath(db.path());
+ job->exec();
+ }
+
+ qDebug() << "Elapsed:" << timer.elapsed();
+ return 0;
+}

File Metadata

Mime Type
text/x-diff
Expires
Sat, Apr 4, 8:16 AM (1 w, 5 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18823186
Default Alt Text
(11 KB)

Event Timeline