Page MenuHomePhorge
Files F117753842

No OneTemporary
Actions

Authored By
Unknown
Size
14 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/src/file/extractor/app.cpp b/src/file/extractor/app.cpp
index 7d20987d..e3a6ca07 100644
--- a/src/file/extractor/app.cpp
+++ b/src/file/extractor/app.cpp
@@ -1,321 +1,322 @@
/*
* This file is part of the KDE Baloo Project
* Copyright (C) 2013-2014 Vishesh Handa <me@vhanda.in>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) version 3, or any
* later version accepted by the membership of KDE e.V. (or its
* successor approved by the membership of KDE e.V.), which shall
* act as a proxy defined in Section 6 of version 3 of the license.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "app.h"
#include "../basicindexingjob.h"
#include "../database.h"
#include "xapiandatabase.h"
#include <QDebug>
#include <QCoreApplication>
#include <QTimer>
#include <QFileInfo>
#include <QDBusMessage>
#include <QSqlQuery>
#include <QSqlError>
#include <QDBusConnection>
#include <KFileMetaData/ExtractorPlugin>
#include <KFileMetaData/PropertyInfo>
#include <iostream>
using namespace Baloo;
App::App(const QString& path, QObject* parent)
: QObject(parent)
, m_path(path)
, m_termCount(0)
{
m_db.setPath(m_path);
if (!m_db.init(true /*sql db only*/)) {
QTimer::singleShot(0, QCoreApplication::instance(), SLOT(quit()));
return;
}
+ m_db.sqlDatabase().transaction();
connect(this, SIGNAL(saved()), this, SLOT(processNextUrl()), Qt::QueuedConnection);
}
void App::startProcessing(const QStringList& args)
{
m_results.reserve(args.size());
Q_FOREACH (const QString& arg, args) {
FileMapping mapping = FileMapping(arg.toUInt());
QString url;
// arg is an id
if (!m_bData && mapping.fetch(m_db.sqlDatabase())) {
url = mapping.url();
if (!QFile::exists(url)) {
mapping.remove(m_db.sqlDatabase());
continue;
}
} else {
// arg is a url
url = QFileInfo(arg).absoluteFilePath();
}
if (QFile::exists(url)) {
m_urls << url;
} else {
// id or url was looked up, but file deleted
qDebug() << url << "does not exist";
// Try to delete it as an id:
// it may have been deleted from the FileMapping db as well.
// The worst that can happen is deleting nothing.
mapping.remove(m_db.sqlDatabase());
m_docsToDelete << mapping.id();
}
}
QTimer::singleShot(0, this, SLOT(processNextUrl()));
}
void App::processNextUrl()
{
if (m_urls.isEmpty()) {
if (m_results.isEmpty() && m_docsToDelete.isEmpty()) {
QCoreApplication::instance()->exit(0);
}
else {
saveChanges();
}
return;
}
const QString url = m_urls.takeFirst();
QString mimetype = m_mimeDb.mimeTypeForFile(url).name();
if (!ignoreConfig()) {
bool shouldIndex = m_config.shouldBeIndexed(url) && m_config.shouldMimeTypeBeIndexed(mimetype);
if (!shouldIndex) {
qDebug() << url << "should not be indexed. Ignoring";
FileMapping mapping(url);
mapping.remove(m_db.sqlDatabase());
m_docsToDelete << mapping.id();
QTimer::singleShot(0, this, SLOT(processNextUrl()));
return;
}
}
//
// HACK: We only want to index plain text files which end with a .txt
// Also, we're ignoring txt files which are greater tha 50 Mb as we
// have trouble processing them
//
if (mimetype == QLatin1String("text/plain")) {
if (!url.endsWith(QLatin1String(".txt"))) {
qDebug() << "text/plain does not end with .txt. Ignoring";
mimetype.clear();
}
QFileInfo fileInfo(url);
if (fileInfo.size() >= 50 * 1024 * 1024 ) {
mimetype.clear();
}
}
FileMapping file(url);
if (!m_bData) {
if (!file.fetch(m_db.sqlDatabase())) {
file.create(m_db.sqlDatabase());
}
}
else {
file.setId(-1);
}
// We always run the basic indexing again. This is mostly so that the proper
// mimetype is set and we get proper type information.
// The mimetype fetched in the BasicIQ is fast but not accurate
BasicIndexingJob basicIndexer(&m_db.sqlDatabase(), file, mimetype);
basicIndexer.index();
file.setId(basicIndexer.id());
Xapian::Document doc = basicIndexer.document();
KFileMetaData::ExtractionResult::Flags flags = KFileMetaData::ExtractionResult::ExtractEverything;
if (m_bData) {
flags = KFileMetaData::ExtractionResult::ExtractMetaData;
}
Result result(url, mimetype, flags);
result.setId(file.id());
result.setDocument(doc);
result.setReadOnly(m_bData);
QList<KFileMetaData::ExtractorPlugin*> exList = m_manager.fetchExtractors(mimetype);
Q_FOREACH (KFileMetaData::ExtractorPlugin* plugin, exList) {
plugin->extract(&result);
}
m_results << result;
m_termCount += result.document().termlist_count();
// Documents with these many terms occupy about 10 mb
if (m_termCount >= 10000 && !m_bData) {
saveChanges();
return;
}
if (m_urls.isEmpty()) {
if (m_bData) {
QByteArray arr;
QDataStream s(&arr, QIODevice::WriteOnly);
Q_FOREACH (const Result& res, m_results) {
QVariantMap map;
QMapIterator<QString, QVariant> it(res.map());
while (it.hasNext()) {
it.next();
int propNum = it.key().toInt();
using namespace KFileMetaData::Property;
Property prop = static_cast<Property>(propNum);
KFileMetaData::PropertyInfo pi(prop);
map.insert(pi.name(), it.value());
}
qDebug() << map;
s << map;
}
std::cout << arr.toBase64().constData();
m_results.clear();
}
else {
saveChanges();
}
}
QTimer::singleShot(0, this, SLOT(processNextUrl()));
}
void App::saveChanges()
{
if (m_results.isEmpty() && m_docsToDelete.isEmpty())
return;
m_updatedFiles.clear();
XapianDatabase xapDb(m_path);
for (int i = 0; i<m_results.size(); ++i) {
Result& res = m_results[i];
res.finish();
xapDb.replaceDocument(res.id(), res.document());
m_updatedFiles << res.inputUrl();
}
Q_FOREACH (int docid, m_docsToDelete) {
xapDb.deleteDocument(docid);
}
m_docsToDelete.clear();
xapDb.commit();
m_db.sqlDatabase().commit();
QDBusMessage message = QDBusMessage::createSignal(QLatin1String("/files"),
QLatin1String("org.kde"),
QLatin1String("changed"));
QVariantList vl;
vl.reserve(1);
vl << QVariant(m_updatedFiles);
message.setArguments(vl);
QDBusConnection::sessionBus().send(message);
m_results.clear();
m_termCount = 0;
m_updatedFiles.clear();
if (m_debugEnabled) {
printDebug();
}
Q_EMIT saved();
}
void App::printDebug()
{
Q_FOREACH (const Result& res, m_results) {
qDebug() << res.inputUrl();
QMapIterator<QString, QVariant> it(res.map());
while (it.hasNext()) {
it.next();
int propNum = it.key().toInt();
using namespace KFileMetaData::Property;
Property prop = static_cast<Property>(propNum);
KFileMetaData::PropertyInfo pi(prop);
qDebug() << pi.name() << it.value();
}
}
// Print the io usage
QFile file(QLatin1String("/proc/self/io"));
file.open(QIODevice::ReadOnly | QIODevice::Text);
QTextStream fs(&file);
QString str = fs.readAll();
qDebug() << "------- IO ---------";
QTextStream stream(&str);
while (!stream.atEnd()) {
QString str = stream.readLine();
QString rchar(QLatin1String("rchar: "));
if (str.startsWith(rchar)) {
ulong amt = str.mid(rchar.size()).toULong();
qDebug() << "Read:" << amt / 1024 << "kb";
}
QString wchar(QLatin1String("wchar: "));
if (str.startsWith(wchar)) {
ulong amt = str.mid(wchar.size()).toULong();
qDebug() << "Write:" << amt / 1024 << "kb";
}
QString read(QLatin1String("read_bytes: "));
if (str.startsWith(read)) {
ulong amt = str.mid(read.size()).toULong();
qDebug() << "Actual Reads:" << amt / 1024 << "kb";
}
QString write(QLatin1String("write_bytes: "));
if (str.startsWith(write)) {
ulong amt = str.mid(write.size()).toULong();
qDebug() << "Actual Writes:" << amt / 1024 << "kb";
}
}
}
bool App::ignoreConfig() const
{
return m_ignoreConfig || m_bData;
}
diff --git a/src/queryparser/completionproposal.h b/src/queryparser/completionproposal.h
index 6f8cf0df..a172acd3 100644
--- a/src/queryparser/completionproposal.h
+++ b/src/queryparser/completionproposal.h
@@ -1,123 +1,123 @@
/* This file is part of the Baloo query parser
Copyright (c) 2013 Denis Steckelmacher <steckdenis@yahoo.fr>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Library General Public
License version 2.1 as published by the Free Software Foundation,
or any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Library General Public License for more details.
You should have received a copy of the GNU Library General Public License
along with this library; see the file COPYING.LIB. If not, write to
the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA.
*/
#ifndef __BALOO_SEARCH_COMPLETION_PROPOSAL_H__
#define __BALOO_SEARCH_COMPLETION_PROPOSAL_H__
#include <QtCore/QtGlobal>
#include <QtCore/QStringList>
#include "queryparser_export.h"
#include "klocalizedstring.h"
namespace Baloo
{
/**
* \class CompletionProposal completionproposal.h Baloo/Query/CompletionProposal
* \brief Information about an auto-completion proposal
*
* When parsing an user query, QueryParser may find that a pattern that nearly
* matches and that the user may want to use. In this case, one or more
* completion proposals are used to describe what patterns can be used.
*/
class BALOO_QUERYPARSER_EXPORT CompletionProposal
{
private:
Q_DISABLE_COPY(CompletionProposal)
public:
/**
* \brief Data-type used by the first placeholder of the pattern
*
- * If the pattern is "sent by ?1", the type of "$1" is Contact. This
+ * If the pattern is "sent by $1", the type of "$1" is Contact. This
* way, a GUI can show to the user a list of his or her contacts.
*/
enum Type
{
NoType, /*!< No specific type (integer, string, something that does not need any auto-completion list) */
DateTime, /*!< A date-time */
Tag, /*!< A valid tag name */
Contact, /*!< Something that can be parsed unambiguously to a contact (a contact name, email, pseudo, etc) */
Email, /*!< An e-mail address */
};
/**
* \param pattern list of terms matched by the proposal ("sent",
* "by", "$1" for instance)
* \param last_matched_part index of the last part of the mattern
* that has been matched against the user
* query
* \param position position in the user query of the pattern matched
* \param length length in the user query of the terms matched
* \param type if the pattern contains "$1", this is the type of
* the value matched by this placeholder
* \param description human description of the pattern
*/
CompletionProposal(const QStringList &pattern,
int last_matched_part,
int position,
int length,
Type type,
const KLocalizedString &description);
~CompletionProposal();
/**
* \return list of terms that make the pattern
*/
QStringList pattern() const;
/**
* \return index of the last matched part of the pattern
*/
int lastMatchedPart() const;
/**
* \return position in the user query of the pattern
*
* As an user query can contain spaces and separators that are
* ignored by the pattern matcher, position() and length() are
* used to find the sub-string of the user query that has matched
* against the pattern.
*/
int position() const;
/**
* \sa position
*/
int length() const;
/**
* \return type of the value represented by the "$1" term in the pattern
*/
Type type() const;
/**
* \return description of the pattern
*/
KLocalizedString description() const;
private:
struct Private;
Private *const d;
};
}
#endif

File Metadata

Mime Type
text/x-diff
Expires
Sat, Apr 4, 6:15 AM (1 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18822812
Default Alt Text
(14 KB)

Event Timeline