Page MenuHomePhorge
Files F117753787

No OneTemporary
Actions

Authored By
Unknown
Size
679 KB
Referenced Files
None
Subscribers
None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.
diff --git a/imap/index.c b/imap/index.c
index 0f47e10dd..e35c69029 100644
--- a/imap/index.c
+++ b/imap/index.c
@@ -1,8011 +1,8008 @@
/* index.c -- Routines for dealing with the index file in the imapd
*
* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The name "Carnegie Mellon University" must not be used to
* endorse or promote products derived from this software without
* prior written permission. For permission or any legal
* details, please contact
* Carnegie Mellon University
* Center for Technology Transfer and Enterprise Creation
* 4615 Forbes Avenue
* Suite 302
* Pittsburgh, PA 15213
* (412) 268-7393, fax: (412) 268-7395
* innovation@andrew.cmu.edu
*
* 4. Redistributions of any form whatsoever must retain the following
* acknowledgment:
* "This product includes software developed by Computing Services
* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
*
* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#include <config.h>
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <sysexits.h>
#include <syslog.h>
#include <ctype.h>
#include <stdlib.h>
#ifdef USE_HTTPD
/* For iCalendar indexing */
#include <libical/ical.h>
#include "vcard_support.h"
#endif
#include "acl.h"
#include "annotate.h"
#include "append.h"
#include "assert.h"
#include "backend.h"
#include "charset.h"
#include "conversations.h"
#include "dlist.h"
#include "hash.h"
#include "hashu64.h"
#include "http_client.h"
#include "jmap_util.h"
#include "global.h"
#include "times.h"
#include "imapd.h"
#include "lsort.h"
#include "mailbox.h"
#include "map.h"
#include "message.h"
#include "msgrecord.h"
#include "parseaddr.h"
#include "search_engines.h"
#include "search_query.h"
#include "seen.h"
#include "statuscache.h"
#include "strhash.h"
#include "user.h"
#include "util.h"
#include "xstats.h"
#include "ptrarray.h"
#include "xmalloc.h"
#include "xstrlcpy.h"
#include "index.h"
#include "sync_log.h"
/* generated headers are not necessarily in current directory */
#include "imap/imap_err.h"
EXPORTED unsigned client_capa;
/* Forward declarations */
static void index_refresh_locked(struct index_state *state);
static void index_tellexists(struct index_state *state);
static int index_lock(struct index_state *state, int readonly);
static void index_unlock(struct index_state *state);
struct index_modified_flags {
int added_flags;
bit32 added_system_flags;
bit32 added_user_flags[MAX_USER_FLAGS/32];
int removed_flags;
bit32 removed_system_flags;
bit32 removed_user_flags[MAX_USER_FLAGS/32];
};
static int index_writeseen(struct index_state *state);
static void index_fetchmsg(struct index_state *state,
const struct buf *msg,
unsigned offset, unsigned size,
unsigned start_octet, unsigned octet_count);
static int index_fetchsection(struct index_state *state, const char *resp,
const struct buf *msg,
char *section, struct body *body, unsigned size,
unsigned start_octet, unsigned octet_count);
static void index_fetchfsection(struct index_state *state,
const char *msg_base, unsigned long msg_size,
struct fieldlist *fsection,
struct body *body,
unsigned start_octet, unsigned octet_count);
static char *index_readheader(const char *msg_base, unsigned long msg_size,
unsigned offset, unsigned size);
static void index_fetchheader(struct index_state *state,
const char *msg_base, unsigned long msg_size,
unsigned size,
const strarray_t *headers,
const strarray_t *headers_not);
static void index_fetchcacheheader(struct index_state *state, struct index_record *record,
const strarray_t *headers, unsigned start_octet,
unsigned octet_count);
static void index_listflags(struct index_state *state);
static void index_fetchflags(struct index_state *state, uint32_t msgno);
static int index_copysetup(struct index_state *state, uint32_t msgno,
struct copyargs *copyargs);
static int index_storeflag(struct index_state *state,
struct index_modified_flags *modified_flags,
uint32_t msgno, msgrecord_t *msgrec,
struct storeargs *storeargs);
static int index_store_annotation(struct index_state *state, uint32_t msgno,
msgrecord_t *mrw, struct storeargs *storeargs,
int *dirty);
static int index_fetchreply(struct index_state *state, uint32_t msgno,
const struct fetchargs *fetchargs);
static void index_printflags(struct index_state *state, uint32_t msgno,
int usinguid, int printmodseq);
static char *get_localpart_addr(const char *header);
static char *get_displayname(const char *header);
static char *index_extract_subject(const char *subj, size_t len, int *is_refwd);
static char *_index_extract_subject(char *s, int *is_refwd);
static void index_get_ids(MsgData *msgdata,
char *envtokens[], const char *headers, unsigned size);
static int index_sort_compare(MsgData *md1, MsgData *md2,
const struct sortcrit *call_data);
static void *index_thread_getnext(Thread *thread);
static void index_thread_setnext(Thread *thread, Thread *next);
static int index_thread_compare(Thread *t1, Thread *t2,
const struct sortcrit *call_data);
static void index_thread_orderedsubj(struct index_state *state,
unsigned *msgno_list, unsigned int nmsg,
int usinguid);
static void index_thread_sort(Thread *root, const struct sortcrit *sortcrit);
static void index_thread_print(struct index_state *state,
Thread *threads, int usinguid);
static void index_thread_references(struct index_state *state,
unsigned *msgno_list, unsigned int nmsg,
int usinguid);
static void index_thread_refs(struct index_state *state,
unsigned *msgno_list, unsigned int nmsg,
int usinguid);
static struct seqset *_parse_sequence(struct index_state *state,
const char *sequence, int usinguid);
static void massage_header(char *hdr);
/* NOTE: Make sure these are listed in CAPABILITY_STRING */
static const struct thread_algorithm thread_algs[] = {
{ "ORDEREDSUBJECT", index_thread_orderedsubj },
{ "REFERENCES", index_thread_references },
{ "REFS", index_thread_refs },
{ NULL, NULL }
};
EXPORTED int index_reload_record(struct index_state *state,
uint32_t msgno,
struct index_record *record)
{
struct index_map *im = &state->map[msgno-1];
int r = 0;
int i;
memset(record, 0, sizeof(struct index_record));
if (!im->recno) {
/* doh, gotta just fill in what we know */
record->uid = im->uid;
}
else {
record->recno = im->recno;
record->uid = im->uid;
r = mailbox_reload_index_record_dirty(state->mailbox, record);
}
/* NOTE: we have released the cyrus.index lock at this point, but are
* still holding the mailbox name relock. This means nobody can rewrite
* the file under us - so the offsets are still guaranteed to be correct,
* and all the immutable fields are unchanged. That said, we can get a
* read of a partially updated record which contains an invalid checksum
* due to incomplete concurrent changes to mutable fields. That's why we
* used the _dirty API which ignores checksums.
* but other errors are still bad */
if (r) return r;
/* better be! */
assert(record->uid == im->uid);
/* restore mutable fields */
record->modseq = im->modseq;
record->system_flags = im->system_flags;
record->internal_flags = im->internal_flags;
record->cache_offset = im->cache_offset;
for (i = 0; i < MAX_USER_FLAGS/32; i++)
record->user_flags[i] = im->user_flags[i];
return 0;
}
static int index_rewrite_record(struct index_state *state,
uint32_t msgno,
struct index_record *record,
int silent)
{
struct index_map *im = &state->map[msgno-1];
int i;
assert(record->uid == im->uid);
if (!silent) {
int r = mailbox_rewrite_index_record(state->mailbox, record);
if (r) return r;
}
/* update tracking of mutable fields */
im->modseq = record->modseq;
im->system_flags = record->system_flags;
im->internal_flags = record->internal_flags;
im->cache_offset = record->cache_offset;
for (i = 0; i < MAX_USER_FLAGS/32; i++)
im->user_flags[i] = record->user_flags[i];
return 0;
}
EXPORTED void index_release(struct index_state *state)
{
if (!state) return;
message_unref(&state->m);
if (state->mailbox) {
mailbox_close(&state->mailbox);
state->mailbox = NULL; /* should be done by close anyway */
}
}
static struct sortcrit *the_sortcrit;
/*
* A mailbox is about to be closed.
*/
EXPORTED void index_close(struct index_state **stateptr)
{
unsigned i;
struct index_state *state = *stateptr;
if (!state) return;
index_release(state);
xfree(state->map);
xfree(state->mboxname);
xfree(state->userid);
for (i = 0; i < MAX_USER_FLAGS; i++)
xfree(state->flagname[i]);
xfree(state);
*stateptr = NULL;
}
/*
* A new mailbox has been selected and already opened, map it into
* memory and do the initial CHECK.
*/
EXPORTED int index_open_mailbox(struct mailbox *mailbox, struct index_init *init,
struct index_state **stateptr)
{
int r;
struct index_state *state = xzmalloc(sizeof(struct index_state));
state->mailbox = mailbox;
state->mboxname = xstrdup(mailbox->name);
if (init) {
state->authstate = init->authstate;
state->examining = init->examine_mode;
state->out = init->out;
state->userid = xstrdupnull(init->userid);
state->want_dav = init->want_dav;
state->want_mbtype = init->want_mbtype;
state->want_expunged = init->want_expunged;
state->myrights = cyrus_acl_myrights(init->authstate,
state->mailbox->acl);
if (state->examining)
state->myrights &= ~ACL_READ_WRITE;
state->internalseen = mailbox_internal_seen(state->mailbox,
state->userid);
}
if (state->mailbox->mbtype & MBTYPES_NONIMAP) {
if (state->want_dav) {
/* User logged in using imapmagicplus token "dav" */
}
else if (state->mailbox->mbtype == state->want_mbtype) {
/* Caller explicitly asks for this NONIMAP type */
}
else {
r = IMAP_MAILBOX_BADTYPE;
goto fail;
}
}
/* initialise the index_state */
index_refresh_locked(state);
/* have to get the vanished list while we're still locked */
if (init)
init->vanishedlist = index_vanished(state, &init->vanished);
index_unlock(state);
*stateptr = state;
return 0;
fail:
xfree(state->mboxname);
xfree(state->userid);
xfree(state);
return r;
}
/*
* A new mailbox has been selected, map it into memory and do the
* initial CHECK.
*/
EXPORTED int index_open(const char *name, struct index_init *init,
struct index_state **stateptr)
{
int r;
struct mailbox *mailbox = NULL;
r = init && init->examine_mode ? mailbox_open_irl(name, &mailbox) :
mailbox_open_iwl(name, &mailbox);
if (r) return r;
r = index_open_mailbox(mailbox, init, stateptr);
if (r) mailbox_close(&mailbox);
return r;
}
EXPORTED int index_expunge(struct index_state *state, char *sequence,
int need_deleted)
{
int r;
uint32_t msgno;
struct index_map *im;
struct seqset *seq = NULL;
struct index_record record;
int numexpunged = 0;
struct mboxevent *mboxevent = NULL;
modseq_t oldmodseq;
r = index_lock(state, /*readonly*/0);
if (r) return r;
/* XXX - check if not mailbox->i.deleted count and need_deleted */
seq = _parse_sequence(state, sequence, 1);
mboxevent = mboxevent_new(EVENT_MESSAGE_EXPUNGE);
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
if (im->internal_flags & FLAG_INTERNAL_EXPUNGED)
continue; /* already expunged */
if (need_deleted && !(im->system_flags & FLAG_DELETED))
continue; /* no \Deleted flag */
/* if there is a sequence list, check it */
if (sequence && !seqset_ismember(seq, im->uid))
continue; /* not in the list */
/* load first once we know we have to process this one */
if (index_reload_record(state, msgno, &record))
continue;
oldmodseq = im->modseq;
if (!im->isseen) {
state->numunseen--;
im->isseen = 1;
}
if (im->isrecent) {
state->numrecent--;
im->isrecent = 0;
}
/* set the flags */
record.system_flags |= FLAG_DELETED;
record.internal_flags |= FLAG_INTERNAL_EXPUNGED;
numexpunged++;
state->num_expunged++;
r = index_rewrite_record(state, msgno, &record, /*silent*/0);
if (r) break;
/* avoid telling again (equivalent to STORE FLAGS.SILENT) */
if (im->told_modseq == oldmodseq)
im->told_modseq = im->modseq;
mboxevent_extract_record(mboxevent, state->mailbox, &record);
}
seqset_free(seq);
mboxevent_extract_mailbox(mboxevent, state->mailbox);
mboxevent_set_access(mboxevent, NULL, NULL, state->userid, state->mailbox->name, 1);
mboxevent_set_numunseen(mboxevent, state->mailbox, state->numunseen);
/* unlock before responding */
index_unlock(state);
if (!r && (numexpunged > 0)) {
syslog(LOG_NOTICE, "Expunged %d messages from %s",
numexpunged, state->mboxname);
/* send the MessageExpunge event notification for "immediate", "default"
* and "delayed" expunge */
mboxevent_notify(&mboxevent);
}
mboxevent_free(&mboxevent);
return r;
}
static char *index_buildseen(struct index_state *state, const char *oldseenuids)
{
struct seqset *outlist;
uint32_t msgno;
unsigned oldmax;
struct index_map *im;
char *out;
outlist = seqset_init(0, SEQ_MERGE);
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
seqset_add(outlist, im->uid, im->isseen);
}
/* there may be future already seen UIDs that this process isn't
* allowed to know about, but we can't blat them either! This is
* a massive pain... */
oldmax = seq_lastnum(oldseenuids, NULL);
if (oldmax > state->last_uid) {
struct seqset *seq = seqset_parse(oldseenuids, NULL, oldmax);
uint32_t uid;
/* for each future UID, copy the state in the old seenuids */
for (uid = state->last_uid + 1; uid <= oldmax; uid++)
seqset_add(outlist, uid, seqset_ismember(seq, uid));
seqset_free(seq);
}
out = seqset_cstring(outlist);
seqset_free(outlist);
return out;
}
static int index_writeseen(struct index_state *state)
{
int r;
struct seen *seendb = NULL;
struct seendata oldsd = SEENDATA_INITIALIZER;
struct seendata sd = SEENDATA_INITIALIZER;
struct mailbox *mailbox = state->mailbox;
const char *userid = (mailbox->i.options & OPT_IMAP_SHAREDSEEN) ? "anyone" : state->userid;
if (!state->seen_dirty)
return 0;
state->seen_dirty = 0;
/* only examining, can't write any changes */
if (state->examining)
return 0;
/* always dirty the mailbox, we may just be updating recent counts which doesn't bump the
* modseq because Recent is all sorts of evil */
mailbox_index_dirty(mailbox);
/* already handled! Just update the header fields */
if (state->internalseen) {
mailbox->i.recenttime = time(0);
if (mailbox->i.recentuid < state->last_uid)
mailbox->i.recentuid = state->last_uid;
return 0;
}
r = seen_open(userid, SEEN_CREATE, &seendb);
if (r) return r;
r = seen_lockread(seendb, mailbox->uniqueid, &oldsd);
if (r) {
oldsd.lastread = 0;
oldsd.lastuid = 0;
oldsd.lastchange = 0;
oldsd.seenuids = xstrdup("");
}
/* fields of interest... */
sd.lastuid = oldsd.lastuid;
sd.seenuids = index_buildseen(state, oldsd.seenuids);
if (!sd.seenuids) sd.seenuids = xstrdup("");
/* make comparison only catch some changes */
sd.lastread = oldsd.lastread;
sd.lastchange = oldsd.lastchange;
/* update \Recent lowmark */
if (sd.lastuid < state->last_uid)
sd.lastuid = state->last_uid;
/* only commit if interesting fields have changed */
if (!seen_compare(&sd, &oldsd)) {
sd.lastread = time(NULL);
sd.lastchange = mailbox->i.last_appenddate;
r = seen_write(seendb, mailbox->uniqueid, &sd);
}
seen_close(&seendb);
seen_freedata(&oldsd);
seen_freedata(&sd);
return r;
}
/* caller must free the list with seqset_free() when done */
static struct seqset *_readseen(struct index_state *state, unsigned *recentuid)
{
struct mailbox *mailbox = state->mailbox;
struct seqset *seenlist = NULL;
/* Obtain seen information */
if (state->internalseen) {
*recentuid = mailbox->i.recentuid;
}
else if (state->userid) {
struct seen *seendb = NULL;
struct seendata sd = SEENDATA_INITIALIZER;
const char *userid = (mailbox->i.options & OPT_IMAP_SHAREDSEEN) ? "anyone" : state->userid;
int r;
r = seen_open(userid, SEEN_CREATE, &seendb);
if (!r) r = seen_read(seendb, mailbox->uniqueid, &sd);
seen_close(&seendb);
/* handle no seen DB gracefully */
if (r) {
*recentuid = mailbox->i.last_uid;
prot_printf(state->out, "* OK (seen state failure) %s: %s\r\n",
error_message(IMAP_NO_CHECKPRESERVE), error_message(r));
syslog(LOG_ERR, "Could not open seen state for %s (%s)",
userid, error_message(r));
}
else {
*recentuid = sd.lastuid;
seenlist = seqset_parse(sd.seenuids, NULL, *recentuid);
seen_freedata(&sd);
}
}
else {
*recentuid = mailbox->i.last_uid; /* nothing is recent! */
}
return seenlist;
}
static void index_refresh_locked(struct index_state *state)
{
struct mailbox *mailbox = state->mailbox;
const message_t *msg;
uint32_t msgno = 1;
uint32_t firstnotseen = 0;
uint32_t numrecent = 0;
uint32_t numunseen = 0;
uint32_t num_expunged = 0;
uint32_t recentuid = 0;
modseq_t delayed_modseq = 0;
struct index_map *im;
uint32_t need_records;
struct seqset *seenlist;
int i;
/* need to start by having enough space for the entire index state
* before telling of any expunges (which happens after this refresh
* if the command allows it). In the update case, where there's
* already a map, we have to theoretically fit the number that existed
* last time plus however many new records might be unEXPUNGEd on the
* end */
need_records = state->want_expunged ? mailbox->i.num_records : mailbox->i.exists;
if (state->last_uid) {
need_records += (mailbox->i.last_uid - state->last_uid);
}
/* make sure we have space */
if (need_records >= state->mapsize) {
state->mapsize = (need_records | 0xff) + 1; /* round up 1-256 */
state->map = xrealloc(state->map,
state->mapsize * sizeof(struct index_map));
}
seenlist = _readseen(state, &recentuid);
/* walk through all records */
struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED);
while ((msg = mailbox_iter_step(iter))) {
const struct index_record *record = msg_record(msg);
im = &state->map[msgno-1];
while (msgno <= state->exists && im->uid < record->uid) {
/* NOTE: this same logic is repeated below for messages
* past the end of recno (repack removing the trailing
* records). Make sure to keep them in sync */
if (!(im->internal_flags & FLAG_INTERNAL_EXPUNGED)) {
/* we don't even know the modseq of when it was wiped,
* but we can be sure it's since the last given highestmodseq,
* so simulate the lowest possible value. This is fine for
* our told_modseq logic, and doesn't have to be exact because
* QRESYNC/CONDSTORE clients will see deletedmodseq and fall
* back to the inefficient codepath anyway */
im->modseq = state->highestmodseq + 1;
}
if (!delayed_modseq || im->modseq < delayed_modseq)
delayed_modseq = im->modseq - 1;
im->recno = 0;
/* simulate expunged flag so we get an EXPUNGE response and
* tell about unlinked so we don't get IO errors trying to
* find the file */
im->internal_flags |= FLAG_INTERNAL_EXPUNGED |
FLAG_INTERNAL_UNLINKED;
im = &state->map[msgno++];
/* this one is expunged */
num_expunged++;
}
/* expunged record not in map, can skip immediately. It's
* never been told to this connection, so it doesn't need to
* get its own msgno */
if (!state->want_expunged
&& (msgno > state->exists || record->uid < im->uid)
&& (record->internal_flags & FLAG_INTERNAL_EXPUNGED))
continue;
/* make sure our UID map is consistent */
if (msgno <= state->exists) {
assert(im->uid == record->uid);
}
else {
im->uid = record->uid;
}
/* copy all mutable fields */
im->recno = record->recno;
im->modseq = record->modseq;
im->system_flags = record->system_flags;
im->internal_flags = record->internal_flags;
im->cache_offset = record->cache_offset;
for (i = 0; i < MAX_USER_FLAGS/32; i++)
im->user_flags[i] = record->user_flags[i];
/* for expunged records, just track the modseq */
if (im->internal_flags & FLAG_INTERNAL_EXPUNGED) {
num_expunged++;
/* http://www.rfc-editor.org/errata_search.php?rfc=5162
* Errata ID: 1809 - if there are expunged records we
* aren't telling about, need to make the highestmodseq
* be one lower so the client can safely resync */
if (!delayed_modseq || im->modseq < delayed_modseq)
delayed_modseq = im->modseq - 1;
}
else {
/* re-calculate seen flags */
if (state->internalseen)
im->isseen = (im->system_flags & FLAG_SEEN) ? 1 : 0;
else
im->isseen = seqset_ismember(seenlist, im->uid) ? 1 : 0;
if (msgno > state->exists) {
/* don't auto-tell new records */
im->told_modseq = im->modseq;
if (im->uid > recentuid) {
/* mark recent if it's newly being added to the index and also
* greater than the recentuid - ensures only one session gets
* the \Recent flag for any one message */
im->isrecent = 1;
state->seen_dirty = 1;
}
else
im->isrecent = 0;
}
/* track select values */
if (!im->isseen) {
numunseen++;
if (!firstnotseen)
firstnotseen = msgno;
}
if (im->isrecent) {
numrecent++;
}
}
msgno++;
/* make sure we don't overflow the memory we mapped */
if (msgno > state->mapsize) {
char buf[2048];
sprintf(buf, "Exists wrong %u %u %u %u", msgno,
state->mapsize, mailbox->i.exists, mailbox->i.num_records);
fatal(buf, EX_IOERR);
}
}
mailbox_iter_done(&iter);
/* may be trailing records which need to be considered for
* delayed_modseq purposes, and to get the count right for
* later expunge processing */
im = &state->map[msgno-1];
while (msgno <= state->exists) {
/* this is the same logic as the block above in the main loop,
* see comments up there, and make sure the blocks are kept
* in sync! */
if (!(im->internal_flags & FLAG_INTERNAL_EXPUNGED))
im->modseq = state->highestmodseq + 1;
if (!delayed_modseq || im->modseq < delayed_modseq)
delayed_modseq = im->modseq - 1;
im->recno = 0;
im->internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED;
im = &state->map[msgno++];
num_expunged++;
}
seqset_free(seenlist);
/* update the header tracking data */
state->oldexists = state->exists; /* we last knew about this many */
state->exists = msgno - 1; /* we actually got this many */
state->delayed_modseq = delayed_modseq;
state->oldhighestmodseq = state->highestmodseq;
state->highestmodseq = mailbox->i.highestmodseq;
state->generation = mailbox->i.generation_no;
state->uidvalidity = mailbox->i.uidvalidity;
state->last_uid = mailbox->i.last_uid;
state->num_records = mailbox->i.num_records;
state->num_expunged = num_expunged;
state->firstnotseen = firstnotseen;
state->numunseen = numunseen;
state->numrecent = numrecent;
}
EXPORTED modseq_t index_highestmodseq(struct index_state *state)
{
if (state->delayed_modseq)
return state->delayed_modseq;
return state->highestmodseq;
}
EXPORTED void index_select(struct index_state *state, struct index_init *init)
{
index_tellexists(state);
/* always print flags */
index_checkflags(state, 1, 1);
if (state->firstnotseen)
prot_printf(state->out, "* OK [UNSEEN %u] Ok\r\n",
state->firstnotseen);
prot_printf(state->out, "* OK [UIDVALIDITY %u] Ok\r\n",
state->mailbox->i.uidvalidity);
prot_printf(state->out, "* OK [UIDNEXT %lu] Ok\r\n",
state->last_uid + 1);
prot_printf(state->out, "* OK [HIGHESTMODSEQ " MODSEQ_FMT "] Ok\r\n",
state->highestmodseq);
prot_printf(state->out, "* OK [URLMECH INTERNAL] Ok\r\n");
/*
* RFC5257. Note that we must report a maximum size for annotations
* but we don't enforce any such limit, so pick a "large" number.
*/
prot_printf(state->out, "* OK [ANNOTATIONS %u] Ok\r\n", 64*1024);
if (init->vanishedlist) {
char *vanished;
const char *sequence = NULL;
struct seqset *seq = NULL;
struct index_map *im;
uint32_t msgno;
/* QRESYNC response:
* UID FETCH seq FLAGS (CHANGEDSINCE modseq VANISHED)
*/
vanished = seqset_cstring(init->vanishedlist);
if (vanished) {
prot_printf(state->out, "* VANISHED (EARLIER) %s\r\n", vanished);
free(vanished);
}
sequence = init->vanished.sequence;
if (sequence) seq = _parse_sequence(state, sequence, 1);
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
if (sequence && !seqset_ismember(seq, im->uid))
continue;
if (im->modseq <= init->vanished.modseq)
continue;
index_printflags(state, msgno, 1, 0);
}
seqset_free(seq);
}
}
/*
* Check for and report updates
*/
EXPORTED int index_check(struct index_state *state, int usinguid, int printuid)
{
int r;
if (!state) return 0;
/* we don't write any records in here, but we want to write the recentuid if
* there were new emails delivered... */
r = index_lock(state, /*readonly*/0);
/* Check for deleted mailbox */
if (r == IMAP_MAILBOX_NONEXISTENT) {
/* Mailbox has been (re)moved */
if (config_getswitch(IMAPOPT_DISCONNECT_ON_VANISHED_MAILBOX)) {
syslog(LOG_WARNING,
"Mailbox %s has been (re)moved out from under client",
state->mboxname);
mailbox_close(&state->mailbox);
fatal("Mailbox has been (re)moved", EX_IOERR);
}
if (state->exists && (client_capa & CAPA_QRESYNC)) {
/* XXX - is it OK to just expand to entire possible range? */
prot_printf(state->out, "* VANISHED 1:%lu\r\n", state->last_uid);
}
else {
int exists;
for (exists = state->exists; exists > 0; exists--) {
prot_printf(state->out, "* 1 EXPUNGE\r\n");
}
}
state->exists = 0;
return IMAP_MAILBOX_NONEXISTENT;
}
if (r) return r;
index_tellchanges(state, usinguid, printuid, 0);
#if TOIMSP
if (state->firstnotseen) {
toimsp(state->mboxname, state->mailbox->i.uidvalidity, "SEENsnn", state->userid,
0, state->mailbox->i.recenttime, 0);
}
else {
toimsp(state->mboxname, state->mailbox->i.uidvalidity, "SEENsnn", state->userid,
state->mailbox->last_uid, state->mailbox->i.recenttime, 0);
}
#endif
index_unlock(state);
return r;
}
/*
* Perform UID FETCH (VANISHED) on a sequence.
*/
struct seqset *index_vanished(struct index_state *state,
struct vanished_params *params)
{
struct mailbox *mailbox = state->mailbox;
struct seqset *outlist;
struct seqset *seq;
/* check uidvalidity match */
if (params->uidvalidity_is_max) {
if (params->uidvalidity < mailbox->i.uidvalidity) return NULL;
}
else {
if (params->uidvalidity != mailbox->i.uidvalidity) return NULL;
}
/* No recently expunged messages */
if (params->modseq >= state->highestmodseq) return NULL;
outlist = seqset_init(0, SEQ_SPARSE);
seq = _parse_sequence(state, params->sequence, 1);
/* XXX - use match_seq and match_uid */
if (params->modseq >= mailbox->i.deletedmodseq) {
const message_t *msg;
/* all records are significant */
/* List only expunged UIDs with MODSEQ > requested */
struct mailbox_iter *iter = mailbox_iter_init(mailbox, params->modseq, 0);
while ((msg = mailbox_iter_step(iter))) {
const struct index_record *record = msg_record(msg);
if (!(record->internal_flags & FLAG_INTERNAL_EXPUNGED))
continue;
if (!params->sequence || seqset_ismember(seq, record->uid))
seqset_add(outlist, record->uid, 1);
}
mailbox_iter_done(&iter);
}
else {
unsigned prevuid = 0;
struct seqset *msgnolist;
struct seqset *uidlist;
uint32_t msgno;
unsigned uid;
syslog(LOG_NOTICE, "inefficient qresync ("
MODSEQ_FMT " > " MODSEQ_FMT ") %s",
mailbox->i.deletedmodseq, params->modseq,
mailbox->name);
/* use the sequence to uid mapping provided by the client to
* skip over any initial matches - see RFC 5162 section 3.1 */
if (params->match_seq && params->match_uid) {
msgnolist = _parse_sequence(state, params->match_seq, 0);
uidlist = _parse_sequence(state, params->match_uid, 1);
while ((msgno = seqset_getnext(msgnolist)) != 0) {
uid = seqset_getnext(uidlist);
/* first non-match, we'll start here */
if (state->map[msgno-1].uid != uid)
break;
/* ok, they matched - so we can start after here */
prevuid = uid;
}
seqset_free(msgnolist);
seqset_free(uidlist);
}
const message_t *msg;
struct mailbox_iter *iter = mailbox_iter_init(mailbox, params->modseq, ITER_SKIP_EXPUNGED);
mailbox_iter_startuid(iter, prevuid);
/* possible efficiency improvement - use "seq_getnext" on seq
* to avoid incrementing through every single number for prevuid.
* Only really an issue if there's a giant block of thousands of
* expunged messages. Only likely to be seen in the wild if
* last_uid winds up being bumped up a few million by a bug... */
/* for the rest of the mailbox, we're just going to have to assume
* every record in the requested range which DOESN'T exist has been
* expunged, so build a complete sequence */
while ((msg = mailbox_iter_step(iter))) {
const struct index_record *record = msg_record(msg);
while (++prevuid < record->uid) {
if (!params->sequence || seqset_ismember(seq, prevuid))
seqset_add(outlist, prevuid, 1);
}
prevuid = record->uid;
}
mailbox_iter_done(&iter);
/* include the space past the final record up to last_uid as well */
while (++prevuid <= mailbox->i.last_uid) {
if (!params->sequence || seqset_ismember(seq, prevuid))
seqset_add(outlist, prevuid, 1);
}
}
seqset_free(seq);
return outlist;
}
static int _fetch_setseen(struct index_state *state,
struct mboxevent *mboxevent,
uint32_t msgno)
{
struct index_map *im = &state->map[msgno-1];
struct index_record record;
int r;
/* already seen */
if (im->isseen)
return 0;
/* no rights to change it */
if (!(state->myrights & ACL_SETSEEN))
return 0;
r = index_reload_record(state, msgno, &record);
if (r) return r;
/* track changes internally */
state->numunseen--;
state->seen_dirty = 1;
im->isseen = 1;
/* also store in the record if it's internal seen */
if (state->internalseen)
record.system_flags |= FLAG_SEEN;
/* need to bump modseq anyway, so always rewrite it */
r = index_rewrite_record(state, msgno, &record, /*silent*/0);
if (r) return r;
mboxevent_extract_record(mboxevent, state->mailbox, &record);
/* RFC2060 says:
* The \Seen flag is implicitly set; if this causes
* the flags to change they SHOULD be included as part
* of the FETCH responses. This is handled later by
* always including flags if the modseq has changed.
*/
return 0;
}
/* seq can be NULL - means "ALL" */
EXPORTED void index_fetchresponses(struct index_state *state,
struct seqset *seq,
int usinguid,
const struct fetchargs *fetchargs,
int *fetchedsomething)
{
uint32_t msgno, start, end;
struct index_map *im;
int fetched = 0;
annotate_db_t *annot_db = NULL;
/* Keep an open reference on the per-mailbox db to avoid
* doing too many slow database opens during the fetch */
if ((fetchargs->fetchitems & FETCH_ANNOTATION))
annotate_getdb(state->mboxname, &annot_db);
start = 1;
end = state->exists;
/* if we haven't told exists and we're fetching something past the end of the
* old size, we need to tell exists now...
* https://github.com/cyrusimap/cyrus-imapd/issues/1967
*/
if (state->exists != state->oldexists) index_tellexists(state);
/* if the modseq hasn't changed then there will be no unsolicited updates
* to send, so we only need to scan messages inside the sequence range.
* https://github.com/cyrusimap/cyrus-imapd/issues/1971
*/
if (state->oldhighestmodseq == state->highestmodseq) {
/* compress the search range down if a sequence was given */
if (seq) {
unsigned first = seqset_first(seq);
unsigned last = seqset_last(seq);
if (usinguid) {
if (first > 1)
start = index_finduid(state, first);
if (first == last)
end = start;
else if (last < state->last_uid)
end = index_finduid(state, last);
}
else {
start = first;
end = last;
}
}
}
/* make sure we didn't go outside the range! */
if (start < 1) start = 1;
if (end > state->exists) end = state->exists;
for (msgno = start; msgno <= end; msgno++) {
im = &state->map[msgno-1];
if (seq && !seqset_ismember(seq, usinguid ? im->uid : msgno)) {
if (im->told_modseq !=0 && im->modseq > im->told_modseq)
index_printflags(state, msgno, usinguid, 0);
continue;
}
if (index_fetchreply(state, msgno, fetchargs))
break;
fetched = 1;
}
/* Update oldhighestmodseq, ensuring we don't have unsolicited updates */
state->oldhighestmodseq = state->highestmodseq;
if (fetchedsomething) *fetchedsomething = fetched;
annotate_putdb(&annot_db);
}
/*
* Perform a FETCH-related command on a sequence.
* Fetchedsomething argument is 0 if nothing was fetched, 1 if something was
* fetched. (A fetch command that fetches nothing is not a valid fetch
* command.)
*/
EXPORTED int index_fetch(struct index_state *state,
const char *sequence,
int usinguid,
const struct fetchargs *fetchargs,
int *fetchedsomething)
{
struct seqset *seq;
struct seqset *vanishedlist = NULL;
struct index_map *im;
uint32_t msgno;
int r;
struct mboxevent *mboxevent = NULL;
r = index_lock(state, /*readonly*/0); // can't be readonly because of FETCH_SETSEEN
if (r) return r;
seq = _parse_sequence(state, sequence, usinguid);
/* set the \Seen flag if necessary - while we still have the lock */
if (fetchargs->fetchitems & FETCH_SETSEEN && !state->examining && state->myrights & ACL_SETSEEN) {
mboxevent = mboxevent_new(EVENT_MESSAGE_READ);
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
if (!seqset_ismember(seq, usinguid ? im->uid : msgno))
continue;
r = _fetch_setseen(state, mboxevent, msgno);
if (r) break;
}
mboxevent_extract_mailbox(mboxevent, state->mailbox);
mboxevent_set_access(mboxevent, NULL, NULL, state->userid, state->mailbox->name, 1);
mboxevent_set_numunseen(mboxevent, state->mailbox,
state->numunseen);
}
if (fetchargs->vanished) {
struct vanished_params v;
v.sequence = sequence;;
v.uidvalidity = state->mailbox->i.uidvalidity;
v.modseq = fetchargs->changedsince;
v.match_seq = fetchargs->match_seq;
v.match_uid = fetchargs->match_uid;
/* XXX - return error unless usinguid? */
vanishedlist = index_vanished(state, &v);
}
index_unlock(state);
/* send MessageRead event notification for successfully rewritten records */
mboxevent_notify(&mboxevent);
mboxevent_free(&mboxevent);
index_checkflags(state, 1, 0);
if (vanishedlist && vanishedlist->len) {
char *vanished = seqset_cstring(vanishedlist);
prot_printf(state->out, "* VANISHED (EARLIER) %s\r\n", vanished);
free(vanished);
}
seqset_free(vanishedlist);
index_fetchresponses(state, seq, usinguid, fetchargs, fetchedsomething);
seqset_free(seq);
index_tellchanges(state, usinguid, usinguid, 0);
return r;
}
/*
* Perform a STORE command on a sequence
*/
EXPORTED int index_store(struct index_state *state, char *sequence,
struct storeargs *storeargs)
{
struct mailbox *mailbox;
int i, r = 0;
uint32_t msgno;
int userflag;
struct seqset *seq;
struct index_map *im;
const strarray_t *flags = &storeargs->flags;
struct mboxevent *mboxevents = NULL;
struct mboxevent *flagsset = NULL, *flagsclear = NULL;
struct index_modified_flags modified_flags;
struct index_record record;
/* First pass at checking permission */
if ((storeargs->seen && !(state->myrights & ACL_SETSEEN)) ||
((storeargs->system_flags & FLAG_DELETED) &&
!(state->myrights & ACL_DELETEMSG)) ||
(((storeargs->system_flags & ~FLAG_DELETED) || flags->count) &&
!(state->myrights & ACL_WRITE))) {
return IMAP_PERMISSION_DENIED;
}
r = index_lock(state, /*readonly*/0);
if (r) return r;
mailbox = state->mailbox;
seq = _parse_sequence(state, sequence, storeargs->usinguid);
for (i = 0; i < flags->count ; i++) {
r = mailbox_user_flag(mailbox, flags->data[i], &userflag, 1);
if (r) goto out;
storeargs->user_flags[userflag/32] |= 1<<(userflag&31);
}
storeargs->update_time = time((time_t *)0);
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
if (!seqset_ismember(seq, storeargs->usinguid ? im->uid : msgno))
continue;
/* if it's expunged already, skip it now */
if ((im->internal_flags & FLAG_INTERNAL_EXPUNGED))
continue;
/* if it's changed already, skip it now */
if (im->modseq > storeargs->unchangedsince) {
if (!storeargs->modified) {
uint32_t maxval = (storeargs->usinguid ?
state->last_uid : state->exists);
storeargs->modified = seqset_init(maxval, SEQ_SPARSE);
}
seqset_add(storeargs->modified,
(storeargs->usinguid ? im->uid : msgno),
/*ismember*/1);
continue;
}
/* TODO(rsto): we need to keep index_reload_record here until we
* can make sure that the index map doesn't contain uncommitted
* changes for this msgno. See the comments in index_reload_record
* on how it releases the cyrus.index lock in the middle of action */
r = index_reload_record(state, msgno, &record);
if (r) goto out;
msgrecord_t *msgrec = msgrecord_from_index_record(state->mailbox, &record);
switch (storeargs->operation) {
case STORE_ADD_FLAGS:
case STORE_REMOVE_FLAGS:
case STORE_REPLACE_FLAGS:
r = index_storeflag(state, &modified_flags, msgno, msgrec, storeargs);
if (r) goto doneloop;
// nothing to do?
if (!(modified_flags.added_flags | modified_flags.removed_flags))
goto doneloop;
r = msgrecord_rewrite(msgrec);
if (r) goto doneloop;
if (modified_flags.added_flags) {
if (flagsset == NULL)
flagsset = mboxevent_enqueue(EVENT_FLAGS_SET, &mboxevents);
mboxevent_add_flags(flagsset, mailbox->flagname,
modified_flags.added_system_flags,
modified_flags.added_user_flags);
mboxevent_extract_msgrecord(flagsset, msgrec);
}
if (modified_flags.removed_flags) {
if (flagsclear == NULL)
flagsclear = mboxevent_enqueue(EVENT_FLAGS_CLEAR, &mboxevents);
mboxevent_add_flags(flagsclear, mailbox->flagname,
modified_flags.removed_system_flags,
modified_flags.removed_user_flags);
mboxevent_extract_msgrecord(flagsclear, msgrec);
}
break;
case STORE_ANNOTATION: {
int dirty = 0;
r = index_store_annotation(state, msgno, msgrec, storeargs, &dirty);
if (r) goto doneloop;
if (!dirty) goto doneloop;
// rewrite message
r = msgrecord_rewrite(msgrec);
if (r) goto doneloop;
// XXX mboxevents?
break;
}
default:
r = IMAP_INTERNAL;
break;
}
if (r) goto doneloop;
/* msgrecord_rewrite already took care of rewriting the index_record,
* but we want to stay up to date of the changes in the index_map.
* Pass the silent flag to index_rewrite_record. */
r = msgrecord_get_index_record(msgrec, &record);
if (r) goto doneloop;
r = index_rewrite_record(state, msgno, &record, /*silent*/1);
doneloop:
msgrecord_unref(&msgrec);
if (r) goto out;
}
/* let mboxevent_notify split FlagsSet into MessageRead, MessageTrash
* and FlagsSet events */
mboxevent_extract_mailbox(flagsset, mailbox);
mboxevent_set_numunseen(flagsset, mailbox, state->numunseen);
mboxevent_set_access(flagsset, NULL, NULL, state->userid, state->mailbox->name, 1);
mboxevent_extract_mailbox(flagsclear, mailbox);
mboxevent_set_access(flagsclear, NULL, NULL, state->userid, state->mailbox->name, 1);
mboxevent_set_numunseen(flagsclear, mailbox, state->numunseen);
mboxevent_notify(&mboxevents);
out:
mboxevent_freequeue(&mboxevents);
if (storeargs->operation == STORE_ANNOTATION && r)
annotate_state_abort(&mailbox->annot_state);
seqset_free(seq);
index_unlock(state);
index_tellchanges(state, storeargs->usinguid, storeargs->usinguid,
(storeargs->unchangedsince != ~0ULL));
return r;
}
static void prefetch_messages(struct index_state *state,
struct seqset *seq,
int usinguid)
{
struct mailbox *mailbox = state->mailbox;
struct index_map *im;
uint32_t msgno;
const char *fname;
struct index_record record;
syslog(LOG_ERR, "Prefetching initial parts of messages\n");
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
if (!seqset_ismember(seq, usinguid ? im->uid : msgno))
continue;
if (index_reload_record(state, msgno, &record))
continue;
fname = mailbox_record_fname(mailbox, &record);
if (!fname)
continue;
warmup_file(fname, 0, 16384);
}
}
/*
* Perform the XRUNANNOTATOR command which runs the
* annotator callout for each message in the given sequence.
*/
EXPORTED int index_run_annotator(struct index_state *state,
const char *sequence, int usinguid,
struct namespace *namespace, int isadmin)
{
struct index_record record;
struct seqset *seq = NULL;
struct index_map *im;
uint32_t msgno;
struct appendstate as;
msgrecord_t *msgrec = NULL;
int r = 0;
/* We do the acl check here rather than in append_setup_mbox()
* to account for the EXAMINE command where state->myrights has
* fewer rights than the ACL actually grants */
if (!(state->myrights & (ACL_WRITE|ACL_ANNOTATEMSG)))
return IMAP_PERMISSION_DENIED;
if (!config_getstring(IMAPOPT_ANNOTATION_CALLOUT))
return 0;
r = index_lock(state, /*readonly*/0);
if (r) return r;
r = append_setup_mbox(&as, state->mailbox,
state->userid, state->authstate,
0, NULL, namespace, isadmin, 0);
if (r) goto out;
seq = _parse_sequence(state, sequence, usinguid);
if (!seq) goto out;
prefetch_messages(state, seq, usinguid);
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
if (!seqset_ismember(seq, usinguid ? im->uid : msgno))
continue;
/* if it's expunged already, skip it now */
if ((im->internal_flags & FLAG_INTERNAL_EXPUNGED))
continue;
r = index_reload_record(state, msgno, &record);
if (r) goto out;
msgrec = msgrecord_from_index_record(state->mailbox, &record);
r = append_run_annotator(&as, msgrec);
if (r) goto out;
/* msgrecord_rewrite already took care of rewriting the index_record,
* but we want to stay up to date of the changes in the index_map.
* Pass the silent flag to index_rewrite_record. */
r = msgrecord_get_index_record(msgrec, &record);
if (r) goto out;
r = index_rewrite_record(state, msgno, &record, /*silent*/1);
if (r) goto out;
msgrecord_unref(&msgrec);
}
out:
seqset_free(seq);
if (msgrec) msgrecord_unref(&msgrec);
if (!r) {
r = append_commit(&as);
}
else {
append_abort(&as);
}
index_unlock(state);
index_tellchanges(state, usinguid, usinguid, 1);
return r;
}
EXPORTED int index_warmup(struct mboxlist_entry *mbentry,
unsigned int warmup_flags,
struct seqset *uids)
{
const char *fname = NULL;
char *userid = NULL;
char *tofree1 = NULL;
char *tofree2 = NULL;
unsigned int uid;
strarray_t files = STRARRAY_INITIALIZER;
int i;
int r = 0;
if (warmup_flags & WARMUP_INDEX) {
fname = mboxname_metapath(mbentry->partition, mbentry->name, mbentry->uniqueid, META_INDEX, 0);
r = warmup_file(fname, 0, 0);
if (r) goto out;
}
if (warmup_flags & WARMUP_CONVERSATIONS) {
if (config_getswitch(IMAPOPT_CONVERSATIONS)) {
fname = tofree1 = conversations_getmboxpath(mbentry->name);
r = warmup_file(fname, 0, 0);
if (r) goto out;
}
}
if (warmup_flags & WARMUP_ANNOTATIONS) {
fname = mboxname_metapath(mbentry->partition, mbentry->name, mbentry->uniqueid, META_ANNOTATIONS, 0);
r = warmup_file(fname, 0, 0);
if (r) goto out;
}
if (warmup_flags & WARMUP_SEARCH) {
userid = mboxname_to_userid(mbentry->name);
r = search_list_files(userid, &files);
if (r) goto out;
for (i = 0 ; i < files.count ; i++) {
fname = strarray_nth(&files, i);
r = warmup_file(fname, 0, 0);
if (r) goto out;
}
}
while ((uid = seqset_getnext(uids))) {
fname = mboxname_datapath(mbentry->partition, mbentry->name, mbentry->uniqueid, uid);
r = warmup_file(fname, 0, 0);
if (r) goto out;
}
out:
if (r == ENOENT || r == ENOSYS)
r = 0;
if (r)
syslog(LOG_ERR, "IOERROR: unable to warmup file %s: %s",
fname, error_message(r));
free(userid);
free(tofree1);
free(tofree2);
strarray_fini(&files);
return r;
}
static void build_query(search_builder_t *bx,
search_expr_t *e,
int remove,
int *nmatchesp)
{
search_expr_t *child;
int bop = -1;
switch (e->op) {
case SEOP_NOT:
bop = SEARCH_OP_NOT;
break;
case SEOP_AND:
bop = SEARCH_OP_AND;
break;
case SEOP_OR:
bop = SEARCH_OP_OR;
break;
case SEOP_FUZZYMATCH:
if (e->attr && e->attr->part >= 0) {
bx->match(bx, e->attr->part, e->value.s);
(*nmatchesp)++;
if (remove && e->attr->part != SEARCH_PART_HEADERS) {
/*
* We're relying on the search engine to correctly
* find matching messages, so we don't need to
* keep this node in the expression tree anymore.
* Rather than remove it we neuter it.
*/
search_expr_neutralise(e);
}
}
return;
default:
return;
}
if (e->children) {
assert(bop != -1);
bx->begin_boolean(bx, bop);
for (child = e->children ; child ; child = child->next)
build_query(bx, child, remove, nmatchesp);
bx->end_boolean(bx, bop);
}
}
static int index_prefilter_messages(unsigned* msg_list,
struct index_state *state,
struct searchargs *searchargs __attribute((unused)))
{
unsigned int msgno;
xstats_inc(SEARCH_TRIVIAL);
/* Just put in all possible messages. This falls back to Cyrus' default
* search. */
for (msgno = 1; msgno <= state->exists; msgno++)
msg_list[msgno-1] = msgno;
return state->exists;
}
static int index_scan_work(const char *s, unsigned long len,
const char *match, unsigned long min)
{
while (len > min) {
if (!strncasecmp(s, match, min)) return(1);
s++;
len--;
}
return(0);
}
/*
* Guts of the SCAN command, lifted from _index_search()
*
* Returns 1 if we get a hit, otherwise returns 0.
*/
EXPORTED int index_scan(struct index_state *state, const char *contents)
{
unsigned *msgno_list;
uint32_t msgno;
int n = 0;
int listindex;
int listcount;
struct searchargs searchargs;
unsigned long length;
struct mailbox *mailbox = state->mailbox;
charset_t ascii = charset_lookupname("US-ASCII");
if (!(contents && contents[0])) return(0);
if (index_check(state, 0, 0))
return 0;
if (state->exists <= 0) return 0;
length = strlen(contents);
memset(&searchargs, 0, sizeof(struct searchargs));
searchargs.root = search_expr_new(NULL, SEOP_MATCH);
searchargs.root->attr = search_attr_find("text");
/* Use US-ASCII to emulate fgrep */
searchargs.root->value.s = charset_convert(contents, ascii, charset_flags);
search_expr_internalise(state, searchargs.root);
msgno_list = (unsigned *) xmalloc(state->exists * sizeof(unsigned));
listcount = index_prefilter_messages(msgno_list, state, &searchargs);
for (listindex = 0; !n && listindex < listcount; listindex++) {
if (!(listindex % 128) && cmd_cancelled(/*insearch*/1))
break;
struct buf buf = BUF_INITIALIZER;
struct index_record record;
msgno = msgno_list[listindex];
if (index_reload_record(state, msgno, &record))
continue;
if (mailbox_map_record(mailbox, &record, &buf))
continue;
n += index_scan_work(buf.s, buf.len, contents, length);
buf_free(&buf);
}
charset_free(&ascii);
search_expr_free(searchargs.root);
free(msgno_list);
return n;
}
EXPORTED uint32_t index_getuid(struct index_state *state, uint32_t msgno)
{
assert(msgno <= state->exists);
return state->map[msgno-1].uid;
}
/* 'uid_list' is malloc'd string representing the hits from searchargs;
returns number of hits */
EXPORTED int index_getuidsequence(struct index_state *state,
struct searchargs *searchargs,
unsigned **uid_list)
{
search_query_t *query = NULL;
search_folder_t *folder = NULL;
int r;
int n = 0;
query = search_query_new(state, searchargs);
r = search_query_run(query);
if (r) goto out;
folder = search_query_find_folder(query, index_mboxname(state));
if (!folder) goto out;
n = search_folder_get_array(folder, uid_list);
out:
search_query_free(query);
return n;
}
static int index_lock(struct index_state *state, int readonly)
{
int r;
if (state->mailbox) {
if (state->examining || readonly) {
r = mailbox_lock_index(state->mailbox, LOCK_SHARED);
if (r) return r;
}
else {
r = mailbox_lock_index(state->mailbox, LOCK_EXCLUSIVE);
if (r) return r;
}
}
else {
if (state->examining || readonly) {
r = mailbox_open_irl(state->mboxname, &state->mailbox);
if (r) return r;
}
else {
r = mailbox_open_iwl(state->mboxname, &state->mailbox);
if (r) return r;
}
}
/* if the UIDVALIDITY has changed, treat as a delete */
if (state->mailbox->i.uidvalidity != state->uidvalidity) {
mailbox_close(&state->mailbox);
return IMAP_MAILBOX_NONEXISTENT;
}
/* if highestmodseq has changed or file is repacked, read updates */
if (state->highestmodseq != state->mailbox->i.highestmodseq
|| state->generation != state->mailbox->i.generation_no)
index_refresh_locked(state);
return 0;
}
EXPORTED int index_status(struct index_state *state, struct statusdata *sdata)
{
int r = index_lock(state, /*readonly*/1);
if (r) return r;
status_fill_mailbox(state->mailbox, sdata);
status_fill_seen(state->userid, sdata, state->numrecent, state->numunseen);
index_unlock(state);
return 0;
}
EXPORTED int index_refresh(struct index_state *state)
{
int r;
r = index_lock(state, /*readonly*/1); /* calls index_refresh_locked */
if (r) return r;
index_unlock(state);
return 0;
}
static void index_unlock(struct index_state *state)
{
// only update seen if we've got a writelocked mailbox
if (mailbox_index_islocked(state->mailbox, 1))
index_writeseen(state); // XXX: errors?
/* grab the latest modseq */
state->highestmodseq = state->mailbox->i.highestmodseq;
if (state->mailbox->i.dirty) {
struct statusdata sdata = STATUSDATA_INIT;
status_fill_mailbox(state->mailbox, &sdata);
// we zero out recent data for everyone else
status_fill_seen(state->userid, &sdata, /*recent*/0, state->numunseen);
mailbox_unlock_index(state->mailbox, &sdata);
}
else {
mailbox_unlock_index(state->mailbox, NULL);
}
}
/*
* RFC 4551 says:
* If client specifies a MODSEQ criterion in a SEARCH command
* and the server returns a non-empty SEARCH result, the server
* MUST also append (to the end of the untagged SEARCH response)
* the highest mod-sequence for all messages being returned.
*/
static int needs_modseq(const struct searchargs *searchargs,
const struct sortcrit *sortcrit)
{
int i;
if (search_expr_uses_attr(searchargs->root, "modseq"))
return 1;
if (sortcrit) {
for (i = 0 ; sortcrit[i].key != SORT_SEQUENCE ; i++)
if (sortcrit[i].key == SORT_MODSEQ)
return 1;
}
return 0;
}
/*
* Performs a SEARCH command.
* This is a wrapper around the search_query API which simply prints the results.
*/
EXPORTED int index_search(struct index_state *state,
struct searchargs *searchargs,
int usinguid)
{
search_query_t *query = NULL;
search_folder_t *folder;
int nmsg = 0;
int i;
modseq_t highestmodseq = 0;
int r;
/* update the index */
if (index_check(state, 0, 0))
return 0;
highestmodseq = needs_modseq(searchargs, NULL);
query = search_query_new(state, searchargs);
r = search_query_run(query);
if (r) goto out; /* search failed */
folder = search_query_find_folder(query, index_mboxname(state));
if (folder) {
if (!usinguid)
search_folder_use_msn(folder, state);
if (highestmodseq)
highestmodseq = search_folder_get_highest_modseq(folder);
nmsg = search_folder_get_count(folder);
}
else
nmsg = 0;
if (searchargs->returnopts) {
/*
* Implement RFC 4731 return options.
*/
prot_printf(state->out, "* ESEARCH");
if (searchargs->tag) {
prot_printf(state->out, " (TAG \"%s\")", searchargs->tag);
}
/* RFC4731: 3.1
* An extended UID SEARCH command MUST cause an ESEARCH response with
* the UID indicator present. */
if (usinguid) prot_printf(state->out, " UID");
if (nmsg) {
if (searchargs->returnopts & SEARCH_RETURN_MIN) {
prot_printf(state->out, " MIN %u", search_folder_get_min(folder));
if (highestmodseq && searchargs->returnopts == SEARCH_RETURN_MIN)
highestmodseq = search_folder_get_first_modseq(folder);
}
if (searchargs->returnopts & SEARCH_RETURN_MAX) {
prot_printf(state->out, " MAX %u", search_folder_get_max(folder));
if (highestmodseq && searchargs->returnopts == SEARCH_RETURN_MAX)
highestmodseq = search_folder_get_last_modseq(folder);
}
if (highestmodseq && searchargs->returnopts == (SEARCH_RETURN_MIN|SEARCH_RETURN_MAX)) {
/* special case min and max should be greatest of the two */
uint64_t last = search_folder_get_last_modseq(folder);
highestmodseq = search_folder_get_first_modseq(folder);
if (last > highestmodseq) highestmodseq = last;
}
if (searchargs->returnopts & SEARCH_RETURN_ALL) {
struct seqset *seq = search_folder_get_seqset(folder);
if (seq->len) {
char *str = seqset_cstring(seq);
prot_printf(state->out, " ALL %s", str);
free(str);
}
seqset_free(seq);
}
if (searchargs->returnopts & SEARCH_RETURN_RELEVANCY) {
prot_printf(state->out, " RELEVANCY (");
for (i = 0; i < nmsg; i++) {
if (i) prot_putc(' ', state->out);
/* for now all messages have relevancy=100 */
prot_printf(state->out, "%u", 100);
}
prot_printf(state->out, ")");
}
if (highestmodseq)
prot_printf(state->out, " MODSEQ " MODSEQ_FMT, highestmodseq);
}
if (searchargs->returnopts & SEARCH_RETURN_COUNT) {
prot_printf(state->out, " COUNT %u", nmsg);
}
}
else {
prot_printf(state->out, "* SEARCH");
if (nmsg) {
search_folder_foreach(folder, i) {
prot_printf(state->out, " %u", i);
}
}
if (highestmodseq)
prot_printf(state->out, " (MODSEQ " MODSEQ_FMT ")", highestmodseq);
}
prot_printf(state->out, "\r\n");
out:
search_query_free(query);
return nmsg;
}
/*
* Performs a SORT command
*/
EXPORTED int index_sort(struct index_state *state,
const struct sortcrit *sortcrit,
struct searchargs *searchargs, int usinguid)
{
int i;
int nmsg = 0;
modseq_t highestmodseq = 0;
search_query_t *query = NULL;
search_folder_t *folder = NULL;
int r;
/* update the index */
if (index_check(state, 0, 0))
return 0;
highestmodseq = needs_modseq(searchargs, NULL);
/* Search for messages based on the given criteria */
query = search_query_new(state, searchargs);
query->sortcrit = sortcrit;
r = search_query_run(query);
if (r) goto out; /* search failed */
folder = search_query_find_folder(query, index_mboxname(state));
if (folder) {
if (highestmodseq)
highestmodseq = search_folder_get_highest_modseq(folder);
nmsg = search_folder_get_count(folder);
}
prot_printf(state->out, "* SORT");
if (nmsg) {
/* Output the sorted messages */
for (i = 0 ; i < query->merged_msgdata.count ; i++) {
MsgData *md = ptrarray_nth(&query->merged_msgdata, i);
prot_printf(state->out, " %u",
(usinguid ? md->uid : md->msgno));
}
}
if (highestmodseq)
prot_printf(state->out, " (MODSEQ " MODSEQ_FMT ")", highestmodseq);
prot_printf(state->out, "\r\n");
out:
search_query_free(query);
return nmsg;
}
#define UNPREDICTABLE (-1)
static int search_predict_total(struct index_state *state,
struct conversations_state *cstate,
const struct searchargs *searchargs,
int conversations,
modseq_t *xconvmodseqp)
{
conv_status_t convstatus = CONV_STATUS_INIT;
uint32_t exists;
if (conversations) {
conversation_getstatus(cstate, index_mboxname(state), &convstatus);
/* always grab xconvmodseq, so we report a growing
* highestmodseq to all callers */
if (xconvmodseqp) *xconvmodseqp = convstatus.threadmodseq;
exists = convstatus.threadexists;
}
else {
if (xconvmodseqp) *xconvmodseqp = state->highestmodseq;
/* we may be in xconvupdates, where expunged are present */
exists = state->exists - state->num_expunged;
}
switch (search_expr_get_countability(searchargs->root)) {
case SEC_EXISTS:
return exists;
case SEC_EXISTS|SEC_NOT:
return 0;
/* we don't try to optimise searches on \Recent */
case SEC_SEEN:
assert(state->exists >= state->numunseen);
return state->exists - state->numunseen;
case SEC_SEEN|SEC_NOT:
return state->numunseen;
case SEC_CONVSEEN:
assert(conversations);
assert(convstatus.threadexists >= convstatus.threadunseen);
return convstatus.threadexists - convstatus.threadunseen;
case SEC_CONVSEEN|SEC_NOT:
assert(conversations);
return convstatus.threadunseen;
default:
return UNPREDICTABLE;
}
}
/*
* Performs a XCONVSORT command
*/
EXPORTED int index_convsort(struct index_state *state,
struct sortcrit *sortcrit,
struct searchargs *searchargs,
const struct windowargs *windowargs)
{
MsgData **msgdata = NULL;
unsigned int mi;
modseq_t xconvmodseq = 0;
int i;
hashu64_table seen_cids = HASHU64_TABLE_INITIALIZER;
uint32_t pos = 0;
int found_anchor = 0;
uint32_t anchor_pos = 0;
uint32_t first_pos = 0;
unsigned int ninwindow = 0;
ptrarray_t results = PTRARRAY_INITIALIZER;
int total = 0;
int r = 0;
struct conversations_state *cstate = NULL;
assert(windowargs);
assert(!windowargs->changedsince);
assert(!windowargs->upto);
/* Check the client didn't specify MULTIANCHOR. */
if (windowargs->anchor && windowargs->anchorfolder)
return IMAP_PROTOCOL_BAD_PARAMETERS;
/* make sure \Deleted messages are expunged. Will also lock the
* mailbox state and read any new information */
r = index_expunge(state, NULL, 1);
if (r) return r;
if (windowargs->conversations) {
cstate = conversations_get_mbox(index_mboxname(state));
if (!cstate)
return IMAP_INTERNAL;
}
search_expr_internalise(state, searchargs->root);
/* this works both with and without conversations */
total = search_predict_total(state, cstate, searchargs,
windowargs->conversations,
&xconvmodseq);
/* not going to match anything? bonus */
if (!total)
goto out;
construct_hashu64_table(&seen_cids, state->exists/4+4, 0);
/* Create/load the msgdata array.
* load data for ALL messages always. We sort before searching so
* we can take advantage of the window arguments to stop searching
* early */
msgdata = index_msgdata_load(state, NULL, state->exists, sortcrit,
windowargs->anchor, &found_anchor);
if (windowargs->anchor && !found_anchor) {
r = IMAP_ANCHOR_NOT_FOUND;
goto out;
}
/* Sort the messages based on the given criteria */
index_msgdata_sort(msgdata, state->exists, sortcrit);
/* One pass through the message list */
for (mi = 0 ; mi < state->exists ; mi++) {
MsgData *msg = msgdata[mi];
struct index_map *im = &state->map[msg->msgno-1];
/* can happen if we didn't "tellchanges" yet */
if (im->internal_flags & FLAG_INTERNAL_EXPUNGED)
continue;
/* run the search program against all messages */
if (!index_search_evaluate(state, searchargs->root, msg->msgno))
continue;
/* figure out whether this message is an exemplar */
if (windowargs->conversations) {
/* in conversations mode => only the first message seen
* with each unique CID is an exemplar */
if (hashu64_lookup(msg->cid, &seen_cids))
continue;
hashu64_insert(msg->cid, (void *)1, &seen_cids);
}
/* else not in conversations mode => all messages are exemplars */
pos++;
if (!anchor_pos &&
windowargs->anchor == msg->uid) {
/* we've found the anchor's position, rejoice! */
anchor_pos = pos;
}
if (windowargs->anchor) {
if (!anchor_pos)
continue;
if (pos < anchor_pos + windowargs->offset)
continue;
}
else if (windowargs->position) {
if (pos < windowargs->position)
continue;
}
if (windowargs->limit &&
++ninwindow > windowargs->limit) {
if (total == UNPREDICTABLE) {
/* the total was not predictable, so we need to keep
* going over the whole list to count it */
continue;
}
break;
}
if (!first_pos)
first_pos = pos;
ptrarray_append(&results, msg);
}
if (total == UNPREDICTABLE) {
/* the total was not predictable prima facie */
total = pos;
}
if (windowargs->anchor && !anchor_pos) {
/* the anchor was present but not an exemplar */
assert(results.count == 0);
r = IMAP_ANCHOR_NOT_FOUND;
goto out;
}
/* Print the resulting list */
/* Yes, we could use a seqset here, but apparently the most common
* sort order seen in the field is reverse date, which is basically
* the worst case for seqset. So we don't bother */
if (results.count) {
prot_printf(state->out, "* SORT"); /* uids */
for (i = 0 ; i < results.count ; i++) {
MsgData *msg = results.data[i];
prot_printf(state->out, " %u", msg->uid);
}
prot_printf(state->out, "\r\n");
}
out:
if (!r) {
if (first_pos)
prot_printf(state->out, "* OK [POSITION %u]\r\n", first_pos);
prot_printf(state->out, "* OK [HIGHESTMODSEQ " MODSEQ_FMT "]\r\n",
MAX(xconvmodseq, state->mailbox->i.highestmodseq));
prot_printf(state->out, "* OK [UIDVALIDITY %u]\r\n",
state->mailbox->i.uidvalidity);
prot_printf(state->out, "* OK [UIDNEXT %u]\r\n",
state->mailbox->i.last_uid + 1);
prot_printf(state->out, "* OK [TOTAL %u]\r\n",
total);
}
/* free all our temporary data */
index_msgdata_free(msgdata, state->exists);
ptrarray_fini(&results);
free_hashu64_table(&seen_cids, NULL);
return r;
}
/*
* Performs a XCONVMULTISORT command
*/
EXPORTED int index_convmultisort(struct index_state *state,
struct sortcrit *sortcrit,
struct searchargs *searchargs,
const struct windowargs *windowargs)
{
int mi;
int fi;
int i;
hashu64_table seen_cids = HASHU64_TABLE_INITIALIZER;
uint32_t pos = 0;
uint32_t anchor_pos = 0;
uint32_t first_pos = 0;
unsigned int ninwindow = 0;
/* array of (arrays of msgdata* with the same CID) */
ptrarray_t results = PTRARRAY_INITIALIZER;
/* Used as a placeholder which provides a non-NULL entry in the
* seen_cids hashtable for conversations which are outside the
* specified window. */
ptrarray_t dummy_response;
int total = UNPREDICTABLE;
int r = 0;
struct mboxname_counters counters;
search_query_t *query = NULL;
search_folder_t *folder = NULL;
search_folder_t *anchor_folder = NULL;
assert(windowargs);
assert(!windowargs->changedsince);
assert(!windowargs->upto);
/* Client needs to have specified MULTIANCHOR which includes
* the folder name instead of just ANCHOR. Check that here
* 'cos it's easier than doing so during parsing */
if (windowargs->anchor && !windowargs->anchorfolder)
return IMAP_PROTOCOL_BAD_PARAMETERS;
/* make sure folder still exists and map in data */
r = index_refresh(state);
if (r) return r;
r = mboxname_read_counters(index_mboxname(state), &counters);
if (r) return r;
query = search_query_new(state, searchargs);
query->multiple = 1;
query->need_ids = 1;
query->need_expunge = 1;
query->sortcrit = sortcrit;
r = search_query_run(query);
if (r) return r;
if (windowargs->anchorfolder) {
anchor_folder = search_query_find_folder(query, windowargs->anchorfolder);
if (!anchor_folder) {
r = IMAP_ANCHOR_NOT_FOUND;
goto out;
}
}
/* going to need to do conversation-level breakdown */
if (windowargs->conversations)
construct_hashu64_table(&seen_cids, query->merged_msgdata.count/4+4, 0);
/* no need */
else
total = query->merged_msgdata.count;
/* Another pass through the merged message list */
for (mi = 0 ; mi < query->merged_msgdata.count ; mi++) {
MsgData *md = ptrarray_nth(&query->merged_msgdata, mi);
ptrarray_t *response = NULL;
/* figure out whether this message is an exemplar */
if (windowargs->conversations) {
response = hashu64_lookup(md->cid, &seen_cids);
/* in conversations mode => only the first message seen
* with each unique CID is an exemplar */
if (response) {
if (response != &dummy_response)
ptrarray_append(response, md);
continue;
}
hashu64_insert(md->cid, &dummy_response, &seen_cids);
}
/* else not in conversations mode => all messages are exemplars */
pos++;
if (!anchor_pos &&
windowargs->anchor == md->uid &&
anchor_folder == md->folder) {
/* we've found the anchor's position, rejoice! */
anchor_pos = pos;
}
if (windowargs->anchor) {
if (!anchor_pos)
continue;
if (pos < anchor_pos + windowargs->offset)
continue;
}
else if (windowargs->position) {
if (pos < windowargs->position)
continue;
}
if (windowargs->limit &&
++ninwindow > windowargs->limit) {
if (total == UNPREDICTABLE) {
/* the total was not predictable, so we need to keep
* going over the whole list to count it */
continue;
}
break;
}
if (!first_pos)
first_pos = pos;
/* the message is the exemplar of a conversation which is inside
* the specified window, so record a non-dummy seen_cids entry
* and a results entry */
response = ptrarray_new();
ptrarray_push(response, md);
ptrarray_push(&results, response);
if (windowargs->conversations) {
hashu64_insert(md->cid, response, &seen_cids);
}
}
if (total == UNPREDICTABLE) {
/* the total was not predictable prima facie */
total = pos;
}
if (windowargs->anchor && !anchor_pos) {
/* the anchor was not found */
assert(results.count == 0);
r = IMAP_ANCHOR_NOT_FOUND;
goto out;
}
/* Print the resulting list */
xstats_add(SEARCH_RESULT, results.count);
if (results.count) {
/* The untagged response would be XCONVMULTISORT but
* Mail::IMAPTalk has an undocumented hack whereby any untagged
* response matching /sort/i is assumed to be a sequence of
* numeric uids. Meh. */
prot_printf(state->out, "* XCONVMULTI (");
for (fi = 0 ; fi < query->folders_by_id.count ; fi++) {
folder = ptrarray_nth(&query->folders_by_id, fi);
char *extname = mboxname_to_external(folder->mboxname, searchargs->namespace, searchargs->userid);
if (fi)
prot_printf(state->out, " ");
prot_printf(state->out, "(");
prot_printstring(state->out, extname);
prot_printf(state->out, " %u)", folder->uidvalidity);
free(extname);
}
prot_printf(state->out, ") (");
for (i = 0 ; i < results.count ; i++) {
ptrarray_t *response = ptrarray_nth(&results, i);
int j;
if (i)
prot_printf(state->out, " ");
for (j = 0; j < response->count; j++) {
MsgData *md = ptrarray_nth(response, j);
if (!j)
prot_printf(state->out, "(%s" , conversation_id_encode(md->cid));
prot_printf(state->out, " (%u %u)", md->folder->id, md->uid);
}
prot_printf(state->out, ")");
}
prot_printf(state->out, ")\r\n");
}
out:
if (!r) {
if (first_pos)
prot_printf(state->out, "* OK [POSITION %u]\r\n", first_pos);
prot_printf(state->out, "* OK [HIGHESTMODSEQ " MODSEQ_FMT "]\r\n",
counters.mailmodseq);
#if 0
prot_printf(state->out, "* OK [UIDNEXT %u]\r\n",
state->mailbox->i.last_uid + 1);
#endif
prot_printf(state->out, "* OK [TOTAL %u]\r\n",
total);
}
/* free all our temporary data */
free_hashu64_table(&seen_cids, NULL);
for (i = 0 ; i < results.count ; i++) {
ptrarray_t *response = ptrarray_nth(&results, i);
ptrarray_free(response);
}
ptrarray_fini(&results);
search_query_free(query);
return r;
}
struct snippet_rock {
struct protstream *out;
struct namespace *namespace;
const char *userid;
};
static int emit_snippet(struct mailbox *mailbox, uint32_t uid,
int part, const char *snippet, void *rock)
{
struct snippet_rock *sr = (struct snippet_rock *)rock;
const char *partname = search_part_as_string(part);
if (!partname) return 0;
char *extname = mboxname_to_external(mailbox->name, sr->namespace, sr->userid);
prot_printf(sr->out, "* SNIPPET ");
prot_printstring(sr->out, extname);
prot_printf(sr->out, " %u %u %s ", mailbox->i.uidvalidity, uid, partname);
prot_printstring(sr->out, snippet);
prot_printf(sr->out, "\r\n");
free(extname);
return 0;
}
EXPORTED int index_snippets(struct index_state *state,
const struct snippetargs *snippetargs,
struct searchargs *searchargs)
{
void *intquery = NULL;
search_builder_t *bx = NULL;
search_text_receiver_t *rx = NULL;
struct mailbox *mailbox = NULL;
int i;
int r = 0;
int nmatches = 0;
struct snippet_rock srock;
/* reload index */
r = index_refresh(state);
if (r) return r;
bx = search_begin_search(state->mailbox, SEARCH_MULTIPLE);
if (!bx) {
r = IMAP_INTERNAL;
goto out;
}
build_query(bx, searchargs->root, 0, &nmatches);
if (!bx->get_internalised) goto out;
intquery = bx->get_internalised(bx);
search_end_search(bx);
if (!intquery) goto out;
srock.out = state->out;
srock.namespace = searchargs->namespace;
srock.userid = searchargs->userid;
rx = search_begin_snippets(intquery, 0/*verbose*/, &default_snippet_markup,
emit_snippet, &srock);
if (!rx) goto out;
for ( ; snippetargs ; snippetargs = snippetargs->next) {
mailbox = NULL;
if (!strcmp(snippetargs->mboxname, index_mboxname(state))) {
mailbox = state->mailbox;
}
else {
r = mailbox_open_irl(snippetargs->mboxname, &mailbox);
if (r) goto out;
mailbox_unlock_index(mailbox, NULL);
}
if (snippetargs->uidvalidity &&
snippetargs->uidvalidity != mailbox->i.uidvalidity) {
r = IMAP_NOTFOUND;
goto out;
}
r = rx->begin_mailbox(rx, mailbox, /*incremental*/0);
for (i = 0 ; i < snippetargs->uids.count ; i++) {
uint32_t uid = snippetargs->uids.data[i];
struct index_record record;
/* It's OK to do a dirty read, because we only care about
* the UID of the message */
r = mailbox_find_index_record(mailbox, uid, &record);
if (r == IMAP_MAILBOX_CHECKSUM) r = 0;
if (r) continue;
if (state->m) message_set_from_record(mailbox, &record, state->m);
else state->m = message_new_from_record(mailbox, &record);
index_getsearchtext(state->m, NULL, rx, /*snippet*/1);
}
r = rx->end_mailbox(rx, mailbox);
if (r) goto out;
if (mailbox != state->mailbox)
mailbox_close(&mailbox);
}
out:
if (rx) search_end_snippets(rx);
if (intquery) search_free_internalised(intquery);
if (mailbox != state->mailbox)
mailbox_close(&mailbox);
return r;
}
static modseq_t get_modseq_of(MsgData *msg,
struct conversations_state *cstate)
{
modseq_t modseq = 0;
if (cstate) {
conversation_get_modseq(cstate, msg->cid, &modseq);
/* TODO: error handling dammit */
} else {
modseq = msg->modseq;
}
return modseq;
}
/*
* Performs a XCONVUPDATES command
*/
EXPORTED int index_convupdates(struct index_state *state,
struct sortcrit *sortcrit,
struct searchargs *searchargs,
const struct windowargs *windowargs)
{
MsgData **msgdata = NULL;
modseq_t xconvmodseq = 0;
unsigned int mi;
int i;
hashu64_table seen_cids = HASHU64_TABLE_INITIALIZER;
hashu64_table old_seen_cids = HASHU64_TABLE_INITIALIZER;
int32_t pos = 0;
uint32_t upto_pos = 0;
ptrarray_t added = PTRARRAY_INITIALIZER;
ptrarray_t removed = PTRARRAY_INITIALIZER;
ptrarray_t changed = PTRARRAY_INITIALIZER;
int total = 0;
struct conversations_state *cstate = NULL;
int is_mutable = search_is_mutable(sortcrit, searchargs);
int r = 0;
assert(windowargs);
assert(windowargs->changedsince);
assert(windowargs->offset == 0);
assert(!windowargs->position);
/* make sure \Deleted messages are expunged. Will also lock the
* mailbox state and read any new information */
r = index_expunge(state, NULL, 1);
if (r) return r;
if (windowargs->conversations) {
cstate = conversations_get_mbox(index_mboxname(state));
if (!cstate)
return IMAP_INTERNAL;
}
search_expr_internalise(state, searchargs->root);
total = search_predict_total(state, cstate, searchargs,
windowargs->conversations,
&xconvmodseq);
/* If there are no current and no expunged messages, we won't
* have any results at all and can short circuit the main loop;
* note that is a righter criterion than for XCONVSORT. */
if (!total && !state->exists)
goto out;
construct_hashu64_table(&seen_cids, state->exists/4+4, 0);
construct_hashu64_table(&old_seen_cids, state->exists/4+4, 0);
/* Create/load the msgdata array
* initial list - load data for ALL messages always */
msgdata = index_msgdata_load(state, NULL, state->exists, sortcrit, 0, NULL);
/* Sort the messages based on the given criteria */
index_msgdata_sort(msgdata, state->exists, sortcrit);
/* Discover exemplars */
for (mi = 0 ; mi < state->exists ; mi++) {
MsgData *msg = msgdata[mi];
struct index_map *im = &state->map[msg->msgno-1];
int was_old_exemplar = 0;
int is_new_exemplar = 0;
int is_deleted = 0;
int is_new = 0;
int was_deleted = 0;
int is_changed = 0;
int in_search = 0;
in_search = index_search_evaluate(state, searchargs->root, msg->msgno);
is_deleted = !!(im->internal_flags & FLAG_INTERNAL_EXPUNGED);
is_new = (im->uid >= windowargs->uidnext);
was_deleted = is_deleted && (im->modseq <= windowargs->modseq);
/* is this message a current exemplar? */
if (!is_deleted &&
in_search &&
(!windowargs->conversations || !hashu64_lookup(msg->cid, &seen_cids))) {
is_new_exemplar = 1;
pos++;
if (windowargs->conversations)
hashu64_insert(msg->cid, (void *)1, &seen_cids);
}
/* optimisation for when the total is
* not known but we've hit 'upto' */
if (upto_pos)
continue;
modseq_t modseq = get_modseq_of(msg, cstate);
is_changed = (modseq > windowargs->modseq);
/* was this message an old exemplar, or in the case of mutable
* searches, possible an old exemplar? */
if (!is_new &&
!was_deleted &&
(in_search || (is_mutable && is_changed)) &&
(!windowargs->conversations || !hashu64_lookup(msg->cid, &old_seen_cids))) {
was_old_exemplar = 1;
if (windowargs->conversations)
hashu64_insert(msg->cid, (void *)1, &old_seen_cids);
}
if (was_old_exemplar && !is_new_exemplar) {
ptrarray_push(&removed, msg);
} else if (!was_old_exemplar && is_new_exemplar) {
msg->msgno = pos; /* hacky: reuse ->msgno for pos */
ptrarray_push(&added, msg);
} else if (was_old_exemplar && is_new_exemplar) {
if (is_changed) {
ptrarray_push(&changed, msg);
if (is_mutable) {
/* is the search is mutable, we're in a whole world of
* uncertainty about the client's state, so we just
* report the exemplar in all three lists and let the
* client sort it out. */
ptrarray_push(&removed, msg);
msg->msgno = pos; /* hacky: reuse ->msgno for pos */
ptrarray_push(&added, msg);
}
}
}
/* if this is the last message the client cares about ('upto')
* then we can break early...unless its a mutable search or
* we need to keep going to calculate an accurate total */
if (!is_mutable &&
!upto_pos &&
msg->uid == windowargs->upto) {
if (total != UNPREDICTABLE)
break;
upto_pos = pos;
}
}
/* unlike 'anchor', the case of not finding 'upto' is not an error */
if (total == UNPREDICTABLE) {
/* the total was not predictable prima facie */
total = pos;
}
/* Print the resulting lists */
if (added.count) {
prot_printf(state->out, "* ADDED"); /* (uid pos) tuples */
for (i = 0 ; i < added.count ; i++) {
MsgData *msg = added.data[i];
prot_printf(state->out, " (%u %u)",
msg->uid, msg->msgno);
}
prot_printf(state->out, "\r\n");
}
if (removed.count) {
prot_printf(state->out, "* REMOVED"); /* uids */
for (i = 0 ; i < removed.count ; i++) {
MsgData *msg = removed.data[i];
prot_printf(state->out, " %u", msg->uid);
}
prot_printf(state->out, "\r\n");
}
if (changed.count) {
prot_printf(state->out, "* CHANGED"); /* cids or uids */
for (i = 0 ; i < changed.count ; i++) {
MsgData *msg = changed.data[i];
if (windowargs->conversations)
prot_printf(state->out, " %s",
conversation_id_encode(msg->cid));
else
prot_printf(state->out, " %u", msg->uid);
}
prot_printf(state->out, "\r\n");
}
out:
if (!r) {
prot_printf(state->out, "* OK [HIGHESTMODSEQ " MODSEQ_FMT "]\r\n",
MAX(xconvmodseq, state->mailbox->i.highestmodseq));
prot_printf(state->out, "* OK [UIDVALIDITY %u]\r\n",
state->mailbox->i.uidvalidity);
prot_printf(state->out, "* OK [UIDNEXT %u]\r\n",
state->mailbox->i.last_uid + 1);
prot_printf(state->out, "* OK [TOTAL %u]\r\n",
total);
}
/* free all our temporary data */
index_msgdata_free(msgdata, state->exists);
ptrarray_fini(&added);
ptrarray_fini(&removed);
ptrarray_fini(&changed);
free_hashu64_table(&seen_cids, NULL);
free_hashu64_table(&old_seen_cids, NULL);
return r;
}
/*
* Performs a THREAD command
*/
EXPORTED int index_thread(struct index_state *state, int algorithm,
struct searchargs *searchargs, int usinguid)
{
search_query_t *query = NULL;
search_folder_t *folder;
unsigned *msgno_list;
int nmsg = 0;
clock_t start;
modseq_t highestmodseq = 0;
int r;
/* update the index */
if (index_check(state, 0, 0))
return 0;
highestmodseq = needs_modseq(searchargs, NULL);
if(CONFIG_TIMING_VERBOSE)
start = clock();
/* Search for messages based on the given criteria */
query = search_query_new(state, searchargs);
r = search_query_run(query);
if (r) goto out; /* search failed */
folder = search_query_find_folder(query, index_mboxname(state));
if (folder) {
search_folder_use_msn(folder, state);
if (highestmodseq)
highestmodseq = search_folder_get_highest_modseq(folder);
nmsg = search_folder_get_array(folder, &msgno_list);
}
if (nmsg) {
/* Thread messages using given algorithm */
(*thread_algs[algorithm].threader)(state, msgno_list, nmsg, usinguid);
free(msgno_list);
if (highestmodseq)
prot_printf(state->out, " (MODSEQ " MODSEQ_FMT ")", highestmodseq);
}
/* print an empty untagged response */
else
index_thread_print(state, NULL, usinguid);
prot_printf(state->out, "\r\n");
if (CONFIG_TIMING_VERBOSE) {
/* debug */
syslog(LOG_DEBUG, "THREAD %s processing time: %d msg in %f sec",
thread_algs[algorithm].alg_name, nmsg,
(clock() - start) / (double) CLOCKS_PER_SEC);
}
out:
search_query_free(query);
return nmsg;
}
/*
* Performs a COPY command
*/
EXPORTED int
index_copy(struct index_state *state,
char *sequence,
int usinguid,
char *name,
char **copyuidp,
int nolink,
struct namespace *namespace,
int isadmin,
int ismove,
int ignorequota)
{
struct copyargs copyargs;
int i;
quota_t qdiffs[QUOTA_NUMRESOURCES] = QUOTA_DIFFS_INITIALIZER;
quota_t *qptr = NULL;
int r;
struct appendstate appendstate;
uint32_t msgno, checkval;
long docopyuid;
struct seqset *seq;
struct mailbox *srcmailbox = NULL;
struct mailbox *destmailbox = NULL;
struct index_map *im;
int is_same_user;
ptrarray_t *msgrecs = ptrarray_new();
*copyuidp = NULL;
memset(&copyargs, 0, sizeof(struct copyargs));
/* let's just see how common this is... */
if (!strcmp(index_mboxname(state), name))
syslog(LOG_NOTICE, "same mailbox copy %s (%s)", name, sequence);
is_same_user = mboxname_same_userid(index_mboxname(state), name);
if (is_same_user < 0)
return is_same_user;
r = index_check(state, usinguid, usinguid);
if (r) return r;
srcmailbox = state->mailbox;
seq = _parse_sequence(state, sequence, usinguid);
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
checkval = usinguid ? im->uid : msgno;
if (!seqset_ismember(seq, checkval))
continue;
index_copysetup(state, msgno, &copyargs);
}
seqset_free(seq);
if (copyargs.nummsg == 0) {
r = IMAP_NO_NOSUCHMSG;
goto done;
}
r = mailbox_open_iwl(name, &destmailbox);
if (r) goto done;
/* make sure copying into this mailbox is enabled */
r = insert_into_mailbox_allowed(destmailbox);
if (r) goto done;
/* not moving or different quota root - need to check quota */
if (!ismove || strcmpsafe(srcmailbox->quotaroot, destmailbox->quotaroot)) {
for (i = 0; i < copyargs.nummsg; i++)
qdiffs[QUOTA_STORAGE] += copyargs.records[i].size;
qdiffs[QUOTA_MESSAGE] = copyargs.nummsg;
qptr = qdiffs;
}
r = append_setup_mbox(&appendstate, destmailbox, state->userid,
state->authstate, ACL_INSERT,
ignorequota ? NULL : qptr, namespace, isadmin,
ismove ? EVENT_MESSAGE_MOVE : EVENT_MESSAGE_COPY);
if (r) goto done;
docopyuid = (appendstate.myrights & ACL_READ);
for (i = 0; i < copyargs.nummsg; i++) {
msgrecord_t *mr = msgrecord_from_index_record(srcmailbox, &copyargs.records[i]);
ptrarray_append(msgrecs, mr);
}
r = append_copy(srcmailbox, &appendstate, msgrecs, nolink, is_same_user);
if (r) {
append_abort(&appendstate);
goto done;
}
r = append_commit(&appendstate);
if (r) goto done;
/* unlock first so we don't hold the lock while expunging
* the source */
mailbox_unlock_index(destmailbox, NULL);
if (docopyuid || ismove) {
char *source;
struct seqset *seq;
unsigned uidvalidity = destmailbox->i.uidvalidity;
seq = seqset_init(0, SEQ_SPARSE);
for (i = 0; i < copyargs.nummsg; i++)
seqset_add(seq, copyargs.records[i].uid, 1);
source = seqset_cstring(seq);
/* remove the source messages */
if (ismove)
r = index_expunge(state, source, 0);
if (docopyuid) {
*copyuidp = xmalloc(strlen(source) + 50);
if (appendstate.nummsg == 1)
sprintf(*copyuidp, "%u %s %u", uidvalidity, source,
appendstate.baseuid);
else
sprintf(*copyuidp, "%u %s %u:%u", uidvalidity, source,
appendstate.baseuid,
appendstate.baseuid + appendstate.nummsg - 1);
}
free(source);
seqset_free(seq);
}
/* we log the first name to get GUID-copy magic */
if (!r)
sync_log_mailbox_double(index_mboxname(state), name);
done:
free(copyargs.records);
mailbox_close(&destmailbox);
for (i = 0; i < msgrecs->count; i++) {
msgrecord_t *mr = ptrarray_nth(msgrecs, i);
msgrecord_unref(&mr);
}
ptrarray_free(msgrecs);
return r;
}
/*
* Helper function to multiappend a message to remote mailbox
*/
static int index_appendremote(struct index_state *state, uint32_t msgno,
struct protstream *pout)
{
struct mailbox *mailbox = state->mailbox;
struct buf buf = BUF_INITIALIZER;
unsigned flag, flagmask = 0;
char datebuf[RFC3501_DATETIME_MAX+1];
char sepchar = '(';
struct index_record record;
int r;
r = index_reload_record(state, msgno, &record);
if (r) return r;
/* Open the message file */
if (mailbox_map_record(mailbox, &record, &buf))
return IMAP_NO_MSGGONE;
/* start the individual append */
prot_printf(pout, " ");
/* add system flags */
if (record.system_flags & FLAG_ANSWERED) {
prot_printf(pout, "%c\\Answered", sepchar);
sepchar = ' ';
}
if (record.system_flags & FLAG_FLAGGED) {
prot_printf(pout, "%c\\Flagged", sepchar);
sepchar = ' ';
}
if (record.system_flags & FLAG_DRAFT) {
prot_printf(pout, "%c\\Draft", sepchar);
sepchar = ' ';
}
if (record.system_flags & FLAG_DELETED) {
prot_printf(pout, "%c\\Deleted", sepchar);
sepchar = ' ';
}
if (record.system_flags & FLAG_SEEN) {
prot_printf(pout, "%c\\Seen", sepchar);
sepchar = ' ';
}
/* add user flags */
for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
if ((flag & 31) == 0) {
flagmask = record.user_flags[flag/32];
}
if (state->flagname[flag] && (flagmask & (1<<(flag & 31)))) {
prot_printf(pout, "%c%s", sepchar, state->flagname[flag]);
sepchar = ' ';
}
}
/* add internal date */
time_to_rfc3501(record.internaldate, datebuf, sizeof(datebuf));
prot_printf(pout, ") \"%s\" ", datebuf);
/* message literal */
index_fetchmsg(state, &buf, 0, record.size, 0, 0);
/* close the message file */
buf_free(&buf);
return 0;
}
/*
* Performs a COPY command from a local mailbox to a remote mailbox
*/
EXPORTED int index_copy_remote(struct index_state *state, char *sequence,
int usinguid, struct protstream *pout)
{
uint32_t msgno;
struct seqset *seq;
struct index_map *im;
int r;
r = index_check(state, usinguid, usinguid);
if (r) return r;
seq = _parse_sequence(state, sequence, usinguid);
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
if (!seqset_ismember(seq, usinguid ? im->uid : msgno))
continue;
index_appendremote(state, msgno, pout);
}
seqset_free(seq);
return 0;
}
/*
* Returns the msgno of the message with UID 'uid'.
* If no message with UID 'uid', returns the message with
* the highest UID not greater than 'uid'.
*/
EXPORTED uint32_t index_finduid(struct index_state *state, uint32_t uid)
{
unsigned low = 1;
unsigned high = state->exists;
unsigned mid;
unsigned miduid;
while (low <= high) {
mid = (high - low)/2 + low;
miduid = index_getuid(state, mid);
if (miduid == uid)
return mid;
else if (miduid > uid)
high = mid - 1;
else
low = mid + 1;
}
return high;
}
/* Helper function to determine domain of data */
enum {
DOMAIN_7BIT = 0,
DOMAIN_8BIT,
DOMAIN_BINARY
};
static int data_domain(const char *p, size_t n)
{
int r = DOMAIN_7BIT;
while (n--) {
if (!*p) return DOMAIN_BINARY;
if (*p & 0x80) r = DOMAIN_8BIT;
p++;
}
return r;
}
/*
* Helper function to fetch data from a message file. Writes a
* quoted-string or literal containing data from 'msg_base', which is
* of size 'msg_size', starting at 'offset' and containing 'size'
* octets. If 'octet_count' is nonzero, the data is
* further constrained by 'start_octet' and 'octet_count' as per the
* IMAP command PARTIAL.
*/
void index_fetchmsg(struct index_state *state, const struct buf *msg,
unsigned offset,
unsigned size, /* this is the correct size for a news message after
having LF translated to CRLF */
unsigned start_octet, unsigned octet_count)
{
unsigned n, domain;
/* If no data, output NIL */
if (!msg || !msg->s) {
prot_printf(state->out, "NIL");
return;
}
/* partial fetch: adjust 'size' */
if (octet_count) {
if (size <= start_octet) {
size = 0;
}
else {
size -= start_octet;
}
if (size > octet_count) size = octet_count;
}
/* If zero-length data, output empty quoted string */
if (size == 0) {
prot_printf(state->out, "\"\"");
return;
}
/* Seek over PARTIAL constraint */
offset += start_octet;
n = size;
if (offset + size > msg->len) {
if (msg->len > offset) {
n = msg->len - offset;
}
else {
prot_printf(state->out, "\"\"");
return;
}
}
/* Get domain of the data */
domain = data_domain(msg->s + offset, n);
if (domain == DOMAIN_BINARY) {
/* Write size of literal8 */
prot_printf(state->out, "~{%u}\r\n", size);
} else {
/* Write size of literal */
prot_printf(state->out, "{%u}\r\n", size);
}
/* Non-text literal -- tell the protstream about it */
if (domain != DOMAIN_7BIT) prot_data_boundary(state->out);
prot_write(state->out, msg->s + offset, n);
while (n++ < size) {
/* File too short, resynch client.
*
* This can only happen if the reported size of the part
* is incorrect and would push us past EOF.
*/
(void)prot_putc(' ', state->out);
}
/* End of non-text literal -- tell the protstream about it */
if (domain != DOMAIN_7BIT) prot_data_boundary(state->out);
}
static int body_is_rfc822(struct body *body)
{
return body &&
!strcasecmpsafe(body->type, "MESSAGE") &&
!strcasecmpsafe(body->subtype, "RFC822");
}
static struct body *find_part(struct body *body, int32_t part)
{
if (body_is_rfc822(body))
body = body->subpart;
else if (!body->numparts)
return NULL;
if (body->numparts) {
/* A multipart */
if (part >= body->numparts + 1)
return NULL;
body = body->subpart + part - 1;
}
else {
/* Every message has at least one part number. */
if (part > 1)
return NULL;
}
return body;
}
/*
* Helper function to fetch a body section
*/
static int index_fetchsection(struct index_state *state, const char *resp,
const struct buf *inmsg,
char *section, struct body *body, unsigned size,
unsigned start_octet, unsigned octet_count)
{
const char *p;
unsigned offset = 0;
char *decbuf = NULL;
struct buf msg = BUF_INITIALIZER;
struct body *top = body;
int wantheader = 0;
int32_t mimenum = 0;
int r;
p = section;
buf_init_ro(&msg, inmsg->s, inmsg->len);
/* Special-case BODY[] */
if (*p == ']') {
if (strstr(resp, "BINARY.SIZE")) {
prot_printf(state->out, "%s%u", resp, size);
} else {
prot_printf(state->out, "%s", resp);
index_fetchmsg(state, &msg, 0, size,
start_octet, octet_count);
}
return 0;
}
/*
section = "[" [section-spec] "]"
section-msgtext = "HEADER" / "HEADER.FIELDS" [".NOT"] SP header-list /
"TEXT"
; top-level or MESSAGE/RFC822 part
section-part = nz-number *("." nz-number)
; body part nesting
section-spec = section-msgtext / (section-part ["." section-text])
section-text = section-msgtext / "MIME"
; text other than actual body part (headers, etc.)
*/
while (*p != ']') {
switch (*p) {
case 'H':
if (!body_is_rfc822(body)) goto badpart;
body = body->subpart;
p += 6;
wantheader = 1;
goto emitpart;
case 'T':
if (!body_is_rfc822(body)) goto badpart;
body = body->subpart;
p += 4;
goto emitpart;
case 'M':
if (body == top) goto badpart;
p += 4;
wantheader = 1;
goto emitpart;
default:
mimenum = 0;
r = parseint32(p, &p, &mimenum);
if (*p == '.') p++;
if (r || !mimenum) goto badpart;
body = find_part(body, mimenum);
if (!body) goto badpart;
break;
}
}
emitpart:
if (*p != ']') goto badpart;
if (wantheader) {
offset = body->header_offset;
size = body->header_size;
}
else {
offset = body->content_offset;
size = body->content_size;
}
if (msg.s && !wantheader && (p = strstr(resp, "BINARY"))) {
/* BINARY or BINARY.SIZE */
int encoding = body->charset_enc & 0xff;
size_t newsize;
/* check that the offset isn't corrupt */
if (offset + size > msg.len) {
syslog(LOG_ERR, "invalid part offset in %s", index_mboxname(state));
return IMAP_IOERROR;
}
msg.s = (char *)charset_decode_mimebody(msg.s + offset, size, encoding,
&decbuf, &newsize);
if (!msg.s) {
/* failed to decode */
if (decbuf) free(decbuf);
return IMAP_NO_UNKNOWN_CTE;
}
else if (p[6] == '.') {
/* BINARY.SIZE */
prot_printf(state->out, "%s%zd", resp, newsize);
if (decbuf) free(decbuf);
return 0;
}
else {
/* BINARY */
offset = 0;
size = newsize;
msg.len = newsize;
}
}
/* Output body part */
prot_printf(state->out, "%s", resp);
index_fetchmsg(state, &msg, offset, size,
start_octet, octet_count);
if (decbuf) free(decbuf);
return 0;
badpart:
if (strstr(resp, "BINARY.SIZE"))
prot_printf(state->out, "%s0", resp);
else
prot_printf(state->out, "%sNIL", resp);
return 0;
}
/*
* Helper function to fetch a HEADER.FIELDS[.NOT] body section
*/
static void index_fetchfsection(struct index_state *state,
const char *msg_base,
unsigned long msg_size,
struct fieldlist *fsection,
struct body *body,
unsigned start_octet, unsigned octet_count)
{
const char *p;
int32_t mimenum = 0;
int fields_not = 0;
const char *crlf = "\r\n";
unsigned crlf_start = 0;
unsigned crlf_size = 2;
char *buf;
unsigned size;
int r;
/* If no data, output null quoted string */
if (!msg_base) {
prot_printf(state->out, "\"\"");
return;
}
p = fsection->section;
while (*p != 'H') {
r = parseint32(p, &p, &mimenum);
if (*p == '.') p++;
if (r || !mimenum) goto badpart;
body = find_part(body, mimenum);
if (!body) goto badpart;
}
if (body_is_rfc822(body)) body = body->subpart;
if (!body->header_size) goto badpart;
if (p[13]) fields_not++; /* Check for "." after "HEADER.FIELDS" */
buf = index_readheader(msg_base, msg_size,
body->header_offset,
body->header_size);
if (fields_not) {
message_pruneheader(buf, 0, fsection->fields);
}
else {
message_pruneheader(buf, fsection->fields, 0);
}
size = strlen(buf);
/* partial fetch: adjust 'size' */
if (octet_count) {
if (size <= start_octet) {
crlf_start = start_octet - size;
size = 0;
start_octet = 0;
if (crlf_size <= crlf_start) {
crlf_size = 0;
}
else {
crlf_size -= crlf_start;
}
}
else {
size -= start_octet;
}
if (size > octet_count) {
size = octet_count;
crlf_size = 0;
}
else if (size + crlf_size > octet_count) {
crlf_size = octet_count - size;
}
}
/* If no data, output null quoted string */
if (size + crlf_size == 0) {
prot_printf(state->out, "\"\"");
return;
}
/* Write literal */
prot_printf(state->out, "{%u}\r\n", size + crlf_size);
prot_write(state->out, buf + start_octet, size);
prot_write(state->out, crlf + crlf_start, crlf_size);
return;
badpart:
prot_printf(state->out, "NIL");
}
/*
* Helper function to read a header section into a static buffer
*/
static char *index_readheader(const char *msg_base, unsigned long msg_size,
unsigned offset, unsigned size)
{
static struct buf buf = BUF_INITIALIZER;
if (offset + size > msg_size) {
/* Message file is too short, truncate request */
if (offset < msg_size) {
size = msg_size - offset;
}
else {
size = 0;
}
}
buf_reset(&buf);
buf_appendmap(&buf, msg_base+offset, size);
return (char *)buf_cstring(&buf);
}
/*
* Handle a FETCH RFC822.HEADER.LINES or RFC822.HEADER.LINES.NOT
* that can't use the cacheheaders in cyrus.cache
*/
static void index_fetchheader(struct index_state *state,
const char *msg_base,
unsigned long msg_size,
unsigned size,
const strarray_t *headers,
const strarray_t *headers_not)
{
char *buf;
/* If no data, output null quoted string */
if (!msg_base) {
prot_printf(state->out, "\"\"");
return;
}
buf = index_readheader(msg_base, msg_size, 0, size);
message_pruneheader(buf, headers, headers_not);
size = strlen(buf);
prot_printf(state->out, "{%u}\r\n%s\r\n", size+2, buf);
}
/*
* Handle a FETCH RFC822.HEADER.LINES that can use the
* cacheheaders in cyrus.cache
*/
static void
index_fetchcacheheader(struct index_state *state, struct index_record *record,
const strarray_t *headers, unsigned start_octet,
unsigned octet_count)
{
static struct buf buf = BUF_INITIALIZER;
unsigned size;
const char *crlf = "\r\n";
unsigned crlf_start = 0;
unsigned crlf_size = 2;
struct mailbox *mailbox = state->mailbox;
if (mailbox_cacherecord(mailbox, record)) {
/* bogus cache record */
prot_printf(state->out, "\"\"");
return;
}
buf_setmap(&buf, cacheitem_base(record, CACHE_HEADERS),
cacheitem_size(record, CACHE_HEADERS));
buf_cstring(&buf);
message_pruneheader(buf.s, headers, 0);
size = strlen(buf.s); /* not buf.len, it has been pruned */
/* partial fetch: adjust 'size' */
if (octet_count) {
if (size <= start_octet) {
crlf_start = start_octet - size;
size = 0;
start_octet = 0;
if (crlf_size <= crlf_start) {
crlf_size = 0;
}
else {
crlf_size -= crlf_start;
}
}
else {
size -= start_octet;
}
if (size > octet_count) {
size = octet_count;
crlf_size = 0;
}
else if (size + crlf_size > octet_count) {
crlf_size = octet_count - size;
}
}
if (size + crlf_size == 0) {
prot_printf(state->out, "\"\"");
}
else {
prot_printf(state->out, "{%u}\r\n", size + crlf_size);
prot_write(state->out, buf.s + start_octet, size);
prot_write(state->out, crlf + crlf_start, crlf_size);
}
}
/*
* Send a * FLAGS response.
*/
static void index_listflags(struct index_state *state)
{
unsigned i;
int cancreate = 0;
char sepchar = '(';
prot_printf(state->out, "* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen");
for (i = 0; i < MAX_USER_FLAGS; i++) {
if (state->flagname[i]) {
prot_printf(state->out, " %s", state->flagname[i]);
}
else cancreate++;
}
prot_printf(state->out, ")\r\n* OK [PERMANENTFLAGS ");
if (!state->examining) {
if (state->myrights & ACL_WRITE) {
prot_printf(state->out, "%c\\Answered \\Flagged \\Draft", sepchar);
sepchar = ' ';
}
if (state->myrights & ACL_DELETEMSG) {
prot_printf(state->out, "%c\\Deleted", sepchar);
sepchar = ' ';
}
if (state->myrights & ACL_SETSEEN) {
prot_printf(state->out, "%c\\Seen", sepchar);
sepchar = ' ';
}
if (state->myrights & ACL_WRITE) {
for (i = 0; i < MAX_USER_FLAGS; i++) {
if (state->flagname[i]) {
prot_printf(state->out, " %s", state->flagname[i]);
}
}
if (cancreate) {
prot_printf(state->out, " \\*");
}
}
}
if (sepchar == '(') prot_printf(state->out, "(");
prot_printf(state->out, ")] Ok\r\n");
}
EXPORTED void index_checkflags(struct index_state *state, int print, int dirty)
{
struct mailbox *mailbox = state->mailbox;
unsigned i;
for (i = 0; i < MAX_USER_FLAGS; i++) {
/* both empty */
if (!mailbox->flagname[i] && !state->flagname[i])
continue;
/* both same */
if (mailbox->flagname[i] && state->flagname[i] &&
!strcmp(mailbox->flagname[i], state->flagname[i]))
continue;
/* ok, got something to change! */
if (state->flagname[i])
free(state->flagname[i]);
if (mailbox->flagname[i])
state->flagname[i] = xstrdup(mailbox->flagname[i]);
else
state->flagname[i] = NULL;
dirty = 1;
}
if (dirty && print)
index_listflags(state);
}
static void index_tellexpunge(struct index_state *state)
{
unsigned oldmsgno;
uint32_t msgno = 1;
struct seqset *vanishedlist;
struct index_map *im;
unsigned exists = state->exists;
// if we want expunged, we can't tell them!
if (state->want_expunged) return;
vanishedlist = seqset_init(0, SEQ_SPARSE);
for (oldmsgno = 1; oldmsgno <= exists; oldmsgno++) {
im = &state->map[oldmsgno-1];
/* inform about expunges */
if (im->internal_flags & FLAG_INTERNAL_EXPUNGED) {
state->exists--;
state->num_expunged--;
/* they never knew about this one, skip */
if (msgno > state->oldexists)
continue;
state->oldexists--;
if ((client_capa & CAPA_QRESYNC))
seqset_add(vanishedlist, im->uid, 1);
else
prot_printf(state->out, "* %u EXPUNGE\r\n", msgno);
continue;
}
/* copy back if necessary (after first expunge) */
if (msgno < oldmsgno)
state->map[msgno-1] = *im;
msgno++;
}
/* report all vanished if we're doing it this way */
if (vanishedlist->len) {
char *vanished = seqset_cstring(vanishedlist);
prot_printf(state->out, "* VANISHED %s\r\n", vanished);
free(vanished);
}
seqset_free(vanishedlist);
/* highestmodseq can now come forward to real-time */
state->delayed_modseq = 0;
}
static void index_tellexists(struct index_state *state)
{
prot_printf(state->out, "* %u EXISTS\r\n", state->exists);
prot_printf(state->out, "* %u RECENT\r\n", state->numrecent);
state->oldexists = state->exists;
}
EXPORTED void index_tellchanges(struct index_state *state, int canexpunge,
int printuid, int printmodseq)
{
uint32_t msgno;
struct index_map *im;
/* must call tellexpunge before tellexists, because tellexpunge changes
* the size of oldexists to mention expunges, and tellexists will reset
* oldexists. If we do these out of order, we will tell the user about
* expunges of messages they never saw, which would be wrong */
if (canexpunge) index_tellexpunge(state);
if (state->oldexists != state->exists) index_tellexists(state);
if (state->oldhighestmodseq == state->highestmodseq) return;
index_checkflags(state, 1, 0);
/* print any changed message flags */
for (msgno = 1; msgno <= state->exists; msgno++) {
im = &state->map[msgno-1];
/* report if it's changed since last told */
if (im->modseq > im->told_modseq)
index_printflags(state, msgno, printuid, printmodseq);
}
}
struct fetch_annotation_rock {
struct protstream *pout;
const char *sep;
};
static void fetch_annotation_response(const char *mboxname
__attribute__((unused)),
uint32_t uid
__attribute__((unused)),
const char *entry,
struct attvaluelist *attvalues,
void *rock)
{
char sep2 = '(';
struct attvaluelist *l;
struct fetch_annotation_rock *frock = rock;
prot_printf(frock->pout, "%s", frock->sep);
prot_printastring(frock->pout, entry);
prot_putc(' ', frock->pout);
for (l = attvalues ; l ; l = l->next) {
prot_putc(sep2, frock->pout);
sep2 = ' ';
prot_printastring(frock->pout, l->attrib);
prot_putc(' ', frock->pout);
prot_printmap(frock->pout, l->value.s, l->value.len);
}
prot_putc(')', frock->pout);
frock->sep = " ";
}
/*
* Helper function to send FETCH data for the ANNOTATION
* fetch item.
*/
static int index_fetchannotations(struct index_state *state,
uint32_t msgno,
const struct fetchargs *fetchargs)
{
annotate_state_t *astate = NULL;
struct fetch_annotation_rock rock;
int r = 0;
r = mailbox_get_annotate_state(state->mailbox,
state->map[msgno-1].uid,
&astate);
if (r) return r;
annotate_state_set_auth(astate, fetchargs->isadmin,
fetchargs->userid, fetchargs->authstate);
memset(&rock, 0, sizeof(rock));
rock.pout = state->out;
rock.sep = "";
r = annotate_state_fetch(astate,
&fetchargs->entries, &fetchargs->attribs,
fetch_annotation_response, &rock);
return r;
}
struct fetch_mailbox_rock {
struct index_state *state;
int sep;
int wantname;
const struct fetchargs *fetchargs;
};
static int fetch_mailbox_cb(const conv_guidrec_t *rec, void *rock)
{
static const int needrights = ACL_READ|ACL_LOOKUP;
struct fetch_mailbox_rock *fmb_rock = (struct fetch_mailbox_rock *) rock;
mbentry_t *mbentry = NULL;
int myrights = 0;
struct mailbox *mailbox = NULL;
msgrecord_t *msgrecord = NULL;
char *extname = NULL;
int r = 0;
assert(fmb_rock->state != NULL);
assert(fmb_rock->fetchargs != NULL);
/* convdb has flags: skip if flag deleted or flag internal expunged */
if (rec->version >= 1) {
if ((rec->system_flags & FLAG_DELETED)
|| (rec->internal_flags & FLAG_INTERNAL_EXPUNGED))
goto done;
}
/* make sure we have appropriate rights */
r = mboxlist_lookup(rec->mboxname, &mbentry, NULL);
if (r) goto done;
myrights = cyrus_acl_myrights(fmb_rock->state->authstate, mbentry->acl);
if ((myrights & needrights) != needrights)
goto done;
/* convdb does not have flags: grab them from message record */
if (rec->version == 0) {
uint32_t system_flags, internal_flags;
r = mailbox_open_irl(rec->mboxname, &mailbox);
if (r) goto done;
r = msgrecord_find(mailbox, rec->uid, &msgrecord);
if (r) goto done;
r = msgrecord_get_systemflags(msgrecord, &system_flags);
if (!r) r = msgrecord_get_internalflags(msgrecord, &internal_flags);
if (r) goto done;
if ((system_flags & FLAG_DELETED)
|| (internal_flags & FLAG_INTERNAL_EXPUNGED))
goto done;
}
if (fmb_rock->wantname) {
extname = mboxname_to_external(rec->mboxname,
fmb_rock->fetchargs->namespace,
fmb_rock->fetchargs->userid);
}
if (fmb_rock->sep)
prot_putc(fmb_rock->sep, fmb_rock->state->out);
prot_printf(fmb_rock->state->out, "%s",
fmb_rock->wantname ? extname : mbentry->uniqueid);
fmb_rock->sep = ' ';
done:
if (extname) free(extname);
if (msgrecord) msgrecord_unref(&msgrecord);
if (mailbox) mailbox_close(&mailbox);
if (mbentry) mboxlist_entry_free(&mbentry);
return r;
}
/*
* Helper function to send FETCH data for the MAILBOXES
* fetch item.
*/
static int index_fetchmailboxes(struct index_state *state,
uint32_t msgno,
const struct fetchargs *fetchargs)
{
struct fetch_mailbox_rock rock = { state, 0, /*wantname*/ 1, fetchargs};
struct index_record record;
int r;
if (!fetchargs->convstate) return 0;
r = index_reload_record(state, msgno, &record);
if (r) return r;
return conversations_guid_foreach(fetchargs->convstate,
message_guid_encode(&record.guid),
&fetch_mailbox_cb,
&rock);
}
/*
* Helper function to send FETCH data for the MAILBOXIDS
* fetch item.
*/
static int index_fetchmailboxids(struct index_state *state,
uint32_t msgno,
const struct fetchargs *fetchargs)
{
struct fetch_mailbox_rock rock = { state, 0, /*wantname*/ 0, fetchargs};
struct index_record record;
int r;
if (!fetchargs->convstate) return 0;
r = index_reload_record(state, msgno, &record);
if (r) return r;
return conversations_guid_foreach(fetchargs->convstate,
message_guid_encode(&record.guid),
&fetch_mailbox_cb,
&rock);
}
/*
* Helper function to send * FETCH (FLAGS data.
* Does not send the terminating close paren or CRLF.
* Also sends preceeding * FLAGS if necessary.
*/
static void index_fetchflags(struct index_state *state,
uint32_t msgno)
{
int sepchar = '(';
unsigned flag;
bit32 flagmask = 0;
struct index_map *im = &state->map[msgno-1];
prot_printf(state->out, "* %u FETCH (FLAGS ", msgno);
if (im->isrecent) {
prot_printf(state->out, "%c\\Recent", sepchar);
sepchar = ' ';
}
if (im->system_flags & FLAG_ANSWERED) {
prot_printf(state->out, "%c\\Answered", sepchar);
sepchar = ' ';
}
if (im->system_flags & FLAG_FLAGGED) {
prot_printf(state->out, "%c\\Flagged", sepchar);
sepchar = ' ';
}
if (im->system_flags & FLAG_DRAFT) {
prot_printf(state->out, "%c\\Draft", sepchar);
sepchar = ' ';
}
if (im->system_flags & FLAG_DELETED) {
prot_printf(state->out, "%c\\Deleted", sepchar);
sepchar = ' ';
}
if (state->want_expunged && (im->internal_flags & FLAG_INTERNAL_EXPUNGED)) {
prot_printf(state->out, "%c\\Expunged", sepchar);
sepchar = ' ';
}
if (im->isseen) {
prot_printf(state->out, "%c\\Seen", sepchar);
sepchar = ' ';
}
for (flag = 0; flag < VECTOR_SIZE(state->flagname); flag++) {
if ((flag & 31) == 0) {
flagmask = im->user_flags[flag/32];
}
if (state->flagname[flag] && (flagmask & (1<<(flag & 31)))) {
prot_printf(state->out, "%c%s", sepchar, state->flagname[flag]);
sepchar = ' ';
}
}
if (sepchar == '(') (void)prot_putc('(', state->out);
(void)prot_putc(')', state->out);
im->told_modseq = im->modseq;
}
static void index_printflags(struct index_state *state,
uint32_t msgno, int usinguid,
int printmodseq)
{
struct index_map *im = &state->map[msgno-1];
index_fetchflags(state, msgno);
/* http://www.rfc-editor.org/errata_search.php?rfc=5162
* Errata ID: 1807 - MUST send UID and MODSEQ to all
* untagged FETCH unsolicited responses */
if (usinguid || (client_capa & CAPA_QRESYNC))
prot_printf(state->out, " UID %u", im->uid);
if (printmodseq || (client_capa & CAPA_CONDSTORE))
prot_printf(state->out, " MODSEQ (" MODSEQ_FMT ")", im->modseq);
prot_printf(state->out, ")\r\n");
}
/* interface message_read_bodystructure which makes sure the cache record
* exists and adds the MESSAGE/RFC822 wrapper to make fetch BODY[*]
* work consistently */
static void loadbody(struct mailbox *mailbox, struct index_record *record,
struct body **bodyp)
{
if (*bodyp) return;
if (mailbox_cacherecord(mailbox, record)) return;
struct body *body = xzmalloc(sizeof(struct body));
message_read_bodystructure(record, &body->subpart);
body->type = xstrdup("MESSAGE");
body->subtype = xstrdup("RFC822");
body->header_offset = 0;
body->header_size = 0;
body->content_offset = 0;
body->content_offset = record->size;
*bodyp = body;
}
/*
* Helper function to send requested * FETCH data for a message
*/
static int index_fetchreply(struct index_state *state, uint32_t msgno,
const struct fetchargs *fetchargs)
{
struct mailbox *mailbox = state->mailbox;
int fetchitems = fetchargs->fetchitems;
struct buf buf = BUF_INITIALIZER;
struct octetinfo *oi = NULL;
int sepchar = '(';
int started = 0;
struct section *section;
struct fieldlist *fsection;
char respbuf[100];
int r = 0;
struct index_map *im = &state->map[msgno-1];
struct index_record record;
struct body *body = NULL;
/* Check the modseq against changedsince */
if (fetchargs->changedsince && im->modseq <= fetchargs->changedsince)
return 0;
/* skip missing records entirely */
if (!im->recno)
return 0;
r = index_reload_record(state, msgno, &record);
if (r) {
prot_printf(state->out, "* OK ");
prot_printf(state->out, error_message(IMAP_NO_MSGGONE), msgno);
prot_printf(state->out, "\r\n");
return 0;
}
/* Check against the CID list filter */
if (fetchargs->cidhash) {
const char *key = conversation_id_encode(record.cid);
if (!hash_lookup(key, fetchargs->cidhash))
return 0;
}
/* Open the message file if we're going to need it */
if ((fetchitems & (FETCH_HEADER|FETCH_TEXT|FETCH_SHA1|FETCH_RFC822)) ||
fetchargs->cache_atleast > record.cache_version ||
fetchargs->binsections || fetchargs->sizesections ||
fetchargs->bodysections) {
if (mailbox_map_record(mailbox, &record, &buf)) {
prot_printf(state->out, "* OK ");
prot_printf(state->out, error_message(IMAP_NO_MSGGONE), msgno);
prot_printf(state->out, "\r\n");
return 0;
}
}
int ischanged = im->told_modseq < record.modseq;
/* display flags if asked _OR_ if they've changed */
if (fetchitems & FETCH_FLAGS || ischanged) {
index_fetchflags(state, msgno);
sepchar = ' ';
}
else if ((fetchitems & ~FETCH_SETSEEN) || fetchargs->fsections ||
fetchargs->headers.count || fetchargs->headers_not.count) {
/* these fetch items will always succeed, so start the response */
prot_printf(state->out, "* %u FETCH ", msgno);
started = 1;
}
if (fetchitems & FETCH_UID || (ischanged && (client_capa & CAPA_QRESYNC))) {
prot_printf(state->out, "%cUID %u", sepchar, record.uid);
sepchar = ' ';
}
if (fetchitems & FETCH_GUID) {
prot_printf(state->out, "%cDIGEST.SHA1 %s", sepchar,
message_guid_encode(&record.guid));
sepchar = ' ';
}
if (fetchitems & FETCH_INTERNALDATE) {
time_t msgdate = record.internaldate;
char datebuf[RFC3501_DATETIME_MAX+1];
time_to_rfc3501(msgdate, datebuf, sizeof(datebuf));
prot_printf(state->out, "%cINTERNALDATE \"%s\"",
sepchar, datebuf);
sepchar = ' ';
}
if (fetchitems & FETCH_LASTUPDATED) {
time_t msgdate = record.last_updated;
char datebuf[RFC3501_DATETIME_MAX+1];
time_to_rfc3501(msgdate, datebuf, sizeof(datebuf));
prot_printf(state->out, "%cLASTUPDATED \"%s\"",
sepchar, datebuf);
sepchar = ' ';
}
if (fetchitems & FETCH_MODSEQ || (ischanged && (client_capa & CAPA_CONDSTORE))) {
prot_printf(state->out, "%cMODSEQ (" MODSEQ_FMT ")",
sepchar, record.modseq);
sepchar = ' ';
}
if (fetchitems & FETCH_SIZE) {
prot_printf(state->out, "%cRFC822.SIZE %u",
sepchar, record.size);
sepchar = ' ';
}
if ((fetchitems & FETCH_ANNOTATION)) {
prot_printf(state->out, "%cANNOTATION (", sepchar);
r = index_fetchannotations(state, msgno, fetchargs);
r = 0;
prot_printf(state->out, ")");
sepchar = ' ';
}
if (fetchitems & FETCH_PREVIEW) {
prot_printf(state->out, "%cPREVIEW (FUZZY ", sepchar);
const char *annot = config_getstring(IMAPOPT_JMAP_PREVIEW_ANNOT);
if (annot && !strncmp(annot, "/shared/", 8)) {
struct buf previewbuf = BUF_INITIALIZER;
annotatemore_msg_lookup(mailbox->name, record.uid, annot+7,
/*userid*/"", &previewbuf);
if (buf_len(&previewbuf) > 256)
buf_truncate(&previewbuf, 256); // XXX - utf8 chars
prot_printastring(state->out, buf_cstring(&previewbuf));
buf_free(&previewbuf);
}
else {
prot_printf(state->out, "NIL");
}
prot_printf(state->out, ")");
sepchar = ' ';
}
if (fetchitems & FETCH_FILESIZE) {
unsigned int msg_size = buf.len;
if (!buf.s) {
const char *fname = mailbox_record_fname(mailbox, &record);
struct stat sbuf;
/* Find the size of the message file */
if (stat(fname, &sbuf) == -1)
syslog(LOG_ERR, "IOERROR: stat on %s: %m", fname);
else
msg_size = sbuf.st_size;
}
prot_printf(state->out, "%cRFC822.FILESIZE %lu", sepchar,
(long unsigned)msg_size);
sepchar = ' ';
}
if (fetchitems & FETCH_SHA1) {
struct message_guid tmpguid;
message_guid_generate(&tmpguid, buf.s, buf.len);
prot_printf(state->out, "%cRFC822.SHA1 %s", sepchar, message_guid_encode(&tmpguid));
sepchar = ' ';
}
if (fetchitems & FETCH_EMAILID) {
char emailid[26];
emailid[0] = 'M';
memcpy(emailid+1, message_guid_encode(&record.guid), 24);
emailid[25] = '\0';
prot_printf(state->out, "%cEMAILID (%s)", sepchar, emailid);
sepchar = ' ';
}
if ((fetchitems & FETCH_CID) &&
config_getswitch(IMAPOPT_CONVERSATIONS)) {
struct buf buf = BUF_INITIALIZER;
if (!record.cid)
buf_appendcstr(&buf, "NIL");
else
buf_printf(&buf, CONV_FMT, record.cid);
prot_printf(state->out, "%cCID %s", sepchar, buf_cstring(&buf));
buf_free(&buf);
sepchar = ' ';
}
if ((fetchitems & FETCH_THREADID)) {
char threadid[18];
if (!record.cid) {
threadid[0] = 'N';
threadid[1] = 'I';
threadid[2] = 'L';
threadid[3] = '\0';
}
else {
threadid[0] = 'T';
memcpy(threadid+1, conversation_id_encode(record.cid), 16);
threadid[17] = '\0';
}
prot_printf(state->out, "%cTHREADID (%s)", sepchar, threadid);
sepchar = ' ';
}
if (fetchitems & FETCH_SAVEDATE) {
time_t msgdate = record.savedate;
char datebuf[RFC3501_DATETIME_MAX+1];
// handle internaldate
if (!msgdate) msgdate = record.internaldate;
time_to_rfc3501(msgdate, datebuf, sizeof(datebuf));
prot_printf(state->out, "%cSAVEDATE \"%s\"",
sepchar, datebuf);
sepchar = ' ';
}
if (fetchitems & FETCH_CREATEDMODSEQ) {
prot_printf(state->out, "%cCREATEDMODSEQ (" MODSEQ_FMT ")",
sepchar, record.createdmodseq);
sepchar = ' ';
}
if ((fetchitems & FETCH_BASECID) &&
config_getswitch(IMAPOPT_CONVERSATIONS)) {
mailbox_read_basecid(mailbox, &record);
struct buf buf = BUF_INITIALIZER;
if (!record.basecid)
buf_appendcstr(&buf, "NIL");
else
buf_printf(&buf, CONV_FMT, record.basecid);
prot_printf(state->out, "%cBASECID %s", sepchar, buf_cstring(&buf));
buf_free(&buf);
sepchar = ' ';
}
if ((fetchitems & FETCH_FOLDER)) {
char *extname = mboxname_to_external(index_mboxname(state),
fetchargs->namespace, fetchargs->userid);
prot_printf(state->out, "%cFOLDER ", sepchar);
prot_printastring(state->out, extname);
sepchar = ' ';
free(extname);
}
if ((fetchitems & FETCH_UIDVALIDITY)) {
prot_printf(state->out, "%cUIDVALIDITY %u", sepchar,
state->mailbox->i.uidvalidity);
sepchar = ' ';
}
if (config_getswitch(IMAPOPT_CONVERSATIONS)) {
if (fetchitems & FETCH_MAILBOXES) {
prot_printf(state->out, "%cMAILBOXES (", sepchar);
r = index_fetchmailboxes(state, msgno, fetchargs);
r = 0;
prot_printf(state->out, ")");
sepchar = ' ';
}
if (fetchitems & FETCH_MAILBOXIDS) {
prot_printf(state->out, "%cMAILBOXIDS (", sepchar);
r = index_fetchmailboxids(state, msgno, fetchargs);
r = 0;
prot_printf(state->out, ")");
sepchar = ' ';
}
}
if (fetchitems & FETCH_ENVELOPE) {
if (!mailbox_cacherecord(mailbox, &record)) {
prot_printf(state->out, "%cENVELOPE ", sepchar);
sepchar = ' ';
prot_putbuf(state->out, cacheitem_buf(&record, CACHE_ENVELOPE));
}
}
if (fetchitems & FETCH_BODYSTRUCTURE) {
if (!mailbox_cacherecord(mailbox, &record)) {
prot_printf(state->out, "%cBODYSTRUCTURE ", sepchar);
sepchar = ' ';
prot_putbuf(state->out, cacheitem_buf(&record, CACHE_BODYSTRUCTURE));
}
}
if (fetchitems & FETCH_BODY) {
if (!mailbox_cacherecord(mailbox, &record)) {
prot_printf(state->out, "%cBODY ", sepchar);
sepchar = ' ';
prot_putbuf(state->out, cacheitem_buf(&record, CACHE_BODY));
}
}
if (fetchitems & FETCH_HEADER) {
prot_printf(state->out, "%cRFC822.HEADER ", sepchar);
sepchar = ' ';
index_fetchmsg(state, &buf, 0,
record.header_size,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->start_octet : 0,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->octet_count : 0);
}
else if (fetchargs->headers.count || fetchargs->headers_not.count) {
prot_printf(state->out, "%cRFC822.HEADER ", sepchar);
sepchar = ' ';
if (fetchargs->cache_atleast > record.cache_version) {
index_fetchheader(state, buf.s, buf.len,
record.header_size,
&fetchargs->headers, &fetchargs->headers_not);
} else {
index_fetchcacheheader(state, &record, &fetchargs->headers, 0, 0);
}
}
if (fetchitems & FETCH_TEXT) {
prot_printf(state->out, "%cRFC822.TEXT ", sepchar);
sepchar = ' ';
index_fetchmsg(state, &buf,
record.header_size, record.size - record.header_size,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->start_octet : 0,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->octet_count : 0);
}
if (fetchitems & FETCH_RFC822) {
prot_printf(state->out, "%cRFC822 ", sepchar);
sepchar = ' ';
index_fetchmsg(state, &buf, 0, record.size,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->start_octet : 0,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->octet_count : 0);
}
for (fsection = fetchargs->fsections; fsection; fsection = fsection->next) {
int i;
prot_printf(state->out, "%cBODY[%s ", sepchar, fsection->section);
sepchar = '(';
for (i = 0 ; i < fsection->fields->count ; i++) {
(void)prot_putc(sepchar, state->out);
sepchar = ' ';
prot_printastring(state->out, fsection->fields->data[i]);
}
(void)prot_putc(')', state->out);
sepchar = ' ';
oi = (struct octetinfo *)fsection->rock;
prot_printf(state->out, "%s ", fsection->trail);
if (fetchargs->cache_atleast > record.cache_version) {
loadbody(mailbox, &record, &body);
if (body) {
index_fetchfsection(state, buf.s, buf.len,
fsection,
body,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->start_octet : oi->start_octet,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->octet_count : oi->octet_count);
} else {
prot_printf(state->out, "NIL");
}
}
else {
index_fetchcacheheader(state, &record, fsection->fields,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->start_octet : oi->start_octet,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->octet_count : oi->octet_count);
}
}
for (section = fetchargs->bodysections; section; section = section->next) {
respbuf[0] = 0;
if (sepchar == '(' && !started) {
/* we haven't output a fetch item yet, so start the response */
snprintf(respbuf, sizeof(respbuf), "* %u FETCH ", msgno);
}
snprintf(respbuf+strlen(respbuf), sizeof(respbuf)-strlen(respbuf),
"%cBODY[%s ", sepchar, section->name);
oi = &section->octetinfo;
loadbody(mailbox, &record, &body);
if (body) {
r = index_fetchsection(state, respbuf, &buf,
section->name, body, record.size,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->start_octet : oi->start_octet,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->octet_count : oi->octet_count);
if (!r) sepchar = ' ';
}
}
for (section = fetchargs->binsections; section; section = section->next) {
respbuf[0] = 0;
if (sepchar == '(' && !started) {
/* we haven't output a fetch item yet, so start the response */
snprintf(respbuf, sizeof(respbuf), "* %u FETCH ", msgno);
}
snprintf(respbuf+strlen(respbuf), sizeof(respbuf)-strlen(respbuf),
"%cBINARY[%s ", sepchar, section->name);
loadbody(mailbox, &record, &body);
if (body) {
oi = &section->octetinfo;
r = index_fetchsection(state, respbuf, &buf,
section->name, body, record.size,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->start_octet : oi->start_octet,
(fetchitems & FETCH_IS_PARTIAL) ?
fetchargs->octet_count : oi->octet_count);
if (!r) sepchar = ' ';
}
}
for (section = fetchargs->sizesections; section; section = section->next) {
respbuf[0] = 0;
if (sepchar == '(' && !started) {
/* we haven't output a fetch item yet, so start the response */
snprintf(respbuf, sizeof(respbuf), "* %u FETCH ", msgno);
}
snprintf(respbuf+strlen(respbuf), sizeof(respbuf)-strlen(respbuf),
"%cBINARY.SIZE[%s ", sepchar, section->name);
loadbody(mailbox, &record, &body);
if (body) {
r = index_fetchsection(state, respbuf, &buf,
section->name, body, record.size,
fetchargs->start_octet, fetchargs->octet_count);
if (!r) sepchar = ' ';
}
}
if (sepchar != '(') {
/* finsh the response if we have one */
prot_printf(state->out, ")\r\n");
}
buf_free(&buf);
if (body) {
message_free_body(body);
free(body);
}
return r;
}
/*
* Fetch the text data associated with an IMAP URL.
*
* If outsize is NULL, the data will be output as a literal (URLFETCH),
* otherwise just the data will be output (CATENATE), and its size returned
* in *outsize.
*
* This is an amalgamation of index_fetchreply(), index_fetchsection()
* and index_fetchmsg().
*/
EXPORTED int index_urlfetch(struct index_state *state, uint32_t msgno,
unsigned params, const char *section,
unsigned long start_octet, unsigned long octet_count,
struct protstream *pout, unsigned long *outsize)
{
/* dumbass eM_Client sends this:
* A4 APPEND "INBOX.Junk Mail" () "14-Jul-2013 17:01:02 +0000"
* CATENATE (URL "/INBOX/;uid=83118/;section=TEXT.MIME"
* URL "/INBOX/;uid=83118/;section=TEXT")
*
* genius. I can sort of see how TEXT.MIME kinda == "HEADER",
* so there we go */
static char text_mime[] = "HEADER";
struct buf buf = BUF_INITIALIZER;
int domain = DOMAIN_7BIT;
const char *data;
size_t size;
int32_t wantheader = 0;
unsigned long n;
int r = 0;
char *decbuf = NULL;
struct index_record record;
struct body *top = NULL, *body = NULL;
size_t section_offset, section_size;
r = index_lock(state, /*readonly*/1); // XXX: do we need to stay locked for this entire function?
if (r) return r;
struct mailbox *mailbox = state->mailbox;
if (!strcasecmpsafe(section, "TEXT.MIME"))
section = text_mime;
if (outsize) *outsize = 0;
r = index_reload_record(state, msgno, &record);
if (r) goto done;
loadbody(mailbox, &record, &body);
if (!body) goto done;
top = body;
/* Open the message file */
if (mailbox_map_record(mailbox, &record, &buf)) {
r = IMAP_NO_MSGGONE;
goto done;
}
data = buf.s;
size = buf.len;
int is_binary = params & URLFETCH_BINARY;
/* Special-case BODY[] */
if (!section || !*section) {
/* whole message, no further parsing */
}
else {
const char *p = ucase((char *) section);
int32_t mimenum = 0;
while (*p) {
switch(*p) {
case 'H':
if (is_binary) goto badpart;
if (!body_is_rfc822(body)) goto badpart;
body = body->subpart;
p += 6;
wantheader = 1;
goto getoffset;
case 'T':
if (is_binary) goto badpart;
if (!body_is_rfc822(body)) goto badpart;
body = body->subpart;
p += 4;
goto getoffset;
case 'M':
if (is_binary) goto badpart;
if (top == body) goto badpart;
p += 4;
wantheader = 1;
goto getoffset;
default:
mimenum = 0;
r = parseint32(p, &p, &mimenum);
if (*p == '.') p++;
if (r || !mimenum) goto badpart;
body = find_part(body, mimenum);
if (!body) goto badpart;
break;
}
}
getoffset:
if (*p) goto badpart;
section_offset = wantheader ? body->header_offset : body->content_offset;
section_size = wantheader ? body->header_size : body->content_size;
if (section_offset + section_size < section_offset
|| section_offset + section_size > size) {
r = IMAP_INTERNAL;
goto done;
}
data += section_offset;
size = section_size;
}
if (is_binary) {
int encoding = body->charset_enc & 0xff;
data = charset_decode_mimebody(data, size, encoding,
&decbuf, &size);
/* update the encoding of this part per RFC5524:3.2 */
if (data && encoding) {
domain = data_domain(data, size);
free(body->encoding);
switch (domain) {
case DOMAIN_BINARY:
body->encoding = xstrdup("BINARY");
break;
case DOMAIN_8BIT:
body->encoding = xstrdup("8BIT");
break;
default:
body->encoding = NULL; // will output 7BIT
break;
}
body->content_size = size;
body->content_lines = 0;
}
}
if (params & URLFETCH_BODYPARTSTRUCTURE) {
struct buf buf = BUF_INITIALIZER;
message_write_body(&buf, body, 1);
prot_puts(pout, " (BODYPARTSTRUCTURE ");
if (buf_len(&buf))
prot_putbuf(pout, &buf);
else
prot_puts(pout, "NIL");
prot_puts(pout, ")");
buf_free(&buf);
}
if (params & URLFETCH_BODY) {
prot_printf(pout, " (BODY");
}
else if (params & URLFETCH_BINARY) {
prot_printf(pout, " (BINARY");
if (!data) {
/* failed to decode */
prot_printf(pout, " NIL)");
r = 0;
goto done;
}
}
else if (params) {
r = 0;
goto done;
}
/* Handle PARTIAL request */
n = octet_count ? octet_count : size;
/* Sanity check the requested size */
if (start_octet > size) {
start_octet = size;
n = 0;
}
else if (start_octet + n < start_octet || start_octet + n > size) {
n = size - start_octet;
}
if (outsize) {
/* Return size (CATENATE) */
*outsize = n;
} else {
domain = data_domain(data + start_octet, n);
if (domain == DOMAIN_BINARY) {
/* Write size of literal8 */
prot_printf(pout, " ~{%lu}\r\n", n);
} else {
/* Write size of literal */
prot_printf(pout, " {%lu}\r\n", n);
}
}
/* Non-text literal -- tell the protstream about it */
if (domain != DOMAIN_7BIT) prot_data_boundary(pout);
if (n) prot_write(pout, data + start_octet, n);
/* End of non-text literal -- tell the protstream about it */
if (domain != DOMAIN_7BIT) prot_data_boundary(pout);
/* Complete extended URLFETCH response */
if (params) prot_printf(pout, ")");
r = 0;
done:
/* Close the message file */
index_unlock(state);
buf_free(&buf);
if (top) {
message_free_body(top);
free(top);
}
if (decbuf) free(decbuf);
return r;
badpart:
r = IMAP_PROTOCOL_BAD_PARAMETERS;
goto done;
}
/*
* Helper function to perform a STORE command for flags.
*/
static int index_storeflag(struct index_state *state,
struct index_modified_flags *modified_flags,
uint32_t msgno, msgrecord_t *msgrec,
struct storeargs *storeargs)
{
uint32_t old, new, keep;
unsigned i;
int dirty = 0;
modseq_t oldmodseq;
struct index_map *im = &state->map[msgno-1];
int r;
memset(modified_flags, 0, sizeof(struct index_modified_flags));
oldmodseq = im->modseq;
/* Change \Seen flag. This gets done on the index first and will only be
copied into the record later if internalseen is set */
if (state->myrights & ACL_SETSEEN) {
old = im->isseen ? 1 : 0;
new = old;
if (storeargs->operation == STORE_REPLACE_FLAGS)
new = storeargs->seen ? 1 : 0;
else if (storeargs->seen)
new = (storeargs->operation == STORE_ADD_FLAGS) ? 1 : 0;
if (new != old) {
state->numunseen += (old - new);
im->isseen = new;
state->seen_dirty = 1;
dirty++;
}
}
uint32_t system_flags;
uint32_t internal_flags;
uint32_t user_flags[MAX_USER_FLAGS/32];
r = msgrecord_get_systemflags(msgrec, &system_flags);
if (r) return r;
r = msgrecord_get_internalflags(msgrec, &internal_flags);
if (r) return r;
r = msgrecord_get_userflags(msgrec, user_flags);
if (r) return r;
keep = internal_flags;
old = system_flags & FLAGS_SYSTEM;
new = storeargs->system_flags & FLAGS_SYSTEM;
/* all other updates happen directly to the record */
if (storeargs->operation == STORE_REPLACE_FLAGS) {
if (!(state->myrights & ACL_WRITE)) {
/* ACL_DELETE handled in index_store() */
if ((old & FLAG_DELETED) != (new & FLAG_DELETED)) {
dirty++;
system_flags = (old & ~FLAG_DELETED) | (new & FLAG_DELETED);
}
}
else {
if (!(state->myrights & ACL_DELETEMSG)) {
if ((old & ~FLAG_DELETED) != (new & ~FLAG_DELETED)) {
dirty++;
system_flags = (old & FLAG_DELETED) | (new & ~FLAG_DELETED);
}
}
else {
if (old != new) {
dirty++;
system_flags = new;
}
}
for (i = 0; i < (MAX_USER_FLAGS/32); i++) {
if (user_flags[i] != storeargs->user_flags[i]) {
uint32_t changed;
dirty++;
changed = ~user_flags[i] & storeargs->user_flags[i];
if (changed) {
modified_flags->added_user_flags[i] = changed;
modified_flags->added_flags++;
}
changed = user_flags[i] & ~storeargs->user_flags[i];
if (changed) {
modified_flags->removed_user_flags[i] = changed;
modified_flags->removed_flags++;
}
user_flags[i] = storeargs->user_flags[i];
}
}
}
}
else if (storeargs->operation == STORE_ADD_FLAGS) {
uint32_t added;
if (~old & new) {
dirty++;
system_flags = old | new;
}
for (i = 0; i < (MAX_USER_FLAGS/32); i++) {
added = ~user_flags[i] & storeargs->user_flags[i];
if (added) {
dirty++;
user_flags[i] |= storeargs->user_flags[i];
modified_flags->added_user_flags[i] = added;
modified_flags->added_flags++;
}
}
}
else { /* STORE_REMOVE_FLAGS */
uint32_t removed;
if (old & new) {
dirty++;
system_flags &= ~storeargs->system_flags;
}
for (i = 0; i < (MAX_USER_FLAGS/32); i++) {
removed = user_flags[i] & storeargs->user_flags[i];
if (removed) {
dirty++;
user_flags[i] &= ~storeargs->user_flags[i];
modified_flags->removed_user_flags[i] = removed;
modified_flags->removed_flags++;
}
}
}
/* rfc4551:
* 3.8. Additional Quality-of-Implementation Issues
*
* Server implementations should follow the following rule, which
* applies to any successfully completed STORE/UID STORE (with and
* without UNCHANGEDSINCE modifier), as well as to a FETCH command that
* implicitly sets \Seen flag:
*
* Adding the flag when it is already present or removing when it is
* not present SHOULD NOT change the mod-sequence.
*
* This will prevent spurious client synchronization requests.
*/
if (!dirty) return 0;
if (state->internalseen) {
/* copy the seen flag from the index */
if (im->isseen)
system_flags |= FLAG_SEEN;
else
system_flags &= ~FLAG_SEEN;
}
/* add back the internal tracking flags */
system_flags |= keep;
modified_flags->added_system_flags = ~old & system_flags & FLAGS_SYSTEM;
if (modified_flags->added_system_flags)
modified_flags->added_flags++;
modified_flags->removed_system_flags = old & ~system_flags & FLAGS_SYSTEM;
if (modified_flags->removed_system_flags)
modified_flags->removed_flags++;
r = msgrecord_set_systemflags(msgrec, system_flags);
if (r) return r;
r = msgrecord_set_internalflags(msgrec, internal_flags);
if (r) return r;
r = msgrecord_set_userflags(msgrec, user_flags);
if (r) return r;
/* if it's silent and unchanged, update the seen value, but
* not if qresync is enabled - RFC 4551 says that the MODSEQ
* must always been told, and we prefer just to tell flags
* as well in this case, it's simpler and not much more
* bandwidth */
if (!(client_capa & CAPA_CONDSTORE) && storeargs->silent && im->told_modseq == oldmodseq)
im->told_modseq = im->modseq;
return 0;
}
/*
* Helper function to perform a STORE command for annotations
*/
static int index_store_annotation(struct index_state *state,
uint32_t msgno,
msgrecord_t *msgrec,
struct storeargs *storeargs,
int *dirty)
{
modseq_t oldmodseq;
struct index_record *record;
struct index_map *im = &state->map[msgno-1];
int r;
r = msgrecord_get_index_record_rw(msgrec, &record);
if (r) goto out;
r = index_reload_record(state, msgno, record);
if (r) goto out;
oldmodseq = record->modseq;
r = msgrecord_annot_set_auth(msgrec, storeargs->isadmin, storeargs->userid,
storeargs->authstate);
if (r) goto out;
r = msgrecord_annot_writeall(msgrec, storeargs->entryatts);
if (r) goto out;
/* It would be nice if the annotate layer told us whether it
* actually made a change to the database, but it doesn't, so
* we have to assume the message is dirty */
*dirty = 1;
r = index_rewrite_record(state, msgno, record, /*silent*/1);
if (r) goto out;
/* if it's silent and unchanged, update the seen value */
if (!(client_capa & CAPA_CONDSTORE) && storeargs->silent && im->told_modseq == oldmodseq)
im->told_modseq = im->modseq;
out:
return r;
}
/*
* Evaluate a searchargs structure on a msgno
*/
EXPORTED int index_search_evaluate(struct index_state *state,
const search_expr_t *e,
uint32_t msgno)
{
struct index_map *im = &state->map[msgno-1];
struct index_record record;
int always = search_expr_always_same(e);
if (always < 0) return 0;
if (always > 0) return 1;
// failure to load is an error!
int r = index_reload_record(state, msgno, &record);
if (r) return 0;
xstats_inc(SEARCH_EVALUATE);
int flags = (im->isrecent ? MESSAGE_RECENT : 0)
| (im->isseen ? MESSAGE_SEEN : 0);
if (state->m) message_set_from_index(state->mailbox, &record, msgno, flags, state->m);
else state->m = message_new_from_index(state->mailbox, &record, msgno, flags);
int match = search_expr_evaluate(state->m, e);
return match;
}
struct extractor_ctx {
struct backend *be;
struct protstream *clientin;
unsigned failed;
};
struct getsearchtext_rock
{
search_text_receiver_t *receiver;
int indexed_headers;
int charset_flags;
const strarray_t *partids;
int snippet_iteration; /* 0..no snippet, 1..first run, 2..second run */
struct extractor_ctx *ext;
};
static void stuff_part(search_text_receiver_t *receiver,
int part,
const struct message_guid *content_guid,
const struct buf *buf)
{
// don't try to index a zero length part
if (!buf_len(buf)) return;
if (part == SEARCH_PART_HEADERS &&
!config_getswitch(IMAPOPT_SEARCH_INDEX_HEADERS))
return;
receiver->begin_part(receiver, part, content_guid);
receiver->append_text(receiver, buf);
receiver->end_part(receiver, part);
}
static void extract_cb(const struct buf *text, void *rock)
{
struct getsearchtext_rock *str = (struct getsearchtext_rock *)rock;
str->receiver->append_text(str->receiver, text);
}
#ifdef USE_HTTPD
static int extract_icalbuf(struct buf *raw, charset_t charset, int encoding,
const struct message_guid *content_guid,
struct getsearchtext_rock *str)
{
icalcomponent *comp = NULL, *ical = NULL;
const char *s;
int r = 0;
struct buf buf = BUF_INITIALIZER;
/* Parse the message into an iCalendar object */
const struct buf *icalbuf = NULL;
if (encoding || strcasecmp(charset_name(charset), "utf-8")) {
char *tmp = charset_to_utf8(buf_cstring(raw), buf_len(raw), charset, encoding);
if (!tmp) return 0; /* could be a bogus header - ignore */
buf_initm(&buf, tmp, strlen(tmp));
icalbuf = &buf;
}
else {
icalbuf = raw;
}
ical = icalparser_parse_string(buf_cstring(icalbuf));
buf_reset(&buf);
if (!ical) {
r = IMAP_INTERNAL;
goto done;
}
for (comp = icalcomponent_get_first_real_component(ical);
comp;
comp = icalcomponent_get_next_component(ical, icalcomponent_isa(comp))) {
icalproperty *prop;
icalparameter *param;
/* description */
if ((s = icalcomponent_get_description(comp))) {
buf_setcstr(&buf, s);
charset_t utf8 = charset_lookupname("utf-8");
str->receiver->begin_part(str->receiver, SEARCH_PART_BODY, content_guid);
charset_extract(extract_cb, str, &buf, utf8, 0, "calendar",
str->charset_flags);
str->receiver->end_part(str->receiver, SEARCH_PART_BODY);
charset_free(&utf8);
buf_reset(&buf);
}
/* summary */
if ((prop = icalcomponent_get_first_property(comp, ICAL_SUMMARY_PROPERTY))) {
if ((s = icalproperty_get_summary(prop))) {
buf_setcstr(&buf, s);
stuff_part(str->receiver, SEARCH_PART_SUBJECT, NULL, &buf);
buf_reset(&buf);
}
}
/* organizer */
if ((prop = icalcomponent_get_first_property(comp, ICAL_ORGANIZER_PROPERTY))) {
if ((s = icalproperty_get_organizer(prop))) {
if (!strncasecmp(s, "mailto:", 7)) {
s += 7;
}
param = icalproperty_get_first_parameter(prop, ICAL_CN_PARAMETER);
if (param) {
buf_printf(&buf, "\"%s\" <%s>", icalparameter_get_cn(param), s);
} else {
buf_setcstr(&buf, s);
}
stuff_part(str->receiver, SEARCH_PART_FROM, NULL, &buf);
buf_reset(&buf);
}
}
/* attendees */
for (prop = icalcomponent_get_first_property(comp, ICAL_ATTENDEE_PROPERTY);
prop;
prop = icalcomponent_get_next_property(comp, ICAL_ATTENDEE_PROPERTY)) {
if ((s = icalproperty_get_attendee(prop))) {
if (!strncasecmp(s, "mailto:", 7)) {
s += 7;
}
param = icalproperty_get_first_parameter(prop, ICAL_CN_PARAMETER);
if (buf.len) {
buf_appendcstr(&buf, ", ");
}
if (param) {
buf_printf(&buf, "\"%s\" <%s>", icalparameter_get_cn(param), s);
} else {
buf_appendcstr(&buf, s);
}
}
}
if (buf.len) {
stuff_part(str->receiver, SEARCH_PART_TO, NULL, &buf);
buf_reset(&buf);
}
/* location */
if ((prop = icalcomponent_get_first_property(comp, ICAL_LOCATION_PROPERTY))) {
if ((s = icalproperty_get_location(prop))) {
buf_setcstr(&buf, s);
stuff_part(str->receiver, SEARCH_PART_LOCATION, content_guid, &buf);
buf_reset(&buf);
}
}
}
done:
if (ical) icalcomponent_free(ical);
buf_free(&buf);
return r;
}
static void _add_vcard_singlval(struct vparse_card *card, const char *key, struct buf *buf)
{
struct vparse_entry *entry;
for (entry = card->properties; entry; entry = entry->next) {
if (strcasecmp(entry->name, key)) continue;
const char *val = entry->v.value;
if (val && val[0]) {
if (buf_len(buf)) buf_putc(buf, ' ');
buf_appendcstr(buf, val);
}
}
}
static void _add_vcard_multival(struct vparse_card *card, const char *key, struct buf *buf)
{
struct vparse_entry *entry;
for (entry = card->properties; entry; entry = entry->next) {
if (strcasecmp(entry->name, key)) continue;
const strarray_t *sa = entry->v.values;
int i;
for (i = 0; i < strarray_size(sa); i++) {
const char *val = strarray_nth(sa, i);
if (val && val[0]) {
if (buf_len(buf)) buf_putc(buf, ' ');
buf_appendcstr(buf, val);
}
}
}
}
static int extract_vcardbuf(struct buf *raw, charset_t charset, int encoding,
const struct message_guid *content_guid,
struct getsearchtext_rock *str)
{
struct vparse_card *vcard = NULL;
int r = 0;
struct buf buf = BUF_INITIALIZER;
/* Parse the message into a vcard object */
const struct buf *vcardbuf = NULL;
if (encoding || strcasecmp(charset_name(charset), "utf-8")) {
char *tmp = charset_to_utf8(buf_cstring(raw), buf_len(raw), charset, encoding);
if (!tmp) return 0; /* could be a bogus header - ignore */
buf_initm(&buf, tmp, strlen(tmp));
vcardbuf = &buf;
}
else {
vcardbuf = raw;
}
vcard = vcard_parse_string(buf_cstring(vcardbuf), /*repair*/1);
if (!vcard || !vcard->objects) {
r = IMAP_INTERNAL;
goto done;
}
buf_reset(&buf);
// these are all the things that we think might be interesting
_add_vcard_singlval(vcard->objects, "fn", &buf);
_add_vcard_singlval(vcard->objects, "email", &buf);
_add_vcard_singlval(vcard->objects, "tel", &buf);
_add_vcard_singlval(vcard->objects, "url", &buf);
_add_vcard_singlval(vcard->objects, "impp", &buf);
_add_vcard_singlval(vcard->objects, "x-social-profile", &buf);
_add_vcard_singlval(vcard->objects, "x-fm-online-other", &buf);
_add_vcard_singlval(vcard->objects, "nickname", &buf);
_add_vcard_singlval(vcard->objects, "note", &buf);
_add_vcard_multival(vcard->objects, "n", &buf);
_add_vcard_multival(vcard->objects, "org", &buf);
_add_vcard_multival(vcard->objects, "adr", &buf);
if (buf.len) {
charset_t utf8 = charset_lookupname("utf-8");
str->receiver->begin_part(str->receiver, SEARCH_PART_BODY, content_guid);
charset_extract(extract_cb, str, &buf, utf8, 0, "vcard",
str->charset_flags);
str->receiver->end_part(str->receiver, SEARCH_PART_BODY);
charset_free(&utf8);
buf_reset(&buf);
}
done:
if (vcard) vparse_free_card(vcard);
buf_free(&buf);
return r;
}
#endif /* USE_HTTPD */
#define IDLE_TIMEOUT (5 * 60) /* 5 min */
struct backend_ctx {
char *path;
int timeout;
};
static int login(struct backend *s __attribute__((unused)),
const char *userid __attribute__((unused)),
sasl_callback_t *cb __attribute__((unused)),
const char **status __attribute__((unused)),
int noauth __attribute__((unused)))
{
return 0;
}
static int ping(struct backend *s,
const char *userid __attribute__((unused)))
{
struct backend_ctx *ctx = (struct backend_ctx *) s->context;
size_t hostlen = strcspn(s->hostname, "/");
hdrcache_t resp_hdrs = NULL;
struct body_t resp_body;
unsigned statuscode = 0;
const char *errstr;
int r = 0;
prot_printf(s->out,
"HEAD %s %s\r\n"
"Host: %.*s\r\n"
"User-Agent: Cyrus/%s\r\n"
"Connection: Keep-Alive\r\n"
"Keep-Alive: timeout=%u\r\n"
"Accept: text/plain\r\n"
"\r\n",
ctx->path, HTTP_VERSION,
(int) hostlen, s->hostname, CYRUS_VERSION, ctx->timeout);
prot_flush(s->out);
/* Read response(s) from backend until final response or error */
do {
resp_body.flags = BODY_DISCARD;
r = http_read_response(s, METH_HEAD, &statuscode,
&resp_hdrs, &resp_body, &errstr);
if (r) {
break;
}
} while (statuscode < 200);
if (r) {
syslog(LOG_DEBUG, "extractor_ping: HEAD %s: failed (%s): %s",
ctx->path, error_message(r), errstr);
}
else {
syslog(LOG_DEBUG, "extractor_ping: HEAD %s: got status %u",
ctx->path, statuscode);
}
if (resp_hdrs) spool_free_hdrcache(resp_hdrs);
return r;
}
static int logout(struct backend *s __attribute__((unused)))
{
return 0;
}
static void extractor_disconnect(struct backend *be)
{
syslog(LOG_DEBUG, "extractor_disconnect(%p)", be);
if (!be || (be->sock == -1)) {
/* already disconnected */
return;
}
/* need to logout of server */
backend_disconnect(be);
/* remove the timeout */
if (be->timeout) prot_removewaitevent(be->clientin, be->timeout);
be->timeout = NULL;
be->clientin = NULL;
}
static struct prot_waitevent *
extractor_timeout(struct protstream *s __attribute__((unused)),
struct prot_waitevent *ev __attribute__((unused)),
void *rock)
{
struct backend *be = (struct backend *) rock;
syslog(LOG_DEBUG, "extractor_timeout(%p)", be);
/* too long since we last used the extractor - disconnect */
extractor_disconnect(be);
return NULL;
}
static struct protocol_t http =
{ "http", "HTTP", TYPE_SPEC, { .spec = { &login, &ping, &logout } } };
static int extractor_connect(const char *exturl, struct extractor_ctx *ext)
{
struct backend *be;
struct backend_ctx *ctx = NULL;
struct buf buf = BUF_INITIALIZER;
const char *hostname;
char path[256];
time_t now = time(NULL);
int r = 0;
syslog(LOG_DEBUG, "extractor_connect(%s)", exturl);
if (!ext || ext->failed > 1) return IMAP_INTERNAL;
be = ext->be;
if (be) {
ctx = be->context;
hostname = be->hostname;
}
else {
char scheme[6], server[100], *p;
unsigned https, port;
/* Parse URL (cheesy parser without having to use libxml2) */
int n = sscanf(exturl, "%5[^:]://%99[^/]%255[^\n]",
scheme, server, path);
if (n != 3 ||
strncmp(lcase(scheme), "http", 4) || (scheme[4] && scheme[4] != 's')) {
syslog(LOG_ERR,
"extract_attachment: unexpected non-HTTP URL %s", exturl);
return IMAP_INTERNAL;
}
/* Normalize URL parts */
https = (scheme[4] == 's');
if (*(p = path + strlen(path) - 1) == '/') *p = '\0';
if ((p = strrchr(server, ':'))) {
*p++ = '\0';
port = atoi(p);
}
else port = https ? 443 : 80;
/* Build servername, port, and options */
buf_printf(&buf, "%s:%u%s/noauth", server, port, https ? "/tls" : "");
hostname = buf_cstring(&buf);
}
if (be && (be->sock != -1)) {
syslog(LOG_DEBUG, "extractor_connect: ping existing connection");
/* ping the extractor */
if (ping(be, NULL)) extractor_disconnect(be);
}
if (!be || (be->sock == -1)) {
/* Connect to extractor service */
syslog(LOG_DEBUG, "extractor_connect: %sconnect to %s",
be ? "re" : "", hostname);
be = ext->be = backend_connect(be, hostname,
&http, NULL, NULL, NULL, -1);
if (!be) {
syslog(LOG_ERR, "extract_connect: failed to connect to %s",
hostname);
ext->failed++;
r = IMAP_IOERROR;
}
else if (!ctx) {
ctx = be->context = xzmalloc(sizeof(struct backend_ctx));
ctx->path = xstrdup(path);
if (ext->clientin) {
/* add a default timeout */
be->clientin = ext->clientin;
be->timeout = prot_addwaitevent(ext->clientin,
now + IDLE_TIMEOUT,
extractor_timeout, be);
}
}
}
if (be->timeout && ctx->timeout) be->timeout->mark = now + ctx->timeout;
buf_free(&buf);
return r;
}
static int extract_attachment(const char *type, const char *subtype,
const struct param *type_params,
const struct buf *data, int encoding,
const struct message_guid *content_guid,
struct getsearchtext_rock *str)
{
struct extractor_ctx *ext = str->ext;
struct backend *be;
struct backend_ctx *ctx;
struct buf buf = BUF_INITIALIZER;
hdrcache_t hdrs = NULL;
struct body_t body = { 0, 0, 0, 0, 0, BUF_INITIALIZER };
const char *guidstr, *errstr = NULL;
size_t hostlen;
int r = 0;
if (!ext) return 0;
be = ext->be;
ctx = (struct backend_ctx *) be->context;
hostlen = strcspn(be->hostname, "/");
if (message_guid_isnull(content_guid)) {
syslog(LOG_DEBUG, "extract_attachment: ignoring null guid for %s/%s",
type ? type : "<null>", subtype ? subtype : "<null>");
return 0;
}
/* Fetch previously extracted text */
unsigned statuscode = 0;
guidstr = message_guid_encode(content_guid);
prot_printf(be->out,
"GET %s/%s %s\r\n"
"Host: %.*s\r\n"
"User-Agent: Cyrus/%s\r\n"
"Connection: Keep-Alive\r\n"
"Keep-Alive: timeout=%u\r\n"
"Accept: text/plain\r\n"
"\r\n",
ctx->path, guidstr, HTTP_VERSION,
(int) hostlen, be->hostname, CYRUS_VERSION,
ctx->timeout ? ctx->timeout : IDLE_TIMEOUT);
prot_flush(be->out);
/* Read GET response */
do {
r = http_read_response(be, METH_GET,
&statuscode, &hdrs, &body, &errstr);
if (r) {
syslog(LOG_ERR,
"extract_attachment: failed to read response for GET %s/%s",
ctx->path, guidstr);
r = IMAP_IOERROR;
goto done;
}
} while (statuscode < 200);
syslog(LOG_DEBUG, "extract_attachment: GET %s/%s: got status %u",
ctx->path, guidstr, statuscode);
/* Abide by server's timeout, if any */
if (!ctx->timeout) {
const char **hdr, *p;
if ((hdr = spool_getheader(hdrs, "Keep-Alive")) &&
(p = strstr(hdr[0], "timeout="))) {
ctx->timeout = atoi(p+8);
syslog(LOG_INFO, "XXXXXXXXXXX extract timeout: %d", ctx->timeout);
}
else ctx->timeout = IDLE_TIMEOUT;
if (be->timeout) be->timeout->mark = time(NULL) + ctx->timeout;
}
if (statuscode == 404) {
/* Decode data */
struct buf decbuf = BUF_INITIALIZER;
if (encoding) {
if (charset_decode(&decbuf, buf_base(data), buf_len(data), encoding)) {
syslog(LOG_ERR, "extract_attachment: failed to decode data");
r = IMAP_IOERROR;
goto done;
}
data = &decbuf;
}
/* Build list of Content-Type parameters */
const struct param *param = type_params;
while (param && param->attribute) {
buf_putc(&buf, ';');
buf_appendcstr(&buf, param->attribute);
if (param->value) {
buf_putc(&buf, '=');
buf_appendcstr(&buf, param->value);
}
param = param->next;
}
/* Send attachment to service for text extraction */
prot_printf(be->out,
"PUT %s/%s %s\r\n"
"Host: %.*s\r\n"
"User-Agent: Cyrus/%s\r\n"
"Connection: Keep-Alive\r\n"
"Keep-Alive: timeout=%u\r\n"
"Accept: text/plain\r\n"
"Content-Type: %s/%s%s\r\n"
"Content-Length: %ld\r\n"
"\r\n",
ctx->path, guidstr, HTTP_VERSION,
(int) hostlen, be->hostname, CYRUS_VERSION, ctx->timeout,
type, subtype, buf_cstring(&buf), buf_len(data));
prot_putbuf(be->out, (struct buf *) data);
prot_flush(be->out);
buf_free(&decbuf);
/* Read PUT response */
body.flags = 0;
do {
r = http_read_response(be, METH_PUT,
&statuscode, &hdrs, &body, &errstr);
if (r) {
syslog(LOG_ERR,
"extract_attachment: failed to read response for PUT %s/%s",
ctx->path, guidstr);
r = IMAP_IOERROR;
goto done;
}
} while (statuscode < 200);
syslog(LOG_DEBUG, "extract_attachment: PUT %s/%s: got status %u",
ctx->path, guidstr, statuscode);
}
if (statuscode >= 400 && statuscode <= 499) {
/* indexer can't extract this for some reason, never try again */
goto done;
}
if (statuscode != 200 && statuscode != 201) {
/* any other status code is an error */
r = IMAP_IOERROR;
goto done;
}
/* Append extracted text */
if (buf_len(&body.payload)) {
str->receiver->begin_part(str->receiver, SEARCH_PART_ATTACHMENTBODY, content_guid);
str->receiver->append_text(str->receiver, &body.payload);
str->receiver->end_part(str->receiver, SEARCH_PART_ATTACHMENTBODY);
}
done:
spool_free_hdrcache(hdrs);
buf_free(&body.payload);
buf_free(&buf);
return r;
}
static struct extractor_ctx *index_text_extractor = NULL;
EXPORTED void index_text_extractor_init(struct protstream *clientin)
{
syslog(LOG_DEBUG, "extractor_init(%p)", clientin);
index_text_extractor = xzmalloc(sizeof(struct extractor_ctx));
index_text_extractor->clientin = clientin;
}
EXPORTED void index_text_extractor_destroy(void)
{
struct extractor_ctx *ext = index_text_extractor;
syslog(LOG_DEBUG, "extractor_destroy(%p)", ext);
if (!ext) return;
if (ext->be) {
struct backend *be = ext->be;
extractor_disconnect(be);
free(((struct backend_ctx *) be->context)->path);
free(be->context);
free(be);
}
free(ext);
index_text_extractor = NULL;
}
static int getsearchtext_cb(int isbody, charset_t charset, int encoding,
const char *type, const char *subtype,
const struct param *type_params __attribute__((unused)),
const char *disposition,
const struct param *disposition_params,
const struct message_guid *content_guid,
const char *part,
struct buf *data,
void *rock)
{
struct getsearchtext_rock *str = (struct getsearchtext_rock *)rock;
char *q;
struct buf text = BUF_INITIALIZER;
int r = 0;
if (isbody && part && str->partids && strarray_find(str->partids, part, 0) < 0) {
/* Skip part */
return 0;
}
if (!isbody) {
if (str->snippet_iteration >= 2) goto done;
if (!str->indexed_headers) {
/* Only index the headers of the top message */
q = charset_decode_mimeheader(buf_cstring(data), str->charset_flags);
buf_init_ro_cstr(&text, q);
stuff_part(str->receiver, SEARCH_PART_HEADERS, NULL, &text);
free(q);
buf_free(&text);
str->indexed_headers = 1;
}
/* Index attachment file names */
const struct param *param;
if (disposition && !strcmp(disposition, "ATTACHMENT")) {
/* Look for "Content-Disposition: attachment;filename=" header */
for (param = disposition_params; param; param = param->next) {
if (!strcmp(param->attribute, "FILENAME")) {
char *tmp = charset_decode_mimeheader(param->value, charset_flags);
buf_init_ro_cstr(&text, tmp);
stuff_part(str->receiver, SEARCH_PART_ATTACHMENTNAME, NULL, &text);
buf_free(&text);
free(tmp);
}
else if (!strcmp(param->attribute, "FILENAME*")) {
char *xval = charset_parse_mimexvalue(param->value, NULL);
if (!xval) xval = xstrdup(param->value);
if (xval) {
char *tmp = charset_decode_mimeheader(xval, charset_flags|CHARSET_MIME_UTF8);
buf_init_ro_cstr(&text, tmp);
stuff_part(str->receiver, SEARCH_PART_ATTACHMENTNAME, NULL, &text);
buf_free(&text);
free(tmp);
free(xval);
}
}
}
}
for(param = type_params; param; param = param->next) {
/* Look for "Content-Type: foo;name=" header */
if (strcmp(param->attribute, "NAME"))
continue;
char *tmp = charset_decode_mimeheader(param->value, charset_flags);
buf_init_ro_cstr(&text, tmp);
stuff_part(str->receiver, SEARCH_PART_ATTACHMENTNAME, NULL, &text);
buf_free(&text);
free(tmp);
}
}
else if (buf_len(data) > 50 && !memcmp(data->s, "-----BEGIN PGP MESSAGE-----", 27)) {
/* PGP encrypted body part - we don't want to index this,
* it's a ton of random base64 noise */
}
else if (isbody && !strcmp(type, "TEXT")) {
if (str->snippet_iteration >= 2) goto done;
if (!strcmp(subtype, "CALENDAR")) {
#ifdef USE_HTTPD
extract_icalbuf(data, charset, encoding, content_guid, str);
#endif /* USE_HTTPD */
}
else if (!strcmp(subtype, "VCARD")) {
#ifdef USE_HTTPD
extract_vcardbuf(data, charset, encoding, content_guid, str);
#endif /* USE_HTTPD */
}
else {
/* body-like */
str->receiver->begin_part(str->receiver, SEARCH_PART_BODY, content_guid);
charset_extract(extract_cb, str, data, charset, encoding, subtype,
str->charset_flags);
str->receiver->end_part(str->receiver, SEARCH_PART_BODY);
}
}
else if (isbody && !strcmp(type, "APPLICATION")) {
/* Ignore attachments in first snippet generation pass */
if (str->snippet_iteration == 1) goto done;
/* Only generate snippets from named attachmend parts */
if (str->snippet_iteration >= 2 && !str->partids) goto done;
r = extract_attachment(type, subtype, type_params, data, encoding,
content_guid, str);
if (r) {
syslog(LOG_ERR, "index: can't extract text from attachment: %s",
error_message(r));
goto done;
}
}
done:
return r;
}
static void append_alnum(struct buf *buf, const char *ss)
{
const unsigned char *s = (const unsigned char *)ss;
for ( ; *s ; ++s) {
if (Uisalnum(*s))
buf_putc(buf, *s);
}
}
EXPORTED int index_getsearchtext(message_t *msg, const strarray_t *partids,
search_text_receiver_t *receiver,
int snippet)
{
const char *exturl =
config_getstring(IMAPOPT_SEARCH_ATTACHMENT_EXTRACTOR_URL);
struct getsearchtext_rock str;
struct buf buf = BUF_INITIALIZER;
int format = MESSAGE_SEARCH;
strarray_t types = STRARRAY_INITIALIZER;
const char *type = NULL, *subtype = NULL;
int i;
int r;
/* Determine Content-Type */
r = message_get_type(msg, &type);
if (r) return r;
r = message_get_subtype(msg, &subtype);
if (r) return r;
/* Set up search receiver */
r = receiver->begin_message(receiver, msg);
if (r) return r;
memset(&str, 0, sizeof(struct getsearchtext_rock));
str.receiver = receiver;
str.indexed_headers = 0;
str.charset_flags = charset_flags;
str.partids = partids;
str.snippet_iteration = 0;
if (exturl) {
str.ext = index_text_extractor;
r = extractor_connect(exturl, str.ext);
if (r) return r;
}
/* Search receiver can override text conversion */
if (receiver->index_charset_flags) {
str.charset_flags = receiver->index_charset_flags(str.charset_flags);
}
if (snippet) {
str.charset_flags |= CHARSET_KEEPCASE;
format = MESSAGE_SNIPPET;
}
/* Choose index scheme for Content=Type */
if (!strcasecmp(type, "TEXT") && !strcasecmp(subtype, "CALENDAR")) {
#ifdef USE_HTTPD
/* An iCalendar entry. */
struct buf buf = BUF_INITIALIZER;
int encoding = 0;
const char *charset_id = NULL;
charset_t charset = CHARSET_UNKNOWN_CHARSET;
const struct body *body = NULL;
r = message_get_field(msg, "rawbody", MESSAGE_RAW, &buf);
if (!r) r = message_get_cachebody(msg, &body);
if (!r) r = message_get_encoding(msg, &encoding);
if (!r) r = message_get_charset_id(msg, &charset_id);
if (!r) charset = charset_lookupname(charset_id);
if (charset != CHARSET_UNKNOWN_CHARSET)
r = extract_icalbuf(&buf, charset, encoding, &body->content_guid, &str);
charset_free(&charset);
buf_free(&buf);
#endif
}
else {
if (!message_get_field(msg, "From", format, &buf))
stuff_part(receiver, SEARCH_PART_FROM, NULL, &buf);
if (!message_get_field(msg, "To", format, &buf))
stuff_part(receiver, SEARCH_PART_TO, NULL, &buf);
if (!message_get_field(msg, "Cc", format, &buf))
stuff_part(receiver, SEARCH_PART_CC, NULL, &buf);
if (!message_get_field(msg, "Bcc", format, &buf))
stuff_part(receiver, SEARCH_PART_BCC, NULL, &buf);
if (!message_get_field(msg, "Subject", format, &buf))
stuff_part(receiver, SEARCH_PART_SUBJECT, NULL, &buf);
if (!message_get_field(msg, "List-Id", format, &buf))
stuff_part(receiver, SEARCH_PART_LISTID, NULL, &buf);
if (!message_get_field(msg, "Mailing-List", format, &buf))
stuff_part(receiver, SEARCH_PART_LISTID, NULL, &buf);
if (!message_get_leaf_types(msg, &types) && types.count) {
/* We add three search terms: the type, subtype, and a combined
* type+subtype string. We carefully control punctuation to
* ensure that each word in indexed as a single term. For
* example if the original message has "application/x-pdf" then
* we index "APPLICATION" "XPDF" "APPLICATION_XPDF". */
receiver->begin_part(receiver, SEARCH_PART_TYPE, NULL);
for (i = 0 ; i < types.count ; i+= 2) {
buf_reset(&buf);
if (i) buf_putc(&buf, ' ');
/* type */
append_alnum(&buf, types.data[i]);
buf_putc(&buf, ' ');
/* subtype */
append_alnum(&buf, types.data[i+1]);
buf_putc(&buf, ' ');
/* combined type_subtype */
append_alnum(&buf, types.data[i]);
buf_putc(&buf, '_');
append_alnum(&buf, types.data[i+1]);
receiver->append_text(receiver, &buf);
}
receiver->end_part(receiver, SEARCH_PART_TYPE);
}
/* A regular message. Generate snippets in two passes. */
str.snippet_iteration = snippet ? 1 : 0;
r = message_foreach_section(msg, getsearchtext_cb, &str);
if (snippet) {
if (receiver->flush) {
r = receiver->flush(receiver);
}
if (!r) {
str.snippet_iteration = 2;
r = message_foreach_section(msg, getsearchtext_cb, &str);
}
if (r == IMAP_OK_COMPLETED) r = 0;
}
if (r) goto done;
}
r = receiver->end_message(receiver);
done:
buf_free(&buf);
strarray_fini(&types);
return r;
}
/*
* Helper function to set up arguments to append_copy()
*/
#define COPYARGSGROW 30
static int index_copysetup(struct index_state *state, uint32_t msgno,
struct copyargs *copyargs)
{
struct index_map *im = &state->map[msgno-1];
int r;
if (copyargs->nummsg == copyargs->msgalloc) {
copyargs->msgalloc += COPYARGSGROW;
copyargs->records = (struct index_record *)
xrealloc((char *)copyargs->records,
copyargs->msgalloc * sizeof(struct index_record));
}
r = index_reload_record(state, msgno, &copyargs->records[copyargs->nummsg]);
if (r) return r;
/* seen is per user - embed it in the record */
if (im->isseen)
copyargs->records[copyargs->nummsg].system_flags |= FLAG_SEEN;
else
copyargs->records[copyargs->nummsg].system_flags &= ~FLAG_SEEN;
if (state->want_expunged && (im->internal_flags & FLAG_INTERNAL_EXPUNGED)) {
copyargs->records[copyargs->nummsg].system_flags &= ~FLAG_DELETED;
copyargs->records[copyargs->nummsg].internal_flags &= ~FLAG_INTERNAL_EXPUNGED;
}
copyargs->nummsg++;
return 0;
}
/*
* Creates a list, and optionally also an array of pointers to, of msgdata.
*
* We fill these structs with the processed info that will be needed
* by the specified sort criteria.
*/
MsgData **index_msgdata_load(struct index_state *state,
unsigned *msgno_list, int n,
const struct sortcrit *sortcrit,
unsigned int anchor, int *found_anchor)
{
MsgData **ptrs, *md, *cur;
int i, j;
char *tmpenv;
char *envtokens[NUMENVTOKENS];
int did_cache, did_env, did_conv;
int label;
struct mailbox *mailbox = state->mailbox;
struct index_record record;
struct conversations_state *cstate = NULL;
conversation_t conv = CONVERSATION_INIT;
if (!n) return NULL;
/* create an array of MsgData */
ptrs = (MsgData **) xzmalloc(n * sizeof(MsgData *) + n * sizeof(MsgData));
md = (MsgData *)(ptrs + n);
xstats_add(MSGDATA_LOAD, n);
if (found_anchor)
*found_anchor = 0;
for (i = 0 ; i < n ; i++) {
cur = &md[i];
ptrs[i] = cur;
/* set msgno */
cur->msgno = (msgno_list ? msgno_list[i] : (unsigned)(i+1));
if (index_reload_record(state, cur->msgno, &record))
continue;
cur->uid = record.uid;
cur->cid = record.cid;
cur->system_flags = record.system_flags;
cur->internal_flags = record.internal_flags;
message_guid_copy(&cur->guid, &record.guid);
if (found_anchor && record.uid == anchor)
*found_anchor = 1;
/* useful for convupdates */
cur->modseq = record.modseq;
did_cache = did_env = did_conv = 0;
tmpenv = NULL;
for (j = 0; sortcrit[j].key; j++) {
label = sortcrit[j].key;
if ((label == SORT_CC ||
label == SORT_FROM || label == SORT_SUBJECT ||
label == SORT_TO || label == LOAD_IDS ||
label == SORT_DISPLAYFROM || label == SORT_DISPLAYTO ||
label == SORT_SPAMSCORE) &&
!did_cache) {
/* fetch cached info */
if (mailbox_cacherecord(mailbox, &record))
continue; /* can't do this with a broken cache */
did_cache++;
}
if ((label == LOAD_IDS) && !did_env) {
/* no point if we don't have enough data */
if (cacheitem_size(&record, CACHE_ENVELOPE) <= 2)
continue;
/* make a working copy of envelope -- strip outer ()'s */
/* +1 -> skip the leading paren */
/* -2 -> don't include the size of the outer parens */
tmpenv = xstrndup(cacheitem_base(&record, CACHE_ENVELOPE) + 1,
cacheitem_size(&record, CACHE_ENVELOPE) - 2);
/* parse envelope into tokens */
parse_cached_envelope(tmpenv, envtokens,
VECTOR_SIZE(envtokens));
did_env++;
}
if ((label == SORT_HASCONVFLAG || label == SORT_CONVMODSEQ ||
label == SORT_CONVEXISTS || label == SORT_CONVSIZE) && !did_conv) {
if (!cstate) cstate = conversations_get_mbox(index_mboxname(state));
assert(cstate);
if (conversation_load_advanced(cstate, record.cid, &conv, /*flags*/0))
continue;
did_conv++;
}
switch (label) {
case SORT_CC:
cur->cc = get_localpart_addr(cacheitem_base(&record, CACHE_CC));
break;
case SORT_DATE:
cur->sentdate = record.gmtime;
/* fall through */
case SORT_ARRIVAL:
cur->internaldate = record.internaldate;
break;
case SORT_FROM:
cur->from = get_localpart_addr(cacheitem_base(&record, CACHE_FROM));
break;
case SORT_MODSEQ:
/* already copied above */
break;
case SORT_SIZE:
cur->size = record.size;
break;
case SORT_SUBJECT:
cur->xsubj = index_extract_subject(cacheitem_base(&record, CACHE_SUBJECT),
cacheitem_size(&record, CACHE_SUBJECT),
&cur->is_refwd);
cur->xsubj_hash = strhash(cur->xsubj);
break;
case SORT_TO:
cur->to = get_localpart_addr(cacheitem_base(&record, CACHE_TO));
break;
case SORT_ANNOTATION: {
struct buf value = BUF_INITIALIZER;
annotatemore_msg_lookup(state->mboxname,
record.uid,
sortcrit[j].args.annot.entry,
sortcrit[j].args.annot.userid,
&value);
/* buf_release() never returns NULL, so if the lookup
* fails for any reason we just get an empty string here */
strarray_appendm(&cur->annot, buf_release(&value));
break;
}
case SORT_SAVEDATE:
if (!strcmpnull(mailbox->uniqueid, sortcrit[j].args.mailbox.id)) {
cur->savedate = record.savedate;
}
else {
/* If not in mailboxId, we use receivedAt */
cur->internaldate = record.internaldate;
}
break;
case SORT_SNOOZEDUNTIL:
if ((record.internal_flags & FLAG_INTERNAL_SNOOZED) &&
- (!sortcrit[j].args.mailbox.id ||
- !strcmp(mailbox->uniqueid, sortcrit[j].args.mailbox.id))) {
+ !strcmpnull(mailbox->uniqueid, sortcrit[j].args.mailbox.id)) {
/* SAVEDATE == snoozed#until */
- cur->snoozed_until = record.savedate;
+ cur->savedate = record.savedate;
- if (!cur->snoozed_until) {
+ if (!cur->savedate) {
/* Try fetching snoozed#until directly */
json_t *snoozed =
jmap_fetch_snoozed(mailbox->name, record.uid);
if (snoozed) {
time_from_iso8601(
json_string_value(json_object_get(snoozed,
"until")),
- &cur->snoozed_until);
+ &cur->savedate);
json_decref(snoozed);
}
}
}
- if (!cur->snoozed_until) {
- /* If not snoozed, we use receivedAt */
- cur->snoozed_until = record.internaldate;
+ if (!cur->savedate) {
+ /* If not snoozed in mailboxId, we use receivedAt */
+ cur->internaldate = record.internaldate;
}
break;
case LOAD_IDS:
index_get_ids(cur, envtokens, cacheitem_base(&record, CACHE_HEADERS),
cacheitem_size(&record, CACHE_HEADERS));
break;
case SORT_DISPLAYFROM:
cur->displayfrom = get_displayname(
cacheitem_base(&record, CACHE_FROM));
break;
case SORT_DISPLAYTO:
cur->displayto = get_displayname(
cacheitem_base(&record, CACHE_TO));
break;
case SORT_SPAMSCORE: {
const char *score = index_getheader(state, cur->msgno, "X-Spam-score");
/* multiply by 100 to give an integer score */
cur->spamscore = (int)((atof(score) * 100) + 0.5);
break;
}
case SORT_HASFLAG: {
const char *name = sortcrit[j].args.flag.name;
if (mailbox_record_hasflag(mailbox, &record, name))
cur->hasflag |= (1<<j);
break;
}
case SORT_HASCONVFLAG: {
const char *name = sortcrit[j].args.flag.name;
int idx = -1;
if (cstate->counted_flags)
idx = strarray_find_case(cstate->counted_flags, name, 0);
/* flag exists in the conversation at all */
if (idx >= 0 && conv.counts[idx] > 0 && j < 31)
cur->hasflag |= (1<<j);
break;
}
case SORT_CONVEXISTS:
cur->convexists = conv.exists;
break;
case SORT_CONVSIZE:
cur->convsize = conv.size;
break;
case SORT_CONVMODSEQ:
cur->convmodseq = conv.modseq;
break;
case SORT_RELEVANCY:
/* for now all messages have relevancy=100 */
break;
}
}
free(tmpenv);
conversation_fini(&conv);
}
return ptrs;
}
static char *get_localpart_addr(const char *header)
{
struct address *addr = NULL;
char *ret = NULL;
parseaddr_list(header, &addr);
if (!addr) return NULL;
if (addr->mailbox)
ret = xstrdup(addr->mailbox);
parseaddr_free(addr);
return ret;
}
/*
* Get the 'display-name' of an address from a header
*/
static char *get_displayname(const char *header)
{
struct address *addr = NULL;
char *ret = NULL;
char *p;
parseaddr_list(header, &addr);
if (!addr) return NULL;
if (addr->name && addr->name[0]) {
/* pure RFC5255 compatible "searchform" conversion */
ret = charset_utf8_to_searchform(addr->name, /*flags*/0);
}
else if (addr->domain && addr->mailbox) {
ret = strconcat(addr->mailbox, "@", addr->domain, (char *)NULL);
/* gotta uppercase mailbox/domain */
for (p = ret; *p; p++)
*p = toupper(*p);
}
else if (addr->mailbox) {
ret = xstrdup(addr->mailbox);
/* gotta uppercase mailbox/domain */
for (p = ret; *p; p++)
*p = toupper(*p);
}
parseaddr_free(addr);
return ret;
}
/*
* Extract base subject from subject header
*
* This is a wrapper around _index_extract_subject() which preps the
* subj NSTRING and checks for Netscape "[Fwd: ]".
*/
static char *index_extract_subject(const char *subj, size_t len, int *is_refwd)
{
char *rawbuf, *buf, *s, *base;
/* parse the subj NSTRING and make a working copy */
if (!strcmp(subj, "NIL")) { /* NIL? */
return xstrdup(""); /* yes, return empty */
} else if (*subj == '"') { /* quoted? */
rawbuf = xstrndup(subj + 1, len - 2); /* yes, strip quotes */
} else {
s = strchr(subj, '}') + 3; /* literal, skip { }\r\n */
rawbuf = xstrndup(s, len - (s - subj));
}
buf = charset_parse_mimeheader(rawbuf, charset_flags);
free(rawbuf);
for (s = buf;;) {
base = _index_extract_subject(s, is_refwd);
/* If we have a Netscape "[Fwd: ...]", extract the contents */
if (!strncasecmp(base, "[fwd:", 5) &&
base[strlen(base) - 1] == ']') {
/* inc refwd counter */
*is_refwd += 1;
/* trim "]" */
base[strlen(base) - 1] = '\0';
/* trim "[fwd:" */
s = base + 5;
}
else /* otherwise, we're done */
break;
}
base = xstrdup(base);
free(buf);
for (s = base; *s; s++) {
*s = toupper(*s);
}
return base;
}
/*
* Guts of subject extraction.
*
* Takes a subject string and returns a pointer to the base.
*/
static char *_index_extract_subject(char *s, int *is_refwd)
{
char *base, *x;
/* trim trailer
*
* start at the end of the string and work towards the front,
* resetting the end of the string as we go.
*/
for (x = s + strlen(s) - 1; x >= s;) {
if (Uisspace(*x)) { /* whitespace? */
*x = '\0'; /* yes, trim it */
x--; /* skip past it */
}
else if (x - s >= 4 &&
!strncasecmp(x-4, "(fwd)", 5)) { /* "(fwd)"? */
*(x-4) = '\0'; /* yes, trim it */
x -= 5; /* skip past it */
*is_refwd += 1; /* inc refwd counter */
}
else
break; /* we're done */
}
/* trim leader
*
* start at the head of the string and work towards the end,
* skipping over stuff we don't care about.
*/
for (base = s; base;) {
if (Uisspace(*base)) base++; /* whitespace? */
/* possible refwd */
else if ((!strncasecmp(base, "re", 2) && /* "re"? */
(x = base + 2)) || /* yes, skip past it */
(!strncasecmp(base, "fwd", 3) && /* "fwd"? */
(x = base + 3)) || /* yes, skip past it */
(!strncasecmp(base, "fw", 2) && /* "fw"? */
(x = base + 2))) { /* yes, skip past it */
int count = 0; /* init counter */
while (Uisspace(*x)) x++; /* skip whitespace */
if (*x == '[') { /* start of blob? */
for (x++; x;) { /* yes, get count */
if (!*x) { /* end of subj, quit */
x = NULL;
break;
}
else if (*x == ']') { /* end of blob, done */
break;
/* if we have a digit, and we're still
counting, keep building the count */
} else if (cyrus_isdigit((int) *x) && count != -1) {
count = count * 10 + *x - '0';
if (count < 0) { /* overflow */
count = -1; /* abort counting */
}
} else { /* no digit, */
count = -1; /* abort counting */
}
x++;
}
if (x) /* end of blob? */
x++; /* yes, skip past it */
else
break; /* no, we're done */
}
while (Uisspace(*x)) x++; /* skip whitespace */
if (*x == ':') { /* ending colon? */
base = x + 1; /* yes, skip past it */
*is_refwd += (count > 0 ? count : 1); /* inc refwd counter
by count or 1 */
}
else
break; /* no, we're done */
}
#if 0 /* do nested blobs - wait for decision on this */
else if (*base == '[') { /* start of blob? */
int count = 1; /* yes, */
x = base + 1; /* find end of blob */
while (count) { /* find matching ']' */
if (!*x) { /* end of subj, quit */
x = NULL;
break;
}
else if (*x == '[') /* new open */
count++; /* inc counter */
else if (*x == ']') /* close */
count--; /* dec counter */
x++;
}
if (!x) /* blob didn't close */
break; /* so quit */
else if (*x) /* end of subj? */
base = x; /* no, skip blob */
#else
else if (*base == '[' && /* start of blob? */
(x = strpbrk(base+1, "[]")) && /* yes, end of blob */
*x == ']') { /* (w/o nesting)? */
if (*(x+1)) /* yes, end of subj? */
base = x + 1; /* no, skip blob */
#endif
else
break; /* yes, return blob */
}
else
break; /* we're done */
}
return base;
}
/* Get message-id, and references/in-reply-to */
void index_get_ids(MsgData *msgdata, char *envtokens[], const char *headers,
unsigned size)
{
static struct buf buf;
strarray_t refhdr = STRARRAY_INITIALIZER;
char *refstr, *ref, *in_reply_to;
buf_reset(&buf);
/* get msgid */
msgdata->msgid = find_msgid(envtokens[ENV_MSGID], NULL);
/* if we don't have one, create one */
if (!msgdata->msgid) {
buf_printf(&buf, "<Empty-ID: %u>", msgdata->msgno);
msgdata->msgid = xstrdup(buf.s);
buf_reset(&buf);
}
/* Copy headers to the buffer */
buf_appendmap(&buf, headers, size);
buf_cstring(&buf);
/* grab the References header */
strarray_append(&refhdr, "references");
message_pruneheader(buf.s, &refhdr, 0);
strarray_fini(&refhdr);
if (buf.s) {
/* allocate some space for refs */
/* find references */
refstr = buf.s;
massage_header(refstr);
while ((ref = find_msgid(refstr, &refstr)) != NULL)
strarray_appendm(&msgdata->ref, ref);
}
/* if we have no references, try in-reply-to */
if (!msgdata->ref.count) {
/* get in-reply-to id */
in_reply_to = find_msgid(envtokens[ENV_INREPLYTO], NULL);
/* if we have an in-reply-to id, make it the ref */
if (in_reply_to)
strarray_appendm(&msgdata->ref, in_reply_to);
}
}
/*
* Function for comparing two integers.
*/
#ifdef HAVE_DECLARE_OPTIMIZE
static inline int numcmp(modseq_t n1, modseq_t n2)
__attribute__((pure, always_inline, optimize("-O3")));
#endif
static int numcmp(modseq_t n1, modseq_t n2)
{
if (n1 < n2) return -1;
if (n1 > n2) return 1;
return 0;
return ((n1 < n2) ? -1 : (n1 > n2) ? 1 : 0);
}
/*
* Comparison function for sorting message lists.
*/
static int index_sort_compare(MsgData *md1, MsgData *md2,
const struct sortcrit *sortcrit)
{
int reverse, ret = 0, i = 0, ann = 0;
do {
/* determine sort order from reverse flag bit */
reverse = sortcrit[i].flags & SORT_REVERSE;
switch (sortcrit[i].key) {
case SORT_SEQUENCE:
ret = numcmp(md1->msgno, md2->msgno);
break;
case SORT_ARRIVAL:
ret = numcmp(md1->internaldate, md2->internaldate);
break;
case SORT_CC:
ret = strcmpsafe(md1->cc, md2->cc);
break;
case SORT_DATE: {
time_t d1 = md1->sentdate ? md1->sentdate : md1->internaldate;
time_t d2 = md2->sentdate ? md2->sentdate : md2->internaldate;
ret = numcmp(d1, d2);
break;
}
+ case SORT_SNOOZEDUNTIL:
case SORT_SAVEDATE: {
time_t d1 = md1->savedate ? md1->savedate : md1->internaldate;
time_t d2 = md2->savedate ? md2->savedate : md2->internaldate;
ret = numcmp(d1, d2);
break;
}
- case SORT_SNOOZEDUNTIL:
- ret = numcmp(md1->snoozed_until, md2->snoozed_until);
- break;
case SORT_FROM:
ret = strcmpsafe(md1->from, md2->from);
break;
case SORT_SIZE:
ret = numcmp(md1->size, md2->size);
break;
case SORT_SUBJECT:
ret = strcmpsafe(md1->xsubj, md2->xsubj);
break;
case SORT_TO:
ret = strcmpsafe(md1->to, md2->to);
break;
case SORT_ANNOTATION:
ret = strcmpsafe(md1->annot.data[ann], md2->annot.data[ann]);
ann++;
break;
case SORT_MODSEQ:
ret = numcmp(md1->modseq, md2->modseq);
break;
case SORT_DISPLAYFROM:
ret = strcmpsafe(md1->displayfrom, md2->displayfrom);
break;
case SORT_DISPLAYTO:
ret = strcmpsafe(md1->displayto, md2->displayto);
break;
case SORT_UID:
ret = numcmp(md1->uid, md2->uid);
break;
case SORT_CONVMODSEQ:
ret = numcmp(md1->convmodseq, md2->convmodseq);
break;
case SORT_CONVEXISTS:
ret = numcmp(md1->convexists, md2->convexists);
break;
case SORT_CONVSIZE:
ret = numcmp(md1->convsize, md2->convsize);
break;
case SORT_SPAMSCORE:
ret = numcmp(md1->spamscore, md2->spamscore);
break;
case SORT_HASFLAG:
case SORT_HASCONVFLAG:
if (i < 31)
ret = numcmp(md1->hasflag & (1<<i),
md2->hasflag & (1<<i));
break;
case SORT_FOLDER:
if (md1->folder && md2->folder)
ret = strcmpsafe(md1->folder->mboxname, md2->folder->mboxname);
break;
case SORT_RELEVANCY:
ret = 0; /* for now all messages have relevancy=100 */
break;
case SORT_GUID:
ret = message_guid_cmp(&md1->guid, &md2->guid);
break;
}
} while (!ret && sortcrit[i++].key != SORT_SEQUENCE);
// must be multi-folder with the same UID! tiebreaker is GUID
if (!ret) return message_guid_cmp(&md1->guid, &md2->guid);
return (reverse ? -ret : ret);
}
static int sortcrit_is_uid(const struct sortcrit *sortcrit)
{
if ((sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_SEQUENCE)) return 0;
return 1;
}
static int sortcrit_is_reverse_uid(const struct sortcrit *sortcrit)
{
if (!(sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_SEQUENCE)) return 0;
return 1;
}
static int sortcrit_is_modseq(const struct sortcrit *sortcrit)
{
if ((sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_MODSEQ)) return 0;
sortcrit++;
if ((sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_SEQUENCE)) return 0;
return 1;
}
static int sortcrit_is_arrival(const struct sortcrit *sortcrit)
{
if ((sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_ARRIVAL)) return 0;
sortcrit++;
if ((sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_SEQUENCE)) return 0;
return 1;
}
static int sortcrit_is_reverse_arrival(const struct sortcrit *sortcrit)
{
if (!(sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_ARRIVAL)) return 0;
sortcrit++;
if ((sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_SEQUENCE)) return 0;
return 1;
}
static int sortcrit_is_reverse_flagged(const struct sortcrit *sortcrit)
{
if (!(sortcrit->flags & SORT_REVERSE)) return 0;
// both HASFLAG and HASCONVFLAG have the same representation internally
if (!(sortcrit->key == SORT_HASFLAG ||
sortcrit->key == SORT_HASCONVFLAG)) return 0;
sortcrit++;
if (!(sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_ARRIVAL)) return 0;
sortcrit++;
if ((sortcrit->flags & SORT_REVERSE)) return 0;
if (!(sortcrit->key == SORT_SEQUENCE)) return 0;
return 1;
}
static int index_sort_compare_generic_qsort(const void *v1, const void *v2)
{
MsgData *md1 = *(MsgData **)v1;
MsgData *md2 = *(MsgData **)v2;
return index_sort_compare(md1, md2, the_sortcrit);
}
#ifdef HAVE_DECLARE_OPTIMIZE
static inline int index_sort_compare_uid(const void *v1, const void *v2)
__attribute__((pure, always_inline, optimize("-O3")));
#endif
static int index_sort_compare_uid(const void *v1, const void *v2)
{
MsgData *md1 = *(MsgData **)v1;
MsgData *md2 = *(MsgData **)v2;
int ret;
ret = md1->uid - md2->uid;
if (ret) return ret;
return message_guid_cmp(&md1->guid, &md2->guid);
}
#ifdef HAVE_DECLARE_OPTIMIZE
static inline int index_sort_compare_reverse_uid(const void *v1, const void *v2)
__attribute__((pure, always_inline, optimize("-O3")));
#endif
static int index_sort_compare_reverse_uid(const void *v1, const void *v2)
{
MsgData *md1 = *(MsgData **)v1;
MsgData *md2 = *(MsgData **)v2;
int ret;
ret = md2->uid - md1->uid;
if (ret) return ret;
return message_guid_cmp(&md1->guid, &md2->guid);
}
#ifdef HAVE_DECLARE_OPTIMIZE
static inline int index_sort_compare_modseq(const void *v1, const void *v2)
__attribute__((pure, always_inline, optimize("-O3")));
#endif
static int index_sort_compare_modseq(const void *v1, const void *v2)
{
MsgData *md1 = *(MsgData **)v1;
MsgData *md2 = *(MsgData **)v2;
int ret;
ret = md1->modseq - md2->modseq;
if (ret) return ret;
ret = md1->uid - md2->uid;
if (ret) return ret;
return message_guid_cmp(&md1->guid, &md2->guid);
}
#ifdef HAVE_DECLARE_OPTIMIZE
static inline int index_sort_compare_arrival(const void *v1, const void *v2)
__attribute__((pure, always_inline, optimize("-O3")));
#endif
static int index_sort_compare_arrival(const void *v1, const void *v2)
{
MsgData *md1 = *(MsgData **)v1;
MsgData *md2 = *(MsgData **)v2;
int ret;
ret = md1->internaldate - md2->internaldate;
if (ret) return ret;
ret = md1->uid - md2->uid;
if (ret) return ret;
return message_guid_cmp(&md1->guid, &md2->guid);
}
#ifdef HAVE_DECLARE_OPTIMIZE
static inline int index_sort_compare_reverse_arrival(const void *v1, const void *v2)
__attribute__((pure, always_inline, optimize("-O3")));
#endif
static int index_sort_compare_reverse_arrival(const void *v1, const void *v2)
{
MsgData *md1 = *(MsgData **)v1;
MsgData *md2 = *(MsgData **)v2;
int ret;
ret = md2->internaldate - md1->internaldate;
if (ret) return ret;
ret = md1->uid - md2->uid;
if (ret) return ret;
return message_guid_cmp(&md1->guid, &md2->guid);
}
#ifdef HAVE_DECLARE_OPTIMIZE
static inline int index_sort_compare_reverse_flagged(const void *v1, const void *v2)
__attribute__((pure, always_inline, optimize("-O3")));
#endif
static int index_sort_compare_reverse_flagged(const void *v1, const void *v2)
{
MsgData *md1 = *(MsgData **)v1;
MsgData *md2 = *(MsgData **)v2;
int ret;
ret = md2->hasflag - md1->hasflag;
if (ret) return ret;
ret = md2->internaldate - md1->internaldate;
if (ret) return ret;
ret = md1->uid - md2->uid;
if (ret) return ret;
return message_guid_cmp(&md1->guid, &md2->guid);
}
void index_msgdata_sort(MsgData **msgdata, int n, const struct sortcrit *sortcrit)
{
if (sortcrit_is_uid(sortcrit)) {
qsort(msgdata, n, sizeof(MsgData *), index_sort_compare_uid);
}
else if (sortcrit_is_reverse_uid(sortcrit)) {
qsort(msgdata, n, sizeof(MsgData *), index_sort_compare_reverse_uid);
}
else if (sortcrit_is_modseq(sortcrit)) {
qsort(msgdata, n, sizeof(MsgData *), index_sort_compare_modseq);
}
else if (sortcrit_is_arrival(sortcrit)) {
qsort(msgdata, n, sizeof(MsgData *), index_sort_compare_arrival);
}
else if (sortcrit_is_reverse_arrival(sortcrit)) {
qsort(msgdata, n, sizeof(MsgData *), index_sort_compare_reverse_arrival);
}
else if (sortcrit_is_reverse_flagged(sortcrit)) {
qsort(msgdata, n, sizeof(MsgData *), index_sort_compare_reverse_flagged);
}
else {
char *tmp = sortcrit_as_string(sortcrit);
syslog(LOG_DEBUG, "GENERICSORT: %s", tmp);
free(tmp);
the_sortcrit = (struct sortcrit *)sortcrit;
qsort(msgdata, n, sizeof(MsgData *), index_sort_compare_generic_qsort);
}
}
/*
* Free an array of MsgData* as built by index_msgdata_load()
*/
void index_msgdata_free(MsgData **msgdata, unsigned int n)
{
unsigned int i;
if (!msgdata)
return;
for (i = 0 ; i < n ; i++) {
MsgData *md = msgdata[i];
if (!md) continue;
xfree(md->cc);
xfree(md->from);
xfree(md->to);
xfree(md->displayfrom);
xfree(md->displayto);
xfree(md->xsubj);
xfree(md->msgid);
xfree(md->listid);
xfree(md->contenttype);
strarray_fini(&md->ref);
strarray_fini(&md->annot);
}
free(msgdata);
}
/*
* Getnext function for sorting thread lists.
*/
static void *index_thread_getnext(Thread *thread)
{
return thread->next;
}
/*
* Setnext function for sorting thread lists.
*/
static void index_thread_setnext(Thread *thread, Thread *next)
{
thread->next = next;
}
/*
* Comparison function for sorting threads.
*/
static int index_thread_compare(Thread *t1, Thread *t2,
const struct sortcrit *call_data)
{
MsgData *md1, *md2;
/* if the container is empty, use the first child's container */
md1 = t1->msgdata ? t1->msgdata : t1->child->msgdata;
md2 = t2->msgdata ? t2->msgdata : t2->child->msgdata;
return index_sort_compare(md1, md2, call_data);
}
/*
* Sort a list of threads.
*/
static void index_thread_sort(Thread *root,
const struct sortcrit *sortcrit)
{
Thread *child;
/* sort the grandchildren */
child = root->child;
while (child) {
/* if the child has children, sort them */
if (child->child)
index_thread_sort(child, sortcrit);
child = child->next;
}
/* sort the children */
root->child = lsort(root->child,
(void * (*)(void*)) index_thread_getnext,
(void (*)(void*,void*)) index_thread_setnext,
(int (*)(void*,void*,void*)) index_thread_compare,
(void *)sortcrit);
}
/*
* Thread a list of messages using the ORDEREDSUBJECT algorithm.
*/
static void index_thread_orderedsubj(struct index_state *state,
unsigned *msgno_list, unsigned int nmsg,
int usinguid)
{
MsgData **msgdata;
unsigned int mi;
static const struct sortcrit sortcrit[] =
{{ SORT_SUBJECT, 0, {{NULL, NULL}} },
{ SORT_DATE, 0, {{NULL, NULL}} },
{ SORT_SEQUENCE, 0, {{NULL, NULL}} }};
unsigned psubj_hash = 0;
char *psubj;
Thread *head, *newnode, *cur, *parent, *last;
/* Create/load the msgdata array */
msgdata = index_msgdata_load(state, msgno_list, nmsg, sortcrit, 0, NULL);
/* Sort messages by subject and date */
index_msgdata_sort(msgdata, nmsg, sortcrit);
/* create an array of Thread to use as nodes of thread tree
*
* we will be building threads under a dummy head,
* so we need (nmsg + 1) nodes
*/
head = (Thread *) xzmalloc((nmsg + 1) * sizeof(Thread));
newnode = head + 1; /* set next newnode to the second
one in the array (skip the head) */
parent = head; /* parent is the head node */
psubj = NULL; /* no previous subject */
cur = NULL; /* no current thread */
last = NULL; /* no last child */
for (mi = 0 ; mi < nmsg ; mi++) {
MsgData *msg = msgdata[mi];
newnode->msgdata = msg;
/* if no previous subj, or
current subj = prev subj (subjs have same hash, and
the strings are equal), then add message to current thread */
if (!psubj ||
(msg->xsubj_hash == psubj_hash &&
!strcmp(msg->xsubj, psubj))) {
/* if no children, create first child */
if (!parent->child) {
last = parent->child = newnode;
if (!cur) /* first thread */
parent = cur = parent->child;
}
/* otherwise, add to siblings */
else {
last->next = newnode;
last = last->next;
}
}
/* otherwise, create a new thread */
else {
cur->next = newnode; /* create and start a new thread */
parent = cur = cur->next; /* now work with the new thread */
}
psubj_hash = msg->xsubj_hash;
psubj = msg->xsubj;
newnode++;
}
/* Sort threads by date */
index_thread_sort(head, sortcrit+1);
/* Output the threaded messages */
index_thread_print(state, head, usinguid);
/* free the thread array */
free(head);
/* free the msgdata array */
index_msgdata_free(msgdata, nmsg);
}
/*
* Guts of thread printing. Recurses over children when necessary.
*
* Frees contents of msgdata as a side effect.
*/
static void _index_thread_print(struct index_state *state,
Thread *thread, int usinguid)
{
Thread *child;
/* for each thread... */
while (thread) {
/* start the thread */
prot_printf(state->out, "(");
/* if we have a message, print its identifier
* (do nothing for empty containers)
*/
if (thread->msgdata && thread->msgdata->uid) {
prot_printf(state->out, "%u",
usinguid ? thread->msgdata->uid :
thread->msgdata->msgno);
/* if we have a child, print the parent-child separator */
if (thread->child) prot_printf(state->out, " ");
}
/* for each child, grandchild, etc... */
child = thread->child;
while (child) {
/* if the child has siblings, print new branch and break */
if (child->next) {
_index_thread_print(state, child, usinguid);
break;
}
/* otherwise print the only child */
else {
prot_printf(state->out, "%u",
usinguid ? child->msgdata->uid :
child->msgdata->msgno);
/* if we have a child, print the parent-child separator */
if (child->child) prot_printf(state->out, " ");
child = child->child;
}
}
/* end the thread */
prot_printf(state->out, ")");
thread = thread->next;
}
}
/*
* Print a list of threads.
*
* This is a wrapper around _index_thread_print() which simply prints the
* start and end of the untagged thread response.
*/
static void index_thread_print(struct index_state *state,
Thread *thread, int usinguid)
{
prot_printf(state->out, "* THREAD");
if (thread) {
prot_printf(state->out, " ");
_index_thread_print(state, thread->child, usinguid);
}
}
/*
* Find threading algorithm for given arg.
* Returns index into thread_algs[], or -1 if not found.
*/
EXPORTED int find_thread_algorithm(char *arg)
{
int alg;
ucase(arg);
for (alg = 0; thread_algs[alg].alg_name; alg++) {
if (!strcmp(arg, thread_algs[alg].alg_name))
return alg;
}
return -1;
}
/*
* The following code is an interpretation of JWZ's description
* and pseudo-code in http://www.jwz.org/doc/threading.html.
*
* It has been modified to match the THREAD=REFERENCES algorithm.
*/
/*
* Determines if child is a descendent of parent.
*
* Returns 1 if yes, 0 otherwise.
*/
static int thread_is_descendent(Thread *parent, Thread *child)
{
Thread *kid;
/* self */
if (parent == child)
return 1;
/* search each child's descendents */
for (kid = parent->child; kid; kid = kid->next) {
if (thread_is_descendent(kid, child))
return 1;
}
return 0;
}
/*
* Links child into parent's children.
*/
static void thread_adopt_child(Thread *parent, Thread *child)
{
child->parent = parent;
child->next = parent->child;
parent->child = child;
}
/*
* Unlinks child from it's parent's children.
*/
static void thread_orphan_child(Thread *child)
{
Thread *prev, *cur;
/* sanity check -- make sure child is actually a child of parent */
for (prev = NULL, cur = child->parent->child;
cur != child && cur != NULL; prev = cur, cur = cur->next);
if (!cur) {
/* uh oh! couldn't find the child in it's parent's children
* we should probably return NO to thread command
*/
return;
}
/* unlink child */
if (!prev) /* first child */
child->parent->child = child->next;
else
prev->next = child->next;
child->parent = child->next = NULL;
}
/*
* Link messages together using message-id and references.
*/
static void ref_link_messages(MsgData **msgdata, unsigned int nmsg,
Thread **newnode, struct hash_table *id_table)
{
Thread *cur, *parent, *ref;
unsigned int mi;
int dup_count = 0;
char buf[100];
int i;
/* for each message... */
for (mi = 0 ; mi < nmsg ; mi++) {
MsgData *msg = msgdata[mi];
/* fill the containers with msgdata
*
* if we already have a container, use it
*/
if ((cur = (Thread *) hash_lookup(msg->msgid, id_table))) {
/* If this container is not empty, then we have a duplicate
* Message-ID. Make this one unique so that we don't stomp
* on the old one.
*/
if (cur->msgdata) {
snprintf(buf, sizeof(buf), "-dup%d", dup_count++);
msg->msgid =
(char *) xrealloc(msg->msgid,
strlen(msg->msgid) + strlen(buf) + 1);
strcat(msg->msgid, buf);
/* clear cur so that we create a new container */
cur = NULL;
}
else
cur->msgdata = msg;
}
/* otherwise, make and index a new container */
if (!cur) {
cur = *newnode;
cur->msgdata = msg;
hash_insert(msg->msgid, cur, id_table);
(*newnode)++;
}
/* Step 1.A */
for (i = 0, parent = NULL; i < msg->ref.count; i++) {
/* if we don't already have a container for the reference,
* make and index a new (empty) container
*/
if (!(ref = (Thread *) hash_lookup(msg->ref.data[i], id_table))) {
ref = *newnode;
hash_insert(msg->ref.data[i], ref, id_table);
(*newnode)++;
}
/* link the references together as parent-child iff:
* - we won't change existing links, AND
* - we won't create a loop
*/
if (!ref->parent &&
parent && !thread_is_descendent(ref, parent)) {
thread_adopt_child(parent, ref);
}
parent = ref;
}
/* Step 1.B
*
* if we have a parent already, it is probably bogus (the result
* of a truncated references field), so unlink from it because
* we now have the actual parent
*/
if (cur->parent) thread_orphan_child(cur);
/* make the last reference the parent of our message iff:
* - we won't create a loop
*/
if (parent && !thread_is_descendent(cur, parent))
thread_adopt_child(parent, cur);
}
}
/*
* Gather orphan messages under the root node.
*/
static void ref_gather_orphans(const char *key __attribute__((unused)),
void *data, void *rock)
{
Thread *node = (Thread *)data;
struct rootset *rootset = (struct rootset *)rock;
/* we only care about nodes without parents */
if (!node->parent) {
if (node->next) {
/* uh oh! a node without a parent should not have a sibling
* we should probably return NO to thread command
*/
return;
}
/* add this node to root's children */
node->next = rootset->root->child;
rootset->root->child = node;
rootset->nroot++;
}
}
/*
* Prune tree of empty containers.
*/
static void ref_prune_tree(Thread *parent)
{
Thread *cur, *prev, *next, *child;
for (prev = NULL, cur = parent->child, next = cur->next;
cur;
prev = cur, cur = next, next = (cur ? cur->next : NULL)) {
retry:
/* if we have an empty container with no children, delete it */
if (!cur->msgdata && !cur->child) {
if (!prev) /* first child */
parent->child = cur->next;
else
prev->next = cur->next;
/* we just removed cur from our list,
* so we need to keep the same prev for the next pass
*/
cur = prev;
}
/* if we have an empty container with children, AND
* we're not at the root OR we only have one child,
* then remove the container but promote its children to this level
* (splice them into the current child list)
*/
else if (!cur->msgdata && cur->child &&
(cur->parent || !cur->child->next)) {
/* move cur's children into cur's place (start the splice) */
if (!prev) /* first child */
parent->child = cur->child;
else
prev->next = cur->child;
/* make cur's parent the new parent of cur's children
* (they're moving in with grandma!)
*/
child = cur->child;
do {
child->parent = cur->parent;
} while (child->next && (child = child->next));
/* make the cur's last child point to cur's next sibling
* (finish the splice)
*/
child->next = cur->next;
/* we just replaced cur with it's children
* so make it's first child the next node to process
*/
next = cur->child;
/* make cur childless and siblingless */
cur->child = cur->next = NULL;
/* we just removed cur from our list,
* so we need to keep the same prev for the next pass
*/
cur = prev;
}
/* if we have a message with children, prune it's children */
else if (cur->child) {
ref_prune_tree(cur);
if (!cur->msgdata && !cur->child) {
/* Did we end up with a completely empty node here?
* Go back and prune it too. See Bug 3784. */
goto retry;
}
}
}
}
/*
* Sort the messages in the root set by date.
*/
static void ref_sort_root(Thread *root)
{
Thread *cur;
static const struct sortcrit sortcrit[] =
{{ SORT_DATE, 0, {{NULL, NULL}} },
{ SORT_SEQUENCE, 0, {{NULL, NULL}} }};
cur = root->child;
while (cur) {
/* if the message is a dummy, sort its children */
if (!cur->msgdata) {
cur->child = lsort(cur->child,
(void * (*)(void*)) index_thread_getnext,
(void (*)(void*,void*)) index_thread_setnext,
(int (*)(void*,void*,void*)) index_thread_compare,
(void *)sortcrit);
}
cur = cur->next;
}
/* sort the root set */
root->child = lsort(root->child,
(void * (*)(void*)) index_thread_getnext,
(void (*)(void*,void*)) index_thread_setnext,
(int (*)(void*,void*,void*)) index_thread_compare,
(void *)sortcrit);
}
/*
* Group threads with same subject.
*/
static void ref_group_subjects(Thread *root, unsigned nroot, Thread **newnode)
{
Thread *cur, *old, *prev, *next, *child;
struct hash_table subj_table;
char *subj;
/* Step 5.A: create a subj_table with one bucket for every possible
* subject in the root set
*/
construct_hash_table(&subj_table, nroot, 1);
/* Step 5.B: populate the table with a container for each subject
* at the root
*/
for (cur = root->child; cur; cur = cur->next) {
/* Step 5.B.i: find subject of the thread
*
* if the container is not empty, use it's subject
*/
if (cur->msgdata)
subj = cur->msgdata->xsubj;
/* otherwise, use the subject of it's first child */
else
subj = cur->child->msgdata->xsubj;
/* Step 5.B.ii: if subject is empty, skip it */
if (!strlen(subj)) continue;
/* Step 5.B.iii: lookup this subject in the table */
old = (Thread *) hash_lookup(subj, &subj_table);
/* Step 5.B.iv: insert the current container into the table iff:
* - this subject is not in the table, OR
* - this container is empty AND the one in the table is not
* (the empty one is more interesting as a root), OR
* - the container in the table is a re/fwd AND this one is not
* (the non-re/fwd is the more interesting of the two)
*/
if (!old ||
(!cur->msgdata && old->msgdata) ||
(old->msgdata && old->msgdata->is_refwd &&
cur->msgdata && !cur->msgdata->is_refwd)) {
hash_insert(subj, cur, &subj_table);
}
}
/* 5.C - group containers with the same subject together */
for (prev = NULL, cur = root->child, next = cur->next;
cur;
prev = cur, cur = next, next = (next ? next->next : NULL)) {
/* Step 5.C.i: find subject of the thread
*
* if container is not empty, use it's subject
*/
if (cur->msgdata)
subj = cur->msgdata->xsubj;
/* otherwise, use the subject of it's first child */
else
subj = cur->child->msgdata->xsubj;
/* Step 5.C.ii: if subject is empty, skip it */
if (!strlen(subj)) continue;
/* Step 5.C.iii: lookup this subject in the table */
old = (Thread *) hash_lookup(subj, &subj_table);
/* Step 5.C.iv: if we found ourselves, skip it */
if (!old || old == cur) continue;
/* ok, we already have a container which contains our current subject,
* so pull this container out of the root set, because we are going to
* merge this node with another one
*/
if (!prev) /* we're at the root */
root->child = cur->next;
else
prev->next = cur->next;
cur->next = NULL;
/* if both containers are dummies, append cur's children to old's */
if (!old->msgdata && !cur->msgdata) {
/* find old's last child */
for (child = old->child; child->next; child = child->next);
/* append cur's children to old's children list */
child->next = cur->child;
/* make old the parent of cur's children */
for (child = cur->child; child; child = child->next)
child->parent = old;
/* make cur childless */
cur->child = NULL;
}
/* if:
* - old container is empty, OR
* - the current message is a re/fwd AND the old one is not,
* make the current container a child of the old one
*
* Note: we don't have to worry about the reverse cases
* because step 5.B guarantees that they won't happen
*/
else if (!old->msgdata ||
(cur->msgdata && cur->msgdata->is_refwd &&
!old->msgdata->is_refwd)) {
thread_adopt_child(old, cur);
}
/* if both messages are re/fwds OR neither are re/fwds,
* then make them both children of a new dummy container
* (we don't want to assume any parent-child relationship between them)
*
* perhaps we can create a parent-child relationship
* between re/fwds by counting the number of re/fwds
*
* Note: we need the hash table to still point to old,
* so we must make old the dummy and make the contents of the
* new container a copy of old's original contents
*/
else {
Thread *new = (*newnode)++;
/* make new a copy of old (except parent and next) */
new->msgdata = old->msgdata;
new->child = old->child;
new->next = NULL;
/* make new the parent of it's newly adopted children */
for (child = new->child; child; child = child->next)
child->parent = new;
/* make old the parent of cur and new */
cur->parent = old;
new->parent = old;
/* empty old and make it have two children (cur and new) */
old->msgdata = NULL;
old->child = cur;
cur->next = new;
}
/* we just removed cur from our list,
* so we need to keep the same prev for the next pass
*/
cur = prev;
}
free_hash_table(&subj_table, NULL);
}
/*
* Guts of thread searching. Recurses over children when necessary.
*/
static int _index_thread_search(struct index_state *state,
Thread *thread, int (*searchproc) (MsgData *))
{
Thread *child;
/* test the head node */
if (thread->msgdata && searchproc(thread->msgdata)) return 1;
/* test the children recursively */
child = thread->child;
while (child) {
if (_index_thread_search(state, child, searchproc)) return 1;
child = child->next;
}
/* if we get here, we struck out */
return 0;
}
/*
* Search a thread to see if it contains a message which matches searchproc().
*
* This is a wrapper around _index_thread_search() which iterates through
* each thread and removes any which fail the searchproc().
*/
static void index_thread_search(struct index_state *state,
Thread *root, int (*searchproc) (MsgData *))
{
Thread *cur, *prev, *next;
for (prev = NULL, cur = root->child, next = cur->next;
cur;
prev = cur, cur= next, next = (cur ? cur->next : NULL)) {
if (!_index_thread_search(state, cur, searchproc)) {
/* unlink the thread from the list */
if (!prev) /* first thread */
root->child = cur->next;
else
prev->next = cur->next;
/* we just removed cur from our list,
* so we need to keep the same prev for the next pass
*/
cur = prev;
}
}
}
/*
* Guts of the REFERENCES algorithms. Behavior is tweaked with loadcrit[],
* threadproc(), searchproc() and sortcrit[].
*/
static void _index_thread_ref(struct index_state *state, unsigned *msgno_list,
unsigned int nmsg,
const struct sortcrit loadcrit[],
MsgData **(*threadproc) (struct rootset *, Thread **),
int (*searchproc) (MsgData *),
const struct sortcrit sortcrit[], int usinguid)
{
MsgData **msgdata, **thrdata = NULL;
unsigned int mi;
int tref, nnode;
Thread *newnode;
struct hash_table id_table;
struct rootset rootset;
/* Create/load the msgdata array */
msgdata = index_msgdata_load(state, msgno_list, nmsg, loadcrit, 0, NULL);
/* calculate the sum of the number of references for all messages */
for (mi = 0, tref = 0 ; mi < nmsg ; mi++)
tref += msgdata[mi]->ref.count;
/* create an array of Thread to use as nodes of thread tree (including
* empty containers)
*
* - We will be building threads under a dummy root, so we need at least
* (nmsg + 1) nodes.
* - We also will need containers for references to non-existent messages.
* To make sure we have enough, we will take the worst case and
* use the sum of the number of references for all messages.
* - Finally, we will need containers to group threads with the same
* subject together. To make sure we have enough, we will take the
* worst case which will be half of the number of messages.
*
* This is overkill, but it is the only way to make sure we have enough
* ahead of time. If we tried to use xrealloc(), the array might be moved,
* and our parent/child/next pointers will no longer be correct
* (been there, done that).
*/
nnode = (int) (1.5 * nmsg + 1 + tref);
rootset.root = (Thread *) xmalloc(nnode * sizeof(Thread));
memset(rootset.root, 0, nnode * sizeof(Thread));
newnode = rootset.root + 1; /* set next newnode to the second
one in the array (skip the root) */
/* Step 0: create an id_table with one bucket for every possible
* message-id and reference (nmsg + tref)
*/
construct_hash_table(&id_table, nmsg + tref, 1);
/* Step 1: link messages together */
ref_link_messages(msgdata, nmsg, &newnode, &id_table);
/* Step 2: find the root set (gather all of the orphan messages) */
rootset.nroot = 0;
hash_enumerate(&id_table, ref_gather_orphans, &rootset);
/* discard id_table */
free_hash_table(&id_table, NULL);
/* Step 3: prune tree of empty containers - get our deposit back :^) */
ref_prune_tree(rootset.root);
/* Steps 4/5: algorithm-specific thread processing */
if (threadproc) thrdata = threadproc(&rootset, &newnode);
/* Optionally search threads (to be used by REFERENCES derivatives) */
if (searchproc) index_thread_search(state, rootset.root, searchproc);
/* Step 6: sort threads */
if (sortcrit) index_thread_sort(rootset.root, sortcrit);
/* Output the threaded messages */
index_thread_print(state, rootset.root, usinguid);
/* free the thread array */
free(rootset.root);
/* free the msgdata array */
index_msgdata_free(msgdata, nmsg);
index_msgdata_free(thrdata, rootset.nroot);
}
/*
* Thread a list of messages using the REFERENCES algorithm.
*/
static MsgData **references_thread_proc(struct rootset *rootset,
Thread **newnode)
{
/* Step 4: sort the root set */
ref_sort_root(rootset->root);
/* Step 5: group root set by subject */
ref_group_subjects(rootset->root, rootset->nroot, newnode);
return NULL;
}
static void index_thread_references(struct index_state *state,
unsigned *msgno_list, unsigned int nmsg,
int usinguid)
{
static const struct sortcrit loadcrit[] =
{{ LOAD_IDS, 0, {{NULL,NULL}} },
{ SORT_SUBJECT, 0, {{NULL,NULL}} },
{ SORT_DATE, 0, {{NULL,NULL}} },
{ SORT_SEQUENCE, 0, {{NULL,NULL}} }};
static const struct sortcrit sortcrit[] =
{{ SORT_DATE, 0, {{NULL,NULL}} },
{ SORT_SEQUENCE, 0, {{NULL,NULL}} }};
_index_thread_ref(state, msgno_list, nmsg, loadcrit,
references_thread_proc, NULL, sortcrit, usinguid);
}
/* Find most recent internaldate of all messages in thread */
static void find_most_recent(Thread *thread, MsgData *recent)
{
Thread *child;
/* test the head node */
if (thread->msgdata->internaldate > recent->internaldate)
recent->internaldate = thread->msgdata->internaldate;
/* test the children recursively */
child = thread->child;
while (child) {
find_most_recent(child, recent);
child = child->next;
}
}
/*
* Tag the root of each thread with the most recent internaldate of all
* messages in the thread. The actual sorting will be done by the calling
* function, but we leverage the fact that sorting by sentdate will fallback
* to internaldate if sentdate == 0.
*/
static MsgData **refs_thread_proc(struct rootset *rootset,
Thread **newnode __attribute((unused)))
{
MsgData **ptrs, *md;
Thread *cur;
int i = 0;
/* Create an array of MsgData for dummy roots */
ptrs = (MsgData **) xzmalloc(rootset->nroot *
(sizeof(MsgData *) + sizeof(MsgData)));
md = (MsgData *)(ptrs + rootset->nroot);
/* Find most recent internaldate in each thread */
cur = rootset->root->child;
while (cur) {
/* If the message is a dummy, assign it MsgData for sorting */
if (!cur->msgdata) {
cur->msgdata = ptrs[i] = &md[i];
i++;
cur->msgdata->internaldate = cur->child->msgdata->internaldate;
}
cur->msgdata->sentdate = 0; /* force date sort to use internaldate */
find_most_recent(cur, cur->msgdata);
cur = cur->next;
}
return ptrs;
}
/*
* Thread a list of messages using the REFS algorithm.
*/
static void index_thread_refs(struct index_state *state,
unsigned *msgno_list, unsigned int nmsg,
int usinguid)
{
static const struct sortcrit loadcrit[] =
{{ LOAD_IDS, 0, {{NULL,NULL}} },
{ SORT_DATE, 0, {{NULL,NULL}} },
{ SORT_ARRIVAL, 0, {{NULL,NULL}} },
{ SORT_SEQUENCE, 0, {{NULL,NULL}} }};
static const struct sortcrit sortcrit[] =
{{ SORT_DATE, 0, {{NULL,NULL}} },
{ SORT_ARRIVAL, 0, {{NULL,NULL}} },
{ SORT_SEQUENCE, 0, {{NULL,NULL}} }};
_index_thread_ref(state, msgno_list, nmsg, loadcrit,
refs_thread_proc, NULL, sortcrit, usinguid);
}
/*
* NNTP specific stuff.
*/
EXPORTED char *index_get_msgid(struct index_state *state,
uint32_t msgno)
{
struct mailbox *mailbox = state->mailbox;
struct index_record record;
if (index_reload_record(state, msgno, &record))
return NULL;
return mailbox_cache_get_env(mailbox, &record, ENV_MSGID);
}
static void massage_header(char *hdr)
{
int n = 0;
char *p, c;
for (p = hdr; *p; p++) {
if (*p == ' ' || *p == '\t' || *p == '\r') {
if (!n || *(p+1) == '\n') {
/* no leading or trailing whitespace */
continue;
}
/* replace with space */
c = ' ';
}
else if (*p == '\n') {
if (*(p+1) == ' ' || *(p+1) == '\t') {
/* folded header */
continue;
}
/* end of header */
break;
}
else
c = *p;
hdr[n++] = c;
}
hdr[n] = '\0';
}
EXPORTED extern struct nntp_overview *index_overview(struct index_state *state,
uint32_t msgno)
{
static struct nntp_overview over;
static char *env = NULL, *from = NULL, *hdr = NULL;
static int envsize = 0, fromsize = 0, hdrsize = 0;
int size;
char *envtokens[NUMENVTOKENS];
struct address addr = { NULL, NULL, NULL, NULL, NULL, NULL, 0 };
strarray_t refhdr = STRARRAY_INITIALIZER;
struct mailbox *mailbox = state->mailbox;
struct index_record record;
/* flush any previous data */
memset(&over, 0, sizeof(struct nntp_overview));
if (index_reload_record(state, msgno, &record))
return NULL;
if (mailbox_cacherecord(mailbox, &record))
return NULL; /* upper layers can cope! */
/* make a working copy of envelope; strip outer ()'s */
/* -2 -> don't include the size of the outer parens */
/* +1 -> leave space for NUL */
size = cacheitem_size(&record, CACHE_ENVELOPE) - 2 + 1;
if (envsize < size) {
envsize = size;
env = xrealloc(env, envsize);
}
/* +1 -> skip the leading paren */
strlcpy(env, cacheitem_base(&record, CACHE_ENVELOPE) + 1, size);
/* make a working copy of headers */
size = cacheitem_size(&record, CACHE_HEADERS);
if (hdrsize < size+2) {
hdrsize = size+100;
hdr = xrealloc(hdr, hdrsize);
}
memcpy(hdr, cacheitem_base(&record, CACHE_HEADERS), size);
hdr[size] = '\0';
parse_cached_envelope(env, envtokens, VECTOR_SIZE(envtokens));
over.uid = record.uid;
over.bytes = record.size;
over.lines = index_getlines(state, msgno);
over.date = envtokens[ENV_DATE];
over.msgid = envtokens[ENV_MSGID];
/* massage subject */
if ((over.subj = envtokens[ENV_SUBJECT]))
massage_header(over.subj);
/* build original From: header */
if (envtokens[ENV_FROM]) /* paranoia */
message_parse_env_address(envtokens[ENV_FROM], &addr);
if (addr.mailbox && addr.domain) { /* paranoia */
/* +3 -> add space for quotes and space */
/* +4 -> add space for < @ > NUL */
size = (addr.name ? strlen(addr.name) + 3 : 0) +
strlen(addr.mailbox) + strlen(addr.domain) + 4;
if (fromsize < size) {
fromsize = size;
from = xrealloc(from, fromsize);
}
from[0] = '\0';
if (addr.name) sprintf(from, "\"%s\" ", addr.name);
snprintf(from + strlen(from), fromsize - strlen(from),
"<%s@%s>", addr.mailbox, addr.domain);
over.from = from;
}
/* massage references */
strarray_append(&refhdr, "references");
message_pruneheader(hdr, &refhdr, 0);
strarray_fini(&refhdr);
if (*hdr) {
over.ref = hdr + 11; /* skip over header name */
massage_header(over.ref);
}
return &over;
}
EXPORTED extern char *index_getheader(struct index_state *state,
uint32_t msgno, char *hdr)
{
static struct buf staticbuf = BUF_INITIALIZER;
strarray_t headers = STRARRAY_INITIALIZER;
struct mailbox *mailbox = state->mailbox;
struct index_record record;
char *buf;
if (index_reload_record(state, msgno, &record))
return NULL;
/* see if the header is cached */
if (mailbox_cached_header(hdr) != BIT32_MAX &&
!mailbox_cacherecord(mailbox, &record)) {
buf_copy(&staticbuf, cacheitem_buf(&record, CACHE_HEADERS));
}
else {
/* uncached header */
struct buf msgbuf = BUF_INITIALIZER;
if (mailbox_map_record(mailbox, &record, &msgbuf))
return NULL;
buf_setcstr(&staticbuf, index_readheader(msgbuf.s, msgbuf.len, 0, record.header_size));
buf_free(&msgbuf);
}
buf_cstring(&staticbuf);
strarray_append(&headers, hdr);
message_pruneheader(staticbuf.s, &headers, NULL);
strarray_fini(&headers);
buf = staticbuf.s;
if (*buf) {
buf += strlen(hdr) + 1; /* skip header: */
massage_header(buf);
}
return buf;
}
EXPORTED extern unsigned long index_getsize(struct index_state *state,
uint32_t msgno)
{
struct index_record record;
if (index_reload_record(state, msgno, &record))
return 0;
return record.size;
}
EXPORTED extern unsigned long index_getlines(struct index_state *state,
uint32_t msgno)
{
struct index_record record;
struct body *body = NULL;
unsigned long lines = 0;
if (index_reload_record(state, msgno, &record))
return 0;
if (mailbox_cacherecord(state->mailbox, &record))
return 0;
message_read_bodystructure(&record, &body);
if (!body) return 0;
lines = body->content_lines;
message_free_body(body);
free(body);
return lines;
}
EXPORTED const char *index_mboxname(const struct index_state *state)
{
if (!state) return NULL;
return state->mboxname;
}
EXPORTED int index_hasrights(const struct index_state *state, int rights)
{
return state->myrights & rights;
}
/*
* Parse a sequence into an array of sorted & merged ranges.
*/
static struct seqset *_parse_sequence(struct index_state *state,
const char *sequence, int usinguid)
{
unsigned maxval = usinguid ? state->last_uid : state->exists;
return seqset_parse(sequence, NULL, maxval);
}
EXPORTED void freesequencelist(struct seqset *l)
{
seqset_free(l);
}
/*
* Create a new search program.
*/
EXPORTED struct searchargs *new_searchargs(const char *tag, int state,
struct namespace *namespace,
const char *userid,
struct auth_state *authstate,
int isadmin)
{
struct searchargs *sa;
sa = (struct searchargs *)xzmalloc(sizeof(struct searchargs));
sa->tag = tag;
sa->state = state;
/* default charset is US-ASCII */
sa->charset = charset_lookupname("US-ASCII");
sa->namespace = namespace;
sa->userid = userid;
sa->authstate = authstate;
sa->isadmin = isadmin;
return sa;
}
/*
* Free the searchargs 's'
*/
EXPORTED void freesearchargs(struct searchargs *s)
{
if (!s) return;
charset_free(&s->charset);
search_expr_free(s->root);
free(s);
}
EXPORTED char *sortcrit_as_string(const struct sortcrit *sortcrit)
{
struct buf b = BUF_INITIALIZER;
static const char * const key_names[] = {
"SEQUENCE", "ARRIVAL", "CC", "DATE",
"DISPLAYFROM", "DISPLAYTO", "FROM",
"SIZE", "SUBJECT", "TO", "ANNOTATION",
"MODSEQ", "UID", "HASFLAG", "CONVMODSEQ",
"CONVEXISTS", "CONVSIZE", "HASCONVFLAG",
"FOLDER", "RELEVANCY", "SPAMSCORE", "GUID",
"EMAILID", "THREADID", "SAVEDATE", "SNOOZEDUNTIL"
};
do {
if (b.len)
buf_putc(&b, ' ');
if (sortcrit->flags & SORT_REVERSE)
buf_appendcstr(&b, "REVERSE ");
if (sortcrit->key < VECTOR_SIZE(key_names))
buf_appendcstr(&b, key_names[sortcrit->key]);
else
buf_printf(&b, "UNKNOWN%u", sortcrit->key);
switch (sortcrit->key) {
case SORT_ANNOTATION:
buf_printf(&b, " \"%s\" \"%s\"",
sortcrit->args.annot.entry,
*sortcrit->args.annot.userid ?
"value.priv" : "value.shared");
break;
}
sortcrit++;
} while (sortcrit->key);
return buf_release(&b);
}
/*
* Free an array of sortcrit
*/
EXPORTED void freesortcrit(struct sortcrit *s)
{
int i = 0;
if (!s) return;
do {
switch (s[i].key) {
case SORT_ANNOTATION:
free(s[i].args.annot.entry);
free(s[i].args.annot.userid);
break;
case SORT_HASFLAG:
case SORT_HASCONVFLAG:
free(s[i].args.flag.name);
break;
case SORT_SAVEDATE:
case SORT_SNOOZEDUNTIL:
free(s[i].args.mailbox.id);
break;
}
i++;
} while (s[i].key != SORT_SEQUENCE);
free(s);
}
EXPORTED int insert_into_mailbox_allowed(struct mailbox *mailbox)
{
int r = 0;
/* prohibit inserting into \Snoozed mailbox */
if (mailbox->i.options & OPT_IMAP_HAS_ALARMS) {
struct buf attrib = BUF_INITIALIZER;
char *userid = mboxname_to_userid(mailbox->name);
r = annotatemore_lookup(mailbox->name, "/specialuse", userid, &attrib);
free(userid);
if (!r && buf_len(&attrib)) {
strarray_t *specialuse =
strarray_split(buf_cstring(&attrib), NULL, 0);
if (strarray_find(specialuse, "\\Snoozed", 0) >= 0) {
r = IMAP_MAILBOX_NOTSUPPORTED;
}
strarray_free(specialuse);
}
buf_free(&attrib);
}
return r;
}
diff --git a/imap/index.h b/imap/index.h
index 5177e1ab1..8c6f59551 100644
--- a/imap/index.h
+++ b/imap/index.h
@@ -1,357 +1,356 @@
/* index.h -- Routines for dealing with the index file in the imapd
*
* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The name "Carnegie Mellon University" must not be used to
* endorse or promote products derived from this software without
* prior written permission. For permission or any legal
* details, please contact
* Carnegie Mellon University
* Center for Technology Transfer and Enterprise Creation
* 4615 Forbes Avenue
* Suite 302
* Pittsburgh, PA 15213
* (412) 268-7393, fax: (412) 268-7395
* innovation@andrew.cmu.edu
*
* 4. Redistributions of any form whatsoever must retain the following
* acknowledgment:
* "This product includes software developed by Computing Services
* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
*
* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/* Header for internal usage of index.c + programs that make raw access
* to index files */
#ifndef INDEX_H
#define INDEX_H
#include <config.h>
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <netinet/in.h>
#include "annotate.h" /* for strlist functionality */
#include "search_engines.h"
#include "message_guid.h"
#include "sequence.h"
#include "strarray.h"
/* Special "sort criteria" to load message-id and references/in-reply-to
* into msgdata array for threaders that need them.
*/
#define LOAD_IDS 256
/* client capabilities by ENABLE command */
extern unsigned client_capa;
struct message;
struct vanished_params {
unsigned long uidvalidity;
modseq_t modseq;
const char *match_seq;
const char *match_uid;
const char *sequence;
int uidvalidity_is_max;
};
struct index_init {
const char *userid;
struct auth_state *authstate;
struct protstream *out;
int examine_mode;
int select;
int want_dav;
uint32_t want_mbtype;
int want_expunged;
struct vanished_params vanished;
struct seqset *vanishedlist;
};
struct index_map {
modseq_t modseq;
modseq_t told_modseq;
uint64_t cache_offset;
uint32_t user_flags[MAX_USER_FLAGS/32];
uint32_t uid;
uint32_t recno;
uint32_t system_flags;
uint32_t internal_flags;
unsigned int isseen:1;
unsigned int isrecent:1;
};
struct index_state {
struct mailbox *mailbox;
unsigned num_records;
unsigned oldexists;
unsigned exists;
unsigned long last_uid;
uint32_t generation; /* to notice repacks */
uint32_t uidvalidity; /* to notice delete/recreate */
modseq_t oldhighestmodseq;
modseq_t highestmodseq;
modseq_t delayed_modseq;
struct index_map *map;
unsigned mapsize;
int internalseen;
int skipped_expunge;
int seen_dirty;
int examining;
int myrights;
unsigned numrecent;
unsigned numunseen;
unsigned firstnotseen;
char *flagname[MAX_USER_FLAGS];
char *userid;
char *mboxname;
struct protstream *out;
struct auth_state *authstate;
int want_dav;
uint32_t want_mbtype;
int want_expunged;
unsigned num_expunged;
message_t *m;
};
struct copyargs {
struct index_record *records;
int nummsg;
int msgalloc;
};
typedef struct msgdata {
struct search_folder *folder; /* search folder (can be NULL) */
/* items from the index_record */
bit32 uid; /* UID for output purposes */
uint32_t msgno; /* message number */
conversation_id_t cid; /* conversation id */
strarray_t ref; /* array of references */
time_t sentdate; /* sent date & time of message
from Date: header (adjusted by time zone) */
time_t internaldate; /* internaldate */
time_t savedate; /* savedate */
size_t size; /* message size */
modseq_t modseq; /* modseq of record*/
bit32 hasflag; /* hasflag values (up to 32 of them) */
struct message_guid guid; /* message guid */
uint32_t system_flags; /* system flags */
uint32_t internal_flags; /* internal flags */
/* items from the conversations database */
modseq_t convmodseq; /* modseq of conversation */
uint32_t convexists; /* exists count of conversation */
uint32_t convsize; /* total size of messages in conversation */
uint32_t spamscore; /* x-spam-score header */
/* items from the cache record */
char *msgid; /* message ID */
char *listid; /* List-Id and Mailing-List fields */
char *contenttype; /* all MIME Content-Types except multipart */
char *cc; /* local-part of first "cc" address */
char *from; /* local-part of first "from" address */
char *to; /* local-part of first "to" address */
char *displayfrom; /* display-name of first "from" address */
char *displayto; /* display-name of first "to" address */
char *xsubj; /* extracted subject text */
unsigned xsubj_hash; /* hash of extracted subject text */
int is_refwd; /* is message a reply or forward? */
/* items from the annotations database */
strarray_t annot; /* array of annotation attribute values
(stored in order of sortcrit) */
- time_t snoozed_until; /* snoozed#until */
} MsgData;
typedef struct thread {
MsgData *msgdata; /* message data */
struct thread *parent; /* parent message */
struct thread *child; /* first child message */
struct thread *next; /* next sibling message */
} Thread;
struct rootset {
Thread *root;
unsigned nroot;
};
struct thread_algorithm {
const char *alg_name;
void (*threader)(struct index_state *state, unsigned *msgno_list,
unsigned int nmsg, int usinguid);
};
struct nntp_overview {
unsigned long uid;
char *subj;
char *from;
char *date;
char *msgid;
char *ref;
unsigned long bytes;
unsigned long lines;
};
enum index_warmup_flags
{
WARMUP_INDEX = (1<<0),
WARMUP_CONVERSATIONS = (1<<1),
WARMUP_ANNOTATIONS = (1<<2),
WARMUP_SEARCH = (1<<3),
WARMUP_ALL = (~WARMUP_SEARCH)
};
/* non-locking, non-updating - just do a fetch on the state
* we already have */
void index_fetchresponses(struct index_state *state,
struct seqset *seq,
int usinguid,
const struct fetchargs *fetchargs,
int *fetchedsomething);
extern int index_fetch(struct index_state *state,
const char* sequence,
int usinguid,
const struct fetchargs* fetchargs,
int* fetchedsomething);
extern int index_store(struct index_state *state,
char *sequence,
struct storeargs *storeargs);
extern int index_run_annotator(struct index_state *state,
const char *sequence, int usinguid,
struct namespace *namespace, int isadmin);
extern int index_warmup(struct mboxlist_entry *, unsigned int warmup_flags,
struct seqset *uids);
extern int index_sort(struct index_state *state, const struct sortcrit *sortcrit,
struct searchargs *searchargs, int usinguid);
extern int index_convsort(struct index_state *state, struct sortcrit *sortcrit,
struct searchargs *searchargs,
const struct windowargs * windowargs);
extern int index_convmultisort(struct index_state *state,
struct sortcrit *sortcrit,
struct searchargs *searchargs,
const struct windowargs * windowargs);
extern int index_snippets(struct index_state *state,
const struct snippetargs *snippetargs,
struct searchargs *searchargs);
extern int index_convupdates(struct index_state *state, struct sortcrit *sortcrit,
struct searchargs *searchargs,
const struct windowargs * windowargs);
extern int index_thread(struct index_state *state, int algorithm,
struct searchargs *searchargs, int usinguid);
extern int index_search(struct index_state *state,
struct searchargs *searchargs,
int usinguid);
extern int index_scan(struct index_state *state,
const char *contents);
extern int index_copy(struct index_state *state,
char *sequence,
int usinguid,
char *name,
char **copyuidp,
int nolink,
struct namespace *namespace,
int isadmin,
int ismove,
int ignorequota);
extern int find_thread_algorithm(char *arg);
extern int index_open(const char *name, struct index_init *init,
struct index_state **stateptr);
extern int index_open_mailbox(struct mailbox *mailbox, struct index_init *init,
struct index_state **stateptr);
extern int index_refresh(struct index_state *state);
extern void index_checkflags(struct index_state *state, int print, int dirty);
extern void index_select(struct index_state *state, struct index_init *init);
extern int index_status(struct index_state *state, struct statusdata *sdata);
extern void index_release(struct index_state *state);
extern void index_close(struct index_state **stateptr);
extern uint32_t index_finduid(struct index_state *state, uint32_t uid);
extern uint32_t index_getuid(struct index_state *state, uint32_t msgno);
extern void index_tellchanges(struct index_state *state, int canexpunge,
int printuid, int printmodseq);
extern modseq_t index_highestmodseq(struct index_state *state);
extern int index_check(struct index_state *state, int usinguid, int printuid);
extern struct seqset *index_vanished(struct index_state *state,
struct vanished_params *params);
extern int index_urlfetch(struct index_state *state, uint32_t msgno,
unsigned params, const char *section,
unsigned long start_octet, unsigned long octet_count,
struct protstream *pout, unsigned long *size);
extern char *index_get_msgid(struct index_state *state, uint32_t msgno);
extern struct nntp_overview *index_overview(struct index_state *state,
uint32_t msgno);
extern char *index_getheader(struct index_state *state, uint32_t msgno,
char *hdr);
extern unsigned long index_getsize(struct index_state *state, uint32_t msgno);
extern unsigned long index_getlines(struct index_state *state, uint32_t msgno);
extern int index_copy_remote(struct index_state *state, char *sequence,
int usinguid, struct protstream *pout);
struct searchargs *new_searchargs(const char *tag, int state,
struct namespace *namespace,
const char *userid,
struct auth_state *authstate,
int isadmin);
void freesequencelist(struct seqset *l);
void freesearchargs(struct searchargs *s);
char *sortcrit_as_string(const struct sortcrit *sortcrit);
void freesortcrit(struct sortcrit *s);
void index_msgdata_sort(MsgData **msgdata, int n, const struct sortcrit *sortcrit);
void index_msgdata_free(MsgData **, unsigned int);
MsgData **index_msgdata_load(struct index_state *state, unsigned *msgno_list, int n,
const struct sortcrit *sortcrit,
unsigned int anchor, int *found_anchor);
extern int index_search_evaluate(struct index_state *state, const search_expr_t *e, uint32_t msgno);
extern int index_expunge(struct index_state *state, char *uidsequence,
int need_deleted);
extern int index_getsearchtext(struct message *msg, const strarray_t *partids,
struct search_text_receiver *receiver,
int snippet);
extern int index_getuidsequence(struct index_state *state,
struct searchargs *searchargs,
unsigned **uid_list);
extern const char *index_mboxname(const struct index_state *state);
extern int index_hasrights(const struct index_state *state, int rights);
extern int index_reload_record(struct index_state *state,
uint32_t msgno,
struct index_record *record);
extern void index_text_extractor_init(struct protstream *clientin);
extern void index_text_extractor_destroy(void);
extern int insert_into_mailbox_allowed(struct mailbox *mailbox);
#endif /* INDEX_H */
diff --git a/imap/jmap_mail.c b/imap/jmap_mail.c
index f5ee34307..87c541d2a 100644
--- a/imap/jmap_mail.c
+++ b/imap/jmap_mail.c
@@ -1,11653 +1,11646 @@
/* jmap_mail.c -- Routines for handling JMAP mail messages
*
* Copyright (c) 1994-2014 Carnegie Mellon University. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The name "Carnegie Mellon University" must not be used to
* endorse or promote products derived from this software without
* prior written permission. For permission or any legal
* details, please contact
* Carnegie Mellon University
* Center for Technology Transfer and Enterprise Creation
* 4615 Forbes Avenue
* Suite 302
* Pittsburgh, PA 15213
* (412) 268-7393, fax: (412) 268-7395
* innovation@andrew.cmu.edu
*
* 4. Redistributions of any form whatsoever must retain the following
* acknowledgment:
* "This product includes software developed by Computing Services
* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
*
* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*
*/
#include <config.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <ctype.h>
#include <string.h>
#include <syslog.h>
#include <assert.h>
#include <sys/mman.h>
#include <sasl/saslutil.h>
#ifdef HAVE_LIBCHARDET
#include <chardet/chardet.h>
#endif
#include "acl.h"
#include "annotate.h"
#include "append.h"
#include "bsearch.h"
#include "carddav_db.h"
#include "hashset.h"
#include "http_dav.h"
#include "http_jmap.h"
#include "http_proxy.h"
#include "jmap_ical.h"
#include "jmap_mail.h"
#include "jmap_mail_query.h"
#include "json_support.h"
#include "mailbox.h"
#include "mappedfile.h"
#include "mboxevent.h"
#include "mboxlist.h"
#include "mboxname.h"
#include "msgrecord.h"
#include "notify.h"
#include "parseaddr.h"
#include "proxy.h"
#include "search_query.h"
#include "seen.h"
#include "smtpclient.h"
#include "statuscache.h"
#include "stristr.h"
#include "sync_log.h"
#include "times.h"
#include "util.h"
#include "xmalloc.h"
#include "xsha1.h"
#include "xstrnchr.h"
/* generated headers are not necessarily in current directory */
#include "imap/http_err.h"
#include "imap/imap_err.h"
static int jmap_email_query(jmap_req_t *req);
static int jmap_email_querychanges(jmap_req_t *req);
static int jmap_email_get(jmap_req_t *req);
static int jmap_email_changes(jmap_req_t *req);
static int jmap_email_import(jmap_req_t *req);
static int jmap_email_parse(jmap_req_t *req);
static int jmap_email_copy(jmap_req_t *req);
static int jmap_email_matchmime_method(jmap_req_t *req);
static int jmap_searchsnippet_get(jmap_req_t *req);
static int jmap_thread_get(jmap_req_t *req);
static int jmap_thread_changes(jmap_req_t *req);
static int jmap_identity_get(jmap_req_t *req);
/*
* Possibly to be implemented:
* - Email/removeAttachments
* - Email/report
* - Identity/changes
* - Identity/set
*/
static jmap_method_t jmap_mail_methods_standard[] = {
{
"Email/query",
JMAP_URN_MAIL,
&jmap_email_query,
JMAP_SHARED_CSTATE
},
{
"Email/queryChanges",
JMAP_URN_MAIL,
&jmap_email_querychanges,
JMAP_SHARED_CSTATE
},
{
"Email/get",
JMAP_URN_MAIL,
&jmap_email_get,
JMAP_SHARED_CSTATE
},
{
"Email/set",
JMAP_URN_MAIL,
&jmap_email_set,
/*flags*/0
},
{
"Email/changes",
JMAP_URN_MAIL,
&jmap_email_changes,
JMAP_SHARED_CSTATE
},
{
"Email/import",
JMAP_URN_MAIL,
&jmap_email_import,
/*flags*/0
},
{
"Email/parse",
JMAP_URN_MAIL,
&jmap_email_parse,
JMAP_SHARED_CSTATE
},
{
"Email/copy",
JMAP_URN_MAIL,
&jmap_email_copy,
/*flags*/ 0
},
{
"Email/matchMime",
JMAP_MAIL_EXTENSION,
&jmap_email_matchmime_method,
JMAP_SHARED_CSTATE
},
{
"SearchSnippet/get",
JMAP_URN_MAIL,
&jmap_searchsnippet_get,
JMAP_SHARED_CSTATE
},
{
"Thread/get",
JMAP_URN_MAIL,
&jmap_thread_get,
JMAP_SHARED_CSTATE
},
{
"Thread/changes",
JMAP_URN_MAIL,
&jmap_thread_changes,
JMAP_SHARED_CSTATE
},
{
"Identity/get",
JMAP_URN_MAIL,
&jmap_identity_get,
JMAP_SHARED_CSTATE
},
{ NULL, NULL, NULL, 0}
};
static jmap_method_t jmap_mail_methods_nonstandard[] = {
{ NULL, NULL, NULL, 0}
};
/* NULL terminated list of supported jmap_email_query sort fields */
static const char *msglist_sortfields_standard[];
static const char *msglist_sortfields_nonstandard[];
#define JMAP_MAIL_MAX_MAILBOXES_PER_EMAIL 20
#define JMAP_MAIL_MAX_KEYWORDS_PER_EMAIL 100 /* defined in mailbox_user_flag */
HIDDEN void jmap_mail_init(jmap_settings_t *settings)
{
jmap_method_t *mp;
for (mp = jmap_mail_methods_standard; mp->name; mp++) {
hash_insert(mp->name, mp, &settings->methods);
}
json_object_set_new(settings->server_capabilities,
JMAP_URN_MAIL, json_object());
if (config_getswitch(IMAPOPT_JMAP_NONSTANDARD_EXTENSIONS)) {
json_object_set_new(settings->server_capabilities,
JMAP_SEARCH_EXTENSION, json_object());
json_object_set_new(settings->server_capabilities,
JMAP_MAIL_EXTENSION, json_object());
for (mp = jmap_mail_methods_nonstandard; mp->name; mp++) {
hash_insert(mp->name, mp, &settings->methods);
}
}
jmap_emailsubmission_init(settings);
jmap_mailbox_init(settings);
jmap_vacation_init(settings);
}
HIDDEN void jmap_mail_capabilities(json_t *account_capabilities)
{
json_t *sortopts = json_array();
const char **sp;
for (sp = msglist_sortfields_standard; *sp; sp++) {
json_array_append_new(sortopts, json_string(*sp));
}
if (config_getswitch(IMAPOPT_JMAP_NONSTANDARD_EXTENSIONS)) {
for (sp = msglist_sortfields_nonstandard; *sp; sp++) {
json_array_append_new(sortopts, json_string(*sp));
}
}
long max_size_attachments_per_email =
config_getint(IMAPOPT_JMAP_MAIL_MAX_SIZE_ATTACHMENTS_PER_EMAIL);
max_size_attachments_per_email *= 1024;
if (max_size_attachments_per_email <= 0) {
syslog(LOG_ERR, "jmap: invalid property value: %s",
imapopts[IMAPOPT_JMAP_MAIL_MAX_SIZE_ATTACHMENTS_PER_EMAIL].optname);
max_size_attachments_per_email = 0;
}
json_t *email_capabilities = json_pack("{s:i s:i s:i s:o}",
"maxMailboxesPerEmail", JMAP_MAIL_MAX_MAILBOXES_PER_EMAIL,
"maxKeywordsPerEmail", JMAP_MAIL_MAX_KEYWORDS_PER_EMAIL,
"maxSizeAttachmentsPerEmail", max_size_attachments_per_email,
"emailsListSortOptions", sortopts);
json_object_set_new(account_capabilities, JMAP_URN_MAIL, email_capabilities);
if (config_getswitch(IMAPOPT_JMAP_NONSTANDARD_EXTENSIONS)) {
json_object_set_new(account_capabilities, JMAP_SEARCH_EXTENSION, json_object());
json_object_set_new(account_capabilities, JMAP_MAIL_EXTENSION, json_object());
}
jmap_mailbox_capabilities(account_capabilities);
}
#define JMAP_HAS_ATTACHMENT_FLAG "$HasAttachment"
typedef enum MsgType {
MSG_IS_ROOT = 0,
MSG_IS_ATTACHED = 1,
} MsgType;
struct _mbentry_by_uniqueid_rock {
const char *uniqueid;
mbentry_t **mbentry;
};
static int _mbentry_by_uniqueid_cb(const mbentry_t *mbentry, void *rock)
{
struct _mbentry_by_uniqueid_rock *data = rock;
if (strcmp(mbentry->uniqueid, data->uniqueid))
return 0;
*(data->mbentry) = mboxlist_entry_copy(mbentry);
return IMAP_OK_COMPLETED;
}
static mbentry_t *_mbentry_by_uniqueid(jmap_req_t *req, const char *id)
{
mbentry_t *mbentry = NULL;
struct _mbentry_by_uniqueid_rock rock = { id, &mbentry };
int r = mboxlist_usermboxtree(req->accountid, req->authstate,
_mbentry_by_uniqueid_cb, &rock,
MBOXTREE_INTERMEDIATES);
if (r != IMAP_OK_COMPLETED && mbentry) {
mboxlist_entry_free(&mbentry);
mbentry = NULL;
}
return mbentry;
}
/*
* Emails
*/
static char *_decode_to_utf8(const char *charset,
const char *data, size_t datalen,
const char *encoding,
int *is_encoding_problem)
{
charset_t cs = charset_lookupname(charset);
char *text = NULL;
int enc = encoding_lookupname(encoding);
if (cs == CHARSET_UNKNOWN_CHARSET || enc == ENCODING_UNKNOWN) {
syslog(LOG_INFO, "decode_to_utf8 error (%s, %s)", charset, encoding);
*is_encoding_problem = 1;
goto done;
}
text = charset_to_utf8(data, datalen, cs, enc);
if (!text) {
*is_encoding_problem = 1;
goto done;
}
size_t textlen = strlen(text);
struct char_counts counts = charset_count_validutf8(text, textlen);
*is_encoding_problem = counts.invalid || counts.replacement;
const char *charset_id = charset_name(cs);
if (!strncasecmp(charset_id, "UTF-32", 6)) {
/* Special-handle UTF-32. Some clients announce the wrong endianess. */
if (counts.invalid || counts.replacement) {
charset_t guess_cs = CHARSET_UNKNOWN_CHARSET;
if (!strcasecmp(charset_id, "UTF-32") || !strcasecmp(charset_id, "UTF-32BE"))
guess_cs = charset_lookupname("UTF-32LE");
else
guess_cs = charset_lookupname("UTF-32BE");
char *guess = charset_to_utf8(data, datalen, guess_cs, enc);
if (guess) {
struct char_counts guess_counts = charset_count_validutf8(guess, strlen(guess));
if (guess_counts.valid > counts.valid) {
free(text);
text = guess;
counts = guess_counts;
}
}
charset_free(&guess_cs);
}
}
#ifdef HAVE_LIBCHARDET
if (counts.invalid || counts.replacement) {
static Detect *d = NULL;
if (!d) d = detect_init();
DetectObj *obj = detect_obj_init();
if (!obj) goto done;
detect_reset(&d);
struct buf buf = BUF_INITIALIZER;
charset_decode(&buf, data, datalen, enc);
buf_cstring(&buf);
if (detect_handledata_r(&d, buf_base(&buf), buf_len(&buf), &obj) == CHARDET_SUCCESS) {
charset_t guess_cs = charset_lookupname(obj->encoding);
if (guess_cs != CHARSET_UNKNOWN_CHARSET) {
char *guess = charset_to_utf8(data, datalen, guess_cs, enc);
if (guess) {
struct char_counts guess_counts = charset_count_validutf8(guess, strlen(guess));
if (guess_counts.valid > counts.valid) {
free(text);
text = guess;
counts = guess_counts;
}
else {
free(guess);
}
}
charset_free(&guess_cs);
}
}
detect_obj_free(&obj);
buf_free(&buf);
}
#endif
done:
charset_free(&cs);
return text;
}
static char *_decode_mimeheader(const char *raw)
{
if (!raw) return NULL;
int is_8bit = 0;
const char *p;
for (p = raw; *p; p++) {
if (*p & 0x80) {
is_8bit = 1;
break;
}
}
char *val = NULL;
if (is_8bit) {
int err = 0;
val = _decode_to_utf8("utf-8", raw, strlen(raw), NULL, &err);
}
if (!val) {
val = charset_decode_mimeheader(raw, CHARSET_KEEPCASE);
}
return val;
}
struct headers {
json_t *raw; /* JSON array of EmailHeader */
json_t *all; /* JSON object: lower-case header name => list of values */
struct buf buf;
};
#define HEADERS_INITIALIZER \
{ json_array(), json_object(), BUF_INITIALIZER }
static void _headers_init(struct headers *headers) {
headers->raw = json_array();
headers->all = json_object();
memset(&headers->buf, 0, sizeof(struct buf));
}
static void _headers_fini(struct headers *headers) {
json_decref(headers->all);
json_decref(headers->raw);
buf_free(&headers->buf);
}
static void _headers_put_new(struct headers *headers, json_t *header, int shift)
{
const char *name = json_string_value(json_object_get(header, "name"));
if (headers->raw == NULL)
headers->raw = json_array();
if (headers->all == NULL)
headers->all = json_object();
/* Append (or shift) the raw header to the in-order header list */
if (shift)
json_array_insert(headers->raw, 0, header);
else
json_array_append(headers->raw, header);
/* Append the raw header to the list of all equal-named headers */
buf_setcstr(&headers->buf, name);
const char *lcasename = buf_lcase(&headers->buf);
json_t *all = json_object_get(headers->all, lcasename);
if (!all) {
all = json_array();
json_object_set_new(headers->all, lcasename, all);
}
if (shift)
json_array_insert_new(all, 0, header);
else
json_array_append_new(all, header);
}
static void _headers_add_new(struct headers *headers, json_t *header)
{
if (!header) return;
_headers_put_new(headers, header, 0);
}
static void _headers_shift_new(struct headers *headers, json_t *header)
{
if (!header) return;
_headers_put_new(headers, header, 1);
}
static json_t* _headers_get(struct headers *headers, const char *name)
{
char *lcasename = lcase(xstrdup(name));
json_t *jheader = json_object_get(headers->all, lcasename);
free(lcasename);
return jheader;
}
static int _headers_have(struct headers *headers, const char *name)
{
return _headers_get(headers, name) != NULL;
}
static int _headers_from_mime_cb(const char *key, const char *val, void *_rock)
{
struct headers *headers = _rock;
_headers_add_new(headers, json_pack("{s:s s:s}", "name", key, "value", val));
return 0;
}
static void _headers_from_mime(const char *base, size_t len, struct headers *headers)
{
message_foreach_header(base, len, _headers_from_mime_cb, headers);
}
static json_t *_header_as_raw(const char *raw)
{
if (!raw) return json_null();
size_t len = strlen(raw);
if (len > 1 && raw[len-1] == '\n' && raw[len-2] == '\r') len -= 2;
return json_stringn(raw, len);
}
static json_t *_header_as_date(const char *raw)
{
if (!raw) return json_null();
time_t t;
if (time_from_rfc5322(raw, &t, DATETIME_FULL) == -1) {
if (!strchr(raw, '\r')) return json_null();
char *tmp = charset_unfold(raw, strlen(raw), CHARSET_UNFOLD_SKIPWS);
int r = time_from_rfc5322(tmp, &t, DATETIME_FULL);
free(tmp);
if (r == -1) return json_null();
}
char cbuf[RFC3339_DATETIME_MAX+1];
cbuf[RFC3339_DATETIME_MAX] = '\0';
time_to_rfc3339(t, cbuf, RFC3339_DATETIME_MAX+1);
return json_string(cbuf);
}
static json_t *_header_as_text(const char *raw)
{
if (!raw) return json_null();
/* TODO this could be optimised to omit unfolding, decoding
* or normalisation, or all, if ASCII */
/* Unfold and remove CRLF */
char *unfolded = charset_unfold(raw, strlen(raw), 0);
char *p = strchr(unfolded, '\r');
while (p && *(p + 1) != '\n') {
p = strchr(p + 1, '\r');
}
if (p) *p = '\0';
/* Trim starting SP */
const char *trimmed = unfolded;
while (isspace(*trimmed)) {
trimmed++;
}
/* Decode header */
char *decoded = _decode_mimeheader(trimmed);
/* Convert to Unicode NFC */
char *normalized = charset_utf8_normalize(decoded);
json_t *result = json_string(normalized);
free(normalized);
free(decoded);
free(unfolded);
return result;
}
static void _remove_ws(char *s)
{
char *d = s;
do {
while (isspace(*s))
s++;
} while ((*d++ = *s++));
}
static json_t *_header_as_messageids(const char *raw)
{
if (!raw) return json_null();
json_t *msgids = json_array();
char *unfolded = charset_unfold(raw, strlen(raw), CHARSET_UNFOLD_SKIPWS);
const char *p = unfolded;
while (*p) {
/* Skip preamble */
while (isspace(*p) || *p == ',') p++;
if (!*p) break;
/* Find end of id */
const char *q = p;
if (*p == '<') {
while (*q && *q != '>') q++;
}
else {
while (*q && !isspace(*q)) q++;
}
/* Read id */
char *val = xstrndup(*p == '<' ? p + 1 : p,
*q == '>' ? q - p - 1 : q - p);
if (*p == '<') {
_remove_ws(val);
}
if (*val) {
/* calculate the value that would be created if this was
* fed back into an Email/set and make sure it would
* validate */
char *msgid = strconcat("<", val, ">", NULL);
int r = conversations_check_msgid(msgid, strlen(msgid));
if (!r) json_array_append_new(msgids, json_string(val));
free(msgid);
}
free(val);
/* Reset iterator */
p = *q ? q + 1 : q;
}
if (!json_array_size(msgids)) {
json_decref(msgids);
msgids = json_null();
}
free(unfolded);
return msgids;
}
static json_t *_emailaddresses_from_addr(struct address *addr)
{
if (!addr) return json_null();
json_t *addresses = json_array();
struct buf buf = BUF_INITIALIZER;
while (addr) {
json_t *e = json_pack("{}");
const char *domain = addr->domain;
if (!strcmpsafe(domain, "unspecified-domain")) {
domain = NULL;
}
if (!addr->name && addr->mailbox && !domain) {
/* That's a group */
json_object_set_new(e, "name", json_string(addr->mailbox));
json_object_set_new(e, "email", json_null());
json_array_append_new(addresses, e);
addr = addr->next;
continue;
}
/* name */
if (addr->name) {
char *tmp = _decode_mimeheader(addr->name);
if (tmp) json_object_set_new(e, "name", json_string(tmp));
free(tmp);
} else {
json_object_set_new(e, "name", json_null());
}
/* email */
if (addr->mailbox) {
buf_setcstr(&buf, addr->mailbox);
if (domain) {
buf_putc(&buf, '@');
buf_appendcstr(&buf, domain);
}
json_object_set_new(e, "email", json_string(buf_cstring(&buf)));
buf_reset(&buf);
} else {
json_object_set_new(e, "email", json_null());
}
json_array_append_new(addresses, e);
addr = addr->next;
}
if (!json_array_size(addresses)) {
json_decref(addresses);
addresses = json_null();
}
buf_free(&buf);
return addresses;
}
static json_t *_header_as_addresses(const char *raw)
{
if (!raw) return json_null();
struct address *addrs = NULL;
parseaddr_list(raw, &addrs);
json_t *result = _emailaddresses_from_addr(addrs);
parseaddr_free(addrs);
return result;
}
static json_t *_header_as_urls(const char *raw)
{
if (!raw) return json_null();
/* A poor man's implementation of RFC 2369, returning anything
* between < and >. */
json_t *urls = json_array();
const char *base = raw;
const char *top = raw + strlen(raw);
while (base < top) {
const char *lo = strchr(base, '<');
if (!lo) break;
const char *hi = strchr(lo, '>');
if (!hi) break;
char *tmp = charset_unfold(lo + 1, hi - lo - 1, CHARSET_UNFOLD_SKIPWS);
_remove_ws(tmp);
if (*tmp) json_array_append_new(urls, json_string(tmp));
free(tmp);
base = hi + 1;
}
if (!json_array_size(urls)) {
json_decref(urls);
urls = json_null();
}
return urls;
}
enum _header_form {
HEADER_FORM_UNKNOWN = 0, /* MUST be zero so we can cast to void* */
HEADER_FORM_RAW,
HEADER_FORM_TEXT,
HEADER_FORM_ADDRESSES,
HEADER_FORM_MESSAGEIDS,
HEADER_FORM_DATE,
HEADER_FORM_URLS
};
struct header_prop {
char *lcasename;
char *name;
const char *prop;
enum _header_form form;
int all;
};
static void _header_prop_fini(struct header_prop *prop)
{
free(prop->lcasename);
free(prop->name);
}
static void _header_prop_free(struct header_prop *prop)
{
_header_prop_fini(prop);
free(prop);
}
static struct header_prop *_header_parseprop(const char *s)
{
strarray_t *fields = strarray_split(s + 7, ":", 0);
const char *f0, *f1, *f2;
int is_valid = 1;
enum _header_form form = HEADER_FORM_RAW;
char *lcasename = NULL, *name = NULL;
/* Initialize allowed header forms by lower-case header name. Any
* header in this map is allowed to be requested either as Raw
* or the form of the map value (casted to void* because C...).
* Any header not found in this map is allowed to be requested
* in any form. */
static hash_table allowed_header_forms = HASH_TABLE_INITIALIZER;
if (allowed_header_forms.size == 0) {
/* TODO initialize with all headers in RFC5322 and RFC2369 */
construct_hash_table(&allowed_header_forms, 32, 0);
hash_insert("bcc", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("cc", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("content-type", (void*) HEADER_FORM_RAW, &allowed_header_forms);
hash_insert("comment", (void*) HEADER_FORM_TEXT, &allowed_header_forms);
hash_insert("date", (void*) HEADER_FORM_DATE, &allowed_header_forms);
hash_insert("from", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("in-reply-to", (void*) HEADER_FORM_MESSAGEIDS, &allowed_header_forms);
hash_insert("list-archive", (void*) HEADER_FORM_URLS, &allowed_header_forms);
hash_insert("list-help", (void*) HEADER_FORM_URLS, &allowed_header_forms);
hash_insert("list-owner", (void*) HEADER_FORM_URLS, &allowed_header_forms);
hash_insert("list-post", (void*) HEADER_FORM_URLS, &allowed_header_forms);
hash_insert("list-subscribe", (void*) HEADER_FORM_URLS, &allowed_header_forms);
hash_insert("list-unsubscribe", (void*) HEADER_FORM_URLS, &allowed_header_forms);
hash_insert("message-id", (void*) HEADER_FORM_MESSAGEIDS, &allowed_header_forms);
hash_insert("references", (void*) HEADER_FORM_MESSAGEIDS, &allowed_header_forms);
hash_insert("reply-to", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("resent-date", (void*) HEADER_FORM_DATE, &allowed_header_forms);
hash_insert("resent-from", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("resent-message-id", (void*) HEADER_FORM_MESSAGEIDS, &allowed_header_forms);
hash_insert("resent-reply-to", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("resent-sender", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("resent-to", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("resent-cc", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("resent-bcc", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("sender", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
hash_insert("subject", (void*) HEADER_FORM_TEXT, &allowed_header_forms);
hash_insert("to", (void*) HEADER_FORM_ADDRESSES, &allowed_header_forms);
}
/* Parse property string into fields */
f0 = f1 = f2 = NULL;
switch (fields->count) {
case 3:
f2 = strarray_nth(fields, 2);
/* fallthrough */
case 2:
f1 = strarray_nth(fields, 1);
/* fallthrough */
case 1:
f0 = strarray_nth(fields, 0);
lcasename = lcase(xstrdup(f0));
name = xstrdup(f0);
break;
default:
strarray_free(fields);
return NULL;
}
if (f2 && (strcmp(f2, "all") || !strcmp(f1, "all"))) {
strarray_free(fields);
free(lcasename);
free(name);
return NULL;
}
if (f1) {
if (!strcmp(f1, "asRaw"))
form = HEADER_FORM_RAW;
else if (!strcmp(f1, "asText"))
form = HEADER_FORM_TEXT;
else if (!strcmp(f1, "asAddresses"))
form = HEADER_FORM_ADDRESSES;
else if (!strcmp(f1, "asMessageIds"))
form = HEADER_FORM_MESSAGEIDS;
else if (!strcmp(f1, "asDate"))
form = HEADER_FORM_DATE;
else if (!strcmp(f1, "asURLs"))
form = HEADER_FORM_URLS;
else if (strcmp(f1, "all"))
is_valid = 0;
}
/* Validate requested header form */
if (is_valid && form != HEADER_FORM_RAW) {
enum _header_form allowed_form = (enum _header_form) \
hash_lookup(lcasename, &allowed_header_forms);
if (allowed_form != HEADER_FORM_UNKNOWN && form != allowed_form) {
is_valid = 0;
}
}
struct header_prop *hprop = NULL;
if (is_valid) {
hprop = xzmalloc(sizeof(struct header_prop));
hprop->lcasename = lcasename;
hprop->name = name;
hprop->prop = s;
hprop->form = form;
hprop->all = f2 != NULL || (f1 && !strcmp(f1, "all"));
}
else {
free(lcasename);
free(name);
}
strarray_free(fields);
return hprop;
}
/* Generate a preview of text of at most len bytes, excluding the zero
* byte.
*
* Consecutive whitespaces, including newlines, are collapsed to a single
* blank. If text is longer than len and len is greater than 4, then return
* a string ending in '...' and holding as many complete UTF-8 characters,
* that the total byte count of non-zero characters is at most len.
*
* The input string must be properly encoded UTF-8 */
static char *_email_extract_preview(const char *text, size_t len)
{
unsigned char *dst, *d, *t;
size_t n;
if (!text) {
return NULL;
}
/* Replace all whitespace with single blanks. */
dst = (unsigned char *) xzmalloc(len+1);
for (t = (unsigned char *) text, d = dst; *t && d < (dst+len); ++t, ++d) {
*d = isspace(*t) ? ' ' : *t;
if (isspace(*t)) {
while(isspace(*++t))
;
--t;
}
}
n = d - dst;
/* Anything left to do? */
if (n < len || len <= 4) {
return (char*) dst;
}
/* Append trailing ellipsis. */
dst[--n] = '.';
dst[--n] = '.';
dst[--n] = '.';
while (n && (dst[n] & 0xc0) == 0x80) {
dst[n+2] = 0;
dst[--n] = '.';
}
if (dst[n] >= 0x80) {
dst[n+2] = 0;
dst[--n] = '.';
}
return (char *) dst;
}
struct _email_mailboxes_rock {
jmap_req_t *req;
json_t *mboxs;
};
static int _email_mailboxes_cb(const conv_guidrec_t *rec, void *rock)
{
struct _email_mailboxes_rock *data = (struct _email_mailboxes_rock*) rock;
json_t *mboxs = data->mboxs;
jmap_req_t *req = data->req;
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
uint32_t system_flags, internal_flags;
int r;
if (rec->part) return 0;
static int needrights = ACL_READ|ACL_LOOKUP;
if (!jmap_hasrights_byname(req, rec->mboxname, needrights))
return 0;
r = jmap_openmbox(req, rec->mboxname, &mbox, 0);
if (r) return r;
// we only want regular mailboxes!
if (mbox->mbtype & MBTYPES_NONIMAP) goto done;
r = msgrecord_find(mbox, rec->uid, &mr);
if (r) goto done;
r = msgrecord_get_systemflags(mr, &system_flags);
if (r) goto done;
r = msgrecord_get_internalflags(mr, &internal_flags);
if (r) goto done;
if (!r) {
char datestr[RFC3339_DATETIME_MAX];
time_t t;
int exists = 1;
if (system_flags & FLAG_DELETED || internal_flags & FLAG_INTERNAL_EXPUNGED) {
exists = 0;
r = msgrecord_get_lastupdated(mr, &t);
}
else {
r = msgrecord_get_savedate(mr, &t);
}
if (r) goto done;
time_to_rfc3339(t, datestr, RFC3339_DATETIME_MAX);
json_t *mboxdata = json_object_get(mboxs, mbox->uniqueid);
if (!mboxdata) {
mboxdata = json_object();
json_object_set_new(mboxs, mbox->uniqueid, mboxdata);
}
if (exists) {
json_t *prev = json_object_get(mboxdata, "added");
if (prev) {
const char *val = json_string_value(prev);
// we want the FIRST date it was added to the mailbox, so skip if this is newer
if (strcmp(datestr, val) >= 0) goto done;
}
json_object_set_new(mboxdata, "added", json_string(datestr));
}
else {
json_t *prev = json_object_get(mboxdata, "removed");
if (prev) {
const char *val = json_string_value(prev);
// we want the LAST date it was removed from the mailbox, so skip if this is older
if (strcmp(datestr, val) <= 0) goto done;
}
json_object_set_new(mboxdata, "removed", json_string(datestr));
}
}
done:
if (mr) msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
return r;
}
static char *_emailbodies_to_plain(struct emailbodies *bodies, const struct buf *msg_buf)
{
if (bodies->textlist.count == 1) {
int is_encoding_problem = 0;
struct body *textbody = ptrarray_nth(&bodies->textlist, 0);
char *text = _decode_to_utf8(textbody->charset_id,
msg_buf->s + textbody->content_offset,
textbody->content_size,
textbody->encoding,
&is_encoding_problem);
return text;
}
/* Concatenate all plain text bodies and replace any
* inlined images with placeholders. */
int i;
struct buf buf = BUF_INITIALIZER;
for (i = 0; i < bodies->textlist.count; i++) {
struct body *part = ptrarray_nth(&bodies->textlist, i);
if (i) buf_appendcstr(&buf, "\n");
if (!strcmp(part->type, "TEXT")) {
int is_encoding_problem = 0;
char *t = _decode_to_utf8(part->charset_id,
msg_buf->s + part->content_offset,
part->content_size,
part->encoding,
&is_encoding_problem);
if (t) buf_appendcstr(&buf, t);
free(t);
}
else if (!strcmp(part->type, "IMAGE")) {
struct param *param;
const char *fname = NULL;
for (param = part->disposition_params; param; param = param->next) {
if (!strncasecmp(param->attribute, "filename", 8)) {
fname =param->value;
break;
}
}
buf_appendcstr(&buf, "[Inline image");
if (fname) {
buf_appendcstr(&buf, ":");
buf_appendcstr(&buf, fname);
}
buf_appendcstr(&buf, "]");
}
}
return buf_release(&buf);
}
/* Replace any <HTML> and </HTML> tags in t with <DIV> and </DIV>,
* writing results into buf */
static void _html_concat_div(struct buf *buf, const char *t)
{
const char *top = t + strlen(t);
const char *p = t, *q = p;
while (*q) {
const char *tag = NULL;
if (q < top - 5 && !strncasecmp(q, "<html", 5) &&
(*(q+5) == '>' || isspace(*(q+5)))) {
/* Found a <HTML> tag */
tag = "<div>";
}
else if (q < top - 6 && !strncasecmp(q, "</html", 6) &&
(*(q+6) == '>' || isspace(*(q+6)))) {
/* Found a </HTML> tag */
tag = "</div>";
}
/* No special tag? */
if (!tag) {
q++;
continue;
}
/* Append whatever we saw since the last HTML tag. */
buf_appendmap(buf, p, q - p);
/* Look for the end of the tag and replace it, even if
* it prematurely ends at the end of the buffer . */
while (*q && *q != '>') { q++; }
buf_appendcstr(buf, tag);
if (*q) q++;
/* Prepare for next loop */
p = q;
}
buf_appendmap(buf, p, q - p);
}
static char *_emailbodies_to_html(struct emailbodies *bodies, const struct buf *msg_buf)
{
if (bodies->htmllist.count == 1) {
const struct body *part = ptrarray_nth(&bodies->htmllist, 0);
int is_encoding_problem = 0;
char *html = _decode_to_utf8(part->charset_id,
msg_buf->s + part->content_offset,
part->content_size,
part->encoding,
&is_encoding_problem);
return html;
}
/* Concatenate all TEXT bodies, enclosing PLAIN text
* in <div> and replacing <html> tags in HTML bodies
* with <div>. */
int i;
struct buf buf = BUF_INITIALIZER;
for (i = 0; i < bodies->htmllist.count; i++) {
struct body *part = ptrarray_nth(&bodies->htmllist, i);
/* XXX htmllist might include inlined images but we
* currently ignore them. After all, there should
* already be an <img> tag for their Content-Id
* header value. If this turns out to be not enough,
* we can insert the <img> tags here. */
if (strcasecmp(part->type, "TEXT")) {
continue;
}
if (!i)
buf_appendcstr(&buf, "<html>"); // XXX use HTML5?
int is_encoding_problem = 0;
char *t = _decode_to_utf8(part->charset_id,
msg_buf->s + part->content_offset,
part->content_size,
part->encoding,
&is_encoding_problem);
if (t && !strcmp(part->subtype, "HTML")) {
_html_concat_div(&buf, t);
}
else if (t) {
buf_appendcstr(&buf, "<div>");
buf_appendcstr(&buf, t);
buf_appendcstr(&buf, "</div>");
}
free(t);
if (i == bodies->htmllist.count - 1)
buf_appendcstr(&buf, "</html>");
}
return buf_release(&buf);
}
static void _html_to_plain_cb(const struct buf *buf, void *rock)
{
struct buf *dst = (struct buf*) rock;
const char *p;
int seenspace = 0;
/* Just merge multiple space into one. That's similar to
* charset_extract's MERGE_SPACE but since we don't want
* it to canonify the text into search form */
for (p = buf_base(buf); p < buf_base(buf) + buf_len(buf) && *p; p++) {
if (*p == ' ') {
if (seenspace) continue;
seenspace = 1;
} else {
seenspace = 0;
}
buf_appendmap(dst, p, 1);
}
}
static char *_html_to_plain(const char *html) {
struct buf src = BUF_INITIALIZER;
struct buf dst = BUF_INITIALIZER;
charset_t utf8 = charset_lookupname("utf8");
char *text;
char *tmp, *q;
const char *p;
/* Replace <br> and <p> with newlines */
q = tmp = xstrdup(html);
p = html;
while (*p) {
if (!strncmp(p, "<br>", 4) || !strncmp(p, "</p>", 4)) {
*q++ = '\n';
p += 4;
}
else if (!strncmp(p, "p>", 3)) {
p += 3;
} else {
*q++ = *p++;
}
}
*q = 0;
/* Strip html tags */
buf_init_ro(&src, tmp, q - tmp);
buf_setcstr(&dst, "");
charset_extract(&_html_to_plain_cb, &dst,
&src, utf8, ENCODING_NONE, "HTML", CHARSET_KEEPCASE);
buf_cstring(&dst);
/* Trim text */
buf_trim(&dst);
text = buf_releasenull(&dst);
if (!strlen(text)) {
free(text);
text = NULL;
}
buf_free(&src);
free(tmp);
charset_free(&utf8);
return text;
}
static const char *_guid_from_id(const char *msgid)
{
return msgid + 1;
}
static conversation_id_t _cid_from_id(const char *thrid)
{
conversation_id_t cid = 0;
if (thrid[0] == 'T')
conversation_id_decode(&cid, thrid+1);
return cid;
}
/*
* Lookup all mailboxes where msgid is contained in.
*
* The return value is a JSON object keyed by the mailbox unique id,
* and its mailbox name as value.
*/
static json_t *_email_mailboxes(jmap_req_t *req, const char *msgid)
{
struct _email_mailboxes_rock data = { req, json_pack("{}") };
conversations_guid_foreach(req->cstate, _guid_from_id(msgid), _email_mailboxes_cb, &data);
return data.mboxs;
}
static void _email_read_annot(const jmap_req_t *req, msgrecord_t *mr,
const char *annot, struct buf *buf)
{
if (!strncmp(annot, "/shared/", 8)) {
msgrecord_annot_lookup(mr, annot+7, /*userid*/"", buf);
}
else if (!strncmp(annot, "/private/", 9)) {
msgrecord_annot_lookup(mr, annot+7, req->userid, buf);
}
else {
msgrecord_annot_lookup(mr, annot, "", buf);
}
}
static json_t *_email_read_jannot(const jmap_req_t *req, msgrecord_t *mr,
const char *annot, int structured)
{
struct buf buf = BUF_INITIALIZER;
json_t *annotvalue = NULL;
_email_read_annot(req, mr, annot, &buf);
if (buf_len(&buf)) {
if (structured) {
json_error_t jerr;
annotvalue = json_loads(buf_cstring(&buf), JSON_DECODE_ANY, &jerr);
/* XXX - log error? */
}
else {
annotvalue = json_string(buf_cstring(&buf));
}
if (!annotvalue) {
syslog(LOG_ERR, "jmap: annotation %s has bogus value", annot);
}
}
buf_free(&buf);
return annotvalue;
}
struct _email_find_rock {
jmap_req_t *req;
char *mboxname;
uint32_t uid;
};
static int _email_find_cb(const conv_guidrec_t *rec, void *rock)
{
struct _email_find_rock *d = (struct _email_find_rock*) rock;
jmap_req_t *req = d->req;
int r = 0;
if (rec->part) return 0;
if (!d->mboxname || jmap_isopenmbox(req, rec->mboxname)) {
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
uint32_t flags;
/* Make sure we are allowed to read this mailbox */
if (!jmap_hasrights_byname(req, rec->mboxname, ACL_READ))
return 0;
/* Prefer to use messages in already opened mailboxes */
r = jmap_openmbox(req, rec->mboxname, &mbox, 0);
if (r) return r;
r = msgrecord_find(mbox, rec->uid, &mr);
if (!r) {
uint32_t internal_flags;
r = msgrecord_get_systemflags(mr, &flags);
if (!r) msgrecord_get_internalflags(mr, &internal_flags);
if (!r && !(flags & FLAG_DELETED || internal_flags & FLAG_INTERNAL_EXPUNGED)) {
if (d->mboxname) {
free(d->mboxname);
r = IMAP_OK_COMPLETED;
}
d->mboxname = xstrdup(rec->mboxname);
d->uid = rec->uid;
}
msgrecord_unref(&mr);
}
jmap_closembox(req, &mbox);
}
return r;
}
static int _email_find_in_account(jmap_req_t *req,
const char *account_id,
const char *email_id,
char **mboxnameptr,
uint32_t *uidptr)
{
struct _email_find_rock rock = { req, NULL, 0 };
int r;
/* must be prefixed with 'M' */
if (email_id[0] != 'M')
return IMAP_NOTFOUND;
/* this is on a 24 character prefix only */
if (strlen(email_id) != 25)
return IMAP_NOTFOUND;
/* Open conversation state, if not already open */
struct conversations_state *mycstate = NULL;
if (strcmp(req->accountid, account_id)) {
r = conversations_open_user(account_id, 1/*shared*/, &mycstate);
if (r) return r;
}
else {
mycstate = req->cstate;
}
r = conversations_guid_foreach(mycstate, _guid_from_id(email_id),
_email_find_cb, &rock);
if (mycstate != req->cstate) {
conversations_commit(&mycstate);
}
/* Set return values */
if (r == IMAP_OK_COMPLETED)
r = 0;
else if (!rock.mboxname)
r = IMAP_NOTFOUND;
*mboxnameptr = rock.mboxname;
*uidptr = rock.uid;
return r;
}
HIDDEN int jmap_email_find(jmap_req_t *req,
const char *email_id,
char **mboxnameptr,
uint32_t *uidptr)
{
return _email_find_in_account(req, req->accountid, email_id, mboxnameptr, uidptr);
}
struct email_getcid_rock {
jmap_req_t *req;
int checkacl;
conversation_id_t cid;
};
static int _email_get_cid_cb(const conv_guidrec_t *rec, void *rock)
{
struct email_getcid_rock *d = (struct email_getcid_rock *)rock;
if (rec->part) return 0;
if (!rec->cid) return 0;
/* Make sure we are allowed to read this mailbox */
if (d->checkacl && !jmap_hasrights_byname(d->req, rec->mboxname, ACL_READ))
return 0;
d->cid = rec->cid;
return IMAP_OK_COMPLETED;
}
static int _email_get_cid(jmap_req_t *req, const char *msgid,
conversation_id_t *cidp)
{
int r;
/* must be prefixed with 'M' */
if (msgid[0] != 'M')
return IMAP_NOTFOUND;
/* this is on a 24 character prefix only */
if (strlen(msgid) != 25)
return IMAP_NOTFOUND;
int checkacl = strcmp(req->userid, req->accountid);
struct email_getcid_rock rock = { req, checkacl, 0 };
r = conversations_guid_foreach(req->cstate, _guid_from_id(msgid), _email_get_cid_cb, &rock);
if (r == IMAP_OK_COMPLETED) {
*cidp = rock.cid;
r = 0;
}
return r;
}
struct email_expunge_check {
jmap_req_t *req;
modseq_t since_modseq;
int status;
};
static int _email_is_expunged_cb(const conv_guidrec_t *rec, void *rock)
{
struct email_expunge_check *check = rock;
msgrecord_t *mr = NULL;
struct mailbox *mbox = NULL;
uint32_t flags;
int r = 0;
if (rec->part) return 0;
r = jmap_openmbox(check->req, rec->mboxname, &mbox, 0);
if (r) return r;
r = msgrecord_find(mbox, rec->uid, &mr);
if (!r) {
uint32_t internal_flags;
modseq_t createdmodseq;
r = msgrecord_get_systemflags(mr, &flags);
if (!r) msgrecord_get_internalflags(mr, &internal_flags);
if (!r) msgrecord_get_createdmodseq(mr, &createdmodseq);
if (!r) {
/* OK, this is a legit record, let's check it out */
if (createdmodseq <= check->since_modseq)
check->status |= 1; /* contains old messages */
if (!((flags & FLAG_DELETED) || (internal_flags & FLAG_INTERNAL_EXPUNGED)))
check->status |= 2; /* contains alive messages */
}
msgrecord_unref(&mr);
}
jmap_closembox(check->req, &mbox);
return 0;
}
static void _email_search_perf_attr(const search_attr_t *attr, strarray_t *perf_filters)
{
const char *cost = NULL;
switch (search_attr_cost(attr)) {
case SEARCH_COST_INDEX:
cost = "index";
break;
case SEARCH_COST_CONV:
cost = "conversations";
break;
case SEARCH_COST_ANNOT:
cost = "annotations";
break;
case SEARCH_COST_CACHE:
cost = "cache";
break;
case SEARCH_COST_BODY:
cost = search_attr_is_fuzzable(attr) ? "xapian" : "body";
break;
default:
; // ignore
}
if (cost) strarray_add(perf_filters, cost);
}
static void _email_search_string(search_expr_t *parent,
const char *s,
const char *name,
strarray_t *perf_filters)
{
charset_t utf8 = charset_lookupname("utf-8");
search_expr_t *e;
const search_attr_t *attr = search_attr_find(name);
enum search_op op;
assert(attr);
op = search_attr_is_fuzzable(attr) ? SEOP_FUZZYMATCH : SEOP_MATCH;
e = search_expr_new(parent, op);
e->attr = attr;
e->value.s = charset_convert(s, utf8, charset_flags);
if (!e->value.s) {
e->op = SEOP_FALSE;
e->attr = NULL;
}
_email_search_perf_attr(attr, perf_filters);
charset_free(&utf8);
}
static void _email_search_type(search_expr_t *parent, const char *s, strarray_t *perf_filters)
{
strarray_t types = STRARRAY_INITIALIZER;
/* Handle type wildcards */
// XXX: due to Xapian's 64 character indexing limitation, we're not prefixing application_
// to the Microsoft types
if (!strcasecmp(s, "image")) {
strarray_append(&types, "image_gif");
strarray_append(&types, "image_jpeg");
strarray_append(&types, "image_pjpeg");
strarray_append(&types, "image_jpg");
strarray_append(&types, "image_png");
strarray_append(&types, "image_bmp");
strarray_append(&types, "image_tiff");
}
else if (!strcasecmp(s, "document")) {
strarray_append(&types, "application_msword");
strarray_append(&types, "vnd.openxmlformats-officedocument.wordprocessingml.document");
strarray_append(&types, "vnd.openxmlformats-officedocument.wordprocessingml.template");
strarray_append(&types, "application_vnd.sun.xml.writer");
strarray_append(&types, "application_vnd.sun.xml.writer.template");
strarray_append(&types, "application_vnd.oasis.opendocument.text");
strarray_append(&types, "application_vnd.oasis.opendocument.text-template");
strarray_append(&types, "application_x-iwork-pages-sffpages");
strarray_append(&types, "application_vnd.apple.pages");
}
else if (!strcasecmp(s, "spreadsheet")) {
strarray_append(&types, "application_vnd.ms-excel");
strarray_append(&types, "vnd.openxmlformats-officedocument.spreadsheetml.sheet");
strarray_append(&types, "vnd.openxmlformats-officedocument.spreadsheetml.template");
strarray_append(&types, "application_vnd.sun.xml.calc");
strarray_append(&types, "application_vnd.sun.xml.calc.template");
strarray_append(&types, "application_vnd.oasis.opendocument.spreadsheet");
strarray_append(&types, "application_vnd.oasis.opendocument.spreadsheet-template");
strarray_append(&types, "application_x-iwork-numbers-sffnumbers");
strarray_append(&types, "application_vnd.apple.numbers");
}
else if (!strcasecmp(s, "presentation")) {
strarray_append(&types, "application_vnd.ms-powerpoint");
strarray_append(&types, "vnd.openxmlformats-officedocument.presentationml.presentation");
strarray_append(&types, "vnd.openxmlformats-officedocument.presentationml.template");
strarray_append(&types, "vnd.openxmlformats-officedocument.presentationml.slideshow");
strarray_append(&types, "application_vnd.sun.xml.impress");
strarray_append(&types, "application_vnd.sun.xml.impress.template");
strarray_append(&types, "application_vnd.oasis.opendocument.presentation");
strarray_append(&types, "application_vnd.oasis.opendocument.presentation-template");
strarray_append(&types, "application_x-iwork-keynote-sffkey");
strarray_append(&types, "application_vnd.apple.keynote");
}
else if (!strcasecmp(s, "email")) {
strarray_append(&types, "message_rfc822");
}
else if (!strcasecmp(s, "pdf")) {
strarray_append(&types, "application_pdf");
}
else {
/* FUZZY contenttype is indexed as `type_subtype` */
char *tmp = xstrdup(s);
char *p = strchr(tmp, '/');
if (p) *p = '_';
strarray_append(&types, tmp);
free(tmp);
}
/* Build expression */
search_expr_t *p = (types.count > 1) ? search_expr_new(parent, SEOP_OR) : parent;
const search_attr_t *attr = search_attr_find("contenttype");
do {
search_expr_t *e = search_expr_new(p, SEOP_FUZZYMATCH);
e->attr = attr;
struct buf buf = BUF_INITIALIZER;
char *orig = strarray_pop(&types);
const unsigned char *s = (const unsigned char *)orig;
for ( ; *s ; ++s) {
if (Uisalnum(*s) || *s == '_')
buf_putc(&buf, *s);
}
e->value.s = buf_release(&buf);
free(orig);
buf_free(&buf);
} while (types.count);
_email_search_perf_attr(attr, perf_filters);
strarray_fini(&types);
}
static void _email_search_keyword(search_expr_t *parent, const char *keyword, strarray_t *perf_filters)
{
search_expr_t *e;
if (!strcasecmp(keyword, "$Seen")) {
e = search_expr_new(parent, SEOP_MATCH);
e->attr = search_attr_find("indexflags");
e->value.u = MESSAGE_SEEN;
}
else if (!strcasecmp(keyword, "$Draft")) {
e = search_expr_new(parent, SEOP_MATCH);
e->attr = search_attr_find("systemflags");
e->value.u = FLAG_DRAFT;
}
else if (!strcasecmp(keyword, "$Flagged")) {
e = search_expr_new(parent, SEOP_MATCH);
e->attr = search_attr_find("systemflags");
e->value.u = FLAG_FLAGGED;
}
else if (!strcasecmp(keyword, "$Answered")) {
e = search_expr_new(parent, SEOP_MATCH);
e->attr = search_attr_find("systemflags");
e->value.u = FLAG_ANSWERED;
}
else {
e = search_expr_new(parent, SEOP_MATCH);
e->attr = search_attr_find("keyword");
e->value.s = xstrdup(keyword);
}
_email_search_perf_attr(e->attr, perf_filters);
}
static void _email_search_threadkeyword(search_expr_t *parent, const char *keyword,
int matchall, strarray_t *perf_filters)
{
const char *flag = jmap_keyword_to_imap(keyword);
if (!flag) return;
search_expr_t *e = search_expr_new(parent, SEOP_MATCH);
e->attr = search_attr_find(matchall ? "allconvflags" : "convflags");
e->value.s = xstrdup(flag);
_email_search_perf_attr(e->attr, perf_filters);
}
static void _email_search_contactgroup(search_expr_t *parent,
const char *groupid,
const char *attrname,
hash_table *contactgroups,
strarray_t *perf_filters)
{
if (!contactgroups || !contactgroups->size) return;
strarray_t *members = hash_lookup(groupid, contactgroups);
if (members && strarray_size(members)) {
search_expr_t *e = search_expr_new(parent, SEOP_OR);
int j;
for (j = 0; j < strarray_size(members); j++) {
_email_search_string(e, strarray_nth(members, j),
attrname, perf_filters);
}
}
}
/* ====================================================================== */
static void _emailsearch_folders_internalise(struct index_state *state,
const union search_value *v,
void **internalisedp)
{
if (state && v) {
*internalisedp = mailbox_get_cstate(state->mailbox);
}
}
struct jmap_search_folder_match_rock {
const strarray_t *folders;
intptr_t is_otherthan;
};
static int _emailsearch_folders_match_cb(const conv_guidrec_t *rec, void *rock)
{
if ((rec->system_flags & FLAG_DELETED) ||
(rec->internal_flags & FLAG_INTERNAL_EXPUNGED)) return 0;
// TODO we could match for mboxid, once the mailbox-id patch lands
struct jmap_search_folder_match_rock *myrock = rock;
int pos = strarray_find(myrock->folders, rec->mboxname, 0);
return ((pos >= 0) == (myrock->is_otherthan == 0)) ? IMAP_OK_COMPLETED : 0;
}
static int _emailsearch_folders_match(message_t *m, const union search_value *v,
void *internalised,
void *data1)
{
struct conversations_state *cstate = internalised;
if (!cstate) return 0;
const struct message_guid *guid = NULL;
int r = message_get_guid(m, &guid);
if (r) return 0;
struct jmap_search_folder_match_rock rock = { v->rock, (intptr_t) data1 };
r = conversations_guid_foreach(cstate, message_guid_encode(guid),
_emailsearch_folders_match_cb, &rock);
return r == IMAP_OK_COMPLETED;
}
static void _emailsearch_folders_serialise(struct buf *buf,
const union search_value *v)
{
char *tmp = strarray_join((strarray_t*)v->rock, " ");
buf_putc(buf, '(');
buf_appendcstr(buf, tmp);
buf_putc(buf, ')');
free(tmp);
}
static int _emailsearch_folders_unserialise(struct protstream* prot,
union search_value *v)
{
struct dlist *dl = NULL;
int c = dlist_parse_asatomlist(&dl, 0, prot);
if (c == EOF) return EOF;
strarray_t *folders = strarray_new();
struct buf tmp = BUF_INITIALIZER;
struct dlist_print_iter *iter = dlist_print_iter_new(dl, /*printkeys*/ 0);
while (iter && dlist_print_iter_step(iter, &tmp)) {
if (buf_len(&tmp)) strarray_append(folders, buf_cstring(&tmp));
buf_reset(&tmp);
}
dlist_print_iter_free(&iter);
buf_free(&tmp);
v->rock = folders;
return c;
}
static void _emailsearch_folders_duplicate(union search_value *new,
const union search_value *old)
{
new->rock = strarray_dup((strarray_t*)old->rock);
}
static void _emailsearch_folders_free(union search_value *v)
{
strarray_free(v->rock);
}
static const search_attr_t _emailsearch_folders_attr = {
"jmap_folders",
SEA_MUTABLE,
SEARCH_PART_NONE,
SEARCH_COST_CONV,
_emailsearch_folders_internalise,
/*cmp*/NULL,
_emailsearch_folders_match,
_emailsearch_folders_serialise,
_emailsearch_folders_unserialise,
/*get_countability*/NULL,
_emailsearch_folders_duplicate,
_emailsearch_folders_free,
(void*)0 /*is_otherthan*/
};
static const search_attr_t _emailsearch_folders_otherthan_attr = {
"jmap_folders_otherthan",
SEA_MUTABLE,
SEARCH_PART_NONE,
SEARCH_COST_CONV,
_emailsearch_folders_internalise,
/*cmp*/NULL,
_emailsearch_folders_match,
_emailsearch_folders_serialise,
_emailsearch_folders_unserialise,
/*get_countability*/NULL,
_emailsearch_folders_duplicate,
_emailsearch_folders_free,
(void*)1 /*is_otherthan*/
};
/* ====================================================================== */
static search_expr_t *_email_buildsearchexpr(jmap_req_t *req, json_t *filter,
search_expr_t *parent,
hash_table *contactgroups,
strarray_t *perf_filters)
{
search_expr_t *this, *e;
json_t *val;
const char *s;
size_t i;
time_t t;
if (!JNOTNULL(filter)) {
return search_expr_new(parent, SEOP_TRUE);
}
if ((s = json_string_value(json_object_get(filter, "operator")))) {
enum search_op op = SEOP_UNKNOWN;
if (!strcmp("AND", s)) {
op = SEOP_AND;
} else if (!strcmp("OR", s)) {
op = SEOP_OR;
} else if (!strcmp("NOT", s)) {
op = SEOP_NOT;
}
this = search_expr_new(parent, op);
e = op == SEOP_NOT ? search_expr_new(this, SEOP_OR) : this;
json_array_foreach(json_object_get(filter, "conditions"), i, val) {
_email_buildsearchexpr(req, val, e, contactgroups, perf_filters);
}
} else {
this = search_expr_new(parent, SEOP_AND);
/* zero properties evaluate to true */
search_expr_new(this, SEOP_TRUE);
if ((s = json_string_value(json_object_get(filter, "after")))) {
time_from_iso8601(s, &t);
e = search_expr_new(this, SEOP_GE);
e->attr = search_attr_find("internaldate");
e->value.u = t;
_email_search_perf_attr(e->attr, perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "before")))) {
time_from_iso8601(s, &t);
e = search_expr_new(this, SEOP_LE);
e->attr = search_attr_find("internaldate");
e->value.u = t;
_email_search_perf_attr(e->attr, perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "body")))) {
_email_search_string(this, s, "body", perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "cc")))) {
_email_search_string(this, s, "cc", perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "bcc")))) {
_email_search_string(this, s, "bcc", perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "from")))) {
_email_search_string(this, s, "from", perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "fromContactGroupId")))) {
_email_search_contactgroup(this, s, "from", contactgroups, perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "toContactGroupId")))) {
_email_search_contactgroup(this, s, "to", contactgroups, perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "ccContactGroupId")))) {
_email_search_contactgroup(this, s, "cc", contactgroups, perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "bccContactGroupId")))) {
_email_search_contactgroup(this, s, "bcc", contactgroups, perf_filters);
}
if (JNOTNULL((val = json_object_get(filter, "hasAttachment")))) {
e = val == json_false() ? search_expr_new(this, SEOP_NOT) : this;
e = search_expr_new(e, SEOP_MATCH);
e->attr = search_attr_find("keyword");
e->value.s = xstrdup(JMAP_HAS_ATTACHMENT_FLAG);
_email_search_perf_attr(e->attr, perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "attachmentName")))) {
_email_search_string(this, s, "attachmentname", perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "attachmentType")))) {
_email_search_type(this, s, perf_filters);
}
if (JNOTNULL((val = json_object_get(filter, "header")))) {
const char *k, *v;
charset_t utf8 = charset_lookupname("utf-8");
search_expr_t *e;
if (json_array_size(val) == 2) {
k = json_string_value(json_array_get(val, 0));
v = json_string_value(json_array_get(val, 1));
} else {
k = json_string_value(json_array_get(val, 0));
v = ""; /* Empty string matches any value */
}
e = search_expr_new(this, SEOP_MATCH);
e->attr = search_attr_find_field(k);
e->value.s = charset_convert(v, utf8, charset_flags);
if (!e->value.s) {
e->op = SEOP_FALSE;
e->attr = NULL;
}
_email_search_perf_attr(e->attr, perf_filters);
charset_free(&utf8);
}
if ((val = json_object_get(filter, "inMailbox"))) {
strarray_t *folders = strarray_new();
const char *mboxid = json_string_value(val);
mbentry_t *mbentry = _mbentry_by_uniqueid(req, mboxid);
if (mbentry && jmap_hasrights(req, mbentry, ACL_LOOKUP)) {
strarray_append(folders, mbentry->name);
}
mboxlist_entry_free(&mbentry);
if (strarray_size(folders)) {
search_expr_t *e = search_expr_new(this, SEOP_MATCH);
e->attr = &_emailsearch_folders_attr;
e->value.rock = folders;
strarray_add(perf_filters, "mailbox");
}
}
if ((val = json_object_get(filter, "inMailboxOtherThan"))) {
strarray_t *folders = strarray_new();
json_t *jmboxid;
json_array_foreach(val, i, jmboxid) {
const char *mboxid = json_string_value(jmboxid);
mbentry_t *mbentry = _mbentry_by_uniqueid(req, mboxid);
if (mbentry && jmap_hasrights(req, mbentry, ACL_LOOKUP)) {
strarray_append(folders, mbentry->name);
}
mboxlist_entry_free(&mbentry);
}
if (strarray_size(folders)) {
search_expr_t *e = search_expr_new(this, SEOP_MATCH);
e->attr = &_emailsearch_folders_otherthan_attr;
e->value.rock = folders;
strarray_add(perf_filters, "mailbox");
}
}
if (JNOTNULL((val = json_object_get(filter, "allInThreadHaveKeyword")))) {
/* This shouldn't happen, validate_sort should have reported
* allInThreadHaveKeyword as unsupported. Let's ignore this
* filter and return false positives. */
_email_search_threadkeyword(this, json_string_value(val), 1, perf_filters);
}
if (JNOTNULL((val = json_object_get(filter, "someInThreadHaveKeyword")))) {
_email_search_threadkeyword(this, json_string_value(val), 0, perf_filters);
}
if (JNOTNULL((val = json_object_get(filter, "noneInThreadHaveKeyword")))) {
e = search_expr_new(this, SEOP_NOT);
_email_search_threadkeyword(e, json_string_value(val), 0, perf_filters);
}
if (JNOTNULL((val = json_object_get(filter, "hasKeyword")))) {
_email_search_keyword(this, json_string_value(val), perf_filters);
}
if (JNOTNULL((val = json_object_get(filter, "notKeyword")))) {
e = search_expr_new(this, SEOP_NOT);
_email_search_keyword(e, json_string_value(val), perf_filters);
}
if (JNOTNULL((val = json_object_get(filter, "maxSize")))) {
e = search_expr_new(this, SEOP_LE);
e->attr = search_attr_find("size");
e->value.u = json_integer_value(val);
_email_search_perf_attr(e->attr, perf_filters);
}
if (JNOTNULL((val = json_object_get(filter, "minSize")))) {
e = search_expr_new(this, SEOP_GE);
e->attr = search_attr_find("size");
e->value.u = json_integer_value(val);
_email_search_perf_attr(e->attr, perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "sinceEmailState")))) {
/* non-standard */
e = search_expr_new(this, SEOP_GT);
e->attr = search_attr_find("modseq");
e->value.u = atomodseq_t(s);
_email_search_perf_attr(e->attr, perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "subject")))) {
_email_search_string(this, s, "subject", perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "text")))) {
_email_search_string(this, s, "text", perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "attachmentBody")))) {
_email_search_string(this, s, "attachmentbody", perf_filters);
}
if ((s = json_string_value(json_object_get(filter, "to")))) {
_email_search_string(this, s, "to", perf_filters);
}
}
return this;
}
static search_expr_t *_email_buildsearch(jmap_req_t *req, json_t *filter,
hash_table *contactgroups,
strarray_t *perf_filters)
{
search_expr_t *root = _email_buildsearchexpr(req, filter, /*parent*/NULL,
contactgroups, perf_filters);
/* The search API internally optimises for IMAP folder queries
* and we'd like to benefit from this also for JMAP. To do so,
* we try to convert as many inMailboxId expressions to IMAP
* mailbox matches as possible. This includes the first
* inMailboxId that is part of a positive AND and any inMailboxId
* that is part of a positive OR expression. */
ptrarray_t todo = PTRARRAY_INITIALIZER;
int found_and_match = 0;
struct work {
search_expr_t *e;
int in_or;
};
struct work *w = xmalloc(sizeof(struct work));
w->e = root;
w->in_or = 0;
ptrarray_push(&todo, w);
while ((w = ptrarray_pop(&todo))) {
if (w->e->op == SEOP_MATCH) {
if (!strcmp(w->e->attr->name, "jmap_folders") &&
w->e->attr->data1 == 0 &&
strarray_size((strarray_t*)w->e->value.rock) == 1) {
/* Its's an inMailboxId expression */
if (w->in_or || !found_and_match) {
char *folder = strarray_pop((strarray_t*)w->e->value.rock);
_emailsearch_folders_free(&w->e->value);
const search_attr_t *attr = search_attr_find("folder");
w->e->value.s = folder;
w->e->attr = attr;
found_and_match = !w->in_or;
}
}
}
else if (w->e->op == SEOP_AND || w->e->op == SEOP_OR) {
search_expr_t *c;
for (c = w->e->children; c; c = c->next) {
struct work *ww = xmalloc(sizeof(struct work));
ww->e = c;
ww->in_or = w->in_or || w->e->op == SEOP_OR;
ptrarray_push(&todo, ww);
}
}
free(w);
}
ptrarray_fini(&todo);
return root;
}
static void _email_contactfilter_initreq(jmap_req_t *req, struct email_contactfilter *cfilter)
{
const char *addressbookid = json_string_value(json_object_get(req->args, "addressbookId"));
jmap_email_contactfilter_init(req->accountid, addressbookid, cfilter);
}
static void _email_parse_filter_cb(jmap_req_t *req,
struct jmap_parser *parser,
json_t *filter,
json_t *unsupported,
void *rock,
json_t **err)
{
struct email_contactfilter *cfilter = rock;
/* Parse filter */
jmap_email_filtercondition_parse(parser, filter, unsupported,
req->using_capabilities);
if (json_array_size(parser->invalid)) return;
/* Gather contactgroups */
int r = jmap_email_contactfilter_from_filtercondition(parser, filter, cfilter);
if (r) {
*err = jmap_server_error(r);
return;
}
const char *field;
json_t *arg;
/* Validate permissions */
json_object_foreach(filter, field, arg) {
if (!strcmp(field, "inMailbox")) {
if (json_is_string(arg)) {
mbentry_t *mbentry = _mbentry_by_uniqueid(req, json_string_value(arg));
if (!mbentry || !jmap_hasrights(req, mbentry, ACL_LOOKUP)) {
jmap_parser_invalid(parser, field);
}
mboxlist_entry_free(&mbentry);
}
}
else if (!strcmp(field, "inMailboxOtherThan")) {
if (json_is_array(arg)) {
size_t i;
json_t *val;
json_array_foreach(arg, i, val) {
const char *s = json_string_value(val);
int is_valid = 0;
if (s) {
mbentry_t *mbentry = _mbentry_by_uniqueid(req, s);
is_valid = mbentry && jmap_hasrights(req, mbentry, ACL_LOOKUP);
mboxlist_entry_free(&mbentry);
}
if (!is_valid) {
jmap_parser_push_index(parser, field, i, s);
jmap_parser_invalid(parser, NULL);
jmap_parser_pop(parser);
}
}
}
}
}
}
static struct sortcrit *_email_buildsort(json_t *sort, int *sort_savedate)
{
json_t *jcomp;
size_t i;
struct sortcrit *sortcrit;
if (!JNOTNULL(sort) || json_array_size(sort) == 0) {
sortcrit = xzmalloc(2 * sizeof(struct sortcrit));
sortcrit[0].flags |= SORT_REVERSE;
sortcrit[0].key = SORT_ARRIVAL;
sortcrit[1].flags |= SORT_REVERSE;
sortcrit[1].key = SORT_SEQUENCE;
return sortcrit;
}
sortcrit = xzmalloc((json_array_size(sort) + 1) * sizeof(struct sortcrit));
json_array_foreach(sort, i, jcomp) {
const char *prop = json_string_value(json_object_get(jcomp, "property"));
if (json_object_get(jcomp, "isAscending") == json_false()) {
sortcrit[i].flags |= SORT_REVERSE;
}
/* Note: add any new sort criteria also to is_supported_msglist_sort */
if (!strcmp(prop, "receivedAt")) {
sortcrit[i].key = SORT_ARRIVAL;
}
if (!strcmp(prop, "sentAt")) {
sortcrit[i].key = SORT_DATE;
}
if (!strcmp(prop, "from")) {
sortcrit[i].key = SORT_DISPLAYFROM;
}
if (!strcmp(prop, "id")) {
sortcrit[i].key = SORT_GUID;
}
if (!strcmp(prop, "emailState")) {
sortcrit[i].key = SORT_MODSEQ;
}
if (!strcmp(prop, "size")) {
sortcrit[i].key = SORT_SIZE;
}
if (!strcmp(prop, "subject")) {
sortcrit[i].key = SORT_SUBJECT;
}
if (!strcmp(prop, "to")) {
sortcrit[i].key = SORT_DISPLAYTO;
}
if (!strcmp(prop, "hasKeyword")) {
const char *name = json_string_value(json_object_get(jcomp, "keyword"));
const char *flagname = jmap_keyword_to_imap(name);
if (flagname) {
sortcrit[i].key = SORT_HASFLAG;
sortcrit[i].args.flag.name = xstrdup(flagname);
}
}
if (!strcmp(prop, "someInThreadHaveKeyword")) {
const char *name = json_string_value(json_object_get(jcomp, "keyword"));
const char *flagname = jmap_keyword_to_imap(name);
if (flagname) {
sortcrit[i].key = SORT_HASCONVFLAG;
sortcrit[i].args.flag.name = xstrdup(flagname);
}
}
// FM specific
- if (!strcmp(prop, "addedDates")) {
+ if (!strcmp(prop, "addedDates") || !strcmp(prop, "snoozedUntil")) {
const char *mboxid =
json_string_value(json_object_get(jcomp, "mailboxId"));
if (sort_savedate) *sort_savedate = 1;
- sortcrit[i].key = SORT_SAVEDATE;
+ sortcrit[i].key = (*prop == 's') ? SORT_SNOOZEDUNTIL : SORT_SAVEDATE;
sortcrit[i].args.mailbox.id = xstrdupnull(mboxid);
}
if (!strcmp(prop, "threadSize")) {
sortcrit[i].key = SORT_CONVSIZE;
}
if (!strcmp(prop, "spamScore")) {
sortcrit[i].key = SORT_SPAMSCORE;
}
- if (!strcmp(prop, "snoozedUntil")) {
- const char *mboxid =
- json_string_value(json_object_get(jcomp, "mailboxId"));
-
- sortcrit[i].key = SORT_SNOOZEDUNTIL;
- sortcrit[i].args.mailbox.id = xstrdupnull(mboxid);
- }
}
i = json_array_size(sort);
sortcrit[i].key = SORT_SEQUENCE;
return sortcrit;
}
static void _email_querychanges_added(struct jmap_querychanges *query,
const char *email_id)
{
json_t *item = json_pack("{s:s,s:i}", "id", email_id, "index", query->total-1);
json_array_append_new(query->added, item);
}
static void _email_querychanges_destroyed(struct jmap_querychanges *query,
const char *email_id)
{
json_array_append_new(query->removed, json_string(email_id));
}
struct emailsearch {
int is_mutable;
char *hash;
strarray_t perf_filters;
/* Internal state */
search_query_t *query;
struct searchargs *args;
struct index_state *state;
struct sortcrit *sortcrit;
struct index_init init;
ptrarray_t *cached_msgdata;
};
static void _emailsearch_free(struct emailsearch *search)
{
if (!search) return;
index_close(&search->state);
search_query_free(search->query);
freesearchargs(search->args);
freesortcrit(search->sortcrit);
free(search->hash);
strarray_fini(&search->perf_filters);
free(search);
}
static char *_emailsearch_hash(struct emailsearch *search)
{
struct buf buf = BUF_INITIALIZER;
if (search->args->root) {
search_expr_t *mysearch = search_expr_duplicate(search->args->root);
search_expr_normalise(&mysearch);
char *tmp = search_expr_serialise(mysearch);
buf_appendcstr(&buf, tmp);
free(tmp);
search_expr_free(mysearch);
}
else {
buf_appendcstr(&buf, "noquery");
}
if (search->query->sortcrit) {
char *tmp = sortcrit_as_string(search->query->sortcrit);
buf_appendcstr(&buf, tmp);
free(tmp);
}
else {
buf_appendcstr(&buf, "nosort");
}
unsigned char raw_sha1[SHA1_DIGEST_LENGTH];
xsha1((const unsigned char *) buf_base(&buf), buf_len(&buf), raw_sha1);
size_t hex_size = (SHA1_DIGEST_LENGTH << 1);
char hex_sha1[hex_size + 1];
bin_to_lchex(raw_sha1, SHA1_DIGEST_LENGTH, hex_sha1);
hex_sha1[hex_size] = '\0';
buf_free(&buf);
return xstrdup(hex_sha1);
}
#define FNAME_EMAILSEARCH_DB "/jmap_emailsearch.db"
#define EMAILSEARCH_DB "twoskip"
static char *emailsearch_getcachepath(void)
{
return xstrdupnull(config_getstring(IMAPOPT_JMAP_EMAILSEARCH_DB_PATH));
}
static int _jmap_checkfolder(const char *mboxname, void *rock)
{
jmap_req_t *req = (jmap_req_t *)rock;
// we only want to look in folders that the user is allowed to read
if (jmap_hasrights_byname(req, mboxname, ACL_READ))
return 1;
return 0;
}
static struct emailsearch* _emailsearch_new(jmap_req_t *req,
json_t *filter,
json_t *sort,
hash_table *contactgroups,
int want_expunged,
int ignore_timer,
int *sort_savedate)
{
struct emailsearch* search = xzmalloc(sizeof(struct emailsearch));
int r = 0;
/* Build search args */
search->args = new_searchargs(NULL/*tag*/, GETSEARCH_CHARSET_FIRST,
&jmap_namespace, req->accountid, req->authstate, 0);
search->args->root = _email_buildsearch(req, filter, contactgroups, &search->perf_filters);
/* Build index state */
search->init.userid = req->accountid;
search->init.authstate = req->authstate;
search->init.want_expunged = want_expunged;
search->init.examine_mode = 1;
char *inboxname = mboxname_user_mbox(req->accountid, NULL);
r = index_open(inboxname, &search->init, &search->state);
free(inboxname);
if (r) {
syslog(LOG_ERR, "jmap: _emailsearch_new: %s", error_message(r));
freesearchargs(search->args);
free(search);
return NULL;
}
/* Build query */
search->query = search_query_new(search->state, search->args);
search->query->sortcrit =
search->sortcrit = _email_buildsort(sort, sort_savedate);
search->query->multiple = 1;
search->query->need_ids = 1;
search->query->verbose = 0;
search->query->want_expunged = want_expunged;
search->query->ignore_timer = ignore_timer;
search->query->checkfolder = _jmap_checkfolder;
search->query->checkfolderrock = req;
search->query->attachments_in_any = jmap_is_using(req, JMAP_SEARCH_EXTENSION);
search->is_mutable = search_is_mutable(search->sortcrit, search->args);
/* Make hash */
search->hash = _emailsearch_hash(search);
return search;
}
static int _emailsearch_run(struct emailsearch *search, const ptrarray_t **msgdataptr)
{
int r = search_query_run(search->query);
if (r) {
syslog(LOG_ERR, "jmap: _emailsearch_run: %s", error_message(r));
return r;
}
*msgdataptr = &search->query->merged_msgdata;
return 0;
}
static const char *msglist_sortfields_standard[] = {
"receivedAt",
"sentAt",
"from",
"id",
"emailstate",
"size",
"subject",
"to",
"hasKeyword",
"someInThreadHaveKeyword",
NULL
};
// FM specific
static const char *msglist_sortfields_nonstandard[] = {
"addedDates",
"threadSize",
"spamScore",
"snoozedUntil",
NULL
};
static int _email_parse_comparator(jmap_req_t *req __attribute__((unused)),
struct jmap_comparator *comp,
void *rock __attribute__((unused)),
json_t **err __attribute__((unused)))
{
/* Reject any collation */
if (comp->collation) {
return 0;
}
/* Search in list of supported sortFields */
const char **sp;
for (sp = msglist_sortfields_standard; *sp; sp++) {
if (!strcmp(*sp, comp->property)) {
return 1;
}
}
// XXX - we should really be looking in "using" to know which to support
for (sp = msglist_sortfields_nonstandard; *sp; sp++) {
if (!strcmp(*sp, comp->property)) {
return 1;
}
}
return 0;
}
static char *_email_make_querystate(modseq_t modseq, uint32_t uid, modseq_t addrbook_modseq)
{
struct buf buf = BUF_INITIALIZER;
buf_printf(&buf, MODSEQ_FMT ":%u", modseq, uid);
if (addrbook_modseq) {
buf_printf(&buf, ",addrbook:" MODSEQ_FMT, addrbook_modseq);
}
return buf_release(&buf);
}
static int _email_read_querystate(const char *s, modseq_t *modseq, uint32_t *uid,
modseq_t *addrbook_modseq)
{
char sentinel = 0;
/* Parse mailbox modseq and uid */
int n = sscanf(s, MODSEQ_FMT ":%u%c", modseq, uid, &sentinel);
if (n <= 2) return n == 2;
else if (sentinel != ',') return 0;
/* Parse addrbook modseq */
s = strchr(s, ',') + 1;
if (strncmp(s, "addrbook:", 9)) return 0;
s += 9;
n = sscanf(s, MODSEQ_FMT "%c", addrbook_modseq, &sentinel);
if (n != 1) return 0;
/* Parsed successfully */
return 1;
}
struct cached_emailquery {
char *ids; /* zero-terminated id strings */
size_t ids_count; /* count of ids in ids array */
size_t id_size; /* byte-length of an id, excluding 0 byte */
};
#define _CACHED_EMAILQUERY_INITIALIZER { NULL, 0, 0 }
static void _cached_emailquery_fini(struct cached_emailquery *cache_record)
{
free(cache_record->ids);
}
#define _EMAILSEARCH_CACHE_VERSION 0x2
static int _email_query_writecache(struct db *cache_db,
const char *cache_key,
modseq_t current_modseq,
strarray_t *email_ids)
{
int r = 0;
/* Serialise cache record preamble */
struct buf buf = BUF_INITIALIZER;
buf_appendbit32(&buf, _EMAILSEARCH_CACHE_VERSION);
buf_appendbit64(&buf, current_modseq);
/* Serialise email ids */
buf_appendbit64(&buf, strarray_size(email_ids));
if (strarray_size(email_ids)) {
const char *email_id = strarray_nth(email_ids, 0);
size_t email_id_len = strlen(email_id);
buf_appendbit64(&buf, email_id_len);
int i;
for (i = 0; i < strarray_size(email_ids); i++) {
const char *email_id = strarray_nth(email_ids, i);
if (strlen(email_id) != email_id_len) {
syslog(LOG_ERR, "jmap: email id %s has length %zd,"
"expected %zd - aborting cache",
email_id, strlen(email_id), email_id_len);
r = CYRUSDB_INTERNAL;
goto done;
}
buf_appendcstr(&buf, email_id);
buf_putc(&buf, '\0');
}
}
/* Store cache record */
r = cyrusdb_store(cache_db, cache_key, strlen(cache_key),
buf_base(&buf), buf_len(&buf), NULL);
done:
buf_free(&buf);
return r;
}
static int _email_query_readcache(struct db *cache_db,
const char *cache_key,
modseq_t current_modseq,
struct cached_emailquery *cache_record)
{
/* Load cache record */
const char *data = NULL;
size_t datalen = 0;
int r = cyrusdb_fetch(cache_db, cache_key, strlen(cache_key), &data, &datalen, NULL);
if (r) {
if (r != CYRUSDB_NOTFOUND) {
syslog(LOG_ERR, "jmap: can't fetch cached email search (%s): %s",
cache_key, cyrusdb_strerror(r));
}
return r;
}
/* Read cache record preamble */
const char *p = data;
uint32_t version = ntohl(((bit32*)(p))[0]); p += 4;
if (version != _EMAILSEARCH_CACHE_VERSION) {
syslog(LOG_ERR, "jmap: unexpected cache version %d (%s)", version, cache_key);
r = CYRUSDB_EXISTS;
goto done;
}
modseq_t cached_modseq = ntohll(((bit64*)(p))[0]); p += 8;
if (cached_modseq != current_modseq) {
r = CYRUSDB_EXISTS;
goto done;
}
/* Read email ids */
size_t ids_count = ntohll(((bit64*)(p))[0]); p += 8;
cache_record->ids_count= ids_count;
if (ids_count) {
size_t id_size = ntohll(((bit64*)(p))[0]); p += 8;
cache_record->id_size = id_size;
size_t ids_size = ids_count * (id_size + 1);
cache_record->ids = xmalloc(ids_size);
memcpy(cache_record->ids, p, ids_size);
p += ids_size;
}
/* Check end of record */
if (p != data + datalen) {
syslog(LOG_ERR, "jmap: invalid query cache entry %s", cache_key);
r = CYRUSDB_NOTFOUND;
goto done;
}
done:
if (r) {
_cached_emailquery_fini(cache_record);
cyrusdb_delete(cache_db, cache_key, strlen(cache_key), NULL, 0);
return r == CYRUSDB_EXISTS? CYRUSDB_NOTFOUND : r;
}
return 0;
}
static void _email_query(jmap_req_t *req, struct jmap_query *query,
int collapse_threads,
hash_table *contactgroups,
json_t **jemailpartids, json_t **err)
{
char *cache_fname = NULL;
char *cache_key = NULL;
struct db *cache_db = NULL;
modseq_t current_modseq = jmap_highestmodseq(req, MBTYPE_EMAIL);
int is_cached = 0;
struct emailsearch *search = _emailsearch_new(req, query->filter, query->sort,
contactgroups, 0, 0,
&query->sort_savedate);
if (!search) {
*err = jmap_server_error(IMAP_INTERNAL);
goto done;
}
/* can calculate changes for mutable sort, but not mutable search */
query->can_calculate_changes = search->is_mutable > 1 ? 0 : 1;
/* make query state */
query->query_state = _email_make_querystate(current_modseq, 0,
contactgroups->size ? jmap_highestmodseq(req, MBTYPE_ADDRESSBOOK) : 0);
/* Open cache */
cache_fname = emailsearch_getcachepath();
if (cache_fname) {
int flags = CYRUSDB_CREATE|CYRUSDB_CONVERT;
int r = cyrusdb_open(EMAILSEARCH_DB, cache_fname, flags, &cache_db);
if (r) {
syslog(LOG_WARNING, "jmap: can't open email search cache %s: %s",
cache_fname, cyrusdb_strerror(r));
}
}
/* Make cache key */
cache_key = strconcat(req->accountid,
"/", collapse_threads ? "collapsed" : "uncollapsed",
"/", search->hash, NULL
);
/* Lookup cache */
if (cache_db) {
struct cached_emailquery cache_record = _CACHED_EMAILQUERY_INITIALIZER;
int r = _email_query_readcache(cache_db, cache_key, current_modseq, &cache_record);
if (!r) {
size_t from = query->position;
if (query->anchor) {
size_t i;
for (i = 0; i < cache_record.ids_count; i++) {
const char *email_id = cache_record.ids + i * (cache_record.id_size + 1);
if (!strcmp(email_id, query->anchor)) {
if (query->anchor_offset < 0) {
size_t neg_offset = (size_t) -query->anchor_offset;
from = neg_offset < i ? i - neg_offset : 0;
}
else {
from = i + query->anchor_offset;
}
break;
}
}
if (i == cache_record.ids_count) {
*err = json_pack("{s:s}", "type", "anchorNotFound");
}
}
else if (query->position < 0) {
ssize_t sposition = (ssize_t) cache_record.ids_count + query->position;
from = sposition < 0 ? 0 : sposition;
}
size_t to = query->limit ? from + query->limit : cache_record.ids_count;
if (to > cache_record.ids_count) to = cache_record.ids_count;
size_t i;
for (i = from; i < to; i++) {
const char *email_id = cache_record.ids + i * (cache_record.id_size + 1);
json_array_append_new(query->ids, json_string(email_id));
}
query->result_position = from;
query->total = cache_record.ids_count;
is_cached = 1;
}
_cached_emailquery_fini(&cache_record);
}
/* Set performance info */
if (jmap_is_using(req, JMAP_PERFORMANCE_EXTENSION)) {
json_object_set_new(req->perf_details, "isCached", json_boolean(is_cached));
int i;
json_t *jfilters = json_array();
for (i = 0; i < strarray_size(&search->perf_filters); i++) {
const char *cost = strarray_nth(&search->perf_filters, i);
json_array_append_new(jfilters, json_string(cost));
}
json_object_set_new(req->perf_details, "filters", jfilters);
}
if (is_cached) goto done;
/* Run search */
const ptrarray_t *msgdata = NULL;
int r = _emailsearch_run(search, &msgdata);
if (r) {
*err = jmap_server_error(r);
goto done;
}
// TODO cache emailId -> threadId on the request context
// TODO support negative positions
assert(query->position >= 0);
/* Initialize search result loop */
size_t anchor_position = (size_t)-1;
char email_id[JMAP_EMAILID_SIZE];
struct hashset *seen_emails = hashset_new(12);
struct hashset *seen_threads = hashset_new(8);
struct hashset *savedates = NULL;
/* List of all matching email ids */
strarray_t email_ids = STRARRAY_INITIALIZER;
int found_anchor = 0;
if (query->sort_savedate) {
/* Build hashset of messages with savedates */
int j;
savedates = hashset_new(12);
for (j = 0; j < msgdata->count; j++) {
MsgData *md = ptrarray_nth(msgdata, j);
/* Skip expunged or hidden messages */
if (md->system_flags & FLAG_DELETED ||
md->internal_flags & FLAG_INTERNAL_EXPUNGED)
continue;
if (md->savedate) hashset_add(savedates, &md->guid.value);
}
}
int i;
for (i = 0 ; i < msgdata->count; i++) {
MsgData *md = ptrarray_nth(msgdata, i);
/* Skip expunged or hidden messages */
if (md->system_flags & FLAG_DELETED ||
md->internal_flags & FLAG_INTERNAL_EXPUNGED)
continue;
/* Is there another copy of this message with a targeted savedate? */
if (!md->savedate &&
savedates && hashset_exists(savedates, &md->guid.value))
continue;
/* Have we seen this message already? */
if (!hashset_add(seen_emails, &md->guid.value))
continue;
if (collapse_threads && !hashset_add(seen_threads, &md->cid))
continue;
/* This message matches the query. */
size_t result_count = json_array_size(query->ids);
query->total++;
jmap_set_emailid(&md->guid, email_id);
if (cache_db) strarray_append(&email_ids, email_id);
/* Apply query window, if any */
if (query->anchor) {
if (!strcmp(email_id, query->anchor)) {
found_anchor = 1;
/* Recalculate the search result */
json_t *anchored_ids = json_pack("[]");
size_t j;
/* Set countdown to enter the anchor window */
if (query->anchor_offset > 0) {
anchor_position = query->anchor_offset;
} else {
anchor_position = 0;
}
/* Readjust the result list */
if (query->anchor_offset < 0) {
size_t neg_offset = (size_t) -query->anchor_offset;
size_t from = neg_offset < result_count ? result_count - neg_offset : 0;
for (j = from; j < result_count; j++) {
json_array_append(anchored_ids, json_array_get(query->ids, j));
}
}
json_decref(query->ids);
query->ids = anchored_ids;
result_count = json_array_size(query->ids);
/* Adjust the window position for this anchor. */
query->result_position = query->total - json_array_size(anchored_ids) - 1;
}
if (anchor_position != (size_t)-1 && anchor_position) {
/* Found the anchor but haven't yet entered its window */
anchor_position--;
/* But this message still counts to the window position */
query->result_position++;
continue;
}
}
else if (query->position > 0 && query->total < ((size_t) query->position) + 1) {
continue;
}
/* Apply limit */
if (query->limit && result_count && query->limit <= result_count)
continue;
/* Add message to result */
json_array_append_new(query->ids, json_string(email_id));
if (*jemailpartids == NULL) {
*jemailpartids = json_object();
}
if (md->folder && md->folder->partids.size) {
const strarray_t *partids = hashu64_lookup(md->uid, &md->folder->partids);
if (partids && strarray_size(partids)) {
json_t *jpartids = json_array();
int k;
for (k = 0; k < strarray_size(partids); k++) {
const char *partid = strarray_nth(partids, k);
json_array_append_new(jpartids, json_string(partid));
}
json_object_set_new(*jemailpartids, email_id, jpartids);
}
}
if (!json_object_get(*jemailpartids, email_id)) {
json_object_set_new(*jemailpartids, email_id, json_null());
}
}
hashset_free(&seen_threads);
hashset_free(&seen_emails);
if (savedates) hashset_free(&savedates);
if (!query->anchor) {
query->result_position = query->position;
}
else if (!found_anchor) {
*err = json_pack("{s:s}", "type", "anchorNotFound");
}
/* Cache search result */
if (cache_db) {
int r = _email_query_writecache(cache_db, cache_key, current_modseq, &email_ids);
if (r) {
syslog(LOG_ERR, "jmap: can't cache email search (%s): %s",
cache_key, cyrusdb_strerror(r));
r = 0;
}
}
strarray_fini(&email_ids);
if (jemailpartids && !*jemailpartids)
*jemailpartids = json_null();
done:
_emailsearch_free(search);
if (cache_db) {
int r = cyrusdb_close(cache_db);
if (r) {
syslog(LOG_ERR, "jmap: can't close email search cache %s: %s",
cache_fname, cyrusdb_strerror(r));
}
}
free(cache_key);
free(cache_fname);
}
static int _email_queryargs_parse(jmap_req_t *req,
struct jmap_parser *parser __attribute__((unused)),
const char *key,
json_t *arg,
void *rock)
{
int *collapse_threads = (int *) rock;
int r = 1;
if (!strcmp(key, "collapseThreads") && json_is_boolean(arg)) {
*collapse_threads = json_boolean_value(arg);
}
else if (!strcmp(key, "addressbookId") && json_is_string(arg) &&
jmap_is_using(req, JMAP_MAIL_EXTENSION)) {
/* Lookup addrbook */
char *addrbookname = carddav_mboxname(req->accountid, json_string_value(arg));
mbentry_t *mbentry = NULL;
int is_valid = 0;
if (!mboxlist_lookup(addrbookname, &mbentry, NULL)) {
is_valid = jmap_hasrights(req, mbentry, ACL_LOOKUP) &&
mbentry->mbtype == MBTYPE_ADDRESSBOOK;
}
mboxlist_entry_free(&mbentry);
free(addrbookname);
return is_valid;
}
else r = 0;
return r;
}
static int jmap_email_query(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_query query;
int collapse_threads = 0;
json_t *jemailpartids = NULL;
struct email_contactfilter contactfilter;
int r = 0;
_email_contactfilter_initreq(req, &contactfilter);
/* Parse request */
json_t *err = NULL;
jmap_query_parse(req, &parser,
_email_queryargs_parse, &collapse_threads,
_email_parse_filter_cb, &contactfilter,
_email_parse_comparator, NULL,
&query, &err);
if (err) {
jmap_error(req, err);
goto done;
}
if (query.position < 0) {
/* we currently don't support negative positions */
jmap_parser_invalid(&parser, "position");
}
if (json_array_size(parser.invalid)) {
err = json_pack("{s:s}", "type", "invalidArguments");
json_object_set(err, "arguments", parser.invalid);
jmap_error(req, err);
goto done;
}
else if (r) {
jmap_error(req, jmap_server_error(r));
goto done;
}
/* Run query */
_email_query(req, &query, collapse_threads, &contactfilter.contactgroups,
&jemailpartids, &err);
if (err) {
jmap_error(req, err);
goto done;
}
/* Build response */
json_t *res = jmap_query_reply(&query);
json_object_set(res, "collapseThreads", json_boolean(collapse_threads));
if (jmap_is_using(req, JMAP_SEARCH_EXTENSION)) {
json_object_set(res, "partIds", jemailpartids); // incref
}
if (jmap_is_using(req, JMAP_DEBUG_EXTENSION)) {
/* List language stats */
const struct search_engine *engine = search_engine();
if (engine->list_lang_stats) {
ptrarray_t lstats = PTRARRAY_INITIALIZER;
int r = engine->list_lang_stats(req->accountid, &lstats);
if (!r) {
json_t *jstats = json_object();
struct search_lang_stats *lstat;
while ((lstat = ptrarray_pop(&lstats))) {
json_t *jstat = json_pack("{s:f}", "weight", lstat->weight);
json_object_set_new(jstats, lstat->iso_lang, jstat);
free(lstat->iso_lang);
free(lstat);
}
json_object_set_new(res, "languageStats", jstats);
}
ptrarray_fini(&lstats);
}
}
jmap_ok(req, res);
done:
jmap_email_contactfilter_fini(&contactfilter);
json_decref(jemailpartids);
jmap_query_fini(&query);
jmap_parser_fini(&parser);
return 0;
}
static void _email_querychanges_collapsed(jmap_req_t *req,
struct jmap_querychanges *query,
struct email_contactfilter *contactfilter,
json_t **err)
{
modseq_t since_modseq;
uint32_t since_uid;
uint32_t num_changes = 0;
modseq_t addrbook_modseq = 0;
if (!_email_read_querystate(query->since_querystate,
&since_modseq, &since_uid,
&addrbook_modseq)) {
*err = json_pack("{s:s}", "type", "cannotCalculateChanges");
return;
}
if (addrbook_modseq && addrbook_modseq != jmap_highestmodseq(req, MBTYPE_ADDRESSBOOK)) {
*err = json_pack("{s:s}", "type", "cannotCalculateChanges");
return;
}
struct emailsearch *search = _emailsearch_new(req, query->filter, query->sort,
&contactfilter->contactgroups,
/*want_expunged*/1, /*ignore_timer*/0,
&query->sort_savedate);
if (!search) {
*err = jmap_server_error(IMAP_INTERNAL);
goto done;
}
/* Run search */
const ptrarray_t *msgdata = NULL;
int r = _emailsearch_run(search, &msgdata);
if (r) {
if (r == IMAP_SEARCH_SLOW) {
*err = json_pack("{s:s, s:s}", "type", "cannotCalculateChanges",
"description", "search too slow");
}
else {
*err = jmap_server_error(r);
}
goto done;
}
/* Prepare result loop */
char email_id[JMAP_EMAILID_SIZE];
int found_up_to = 0;
size_t mdcount = msgdata->count;
hash_table touched_ids = HASH_TABLE_INITIALIZER;
memset(&touched_ids, 0, sizeof(hash_table));
construct_hash_table(&touched_ids, mdcount + 1, 0);
hashu64_table touched_cids = HASH_TABLE_INITIALIZER;
memset(&touched_cids, 0, sizeof(hashu64_table));
construct_hashu64_table(&touched_cids, mdcount + 1, 0);
/* touched_ids contains values for each email_id:
* 1 - email has been modified
* 2 - email has been seen (aka: non-expunged record shown)
* 4 - email has been reported removed
* 8 - email has been reported added
*/
/* touched_cids contains values for each thread
* 1 - thread has been modified
* 2 - thread has been seen (aka: exemplar shown)
* 4 - thread has had an expunged shown (aka: possible old exemplar passed)
* 8 - thread is finished (aka: old exemplar definitely passed)
*/
// phase 1: find messages and threads which have been modified
size_t i;
for (i = 0 ; i < mdcount; i++) {
MsgData *md = ptrarray_nth(msgdata, i);
// for this phase, we only care that it has a change
if (md->modseq <= since_modseq) {
if (search->is_mutable) {
modseq_t modseq = 0;
conversation_get_modseq(req->cstate, md->cid, &modseq);
if (modseq > since_modseq)
hashu64_insert(md->cid, (void*)1, &touched_cids);
}
continue;
}
jmap_set_emailid(&md->guid, email_id);
hash_insert(email_id, (void*)1, &touched_ids);
hashu64_insert(md->cid, (void*)1, &touched_cids);
}
// phase 2: report messages that need it
for (i = 0 ; i < mdcount; i++) {
MsgData *md = ptrarray_nth(msgdata, i);
jmap_set_emailid(&md->guid, email_id);
int is_expunged = (md->system_flags & FLAG_DELETED) ||
(md->internal_flags & FLAG_INTERNAL_EXPUNGED);
size_t touched_id = (size_t)hash_lookup(email_id, &touched_ids);
size_t new_touched_id = touched_id;
size_t touched_cid = (size_t)hashu64_lookup(md->cid, &touched_cids);
size_t new_touched_cid = touched_cid;
if (is_expunged) {
// don't need to tell changes any more
if (found_up_to) goto doneloop;
// nothing to do if not changed (could not be old exemplar)
if (!(touched_id & 1)) goto doneloop;
// could not possibly be old exemplar
if (!search->is_mutable && (touched_cid & 8)) goto doneloop;
// add the destroy notice
if (!(touched_id & 4)) {
_email_querychanges_destroyed(query, email_id);
new_touched_id |= 4;
new_touched_cid |= 4;
}
goto doneloop;
}
// this is the exemplar for the cid
if (!(touched_cid & 2)) {
query->total++;
new_touched_cid |= 2;
}
if (found_up_to) goto doneloop;
// if it's a changed cid, see if we should tell
if ((touched_cid & 1)) {
// haven't told the exemplar yet? This is the exemplar!
if (!(touched_cid & 2)) {
// not yet told in any way, and this ID hasn't been told at all
if (touched_cid == 1 && touched_id == 0 && !search->is_mutable) {
// this is both old AND new exemplar, horray. We don't
// need to tell anything
new_touched_cid |= 8;
goto doneloop;
}
// have to tell both a remove and an add for the exemplar
if (!(touched_id & 4)) {
_email_querychanges_destroyed(query, email_id);
new_touched_id |= 4;
num_changes++;
}
if (!(touched_id & 8)) {
_email_querychanges_added(query, email_id);
new_touched_id |= 8;
num_changes++;
}
new_touched_cid |= 4;
goto doneloop;
}
// otherwise we've already told the exemplar.
// could not possibly be old exemplar
if (!search->is_mutable && (touched_cid & 8)) goto doneloop;
// OK, maybe this alive message WAS the old examplar
if (!(touched_id & 4)) {
_email_querychanges_destroyed(query, email_id);
new_touched_id |= 4;
new_touched_cid |= 4;
}
// and if this message is a stopper (must have been a candidate
// for old exemplar) then stop
if (!(touched_id & 1)) {
new_touched_cid |= 8;
}
}
doneloop:
if (query->max_changes && (num_changes > query->max_changes)) {
*err = json_pack("{s:s}", "type", "tooManyChanges");
break;
}
if (new_touched_id != touched_id)
hash_insert(email_id, (void*)new_touched_id, &touched_ids);
if (new_touched_cid != touched_cid)
hashu64_insert(md->cid, (void*)new_touched_cid, &touched_cids);
// if the search is mutable, later changes could have
// been earlier once, so no up_to_id is possible
if (!found_up_to && !search->is_mutable
&& query->up_to_id
&& !strcmp(email_id, query->up_to_id)) {
found_up_to = 1;
}
}
free_hash_table(&touched_ids, NULL);
free_hashu64_table(&touched_cids, NULL);
modseq_t modseq = jmap_highestmodseq(req, MBTYPE_EMAIL);
query->new_querystate = _email_make_querystate(modseq, 0, addrbook_modseq);
done:
_emailsearch_free(search);
}
static void _email_querychanges_uncollapsed(jmap_req_t *req,
struct jmap_querychanges *query,
struct email_contactfilter *contactfilter,
json_t **err)
{
modseq_t since_modseq;
uint32_t since_uid;
uint32_t num_changes = 0;
modseq_t addrbook_modseq = 0;
if (!_email_read_querystate(query->since_querystate,
&since_modseq, &since_uid,
&addrbook_modseq)) {
*err = json_pack("{s:s}", "type", "cannotCalculateChanges");
return;
}
if (addrbook_modseq && addrbook_modseq != jmap_highestmodseq(req, MBTYPE_ADDRESSBOOK)) {
*err = json_pack("{s:s}", "type", "cannotCalculateChanges");
return;
}
struct emailsearch *search = _emailsearch_new(req, query->filter, query->sort,
&contactfilter->contactgroups,
/*want_expunged*/1, /*ignore_timer*/0,
&query->sort_savedate);
if (!search) {
*err = jmap_server_error(IMAP_INTERNAL);
goto done;
}
/* Run search */
const ptrarray_t *msgdata = NULL;
int r = _emailsearch_run(search, &msgdata);
if (r) {
if (r == IMAP_SEARCH_SLOW) {
*err = json_pack("{s:s, s:s}", "type", "cannotCalculateChanges",
"description", "search too slow");
}
else {
*err = jmap_server_error(r);
}
goto done;
}
/* Prepare result loop */
char email_id[JMAP_EMAILID_SIZE];
int found_up_to = 0;
size_t mdcount = msgdata->count;
hash_table touched_ids = HASH_TABLE_INITIALIZER;
memset(&touched_ids, 0, sizeof(hash_table));
construct_hash_table(&touched_ids, mdcount + 1, 0);
/* touched_ids contains values for each email_id:
* 1 - email has been modified
* 2 - email has been seen (aka: non-expunged record shown)
* 4 - email has been reported removed
* 8 - email has been reported added
*/
// phase 1: find messages which have been modified
size_t i;
for (i = 0 ; i < mdcount; i++) {
MsgData *md = ptrarray_nth(msgdata, i);
// for this phase, we only care that it has a change
if (md->modseq <= since_modseq) continue;
jmap_set_emailid(&md->guid, email_id);
hash_insert(email_id, (void*)1, &touched_ids);
}
// phase 2: report messages that need it
for (i = 0 ; i < mdcount; i++) {
MsgData *md = ptrarray_nth(msgdata, i);
jmap_set_emailid(&md->guid, email_id);
int is_expunged = (md->system_flags & FLAG_DELETED) ||
(md->internal_flags & FLAG_INTERNAL_EXPUNGED);
size_t touched_id = (size_t)hash_lookup(email_id, &touched_ids);
size_t new_touched_id = touched_id;
if (is_expunged) {
// don't need to tell changes any more
if (found_up_to) continue;
// nothing to do if not changed
if (!(touched_id & 1)) continue;
// add the destroy notice
if (!(touched_id & 4)) {
_email_querychanges_destroyed(query, email_id);
new_touched_id |= 4;
}
goto doneloop;
}
// this is an exemplar
if (!(touched_id & 2)) {
query->total++;
new_touched_id |= 2;
}
if (found_up_to) goto doneloop;
// if it's changed, tell about that
if ((touched_id & 1)) {
if (!search->is_mutable && touched_id == 1 && md->modseq <= since_modseq) {
// this is the exemplar, and it's unchanged,
// and we haven't told a removed yet, so we
// can just suppress everything
new_touched_id |= 4 | 8;
goto doneloop;
}
// otherwise we're going to have to tell both, if we haven't already
if (!(touched_id & 4)) {
_email_querychanges_destroyed(query, email_id);
new_touched_id |= 4;
num_changes++;
}
if (!(touched_id & 8)) {
_email_querychanges_added(query, email_id);
new_touched_id |= 8;
num_changes++;
}
}
doneloop:
if (query->max_changes && (num_changes > query->max_changes)) {
*err = json_pack("{s:s}", "type", "tooManyChanges");
break;
}
if (new_touched_id != touched_id)
hash_insert(email_id, (void*)new_touched_id, &touched_ids);
// if the search is mutable, later changes could have
// been earlier once, so no up_to_id is possible
if (!found_up_to && !search->is_mutable
&& query->up_to_id
&& !strcmp(email_id, query->up_to_id)) {
found_up_to = 1;
}
}
free_hash_table(&touched_ids, NULL);
modseq_t modseq = jmap_highestmodseq(req, MBTYPE_EMAIL);
query->new_querystate = _email_make_querystate(modseq, 0, addrbook_modseq);
done:
_emailsearch_free(search);
}
static int jmap_email_querychanges(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_querychanges query;
struct email_contactfilter contactfilter;
int collapse_threads = 0;
_email_contactfilter_initreq(req, &contactfilter);
/* Parse arguments */
json_t *err = NULL;
jmap_querychanges_parse(req, &parser,
_email_queryargs_parse, &collapse_threads,
_email_parse_filter_cb, &contactfilter,
_email_parse_comparator, NULL,
&query, &err);
if (err) {
jmap_error(req, err);
goto done;
}
if (json_array_size(parser.invalid)) {
err = json_pack("{s:s}", "type", "invalidArguments");
json_object_set(err, "arguments", parser.invalid);
jmap_error(req, err);
goto done;
}
/* Query changes */
if (collapse_threads)
_email_querychanges_collapsed(req, &query, &contactfilter, &err);
else
_email_querychanges_uncollapsed(req, &query, &contactfilter, &err);
if (err) {
jmap_error(req, err);
goto done;
}
/* Build response */
json_t *res = jmap_querychanges_reply(&query);
json_object_set(res, "collapseThreads", json_boolean(collapse_threads));
jmap_ok(req, res);
done:
jmap_email_contactfilter_fini(&contactfilter);
jmap_querychanges_fini(&query);
jmap_parser_fini(&parser);
return 0;
}
static void _email_changes(jmap_req_t *req, struct jmap_changes *changes, json_t **err)
{
/* Run search */
json_t *filter = json_pack("{s:o}", "sinceEmailState",
jmap_fmtstate(changes->since_modseq));
json_t *sort = json_pack("[{s:s}]", "property", "emailState");
struct emailsearch *search = _emailsearch_new(req, filter, sort,
/*contactgroups*/NULL,
/*want_expunged*/1,
/*ignore_timer*/1,
NULL);
if (!search) {
*err = jmap_server_error(IMAP_INTERNAL);
goto done;
}
const ptrarray_t *msgdata = NULL;
int r = _emailsearch_run(search, &msgdata);
if (r) {
*err = jmap_server_error(r);
goto done;
}
/* Process results */
char email_id[JMAP_EMAILID_SIZE];
size_t changes_count = 0;
modseq_t highest_modseq = 0;
int i;
hash_table seen_ids = HASH_TABLE_INITIALIZER;
memset(&seen_ids, 0, sizeof(hash_table));
construct_hash_table(&seen_ids, msgdata->count + 1, 0);
for (i = 0 ; i < msgdata->count; i++) {
MsgData *md = ptrarray_nth(msgdata, i);
jmap_set_emailid(&md->guid, email_id);
/* Skip already seen messages */
if (hash_lookup(email_id, &seen_ids)) continue;
hash_insert(email_id, (void*)1, &seen_ids);
/* Apply limit, if any */
if (changes->max_changes && ++changes_count > changes->max_changes) {
changes->has_more_changes = 1;
break;
}
/* Keep track of the highest modseq */
if (highest_modseq < md->modseq)
highest_modseq = md->modseq;
struct email_expunge_check rock = { req, changes->since_modseq, 0 };
int r = conversations_guid_foreach(req->cstate, _guid_from_id(email_id),
_email_is_expunged_cb, &rock);
if (r) {
*err = jmap_server_error(r);
goto done;
}
/* Check the message status - status is a bitfield with:
* 1: a message exists which was created on or before since_modseq
* 2: a message exists which is not deleted
*
* from those facts we can determine ephemeral / destroyed / created / updated
* and we don't need to tell about ephemeral (all created since last time, but none left)
*/
switch (rock.status) {
default:
break; /* all messages were created AND deleted since previous state! */
case 1:
/* only expunged messages exist */
json_array_append_new(changes->destroyed, json_string(email_id));
break;
case 2:
/* alive, and all messages are created since previous modseq */
json_array_append_new(changes->created, json_string(email_id));
break;
case 3:
/* alive, and old */
json_array_append_new(changes->updated, json_string(email_id));
break;
}
}
free_hash_table(&seen_ids, NULL);
/* Set new state */
changes->new_modseq = changes->has_more_changes ?
highest_modseq : jmap_highestmodseq(req, MBTYPE_EMAIL);
done:
json_decref(filter);
json_decref(sort);
_emailsearch_free(search);
}
static int jmap_email_changes(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_changes changes;
/* Parse request */
json_t *err = NULL;
jmap_changes_parse(req, &parser, NULL, NULL, &changes, &err);
if (err) {
jmap_error(req, err);
goto done;
}
/* Search for updates */
_email_changes(req, &changes, &err);
if (err) {
jmap_error(req, err);
goto done;
}
/* Build response */
jmap_ok(req, jmap_changes_reply(&changes));
done:
jmap_changes_fini(&changes);
jmap_parser_fini(&parser);
return 0;
}
static void _thread_changes(jmap_req_t *req, struct jmap_changes *changes, json_t **err)
{
conversation_t conv = CONVERSATION_INIT;
/* Run search */
json_t *filter = json_pack("{s:o}", "sinceEmailState",
jmap_fmtstate(changes->since_modseq));
json_t *sort = json_pack("[{s:s}]", "property", "emailState");
struct emailsearch *search = _emailsearch_new(req, filter, sort,
/*contactgroups*/NULL,
/*want_expunged*/1,
/*ignore_timer*/1,
NULL);
if (!search) {
*err = jmap_server_error(IMAP_INTERNAL);
goto done;
}
const ptrarray_t *msgdata = NULL;
int r = _emailsearch_run(search, &msgdata);
if (r) {
*err = jmap_server_error(r);
goto done;
}
/* Process results */
size_t changes_count = 0;
modseq_t highest_modseq = 0;
int i;
struct hashset *seen_threads = hashset_new(8);
char thread_id[JMAP_THREADID_SIZE];
for (i = 0 ; i < msgdata->count; i++) {
MsgData *md = ptrarray_nth(msgdata, i);
/* Skip already seen threads */
if (!hashset_add(seen_threads, &md->cid)) continue;
/* Apply limit, if any */
if (changes->max_changes && ++changes_count > changes->max_changes) {
changes->has_more_changes = 1;
break;
}
/* Keep track of the highest modseq */
if (highest_modseq < md->modseq)
highest_modseq = md->modseq;
/* Determine if the thread got changed or destroyed */
if (conversation_load_advanced(req->cstate, md->cid, &conv, /*flags*/0))
continue;
/* Report thread */
jmap_set_threadid(md->cid, thread_id);
if (conv.exists) {
if (conv.createdmodseq <= changes->since_modseq)
json_array_append_new(changes->updated, json_string(thread_id));
else
json_array_append_new(changes->created, json_string(thread_id));
}
else {
if (conv.createdmodseq <= changes->since_modseq)
json_array_append_new(changes->destroyed, json_string(thread_id));
}
conversation_fini(&conv);
memset(&conv, 0, sizeof(conversation_t));
}
hashset_free(&seen_threads);
/* Set new state */
changes->new_modseq = changes->has_more_changes ?
highest_modseq : jmap_highestmodseq(req, MBTYPE_EMAIL);
done:
conversation_fini(&conv);
json_decref(filter);
json_decref(sort);
_emailsearch_free(search);
}
static int jmap_thread_changes(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_changes changes;
/* Parse request */
json_t *err = NULL;
jmap_changes_parse(req, &parser, NULL, NULL, &changes, &err);
if (err) {
jmap_error(req, err);
goto done;
}
/* Search for updates */
_thread_changes(req, &changes, &err);
if (err) {
jmap_error(req, err);
goto done;
}
/* Build response */
jmap_ok(req, jmap_changes_reply(&changes));
done:
jmap_changes_fini(&changes);
jmap_parser_fini(&parser);
return 0;
}
static int _snippet_get_cb(struct mailbox *mbox __attribute__((unused)),
uint32_t uid __attribute__((unused)),
int part, const char *s, void *rock)
{
const char *propname = NULL;
json_t *snippet = rock;
if (part == SEARCH_PART_SUBJECT) {
propname = "subject";
}
else if (part == SEARCH_PART_BODY || part == SEARCH_PART_ATTACHMENTBODY) {
propname = "preview";
}
if (propname) {
json_object_set_new(snippet, propname, json_string(s));
}
/* Avoid costly attachment body snippets, if possible */
return part == SEARCH_PART_BODY ? IMAP_OK_COMPLETED : 0;
}
static int _snippet_get(jmap_req_t *req, json_t *filter,
json_t *messageids, json_t *jemailpartids,
json_t **snippets, json_t **notfound)
{
struct index_state *state = NULL;
void *intquery = NULL;
search_builder_t *bx = NULL;
search_text_receiver_t *rx = NULL;
struct mailbox *mbox = NULL;
struct searchargs *searchargs = NULL;
struct index_init init;
json_t *snippet = NULL;
int r = 0;
json_t *val;
size_t i;
char *mboxname = NULL;
static search_snippet_markup_t markup = { "<mark>", "</mark>", "..." };
strarray_t partids = STRARRAY_INITIALIZER;
*snippets = json_pack("[]");
*notfound = json_pack("[]");
/* Build searchargs */
strarray_t perf_filters = STRARRAY_INITIALIZER;
searchargs = new_searchargs(NULL/*tag*/, GETSEARCH_CHARSET_FIRST,
&jmap_namespace, req->userid, req->authstate, 0);
searchargs->root = _email_buildsearch(req, filter, /*contactgroups*/NULL, &perf_filters);
strarray_fini(&perf_filters);
/* Build the search query */
memset(&init, 0, sizeof(init));
init.userid = req->userid;
init.authstate = req->authstate;
init.examine_mode = 1;
char *inboxname = mboxname_user_mbox(req->accountid, NULL);
r = index_open(inboxname, &init, &state);
free(inboxname);
if (r) goto done;
bx = search_begin_search(state->mailbox, SEARCH_MULTIPLE);
if (!bx) {
r = IMAP_INTERNAL;
goto done;
}
search_build_query(bx, searchargs->root);
if (!bx->get_internalised) {
r = IMAP_INTERNAL;
goto done;
}
intquery = bx->get_internalised(bx);
search_end_search(bx);
if (!intquery) {
r = IMAP_INTERNAL;
goto done;
}
/* Set up snippet callback context */
snippet = json_pack("{}");
rx = search_begin_snippets(intquery, 0, &markup, _snippet_get_cb, snippet);
if (!rx) {
r = IMAP_INTERNAL;
goto done;
}
/* Convert the snippets */
json_array_foreach(messageids, i, val) {
message_t *msg;
msgrecord_t *mr = NULL;
uint32_t uid;
const char *msgid = json_string_value(val);
r = jmap_email_find(req, msgid, &mboxname, &uid);
if (r) {
if (r == IMAP_NOTFOUND) {
json_array_append_new(*notfound, json_string(msgid));
}
r = 0;
continue;
}
r = jmap_openmbox(req, mboxname, &mbox, 0);
if (r) goto done;
r = msgrecord_find(mbox, uid, &mr);
if (r) goto doneloop;
r = msgrecord_get_message(mr, &msg);
if (r) goto doneloop;
json_t *jpartids = json_object_get(jemailpartids, msgid);
if (jpartids) {
json_t *jpartid;
size_t j;
json_array_foreach(jpartids, j, jpartid) {
strarray_append(&partids, json_string_value(jpartid));
}
}
json_object_set_new(snippet, "emailId", json_string(msgid));
json_object_set_new(snippet, "subject", json_null());
json_object_set_new(snippet, "preview", json_null());
r = rx->begin_mailbox(rx, mbox, /*incremental*/0);
r = index_getsearchtext(msg, jpartids ? &partids : NULL, rx, 1);
if (!r || r == IMAP_OK_COMPLETED) {
json_array_append_new(*snippets, json_deep_copy(snippet));
r = 0;
}
int r2 = rx->end_mailbox(rx, mbox);
if (!r) r = r2;
json_object_clear(snippet);
strarray_truncate(&partids, 0);
msgrecord_unref(&mr);
doneloop:
if (mr) msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
free(mboxname);
mboxname = NULL;
if (r) goto done;
}
if (!json_array_size(*notfound)) {
json_decref(*notfound);
*notfound = json_null();
}
done:
if (rx) search_end_snippets(rx);
if (snippet) json_decref(snippet);
if (intquery) search_free_internalised(intquery);
if (mboxname) free(mboxname);
if (mbox) jmap_closembox(req, &mbox);
if (searchargs) freesearchargs(searchargs);
strarray_fini(&partids);
index_close(&state);
return r;
}
static int _email_filter_contains_text(json_t *filter)
{
if (JNOTNULL(filter)) {
json_t *val;
size_t i;
if (JNOTNULL(json_object_get(filter, "text"))) {
return 1;
}
if (JNOTNULL(json_object_get(filter, "subject"))) {
return 1;
}
if (JNOTNULL(json_object_get(filter, "body"))) {
return 1;
}
if (JNOTNULL(json_object_get(filter, "attachmentBody"))) {
return 1;
}
/* We don't generate snippets for headers, but we
* might find header text in the body or subject again. */
if (JNOTNULL(json_object_get(filter, "header"))) {
return 1;
}
if (JNOTNULL(json_object_get(filter, "from"))) {
return 1;
}
if (JNOTNULL(json_object_get(filter, "to"))) {
return 1;
}
if (JNOTNULL(json_object_get(filter, "cc"))) {
return 1;
}
if (JNOTNULL(json_object_get(filter, "bcc"))) {
return 1;
}
json_array_foreach(json_object_get(filter, "conditions"), i, val) {
if (_email_filter_contains_text(val)) {
return 1;
}
}
}
return 0;
}
static int jmap_searchsnippet_get(jmap_req_t *req)
{
int r = 0;
const char *key;
json_t *arg, *jfilter = NULL, *jmessageids = NULL, *jemailpartids = NULL;
json_t *snippets, *notfound;
struct buf buf = BUF_INITIALIZER;
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct email_contactfilter contactfilter;
json_t *err = NULL;
_email_contactfilter_initreq(req, &contactfilter);
/* Parse and validate arguments. */
json_t *unsupported_filter = json_pack("[]");
json_object_foreach(req->args, key, arg) {
if (!strcmp(key, "accountId")) {
/* already handled in jmap_api() */
}
/* filter */
else if (!strcmp(key, "filter")) {
jfilter = arg;
if (JNOTNULL(jfilter)) {
jmap_parser_push(&parser, "filter");
jmap_filter_parse(req, &parser, jfilter, unsupported_filter,
_email_parse_filter_cb, &contactfilter, &err);
jmap_parser_pop(&parser);
if (err) break;
}
}
/* messageIds */
else if (!strcmp(key, "emailIds")) {
jmessageids = arg;
if (json_array_size(jmessageids)) {
jmap_parse_strings(jmessageids, &parser, "emailIds");
}
else if (!json_is_array(jmessageids)) {
jmap_parser_invalid(&parser, "emailIds");
}
}
/* partIds */
else if (jmap_is_using(req, JMAP_SEARCH_EXTENSION) && !strcmp(key, "partIds")) {
jemailpartids = arg;
int is_valid = 1;
if (json_is_object(jemailpartids)) {
const char *email_id;
json_t *jpartids;
json_object_foreach(jemailpartids, email_id, jpartids) {
if (json_is_array(jpartids)) {
size_t i;
json_t *jpartid;
json_array_foreach(jpartids, i, jpartid) {
if (!json_is_string(jpartid)) {
is_valid = 0;
break;
}
}
}
else if (json_is_null(jpartids)) {
/* JSON null means: no parts */
continue;
}
if (!is_valid) break;
}
}
else is_valid = json_is_null(jemailpartids);
if (!is_valid) {
jmap_parser_invalid(&parser, "partIds");
}
}
else jmap_parser_invalid(&parser, key);
if (!json_object_size(jemailpartids)) {
jemailpartids = NULL;
}
}
/* Bail out for argument errors */
if (err) {
jmap_error(req, err);
json_decref(unsupported_filter);
goto done;
}
else if (json_array_size(parser.invalid)) {
jmap_error(req, json_pack("{s:s, s:O}", "type", "invalidArguments",
"arguments", parser.invalid));
json_decref(unsupported_filter);
goto done;
}
/* Report unsupported filters */
if (json_array_size(unsupported_filter)) {
jmap_error(req, json_pack("{s:s, s:o}", "type", "unsupportedFilter",
"filters", unsupported_filter));
goto done;
}
json_decref(unsupported_filter);
if (json_array_size(jmessageids) && _email_filter_contains_text(jfilter)) {
/* Render snippets */
r = _snippet_get(req, jfilter, jmessageids, jemailpartids, &snippets, &notfound);
if (r) goto done;
} else {
/* Trivial, snippets cant' match */
size_t i;
json_t *val;
snippets = json_pack("[]");
notfound = json_null();
json_array_foreach(jmessageids, i, val) {
json_array_append_new(snippets, json_pack("{s:s s:n s:n}",
"emailId", json_string_value(val),
"subject", "preview"));
}
}
/* Prepare response. */
json_t *res = json_pack("{s:o s:o}",
"list", snippets, "notFound", notfound);
if (jfilter) json_object_set(res, "filter", jfilter);
jmap_ok(req, res);
done:
jmap_email_contactfilter_fini(&contactfilter);
jmap_parser_fini(&parser);
buf_free(&buf);
return r;
}
static int _thread_is_shared_cb(const conv_guidrec_t *rec, void *rock)
{
if (rec->part) return 0;
jmap_req_t *req = (jmap_req_t *)rock;
static int needrights = ACL_READ|ACL_LOOKUP;
if (jmap_hasrights_byname(req, rec->mboxname, needrights))
return IMAP_OK_COMPLETED;
return 0;
}
static int _thread_get(jmap_req_t *req, json_t *ids,
json_t *list, json_t *not_found)
{
conversation_t conv = CONVERSATION_INIT;
json_t *val;
size_t i;
int r = 0;
json_array_foreach(ids, i, val) {
conv_thread_t *thread;
char email_id[JMAP_EMAILID_SIZE];
const char *threadid = json_string_value(val);
memset(&conv, 0, sizeof(conversation_t));
r = conversation_load_advanced(req->cstate, _cid_from_id(threadid),
&conv, CONV_WITHTHREAD);
if (r || !conv.thread) {
json_array_append_new(not_found, json_string(threadid));
continue;
}
int is_own_account = !strcmp(req->userid, req->accountid);
json_t *ids = json_pack("[]");
for (thread = conv.thread; thread; thread = thread->next) {
if (!is_own_account) {
const char *guidrep = message_guid_encode(&thread->guid);
int r = conversations_guid_foreach(req->cstate, guidrep,
_thread_is_shared_cb, req);
if (r != IMAP_OK_COMPLETED) {
if (r) {
syslog(LOG_ERR, "jmap: _thread_is_shared_cb(%s): %s",
guidrep, error_message(r));
}
continue;
}
}
jmap_set_emailid(&thread->guid, email_id);
json_array_append_new(ids, json_string(email_id));
}
/* if we didn't find any visible IDs, then the thread doesn't really
exist for this user */
if (!json_array_size(ids)) {
json_decref(ids);
json_array_append_new(not_found, json_string(threadid));
continue;
}
json_t *jthread = json_pack("{s:s s:o}", "id", threadid, "emailIds", ids);
json_array_append_new(list, jthread);
conversation_fini(&conv);
}
r = 0;
conversation_fini(&conv);
return r;
}
static const jmap_property_t thread_props[] = {
{
"id",
NULL,
JMAP_PROP_IMMUTABLE | JMAP_PROP_ALWAYS_GET
},
{
"emailIds",
NULL,
0
},
{ NULL, NULL, 0 }
};
static int jmap_thread_get(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_get get;
json_t *err = NULL;
/* Parse request */
jmap_get_parse(req, &parser, thread_props, /*allow_null_ids*/0,
NULL, NULL, &get, &err);
if (err) {
jmap_error(req, err);
goto done;
}
/* Find threads */
int r = _thread_get(req, get.ids, get.list, get.not_found);
if (r) {
syslog(LOG_ERR, "jmap: Thread/get: %s", error_message(r));
jmap_error(req, jmap_server_error(r));
goto done;
}
json_t *jstate = jmap_getstate(req, MBTYPE_EMAIL, /*refresh*/0);
get.state = xstrdup(json_string_value(jstate));
json_decref(jstate);
jmap_ok(req, jmap_get_reply(&get));
done:
jmap_parser_fini(&parser);
jmap_get_fini(&get);
return 0;
}
struct email_getcontext {
struct seen *seendb; /* Seen database for shared accounts */
hash_table seenseq_by_mbox_id; /* Cached seen sequences */
};
static void _email_getcontext_fini(struct email_getcontext *ctx)
{
free_hash_table(&ctx->seenseq_by_mbox_id, (void(*)(void*))seqset_free);
seen_close(&ctx->seendb);
}
struct email_getargs {
/* Email/get arguments */
hash_table *props; /* owned by JMAP get or process stack */
hash_table *bodyprops;
ptrarray_t want_headers; /* array of header_prop */
ptrarray_t want_bodyheaders; /* array of header_prop */
short fetch_text_body;
short fetch_html_body;
short fetch_all_body;
size_t max_body_bytes;
/* Request-scoped context */
struct email_getcontext ctx;
};
#define _EMAIL_GET_ARGS_INITIALIZER \
{ \
NULL, \
NULL, \
PTRARRAY_INITIALIZER, \
PTRARRAY_INITIALIZER, \
0, \
0, \
0, \
0, \
{ \
NULL, \
HASH_TABLE_INITIALIZER \
} \
};
/* Initialized in email_get_parse. *Not* thread-safe */
static hash_table _email_get_default_bodyprops = HASH_TABLE_INITIALIZER;
static hash_table _email_parse_default_props = HASH_TABLE_INITIALIZER;
static void _email_getargs_fini(struct email_getargs *args)
{
if (args->bodyprops && args->bodyprops != &_email_get_default_bodyprops) {
free_hash_table(args->bodyprops, NULL);
free(args->bodyprops);
}
args->bodyprops = NULL;
struct header_prop *prop;
while ((prop = ptrarray_pop(&args->want_headers))) {
_header_prop_fini(prop);
free(prop);
}
ptrarray_fini(&args->want_headers);
while ((prop = ptrarray_pop(&args->want_bodyheaders))) {
_header_prop_fini(prop);
free(prop);
}
ptrarray_fini(&args->want_bodyheaders);
_email_getcontext_fini(&args->ctx);
}
/* A wrapper to aggregate JMAP keywords over a set of message records.
* Notably the $seen keyword is a pain to map from IMAP to JMAP:
* (1) it must only be reported if the IMAP \Seen flag is set on
* all non-deleted index records.
* (2) it must be read from seen.db for shared mailboxes
*/
struct email_keywords {
const char *userid;
hash_table counts;
size_t totalmsgs;
hash_table *seenseq_by_mbox_id;
struct seen *seendb;
};
#define _EMAIL_KEYWORDS_INITIALIZER { NULL, HASH_TABLE_INITIALIZER, 0, NULL, NULL }
/* Initialize the keyword aggregator for the authenticated userid.
*
* The seenseq hash table is used to read cached sequence sets
* read from seen.db per mailbox. If the hash table does not
* contain a sequence for the respective mailbox id, it is read
* from seen.db and stored in the map.
* Callers must free any entries in seenseq_by_mbox_id. */
static void _email_keywords_init(struct email_keywords *keywords,
const char *userid,
struct seen *seendb,
hash_table *seenseq_by_mbox_id)
{
construct_hash_table(&keywords->counts, 64, 0);
keywords->userid = userid;
keywords->seendb = seendb;
keywords->seenseq_by_mbox_id = seenseq_by_mbox_id;
}
static void _email_keywords_fini(struct email_keywords *keywords)
{
free_hash_table(&keywords->counts, NULL);
}
static void _email_keywords_add_keyword(struct email_keywords *keywords,
const char *keyword)
{
uintptr_t count = (uintptr_t) hash_lookup(keyword, &keywords->counts);
hash_insert(keyword, (void*) count+1, &keywords->counts);
}
static int _email_keywords_add_msgrecord(struct email_keywords *keywords,
msgrecord_t *mr)
{
uint32_t uid, system_flags, internal_flags;
uint32_t user_flags[MAX_USER_FLAGS/32];
struct mailbox *mbox = NULL;
int r = msgrecord_get_uid(mr, &uid);
if (r) goto done;
r = msgrecord_get_mailbox(mr, &mbox);
if (r) goto done;
r = msgrecord_get_systemflags(mr, &system_flags);
if (r) goto done;
r = msgrecord_get_internalflags(mr, &internal_flags);
if (r) goto done;
if (system_flags & FLAG_DELETED || internal_flags & FLAG_INTERNAL_EXPUNGED) goto done;
r = msgrecord_get_userflags(mr, user_flags);
if (r) goto done;
int read_seendb = !mailbox_internal_seen(mbox, keywords->userid);
/* Read system flags */
if ((system_flags & FLAG_DRAFT))
_email_keywords_add_keyword(keywords, "$draft");
if ((system_flags & FLAG_FLAGGED))
_email_keywords_add_keyword(keywords, "$flagged");
if ((system_flags & FLAG_ANSWERED))
_email_keywords_add_keyword(keywords, "$answered");
if (!read_seendb && system_flags & FLAG_SEEN)
_email_keywords_add_keyword(keywords, "$seen");
/* Read user flags */
struct buf buf = BUF_INITIALIZER;
int i;
for (i = 0 ; i < MAX_USER_FLAGS ; i++) {
if (mbox->flagname[i] && (user_flags[i/32] & 1<<(i&31))) {
buf_setcstr(&buf, mbox->flagname[i]);
_email_keywords_add_keyword(keywords, buf_lcase(&buf));
}
}
buf_free(&buf);
if (read_seendb) {
/* Read $seen keyword from seen.db for shared accounts */
struct seqset *seenseq = hash_lookup(mbox->uniqueid, keywords->seenseq_by_mbox_id);
if (!seenseq) {
struct seendata sd = SEENDATA_INITIALIZER;
int r = seen_read(keywords->seendb, mbox->uniqueid, &sd);
if (!r) {
seenseq = seqset_parse(sd.seenuids, NULL, sd.lastuid);
hash_insert(mbox->uniqueid, seenseq, keywords->seenseq_by_mbox_id);
seen_freedata(&sd);
}
else {
syslog(LOG_ERR, "Could not read seen state for %s (%s)",
keywords->userid, error_message(r));
}
}
if (seenseq && seqset_ismember(seenseq, uid))
_email_keywords_add_keyword(keywords, "$seen");
}
/* Count message */
keywords->totalmsgs++;
done:
return r;
}
static json_t *_email_keywords_to_jmap(struct email_keywords *keywords)
{
json_t *jkeywords = json_object();
hash_iter *kwiter = hash_table_iter(&keywords->counts);
while (hash_iter_next(kwiter)) {
const char *keyword = hash_iter_key(kwiter);
uintptr_t count = (uintptr_t) hash_iter_val(kwiter);
if (strcasecmp(keyword, "$seen") || count == keywords->totalmsgs) {
json_object_set_new(jkeywords, keyword, json_true());
}
}
hash_iter_free(&kwiter);
return jkeywords;
}
struct email_get_keywords_rock {
jmap_req_t *req;
struct email_keywords keywords;
};
static int _email_get_keywords_cb(const conv_guidrec_t *rec, void *vrock)
{
struct email_get_keywords_rock *rock = vrock;
jmap_req_t *req = rock->req;
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
if (rec->part) return 0;
if (!jmap_hasrights_byname(req, rec->mboxname, ACL_READ|ACL_LOOKUP)) return 0;
/* Fetch system flags */
int r = jmap_openmbox(req, rec->mboxname, &mbox, 0);
if (r) return r;
r = msgrecord_find(mbox, rec->uid, &mr);
if (r) goto done;
r = _email_keywords_add_msgrecord(&rock->keywords, mr);
done:
msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
return r;
}
static int _email_get_keywords(jmap_req_t *req,
struct email_getcontext *ctx,
const char *msgid,
json_t **jkeywords)
{
/* Initialize seen.db and sequence set cache */
if (ctx->seendb == NULL && strcmp(req->accountid, req->userid)) {
int r = seen_open(req->userid, SEEN_CREATE, &ctx->seendb);
if (r) return r;
}
if (ctx->seenseq_by_mbox_id.size == 0) {
construct_hash_table(&ctx->seenseq_by_mbox_id, 128, 0);
}
/* Gather keywords for all message records */
struct email_get_keywords_rock rock = { req, _EMAIL_KEYWORDS_INITIALIZER };
_email_keywords_init(&rock.keywords, req->userid, ctx->seendb, &ctx->seenseq_by_mbox_id);
int r = conversations_guid_foreach(req->cstate, _guid_from_id(msgid),
_email_get_keywords_cb, &rock);
*jkeywords = _email_keywords_to_jmap(&rock.keywords);
_email_keywords_fini(&rock.keywords);
return r;
}
struct email_get_snoozed_rock {
jmap_req_t *req;
json_t *snoozed;
};
static int _email_get_snoozed_cb(const conv_guidrec_t *rec, void *vrock)
{
struct email_get_snoozed_rock *rock = vrock;
if (rec->part) return 0;
if (!jmap_hasrights_byname(rock->req, rec->mboxname, ACL_READ|ACL_LOOKUP))
return 0;
if (FLAG_INTERNAL_SNOOZED ==
(rec->internal_flags & (FLAG_INTERNAL_SNOOZED|FLAG_INTERNAL_EXPUNGED))) {
/* Fetch snoozed annotation */
rock->snoozed = jmap_fetch_snoozed(rec->mboxname, rec->uid);
}
/* Short-circuit the foreach if we find a snoozed message */
return (rock->snoozed != NULL);
}
static void _email_parse_wantheaders(json_t *jprops,
struct jmap_parser *parser,
const char *prop_name,
ptrarray_t *want_headers)
{
size_t i;
json_t *jval;
json_array_foreach(jprops, i, jval) {
const char *s = json_string_value(jval);
if (!s || strncmp(s, "header:", 7))
continue;
struct header_prop *hprop;
if ((hprop = _header_parseprop(s))) {
ptrarray_append(want_headers, hprop);
}
else {
jmap_parser_push_index(parser, prop_name, i, s);
jmap_parser_invalid(parser, NULL);
jmap_parser_pop(parser);
}
}
}
static void _email_init_default_props(hash_table *props)
{
/* Initialize process-owned default property list */
construct_hash_table(props, 32, 0);
if (props == &_email_get_default_bodyprops) {
hash_insert("blobId", (void*)1, props);
hash_insert("charset", (void*)1, props);
hash_insert("cid", (void*)1, props);
hash_insert("disposition", (void*)1, props);
hash_insert("language", (void*)1, props);
hash_insert("location", (void*)1, props);
hash_insert("name", (void*)1, props);
hash_insert("partId", (void*)1, props);
hash_insert("size", (void*)1, props);
hash_insert("type", (void*)1, props);
}
else {
hash_insert("attachments", (void*)1, props);
hash_insert("bcc", (void*)1, props);
hash_insert("bodyValues", (void*)1, props);
hash_insert("cc", (void*)1, props);
hash_insert("from", (void*)1, props);
hash_insert("hasAttachment", (void*)1, props);
hash_insert("htmlBody", (void*)1, props);
hash_insert("inReplyTo", (void*)1, props);
hash_insert("messageId", (void*)1, props);
hash_insert("preview", (void*)1, props);
hash_insert("references", (void*)1, props);
hash_insert("replyTo", (void*)1, props);
hash_insert("sender", (void*)1, props);
hash_insert("sentAt", (void*)1, props);
hash_insert("subject", (void*)1, props);
hash_insert("textBody", (void*)1, props);
hash_insert("to", (void*)1, props);
}
}
static int _email_getargs_parse(jmap_req_t *req __attribute__((unused)),
struct jmap_parser *parser,
const char *key,
json_t *arg,
void *rock)
{
struct email_getargs *args = (struct email_getargs *) rock;
int r = 1;
/* bodyProperties */
if (!strcmp(key, "bodyProperties")) {
if (jmap_parse_strings(arg, parser, "bodyProperties")) {
size_t i;
json_t *val;
args->bodyprops = xzmalloc(sizeof(hash_table));
construct_hash_table(args->bodyprops, json_array_size(arg) + 1, 0);
json_array_foreach(arg, i, val) {
hash_insert(json_string_value(val), (void*)1, args->bodyprops);
}
}
/* header:Xxx properties */
_email_parse_wantheaders(arg, parser, "bodyProperties",
&args->want_bodyheaders);
}
/* fetchTextBodyValues */
else if (!strcmp(key, "fetchTextBodyValues") && json_is_boolean(arg)) {
args->fetch_text_body = json_boolean_value(arg);
}
/* fetchHTMLBodyValues */
else if (!strcmp(key, "fetchHTMLBodyValues") && json_is_boolean(arg)) {
args->fetch_html_body = json_boolean_value(arg);
}
/* fetchAllBodyValues */
else if (!strcmp(key, "fetchAllBodyValues") && json_is_boolean(arg)) {
args->fetch_all_body = json_boolean_value(arg);
}
/* maxBodyValueBytes */
else if (!strcmp(key, "maxBodyValueBytes") &&
json_is_integer(arg) && json_integer_value(arg) > 0) {
args->max_body_bytes = json_integer_value(arg);
}
else r = 0;
return r;
}
struct cyrusmsg {
msgrecord_t *mr; /* Message record for top-level message */
const struct body *part0; /* Root body-part */
const struct body *rfc822part; /* RFC822 root part for embedded message */
const struct buf *mime; /* Raw MIME buffer */
json_t *imagesize_by_part; /* FastMail-specific extension */
message_t *_m; /* Message loaded from message record */
struct body *_mybody; /* Bodystructure */
struct buf _mymime; /* Raw MIME buffer */
struct headers *_headers; /* Parsed part0 headers. Don't free. */
hash_table *_headers_by_part_id; /* Parsed subpart headers. Don't free. */
ptrarray_t _headers_mempool; /* Allocated headers memory */
};
static void _cyrusmsg_free(struct cyrusmsg **msgptr)
{
if (msgptr == NULL || *msgptr == NULL) return;
struct cyrusmsg *msg = *msgptr;
if (msg->_mybody) {
message_free_body(msg->_mybody);
free(msg->_mybody);
}
buf_free(&msg->_mymime);
json_decref(msg->imagesize_by_part);
if (msg->_headers_by_part_id) {
free_hash_table(msg->_headers_by_part_id, NULL);
free(msg->_headers_by_part_id);
}
struct headers *hdrs;
while ((hdrs = ptrarray_pop(&msg->_headers_mempool))) {
_headers_fini(hdrs);
free(hdrs);
}
ptrarray_fini(&msg->_headers_mempool);
free(*msgptr);
*msgptr = NULL;
}
static int _cyrusmsg_from_record(msgrecord_t *mr, struct cyrusmsg **msgptr)
{
struct cyrusmsg *msg = xzmalloc(sizeof(struct cyrusmsg));
msg->mr = mr;
*msgptr = msg;
return 0;
}
static int _cyrusmsg_from_bodypart(msgrecord_t *mr,
struct body *body,
const struct body *part,
struct cyrusmsg **msgptr)
{
struct cyrusmsg *msg = xzmalloc(sizeof(struct cyrusmsg));
msg->mr = mr;
msg->part0 = body;
msg->rfc822part = part;
*msgptr = msg;
return 0;
}
static void _cyrusmsg_init_partids(struct body *body, const char *part_id)
{
if (!body) return;
if (!strcmp(body->type, "MULTIPART")) {
struct buf buf = BUF_INITIALIZER;
int i;
for (i = 0; i < body->numparts; i++) {
struct body *subpart = body->subpart + i;
if (part_id) buf_printf(&buf, "%s.", part_id);
buf_printf(&buf, "%d", i + 1);
subpart->part_id = buf_release(&buf);
_cyrusmsg_init_partids(subpart, subpart->part_id);
}
free(body->part_id);
body->part_id = NULL;
buf_free(&buf);
}
else {
struct buf buf = BUF_INITIALIZER;
if (!body->part_id) {
if (part_id) buf_printf(&buf, "%s.", part_id);
buf_printf(&buf, "%d", 1);
body->part_id = buf_release(&buf);
}
buf_free(&buf);
if (!strcmp(body->type, "MESSAGE") &&
!strcmp(body->subtype, "RFC822")) {
_cyrusmsg_init_partids(body->subpart, body->part_id);
}
}
}
static int _cyrusmsg_from_buf(const struct buf *buf, struct cyrusmsg **msgptr)
{
/* No more return from here */
struct body *mybody = xzmalloc(sizeof(struct body));
struct protstream *pr = prot_readmap(buf_base(buf), buf_len(buf));
/* Pre-run compliance check */
int r = message_copy_strict(pr, /*to*/NULL, buf_len(buf), /*allow_null*/0);
if (r) goto done;
/* Parse message */
r = message_parse_mapped(buf_base(buf), buf_len(buf), mybody);
if (r || !mybody->subpart) {
r = IMAP_MESSAGE_BADHEADER;
goto done;
}
/* parse_mapped doesn't set part ids */
_cyrusmsg_init_partids(mybody->subpart, NULL);
struct cyrusmsg *msg = xzmalloc(sizeof(struct cyrusmsg));
msg->_mybody = mybody;
msg->part0 = mybody;
msg->rfc822part = mybody;
msg->mime = buf;
*msgptr = msg;
done:
if (pr) prot_free(pr);
if (r && mybody) {
message_free_body(mybody);
free(mybody);
}
return r;
}
static int _cyrusmsg_need_part0(struct cyrusmsg *msg)
{
if (msg->part0)
return 0;
if (!msg->mr)
return IMAP_INTERNAL;
int r = msgrecord_extract_bodystructure(msg->mr, &msg->_mybody);
if (r) return r;
msg->part0 = msg->_mybody;
return 0;
}
static int _cyrusmsg_need_mime(struct cyrusmsg *msg)
{
if (msg->mime) return 0;
if (msg->mr == NULL) {
return IMAP_INTERNAL;
}
int r = msgrecord_get_body(msg->mr, &msg->_mymime);
msg->mime = &msg->_mymime;
if (r) return r;
return 0;
}
static int _cyrusmsg_get_headers(struct cyrusmsg *msg,
const struct body *part,
struct headers **headersptr)
{
if (part == NULL && msg->_headers) {
*headersptr = msg->_headers;
return 0;
}
else if (part && part->part_id) {
if (msg->_headers_by_part_id) {
*headersptr = hash_lookup(part->part_id, msg->_headers_by_part_id);
if (*headersptr) return 0;
}
else {
msg->_headers_by_part_id = xzmalloc(sizeof(hash_table));
construct_hash_table(msg->_headers_by_part_id, 64, 0);
}
}
/* Prefetch body structure */
int r = _cyrusmsg_need_part0(msg);
if (r) return r;
/* Prefetch MIME message */
r = _cyrusmsg_need_mime(msg);
if (r) return r;
const struct body *header_part = part ? part : msg->part0;
struct headers *headers = xmalloc(sizeof(struct headers));
_headers_init(headers);
_headers_from_mime(msg->mime->s + header_part->header_offset,
header_part->header_size, headers);
if (part && part->part_id)
hash_insert(part->part_id, headers, msg->_headers_by_part_id);
else if (part == NULL)
msg->_headers = headers;
ptrarray_append(&msg->_headers_mempool, headers);
*headersptr = headers;
return 0;
}
static json_t * _email_get_header(struct cyrusmsg *msg,
const struct body *part,
const char *lcasename,
enum _header_form want_form,
int want_all)
{
if (!part) {
/* Fetch bodypart */
int r = _cyrusmsg_need_part0(msg);
if (r) return json_null();
part = msg->part0;
}
/* Try to read the header from the parsed body part */
if (part && !want_all && want_form != HEADER_FORM_RAW) {
json_t *jval = NULL;
if (!strcmp("messageId", lcasename)) {
jval = want_form == HEADER_FORM_MESSAGEIDS ?
_header_as_messageids(part->message_id) : json_null();
}
else if (!strcmp("inReplyTo", lcasename)) {
jval = want_form == HEADER_FORM_MESSAGEIDS ?
_header_as_messageids(part->in_reply_to) : json_null();
}
if (!strcmp("subject", lcasename)) {
jval = want_form == HEADER_FORM_TEXT ?
_header_as_text(part->subject) : json_null();
}
if (!strcmp("from", lcasename)) {
jval = want_form == HEADER_FORM_ADDRESSES ?
_emailaddresses_from_addr(part->from) : json_null();
}
else if (!strcmp("to", lcasename)) {
jval = want_form == HEADER_FORM_ADDRESSES ?
_emailaddresses_from_addr(part->to) : json_null();
}
else if (!strcmp("cc", lcasename)) {
jval = want_form == HEADER_FORM_ADDRESSES ?
_emailaddresses_from_addr(part->cc) : json_null();
}
else if (!strcmp("bcc", lcasename)) {
jval = want_form == HEADER_FORM_ADDRESSES ?
_emailaddresses_from_addr(part->bcc) : json_null();
}
else if (!strcmp("sentAt", lcasename)) {
jval = json_null();
if (want_form == HEADER_FORM_DATE) {
struct offsettime t;
if (offsettime_from_rfc5322(part->date, &t, DATETIME_FULL) != -1) {
char datestr[30];
offsettime_to_iso8601(&t, datestr, 30, 1);
jval = json_string(datestr);
}
}
}
if (jval) return jval;
}
/* Determine header form converter */
json_t* (*conv)(const char *raw);
switch (want_form) {
case HEADER_FORM_TEXT:
conv = _header_as_text;
break;
case HEADER_FORM_DATE:
conv = _header_as_date;
break;
case HEADER_FORM_ADDRESSES:
conv = _header_as_addresses;
break;
case HEADER_FORM_MESSAGEIDS:
conv = _header_as_messageids;
break;
case HEADER_FORM_URLS:
conv = _header_as_urls;
break;
default:
conv = _header_as_raw;
}
/* Try to read the value from the index record or header cache */
if (msg->mr && part == msg->part0 && !want_all && want_form != HEADER_FORM_RAW) {
if (!msg->_m) {
int r = msgrecord_get_message(msg->mr, &msg->_m);
if (r) return json_null();
}
struct buf buf = BUF_INITIALIZER;
int r = message_get_field(msg->_m, lcasename, MESSAGE_RAW|MESSAGE_LAST, &buf);
if (r) return json_null();
json_t *jval = NULL;
if (buf_len(&buf)) jval = conv(buf_cstring(&buf));
buf_free(&buf);
if (jval) return jval;
}
/* Read the raw MIME headers */
struct headers *partheaders = NULL;
int r = _cyrusmsg_get_headers(msg, part, &partheaders);
if (r) return json_null();
/* Lookup array of EmailHeader objects by name */
json_t *jheaders = json_object_get(partheaders->all, lcasename);
if (!jheaders || !json_array_size(jheaders)) {
return want_all ? json_array() : json_null();
}
/* Convert header values */
if (want_all) {
json_t *allvals = json_array();
size_t i;
for (i = 0; i < json_array_size(jheaders); i++) {
json_t *jheader = json_array_get(jheaders, i);
json_t *jheaderval = json_object_get(jheader, "value");
json_array_append_new(allvals, conv(json_string_value(jheaderval)));
}
return allvals;
}
json_t *jheader = json_array_get(jheaders, json_array_size(jheaders) - 1);
json_t *jheaderval = json_object_get(jheader, "value");
return conv(json_string_value(jheaderval));
}
static int _email_get_meta(jmap_req_t *req,
struct email_getargs *args,
struct cyrusmsg *msg,
json_t *email)
{
int r = 0;
hash_table *props = args->props;
char email_id[JMAP_EMAILID_SIZE];
if (msg->rfc822part) {
if (jmap_wantprop(props, "id")) {
json_object_set_new(email, "id", json_null());
}
if (jmap_wantprop(props, "blobId")) {
char blob_id[JMAP_BLOBID_SIZE];
jmap_set_blobid(&msg->rfc822part->content_guid, blob_id);
json_object_set_new(email, "blobId", json_string(blob_id));
}
if (jmap_wantprop(props, "threadId"))
json_object_set_new(email, "threadId", json_null());
if (jmap_wantprop(props, "mailboxIds"))
json_object_set_new(email, "mailboxIds", json_null());
if (jmap_wantprop(props, "keywords"))
json_object_set_new(email, "keywords", json_object());
if (jmap_wantprop(props, "size")) {
size_t size = msg->rfc822part->header_size + msg->rfc822part->content_size;
json_object_set_new(email, "size", json_integer(size));
}
if (jmap_wantprop(props, "receivedAt"))
json_object_set_new(email, "receivedAt", json_null());
return 0;
}
/* This is a top-level messages with a regular index record. */
/* Determine message id */
struct message_guid guid;
r = msgrecord_get_guid(msg->mr, &guid);
if (r) goto done;
jmap_set_emailid(&guid, email_id);
/* id */
if (jmap_wantprop(props, "id")) {
json_object_set_new(email, "id", json_string(email_id));
}
/* blobId */
if (jmap_wantprop(props, "blobId")) {
char blob_id[JMAP_BLOBID_SIZE];
jmap_set_blobid(&guid, blob_id);
json_object_set_new(email, "blobId", json_string(blob_id));
}
/* threadid */
if (jmap_wantprop(props, "threadId")) {
bit64 cid;
r = msgrecord_get_cid(msg->mr, &cid);
if (r) goto done;
char thread_id[JMAP_THREADID_SIZE];
jmap_set_threadid(cid, thread_id);
json_object_set_new(email, "threadId", json_string(thread_id));
}
/* mailboxIds */
if (jmap_wantprop(props, "mailboxIds") ||
jmap_wantprop(props, "addedDates") || jmap_wantprop(props, "removedDates")) {
json_t *mboxids =
jmap_wantprop(props, "mailboxIds") ? json_object() : NULL;
json_t *added =
jmap_wantprop(props, "addedDates") ? json_object() : NULL;
json_t *removed =
jmap_wantprop(props, "removedDates") ? json_object() : NULL;
json_t *mailboxes = _email_mailboxes(req, email_id);
json_t *val;
const char *mboxid;
json_object_foreach(mailboxes, mboxid, val) {
json_t *exists = json_object_get(val, "added");
if (exists) {
if (mboxids) json_object_set_new(mboxids, mboxid, json_true());
if (added) json_object_set(added, mboxid, exists);
}
else if (removed) {
json_object_set(removed, mboxid, json_object_get(val, "removed"));
}
}
json_decref(mailboxes);
if (mboxids) json_object_set_new(email, "mailboxIds", mboxids);
if (removed) json_object_set_new(email, "removedDates", removed);
if (added) json_object_set_new(email, "addedDates", added);
}
/* keywords */
if (jmap_wantprop(props, "keywords")) {
json_t *keywords = NULL;
r = _email_get_keywords(req, &args->ctx, email_id, &keywords);
if (r) goto done;
json_object_set_new(email, "keywords", keywords);
}
/* size */
if (jmap_wantprop(props, "size")) {
uint32_t size;
r = msgrecord_get_size(msg->mr, &size);
if (r) goto done;
json_object_set_new(email, "size", json_integer(size));
}
/* receivedAt */
if (jmap_wantprop(props, "receivedAt")) {
char datestr[RFC3339_DATETIME_MAX];
time_t t;
r = msgrecord_get_internaldate(msg->mr, &t);
if (r) goto done;
time_to_rfc3339(t, datestr, RFC3339_DATETIME_MAX);
json_object_set_new(email, "receivedAt", json_string(datestr));
}
/* FastMail-extension properties */
if (jmap_wantprop(props, "trustedSender")) {
json_t *trusted_sender = NULL;
int has_trusted_flag = 0;
r = msgrecord_hasflag(msg->mr, "$IsTrusted", &has_trusted_flag);
if (r) goto done;
if (has_trusted_flag) {
struct buf buf = BUF_INITIALIZER;
_email_read_annot(req, msg->mr, "/vendor/messagingengine.com/trusted", &buf);
if (buf_len(&buf)) {
trusted_sender = json_string(buf_cstring(&buf));
}
buf_free(&buf);
}
json_object_set_new(email, "trustedSender", trusted_sender ?
trusted_sender : json_null());
}
if (jmap_wantprop(props, "spamScore")) {
int r = 0;
struct buf buf = BUF_INITIALIZER;
json_t *jval = json_null();
if (!msg->_m) r = msgrecord_get_message(msg->mr, &msg->_m);
if (!r) r = message_get_field(msg->_m, "x-spam-score", MESSAGE_RAW, &buf);
if (!r && buf_len(&buf)) jval = json_real(atof(buf_cstring(&buf)));
json_object_set_new(email, "spamScore", jval);
buf_free(&buf);
}
if (jmap_wantprop(props, "snoozed")) {
struct email_get_snoozed_rock rock = { req, NULL };
/* Look for the first snoozed copy of this email_id */
conversations_guid_foreach(req->cstate, _guid_from_id(email_id),
_email_get_snoozed_cb, &rock);
json_object_set_new(email, "snoozed",
rock.snoozed ? rock.snoozed : json_null());
}
done:
return r;
}
static int _email_get_headers(jmap_req_t *req __attribute__((unused)),
struct email_getargs *args,
struct cyrusmsg *msg,
json_t *email)
{
int r = 0;
hash_table *props = args->props;
if (jmap_wantprop(props, "headers") || args->want_headers.count) {
/* headers */
if (jmap_wantprop(props, "headers")) {
struct headers *headers = NULL;
r = _cyrusmsg_get_headers(msg, NULL, &headers);
if (r) return r;
json_object_set(email, "headers", headers->raw); /* incref! */
}
/* headers:Xxx */
if (ptrarray_size(&args->want_headers)) {
int i;
for (i = 0; i < ptrarray_size(&args->want_headers); i++) {
struct header_prop *want_header = ptrarray_nth(&args->want_headers, i);
json_t *jheader = _email_get_header(msg, NULL, want_header->lcasename,
want_header->form, want_header->all);
json_object_set_new(email, want_header->prop, jheader);
}
}
}
/* references */
if (jmap_wantprop(props, "references")) {
json_t *references = _email_get_header(msg, NULL, "references",
HEADER_FORM_MESSAGEIDS,/*all*/0);
json_object_set_new(email, "references", references);
}
/* sender */
if (jmap_wantprop(props, "sender")) {
json_t *sender = _email_get_header(msg, NULL, "sender",
HEADER_FORM_ADDRESSES,/*all*/0);
json_object_set_new(email, "sender", sender);
}
/* replyTo */
if (jmap_wantprop(props, "replyTo")) {
json_t *replyTo = _email_get_header(msg, NULL, "reply-to",
HEADER_FORM_ADDRESSES, /*all*/0);
json_object_set_new(email, "replyTo", replyTo);
}
/* The following fields are all read from the body-part structure */
const struct body *part = NULL;
if (jmap_wantprop(props, "messageId") ||
jmap_wantprop(props, "inReplyTo") ||
jmap_wantprop(props, "from") ||
jmap_wantprop(props, "to") ||
jmap_wantprop(props, "cc") ||
jmap_wantprop(props, "bcc") ||
jmap_wantprop(props, "subject") ||
jmap_wantprop(props, "sentAt")) {
if (msg->rfc822part) {
part = msg->rfc822part->subpart;
}
else {
r = _cyrusmsg_need_part0(msg);
if (r) return r;
part = msg->part0;
}
}
/* messageId */
if (jmap_wantprop(props, "messageId")) {
json_object_set_new(email, "messageId",
_header_as_messageids(part->message_id));
}
/* inReplyTo */
if (jmap_wantprop(props, "inReplyTo")) {
json_object_set_new(email, "inReplyTo",
_header_as_messageids(part->in_reply_to));
}
/* from */
if (jmap_wantprop(props, "from")) {
json_object_set_new(email, "from",
_emailaddresses_from_addr(part->from));
}
/* to */
if (jmap_wantprop(props, "to")) {
json_object_set_new(email, "to",
_emailaddresses_from_addr(part->to));
}
/* cc */
if (jmap_wantprop(props, "cc")) {
json_object_set_new(email, "cc",
_emailaddresses_from_addr(part->cc));
}
/* bcc */
if (jmap_wantprop(props, "bcc")) {
json_object_set_new(email, "bcc",
_emailaddresses_from_addr(part->bcc));
}
/* subject */
if (jmap_wantprop(props, "subject")) {
json_object_set_new(email, "subject",
_header_as_text(part->subject));
}
/* sentAt */
if (jmap_wantprop(props, "sentAt")) {
json_t *jsent_at = json_null();
struct offsettime t;
if (offsettime_from_rfc5322(part->date, &t, DATETIME_FULL) != -1) {
char datestr[30];
offsettime_to_iso8601(&t, datestr, 30, 1);
jsent_at = json_string(datestr);
}
json_object_set_new(email, "sentAt", jsent_at);
}
return r;
}
static json_t *_email_get_bodypart(jmap_req_t *req,
struct email_getargs *args,
struct cyrusmsg *msg,
const struct body *part)
{
struct buf buf = BUF_INITIALIZER;
struct param *param;
hash_table *bodyprops = args->bodyprops;
ptrarray_t *want_bodyheaders = &args->want_bodyheaders;
json_t *jbodypart = json_object();
/* partId */
if (jmap_wantprop(bodyprops, "partId")) {
json_t *jpart_id = json_null();
if (strcasecmp(part->type, "MULTIPART"))
jpart_id = json_string(part->part_id);
json_object_set_new(jbodypart, "partId", jpart_id);
}
/* blobId */
if (jmap_wantprop(bodyprops, "blobId")) {
json_t *jblob_id = json_null();
if (!message_guid_isnull(&part->content_guid)) {
char blob_id[JMAP_BLOBID_SIZE];
jmap_set_blobid(&part->content_guid, blob_id);
jblob_id = json_string(blob_id);
}
json_object_set_new(jbodypart, "blobId", jblob_id);
}
/* size */
if (jmap_wantprop(bodyprops, "size")) {
size_t size = 0;
if (part->numparts && strcasecmp(part->type, "MESSAGE")) {
/* Multipart */
size = 0;
}
else if (part->charset_enc & 0xff) {
if (part->decoded_content_size == 0) {
char *tmp = NULL;
size_t tmp_size;
int r = _cyrusmsg_need_mime(msg);
if (!r) {
charset_decode_mimebody(msg->mime->s + part->content_offset,
part->content_size, part->charset_enc, &tmp, &tmp_size);
size = tmp_size;
free(tmp);
}
}
else {
size = part->decoded_content_size;
}
}
else {
size = part->content_size;
}
json_object_set_new(jbodypart, "size", json_integer(size));
}
/* headers */
if (jmap_wantprop(bodyprops, "headers") || want_bodyheaders->count) {
/* headers */
if (jmap_wantprop(bodyprops, "headers")) {
struct headers *headers = NULL;
int r = _cyrusmsg_get_headers(msg, part, &headers);
if (!r) {
json_object_set(jbodypart, "headers", headers->raw); /* incref! */
}
else {
json_object_set(jbodypart, "headers", json_null());
}
}
/* headers:Xxx */
if (ptrarray_size(want_bodyheaders)) {
int i;
for (i = 0; i < ptrarray_size(want_bodyheaders); i++) {
struct header_prop *want_header = ptrarray_nth(want_bodyheaders, i);
json_t *jheader = _email_get_header(msg, part, want_header->lcasename,
want_header->form, want_header->all);
json_object_set_new(jbodypart, want_header->prop, jheader);
}
}
}
/* name */
if (jmap_wantprop(bodyprops, "name")) {
const char *fname = NULL;
char *val = NULL;
int is_extended = 0;
/* Lookup name parameter. Disposition header has precedence */
for (param = part->disposition_params; param; param = param->next) {
if (!strncasecmp(param->attribute, "filename", 8)) {
is_extended = param->attribute[8] == '*';
fname = param->value;
break;
}
}
/* Lookup Content-Type parameters */
if (!fname) {
for (param = part->params; param; param = param->next) {
if (!strncasecmp(param->attribute, "name", 4)) {
is_extended = param->attribute[4] == '*';
fname = param->value;
break;
}
}
}
/* Decode header value */
if (fname && is_extended) {
val = charset_parse_mimexvalue(fname, NULL);
}
if (fname && !val) {
val = charset_parse_mimeheader(fname, CHARSET_KEEPCASE|CHARSET_MIME_UTF8);
}
json_object_set_new(jbodypart, "name", val ?
json_string(val) : json_null());
free(val);
}
/* type */
if (jmap_wantprop(bodyprops, "type")) {
buf_setcstr(&buf, part->type);
if (part->subtype) {
buf_appendcstr(&buf, "/");
buf_appendcstr(&buf, part->subtype);
}
json_object_set_new(jbodypart, "type", json_string(buf_lcase(&buf)));
}
/* charset */
if (jmap_wantprop(bodyprops, "charset")) {
const char *charset_id = NULL;
if (part->charset_id) {
charset_id = part->charset_id;
}
else if (!strcasecmp(part->type, "TEXT")) {
charset_id = "us-ascii";
}
json_object_set_new(jbodypart, "charset", charset_id ?
json_string(charset_id) : json_null());
}
/* disposition */
if (jmap_wantprop(bodyprops, "disposition")) {
json_t *jdisp = json_null();
if (part->disposition) {
char *disp = lcase(xstrdup(part->disposition));
jdisp = json_string(disp);
free(disp);
}
json_object_set_new(jbodypart, "disposition", jdisp);
}
/* cid */
if (jmap_wantprop(bodyprops, "cid")) {
json_t *jcid = _email_get_header(msg, part, "content-id",
HEADER_FORM_MESSAGEIDS, /*all*/0);
json_object_set(jbodypart, "cid", json_array_size(jcid) ?
json_array_get(jcid, 0) : json_null());
json_decref(jcid);
}
/* language */
if (jmap_wantprop(bodyprops, "language")) {
json_t *jlanguage = json_null();
json_t *jrawheader = _email_get_header(msg, part, "content-language",
HEADER_FORM_RAW, /*all*/0);
if (JNOTNULL(jrawheader)) {
/* Split by space and comma and aggregate into array */
const char *s = json_string_value(jrawheader);
jlanguage = json_array();
int i;
char *tmp = charset_unfold(s, strlen(s), 0);
strarray_t *ls = strarray_split(tmp, "\t ,", STRARRAY_TRIM);
for (i = 0; i < ls->count; i++) {
json_array_append_new(jlanguage, json_string(strarray_nth(ls, i)));
}
strarray_free(ls);
free(tmp);
}
if (!json_array_size(jlanguage)) {
json_decref(jlanguage);
jlanguage = json_null();
}
json_object_set_new(jbodypart, "language", jlanguage);
json_decref(jrawheader);
}
/* location */
if (jmap_wantprop(bodyprops, "location")) {
json_object_set_new(jbodypart, "location", part->location ?
json_string(part->location) : json_null());
}
/* subParts */
if (!strcmp(part->type, "MULTIPART")) {
json_t *subparts = json_array();
int i;
for (i = 0; i < part->numparts; i++) {
struct body *subpart = part->subpart + i;
json_array_append_new(subparts,
_email_get_bodypart(req, args, msg, subpart));
}
json_object_set_new(jbodypart, "subParts", subparts);
}
else if (jmap_wantprop(bodyprops, "subParts")) {
json_object_set_new(jbodypart, "subParts", json_array());
}
/* FastMail extension properties */
if (jmap_wantprop(bodyprops, "imageSize")) {
json_t *imagesize = json_null();
if (msg->mr && msg->imagesize_by_part == NULL) {
/* This is the first attempt to read the vendor annotation.
* Load the annotation value, if any, for top-level messages.
* Use JSON null for an unsuccessful attempt, so we know not
* to try again. */
msg->imagesize_by_part = _email_read_jannot(req, msg->mr,
"/vendor/messagingengine.com/imagesize", 1);
if (!msg->imagesize_by_part)
msg->imagesize_by_part = json_null();
}
imagesize = json_object_get(msg->imagesize_by_part, part->part_id);
json_object_set(jbodypart, "imageSize", imagesize ? imagesize : json_null());
}
if (jmap_wantprop(bodyprops, "isDeleted")) {
json_object_set_new(jbodypart, "isDeleted",
json_boolean(!strcmp(part->type, "TEXT") &&
!strcmp(part->subtype, "X-ME-REMOVED-FILE")));
}
buf_free(&buf);
return jbodypart;
}
static json_t * _email_get_bodyvalue(struct body *part,
const struct buf *msg_buf,
size_t max_body_bytes,
int is_html)
{
json_t *jbodyvalue = NULL;
int is_encoding_problem = 0;
int is_truncated = 0;
struct buf buf = BUF_INITIALIZER;
/* Decode into UTF-8 buffer */
char *raw = _decode_to_utf8(part->charset_id,
msg_buf->s + part->content_offset,
part->content_size, part->encoding,
&is_encoding_problem);
if (!raw) goto done;
/* In-place remove CR characters from buffer */
size_t i, j, rawlen = strlen(raw);
for (i = 0, j = 0; j < rawlen; j++) {
if (raw[j] != '\r') raw[i++] = raw[j];
}
raw[i] = '\0';
/* Initialize return value */
buf_initm(&buf, raw, rawlen);
/* Truncate buffer */
if (buf_len(&buf) && max_body_bytes && max_body_bytes < buf_len(&buf)) {
/* Cut of excess bytes */
buf_truncate(&buf, max_body_bytes);
is_truncated = 1;
/* Clip to sane UTF-8 */
/* XXX do not split between combining characters */
const unsigned char *base = (unsigned char *) buf_base(&buf);
const unsigned char *top = base + buf_len(&buf);
const unsigned char *p = top - 1;
while (p >= base && ((*p & 0xc0) == 0x80))
p--;
if (p >= base) {
ssize_t have_bytes = top - p;
ssize_t need_bytes = 0;
unsigned char hi_nibble = *p & 0xf0;
switch (hi_nibble) {
case 0xf0:
need_bytes = 4;
break;
case 0xe0:
need_bytes = 3;
break;
case 0xc0:
need_bytes = 2;
break;
default:
need_bytes = 1;
}
if (have_bytes < need_bytes)
buf_truncate(&buf, p - base);
}
else {
buf_reset(&buf);
}
}
/* Truncate HTML */
if (buf_len(&buf) && max_body_bytes && is_html) {
/* Truncate any trailing '<' start tag character without closing '>' */
const char *base = buf_base(&buf);
const char *top = base + buf_len(&buf);
const char *p;
for (p = top - 1; *p != '>' && p >= base; p--) {
if (*p == '<') {
buf_truncate(&buf, p - base + 1);
is_truncated = 1;
break;
}
}
}
done:
jbodyvalue = json_pack("{s:s s:b s:b}",
"value", buf_cstring(&buf),
"isEncodingProblem", is_encoding_problem,
"isTruncated", is_truncated);
buf_free(&buf);
return jbodyvalue;
}
static int _email_get_bodies(jmap_req_t *req,
struct email_getargs *args,
struct cyrusmsg *msg,
json_t *email)
{
struct emailbodies bodies = EMAILBODIES_INITIALIZER;
hash_table *props = args->props;
int r = 0;
const struct body *part;
if (msg->rfc822part) {
part = msg->rfc822part->subpart;
}
else {
r = _cyrusmsg_need_part0(msg);
if (r) return r;
part = msg->part0;
}
/* Dissect message into its parts */
r = jmap_emailbodies_extract(part, &bodies);
if (r) goto done;
/* bodyStructure */
if (jmap_wantprop(props, "bodyStructure")) {
json_object_set_new(email, "bodyStructure",
_email_get_bodypart(req, args, msg, part));
}
/* bodyValues */
if (jmap_wantprop(props, "bodyValues")) {
json_t *body_values = json_object();
/* Determine which body value parts to fetch */
int i;
ptrarray_t parts = PTRARRAY_INITIALIZER;
if (args->fetch_text_body || args->fetch_all_body) {
for (i = 0; i < bodies.textlist.count; i++)
ptrarray_append(&parts, ptrarray_nth(&bodies.textlist, i));
}
if (args->fetch_html_body || args->fetch_all_body) {
for (i = 0; i < bodies.htmllist.count; i++)
ptrarray_append(&parts, ptrarray_nth(&bodies.htmllist, i));
}
if (parts.count) {
r = _cyrusmsg_need_mime(msg);
if (r) goto done;
}
/* Fetch body values */
for (i = 0; i < parts.count; i++) {
struct body *part = ptrarray_nth(&parts, i);
if (strcmp("TEXT", part->type)) {
continue;
}
if (part->part_id && json_object_get(body_values, part->part_id)) {
continue;
}
json_object_set_new(body_values, part->part_id,
_email_get_bodyvalue(part, msg->mime, args->max_body_bytes,
!strcmp("HTML", part->subtype)));
}
ptrarray_fini(&parts);
json_object_set_new(email, "bodyValues", body_values);
}
/* textBody */
if (jmap_wantprop(props, "textBody")) {
json_t *text_body = json_array();
int i;
for (i = 0; i < bodies.textlist.count; i++) {
struct body *part = ptrarray_nth(&bodies.textlist, i);
json_array_append_new(text_body,
_email_get_bodypart(req, args, msg, part));
}
json_object_set_new(email, "textBody", text_body);
}
/* htmlBody */
if (jmap_wantprop(props, "htmlBody")) {
json_t *html_body = json_array();
int i;
for (i = 0; i < bodies.htmllist.count; i++) {
struct body *part = ptrarray_nth(&bodies.htmllist, i);
json_array_append_new(html_body,
_email_get_bodypart(req, args, msg, part));
}
json_object_set_new(email, "htmlBody", html_body);
}
/* attachments */
if (jmap_wantprop(props, "attachments")) {
json_t *attachments = json_array();
int i;
for (i = 0; i < bodies.attslist.count; i++) {
struct body *part = ptrarray_nth(&bodies.attslist, i);
json_array_append_new(attachments,
_email_get_bodypart(req, args, msg, part));
}
json_object_set_new(email, "attachments", attachments);
}
/* calendarEvents -- non-standard */
if (jmap_wantprop(props, "calendarEvents")) {
json_t *calendar_events = json_object();
int i;
for (i = 0; i < bodies.attslist.count; i++) {
struct body *part = ptrarray_nth(&bodies.attslist, i);
/* Process text/calendar attachments and files ending with .ics */
if (strcmp(part->type, "TEXT") || strcmp(part->subtype, "CALENDAR")) {
int has_ics_attachment = 0;
struct param *param = part->disposition_params;
while (param) {
if (!strcasecmp(param->attribute, "FILENAME")) {
size_t len = strlen(param->value);
if (len > 4 && !strcasecmp(param->value + len-4, ".ICS")) {
has_ics_attachment = 1;
}
}
param = param->next;
}
if (!has_ics_attachment)
continue;
}
/* Parse decoded data to iCalendar object */
r = _cyrusmsg_need_mime(msg);
if (r) goto done;
char *decbuf = NULL;
size_t declen = 0;
const char *rawical = charset_decode_mimebody(msg->mime->s + part->content_offset,
part->content_size, part->charset_enc, &decbuf, &declen);
if (!rawical) continue;
struct buf buf = BUF_INITIALIZER;
buf_setmap(&buf, rawical, declen);
icalcomponent *ical = ical_string_as_icalcomponent(&buf);
buf_free(&buf);
free(decbuf);
if (!ical) continue;
/* Parse iCalendar object to JSCalendar */
json_t *jsevents = jmapical_tojmap_all(ical, NULL);
if (json_array_size(jsevents)) {
json_object_set_new(calendar_events, part->part_id, jsevents);
}
icalcomponent_free(ical);
}
if (!json_object_size(calendar_events)) {
json_decref(calendar_events);
calendar_events = json_null();
}
json_object_set_new(email, "calendarEvents", calendar_events);
}
/* hasAttachment */
if (jmap_wantprop(props, "hasAttachment")) {
int has_att = 0;
if (msg->rfc822part == NULL) {
msgrecord_hasflag(msg->mr, JMAP_HAS_ATTACHMENT_FLAG, &has_att);
}
else {
has_att = bodies.attslist.count > 0;
}
json_object_set_new(email, "hasAttachment", json_boolean(has_att));
}
/* preview */
if (jmap_wantprop(props, "preview")) {
const char *preview_annot = config_getstring(IMAPOPT_JMAP_PREVIEW_ANNOT);
if (preview_annot && msg->rfc822part == NULL) {
json_t *preview = _email_read_jannot(req, msg->mr, preview_annot, /*structured*/0);
json_object_set_new(email, "preview", preview ? preview : json_string(""));
}
else {
r = _cyrusmsg_need_mime(msg);
if (r) goto done;
/* TODO optimise for up to PREVIEW_LEN bytes */
char *text = _emailbodies_to_plain(&bodies, msg->mime);
if (!text) {
char *html = _emailbodies_to_html(&bodies, msg->mime);
if (html) text = _html_to_plain(html);
free(html);
}
if (text) {
size_t len = config_getint(IMAPOPT_JMAP_PREVIEW_LENGTH);
char *preview = _email_extract_preview(text, len);
json_object_set_new(email, "preview", json_string(preview));
free(preview);
free(text);
}
}
}
done:
jmap_emailbodies_fini(&bodies);
return r;
}
static int _email_from_msg(jmap_req_t *req,
struct email_getargs *args,
struct cyrusmsg *msg,
json_t **emailptr)
{
json_t *email = json_object();
int r = 0;
r = _email_get_meta(req, args, msg, email);
if (r) goto done;
r = _email_get_headers(req, args, msg, email);
if (r) goto done;
r = _email_get_bodies(req, args, msg, email);
if (r) goto done;
*emailptr = email;
done:
if (r) json_decref(email);
return r;
}
static int _email_from_record(jmap_req_t *req,
struct email_getargs *args,
msgrecord_t *mr,
json_t **emailptr)
{
struct cyrusmsg *msg = NULL;
int r = _cyrusmsg_from_record(mr, &msg);
if (!r) r = _email_from_msg(req, args, msg, emailptr);
_cyrusmsg_free(&msg);
return r;
}
static int _email_from_body(jmap_req_t *req,
struct email_getargs *args,
msgrecord_t *mr,
struct body *body,
const struct body *part,
json_t **emailptr)
{
struct cyrusmsg *msg = NULL;
int r = _cyrusmsg_from_bodypart(mr, body, part, &msg);
if (!r) r = _email_from_msg(req, args, msg, emailptr);
_cyrusmsg_free(&msg);
return r;
}
static int _email_from_buf(jmap_req_t *req,
struct email_getargs *args,
const struct buf *buf,
const char *encoding,
json_t **emailptr)
{
struct buf mybuf = BUF_INITIALIZER;
buf_setcstr(&mybuf, "Content-Type: message/rfc822\r\n");
if (encoding) {
if (!strcasecmp(encoding, "BASE64")) {
char *tmp = NULL;
size_t tmp_size = 0;
charset_decode_mimebody(buf_base(buf), buf_len(buf),
ENCODING_BASE64, &tmp, &tmp_size);
buf_appendcstr(&mybuf, "Content-Transfer-Encoding: binary\r\n");
/* Append base64-decoded body */
buf_appendcstr(&mybuf, "\r\n");
buf_appendmap(&mybuf, tmp, tmp_size);
free(tmp);
}
else {
buf_appendcstr(&mybuf, "Content-Transfer-Encoding: ");
buf_appendcstr(&mybuf, encoding);
buf_appendcstr(&mybuf, "\r\n");
/* Append encoded body */
buf_appendcstr(&mybuf, "\r\n");
buf_append(&mybuf, buf);
}
}
else {
/* Append raw body */
buf_appendcstr(&mybuf, "\r\n");
buf_append(&mybuf, buf);
}
struct cyrusmsg *msg = NULL;
int r = _cyrusmsg_from_buf(&mybuf, &msg);
if (!r) r = _email_from_msg(req, args, msg, emailptr);
buf_free(&mybuf);
_cyrusmsg_free(&msg);
return r;
}
HIDDEN int jmap_email_get_with_props(jmap_req_t *req,
hash_table *props,
msgrecord_t *mr,
json_t **msgp)
{
struct email_getargs args = _EMAIL_GET_ARGS_INITIALIZER;
args.props = props;
int r = _email_from_record(req, &args, mr, msgp);
args.props = NULL;
_email_getargs_fini(&args);
return r;
}
static int _isthreadsonly(json_t *jargs)
{
json_t *arg = json_object_get(jargs, "properties");
if (!json_is_array(arg)) return 0;
if (json_array_size(arg) != 1) return 0;
const char *s = json_string_value(json_array_get(arg, 0));
if (strcmpsafe(s, "threadId")) return 0;
return 1;
}
static void jmap_email_get_threadsonly(jmap_req_t *req, struct jmap_get *get)
{
size_t i;
json_t *val;
json_array_foreach(get->ids, i, val) {
const char *id = json_string_value(val);
conversation_id_t cid = 0;
int r = _email_get_cid(req, id, &cid);
if (!r && cid) {
char thread_id[JMAP_THREADID_SIZE];
jmap_set_threadid(cid, thread_id);
json_t *msg = json_pack("{s:s, s:s}", "id", id, "threadId", thread_id);
json_array_append_new(get->list, msg);
}
else {
json_array_append_new(get->not_found, json_string(id));
}
if (r) {
syslog(LOG_ERR, "jmap: Email/get(%s): %s", id, error_message(r));
}
}
}
struct _warmup_mboxcache_cb_rock {
jmap_req_t *req;
ptrarray_t mboxes;
};
static int _warmup_mboxcache_cb(const conv_guidrec_t *rec, void* vrock)
{
struct _warmup_mboxcache_cb_rock *rock = vrock;
int i;
for (i = 0; i < ptrarray_size(&rock->mboxes); i++) {
struct mailbox *mbox = ptrarray_nth(&rock->mboxes, i);
if (!strcmp(rec->mboxname, mbox->name)) {
return 0;
}
}
struct mailbox *mbox = NULL;
int r = jmap_openmbox(rock->req, rec->mboxname, &mbox, /*rw*/0);
if (!r) {
ptrarray_append(&rock->mboxes, mbox);
}
return r;
}
static void jmap_email_get_full(jmap_req_t *req, struct jmap_get *get, struct email_getargs *args)
{
size_t i;
json_t *val;
/* Warm up the mailbox cache by opening all mailboxes */
struct _warmup_mboxcache_cb_rock rock = { req, PTRARRAY_INITIALIZER };
json_array_foreach(get->ids, i, val) {
const char *email_id = json_string_value(val);
if (email_id[0] != 'M' || strlen(email_id) != 25) {
continue;
}
int r = conversations_guid_foreach(req->cstate, _guid_from_id(email_id),
_warmup_mboxcache_cb, &rock);
if (r) {
/* Ignore errors, they'll be handled in email_find */
syslog(LOG_WARNING, "__warmup_mboxcache_cb(%s): %s", email_id, error_message(r));
continue;
}
}
/* Process emails one after the other */
json_array_foreach(get->ids, i, val) {
const char *id = json_string_value(val);
char *mboxname = NULL;
msgrecord_t *mr = NULL;
json_t *msg = NULL;
struct mailbox *mbox = NULL;
uint32_t uid;
int r = jmap_email_find(req, id, &mboxname, &uid);
if (!r) {
r = jmap_openmbox(req, mboxname, &mbox, 0);
if (!r) {
r = msgrecord_find(mbox, uid, &mr);
if (!r) {
r = _email_from_record(req, args, mr, &msg);
}
jmap_closembox(req, &mbox);
}
}
if (!r && msg) {
json_array_append_new(get->list, msg);
}
else {
json_array_append_new(get->not_found, json_string(id));
}
if (r) {
syslog(LOG_ERR, "jmap: Email/get(%s): %s", id, error_message(r));
}
free(mboxname);
msgrecord_unref(&mr);
}
/* Close cached mailboxes */
struct mailbox *mbox = NULL;
while ((mbox = ptrarray_pop(&rock.mboxes))) {
jmap_closembox(req, &mbox);
}
ptrarray_fini(&rock.mboxes);
}
static const jmap_property_t email_props[] = {
{
"id",
NULL,
JMAP_PROP_SERVER_SET | JMAP_PROP_IMMUTABLE | JMAP_PROP_ALWAYS_GET
},
{
"blobId",
NULL,
JMAP_PROP_SERVER_SET | JMAP_PROP_IMMUTABLE
},
{
"threadId",
NULL,
JMAP_PROP_SERVER_SET | JMAP_PROP_IMMUTABLE
},
{
"mailboxIds",
NULL,
0
},
{
"keywords",
NULL,
0
},
{
"size",
NULL,
JMAP_PROP_SERVER_SET | JMAP_PROP_IMMUTABLE
},
{
"receivedAt",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"headers",
NULL,
JMAP_PROP_IMMUTABLE | JMAP_PROP_SKIP_GET
},
{
"header:*",
NULL,
JMAP_PROP_IMMUTABLE | JMAP_PROP_SKIP_GET
},
{
"messageId",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"inReplyTo",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"references",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"sender",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"from",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"to",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"cc",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"bcc",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"replyTo",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"subject",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"sentAt",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"bodyStructure",
NULL,
JMAP_PROP_IMMUTABLE | JMAP_PROP_SKIP_GET
},
{
"bodyValues",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"textBody",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"htmlBody",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"attachments",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"hasAttachment",
NULL,
JMAP_PROP_SERVER_SET | JMAP_PROP_IMMUTABLE
},
{
"preview",
NULL,
JMAP_PROP_SERVER_SET | JMAP_PROP_IMMUTABLE
},
/* FM extensions (do ALL of these get through to Cyrus?) */
{
"addedDates",
JMAP_MAIL_EXTENSION,
0
},
{
"removedDates",
JMAP_MAIL_EXTENSION,
0
},
{
"trustedSender",
JMAP_MAIL_EXTENSION,
JMAP_PROP_SERVER_SET | JMAP_PROP_IMMUTABLE
},
{
"spamScore",
JMAP_MAIL_EXTENSION,
JMAP_PROP_IMMUTABLE
},
{
"calendarEvents",
JMAP_MAIL_EXTENSION,
JMAP_PROP_IMMUTABLE
},
{
"isDeleted",
JMAP_MAIL_EXTENSION,
0
},
{
"imageSize",
JMAP_MAIL_EXTENSION,
0
},
{
"snoozed",
JMAP_MAIL_EXTENSION,
0
},
{ NULL, NULL, 0 }
};
static int jmap_email_get(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_get get;
struct email_getargs args = _EMAIL_GET_ARGS_INITIALIZER;
json_t *err = NULL;
/* Parse request */
jmap_get_parse(req, &parser, email_props, /*allow_null_ids*/0,
&_email_getargs_parse, &args, &get, &err);
if (!err) {
/* header:Xxx properties */
json_t *jprops = json_object_get(req->args, "properties");
if (JNOTNULL(jprops)) {
_email_parse_wantheaders(jprops, &parser, "properties",
&args.want_headers);
}
if (json_array_size(parser.invalid)) {
err = json_pack("{s:s s:O}", "type", "invalidArguments",
"arguments", parser.invalid);
}
}
if (err) {
jmap_error(req, err);
goto done;
}
/* properties - already parsed in jmap_get_parse */
args.props = get.props;
/* Set default body properties, if not set by client */
if (args.bodyprops == NULL) {
args.bodyprops = &_email_get_default_bodyprops;
if (args.bodyprops->size == 0) {
_email_init_default_props(args.bodyprops);
}
}
if (_isthreadsonly(req->args))
jmap_email_get_threadsonly(req, &get);
else
jmap_email_get_full(req, &get, &args);
json_t *jstate = jmap_getstate(req, MBTYPE_EMAIL, /*refresh*/0);
get.state = xstrdup(json_string_value(jstate));
json_decref(jstate);
jmap_ok(req, jmap_get_reply(&get));
done:
_email_getargs_fini(&args);
jmap_parser_fini(&parser);
jmap_get_fini(&get);
return 0;
}
static int jmap_email_parse(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct email_getargs getargs = _EMAIL_GET_ARGS_INITIALIZER;
const char *key;
json_t *arg, *jblobIds = NULL;
hash_table *props = NULL;
/* Parse request */
json_object_foreach(req->args, key, arg) {
if (!strcmp(key, "accountId")) {
/* already handled in jmap_api() */
}
else if (!strcmp(key, "blobIds")) {
jblobIds = arg;
jmap_parse_strings(jblobIds, &parser, "blobIds");
}
else if (!strcmp(key, "properties")) {
if (json_is_array(arg)) {
size_t i;
json_t *val;
props = xzmalloc(sizeof(hash_table));
construct_hash_table(props, json_array_size(arg) + 1, 0);
json_array_foreach(arg, i, val) {
const char *s = json_string_value(val);
if (!s) {
jmap_parser_push_index(&parser, "properties", i, s);
jmap_parser_invalid(&parser, NULL);
jmap_parser_pop(&parser);
continue;
}
hash_insert(s, (void*)1, props);
}
getargs.props = props;
}
else if (JNOTNULL(arg)) {
jmap_parser_invalid(&parser, "properties");
}
}
else if (!_email_getargs_parse(req, &parser, key, arg, &getargs)) {
jmap_parser_invalid(&parser, key);
}
}
if (json_array_size(parser.invalid)) {
json_t *err = json_pack("{s:s s:O}", "type", "invalidArguments",
"arguments", parser.invalid);
jmap_error(req, err);
goto done;
}
/* Set default properties, if not set by client */
if (getargs.props == NULL) {
getargs.props = &_email_parse_default_props;
if (getargs.props->size == 0) {
_email_init_default_props(getargs.props);
}
}
/* Set default body properties, if not set by client */
if (getargs.bodyprops == NULL) {
getargs.bodyprops = &_email_get_default_bodyprops;
if (getargs.bodyprops->size == 0) {
_email_init_default_props(getargs.bodyprops);
}
}
/* Process request */
json_t *parsed = json_object();
json_t *notParsable = json_array();
json_t *notFound = json_array();
json_t *jval;
size_t i;
json_array_foreach(jblobIds, i, jval) {
const char *blobid = json_string_value(jval);
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
struct body *body = NULL;
const struct body *part = NULL;
int r = jmap_findblob(req, NULL/*accountid*/, blobid,
&mbox, &mr, &body, &part, NULL);
if (r) {
json_array_append_new(notFound, json_string(blobid));
continue;
}
json_t *email = NULL;
if (part && (strcmp(part->type, "MESSAGE") || !strcmpnull(part->encoding, "BASE64"))) {
struct buf msg_buf = BUF_INITIALIZER;
r = msgrecord_get_body(mr, &msg_buf);
if (!r) {
struct buf buf = BUF_INITIALIZER;
buf_init_ro(&buf, buf_base(&msg_buf) + part->content_offset,
part->content_size);
r = _email_from_buf(req, &getargs, &buf, part->encoding, &email);
buf_free(&buf);
}
buf_free(&msg_buf);
if (r) {
syslog(LOG_ERR, "jmap: Email/parse(%s): %s", blobid, error_message(r));
}
}
else {
_email_from_body(req, &getargs, mr, body, part, &email);
}
if (email) {
json_object_set_new(parsed, blobid, email);
}
else {
json_array_append_new(notParsable, json_string(blobid));
}
msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
message_free_body(body);
free(body);
}
/* Build response */
json_t *res = json_object();
if (!json_object_size(parsed)) {
json_decref(parsed);
parsed = json_null();
}
if (!json_array_size(notParsable)) {
json_decref(notParsable);
notParsable = json_null();
}
if (!json_array_size(notFound)) {
json_decref(notFound);
notFound = json_null();
}
json_object_set_new(res, "parsed", parsed);
json_object_set_new(res, "notParsable", notParsable);
json_object_set_new(res, "notFound", notFound);
jmap_ok(req, res);
done:
_email_getargs_fini(&getargs);
jmap_parser_fini(&parser);
free_hash_table(props, NULL);
free(props);
return 0;
}
static char *_mime_make_boundary()
{
char *boundary, *p, *q;
boundary = xstrdup(makeuuid());
for (p = boundary, q = boundary; *p; p++) {
if (*p != '-') *q++ = *p;
}
*q = 0;
return boundary;
}
/* A soft limit for MIME header lengths when generating MIME from JMAP.
* See the header_from_Xxx functions for usage. */
#define MIME_MAX_HEADER_LENGTH 78
static void _mime_write_xparam(struct buf *buf, const char *name, const char *value)
{
int is_7bit = 1;
int is_fold = 0;
const char *p = value;
for (p = value; *p && (is_7bit || !is_fold); p++) {
if (*p & 0x80)
is_7bit = 0;
if (*p == '\n')
is_fold = 1;
}
char *xvalue = is_7bit ? xstrdup(value) : charset_encode_mimexvalue(value, NULL);
if (strlen(name) + strlen(xvalue) + 1 < MIME_MAX_HEADER_LENGTH) {
if (is_7bit)
buf_printf(buf, ";%s=\"%s\"", name, xvalue);
else
buf_printf(buf, ";%s*=%s", name, xvalue);
goto done;
}
/* Break value into continuations */
int section = 0;
p = xvalue;
struct buf line = BUF_INITIALIZER;
buf_appendcstr(&line, ";\r\n ");
while (*p) {
/* Build parameter continuation line. */
buf_printf(&line, "%s*%d*=", name, section);
/* Write at least one character of the value */
if (is_7bit)
buf_putc(&line, '"');
int n = buf_len(&line) + 1;
do {
buf_putc(&line, *p);
n++;
p++;
if (!is_7bit && *p == '%' && n >= MIME_MAX_HEADER_LENGTH - 2)
break;
} while (*p && n < MIME_MAX_HEADER_LENGTH);
if (is_7bit)
buf_putc(&line, '"');
/* Write line */
buf_append(buf, &line);
/* Prepare next iteration */
if (*p) buf_appendcstr(buf, ";\r\n ");
buf_reset(&line);
section++;
}
buf_free(&line);
done:
free(xvalue);
}
static int _copy_msgrecords(struct auth_state *authstate,
const char *user_id,
struct namespace *namespace,
struct mailbox *src,
struct mailbox *dst,
ptrarray_t *msgrecs)
{
struct appendstate as;
int r;
int nolink = !config_getswitch(IMAPOPT_SINGLEINSTANCESTORE);
r = append_setup_mbox(&as, dst, user_id, authstate,
ACL_INSERT, NULL, namespace, 0, EVENT_MESSAGE_COPY);
if (r) goto done;
r = append_copy(src, &as, msgrecs, nolink,
mboxname_same_userid(src->name, dst->name));
if (r) {
append_abort(&as);
goto done;
}
r = append_commit(&as);
if (r) goto done;
sync_log_mailbox_double(src->name, dst->name);
done:
return r;
}
static int _copy_msgrecord(struct auth_state *authstate,
const char *user_id,
struct namespace *namespace,
struct mailbox *src,
struct mailbox *dst,
msgrecord_t *mrw)
{
if (!strcmp(src->uniqueid, dst->uniqueid))
return 0;
ptrarray_t msgrecs = PTRARRAY_INITIALIZER;
ptrarray_add(&msgrecs, mrw);
int r = _copy_msgrecords(authstate, user_id, namespace, src, dst, &msgrecs);
ptrarray_fini(&msgrecs);
return r;
}
/* A subset of all messages within an IMAP mailbox. */
struct email_mboxrec {
char *mboxname; /* IMAP mailbox name */
char *mbox_id; /* Cyrus-internal unique mailbox id */
ptrarray_t uidrecs; /* Array of struct email_uidrec */
};
/* A single mailboxname/UID pair of the JMAP email identified by
* email_id. Each email has 1 or more uid records, but uid
* records may represent expunged messages. */
struct email_uidrec {
struct email_mboxrec *mboxrec; /* owning mailboxrec */
char *email_id; /* JMAP email id */
uint32_t uid; /* IMAP uid in mbox */
int is_new; /* Used by Email/set{update} */
int is_snoozed; /* Used by Email/set{update} */
};
static void _email_multiexpunge(jmap_req_t *req, struct mailbox *mbox,
ptrarray_t *uidrecs, json_t *errors)
{
int r;
struct mboxevent *mboxevent = NULL;
msgrecord_t *mrw = NULL;
uint32_t system_flags, internal_flags;
mboxevent = mboxevent_new(EVENT_MESSAGE_EXPUNGE);
int j;
int didsome = 0;
for (j = 0; j < ptrarray_size(uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(uidrecs, j);
// skip known errors
if (json_object_get(errors, uidrec->email_id)) {
continue;
}
// load the record
if (mrw) msgrecord_unref(&mrw);
r = msgrecord_find(mbox, uidrec->uid, &mrw);
if (!r) r = msgrecord_get_systemflags(mrw, &system_flags);
if (!r) r = msgrecord_get_internalflags(mrw, &internal_flags);
// already expunged, skip (aka: will be reported as success)
if (internal_flags & FLAG_INTERNAL_EXPUNGED) continue;
// update the flags
if (!r) r = msgrecord_add_systemflags(mrw, FLAG_DELETED);
if (!r) r = msgrecord_add_internalflags(mrw, FLAG_INTERNAL_EXPUNGED);
if (!r) r = msgrecord_rewrite(mrw);
if (!r) {
mboxevent_extract_msgrecord(mboxevent, mrw);
didsome++;
}
// if errors, record the issue
if (r) json_object_set_new(errors, uidrec->email_id, jmap_server_error(r));
}
if (mrw) msgrecord_unref(&mrw);
/* Report mailbox event if anything to say */
if (didsome) {
mboxevent_extract_mailbox(mboxevent, mbox);
mboxevent_set_numunseen(mboxevent, mbox, -1);
mboxevent_set_access(mboxevent, NULL, NULL, req->userid, mbox->name, 0);
mboxevent_notify(&mboxevent);
}
mboxevent_free(&mboxevent);
}
struct email_append_detail {
char blob_id[JMAP_BLOBID_SIZE];
char email_id[JMAP_EMAILID_SIZE];
char thread_id[JMAP_THREADID_SIZE];
size_t size;
};
static void _email_append(jmap_req_t *req,
json_t *mailboxids,
strarray_t *keywords,
time_t internaldate,
json_t *snoozed,
int has_attachment,
const char *sourcefile,
int(*writecb)(jmap_req_t* req, FILE* fp, void* rock, json_t **err),
void *rock,
struct email_append_detail *detail,
json_t **err)
{
int fd;
void *addr;
FILE *f = NULL;
char *mboxname = NULL, *last = NULL;
const char *id;
struct stagemsg *stage = NULL;
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
quota_t qdiffs[QUOTA_NUMRESOURCES] = QUOTA_DIFFS_INITIALIZER;
json_t *val, *mailboxes = NULL;
size_t len;
int r = 0;
time_t savedate = 0;
if (json_object_size(mailboxids) > JMAP_MAIL_MAX_MAILBOXES_PER_EMAIL) {
*err = json_pack("{s:s}", "type", "tooManyMailboxes");
goto done;
}
else if (strarray_size(keywords) > JMAP_MAIL_MAX_KEYWORDS_PER_EMAIL) {
*err = json_pack("{s:s}", "type", "tooManyKeywords");
goto done;
}
if (!internaldate) internaldate = time(NULL);
/* Pick the mailbox to create the message in, prefer \Snoozed then \Drafts */
mailboxes = json_pack("{}"); /* maps mailbox ids to mboxnames */
json_object_foreach(mailboxids, id, val) {
const char *mboxid = id;
/* Lookup mailbox */
if (mboxid && mboxid[0] == '#') {
mboxid = jmap_lookup_id(req, mboxid + 1);
}
if (!mboxid) continue;
mbentry_t *mbentry = _mbentry_by_uniqueid(req, mboxid);
if (!mbentry || !jmap_hasrights(req, mbentry, ACL_LOOKUP)) {
r = IMAP_MAILBOX_NONEXISTENT;
goto done;
}
/* Convert intermediary mailbox to real mailbox */
if (mbentry->mbtype & MBTYPE_INTERMEDIATE) {
r = mboxlist_promote_intermediary(mbentry->name);
if (r) goto done;
}
if (json_is_string(val)) {
/* We flagged this mailboxId as the $snoozed mailbox */
if (mboxname) free(mboxname);
mboxname = xstrdup(mbentry->name);
}
else if (!mboxname) {
mbname_t *mbname = mbname_from_intname(mbentry->name);
/* Is this the draft mailbox? */
struct buf buf = BUF_INITIALIZER;
annotatemore_lookup(mbname_intname(mbname), "/specialuse",
req->accountid, &buf);
if (buf.len) {
strarray_t *uses = strarray_split(buf_cstring(&buf), " ", STRARRAY_TRIM);
if (strarray_find_case(uses, "\\Drafts", 0)) {
if (mboxname) free(mboxname);
mboxname = xstrdup(mbentry->name);
}
strarray_free(uses);
}
buf_free(&buf);
mbname_free(&mbname);
}
/* If we haven't picked a mailbox, remember the last one. */
if (last) free(last);
if (!mboxname) last = xstrdup(mbentry->name);
/* Map mailbox id to mailbox name. */
json_object_set_new(mailboxes, mboxid, json_string(mbentry->name));
mboxlist_entry_free(&mbentry);
}
/* If we haven't picked a mailbox, pick the last one. */
if (!mboxname) mboxname = last;
if (!mboxname) {
char *s = json_dumps(mailboxids, 0);
syslog(LOG_ERR, "_email_append: invalid mailboxids: %s", s);
free(s);
r = IMAP_INTERNAL;
goto done;
}
/* Create the message in the destination mailbox */
r = jmap_openmbox(req, mboxname, &mbox, 1);
if (r) goto done;
if (sourcefile) {
if (!(f = append_newstage_full(mbox->name, internaldate, 0, &stage, sourcefile))) {
syslog(LOG_ERR, "append_newstage(%s) failed", mbox->name);
r = HTTP_SERVER_ERROR;
goto done;
}
}
else {
/* Write the message to the filesystem */
if (!(f = append_newstage(mbox->name, internaldate, 0, &stage))) {
syslog(LOG_ERR, "append_newstage(%s) failed", mbox->name);
r = HTTP_SERVER_ERROR;
goto done;
}
r = writecb(req, f, rock, err);
if (r) goto done;
if (fflush(f)) {
r = IMAP_IOERROR;
goto done;
}
}
fseek(f, 0L, SEEK_END);
len = ftell(f);
/* Generate a GUID from the raw file content */
fd = fileno(f);
if ((addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0))) {
struct message_guid guid;
message_guid_generate(&guid, addr, len);
jmap_set_emailid(&guid, detail->email_id);
jmap_set_blobid(&guid, detail->blob_id);
detail->size = len;
munmap(addr, len);
} else {
r = IMAP_IOERROR;
goto done;
}
fclose(f);
f = NULL;
/* Check if a message with this GUID already exists and is
* visible for the authenticated user. */
char *exist_mboxname = NULL;
uint32_t exist_uid;
r = jmap_email_find(req, detail->email_id, &exist_mboxname, &exist_uid);
free(exist_mboxname);
if (r != IMAP_NOTFOUND) {
if (!r) r = IMAP_MAILBOX_EXISTS;
goto done;
}
/* Great, that's a new message! */
struct body *body = NULL;
struct appendstate as;
/* Prepare flags */
strarray_t flags = STRARRAY_INITIALIZER;
int i;
for (i = 0; i < strarray_size(keywords); i++) {
const char *flag = jmap_keyword_to_imap(strarray_nth(keywords, i));
if (flag) strarray_append(&flags, flag);
}
if (has_attachment) {
strarray_add(&flags, "$hasattachment");
}
/* Append the message to the mailbox. */
qdiffs[QUOTA_MESSAGE] = 1;
qdiffs[QUOTA_STORAGE] = len;
r = append_setup_mbox(&as, mbox, req->userid, httpd_authstate,
0, qdiffs, 0, 0, EVENT_MESSAGE_NEW);
if (r) goto done;
struct entryattlist *annots = NULL;
if (json_is_object(snoozed)) {
const char *annot = IMAP_ANNOT_NS "snoozed";
const char *attrib = "value.shared";
struct buf buf = BUF_INITIALIZER;
char *json = json_dumps(snoozed, JSON_COMPACT);
buf_initm(&buf, json, strlen(json));
setentryatt(&annots, annot, attrib, &buf);
buf_free(&buf);
/* Add \snoozed pseudo-flag */
strarray_add(&flags, "\\snoozed");
/* Extract until and use it as savedate */
time_from_iso8601(json_string_value(json_object_get(snoozed, "until")),
&savedate);
}
r = append_fromstage_full(&as, &body, stage, internaldate, savedate, 0,
flags.count ? &flags : NULL, 0, annots);
freeentryatts(annots);
if (r) {
append_abort(&as);
goto done;
}
strarray_fini(&flags);
message_free_body(body);
free(body);
r = append_commit(&as);
if (r) goto done;
/* Load message record */
r = msgrecord_find(mbox, mbox->i.last_uid, &mr);
if (r) goto done;
bit64 cid;
r = msgrecord_get_cid(mr, &cid);
if (r) goto done;
jmap_set_threadid(cid, detail->thread_id);
/* Complete message creation */
if (stage) {
append_removestage(stage);
stage = NULL;
}
json_object_del(mailboxes, mbox->uniqueid);
/* Make sure there is enough quota for all mailboxes */
qdiffs[QUOTA_STORAGE] = len;
if (json_object_size(mailboxes)) {
char foundroot[MAX_MAILBOX_BUFFER];
json_t *deltas = json_pack("{}");
const char *mbname;
/* Count message delta for each quota root */
json_object_foreach(mailboxes, id, val) {
mbname = json_string_value(val);
if (quota_findroot(foundroot, sizeof(foundroot), mbname)) {
json_t *delta = json_object_get(deltas, mbname);
delta = json_integer(json_integer_value(delta) + 1);
json_object_set_new(deltas, mbname, delta);
}
}
/* Check quota for each quota root. */
json_object_foreach(deltas, mbname, val) {
struct quota quota;
quota_t delta = json_integer_value(val);
quota_init(&quota, mbname);
r = quota_check(&quota, QUOTA_STORAGE, delta * qdiffs[QUOTA_STORAGE]);
if (!r) r = quota_check(&quota, QUOTA_MESSAGE, delta);
quota_free(&quota);
if (r) break;
}
json_decref(deltas);
if (r) goto done;
}
/* Copy the message to all remaining mailboxes */
json_object_foreach(mailboxes, id, val) {
const char *dstname = json_string_value(val);
struct mailbox *dst = NULL;
if (!strcmp(mboxname, dstname))
continue;
r = jmap_openmbox(req, dstname, &dst, 1);
if (r) goto done;
r = _copy_msgrecord(httpd_authstate, req->userid, &jmap_namespace,
mbox, dst, mr);
jmap_closembox(req, &dst);
if (r) goto done;
}
done:
if (f) fclose(f);
append_removestage(stage);
msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
free(mboxname);
json_decref(mailboxes);
if (r && *err == NULL) {
switch (r) {
case IMAP_PERMISSION_DENIED:
*err = json_pack("{s:s}", "type", "forbidden");
break;
case IMAP_MAILBOX_EXISTS:
*err = json_pack("{s:s s:s}", "type", "alreadyExists", "existingId", detail->email_id);
break;
case IMAP_QUOTA_EXCEEDED:
*err = json_pack("{s:s}", "type", "overQuota");
break;
case IMAP_MESSAGE_CONTAINSNULL:
case IMAP_MESSAGE_CONTAINSNL:
case IMAP_MESSAGE_CONTAINS8BIT:
case IMAP_MESSAGE_BADHEADER:
case IMAP_MESSAGE_NOBLANKLINE:
*err = json_pack("{s:s s:s}", "type", "invalidEmail",
"description", error_message(r));
break;
default:
*err = jmap_server_error(r);
}
}
}
struct emailpart {
/* Mandatory fields */
struct headers headers; /* raw headers */
/* Optional fields */
json_t *jpart; /* original EmailBodyPart JSON object */
json_t *jbody; /* EmailBodyValue for text bodies */
char *blob_id; /* blobId to dump contents from */
ptrarray_t subparts; /* array of emailpart pointers */
char *type; /* Content-Type main type */
char *subtype; /* Content-Type subtype */
char *charset; /* Content-Type charset parameter */
char *boundary; /* Content-Type boundary parameter */
char *disposition; /* Content-Disposition without parameters */
char *filename; /* Content-Disposition filename parameter */
};
static void _emailpart_fini(struct emailpart *part)
{
if (!part) return;
struct emailpart *subpart;
while ((subpart = ptrarray_pop(&part->subparts))) {
_emailpart_fini(subpart);
free(subpart);
}
ptrarray_fini(&part->subparts);
json_decref(part->jpart);
json_decref(part->jbody);
_headers_fini(&part->headers);
free(part->type);
free(part->subtype);
free(part->boundary);
free(part->charset);
free(part->disposition);
free(part->filename);
free(part->blob_id);
}
struct email {
struct headers headers; /* parsed headers */
json_t *jemail; /* original Email JSON object */
struct emailpart *body; /* top-level MIME part */
time_t internaldate; /* RFC 3501 internaldate aka receivedAt */
int has_attachment; /* set the HasAttachment flag */
json_t *snoozed; /* set snoozed annotation */
};
static void _email_fini(struct email *email)
{
if (!email) return;
_headers_fini(&email->headers);
json_decref(email->jemail);
_emailpart_fini(email->body);
free(email->body);
}
static json_t *_header_make(const char *header_name, const char *prop_name, struct buf *val)
{
char *tmp = buf_release(val);
json_t *jheader = json_pack("{s:s s:s}", "name", header_name, "value", tmp);
free(tmp);
if (prop_name) json_object_set_new(jheader, "prop", json_string(prop_name));
return jheader;
}
typedef json_t* (*header_from_t)(json_t *jval,
struct jmap_parser *parser,
const char *prop_name,
const char *header_name);
static json_t *_header_from_raw(json_t *jraw,
struct jmap_parser *parser,
const char *prop_name,
const char *header_name)
{
/* Verbatim use header value in raw form */
if (json_is_string(jraw)) {
json_t *jheader = json_pack("{s:s s:O s:s}",
"name", header_name, "value", jraw, "prop", prop_name);
return jheader;
}
else {
jmap_parser_invalid(parser, prop_name);
return NULL;
}
}
static json_t *_header_from_text(json_t *jtext,
struct jmap_parser *parser,
const char *prop_name,
const char *header_name)
{
/* Parse a Text header into raw form */
if (json_is_string(jtext)) {
size_t prefix_len = strlen(header_name) + 2;
const char *s = json_string_value(jtext);
/* Q-encoding will fold lines for us */
int force_quote = prefix_len + strlen(s) > MIME_MAX_HEADER_LENGTH;
char *tmp = charset_encode_mimeheader(s, strlen(s), force_quote);
struct buf val = BUF_INITIALIZER;
/* If text got force-quoted the first line of the Q-encoded
* text might spill over the soft 78-character limit due to
* the Header name prefix. Looking at how most of the mail
* clients are doing this, this seems not to be an issue and
* allows us to not start the header value with a line fold. */
buf_setcstr(&val, tmp);
free(tmp);
return _header_make(header_name, prop_name, &val);
}
else {
jmap_parser_invalid(parser, prop_name);
return NULL;
}
}
static json_t *_header_from_jstrings(json_t *jstrings,
struct jmap_parser *parser,
const char *prop_name,
const char *header_name,
char sep)
{
if (!json_array_size(jstrings)) {
jmap_parser_invalid(parser, prop_name);
return NULL;
}
size_t sep_len = sep ? 1 : 0;
size_t line_len = strlen(header_name) + 2;
size_t i;
json_t *jval;
struct buf val = BUF_INITIALIZER;
json_array_foreach(jstrings, i, jval) {
const char *s = json_string_value(jval);
if (!s) {
jmap_parser_invalid(parser, prop_name);
goto fail;
}
size_t s_len = strlen(s);
if (i && sep) {
buf_putc(&val, sep);
line_len++;
}
if (line_len + s_len + sep_len + 1 > MIME_MAX_HEADER_LENGTH) {
buf_appendcstr(&val, "\r\n ");
line_len = 1;
}
else if (i) {
buf_putc(&val, ' ');
line_len++;
}
buf_appendcstr(&val, s);
line_len += s_len;
}
return _header_make(header_name, prop_name, &val);
fail:
buf_free(&val);
return NULL;
}
static json_t *_header_from_addresses(json_t *addrs,
struct jmap_parser *parser,
const char *prop_name,
const char *header_name)
{
if (!json_array_size(addrs)) {
jmap_parser_invalid(parser, prop_name);
return NULL;
}
size_t i;
json_t *addr;
struct buf emailbuf = BUF_INITIALIZER;
struct buf buf = BUF_INITIALIZER;
json_t *jstrings = json_array();
json_t *ret = NULL;
json_array_foreach(addrs, i, addr) {
json_t *jname = json_object_get(addr, "name");
if (!json_is_string(jname) && JNOTNULL(jname)) {
jmap_parser_push_index(parser, prop_name, i, NULL);
jmap_parser_invalid(parser, "name");
jmap_parser_pop(parser);
}
json_t *jemail = json_object_get(addr, "email");
if (!json_is_string(jemail) && JNOTNULL(jemail)) {
jmap_parser_push_index(parser, prop_name, i, NULL);
jmap_parser_invalid(parser, "email");
jmap_parser_pop(parser);
}
if (json_array_size(parser->invalid))
goto done;
if (!JNOTNULL(jname) && !JNOTNULL(jemail))
continue;
const char *name = json_string_value(jname);
const char *email = json_string_value (jemail);
if (!name && !email) continue;
/* Trim whitespace from email */
if (email) {
buf_setcstr(&emailbuf, email);
buf_trim(&emailbuf);
email = buf_cstring(&emailbuf);
}
if (name && strlen(name) && email) {
enum name_type { ATOM, QUOTED_STRING, HIGH_BIT } name_type = ATOM;
const char *p;
for (p = name; *p; p++) {
char c = *p;
/* Check for ATOM characters */
if (('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'))
continue;
if ('0' <= c && c <= '9')
continue;
if (strchr("!#$%&'*+-/=?^_`{|}~", c))
continue;
if (c < 0) {
/* Contains at least one high bit character. */
name_type = HIGH_BIT;
break;
}
else {
/* Requires at least a quoted string, but could
* still contain a high bit at a later position. */
name_type = QUOTED_STRING;
}
}
if (name_type == ATOM) {
buf_setcstr(&buf, name);
}
else if (name_type == QUOTED_STRING) {
buf_putc(&buf, '"');
for (p = name; *p; p++) {
if (*p == '"' || *p == '\\' || *p == '\r') {
buf_putc(&buf, '\\');
}
buf_putc(&buf, *p);
}
buf_putc(&buf, '"');
}
else if (name_type == HIGH_BIT) {
char *xname = charset_encode_mimephrase(name);
buf_appendcstr(&buf, xname);
free(xname);
}
buf_printf(&buf, " <%s>", email);
} else if (email) {
buf_setcstr(&buf, email);
}
json_array_append_new(jstrings, json_string(buf_cstring(&buf)));
buf_reset(&emailbuf);
buf_reset(&buf);
}
ret = _header_from_jstrings(jstrings, parser, prop_name, header_name, ',');
done:
json_decref(jstrings);
buf_free(&emailbuf);
buf_free(&buf);
return ret;
}
static json_t *_header_from_messageids(json_t *jmessageids,
struct jmap_parser *parser,
const char *prop_name,
const char *header_name)
{
if (!json_array_size(jmessageids)) {
jmap_parser_invalid(parser, prop_name);
return NULL;
}
size_t i;
json_t *jval;
struct buf val = BUF_INITIALIZER;
struct buf msgid = BUF_INITIALIZER;
json_t *jstrings = json_array();
json_t *ret = NULL;
json_array_foreach(jmessageids, i, jval) {
const char *s = json_string_value(jval);
if (!s) {
jmap_parser_invalid(parser, prop_name);
goto done;
}
buf_setcstr(&msgid, s);
buf_trim(&msgid);
buf_putc(&val, '<');
buf_appendcstr(&val, buf_cstring(&msgid));
buf_putc(&val, '>');
if (conversations_check_msgid(buf_base(&val), buf_len(&val))) {
jmap_parser_invalid(parser, prop_name);
goto done;
}
json_array_append_new(jstrings, json_string(buf_cstring(&val)));
buf_reset(&val);
}
ret = _header_from_jstrings(jstrings, parser, prop_name, header_name, 0);
done:
json_decref(jstrings);
buf_free(&msgid);
buf_free(&val);
return ret;
}
static json_t *_header_from_date(json_t *jdate,
struct jmap_parser *parser,
const char *prop_name,
const char *header_name)
{
const char *s = json_string_value(jdate);
if (!s) {
jmap_parser_invalid(parser, prop_name);
return NULL;
}
struct offsettime t;
int n = offsettime_from_iso8601(s, &t);
if (n <= 0 || s[n] != '\0') {
jmap_parser_invalid(parser, prop_name);
return NULL;
}
char fmt[RFC5322_DATETIME_MAX+1];
memset(fmt, 0, RFC5322_DATETIME_MAX+1);
offsettime_to_rfc5322(&t, fmt, RFC5322_DATETIME_MAX+1);
struct buf val = BUF_INITIALIZER;
buf_setcstr(&val, fmt);
return _header_make(header_name, prop_name, &val);
}
static json_t *_header_from_urls(json_t *jurls,
struct jmap_parser *parser,
const char *prop_name,
const char *header_name)
{
if (!json_array_size(jurls)) {
jmap_parser_invalid(parser, prop_name);
return NULL;
}
size_t i;
json_t *jval;
struct buf val = BUF_INITIALIZER;
json_t *jstrings = json_array();
json_t *ret = NULL;
json_array_foreach(jurls, i, jval) {
const char *s = json_string_value(jval);
if (!s) {
jmap_parser_invalid(parser, prop_name);
goto done;
}
buf_appendcstr(&val, "<");
buf_appendcstr(&val, s);
buf_appendcstr(&val, ">");
json_array_append_new(jstrings, json_string(buf_cstring(&val)));
buf_reset(&val);
}
ret = _header_from_jstrings(jstrings, parser, prop_name, header_name, ',');
done:
json_decref(jstrings);
buf_free(&val);
return ret;
}
static void _headers_parseprops(json_t *jobject,
struct jmap_parser *parser,
struct headers *headers)
{
const char *field;
json_t *jval;
json_object_foreach(jobject, field, jval) {
if (strncmp(field, "header:", 7))
continue;
/* Parse header or reject if invalid form */
struct header_prop *hprop = _header_parseprop(field);
if (!hprop) {
jmap_parser_invalid(parser, field);
continue;
}
/* Reject redefinition of header */
if (json_object_get(headers->all, hprop->lcasename)) {
_header_prop_free(hprop);
jmap_parser_invalid(parser, field);
continue;
}
/* Parse header value */
header_from_t cb = NULL;
switch (hprop->form) {
case HEADER_FORM_RAW:
cb = _header_from_raw;
break;
case HEADER_FORM_TEXT:
cb = _header_from_text;
break;
case HEADER_FORM_ADDRESSES:
cb = _header_from_addresses;
break;
case HEADER_FORM_MESSAGEIDS:
cb = _header_from_messageids;
break;
case HEADER_FORM_DATE:
cb = _header_from_date;
break;
case HEADER_FORM_URLS:
cb = _header_from_urls;
break;
default:
syslog(LOG_ERR, "jmap: unknown header form: %d", hprop->form);
jmap_parser_invalid(parser, field);
}
if (!jval || jval == json_null()) {
/* ignore null headers */
_header_prop_free(hprop);
continue;
}
if (hprop->all) {
size_t i;
json_t *jall = jval;
json_array_foreach(jall, i, jval) {
jmap_parser_push_index(parser, field, i, NULL);
json_t *jheader = cb(jval, parser, field, hprop->name);
if (jheader) _headers_add_new(headers, jheader);
jmap_parser_pop(parser);
}
}
else {
json_t *jheader = cb(jval, parser, field, hprop->name);
if (jheader) _headers_add_new(headers, jheader);
}
_header_prop_free(hprop);
}
}
static void _emailpart_parse_headers(json_t *jpart,
struct jmap_parser *parser,
struct emailpart *part)
{
/* headers */
if (JNOTNULL(json_object_get(jpart, "headers"))) {
jmap_parser_invalid(parser, "headers");
}
/* header:Xxx */
const char *lcasename = NULL;
json_t *jheaders;
_headers_parseprops(jpart, parser, &part->headers);
/* Validate Content-Xxx headers */
json_object_foreach(part->headers.all, lcasename, jheaders) {
if (strncmp(lcasename, "content-", 8))
continue;
json_t *jheader = json_array_get(jheaders, 0);
const char *name = json_string_value(json_object_get(jheader, "name"));
const char *val = json_string_value(json_object_get(jheader, "value"));
const char *prop = json_string_value(json_object_get(jheader, "prop"));
/* Reject re-definition of Content-Xxx headers */
if (json_array_size(jheaders) > 1) {
size_t j;
json_array_foreach(jheaders, j, jheader) {
prop = json_string_value(json_object_get(jheader, "prop"));
jmap_parser_invalid(parser, prop);
}
continue;
}
if (!strcasecmp(name, "Content-Type")) {
/* Validate Content-Type */
struct param *type_params = NULL;
message_parse_type(val, &part->type, &part->subtype, &type_params);
if (part->type && part->subtype) {
struct param *param = type_params;
while (param) {
if (!strcasecmp(param->attribute, "BOUNDARY")) {
part->boundary = xstrdupnull(param->value);
}
if (!strcasecmp(param->attribute, "CHARSET")) {
part->charset = xstrdupnull(param->value);
}
param = param->next;
}
/* Headers for multipart MUST specify a boundary */
if (!strcasecmp(part->type, "MULTIPART") && !part->boundary)
jmap_parser_invalid(parser, prop);
/* Headers for bodyparts with partId MUST NOT specify a charset */
if (JNOTNULL(json_object_get(jpart, "partId")) && part->charset)
jmap_parser_invalid(parser, prop);
}
else {
jmap_parser_invalid(parser, prop);
}
param_free(&type_params);
}
else if (!strcasecmp(name, "Content-Disposition")) {
/* Validate Content-Disposition */
struct param *disp_params = NULL;
message_parse_disposition(val, &part->disposition, &disp_params);
if (!part->disposition) {
jmap_parser_invalid(parser, prop);
continue;
}
param_free(&disp_params);
}
else if (!strcasecmp(name, "Content-Transfer-Encoding")) {
/* Always reject Content-Transfer-Encoding */
jmap_parser_invalid(parser, prop);
}
}
}
static struct emailpart *_emailpart_parse(json_t *jpart,
struct jmap_parser *parser,
json_t *bodies)
{
if (!json_is_object(jpart)) {
jmap_parser_invalid(parser, NULL);
return NULL;
}
struct buf buf = BUF_INITIALIZER;
struct emailpart *part = xzmalloc(sizeof(struct emailpart));
part->jpart = json_incref(jpart);
json_t *jval;
/* partId */
json_t *jpartId = json_object_get(jpart, "partId");
if (JNOTNULL(jpartId) && !json_is_string(jpartId)) {
jmap_parser_invalid(parser, "partId");
}
/* blobId */
jval = json_object_get(jpart, "blobId");
if (JNOTNULL(jval) && json_is_string(jval)) {
part->blob_id = xstrdup(json_string_value(jval));
}
else if (JNOTNULL(jval)) {
jmap_parser_invalid(parser, "blobId");
}
/* size */
jval = json_object_get(jpart, "size");
if (JNOTNULL(jval) && (!json_is_integer(jval) || JNOTNULL(jpartId))) {
jmap_parser_invalid(parser, "size");
}
/* Parse headers */
_emailpart_parse_headers(jpart, parser, part);
/* Parse convenience header properties */
int seen_header;
/* cid */
json_t *jcid = json_object_get(jpart, "cid");
seen_header = _headers_have(&part->headers, "Content-Id");
if (json_is_string(jcid) && !seen_header) {
const char *cid = json_string_value(jcid);
buf_setcstr(&buf, "<");
buf_appendcstr(&buf, cid);
buf_appendcstr(&buf, ">");
_headers_add_new(&part->headers, _header_make("Content-Id", "cid", &buf));
}
else if (JNOTNULL(jcid)) {
jmap_parser_invalid(parser, "cid");
}
/* language */
json_t *jlanguage = json_object_get(jpart, "language");
seen_header = _headers_have(&part->headers, "Content-Language");
if (json_is_array(jlanguage) && !seen_header) {
_headers_add_new(&part->headers, _header_from_jstrings(jlanguage,
parser, "language", "Content-Language", ','));
}
else if (JNOTNULL(jlanguage)) {
jmap_parser_invalid(parser, "language");
}
/* location */
json_t *jlocation = json_object_get(jpart, "location");
seen_header = _headers_have(&part->headers, "Content-Location");
if (json_is_string(jlocation) && !seen_header) {
buf_setcstr(&buf, json_string_value(jlocation));
_headers_add_new(&part->headers, _header_make("Content-Location", "location", &buf));
}
else if (JNOTNULL(jlocation)) {
jmap_parser_invalid(parser, "location");
}
/* Check Content-Type and Content-Disposition header properties */
int have_type_header = _headers_have(&part->headers, "Content-Type");
int have_disp_header = _headers_have(&part->headers, "Content-Disposition");
/* name */
json_t *jname = json_object_get(jpart, "name");
if (json_is_string(jname) && !have_type_header && !have_disp_header) {
part->filename = xstrdup(json_string_value(jname));
}
else if (JNOTNULL(jname)) {
jmap_parser_invalid(parser, "name");
}
/* disposition */
json_t *jdisposition = json_object_get(jpart, "disposition");
if (json_is_string(jdisposition) && !have_disp_header) {
/* Build Content-Disposition header */
part->disposition = xstrdup(json_string_value(jdisposition));
buf_setcstr(&buf, part->disposition);
if (part->filename) {
_mime_write_xparam(&buf, "filename", part->filename);
}
_headers_add_new(&part->headers,
_header_make("Content-Disposition", "disposition", &buf));
}
else if (JNOTNULL(jdisposition)) {
jmap_parser_invalid(parser, "disposition");
}
else if (jname) {
/* Make Content-Disposition header */
part->disposition = xstrdup("attachment");
buf_printf(&buf, "attachment");
_mime_write_xparam(&buf, "filename", part->filename);
_headers_add_new(&part->headers,
_header_make("Content-Disposition", "name", &buf));
}
/* charset */
json_t *jcharset = json_object_get(jpart, "charset");
if (json_is_string(jcharset) && !have_type_header && !JNOTNULL(jpartId)) {
part->charset = xstrdup(json_string_value(jcharset));
}
else if (JNOTNULL(jcharset)) {
jmap_parser_invalid(parser, "charset");
}
/* type */
json_t *jtype = json_object_get(jpart, "type");
if (JNOTNULL(jtype) && json_is_string(jtype) && !have_type_header) {
const char *type = json_string_value(jtype);
struct param *type_params = NULL;
/* Validate type value */
message_parse_type(type, &part->type, &part->subtype, &type_params);
if (part->type && part->subtype && !type_params) {
/* Build Content-Type header */
if (!strcasecmp(part->type, "MULTIPART")) {
/* Make boundary */
part->boundary = _mime_make_boundary();
}
buf_reset(&buf);
buf_printf(&buf, "%s/%s", part->type, part->subtype);
buf_lcase(&buf);
if (part->charset) {
buf_appendcstr(&buf, "; charset=");
buf_appendcstr(&buf, part->charset);
}
if (part->filename) {
int force_quote = strlen(part->filename) > MIME_MAX_HEADER_LENGTH;
char *tmp = charset_encode_mimeheader(part->filename, 0, force_quote);
if (force_quote)
buf_appendcstr(&buf, ";\r\n ");
else
buf_appendcstr(&buf, "; ");
buf_appendcstr(&buf, "name=\"");
buf_appendcstr(&buf, tmp);
buf_appendcstr(&buf, "\"");
free(tmp);
}
if (part->boundary) {
buf_appendcstr(&buf, ";\r\n boundary=");
buf_appendcstr(&buf, part->boundary);
}
_headers_add_new(&part->headers,
_header_make("Content-Type", "type", &buf));
}
else {
jmap_parser_invalid(parser, "type");
}
param_free(&type_params);
}
else if (JNOTNULL(jtype)) {
jmap_parser_invalid(parser, "type");
}
/* Validate by type */
const char *part_id = json_string_value(json_object_get(jpart, "partId"));
const char *blob_id = json_string_value(json_object_get(jpart, "blobId"));
json_t *subParts = json_object_get(jpart, "subParts");
json_t *bodyValue = part_id ? json_object_get(bodies, part_id) : NULL;
if (part_id && blob_id)
jmap_parser_invalid(parser, "blobId");
if (part_id && !bodyValue)
jmap_parser_invalid(parser, "partId");
if (subParts || (part->type && !strcasecmp(part->type, "MULTIPART"))) {
/* Parse sub parts */
if (json_array_size(subParts)) {
size_t i;
json_t *subPart;
json_array_foreach(subParts, i, subPart) {
jmap_parser_push_index(parser, "subParts", i, NULL);
struct emailpart *subpart = _emailpart_parse(subPart, parser, bodies);
if (subpart) ptrarray_append(&part->subparts, subpart);
jmap_parser_pop(parser);
}
}
else {
jmap_parser_invalid(parser, "subParts");
}
/* Must not have a body value */
if (JNOTNULL(bodyValue))
jmap_parser_invalid(parser, "partId");
/* Must not have a blobId */
if (blob_id)
jmap_parser_invalid(parser, "blobId");
}
else if (part_id || (part->type && !strcasecmp(part->type, "TEXT"))) {
/* Must have a text body as blob or bodyValue */
if ((bodyValue == NULL) == (blob_id == NULL))
jmap_parser_invalid(parser, "blobId");
/* Must not have sub parts */
if (JNOTNULL(subParts))
jmap_parser_invalid(parser, "subParts");
}
else {
/* Must have a blob id */
if (!blob_id)
jmap_parser_invalid(parser, "blobId");
/* Must not have a text body */
if (bodyValue)
jmap_parser_invalid(parser, "partId");
/* Must not have sub parts */
if (JNOTNULL(subParts))
jmap_parser_invalid(parser, "subParts");
}
buf_free(&buf);
if (json_array_size(parser->invalid)) {
_emailpart_fini(part);
free(part);
return NULL;
}
/* Finalize part definition */
part->jbody = json_incref(bodyValue);
return part;
}
static struct emailpart *_emailpart_new_multi(const char *subtype,
ptrarray_t *subparts)
{
struct emailpart *part = xzmalloc(sizeof(struct emailpart));
int i;
part->type = xstrdup("multipart");
part->subtype = xstrdup(subtype);
part->boundary = _mime_make_boundary();
struct buf val = BUF_INITIALIZER;
buf_printf(&val, "%s/%s;boundary=%s",
part->type, part->subtype, part->boundary);
_headers_add_new(&part->headers,
_header_make("Content-Type", NULL, &val));
for (i = 0; i < subparts->count; i++)
ptrarray_append(&part->subparts, ptrarray_nth(subparts, i));
return part;
}
static struct emailpart *_email_buildbody(struct emailpart *text_body,
struct emailpart *html_body,
ptrarray_t *attachments)
{
struct emailpart *root = NULL;
/* Split attachments into inlined, emails and other files */
ptrarray_t attached_emails = PTRARRAY_INITIALIZER;
ptrarray_t attached_files = PTRARRAY_INITIALIZER;
ptrarray_t inlined_files = PTRARRAY_INITIALIZER;
int i;
for (i = 0; i < attachments->count; i++) {
struct emailpart *part = ptrarray_nth(attachments, i);
if (part->type && !strcasecmp(part->type, "MESSAGE")) {
ptrarray_append(&attached_emails, part);
}
else if (part->disposition && !strcasecmp(part->disposition, "INLINE") &&
(text_body || html_body)) {
ptrarray_append(&inlined_files, part);
}
else {
ptrarray_append(&attached_files, part);
}
}
/* Make MIME part for embedded emails. */
struct emailpart *emails = NULL;
if (attached_emails.count >= 2)
emails = _emailpart_new_multi("digest", &attached_emails);
else if (attached_emails.count == 1)
emails = ptrarray_nth(&attached_emails, 0);
/* Make MIME part for text bodies. */
struct emailpart *text = NULL;
if (text_body && html_body) {
ptrarray_t alternatives = PTRARRAY_INITIALIZER;
ptrarray_append(&alternatives, text_body);
ptrarray_append(&alternatives, html_body);
text = _emailpart_new_multi("alternative", &alternatives);
ptrarray_fini(&alternatives);
}
else if (text_body)
text = text_body;
else if (html_body)
text = html_body;
/* Make MIME part for inlined attachments, if any. */
if (text && inlined_files.count) {
struct emailpart *related = _emailpart_new_multi("related", &inlined_files);
ptrarray_insert(&related->subparts, 0, text);
text = related;
}
/* Choose top-level MIME part. */
if (attached_files.count) {
struct emailpart *mixed = _emailpart_new_multi("mixed", &attached_files);
if (emails) ptrarray_insert(&mixed->subparts, 0, emails);
if (text) ptrarray_insert(&mixed->subparts, 0, text);
root = mixed;
}
else if (text && emails) {
ptrarray_t wrapped = PTRARRAY_INITIALIZER;
ptrarray_append(&wrapped, text);
ptrarray_append(&wrapped, emails);
root = _emailpart_new_multi("mixed", &wrapped);
ptrarray_fini(&wrapped);
}
else if (text)
root = text;
else if (emails)
root = emails;
else
root = NULL;
ptrarray_fini(&attached_emails);
ptrarray_fini(&attached_files);
ptrarray_fini(&inlined_files);
return root;
}
static void _email_parse_bodies(json_t *jemail,
struct jmap_parser *parser,
struct email *email)
{
/* bodyValues */
json_t *bodyValues = json_object_get(jemail, "bodyValues");
if (json_is_object(bodyValues)) {
const char *part_id;
json_t *bodyValue;
jmap_parser_push(parser, "bodyValues");
json_object_foreach(bodyValues, part_id, bodyValue) {
jmap_parser_push(parser, part_id);
if (json_is_object(bodyValue)) {
json_t *jval = json_object_get(bodyValue, "value");
if (!json_is_string(jval)) {
jmap_parser_invalid(parser, "value");
}
jval = json_object_get(bodyValue, "isEncodingProblem");
if (JNOTNULL(jval) && jval != json_false()) {
jmap_parser_invalid(parser, "isEncodingProblem");
}
jval = json_object_get(bodyValue, "isTruncated");
if (JNOTNULL(jval) && jval != json_false()) {
jmap_parser_invalid(parser, "isTruncated");
}
}
else {
jmap_parser_invalid(parser, NULL);
}
jmap_parser_pop(parser);
}
jmap_parser_pop(parser);
}
else if (JNOTNULL(bodyValues)) {
jmap_parser_invalid(parser, "bodyValues");
}
/* bodyStructure */
json_t *jbody = json_object_get(jemail, "bodyStructure");
if (json_is_object(jbody)) {
jmap_parser_push(parser, "bodyStructure");
email->body = _emailpart_parse(jbody, parser, bodyValues);
jmap_parser_pop(parser);
/* Top-level body part MUST NOT redefine headers in Email */
if (email->body) {
const char *name;
json_t *jheader;
json_object_foreach(email->body->headers.all, name, jheader) {
if (json_object_get(email->headers.all, name)) {
/* Report offending header property */
json_t *jprop = json_object_get(jheader, "prop");
const char *prop = json_string_value(jprop);
if (prop) prop = "bodyStructure";
jmap_parser_invalid(parser, prop);
}
}
}
}
else if (JNOTNULL(jbody)) {
jmap_parser_invalid(parser, "bodyStructure");
}
json_t *jtextBody = json_object_get(jemail, "textBody");
json_t *jhtmlBody = json_object_get(jemail, "htmlBody");
json_t *jattachments = json_object_get(jemail, "attachments");
struct emailpart *text_body = NULL;
struct emailpart *html_body = NULL;
ptrarray_t attachments = PTRARRAY_INITIALIZER; /* array of struct emailpart* */
if (JNOTNULL(jbody)) {
/* bodyStructure and fooBody are mutually exclusive */
if (JNOTNULL(jtextBody)) {
jmap_parser_invalid(parser, "textBody");
}
if (JNOTNULL(jhtmlBody)) {
jmap_parser_invalid(parser, "htmlBody");
}
if (JNOTNULL(jattachments)) {
jmap_parser_invalid(parser, "attachments");
}
}
else {
/* textBody */
if (json_array_size(jtextBody) == 1) {
json_t *jpart = json_array_get(jtextBody, 0);
jmap_parser_push_index(parser, "textBody", 0, NULL);
text_body = _emailpart_parse(jpart, parser, bodyValues);
if (text_body) {
if (!text_body->type) {
/* Set default type */
text_body->type = xstrdup("text");
text_body->subtype = xstrdup("plain");
struct buf val = BUF_INITIALIZER;
buf_setcstr(&val, "text/plain");
_headers_add_new(&text_body->headers,
_header_make("Content-Type", NULL, &val));
}
else if (strcasecmp(text_body->type, "text") ||
strcasecmp(text_body->subtype, "plain")) {
jmap_parser_invalid(parser, "type");
}
}
jmap_parser_pop(parser);
}
else if (JNOTNULL(jtextBody)) {
jmap_parser_invalid(parser, "textBody");
}
/* htmlBody */
if (json_array_size(jhtmlBody) == 1) {
json_t *jpart = json_array_get(jhtmlBody, 0);
jmap_parser_push_index(parser, "htmlBody", 0, NULL);
html_body = _emailpart_parse(jpart, parser, bodyValues);
jmap_parser_pop(parser);
if (html_body) {
if (!html_body->type) {
/* Set default type */
html_body->type = xstrdup("text");
html_body->subtype = xstrdup("html");
struct buf val = BUF_INITIALIZER;
buf_setcstr(&val, "text/html");
_headers_add_new(&html_body->headers,
_header_make("Content-Type", NULL, &val));
}
else if (strcasecmp(html_body->type, "text") ||
strcasecmp(html_body->subtype, "html")) {
jmap_parser_invalid(parser, "htmlBody");
}
}
}
else if (JNOTNULL(jhtmlBody)) {
jmap_parser_invalid(parser, "htmlBody");
}
/* attachments */
if (json_is_array(jattachments)) {
size_t i;
json_t *jpart;
struct emailpart *attpart;
int have_inlined = 0;
json_array_foreach(jattachments, i, jpart) {
jmap_parser_push_index(parser, "attachments", i, NULL);
attpart = _emailpart_parse(jpart, parser, bodyValues);
if (attpart) {
if (!have_inlined && attpart->disposition) {
have_inlined = !strcasecmp(attpart->disposition, "INLINE");
}
ptrarray_append(&attachments, attpart);
}
jmap_parser_pop(parser);
}
if (have_inlined && !html_body) {
/* Reject inlined attachments without a HTML body. The client
* is free to produce whatever it wants by setting bodyStructure,
* but for the convenience properties we require sane inputs. */
jmap_parser_invalid(parser, "htmlBody");
}
}
else if (JNOTNULL(jattachments)) {
jmap_parser_invalid(parser, "attachments");
}
}
/* calendarEvents is read-only */
if (JNOTNULL(json_object_get(jemail, "calendarEvents"))) {
jmap_parser_invalid(parser, "calendarEvents");
}
if (!email->body) {
/* Build email body from convenience body properties */
email->body = _email_buildbody(text_body, html_body, &attachments);
}
ptrarray_fini(&attachments);
/* Look through all parts if any part is an attachment.
* If so, set the hasAttachment flag. */
if (email->body) {
ptrarray_t work = PTRARRAY_INITIALIZER;
ptrarray_append(&work, email->body);
while (!email->has_attachment && work.count) {
struct emailpart *part = ptrarray_pop(&work);
if (part->disposition && strcasecmp(part->disposition, "INLINE")) {
email->has_attachment = 1;
}
else if (part->filename) {
email->has_attachment = 1;
}
else if (part->type && strcasecmp(part->type, "TEXT") &&
strcasecmp(part->type, "MULTIPART") &&
(!part->disposition || strcasecmp(part->disposition, "INLINE"))) {
email->has_attachment = 1;
}
else if (part->blob_id && (!part->type || strcasecmp(part->type, "TEXT"))) {
email->has_attachment = 1;
}
else {
int i;
for (i = 0; i < part->subparts.count; i++) {
struct emailpart *subpart = ptrarray_nth(&part->subparts, i);
ptrarray_append(&work, subpart);
}
}
}
ptrarray_fini(&work);
}
}
static void _email_snoozed_parse(json_t *snoozed,
struct jmap_parser *parser)
{
const char *field;
json_t *jval;
jmap_parser_push(parser, "snoozed");
json_object_foreach(snoozed, field, jval) {
if (!strcmp(field, "until")) {
if (!json_is_utcdate(jval)) {
jmap_parser_invalid(parser, "until");
}
}
else if (!strcmp(field, "setKeywords")) {
const char *keyword;
json_t *jbool;
jmap_parser_push(parser, "setKeywords");
json_object_foreach(jval, keyword, jbool) {
if (!json_is_boolean(jbool) ||
!jmap_email_keyword_is_valid(keyword)) {
jmap_parser_invalid(parser, keyword);
}
}
jmap_parser_pop(parser);
}
else if (strcmp(field, "moveToMailboxId")) {
jmap_parser_invalid(parser, field);
}
}
jmap_parser_pop(parser);
}
/* Parse a JMAP Email into its internal representation for creation. */
static void _parse_email(json_t *jemail,
struct jmap_parser *parser,
struct email *email)
{
email->jemail = json_incref(jemail);
/* mailboxIds */
json_t *jmailboxIds = json_object_get(jemail, "mailboxIds");
if (json_object_size(jmailboxIds)) {
const char *mailboxid;
json_t *jval;
jmap_parser_push(parser, "mailboxIds");
json_object_foreach(jmailboxIds, mailboxid, jval) {
if (*mailboxid == '\0' || jval != json_true()) {
jmap_parser_invalid(parser, NULL);
break;
}
}
jmap_parser_pop(parser);
}
else {
jmap_parser_invalid(parser, "mailboxIds");
}
/* keywords */
json_t *jkeywords = json_object_get(jemail, "keywords");
if (json_is_object(jkeywords)) {
const char *keyword;
json_t *jval;
jmap_parser_push(parser, "keywords");
json_object_foreach(jkeywords, keyword, jval) {
if (jval != json_true() || !jmap_email_keyword_is_valid(keyword)) {
jmap_parser_invalid(parser, keyword);
}
}
jmap_parser_pop(parser);
}
else if (JNOTNULL(jkeywords)) {
jmap_parser_invalid(parser, "keywords");
}
/* headers */
if (JNOTNULL(json_object_get(jemail, "headers"))) {
jmap_parser_invalid(parser, "headers");
}
/* header:Xxx */
_headers_parseprops(jemail, parser, &email->headers);
size_t i;
json_t *jheader;
json_array_foreach(email->headers.raw, i, jheader) {
const char *name = json_string_value(json_object_get(jheader, "name"));
const char *val = json_string_value(json_object_get(jheader, "value"));
/* Reject Content-Xxx headers in Email/headers */
if (!strncasecmp("Content-", name, 8)) {
char *tmp = strconcat("header:", name, NULL);
jmap_parser_invalid(parser, tmp);
free(tmp);
}
else if (!strcasecmp("Message-ID", name) || !strcasecmp("In-Reply-To", name)) {
/* conversations.db will barf if these are invalid raw headers,
* so make sure we reject invalid values here. */
if (conversations_check_msgid(val, strlen(val))) {
char *tmp = strconcat("header:", name, NULL);
jmap_parser_invalid(parser, tmp);
free(tmp);
}
}
}
/* Parse convenience header properties - in order as serialised */
struct buf buf = BUF_INITIALIZER;
json_t *prop;
int seen_header;
/* messageId */
prop = json_object_get(jemail, "messageId");
seen_header = _headers_have(&email->headers, "Message-Id");
if (json_array_size(prop) == 1 && !seen_header) {
_headers_add_new(&email->headers, _header_from_messageids(prop,
parser, "messageId", "Message-Id"));
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "messageId");
}
/* inReplyTo */
prop = json_object_get(jemail, "inReplyTo");
seen_header = _headers_have(&email->headers, "In-Reply-To");
if (json_is_array(prop) && !seen_header) {
_headers_add_new(&email->headers, _header_from_messageids(prop,
parser, "inReplyTo", "In-Reply-To"));
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "inReplyTo");
}
/* references */
prop = json_object_get(jemail, "references");
seen_header = _headers_have(&email->headers, "References");
if (json_is_array(prop) && !seen_header) {
_headers_add_new(&email->headers, _header_from_messageids(prop,
parser, "references", "References"));
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "references");
}
/* sentAt */
prop = json_object_get(jemail, "sentAt");
seen_header = _headers_have(&email->headers, "Date");
if (json_is_string(prop) && !seen_header) {
_headers_add_new(&email->headers, _header_from_date(prop,
parser, "sentAt", "Date"));
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "sentAt");
}
/* receivedAt */
prop = json_object_get(jemail, "receivedAt");
if (json_is_utcdate(prop)) {
time_from_iso8601(json_string_value(prop), &email->internaldate);
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "receivedAt");
}
/* from */
prop = json_object_get(jemail, "from");
seen_header = _headers_have(&email->headers, "From");
if (json_is_array(prop) && !seen_header) {
if ((jheader = _header_from_addresses(prop, parser, "from", "From"))) {
_headers_add_new(&email->headers, jheader);
}
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "from");
}
/* replyTo */
prop = json_object_get(jemail, "replyTo");
seen_header = _headers_have(&email->headers, "Reply-To");
if (json_is_array(prop) && !seen_header) {
if ((jheader = _header_from_addresses(prop, parser, "replyTo", "Reply-To"))) {
_headers_add_new(&email->headers, jheader);
}
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "replyTo");
}
/* sender */
prop = json_object_get(jemail, "sender");
seen_header = _headers_have(&email->headers, "Sender");
if (json_is_array(prop) && !seen_header) {
if ((jheader = _header_from_addresses(prop, parser, "sender", "Sender"))) {
_headers_add_new(&email->headers, jheader);
}
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "sender");
}
/* to */
prop = json_object_get(jemail, "to");
seen_header = _headers_have(&email->headers, "To");
if (json_is_array(prop) && !seen_header) {
if ((jheader = _header_from_addresses(prop, parser, "to", "To"))) {
_headers_add_new(&email->headers, jheader);
}
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "to");
}
/* cc */
prop = json_object_get(jemail, "cc");
seen_header = _headers_have(&email->headers, "Cc");
if (json_is_array(prop) && !seen_header) {
if ((jheader = _header_from_addresses(prop, parser, "cc", "Cc"))) {
_headers_add_new(&email->headers, jheader);
}
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "cc");
}
/* bcc */
prop = json_object_get(jemail, "bcc");
seen_header = _headers_have(&email->headers, "Bcc");
if (json_is_array(prop) && !seen_header) {
if ((jheader = _header_from_addresses(prop, parser, "bcc", "Bcc"))) {
_headers_add_new(&email->headers, jheader);
}
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "bcc");
}
/* subject */
prop = json_object_get(jemail, "subject");
seen_header = _headers_have(&email->headers, "Subject");
if (json_is_string(prop) && !seen_header) {
if ((jheader = _header_from_text(prop, parser, "subject", "Subject"))) {
_headers_add_new(&email->headers, jheader);
}
}
else if (JNOTNULL(prop)) {
jmap_parser_invalid(parser, "subject");
}
buf_free(&buf);
/* Parse bodies */
_email_parse_bodies(jemail, parser, email);
/* Is snoozed being set? */
json_t *snoozed = json_object_get(jemail, "snoozed");
if (json_is_object(snoozed)) {
_email_snoozed_parse(snoozed, parser);
}
else if (JNOTNULL(snoozed)) {
jmap_parser_invalid(parser, "snoozed");
}
email->snoozed = snoozed;
}
static void _emailpart_blob_to_mime(jmap_req_t *req,
FILE *fp,
struct emailpart *emailpart,
json_t *missing_blobs)
{
struct buf blob_buf = BUF_INITIALIZER;
msgrecord_t *mr = NULL;
struct mailbox *mbox = NULL;
struct body *body = NULL;
const struct body *part = NULL;
/* Find body part containing blob */
int r = jmap_findblob(req, NULL/*accountid*/, emailpart->blob_id,
&mbox, &mr, &body, &part, &blob_buf);
if (r) goto done;
uint32_t size;
r = msgrecord_get_size(mr, &size);
if (r) goto done;
/* Fetch blob contents and headers */
const char *content = blob_buf.s;
size_t content_size = blob_buf.len;
const char *src_encoding = NULL;
if (part) {
content += part->content_offset;
content_size = part->content_size;
src_encoding = part->encoding;
}
/* Determine target encoding */
const char *encoding = src_encoding;
char *encbuf = NULL;
if (!strcasecmpsafe(emailpart->type, "MESSAGE")) {
if (!strcasecmpsafe(src_encoding, "BASE64")) {
/* This is a MESSAGE and hence it is only allowed
* to be in 7bit, 8bit or binary encoding. Base64
* is not allowed, so let's decode the blob and
* assume it to be in binary encoding. */
encoding = "BINARY";
content = charset_decode_mimebody(content, content_size,
ENCODING_BASE64, &encbuf, &content_size);
}
}
else if (strcasecmpsafe(src_encoding, "QUOTED-PRINTABLE") &&
strcasecmpsafe(src_encoding, "BASE64")) {
/* Encode text to quoted-printable, if it isn't an attachment */
if (!strcasecmpsafe(emailpart->type, "TEXT") &&
(!strcasecmpsafe(emailpart->subtype, "PLAIN") ||
!strcasecmpsafe(emailpart->subtype, "HTML")) &&
(!emailpart->disposition || !strcasecmp(emailpart->disposition, "INLINE"))) {
encoding = "QUOTED-PRINTABLE";
size_t lenqp = 0;
encbuf = charset_qpencode_mimebody(content, content_size, 0, &lenqp);
content = encbuf;
content_size = lenqp;
}
/* Encode all other types to base64 */
else {
encoding = "BASE64";
size_t len64 = 0;
/* Pre-flight encoder to determine length */
charset_encode_mimebody(NULL, content_size, NULL, &len64, NULL, 1 /* wrap */);
if (len64) {
/* Now encode the body */
encbuf = xmalloc(len64);
charset_encode_mimebody(content, content_size, encbuf, &len64, NULL, 1 /* wrap */);
}
content = encbuf;
content_size = len64;
}
}
/* Write headers defined by client. */
size_t i;
json_t *jheader;
json_array_foreach(emailpart->headers.raw, i, jheader) {
json_t *jval = json_object_get(jheader, "name");
const char *name = json_string_value(jval);
jval = json_object_get(jheader, "value");
const char *value = json_string_value(jval);
fprintf(fp, "%s: %s\r\n", name, value);
}
/* Write encoding header, if required */
if (encoding) {
fputs("Content-Transfer-Encoding: ", fp);
fputs(encoding, fp);
fputs("\r\n", fp);
}
/* Write body */
fputs("\r\n", fp);
if (content_size) fwrite(content, 1, content_size, fp);
free(encbuf);
done:
if (r) json_array_append_new(missing_blobs, json_string(emailpart->blob_id));
if (body) {
message_free_body(body);
free(body);
}
msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
buf_free(&blob_buf);
}
static void _emailpart_text_to_mime(FILE *fp, struct emailpart *part)
{
json_t *jval = json_object_get(part->jbody, "value");
const char *text = json_string_value(jval);
size_t len = strlen(text);
/* Check and sanitise text */
int has_long_lines = 0;
int is_7bit = 1;
const char *p = text;
const char *base = text;
const char *top = text + len;
const char *last_lf = p;
struct buf txtbuf = BUF_INITIALIZER;
for (p = base; p < top; p++) {
/* Keep track of line-length and high-bit bytes */
if (p - last_lf > 998)
has_long_lines = 1;
if (*p == '\n')
last_lf = p;
if (*p & 0x80)
is_7bit = 0;
/* Omit CR */
if (*p == '\r')
continue;
/* Expand LF to CRLF */
if (*p == '\n')
buf_putc(&txtbuf, '\r');
buf_putc(&txtbuf, *p);
}
const char *charset = NULL;
if (!is_7bit) charset = "utf-8";
/* Write headers */
size_t i;
json_t *jheader;
json_array_foreach(part->headers.raw, i, jheader) {
json_t *jval = json_object_get(jheader, "name");
const char *name = json_string_value(jval);
jval = json_object_get(jheader, "value");
const char *value = json_string_value(jval);
if (!strcasecmp(name, "Content-Type") && charset) {
/* Clients are forbidden to set charset on TEXT bodies,
* so make sure we properly set the parameter value. */
fprintf(fp, "%s: %s;charset=%s\r\n", name, value, charset);
}
else {
fprintf(fp, "%s: %s\r\n", name, value);
}
}
/* Write body */
if (!is_7bit || has_long_lines) {
/* Write quoted printable */
size_t qp_len = 0;
char *qp_text = charset_qpencode_mimebody(txtbuf.s, txtbuf.len, 1, &qp_len);
fputs("Content-Transfer-Encoding: quoted-printable\r\n", fp);
fputs("\r\n", fp);
fwrite(qp_text, 1, qp_len, fp);
free(qp_text);
}
else {
/* Write plain */
fputs("\r\n", fp);
fwrite(buf_cstring(&txtbuf), 1, buf_len(&txtbuf), fp);
}
buf_free(&txtbuf);
}
static void _emailpart_to_mime(jmap_req_t *req, FILE *fp,
struct emailpart *part,
json_t *missing_blobs)
{
if (part->subparts.count) {
/* Write raw headers */
size_t i;
json_t *jheader;
json_array_foreach(part->headers.raw, i, jheader) {
json_t *jval = json_object_get(jheader, "name");
const char *name = json_string_value(jval);
jval = json_object_get(jheader, "value");
const char *value = json_string_value(jval);
fprintf(fp, "%s: %s\r\n", name, value);
}
/* Write default Content-Type, if not set */
if (!_headers_have(&part->headers, "Content-Type")) {
part->boundary = _mime_make_boundary();
fputs("Content-Type: multipart/mixed;boundary=", fp);
fputs(part->boundary, fp);
fputs("\r\n", fp);
}
/* Write sub parts */
int j;
int is_digest = part->type && !strcasecmp(part->type, "MULTIPART") &&
part->subtype && !strcasecmp(part->subtype, "DIGEST");
for (j = 0; j < part->subparts.count; j++) {
struct emailpart *subpart = ptrarray_nth(&part->subparts, j);
if (is_digest && !subpart->type && subpart->blob_id) {
/* multipart/digest changes the default content type of this
* part from text/plain to message/rfc822, so make sure that
* emailpart_blob_to_mime will properly deal with it */
subpart->type = xstrdup("MESSAGE");
subpart->subtype = xstrdup("RFC822");
}
fprintf(fp, "\r\n--%s\r\n", part->boundary);
_emailpart_to_mime(req, fp, subpart, missing_blobs);
}
fprintf(fp, "\r\n--%s--\r\n", part->boundary);
}
else if (part->jbody) {
_emailpart_text_to_mime(fp, part);
}
else if (part->blob_id) {
_emailpart_blob_to_mime(req, fp, part, missing_blobs);
}
}
static int _email_have_toplevel_header(struct email *email, const char *lcasename)
{
json_t *header = json_object_get(email->headers.all, lcasename);
if (!header && email->body) {
header = json_object_get(email->body->headers.all, lcasename);
}
return JNOTNULL(header);
}
static int _email_to_mime(jmap_req_t *req, FILE *fp, void *rock, json_t **err)
{
struct email *email = rock;
json_t *header;
size_t i;
/* Set mandatory and quasi-mandatory headers */
if (!_email_have_toplevel_header(email, "mime-version")) {
header = json_pack("{s:s s:s}", "name", "Mime-Version", "value", "1.0");
_headers_shift_new(&email->headers, header);
}
if (!_email_have_toplevel_header(email, "user-agent")) {
char *tmp = strconcat("Cyrus-JMAP/", CYRUS_VERSION, NULL);
header = json_pack("{s:s s:s}", "name", "User-Agent", "value", tmp);
_headers_shift_new(&email->headers, header);
free(tmp);
}
if (!_email_have_toplevel_header(email, "message-id")) {
struct buf buf = BUF_INITIALIZER;
buf_printf(&buf, "<%s@%s>", makeuuid(), config_servername);
header = json_pack("{s:s s:s}", "name", "Message-Id", "value", buf_cstring(&buf));
_headers_shift_new(&email->headers, header);
buf_free(&buf);
}
if (!_email_have_toplevel_header(email, "date")) {
char fmt[RFC5322_DATETIME_MAX+1];
memset(fmt, 0, RFC5322_DATETIME_MAX+1);
time_to_rfc5322(time(NULL), fmt, RFC5322_DATETIME_MAX+1);
header = json_pack("{s:s s:s}", "name", "Date", "value", fmt);
_headers_shift_new(&email->headers, header);
}
if (!_email_have_toplevel_header(email, "from")) {
header = json_pack("{s:s s:s}", "name", "From", "value", req->userid);
_headers_shift_new(&email->headers, header);
}
/* Write headers */
json_array_foreach(email->headers.raw, i, header) {
json_t *jval;
jval = json_object_get(header, "name");
const char *name = json_string_value(jval);
jval = json_object_get(header, "value");
const char *value = json_string_value(jval);
fprintf(fp, "%s: %s\r\n", name, value);
}
json_t *missing_blobs = json_array();
if (email->body) _emailpart_to_mime(req, fp, email->body, missing_blobs);
if (json_array_size(missing_blobs)) {
*err = json_pack("{s:s s:o}", "type", "blobNotFound",
"notFound", missing_blobs);
}
else {
json_decref(missing_blobs);
}
return 0;
}
static void _append_validate_mboxids(jmap_req_t *req,
json_t *jmailboxids,
struct jmap_parser *parser,
int *have_snoozed_mboxid)
{
char *snoozed_mboxname = NULL, *snoozed_uniqueid = NULL;
const char *mbox_id;
json_t *jval;
void *tmp;
jmap_mailbox_find_role(req, "snoozed", &snoozed_mboxname, &snoozed_uniqueid);
jmap_parser_push(parser, "mailboxIds");
json_object_foreach_safe(jmailboxids, tmp, mbox_id, jval) {
int need_rights = ACL_LOOKUP|ACL_INSERT;
int is_valid = 1;
if (*mbox_id == '$') {
/* Lookup mailbox by role */
const char *role = mbox_id + 1;
char *uniqueid = NULL;
char *mboxname = NULL;
if (snoozed_uniqueid && !strcmp(role, "snoozed") &&
jmap_hasrights_byname(req, snoozed_mboxname, need_rights)) {
/* Flag this mailboxId as being $snoozed */
json_object_del(jmailboxids, mbox_id);
json_object_set_new(jmailboxids,
snoozed_uniqueid, json_string("$snoozed"));
*have_snoozed_mboxid = 1;
}
else if (!jmap_mailbox_find_role(req, role, &mboxname, &uniqueid) &&
jmap_hasrights_byname(req, mboxname, need_rights)) {
json_object_del(jmailboxids, mbox_id);
json_object_set_new(jmailboxids, uniqueid, json_true());
}
else {
jmap_parser_invalid(parser, NULL);
is_valid = 0;
}
free(uniqueid);
free(mboxname);
}
else {
/* Lookup mailbox by id */
mbentry_t *mbentry = NULL;
if (*mbox_id == '#') {
mbox_id = jmap_lookup_id(req, mbox_id + 1);
}
if (mbox_id) {
mbentry = _mbentry_by_uniqueid(req, mbox_id);
}
if (!mbentry || !jmap_hasrights(req, mbentry, need_rights)) {
jmap_parser_invalid(parser, NULL);
is_valid = 0;
}
else if (!strcmpnull(snoozed_uniqueid, mbentry->uniqueid)) {
/* Flag this mailboxId as being $snoozed */
json_object_set_new(jmailboxids,
mbox_id, json_string("$snoozed"));
*have_snoozed_mboxid = 1;
}
mboxlist_entry_free(&mbentry);
}
if (!is_valid) break;
}
jmap_parser_pop(parser);
free(snoozed_mboxname);
free(snoozed_uniqueid);
}
static void _email_create(jmap_req_t *req,
json_t *jemail,
json_t **new_email,
json_t **set_err)
{
strarray_t keywords = STRARRAY_INITIALIZER;
int r = 0, have_snoozed_mboxid = 0;
*set_err = NULL;
struct email_append_detail detail;
memset(&detail, 0, sizeof(struct email_append_detail));
/* Parse Email object into internal representation */
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct email email = { HEADERS_INITIALIZER, NULL, NULL, time(NULL), 0, NULL };
_parse_email(jemail, &parser, &email);
/* Validate mailboxIds */
json_t *jmailboxids = json_copy(json_object_get(jemail, "mailboxIds"));
_append_validate_mboxids(req, jmailboxids, &parser, &have_snoozed_mboxid);
/* Validate snoozed + mailboxIds */
if (json_is_object(email.snoozed) && !have_snoozed_mboxid) {
jmap_parser_invalid(&parser, "snoozed");
}
if (json_array_size(parser.invalid)) {
*set_err = json_pack("{s:s s:O}", "type", "invalidProperties",
"properties", parser.invalid);
goto done;
}
/* Gather keywords */
json_t *jkeywords = json_object_get(jemail, "keywords");
if (json_object_size(jkeywords)) {
json_t *jval;
const char *keyword;
json_object_foreach(jkeywords, keyword, jval) {
if (!strcasecmp(keyword, JMAP_HAS_ATTACHMENT_FLAG)) {
continue;
}
strarray_append(&keywords, keyword);
}
}
if (keywords.count > MAX_USER_FLAGS) {
*set_err = json_pack("{s:s}", "type", "tooManyKeywords");
goto done;
}
/* Append MIME-encoded Email to mailboxes and write keywords */
_email_append(req, jmailboxids, &keywords, email.internaldate, email.snoozed,
config_getswitch(IMAPOPT_JMAP_SET_HAS_ATTACHMENT) ?
email.has_attachment : 0, NULL, _email_to_mime, &email,
&detail, set_err);
if (*set_err) goto done;
/* Return newly created Email object */
*new_email = json_pack("{s:s, s:s, s:s, s:i}",
"id", detail.email_id,
"blobId", detail.blob_id,
"threadId", detail.thread_id,
"size", detail.size);
*set_err = NULL;
done:
if (r && *set_err == NULL) {
syslog(LOG_ERR, "jmap: email_create: %s", error_message(r));
if (r == IMAP_QUOTA_EXCEEDED)
*set_err = json_pack("{s:s}", "type", "overQuota");
else
*set_err = jmap_server_error(r);
}
json_decref(jmailboxids);
strarray_fini(&keywords);
jmap_parser_fini(&parser);
_email_fini(&email);
}
static int _email_uidrec_compareuid_cb(const void **pa, const void **pb)
{
const struct email_uidrec *a = *pa;
const struct email_uidrec *b = *pb;
if (a->uid < b->uid)
return -1;
else if (a->uid > b->uid)
return 1;
else
return 0;
}
static void _email_mboxrec_free(struct email_mboxrec *mboxrec)
{
struct email_uidrec *uidrec;
while ((uidrec = ptrarray_pop(&mboxrec->uidrecs))) {
free(uidrec->email_id);
free(uidrec);
}
ptrarray_fini(&mboxrec->uidrecs);
free(mboxrec->mboxname);
free(mboxrec->mbox_id);
free(mboxrec);
}
static void _email_mboxrecs_free(ptrarray_t **mboxrecsptr)
{
if (mboxrecsptr == NULL || *mboxrecsptr == NULL) return;
ptrarray_t *mboxrecs = *mboxrecsptr;
int i;
for (i = 0; i < ptrarray_size(mboxrecs); i++) {
_email_mboxrec_free(ptrarray_nth(mboxrecs, i));
}
ptrarray_free(mboxrecs);
*mboxrecsptr = NULL;
}
struct email_mboxrecs_make_rock {
jmap_req_t *req;
const char *email_id;
ptrarray_t *mboxrecs;
};
static int _email_mboxrecs_read_cb(const conv_guidrec_t *rec, void *_rock)
{
struct email_mboxrecs_make_rock *rock = _rock;
ptrarray_t *mboxrecs = rock->mboxrecs;
/* don't process emails that have this email attached! */
if (rec->part) return 0;
if (!jmap_hasrights_byname(rock->req, rec->mboxname, ACL_READ|ACL_LOOKUP)) return 0;
/* Check if there's already a mboxrec for this mailbox. */
int i;
struct email_mboxrec *mboxrec = NULL;
for (i = 0; i < ptrarray_size(mboxrecs); i++) {
struct email_mboxrec *p = ptrarray_nth(mboxrecs, i);
if (!strcmp(rec->mboxname, p->mboxname)) {
mboxrec = p;
break;
}
}
if (mboxrec == NULL) {
mbentry_t *mbentry = NULL;
int r = mboxlist_lookup(rec->mboxname, &mbentry, NULL);
if (r) return r;
mboxrec = xzmalloc(sizeof(struct email_mboxrec));
mboxrec->mboxname = xstrdup(rec->mboxname);
mboxrec->mbox_id = xstrdup(mbentry->uniqueid);
ptrarray_append(mboxrecs, mboxrec);
mboxlist_entry_free(&mbentry);
}
struct email_uidrec *uidrec = xzmalloc(sizeof(struct email_uidrec));
uidrec->mboxrec = mboxrec;
uidrec->email_id = xstrdup(rock->email_id);
uidrec->uid = rec->uid;
uidrec->is_snoozed =
((rec->internal_flags & (FLAG_INTERNAL_SNOOZED | FLAG_INTERNAL_EXPUNGED))
== FLAG_INTERNAL_SNOOZED);
ptrarray_append(&mboxrec->uidrecs, uidrec);
return 0;
}
static void _email_mboxrecs_read(jmap_req_t *req,
struct conversations_state *cstate,
strarray_t *email_ids,
json_t *set_errors,
ptrarray_t **mboxrecsptr)
{
ptrarray_t *mboxrecs = ptrarray_new();
int i;
for (i = 0; i < strarray_size(email_ids); i++) {
const char *email_id = strarray_nth(email_ids, i);
struct email_mboxrecs_make_rock rock = { req, email_id, mboxrecs };
int r = conversations_guid_foreach(cstate, _guid_from_id(email_id),
_email_mboxrecs_read_cb, &rock);
if (r) {
json_t *err = (r == IMAP_NOTFOUND || r == IMAP_PERMISSION_DENIED) ?
json_pack("{s:s}", "notFound") : jmap_server_error(r);
json_object_set_new(set_errors, email_id, err);
_email_mboxrecs_free(&mboxrecs);
}
}
// sort the UID lists
for (i = 0; i < ptrarray_size(mboxrecs); i++) {
struct email_mboxrec *p = ptrarray_nth(mboxrecs, i);
ptrarray_sort(&p->uidrecs, _email_uidrec_compareuid_cb);
}
*mboxrecsptr = mboxrecs;
}
/* Parsed JMAP Email/set#{update} argument. */
struct email_update {
const char *email_id; /* Id of updated JMAP email */
json_t *keywords; /* JMAP Email/set keywords argument */
int patch_keywords; /* True if keywords is a patch object */
json_t *full_keywords; /* Patched JMAP keywords across index records */
json_t *mailboxids; /* JMAP Email/set mailboxIds argument */
int patch_mailboxids; /* True if mailboxids is a patch object */
json_t *snoozed; /* JMAP Email/set snoozed argument */
int patch_snoozed; /* True if snoozed is a patch object */
struct email_uidrec *snoozed_uidrec; /* Currently snoozed email */
char *snooze_in_mboxid; /* Snooze the email in this mailboxid */
};
static void _email_update_free(struct email_update *update)
{
json_decref(update->keywords);
json_decref(update->full_keywords);
json_decref(update->mailboxids);
json_decref(update->snoozed);
free(update->snooze_in_mboxid);
free(update);
}
struct modified_flags {
int added_flags;
bit32 added_system_flags;
bit32 added_user_flags[MAX_USER_FLAGS/32];
int removed_flags;
bit32 removed_system_flags;
bit32 removed_user_flags[MAX_USER_FLAGS/32];
};
/* Overwrite or patch the JMAP keywords on message record mrw.
* If add_seen_uids or del_seen_uids is not NULL, then
* the record UID is added to respective sequence set,
* if the flag must be set or deleted. */
static int _email_setflags(json_t *keywords, int patch_keywords,
msgrecord_t *mrw,
struct seqset *add_seen_uids,
struct seqset *del_seen_uids,
struct modified_flags *modflags)
{
uint32_t internal_flags = 0;
struct mailbox *mbox = NULL;
uint32_t uid = 0;
int r = msgrecord_get_mailbox(mrw, &mbox);
if (r) return r;
r = msgrecord_get_uid(mrw, &uid);
if (r) goto done;
r = msgrecord_get_internalflags(mrw, &internal_flags);
if (r) goto done;
if (internal_flags & FLAG_INTERNAL_EXPUNGED) goto done;
uint32_t old_system_flags = 0;
r = msgrecord_get_systemflags(mrw, &old_system_flags);
if (r) goto done;
uint32_t old_user_flags[MAX_USER_FLAGS/32];
r = msgrecord_get_userflags(mrw, old_user_flags);
if (r) goto done;
/* Determine if to patch or reset flags */
uint32_t new_system_flags = 0;
uint32_t new_user_flags[MAX_USER_FLAGS/32];
if (patch_keywords) {
new_system_flags = old_system_flags;
memcpy(new_user_flags, old_user_flags, sizeof(old_user_flags));
}
else {
new_system_flags = (old_system_flags & ~FLAGS_SYSTEM) |
(old_system_flags & FLAG_DELETED);
memset(new_user_flags, 0, sizeof(new_user_flags));
}
/* Update flags */
json_t *jval;
const char *keyword;
json_object_foreach(keywords, keyword, jval) {
if (!strcasecmp(keyword, "$Flagged")) {
if (jval == json_true())
new_system_flags |= FLAG_FLAGGED;
else
new_system_flags &= ~FLAG_FLAGGED;
}
else if (!strcasecmp(keyword, "$Answered")) {
if (jval == json_true())
new_system_flags |= FLAG_ANSWERED;
else
new_system_flags &= ~FLAG_ANSWERED;
}
else if (!strcasecmp(keyword, "$Seen")) {
if (jval == json_true()) {
if (add_seen_uids)
seqset_add(add_seen_uids, uid, 1);
else
new_system_flags |= FLAG_SEEN;
}
else {
if (del_seen_uids)
seqset_add(del_seen_uids, uid, 1);
else
new_system_flags &= ~FLAG_SEEN;
}
}
else if (!strcasecmp(keyword, "$Draft")) {
if (jval == json_true())
new_system_flags |= FLAG_DRAFT;
else
new_system_flags &= ~FLAG_DRAFT;
}
else {
int userflag;
r = mailbox_user_flag(mbox, keyword, &userflag, 1);
if (r) goto done;
if (jval == json_true())
new_user_flags[userflag/32] |= 1<<(userflag&31);
else
new_user_flags[userflag/32] &= ~(1<<(userflag&31));
}
}
if (!patch_keywords && del_seen_uids) {
if (json_object_get(keywords, "$seen") == NULL) {
seqset_add(del_seen_uids, uid, 1);
}
}
/* Write flags to record */
r = msgrecord_set_systemflags(mrw, new_system_flags);
if (r) goto done;
r = msgrecord_set_userflags(mrw, new_user_flags);
if (r) goto done;
r = msgrecord_rewrite(mrw);
/* Determine flag delta */
memset(modflags, 0, sizeof(struct modified_flags));
modflags->added_system_flags = ~old_system_flags & new_system_flags & FLAGS_SYSTEM;
if (modflags->added_system_flags) {
modflags->added_flags = 1;
}
modflags->removed_system_flags = old_system_flags & ~new_system_flags & FLAGS_SYSTEM;
if (modflags->removed_system_flags) {
modflags->removed_flags = 1;
}
size_t i;
for (i = 0; i < MAX_USER_FLAGS/32; i++) {
modflags->added_user_flags[i] = ~old_user_flags[i] & new_user_flags[i];
if (modflags->added_user_flags[i]) {
modflags->added_flags = 1;
}
modflags->removed_user_flags[i] = old_user_flags[i] & ~new_user_flags[i];
if (modflags->removed_user_flags[i]) {
modflags->removed_flags = 1;
}
}
done:
return r;
}
struct email_bulkupdate {
jmap_req_t *req; /* JMAP Email/set request context */
hash_table updates_by_email_id; /* Map to ptrarray of email_update */
hash_table uidrecs_by_email_id; /* Map to ptrarray of email_uidrec, excluding expunged */
hash_table plans_by_mbox_id; /* Map to email_updateplan */
json_t *set_errors; /* JMAP SetError by email id */
struct seen *seendb; /* Seen database for shared mailboxes, or NULL */
ptrarray_t *cur_mboxrecs; /* List of current mbox and UI recs, including expunged */
ptrarray_t *new_mboxrecs; /* New mbox and UID records allocated by planner */
};
#define _EMAIL_BULKUPDATE_INITIALIZER {\
NULL, \
HASH_TABLE_INITIALIZER, \
HASH_TABLE_INITIALIZER, \
HASH_TABLE_INITIALIZER, \
json_object(), \
NULL, \
NULL, \
ptrarray_new() \
}
static void _email_update_parse(json_t *jemail,
struct jmap_parser *parser,
struct email_update *update)
{
struct buf buf = BUF_INITIALIZER;
/* Are keywords overwritten or patched? */
json_t *keywords = json_object_get(jemail, "keywords");
if (keywords == NULL) {
/* Collect keywords as patch */
const char *field = NULL;
json_t *jval;
keywords = json_object();
json_object_foreach(jemail, field, jval) {
if (strncmp(field, "keywords/", 9)) {
continue;
}
const char *keyword = field + 9;
if (!jmap_email_keyword_is_valid(keyword) || (jval != json_true() && jval != json_null())) {
jmap_parser_push(parser, "keywords");
jmap_parser_invalid(parser, keyword);
jmap_parser_pop(parser);
continue;
}
else if (!strcasecmp(keyword, JMAP_HAS_ATTACHMENT_FLAG)) {
continue;
}
/* At least one keyword gets patched */
update->patch_keywords = 1;
/* Normalize keywords to lowercase */
buf_setcstr(&buf, keyword);
buf_lcase(&buf);
json_object_set(keywords, buf_cstring(&buf), jval);
}
if (!json_object_size(keywords)) {
json_decref(keywords);
keywords = NULL;
}
}
else if (json_is_object(keywords)) {
/* Overwrite keywords */
json_t *normalized_keywords = json_object();
const char *keyword;
json_t *jval;
json_object_foreach(keywords, keyword, jval) {
if (!jmap_email_keyword_is_valid(keyword) || jval != json_true()) {
jmap_parser_push(parser, "keywords");
jmap_parser_invalid(parser, keyword);
jmap_parser_pop(parser);
continue;
}
else if (!strcasecmp(keyword, JMAP_HAS_ATTACHMENT_FLAG)) {
continue;
}
buf_setcstr(&buf, keyword);
buf_lcase(&buf);
json_object_set(normalized_keywords, buf_cstring(&buf), jval);
}
keywords = normalized_keywords;
}
else if (JNOTNULL(keywords)) {
jmap_parser_invalid(parser, "keywords");
}
update->keywords = keywords;
/* Are mailboxes being overwritten or patched? */
json_t *mailboxids = json_copy(json_object_get(jemail, "mailboxIds"));
if (mailboxids == NULL) {
/* Collect mailboxids as patch */
const char *field = NULL;
json_t *jval;
mailboxids = json_object();
/* Check if mailboxIds are patched */
json_object_foreach(jemail, field, jval) {
if (strncmp(field, "mailboxIds/", 11)) {
continue;
}
update->patch_mailboxids = 1;
if (jval == json_true() || jval == json_null()) {
json_object_set(mailboxids, field + 11, jval);
}
else {
jmap_parser_invalid(parser, field);
}
}
if (json_object_size(mailboxids) == 0) {
json_decref(mailboxids);
mailboxids = NULL;
}
}
update->mailboxids = mailboxids;
/* Is snoozed being overwritten or patched? */
json_t *snoozed = json_copy(json_object_get(jemail, "snoozed"));
if (snoozed == NULL) {
/* Collect fields as patch */
const char *field = NULL;
json_t *jval;
snoozed = json_object();
json_object_foreach(jemail, field, jval) {
int invalid = 0;
if (strncmp(field, "snoozed/", 8)) {
continue;
}
const char *subfield = field +8;
if (!strcmp(subfield, "until")) {
if (!json_is_utcdate(jval)) invalid = 1;
}
else if (!strncmp(subfield, "setKeywords/", 12)) {
const char *keyword = subfield + 12;
if (!(json_is_boolean(jval) || json_is_null(jval)) ||
!jmap_email_keyword_is_valid(keyword)) invalid = 1;
}
else if (!strcmp(subfield, "moveToMailboxId")) {
if (!json_is_string(jval)) invalid = 1;
}
else invalid = 1;
if (invalid) {
jmap_parser_invalid(parser, field);
}
else {
/* At least one field gets patched */
update->patch_snoozed = 1;
json_object_set(snoozed, subfield, jval);
}
}
if (json_object_size(snoozed) == 0) {
json_decref(snoozed);
snoozed = NULL;
}
}
else if (json_is_object(snoozed)) {
_email_snoozed_parse(snoozed, parser);
}
else if (JNOTNULL(snoozed)) {
jmap_parser_invalid(parser, "snoozed");
}
update->snoozed = snoozed;
buf_free(&buf);
}
/* A plan to create, update or destroy messages per mailbox */
struct email_updateplan {
char *mboxname; /* Mailbox IMAP name */
char *mbox_id; /* Mailbox unique id */
struct mailbox *mbox; /* Write-locked mailbox */
ptrarray_t copy; /* Array of array of email_uidrec, grouped by mailbox */
ptrarray_t setflags; /* Array of email_uidrec */
ptrarray_t delete; /* Array of email_uidrec */
ptrarray_t snooze; /* Array of email_uidrec */
int needrights; /* Required ACL bits set */
int use_seendb; /* Set if this mailbox requires seen.db */
struct email_mboxrec *mboxrec; /* Mailbox record */
struct seendata old_seendata; /* Lock-read seen data from database */
struct seqset *old_seenseq; /* Parsed seen sequence before update */
};
void _email_updateplan_free_p(void* p)
{
struct email_updateplan *plan = p;
seqset_free(plan->old_seenseq);
seen_freedata(&plan->old_seendata);
free(plan->mboxname);
free(plan->mbox_id);
ptrarray_t *tmp;
while ((tmp = ptrarray_pop(&plan->copy))) {
ptrarray_free((ptrarray_t*)tmp);
}
ptrarray_fini(&plan->copy);
ptrarray_fini(&plan->setflags);
ptrarray_fini(&plan->delete);
ptrarray_fini(&plan->snooze);
free(plan);
}
void _ptrarray_free_p(void *p)
{
ptrarray_free((ptrarray_t*)p);
}
void _email_bulkupdate_close(struct email_bulkupdate *bulk)
{
hash_iter *iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
jmap_closembox(bulk->req, &plan->mbox);
}
seen_close(&bulk->seendb); /* force-close on error */
hash_iter_free(&iter);
free_hash_table(&bulk->uidrecs_by_email_id, _ptrarray_free_p);
free_hash_table(&bulk->updates_by_email_id, NULL);
free_hash_table(&bulk->plans_by_mbox_id, _email_updateplan_free_p);
_email_mboxrecs_free(&bulk->cur_mboxrecs);
_email_mboxrecs_free(&bulk->new_mboxrecs);
json_decref(bulk->set_errors);
}
static struct email_updateplan *_email_bulkupdate_addplan(struct email_bulkupdate *bulk,
struct mailbox *mbox,
struct email_mboxrec *mboxrec)
{
struct email_updateplan *plan = xzmalloc(sizeof(struct email_updateplan));
plan->mbox = mbox;
plan->mbox_id = xstrdup(mbox->uniqueid);
plan->mboxname = xstrdup(mbox->name);
plan->mboxrec = mboxrec;
plan->use_seendb = !mailbox_internal_seen(plan->mbox, bulk->req->userid);
hash_insert(plan->mbox_id, plan, &bulk->plans_by_mbox_id);
return plan;
}
static void _email_updateplan_error(struct email_updateplan *plan, int errcode, json_t *set_errors)
{
json_t *err;
switch (errcode) {
case IMAP_PERMISSION_DENIED:
err = json_pack("{s:s}", "type", "forbidden");
break;
case IMAP_QUOTA_EXCEEDED:
err = json_pack("{s:s}", "type", "overQuota");
break;
default:
err = jmap_server_error(errcode);
}
int i;
for (i = 0; i < ptrarray_size(&plan->copy); i++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->copy, i);
if (json_object_get(set_errors, uidrec->email_id)) {
continue;
}
json_object_set(set_errors, uidrec->email_id, err);
}
for (i = 0; i < ptrarray_size(&plan->setflags); i++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->setflags, i);
if (json_object_get(set_errors, uidrec->email_id)) {
continue;
}
json_object_set(set_errors, uidrec->email_id, err);
}
for (i = 0; i < ptrarray_size(&plan->delete); i++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->delete, i);
if (json_object_get(set_errors, uidrec->email_id)) {
continue;
}
json_object_set(set_errors, uidrec->email_id, err);
}
for (i = 0; i < ptrarray_size(&plan->snooze); i++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->snooze, i);
if (json_object_get(set_errors, uidrec->email_id)) {
continue;
}
json_object_set(set_errors, uidrec->email_id, err);
}
json_decref(err);
}
static void _email_bulkupdate_plan_mailboxids(struct email_bulkupdate *bulk, ptrarray_t *updates)
{
hash_table copyupdates_by_mbox_id = HASH_TABLE_INITIALIZER;
construct_hash_table(&copyupdates_by_mbox_id, ptrarray_size(updates)+1, 0);
int i;
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
const char *email_id = update->email_id;
if (json_object_get(bulk->set_errors, email_id)) {
continue;
}
ptrarray_t *current_uidrecs = hash_lookup(email_id, &bulk->uidrecs_by_email_id);
if (update->snooze_in_mboxid) {
/* Update/delete existing snoozeDetails */
if (update->snoozed) {
/* Make a new copy of current snoozed message */
ptrarray_t *copyupdates =
hash_lookup(update->snooze_in_mboxid,
&copyupdates_by_mbox_id);
if (copyupdates == NULL) {
copyupdates = ptrarray_new();
hash_insert(update->snooze_in_mboxid,
copyupdates, &copyupdates_by_mbox_id);
}
ptrarray_append(copyupdates, update);
}
if (update->snoozed_uidrec &&
!strcmp(update->snooze_in_mboxid,
update->snoozed_uidrec->mboxrec->mbox_id)) {
/* Delete current snoozed message from mailbox */
struct email_updateplan *plan =
hash_lookup(update->snooze_in_mboxid, &bulk->plans_by_mbox_id);
ptrarray_append(&plan->delete, update->snoozed_uidrec);
plan->needrights |= ACL_EXPUNGE|ACL_DELETEMSG;
}
}
if (!update->mailboxids) {
continue;
}
if (update->patch_mailboxids) {
const char *mbox_id = NULL;
json_t *jval = NULL;
json_object_foreach(update->mailboxids, mbox_id, jval) {
int j;
/* Lookup the uid record of this email in this mailbox, can be NULL. */
struct email_uidrec *uidrec = NULL;
struct email_updateplan *plan = hash_lookup(mbox_id, &bulk->plans_by_mbox_id);
for (j = 0; j < ptrarray_size(current_uidrecs); j++) {
struct email_uidrec *tmp = ptrarray_nth(current_uidrecs, j);
if (!strcmp(mbox_id, tmp->mboxrec->mbox_id)) {
uidrec = tmp;
break;
}
}
/* Patch the mailbox */
if (jval == json_true()) {
if (uidrec) {
/* This email is patched to stay in it's mailbox. Whatever. */
}
else {
/* This is a new mailbox for this email. Copy it over. */
ptrarray_t *copyupdates = hash_lookup(mbox_id, &copyupdates_by_mbox_id);
if (copyupdates == NULL) {
copyupdates = ptrarray_new();
hash_insert(mbox_id, copyupdates, &copyupdates_by_mbox_id);
}
/* XXX Use ptrarray_add() here to avoid duplicating
an update already done above via snooze */
ptrarray_add(copyupdates, update);
}
}
else {
if (uidrec) {
/* Delete the email from this mailbox. */
ptrarray_append(&plan->delete, uidrec);
plan->needrights |= ACL_EXPUNGE|ACL_DELETEMSG;
}
}
}
}
else {
json_t *mailboxids = json_deep_copy(update->mailboxids);
int j;
/* For all current uid records of this email, determine if to
* keep, create or delete them in their respective mailbox. */
for (j = 0; j < ptrarray_size(current_uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(current_uidrecs, j);
struct email_mboxrec *mboxrec = uidrec->mboxrec;
struct email_updateplan *plan = hash_lookup(mboxrec->mbox_id, &bulk->plans_by_mbox_id);
json_t *keep = json_object_get(mailboxids, mboxrec->mbox_id);
if (keep) {
/* Keep message in mailbox */
json_object_del(mailboxids, mboxrec->mbox_id);
}
else {
/* Delete message from mailbox */
ptrarray_append(&plan->delete, uidrec);
plan->needrights |= ACL_EXPUNGE|ACL_DELETEMSG;
}
}
/* Copy message to any new mailboxes which weren't seen in uidrecs */
const char *mbox_id;
json_t *jval;
json_object_foreach(mailboxids, mbox_id, jval) {
ptrarray_t *copyupdates = hash_lookup(mbox_id, &copyupdates_by_mbox_id);
if (copyupdates == NULL) {
copyupdates = ptrarray_new();
hash_insert(mbox_id, copyupdates, &copyupdates_by_mbox_id);
}
ptrarray_append(copyupdates, update);
}
json_decref(mailboxids);
}
}
/* Cluster copy operations by mailbox */
hash_iter *iter = hash_table_iter(&copyupdates_by_mbox_id);
while (hash_iter_next(iter)) {
const char *dst_mbox_id = hash_iter_key(iter);
ptrarray_t *updates = hash_iter_val(iter);
/* Determine the number of messages per source mailbox which
* could be copied into the destination mailbox. */
hash_table src_mbox_id_counts = HASH_TABLE_INITIALIZER;
construct_hash_table(&src_mbox_id_counts, ptrarray_size(updates)*8+1, 0);
int i;
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
ptrarray_t *current_uidrecs = hash_lookup(update->email_id, &bulk->uidrecs_by_email_id);
int j;
for (j = 0; j < ptrarray_size(current_uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(current_uidrecs, j);
const char *src_mbox_id = uidrec->mboxrec->mbox_id;
uintptr_t count = (uintptr_t) hash_lookup(src_mbox_id, &src_mbox_id_counts);
hash_insert(src_mbox_id, (void*) count++, &src_mbox_id_counts);
}
}
/* For each copy update, pick the uid record from the source mailbox
* that minimizes append_copy calls between mailboxes. */
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
ptrarray_t *current_uidrecs = hash_lookup(update->email_id, &bulk->uidrecs_by_email_id);
int j;
struct email_uidrec *pick_uidrec = ptrarray_nth(current_uidrecs, 0);
uintptr_t best_count = (uintptr_t) hash_lookup(pick_uidrec->mboxrec->mbox_id, &src_mbox_id_counts);
for (j = 1; j < ptrarray_size(current_uidrecs); j++) {
struct email_uidrec *tmp = ptrarray_nth(current_uidrecs, j);
uintptr_t count = (uintptr_t) hash_lookup(tmp->mboxrec->mbox_id, &src_mbox_id_counts);
if (count > best_count) {
pick_uidrec = tmp;
best_count = count;
}
}
/* Add the picked uid record to its slot in the copy plan */
struct email_updateplan *plan = hash_lookup(dst_mbox_id, &bulk->plans_by_mbox_id);
ptrarray_t *pick_uidrecs = NULL;
for (j = 0; j < ptrarray_size(&plan->copy); j++) {
ptrarray_t *copy_uidrecs = ptrarray_nth(&plan->copy, j);
struct email_uidrec *tmp = ptrarray_nth(copy_uidrecs, 0);
if (!strcmp(tmp->mboxrec->mbox_id, pick_uidrec->mboxrec->mbox_id)) {
/* We found an existing slot in the copy plan */
pick_uidrecs = copy_uidrecs;
break;
}
}
if (pick_uidrecs == NULL) {
pick_uidrecs = ptrarray_new();
ptrarray_append(&plan->copy, pick_uidrecs);
}
ptrarray_append(pick_uidrecs, pick_uidrec);
plan->needrights |= ACL_INSERT;
}
free_hash_table(&src_mbox_id_counts, NULL);
}
hash_iter_free(&iter);
free_hash_table(&copyupdates_by_mbox_id, _ptrarray_free_p);
}
static void _email_bulkupdate_checklimits(struct email_bulkupdate *bulk)
{
/* Validate mailbox counts per email */
hash_table mbox_ids_by_email_id = HASH_TABLE_INITIALIZER;
construct_hash_table(&mbox_ids_by_email_id, hash_numrecords(&bulk->uidrecs_by_email_id)+1, 0);
/* Collect current mailboxes per email */
hash_iter *iter = hash_table_iter(&bulk->uidrecs_by_email_id);
while (hash_iter_next(iter)) {
ptrarray_t *uidrecs = hash_iter_val(iter);
int i;
for (i = 0; i < ptrarray_size(uidrecs); i++) {
struct email_uidrec *uidrec = ptrarray_nth(uidrecs, i);
strarray_t *mbox_ids = hash_lookup(uidrec->email_id, &mbox_ids_by_email_id);
if (!mbox_ids) {
mbox_ids = strarray_new();
hash_insert(uidrec->email_id, mbox_ids, &mbox_ids_by_email_id);
}
strarray_add(mbox_ids, uidrec->mboxrec->mbox_id);
}
}
hash_iter_free(&iter);
/* Apply plans to mailbox counts */
iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
int i;
for (i = 0; i < ptrarray_size(&plan->copy); i++) {
ptrarray_t *uidrecs = ptrarray_nth(&plan->copy, i);
int j;
for (j = 0; j < ptrarray_size(uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(uidrecs, j);
strarray_t *mbox_ids = hash_lookup(uidrec->email_id, &mbox_ids_by_email_id);
if (!mbox_ids) {
mbox_ids = strarray_new();
hash_insert(uidrec->email_id, mbox_ids, &mbox_ids_by_email_id);
}
strarray_add(mbox_ids, plan->mbox_id);
}
}
for (i = 0; i < ptrarray_size(&plan->delete); i++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->delete, i);
strarray_t *mbox_ids = hash_lookup(uidrec->email_id, &mbox_ids_by_email_id);
if (!mbox_ids) {
mbox_ids = strarray_new();
hash_insert(uidrec->email_id, mbox_ids, &mbox_ids_by_email_id);
}
strarray_remove_all(mbox_ids, plan->mbox_id);
}
}
hash_iter_free(&iter);
/* Validate mailbox counts */
iter = hash_table_iter(&mbox_ids_by_email_id);
while (hash_iter_next(iter)) {
const char *email_id = hash_iter_key(iter);
strarray_t *mbox_ids = hash_iter_val(iter);
if (!mbox_ids) continue;
if (strarray_size(mbox_ids) > JMAP_MAIL_MAX_MAILBOXES_PER_EMAIL) {
if (json_object_get(bulk->set_errors, email_id) == NULL) {
json_object_set_new(bulk->set_errors, email_id,
json_pack("{s:s}", "type", "tooManyMailboxes"));
}
}
strarray_free(mbox_ids);
}
hash_iter_free(&iter);
free_hash_table(&mbox_ids_by_email_id, NULL);
/* Validate keyword counts. This assumes keyword patches already
* have been replaced with the complete set of patched keywords. */
iter = hash_table_iter(&bulk->updates_by_email_id);
while (hash_iter_next(iter)) {
struct email_update *update = hash_iter_val(iter);
if (json_object_get(bulk->set_errors, update->email_id)) {
continue;
}
if (json_object_size(update->keywords) > JMAP_MAIL_MAX_KEYWORDS_PER_EMAIL) {
json_object_set_new(bulk->set_errors, update->email_id,
json_pack("{s:s}", "type", "tooManyKeywords"));
}
}
hash_iter_free(&iter);
}
static json_t *_email_bulkupdate_aggregate_keywords(struct email_bulkupdate *bulk,
const char *email_id,
hash_table *seenseq_by_mbox_id)
{
ptrarray_t *current_uidrecs = hash_lookup(email_id, &bulk->uidrecs_by_email_id);
struct email_keywords keywords = _EMAIL_KEYWORDS_INITIALIZER;
_email_keywords_init(&keywords, bulk->req->userid, bulk->seendb, seenseq_by_mbox_id);
int i;
for (i = 0; i < ptrarray_size(current_uidrecs); i++) {
struct email_uidrec *uidrec = ptrarray_nth(current_uidrecs, i);
struct email_updateplan *plan = hash_lookup(uidrec->mboxrec->mbox_id,
&bulk->plans_by_mbox_id);
msgrecord_t *mr = NULL;
int r = msgrecord_find(plan->mbox, uidrec->uid, &mr);
if (!r) _email_keywords_add_msgrecord(&keywords, mr);
if (r) {
if (!json_object_get(bulk->set_errors, uidrec->email_id)) {
json_object_set_new(bulk->set_errors, uidrec->email_id,
jmap_server_error(r));
}
}
msgrecord_unref(&mr);
}
json_t *aggregated_keywords = _email_keywords_to_jmap(&keywords);
_email_keywords_fini(&keywords);
return aggregated_keywords;
}
static int _flag_update_changes_seen(json_t *new, json_t *old)
{
int is_seen = json_object_get(new, "$seen") ? 1 : 0;
int was_seen = (old && json_object_get(old, "$seen")) ? 1 : 0;
return is_seen != was_seen;
}
static int _flag_update_changes_not_seen(json_t *new, json_t *old)
{
const char *name;
json_t *val;
json_object_foreach(new, name, val) {
if (!strcmp(name, "$seen")) continue;
int was_seen = (old && json_object_get(old, name)) ? 1 : 0;
if (!was_seen) return 1;
}
if (old) {
json_object_foreach(old, name, val) {
if (!strcmp(name, "$seen")) continue;
int is_seen = json_object_get(new, name) ? 1 : 0;
if (!is_seen) return 1;
}
}
return 0;
}
static void _email_bulkupdate_plan_keywords(struct email_bulkupdate *bulk, ptrarray_t *updates)
{
int i;
/* Open seen.db, if required */
if (strcmp(bulk->req->accountid, bulk->req->userid)) {
int r = seen_open(bulk->req->userid, SEEN_CREATE, &bulk->seendb);
if (r) {
/* There's something terribly wrong. Abort all updates. */
syslog(LOG_ERR, "_email_bulkupdate_plan_keywords: can't open seen.db: %s",
error_message(r));
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
if (json_object_get(bulk->set_errors, update->email_id) == NULL) {
json_object_set_new(bulk->set_errors, update->email_id,
jmap_server_error(r));
}
}
return;
}
}
/* Add uid records to each mailboxes setflags plan */
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
const char *email_id = update->email_id;
if (!update->keywords || json_object_get(bulk->set_errors, email_id)) {
continue;
}
ptrarray_t *current_uidrecs = hash_lookup(email_id, &bulk->uidrecs_by_email_id);
int j;
for (j = 0; j < ptrarray_size(current_uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(current_uidrecs, j);
struct email_mboxrec *mboxrec = uidrec->mboxrec;
struct email_updateplan *plan = hash_lookup(mboxrec->mbox_id, &bulk->plans_by_mbox_id);
if (!jmap_hasrights_byname(bulk->req, plan->mboxname, ACL_READ|ACL_LOOKUP)) {
continue;
}
if (!update->mailboxids) {
/* Add keyword update to all current uid records */
ptrarray_append(&plan->setflags, uidrec);
}
else {
/* Add keyword update to all current records that won't be deleted */
json_t *jval = json_object_get(update->mailboxids, mboxrec->mbox_id);
if (jval == json_true() || (jval == NULL && update->patch_mailboxids)) {
ptrarray_append(&plan->setflags, uidrec);
}
}
}
}
hash_table seenseq_by_mbox_id = HASH_TABLE_INITIALIZER;
construct_hash_table(&seenseq_by_mbox_id, hash_numrecords(&bulk->plans_by_mbox_id)+1, 0);
/* Plan keyword updates per mailbox */
hash_iter *iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
if (plan->use_seendb) {
/* Read seen sequence set */
int r = seen_lockread(bulk->seendb, plan->mbox->uniqueid, &plan->old_seendata);
if (!r) {
plan->old_seenseq = seqset_parse(plan->old_seendata.seenuids, NULL, 0);
if (!hash_lookup(plan->mbox_id, &seenseq_by_mbox_id))
hash_insert(plan->mbox_id, seqset_dup(plan->old_seenseq), &seenseq_by_mbox_id);
}
else {
int j;
for (j = 0; j < ptrarray_size(&plan->setflags); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->setflags, j);
if (json_object_get(bulk->set_errors, uidrec->email_id) == NULL) {
json_object_set_new(bulk->set_errors, uidrec->email_id,
jmap_server_error(r));
}
}
}
}
/* Determine the ACL and keywords for all new uid records */
for (i = 0; i < ptrarray_size(&plan->copy); i++) {
ptrarray_t *copy_uidrecs = ptrarray_nth(&plan->copy, i);
int j;
for (j = 0; j < ptrarray_size(copy_uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(copy_uidrecs, j);
if (json_object_get(bulk->set_errors, uidrec->email_id)) {
continue;
}
struct email_update *update = hash_lookup(uidrec->email_id,
&bulk->updates_by_email_id);
if (!update->full_keywords) {
/* Determine the full set of keywords to write on this record */
if (!update->keywords) {
/* Write the combined keywords of all records of this email */
update->full_keywords = _email_bulkupdate_aggregate_keywords(bulk,
uidrec->email_id, &seenseq_by_mbox_id);
}
else if (update->patch_keywords) {
/* Write the patched, combined keywords */
json_t *aggregated_keywords = _email_bulkupdate_aggregate_keywords(bulk,
uidrec->email_id, &seenseq_by_mbox_id);
update->full_keywords = jmap_patchobject_apply(aggregated_keywords,
update->keywords);
json_decref(aggregated_keywords);
}
else {
/* Write the keywords defined in the update */
update->full_keywords = json_incref(update->keywords);
}
}
/* Determine required ACL rights */
if (_flag_update_changes_seen(update->full_keywords, NULL))
plan->needrights |= ACL_SETSEEN;
if (_flag_update_changes_not_seen(update->full_keywords, NULL))
plan->needrights |= ACL_WRITE;
/* XXX - what about annotations? */
}
}
/* Determine ACL for all existing uid records with updated keywords */
for (i = 0; i < ptrarray_size(&plan->setflags); i++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->setflags, i);
if (json_object_get(bulk->set_errors, uidrec->email_id)) {
continue;
}
struct email_update *update = hash_lookup(uidrec->email_id,
&bulk->updates_by_email_id);
/* Convert flags to JMAP keywords */
struct email_keywords keywords = _EMAIL_KEYWORDS_INITIALIZER;
_email_keywords_init(&keywords, bulk->req->userid, bulk->seendb, &seenseq_by_mbox_id);
msgrecord_t *mr = NULL;
int r = msgrecord_find(plan->mbox, uidrec->uid, &mr);
if (!r) _email_keywords_add_msgrecord(&keywords, mr);
if (r) {
json_object_set_new(bulk->set_errors, uidrec->email_id,
jmap_server_error(r));
}
msgrecord_unref(&mr);
json_t *current_keywords = _email_keywords_to_jmap(&keywords);
_email_keywords_fini(&keywords);
json_t *new_keywords;
if (update->patch_keywords) {
new_keywords = jmap_patchobject_apply(current_keywords, update->keywords);
}
else {
new_keywords = json_incref(update->keywords);
}
/* Determine required ACL rights */
if (_flag_update_changes_seen(new_keywords, current_keywords))
plan->needrights |= ACL_SETSEEN;
if (_flag_update_changes_not_seen(new_keywords, current_keywords))
plan->needrights |= ACL_WRITE;
/* XXX - what about annotations? */
json_decref(new_keywords);
json_decref(current_keywords);
}
}
hash_iter_free(&iter);
free_hash_table(&seenseq_by_mbox_id, (void(*)(void*))seqset_free);
}
static void _email_bulkupdate_plan_snooze(struct email_bulkupdate *bulk,
ptrarray_t *updates)
{
char *snoozed_mboxid = NULL, *inboxid = NULL;
jmap_mailbox_find_role(bulk->req, "snoozed", NULL, &snoozed_mboxid);
jmap_mailbox_find_role(bulk->req, "inbox", NULL, &inboxid);
int i;
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
const char *email_id = update->email_id;
int invalid = 0, find_mbox = 0;
if (json_object_get(bulk->set_errors, email_id)) {
continue;
}
ptrarray_t *current_uidrecs =
hash_lookup(email_id, &bulk->uidrecs_by_email_id);
/* Lookup the currently snoozed copy of this email_id */
int j;
for (j = 0; j < ptrarray_size(current_uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(current_uidrecs, j);
if (uidrec->is_snoozed) {
update->snoozed_uidrec = uidrec;
break;
}
}
if (update->snoozed_uidrec) {
const char *current_mboxid =
update->snoozed_uidrec->mboxrec->mbox_id;
struct email_updateplan *plan =
hash_lookup(current_mboxid, &bulk->plans_by_mbox_id);
if (update->snoozed) {
/* Updating/removing snoozeDetails */
json_t *jval = NULL;
if (update->patch_snoozed) {
const char *current_mboxname =
update->snoozed_uidrec->mboxrec->mboxname;
json_t *orig =
jmap_fetch_snoozed(current_mboxname,
update->snoozed_uidrec->uid);
json_t *patch = update->snoozed;
update->snoozed = jmap_patchobject_apply(orig, patch);
json_decref(orig);
json_decref(patch);
}
if (update->mailboxids) {
jval = json_object_get(update->mailboxids, current_mboxid);
}
if (!update->mailboxids || jval == json_true() ||
(jval == NULL && update->patch_mailboxids)) {
/* This message is NOT being deleted -
Update/remove snoozed */
update->snooze_in_mboxid = xstrdup(current_mboxid);
update->snoozed_uidrec->is_snoozed = 0;
ptrarray_append(&plan->snooze, update->snoozed_uidrec);
}
else {
/* Determine which mailbox to use for the snoozed email */
find_mbox = 1;
}
}
else if (update->mailboxids) {
/* No change to snoozeDetails -
Check if this message is being deleted */
json_t *jval =
json_object_get(update->mailboxids, current_mboxid);
if (jval == json_null() ||
(jval == NULL && !update->patch_mailboxids)) {
/* This message is being deleted - Remove snoozed */
update->snooze_in_mboxid = xstrdup(current_mboxid);
update->snoozed_uidrec->is_snoozed = 0;
ptrarray_append(&plan->snooze, update->snoozed_uidrec);
}
}
}
else if (json_is_object(update->snoozed)) {
/* Setting snoozeDetails */
if (!update->mailboxids) {
/* invalidArguments */
invalid = 1;
}
else {
/* Determine which mailbox to use for the snoozed email */
find_mbox = 1;
}
}
if (find_mbox) {
/* Determine which mailbox to use for the snoozed email */
const char *movetoid =
json_string_value(json_object_get(update->snoozed,
"moveToMailboxId"));
if (json_is_true(json_object_get(update->mailboxids,
snoozed_mboxid))) {
/* Being added to \snoozed mailbox */
update->snooze_in_mboxid = xstrdup(snoozed_mboxid);
}
else if (movetoid &&
json_is_true(json_object_get(update->mailboxids,
movetoid))) {
/* Being added to moveToMailboxId */
update->snooze_in_mboxid = xstrdup(movetoid);
}
else if (json_is_true(json_object_get(update->mailboxids,
inboxid))) {
/* Being added to Inbox */
update->snooze_in_mboxid = xstrdup(inboxid);
}
else {
const char *mbox_id = NULL;
json_t *jval = NULL;
json_object_foreach(update->mailboxids, mbox_id, jval) {
if (json_is_true(jval)) {
/* Use the first mailbox being added to */
update->snooze_in_mboxid = xstrdup(mbox_id);
break;
}
}
if (!update->snooze_in_mboxid) {
/* invalidArguments */
invalid = 1;
}
}
}
if (invalid) {
json_object_set_new(bulk->set_errors, email_id,
json_pack("{s:s s:[s,s]}",
"type", "invalidArguments",
"arguments", "mailboxIds", "snoozed"));
}
}
free(snoozed_mboxid);
free(inboxid);
}
static void _email_bulkupdate_plan(struct email_bulkupdate *bulk, ptrarray_t *updates)
{
int i;
/* Pre-process snooze updates */
_email_bulkupdate_plan_snooze(bulk, updates);
/* Plan mailbox copies, moves and deletes */
_email_bulkupdate_plan_mailboxids(bulk, updates);
/* Pre-process keyword updates */
_email_bulkupdate_plan_keywords(bulk, updates);
/* Check mailbox count and keyword limits per email */
_email_bulkupdate_checklimits(bulk);
/* Validate plans */
strarray_t erroneous_plans = STRARRAY_INITIALIZER;
/* Check permissions */
hash_iter *iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
if (!jmap_hasrights_byname(bulk->req, plan->mboxname, plan->needrights)) {
_email_updateplan_error(plan, IMAP_PERMISSION_DENIED, bulk->set_errors);
strarray_append(&erroneous_plans, plan->mbox_id);
}
}
hash_iter_reset(iter);
if (!config_getswitch(IMAPOPT_SINGLEINSTANCESTORE)) {
/* Check quota */
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
quota_t qdiffs[QUOTA_NUMRESOURCES] = QUOTA_DIFFS_INITIALIZER;
int i;
for (i = 0; i < ptrarray_size(&plan->copy); i++) {
qdiffs[QUOTA_MESSAGE] += ptrarray_size(ptrarray_nth(&plan->copy, i));
}
qdiffs[QUOTA_MESSAGE] -= ptrarray_size(&plan->delete);
int r = mailbox_quota_check(plan->mbox, qdiffs);
if (r) {
_email_updateplan_error(plan, r, bulk->set_errors);
strarray_append(&erroneous_plans, plan->mbox_id);
}
}
}
hash_iter_free(&iter);
/* Remove erroneous plans */
for (i = 0; i < strarray_size(&erroneous_plans); i++) {
const char *mbox_id = strarray_nth(&erroneous_plans, i);
struct email_updateplan *plan = hash_del(mbox_id, &bulk->plans_by_mbox_id);
if (!plan) continue;
jmap_closembox(bulk->req, &plan->mbox);
_email_updateplan_free_p(plan);
}
strarray_fini(&erroneous_plans);
/* Sort UID records arrays */
iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
for (i = 0; i < ptrarray_size(&plan->copy); i++) {
ptrarray_t *uidrecs = ptrarray_nth(&plan->copy, i);
ptrarray_sort(uidrecs, _email_uidrec_compareuid_cb);
}
ptrarray_sort(&plan->setflags, _email_uidrec_compareuid_cb);
ptrarray_sort(&plan->delete, _email_uidrec_compareuid_cb);
ptrarray_sort(&plan->snooze, _email_uidrec_compareuid_cb);
}
hash_iter_free(&iter);
}
static void _email_bulkupdate_open(jmap_req_t *req, struct email_bulkupdate *bulk, ptrarray_t *updates)
{
int i;
bulk->req = req;
/* Map mailbox creation ids and role to mailbox identifiers */
for(i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
if (!update->mailboxids) continue;
void *tmp;
const char *mbox_id;
json_t *jval;
json_object_foreach_safe(update->mailboxids, tmp, mbox_id, jval) {
int is_valid = 1;
if (*mbox_id == '$') {
const char *role = mbox_id + 1;
char *mboxname = NULL;
char *uniqueid = NULL;
if (!jmap_mailbox_find_role(bulk->req, role, &mboxname, &uniqueid)) {
json_object_del(update->mailboxids, mbox_id);
json_object_set(update->mailboxids, uniqueid, jval);
}
else is_valid = 0;
free(uniqueid);
free(mboxname);
}
else if (*mbox_id == '#') {
const char *resolved_mbox_id = jmap_lookup_id(req, mbox_id + 1);
if (resolved_mbox_id) {
json_object_del(update->mailboxids, mbox_id);
json_object_set(update->mailboxids, resolved_mbox_id, jval);
}
else is_valid = 0;
}
if (!is_valid) {
if (json_object_get(bulk->set_errors, update->email_id) == NULL) {
json_object_set_new(bulk->set_errors, update->email_id,
json_pack("{s:s s:[s]}", "type", "invalidProperties",
"properties", "mailboxIds"));
}
}
}
}
/* Map updates to their email id */
construct_hash_table(&bulk->updates_by_email_id, ptrarray_size(updates)+1, 0);
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
hash_insert(update->email_id, update, &bulk->updates_by_email_id);
}
/* Determine uid records per mailbox */
strarray_t email_ids = STRARRAY_INITIALIZER;
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
if (json_object_get(bulk->set_errors, update->email_id)) {
continue;
}
strarray_append(&email_ids, update->email_id);
}
_email_mboxrecs_read(req, req->cstate, &email_ids, bulk->set_errors, &bulk->cur_mboxrecs);
/* Open current mailboxes */
size_t mboxhash_size = ptrarray_size(updates) * JMAP_MAIL_MAX_MAILBOXES_PER_EMAIL + 1;
construct_hash_table(&bulk->plans_by_mbox_id, mboxhash_size, 0);
construct_hash_table(&bulk->uidrecs_by_email_id, strarray_size(&email_ids)+1, 0);
for (i = 0; i < ptrarray_size(bulk->cur_mboxrecs); i++) {
struct email_mboxrec *mboxrec = ptrarray_nth(bulk->cur_mboxrecs, i);
struct email_updateplan *plan = hash_lookup(mboxrec->mbox_id, &bulk->plans_by_mbox_id);
if (!plan) {
struct mailbox *mbox = NULL;
mbentry_t *mbentry = _mbentry_by_uniqueid(req, mboxrec->mbox_id);
int r = 0;
if (mbentry && mbentry->mbtype & MBTYPE_INTERMEDIATE) {
r = mboxlist_promote_intermediary(mbentry->name);
}
else if (!mbentry) {
r = IMAP_MAILBOX_NONEXISTENT;
}
mboxlist_entry_free(&mbentry);
if (!r) r = jmap_openmbox(req, mboxrec->mboxname, &mbox, /*rw*/1);
if (r) {
int j;
for (j = 0; j < ptrarray_size(&mboxrec->uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&mboxrec->uidrecs, j);
if (json_object_get(bulk->set_errors, uidrec->email_id) == NULL) {
json_object_set_new(bulk->set_errors, uidrec->email_id,
json_pack("{s:s s:[s]}", "type", "invalidProperties",
"properties", "mailboxIds"));
}
}
continue;
}
plan = _email_bulkupdate_addplan(bulk, mbox, mboxrec);
}
/* Map email ids to their list of non-deleted uid records. */
int j;
for (j = 0; j < ptrarray_size(&mboxrec->uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&mboxrec->uidrecs, j);
if (json_object_get(bulk->set_errors, uidrec->email_id)) {
continue;
}
/* Check if the uid record is expunged */
msgrecord_t *mr = NULL;
uint32_t system_flags = 0, internal_flags = 0;
int r = msgrecord_find(plan->mbox, uidrec->uid, &mr);
if (!r) r = msgrecord_get_systemflags(mr, &system_flags);
if (!r) r = msgrecord_get_internalflags(mr, &internal_flags);
if ((system_flags & FLAG_DELETED) || (internal_flags & FLAG_INTERNAL_EXPUNGED)) {
r = IMAP_NOTFOUND;
}
msgrecord_unref(&mr);
if (r) continue;
ptrarray_t *current_uidrecs = hash_lookup(uidrec->email_id, &bulk->uidrecs_by_email_id);
if (current_uidrecs == NULL) {
current_uidrecs = ptrarray_new();
hash_insert(uidrec->email_id, current_uidrecs, &bulk->uidrecs_by_email_id);
}
ptrarray_append(current_uidrecs, uidrec);
}
}
/* An email with no current uidrecs is expunged */
for (i = 0; i < strarray_size(&email_ids); i++) {
const char *email_id = strarray_nth(&email_ids, i);
if (json_object_get(bulk->set_errors, email_id)) {
continue;
}
if (!hash_lookup(email_id, &bulk->uidrecs_by_email_id)) {
json_object_set_new(bulk->set_errors, email_id,
json_pack("{s:s}", "type", "notFound"));
}
}
strarray_fini(&email_ids);
/* Open new mailboxes that haven't been opened already */
for (i = 0; i < ptrarray_size(updates); i++) {
struct email_update *update = ptrarray_nth(updates, i);
if (!update->mailboxids) {
continue;
}
json_t *jval;
const char *mbox_id;
void *tmp;
json_object_foreach_safe(update->mailboxids, tmp, mbox_id, jval) {
struct mailbox *mbox = NULL;
mbentry_t *mbentry = _mbentry_by_uniqueid(req, mbox_id);
if (mbentry) {
int r = 0;
if (mbentry->mbtype & MBTYPE_INTERMEDIATE) {
r = mboxlist_promote_intermediary(mbentry->name);
}
if (!r) jmap_openmbox(req, mbentry->name, &mbox, /*rw*/1);
}
mboxlist_entry_free(&mbentry);
if (mbox) {
if (!hash_lookup(mbox->uniqueid, &bulk->plans_by_mbox_id)) {
struct email_mboxrec *mboxrec = xzmalloc(sizeof(struct email_mboxrec));
mboxrec->mboxname = xstrdup(mbox->name);
mboxrec->mbox_id = xstrdup(mbox->uniqueid);
ptrarray_append(bulk->new_mboxrecs, mboxrec);
_email_bulkupdate_addplan(bulk, mbox, mboxrec);
}
else jmap_closembox(req, &mbox); // already reference counted
}
else {
json_object_set_new(bulk->set_errors, update->email_id,
json_pack("{s:s s:[s]}", "type", "invalidProperties",
"properties", "mailboxIds"));
}
}
}
/* Map updates to update plan */
_email_bulkupdate_plan(bulk, updates);
}
static void _email_bulkupdate_dump(struct email_bulkupdate *bulk, json_t *jdump)
{
int i;
struct buf buf = BUF_INITIALIZER;
json_t *jcur_mboxrecs = json_object();
for (i = 0; i < ptrarray_size(bulk->cur_mboxrecs); i++) {
struct email_mboxrec *mboxrec = ptrarray_nth(bulk->cur_mboxrecs, i);
json_t *jrecs = json_array();
int j;
for (j = 0; j < ptrarray_size(&mboxrec->uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&mboxrec->uidrecs, j);
json_array_append_new(jrecs, json_pack("{s:s s:i}",
"emailId", uidrec->email_id, "uid", uidrec->uid));
}
json_object_set_new(jcur_mboxrecs, mboxrec->mboxname, jrecs);
}
json_object_set_new(jdump, "curMboxrecs", jcur_mboxrecs);
json_t *jemails = json_object();
hash_iter *iter = hash_table_iter(&bulk->uidrecs_by_email_id);
while (hash_iter_next(iter)) {
const char *email_id = hash_iter_key(iter);
ptrarray_t *uidrecs = hash_iter_val(iter);
json_t *juidrecs = json_array();
int j;
for (j = 0; j < ptrarray_size(uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(uidrecs, j);
buf_printf(&buf, "%s:%d", uidrec->mboxrec->mboxname, uidrec->uid);
json_array_append_new(juidrecs, json_string(buf_cstring(&buf)));
buf_reset(&buf);
}
json_object_set_new(jemails, email_id, juidrecs);
}
hash_iter_free(&iter);
json_object_set_new(jdump, "emails", jemails);
json_t *jmailboxes = json_object();
json_t *jplans = json_object();
iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
json_object_set_new(jmailboxes, plan->mboxname, json_string(plan->mbox_id));
json_t *jplan = json_object();
json_t *jcopy = json_array();
int j;
for (j = 0; j < ptrarray_size(&plan->copy); j++) {
ptrarray_t *uidrecs = ptrarray_nth(&plan->copy, j);
int k;
for (k = 0; k < ptrarray_size(uidrecs); k++) {
struct email_uidrec *uidrec = ptrarray_nth(uidrecs, k);
buf_printf(&buf, "%s:%d", uidrec->mboxrec->mboxname, uidrec->uid);
json_array_append_new(jcopy, json_string(buf_cstring(&buf)));
buf_reset(&buf);
}
}
json_object_set_new(jplan, "copy", jcopy);
json_t *jsetflags = json_array();
for (j = 0; j < ptrarray_size(&plan->setflags); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->setflags, j);
json_array_append_new(jsetflags, json_integer(uidrec->uid));
}
json_object_set_new(jplan, "setflags", jsetflags);
json_t *jdelete = json_array();
for (j = 0; j < ptrarray_size(&plan->delete); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->delete, j);
json_array_append_new(jdelete, json_integer(uidrec->uid));
}
json_object_set_new(jplan, "delete", jdelete);
json_t *jsnooze = json_array();
for (j = 0; j < ptrarray_size(&plan->snooze); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->snooze, j);
json_array_append_new(jsnooze, json_integer(uidrec->uid));
}
json_object_set_new(jplan, "snooze", jsnooze);
json_object_set_new(jplans, plan->mboxname, jplan);
}
hash_iter_free(&iter);
json_object_set_new(jdump, "plans", jplans);
json_object_set_new(jdump, "mailboxes", jmailboxes);
buf_free(&buf);
}
static void _email_bulkupdate_exec_copy(struct email_bulkupdate *bulk)
{
hash_iter *iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
struct mailbox *dst_mbox = plan->mbox;
int j;
for (j = 0; j < ptrarray_size(&plan->copy); j++) {
ptrarray_t *src_uidrecs = ptrarray_nth(&plan->copy, j);
ptrarray_t src_msgrecs = PTRARRAY_INITIALIZER;
if (!ptrarray_size(src_uidrecs)) continue;
/* Lookup the source mailbox plan of the first entry. */
struct email_uidrec *tmp = ptrarray_nth(src_uidrecs, 0);
const char *src_mbox_id = tmp->mboxrec->mbox_id;
struct email_updateplan *src_plan = hash_lookup(src_mbox_id, &bulk->plans_by_mbox_id);
struct mailbox *src_mbox = src_plan->mbox;
uint32_t last_uid_before_copy = dst_mbox->i.last_uid;
/* Bulk copy messages per mailbox to destination */
int k;
for (k = 0; k < ptrarray_size(src_uidrecs); k++) {
struct email_uidrec *src_uidrec = ptrarray_nth(src_uidrecs, k);
if (json_object_get(bulk->set_errors, src_uidrec->email_id)) {
continue;
}
msgrecord_t *mrw = msgrecord_from_uid(src_mbox, src_uidrec->uid);
ptrarray_append(&src_msgrecs, mrw);
}
int r = _copy_msgrecords(httpd_authstate, bulk->req->userid, &jmap_namespace,
src_mbox, dst_mbox, &src_msgrecs);
if (r) {
for (k = 0; k < ptrarray_size(src_uidrecs); k++) {
struct email_uidrec *src_uidrec = ptrarray_nth(src_uidrecs, k);
if (json_object_get(bulk->set_errors, src_uidrec->email_id)) {
continue;
}
json_object_set_new(bulk->set_errors, src_uidrec->email_id, jmap_server_error(r));
}
}
for (k = 0; k < ptrarray_size(&src_msgrecs); k++) {
msgrecord_t *mrw = ptrarray_nth(&src_msgrecs, k);
msgrecord_unref(&mrw);
}
ptrarray_fini(&src_msgrecs);
for (k = 0; k < ptrarray_size(src_uidrecs); k++) {
struct email_uidrec *src_uidrec = ptrarray_nth(src_uidrecs, k);
if (json_object_get(bulk->set_errors, src_uidrec->email_id)) {
continue;
}
/* Create new uid record */
struct email_uidrec *new_uidrec = xzmalloc(sizeof(struct email_uidrec));
new_uidrec->email_id = xstrdup(src_uidrec->email_id);
new_uidrec->uid = last_uid_before_copy + k + 1;
new_uidrec->mboxrec = plan->mboxrec;
new_uidrec->is_new = 1;
ptrarray_append(&plan->mboxrec->uidrecs, new_uidrec);
/* Add new record to setflags plan if keywords are updated */
/* XXX append_copy should take new flags as parameter */
struct email_update *update = hash_lookup(src_uidrec->email_id,
&bulk->updates_by_email_id);
if (update->keywords || update->full_keywords) {
ptrarray_append(&plan->setflags, new_uidrec);
}
/* Add new record to snooze plan if copied to snooze folder */
if (update->snoozed) {
if (json_is_object(update->snoozed) &&
!strcmpnull(plan->mbox_id, update->snooze_in_mboxid)) {
/* Only flag as snoozed (add \snoozed and annotation) if:
- SnoozeDetails != json_null() AND
- this mailbox is the mailbox in which it is snoozed
*/
new_uidrec->is_snoozed = 1;
}
ptrarray_append(&plan->snooze, new_uidrec);
}
}
}
}
hash_iter_free(&iter);
}
static void _email_bulkupdate_exec_setflags(struct email_bulkupdate *bulk)
{
hash_iter *iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
int j;
struct seqset *add_seenseq = NULL;
struct seqset *del_seenseq = NULL;
uint32_t last_uid = plan->mbox->i.last_uid;
struct mboxevent *flagsset = mboxevent_new(EVENT_FLAGS_SET);
int notify_flagsset = 0;
struct mboxevent *flagsclear = mboxevent_new(EVENT_FLAGS_CLEAR);
int notify_flagsclear = 0;
if (plan->use_seendb) {
add_seenseq = seqset_init(0, SEQ_SPARSE);
del_seenseq = seqset_init(0, SEQ_SPARSE);
}
/* Re-sort uid records before processing. */
ptrarray_sort(&plan->setflags, _email_uidrec_compareuid_cb);
/* Process uid records */
for (j = 0; j < ptrarray_size(&plan->setflags); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->setflags, j);
if (json_object_get(bulk->set_errors, uidrec->email_id)) {
continue;
}
const char *email_id = uidrec->email_id;
struct email_update *update = hash_lookup(email_id, &bulk->updates_by_email_id);
/* Determine if to write the aggregated or updated JMAP keywords */
json_t *keywords = uidrec->is_new ? update->full_keywords : update->keywords;
int patch_keywords = uidrec->is_new ? 0 : update->patch_keywords;
/* Write keywords */
struct modified_flags modflags;
memset(&modflags, 0, sizeof(struct modified_flags));
msgrecord_t *mrw = msgrecord_from_uid(plan->mbox, uidrec->uid);
int r = _email_setflags(keywords, patch_keywords, mrw,
add_seenseq, del_seenseq, &modflags);
if (!r) {
if (modflags.added_flags) {
mboxevent_add_flags(flagsset, plan->mbox->flagname,
modflags.added_system_flags,
modflags.added_user_flags);
mboxevent_extract_msgrecord(flagsset, mrw);
notify_flagsset = 1;
}
if (modflags.removed_flags) {
mboxevent_add_flags(flagsclear, plan->mbox->flagname,
modflags.removed_system_flags,
modflags.removed_user_flags);
mboxevent_extract_msgrecord(flagsclear, mrw);
notify_flagsclear = 1;
}
msgrecord_unref(&mrw);
if (last_uid < uidrec->uid) {
last_uid = uidrec->uid;
}
}
else {
json_object_set_new(bulk->set_errors, email_id, jmap_server_error(r));
}
}
/* Write seen db for shared mailboxes */
if (plan->use_seendb) {
if (add_seenseq || del_seenseq) {
struct seqset *new_seenseq = seqset_init(0, SEQ_SPARSE);
if (del_seenseq->len) {
uint32_t uid;
while ((uid = seqset_getnext(plan->old_seenseq)))
if (!seqset_ismember(del_seenseq, uid))
seqset_add(new_seenseq, uid, 1);
}
else {
seqset_join(new_seenseq, plan->old_seenseq);
}
if (add_seenseq->len)
seqset_join(new_seenseq, add_seenseq);
struct seendata sd = SEENDATA_INITIALIZER;
sd.seenuids = seqset_cstring(new_seenseq);
if (!sd.seenuids) sd.seenuids = xstrdup("");
sd.lastread = time(NULL);
sd.lastchange = plan->mbox->i.last_appenddate;
sd.lastuid = last_uid;
int r = seen_write(bulk->seendb, plan->mbox->uniqueid, &sd);
if (r) {
for (j = 0; j < ptrarray_size(&plan->setflags); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->setflags, j);
if (json_object_get(bulk->set_errors, uidrec->email_id) == NULL) {
json_object_set_new(bulk->set_errors, uidrec->email_id,
jmap_server_error(r));
}
}
}
seqset_free(add_seenseq);
seqset_free(del_seenseq);
seqset_free(new_seenseq);
seen_freedata(&sd);
}
}
if (notify_flagsset) {
mboxevent_extract_mailbox(flagsset, plan->mbox);
mboxevent_set_numunseen(flagsset, plan->mbox, -1);
mboxevent_set_access(flagsset, NULL, NULL, bulk->req->userid,
plan->mbox->name, 0);
mboxevent_notify(&flagsset);
}
if (notify_flagsclear) {
mboxevent_extract_mailbox(flagsclear, plan->mbox);
mboxevent_set_numunseen(flagsclear, plan->mbox, -1);
mboxevent_set_access(flagsclear, NULL, NULL, bulk->req->userid,
plan->mbox->name, 0);
mboxevent_notify(&flagsclear);
}
mboxevent_free(&flagsset);
mboxevent_free(&flagsclear);
}
hash_iter_free(&iter);
}
static void _email_bulkupdate_exec_delete(struct email_bulkupdate *bulk)
{
hash_iter *iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
_email_multiexpunge(bulk->req, plan->mbox, &plan->delete, bulk->set_errors);
}
hash_iter_free(&iter);
}
static void _email_bulkupdate_exec_snooze(struct email_bulkupdate *bulk)
{
hash_iter *iter = hash_table_iter(&bulk->plans_by_mbox_id);
while (hash_iter_next(iter)) {
struct email_updateplan *plan = hash_iter_val(iter);
int j;
/* Process uid records */
for (j = 0; j < ptrarray_size(&plan->snooze); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&plan->snooze, j);
if (json_object_get(bulk->set_errors, uidrec->email_id)) {
continue;
}
const char *email_id = uidrec->email_id;
struct email_update *update =
hash_lookup(email_id, &bulk->updates_by_email_id);
/* Write annotation */
msgrecord_t *mrw = msgrecord_from_uid(plan->mbox, uidrec->uid);
uint32_t internalflags = 0;
int r = IMAP_MAILBOX_NONEXISTENT;
time_t savedate = 0;
if (mrw) r = msgrecord_get_internalflags(mrw, &internalflags);
if (!r) r = msgrecord_get_savedate(mrw, &savedate);
if (!r) {
const char *annot = IMAP_ANNOT_NS "snoozed";
struct buf val = BUF_INITIALIZER;
if (uidrec->is_snoozed) {
/* Set/update annotation */
char *json = json_dumps(update->snoozed, JSON_COMPACT);
buf_initm(&val, json, strlen(json));
/* Extract until and use it as savedate */
time_from_iso8601(json_string_value(json_object_get(update->snoozed, "until")),
&savedate);
internalflags |= FLAG_INTERNAL_SNOOZED;
}
else {
/* Delete annotation */
internalflags &= ~FLAG_INTERNAL_SNOOZED;
}
r = msgrecord_annot_write(mrw, annot, "", &val);
if (!r) r = msgrecord_set_internalflags(mrw, internalflags);
if (!r) r = msgrecord_set_savedate(mrw, savedate);
if (!r) r = msgrecord_rewrite(mrw);
msgrecord_unref(&mrw);
buf_free(&val);
}
if (r) {
json_object_set_new(bulk->set_errors, email_id, jmap_server_error(r));
}
}
}
hash_iter_free(&iter);
}
static void _email_bulkupdate_exec(struct email_bulkupdate *bulk,
json_t *updated,
json_t *not_updated,
json_t *debug)
{
/* Execute plans */
_email_bulkupdate_exec_copy(bulk);
_email_bulkupdate_exec_setflags(bulk);
_email_bulkupdate_exec_snooze(bulk);
_email_bulkupdate_exec_delete(bulk);
/* Report results */
hash_iter *iter = hash_table_iter(&bulk->updates_by_email_id);
while (hash_iter_next(iter)) {
const char *email_id = hash_iter_key(iter);
json_t *err = json_object_get(bulk->set_errors, email_id);
if (err) {
json_object_set(not_updated, email_id, err);
}
else {
json_object_set_new(updated, email_id, json_pack("{s:s}", "id", email_id));
}
}
hash_iter_free(&iter);
if (debug) _email_bulkupdate_dump(bulk, debug);
}
/* Execute the sequence of JMAP Email/set/{update} arguments in bulk. */
static void _email_update_bulk(jmap_req_t *req,
json_t *update,
json_t *updated,
json_t *not_updated,
json_t *debug)
{
struct email_bulkupdate bulkupdate = _EMAIL_BULKUPDATE_INITIALIZER;
ptrarray_t updates = PTRARRAY_INITIALIZER;
int i;
/* Bulk updates are implemented to minimize the number of Cyrus mailbox
* open and close calls. This is achieved by the following steps:
*
* (1) Parse all update arguments.
*
* (2) For each updated email, determine its mailboxes and UID records.
* Keep note of all mailboxes where the email is currently contained
* in, deleted from or copied to. Open all these mailboxes and create
* an empty update plan for each mailbox.
*
* (3) For each email, determine the required update operations per
* mailbox:
*
* (3.1) If the email keywords are updated, add the email's
* UID record to the `setflags` field of the according
* mailbox update plan.
*
* (3.2) If the email mailboxes are updated, determine which
* mailboxes to delete the UID record from, and where
* to copy an existing UID record to. For each mailbox,
* determine all the UID records that are copied into it.
* Cluster all these UID records such that the number
* of mailboxes to copy from is minimized.
*
* (4) Check required permissions and quota for each mailbox. For
* any failed mailbox check, mark all updates of emails that
* affect this mailbox as failed.
*
* (5) Check email-scoped limits such as mailbox count and keywords per
* email. Mark all erroneous emails as not updated.
*
* (6) Execute the update plans. Apply copy first, followed by setflags,
* followed by deletes.
*
* (7) Report all updated emails and close all mailboxes.
*
*/
/* Parse updates and add valid updates to todo list. */
const char *email_id;
json_t *jval;
bulkupdate.req = req;
json_object_foreach(update, email_id, jval) {
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct email_update *update = xzmalloc(sizeof(struct email_update));
update->email_id = email_id;
_email_update_parse(jval, &parser, update);
if (json_array_size(parser.invalid)) {
json_object_set_new(not_updated, email_id,
json_pack("{s:s s:O}", "type", "invalidProperties",
"properties", parser.invalid));
_email_update_free(update);
}
else {
ptrarray_append(&updates, update);
}
jmap_parser_fini(&parser);
}
if (ptrarray_size(&updates)) {
/* Build and execute bulk update */
_email_bulkupdate_open(req, &bulkupdate, &updates);
_email_bulkupdate_exec(&bulkupdate, updated, not_updated, debug);
_email_bulkupdate_close(&bulkupdate);
}
else {
/* just clean up the memory we allocated above */
_email_mboxrecs_free(&bulkupdate.new_mboxrecs);
json_decref(bulkupdate.set_errors);
}
for (i = 0; i < ptrarray_size(&updates); i++) {
_email_update_free(ptrarray_nth(&updates, i));
}
ptrarray_fini(&updates);
}
static void _email_destroy_bulk(jmap_req_t *req,
json_t *destroy,
json_t *destroyed,
json_t *not_destroyed)
{
ptrarray_t *mboxrecs = NULL;
strarray_t email_ids = STRARRAY_INITIALIZER;
size_t iz;
json_t *jval;
int i;
/* Map email ids to mailbox name and UID */
json_array_foreach(destroy, iz, jval) {
strarray_append(&email_ids, json_string_value(jval));
}
_email_mboxrecs_read(req, req->cstate, &email_ids, not_destroyed, &mboxrecs);
/* Check mailbox ACL for shared accounts. */
if (strcmp(req->accountid, req->userid)) {
for (i = 0; i < ptrarray_size(mboxrecs); i++) {
struct email_mboxrec *mboxrec = ptrarray_nth(mboxrecs, i);
if (!jmap_hasrights_byname(req, mboxrec->mboxname, ACL_DELETEMSG)) {
/* Mark all messages of this mailbox as failed */
int j;
for (j = 0; j < ptrarray_size(&mboxrec->uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&mboxrec->uidrecs, j);
if (!json_object_get(not_destroyed, uidrec->email_id)) {
json_object_set_new(not_destroyed, uidrec->email_id,
json_pack("{s:s}", "type", "forbidden"));
}
}
/* Remove this mailbox from the todo list */
ptrarray_remove(mboxrecs, i--);
_email_mboxrec_free(mboxrec);
}
}
}
/* Expunge messages in bulk per mailbox */
for (i = 0; i < ptrarray_size(mboxrecs); i++) {
struct email_mboxrec *mboxrec = ptrarray_nth(mboxrecs, i);
struct mailbox *mbox = NULL;
int j;
int r = jmap_openmbox(req, mboxrec->mboxname, &mbox, 1);
if (!r) {
/* Expunge messages one by one, marking any failed message */
_email_multiexpunge(req, mbox, &mboxrec->uidrecs, not_destroyed);
}
else {
/* Mark all messages of this mailbox as failed */
for (j = 0; j < ptrarray_size(&mboxrec->uidrecs); j++) {
struct email_uidrec *uidrec = ptrarray_nth(&mboxrec->uidrecs, j);
if (!json_object_get(not_destroyed, uidrec->email_id)) {
json_object_set_new(not_destroyed, uidrec->email_id,
jmap_server_error(r));
}
}
}
jmap_closembox(req, &mbox);
}
/* Report successful destroys */
json_array_foreach(destroy, iz, jval) {
const char *email_id = json_string_value(jval);
if (!json_object_get(not_destroyed, email_id))
json_array_append(destroyed, jval);
}
_email_mboxrecs_free(&mboxrecs);
strarray_fini(&email_ids);
}
HIDDEN int jmap_email_set(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_set set;
json_t *err = NULL;
jmap_set_parse(req, &parser, email_props, NULL, NULL, &set, &err);
if (err) {
jmap_error(req, err);
goto done;
}
if (set.if_in_state) {
/* TODO rewrite state function to use char* not json_t* */
json_t *jstate = json_string(set.if_in_state);
if (jmap_cmpstate(req, jstate, MBTYPE_EMAIL)) {
jmap_error(req, json_pack("{s:s}", "type", "stateMismatch"));
goto done;
}
json_decref(jstate);
set.old_state = xstrdup(set.if_in_state);
}
else {
json_t *jstate = jmap_getstate(req, MBTYPE_EMAIL, /*refresh*/0);
set.old_state = xstrdup(json_string_value(jstate));
json_decref(jstate);
}
json_t *email;
const char *creation_id;
json_object_foreach(set.create, creation_id, email) {
json_t *set_err = NULL;
json_t *new_email = NULL;
/* Create message */
_email_create(req, email, &new_email, &set_err);
if (set_err) {
json_object_set_new(set.not_created, creation_id, set_err);
continue;
}
/* Report message as created */
json_object_set_new(set.created, creation_id, new_email);
const char *msg_id = json_string_value(json_object_get(new_email, "id"));
jmap_add_id(req, creation_id, msg_id);
}
json_t *debug_bulkupdate = NULL;
if (jmap_is_using(req, JMAP_DEBUG_EXTENSION)) {
debug_bulkupdate = json_object();
}
_email_update_bulk(req, set.update, set.updated, set.not_updated, debug_bulkupdate);
_email_destroy_bulk(req, set.destroy, set.destroyed, set.not_destroyed);
// TODO refactor jmap_getstate to return a string, once
// all code has been migrated to the new JMAP parser.
json_t *jstate = jmap_getstate(req, MBTYPE_EMAIL, /*refresh*/1);
set.new_state = xstrdup(json_string_value(jstate));
json_decref(jstate);
json_t *reply = jmap_set_reply(&set);
if (jmap_is_using(req, JMAP_DEBUG_EXTENSION)) {
json_object_set_new(reply, "debugBulkUpdate", debug_bulkupdate); // takes ownership
}
jmap_ok(req, reply);
done:
jmap_parser_fini(&parser);
jmap_set_fini(&set);
return 0;
}
struct _email_import_rock {
struct buf buf;
};
static int _email_import_cb(jmap_req_t *req __attribute__((unused)),
FILE *out,
void *rock,
json_t **err __attribute__((unused)))
{
struct _email_import_rock *data = (struct _email_import_rock*) rock;
const char *base = data->buf.s;
size_t len = data->buf.len;
struct protstream *stream = prot_readmap(base, len);
int r = message_copy_strict(stream, out, len, 0);
prot_free(stream);
return r;
}
struct msgimport_checkacl_rock {
jmap_req_t *req;
json_t *mailboxes;
};
static int msgimport_checkacl_cb(const mbentry_t *mbentry, void *xrock)
{
struct msgimport_checkacl_rock *rock = xrock;
jmap_req_t *req = rock->req;
if (!json_object_get(rock->mailboxes, mbentry->uniqueid))
return 0;
int needrights = ACL_INSERT|ACL_ANNOTATEMSG;
if (!jmap_hasrights(req, mbentry, needrights))
return IMAP_PERMISSION_DENIED;
return 0;
}
static void _email_import(jmap_req_t *req,
json_t *jemail_import,
json_t **new_email,
json_t **err)
{
const char *blob_id = json_string_value(json_object_get(jemail_import, "blobId"));
json_t *jmailbox_ids = json_object_get(jemail_import, "mailboxIds");
char *mboxname = NULL;
struct _email_import_rock content = { BUF_INITIALIZER };
int has_attachment = 0;
/* Force write locks on mailboxes. */
req->force_openmbox_rw = 1;
/* Gather keywords */
strarray_t keywords = STRARRAY_INITIALIZER;
const json_t *val;
const char *keyword;
json_object_foreach(json_object_get(jemail_import, "keywords"), keyword, val) {
if (!strcasecmp(keyword, JMAP_HAS_ATTACHMENT_FLAG)) {
continue;
}
strarray_append(&keywords, keyword);
}
/* check for internaldate */
time_t internaldate = 0;
const char *received_at = json_string_value(json_object_get(jemail_import, "receivedAt"));
if (received_at) {
time_from_iso8601(received_at, &internaldate);
}
/* check for snoozed */
json_t *snoozed = json_object_get(jemail_import, "snoozed");
/* Check mailboxes for ACL */
if (strcmp(req->userid, req->accountid)) {
struct msgimport_checkacl_rock rock = { req, jmailbox_ids };
int r = mboxlist_usermboxtree(req->accountid, req->authstate, msgimport_checkacl_cb, &rock, MBOXTREE_INTERMEDIATES);
if (r) {
*err = json_pack("{s:s s:[s]}", "type", "invalidProperties",
"properties", "mailboxIds");
goto done;
}
}
/* Start import */
struct email_append_detail detail;
memset(&detail, 0, sizeof(struct email_append_detail));
/* Lookup blob */
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
/* see if we can get a direct email! */
const char *sourcefile = NULL;
int r = jmap_findblob_exact(req, NULL/*accountid*/, blob_id,
&mbox, &mr);
if (!r) r = msgrecord_get_fname(mr, &sourcefile);
if (!r) r = msgrecord_get_body(mr, &content.buf);
if (!r) goto gotrecord;
/* better clean up before we go the slow path */
buf_reset(&content.buf);
sourcefile = NULL;
msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
struct body *body = NULL;
const struct body *part = NULL;
struct buf msg_buf = BUF_INITIALIZER;
r = jmap_findblob(req, NULL/*accountid*/, blob_id,
&mbox, &mr, &body, &part, &msg_buf);
if (r) {
if (r == IMAP_NOTFOUND || r == IMAP_PERMISSION_DENIED)
*err = json_pack("{s:s s:[s]}", "type", "invalidProperties",
"properties", "blobId");
else
*err = jmap_server_error(r);
goto done;
}
/* Decode blob */
const char *blob_base = buf_base(&msg_buf);
size_t blob_len = buf_len(&msg_buf);
if (part) {
blob_base += part->content_offset;
blob_len = part->content_size;
int enc = encoding_lookupname(part->encoding);
if (enc != ENCODING_NONE) {
char *tmp;
size_t dec_len;
const char *dec = charset_decode_mimebody(blob_base, blob_len, enc, &tmp, &dec_len);
buf_setmap(&content.buf, dec, dec_len);
free(tmp);
}
else {
buf_setmap(&content.buf, blob_base, blob_len);
}
}
else {
buf_setmap(&content.buf, blob_base, blob_len);
}
message_free_body(body);
free(body);
gotrecord:
/* Determine $hasAttachment flag */
if (config_getswitch(IMAPOPT_JMAP_SET_HAS_ATTACHMENT)) {
/* Parse email */
json_t *email = NULL;
struct email_getargs getargs = _EMAIL_GET_ARGS_INITIALIZER;
getargs.props = xzmalloc(sizeof(hash_table));
construct_hash_table(getargs.props, 1, 0);
hash_insert("hasAttachment", (void*)1, getargs.props);
_email_from_buf(req, &getargs, &content.buf, NULL, &email);
has_attachment = json_boolean_value(json_object_get(email, "hasAttachment"));
free_hash_table(getargs.props, NULL);
free(getargs.props);
getargs.props = NULL;
_email_getargs_fini(&getargs);
json_decref(email);
}
/* Write the message to the file system */
_email_append(req, jmailbox_ids, &keywords, internaldate, snoozed,
has_attachment, sourcefile, _email_import_cb, &content, &detail, err);
msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
if (*err) goto done;
*new_email = json_pack("{s:s, s:s, s:s, s:i}",
"id", detail.email_id,
"blobId", detail.blob_id,
"threadId", detail.thread_id,
"size", detail.size);
done:
strarray_fini(&keywords);
buf_free(&content.buf);
free(mboxname);
}
static int jmap_email_import(jmap_req_t *req)
{
json_t *created = json_pack("{}");
json_t *not_created = json_pack("{}");
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
const char *key;
json_t *arg, *emails = NULL;
const char *id;
json_t *jemail_import;
int have_snoozed_mboxid = 0;
/* Parse request */
json_object_foreach(req->args, key, arg) {
if (!strcmp(key, "accountId")) {
/* already handled in jmap_api() */
}
else if (!strcmp(key, "emails")) {
if (json_is_object(arg)) {
emails = arg;
jmap_parser_push(&parser, "emails");
json_object_foreach(emails, id, jemail_import) {
if (!json_is_object(jemail_import)) {
jmap_parser_invalid(&parser, id);
}
}
jmap_parser_pop(&parser);
}
}
else {
jmap_parser_invalid(&parser, key);
}
}
/* emails is a required argument */
if (!emails) jmap_parser_invalid(&parser, "emails");
if (json_array_size(parser.invalid)) {
json_t *err = json_pack("{s:s s:O}", "type", "invalidArguments",
"arguments", parser.invalid);
jmap_error(req, err);
goto done;
}
json_object_foreach(emails, id, jemail_import) {
/* Parse import */
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
json_t *jval;
const char *s;
/* blobId */
s = json_string_value(json_object_get(jemail_import, "blobId"));
if (!s) {
jmap_parser_invalid(&parser, "blobId");
}
/* keywords */
json_t *keywords = json_object_get(jemail_import, "keywords");
if (json_is_object(keywords)) {
jmap_parser_push(&parser, "keywords");
json_object_foreach(keywords, s, jval) {
if (jval != json_true() || !jmap_email_keyword_is_valid(s)) {
jmap_parser_invalid(&parser, s);
}
}
jmap_parser_pop(&parser);
}
else if (JNOTNULL(keywords)) {
jmap_parser_invalid(&parser, "keywords");
}
/* receivedAt */
json_t *jrecv = json_object_get(jemail_import, "receivedAt");
if (JNOTNULL(jrecv)) {
if (!json_is_utcdate(jrecv)) {
jmap_parser_invalid(&parser, "receivedAt");
}
}
/* Validate mailboxIds */
json_t *jmailboxids = json_copy(json_object_get(jemail_import, "mailboxIds"));
if (json_object_size(jmailboxids)) {
_append_validate_mboxids(req, jmailboxids, &parser, &have_snoozed_mboxid);
}
else {
jmap_parser_invalid(&parser, "mailboxIds");
}
/* Validate snoozed + mailboxIds */
json_t *snoozed = json_object_get(jemail_import, "snoozed");
if (JNOTNULL(snoozed) &&
!(json_is_utcdate(json_object_get(snoozed, "until")) &&
have_snoozed_mboxid)) {
jmap_parser_invalid(&parser, "snoozed");
}
json_t *invalid = json_incref(parser.invalid);
jmap_parser_fini(&parser);
if (json_array_size(invalid)) {
json_t *err = json_pack("{s:s}", "type", "invalidProperties");
json_object_set_new(err, "properties", invalid);
json_object_set_new(not_created, id, err);
json_decref(jmailboxids);
continue;
}
json_decref(invalid);
/* Process import */
json_t *orig_mailboxids = json_incref(json_object_get(jemail_import, "mailboxIds"));
json_object_set_new(jemail_import, "mailboxIds", jmailboxids);
json_t *new_email = NULL;
json_t *err = NULL;
_email_import(req, jemail_import, &new_email, &err);
if (err) {
json_object_set_new(not_created, id, err);
}
else {
/* Successful import */
json_object_set_new(created, id, new_email);
const char *newid = json_string_value(json_object_get(new_email, "id"));
jmap_add_id(req, id, newid);
}
json_object_set_new(jemail_import, "mailboxIds", orig_mailboxids);
}
/* Reply */
jmap_ok(req, json_pack("{s:s s:O s:O}",
"accountId", req->accountid,
"created", created,
"notCreated", not_created));
done:
json_decref(created);
json_decref(not_created);
jmap_parser_fini(&parser);
return 0;
}
struct _email_copy_checkmbox_rock {
jmap_req_t *req; /* JMAP request context */
json_t *dst_mboxids; /* mailboxIds argument */
strarray_t *dst_mboxnames; /* array of destination mailbox names */
};
static int _email_copy_checkmbox_cb(const mbentry_t *mbentry, void *_rock)
{
struct _email_copy_checkmbox_rock *rock = _rock;
/* Ignore anything but regular and intermediate mailboxes */
if (!mbentry || (mbentry->mbtype & ~MBTYPE_INTERMEDIATE)) {
return 0;
}
if (!json_object_get(rock->dst_mboxids, mbentry->uniqueid)) {
return 0;
}
/* Check read-write ACL rights */
int needrights = ACL_LOOKUP|ACL_READ|ACL_WRITE|ACL_INSERT|
ACL_SETSEEN|ACL_ANNOTATEMSG;
if (!jmap_hasrights(rock->req, mbentry, needrights))
return IMAP_PERMISSION_DENIED;
/* Mark this mailbox as found */
strarray_append(rock->dst_mboxnames, mbentry->name);
size_t want_count = json_object_size(rock->dst_mboxids);
size_t have_count = strarray_size(rock->dst_mboxnames);
return want_count == have_count ? IMAP_OK_COMPLETED : 0;
}
struct _email_copy_writeprops_rock {
/* Input values */
jmap_req_t *req;
const char *received_at;
json_t *keywords;
struct seen *seendb;
/* Return values */
conversation_id_t cid; /* Thread id of copied message */
uint32_t size; /* Byte size of copied message */
};
static int _email_copy_writeprops_cb(const conv_guidrec_t* rec, void* _rock)
{
struct _email_copy_writeprops_rock *rock = _rock;
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
jmap_req_t *req = rock->req;
struct mboxevent *flagsset = mboxevent_new(EVENT_FLAGS_SET);
struct mboxevent *flagsclear = mboxevent_new(EVENT_FLAGS_CLEAR);
int notify_flagsset = 0;
int notify_flagsclear = 0;
if (rec->part) {
return 0;
}
/* Overwrite message record */
int r = jmap_openmbox(rock->req, rec->mboxname, &mbox, /*rw*/1);
if (!r) r = msgrecord_find(mbox, rec->uid, &mr);
if (!r && rock->received_at) {
time_t internal_date;
time_from_iso8601(rock->received_at, &internal_date);
r = msgrecord_set_internaldate(mr, internal_date);
}
if (!r) {
/* Write the keywords. There's lots of ceremony around seen.db */
struct seqset *seenseq = NULL;
struct seqset *addseen = NULL;
struct seqset *delseen = NULL;
/* Read the current seen sequence from seen.db */
int need_seendb = !mailbox_internal_seen(mbox, req->userid);
if (need_seendb) {
delseen = seqset_init(0, SEQ_SPARSE);
addseen = seqset_init(0, SEQ_SPARSE);
struct seendata sd = SEENDATA_INITIALIZER;
int r = seen_lockread(rock->seendb, mbox->uniqueid, &sd);
if (!r) {
seenseq = seqset_parse(sd.seenuids, NULL, sd.lastuid);
seen_freedata(&sd);
}
}
/* Write the flags on the record */
struct modified_flags modflags;
memset(&modflags, 0, sizeof(struct modified_flags));
if (!r) r = _email_setflags(rock->keywords, 0, mr, addseen, delseen, &modflags);
if (!r) {
if (modflags.added_flags) {
mboxevent_extract_msgrecord(flagsset, mr);
mboxevent_add_flags(flagsset, mbox->flagname,
modflags.added_system_flags,
modflags.added_user_flags);
notify_flagsset = 1;
}
if (modflags.removed_flags) {
mboxevent_extract_msgrecord(flagsclear, mr);
mboxevent_add_flags(flagsclear, mbox->flagname,
modflags.removed_system_flags,
modflags.removed_user_flags);
notify_flagsclear = 1;
}
}
/* Write back changes to seen.db */
if (!r && need_seendb && (addseen->len || delseen->len)) {
if (delseen->len) {
struct seqset *newseen = seqset_init(0, SEQ_SPARSE);
uint32_t uid;
while ((uid = seqset_getnext(seenseq))) {
if (!seqset_ismember(delseen, uid)) {
seqset_add(newseen, uid, 1);
}
}
seqset_free(seenseq);
seenseq = newseen;
}
else if (addseen->len) {
seqset_add(seenseq, rec->uid, 1);
}
struct seendata sd = SEENDATA_INITIALIZER;
sd.seenuids = seqset_cstring(seenseq);
if (!sd.seenuids) sd.seenuids = xstrdup("");
sd.lastread = time(NULL);
sd.lastchange = mbox->i.last_appenddate;
sd.lastuid = mbox->i.last_uid;
r = seen_write(rock->seendb, mbox->uniqueid, &sd);
seen_freedata(&sd);
}
seqset_free(delseen);
seqset_free(addseen);
seqset_free(seenseq);
}
if (!r) r = msgrecord_rewrite(mr);
if (r) goto done;
/* Write mboxevents */
if (notify_flagsset) {
mboxevent_extract_mailbox(flagsset, mbox);
mboxevent_set_numunseen(flagsset, mbox, -1);
mboxevent_set_access(flagsset, NULL, NULL, req->userid, mbox->name, 0);
mboxevent_notify(&flagsset);
}
if (notify_flagsclear) {
mboxevent_extract_mailbox(flagsclear, mbox);
mboxevent_set_numunseen(flagsclear, mbox, -1);
mboxevent_set_access(flagsclear, NULL, NULL, req->userid, mbox->name, 0);
mboxevent_notify(&flagsclear);
}
/* Read output values */
if (!rock->cid) rock->cid = rec->cid;
if (!rock->size) r = msgrecord_get_size(mr, &rock->size);
done:
mboxevent_free(&flagsset);
mboxevent_free(&flagsclear);
if (mr) msgrecord_unref(&mr);
jmap_closembox(rock->req, &mbox);
return r;
}
struct _email_exists_rock {
jmap_req_t *req;
int exists;
};
static int _email_exists_cb(const conv_guidrec_t *rec, void *rock)
{
struct _email_exists_rock *data = (struct _email_exists_rock*) rock;
jmap_req_t *req = data->req;
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
uint32_t internal_flags;
int r = 0;
r = jmap_openmbox(req, rec->mboxname, &mbox, 0);
if (r) return r;
r = msgrecord_find(mbox, rec->uid, &mr);
if (r) goto done;
r = msgrecord_get_internalflags(mr, &internal_flags);
if (r) goto done;
if (!(internal_flags & FLAG_INTERNAL_EXPUNGED)) {
data->exists = 1;
r = CYRUSDB_DONE;
}
done:
if (mr) msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
return r;
}
struct emailcopy_pickrecord_rock {
jmap_req_t *req;
struct mailbox *mbox;
msgrecord_t *mr;
struct email_keywords keywords;
int gather_keywords;
};
static int _email_copy_pickrecord_cb(const conv_guidrec_t *rec, void *vrock)
{
struct emailcopy_pickrecord_rock *rock = vrock;
jmap_req_t *req = rock->req;
/* Make sure we are allowed to read this mailbox */
if (!jmap_hasrights_byname(req, rec->mboxname, ACL_READ)) return 0;
struct mailbox *mbox = NULL;
msgrecord_t *mr = NULL;
/* Check if this record is expunged */
int r = jmap_openmbox(req, rec->mboxname, &mbox, 0);
if (r) goto done;
r = msgrecord_find(mbox, rec->uid, &mr);
if (r) goto done;
uint32_t system_flags;
uint32_t internal_flags;
r = msgrecord_get_systemflags(mr, &system_flags);
if (!r) msgrecord_get_internalflags(mr, &internal_flags);
if (system_flags & FLAG_DELETED || internal_flags & FLAG_INTERNAL_EXPUNGED) {
msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
goto done;
}
/* Aggregate message record flags into JMAP keywords */
if (rock->gather_keywords) {
_email_keywords_add_msgrecord(&rock->keywords, mr);
}
/* Keep this message record as source to copy from? */
if (!rock->mbox) {
rock->mbox = mbox;
rock->mr = mr;
}
else {
msgrecord_unref(&mr);
jmap_closembox(req, &mbox);
}
done:
return r;
}
static void _email_copy(jmap_req_t *req, json_t *copy_email,
const char *from_account_id,
struct seen *seendb,
json_t **new_email, json_t **err)
{
strarray_t dst_mboxnames = STRARRAY_INITIALIZER;
struct mailbox *src_mbox = NULL;
msgrecord_t *src_mr = NULL;
char *src_mboxname = NULL;
int r = 0;
char *blob_id = NULL;
json_t *new_keywords = NULL;
json_t *jmailboxids = json_copy(json_object_get(copy_email, "mailboxIds"));
/* Support mailboxids by role */
const char *mbox_id;
json_t *jval;
void *tmp;
json_object_foreach_safe(jmailboxids, tmp, mbox_id, jval) {
if (*mbox_id != '$') continue;
const char *role = mbox_id + 1;
char *uniqueid = NULL;
if (jmap_mailbox_find_role(req, role, NULL, &uniqueid) == 0) {
json_object_del(jmailboxids, mbox_id);
json_object_set_new(jmailboxids, uniqueid, jval);
}
else {
*err = json_pack("{s:s s:[s]}", "type", "invalidProperties",
"properties", "mailboxIds");
}
free(uniqueid);
if (*err) goto done;
}
const char *email_id = json_string_value(json_object_get(copy_email, "id"));
uint32_t src_size = 0;
/* Lookup source message record and gather JMAP keywords */
new_keywords = json_deep_copy(json_object_get(copy_email, "keywords"));
if (json_is_null(new_keywords)) {
new_keywords = NULL;
}
hash_table seenseq_by_mbox_id = HASH_TABLE_INITIALIZER;
construct_hash_table(&seenseq_by_mbox_id, 32, 0);
struct emailcopy_pickrecord_rock pickrecord_rock = {
req, NULL, NULL, _EMAIL_KEYWORDS_INITIALIZER, 0
};
if (!new_keywords) {
_email_keywords_init(&pickrecord_rock.keywords, req->userid, seendb, &seenseq_by_mbox_id);
pickrecord_rock.gather_keywords = 1;
}
if (strcmp(from_account_id, req->accountid)) {
struct conversations_state *mycstate = NULL;
r = conversations_open_user(from_account_id, 0/*shared*/, &mycstate);
if (!r) r = conversations_guid_foreach(mycstate, _guid_from_id(email_id),
_email_copy_pickrecord_cb, &pickrecord_rock);
if (!r) r = conversations_commit(&mycstate);
}
else {
r = conversations_guid_foreach(req->cstate, _guid_from_id(email_id),
_email_copy_pickrecord_cb, &pickrecord_rock);
}
if (!r && pickrecord_rock.mbox) {
src_mbox = pickrecord_rock.mbox;
src_mr = pickrecord_rock.mr;
if (!new_keywords) {
new_keywords = _email_keywords_to_jmap(&pickrecord_rock.keywords);
}
r = msgrecord_get_size(src_mr, &src_size);
}
else if (!r) {
r = IMAP_NOTFOUND;
}
free_hash_table(&seenseq_by_mbox_id, (void (*)(void *)) seqset_free);
_email_keywords_fini(&pickrecord_rock.keywords);
if (r) {
if (r == IMAP_NOTFOUND || r == IMAP_PERMISSION_DENIED) {
*err = json_pack("{s:s s:[s]}", "type", "invalidProperties",
"properties", "id");
r = 0;
}
goto done;
}
/* Override hasAttachment flag */
int has_attachment = 0;
r = msgrecord_hasflag(src_mr, JMAP_HAS_ATTACHMENT_FLAG, &has_attachment);
if (r) goto done;
if (has_attachment)
json_object_set_new(new_keywords, JMAP_HAS_ATTACHMENT_FLAG, json_true());
else
json_object_del(new_keywords, JMAP_HAS_ATTACHMENT_FLAG);
struct message_guid guid;
r = msgrecord_get_guid(src_mr, &guid);
if (r) goto done;
blob_id = xstrdup(message_guid_encode(&guid));
/* Check if email already exists in to_account */
struct _email_exists_rock data = { req, 0 };
conversations_guid_foreach(req->cstate, blob_id, _email_exists_cb, &data);
if (data.exists) {
*err = json_pack("{s:s s:s}", "type", "alreadyExists", "existingId", email_id);
goto done;
}
/* Lookup mailbox names and make sure they are all writeable */
struct _email_copy_checkmbox_rock checkmbox_rock = {
req, jmailboxids, &dst_mboxnames
};
r = mboxlist_usermboxtree(req->accountid, httpd_authstate,
_email_copy_checkmbox_cb, &checkmbox_rock,
MBOXTREE_INTERMEDIATES);
if (r != IMAP_OK_COMPLETED) {
if (r == 0 || r == IMAP_PERMISSION_DENIED) {
*err = json_pack("{s:s s:[s]}", "type", "invalidProperties",
"properties", "mailboxIds");
r = 0;
}
goto done;
}
/* Copy message record to mailboxes */
char *dst_mboxname;
while ((dst_mboxname = strarray_pop(&dst_mboxnames))) {
mbentry_t *mbentry = NULL;
r = jmap_mboxlist_lookup(dst_mboxname, &mbentry, NULL);
if (!r && (mbentry->mbtype & MBTYPE_INTERMEDIATE)) {
r = mboxlist_promote_intermediary(dst_mboxname);
}
if (!r) {
struct mailbox *dst_mbox = NULL;
r = jmap_openmbox(req, dst_mboxname, &dst_mbox, /*rw*/1);
if (!r) {
r = _copy_msgrecord(httpd_authstate, req->accountid,
&jmap_namespace, src_mbox, dst_mbox, src_mr);
}
jmap_closembox(req, &dst_mbox);
free(dst_mboxname);
}
mboxlist_entry_free(&mbentry);
if (r) goto done;
}
/* Rewrite new message record properties and lookup thread id */
const char *receivedAt = json_string_value(json_object_get(copy_email, "receivedAt"));
struct _email_copy_writeprops_rock writeprops_rock = {
req, receivedAt, new_keywords, seendb, /*cid*/0, /*size*/0
};
r = conversations_guid_foreach(req->cstate, email_id + 1,
_email_copy_writeprops_cb, &writeprops_rock);
if (!r) {
char thread_id[JMAP_THREADID_SIZE];
jmap_set_threadid(writeprops_rock.cid, thread_id);
*new_email = json_pack("{s:s s:s s:s s:i}",
"id", email_id,
"blobId", blob_id,
"threadId", thread_id,
"size", src_size);
}
done:
json_decref(jmailboxids);
if (r && *err == NULL) {
*err = jmap_server_error(r);
}
free(src_mboxname);
free(blob_id);
strarray_fini(&dst_mboxnames);
if (src_mr) msgrecord_unref(&src_mr);
jmap_closembox(req, &src_mbox);
json_decref(new_keywords);
}
static void _email_copy_validate_props(json_t *jemail, json_t **err)
{
struct jmap_parser myparser = JMAP_PARSER_INITIALIZER;
/* Validate properties */
json_t *prop, *id = NULL, *mailboxids = NULL;;
const char *pname;
json_object_foreach(jemail, pname, prop) {
if (!strcmp(pname, "id")) {
if (!json_is_string(prop)) {
jmap_parser_invalid(&myparser, "id");
}
id = prop;
}
else if (!strcmp(pname, "mailboxIds")) {
jmap_parser_push(&myparser, "mailboxIds");
const char *mbox_id;
json_t *jbool;
json_object_foreach(prop, mbox_id, jbool) {
if (!strlen(mbox_id) || jbool != json_true()) {
jmap_parser_invalid(&myparser, NULL);
break;
}
}
jmap_parser_pop(&myparser);
mailboxids = prop;
}
else if (!strcmp(pname, "keywords")) {
if (json_is_object(prop)) {
jmap_parser_push(&myparser, "keywords");
const char *keyword;
json_t *jbool;
json_object_foreach(prop, keyword, jbool) {
if (!jmap_email_keyword_is_valid(keyword) ||
jbool != json_true()) {
jmap_parser_invalid(&myparser, keyword);
}
}
jmap_parser_pop(&myparser);
}
else {
jmap_parser_invalid(&myparser, "keywords");
}
}
else if (!strcmp(pname, "receivedAt")) {
if (!json_is_utcdate(prop)) {
jmap_parser_invalid(&myparser, "receivedAt");
}
}
else {
jmap_parser_invalid(&myparser, pname);
}
}
/* Check mandatory properties */
if (!id) {
jmap_parser_invalid(&myparser, "id");
}
if (!mailboxids) {
jmap_parser_invalid(&myparser, "mailboxIds");
}
/* Reject invalid properties... */
if (json_array_size(myparser.invalid)) {
*err = json_pack("{s:s s:O}",
"type", "invalidProperties",
"properties", myparser.invalid);
}
jmap_parser_fini(&myparser);
}
static int jmap_email_copy(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_copy copy;
json_t *err = NULL;
struct seen *seendb = NULL;
json_t *destroy_emails = json_array();
/* Parse request */
jmap_copy_parse(req, &parser, NULL, NULL, &copy, &err);
if (err) {
jmap_error(req, err);
goto done;
}
int r = seen_open(req->userid, SEEN_CREATE, &seendb);
if (r) {
syslog(LOG_ERR, "jmap_email_copy: can't open seen.db: %s",
error_message(r));
jmap_error(req, jmap_server_error(r));
goto done;
}
/* Process request */
const char *creation_id;
json_t *copy_email;
json_object_foreach(copy.create, creation_id, copy_email) {
json_t *set_err = NULL;
json_t *new_email = NULL;
/* Validate create */
_email_copy_validate_props(copy_email, &set_err);
if (set_err) {
json_object_set_new(copy.not_created, creation_id, set_err);
continue;
}
/* Copy message */
_email_copy(req, copy_email, copy.from_account_id,
seendb, &new_email, &set_err);
if (set_err) {
json_object_set_new(copy.not_created, creation_id, set_err);
continue;
}
/* Note the source ID for deletion */
json_array_append(destroy_emails, json_object_get(copy_email, "id"));
/* Report the message as created */
json_object_set_new(copy.created, creation_id, new_email);
const char *msg_id = json_string_value(json_object_get(new_email, "id"));
jmap_add_id(req, creation_id, msg_id);
}
/* Build response */
jmap_ok(req, jmap_copy_reply(&copy));
/* Destroy originals, if requested */
if (copy.on_success_destroy_original && json_array_size(destroy_emails)) {
json_t *subargs = json_object();
json_object_set(subargs, "destroy", destroy_emails);
json_object_set_new(subargs, "accountId", json_string(copy.from_account_id));
jmap_add_subreq(req, "Email/set", subargs, NULL);
}
done:
json_decref(destroy_emails);
jmap_parser_fini(&parser);
jmap_copy_fini(&copy);
seen_close(&seendb);
return 0;
}
/* Identity/get method */
static const jmap_property_t identity_props[] = {
{
"id",
NULL,
JMAP_PROP_SERVER_SET | JMAP_PROP_IMMUTABLE | JMAP_PROP_ALWAYS_GET
},
{
"name",
NULL,
0
},
{
"email",
NULL,
JMAP_PROP_IMMUTABLE
},
{
"replyTo",
NULL,
0
},
{
"bcc",
NULL,
0
},
{
"textSignature",
NULL,
0
},
{
"htmlSignature",
NULL,
0
},
{
"mayDelete",
NULL,
JMAP_PROP_SERVER_SET
},
/* FM extensions (do ALL of these get through to Cyrus?) */
{
"displayName",
JMAP_MAIL_EXTENSION,
0
},
{
"addBccOnSMTP",
JMAP_MAIL_EXTENSION,
0
},
{
"saveSentToMailboxId",
JMAP_MAIL_EXTENSION,
0
},
{
"saveOnSMTP",
JMAP_MAIL_EXTENSION,
0
},
{
"useForAutoReply",
JMAP_MAIL_EXTENSION,
0
},
{
"isAutoConfigured",
JMAP_MAIL_EXTENSION,
0
},
{
"enableExternalSMTP",
JMAP_MAIL_EXTENSION,
0
},
{
"smtpServer",
JMAP_MAIL_EXTENSION,
0
},
{
"smtpPort",
JMAP_MAIL_EXTENSION,
0
},
{
"smtpSSL",
JMAP_MAIL_EXTENSION,
0
},
{
"smtpUser",
JMAP_MAIL_EXTENSION,
0
},
{
"smtpPassword",
JMAP_MAIL_EXTENSION,
0
},
{
"smtpRemoteService",
JMAP_MAIL_EXTENSION,
0
},
{
"popLinkId",
JMAP_MAIL_EXTENSION,
0
},
{ NULL, NULL, 0 }
};
static int jmap_identity_get(jmap_req_t *req)
{
struct jmap_parser parser = JMAP_PARSER_INITIALIZER;
struct jmap_get get;
json_t *err = NULL;
/* Parse request */
jmap_get_parse(req, &parser, identity_props, /*allow_null_ids*/1,
NULL, NULL, &get, &err);
if (err) {
jmap_error(req, err);
goto done;
}
/* Build response */
json_t *me = json_pack("{s:s}", "id", req->userid);
if (jmap_wantprop(get.props, "name")) {
json_object_set_new(me, "name", json_string(""));
}
if (jmap_wantprop(get.props, "email")) {
json_object_set_new(me, "email",
json_string(strchr(req->userid, '@') ? req->userid : ""));
}
if (jmap_wantprop(get.props, "mayDelete")) {
json_object_set_new(me, "mayDelete", json_false());
}
if (json_array_size(get.ids)) {
size_t i;
json_t *val;
json_array_foreach(get.ids, i, val) {
if (strcmp(json_string_value(val), req->userid)) {
json_array_append(get.not_found, val);
}
else {
json_array_append(get.list, me);
}
}
} else if (!JNOTNULL(get.ids)) {
json_array_append(get.list, me);
}
json_decref(me);
/* Reply */
get.state = xstrdup("0");
jmap_ok(req, jmap_get_reply(&get));
done:
jmap_parser_fini(&parser);
jmap_get_fini(&get);
return 0;
}
static int jmap_email_matchmime_method(jmap_req_t *req)
{
json_t *jfilter = json_object_get(req->args, "filter");
json_t *jmime = json_object_get(req->args, "mime");
json_t *err = NULL;
struct buf mime = BUF_INITIALIZER;
buf_setcstr(&mime, json_string_value(jmime));
int matches = jmap_email_matchmime(&mime, jfilter, req->accountid, time(NULL), &err);
buf_free(&mime);
if (!err) {
json_t *res = json_pack("{s:O s:b}", "filter", jfilter, "matches", matches);
jmap_ok(req, res);
}
else {
jmap_error(req, err);
}
return 0;
}

File Metadata

Mime Type
text/x-diff
Expires
Sat, Apr 4, 6:08 AM (1 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
18821385
Default Alt Text
(679 KB)

Event Timeline