diff --git a/imap/mailbox.c b/imap/mailbox.c old mode 100644 new mode 100755 index 63aca0eff..278bd090a --- a/imap/mailbox.c +++ b/imap/mailbox.c @@ -1,8289 +1,8292 @@ /* mailbox.c -- Mailbox manipulation routines * * Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #ifdef HAVE_UNISTD_H #include #endif #include #include #ifdef HAVE_INTTYPES_H # include #elif defined(HAVE_STDINT_H) # include #endif #include #include #include #include #include #include #ifdef HAVE_DIRENT_H # include # define NAMLEN(dirent) strlen((dirent)->d_name) #else # define dirent direct # define NAMLEN(dirent) (dirent)->d_namlen # if HAVE_SYS_NDIR_H # include # endif # if HAVE_SYS_DIR_H # include # endif # if HAVE_NDIR_H # include # endif #endif #include "annotate.h" #include "assert.h" #ifdef WITH_DAV #include "caldav_db.h" #include "caldav_alarm.h" #include "carddav_db.h" #include "webdav_db.h" #include "ical_support.h" #include "vcard_support.h" #endif /* WITH_DAV */ #ifdef USE_SIEVE #include "sieve_db.h" #include "sievedir.h" #endif #include "crc32.h" #include "md5.h" #include "global.h" #include "imparse.h" #include "cyr_lock.h" #include "mailbox.h" #include "mappedfile.h" #include "message.h" #include "map.h" #include "mboxevent.h" #include "mboxlist.h" #include "parseaddr.h" #include "proc.h" #include "retry.h" #include "seen.h" #include "user.h" #include "util.h" #include "seqset.h" #include "statuscache.h" #include "strarray.h" #include "sync_log.h" #include "xmalloc.h" #include "xstrlcpy.h" #include "xstrlcat.h" #include "xstats.h" #if defined ENABLE_OBJECTSTORE #include "objectstore.h" #include "objectstore_db.h" #endif /* generated headers are not necessarily in current directory */ #include "imap/imap_err.h" #include "imap/mailbox_header_cache.h" static mailbox_wait_cb_t *mailbox_wait_cb = NULL; static void *mailbox_wait_cb_rock = NULL; struct mailbox_iter { struct mailbox *mailbox; message_t *msg; modseq_t changedsince; uint32_t recno; uint32_t num_records; unsigned skipflags; seqset_t *uidset; }; struct mailboxlist { struct mailboxlist *next; char *name; struct mailbox m; struct mboxlock *l; int nopen; }; static struct mailboxlist *open_mailboxes = NULL; #define zeromailbox(m) do { memset(&m, 0, sizeof(struct mailbox)); \ (m).index_fd = -1; \ (m).header_fd = -1; \ } while (0) /* for repack */ struct mailbox_repack { struct mailbox *mailbox; struct mailbox newmailbox; seqset_t *seqset; struct synccrcs crcs; char *userid; ptrarray_t caches; }; static struct MsgFlagMap msgflagmap[] = { {"AN", (int)FLAG_ANSWERED}, {"FL", (int)FLAG_FLAGGED}, {"DE", (int)FLAG_DELETED}, {"DR", (int)FLAG_DRAFT}, {"SE", (int)FLAG_SEEN}, {"SN", (int)FLAG_INTERNAL_SNOOZED}, {"SP", (int)FLAG_INTERNAL_SPLITCONVERSATION}, {"NC", (int)FLAG_INTERNAL_NEEDS_CLEANUP}, {"AR", (int)FLAG_INTERNAL_ARCHIVED}, {"UN", (int)FLAG_INTERNAL_UNLINKED}, {"EX", (int)FLAG_INTERNAL_EXPUNGED} /* ZZ -> invalid file found on disk */ }; /* The length of the msgflagmap list * 2 + * Total number of separators possible * = 33 bytes */ #define FLAGMAPSTR_MAXLEN (1 + 3 *(sizeof(msgflagmap) / sizeof(struct MsgFlagMap))) static int mailbox_index_unlink(struct mailbox *mailbox); static int mailbox_index_repack(struct mailbox *mailbox, int version); static void mailbox_repack_abort(struct mailbox_repack **repackptr); static int mailbox_lock_index_internal(struct mailbox *mailbox, int locktype); static void cleanup_stale_expunged(struct mailbox *mailbox); static bit32 mailbox_index_record_to_buf(struct index_record *record, int version, unsigned char *buf); #ifdef WITH_DAV static struct webdav_db *mailbox_open_webdav(struct mailbox *); static int mailbox_commit_dav(struct mailbox *mailbox); static int mailbox_abort_dav(struct mailbox *mailbox); #endif #ifdef USE_SIEVE static int mailbox_commit_sieve(struct mailbox *mailbox); static int mailbox_abort_sieve(struct mailbox *mailbox); static int mailbox_delete_sieve(struct mailbox *mailbox); #endif static inline void flags_to_str_internal(uint32_t flags, char *flagstr) { size_t i, len = 0; size_t map_size; char *p; p = flagstr; memset(p, 0, FLAGMAPSTR_MAXLEN); map_size = sizeof(msgflagmap) / sizeof(struct MsgFlagMap); for (i = 0; i < map_size && len < FLAGMAPSTR_MAXLEN; i++) { if (flags & msgflagmap[i].flag) { if (p != flagstr) { *p++ = '|'; len++; } *p++ = msgflagmap[i].code[0]; *p++ = msgflagmap[i].code[1]; len += 2; } } } static void flags_to_str(struct index_record *record, char *flagstr) { uint32_t flags = record->system_flags | record->internal_flags; flags_to_str_internal(flags, flagstr); } EXPORTED int open_mailboxes_exist() { return open_mailboxes ? 1 : 0; } static struct mailboxlist *create_listitem(const char *name) { struct mailboxlist *item = xzmalloc(sizeof(struct mailboxlist)); item->next = open_mailboxes; open_mailboxes = item; item->nopen = 1; zeromailbox(item->m); item->name = xstrdup(name); /* ensure we never print insane times */ gettimeofday(&item->m.starttime, 0); #if defined ENABLE_OBJECTSTORE if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)) keep_user_message_db_open (1); #endif return item; } static struct mailboxlist *find_listitem(const char *name) { struct mailboxlist *item; for (item = open_mailboxes; item; item = item->next) { if (!strcmp(name, item->name)) return item; } return NULL; } static void remove_listitem(struct mailboxlist *remitem) { struct mailboxlist *item; struct mailboxlist *previtem = NULL; for (item = open_mailboxes; item; item = item->next) { if (item == remitem) { if (previtem) previtem->next = item->next; else open_mailboxes = item->next; free(item->name); free(item); #if defined ENABLE_OBJECTSTORE if (!open_mailboxes && config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)) // time to close the database keep_user_message_db_open (0); #endif return; } previtem = item; } fatal("didn't find item in list", EX_SOFTWARE); } EXPORTED const char *mailbox_meta_fname(const struct mailbox *mailbox, int metafile) { static char fnamebuf[MAX_MAILBOX_PATH]; uint32_t legacy_dirs = (mailbox_mbtype(mailbox) & MBTYPE_LEGACY_DIRS); const char *src; src = mboxname_metapath(mailbox_partition(mailbox), mailbox_name(mailbox), legacy_dirs ? NULL : mailbox_uniqueid(mailbox), metafile, 0); if (!src) return NULL; xstrncpy(fnamebuf, src, MAX_MAILBOX_PATH); return fnamebuf; } EXPORTED const char *mailbox_meta_newfname(const struct mailbox *mailbox, int metafile) { static char fnamebuf[MAX_MAILBOX_PATH]; uint32_t legacy_dirs = (mailbox_mbtype(mailbox) & MBTYPE_LEGACY_DIRS); const char *src; src = mboxname_metapath(mailbox_partition(mailbox), mailbox_name(mailbox), legacy_dirs ? NULL : mailbox_uniqueid(mailbox), metafile, 1); if (!src) return NULL; xstrncpy(fnamebuf, src, MAX_MAILBOX_PATH); return fnamebuf; } EXPORTED int mailbox_meta_rename(struct mailbox *mailbox, int metafile) { const char *fname = mailbox_meta_fname(mailbox, metafile); const char *newfname = mailbox_meta_newfname(mailbox, metafile); if (rename(newfname, fname)) return IMAP_IOERROR; return 0; } static const char *mailbox_spool_fname(struct mailbox *mailbox, uint32_t uid) { uint32_t legacy_dirs = (mailbox_mbtype(mailbox) & MBTYPE_LEGACY_DIRS); return mboxname_datapath(mailbox_partition(mailbox), mailbox_name(mailbox), legacy_dirs ? NULL : mailbox_uniqueid(mailbox), uid); } static const char *mailbox_archive_fname(struct mailbox *mailbox, uint32_t uid) { uint32_t legacy_dirs = (mailbox_mbtype(mailbox) & MBTYPE_LEGACY_DIRS); return mboxname_archivepath(mailbox_partition(mailbox), mailbox_name(mailbox), legacy_dirs ? NULL : mailbox_uniqueid(mailbox), uid); } EXPORTED const char *mailbox_record_fname(struct mailbox *mailbox, const struct index_record *record) { int object_storage_enabled = 0 ; #if defined ENABLE_OBJECTSTORE object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ; #endif if (!object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED)) return mailbox_archive_fname(mailbox, record->uid); else return mailbox_spool_fname(mailbox, record->uid); } EXPORTED const char *mailbox_datapath(struct mailbox *mailbox, uint32_t uid) { static char localbuf[MAX_MAILBOX_PATH]; uint32_t legacy_dirs = (mailbox_mbtype(mailbox) & MBTYPE_LEGACY_DIRS); const char *src; src = mboxname_datapath(mailbox_partition(mailbox), mailbox_name(mailbox), legacy_dirs ? NULL : mailbox_uniqueid(mailbox), uid); if (!src) return NULL; xstrncpy(localbuf, src, MAX_MAILBOX_PATH); return localbuf; } /* * Function to test if a header is in the cache * * Assume cache entry version 1, unless other data is found * in the table. */ static inline unsigned is_cached_header(const char *hdr) { size_t len; struct mailbox_header_cache *thdr; len = strlen(hdr); if (len >= MAX_CACHED_HEADER_SIZE) return BIT32_MAX; thdr = mailbox_header_cache_lookup(hdr, len); if (thdr) return thdr->min_cache_version; /* Don't Cache X- headers unless explicitly configured to*/ if (((hdr[0] == 'x') || (hdr[0] == 'X')) && (hdr[1] == '-')) return BIT32_MAX; /* Everything else we cache in version 1 */ return 1; } /* External API to is_cached_header that prepares the string * * Returns minimum version required for lookup to succeed * or BIT32_MAX if header not cached */ EXPORTED unsigned mailbox_cached_header(const char *s) { return is_cached_header(s); } /* Same as mailbox_cached_header, but for use on a header * as it appears in the message (i.e. :-terminated, not NUL-terminated) */ HIDDEN unsigned mailbox_cached_header_inline(const char *text) { char buf[MAX_CACHED_HEADER_SIZE]; int i; /* Scan for header */ for (i=0; i < MAX_CACHED_HEADER_SIZE; i++) { if (!text[i] || text[i] == '\r' || text[i] == '\n') break; if (text[i] == ':') { buf[i] = '\0'; return is_cached_header(buf); } else { buf[i] = text[i]; } } return BIT32_MAX; } static const char *cache_base(const struct index_record *record) { const char *base = record->crec.buf->s; return base + record->crec.offset; } static size_t cache_len(const struct index_record *record) { return record->crec.len; } static struct buf *cache_buf(const struct index_record *record) { static struct buf staticbuf; buf_init_ro(&staticbuf, cache_base(record), cache_len(record)); return &staticbuf; } EXPORTED const char *cacheitem_base(const struct index_record *record, int field) { const char *base = record->crec.buf->s; return base + record->crec.item[field].offset; } EXPORTED unsigned cacheitem_size(const struct index_record *record, int field) { return record->crec.item[field].len; } EXPORTED struct buf *cacheitem_buf(const struct index_record *record, int field) { static struct buf staticbuf; buf_init_ro(&staticbuf, cacheitem_base(record, field), cacheitem_size(record, field)); return &staticbuf; } /* parse a single cache record from the mapped file - creates buf * records which point into the map, so you can't free it while * you still have them around! */ static int cache_parserecord(struct mappedfile *cachefile, uint64_t cache_offset, struct cacherecord *crec) { const struct buf *buf = mappedfile_buf(cachefile); size_t buf_size = mappedfile_size(cachefile); const char *cacheitem, *next; size_t offset; int cache_ent; offset = cache_offset; for (cache_ent = 0; cache_ent < NUM_CACHE_FIELDS; cache_ent++) { cacheitem = buf->s + offset; /* bounds checking */ if (offset >= buf_size) { xsyslog(LOG_ERR, "IOERROR: offset greater than cache size", "offset=<" SIZE_T_FMT "> " "buf_size=<" SIZE_T_FMT "> " "cache_ent=<%d>", offset, buf_size, cache_ent); return IMAP_IOERROR; } if (offset + CACHE_ITEM_SIZE_SKIP + CACHE_ITEM_LEN(cacheitem) > buf_size) { xsyslog(LOG_ERR, "IOERROR: cache entry truncated", "offset=<" SIZE_T_FMT "> " "length=<%u> " "buf_size=<" SIZE_T_FMT "> " "cache_ent=<%d>", offset, CACHE_ITEM_LEN(cacheitem), buf_size, cache_ent); return IMAP_IOERROR; } /* copy locations */ crec->item[cache_ent].len = CACHE_ITEM_LEN(cacheitem); crec->item[cache_ent].offset = offset + CACHE_ITEM_SIZE_SKIP; /* moving on */ next = CACHE_ITEM_NEXT(cacheitem); if (next < cacheitem) { xsyslog(LOG_ERR, "IOERROR: cache offset negative", NULL); return IMAP_IOERROR; } offset = next - buf->s; } /* all fit within the cache, it's gold as far as we can tell */ crec->buf = buf; crec->len = offset - cache_offset; crec->offset = cache_offset; return 0; } EXPORTED char *mailbox_cache_get_env(struct mailbox *mailbox, const struct index_record *record, int token) { char *env; char *envtokens[NUMENVTOKENS]; char *field; if (mailbox_cacherecord(mailbox, record)) return NULL; if (cacheitem_size(record, CACHE_ENVELOPE) <= 2) return NULL; /* get field out of the envelope * * get a working copy; strip outer ()'s * +1 -> skip the leading paren * -2 -> don't include the size of the outer parens */ env = xstrndup(cacheitem_base(record, CACHE_ENVELOPE) + 1, cacheitem_size(record, CACHE_ENVELOPE) - 2); parse_cached_envelope(env, envtokens, VECTOR_SIZE(envtokens)); field = xstrdupnull(envtokens[token]); /* free stuff */ free(env); return field; } EXPORTED int mailbox_index_islocked(struct mailbox *mailbox, int write) { if (mailbox->index_locktype == LOCK_EXCLUSIVE) return 1; if (mailbox->index_locktype == LOCK_SHARED && !write) return 1; return 0; } static int cache_append_record(struct mappedfile *mf, struct index_record *record) { const struct buf *buf = cache_buf(record); size_t offset = mappedfile_size(mf); int n; n = mappedfile_pwritebuf(mf, buf, offset); if (n < 0) { syslog(LOG_ERR, "failed to append " SIZE_T_FMT " bytes to cache", buf->len); return IMAP_IOERROR; } record->cache_offset = offset; return 0; } static struct mappedfile *cache_getfile(ptrarray_t *list, const char *fname, int readonly, uint32_t generation) { struct mappedfile *cachefile = NULL; int openflags = readonly ? 0 : MAPPEDFILE_CREATE | MAPPEDFILE_RW; int i; for (i = 0; i < list->count; i++) { cachefile = ptrarray_nth(list, i); if (!strcmp(fname, mappedfile_fname(cachefile))) return cachefile; } /* guess we didn't find it - open a new one */ cachefile = NULL; if (mappedfile_open(&cachefile, fname, openflags)) { xsyslog(LOG_ERR, "IOERROR: failed to open cache file", "fname=<%s>", fname); return NULL; } if (!readonly && !mappedfile_size(cachefile)) { /* zero byte file? Set the generation */ uint32_t netgen = htonl(generation); mappedfile_pwrite(cachefile, (const char *) &netgen, 4, 0); mappedfile_commit(cachefile); } ptrarray_append(list, cachefile); return cachefile; } static struct mappedfile *mailbox_cachefile(struct mailbox *mailbox, const struct index_record *record) { const char *fname; if (record->internal_flags & FLAG_INTERNAL_ARCHIVED) fname = mailbox_meta_fname(mailbox, META_ARCHIVECACHE); else fname = mailbox_meta_fname(mailbox, META_CACHE); return cache_getfile(&mailbox->caches, fname, mailbox->is_readonly, mailbox->i.generation_no); } static struct mappedfile *repack_cachefile(struct mailbox_repack *repack, const struct index_record *record) { const char *fname; if (record->internal_flags & FLAG_INTERNAL_ARCHIVED) fname = mailbox_meta_newfname(repack->mailbox, META_ARCHIVECACHE); else fname = mailbox_meta_newfname(repack->mailbox, META_CACHE); return cache_getfile(&repack->caches, fname, /*readonly*/0, repack->newmailbox.i.generation_no); } /* return the offset for the start of the record! */ static int mailbox_append_cache(struct mailbox *mailbox, struct index_record *record) { struct mappedfile *cachefile; int r; assert(mailbox_index_islocked(mailbox, 1)); /* already been written */ if (record->cache_offset) return 0; /* no cache content */ if (!record->crec.len) { /* make one! */ const char *fname = mailbox_record_fname(mailbox, record); xsyslog(LOG_ERR, "IOERROR: no cache content, parsing and saving", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), record->uid); r = message_parse(fname, record); if (r) return r; mailbox_index_dirty(mailbox); mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK; } cachefile = mailbox_cachefile(mailbox, record); if (!cachefile) { syslog(LOG_ERR, "Failed to open cache to %s for %u", mailbox_name(mailbox), record->uid); return IMAP_IOERROR; /* unable to append */ } r = cache_append_record(cachefile, record); if (r) { syslog(LOG_ERR, "Failed to append cache to %s for %u", mailbox_name(mailbox), record->uid); return r; } return 0; } enum mbcache_rewrite_t { MBCACHE_NOPARSE, MBCACHE_PARSEONLY, MBCACHE_REWRITE }; static int mailbox_cacherecord_internal(struct mailbox *mailbox, const struct index_record *record, enum mbcache_rewrite_t rewrite) { struct mappedfile *cachefile; bit32 crc = 0; int r = IMAP_IOERROR; /* we do something nasty here to work around lazy loading while still * giving const protection to records which are only used for read */ struct index_record *backdoor = (struct index_record *)record; /* do we already have a record loaded? */ if (record->crec.len) return 0; /* make sure there's a file to read from */ cachefile = mailbox_cachefile(mailbox, record); if (!cachefile) goto err; /* do we have an offset? */ if (!record->cache_offset) goto err; /* try to parse the cache record */ r = cache_parserecord(cachefile, record->cache_offset, &backdoor->crec); if (r) goto err; /* old-style record */ if (!record->cache_crc) goto err; crc = crc32_buf(cache_buf(record)); if (crc != record->cache_crc) r = IMAP_MAILBOX_CHECKSUM; if (r) goto err; return 0; err: if (!cachefile) xsyslog(LOG_ERR, "IOERROR: missing cache file", "mailbox=<%s> uid=<%u>", mailbox_name(mailbox), record->uid); else if (!record->cache_offset) xsyslog(LOG_ERR, "IOERROR: missing cache offset", "mailbox=<%s> uid=<%u>", mailbox_name(mailbox), record->uid); else if (r) xsyslog(LOG_ERR, "IOERROR invalid cache record", "mailbox=<%s> uid=<%u> error=<%s> crc=<%d> cache_offset=<%llu>", mailbox_name(mailbox), record->uid, error_message(r), crc, (unsigned long long) record->cache_offset); if (rewrite == MBCACHE_NOPARSE) return r; /* parse the file again */ /* parse directly into the cache for this record */ const char *fname = mailbox_record_fname(mailbox, record); if (!fname) { xsyslog(LOG_ERR, "IOERROR: no spool file", "mailbox=<%s> uid=<%u>", mailbox_name(mailbox), record->uid); return IMAP_IOERROR; } /* parse into the file and zero the cache offset */ r = message_parse(fname, backdoor); if (r) { xsyslog(LOG_ERR, "IOERROR: failed to parse message", "mailbox=<%s> uid=<%u>", mailbox_name(mailbox), record->uid); return r; } backdoor->cache_offset = 0; if (rewrite == MBCACHE_REWRITE) { r = mailbox_append_cache(mailbox, backdoor); if (r) { xsyslog(LOG_ERR, "IOERROR: failed to append cache", "mailbox=<%s> uid=<%u>", mailbox_name(mailbox), record->uid); return r; } } return 0; } EXPORTED int mailbox_cacherecord(struct mailbox *mailbox, const struct index_record *record) { enum mbcache_rewrite_t rewrite = mailbox_index_islocked(mailbox, 1) ? MBCACHE_REWRITE : MBCACHE_PARSEONLY; return mailbox_cacherecord_internal(mailbox, record, rewrite); } static int mailbox_abort_cache(struct mailbox *mailbox) { if (!mailbox->caches.count) return 0; /* XXX - we need offsets into each file to use mappedfile_truncate */ return 0; } static int mailbox_commit_cache(struct mailbox *mailbox) { int i; for (i = 0; i < mailbox->caches.count; i++) { struct mappedfile *cachefile = ptrarray_nth(&mailbox->caches, i); int r = mappedfile_commit(cachefile); if (r) return r; } return 0; } /* function to be used for notification of mailbox changes/updates */ static mailbox_notifyproc_t *updatenotifier = NULL; /* * Set the updatenotifier function */ HIDDEN void mailbox_set_updatenotifier(mailbox_notifyproc_t *notifyproc) { updatenotifier = notifyproc; } /* * Get the updatenotifier function */ mailbox_notifyproc_t *mailbox_get_updatenotifier(void) { return updatenotifier; } EXPORTED void mailbox_set_uniqueid(struct mailbox *mailbox, const char *uniqueid) { free(mailbox->h.uniqueid); mailbox->h.uniqueid = xstrdup(uniqueid); mailbox->header_dirty = 1; } EXPORTED void mailbox_set_mbtype(struct mailbox *mailbox, uint32_t mbtype) { mailbox->h.mbtype = mbtype; mailbox->header_dirty = 1; } static int _map_local_record(const struct mailbox *mailbox, const char *fname, struct buf *buf) { struct stat sbuf; int msgfd; msgfd = open(fname, O_RDONLY, 0666); if (msgfd == -1) return errno; if (fstat(msgfd, &sbuf) == -1) { xsyslog(LOG_ERR, "IOERROR: fstat failed", "fname=<%s>", fname); fatal("can't fstat message file", EX_OSFILE); } buf_refresh_mmap(buf, /*onceonly*/1, msgfd, fname, sbuf.st_size, mailbox_name(mailbox)); close(msgfd); return 0; } EXPORTED int mailbox_map_record(struct mailbox *mailbox, const struct index_record *record, struct buf *buf) { xstats_inc(MESSAGE_MAP); const char *fname = mailbox_record_fname(mailbox, record); int r = _map_local_record(mailbox, fname, buf); if (!r) return 0; #if defined ENABLE_OBJECTSTORE if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)){ r = objectstore_get(mailbox, record, fname); if (r) return r; r = _map_local_record(mailbox, fname, buf); remove(fname); } #endif return r; } static void mailbox_release_resources(struct mailbox *mailbox) { int i; if (mailbox->i.dirty) abort(); /* just close the header */ xclose(mailbox->header_fd); /* release and unmap index */ xclose(mailbox->index_fd); mailbox->index_locktype = 0; /* lock was released by closing fd */ if (mailbox->index_base) map_free(&mailbox->index_base, &mailbox->index_len); /* release caches */ for (i = 0; i < mailbox->caches.count; i++) { struct mappedfile *cachefile = ptrarray_nth(&mailbox->caches, i); mappedfile_close(&cachefile); } ptrarray_fini(&mailbox->caches); } /* * Open the index file for 'mailbox' */ static int mailbox_open_index(struct mailbox *mailbox) { struct stat sbuf; const char *fname; int openflags = mailbox->is_readonly ? O_RDONLY : O_RDWR; mailbox_release_resources(mailbox); /* open and map the index file */ fname = mailbox_meta_fname(mailbox, META_INDEX); if (!fname) return IMAP_MAILBOX_BADNAME; mailbox->index_fd = open(fname, openflags, 0); if (mailbox->index_fd == -1) return IMAP_IOERROR; /* don't open the cache yet, it will be loaded by lazy-loading * later */ fstat(mailbox->index_fd, &sbuf); mailbox->index_ino = sbuf.st_ino; mailbox->index_mtime = sbuf.st_mtime; mailbox->index_size = sbuf.st_size; map_refresh(mailbox->index_fd, 0, &mailbox->index_base, &mailbox->index_len, mailbox->index_size, "index", mailbox_name(mailbox)); return 0; } static int mailbox_mboxlock_reopen(struct mailboxlist *listitem, int locktype, int index_locktype) { struct mailbox *mailbox = &listitem->m; uint32_t legacy_dirs = (mailbox_mbtype(mailbox) & MBTYPE_LEGACY_DIRS); int r; mailbox_release_resources(mailbox); mboxname_release(&listitem->l); mboxname_release(&mailbox->local_namespacelock); char *userid = mboxname_to_userid(mailbox_name(mailbox)); if (userid) { int haslock = user_isnamespacelocked(userid); if (haslock) { if (index_locktype != LOCK_SHARED) assert(haslock != LOCK_SHARED); } else { mailbox->local_namespacelock = user_namespacelock_full(userid, index_locktype); } free(userid); } r = mboxname_lock(legacy_dirs ? mailbox_name(mailbox) : mailbox_uniqueid(mailbox), &listitem->l, locktype); if (r) return r; return r; } /* * Open and read the header of the mailbox with name 'name' * The structure pointed to by 'mailbox' is initialized. */ static int mailbox_open_advanced(const char *name, int locktype, int index_locktype, const mbentry_t *mbe, struct mailbox **mailboxptr) { mbentry_t *mbentry = NULL; struct mailboxlist *listitem; struct mailbox *mailbox = NULL; int r = 0; assert(*mailboxptr == NULL); listitem = find_listitem(name); /* already open? just use this one */ if (listitem) { /* can't reuse an exclusive locked mailbox */ if (listitem->l->locktype & LOCK_EXCLUSIVE) return IMAP_MAILBOX_LOCKED; if (locktype & LOCK_EXCLUSIVE) return IMAP_MAILBOX_LOCKED; /* can't reuse an already locked index */ if (listitem->m.index_locktype) return IMAP_MAILBOX_LOCKED; listitem->nopen++; mailbox = &listitem->m; goto lockindex; } listitem = create_listitem(name); mailbox = &listitem->m; // lock the user namespace FIRST before the mailbox namespace char *userid = mboxname_to_userid(name); int haslock = user_isnamespacelocked(userid); if (haslock) { if (index_locktype & LOCK_EXCLUSIVE) assert(haslock & LOCK_EXCLUSIVE); } else { mailbox->local_namespacelock = user_namespacelock_full(userid, index_locktype); } free(userid); if (mbe) mbentry = mboxlist_entry_copy(mbe); else r = mboxlist_lookup_allow_all(name, &mbentry, NULL); /* pre-check for some conditions which mean that we don't want to go ahead and open this mailbox */ if (!r && mbentry->mbtype & MBTYPE_DELETED) r = IMAP_MAILBOX_NONEXISTENT; if (!r && mbentry->mbtype & MBTYPE_MOVING) r = IMAP_MAILBOX_MOVED; if (!r && mbentry->mbtype & MBTYPE_INTERMEDIATE) r = IMAP_MAILBOX_NONEXISTENT; if (!r && !mbentry->partition) r = IMAP_MAILBOX_NONEXISTENT; if (r) { if (mailbox->local_namespacelock) mboxname_release(&mailbox->local_namespacelock); mboxlist_entry_free(&mbentry); remove_listitem(listitem); return r; } /* XXX can we even get here for remote mbentries? */ if (!mbentry->uniqueid && mbentry_is_local_mailbox(mbentry)) { /* Theoretically it shouldn't be possible for an mbentry to not * have a uniqueid... so if it happens, complain loudly. */ xsyslog(LOG_ERR, "mbentry has no uniqueid, needs reconstruct", "mboxname=<%s>", name); } uint32_t legacy_dirs = (mbentry->mbtype & MBTYPE_LEGACY_DIRS); r = mboxname_lock(legacy_dirs ? name : mbentry->uniqueid, &listitem->l, locktype); if (r) { /* locked is not an error - just means we asked for NONBLOCKING */ if (r != IMAP_MAILBOX_LOCKED) xsyslog(LOG_ERR, "IOERROR: lock failed", "mailbox=<%s> error=<%s>", name, error_message(r)); if (mailbox->local_namespacelock) mboxname_release(&mailbox->local_namespacelock); mboxlist_entry_free(&mbentry); remove_listitem(listitem); return r; } if (!mbentry->name) mbentry->name = xstrdup(name); mailbox->mbentry = mbentry; if (index_locktype == LOCK_SHARED) mailbox->is_readonly = 1; r = mailbox_open_index(mailbox); if (r) { xsyslog(LOG_ERR, "IOERROR: opening index failed", "mailbox=<%s> error=<%s>", mailbox_name(mailbox), error_message(r)); goto done; } lockindex: r = mailbox_lock_index(mailbox, index_locktype); /* we always nuke expunged if the version is less than 12 */ if (mailbox->i.minor_version < 12) cleanup_stale_expunged(mailbox); done: if (r) mailbox_close(&mailbox); else *mailboxptr = mailbox; return r; } EXPORTED int mailbox_open_irl(const char *name, struct mailbox **mailboxptr) { return mailbox_open_advanced(name, LOCK_SHARED, LOCK_SHARED, NULL, mailboxptr); } EXPORTED int mailbox_open_iwl(const char *name, struct mailbox **mailboxptr) { return mailbox_open_advanced(name, LOCK_SHARED, LOCK_EXCLUSIVE, NULL, mailboxptr); } EXPORTED int mailbox_open_irlnb(const char *name, struct mailbox **mailboxptr) { return mailbox_open_advanced(name, LOCK_SHARED|LOCK_NONBLOCK, /* cannot do nonblocking lock on index...why? */ LOCK_SHARED, NULL, mailboxptr); } EXPORTED int mailbox_open_exclusive(const char *name, struct mailbox **mailboxptr) { return mailbox_open_advanced(name, LOCK_EXCLUSIVE, LOCK_EXCLUSIVE, NULL, mailboxptr); } EXPORTED int mailbox_open_from_mbe(const mbentry_t *mbe, struct mailbox **mailboxptr) { return mailbox_open_advanced(mbe->name, LOCK_EXCLUSIVE, LOCK_EXCLUSIVE, mbe, mailboxptr); } EXPORTED const char *mailbox_name(const struct mailbox *mailbox) { return mailbox->mbentry->name; } EXPORTED const char *mailbox_uniqueid(const struct mailbox *mailbox) { mbentry_t *mbentry = mailbox->mbentry; return mbentry->uniqueid ? mbentry->uniqueid : mailbox->h.uniqueid; } EXPORTED const char *mailbox_partition(const struct mailbox *mailbox) { return mailbox->mbentry->partition; } EXPORTED const char *mailbox_acl(const struct mailbox *mailbox) { return mailbox->mbentry->acl; } EXPORTED uint32_t mailbox_mbtype(const struct mailbox *mailbox) { return mailbox->mbentry->mbtype; } EXPORTED modseq_t mailbox_foldermodseq(const struct mailbox *mailbox) { return mailbox->mbentry->foldermodseq; } EXPORTED const char *mailbox_quotaroot(const struct mailbox *mailbox) { return mailbox->h.quotaroot; } EXPORTED void mailbox_index_dirty(struct mailbox *mailbox) { assert(mailbox_index_islocked(mailbox, 1)); mailbox->i.dirty = 1; } EXPORTED modseq_t mailbox_modseq_dirty(struct mailbox *mailbox) { assert(mailbox_index_islocked(mailbox, 1)); if (mailbox->silentchanges) { mailbox->modseq_dirty = 1; mailbox_index_dirty(mailbox); return mailbox->i.highestmodseq; } if (!mailbox->modseq_dirty) { mailbox->i.highestmodseq = mboxname_setmodseq(mailbox_name(mailbox), mailbox->i.highestmodseq, mailbox_mbtype(mailbox), /*flags*/0); mailbox->last_updated = time(0); mailbox->modseq_dirty = 1; mailbox_index_dirty(mailbox); } mailbox->i.highestmodseq++; return mailbox->i.highestmodseq; } EXPORTED int mailbox_setversion(struct mailbox *mailbox, int version) { int r = 0; if (version && mailbox->i.minor_version != version) { /* need to re-set the version! */ struct mailboxlist *listitem = find_listitem(mailbox_name(mailbox)); assert(listitem); assert(&listitem->m == mailbox); /* we need an exclusive lock on the listitem because we're renaming * index files, so release locks and then go full exclusive */ mailbox_unlock_index(mailbox, NULL); r = mailbox_mboxlock_reopen(listitem, LOCK_EXCLUSIVE, LOCK_EXCLUSIVE); /* we need to re-open the index because we dropped the mboxname lock, * so the file may have changed */ if (!r) r = mailbox_open_index(mailbox); /* lock_internal so DELETED doesn't cause it to appear to be * NONEXISTENT */ if (!r) r = mailbox_lock_index_internal(mailbox, LOCK_EXCLUSIVE); /* perform the actual repack! */ if (!r) r = mailbox_index_repack(mailbox, version); /* NOTE: this leaves the mailbox in an unlocked state internally, so * let's release all the acutal locks */ mailbox_unlock_index(mailbox, NULL); /* we're also still holding an exclusive namelock in the listitem, * but that's OK because the only caller will be calling mailbox_close * immediately afterwards */ } return r; } static void _delayed_cleanup(void *rock) { const char *mboxname = (const char *)rock; struct mailbox *mailbox = NULL; /* don't do the potentially expensive work of repacking mailboxes * if we are in the middle of a shutdown */ if (in_shutdown) goto done; int r = mailbox_open_advanced(mboxname, LOCK_EXCLUSIVE|LOCK_NONBLOCK, LOCK_EXCLUSIVE, NULL, &mailbox); if (r) goto done; if (mailbox->i.options & OPT_MAILBOX_NEEDS_REPACK) { mailbox_index_repack(mailbox, mailbox->i.minor_version); // clear the flags here too so we don't try to repack again mailbox->i.options &= ~(OPT_MAILBOX_NEEDS_REPACK|OPT_MAILBOX_NEEDS_UNLINK); } else if (mailbox->i.options & OPT_MAILBOX_NEEDS_UNLINK) { mailbox_index_unlink(mailbox); } /* or we missed out - someone else beat us to it, all good */ done: mailbox_close(&mailbox); } /* * Close the mailbox 'mailbox', freeing all associated resources. */ EXPORTED void mailbox_close(struct mailbox **mailboxptr) { int flag; struct mailbox *mailbox = *mailboxptr; struct mailboxlist *listitem; /* be safe against double-close */ if (!mailbox) return; listitem = find_listitem(mailbox_name(mailbox)); assert(listitem && &listitem->m == mailbox); *mailboxptr = NULL; /* open multiple times? Just close this one */ if (listitem->nopen > 1) { listitem->nopen--; mailbox_unlock_index(mailbox, NULL); return; } if (mailbox->index_fd != -1) { /* drop the index lock here because we'll lose our right to it * when try to upgrade the mboxlock anyway. */ mailbox_unlock_index(mailbox, NULL); } if (mailbox->i.options & OPT_MAILBOX_DELETED) { int r = mailbox_mboxlock_reopen(listitem, LOCK_EXCLUSIVE, LOCK_EXCLUSIVE); if (!r) r = mailbox_open_index(mailbox); /* lock_internal so DELETED doesn't cause it to appear to be * NONEXISTENT - but we still need conversations so we can write changes! */ if (!r) r = mailbox_lock_index_internal(mailbox, LOCK_EXCLUSIVE); /* double check just in case a new mailbox with the same name got created * in a race condition and isn't deleted! */ if (!r && (mailbox->i.options & OPT_MAILBOX_DELETED)) { mailbox_delete_cleanup(mailbox, mailbox_partition(mailbox), mailbox_name(mailbox), (mailbox_mbtype(mailbox) & MBTYPE_LEGACY_DIRS) ? NULL : mailbox_uniqueid(mailbox)); } mailbox_unlock_index(mailbox, NULL); } else if (!in_shutdown && (mailbox->i.options & MAILBOX_CLEANUP_MASK)) { // there's cleanup to do! Schedule it for after we've replied to the user libcyrus_delayed_action(mailbox_meta_fname(mailbox, META_HEADER), _delayed_cleanup, free, xstrdup(mailbox_name(mailbox))); } mailbox_release_resources(mailbox); mboxlist_entry_free(&mailbox->mbentry); xzfree(mailbox->h.name); xzfree(mailbox->h.uniqueid); xzfree(mailbox->h.quotaroot); xzfree(mailbox->h.acl); for (flag = 0; flag < MAX_USER_FLAGS; flag++) { xzfree(mailbox->h.flagname[flag]); } if (listitem->l) mboxname_release(&listitem->l); if (mailbox->local_namespacelock) mboxname_release(&mailbox->local_namespacelock); remove_listitem(listitem); } struct parseentry_rock { struct mailbox_header *h; struct buf aclbuf; int doingacl; int doingflags; int nflags; }; static int parseentry_cb(int type, struct dlistsax_data *d) { struct parseentry_rock *rock = (struct parseentry_rock *)d->rock; const char *key = buf_cstring(&d->kbuf); switch(type) { case DLISTSAX_KVLISTSTART: if (!strcmp(key, "A")) { rock->doingacl = 1; } break; case DLISTSAX_KVLISTEND: if (rock->doingacl) { xzfree(rock->h->acl); rock->h->acl = buf_release(&rock->aclbuf); rock->doingacl = 0; } break; case DLISTSAX_LISTSTART: if (!strcmp(key, "U")) { rock->doingflags = 1; rock->nflags = 0; } break; case DLISTSAX_LISTEND: if (rock->doingflags) { rock->doingflags = 0; /* zero out the rest */ while (rock->nflags < MAX_USER_FLAGS) { xzfree(rock->h->flagname[rock->nflags]); rock->nflags++; } } break; case DLISTSAX_STRING: if (rock->doingacl) { buf_append(&rock->aclbuf, &d->kbuf); buf_putc(&rock->aclbuf, '\t'); buf_appendcstr(&rock->aclbuf, d->data); buf_putc(&rock->aclbuf, '\t'); } else if (rock->doingflags) { xzfree(rock->h->flagname[rock->nflags]); rock->h->flagname[rock->nflags++] = xstrdupnull(d->data); } else { if (!strcmp(key, "I")) { xzfree(rock->h->uniqueid); rock->h->uniqueid = xstrdupnull(d->data); } else if (!strcmp(key, "N")) { xzfree(rock->h->name); rock->h->name = xstrdupnull(d->data); } else if (!strcmp(key, "T")) { rock->h->mbtype = mboxlist_string_to_mbtype(d->data); } else if (!strcmp(key, "Q")) { xzfree(rock->h->quotaroot); rock->h->quotaroot = xstrdupnull(d->data); } } } return 0; } /* * parse data read from cyrus.header into its parts. * * full dlist format is: * A: _a_cl * I: unique_i_d * N: _n_ame * Q: _q_uotaroot * T: _t_ype * U: user_f_lags */ static int _parse_header_data(struct mailbox *mailbox, const char *data, size_t datalen) { if (!datalen) return IMAP_MAILBOX_BADFORMAT; struct parseentry_rock rock = { &mailbox->h, BUF_INITIALIZER, 0, 0, 0 }; int r = dlist_parsesax(data, datalen, 0, parseentry_cb, &rock); buf_free(&rock.aclbuf); // should be noop, but cleans up after errors return r; } /* * Read the header of 'mailbox' * new format: * MAGIC * dlist (see above) * * old format: * MAGIC * quotaroot TAB uniqueid * userflag1 SPACE userflag2 SPACE userflag3 [...] (with no trailing space) * user1 TAB user1acl TAB user2 TAB user2acl TAB (with trailing tab!) */ static int mailbox_read_header(struct mailbox *mailbox, const char *fname) { int r = 0; int flag; const char *name, *p, *tab, *eol; struct stat sbuf; const char *base = NULL; size_t len = 0; unsigned magic_size = sizeof(MAILBOX_HEADER_MAGIC) - 1; /* can't be dirty if we're reading it */ if (mailbox->header_dirty) abort(); xclose(mailbox->header_fd); if (!fname) fname = mailbox_meta_fname(mailbox, META_HEADER); mailbox->header_fd = open(fname, O_RDONLY, 0); if (mailbox->header_fd == -1) { r = IMAP_IOERROR; goto done; } if (fstat(mailbox->header_fd, &sbuf) == -1) { xclose(mailbox->header_fd); r = IMAP_IOERROR; goto done; } map_refresh(mailbox->header_fd, 1, &base, &len, sbuf.st_size, "header", mailbox_name(mailbox)); mailbox->header_file_ino = sbuf.st_ino; mailbox->header_file_crc = crc32_map(base, sbuf.st_size); /* Check magic number */ if ((unsigned) sbuf.st_size < magic_size || strncmp(base, MAILBOX_HEADER_MAGIC, magic_size)) { r = IMAP_MAILBOX_BADFORMAT; goto done; } /* Read quota data line */ p = base + sizeof(MAILBOX_HEADER_MAGIC)-1; tab = memchr(p, '\t', sbuf.st_size - (p - base)); eol = memchr(p, '\n', sbuf.st_size - (p - base)); if (!eol) { r = IMAP_MAILBOX_BADFORMAT; goto done; } xzfree(mailbox->h.quotaroot); xzfree(mailbox->h.uniqueid); xzfree(mailbox->h.acl); /* check for DLIST mboxlist */ if (*p == '%') { r = _parse_header_data(mailbox, p, eol - p); goto done; } /* quotaroot (if present) */ if (!tab || tab > eol) { syslog(LOG_DEBUG, "mailbox '%s' has old cyrus.header", mailbox_name(mailbox)); tab = eol; } if (p < tab) { mailbox->h.quotaroot = xstrndup(p, tab - p); } /* read uniqueid (should always exist unless old format) */ if (tab < eol) { p = tab + 1; if (p == eol) { r = IMAP_MAILBOX_BADFORMAT; goto done; } tab = memchr(p, '\t', sbuf.st_size - (p - base)); if (!tab || tab > eol) tab = eol; mailbox->h.uniqueid = xstrndup(p, tab - p); } else { /* ancient cyrus.header file without a uniqueid field! */ xsyslog(LOG_ERR, "mailbox header has no uniqueid, needs reconstruct", "mboxname=<%s>", mailbox_name(mailbox)); } /* Read names of user flags */ p = eol + 1; eol = memchr(p, '\n', sbuf.st_size - (p - base)); if (!eol) { r = IMAP_MAILBOX_BADFORMAT; goto done; } name = p; /* read the names of flags */ for (flag = 0; name <= eol && flag < MAX_USER_FLAGS; flag++) { xzfree(mailbox->h.flagname[flag]); p = memchr(name, ' ', eol-name); if (!p) p = eol; if (name != p) mailbox->h.flagname[flag] = xstrndup(name, p-name); name = p+1; } /* zero out the rest */ for (; flag < MAX_USER_FLAGS; flag++) { xzfree(mailbox->h.flagname[flag]); } /* Read ACL */ p = eol + 1; eol = memchr(p, '\n', sbuf.st_size - (p - base)); if (!eol) { r = IMAP_MAILBOX_BADFORMAT; goto done; } mailbox->h.acl = xstrndup(p, eol-p); done: if (base) map_free(&base, &len); return r; } /* set a new ACL - only dirty if changed */ EXPORTED void mailbox_set_acl(struct mailbox *mailbox, const char *acl) { if (!strcmpsafe(mailbox->h.acl, acl)) return; /* no change */ /* patch our mbentry copy: XXX: this really should be the other way * around that we update and then WRITE our entry! */ free(mailbox->mbentry->acl); mailbox->mbentry->acl = xstrdup(acl); /* update the copy in the header and mark the header dirty */ free(mailbox->h.acl); mailbox->h.acl = xstrdup(acl); mailbox->header_dirty = 1; } /* set a new QUOTAROOT - only dirty if changed */ EXPORTED void mailbox_set_quotaroot(struct mailbox *mailbox, const char *quotaroot) { if (mailbox->h.quotaroot) { if (quotaroot && !strcmp(mailbox->h.quotaroot, quotaroot)) return; /* no change */ xzfree(mailbox->h.quotaroot); } else { if (!quotaroot) return; /* no change */ } if (quotaroot) mailbox->h.quotaroot = xstrdup(quotaroot); /* either way, it's changed, so dirty */ mailbox->header_dirty = 1; } static int _too_many_flags(const char *flag, int num) { if (num < 100) return 0; if (num >= MAX_USER_FLAGS) return 1; int too_many = 1; /* check if this is a required user flag */ const char *val = config_getstring(IMAPOPT_MAILBOX_INITIAL_FLAGS); if (val) { strarray_t *flags = strarray_split(val, NULL, 0); // it's not too many if there's still space and it's an initial flag if (strarray_find_case(flags, flag, 0) >= 0) too_many = 0; strarray_free(flags); } return too_many; } /* find or create a user flag - dirty header if change needed. If 'create' * is 1, then only 100 flags may be created. If >1, then you can use all 128 */ EXPORTED int mailbox_user_flag(struct mailbox *mailbox, const char *flag, int *flagnum, int create) { int userflag; int emptyflag = -1; for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) { // keep track of the first empty slot if (!mailbox->h.flagname[userflag]) { if (emptyflag == -1) emptyflag = userflag; continue; } // name matches? We've found the flag if (!strcasecmp(flag, mailbox->h.flagname[userflag])) break; } if (userflag == MAX_USER_FLAGS) { if (!create) return IMAP_NOTFOUND; if (emptyflag == -1) return IMAP_USERFLAG_EXHAUSTED; /* stop imapd exhausting flags */ if (create == 1 && _too_many_flags(flag, emptyflag)) { xsyslog(LOG_ERR, "IOERROR: out of flags", "mailbox=<%s> flag=<%s>", mailbox_name(mailbox), flag); return IMAP_USERFLAG_EXHAUSTED; } /* need to be index locked to make flag changes */ if (!mailbox_index_islocked(mailbox, 1)) return IMAP_MAILBOX_LOCKED; if (!imparse_isatom(flag)) return IMAP_INVALID_IDENTIFIER; /* set the flag and mark the header dirty */ userflag = emptyflag; mailbox->h.flagname[emptyflag] = xstrdup(flag); mailbox->header_dirty = 1; } if (flagnum) *flagnum = userflag; return 0; } /* Remove a user flag from the mailbox, so that the slot can * be reused. Called from cyr_expire when we've made certain * that no record uses the flag anymore. */ EXPORTED int mailbox_remove_user_flag(struct mailbox *mailbox, int flagnum) { if (flagnum < 0 || flagnum >= MAX_USER_FLAGS) return IMAP_INTERNAL; /* invalid flag number */ if (!mailbox->h.flagname[flagnum]) return 0; /* already gone */ /* need to be index locked to make flag changes */ if (!mailbox_index_islocked(mailbox, 1)) return IMAP_MAILBOX_LOCKED; xzfree(mailbox->h.flagname[flagnum]); mailbox->header_dirty = 1; return 0; } EXPORTED int mailbox_record_hasflag(struct mailbox *mailbox, const struct index_record *record, const char *flag) { int userflag; if (!mailbox) return 0; if (!flag) return 0; if (!record) return 0; if (flag[0] == '\\') { if (!strcasecmp(flag, "\\answered")) return ((record->system_flags & FLAG_ANSWERED) ? 1 : 0); if (!strcasecmp(flag, "\\deleted")) return ((record->system_flags & FLAG_DELETED) ? 1 : 0); if (!strcasecmp(flag, "\\draft")) return ((record->system_flags & FLAG_DRAFT) ? 1 : 0); if (!strcasecmp(flag, "\\flagged")) return ((record->system_flags & FLAG_FLAGGED) ? 1 : 0); if (!strcasecmp(flag, "\\seen")) { /* NOTE: this is a special case because it depends * who the userid is. We will only return the user * or global seen value */ return ((record->system_flags & FLAG_SEEN) ? 1 : 0); } /* unknown system flag is never present */ return 0; } if (mailbox_user_flag(mailbox, flag, &userflag, 0)) return 0; return ((record->user_flags[userflag/32] & (1<<(userflag&31))) ? 1 : 0); } EXPORTED strarray_t *mailbox_extract_flags(const struct mailbox *mailbox, const struct index_record *record, const char *userid) { int i; strarray_t *flags = strarray_new(); /* Note: we don't handle the external seen db here, on * the grounds that it would add complexity without * actually being useful to annotators */ if (mailbox_internal_seen(mailbox, userid) && (record->system_flags & FLAG_SEEN)) strarray_append(flags, "\\Seen"); if ((record->system_flags & FLAG_DELETED)) strarray_append(flags, "\\Deleted"); if ((record->system_flags & FLAG_DRAFT)) strarray_append(flags, "\\Draft"); if ((record->system_flags & FLAG_FLAGGED)) strarray_append(flags, "\\Flagged"); if ((record->system_flags & FLAG_ANSWERED)) strarray_append(flags, "\\Answered"); for (i = 0 ; i < MAX_USER_FLAGS ; i++) { if (mailbox->h.flagname[i] && (record->user_flags[i/32] & 1<<(i&31))) strarray_append(flags, mailbox->h.flagname[i]); } return flags; } static int load_annot_cb(const char *mailbox __attribute__((unused)), uint32_t uid __attribute__((unused)), const char *entry, const char *userid, const struct buf *value, const struct annotate_metadata *mdata __attribute__((unused)), void *rock) { struct entryattlist **eal = (struct entryattlist **)rock; const char *attrib = (userid[0] ? "value.priv" : "value.shared"); setentryatt(eal, entry, attrib, value); return 0; } EXPORTED struct entryattlist *mailbox_extract_annots(const struct mailbox *mailbox, const struct index_record *record) { struct entryattlist *annots = NULL; int r = annotatemore_findall_mailbox(mailbox, record->uid, "*", /*modseq*/0, load_annot_cb, &annots, /*flags*/0); if (r) return NULL; return annots; } static int mailbox_buf_to_index_header(const char *buf, size_t len, struct index_header *i) { uint32_t crc; bit32 qannot; size_t headerlen; if (len < OFFSET_MINOR_VERSION+4) return IMAP_MAILBOX_BADFORMAT; memset(i, 0, sizeof(struct index_header)); i->generation_no = ntohl(*((bit32 *)(buf+OFFSET_GENERATION_NO))); i->format = ntohl(*((bit32 *)(buf+OFFSET_FORMAT))); i->minor_version = ntohl(*((bit32 *)(buf+OFFSET_MINOR_VERSION))); switch (i->minor_version) { case 6: case 7: headerlen = 76; break; case 8: headerlen = 92; break; case 9: case 10: headerlen = 96; break; case 12: case 13: headerlen = 128; break; case 14: case 15: case 16: case 17: headerlen = 160; break; default: return IMAP_MAILBOX_BADFORMAT; } if (len < headerlen) return IMAP_MAILBOX_BADFORMAT; i->start_offset = ntohl(*((bit32 *)(buf+OFFSET_START_OFFSET))); i->record_size = ntohl(*((bit32 *)(buf+OFFSET_RECORD_SIZE))); i->num_records = ntohl(*((bit32 *)(buf+OFFSET_NUM_RECORDS))); i->last_appenddate = ntohl(*((bit32 *)(buf+OFFSET_LAST_APPENDDATE))); i->last_uid = ntohl(*((bit32 *)(buf+OFFSET_LAST_UID))); i->quota_mailbox_used = align_ntohll(buf+OFFSET_QUOTA_MAILBOX_USED); i->pop3_last_login = ntohl(*((bit32 *)(buf+OFFSET_POP3_LAST_LOGIN))); i->uidvalidity = ntohl(*((bit32 *)(buf+OFFSET_UIDVALIDITY))); i->deleted = ntohl(*((bit32 *)(buf+OFFSET_DELETED))); i->answered = ntohl(*((bit32 *)(buf+OFFSET_ANSWERED))); i->flagged = ntohl(*((bit32 *)(buf+OFFSET_FLAGGED))); i->options = ntohl(*((bit32 *)(buf+OFFSET_MAILBOX_OPTIONS))); i->leaked_cache_records = ntohl(*((bit32 *)(buf+OFFSET_LEAKED_CACHE))); if (i->minor_version < 8) goto done; i->highestmodseq = align_ntohll(buf+OFFSET_HIGHESTMODSEQ); if (i->minor_version < 12) goto done; i->deletedmodseq = align_ntohll(buf+OFFSET_DELETEDMODSEQ); i->exists = ntohl(*((bit32 *)(buf+OFFSET_EXISTS))); i->first_expunged = ntohl(*((bit32 *)(buf+OFFSET_FIRST_EXPUNGED))); i->last_repack_time = ntohl(*((bit32 *)(buf+OFFSET_LAST_REPACK_TIME))); i->header_file_crc = ntohl(*((bit32 *)(buf+OFFSET_HEADER_FILE_CRC))); i->synccrcs.basic = ntohl(*((bit32 *)(buf+OFFSET_SYNCCRCS_BASIC))); i->recentuid = ntohl(*((bit32 *)(buf+OFFSET_RECENTUID))); i->recenttime = ntohl(*((bit32 *)(buf+OFFSET_RECENTTIME))); if (i->minor_version < 13) goto crc; i->pop3_show_after = ntohl(*((bit32 *)(buf+OFFSET_POP3_SHOW_AFTER))); qannot = ntohl(*((bit32 *)(buf+OFFSET_QUOTA_ANNOT_USED))); /* this field is stored as a 32b unsigned on disk but 64b signed * in memory, so we need to be careful about sign extension */ i->quota_annot_used = (quota_t)((unsigned long long)qannot); i->synccrcs.annot = ntohl(*((bit32 *)(buf+OFFSET_SYNCCRCS_ANNOT))); if (i->minor_version < 14) goto crc; i->unseen = ntohl(*((bit32 *)(buf+OFFSET_UNSEEN))); if (i->minor_version < 16) goto crc; i->createdmodseq = align_ntohll(buf+OFFSET_MAILBOX_CREATEDMODSEQ); if (i->minor_version < 17) goto crc; i->changes_epoch = ntohl(*((bit32 *)(buf+OFFSET_CHANGES_EPOCH))); crc: /* CRC is always the last 4 bytes */ crc = ntohl(*((bit32 *)(buf+headerlen-4))); if (crc != crc32_map(buf, headerlen-4)) return IMAP_MAILBOX_CHECKSUM; done: if (!i->exists) i->options |= OPT_POP3_NEW_UIDL; if (!i->highestmodseq) i->highestmodseq = 1; if (i->minor_version < 12) { i->deletedmodseq = i->highestmodseq; i->exists = i->num_records; } if (i->minor_version < 17) { i->changes_epoch = i->last_repack_time ? i->last_repack_time : time(0); } return 0; } static int mailbox_refresh_index_map(struct mailbox *mailbox) { size_t need_size; struct stat sbuf; /* check if we need to extend the mmaped space for the index file * (i.e. new records appended since last read) */ need_size = mailbox->i.start_offset + mailbox->i.num_records * mailbox->i.record_size; if (mailbox->index_size < need_size) { if (fstat(mailbox->index_fd, &sbuf) == -1) return IMAP_IOERROR; if (sbuf.st_size < (int)need_size) return IMAP_MAILBOX_BADFORMAT; mailbox->index_size = sbuf.st_size; } /* always refresh, we may be using map_nommap */ map_refresh(mailbox->index_fd, 1, &mailbox->index_base, &mailbox->index_len, mailbox->index_size, "index", mailbox_name(mailbox)); return 0; } static int mailbox_read_index_header(struct mailbox *mailbox) { int r; /* no dirty mailboxes please */ if (mailbox->i.dirty) abort(); /* need to be locked to ensure a consistent read - otherwise * a busy mailbox will get CRC errors due to rewrite happening * under our feet! */ if (!mailbox_index_islocked(mailbox, 0)) return IMAP_MAILBOX_LOCKED; /* and of course it needs to exist and have at least enough * header to read the version number */ if (!mailbox->index_base) return IMAP_MAILBOX_BADFORMAT; /* need to make sure we're reading fresh data! */ map_refresh(mailbox->index_fd, 1, &mailbox->index_base, &mailbox->index_len, mailbox->index_size, "index", mailbox_name(mailbox)); r = mailbox_buf_to_index_header(mailbox->index_base, mailbox->index_len, &mailbox->i); if (r) return r; r = mailbox_refresh_index_map(mailbox); if (r) return r; return 0; } /* * Read an index record from a mapped index file */ #ifdef HAVE_DECLARE_OPTIMIZE static int mailbox_buf_to_index_record(const char *buf, int version, struct index_record *record, int dirty) __attribute__((optimize("-O3"))); #endif static int mailbox_buf_to_index_record(const char *buf, int version, struct index_record *record, int dirty) { uint32_t crc; uint32_t stored_system_flags = 0; int n; /* tracking fields - initialise */ memset(record, 0, sizeof(struct index_record)); /* parse the shared bits first */ record->uid = ntohl(*((bit32 *)(buf+OFFSET_UID))); record->internaldate = ntohl(*((bit32 *)(buf+OFFSET_INTERNALDATE))); record->sentdate = ntohl(*((bit32 *)(buf+OFFSET_SENTDATE))); record->size = ntohl(*((bit32 *)(buf+OFFSET_SIZE))); record->header_size = ntohl(*((bit32 *)(buf+OFFSET_HEADER_SIZE))); record->gmtime = ntohl(*((bit32 *)(buf+OFFSET_GMTIME))); uint64_t cache_offset_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_OFFSET))); record->last_updated = ntohl(*((bit32 *)(buf+OFFSET_LAST_UPDATED))); stored_system_flags = ntohl(*((bit32 *)(buf+OFFSET_SYSTEM_FLAGS))); /* de-serialise system flags and internal flags */ record->system_flags = stored_system_flags & 0x0000ffff; record->internal_flags = stored_system_flags & 0xffff0000; for (n = 0; n < MAX_USER_FLAGS/32; n++) { record->user_flags[n] = ntohl(*((bit32 *)(buf+OFFSET_USER_FLAGS+4*n))); } uint64_t cache_version_field = ntohl(*((bit32 *)(buf+OFFSET_CACHE_VERSION))); /* keep the low bits of the version field for the version */ record->cache_version = cache_version_field & 0xffff; /* use the high bits of the version field to extend the cache offset */ record->cache_offset = cache_offset_field | ((cache_version_field & 0xffff0000) << 16); if (version < 8) return 0; if (version < 10) { /* modseq was at 72 before the GUID move */ record->modseq = ntohll(*((bit64 *)(buf+72))); return 0; } message_guid_import(&record->guid, buf+OFFSET_MESSAGE_GUID); record->modseq = ntohll(*((bit64 *)(buf+OFFSET_MODSEQ))); if (version < 12) return 0; /* THRID got inserted before cache_crc32 in version 12 */ if (version < 13) { record->cache_crc = ntohl(*((bit32 *)(buf+88))); if (dirty) return 0; /* check CRC32 */ crc = crc32_map(buf, 92); if (crc != ntohl(*((bit32 *)(buf+92)))) return IMAP_MAILBOX_CHECKSUM; return 0; } record->cid = ntohll(*(bit64 *)(buf+OFFSET_THRID)); if (version > 14) { record->savedate = ntohl(*((bit32 *)(buf+OFFSET_SAVEDATE))); } /* createdmodseq was added in version 16, pushing the CRCs down */ if (version < 16) { record->cache_crc = ntohl(*((bit32 *)(buf+96))); if (dirty) return 0; /* check CRC32 */ crc = crc32_map(buf, 100); if (crc != ntohl(*((bit32 *)(buf+100)))) return IMAP_MAILBOX_CHECKSUM; return 0; } record->createdmodseq = ntohll(*(bit64 *)(buf+OFFSET_CREATEDMODSEQ)); record->cache_crc = ntohl(*((bit32 *)(buf+OFFSET_CACHE_CRC))); if (dirty) return 0; /* check CRC32 */ crc = crc32_map(buf, OFFSET_RECORD_CRC); if (crc != ntohl(*((bit32 *)(buf+OFFSET_RECORD_CRC)))) return IMAP_MAILBOX_CHECKSUM; return 0; } static struct index_change *_find_change(struct mailbox *mailbox, uint32_t recno) { uint32_t changeno = mailbox->index_change_map[recno % INDEX_MAP_SIZE]; while (changeno) { if (mailbox->index_changes[changeno-1].record.recno == recno) return &mailbox->index_changes[changeno-1]; changeno = mailbox->index_changes[changeno-1].mapnext; } return NULL; } static int _store_change(struct mailbox *mailbox, struct index_record *record, int flags) { struct index_change *change = _find_change(mailbox, record->recno); if (!change) { mailbox->index_change_count++; /* allocate a space if required */ if (mailbox->index_change_count > mailbox->index_change_alloc) { mailbox->index_change_alloc += 256; mailbox->index_changes = xrealloc(mailbox->index_changes, sizeof(struct index_change) * mailbox->index_change_alloc); } change = &mailbox->index_changes[mailbox->index_change_count-1]; memset(change, 0, sizeof(struct index_change)); /* stitch into place */ uint32_t pos = record->recno % INDEX_MAP_SIZE; change->mapnext = mailbox->index_change_map[pos]; mailbox->index_change_map[pos] = mailbox->index_change_count; /* always non-zero */ } /* finally always copy the data into place */ change->record = *record; change->flags = flags; if (mailbox_cacherecord(mailbox, record)) { /* failed to load cache record */ free(change->msgid); change->msgid = xstrdup("unknown"); } else { char *c_env = xstrndup(cacheitem_base(record, CACHE_ENVELOPE) + 1, cacheitem_size(record, CACHE_ENVELOPE) - 2); char *envtokens[NUMENVTOKENS]; parse_cached_envelope(c_env, envtokens, NUMENVTOKENS); free(change->msgid); change->msgid = xstrdup(envtokens[ENV_MSGID] ? envtokens[ENV_MSGID] : "unknown"); free(c_env); } annotate_state_t *astate = NULL; int r = mailbox_get_annotate_state(mailbox, record->uid, &astate); if (r) return r; struct buf annotval = BUF_INITIALIZER; if (record->cid && record->basecid && record->basecid != record->cid) buf_printf(&annotval, "%016llx", record->basecid); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "basethrid", "", &annotval); buf_free(&annotval); if (r) return r; return 0; } static int _commit_one(struct mailbox *mailbox, struct index_change *change) { indexbuffer_t ibuf; unsigned char *buf = ibuf.buf; size_t offset; struct index_record *record = &change->record; uint32_t recno = record->recno; mailbox_index_record_to_buf(&change->record, mailbox->i.minor_version, buf); offset = mailbox->i.start_offset + ((recno-1) * mailbox->i.record_size); /* any failure here is a disaster! */ if (lseek(mailbox->index_fd, offset, SEEK_SET) == -1) { xsyslog(LOG_ERR, "IOERROR: seeking index record failed", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), recno); return IMAP_IOERROR; } if (retry_write(mailbox->index_fd, buf, mailbox->i.record_size) != mailbox->i.record_size) { xsyslog(LOG_ERR, "IOERROR: writing index record failed", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), recno); return IMAP_IOERROR; } /* audit logging */ if (config_auditlog) { char flagstr[FLAGMAPSTR_MAXLEN]; flags_to_str(record, flagstr); if (change->flags & CHANGE_ISAPPEND) /* note: messageid doesn't have <> wrappers because it already includes them */ syslog(LOG_NOTICE, "auditlog: append sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> " "sysflags=<%s> guid=<%s> messageid=%s size=<%u>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, record->modseq, flagstr, message_guid_encode(&record->guid), change->msgid, record->size); if ((record->internal_flags & FLAG_INTERNAL_EXPUNGED) && !(change->flags & CHANGE_WASEXPUNGED)) syslog(LOG_NOTICE, "auditlog: expunge sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> " "sysflags=<%s> guid=<%s> size=<%u>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, record->modseq, flagstr, message_guid_encode(&record->guid), record->size); if ((record->internal_flags & FLAG_INTERNAL_UNLINKED) && !(change->flags & CHANGE_WASUNLINKED)) syslog(LOG_NOTICE, "auditlog: unlink sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uid=<%u> modseq=<%llu> " "sysflags=<%s> guid=<%s>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, record->modseq, flagstr, message_guid_encode(&record->guid)); } return 0; } static void _cleanup_changes(struct mailbox *mailbox) { uint32_t i; for (i = 0; i < mailbox->index_change_count; i++) { free(mailbox->index_changes[i].msgid); } free(mailbox->index_changes); mailbox->index_changes = NULL; mailbox->index_change_count = 0; mailbox->index_change_alloc = 0; memset(mailbox->index_change_map, 0, sizeof(uint32_t)*INDEX_MAP_SIZE); } /* qsort function for changes */ static int change_compar(const void *a, const void *b) { struct index_change *ac = (struct index_change *)a; struct index_change *bc = (struct index_change *)b; if (ac->record.recno > bc->record.recno) return 1; if (ac->record.recno < bc->record.recno) return -1; return 0; } static int _commit_changes(struct mailbox *mailbox) { uint32_t i; int r; if (!mailbox->index_change_count) return 0; mailbox->i.dirty = 1; /* in which we throw away all our next pointers, but we don't care any more. * we just want to write in sensible order. Otherwise, there's no need to * do this sort at all */ qsort(mailbox->index_changes, mailbox->index_change_count, sizeof(struct index_change), change_compar); for (i = 0; i < mailbox->index_change_count; i++) { r = _commit_one(mailbox, &mailbox->index_changes[i]); if (r) return r; /* DAMN, we're screwed */ } _cleanup_changes(mailbox); /* recalculate the size */ mailbox->index_size = mailbox->i.start_offset + (mailbox->i.num_records * mailbox->i.record_size); r = mailbox_refresh_index_map(mailbox); if (r) return r; return 0; } EXPORTED int mailbox_reload_index_record_dirty(struct mailbox *mailbox, struct index_record *record) { unsigned recno = record->recno; unsigned offset = mailbox->i.start_offset + (recno-1) * mailbox->i.record_size; if (offset + mailbox->i.record_size > mailbox->index_size) { xsyslog(LOG_ERR, "IOERROR: index record past end of file", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), recno); return IMAP_IOERROR; } const char *buf = mailbox->index_base + offset; mailbox_buf_to_index_record(buf, mailbox->i.minor_version, record, 1); record->recno = recno; return 0; } /* * Read an index record from a mailbox */ static int mailbox_read_index_record(struct mailbox *mailbox, uint32_t recno, struct index_record *record) { const char *buf; unsigned offset; int r; struct index_change *change = _find_change(mailbox, recno); if (change) { *record = change->record; return 0; } offset = mailbox->i.start_offset + (recno-1) * mailbox->i.record_size; if (offset + mailbox->i.record_size > mailbox->index_size) { xsyslog(LOG_ERR, "IOERROR: index record past end of file", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), recno); return IMAP_IOERROR; } buf = mailbox->index_base + offset; r = mailbox_buf_to_index_record(buf, mailbox->i.minor_version, record, 0); record->recno = recno; return r; } EXPORTED int mailbox_read_basecid(struct mailbox *mailbox, const struct index_record *record) { if (record->basecid) return 0; if (record->internal_flags & FLAG_INTERNAL_SPLITCONVERSATION) { struct buf annotval = BUF_INITIALIZER; mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "basethrid", "", &annotval); if (annotval.len == 16) { const char *p = buf_cstring(&annotval); /* we have a new canonical CID */ struct index_record *backdoor = (struct index_record *)record; parsehex(p, &p, 16, &backdoor->basecid); } buf_free(&annotval); } return 0; } EXPORTED int mailbox_has_conversations_full(struct mailbox *mailbox, int allow_deleted) { char *path; /* not needed */ if (!config_getswitch(IMAPOPT_CONVERSATIONS)) return 0; /* we never store data about deleted mailboxes */ if (!allow_deleted && mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) return 0; /* we never store data about submission mailboxes */ if (mboxname_issubmissionmailbox(mailbox_name(mailbox), mailbox_mbtype(mailbox))) return 0; path = conversations_getmboxpath(mailbox_name(mailbox)); if (!path) return 0; free(path); return 1; } #ifdef WITH_DAV HIDDEN struct caldav_db *mailbox_open_caldav(struct mailbox *mailbox) { if (!mailbox->local_caldav) { mailbox->local_caldav = caldav_open_mailbox(mailbox); int r = caldav_begin(mailbox->local_caldav); if (r) { caldav_abort(mailbox->local_caldav); caldav_close(mailbox->local_caldav); mailbox->local_caldav = NULL; } } return mailbox->local_caldav; } HIDDEN struct carddav_db *mailbox_open_carddav(struct mailbox *mailbox) { if (!mailbox->local_carddav) { mailbox->local_carddav = carddav_open_mailbox(mailbox); int r = carddav_begin(mailbox->local_carddav); if (r) { carddav_abort(mailbox->local_carddav); carddav_close(mailbox->local_carddav); mailbox->local_carddav = NULL; } } return mailbox->local_carddav; } static struct webdav_db *mailbox_open_webdav(struct mailbox *mailbox) { if (!mailbox->local_webdav) { mailbox->local_webdav = webdav_open_mailbox(mailbox); int r = webdav_begin(mailbox->local_webdav); if (r) { webdav_abort(mailbox->local_webdav); webdav_close(mailbox->local_webdav); mailbox->local_webdav = NULL; } } return mailbox->local_webdav; } #endif static uint32_t mailbox_getuid(struct mailbox *mailbox, uint32_t recno) { struct index_record record; record.uid = 0; /* XXX - cheaper memory-access reads? */ mailbox_read_index_record(mailbox, recno, &record); return record.uid; } /* * Returns the recno of the message with UID 'uid'. * If no message with UID 'uid', returns the message with * the highest UID not greater than 'uid'. */ static uint32_t mailbox_finduid(struct mailbox *mailbox, uint32_t uid) { uint32_t low = 1; uint32_t high = mailbox->i.num_records; uint32_t mid; uint32_t miduid; while (low <= high) { mid = (high - low)/2 + low; miduid = mailbox_getuid(mailbox, mid); if (miduid == uid) return mid; else if (miduid > uid) high = mid - 1; else low = mid + 1; } return high; } /* * Perform a binary search on the mailbox index file to read the record * for uid 'uid' into 'record'. If 'oldrecord' is not NULL then it is * assumed to point a correct and current index record from an earlier * call, and the search is bounded by that record. Returns 0 on success * or an IMAP error code on failure. */ EXPORTED int mailbox_find_index_record(struct mailbox *mailbox, uint32_t uid, struct index_record *record) { uint32_t recno = mailbox_finduid(mailbox, uid); /* no records? */ if (!recno) return IMAP_NOTFOUND; int r = mailbox_read_index_record(mailbox, recno, record); /* failed read? */ if (r) return r; /* wasn't the actual record? */ if (record->uid != uid) return IMAP_NOTFOUND; return 0; } /* * Lock the index file for 'mailbox'. Reread index file header if necessary. */ static int mailbox_lock_index_internal(struct mailbox *mailbox, int locktype) { struct stat sbuf; int r = 0; const char *header_fname = mailbox_meta_fname(mailbox, META_HEADER); const char *index_fname = mailbox_meta_fname(mailbox, META_INDEX); assert(mailbox->index_fd != -1); assert(!mailbox->index_locktype); char *userid = mboxname_to_userid(mailbox_name(mailbox)); if (userid) { if (!user_isnamespacelocked(userid)) { struct mailboxlist *listitem = find_listitem(mailbox_name(mailbox)); assert(listitem); assert(&listitem->m == mailbox); r = mailbox_mboxlock_reopen(listitem, LOCK_SHARED, locktype); if (locktype == LOCK_SHARED) mailbox->is_readonly = 1; if (!r) r = mailbox_open_index(mailbox); } free(userid); if (r) return r; } if (locktype == LOCK_EXCLUSIVE) { /* handle read-only case cleanly - we need to re-open read-write first! */ if (mailbox->is_readonly) { mailbox->is_readonly = 0; r = mailbox_open_index(mailbox); } if (!r) r = lock_blocking(mailbox->index_fd, index_fname); } else if (locktype == LOCK_SHARED) { r = lock_shared(mailbox->index_fd, index_fname); } else { /* this function does not support nonblocking locks */ fatal("invalid locktype for index", EX_SOFTWARE); } /* double check that the index exists and has at least enough * data to check the version number */ if (!r) { if (!mailbox->index_base) r = IMAP_MAILBOX_BADFORMAT; else if (mailbox->index_size < OFFSET_NUM_RECORDS) r = IMAP_MAILBOX_BADFORMAT; if (r) lock_unlock(mailbox->index_fd, index_fname); } if (r) { xsyslog(LOG_ERR, "IOERROR: lock index failed", "mailbox=<%s> error=<%s>", mailbox_name(mailbox), error_message(r)); return IMAP_IOERROR; } mailbox->index_locktype = locktype; gettimeofday(&mailbox->starttime, 0); r = stat(header_fname, &sbuf); if (r == -1) { xsyslog(LOG_ERR, "IOERROR: stat header failed", "mailbox=<%s> header=<%s>", mailbox_name(mailbox), header_fname); mailbox_unlock_index(mailbox, NULL); return IMAP_IOERROR; } /* has the header file changed? */ if (sbuf.st_ino != mailbox->header_file_ino) { r = mailbox_read_header(mailbox, NULL); if (r) { xsyslog(LOG_ERR, "IOERROR: read header failed", "mailbox=<%s> error=<%s>", mailbox_name(mailbox), error_message(r)); mailbox_unlock_index(mailbox, NULL); return r; } } /* release caches */ int i; for (i = 0; i < mailbox->caches.count; i++) { struct mappedfile *cachefile = ptrarray_nth(&mailbox->caches, i); mappedfile_close(&cachefile); } ptrarray_fini(&mailbox->caches); /* note: it's guaranteed by our outer cyrus.lock lock that the * cyrus.index and cyrus.cache files are never rewritten, so * we're safe to just extend the map if needed */ r = mailbox_read_index_header(mailbox); if (r) { xsyslog(LOG_ERR, "IOERROR: refreshing index failed", "mailbox=<%s> error=<%s>", mailbox_name(mailbox), error_message(r)); mailbox_unlock_index(mailbox, NULL); return r; } /* check the CRC */ if (mailbox->header_file_crc && mailbox->i.header_file_crc && mailbox->header_file_crc != mailbox->i.header_file_crc) { syslog(LOG_WARNING, "Header CRC mismatch for mailbox %s: %08X %08X", mailbox_name(mailbox), (unsigned int)mailbox->header_file_crc, (unsigned int)mailbox->i.header_file_crc); } return 0; } EXPORTED int mailbox_lock_index(struct mailbox *mailbox, int locktype) { int r = 0; r = mailbox_lock_index_internal(mailbox, locktype); if (r) return r; /* otherwise, sanity checks for regular use, but not for internal * use during cleanup */ /* we may be in the process of deleting this mailbox */ if (mailbox->i.options & OPT_MAILBOX_DELETED) { mailbox_unlock_index(mailbox, NULL); return IMAP_MAILBOX_NONEXISTENT; } return 0; } /* * Release lock on the index file for 'mailbox' */ EXPORTED void mailbox_unlock_index(struct mailbox *mailbox, struct statusdata *sdata) { struct statusdata mysdata = STATUSDATA_INIT; struct timeval endtime; double timediff; int r; const char *index_fname = mailbox_meta_fname(mailbox, META_INDEX); /* this is kinda awful, but too much code expects it to work, and the * refcounting isn't good about partial commit/abort and all the * unwinding, so here you are. At least if you mailbox_abort, then * it resets the dirty flags, so this becomes a NOOP during close */ /* naughty - you can't unlock a dirty mailbox! */ r = mailbox_commit(mailbox); if (r) { xsyslog(LOG_ERR, "IOERROR: failed to commit mailbox, " "probably need to reconstruct", "mailbox=<%s>", mailbox_name(mailbox)); abort(); } if (mailbox->has_changed) { if (updatenotifier) updatenotifier(mailbox_name(mailbox)); sync_log_mailbox(mailbox_name(mailbox)); if (!sdata) { status_fill_mailbox(mailbox, &mysdata); sdata = &mysdata; } mailbox->has_changed = 0; } // we always write if given new statusdata, or if we changed the mailbox if (sdata) statuscache_invalidate(mailbox_name(mailbox), sdata); if (mailbox->index_locktype) { if (lock_unlock(mailbox->index_fd, index_fname)) xsyslog(LOG_ERR, "IOERROR: unlocking index failed", "mailbox=<%s>", mailbox_name(mailbox)); mailbox->index_locktype = 0; gettimeofday(&endtime, 0); timediff = timesub(&mailbox->starttime, &endtime); if (timediff > 1.0) { syslog(LOG_NOTICE, "mailbox: longlock %s for %0.1f seconds", mailbox_name(mailbox), timediff); } } if (mailbox->local_cstate) { int r = conversations_commit(&mailbox->local_cstate); if (r) syslog(LOG_ERR, "Error committing to conversations database for mailbox %s: %s", mailbox_name(mailbox), error_message(r)); } // release the namespacelock here if (mailbox->local_namespacelock) { mboxname_release(&mailbox->local_namespacelock); } } static char *mailbox_header_data_cstring(struct mailbox *mailbox) { struct buf buf = BUF_INITIALIZER; struct dlist *dl = dlist_newkvlist(NULL, NULL); int nflags = 0; int flag; dlist_setatom(dl, "T", mboxlist_mbtype_to_string(mailbox->h.mbtype)); dlist_setatom(dl, "N", mailbox->h.name); dlist_setatom(dl, "I", mailbox->h.uniqueid); if (mailbox->h.quotaroot) dlist_setatom(dl, "Q", mailbox->h.quotaroot); if (mailbox->h.acl) dlist_stitch(dl, mailbox_acl_to_dlist(mailbox->h.acl)); for (flag = 0; flag < MAX_USER_FLAGS; flag++) { if (mailbox->h.flagname[flag]) nflags = flag+1; } if (nflags) { struct dlist *fl = dlist_newlist(dl, "U"); // this could be inserting NIL values for removed flags for (flag = 0; flag < nflags; flag++) { dlist_setatom(fl, NULL, mailbox->h.flagname[flag]); } } dlist_printbuf(dl, 0, &buf); dlist_free(&dl); return buf_release(&buf); } /* * Write the header file for 'mailbox' */ static int mailbox_commit_header(struct mailbox *mailbox) { int fd; int r = 0; const char *newfname; struct iovec iov[10]; int niov; if (!mailbox->header_dirty) return 0; /* nothing to write! */ /* we actually do all header actions under an INDEX lock, because * we need to write the crc32 to be consistent! */ assert(mailbox_index_islocked(mailbox, 1)); newfname = mailbox_meta_newfname(mailbox, META_HEADER); fd = open(newfname, O_CREAT | O_TRUNC | O_RDWR, 0666); if (fd == -1) { xsyslog(LOG_ERR, "IOERROR: open failed", "newfname=<%s>", newfname); return IMAP_IOERROR; } /* Write magic header, do NOT write the trailing NUL */ r = write(fd, MAILBOX_HEADER_MAGIC, sizeof(MAILBOX_HEADER_MAGIC) - 1); if (r != -1) { char *data = mailbox_header_data_cstring(mailbox); niov = 0; WRITEV_ADDSTR_TO_IOVEC(iov, niov, data); WRITEV_ADD_TO_IOVEC(iov, niov, "\n", 1); r = retry_writev(fd, iov, niov); free(data); } if (r == -1 || fsync(fd)) { xsyslog(LOG_ERR, "IOERROR: write failed", "newfname=<%s>", newfname); close(fd); unlink(newfname); return IMAP_IOERROR; } close(fd); /* rename the new header file over the old one */ r = mailbox_meta_rename(mailbox, META_HEADER); if (r) return r; mailbox->header_dirty = 0; /* we wrote it out, so not dirty any more */ /* re-read the header */ r = mailbox_read_header(mailbox, NULL); if (r) return r; /* copy the new CRC into the index header */ mailbox->i.header_file_crc = mailbox->header_file_crc; mailbox_index_dirty(mailbox); return 0; } static bit32 mailbox_index_header_to_buf(struct index_header *i, unsigned char *buf) { bit32 crc; bit32 options = i->options & MAILBOX_OPT_VALID; size_t headerlen = INDEX_HEADER_SIZE; memset(buf, 0, INDEX_HEADER_SIZE); /* buffer is always this big, and aligned */ assert (i->minor_version >= 6); *((bit32 *)(buf+OFFSET_GENERATION_NO)) = htonl(i->generation_no); *((bit32 *)(buf+OFFSET_FORMAT)) = htonl(i->format); *((bit32 *)(buf+OFFSET_MINOR_VERSION)) = htonl(i->minor_version); *((bit32 *)(buf+OFFSET_START_OFFSET)) = htonl(i->start_offset); *((bit32 *)(buf+OFFSET_RECORD_SIZE)) = htonl(i->record_size); /* this was moved to make upgrades clean, because num_records was * the same as exists back then, we didn't keep expunged in the * record - but we always have to write NUM_RECORDS so that expunged * handing over repack works */ *((bit32 *)(buf+OFFSET_NUM_RECORDS)) = htonl(i->num_records); *((bit32 *)(buf+OFFSET_LAST_APPENDDATE)) = htonl(i->last_appenddate); *((bit32 *)(buf+OFFSET_LAST_UID)) = htonl(i->last_uid); /* quotas may be 64bit now */ align_htonll(buf+OFFSET_QUOTA_MAILBOX_USED, i->quota_mailbox_used); *((bit32 *)(buf+OFFSET_POP3_LAST_LOGIN)) = htonl(i->pop3_last_login); *((bit32 *)(buf+OFFSET_UIDVALIDITY)) = htonl(i->uidvalidity); *((bit32 *)(buf+OFFSET_DELETED)) = htonl(i->deleted); *((bit32 *)(buf+OFFSET_ANSWERED)) = htonl(i->answered); *((bit32 *)(buf+OFFSET_FLAGGED)) = htonl(i->flagged); if (i->minor_version < 8) { /* this was called OFFSET_POP3_NEW_UIDL and was only zero or one */ *((bit32 *)(buf+OFFSET_MAILBOX_OPTIONS)) = htonl(options&1); return 0; /* no CRC32 support */ } /* otherwise we have options and modseqs */ *((bit32 *)(buf+OFFSET_MAILBOX_OPTIONS)) = htonl(options); *((bit32 *)(buf+OFFSET_LEAKED_CACHE)) = htonl(i->leaked_cache_records); align_htonll(buf+OFFSET_HIGHESTMODSEQ, i->highestmodseq); /* and that's where it stopped until version 2.4.0 with index version 12 (ignoring * version 11, which doesn't exist in the wild */ if (i->minor_version < 12) { return 0; } align_htonll(buf+OFFSET_DELETEDMODSEQ, i->deletedmodseq); *((bit32 *)(buf+OFFSET_EXISTS)) = htonl(i->exists); *((bit32 *)(buf+OFFSET_FIRST_EXPUNGED)) = htonl(i->first_expunged); *((bit32 *)(buf+OFFSET_LAST_REPACK_TIME)) = htonl(i->last_repack_time); *((bit32 *)(buf+OFFSET_HEADER_FILE_CRC)) = htonl(i->header_file_crc); *((bit32 *)(buf+OFFSET_SYNCCRCS_BASIC)) = htonl(i->synccrcs.basic); *((bit32 *)(buf+OFFSET_RECENTUID)) = htonl(i->recentuid); *((bit32 *)(buf+OFFSET_RECENTTIME)) = htonl(i->recenttime); if (i->minor_version > 12) { /* these were added in version 13, but replaced zero-byte fields in * in version 12, so if we don't write them then the CRC will still * be correct for version 12, since the header size didn't change */ *((bit32 *)(buf+OFFSET_POP3_SHOW_AFTER)) = htonl(i->pop3_show_after); /* this field is 64b in memory but 32b on disk - as it counts * bytes stored in dbs and the dbs are 32b anyway there should * be no problem */ *((bit32 *)(buf+OFFSET_QUOTA_ANNOT_USED)) = htonl((bit32)i->quota_annot_used); *((bit32 *)(buf+OFFSET_SYNCCRCS_ANNOT)) = htonl(i->synccrcs.annot); } if (i->minor_version > 13) { *((bit32 *)(buf+OFFSET_UNSEEN)) = htonl(i->unseen); } else { /* we expanded the file size in version 14 */ headerlen = 128; } if (i->minor_version > 15) { align_htonll(buf+OFFSET_MAILBOX_CREATEDMODSEQ, i->createdmodseq); } if (i->minor_version > 16) { *((bit32 *)(buf+OFFSET_CHANGES_EPOCH)) = htonl(i->changes_epoch); } /* Update checksum */ crc = htonl(crc32_map((char *)buf, headerlen-4)); *((bit32 *)(buf+headerlen-4)) = crc; return crc; } HIDDEN int mailbox_commit_quota(struct mailbox *mailbox) { int res; int changed = 0; quota_t quota_usage[QUOTA_NUMRESOURCES]; /* not dirty */ if (!mailbox->quota_dirty) return 0; mailbox->quota_dirty = 0; /* no quota root means we don't track quota. That's OK */ if (!mailbox_quotaroot(mailbox)) return 0; mailbox_get_usage(mailbox, quota_usage); for (res = 0; res < QUOTA_NUMRESOURCES; res++) { quota_usage[res] -= mailbox->quota_previously_used[res]; if (quota_usage[res] != 0) { changed++; } } /* unchanged */ if (!changed) return 0; assert(mailbox_index_islocked(mailbox, 1)); quota_update_useds(mailbox_quotaroot(mailbox), quota_usage, mailbox_name(mailbox), mailbox->silentchanges); /* XXX - fail upon issue? It's tempting */ return 0; } /* * Abort the changes to a mailbox */ EXPORTED int mailbox_abort(struct mailbox *mailbox) { int r; #ifdef WITH_DAV r = mailbox_abort_dav(mailbox); if (r) return r; #endif #ifdef USE_SIEVE r = mailbox_abort_sieve(mailbox); if (r) return r; #endif /* try to commit sub parts first */ r = mailbox_abort_cache(mailbox); if (r) return r; annotate_state_abort(&mailbox->annot_state); if (mailbox->local_cstate) conversations_abort(&mailbox->local_cstate); if (!mailbox->i.dirty) return 0; assert(mailbox_index_islocked(mailbox, 1)); /* remove all dirty flags! */ mailbox->i.dirty = 0; mailbox->modseq_dirty = 0; mailbox->header_dirty = 0; /* removed cached changes */ _cleanup_changes(mailbox); /* we re-read the header and index header to wipe * away all the changed values */ r = mailbox_read_header(mailbox, NULL); if (r) return r; r = mailbox_read_index_header(mailbox); if (r) return r; return 0; } /* * Write the index header for 'mailbox' */ EXPORTED int mailbox_commit(struct mailbox *mailbox) { /* XXX - ibuf for alignment? */ static unsigned char buf[INDEX_HEADER_SIZE]; int n, r; /* try to commit sub parts first */ #ifdef WITH_DAV r = mailbox_commit_dav(mailbox); if (r) return r; #endif #ifdef USE_SIEVE r = mailbox_commit_sieve(mailbox); if (r) return r; #endif r = mailbox_commit_cache(mailbox); if (r) return r; r = mailbox_commit_quota(mailbox); if (r) return r; r = annotate_state_commit(&mailbox->annot_state); if (r) return r; r = mailbox_commit_header(mailbox); if (r) return r; if (!mailbox->i.dirty) return 0; if (!mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) { mboxname_setmodseq(mailbox_name(mailbox), mailbox->i.highestmodseq, mailbox_mbtype(mailbox), /*flags*/0); } assert(mailbox_index_islocked(mailbox, 1)); r = _commit_changes(mailbox); if (r) return r; /* always update xconvmodseq, it might have been done by annotations */ r = mailbox_update_xconvmodseq(mailbox, mailbox->i.highestmodseq, /*force*/0); if (r) return r; mailbox_index_header_to_buf(&mailbox->i, buf); lseek(mailbox->index_fd, 0, SEEK_SET); n = retry_write(mailbox->index_fd, buf, mailbox->i.start_offset); if (n < 0 || fsync(mailbox->index_fd)) { xsyslog(LOG_ERR, "IOERROR: writing index header failed", "mailbox=<%s>", mailbox_name(mailbox)); return IMAP_IOERROR; } if (config_auditlog && mailbox->modseq_dirty) syslog(LOG_NOTICE, "auditlog: modseq sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> highestmodseq=<" MODSEQ_FMT "> deletedmodseq=<" MODSEQ_FMT ">", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), mailbox->i.highestmodseq, mailbox->i.deletedmodseq); if (mailbox->modseq_dirty) { struct mboxevent *mboxevent = mboxevent_new(EVENT_MAILBOX_MODSEQ); mboxevent_extract_mailbox(mboxevent, mailbox); mboxevent_set_access(mboxevent, NULL, NULL, "", mailbox_name(mailbox), 0); mboxevent_notify(&mboxevent); mboxevent_free(&mboxevent); } /* remove all dirty flags! */ mailbox->i.dirty = 0; mailbox->modseq_dirty = 0; mailbox->header_dirty = 0; /* label changes for later logging */ mailbox->has_changed = 1; return 0; } /* * Put an index record into a buffer suitable for writing to a file. */ static bit32 mailbox_index_record_to_buf(struct index_record *record, int version, unsigned char *buf) { int n; bit32 crc; uint32_t system_flags = 0; memset(buf, 0, INDEX_RECORD_SIZE); /* keep the low bits of the offset in the offset field */ uint32_t cache_offset_field = record->cache_offset & 0xffffffff; /* mix in the high bits of the offset to the top half of the version field */ uint32_t cache_version_field = (uint32_t)record->cache_version | (record->cache_offset & 0xffff00000000) >> 16; *((bit32 *)(buf+OFFSET_UID)) = htonl(record->uid); *((bit32 *)(buf+OFFSET_INTERNALDATE)) = htonl(record->internaldate); *((bit32 *)(buf+OFFSET_SENTDATE)) = htonl(record->sentdate); *((bit32 *)(buf+OFFSET_SIZE)) = htonl(record->size); *((bit32 *)(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size); if (version >= 12) { *((bit32 *)(buf+OFFSET_GMTIME)) = htonl(record->gmtime); } else { /* content_offset was always the same */ *((bit32 *)(buf+OFFSET_GMTIME)) = htonl(record->header_size); } *((bit32 *)(buf+OFFSET_CACHE_OFFSET)) = htonl(cache_offset_field); *((bit32 *)(buf+OFFSET_LAST_UPDATED)) = htonl(record->last_updated); /* serialise system flags and internal flags */ system_flags = record->system_flags | record->internal_flags; *((bit32 *)(buf+OFFSET_SYSTEM_FLAGS)) = htonl(system_flags); for (n = 0; n < MAX_USER_FLAGS/32; n++) { *((bit32 *)(buf+OFFSET_USER_FLAGS+4*n)) = htonl(record->user_flags[n]); } if (version > 14) { *((bit32 *)(buf+OFFSET_SAVEDATE)) = htonl(record->savedate); } else { *((bit32 *)(buf+OFFSET_SAVEDATE)) = 0; // blank out old content_lines } *((bit32 *)(buf+OFFSET_CACHE_VERSION)) = htonl(cache_version_field); /* versions less than 8 had no modseq */ if (version < 8) { return 0; } /* versions 8 and 9 only had a smaller UUID, which we will ignore, * but the modseq existed and was at offset 72 and 76 */ if (version < 10) { *((bit64 *)(buf+72)) = htonll(record->modseq); return 0; } /* otherwise we have the GUID and MODSEQ in their current place */ message_guid_export(&record->guid, (char *)buf+OFFSET_MESSAGE_GUID); *((bit64 *)(buf+OFFSET_MODSEQ)) = htonll(record->modseq); /* version 12 added the CACHE_CRC and RECORD_CRC, but at a lower point */ if (version < 13) { *((bit32 *)(buf+88)) = htonl(record->cache_crc); /* calculate the checksum */ crc = crc32_map((char *)buf, 92); *((bit32 *)(buf+92)) = htonl(crc); return crc; } *((bit64 *)(buf+OFFSET_THRID)) = htonll(record->cid); /* version 16 added createdmodseq, pushing the CRCs down */ if (version < 16) { *((bit32 *)(buf+96)) = htonl(record->cache_crc); crc = crc32_map((char *)buf, 100); *((bit32 *)(buf+100)) = htonl(crc); return crc; } *((bit64 *)(buf+OFFSET_CREATEDMODSEQ)) = htonll(record->createdmodseq); *((bit32 *)(buf+OFFSET_CACHE_CRC)) = htonl(record->cache_crc); /* calculate the checksum */ crc = crc32_map((char *)buf, OFFSET_RECORD_CRC); *((bit32 *)(buf+OFFSET_RECORD_CRC)) = htonl(crc); return crc; } static void mailbox_quota_dirty(struct mailbox *mailbox) { /* track quota use */ if (!mailbox->quota_dirty) { mailbox->quota_dirty = 1; mailbox_get_usage(mailbox, mailbox->quota_previously_used); } } static void header_update_counts(struct index_header *i, const struct index_record *record, int is_add) { int num = is_add ? 1 : -1; /* we don't track counts for EXPUNGED records */ if (record->internal_flags & FLAG_INTERNAL_EXPUNGED) return; /* update mailbox header fields */ if (record->system_flags & FLAG_ANSWERED) i->answered += num; if (record->system_flags & FLAG_FLAGGED) i->flagged += num; if (record->system_flags & FLAG_DELETED) i->deleted += num; if (!(record->system_flags & FLAG_SEEN)) i->unseen += num; if (is_add) { i->exists++; i->quota_mailbox_used += record->size; } else { if (i->exists) i->exists--; /* corruption prevention - check we don't go negative */ if (i->quota_mailbox_used > record->size) i->quota_mailbox_used -= record->size; else i->quota_mailbox_used = 0; } } /*************************** Sync CRC ***************************/ struct annot_calc_rock { struct mailbox *mailbox; uint32_t annot; quota_t used; }; /* this is the algorithm from version 2.4, it's locked in */ static uint32_t crc_basic(const struct mailbox *mailbox, const struct index_record *record) { char buf[4096]; uint32_t flagcrc = 0; int flag; /* expunged flags have no sync CRC */ if (record->internal_flags & FLAG_INTERNAL_EXPUNGED) return 0; /* calculate an XORed CRC32 over all the flags on the message, so no * matter what order they are store in the header, the final value * is the same */ if (record->system_flags & FLAG_DELETED) flagcrc ^= crc32_cstring("\\deleted"); if (record->system_flags & FLAG_ANSWERED) flagcrc ^= crc32_cstring("\\answered"); if (record->system_flags & FLAG_FLAGGED) flagcrc ^= crc32_cstring("\\flagged"); if (record->system_flags & FLAG_DRAFT) flagcrc ^= crc32_cstring("\\draft"); if (record->system_flags & FLAG_SEEN) flagcrc ^= crc32_cstring("\\seen"); for (flag = 0; flag < MAX_USER_FLAGS; flag++) { if (!mailbox->h.flagname[flag]) continue; if (!(record->user_flags[flag/32] & (1<<(flag&31)))) continue; /* need to compare without case being significant */ strlcpy(buf, mailbox->h.flagname[flag], 4096); lcase(buf); flagcrc ^= crc32_cstring(buf); } snprintf(buf, sizeof(buf), "%u " MODSEQ_FMT " " TIME_T_FMT " (%u) " TIME_T_FMT " %s", record->uid, record->modseq, record->last_updated, flagcrc, record->internaldate, message_guid_encode(&record->guid)); return crc32_cstring(buf); } static uint32_t crc_annot(unsigned int uid, const char *entry, const char *userid, const struct buf *value) { struct buf buf = BUF_INITIALIZER; uint32_t res = 0; // ignore everything with a NULL userid, it's bogus! if (!userid) return 0; buf_printf(&buf, "%u %s %s ", uid, entry, userid); buf_append(&buf, value); res = crc32_buf(&buf); buf_free(&buf); return res; } static int mailbox_is_virtannot(struct mailbox *mailbox, const char *entry) { if (mailbox->i.minor_version < 13) return 0; // thrid was introduced in v13 if (!strcmp(entry, IMAP_ANNOT_NS "thrid")) return 1; if (mailbox->i.minor_version < 15) return 0; // savedate was introduced in v15 if (!strcmp(entry, IMAP_ANNOT_NS "savedate")) return 1; if (mailbox->i.minor_version < 16) return 0; // createdmodseq was introduced in v16 if (!strcmp(entry, IMAP_ANNOT_NS "createdmodseq")) return 1; return 0; } static uint32_t crc_virtannot(struct mailbox *mailbox, const struct index_record *record) { if (record->internal_flags & FLAG_INTERNAL_EXPUNGED) return 0; uint32_t crc = 0; struct buf buf = BUF_INITIALIZER; if (record->cid && mailbox->i.minor_version >= 13) { buf_printf(&buf, "%llx", record->cid); crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "thrid", "", &buf); buf_reset(&buf); } if (record->savedate && mailbox->i.minor_version >= 15) { buf_printf(&buf, TIME_T_FMT, record->savedate); crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "savedate", "", &buf); buf_reset(&buf); } if (record->createdmodseq && mailbox->i.minor_version >= 16) { buf_printf(&buf, "%llu", record->createdmodseq); crc ^= crc_annot(record->uid, IMAP_ANNOT_NS "createdmodseq", "", &buf); buf_reset(&buf); } buf_free(&buf); return crc; } EXPORTED void mailbox_annot_changed(struct mailbox *mailbox, unsigned int uid, const char *entry, const char *userid, const struct buf *oldval, const struct buf *newval, int silent) { /* update sync_crc - NOTE, only per-message annotations count */ if (uid) { /* check that the record isn't already expunged */ struct index_record record; int r = mailbox_find_index_record(mailbox, uid, &record); if (r || record.internal_flags & FLAG_INTERNAL_EXPUNGED) return; if (!mailbox_is_virtannot(mailbox, entry)) { if (oldval->len) mailbox->i.synccrcs.annot ^= crc_annot(uid, entry, userid, oldval); if (newval->len) mailbox->i.synccrcs.annot ^= crc_annot(uid, entry, userid, newval); } } if (!silent) { /* we are dirtying modseq for any annotation change */ mailbox_modseq_dirty(mailbox); /* and we're dirtying foldermodseq if it's a mailbox level annotation */ if (!uid) mboxlist_update_foldermodseq(mailbox_name(mailbox), mailbox->i.highestmodseq); } /* we always dirty the quota */ mailbox_quota_dirty(mailbox); /* corruption prevention - check we don't go negative */ if (mailbox->i.quota_annot_used > (quota_t)oldval->len) mailbox->i.quota_annot_used -= oldval->len; else mailbox->i.quota_annot_used = 0; mailbox->i.quota_annot_used += newval->len; } static int calc_one_annot(const char *mboxname __attribute__((unused)), uint32_t uid, const char *entry, const char *userid, const struct buf *value, const struct annotate_metadata *mdata __attribute__((unused)), void *rock) { struct annot_calc_rock *cr = (struct annot_calc_rock *)rock; /* update sync_crc - NOTE, only per-message annotations count */ if (uid && !mailbox_is_virtannot(cr->mailbox, entry)) cr->annot ^= crc_annot(uid, entry, userid, value); /* always count the size */ cr->used += value->len; return 0; } static void mailbox_annot_update_counts(struct mailbox *mailbox, const struct index_record *record, int is_add) { struct annot_calc_rock cr = { mailbox, 0, 0 }; /* expunged records don't count */ if (record && record->internal_flags & FLAG_INTERNAL_EXPUNGED) return; annotatemore_findall_mailbox(mailbox, record ? record->uid : 0, /* all entries*/"*", /*modseq*/0, calc_one_annot, &cr, /*flags*/0); if (record) mailbox->i.synccrcs.annot ^= cr.annot; if (is_add) mailbox->i.quota_annot_used += cr.used; else { /* corruption prevention - check we don't go negative */ if (mailbox->i.quota_annot_used > cr.used) mailbox->i.quota_annot_used -= cr.used; else mailbox->i.quota_annot_used = 0; } } /* * Calculate a sync CRC for the entire @mailbox using CRC algorithm * version @vers, optionally forcing recalculation */ EXPORTED struct synccrcs mailbox_synccrcs(struct mailbox *mailbox, int force) { annotate_state_t *astate = NULL; const message_t *msg; struct synccrcs crcs = { CRC_INIT_BASIC, CRC_INIT_ANNOT }; if (!force) return mailbox->i.synccrcs; /* hold annotations DB open - failure to load is an error */ if (mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, &astate)) return crcs; /* and make sure it stays locked for the whole process */ annotate_state_begin(astate); struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); crcs.basic ^= crc_basic(mailbox, record); crcs.annot ^= crc_virtannot(mailbox, record); struct annot_calc_rock cr = { mailbox, 0, 0 }; annotatemore_findall_mailbox(mailbox, record->uid, /* all entries*/"*", /*modseq*/0, calc_one_annot, &cr, /*flags*/0); crcs.annot ^= cr.annot; } mailbox_iter_done(&iter); /* possibly upgrade the stored value */ if (mailbox_index_islocked(mailbox, /*write*/1)) { mailbox->i.synccrcs = crcs; mailbox_index_dirty(mailbox); } /* return the newly calculated CRC */ return crcs; } static void mailbox_index_update_counts(struct mailbox *mailbox, const struct index_record *record, int is_add) { mailbox_quota_dirty(mailbox); mailbox_index_dirty(mailbox); header_update_counts(&mailbox->i, record, is_add); mailbox->i.synccrcs.basic ^= crc_basic(mailbox, record); mailbox->i.synccrcs.annot ^= crc_virtannot(mailbox, record); } EXPORTED int mailbox_index_recalc(struct mailbox *mailbox) { annotate_state_t *astate = NULL; const message_t *msg; int r = 0; assert(mailbox_index_islocked(mailbox, 1)); /* cache the old used quota */ mailbox_quota_dirty(mailbox); mailbox_index_dirty(mailbox); mailbox->i.answered = 0; mailbox->i.flagged = 0; mailbox->i.deleted = 0; mailbox->i.unseen = 0; mailbox->i.exists = 0; mailbox->i.quota_mailbox_used = 0; mailbox->i.quota_annot_used = 0; mailbox->i.synccrcs.basic = CRC_INIT_BASIC; mailbox->i.synccrcs.annot = CRC_INIT_ANNOT; /* mailbox level annotations */ mailbox_annot_update_counts(mailbox, NULL, 1); /* hold annotations DB open */ r = mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, &astate); if (r) goto out; /* and make sure it stays locked for the whole process */ annotate_state_begin(astate); struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); mailbox_index_update_counts(mailbox, record, 1); mailbox_annot_update_counts(mailbox, record, 1); } mailbox_iter_done(&iter); out: return r; } #ifdef WITH_DAV static int mailbox_update_carddav(struct mailbox *mailbox, const struct index_record *old, const struct index_record *new) { struct carddav_db *carddavdb = NULL; struct param *param; struct body *body = NULL; struct carddav_data *cdata = NULL; const char *resource = NULL; int r = 0; /* conditions in which there's nothing to do */ if (!new) goto done; /* phantom record - never really existed here */ if (!old && (new->internal_flags & FLAG_INTERNAL_UNLINKED)) goto done; r = mailbox_cacherecord(mailbox, new); if (r) goto done; /* Get resource URL from filename param in Content-Disposition header */ message_read_bodystructure(new, &body); for (param = body->disposition_params; param; param = param->next) { if (!strcmp(param->attribute, "FILENAME")) { resource = param->value; } } assert(resource); carddavdb = mailbox_open_carddav(mailbox); /* find existing record for this resource */ const mbentry_t mbentry = { .name = (char *)mailbox_name(mailbox), .uniqueid = (char *)mailbox_uniqueid(mailbox) }; carddav_lookup_resource(carddavdb, &mbentry, resource, &cdata, /*tombstones*/1); /* does it still come from this UID? */ if (cdata->dav.imap_uid > new->uid) goto done; if (new->internal_flags & FLAG_INTERNAL_UNLINKED) { /* is there an existing record? */ if (!cdata->dav.imap_uid) goto done; /* delete entry */ r = carddav_delete(carddavdb, cdata->dav.rowid); } else if (cdata->dav.imap_uid == new->uid) { /* just a flag change on an existing record */ struct vparse_card *vcard = record_to_vcard(mailbox, new); int ispinned = (new->system_flags & FLAG_FLAGGED) ? 1 : 0; if (!vcard || !vcard->objects) { syslog(LOG_ERR, "record_to_vcard failed for record %u:%s", cdata->dav.imap_uid, mailbox_name(mailbox)); r = IMAP_MAILBOX_BADFORMAT; // XXX better error? vparse_free_card(vcard); goto done; } cdata->dav.modseq = new->modseq; cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1; r = carddav_writecard(carddavdb, cdata, vcard->objects, ispinned); vparse_free_card(vcard); } else { /* Load message containing the resource and parse vcard data */ struct vparse_card *vcard = record_to_vcard(mailbox, new); int ispinned = (new->system_flags & FLAG_FLAGGED) ? 1 : 0; if (!vcard || !vcard->objects) { syslog(LOG_ERR, "record_to_vcard failed for record %u:%s", cdata->dav.imap_uid, mailbox_name(mailbox)); r = IMAP_MAILBOX_BADFORMAT; // XXX better error? vparse_free_card(vcard); goto done; } /* Create mapping entry from resource name to UID */ cdata->dav.imap_uid = new->uid; cdata->dav.modseq = new->modseq; cdata->dav.createdmodseq = new->createdmodseq; cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1; if (!cdata->dav.creationdate) cdata->dav.creationdate = new->internaldate; r = carddav_writecard(carddavdb, cdata, vcard->objects, ispinned); vparse_free_card(vcard); } done: message_free_body(body); free(body); return r; } static int mailbox_update_caldav(struct mailbox *mailbox, const struct index_record *old, const struct index_record *new) { struct caldav_db *caldavdb = NULL; struct param *param; struct body *body = NULL; struct caldav_data *cdata = NULL; const char *resource = NULL; const char *sched_tag = NULL; unsigned tzbyref = 0, shared = 0; int r = 0; /* conditions in which there's nothing to do */ if (!new) goto done; /* phantom record - never really existed at all */ if (!old && (new->internal_flags & FLAG_INTERNAL_UNLINKED)) goto done; r = mailbox_cacherecord(mailbox, new); if (r) goto done; /* get resource URL from filename param in Content-Disposition header */ message_read_bodystructure(new, &body); for (param = body->disposition_params; param; param = param->next) { if (!strcmp(param->attribute, "FILENAME")) { resource = param->value; } else if (!strcmp(param->attribute, "SCHEDULE-TAG")) { sched_tag = param->value; } else if (!strcmp(param->attribute, "TZ-BY-REF")) { tzbyref = !strcasecmp(param->value, "TRUE"); } else if (!strcmp(param->attribute, "PER-USER-DATA")) { shared = !strcasecmp(param->value, "TRUE"); } } caldavdb = mailbox_open_caldav(mailbox); /* Find existing record for this resource */ const mbentry_t mbentry = { .name = (char *)mailbox_name(mailbox), .uniqueid = (char *)mailbox_uniqueid(mailbox) }; caldav_lookup_resource(caldavdb, &mbentry, resource, &cdata, /*tombstones*/1); /* has this record already been replaced? Don't write anything */ if (cdata->dav.imap_uid > new->uid) goto done; if (new->internal_flags & FLAG_INTERNAL_UNLINKED) { /* is there an existing record? */ if (!cdata->dav.imap_uid) goto done; /* remove associated alarms */ caldav_alarm_delete_record(cdata->dav.mailbox, cdata->dav.imap_uid); /* delete entry */ r = caldav_delete(caldavdb, cdata->dav.rowid); } else if (cdata->dav.imap_uid == new->uid) { if (new->internal_flags & FLAG_INTERNAL_EXPUNGED) { /* remove associated alarms */ caldav_alarm_delete_record(cdata->dav.mailbox, cdata->dav.imap_uid); } else if (!new->silentupdate) { /* make sure record is up to date - see add below for description of * why we don't touch silent records */ caldav_alarm_touch_record(mailbox, new); } /* just a flags update to an existing record */ cdata->dav.modseq = new->modseq; cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1; r = caldav_write(caldavdb, cdata); } else { /* Load message containing the resource and parse ical data */ icalcomponent *ical = record_to_ical(mailbox, new, NULL); if (!ical) { syslog(LOG_ERR, "record_to_ical failed for record %u:%s", cdata->dav.imap_uid, mailbox_name(mailbox)); r = IMAP_MAILBOX_BADFORMAT; // XXX better error? goto done; } /* remove old ones */ if (cdata->dav.imap_uid) { r = caldav_alarm_delete_record(cdata->dav.mailbox, cdata->dav.imap_uid); if (r) goto alarmdone; } cdata->dav.creationdate = new->internaldate; cdata->dav.imap_uid = new->uid; cdata->dav.modseq = new->modseq; cdata->dav.createdmodseq = new->createdmodseq; cdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1; cdata->sched_tag = sched_tag; cdata->comp_flags.tzbyref = tzbyref; cdata->comp_flags.shared = shared; /* add new ones unless this record is expunged */ /* we need to skip silent records (replication) * because the lastalarm annotation won't be set yet - * instead, we have an explicit sync from the annotation * which is done after the annotations are written in sync_support.c */ if (cdata->dav.alive && !new->silentupdate) { r = caldav_alarm_add_record(mailbox, new, ical); if (r) goto alarmdone; } r = caldav_writeentry(caldavdb, cdata, ical); alarmdone: icalcomponent_free(ical); } done: message_free_body(body); free(body); return r; } static int mailbox_update_webdav(struct mailbox *mailbox, const struct index_record *old, const struct index_record *new) { struct webdav_db *webdavdb = NULL; struct param *param; struct body *body = NULL; struct webdav_data *wdata = NULL; const char *resource = NULL; int r = 0; /* conditions in which there's nothing to do */ if (!new) goto done; /* phantom record - never really existed here */ if (!old && new->internal_flags & FLAG_INTERNAL_EXPUNGED) goto done; r = mailbox_cacherecord(mailbox, new); if (r) goto done; /* Get resource URL from filename param in Content-Disposition header */ message_read_bodystructure(new, &body); for (param = body->disposition_params; param; param = param->next) { if (!strcmp(param->attribute, "FILENAME")) { resource = param->value; } } webdavdb = mailbox_open_webdav(mailbox); /* Find existing record for this resource */ const mbentry_t mbentry = { .name = (char *)mailbox_name(mailbox), .uniqueid = (char *)mailbox_uniqueid(mailbox) }; webdav_lookup_resource(webdavdb, &mbentry, resource, &wdata, /*tombstones*/1); /* if updated by a newer UID, skip - this record doesn't refer to the current item */ if (wdata->dav.imap_uid > new->uid) goto done; if (new->internal_flags & FLAG_INTERNAL_UNLINKED) { /* is there an existing record? */ if (!wdata->dav.imap_uid) goto done; /* delete entry */ r = webdav_delete(webdavdb, wdata->dav.rowid); } else if (wdata->dav.imap_uid == new->uid) { /* just a flags update to an existing record */ wdata->dav.modseq = new->modseq; wdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1; wdata->ref_count *= wdata->dav.alive; r = webdav_write(webdavdb, wdata); } else { struct buf msg_buf = BUF_INITIALIZER; struct message_guid guid; /* Load message containing the resource */ r = mailbox_map_record(mailbox, new, &msg_buf); if (r) goto done; /* Calculate GUID for body content only */ message_guid_generate(&guid, buf_base(&msg_buf) + new->header_size, buf_len(&msg_buf) - new->header_size); buf_free(&msg_buf); wdata->dav.creationdate = new->internaldate; wdata->dav.imap_uid = new->uid; wdata->dav.modseq = new->modseq; wdata->dav.createdmodseq = new->createdmodseq; wdata->dav.alive = (new->internal_flags & FLAG_INTERNAL_EXPUNGED) ? 0 : 1; wdata->ref_count *= wdata->dav.alive; wdata->filename = body->description; wdata->type = lcase(body->type); wdata->subtype = lcase(body->subtype); wdata->res_uid = message_guid_encode(&guid); r = webdav_write(webdavdb, wdata); } done: if (body) { message_free_body(body); free(body); } return r; } static int mailbox_update_dav(struct mailbox *mailbox, const struct index_record *old, const struct index_record *new) { /* never have DAV on deleted mailboxes */ if (mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) return 0; switch (mbtype_isa(mailbox_mbtype(mailbox))) { case MBTYPE_ADDRESSBOOK: return mailbox_update_carddav(mailbox, old, new); case MBTYPE_CALENDAR: return mailbox_update_caldav(mailbox, old, new); case MBTYPE_COLLECTION: return mailbox_update_webdav(mailbox, old, new); } return 0; } static int mailbox_commit_dav(struct mailbox *mailbox) { int r; if (mailbox->local_caldav) { r = caldav_commit(mailbox->local_caldav); caldav_close(mailbox->local_caldav); mailbox->local_caldav = NULL; if (r) return r; } if (mailbox->local_carddav) { r = carddav_commit(mailbox->local_carddav); carddav_close(mailbox->local_carddav); mailbox->local_carddav = NULL; if (r) return r; } if (mailbox->local_webdav) { r = webdav_commit(mailbox->local_webdav); webdav_close(mailbox->local_webdav); mailbox->local_webdav = NULL; if (r) return r; } return 0; } static int mailbox_abort_dav(struct mailbox *mailbox) { int r; if (mailbox->local_caldav) { r = caldav_abort(mailbox->local_caldav); caldav_close(mailbox->local_caldav); mailbox->local_caldav = NULL; if (r) return r; } if (mailbox->local_carddav) { r = carddav_abort(mailbox->local_carddav); carddav_close(mailbox->local_carddav); mailbox->local_carddav = NULL; if (r) return r; } if (mailbox->local_webdav) { r = webdav_abort(mailbox->local_webdav); webdav_close(mailbox->local_webdav); mailbox->local_webdav = NULL; if (r) return r; } return 0; } #endif // WITH_DAV #ifdef WITH_JMAP static int mailbox_update_email_alarms(struct mailbox *mailbox, const struct index_record *old, const struct index_record *new) { int r = 0; if (!(mailbox->i.options & OPT_IMAP_HAS_ALARMS)) return 0; /* never have alarms on deleted mailboxes */ if (mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) return 0; /* remove associated alarms if deleted */ if (!new || new->internal_flags & FLAG_INTERNAL_EXPUNGED) { r = caldav_alarm_delete_record(mailbox_name(mailbox), new->uid); } /* remove associated alarms if canceled or final */ else if (mbtype_isa(mailbox_mbtype(mailbox)) == MBTYPE_JMAPSUBMIT && (new->system_flags & (FLAG_FLAGGED | FLAG_ANSWERED))) { r = caldav_alarm_delete_record(mailbox_name(mailbox), new->uid); } /* touch or create otherwise */ else if (old && (old->uid == new->uid)) { r = caldav_alarm_touch_record(mailbox, new); } else { r = caldav_alarm_add_record(mailbox, new, NULL); } return r; } EXPORTED int mailbox_add_email_alarms(struct mailbox *mailbox) { const message_t *msg; int r = 0; if (!(mailbox->i.options & OPT_IMAP_HAS_ALARMS)) return 0; if (mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) return 0; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); r = mailbox_update_email_alarms(mailbox, NULL, record); if (r) break; } mailbox_iter_done(&iter); return r; } #endif // WITH_JMAP #ifdef USE_SIEVE static struct sieve_db *mailbox_open_sieve(struct mailbox *mailbox) { if (!mailbox->sievedir) { char *userid = mboxname_to_userid(mailbox_name(mailbox)); mailbox->sievedir = xstrdup(user_sieve_path(userid)); free(userid); } if (!mailbox->local_sieve) { mailbox->local_sieve = sievedb_open_mailbox(mailbox); int r = sievedb_begin(mailbox->local_sieve); if (r) { sievedb_abort(mailbox->local_sieve); sievedb_close(mailbox->local_sieve); mailbox->local_sieve = NULL; } } return mailbox->local_sieve; } static int mailbox_update_sieve(struct mailbox *mailbox, const struct index_record *old, const struct index_record *new) { struct sieve_db *sievedb = NULL; struct param *param; struct body *body = NULL; struct sieve_data *sdata = NULL; char *id = NULL; const char *name = NULL; struct buf msg_buf = BUF_INITIALIZER; int isexpunged = (new->internal_flags & FLAG_INTERNAL_EXPUNGED ? 1 : 0); int isactive = (new->system_flags & FLAG_FLAGGED ? 1 : 0); int r = 0; if (mbtype_isa(mailbox_mbtype(mailbox)) != MBTYPE_SIEVE) return 0; /* never have Sieve on deleted mailboxes */ if (mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) return 0; /* conditions in which there's nothing to do */ if (!new) return 0; /* phantom record - never really existed here */ if (!old && isexpunged) return 0; r = mailbox_cacherecord(mailbox, new); if (r) goto done; /* Get script id from filename param in Content-Disposition header */ message_read_bodystructure(new, &body); for (param = body->disposition_params; param; param = param->next) { if (!strcmp(param->attribute, "FILENAME")) { ssize_t idlen = strlen(param->value) - strlen(SIEVE_EXTENSION); if (idlen > 0 && !strcmp(param->value + idlen, SIEVE_EXTENSION)) { id = xstrndup(param->value, idlen); } } } assert(id); /* Get script name from Subject */ name = body->subject; sievedb = mailbox_open_sieve(mailbox); /* Find existing record for this id */ sievedb_lookup_id(sievedb, id, &sdata, /*tombstones*/1); /* if updated by a newer UID, skip - this record doesn't refer to the current item */ if (sdata->imap_uid > new->uid) goto done; if (new->internal_flags & FLAG_INTERNAL_UNLINKED) { /* is there an existing record? */ if (!sdata->imap_uid) goto done; /* delete entry */ r = sievedb_delete(sievedb, sdata->rowid); } else if (sdata->imap_uid == new->uid) { /* just a flags update to an existing record */ if (isexpunged) { r = sievedir_delete_script(mailbox->sievedir, name); } else if (isactive) { if (!sdata->isactive) { r = sievedir_activate_script(mailbox->sievedir, name); } } else if (sdata->isactive) { r = sievedir_deactivate_script(mailbox->sievedir); } if (r) { r = IMAP_IOERROR; goto done; } sdata->modseq = new->modseq; sdata->alive = !isexpunged; sdata->isactive = isactive; r = sievedb_write(sievedb, sdata); } else { /* Load message containing the script */ r = mailbox_map_record(mailbox, new, &msg_buf); if (r) goto done; const char *content = buf_cstring(&msg_buf) + new->header_size; char *errors = NULL; r = sievedir_put_script(mailbox->sievedir, name, content, &errors); + if (errors) { + syslog(LOG_ERR, "Error during sievedir_put_script for %s %s", mailbox_name(mailbox), errors); + } if (!r && isactive) { r = sievedir_activate_script(mailbox->sievedir, name); } if (r) { r = IMAP_IOERROR; goto done; } sdata->lastupdated = new->internaldate; sdata->mailbox = mailbox_uniqueid(mailbox); sdata->imap_uid = new->uid; sdata->modseq = new->modseq; sdata->createdmodseq = new->createdmodseq; sdata->alive = !isexpunged; sdata->isactive = isactive; sdata->id = id; sdata->name = name; sdata->contentid = message_guid_encode(&body->content_guid); if (!sdata->creationdate) sdata->creationdate = new->internaldate; r = sievedb_write(sievedb, sdata); } done: if (body) { message_free_body(body); free(body); } buf_free(&msg_buf); free(id); return r; } static int mailbox_delete_sieve(struct mailbox *mailbox) { struct sieve_db *sievedb = NULL; if (mbtype_isa(mailbox_mbtype(mailbox)) != MBTYPE_SIEVE) return 0; sievedb = sievedb_open_mailbox(mailbox); if (sievedb) { int r = sievedb_delmbox(sievedb); sievedb_close(sievedb); if (r) return r; } return 0; } static int mailbox_commit_sieve(struct mailbox *mailbox) { int r; free(mailbox->sievedir); mailbox->sievedir = NULL; if (mailbox->local_sieve) { r = sievedb_commit(mailbox->local_sieve); sievedb_close(mailbox->local_sieve); mailbox->local_sieve = NULL; if (r) return r; } return 0; } static int mailbox_abort_sieve(struct mailbox *mailbox) { int r; free(mailbox->sievedir); mailbox->sievedir = NULL; if (mailbox->local_sieve) { r = sievedb_abort(mailbox->local_sieve); sievedb_close(mailbox->local_sieve); mailbox->local_sieve = NULL; if (r) return r; } return 0; } EXPORTED int mailbox_add_sieve(struct mailbox *mailbox) { const message_t *msg; int r = 0; if (mbtype_isa(mailbox_mbtype(mailbox)) != MBTYPE_SIEVE) return 0; if (mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) return 0; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); r = mailbox_update_sieve(mailbox, NULL, record); if (r) break; /* in THEORY there maybe changes here that we should be saving... */ } mailbox_iter_done(&iter); return r; } #endif // USE_SIEVE EXPORTED struct conversations_state *mailbox_get_cstate_full(struct mailbox *mailbox, int allow_deleted) { if (!mailbox_has_conversations_full(mailbox, allow_deleted)) return NULL; /* we already own it? */ if (mailbox->local_cstate) return mailbox->local_cstate; /* already exists, use that one */ struct conversations_state *cstate = conversations_get_mbox(mailbox_name(mailbox)); if (cstate) { /* but make sure it's not read-only unless we are */ if (!mailbox->is_readonly) assert (!cstate->is_shared); return cstate; } /* open the conversations DB - don't bother checking return code since it'll * only be set if it opens successfully, and we can only return NULL or an * object */ conversations_open_mbox(mailbox_name(mailbox), mailbox->is_readonly, &mailbox->local_cstate); return mailbox->local_cstate; } static int mailbox_update_conversations(struct mailbox *mailbox, const struct index_record *old, struct index_record *new) { struct conversations_state *cstate = mailbox_get_cstate(mailbox); if (!cstate) return 0; /* handle unlinked items as if they didn't exist */ if (old && (old->internal_flags & FLAG_INTERNAL_UNLINKED)) old = NULL; if (new && (new->internal_flags & FLAG_INTERNAL_UNLINKED)) new = NULL; if (!old && !new) return 0; int ignorelimits = new ? new->ignorelimits : 1; return conversations_update_record(cstate, mailbox, old, new, /*allowrenumber*/1, ignorelimits); } EXPORTED int mailbox_get_xconvmodseq(struct mailbox *mailbox, modseq_t *modseqp) { conv_status_t status = CONV_STATUS_INIT; int r; if (modseqp) *modseqp = 0; struct conversations_state *cstate = mailbox_get_cstate(mailbox); if (!cstate) return 0; r = conversation_getstatus(cstate, CONV_FOLDER_KEY_MBOX(cstate, mailbox), &status); if (r) return r; *modseqp = status.threadmodseq; return 0; } /* Used in replication */ EXPORTED int mailbox_update_xconvmodseq(struct mailbox *mailbox, modseq_t newmodseq, int force) { conv_status_t status = CONV_STATUS_INIT; int r; struct conversations_state *cstate = mailbox_get_cstate(mailbox); if (!cstate) return 0; r = conversation_getstatus(cstate, CONV_FOLDER_KEY_MBOX(cstate, mailbox), &status); if (r) return r; if (newmodseq > status.threadmodseq || (force && newmodseq < status.threadmodseq)) { status.threadmodseq = newmodseq; r = conversation_setstatus(cstate, CONV_FOLDER_KEY_MBOX(cstate, mailbox), &status); } return r; } /* NOTE: maybe make this able to return error codes if we have * support for transactional mailbox updates later. For now, * we expect callers to have already done all sanity checking */ static int mailbox_update_indexes(struct mailbox *mailbox, const struct index_record *old, struct index_record *new) { int r = 0; // 'new' is not static because conversations might change the CID r = mailbox_update_conversations(mailbox, old, new); if (r) return r; #ifdef WITH_DAV r = mailbox_update_dav(mailbox, old, new); if (r) return r; #endif #ifdef WITH_JMAP r = mailbox_update_email_alarms(mailbox, old, new); if (r) return r; #endif #ifdef USE_SIEVE r = mailbox_update_sieve(mailbox, old, new); if (r) return r; #endif /* NOTE - we do these last, once the counts are updated */ if (old) mailbox_index_update_counts(mailbox, old, 0); if (new) mailbox_index_update_counts(mailbox, new, 1); return 0; } EXPORTED int mailbox_reload_index_record(struct mailbox *mailbox, struct index_record *record) { if (record->recno) return mailbox_read_index_record(mailbox, record->recno, record); else return mailbox_find_index_record(mailbox, record->uid, record); } /* * Rewrite an index record in a mailbox - updates all * necessary tracking fields automatically. */ EXPORTED int mailbox_rewrite_index_record(struct mailbox *mailbox, struct index_record *record) { int r; struct index_record oldrecord; int expunge_mode = config_getenum(IMAPOPT_EXPUNGE_MODE); int immediate = (expunge_mode == IMAP_ENUM_EXPUNGE_MODE_IMMEDIATE || expunge_mode == IMAP_ENUM_EXPUNGE_MODE_SEMIDELAYED || mailbox->i.minor_version < 12); int changeflags = 0; assert(mailbox_index_islocked(mailbox, 1)); assert(record->recno > 0 && record->recno <= mailbox->i.num_records); r = mailbox_read_index_record(mailbox, record->recno, &oldrecord); if (r) { xsyslog(LOG_ERR, "IOERROR: re-reading record failed", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), record->uid); return r; } mailbox_read_basecid(mailbox, &oldrecord); /* OK, we need to decide how to handle basecid. Three cases here: * 1) basecid is already set - keep it * 2) basecid is zero, but the system flag says it should be readable, set it from oldrecord * 3) basecid is zero, system flag is zero - keep zero * we only need code for case #2 */ if (!record->basecid && (record->internal_flags & FLAG_INTERNAL_SPLITCONVERSATION)) record->basecid = oldrecord.basecid; if (oldrecord.internal_flags & FLAG_INTERNAL_EXPUNGED) changeflags |= CHANGE_WASEXPUNGED; if (oldrecord.internal_flags & FLAG_INTERNAL_UNLINKED) changeflags |= CHANGE_WASUNLINKED; /* the UID has to match, of course, for it to be the same * record. XXX - test fields like "internaldate", etc here * too? Maybe replication should be more strict about it */ assert(record->uid == oldrecord.uid); assert(message_guid_equal(&oldrecord.guid, &record->guid)); if (oldrecord.internal_flags & FLAG_INTERNAL_EXPUNGED) { /* it is a sin to unexpunge a message. unexpunge.c copies * the data from the old record and appends it with a new * UID, which is righteous in the eyes of the IMAP client */ assert(record->internal_flags & FLAG_INTERNAL_EXPUNGED); } if (oldrecord.internal_flags & FLAG_INTERNAL_ARCHIVED) { /* it is also a sin to unarchive a message, except in the * the very odd case of a reconstruct. So let's see about * that */ if (!(record->internal_flags & FLAG_INTERNAL_ARCHIVED)) xsyslog(LOG_ERR, "IOERROR: bogus removal of archived flag", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), record->uid); } /* handle immediate expunges here... */ if (immediate && (record->internal_flags & FLAG_INTERNAL_EXPUNGED)) { record->internal_flags |= FLAG_INTERNAL_UNLINKED | FLAG_INTERNAL_NEEDS_CLEANUP; } /* make sure highestmodseq gets updated unless we're * being silent about it (i.e. marking an already EXPUNGED * message as UNLINKED, or just updating the content_lines * field or cache_offset) */ if (record->silentupdate) { mailbox_index_dirty(mailbox); } else { mailbox_modseq_dirty(mailbox); record->modseq = mailbox->i.highestmodseq; record->last_updated = mailbox->last_updated; } assert(record->modseq >= oldrecord.modseq); if (record->internal_flags & FLAG_INTERNAL_UNLINKED) { /* mark required actions */ if (expunge_mode == IMAP_ENUM_EXPUNGE_MODE_IMMEDIATE || mailbox->i.minor_version < 12) mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK; mailbox->i.options |= OPT_MAILBOX_NEEDS_UNLINK; } else { /* rewrite the cache record if required anyway */ r = mailbox_append_cache(mailbox, record); if (r) return r; } r = mailbox_update_indexes(mailbox, &oldrecord, record); if (r) return r; if ((record->internal_flags & FLAG_INTERNAL_EXPUNGED) && !(changeflags & CHANGE_WASEXPUNGED)) { if (!mailbox->i.first_expunged || mailbox->i.first_expunged > record->last_updated) mailbox->i.first_expunged = record->last_updated; mailbox_annot_update_counts(mailbox, &oldrecord, 0); } r = _store_change(mailbox, record, changeflags); if (r) return r; if (config_auditlog) { char oldflags[FLAGMAPSTR_MAXLEN], sysflags[FLAGMAPSTR_MAXLEN]; flags_to_str(&oldrecord, oldflags); flags_to_str(record, sysflags); syslog(LOG_NOTICE, "auditlog: touched sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uid=<%u> guid=<%s> cid=<%s> " "modseq=<" MODSEQ_FMT "> oldflags=<%s> sysflags=<%s>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), record->uid, message_guid_encode(&record->guid), conversation_id_encode(record->cid), record->modseq, oldflags, sysflags); } /* expunged tracking */ if (record->internal_flags & FLAG_INTERNAL_EXPUNGED && (!mailbox->i.first_expunged || mailbox->i.first_expunged > record->last_updated)) mailbox->i.first_expunged = record->last_updated; return 0; } /* append a single message to a mailbox - also updates everything * automatically. These two functions are the ONLY way to modify * the contents or tracking fields of a message */ EXPORTED int mailbox_append_index_record(struct mailbox *mailbox, struct index_record *record) { int r; struct utimbuf settime; uint32_t changeflags = CHANGE_ISAPPEND; assert(mailbox_index_islocked(mailbox, 1)); /* Append MUST be a higher UID than any we've yet seen */ assert(record->uid > mailbox->i.last_uid); /* Append MUST have a message with data */ assert(record->size); /* GUID must not be null */ assert(!message_guid_isnull(&record->guid)); /* Check mailbox type size limits */ if (mailbox->i.exists && !record->ignorelimits) { if (mbtype_isa(mailbox_mbtype(mailbox)) == MBTYPE_ADDRESSBOOK) { int limit = config_getint(IMAPOPT_MAILBOX_MAXMESSAGES_ADDRESSBOOK); if (limit > 0 && limit <= (int)mailbox->i.exists) { xsyslog(LOG_ERR, "IOERROR: client hit per-addressbook exists limit", "mailbox=<%s>", mailbox_name(mailbox)); return IMAP_NO_OVERQUOTA; } } else if (mbtype_isa(mailbox_mbtype(mailbox)) == MBTYPE_CALENDAR) { int limit = config_getint(IMAPOPT_MAILBOX_MAXMESSAGES_CALENDAR); if (limit > 0 && limit <= (int)mailbox->i.exists) { xsyslog(LOG_ERR, "IOERROR: client hit per-calendar exists limit", "mailbox=<%s>", mailbox_name(mailbox)); return IMAP_NO_OVERQUOTA; } } else if (mbtype_isa(mailbox_mbtype(mailbox)) == MBTYPE_EMAIL) { int limit = config_getint(IMAPOPT_MAILBOX_MAXMESSAGES_EMAIL); if (limit > 0 && limit <= (int)mailbox->i.exists) { xsyslog(LOG_ERR, "IOERROR: client hit per-mailbox exists limit", "mailbox=<%s>", mailbox_name(mailbox)); return IMAP_NO_OVERQUOTA; } } else { /* no limits for other types defined yet */ } } /* belt AND suspenders - check the previous record too */ if (mailbox->i.num_records) { struct index_record prev; r = mailbox_read_index_record(mailbox, mailbox->i.num_records, &prev); if (r) return r; assert(prev.uid <= mailbox->i.last_uid); if (message_guid_equal(&prev.guid, &record->guid)) { syslog(LOG_INFO, "%s: same message appears twice %u %u", mailbox_name(mailbox), prev.uid, record->uid); /* but it's OK, we won't reject it */ } } if (!record->internaldate) record->internaldate = time(NULL); if (!record->gmtime) record->gmtime = record->internaldate; if (!record->sentdate) { struct tm *tm = localtime(&record->internaldate); /* truncate to the day */ tm->tm_sec = 0; tm->tm_min = 0; tm->tm_hour = 0; record->sentdate = mktime(tm); } /* update the highestmodseq if needed */ if (record->silentupdate) { mailbox_index_dirty(mailbox); } else { mailbox_modseq_dirty(mailbox); record->modseq = mailbox->i.highestmodseq; if (!record->createdmodseq || record->createdmodseq > record->modseq) record->createdmodseq = record->modseq; record->last_updated = mailbox->last_updated; if (!record->savedate) { // store the time of actual append if requested record->savedate = mailbox->last_updated; } } int object_storage_enabled = 0 ; #if defined ENABLE_OBJECTSTORE object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ; #endif if (!(record->internal_flags & FLAG_INTERNAL_UNLINKED)) { /* make the file timestamp correct */ settime.actime = settime.modtime = record->internaldate; if (!(object_storage_enabled && (record->internal_flags & FLAG_INTERNAL_ARCHIVED))) // mabe there is no file in directory. if (utime(mailbox_record_fname(mailbox, record), &settime) == -1) return IMAP_IOERROR; /* write the cache record before buffering the message, it * will set the cache_offset field. */ r = mailbox_append_cache(mailbox, record); if (r) return r; } r = mailbox_update_indexes(mailbox, NULL, record); if (r) return r; record->recno = mailbox->i.num_records + 1; mailbox->i.last_uid = record->uid; mailbox->i.num_records = record->recno; r = _store_change(mailbox, record, changeflags); if (r) return r; /* expunged tracking */ if ((record->internal_flags & FLAG_INTERNAL_EXPUNGED) && (!mailbox->i.first_expunged || mailbox->i.first_expunged > record->last_updated)) mailbox->i.first_expunged = record->last_updated; return 0; } EXPORTED void mailbox_cleanup_uid(struct mailbox *mailbox, uint32_t uid, const char *flagstr) { const char *spoolfname = mailbox_spool_fname(mailbox, uid); const char *archivefname = mailbox_archive_fname(mailbox, uid); if (unlink(spoolfname) == 0) { if (config_auditlog) { syslog(LOG_NOTICE, "auditlog: unlink sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uid=<%u> sysflags=<%s>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), uid, flagstr); } } if (strcmp(spoolfname, archivefname)) { if (unlink(archivefname) == 0) { if (config_auditlog) { syslog(LOG_NOTICE, "auditlog: unlinkarchive sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uid=<%u> sysflags=<%s>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), uid, flagstr); } } } } static void mailbox_record_cleanup(struct mailbox *mailbox, struct index_record *record) { #if defined ENABLE_OBJECTSTORE if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)) { /* we always remove the spool file here, because we've archived it */ if (record->system_flags & FLAG_INTERNAL_ARCHIVED) unlink(spoolfname); /* if the record is also deleted, we remove the objectstore copy */ if (record->system_flags & FLAG_INTERNAL_UNLINKED) objectstore_delete(mailbox, record); return; } #endif if (record->internal_flags & FLAG_INTERNAL_UNLINKED) { char flagstr[FLAGMAPSTR_MAXLEN]; flags_to_str(record, flagstr); /* remove both files */ mailbox_cleanup_uid(mailbox, record->uid, flagstr); int r = mailbox_get_annotate_state(mailbox, record->uid, NULL); if (r) { xsyslog(LOG_ERR, "IOERROR: failed to open annotations", "mailbox=<%s> record=<%u> error=<%s>", mailbox_name(mailbox), record->uid, error_message(r)); } r = annotate_msg_cleanup(mailbox, record->uid); if (r) { xsyslog(LOG_ERR, "IOERROR: failed to cleanup annotations", "mailbox=<%s> record=<%u> error=<%s>", mailbox_name(mailbox), record->uid, error_message(r)); } return; } /* file is still alive - check if there's anything to clean up */ const char *spoolfname = mailbox_spool_fname(mailbox, record->uid); const char *archivefname = mailbox_archive_fname(mailbox, record->uid); /* don't cleanup if it's the same file! */ if (strcmp(spoolfname, archivefname)) { if (record->internal_flags & FLAG_INTERNAL_ARCHIVED) { /* XXX - stat to make sure the other file exists first? - we mostly * trust that we didn't do stupid things everywhere else, so maybe not */ unlink(spoolfname); } else { unlink(archivefname); } } } /* need a mailbox exclusive lock, we're removing files */ static int mailbox_index_unlink(struct mailbox *mailbox) { syslog(LOG_INFO, "Unlinking files in mailbox %s", mailbox_name(mailbox)); /* NOTE: this gets called for two different cases: * 1) file is actually ready for unlinking (immediate expunge or * cyr_expire). * 2) file has been archived/unarchived, and the other one needs * to be removed. */ const message_t *msg; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, 0); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); /* still gotta check for FLAG_INTERNAL_UNLINKED, because it may have been * created by old code. Woot */ if ((record->internal_flags & FLAG_INTERNAL_NEEDS_CLEANUP) || record->internal_flags & FLAG_INTERNAL_UNLINKED) { struct index_record copyrecord = *record; mailbox_record_cleanup(mailbox, ©record); copyrecord.internal_flags &= ~FLAG_INTERNAL_NEEDS_CLEANUP; copyrecord.silentupdate = 1; copyrecord.ignorelimits = 1; /* XXX - error handling */ mailbox_rewrite_index_record(mailbox, ©record); } } mailbox_iter_done(&iter); /* need to clear the flag, even if nothing needed unlinking! */ mailbox_index_dirty(mailbox); mailbox->i.options &= ~OPT_MAILBOX_NEEDS_UNLINK; mailbox_commit(mailbox); return 0; } static int mailbox_repack_setup(struct mailbox *mailbox, int version, struct mailbox_repack **repackptr) { struct mailbox_repack *repack = xzmalloc(sizeof(struct mailbox_repack)); const char *fname; indexbuffer_t ibuf; unsigned char *buf = ibuf.buf; int n; /* if we're changing version at all, recalculate counts up-front */ if (version != mailbox->i.minor_version) { /* NOTE: this maps in annot_state in mailbox, which will get copied * into newmailbox below, which means all the annotate stuff later * is going to be good! The counters will be updated in the original * mailbox index, but that's OK because we recalc everything at the * end */ int r = mailbox_index_recalc(mailbox); if (r) goto fail; } /* init */ repack->mailbox = mailbox; repack->crcs = mailbox->i.synccrcs; repack->newmailbox = *mailbox; // struct copy repack->newmailbox.index_fd = -1; /* new files */ fname = mailbox_meta_newfname(mailbox, META_INDEX); repack->newmailbox.index_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666); if (repack->newmailbox.index_fd == -1) { xsyslog(LOG_ERR, "IOERROR: create failed", "fname=<%s>", fname); goto fail; } /* update the generation number */ repack->newmailbox.i.generation_no++; /* track the version number */ repack->newmailbox.i.minor_version = version; switch (version) { case 6: repack->newmailbox.i.start_offset = 76; repack->newmailbox.i.record_size = 60; break; case 7: repack->newmailbox.i.start_offset = 76; repack->newmailbox.i.record_size = 72; break; case 8: repack->newmailbox.i.start_offset = 92; repack->newmailbox.i.record_size = 80; break; case 9: repack->newmailbox.i.start_offset = 96; repack->newmailbox.i.record_size = 80; break; case 10: repack->newmailbox.i.start_offset = 96; repack->newmailbox.i.record_size = 88; break; /* 11 was FastMail internal */ case 12: repack->newmailbox.i.start_offset = 128; repack->newmailbox.i.record_size = 96; break; case 13: repack->newmailbox.i.start_offset = 128; repack->newmailbox.i.record_size = 104; break; case 14: /* version 15 just repurposed a field */ case 15: repack->newmailbox.i.start_offset = 160; repack->newmailbox.i.record_size = 104; break; case 16: case 17: repack->newmailbox.i.start_offset = 160; repack->newmailbox.i.record_size = 112; break; default: fatal("index version not supported", EX_SOFTWARE); } /* upgrades or downgrades across version 12 boundary? Sort out seen state */ if (version >= 12 && mailbox->i.minor_version < 12) { /* we need to read the current seen state for the owner */ struct seendata sd = SEENDATA_INITIALIZER; int r = IMAP_MAILBOX_NONEXISTENT; if (mailbox->i.options & OPT_IMAP_SHAREDSEEN) repack->userid = xstrdup("anyone"); else repack->userid = mboxname_to_userid(mailbox_name(mailbox)); if (repack->userid) { struct seen *seendb = NULL; r = seen_open(repack->userid, SEEN_SILENT, &seendb); if (!r) r = seen_read(seendb, mailbox_uniqueid(mailbox), &sd); seen_close(&seendb); } if (!r) { repack->newmailbox.i.recentuid = sd.lastuid; repack->newmailbox.i.recenttime = sd.lastchange; repack->seqset = seqset_parse(sd.seenuids, NULL, sd.lastuid); seen_freedata(&sd); } } else if (version < 12 && mailbox->i.minor_version >= 12) { if (mailbox->i.options & OPT_IMAP_SHAREDSEEN) repack->userid = xstrdup("anyone"); else repack->userid = mboxname_to_userid(mailbox_name(mailbox)); /* we need to create the seen state for the owner from the mailbox */ if (repack->userid) repack->seqset = seqset_init(mailbox->i.last_uid, SEQ_MERGE); } /* we'll count the records as they get added */ repack->newmailbox.i.num_records = 0; /* we're recreating caches, so there'll be nothing leaked */ repack->newmailbox.i.leaked_cache_records = 0; /* prepare initial header buffer */ mailbox_index_header_to_buf(&repack->newmailbox.i, buf); n = retry_write(repack->newmailbox.index_fd, buf, repack->newmailbox.i.start_offset); if (n == -1) goto fail; if (version != mailbox->i.minor_version) { /* we're might be reading this file as we go, so let's * start the job by mapping the new file into the newmailbox! */ repack->newmailbox.index_size = 0; repack->newmailbox.index_base = NULL; repack->newmailbox.index_len = 0; int r = mailbox_refresh_index_map(&repack->newmailbox); if (r) goto fail; } *repackptr = repack; return 0; fail: mailbox_repack_abort(&repack); return IMAP_IOERROR; } static int mailbox_repack_add(struct mailbox_repack *repack, struct index_record *record) { struct mappedfile *cachefile; indexbuffer_t ibuf; unsigned char *buf = ibuf.buf; int r; int n; cachefile = repack_cachefile(repack, record); /* write out the new cache record - need to clear the cache_offset * so it gets reset in the new record */ record->cache_offset = 0; r = cache_append_record(cachefile, record); if (r) return r; /* write the index record out */ mailbox_index_record_to_buf(record, repack->newmailbox.i.minor_version, buf); n = retry_write(repack->newmailbox.index_fd, buf, repack->newmailbox.i.record_size); if (n == -1) return IMAP_IOERROR; repack->newmailbox.i.num_records++; return 0; } static void mailbox_repack_abort(struct mailbox_repack **repackptr) { struct mailbox_repack *repack = *repackptr; int i; if (!repack) return; /* safe against double-free */ seqset_free(&repack->seqset); /* close and remove index */ xclose(repack->newmailbox.index_fd); unlink(mailbox_meta_newfname(repack->mailbox, META_INDEX)); /* close and remove all new caches */ for (i = 0; i < repack->caches.count; i++) { struct mappedfile *cachefile = ptrarray_nth(&repack->caches, i); char *fname = xstrdup(mappedfile_fname(cachefile)); mappedfile_commit(cachefile); /* gotta commit to clear the dirty flag. Alternative would be an unlink function */ mappedfile_unlock(cachefile); mappedfile_close(&cachefile); unlink(fname); free(fname); } ptrarray_fini(&repack->caches); // drop the map if we've mapped in the newmailbox index separately if (repack->newmailbox.index_base != repack->mailbox->index_base) { map_free(&repack->newmailbox.index_base, &repack->newmailbox.index_len); } mailbox_abort(repack->mailbox); free(repack->userid); free(repack); *repackptr = NULL; return; } HIDDEN int mailbox_repack_commit(struct mailbox_repack **repackptr) { strarray_t cachefiles = STRARRAY_INITIALIZER; indexbuffer_t ibuf; unsigned char *buf = ibuf.buf; struct mailbox_repack *repack = *repackptr; int r = IMAP_IOERROR; int i; assert(repack); /* if we changed versions, we'll re-calculate counts on the new mailbox too */ if (repack->newmailbox.i.minor_version != repack->mailbox->i.minor_version) { r = mailbox_refresh_index_map(&repack->newmailbox); if (r) goto fail; r = mailbox_index_recalc(&repack->newmailbox); if (r) goto fail; } if (repack->newmailbox.i.minor_version >= 10 && repack->mailbox->i.minor_version >= 10 && !mailbox_crceq(repack->newmailbox.i.synccrcs, repack->crcs)) { xsyslog(LOG_ERR, "IOERROR: CRC mismatch on repack commit", "mailbox=<%s> oldbasic=<%u> newbasic=<%u> " "oldannot=<%u> newannot=<%u>", mailbox_name(repack->mailbox), repack->crcs.basic, repack->newmailbox.i.synccrcs.basic, repack->crcs.annot, repack->newmailbox.i.synccrcs.annot); r = IMAP_MAILBOX_CHECKSUM; goto fail; } repack->newmailbox.i.last_repack_time = time(0); if (repack->mailbox->i.minor_version >= 12 && repack->newmailbox.i.minor_version < 12 && repack->seqset && repack->userid) { struct seendata sd = SEENDATA_INITIALIZER; struct seen *seendb = NULL; int r = seen_open(repack->userid, SEEN_CREATE, &seendb); if (!r) r = seen_lockread(seendb, mailbox_uniqueid(repack->mailbox), &sd); if (!r) { sd.lastuid = repack->newmailbox.i.last_uid; sd.seenuids = seqset_cstring(repack->seqset); if (!sd.seenuids) sd.seenuids = xstrdup(""); sd.lastread = time(NULL); sd.lastchange = repack->newmailbox.i.last_appenddate; r = seen_write(seendb, mailbox_uniqueid(repack->mailbox), &sd); /* XXX - syslog on errors? */ } seen_close(&seendb); seen_freedata(&sd); } /* rewrite the header with updated details */ mailbox_index_header_to_buf(&repack->newmailbox.i, buf); if (lseek(repack->newmailbox.index_fd, 0, SEEK_SET) < 0) goto fail; if (retry_write(repack->newmailbox.index_fd, buf, repack->newmailbox.i.start_offset) < 0) goto fail; /* ensure everything is committed to disk */ if (fsync(repack->newmailbox.index_fd) < 0) goto fail; xclose(repack->newmailbox.index_fd); /* NOTE: cache files need committing before index is renamed */ for (i = 0; i < repack->caches.count; i++) { struct mappedfile *cachefile = ptrarray_nth(&repack->caches, i); r = mappedfile_commit(cachefile); if (r) goto fail; } /* rename index first - loader will handle un-renamed cache if * the generation is lower */ r = mailbox_meta_rename(repack->mailbox, META_INDEX); if (r) goto fail; /* which cache files might currently exist? */ strarray_add(&cachefiles, mailbox_meta_fname(repack->mailbox, META_CACHE)); strarray_add(&cachefiles, mailbox_meta_fname(repack->mailbox, META_ARCHIVECACHE)); /* now the cache files can be renamed */ for (i = 0; i < repack->caches.count; i++) { struct mappedfile *cachefile = ptrarray_nth(&repack->caches, i); char *newname = xstrdup(mappedfile_fname(cachefile)); size_t len = strlen(newname)-4; assert(!strcmp(newname+len, ".NEW")); newname[len] = '\0'; /* STRIP .NEW */ mappedfile_rename(cachefile, newname); mappedfile_close(&cachefile); strarray_remove_all(&cachefiles, newname); free(newname); } ptrarray_fini(&repack->caches); for (i = 0; i < cachefiles.count; i++) { const char *fname = strarray_nth(&cachefiles, i); if (!unlink(fname)) syslog(LOG_NOTICE, "Removed unused cache file %s", fname); } strarray_fini(&cachefiles); // drop the map if we've mapped in the newmailbox index separately if (repack->newmailbox.index_base != repack->mailbox->index_base) { map_free(&repack->newmailbox.index_base, &repack->newmailbox.index_len); } seqset_free(&repack->seqset); free(repack->userid); free(repack); *repackptr = NULL; return 0; fail: strarray_fini(&cachefiles); mailbox_repack_abort(repackptr); if (!r) r = IMAP_IOERROR; return r; } /* need a mailbox exclusive lock, we're rewriting files */ static int mailbox_index_repack(struct mailbox *mailbox, int version) { struct mailbox_repack *repack = NULL; const message_t *msg; struct mailbox_iter *iter = NULL; struct buf buf = BUF_INITIALIZER; int r = IMAP_IOERROR; syslog(LOG_INFO, "Repacking mailbox %s version %d", mailbox_name(mailbox), version); r = mailbox_repack_setup(mailbox, version, &repack); if (r) goto done; iter = mailbox_iter_init(mailbox, 0, 0); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); struct index_record copyrecord = *record; int needs_cache_upgrade = 0; annotate_state_t *astate = NULL; r = mailbox_get_annotate_state(mailbox, record->uid, &astate); if (r) goto done; /* version changes? */ if (mailbox->i.minor_version < 12 && repack->newmailbox.i.minor_version >= 12) { if (seqset_ismember(repack->seqset, copyrecord.uid)) copyrecord.system_flags |= FLAG_SEEN; else copyrecord.system_flags &= ~FLAG_SEEN; needs_cache_upgrade = 1; } if (mailbox->i.minor_version >= 12 && repack->newmailbox.i.minor_version < 12) { if (repack->seqset) seqset_add(repack->seqset, copyrecord.uid, copyrecord.system_flags & FLAG_SEEN ? 1 : 0); copyrecord.system_flags &= ~FLAG_SEEN; } /* force cache upgrade across version 15 repack */ if (repack->newmailbox.i.minor_version >= 15 && record->cache_version < 9) needs_cache_upgrade = 1; /* better handle the cleanup just in case it's unlinked too */ /* still gotta check for FLAG_INTERNAL_UNLINKED, because it may have been * created by old code. Woot */ if (copyrecord.internal_flags & (FLAG_INTERNAL_NEEDS_CLEANUP | FLAG_INTERNAL_UNLINKED)) { mailbox_record_cleanup(mailbox, ©record); copyrecord.internal_flags &= ~FLAG_INTERNAL_NEEDS_CLEANUP; /* no need to rewrite - it's already being written to the new file */ } /* we aren't keeping unlinked files, that's kind of the point */ if (copyrecord.internal_flags & FLAG_INTERNAL_UNLINKED) { /* track the modseq for QRESYNC purposes */ if (copyrecord.modseq > repack->newmailbox.i.deletedmodseq) { repack->newmailbox.i.deletedmodseq = copyrecord.modseq; repack->newmailbox.i.changes_epoch = copyrecord.last_updated; } continue; } if (needs_cache_upgrade) { const char *fname = mailbox_record_fname(mailbox, ©record); if (message_parse(fname, ©record)) { /* failed to parse, don't try to write out record */ copyrecord.crec.len = 0; /* and the record is expunged too! */ copyrecord.internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED; xsyslog(LOG_ERR, "IOERROR: FATAL - failed to parse file, expunging", "mailbox=<%s> record=<%u>", mailbox_name(repack->mailbox), copyrecord.uid); } } /* virtual annotations */ if (mailbox->i.minor_version < 13 && repack->newmailbox.i.minor_version >= 13) { /* extract CID */ buf_reset(&buf); mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "thrid", "", &buf); if (buf.len == 16) { const char *p = buf_cstring(&buf); parsehex(p, &p, 16, ©record.cid); } buf_reset(&buf); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "thrid", "", &buf); if (r) goto done; } if (mailbox->i.minor_version >= 13 && repack->newmailbox.i.minor_version < 13) { if (record->cid) { buf_reset(&buf); buf_printf(&buf, "%llx", record->cid); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "thrid", "", &buf); if (r) goto done; } } if (mailbox->i.minor_version < 15 && repack->newmailbox.i.minor_version >= 15) { /* extract CID */ buf_reset(&buf); mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "savedate", "", &buf); if (buf.len) { const char *p = buf_cstring(&buf); bit64 newval; parsenum(p, &p, 0, &newval); copyrecord.savedate = newval; } buf_reset(&buf); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "savedate", "", &buf); if (r) goto done; } if (mailbox->i.minor_version >= 15 && repack->newmailbox.i.minor_version < 15) { if (record->savedate) { buf_reset(&buf); buf_printf(&buf, TIME_T_FMT, record->savedate); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "savedate", "", &buf); if (r) goto done; } } if (mailbox->i.minor_version < 16 && repack->newmailbox.i.minor_version >= 16) { /* extract CID */ buf_reset(&buf); mailbox_annotation_lookup(mailbox, record->uid, IMAP_ANNOT_NS "createdmodseq", "", &buf); if (buf.len) { const char *p = buf_cstring(&buf); bit64 newval; parsenum(p, &p, 0, &newval); copyrecord.createdmodseq = newval; } buf_reset(&buf); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "createdmodseq", "", &buf); if (r) goto done; } if (mailbox->i.minor_version >= 16 && repack->newmailbox.i.minor_version < 16) { if (record->createdmodseq) { buf_reset(&buf); buf_printf(&buf, "%llu", record->createdmodseq); r = annotate_state_writesilent(astate, IMAP_ANNOT_NS "createdmodseq", "", &buf); if (r) goto done; } } /* read in the old cache record */ r = mailbox_cacherecord(mailbox, ©record); if (r) goto done; r = mailbox_repack_add(repack, ©record); if (r) goto done; } /* we unlinked any "needs unlink" in the process */ repack->newmailbox.i.options &= ~(OPT_MAILBOX_NEEDS_REPACK|OPT_MAILBOX_NEEDS_UNLINK); done: mailbox_iter_done(&iter); buf_free(&buf); if (r) mailbox_repack_abort(&repack); else { modseq_t deletedmodseq = repack->newmailbox.i.deletedmodseq; r = mailbox_repack_commit(&repack); if (!r) { mboxname_setmodseq(mailbox_name(mailbox), deletedmodseq, mailbox_mbtype(mailbox), MBOXMODSEQ_ISDELETE); } } return r; } /* * Expunge decision proc used by mailbox_expunge() * to expunge \Deleted messages. */ static unsigned expungedeleted(struct mailbox *mailbox __attribute__((unused)), const struct index_record *record, void *rock __attribute__((unused))) { if (record->system_flags & FLAG_DELETED) return 1; return 0; } EXPORTED unsigned mailbox_should_archive(struct mailbox *mailbox, const struct index_record *record, void *rock) { int archive_after = config_getduration(IMAPOPT_ARCHIVE_AFTER, 'd'); time_t cutoff = time(0) - archive_after; if (rock) cutoff = *((time_t *)rock); int archive_size = config_getint(IMAPOPT_ARCHIVE_MAXSIZE); size_t maxsize = archive_size * 1024; int keepflagged = config_getswitch(IMAPOPT_ARCHIVE_KEEPFLAGGED); /* never pull messages back from the archives */ if (record->internal_flags & FLAG_INTERNAL_ARCHIVED) return 1; /* first check if we're archiving anything */ if (!config_getswitch(IMAPOPT_ARCHIVE_ENABLED)) return 0; /* always archive big messages */ if (record->size >= maxsize) return 1; /* archive everything in DELETED mailboxes */ if (mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) return 1; /* Calendar and Addressbook are small files and need to be hot */ switch (mbtype_isa(mailbox_mbtype(mailbox))) { case MBTYPE_ADDRESSBOOK: return 0; case MBTYPE_CALENDAR: return 0; } /* don't archive flagged messages */ if (keepflagged && (record->system_flags & FLAG_FLAGGED)) return 0; /* archive all other old messages */ if (record->internaldate <= cutoff) return 1; /* and don't archive anything else! */ return 0; } /* * Move messages between spool and archive partition * function pointed to by 'decideproc' is called (with 'deciderock') to * determine which messages to move. If deciderock return 0, the message * should be in the spool - if 1, the message should be in the archive. */ EXPORTED void mailbox_archive(struct mailbox *mailbox, mailbox_decideproc_t *decideproc, void *deciderock, unsigned flags) { int r; int dirtycache = 0; const message_t *msg; struct index_record copyrecord; const char *srcname; const char *destname; char *spoolcache = xstrdup(mailbox_meta_fname(mailbox, META_CACHE)); char *archivecache = xstrdup(mailbox_meta_fname(mailbox, META_ARCHIVECACHE)); int differentcache = strcmp(spoolcache, archivecache); int object_storage_enabled = 0; #if defined ENABLE_OBJECTSTORE object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ; #endif free(spoolcache); free(archivecache); assert(mailbox_index_islocked(mailbox, 1)); if (!decideproc) decideproc = &mailbox_should_archive; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, flags); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); const char *action = NULL; if (decideproc(mailbox, record, deciderock)) { if (record->internal_flags & FLAG_INTERNAL_ARCHIVED) continue; copyrecord = *record; srcname = mailbox_spool_fname(mailbox, copyrecord.uid); destname = mailbox_archive_fname(mailbox, copyrecord.uid); /* load cache before changing the flags */ r = mailbox_cacherecord(mailbox, ©record); if (r) { xsyslog(LOG_ERR, "IOERROR: failed to read cache", "mailbox=<%s> record=<%u> error=<%s>", mailbox_name(mailbox), copyrecord.uid, error_message(r)); continue; } #if defined ENABLE_OBJECTSTORE if (object_storage_enabled){ /* upload on the blob store */ r = objectstore_put(mailbox, ©record, srcname); if (r) { xsyslog(LOG_ERR, "IOERROR: objectstorage put failed", "mailbox=<%s> record=<%u> " "srcname=<%s> error=<%s>", mailbox_name(mailbox), copyrecord.uid, srcname, error_message(r)); // didn't manage to store it, so remove the ARCHIVED flag continue; } r = unlink (srcname); if (r < 0) syslog(LOG_ERR, "unlink(%s) failed: %m", srcname); } #endif copyrecord.internal_flags |= FLAG_INTERNAL_ARCHIVED | FLAG_INTERNAL_NEEDS_CLEANUP; action = "archive"; } else { if (!(record->internal_flags & FLAG_INTERNAL_ARCHIVED)) continue; copyrecord = *record; destname = mailbox_spool_fname(mailbox, copyrecord.uid); srcname = mailbox_archive_fname(mailbox, copyrecord.uid); /* load cache before changing the flags */ r = mailbox_cacherecord(mailbox, ©record); if (r) { xsyslog(LOG_ERR, "IOERROR: failed to read cache", "mailbox=<%s> record=<%u> error=<%s>", mailbox_name(mailbox), copyrecord.uid, error_message(r)); continue; } #if defined ENABLE_OBJECTSTORE if (object_storage_enabled){ /* recover from the blob store */ r = objectstore_get(mailbox, ©record, destname); if (r) { xsyslog(LOG_ERR, "IOERROR: objectstorage get failed", "mailbox=<%s> record=<%u> " "destname=<%s> error=<%s>", mailbox_name(mailbox), copyrecord.uid, destname, error_message(r)); continue; } objectstore_delete(mailbox, ©record); // this should only lower ref count. } #endif copyrecord.internal_flags &= ~FLAG_INTERNAL_ARCHIVED; copyrecord.internal_flags |= FLAG_INTERNAL_NEEDS_CLEANUP; action = "unarchive"; } if (!object_storage_enabled){ /* got a file to copy! */ if (strcmp(srcname, destname)) { r = cyrus_copyfile(srcname, destname, COPYFILE_MKDIR|COPYFILE_KEEPTIME); if (r) { xsyslog(LOG_ERR, "IOERROR: copyfile failed", "mailbox=<%s> record=<%u> " "srcname=<%s> destname=<%s> error=<%s>", mailbox_name(mailbox), copyrecord.uid, srcname, destname, error_message(r)); continue; } } } /* got a new cache record to write */ if (differentcache) { dirtycache = 1; copyrecord.cache_offset = 0; if (mailbox_append_cache(mailbox, ©record)) continue; } /* rewrite the index record */ copyrecord.silentupdate = 1; copyrecord.ignorelimits = 1; if (mailbox_rewrite_index_record(mailbox, ©record)) continue; mailbox->i.options |= OPT_MAILBOX_NEEDS_UNLINK; if (config_auditlog) { char flagstr[FLAGMAPSTR_MAXLEN]; flags_to_str(©record, flagstr); syslog(LOG_NOTICE, "auditlog: %s sessionid=<%s> mailbox=<%s> " "uniqueid=<%s> uid=<%u> guid=<%s> cid=<%s> sysflags=<%s>", action, session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), copyrecord.uid, message_guid_encode(©record.guid), conversation_id_encode(copyrecord.cid), flagstr); } } mailbox_iter_done(&iter); /* if we have stale cache records, we'll need a repack */ if (dirtycache) { mailbox_index_dirty(mailbox); mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK; } } EXPORTED void mailbox_remove_files_from_object_storage(struct mailbox *mailbox, unsigned flags) { const message_t *msg; #if defined ENABLE_OBJECTSTORE int object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ; #endif assert(mailbox_index_islocked(mailbox, 1)); struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, flags); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); if (!(record->internal_flags & FLAG_INTERNAL_ARCHIVED)) continue; #if defined ENABLE_OBJECTSTORE if (object_storage_enabled) objectstore_delete(mailbox, record); // this should only lower ref count. #endif } mailbox_iter_done(&iter); } /* * Perform an expunge operation on 'mailbox'. If nonzero, the * function pointed to by 'decideproc' is called (with 'deciderock') to * determine which messages to expunge. If 'decideproc' is a null pointer, * then messages with the \Deleted flag are expunged. * * event_type - the event among MessageExpunge, MessageExpire (zero means * don't send notification) */ EXPORTED int mailbox_expunge(struct mailbox *mailbox, mailbox_decideproc_t *decideproc, void *deciderock, unsigned *nexpunged, int event_type) { int r = 0; int numexpunged = 0; const message_t *msg; struct mboxevent *mboxevent = NULL; assert(mailbox_index_islocked(mailbox, 1)); /* anything to do? */ if (!mailbox->i.num_records) { if (nexpunged) *nexpunged = 0; return 0; } if (event_type) mboxevent = mboxevent_new(event_type); if (!decideproc) decideproc = expungedeleted; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); if (decideproc(mailbox, record, deciderock)) { numexpunged++; struct index_record copyrecord = *record; /* mark deleted */ copyrecord.internal_flags |= FLAG_INTERNAL_EXPUNGED; r = mailbox_rewrite_index_record(mailbox, ©record); if (r) { mboxevent_free(&mboxevent); mailbox_iter_done(&iter); return IMAP_IOERROR; } mboxevent_extract_record(mboxevent, mailbox, ©record); } } mailbox_iter_done(&iter); if (numexpunged > 0) { syslog(LOG_NOTICE, "Expunged %d messages from %s", numexpunged, mailbox_name(mailbox)); /* send the MessageExpunge or MessageExpire event notification */ mboxevent_extract_mailbox(mboxevent, mailbox); mboxevent_set_access(mboxevent, NULL, NULL, "", mailbox_name(mailbox), 0); mboxevent_set_numunseen(mboxevent, mailbox, -1); mboxevent_notify(&mboxevent); } mboxevent_free(&mboxevent); if (nexpunged) *nexpunged = numexpunged; return 0; } EXPORTED int mailbox_expunge_cleanup(struct mailbox *mailbox, time_t expunge_mark, unsigned *ndeleted) { int dirty = 0; unsigned numdeleted = 0; const message_t *msg; time_t first_expunged = 0; int r = 0; /* run the actual expunge phase */ struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, 0); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); /* already unlinked, skip it (but dirty so we mark a repack is needed) */ if (record->internal_flags & FLAG_INTERNAL_UNLINKED) { dirty = 1; continue; } /* not actually expunged, skip it */ if (!(record->internal_flags & FLAG_INTERNAL_EXPUNGED)) continue; /* not stale enough yet, skip it - but track the updated time * so we know when to run again */ if (record->last_updated > expunge_mark) { if (!first_expunged || (first_expunged > record->last_updated)) first_expunged = record->last_updated; continue; } dirty = 1; numdeleted++; struct index_record copyrecord = *record; copyrecord.internal_flags |= FLAG_INTERNAL_UNLINKED; copyrecord.silentupdate = 1; copyrecord.ignorelimits = 1; if (mailbox_rewrite_index_record(mailbox, ©record)) { xsyslog(LOG_ERR, "IOERROR: failed to mark unlinked", "mailbox=<%s> uid=<%u> recno=<%u>", mailbox_name(mailbox), copyrecord.uid, copyrecord.recno); break; } } mailbox_iter_done(&iter); if (dirty) { mailbox_index_dirty(mailbox); mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK; mailbox->i.first_expunged = first_expunged; } if (ndeleted) *ndeleted = numdeleted; return r; } EXPORTED int mailbox_internal_seen(const struct mailbox *mailbox, const char *userid) { /* old mailboxes don't have internal seen at all */ if (mailbox->i.minor_version < 12) return 0; /* shared seen - everyone's state is internal */ if (mailbox->i.options & OPT_IMAP_SHAREDSEEN) return 1; /* no username => use internal as well */ if (!userid) return 1; /* otherwise the owner's seen state is internal */ return mboxname_userownsmailbox(userid, mailbox_name(mailbox)); } /* * Return the number of message without \Seen flag in a mailbox. * Suppose that authenticated user is the owner or sharedseen is enabled */ unsigned mailbox_count_unseen(struct mailbox *mailbox) { assert(mailbox_index_islocked(mailbox, 0)); if (mailbox->i.minor_version > 13) return mailbox->i.unseen; const message_t *msg; unsigned count = 0; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_EXPUNGED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); if (!(record->system_flags & FLAG_SEEN)) count++; } mailbox_iter_done(&iter); return count; } /* returns a mailbox locked in MAILBOX EXCLUSIVE mode, so you * don't need to lock the index file to work with it :) */ EXPORTED int mailbox_create(const char *name, uint32_t mbtype, const char *part, const char *acl, const char *uniqueid, int options, unsigned uidvalidity, modseq_t createdmodseq, modseq_t highestmodseq, struct mailbox **mailboxptr) { int r = 0; char quotaroot[MAX_MAILBOX_BUFFER]; int hasquota; const char *fname; struct mailbox *mailbox = NULL; int n; int createfnames[] = { META_INDEX, META_HEADER, 0 }; struct mailboxlist *listitem; if (!uniqueid) uniqueid = makeuuid(); /* if we already have this name open then that's an error too */ listitem = find_listitem(name); if (listitem) return IMAP_MAILBOX_LOCKED; listitem = create_listitem(name); mailbox = &listitem->m; /* needs to be an exclusive namelock to create a mailbox */ char *userid = mboxname_to_userid(name); if (userid) { int haslock = user_isnamespacelocked(userid); assert(haslock == LOCK_EXCLUSIVE); free(userid); } uint32_t legacy_dirs = (mbtype & MBTYPE_LEGACY_DIRS); r = mboxname_lock(legacy_dirs ? name : uniqueid, &listitem->l, LOCK_EXCLUSIVE); if (r) { if (mailbox->local_namespacelock) mboxname_release(&mailbox->local_namespacelock); remove_listitem(listitem); return r; } // fill out the initial mbentry (XXX: pass it in from mboxlist.c) mailbox->mbentry = mboxlist_entry_create(); mailbox->mbentry->name = xstrdup(name); mailbox->mbentry->partition = xstrdup(part); mailbox->mbentry->acl = xstrdup(acl); mailbox->mbentry->mbtype = mbtype; mailbox->mbentry->uniqueid = xstrdup(uniqueid); // fill out the header too mailbox->h.name = xstrdup(name); mailbox->h.acl = xstrdup(acl); mailbox->h.mbtype = mbtype; mailbox->h.uniqueid = xstrdup(uniqueid); // if we've been given a highestmodseq, we don't update it if (highestmodseq) mailbox->silentchanges = 1; // set the quotaroot if any hasquota = quota_findroot(quotaroot, sizeof(quotaroot), name); if (hasquota) { mailbox->h.quotaroot = xstrdup(quotaroot); mailbox->quota_dirty = 1; } /* ensure all paths exist */ for (n = 0; createfnames[n]; n++) { fname = mailbox_meta_fname(mailbox, createfnames[n]); if (!fname) { xsyslog(LOG_ERR, "IOERROR: Mailbox name too long", "mailbox=<%s>", mailbox_name(mailbox)); r = IMAP_MAILBOX_BADNAME; goto done; } if (cyrus_mkdir(fname, 0755) == -1) { xsyslog(LOG_ERR, "IOERROR: mkdir failed", "fname=<%s>", fname); r = IMAP_IOERROR; goto done; } } /* ensure we can fit the longest possible file name */ fname = mailbox_datapath(mailbox, 0); if (!fname) { xsyslog(LOG_ERR, "IOERROR: Mailbox name too long", "mailbox=<%s>", mailbox_name(mailbox)); r = IMAP_MAILBOX_BADNAME; goto done; } /* and create the directory too :) */ if (cyrus_mkdir(fname, 0755) == -1) { xsyslog(LOG_ERR, "IOERROR: mkdir failed", "fname=<%s>", fname); r = IMAP_IOERROR; goto done; } /* create initial mbentry for new users -- the uniqueid in the record is required to open user metadata files (conversations, counters) */ if (mboxname_isusermailbox(mailbox_name(mailbox), 1) && mboxlist_lookup_by_uniqueid(mailbox_uniqueid(mailbox), NULL, NULL) != 0) { mbentry_t mbentry; memset(&mbentry, 0, sizeof(mbentry_t)); mbentry.mbtype = mbtype | MBTYPE_INTERMEDIATE; mbentry.name = (char *)mailbox_name(mailbox); mbentry.uniqueid = (char *)mailbox_uniqueid(mailbox); r = mboxlist_update(&mbentry, 1 /* localonly */); if (r) { syslog(LOG_ERR, "IOERROR: creating initial mbentry %s %s", mailbox_name(mailbox), error_message(r)); r = IMAP_IOERROR; goto done; } } fname = mailbox_meta_fname(mailbox, META_INDEX); if (!fname) { xsyslog(LOG_ERR, "IOERROR: Mailbox name too long", "mailbox=<%s>", mailbox_name(mailbox)); r = IMAP_MAILBOX_BADNAME; goto done; } mailbox->index_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666); if (mailbox->index_fd == -1) { xsyslog(LOG_ERR, "IOERROR: create index failed", "fname=<%s>", fname); r = IMAP_IOERROR; goto done; } r = lock_blocking(mailbox->index_fd, fname); if (r) { xsyslog(LOG_ERR, "IOERROR: lock index failed", "fname=<%s>", fname); r = IMAP_IOERROR; goto done; } mailbox->index_locktype = LOCK_EXCLUSIVE; /* ensure a UIDVALIDITY is set */ if (!uidvalidity) uidvalidity = mboxname_nextuidvalidity(name, time(0)); else mboxname_setuidvalidity(mailbox_name(mailbox), uidvalidity); mailbox->mbentry->uidvalidity = uidvalidity; /* and highest modseq */ if (!highestmodseq) highestmodseq = mboxname_nextmodseq(mailbox_name(mailbox), 0, mbtype, MBOXMODSEQ_ISFOLDER); else mboxname_setmodseq(mailbox_name(mailbox), highestmodseq, mbtype, MBOXMODSEQ_ISFOLDER); /* and created modseq */ if (!createdmodseq || createdmodseq > highestmodseq) createdmodseq = highestmodseq; /* init non-zero fields */ mailbox_index_dirty(mailbox); mailbox->i.minor_version = MAILBOX_MINOR_VERSION; mailbox->i.start_offset = INDEX_HEADER_SIZE; mailbox->i.record_size = INDEX_RECORD_SIZE; mailbox->i.options = options; mailbox->i.uidvalidity = uidvalidity; mailbox->i.createdmodseq = createdmodseq; mailbox->i.highestmodseq = highestmodseq; mailbox->i.changes_epoch = time(0); mailbox->i.synccrcs.basic = CRC_INIT_BASIC; mailbox->i.synccrcs.annot = CRC_INIT_ANNOT; /* initialise header size field so appends calculate the * correct map size */ mailbox->index_size = INDEX_HEADER_SIZE; mailbox->header_dirty = 1; r = seen_create_mailbox(NULL, mailbox); if (r) goto done; r = mailbox_commit(mailbox); if (r) goto done; if (config_auditlog) syslog(LOG_NOTICE, "auditlog: create sessionid=<%s> " "mailbox=<%s> uniqueid=<%s> uidvalidity=<%u>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox), mailbox->i.uidvalidity); done: if (!r && mailboxptr) *mailboxptr = mailbox; else mailbox_close(&mailbox); return r; } /* * Remove all files in directory */ static void mailbox_delete_files(const char *path) { DIR *dirp; struct dirent *f; char buf[MAX_MAILBOX_PATH+1]; char *tail; strlcpy(buf, path, sizeof(buf)); if (strlen(buf) >= sizeof(buf) - 2) { xsyslog(LOG_ERR, "IOERROR: path too long", "buf=<%s>", buf); fatal("path too long", EX_OSFILE); } tail = buf + strlen(buf); *tail++ = '/'; *tail = '\0'; dirp = opendir(path); if (dirp) { while ((f = readdir(dirp))!=NULL) { if (f->d_name[0] == '.' && (f->d_name[1] == '\0' || (f->d_name[1] == '.' && f->d_name[2] == '\0'))) { /* readdir() can return "." or "..", and I got a bug report that SCO might blow the file system to smithereens if we unlink(".."). Let's not do that. */ continue; } if(strlen(buf) + strlen(f->d_name) >= sizeof(buf)) { xsyslog(LOG_ERR, "IOERROR: path too long", "buf=<%s> d_name=<%s>", buf, f->d_name); fatal("Path too long", EX_OSFILE); } strcpy(tail, f->d_name); unlink(buf); *tail = '\0'; } closedir(dirp); } } /* Callback for use by cmd_delete */ static int chkchildren(const mbentry_t *mbentry, void *rock) { const char *part = (const char *)rock; if (!strcmpnull(part, mbentry->partition)) return CYRUSDB_DONE; return 0; } #ifdef WITH_DAV EXPORTED int mailbox_add_dav(struct mailbox *mailbox) { const message_t *msg; int r = 0; if (!mbtypes_dav(mailbox_mbtype(mailbox))) return 0; if (mboxname_isdeletedmailbox(mailbox_name(mailbox), NULL)) return 0; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); r = mailbox_update_dav(mailbox, NULL, record); if (r) break; } mailbox_iter_done(&iter); return r; } #endif /* WITH_DAV */ EXPORTED int mailbox_add_conversations(struct mailbox *mailbox, int silent) { const message_t *msg; int r = 0; struct conversations_state *cstate = mailbox_get_cstate(mailbox); if (!cstate) return 0; /* add record for mailbox */ conv_status_t status = CONV_STATUS_INIT; status.threadmodseq = mailbox->i.highestmodseq; r = conversation_setstatus(cstate, CONV_FOLDER_KEY_MBOX(cstate, mailbox), &status); if (r) return r; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); /* not assigned, skip */ if (!record->cid) continue; struct index_record copyrecord = *record; copyrecord.silentupdate = silent; r = conversations_update_record(cstate, mailbox, NULL, ©record, 1, /*ignorelimits*/1); if (r) break; if (copyrecord.cid == record->cid) continue; assert(!silent); // can't change cid if silent! /* remove this record again */ r = conversations_update_record(cstate, mailbox, ©record, NULL, 0, /*ignorelimits*/1); if (r) break; /* we had a cid change, so rewrite will try to correct the counts, so we * need to add this one in again */ struct index_record oldrecord = *record; /* add the old record that's going away */ r = conversations_update_record(cstate, mailbox, NULL, &oldrecord, 0, /*ignorelimits*/1); if (r) break; /* and finally to the update that will reverse those two actions again */ r = mailbox_rewrite_index_record(mailbox, ©record); if (r) break; } mailbox_iter_done(&iter); return r; } static int mailbox_delete_conversations(struct mailbox *mailbox) { struct conversations_state *cstate = mailbox_get_cstate(mailbox); const message_t *msg; int r = 0; if (!cstate) return 0; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); /* not assigned, skip */ if (!record->cid) continue; r = conversations_update_record(cstate, mailbox, record, NULL, /*allowrenumber*/0, /*ignorelimits*/1); if (r) break; } mailbox_iter_done(&iter); if (r) return r; return conversations_rename_folder(cstate, CONV_FOLDER_KEY_MBOX(cstate, mailbox), NULL); } static int mailbox_delete_internal(struct mailbox **mailboxptr) { int r = 0; struct mailbox *mailbox = *mailboxptr; /* mark the quota removed */ mailbox_quota_dirty(mailbox); /* mark the mailbox deleted */ mailbox_index_dirty(mailbox); mailbox->i.options |= OPT_MAILBOX_DELETED; #ifdef WITH_DAV /* remove any DAV records */ r = mailbox_delete_dav(mailbox); if (r) return r; #endif #ifdef USE_SIEVE /* remove any Sieve records */ r = mailbox_delete_sieve(mailbox); if (r) return r; #endif /* clean up annotations */ r = annotate_delete_mailbox(mailbox); if (r) return r; /* commit the changes */ r = mailbox_commit(mailbox); if (r) return r; /* remove any seen */ seen_delete_mailbox(NULL, mailbox); /* can't unlink any files yet, because our promise to other * users of the mailbox applies! Can only unlink with an * exclusive lock. mailbox_close will try to get one of * those. */ syslog(LOG_NOTICE, "Deleted mailbox %s", mailbox_name(mailbox)); if (config_auditlog) syslog(LOG_NOTICE, "auditlog: delete sessionid=<%s> " "mailbox=<%s> uniqueid=<%s>", session_id(), mailbox_name(mailbox), mailbox_uniqueid(mailbox)); proc_killmbox(mailbox_name(mailbox)); mailbox_close(mailboxptr); return 0; } #ifdef WITH_JMAP static int mailbox_delete_alarms(struct mailbox *mailbox) { if (!(mailbox->i.options & OPT_IMAP_HAS_ALARMS)) return 0; return caldav_alarm_delete_mailbox(mailbox_name(mailbox)); } #endif /* WITH_JMAP */ #ifdef WITH_DAV static int mailbox_delete_caldav(struct mailbox *mailbox) { struct caldav_db *caldavdb = NULL; caldavdb = caldav_open_mailbox(mailbox); if (caldavdb) { const mbentry_t mbentry = { .name = (char *)mailbox_name(mailbox), .uniqueid = (char *)mailbox_uniqueid(mailbox) }; int r = caldav_delmbox(caldavdb, &mbentry); caldav_close(caldavdb); if (r) return r; } int r = caldav_alarm_delete_mailbox(mailbox_name(mailbox)); if (r) return r; return 0; } static int mailbox_delete_carddav(struct mailbox *mailbox) { struct carddav_db *carddavdb = NULL; carddavdb = carddav_open_mailbox(mailbox); if (carddavdb) { const mbentry_t mbentry = { .name = (char *)mailbox_name(mailbox), .uniqueid = (char *)mailbox_uniqueid(mailbox) }; int r = carddav_delmbox(carddavdb, &mbentry); carddav_close(carddavdb); if (r) return r; } return 0; } static int mailbox_delete_webdav(struct mailbox *mailbox) { struct webdav_db *webdavdb = NULL; webdavdb = webdav_open_mailbox(mailbox); if (webdavdb) { const mbentry_t mbentry = { .name = (char *)mailbox_name(mailbox), .uniqueid = (char *)mailbox_uniqueid(mailbox) }; int r = webdav_delmbox(webdavdb, &mbentry); webdav_close(webdavdb); if (r) return r; } return 0; } EXPORTED int mailbox_delete_dav(struct mailbox *mailbox) { switch (mbtype_isa(mailbox_mbtype(mailbox))) { case MBTYPE_ADDRESSBOOK: return mailbox_delete_carddav(mailbox); case MBTYPE_CALENDAR: return mailbox_delete_caldav(mailbox); case MBTYPE_COLLECTION: return mailbox_delete_webdav(mailbox); } return 0; } #endif /* WITH_DAV */ /* * Delete and close the mailbox 'mailbox'. Closes 'mailbox' whether * or not the deletion was successful. Requires a locked mailbox. */ EXPORTED int mailbox_delete(struct mailbox **mailboxptr) { struct mailbox *mailbox = *mailboxptr; int r; r = mailbox_delete_conversations(mailbox); if (r) return r; #ifdef WITH_JMAP r = mailbox_delete_alarms(mailbox); if (r) return r; #endif /* WITH_JMAP */ #ifdef WITH_DAV r = mailbox_delete_dav(mailbox); if (r) return r; #endif /* WITH_DAV */ #ifdef USE_SIEVE r = mailbox_delete_sieve(mailbox); if (r) return r; #endif /* USE_SIEVE */ return mailbox_delete_internal(mailboxptr); } struct meta_file { unsigned long metaflag; int optional; int nolink; }; static struct meta_file meta_files[] = { { META_HEADER, 0, 1 }, { META_INDEX, 0, 1 }, { META_CACHE, 1, 1 }, { META_SQUAT, 1, 0 }, { META_ANNOTATIONS, 1, 1 }, { META_ARCHIVECACHE, 1, 1 }, { 0, 0, 0 } }; /* XXX - move this part of cleanup into mboxlist. Really * needs to be done with mailboxes.db locked so nobody can * try to create a mailbox while the delete is underway. * VERY tight race condition exists right now... */ /* we need an exclusive namelock for this */ HIDDEN int mailbox_delete_cleanup(struct mailbox *mailbox, const char *part, const char *name, const char *uniqueid) { strarray_t paths = STRARRAY_INITIALIZER; int i; mbentry_t *mbentry; struct meta_file *mf; int r; char nbuf[MAX_MAILBOX_NAME]; char *ntail; char *p; int object_storage_enabled = 0 ; #if defined ENABLE_OBJECTSTORE object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED); #endif xstrncpy(nbuf, name, sizeof(nbuf)); ntail = nbuf + strlen(nbuf); if (mailbox && object_storage_enabled){ mailbox_remove_files_from_object_storage (mailbox, 0) ; } /* XXX - double XXX - this is a really ugly function. It should be * using mboxname_parts and walking back up the 'boxes' list */ strarray_add(&paths, mboxname_datapath(part, name, uniqueid, 0)); /* find the directory for every one of the meta files */ for (mf = meta_files; mf->metaflag; mf++) { char *fname = xstrdup(mboxname_metapath(part, name, uniqueid, mf->metaflag, 0)); p = strrchr(fname, '/'); if (p) *p = '\0'; strarray_add(&paths, fname); free(fname); } for (i = 0; i < paths.count; i++) { const char *path = strarray_nth(&paths, i); mailbox_delete_files(path); } do { if (!uniqueid) { /* paths by mboxname - Check if the mailbox has children */ r = mboxlist_mboxtree(nbuf, chkchildren, (void *)part, MBOXTREE_SKIP_ROOT); if (r != 0) break; /* We short-circuit with CYRUSDB_DONE */ } /* no children, remove the directories */ for (i = 0; i < paths.count; i++) { char *path = paths.data[i]; /* need direct reference, because we're fiddling */ r = rmdir(path); if (r && errno != ENOENT) syslog(LOG_NOTICE, "Remove of supposedly empty directory %s failed: %m", path); p = strrchr(path, '/'); if (p) *p = '\0'; } /* Check if parent mailbox exists */ ntail = strrchr(nbuf, '.'); if (!ntail || strchr(ntail, '!')) { /* Hit top of hierarchy or domain separator */ break; } *ntail = '\0'; if (!strcmp(nbuf, "user") || ((ntail - nbuf > 5) && !strcmp(ntail-5, "!user"))) { /* Hit top of 'user' hierarchy */ break; } r = mboxlist_lookup(nbuf, &mbentry, NULL); /* if it's not being moved, and not the same partition, then it's safe to * clean up the parent directory too */ if (!r) { if (!(mbentry->mbtype & MBTYPE_MOVING) && strcmp(mbentry->partition, part)) r = IMAP_MAILBOX_NONEXISTENT; mboxlist_entry_free(&mbentry); } } while (r == IMAP_MAILBOX_NONEXISTENT); strarray_fini(&paths); return 0; } EXPORTED int mailbox_copy_files(struct mailbox *mailbox, const char *newpart, const char *newname, const char *newuniqueid) { char oldbuf[MAX_MAILBOX_PATH], newbuf[MAX_MAILBOX_PATH]; struct meta_file *mf; const message_t *msg; int r = 0; int object_storage_enabled = 0 ; #if defined ENABLE_OBJECTSTORE object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED); #endif /* Copy over meta files */ for (mf = meta_files; mf->metaflag; mf++) { struct stat sbuf; xstrncpy(oldbuf, mailbox_meta_fname(mailbox, mf->metaflag), MAX_MAILBOX_PATH); xstrncpy(newbuf, mboxname_metapath(newpart, newname, newuniqueid, mf->metaflag, 0), MAX_MAILBOX_PATH); unlink(newbuf); /* Make link() possible */ if (!mf->optional || stat(oldbuf, &sbuf) != -1) { r = mailbox_copyfile(oldbuf, newbuf, mf->nolink); if (r) return r; } } struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); xstrncpy(oldbuf, mailbox_record_fname(mailbox, record), MAX_MAILBOX_PATH); if (!object_storage_enabled && record->internal_flags & FLAG_INTERNAL_ARCHIVED) xstrncpy(newbuf, mboxname_archivepath(newpart, newname, newuniqueid, record->uid), MAX_MAILBOX_PATH); else xstrncpy(newbuf, mboxname_datapath(newpart, newname, newuniqueid, record->uid), MAX_MAILBOX_PATH); if (!(object_storage_enabled && record->internal_flags & FLAG_INTERNAL_ARCHIVED)) // if object storage do not move file r = mailbox_copyfile(oldbuf, newbuf, 0); if (r) break; #if defined ENABLE_OBJECTSTORE if (object_storage_enabled && record->internal_flags & FLAG_INTERNAL_ARCHIVED) { static struct mailbox new_mailbox; memset(&new_mailbox, 0, sizeof(struct mailbox)); new_mailbox.name = (char*) newname; new_mailbox.part = (char*) config_defpartition ; r = objectstore_put(&new_mailbox, record, newbuf); // put should just add to refcount. if (r) break; } #endif } mailbox_iter_done(&iter); return r; } HIDDEN int mailbox_rename_nocopy(struct mailbox *oldmailbox, const char *newname, int silent) { char quotaroot[MAX_MAILBOX_BUFFER]; int hasquota = quota_findroot(quotaroot, sizeof(quotaroot), newname); /* Move any quota usage */ int r = mailbox_changequotaroot(oldmailbox, hasquota ? quotaroot: NULL, silent); if (!r) { /* copy any mailbox annotations */ mbentry_t newmbentry = MBENTRY_INITIALIZER; newmbentry.name = (char *) newname; struct mailbox newmailbox = { .mbentry = &newmbentry, .index_locktype = LOCK_EXCLUSIVE }; r = annotate_rename_mailbox(oldmailbox, &newmailbox); } if (!r && mailbox_has_conversations(oldmailbox)) { struct conversations_state *oldcstate = mailbox_get_cstate(oldmailbox); assert(oldcstate); if (mboxname_isdeletedmailbox(newname, NULL)) { /* we never store data about deleted mailboxes */ r = mailbox_delete_conversations(oldmailbox); } else if (oldcstate->folders_byname) { /* we can just rename within the same user */ r = conversations_rename_folder(oldcstate, mailbox_name(oldmailbox), newname); } // otherwise, we don't need to rename because it's the same uniqueid } /* unless on a replica, bump the modseq */ if (!silent) mailbox_modseq_dirty(oldmailbox); /* update the name in the header */ xzfree(oldmailbox->h.name); oldmailbox->h.name = xstrdup(newname); oldmailbox->header_dirty = 1; return r; } /* if 'userid' is set, we perform the funky RENAME INBOX INBOX.old semantics, regardless of whether or not the name of the mailbox is 'user.foo'.*/ /* requires a write-locked oldmailbox pointer, since we delete it immediately afterwards */ /* This function ONLY WORKS if the type is legacy */ HIDDEN int mailbox_rename_copy(struct mailbox *oldmailbox, const char *newname, const char *newpartition, unsigned uidvalidity, int ignorequota, int silent, struct mailbox **newmailboxptr) { int r; struct mailbox *newmailbox = NULL; struct conversations_state *oldcstate = NULL; struct conversations_state *newcstate = NULL; char *newquotaroot = NULL; char *newuniqueid = NULL; assert(mailbox_index_islocked(oldmailbox, 1)); assert(mailbox_mbtype(oldmailbox) & MBTYPE_LEGACY_DIRS); /* we can't rename back from a deleted mailbox, because the conversations * information will be wrong. Ideally we might re-calculate, but for now * we just throw a big fat error */ if (config_getswitch(IMAPOPT_CONVERSATIONS) && mboxname_isdeletedmailbox(mailbox_name(oldmailbox), NULL)) { syslog(LOG_ERR, "can't rename a deleted mailbox %s", mailbox_name(oldmailbox)); return IMAP_MAILBOX_BADNAME; } /* create uidvalidity if not explicitly requested */ if (!uidvalidity) uidvalidity = mboxname_nextuidvalidity(newname, oldmailbox->i.uidvalidity); modseq_t highestmodseq = silent ? oldmailbox->i.highestmodseq : 0; /* Create new mailbox */ r = mailbox_create(newname, mailbox_mbtype(oldmailbox), newpartition, mailbox_acl(oldmailbox), mailbox_uniqueid(oldmailbox), oldmailbox->i.options, uidvalidity, oldmailbox->i.createdmodseq, highestmodseq, &newmailbox); if (r) return r; /* Check quota if necessary */ if (!ignorequota && newmailbox->h.quotaroot && strcmpsafe(oldmailbox->h.quotaroot, newmailbox->h.quotaroot)) { quota_t usage[QUOTA_NUMRESOURCES]; mailbox_get_usage(oldmailbox, usage); r = mailbox_quota_check(newmailbox, usage); /* then we abort - no space to rename */ if (r) goto fail; } newquotaroot = xstrdupnull(newmailbox->h.quotaroot); r = mailbox_copy_files(oldmailbox, newpartition, newname, mailbox_mbtype(newmailbox) & MBTYPE_LEGACY_DIRS ? NULL : mailbox_uniqueid(newmailbox)); if (r) goto fail; /* Re-open index file */ r = mailbox_open_index(newmailbox); if (r) goto fail; /* cyrus.header has been copied with old uniqueid. make a copy of new uniqueid so we can reset it */ newuniqueid = xstrdup(mailbox_uniqueid(newmailbox)); /* Re-lock index */ r = mailbox_lock_index_internal(newmailbox, LOCK_EXCLUSIVE); /* Reset new uniqueid */ free(newmailbox->h.uniqueid); newmailbox->h.uniqueid = xstrdup(newuniqueid); newmailbox->header_dirty = 1; /* update mailbox annotations if necessary */ r = annotate_rename_mailbox(oldmailbox, newmailbox); if (r) goto fail; /* mark the "used" back to zero, so it updates the new quota! */ mailbox_set_quotaroot(newmailbox, newquotaroot); mailbox_quota_dirty(newmailbox); memset(newmailbox->quota_previously_used, 0, sizeof(newmailbox->quota_previously_used)); /* except this one... because we've counted it when we created the folder */ newmailbox->quota_previously_used[QUOTA_NUMFOLDERS] = 1; /* re-set the UIDVALIDITY, it will have been the old one in the index header */ mailbox_index_dirty(newmailbox); newmailbox->i.uidvalidity = uidvalidity; /* unless on a replica, bump the modseq too */ if (!silent) mailbox_modseq_dirty(newmailbox); /* NOTE: in the case of renaming a user to another user, we * don't rename the conversations DB - instead we re-create * the records in the target user. Sorry, was too complex * otherwise handling all the special cases */ if (mailbox_has_conversations(oldmailbox)) { oldcstate = mailbox_get_cstate(oldmailbox); assert(oldcstate); } if (mailbox_has_conversations(newmailbox)) { newcstate = mailbox_get_cstate(newmailbox); assert(newcstate); } if (oldcstate && newcstate && !strcmp(oldcstate->path, newcstate->path)) { /* we can just rename within the same user */ if (oldcstate->folders_byname) { r = conversations_rename_folder(oldcstate, mailbox_name(oldmailbox), newname); } } else { /* have to handle each one separately */ if (newcstate) r = mailbox_add_conversations(newmailbox, /*silent*/0); if (oldcstate) r = mailbox_delete_conversations(oldmailbox); } if (r) goto fail; /* commit the index changes */ r = mailbox_commit(newmailbox); if (r) goto fail; if (config_auditlog) syslog(LOG_NOTICE, "auditlog: rename sessionid=<%s> " "oldmailbox=<%s> newmailbox=<%s> uniqueid=<%s>", session_id(), mailbox_name(oldmailbox), newname, mailbox_uniqueid(newmailbox)); if (newmailboxptr) *newmailboxptr = newmailbox; else mailbox_close(&newmailbox); free(newquotaroot); free(newuniqueid); return 0; fail: /* first unlock so we don't need to write anything new down */ mailbox_unlock_index(newmailbox, NULL); /* then remove all the files */ mailbox_delete_cleanup(NULL, mailbox_partition(newmailbox), mailbox_name(newmailbox), (mailbox_mbtype(newmailbox) & MBTYPE_LEGACY_DIRS) ? NULL : mailbox_uniqueid(newmailbox)); /* and finally, abort */ mailbox_abort(newmailbox); mailbox_close(&newmailbox); free(newquotaroot); free(newuniqueid); return r; } EXPORTED int mailbox_rename_cleanup(struct mailbox **mailboxptr) { int r = 0; struct mailbox *oldmailbox = *mailboxptr; char *name = xstrdup(mailbox_name(oldmailbox)); r = mailbox_delete_internal(mailboxptr); if (r) { syslog(LOG_CRIT, "Rename Failure during mailbox_rename_cleanup (%s), " \ "potential leaked space (%s)", name, error_message(r)); } free(name); return r; } /* * Copy (or link) the file 'from' to the file 'to' */ EXPORTED int mailbox_copyfile(const char *from, const char *to, int nolink) { int flags = COPYFILE_MKDIR|COPYFILE_KEEPTIME; if (nolink) flags |= COPYFILE_NOLINK; if (mailbox_wait_cb) mailbox_wait_cb(mailbox_wait_cb_rock); if (cyrus_copyfile(from, to, flags)) return IMAP_IOERROR; return 0; } /* ---------------------------------------------------------------------- */ /* RECONSTRUCT SUPPORT */ /* ---------------------------------------------------------------------- */ #define UIDGROW 300 struct found_uid { uint32_t uid; unsigned isarchive:1; unsigned issnoozed:1; }; struct found_uids { struct found_uid *found; unsigned nalloc; unsigned nused; unsigned pos; }; #define FOUND_UIDS_INITIALIZER \ { NULL, 0, 0, 0 } static int sort_found(const void *a, const void *b) { struct found_uid *fa = (struct found_uid *)a; struct found_uid *fb = (struct found_uid *)b; if (fa->uid != fb->uid) return fa->uid - fb->uid; return fa->isarchive - fb->isarchive; } static void add_found(struct found_uids *ff, uint32_t uid, int isarchive) { /* make sure there's space */ if (ff->nused >= ff->nalloc) { ff->nalloc += UIDGROW; ff->found = xrealloc(ff->found, ff->nalloc * sizeof(ff->found[0])); } ff->found[ff->nused].uid = uid; ff->found[ff->nused].isarchive = !!isarchive; ff->found[ff->nused].issnoozed = 0; ff->nused++; } static void free_found(struct found_uids *ff) { free(ff->found); ff->found = NULL; ff->nalloc = 0; ff->nused = 0; ff->pos = 0; } EXPORTED int mailbox_parse_datafilename(const char *name, uint32_t *uidp) { const char *p = name; /* must be at least one digit */ if (!cyrus_isdigit(*p)) return IMAP_MAILBOX_BADNAME; do { p++; } while cyrus_isdigit(*p); /* has to end with a dot */ if (*p != '.') return IMAP_MAILBOX_BADNAME; if (p[1]) return IMAP_MAILBOX_BADNAME; return parseuint32(name, &p, uidp); } static int find_files(struct mailbox *mailbox, struct found_uids *files, int flags) { strarray_t paths = STRARRAY_INITIALIZER; DIR *dirp; struct dirent *dirent; uint32_t uid; const char *p; char buf[MAX_MAILBOX_PATH]; struct stat sbuf; int r; int i; strarray_add(&paths, mailbox_spool_fname(mailbox, 0)); #if defined ENABLE_OBJECTSTORE if (config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED)) { uint32_t i , count = 0 ; struct message *list = get_list_of_message (mailbox, &count) ; for (i = 0; i < count; i++) { add_found(files, list [i].message_uid, 1 /* isarchive */ ); } discard_list( ) ; } else #endif strarray_add(&paths, mailbox_archive_fname(mailbox, 0)); for (i = 0; i < paths.count; i++) { const char *dirpath = strarray_nth(&paths, i); int isarchive = strcmp(dirpath, mailbox_datapath(mailbox, 0)); dirp = opendir(dirpath); if (!dirp) continue; /* data directory is fine */ while ((dirent = readdir(dirp)) != NULL) { p = dirent->d_name; if (*p == '.') continue; /* dot files */ if (!strncmp(p, "cyrus.", 6)) continue; /* cyrus.* files */ r = mailbox_parse_datafilename(p, &uid); if (r) { /* check if it's a directory */ snprintf(buf, MAX_MAILBOX_PATH, "%s/%s", dirpath, dirent->d_name); if (stat(buf, &sbuf) == -1) continue; /* ignore ephemeral */ if (!S_ISDIR(sbuf.st_mode)) { if (!(flags & RECONSTRUCT_IGNORE_ODDFILES)) { printf("%s odd file %s\n", mailbox_name(mailbox), buf); syslog(LOG_ERR, "%s odd file %s", mailbox_name(mailbox), buf); if (flags & RECONSTRUCT_REMOVE_ODDFILES) unlink(buf); else { printf("run reconstruct with -O to remove odd files\n"); syslog(LOG_ERR, "run reconstruct with -O to " "remove odd files"); } } } } else { /* it's one of ours :) */ add_found(files, uid, isarchive); } } closedir(dirp); } /* make sure UIDs are sorted for comparison */ qsort(files->found, files->nused, sizeof(files->found[0]), sort_found); strarray_fini(&paths); return 0; } static void cleanup_stale_expunged(struct mailbox *mailbox) { const char *fname; int expunge_fd = -1; const char *expunge_base = NULL; size_t expunge_len = 0; /* mapped size */ unsigned long expunge_num; unsigned long emapnum; uint32_t erecno; uint32_t eversion; bit32 eoffset, expungerecord_size; const char *bufp; struct stat sbuf; int r; /* it's always read-writes */ fname = mailbox_meta_fname(mailbox, META_EXPUNGE); expunge_fd = open(fname, O_RDWR, 0); if (expunge_fd == -1) goto done; /* yay, no crappy expunge file */ /* boo - gotta read and find out the UIDs */ r = fstat(expunge_fd, &sbuf); if (r == -1) goto done; if (sbuf.st_size < INDEX_HEADER_SIZE) goto done; map_refresh(expunge_fd, 1, &expunge_base, &expunge_len, sbuf.st_size, "expunge", mailbox_name(mailbox)); /* use the expunge file's header information just in case * versions are skewed for some reason */ eoffset = ntohl(*((bit32 *)(expunge_base+OFFSET_START_OFFSET))); expungerecord_size = ntohl(*((bit32 *)(expunge_base+OFFSET_RECORD_SIZE))); /* bogus data at the start of the expunge file? */ if (!eoffset || !expungerecord_size) goto done; expunge_num = ntohl(*((bit32 *)(expunge_base+OFFSET_NUM_RECORDS))); eversion = ntohl(*((bit32 *)(expunge_base+OFFSET_MINOR_VERSION))); emapnum = (sbuf.st_size - eoffset) / expungerecord_size; if (emapnum < expunge_num) { expunge_num = emapnum; } /* add every UID to the files list */ for (erecno = 1; erecno <= expunge_num; erecno++) { struct index_record record; bufp = expunge_base + eoffset + (erecno-1)*expungerecord_size; mailbox_buf_to_index_record(bufp, eversion, &record, 0); record.internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED; mailbox_record_cleanup(mailbox, &record); } fname = mailbox_meta_fname(mailbox, META_EXPUNGE); unlink(fname); done: if (expunge_base) map_free(&expunge_base, &expunge_len); xclose(expunge_fd); } /* this is kind of like mailbox_create, but we try to rescue * what we can from the filesystem! */ static int mailbox_reconstruct_create(const char *name, struct mailbox **mbptr) { struct mailbox *mailbox = NULL; int options = config_getint(IMAPOPT_MAILBOX_DEFAULT_OPTIONS) | OPT_POP3_NEW_UIDL; mbentry_t *mbentry = NULL; struct mailboxlist *listitem; int r; /* make sure it's not already open. Very odd, since we already * discovered it's not openable! */ listitem = find_listitem(name); if (listitem) return IMAP_MAILBOX_LOCKED; listitem = create_listitem(name); mailbox = &listitem->m; // lock the user namespace FIRST before the mailbox namespace char *userid = mboxname_to_userid(name); if (userid) { int haslock = user_isnamespacelocked(userid); if (haslock) { assert(haslock != LOCK_SHARED); } else { int locktype = LOCK_EXCLUSIVE; mailbox->local_namespacelock = user_namespacelock_full(userid, locktype); } free(userid); } /* Start by looking up current data in mailbox list */ /* XXX - no mboxlist entry? Can we recover? */ r = mboxlist_lookup(name, &mbentry, NULL); if (r) goto done; /* if we can't get an exclusive lock first try, there's something * racy going on! */ uint32_t legacy_dirs = (mbentry->mbtype & MBTYPE_LEGACY_DIRS); r = mboxname_lock(legacy_dirs ? name : mbentry->uniqueid, &listitem->l, LOCK_EXCLUSIVE); if (r) goto done; mailbox->mbentry = mboxlist_entry_copy(mbentry); syslog(LOG_NOTICE, "create new mailbox %s", name); /* Attempt to open index */ r = mailbox_open_index(mailbox); if (!r) r = mailbox_read_index_header(mailbox); if (r) { printf("%s: failed to read index header\n", mailbox_name(mailbox)); syslog(LOG_ERR, "failed to read index header for %s", mailbox_name(mailbox)); /* no cyrus.index file at all - well, we're in a pickle! * no point trying to rescue anything else... */ mailbox_close(&mailbox); r = mailbox_create(name, mbentry->mbtype, mbentry->partition, mbentry->acl, mbentry->uniqueid, options, 0, 0, 0, mbptr); mboxlist_entry_free(&mbentry); return r; } mboxlist_entry_free(&mbentry); /* read header, if it is not there, we need to create it */ r = mailbox_read_header(mailbox, NULL); if (r) { /* Header failed to read - recreate it */ printf("%s: failed to read header file\n", mailbox_name(mailbox)); syslog(LOG_ERR, "failed to read header file for %s", mailbox_name(mailbox)); mailbox_make_uniqueid(mailbox); r = mailbox_commit(mailbox); if (r) goto done; } if (mailbox->header_file_crc != mailbox->i.header_file_crc) { mailbox->i.header_file_crc = mailbox->header_file_crc; printf("%s: header file CRC mismatch, correcting\n", mailbox_name(mailbox)); syslog(LOG_ERR, "%s: header file CRC mismatch, correcting", mailbox_name(mailbox)); mailbox_index_dirty(mailbox); r = mailbox_commit(mailbox); if (r) goto done; } done: if (r) mailbox_close(&mailbox); else *mbptr = mailbox; return r; } static int mailbox_reconstruct_acl(struct mailbox *mailbox, int flags) { int make_changes = flags & RECONSTRUCT_MAKE_CHANGES; int r; r = mailbox_read_header(mailbox, NULL); if (r) return r; if (strcmp(mailbox_acl(mailbox), mailbox->h.acl)) { printf("%s: update acl from header %s => %s\n", mailbox_name(mailbox), mailbox_acl(mailbox), mailbox->h.acl); if (make_changes) printf("XXX - this is a noop right now - needs to update mailboxes.db\n"); } return r; } static int records_match(const char *mboxname, struct index_record *old, struct index_record *new) { int i; int match = 1; int userflags_dirty = 0; if (old->internaldate != new->internaldate) { printf("%s uid %u mismatch: internaldate\n", mboxname, new->uid); match = 0; } if (old->sentdate != new->sentdate) { printf("%s uid %u mismatch: sentdate\n", mboxname, new->uid); match = 0; } if (old->size != new->size) { printf("%s uid %u mismatch: size\n", mboxname, new->uid); match = 0; } if (old->header_size != new->header_size) { printf("%s uid %u mismatch: header_size\n", mboxname, new->uid); match = 0; } if (old->gmtime != new->gmtime) { printf("%s uid %u mismatch: gmtime\n", mboxname, new->uid); match = 0; } if (old->savedate != new->savedate) { printf("%s uid %u mismatch: savedate\n", mboxname, new->uid); match = 0; } if (old->createdmodseq != new->createdmodseq) { printf("%s uid %u mismatch: createdmodseq\n", mboxname, new->uid); match = 0; } if (old->system_flags != new->system_flags) { printf("%s uid %u mismatch: systemflags\n", mboxname, new->uid); match = 0; } if (old->internal_flags != new->internal_flags) { printf("%s uid %u mismatch: internalflags\n", mboxname, new->uid); match = 0; } for (i = 0; i < MAX_USER_FLAGS/32; i++) { if (old->user_flags[i] != new->user_flags[i]) userflags_dirty = 1; } if (userflags_dirty) { printf("%s uid %u mismatch: userflags\n", mboxname, new->uid); match = 0; } if (!message_guid_equal(&old->guid, &new->guid)) { printf("%s uid %u mismatch: guid\n", mboxname, new->uid); match = 0; } if (!match) { syslog(LOG_ERR, "%s uid %u record mismatch, rewriting", mboxname, new->uid); } /* cache issues - don't print, probably just a version * upgrade... */ if (old->cache_version != new->cache_version) { match = 0; } if (old->cache_crc != new->cache_crc) { match = 0; } if (cache_len(old) != cache_len(new)) { match = 0; } /* only compare cache records if size matches */ else if (memcmp(cache_base(old), cache_base(new), cache_len(new))) { match = 0; } return match; } static int mailbox_reconstruct_compare_update(struct mailbox *mailbox, struct index_record *record, bit32 *valid_user_flags, int flags, int have_file, int has_snoozedannot, struct found_uids *discovered) { const char *fname = mailbox_record_fname(mailbox, record); int r = 0; int i; struct index_record copy; struct stat sbuf; int make_changes = flags & RECONSTRUCT_MAKE_CHANGES; int re_parse = flags & RECONSTRUCT_ALWAYS_PARSE; int do_stat = flags & RECONSTRUCT_DO_STAT; int re_pack = 0; int did_stat = 0; int remove_temp_spool_file = 0 ; #if defined ENABLE_OBJECTSTORE int object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED) ; if (object_storage_enabled && record->system_flags & FLAG_INTERNAL_ARCHIVED){ // put the file on spool temporarily remove_temp_spool_file = 1; r = objectstore_get(mailbox, record, fname); } #endif /* does the file actually exist? */ if (have_file && do_stat) { if (stat(fname, &sbuf) == -1 || (sbuf.st_size == 0)) { have_file = 0; } else if (record->size != (unsigned) sbuf.st_size) { re_parse = 1; } did_stat = 1; } if (!have_file) { /* well, that's OK if it's supposed to be missing! */ if (record->internal_flags & FLAG_INTERNAL_UNLINKED) { r = 0 ; goto out; } printf("%s uid %u not found\n", mailbox_name(mailbox), record->uid); syslog(LOG_ERR, "%s uid %u not found", mailbox_name(mailbox), record->uid); if (!make_changes) { r = 0 ; goto out; } /* otherwise we have issues, mark it unlinked */ unlink(fname); record->internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED; mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK; r = mailbox_rewrite_index_record(mailbox, record); goto out; } if (mailbox_cacherecord_internal(mailbox, record, MBCACHE_NOPARSE) || record->crec.len == 0) { re_parse = 1; re_pack = 1; /* cache record will have to be rewritten */ } /* copy once the cache record is read in... */ copy = *record; if (!record->internaldate) { re_parse = 1; } /* re-calculate all the "derived" fields by parsing the file on disk */ if (re_parse) { /* set NULL in case parse finds a new value */ record->internaldate = 0; r = message_parse(fname, record); if (r) goto out; /* unchanged, keep the old value */ if (!record->internaldate) record->internaldate = copy.internaldate; /* it's not the same message! */ if (!message_guid_equal(&record->guid, ©.guid)) { int do_unlink = 0; printf("%s uid %u guid mismatch\n", mailbox_name(mailbox), record->uid); syslog(LOG_ERR, "%s uid %u guid mismatch", mailbox_name(mailbox), record->uid); if (!make_changes) { r = 0 ; goto out; } if (record->internal_flags & FLAG_INTERNAL_EXPUNGED) { /* already expunged, just unlink it */ printf("%s uid %u already expunged, unlinking\n", mailbox_name(mailbox), record->uid); syslog(LOG_ERR, "%s uid %u already expunged, unlinking", mailbox_name(mailbox), record->uid); do_unlink = 1; } else if (flags & RECONSTRUCT_GUID_REWRITE) { /* treat this file as discovered */ add_found(discovered, record->uid, (record->internal_flags & FLAG_INTERNAL_ARCHIVED)); printf("%s uid %u marking for uid upgrade\n", mailbox_name(mailbox), record->uid); syslog(LOG_ERR, "%s uid %u marking for uid upgrade", mailbox_name(mailbox), record->uid); do_unlink = 1; } else if (flags & RECONSTRUCT_GUID_UNLINK) { printf("%s uid %u unlinking as requested with -U\n", mailbox_name(mailbox), record->uid); syslog(LOG_ERR, "%s uid %u unlinking as requested with -U", mailbox_name(mailbox), record->uid); do_unlink = 1; } if (do_unlink) { /* rewrite with the original so we don't break the * expectation that GUID never changes */ copy.internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED; mailbox->i.options |= OPT_MAILBOX_NEEDS_UNLINK; r = mailbox_rewrite_index_record(mailbox, ©); goto out; } /* otherwise we just report it and move on - hopefully the * correct file can be restored from backup or something */ printf("run reconstruct with -R to fix or -U to remove\n"); syslog(LOG_ERR, "run reconstruct with -R to fix or -U to remove"); r = 0 ; goto out; } } if (!record->size) { /* dang, guess it failed to parse */ printf("%s uid %u failed to parse\n", mailbox_name(mailbox), record->uid); syslog(LOG_ERR, "%s uid %u failed to parse", mailbox_name(mailbox), record->uid); if (!make_changes) { r = 0 ; goto out; } /* otherwise we have issues, mark it unlinked */ unlink(fname); record->internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED; mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK; r = mailbox_rewrite_index_record(mailbox, record); goto out; } /* get internaldate from the file if not set */ if (!record->internaldate) { if (did_stat || stat(fname, &sbuf) != -1) record->internaldate = sbuf.st_mtime; else record->internaldate = time(NULL); } if (!record->gmtime) record->gmtime = record->internaldate; if (!record->sentdate) { struct tm *tm = localtime(&record->internaldate); /* truncate to the day */ tm->tm_sec = 0; tm->tm_min = 0; tm->tm_hour = 0; record->sentdate = mktime(tm); } /* XXX - conditions under which modseq or uid or internaldate could be bogus? */ if (record->modseq > mailbox->i.highestmodseq) { printf("%s uid %u future modseq " MODSEQ_FMT " found\n", mailbox_name(mailbox), record->uid, record->modseq); syslog(LOG_ERR, "%s uid %u future modseq " MODSEQ_FMT " found", mailbox_name(mailbox), record->uid, record->modseq); mailbox_index_dirty(mailbox); mailbox->i.highestmodseq = mboxname_setmodseq(mailbox_name(mailbox), record->modseq, mailbox_mbtype(mailbox), /*flags*/0); } if (record->uid > mailbox->i.last_uid) { printf("%s future uid %u found\n", mailbox_name(mailbox), record->uid); syslog(LOG_ERR, "%s future uid %u found", mailbox_name(mailbox), record->uid); mailbox_index_dirty(mailbox); mailbox->i.last_uid = record->uid; } /* remove any user_flags that are missing from the header */ for (i = 0; i < MAX_USER_FLAGS/32; i++) { record->user_flags[i] &= valid_user_flags[i]; } /* check if the snoozed status matches (unless expunged, which shouldn't be snoozed) */ if (!(record->internal_flags & FLAG_INTERNAL_EXPUNGED) && !!(record->internal_flags & FLAG_INTERNAL_SNOOZED) != !!has_snoozedannot) { printf("%s uid %u snoozed mismatch\n", mailbox_name(mailbox), record->uid); syslog(LOG_ERR, "%s uid %u snoozed mismatch", mailbox_name(mailbox), record->uid); if (has_snoozedannot) record->internal_flags |= FLAG_INTERNAL_SNOOZED; else record->internal_flags &= ~FLAG_INTERNAL_SNOOZED; } /* after all this - if it still matches in every respect, we don't need * to rewrite the record - just return */ if (records_match(mailbox_name(mailbox), ©, record)) { r = 0 ; goto out; } /* XXX - inform of changes */ if (!make_changes) { r = 0 ; goto out; } /* rewrite the cache record */ if (re_pack || !record->cache_offset || record->cache_crc != copy.cache_crc) { int32_t oldcrc = copy.cache_crc; int32_t newcrc = record->cache_crc; size_t oldoff = record->cache_offset; mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK; record->cache_offset = 0; r = mailbox_append_cache(mailbox, record); if (r) goto out ; printf("%s rewrote cache for %u (offset %llu to %llu, crc %u to %u/%u)\n", mailbox_name(mailbox), record->uid, (long long unsigned)oldoff, (long long unsigned)record->cache_offset, oldcrc, newcrc, record->cache_crc); syslog(LOG_NOTICE, "%s rewrote cache for %u (offset %llu to %llu, crc %u to %u/%u)", mailbox_name(mailbox), record->uid, (long long unsigned)oldoff, (long long unsigned)record->cache_offset, oldcrc, newcrc, record->cache_crc); } r = mailbox_rewrite_index_record(mailbox, record); out: if (remove_temp_spool_file) remove(fname); return r; } static int mailbox_reconstruct_append(struct mailbox *mailbox, uint32_t uid, int isarchive, int has_snoozedannot, int flags) { /* XXX - support archived */ const char *fname; int r = 0; struct index_record record; struct stat sbuf; int make_changes = flags & RECONSTRUCT_MAKE_CHANGES; memset(&record, 0, sizeof(struct index_record)); int remove_temp_spool_file = 0; int object_storage_enabled = 0; #if defined ENABLE_OBJECTSTORE object_storage_enabled = config_getswitch(IMAPOPT_OBJECT_STORAGE_ENABLED); #endif if (isarchive && !object_storage_enabled) fname = mailbox_archive_fname(mailbox, uid); else fname = mailbox_spool_fname(mailbox, uid); #if defined ENABLE_OBJECTSTORE if (object_storage_enabled) { uint32_t i , count = 0; struct message *list = get_list_of_message (mailbox, &count); for (i = 0; i < count; i++) { if (uid == list[i].message_uid){ record.uid = uid; record.guid = list[i].message_guid; remove_temp_spool_file = 1; r = objectstore_get(mailbox, &record, fname); break; } } discard_list(); } #endif /* possible if '0.' file exists */ if (!uid) { /* filthy hack - copy the path to '1.' and replace 1 with 0 */ char *hack; if (isarchive && !object_storage_enabled) fname = mailbox_archive_fname(mailbox, 1); else fname = mailbox_spool_fname(mailbox, 1); hack = (char *)fname; hack[strlen(fname)-2] = '0'; } if (stat(fname, &sbuf) == -1) r = IMAP_MAILBOX_NONEXISTENT; else if (sbuf.st_size == 0) r = IMAP_MAILBOX_NONEXISTENT; /* no file, nothing to do! */ if (r) { syslog(LOG_ERR, "%s uid %u not found", mailbox_name(mailbox), uid); printf("%s uid %u not found", mailbox_name(mailbox), uid); r = 0; if (make_changes) unlink(fname); goto out; } r = message_parse(fname, &record); if (r) goto out; if (isarchive) record.internal_flags |= FLAG_INTERNAL_ARCHIVED; if (has_snoozedannot) record.internal_flags |= FLAG_INTERNAL_SNOOZED; /* copy the timestamp from the file if not calculated */ if (!record.internaldate) record.internaldate = sbuf.st_mtime; if (uid > mailbox->i.last_uid) { printf("%s uid %u found - adding\n", mailbox_name(mailbox), uid); syslog(LOG_ERR, "%s uid %u found - adding", mailbox_name(mailbox), uid); record.uid = uid; } else { char *oldfname; char *newfname; printf("%s uid %u rediscovered - appending\n", mailbox_name(mailbox), uid); syslog(LOG_ERR, "%s uid %u rediscovered - appending", mailbox_name(mailbox), uid); /* XXX - check firstexpunged? */ record.uid = mailbox->i.last_uid + 1; if (!make_changes) { r = 0 ; goto out; } oldfname = xstrdup(fname); newfname = xstrdup(mailbox_record_fname(mailbox, &record)); r = rename(oldfname, newfname); free(oldfname); free(newfname); if (r) { r = IMAP_IOERROR; goto out ; } } /* XXX - inform of changes */ if (!make_changes){ r = 0 ; goto out; } /* always allow reconstruct to add records, even if we'd hit GUID limits */ record.ignorelimits = 1; r = mailbox_append_index_record(mailbox, &record); /* XXX - copy per-message annotations? */ out: if (remove_temp_spool_file) remove(fname); return r; } static void reconstruct_compare_headers(struct mailbox *mailbox, struct index_header *old, struct index_header *new) { if (old->quota_mailbox_used != new->quota_mailbox_used) { printf("%s updating quota_mailbox_used: " QUOTA_T_FMT " => " QUOTA_T_FMT "\n", mailbox_name(mailbox), old->quota_mailbox_used, new->quota_mailbox_used); syslog(LOG_ERR, "%s updating quota_mailbox_used: " QUOTA_T_FMT " => " QUOTA_T_FMT, mailbox_name(mailbox), old->quota_mailbox_used, new->quota_mailbox_used); } if (old->quota_annot_used != new->quota_annot_used) { printf("%s updating quota_annot_used: " QUOTA_T_FMT " => " QUOTA_T_FMT "\n", mailbox_name(mailbox), old->quota_annot_used, new->quota_annot_used); syslog(LOG_ERR, "%s updating quota_annot_used: " QUOTA_T_FMT " => " QUOTA_T_FMT, mailbox_name(mailbox), old->quota_annot_used, new->quota_annot_used); } if (old->answered != new->answered) { syslog(LOG_ERR, "%s: updating answered %u => %u", mailbox_name(mailbox), old->answered, new->answered); printf("%s: updating answered %u => %u\n", mailbox_name(mailbox), old->answered, new->answered); } if (old->flagged != new->flagged) { syslog(LOG_ERR, "%s: updating flagged %u => %u", mailbox_name(mailbox), old->flagged, new->flagged); printf("%s: updating flagged %u => %u\n", mailbox_name(mailbox), old->flagged, new->flagged); } if (old->deleted != new->deleted) { syslog(LOG_ERR, "%s: updating deleted %u => %u", mailbox_name(mailbox), old->deleted, new->deleted); printf("%s: updating deleted %u => %u\n", mailbox_name(mailbox), old->deleted, new->deleted); } if (old->exists != new->exists) { syslog(LOG_ERR, "%s: updating exists %u => %u", mailbox_name(mailbox), old->exists, new->exists); printf("%s: updating exists %u => %u\n", mailbox_name(mailbox), old->exists, new->exists); } if (old->synccrcs.basic != new->synccrcs.basic) { syslog(LOG_ERR, "%s: updating sync_crc %u => %u", mailbox_name(mailbox), old->synccrcs.basic, new->synccrcs.basic); printf("%s: updating sync_crc %u => %u\n", mailbox_name(mailbox), old->synccrcs.basic, new->synccrcs.basic); } if (old->synccrcs.annot != new->synccrcs.annot) { syslog(LOG_ERR, "%s: updating sync_crc_annot %u => %u", mailbox_name(mailbox), old->synccrcs.annot, new->synccrcs.annot); printf("%s: updating sync_crc_annot %u => %u\n", mailbox_name(mailbox), old->synccrcs.annot, new->synccrcs.annot); } } static int mailbox_wipe_index_record(struct mailbox *mailbox, struct index_record *record) { int n; indexbuffer_t ibuf; unsigned char *buf = ibuf.buf; size_t offset; assert(mailbox_index_islocked(mailbox, 1)); assert(record->recno > 0 && record->recno <= mailbox->i.num_records); record->uid = 0; record->internal_flags |= FLAG_INTERNAL_EXPUNGED | FLAG_INTERNAL_UNLINKED; mailbox->i.options |= OPT_MAILBOX_NEEDS_REPACK; mailbox_index_dirty(mailbox); mailbox_index_record_to_buf(record, mailbox->i.minor_version, buf); offset = mailbox->i.start_offset + (record->recno-1) * mailbox->i.record_size; off_t p = lseek(mailbox->index_fd, offset, SEEK_SET); if (p == -1) { xsyslog(LOG_ERR, "IOERROR: seeking index record failed", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), record->recno); return IMAP_IOERROR; } n = retry_write(mailbox->index_fd, buf, mailbox->i.record_size); if (n < 0) { xsyslog(LOG_ERR, "IOERROR: writing index record failed", "mailbox=<%s> record=<%u>", mailbox_name(mailbox), record->recno); return IMAP_IOERROR; } return 0; } static int addannot_uid(const char *mailbox __attribute__((unused)), uint32_t uid, const char *entry, const char *userid, const struct buf *value, const struct annotate_metadata *mdata __attribute__((unused)), void *rock) { struct found_uids *annots = (struct found_uids *)rock; /* take advantage of the guarantee that all annotations with the same UID * will be together in a 'foreach' response */ if (!annots->nused || annots->found[annots->nused-1].uid != uid) { /* we don't support an archive annotations DB yet */ add_found(annots, uid, /*isarchive*/0); } /* the last item will be for this UID regardless, check if we have the snoozed annotation */ if (!strcmpsafe(userid, "") && buf_len(value) && !strcmp(entry, IMAP_ANNOT_NS "snoozed")) { annots->found[annots->nused-1].issnoozed = 1; } return 0; } static int find_annots(struct mailbox *mailbox, struct found_uids *annots) { int r = 0; r = annotatemore_findall_mailbox(mailbox, ANNOTATE_ANY_UID, "*", /*modseq*/0, addannot_uid, annots, /*flags*/0); if (r) return r; /* make sure UIDs are sorted for comparison */ qsort(annots->found, annots->nused, sizeof(annots->found[0]), sort_found); return 0; } static int reconstruct_delannots(struct mailbox *mailbox, struct found_uids *delannots, int flags) { int make_changes = (flags & RECONSTRUCT_MAKE_CHANGES); int r = 0; r = mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, NULL); if (r) { syslog(LOG_ERR, "IOERROR: failed to open annotations %s: %s", mailbox_name(mailbox), error_message(r)); goto out; } while (delannots->pos < delannots->nused) { uint32_t uid = delannots->found[delannots->pos].uid; syslog(LOG_NOTICE, "removing stale annotations for %u", uid); printf("removing stale annotations for %u\n", uid); if (make_changes) { r = annotate_msg_cleanup(mailbox, uid); if (r) goto out; } delannots->pos++; } out: return r; } /* * Reconstruct the single mailbox named 'name' */ EXPORTED int mailbox_reconstruct(const char *name, int flags, struct mailbox **mboxptr) { /* settings */ int make_changes = (flags & RECONSTRUCT_MAKE_CHANGES); int r = 0; int i, flag; struct mailbox *mailbox = NULL; struct found_uids files = FOUND_UIDS_INITIALIZER; struct found_uids discovered = FOUND_UIDS_INITIALIZER; struct found_uids annots = FOUND_UIDS_INITIALIZER; struct found_uids delannots = FOUND_UIDS_INITIALIZER; struct index_header old_header; int have_file; uint32_t last_seen_uid = 0; bit32 valid_user_flags[MAX_USER_FLAGS/32]; struct buf buf = BUF_INITIALIZER; if (make_changes && !(flags & RECONSTRUCT_QUIET)) { syslog(LOG_NOTICE, "reconstructing %s", name); } r = mailbox_open_iwl(name, &mailbox); if (r) { if (!make_changes) return r; /* returns a locktype == LOCK_EXCLUSIVE mailbox */ r = mailbox_reconstruct_create(name, &mailbox); } if (r) return r; /* NOTE: we have to do this first, because it reads the header */ r = mailbox_reconstruct_acl(mailbox, flags); if (r) goto close; /* open and lock the annotation state */ r = mailbox_get_annotate_state(mailbox, ANNOTATE_ANY_UID, NULL); if (r) { syslog(LOG_ERR, "IOERROR: failed to open annotations %s: %s", mailbox_name(mailbox), error_message(r)); goto close; } /* Validate user flags */ for (i = 0; i < MAX_USER_FLAGS/32; i++) { valid_user_flags[i] = 0; } for (flag = 0; flag < MAX_USER_FLAGS; flag++) { if (!mailbox->h.flagname[flag]) continue; if (!imparse_isatom(mailbox->h.flagname[flag])) { printf("%s: bogus flag name %d:%s", mailbox_name(mailbox), flag, mailbox->h.flagname[flag]); syslog(LOG_ERR, "%s: bogus flag name %d:%s", mailbox_name(mailbox), flag, mailbox->h.flagname[flag]); xzfree(mailbox->h.flagname[flag]); mailbox->header_dirty = 1; continue; } valid_user_flags[flag/32] |= 1<<(flag&31); } /* find cyrus.expunge file if present */ cleanup_stale_expunged(mailbox); r = find_files(mailbox, &files, flags); if (r) goto close; r = find_annots(mailbox, &annots); if (r) goto close; uint32_t recno; struct index_record record; for (recno = 1; recno <= mailbox->i.num_records; recno++) { r = mailbox_read_index_record(mailbox, recno, &record); if (r || record.uid <= last_seen_uid) { if (!r && record.uid) syslog(LOG_ERR, "%s out of order uid %u at record %u, wiping", mailbox_name(mailbox), record.uid, record.recno); if (r) syslog(LOG_ERR, "%s failed to read at record %u (%s), wiping", mailbox_name(mailbox), record.recno, error_message(r)); mailbox_wipe_index_record(mailbox, &record); continue; } last_seen_uid = record.uid; /* bogus annotations? XXX: should we try to keep them if we found a file? */ while (annots.pos < annots.nused && annots.found[annots.pos].uid < record.uid) { add_found(&delannots, annots.found[annots.pos].uid, /*isarchive*/0); annots.pos++; } /* skip over current */ int has_snoozedannot = 0; while (annots.pos < annots.nused && annots.found[annots.pos].uid == record.uid) { if (annots.found[annots.pos].issnoozed) has_snoozedannot = 1; annots.pos++; } /* lower UID file exists */ while (files.pos < files.nused && files.found[files.pos].uid < record.uid) { add_found(&discovered, files.found[files.pos].uid, files.found[files.pos].isarchive); files.pos++; } /* if they match, advance the pointer */ have_file = 0; while (files.pos < files.nused && files.found[files.pos].uid == record.uid) { if (have_file) { /* we can just unlink this one, already processed one copy */ const char *fname = mailbox_archive_fname(mailbox, record.uid); printf("Removing duplicate archive file %s\n", fname); unlink(fname); } else { if (files.found[files.pos].isarchive) { if (!(record.internal_flags & FLAG_INTERNAL_ARCHIVED)) { /* oops, it's really archived - let's fix that right now */ record.internal_flags |= FLAG_INTERNAL_ARCHIVED; printf("Marking file as archived %s %u\n", mailbox_name(mailbox), record.uid); mailbox_rewrite_index_record(mailbox, &record); } } else { if (record.internal_flags & FLAG_INTERNAL_ARCHIVED) { /* oops, non-archived copy exists, let's use that */ record.internal_flags &= ~FLAG_INTERNAL_ARCHIVED; printf("Marking file as not archived %s %u\n", mailbox_name(mailbox), record.uid); mailbox_rewrite_index_record(mailbox, &record); } } have_file = 1; } files.pos++; } r = mailbox_reconstruct_compare_update(mailbox, &record, valid_user_flags, flags, have_file, has_snoozedannot, &discovered); if (r) goto close; if (mailbox->i.minor_version >= 13) { buf_reset(&buf); mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "thrid", "", &buf); if (!buf.len) mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "thrid", NULL, &buf); if (buf.len) { syslog(LOG_NOTICE, "removing stale thrid for %u", record.uid); printf("removing stale thrid for %u\n", record.uid); buf_reset(&buf); r = mailbox_annotation_write(mailbox, record.uid, IMAP_ANNOT_NS "thrid", "", &buf); if (r) goto close; } } if (mailbox->i.minor_version >= 15) { buf_reset(&buf); mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "savedate", "", &buf); if (!buf.len) mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "savedate", NULL, &buf); if (buf.len) { syslog(LOG_NOTICE, "removing stale savedate for %u", record.uid); printf("removing stale savedate for %u\n", record.uid); buf_reset(&buf); r = mailbox_annotation_write(mailbox, record.uid, IMAP_ANNOT_NS "savedate", "", &buf); if (r) goto close; } } if (mailbox->i.minor_version >= 16) { buf_reset(&buf); mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "createdmodseq", "", &buf); if (!buf.len) mailbox_annotation_lookup(mailbox, record.uid, IMAP_ANNOT_NS "createdmodseq", NULL, &buf); if (buf.len) { syslog(LOG_NOTICE, "removing stale createdmodseq for %u", record.uid); printf("removing stale createdmodseq for %u\n", record.uid); buf_reset(&buf); r = mailbox_annotation_write(mailbox, record.uid, IMAP_ANNOT_NS "createdmodseq", "", &buf); if (r) goto close; } } } /* make sure appends will be allowed, fixing last_uid to be at least * the highest discovered uid that we'll keep */ if (mailbox->i.last_uid < last_seen_uid) { syslog(LOG_ERR, "%s last_uid too small %u with highest uid %u, updating", mailbox_name(mailbox), mailbox->i.last_uid, last_seen_uid); mailbox_index_dirty(mailbox); mailbox->i.last_uid = last_seen_uid; } /* add discovered messages before last_uid to the list in order */ while (files.pos < files.nused && files.found[files.pos].uid <= mailbox->i.last_uid) { add_found(&discovered, files.found[files.pos].uid, files.found[files.pos].isarchive); files.pos++; } /* messages AFTER last_uid can keep the same UID (see also, restore * from lost .index file) - so don't bother moving those */ while (files.pos < files.nused) { uint32_t uid = files.found[files.pos].uid; /* bogus annotations? */ while (annots.pos < annots.nused && annots.found[annots.pos].uid < uid) { add_found(&delannots, annots.found[annots.pos].uid, /*isarchive*/0); annots.pos++; } int has_snoozedannot = 0; /* we can keep this annotation too... */ while (annots.pos < annots.nused && annots.found[annots.pos].uid == uid) { if (annots.found[annots.pos].issnoozed) has_snoozedannot = 1; annots.pos++; } r = mailbox_reconstruct_append(mailbox, files.found[files.pos].uid, files.found[files.pos].isarchive, has_snoozedannot, flags); if (r) goto close; files.pos++; } /* bogus annotations after the end? */ while (annots.pos < annots.nused) { add_found(&delannots, annots.found[annots.pos].uid, /*isarchive*/0); annots.pos++; } /* handle new list - note, we don't copy annotations for these */ while (discovered.pos < discovered.nused) { r = mailbox_reconstruct_append(mailbox, discovered.found[discovered.pos].uid, discovered.found[discovered.pos].isarchive, /*has_snoozedannot*/0, flags); if (r) goto close; discovered.pos++; } if (delannots.nused) { r = reconstruct_delannots(mailbox, &delannots, flags); if (r) goto close; } /* make sure we have enough index file mmaped */ r = mailbox_refresh_index_map(mailbox); old_header = mailbox->i; /* re-calculate derived fields */ r = mailbox_index_recalc(mailbox); if (r) goto close; /* inform users of any changed header fields */ reconstruct_compare_headers(mailbox, &old_header, &mailbox->i); /* fix up 2.4.0 bug breakage */ if (!mailbox->i.uidvalidity) { if (make_changes) { mailbox->i.uidvalidity = mboxname_nextuidvalidity(mailbox_name(mailbox), time(0)); mailbox_index_dirty(mailbox); } syslog(LOG_ERR, "%s: zero uidvalidity", mailbox_name(mailbox)); } if (!mailbox->i.highestmodseq) { if (make_changes) { mailbox_index_dirty(mailbox); mailbox->i.highestmodseq = mboxname_nextmodseq(mailbox_name(mailbox), 0, mailbox_mbtype(mailbox), MBOXMODSEQ_ISFOLDER); } syslog(LOG_ERR, "%s: zero highestmodseq", mailbox_name(mailbox)); } else { mboxname_setmodseq(mailbox_name(mailbox), mailbox->i.highestmodseq, mailbox_mbtype(mailbox), MBOXMODSEQ_ISFOLDER); } if (mboxptr) { *mboxptr = mailbox; mailbox = NULL; } else { if (make_changes) { r = mailbox_commit(mailbox); } else { r = mailbox_abort(mailbox); } } close: free_found(&files); free_found(&discovered); free_found(&annots); free_found(&delannots); mailbox_close(&mailbox); buf_free(&buf); return r; } EXPORTED struct mailbox_iter *mailbox_iter_init(struct mailbox *mailbox, modseq_t changedsince, unsigned flags) { struct mailbox_iter *iter = xzmalloc(sizeof(struct mailbox_iter)); iter->mailbox = mailbox; iter->changedsince = changedsince; iter->num_records = mailbox->i.num_records; iter->msg = message_new(); /* calculate which system_flags to skip over */ if (flags & ITER_SKIP_UNLINKED) iter->skipflags |= FLAG_INTERNAL_UNLINKED; if (flags & ITER_SKIP_EXPUNGED) iter->skipflags |= FLAG_INTERNAL_EXPUNGED; if (flags & ITER_SKIP_DELETED) iter->skipflags |= FLAG_DELETED; return iter; } EXPORTED void mailbox_iter_startuid(struct mailbox_iter *iter, uint32_t uid) { struct mailbox *mailbox = iter->mailbox; iter->recno = uid ? mailbox_finduid(mailbox, uid-1) : 0; } EXPORTED void mailbox_iter_uidset(struct mailbox_iter *iter, seqset_t *seq) { iter->uidset = seq; mailbox_iter_startuid(iter, seqset_first(seq)); } EXPORTED const message_t *mailbox_iter_step(struct mailbox_iter *iter) { if (mailbox_wait_cb) mailbox_wait_cb(mailbox_wait_cb_rock); for (iter->recno++; iter->recno <= iter->num_records; iter->recno++) { message_set_from_mailbox(iter->mailbox, iter->recno, iter->msg); const struct index_record *record = msg_record(iter->msg); if (!record->uid) continue; /* can happen on damaged mailboxes */ if ((record->system_flags & iter->skipflags)) continue; if ((record->internal_flags & iter->skipflags)) continue; if (iter->changedsince && record->modseq <= iter->changedsince) continue; if (iter->uidset) { if (record->uid > seqset_last(iter->uidset)) return NULL; if (!seqset_ismember(iter->uidset, record->uid)) continue; } return iter->msg; } /* guess we're done */ return NULL; } EXPORTED void mailbox_iter_done(struct mailbox_iter **iterp) { struct mailbox_iter *iter = *iterp; if (!iter) return; message_unref(&iter->msg); free(iter); *iterp = NULL; } /* * Gets messages usage. */ EXPORTED void mailbox_get_usage(struct mailbox *mailbox, quota_t usage[QUOTA_NUMRESOURCES]) { int res; for (res = 0; res < QUOTA_NUMRESOURCES; res++) { usage[res] = 0; } if (!(mailbox->i.options & OPT_MAILBOX_DELETED)) { usage[QUOTA_STORAGE] = mailbox->i.quota_mailbox_used; usage[QUOTA_MESSAGE] = mailbox->i.exists; usage[QUOTA_ANNOTSTORAGE] = mailbox->i.quota_annot_used; usage[QUOTA_NUMFOLDERS] = 1; } /* else: mailbox is being deleted, thus its new usage is 0 */ } EXPORTED int mailbox_get_annotate_state(struct mailbox *mailbox, unsigned int uid, annotate_state_t **statep) { int r = 0; if (statep) *statep = NULL; if (!mailbox->annot_state) mailbox->annot_state = annotate_state_new(); r = annotate_state_set_message(mailbox->annot_state, mailbox, uid); if (r) return r; /* lock immediately if we have a write lock */ if (mailbox_index_islocked(mailbox, /*write*/1)) annotate_state_begin(mailbox->annot_state); if (statep) *statep = mailbox->annot_state; return 0; } EXPORTED int mailbox_annotation_write(struct mailbox *mailbox, uint32_t uid, const char *entry, const char *userid, const struct buf *value) { annotate_state_t *state = NULL; int r = 0; struct buf oldvalue = BUF_INITIALIZER; annotatemore_msg_lookup(mailbox, uid, entry, userid, &oldvalue); if (oldvalue.len == value->len && (!value->len || !memcmp(oldvalue.s, value->s, value->len))) goto done; struct index_record record; memset(&record, 0, sizeof(struct index_record)); r = mailbox_find_index_record(mailbox, uid, &record); if (r) goto done; r = mailbox_get_annotate_state(mailbox, uid, &state); if (r) goto done; r = annotate_state_write(state, entry, userid, value); if (r) goto done; /* need to touch the modseq */ r = mailbox_rewrite_index_record(mailbox, &record); if (r) goto done; done: buf_free(&oldvalue); return r; } EXPORTED int mailbox_annotation_writemask(struct mailbox *mailbox, uint32_t uid, const char *entry, const char *userid, const struct buf *value) { annotate_state_t *state = NULL; int r = 0; struct buf oldvalue = BUF_INITIALIZER; /* we don't lookupmask here - because we want to still write the value as the * user's own value rather than the masked value, regardless of whether they * have the same content */ annotatemore_msg_lookup(mailbox, uid, entry, userid, &oldvalue); if (oldvalue.len == value->len && (!value->len || !memcmp(oldvalue.s, value->s, value->len))) goto done; struct index_record record; memset(&record, 0, sizeof(struct index_record)); r = mailbox_find_index_record(mailbox, uid, &record); if (r) goto done; r = mailbox_get_annotate_state(mailbox, uid, &state); if (r) goto done; r = annotate_state_writemask(state, entry, userid, value); if (r) goto done; /* need to touch the modseq */ r = mailbox_rewrite_index_record(mailbox, &record); if (r) goto done; done: buf_free(&oldvalue); return r; } EXPORTED int mailbox_annotation_lookup(struct mailbox *mailbox, uint32_t uid, const char *entry, const char *userid, struct buf *value) { return annotatemore_msg_lookup(mailbox, uid, entry, userid, value); } EXPORTED int mailbox_annotation_lookupmask(struct mailbox *mailbox, uint32_t uid, const char *entry, const char *userid, struct buf *value) { return annotatemore_msg_lookupmask(mailbox, uid, entry, userid, value); } int mailbox_cid_rename(struct mailbox *mailbox, conversation_id_t from_cid, conversation_id_t to_cid) { const message_t *msg; int r = 0; if (!config_getswitch(IMAPOPT_CONVERSATIONS)) return 0; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); if (record->cid != from_cid) continue; /* * Just rename the CID in place - injecting a copy at the end * messes with clients that just use UID ordering, like Apple's * IOS email client */ struct index_record copyrecord = *record; copyrecord.cid = to_cid; r = mailbox_rewrite_index_record(mailbox, ©record); if (r) { syslog(LOG_ERR, "mailbox_cid_rename: error " "rewriting record %u, mailbox %s: %s from %llu to %llu", record->recno, mailbox_name(mailbox), error_message(r), from_cid, to_cid); break; } } mailbox_iter_done(&iter); return r; } EXPORTED void mailbox_set_wait_cb(void (*cb)(void *), void *rock) { mailbox_wait_cb = cb; mailbox_wait_cb_rock = rock; } /* if either CRC is zero for a field, then we consider it to match. * this lets us bootstrap the case where CRCs weren't being calculated, * and also allows a client with incomplete local information to request * a change be made on a sync_server without having to fetch all the * data first just to calculate the CRC */ EXPORTED int mailbox_crceq(struct synccrcs a, struct synccrcs b) { if (a.basic && b.basic && a.basic != b.basic) return 0; if (a.annot && b.annot && a.annot != b.annot) return 0; return 1; } /* XXX should this be moved to lib/acl.c? not easily, because * XXX acl is in libcyrus but struct dlist is in libcyrus_imap! */ EXPORTED struct dlist *mailbox_acl_to_dlist(const char *aclstr) { const char *p, *q; struct dlist *al = dlist_newkvlist(NULL, "A"); p = aclstr; while (p && *p) { char *name,*val; q = strchr(p, '\t'); if (!q) break; name = xstrndup(p, q-p); q++; p = strchr(q, '\t'); if (p) { val = xstrndup(q, p-q); p++; } else val = xstrdup(q); dlist_setatom(al, name, val); free(name); free(val); } return al; } HIDDEN int mailbox_changequotaroot(struct mailbox *mailbox, const char *root, int silent) { int r = 0; int res; quota_t quota_usage[QUOTA_NUMRESOURCES]; mailbox_get_usage(mailbox, quota_usage); if (mailbox->h.quotaroot) { quota_t quota_diff[QUOTA_NUMRESOURCES]; if (root) { size_t len = strlen(root); if (strlen(mailbox->h.quotaroot) >= len && !strncmp(mailbox->h.quotaroot, root, len) && (mailbox->h.quotaroot[len] == '\0' || mailbox->h.quotaroot[len] == '.')) { /* Part of a child quota root - skip */ goto done; } } /* remove usage from the old quotaroot */ for (res = 0; res < QUOTA_NUMRESOURCES ; res++) { quota_diff[res] = -quota_usage[res]; } r = quota_update_useds(mailbox->h.quotaroot, quota_diff, mailbox_name(mailbox), silent); } /* update (or set) the quotaroot */ mailbox_set_quotaroot(mailbox, root); if (root) { /* update the new quota root */ r = quota_update_useds(root, quota_usage, mailbox_name(mailbox), silent); } done: return r; } struct part_rock { const char *path; unsigned long is_meta; char **partition; }; static void get_partition(const char *key, const char *val, void *rock) { struct part_rock *prock = (struct part_rock *) rock; if (*prock->partition) return; if (prock->is_meta) { if (strncmp("meta", key, 4)) return; key += 4; } if (strncmp("partition-", key, 10)) return; size_t vlen = strlen(val); if (!strncmp(prock->path, val, vlen) && (val[vlen-1] == '/' || prock->path[vlen] == '/')) { *prock->partition = xstrdup(key+10); } } EXPORTED struct mboxlist_entry *mailbox_mbentry_from_path(const char *header_path) { struct mboxlist_entry *mbentry = mboxlist_entry_create(); struct mailbox mailbox; int r; zeromailbox(mailbox); mailbox.mbentry = mbentry; r = mailbox_read_header(&mailbox, header_path); if (!r) { mbentry->mbtype = mailbox.h.mbtype; /* Steal strings from mailbox_header */ mbentry->name = mailbox.h.name; mbentry->uniqueid = mailbox.h.uniqueid; mbentry->acl = mailbox.h.acl; /* Cleanup remaining mailbox_header */ for (int flag = 0; flag < MAX_USER_FLAGS; flag++) { xzfree(mailbox.h.flagname[flag]); } xzfree(mailbox.h.quotaroot); xclose(mailbox.header_fd); /* Get partition name */ struct part_rock prock = { header_path, config_metapartition_files & IMAP_ENUM_METAPARTITION_FILES_HEADER, &mbentry->partition }; config_foreachoverflowstring(&get_partition, &prock); } else { mboxlist_entry_free(&mbentry); } return mbentry; } diff --git a/imap/sieve_db.c b/imap/sieve_db.c old mode 100644 new mode 100755 index cfaa0b4af..3b787a740 --- a/imap/sieve_db.c +++ b/imap/sieve_db.c @@ -1,1081 +1,1084 @@ /* sieve_db.c -- implementation of per-user Sieve database * * Copyright (c) 1994-2020 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * */ #include #include #include #include "append.h" #include "cyrusdb.h" #include "dav_db.h" #include #include "global.h" #include "jmap_api.h" #include "libconfig.h" #include "mboxlist.h" #include "sieve_db.h" #include "sievedir.h" #include "times.h" #include "user.h" #include "util.h" #include "xmalloc.h" #include "sieve/bytecode.h" #include "sieve/bc_parse.h" #include "sieve/script.h" #include "sieve/sieve_interface.h" /* generated headers are not necessarily in current directory */ #include "imap/imap_err.h" enum { STMT_SELRSRC, STMT_SELUID, STMT_SELMBOX, STMT_INSERT, STMT_UPDATE, STMT_DELETE, STMT_DELMBOX, STMT_BEGIN, STMT_COMMIT, STMT_ROLLBACK }; #define NUM_STMT 10 struct sieve_db { sqldb_t *db; /* DB handle */ struct buf mailbox; /* buffers for copies of column text */ struct buf id; struct buf name; struct buf contentid; }; static int sieve_initialized = 0; static void done_cb(void *rock __attribute__((unused))) { sievedb_done(); } static void init_internal() { if (!sieve_initialized) { sievedb_init(); cyrus_modules_add(done_cb, NULL); } } EXPORTED int sievedb_init(void) { int r = sqldb_init(); if (!r) sieve_initialized = 1; return r; } EXPORTED int sievedb_done(void) { int r = sqldb_done(); if (!r) sieve_initialized = 0; return r; } /* Open Sieve DB corresponding to userid */ EXPORTED struct sieve_db *sievedb_open_userid(const char *userid) { struct sieve_db *sievedb = NULL; init_internal(); sqldb_t *db = dav_open_userid(userid); if (!db) return NULL; sievedb = xzmalloc(sizeof(struct sieve_db)); sievedb->db = db; return sievedb; } /* Open Sieve DB corresponding to mailbox */ EXPORTED struct sieve_db *sievedb_open_mailbox(struct mailbox *mailbox) { struct sieve_db *sievedb = NULL; char *userid = mboxname_to_userid(mailbox_name(mailbox)); init_internal(); if (userid) { sievedb = sievedb_open_userid(userid); free(userid); return sievedb; } sqldb_t *db = dav_open_mailbox(mailbox); if (!db) return NULL; sievedb = xzmalloc(sizeof(struct sieve_db)); sievedb->db = db; return sievedb; } /* Close Sieve DB */ EXPORTED int sievedb_close(struct sieve_db *sievedb) { int r = 0; if (!sievedb) return 0; buf_free(&sievedb->mailbox); buf_free(&sievedb->id); buf_free(&sievedb->name); buf_free(&sievedb->contentid); r = dav_close(&sievedb->db); free(sievedb); return r; } EXPORTED int sievedb_begin(struct sieve_db *sievedb) { return sqldb_begin(sievedb->db, "sieve"); } EXPORTED int sievedb_commit(struct sieve_db *sievedb) { return sqldb_commit(sievedb->db, "sieve"); } EXPORTED int sievedb_abort(struct sieve_db *sievedb) { return sqldb_rollback(sievedb->db, "sieve"); } struct read_rock { struct sieve_db *db; struct sieve_data *sdata; int tombstones; sieve_cb_t *cb; void *rock; }; static const char *column_text_to_buf(const char *text, struct buf *buf) { if (text) { buf_setcstr(buf, text); text = buf_cstring(buf); } return text; } static int read_cb(sqlite3_stmt *stmt, void *rock) { struct read_rock *rrock = (struct read_rock *) rock; struct sieve_db *db = rrock->db; struct sieve_data *sdata = rrock->sdata; int r = 0; memset(sdata, 0, sizeof(struct sieve_data)); sdata->modseq = sqlite3_column_int64(stmt, 5); sdata->createdmodseq = sqlite3_column_int64(stmt, 6); sdata->alive = sqlite3_column_int(stmt, 11); if (!rrock->tombstones && !sdata->alive) return 0; sdata->rowid = sqlite3_column_int(stmt, 0); sdata->creationdate = sqlite3_column_int(stmt, 1); sdata->lastupdated = sqlite3_column_int(stmt, 2); sdata->imap_uid = sqlite3_column_int(stmt, 4); sdata->isactive = sqlite3_column_int(stmt, 10); if (rrock->cb) { /* We can use the column data directly for the callback */ sdata->mailbox = (const char *) sqlite3_column_text(stmt, 3); sdata->id = (const char *) sqlite3_column_text(stmt, 7); sdata->name = (const char *) sqlite3_column_text(stmt, 8); sdata->contentid = (const char *) sqlite3_column_text(stmt, 9); r = rrock->cb(rrock->rock, sdata); } else { /* For single row SELECTs like sieve_read(), * we need to make a copy of the column data before * it gets flushed by sqlite3_step() or sqlite3_reset() */ sdata->mailbox = column_text_to_buf((const char *) sqlite3_column_text(stmt, 3), &db->mailbox); sdata->id = column_text_to_buf((const char *) sqlite3_column_text(stmt, 7), &db->id); sdata->name = column_text_to_buf((const char *) sqlite3_column_text(stmt, 8), &db->name); sdata->contentid = column_text_to_buf((const char *) sqlite3_column_text(stmt, 9), &db->contentid); } return r; } #define CMD_GETFIELDS \ "SELECT rowid, creationdate, lastupdated, mailbox, imap_uid," \ " modseq, createdmodseq, id, name, contentid, isactive, alive" \ " FROM sieve_scripts" #define CMD_SELNAME CMD_GETFIELDS \ " WHERE name = :name;" EXPORTED int sievedb_lookup_name(struct sieve_db *sievedb, const char *name, struct sieve_data **result, int tombstones) { struct sqldb_bindval bval[] = { { ":name", SQLITE_TEXT, { .s = name } }, { NULL, SQLITE_NULL, { .s = NULL } } }; static struct sieve_data sdata; struct read_rock rrock = { sievedb, &sdata, tombstones, NULL, NULL }; int r; *result = memset(&sdata, 0, sizeof(struct sieve_data)); r = sqldb_exec(sievedb->db, CMD_SELNAME, bval, &read_cb, &rrock); if (!r && !sdata.rowid) r = CYRUSDB_NOTFOUND; return r; } #define CMD_SELID CMD_GETFIELDS \ " WHERE id = :id;" EXPORTED int sievedb_lookup_id(struct sieve_db *sievedb, const char *id, struct sieve_data **result, int tombstones) { struct sqldb_bindval bval[] = { { ":id", SQLITE_TEXT, { .s = id } }, { NULL, SQLITE_NULL, { .s = NULL } } }; static struct sieve_data sdata; struct read_rock rrock = { sievedb, &sdata, tombstones, NULL, NULL }; int r; *result = memset(&sdata, 0, sizeof(struct sieve_data)); r = sqldb_exec(sievedb->db, CMD_SELID, bval, &read_cb, &rrock); if (!r && !sdata.rowid) r = CYRUSDB_NOTFOUND; return r; } #define CMD_SELIMAPUID CMD_GETFIELDS \ " WHERE imap_uid = :imap_uid;" EXPORTED int sievedb_lookup_imapuid(struct sieve_db *sievedb, int imap_uid, struct sieve_data **result, int tombstones) { struct sqldb_bindval bval[] = { { ":imap_uid", SQLITE_INTEGER, { .i = imap_uid } }, { NULL, SQLITE_NULL, { .s = NULL } } }; static struct sieve_data sdata; struct read_rock rrock = { sievedb, &sdata, tombstones, NULL, NULL }; int r; *result = memset(&sdata, 0, sizeof(struct sieve_data)); r = sqldb_exec(sievedb->db, CMD_SELIMAPUID, bval, &read_cb, &rrock); if (!r && !sdata.rowid) r = CYRUSDB_NOTFOUND; sdata.imap_uid = imap_uid; return r; } #define CMD_SELACTIVE CMD_GETFIELDS " WHERE isactive = 1 AND alive = 1;" EXPORTED int sievedb_lookup_active(struct sieve_db *sievedb, struct sieve_data **result) { static struct sieve_data sdata; struct read_rock rrock = { sievedb, &sdata, 0, NULL, NULL }; int r; *result = memset(&sdata, 0, sizeof(struct sieve_data)); r = sqldb_exec(sievedb->db, CMD_SELACTIVE, NULL, &read_cb, &rrock); if (!r && !sdata.rowid) r = CYRUSDB_NOTFOUND; return r; } #define CMD_SELMBOX CMD_GETFIELDS \ " WHERE alive = 1;" EXPORTED int sievedb_foreach(struct sieve_db *sievedb, int (*cb)(void *rock, struct sieve_data *data), void *rock) { struct sieve_data sdata; struct read_rock rrock = { sievedb, &sdata, 0, cb, rock }; return sqldb_exec(sievedb->db, CMD_SELMBOX, NULL, &read_cb, &rrock); } #define CMD_INSERT \ "INSERT INTO sieve_scripts (" \ " creationdate, lastupdated, mailbox, imap_uid," \ " modseq, createdmodseq, id, name, contentid, isactive, alive )" \ " VALUES (" \ " :creationdate, :lastupdated, :mailbox, :imap_uid," \ " :modseq, :createdmodseq, :id, :name, :contentid, :isactive, :alive );" #define CMD_UPDATE \ "UPDATE sieve_scripts SET" \ " lastupdated = :lastupdated," \ " imap_uid = :imap_uid," \ " modseq = :modseq," \ " name = :name," \ " contentid = :contentid," \ " isactive = :isactive," \ " alive = :alive" \ " WHERE rowid = :rowid;" EXPORTED int sievedb_write(struct sieve_db *sievedb, struct sieve_data *sdata) { struct sqldb_bindval bval[] = { { ":lastupdated", SQLITE_INTEGER, { .i = sdata->lastupdated } }, { ":imap_uid", SQLITE_INTEGER, { .i = sdata->imap_uid } }, { ":modseq", SQLITE_INTEGER, { .i = sdata->modseq } }, { ":name", SQLITE_TEXT, { .s = sdata->name } }, { ":contentid", SQLITE_TEXT, { .s = sdata->contentid } }, { ":isactive", SQLITE_INTEGER, { .i = sdata->isactive } }, { ":alive", SQLITE_INTEGER, { .i = sdata->alive } }, { NULL, SQLITE_NULL, { .s = NULL } }, { NULL, SQLITE_NULL, { .s = NULL } }, { NULL, SQLITE_NULL, { .s = NULL } }, { NULL, SQLITE_NULL, { .s = NULL } }, { NULL, SQLITE_NULL, { .s = NULL } } }; const char *cmd; int r; if (sdata->rowid) { cmd = CMD_UPDATE; bval[7].name = ":rowid"; bval[7].type = SQLITE_INTEGER; bval[7].val.i = sdata->rowid; } else { cmd = CMD_INSERT; bval[7].name = ":creationdate"; bval[7].type = SQLITE_INTEGER; bval[7].val.i = sdata->creationdate; bval[8].name = ":createdmodseq"; bval[8].type = SQLITE_INTEGER; bval[8].val.i = sdata->createdmodseq; bval[9].name = ":id"; bval[9].type = SQLITE_TEXT; bval[9].val.s = sdata->id; bval[10].name = ":mailbox"; bval[10].type = SQLITE_TEXT; bval[10].val.s = sdata->mailbox; } r = sqldb_exec(sievedb->db, cmd, bval, NULL, NULL); return r; } #define CMD_DELETE "DELETE FROM sieve_scripts WHERE rowid = :rowid;" EXPORTED int sievedb_delete(struct sieve_db *sievedb, unsigned rowid) { struct sqldb_bindval bval[] = { { ":rowid", SQLITE_INTEGER, { .i = rowid } }, { NULL, SQLITE_NULL, { .s = NULL } } }; int r; r = sqldb_exec(sievedb->db, CMD_DELETE, bval, NULL, NULL); return r; } #define CMD_DELMBOX "DELETE FROM sieve_scripts;" HIDDEN int sievedb_delmbox(struct sieve_db *sievedb) { int r; r = sqldb_exec(sievedb->db, CMD_DELMBOX, NULL, NULL, NULL); return r; } EXPORTED int sievedb_get_updates(struct sieve_db *sievedb, modseq_t oldmodseq, int limit, int (*cb)(void *rock, struct sieve_data *sdata), void *rock) { struct sqldb_bindval bval[] = { { ":modseq", SQLITE_INTEGER, { .i = oldmodseq } }, /* SQLite interprets a negative limit as unbounded. */ { ":limit", SQLITE_INTEGER, { .i = limit > 0 ? limit : -1 } }, { NULL, SQLITE_NULL, { .s = NULL } } }; static struct sieve_data sdata; struct read_rock rrock = { sievedb, &sdata, 1 /* tombstones */, cb, rock }; struct buf sqlbuf = BUF_INITIALIZER; int r; buf_setcstr(&sqlbuf, CMD_GETFIELDS " WHERE"); if (!oldmodseq) buf_appendcstr(&sqlbuf, " alive = 1 AND"); buf_appendcstr(&sqlbuf, " modseq > :modseq ORDER BY modseq LIMIT :limit;"); r = sqldb_exec(sievedb->db, buf_cstring(&sqlbuf), bval, &read_cb, &rrock); buf_free(&sqlbuf); return r; } static int count_cb(sqlite3_stmt *stmt, void *rock) { int *count = (int *) rock; *count = sqlite3_column_int(stmt, 0); return 0; } #define CMD_COUNTREC "SELECT COUNT(name) FROM sieve_scripts WHERE alive = 1;" EXPORTED int sievedb_count(struct sieve_db *sievedb, int *count) { int r; *count = 0; r = sqldb_exec(sievedb->db, CMD_COUNTREC, NULL, &count_cb, count); return r; } static int lock_and_execute(struct mailbox *mailbox, struct sieve_data *sdata, void *rock, int (*proc)(struct mailbox *mailbox, struct sieve_data *sdata, void *rock)) { int r, unlock = 0; if (!mailbox_index_islocked(mailbox, 1/*write*/)) { r = mailbox_lock_index(mailbox, LOCK_EXCLUSIVE); if (r) { syslog(LOG_ERR, "locking mailbox %s failed: %s", mailbox_name(mailbox), error_message(r)); return r; } unlock = 1; } r = proc(mailbox, sdata, rock); if (unlock) mailbox_unlock_index(mailbox, NULL); return r; } static int remove_uid(struct mailbox *mailbox, uint32_t uid) { struct index_record record; int r; r = mailbox_find_index_record(mailbox, uid, &record); if (!r) { record.internal_flags |= FLAG_INTERNAL_EXPUNGED; r = mailbox_rewrite_index_record(mailbox, &record); } if (r) { syslog(LOG_ERR, "expunging record (%s:%u) failed: %s", mailbox_name(mailbox), uid, error_message(r)); } return r; } static int store_script(struct mailbox *mailbox, struct sieve_data *sdata, void *rock) { const struct buf *databuf = (const struct buf *) rock; strarray_t flags = STRARRAY_INITIALIZER; struct auth_state *authstate = NULL; struct buf buf = BUF_INITIALIZER; uint32_t old_uid = sdata->imap_uid; const char *data = buf_base(databuf); size_t datalen = buf_len(databuf); struct stagemsg *stage; struct appendstate as; time_t now = time(0); FILE *f = NULL; char *mimehdr; int r = 0; init_internal(); /* Prepare to stage the message */ if (!(f = append_newstage(mailbox_name(mailbox), now, 0, &stage))) { syslog(LOG_ERR, "append_newstage(%s) failed", mailbox_name(mailbox)); return CYRUSDB_IOERROR; } /* Create RFC 5322 header for script */ char *userid = mboxname_to_userid(mailbox_name(mailbox)); if (strchr(userid, '@')) { buf_printf(&buf, "<%s>", userid); } else { buf_printf(&buf, "<%s@%s>", userid, config_servername); } mimehdr = charset_encode_mimeheader(buf_cstring(&buf), buf_len(&buf), 0); fprintf(f, "From: %s\r\n", mimehdr); free(mimehdr); mimehdr = charset_encode_mimeheader(sdata->name, 0, 0); fprintf(f, "Subject: %s\r\n", mimehdr); free(mimehdr); char datestr[80]; time_to_rfc5322(now, datestr, sizeof(datestr)); fprintf(f, "Date: %s\r\n", datestr); /* Use SHA1(script) as contentid */ struct message_guid uuid; message_guid_generate(&uuid, data, datalen); sdata->contentid = message_guid_encode(&uuid); /* Use scriptid@servername as Message-ID */ fprintf(f, "Message-ID: <%s@%s>\r\n", sdata->contentid, config_servername); fprintf(f, "Content-Type: application/sieve; charset=utf-8\r\n"); fprintf(f, "Content-Length: %lu\r\n", datalen); fprintf(f, "Content-Disposition: attachment;\r\n\tfilename=\"%s%s\"\r\n", sdata->id ? sdata->id : makeuuid(), SIEVE_EXTENSION); fputs("MIME-Version: 1.0\r\n", f); fputs("\r\n", f); /* Write the script data to the file */ fwrite(data, datalen, 1, f); fclose(f); if (sdata->isactive) { /* Flag script as active */ strarray_append(&flags, "\\Flagged"); /* Need authstate in order to set flags */ authstate = auth_newstate(userid); } if ((r = append_setup_mbox(&as, mailbox, userid, authstate, 0, NULL, NULL, 0, 0))) { syslog(LOG_ERR, "append_setup(%s) failed: %s", mailbox_name(mailbox), error_message(r)); } else { struct body *body = NULL; r = append_fromstage(&as, &body, stage, now, sdata->createdmodseq, &flags, 0, NULL); if (body) { message_free_body(body); free(body); } if (r) { syslog(LOG_ERR, "append_fromstage() failed: %s", error_message(r)); append_abort(&as); } else { /* Commit the append to the sieve mailbox */ r = append_commit(&as); if (r) { syslog(LOG_ERR, "append_commit() failed: %s", error_message(r)); } else if (old_uid) { /* Now that we have the replacement script in place expunge the old one. */ r = remove_uid(mailbox, old_uid); } } } append_removestage(stage); auth_freestate(authstate); strarray_fini(&flags); buf_free(&buf); free(userid); return r; } EXPORTED int sieve_script_store(struct mailbox *mailbox, struct sieve_data *sdata, const struct buf *content) { return lock_and_execute(mailbox, sdata, (void *) content, &store_script); } static int activate_script(struct mailbox *mailbox, struct sieve_data *sdata, void *rock __attribute__((unused))) { struct index_record record; int activate = (sdata != NULL); int r; init_internal(); if (activate) { if (sdata->isactive) return 0; r = mailbox_find_index_record(mailbox, sdata->imap_uid, &record); if (r) { syslog(LOG_ERR, "fetching record (%s:%u) failed: %s", mailbox_name(mailbox), sdata->imap_uid, error_message(r)); return r; } } struct sieve_db *sievedb = sievedb_open_mailbox(mailbox); if (!sievedb) { syslog(LOG_ERR, "opening sieve_db for %s failed", mailbox_name(mailbox)); return CYRUSDB_IOERROR; } struct sieve_data *mydata = NULL; r = sievedb_lookup_active(sievedb, &mydata); if (r == CYRUSDB_NOTFOUND) { /* No active script to deactivate */ r = 0; } else if (!r) { struct index_record oldactive; r = mailbox_find_index_record(mailbox, mydata->imap_uid, &oldactive); if (r) { syslog(LOG_ERR, "fetching record (%s:%u) failed: %s", mailbox_name(mailbox), mydata->imap_uid, error_message(r)); } else { oldactive.system_flags &= ~FLAG_FLAGGED; r = mailbox_rewrite_index_record(mailbox, &oldactive); if (r) { syslog(LOG_ERR, "unflagging record (%s:%u) failed: %s", mailbox_name(mailbox), oldactive.uid, error_message(r)); } } } sievedb_close(sievedb); if (!r && activate) { record.system_flags |= FLAG_FLAGGED; r = mailbox_rewrite_index_record(mailbox, &record); if (r) { syslog(LOG_ERR, "flagging record (%s:%u) failed: %s", mailbox_name(mailbox), record.uid, error_message(r)); } } return r; } EXPORTED int sieve_script_activate(struct mailbox *mailbox, struct sieve_data *sdata) { return lock_and_execute(mailbox, sdata, NULL, &activate_script); } static int remove_script(struct mailbox *mailbox, struct sieve_data *sdata, void *rock __attribute__((unused))) { init_internal(); return remove_uid(mailbox, sdata->imap_uid); } EXPORTED int sieve_script_remove(struct mailbox *mailbox, struct sieve_data *sdata) { return lock_and_execute(mailbox, sdata, NULL, &remove_script); } EXPORTED int sieve_script_rename(struct mailbox *mailbox, struct sieve_data *sdata, const char *newname) { struct buf content = BUF_INITIALIZER; int r; r = sieve_script_fetch(mailbox, sdata, &content); if (!r) { sdata->name = newname; r = sieve_script_store(mailbox, sdata, &content); } buf_free(&content); return r; } EXPORTED int sieve_script_fetch(struct mailbox *mailbox, const struct sieve_data *sdata, struct buf *content) { struct index_record record; int r; r = mailbox_find_index_record(mailbox, sdata->imap_uid, &record); if (!r) { /* Load message containing the resource */ message_t *m = message_new_from_record(mailbox, &record); r = message_get_field(m, "rawbody", MESSAGE_RAW, content); message_unref(&m); } if (r) { syslog(LOG_ERR, "fetching message (%s:%u) failed: %s", mailbox_name(mailbox), sdata->imap_uid, error_message(r)); } return r; } struct migrate_rock { struct mailbox *mailbox; char *active; }; static int migrate_cb(const char *sievedir, const char *fname, struct stat *sbuf, const char *link_target __attribute__((unused)), void *rock) { struct migrate_rock *mrock = (struct migrate_rock *) rock; struct buf *content = sievedir_get_script(sievedir, fname); if (content) { struct sieve_data sdata; char *myname = xstrndup(fname, strlen(fname) - SCRIPT_SUFFIX_LEN); int deletebc = 0; memset(&sdata, 0, sizeof(sdata)); if (!strcmp(myname, "jmap_vacation")) { sdata.id = sdata.name = JMAP_URN_VACATION; deletebc = 1; } else { sdata.name = myname; } sdata.lastupdated = sbuf->st_mtime; sdata.isactive = !strcmpnull(myname, mrock->active); if (!store_script(mrock->mailbox, &sdata, content)) { char path[PATH_MAX]; /* delete script */ snprintf(path, sizeof(path), "%s/%s", sievedir, fname); unlink(path); if (deletebc) { sievedir_delete_script(sievedir, myname); } } buf_destroy(content); free(myname); } return SIEVEDIR_OK; } EXPORTED int sieve_ensure_folder(const char *userid, struct mailbox **mailboxptr) { const char *sievedir = user_sieve_path(userid); struct stat sbuf; int r; r = stat(sievedir, &sbuf); if (r && errno == ENOENT) { if (!mailboxptr) { /* Don't bother continuing if sievedir doesn't currently exist */ return 0; } r = cyrus_mkdir(sievedir, 0755); if (!r) { r = mkdir(sievedir, 0755); } } if (r) return IMAP_IOERROR; struct mboxlock *namespacelock = NULL; char *mboxname = sieve_mboxname(userid); r = mboxlist_lookup(mboxname, NULL, NULL); if (r == IMAP_MAILBOX_NONEXISTENT) { namespacelock = user_namespacelock(userid); if (!namespacelock) { r = IMAP_MAILBOX_LOCKED; goto done; } /* maybe we lost the race on this one */ r = mboxlist_lookup(mboxname, NULL, NULL); } if (r == IMAP_MAILBOX_NONEXISTENT) { /* Create locally */ struct mailbox *mailbox = NULL; mbentry_t mbentry = MBENTRY_INITIALIZER; mbentry.name = (char *) mboxname; mbentry.mbtype = MBTYPE_SIEVE; r = mboxlist_createmailbox(&mbentry, 0/*options*/, 0/*highestmodseq*/, 1/*isadmin*/, userid, NULL/*auth_state*/, 0/*flags*/, &mailbox); if (r) { syslog(LOG_ERR, "IOERROR: failed to create %s (%s)", mboxname, error_message(r)); goto done; } /* Migrate scripts from sievedir into mailbox */ struct migrate_rock mrock = { mailbox, xstrdupnull(sievedir_get_active(sievedir)) }; sievedir_foreach(sievedir, SIEVEDIR_SCRIPTS_ONLY, &migrate_cb, &mrock); free(mrock.active); // close the mailbox here, we'll re-open once we've released the namespace lock mailbox_close(&mailbox); } mboxname_release(&namespacelock); if (!r && mailboxptr) { r = mailbox_open_iwl(mboxname, mailboxptr); if (r) { syslog(LOG_ERR, "IOERROR: failed to open %s (%s)", mboxname, error_message(r)); goto done; } } done: mboxname_release(&namespacelock); free(mboxname); return r; } EXPORTED int sieve_script_rebuild(const char *userid, const char *sievedir, const char *script) { struct buf namebuf = BUF_INITIALIZER, *content_buf = NULL; struct sieve_data *sdata = NULL; struct sieve_db *db = NULL; const char *content = NULL; time_t lastupdated = 0; struct stat bc_stat; int r; db = sievedb_open_userid(userid); if (!db) { r = IMAP_INTERNAL; goto done; } /* Lookup script in Sieve DB */ r = sievedb_lookup_name(db, script, &sdata, 0); if (!r) { char *mboxname = sieve_mboxname(userid); struct mailbox *mailbox = NULL; lastupdated = sdata->lastupdated; content_buf = buf_new(); r = mailbox_open_irl(mboxname, &mailbox); if (r) { syslog(LOG_ERR, "IOERROR: failed to open %s (%s)", mboxname, error_message(r)); } else { r = sieve_script_fetch(mailbox, sdata, content_buf); if (!r) { content = buf_cstring(content_buf); } } mailbox_close(&mailbox); free(mboxname); if (r) goto done; } else if (r == CYRUSDB_NOTFOUND) { /* Get mtime of script file */ struct stat sbuf; buf_printf(&namebuf, "%s/%s%s", sievedir, script, SCRIPT_SUFFIX); r = stat(buf_cstring(&namebuf), &sbuf); if (!r) { lastupdated = sbuf.st_mtime; } else { syslog(LOG_ERR, "IOERROR: stat %s: %m", buf_cstring(&namebuf)); } } if (r) { r = IMAP_IOERROR; goto done; } /* Get mtime of bytecode file */ buf_reset(&namebuf); buf_printf(&namebuf, "%s/%s%s", sievedir, script, BYTECODE_SUFFIX); r = stat(buf_cstring(&namebuf), &bc_stat); if (r && errno != ENOENT) { syslog(LOG_ERR, "IOERROR: stat %s: %m", buf_cstring(&namebuf)); r = IMAP_IOERROR; goto done; } if (!r && bc_stat.st_mtime >= lastupdated) { /* Check version of bytecode file */ sieve_execute_t *exe = NULL; int version = -1; r = sieve_script_load(buf_cstring(&namebuf), &exe); if (!r) { bc_header_parse((bytecode_input_t *) exe->bc_cur->data, &version, NULL); } sieve_script_unload(&exe); if (version == BYTECODE_VERSION) { syslog(LOG_DEBUG, "%s: %s is up to date\n", __func__, buf_cstring(&namebuf)); goto done; } } if (!content) { /* Fetch content from script file */ buf_reset(&namebuf); buf_printf(&namebuf, "%s%s", script, SCRIPT_SUFFIX); content_buf = sievedir_get_script(sievedir, buf_cstring(&namebuf)); if (!content_buf) { r = IMAP_IOERROR; goto done; } content = buf_cstring(content_buf); } /* Update bytecode */ char *errors = NULL; r = sievedir_put_script(sievedir, script, content, &errors); + if (errors) { + syslog(LOG_ERR, "Error during sievedir_put_script for %s %s", userid, errors); + } free(errors); done: buf_destroy(content_buf); buf_free(&namebuf); sievedb_close(db); return r; } EXPORTED char *sieve_mboxname(const char *userid) { struct buf boxbuf = BUF_INITIALIZER; char *res = NULL; init_internal(); buf_setcstr(&boxbuf, config_getstring(IMAPOPT_SIEVE_FOLDER)); res = mboxname_user_mbox(userid, buf_cstring(&boxbuf)); buf_free(&boxbuf); return res; }