reconstruct.c
No OneTemporary
Actions

Authored By

Unknown

Size

41 KB

Referenced Files

None

Subscribers

None

reconstruct.c
View Options

	/* reconstruct.c -- program to reconstruct a mailbox
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: reconstruct.c,v 1.110 2009/03/31 04:11:20 brong Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdlib.h>
	#include <stdio.h>
	#include <errno.h>
	#include <string.h>
	#include <fcntl.h>
	#include <ctype.h>
	#include <syslog.h>
	#include <sys/types.h>
	#include <netinet/in.h>
	#include <sys/stat.h>
	#include <stdlib.h>

	#if HAVE_DIRENT_H
	# include <dirent.h>
	# define NAMLEN(dirent) strlen((dirent)->d_name)
	#else
	# define dirent direct
	# define NAMLEN(dirent) (dirent)->d_namlen
	# if HAVE_SYS_NDIR_H
	# include <sys/ndir.h>
	# endif
	# if HAVE_SYS_DIR_H
	# include <sys/dir.h>
	# endif
	# if HAVE_NDIR_H
	# include <ndir.h>
	# endif
	#endif

	#include "acl.h"
	#include "assert.h"
	#include "bsearch.h"
	#include "imparse.h"
	#include "global.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "map.h"
	#include "message.h"
	#include "message_guid.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"
	#include "global.h"
	#include "mboxname.h"
	#include "mboxlist.h"
	#include "quota.h"
	#include "seen.h"
	#include "retry.h"
	#include "convert_code.h"
	#include "util.h"
	#include "sync_log.h"
	#include "lock.h"

	extern int optind;
	extern char *optarg;

	struct discovered {
	char *name;
	struct discovered *next;
	};
	struct uniqmailid {
	char * uniqmbxid;
	char *uniqname;
	struct uniqmailid *uniqnext;
	};

	struct uniqmailid *uniqmid_head;

	/* current namespace */
	static struct namespace recon_namespace;

	/* config.c stuff */
	const int config_need_data = CONFIG_NEED_PARTITION_DATA;

	/* forward declarations */
	void do_mboxlist(void);
	int do_reconstruct(char name, int matchlen, int maycreate, void rock);
	int reconstruct(char name, struct discovered l);
	void usage(void);
	char * getmailname (char * mailboxname);
	struct uniqmailid * add_uniqid (char * mailboxname, char * mailboxid);
	struct uniqmailid * find_uniqid (char * mailboxname, char * mailboxid);

	extern cyrus_acl_canonproc_t mboxlist_ensureOwnerRights;

	int code = 0;
	int keepflag = 0;
	int syncflag = 0;
	int guid_clear = 0;
	int guid_set = 0;

	int main(int argc, char **argv)
	{
	int opt, i, r;
	int rflag = 0;
	int mflag = 0;
	int fflag = 0;
	int xflag = 0;
	char buf[MAX_MAILBOX_PATH+1];
	char mbbuf[MAX_MAILBOX_PATH+1];
	struct discovered head;
	char *alt_config = NULL;
	char *start_part = NULL;
	const char start_part_path = NULL, start_part_mpath = NULL, *path;

	memset(&head, 0, sizeof(head));

	if ((geteuid()) == 0 && (become_cyrus() != 0)) {
	fatal("must run as the Cyrus user", EC_USAGE);
	}

	/* Ensure we're up-to-date on the index file format */
	assert(INDEX_HEADER_SIZE == (OFFSET_SPARE4+4));
	assert(INDEX_RECORD_SIZE == (OFFSET_MODSEQ+4));

	while ((opt = getopt(argc, argv, "C:kp:rmfsxgG")) != EOF) {
	switch (opt) {
	case 'C': /* alt config file */
	alt_config = optarg;
	break;

	case 'k':
	keepflag = 1;
	break;

	case 'p':
	start_part = optarg;
	break;

	case 'r':
	rflag = 1;
	break;

	case 'm':
	mflag = 1;
	break;

	case 'f':
	fflag = 1;
	break;

	case 's':
	syncflag = 1;
	break;

	case 'x':
	xflag = 1;
	break;

	case 'g':
	guid_clear = 1;
	break;

	case 'G':
	guid_set = 1;
	break;

	default:
	usage();
	}
	}

	cyrus_init(alt_config, "reconstruct", 0);
	global_sasl_init(1,0,NULL);

	/* Set namespace -- force standard (internal) */
	if ((r = mboxname_init_namespace(&recon_namespace, 1)) != 0) {
	syslog(LOG_ERR, error_message(r));
	fatal(error_message(r), EC_CONFIG);
	}
	if (syncflag)
	sync_log_init();

	if(start_part) {
	/* Get partition's path */
	start_part_path = config_partitiondir(start_part);
	if (!start_part_path) {
	fatal(error_message(IMAP_PARTITION_UNKNOWN), EC_USAGE);
	}
	start_part_mpath = config_metapartitiondir(start_part);
	}

	if (mflag) {
	if (rflag \|\| fflag \|\| optind != argc) {
	cyrus_done();
	usage();
	}
	do_mboxlist();
	}

	mboxlist_init(0);
	mboxlist_open(NULL);

	quotadb_init(0);
	quotadb_open(NULL);

	mailbox_reconstructmode();

	/* Deal with nonexistent mailboxes */
	if (start_part) {
	/* We were handed a mailbox that does not exist currently */
	if(optind == argc) {
	fprintf(stderr,
	"When using -p, you must specify a mailbox to attempt to reconstruct.");
	exit(EC_USAGE);
	}

	/* do any of the mailboxes exist in mboxlist already? */
	/* Do they look like mailboxes? */
	for (i = optind; i < argc; i++) {
	struct stat sbuf;

	if(strchr(argv[i],'%') \|\| strchr(argv[i],'*')) {
	fprintf(stderr, "Using wildcards with -p is not supported.\n");
	exit(EC_USAGE);
	}

	/* Translate mailboxname */
	(*recon_namespace.mboxname_tointernal)(&recon_namespace, argv[i],
	NULL, buf);

	/* Does it exist */
	do {
	r = mboxlist_lookup(buf, NULL, NULL);
	} while (r == IMAP_AGAIN);

	if(r != IMAP_MAILBOX_NONEXISTENT) {
	fprintf(stderr,
	"Mailbox %s already exists. Cannot specify -p.\n",
	argv[i]);
	exit(EC_USAGE);
	}

	/* Does the suspected path look like a mailbox? */
	path = (start_part_mpath &&
	(config_metapartition_files &
	IMAP_ENUM_METAPARTITION_FILES_HEADER)) ?
	start_part_mpath : start_part_path;
	mailbox_hash_mbox(mbbuf, sizeof(mbbuf), path, buf);
	strlcat(mbbuf, FNAME_HEADER, sizeof(mbbuf));
	if(stat(mbbuf, &sbuf) < 0) {
	fprintf(stderr,
	"%s does not appear to be a mailbox (no %s).\n",
	argv[i], mbbuf);
	exit(EC_USAGE);
	}
	}

	/* None of them exist. Create them. */
	for (i = optind; i < argc; i++) {
	/* Translate mailboxname */
	(*recon_namespace.mboxname_tointernal)(&recon_namespace, argv[i],
	NULL, buf);

	r = mboxlist_createmailbox(buf, 0, start_part, 1,
	"cyrus", NULL, 0, 0, !xflag);
	if(r) {
	fprintf(stderr, "could not create %s\n", argv[i]);
	}
	}
	}

	/* Normal Operation */
	if (optind == argc) {
	if (rflag) {
	fprintf(stderr, "please specify a mailbox to recurse from\n");
	cyrus_done();
	exit(EC_USAGE);
	}
	assert(!rflag);
	strlcpy(buf, "*", sizeof(buf));
	(*recon_namespace.mboxlist_findall)(&recon_namespace, buf, 1, 0, 0,
	do_reconstruct, NULL);
	}

	for (i = optind; i < argc; i++) {
	char *domain = NULL;

	/* save domain */
	if (config_virtdomains) domain = strchr(argv[i], '@');

	strlcpy(buf, argv[i], sizeof(buf));
	/* Translate any separators in mailboxname */
	mboxname_hiersep_tointernal(&recon_namespace, buf,
	config_virtdomains ?
	strcspn(buf, "@") : 0);

	/* reconstruct the first mailbox/pattern */
	(*recon_namespace.mboxlist_findall)(&recon_namespace, buf, 1, 0,
	0, do_reconstruct,
	fflag ? &head : NULL);
	if (rflag) {
	/* build a pattern for submailboxes */
	char *p = strchr(buf, '@');
	if (p) *p = '\0';
	strlcat(buf, ".*", sizeof(buf));

	/* append the domain */
	if (domain) strlcat(buf, domain, sizeof(buf));

	/* reconstruct the submailboxes */
	(*recon_namespace.mboxlist_findall)(&recon_namespace, buf, 1, 0,
	0, do_reconstruct,
	fflag ? &head : NULL);
	}
	}

	/* examine our list to see if we discovered anything */
	while (head.next) {
	struct discovered *p;
	int r = 0;

	p = head.next;
	head.next = p->next;

	/* create p (database only) and reconstruct it */
	/* partition is defined by the parent mailbox */
	r = mboxlist_createmailbox(p->name, 0, NULL, 1,
	"cyrus", NULL, 0, 0, !xflag);
	if (!r) {
	do_reconstruct(p->name, strlen(p->name), 0, &head);
	} else {
	fprintf(stderr, "createmailbox %s: %s\n",
	p->name, error_message(r));
	}
	/* may have added more things into our list */

	free(p->name);
	free(p);
	}

	mboxlist_close();
	mboxlist_done();

	quotadb_close();
	quotadb_done();

	cyrus_done();

	return code;
	}

	void usage(void)
	{
	fprintf(stderr,
	"usage: reconstruct [-C <alt_config>] [-p partition] [-ksrfx] mailbox...\n");
	fprintf(stderr, " reconstruct [-C <alt_config>] -m\n");
	exit(EC_USAGE);
	}

	int compare_uid(const void a, const void b)
	{
	return (unsigned long )a - (unsigned long )b;
	}

	#define UIDGROW 300


	/*
	* mboxlist_findall() callback function to reconstruct a mailbox
	*/
	int
	do_reconstruct(char *name,
	int matchlen,
	int maycreate __attribute__((unused)),
	void *rock)
	{
	int r;
	char buf[MAX_MAILBOX_PATH+1];
	static char lastname[MAX_MAILBOX_PATH+1] = "";

	signals_poll();

	/* don't repeat */
	if (matchlen == (int) strlen(lastname) &&
	!strncmp(name, lastname, matchlen)) return 0;

	if(matchlen >= (int) sizeof(lastname))
	matchlen = sizeof(lastname) - 1;

	strncpy(lastname, name, matchlen);
	lastname[matchlen] = '\0';

	r = reconstruct(lastname, rock);
	if (r) {
	com_err(lastname, r,
	(r == IMAP_IOERROR) ? error_message(errno) : NULL);
	code = convert_code(r);
	} else {
	/* Convert internal name to external */
	(*recon_namespace.mboxname_toexternal)(&recon_namespace, lastname,
	NULL, buf);
	printf("%s\n", buf);
	}

	return 0;
	}

	char *
	getmailname (char * mailboxname)
	{
	static char namebuf[MAX_MAILBOX_PATH + 1];

	char * pname;

	strlcpy (namebuf, mailboxname, sizeof (namebuf));
	pname = strchr (namebuf, '.');
	if (pname) {
	pname = strchr(pname + 1, '.');
	if (pname)
	*pname = '\0';
	}
	return (namebuf);
	}

	struct uniqmailid *
	find_uniqid ( char * mailboxname, char * mailboxid)
	{
	struct uniqmailid *puniq;
	char * nameptr;

	nameptr = getmailname (mailboxname);
	for (puniq = uniqmid_head; puniq != NULL; puniq = puniq->uniqnext) {
	if (strcmp (puniq->uniqmbxid, mailboxid) == 0) {
	if (strcmp (puniq->uniqname, nameptr) == 0) {
	return (puniq);
	}
	}
	}
	return NULL;
	}
	struct uniqmailid *
	add_uniqid ( char * mailboxname, char * mailboxid)
	{
	struct uniqmailid *puniq;
	char *pboxname;

	pboxname = getmailname (mailboxname);

	puniq = xmalloc (sizeof (struct uniqmailid));
	puniq->uniqmbxid = xstrdup(mailboxid);
	puniq->uniqname = xstrdup(pboxname);

	puniq->uniqnext = uniqmid_head;
	uniqmid_head = puniq;

	return (puniq);
	}

	/* ---------------------------------------------------------------------- */

	/* Code which is typically reused for index,expunge and cache files */

	static void reconstruct_make_path(char *buf, int size,
	struct mailbox *mailbox,
	int mask, char name, char suffix)
	{
	char *path = (mailbox->mpath && (config_metapartition_files & mask))
	? mailbox->mpath : mailbox->path;

	strlcpy(buf, path, size);
	strlcat(buf, name, size);

	if (suffix && suffix[0])
	strlcat(buf, suffix, size);
	}

	static int reconstruct_open_expunge(struct mailbox mailbox, int fdp,
	unsigned long *lenp)
	{
	char fnamebuf[MAX_MAILBOX_PATH+1];
	struct stat sbuf;
	const char *lockfailaction;
	int r;

	reconstruct_make_path(fnamebuf, sizeof(fnamebuf), mailbox,
	IMAP_ENUM_METAPARTITION_FILES_EXPUNGE,
	FNAME_EXPUNGE_INDEX, NULL);
	*fdp = -1;
	if ((stat(fnamebuf, &sbuf) < 0) \|\|
	(sbuf.st_size < (int) INDEX_HEADER_SIZE) \|\|
	((*fdp = open(fnamebuf, O_RDWR, 0666)) < 0)) {
	unlink(fnamebuf);
	return(0);
	}
	*lenp = sbuf.st_size;

	if ((r = lock_reopen(*fdp, fnamebuf, &sbuf, &lockfailaction)))
	syslog(LOG_ERR, "IOERROR: %s expunge index for %s: %m",
	lockfailaction, mailbox->name);
	return(r);
	}

	/* Commit a single index/expunge/cache file */
	static int reconstruct_rename_single(struct mailbox *mailbox,
	int mask, char *filename)
	{
	char fnamebuf[MAX_MAILBOX_PATH+1], fnamebufnew[MAX_MAILBOX_PATH+1];

	reconstruct_make_path(fnamebuf, sizeof(fnamebuf),
	mailbox, mask, filename, NULL);
	strlcpy(fnamebufnew, fnamebuf, sizeof(fnamebufnew));
	strlcat(fnamebufnew, ".NEW", sizeof(fnamebufnew));

	if (rename(fnamebufnew, fnamebuf)) {
	syslog(LOG_ERR, "IOERROR: renaming %s for %s: %m",
	filename, mailbox->name);
	return(IMAP_IOERROR);
	}
	return(0);
	}

	/* Delete a single index/expunge/cache file */
	static int reconstruct_delete_single(struct mailbox *mailbox, int mask,
	char filename, char suffix)
	{
	char fnamebuf[MAX_MAILBOX_PATH+1];

	reconstruct_make_path(fnamebuf, sizeof(fnamebuf), mailbox, mask,
	filename, suffix);
	return ((unlink(fnamebuf) < 0) ? IMAP_IOERROR : 0);
	}

	/* ---------------------------------------------------------------------- */

	/* uiditem used to generate list of msgnos sorted by ascending UID */
	struct uiditem {
	unsigned long msgno;
	unsigned long uid;
	};

	static int compare_uiditem(const void a0, const void b0)
	{
	struct uiditem a = (struct uiditem ) a0;
	struct uiditem b = (struct uiditem ) b0;

	/* If duplicate UIDs appear list lowest msgnos first */
	if (a->uid == b->uid)
	return((a->msgno) - (b->msgno));

	return ((a->uid) - (b->uid));
	}

	/* ---------------------------------------------------------------------- */

	/* Running counts which will go into the index and expunge headers */
	struct reconstruct_counts {
	unsigned long newexists;
	unsigned long newanswered;
	unsigned long newflagged;
	unsigned long newdeleted;
	uquota_t newquota_used;
	};

	static void reconstruct_counts_clear(struct reconstruct_counts *c)
	{
	memset(c, 0, sizeof(struct reconstruct_counts));
	}


	static void reconstruct_counts_update(struct reconstruct_counts *c,
	struct index_record *p)
	{
	c->newexists++;
	c->newquota_used += p->size;

	if (p->system_flags & FLAG_ANSWERED) c->newanswered++;
	if (p->system_flags & FLAG_DELETED) c->newdeleted++;
	if (p->system_flags & FLAG_FLAGGED) c->newflagged++;
	}

	static void reconstruct_counts_tobuf(unsigned char *buf,
	struct mailbox *mailbox,
	struct reconstruct_counts *c)
	{
	((bit32 )(buf+OFFSET_GENERATION_NO)) = htonl(mailbox->generation_no + 1);
	((bit32 )(buf+OFFSET_FORMAT)) = htonl(mailbox->format);
	((bit32 )(buf+OFFSET_MINOR_VERSION)) = htonl(MAILBOX_MINOR_VERSION);
	((bit32 )(buf+OFFSET_START_OFFSET)) = htonl(INDEX_HEADER_SIZE);
	((bit32 )(buf+OFFSET_RECORD_SIZE)) = htonl(INDEX_RECORD_SIZE);
	((bit32 )(buf+OFFSET_EXISTS)) = htonl(c->newexists);
	((bit32 )(buf+OFFSET_LAST_APPENDDATE)) = htonl(mailbox->last_appenddate);
	((bit32 )(buf+OFFSET_LAST_UID)) = htonl(mailbox->last_uid);

	/* quotas may be 64bit now */
	#ifdef HAVE_LONG_LONG_INT
	((bit64 )(buf+OFFSET_QUOTA_MAILBOX_USED64)) = htonll(c->newquota_used);
	#else
	/* zero the unused 32bits */
	((bit32 )(buf+OFFSET_QUOTA_MAILBOX_USED64)) = htonl(0);
	((bit32 )(buf+OFFSET_QUOTA_MAILBOX_USED)) = htonl(c->newquota_used);
	#endif

	((bit32 )(buf+OFFSET_POP3_LAST_LOGIN)) = htonl(mailbox->pop3_last_login);
	((bit32 )(buf+OFFSET_UIDVALIDITY)) = htonl(mailbox->uidvalidity);
	((bit32 )(buf+OFFSET_DELETED)) = htonl(c->newdeleted);
	((bit32 )(buf+OFFSET_ANSWERED)) = htonl(c->newanswered);
	((bit32 )(buf+OFFSET_FLAGGED)) = htonl(c->newflagged);
	((bit32 )(buf+OFFSET_MAILBOX_OPTIONS)) = htonl(mailbox->options);
	((bit32 )(buf+OFFSET_LEAKED_CACHE)) = htonl(0);
	#ifdef HAVE_LONG_LONG_INT
	align_htonll(buf+OFFSET_HIGHESTMODSEQ_64, mailbox->highestmodseq);
	#else
	/* zero the unused 32bits */
	((bit32 )(buf+OFFSET_HIGHESTMODSEQ_64)) = htonl(0);
	((bit32 )(buf+OFFSET_HIGHESTMODSEQ)) = htonl(mailbox->highestmodseq);
	#endif
	((bit32 )(buf+OFFSET_SPARE0)) = htonl(0); /* RESERVED */
	((bit32 )(buf+OFFSET_SPARE1)) = htonl(0); /* RESERVED */
	((bit32 )(buf+OFFSET_SPARE2)) = htonl(0); /* RESERVED */
	((bit32 )(buf+OFFSET_SPARE3)) = htonl(0); /* RESERVED */
	((bit32 )(buf+OFFSET_SPARE4)) = htonl(0); /* RESERVED */
	}

	/* ---------------------------------------------------------------------- */

	static int reconstruct_header_isvalid(const char *index_base,
	unsigned long index_len)
	{
	int format;
	int minor_version;
	unsigned long start_offset;
	unsigned long record_size;
	unsigned long exists;

	if ((index_base == NULL) \|\| (index_len < INDEX_HEADER_SIZE))
	return(0);

	format = ntohl(((bit32 )(index_base+OFFSET_FORMAT)));
	minor_version = ntohl(((bit32 )(index_base+OFFSET_MINOR_VERSION)));
	start_offset = ntohl(((bit32 )(index_base+OFFSET_START_OFFSET)));
	record_size = ntohl(((bit32 )(index_base+OFFSET_RECORD_SIZE)));
	exists = ntohl(((bit32 )(index_base+OFFSET_EXISTS)));

	if ((format != 0) \|\|
	(minor_version == 0) \|\| (minor_version > MAILBOX_MINOR_VERSION) \|\|
	(start_offset == 0) \|\| (start_offset > INDEX_HEADER_SIZE) \|\|
	(record_size == 0) \|\| (record_size > INDEX_RECORD_SIZE) \|\|
	((unsigned) index_len < (start_offset + exists * record_size)))
	return(0);

	return(1);
	}

	static void reconstruct_clear_expunged(struct mailbox *mailbox,
	struct uiditem *expunge_uidmap,
	unsigned expunge_exists)
	{
	char msgfname[MAILBOX_FNAME_LEN+1];
	unsigned long uid;
	unsigned msgno;

	for (msgno = 1; msgno <= expunge_exists; msgno++) {
	if ((uid = expunge_uidmap[msgno-1].uid) > 0) {
	mailbox_message_get_fname(mailbox, uid, msgfname, sizeof(msgfname));
	unlink(msgfname);
	}
	}
	}

	/* Read an index record from a mapped index file without struct mailbox */
	int reconstruct_read_index_record(const char *name,
	const char *index_base,
	unsigned long index_len,
	unsigned msgno,
	struct index_record *record)
	{
	unsigned long start_offset;
	unsigned long record_size;
	unsigned long exists;
	unsigned long offset;
	unsigned const char *buf;
	int n;

	/* Following would normally come from struct mailbox */
	start_offset = ntohl(((bit32 )(index_base+OFFSET_START_OFFSET)));
	record_size = ntohl(((bit32 )(index_base+OFFSET_RECORD_SIZE)));
	exists = ntohl(((bit32 )(index_base+OFFSET_EXISTS)));

	offset = start_offset + (msgno-1) * record_size;
	if (offset + INDEX_RECORD_SIZE > index_len) {
	syslog(LOG_ERR,
	"IOERROR: index record %u for %s past end of file",
	msgno, name);
	return IMAP_IOERROR;
	}

	buf = (unsigned char*) index_base + offset;

	record->uid = ntohl(((bit32 )(buf+OFFSET_UID)));
	record->internaldate = ntohl(((bit32 )(buf+OFFSET_INTERNALDATE)));
	record->sentdate = ntohl(((bit32 )(buf+OFFSET_SENTDATE)));
	record->size = ntohl(((bit32 )(buf+OFFSET_SIZE)));
	record->header_size = ntohl(((bit32 )(buf+OFFSET_HEADER_SIZE)));
	record->content_offset = ntohl(((bit32 )(buf+OFFSET_CONTENT_OFFSET)));
	record->cache_offset = ntohl(((bit32 )(buf+OFFSET_CACHE_OFFSET)));
	record->last_updated = ntohl(((bit32 )(buf+OFFSET_LAST_UPDATED)));
	record->system_flags = ntohl(((bit32 )(buf+OFFSET_SYSTEM_FLAGS)));
	for (n = 0; n < MAX_USER_FLAGS/32; n++) {
	record->user_flags[n] = ntohl(((bit32 )(buf+OFFSET_USER_FLAGS+4*n)));
	}
	record->content_lines = ntohl(((bit32 )(buf+OFFSET_CONTENT_LINES)));
	record->cache_version = ntohl(((bit32 )(buf+OFFSET_CACHE_VERSION)));
	message_guid_import(&record->guid, buf+OFFSET_MESSAGE_GUID);
	#ifdef HAVE_LONG_LONG_INT
	record->modseq = ntohll(((bit64 )(buf+OFFSET_MODSEQ_64)));
	#else
	record->modseq = ntohl(((bit32 )(buf+OFFSET_MODSEQ)));
	#endif
	return 0;
	}

	/* ---------------------------------------------------------------------- */

	/*
	* Reconstruct the single mailbox named 'name'
	*/
	int reconstruct(char name, struct discovered found)
	{
	indexbuffer_t ibuf;
	unsigned char *buf = ibuf.buf;
	char quota_root[MAX_MAILBOX_BUFFER];
	bit32 valid_user_flags[MAX_USER_FLAGS/32];

	struct mailbox mailbox;

	int expunge_fd = -1;
	const char *expunge_base = NULL;
	unsigned long expunge_len = 0;
	unsigned long expunge_exists = 0;
	struct uiditem *expunge_uidmap = NULL;
	unsigned long expunge_size = 0;
	unsigned expmsg;

	int r = 0;
	int i, n, hasquota, flag;
	int format = MAILBOX_FORMAT_NORMAL;

	char *p;

	char fnamebuf[MAX_MAILBOX_PATH+1];
	FILE newindex = NULL, newexpunge = NULL, *msgfile = NULL;
	DIR *dirp;
	struct dirent *dirent;
	struct stat sbuf;
	int newcache_fd = -1;

	unsigned long *uid = NULL;
	unsigned uid_num, uid_alloc;
	unsigned msg, oldmsg;

	struct reconstruct_counts index_counts;
	struct reconstruct_counts expunge_counts;

	char list_acl, list_part;
	int list_type;

	struct index_record message_index, tmp_index, old_index;
	struct body *body = NULL;

	char mypath, mympath, mypart, myacl;
	int mytype;
	char mbpath[MAX_MAILBOX_PATH+1], *path;

	int expunge_found, index_found;
	char unique_buf[32];

	time_t now = time(0);
	modseq_t highestmodseq = 0;

	/* Start by looking up current data in mailbox list */
	r = mboxlist_detail(name, &mytype, &mypath, &mympath,
	&mypart, &myacl, NULL);
	if(r) return r;

	/* stat for header, if it is not there, we need to create it
	* note that we do not want to wind up with a fully-open mailbox,
	* so we will re-open. */
	path = (mympath &&
	(config_metapartition_files &
	IMAP_ENUM_METAPARTITION_FILES_HEADER)) ?
	mympath : mypath;
	snprintf(mbpath, sizeof(mbpath), "%s%s", path, FNAME_HEADER);
	if(stat(mbpath, &sbuf) == -1) {
	/* Header doesn't exist, create it! */
	r = mailbox_create(name, mypart, myacl, NULL,
	((mytype & MBTYPE_NETNEWS) ?
	MAILBOX_FORMAT_NETNEWS :
	MAILBOX_FORMAT_NORMAL), NULL);
	if(r) return r;
	}

	/* Now open just the header (it will hopefully be valid) */
	r = mailbox_open_header(name, 0, &mailbox);
	if (r) return r;

	if (mailbox.header_fd != -1) {
	(void) mailbox_lock_header(&mailbox);
	}
	mailbox.header_lock_count = 1;

	if (chdir(mailbox.path) == -1) {
	mailbox_close(&mailbox);
	return IMAP_IOERROR;
	}

	/* Fix quota root */
	hasquota = quota_findroot(quota_root, sizeof(quota_root), mailbox.name);
	if (mailbox.quota.root) free(mailbox.quota.root);
	if (hasquota) {
	mailbox.quota.root = xstrdup(quota_root);
	}
	else {
	mailbox.quota.root = 0;
	}

	/* Validate user flags */
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	valid_user_flags[i] = 0;
	}
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (!mailbox.flagname[flag]) continue;
	if ((flag && !mailbox.flagname[flag-1]) \|\|
	!imparse_isatom(mailbox.flagname[flag])) {
	free(mailbox.flagname[flag]);
	mailbox.flagname[flag] = 0;
	}
	valid_user_flags[flag/32] \|= 1<<(flag&31);
	}

	/* Verify ACL and update mboxlist if needed */
	r = mailbox_read_header_acl(&mailbox);
	if (r) {
	mailbox_close(&mailbox);
	return r;
	}

	r = mboxlist_detail(name, &list_type, NULL, NULL,
	&list_part, &list_acl, NULL);
	if (r) {
	mailbox_close(&mailbox);
	return r;
	}

	if(strcmp(list_acl, mailbox.acl)) {
	r = mboxlist_update(name, list_type, list_part, mailbox.acl, 0);
	}
	if (r) {
	mailbox_close(&mailbox);
	return r;
	}

	/* Attempt to open/lock index */
	r = mailbox_open_index(&mailbox);
	if (r) {
	mailbox.exists = 0;
	mailbox.last_uid = 0;
	mailbox.last_appenddate = 0;
	mailbox.uidvalidity = now;
	/* If we can't read the index, assume new UIDL so that stupid clients
	will retrieve all of the messages in the mailbox. */
	mailbox.options = OPT_POP3_NEW_UIDL;
	mailbox.highestmodseq = 1;
	}
	else {
	(void) mailbox_lock_index(&mailbox);
	}
	mailbox.index_lock_count = 1;
	mailbox.pop3_last_login = 0;

	/* Open, lock and then map cyrus.expunge file if it exists */
	r = reconstruct_open_expunge(&mailbox, &expunge_fd, &expunge_size);
	if (r) {
	mailbox_close(&mailbox);
	return IMAP_IOERROR;
	}
	if (expunge_fd != -1) {
	map_refresh(expunge_fd, 1, &expunge_base, &expunge_len, expunge_size,
	"expunge", mailbox.name);
	if (!reconstruct_header_isvalid(expunge_base, expunge_len)) {
	map_free(&expunge_base, &expunge_len);
	close(expunge_fd);
	expunge_fd = -1;
	syslog(LOG_ERR, "Unable to verify expunge header - deleting: %s",
	mailbox.name);
	reconstruct_delete_single(&mailbox,
	IMAP_ENUM_METAPARTITION_FILES_EXPUNGE,
	FNAME_EXPUNGE_INDEX, NULL);
	}
	}
	if (expunge_base && (expunge_len >= INDEX_HEADER_SIZE))
	expunge_exists = ntohl(((bit32 )(expunge_base+OFFSET_EXISTS)));

	/* expunge_uidmap is list of msgnos sorted in ascending UID */
	if (expunge_exists > 0) {
	unsigned msgno;

	expunge_uidmap = xmalloc(expunge_exists * sizeof(struct uiditem));

	for (msgno = 1 ; msgno <= expunge_exists; msgno++) {
	/* Can't use mailbox_read_index_record_from_mapped() safely */
	r = reconstruct_read_index_record(mailbox.name,
	expunge_base, expunge_len,
	msgno, &message_index);
	if (r) {
	r = IMAP_IOERROR;
	goto bail;
	}
	expunge_uidmap[msgno-1].msgno = msgno;
	expunge_uidmap[msgno-1].uid = message_index.uid;
	}
	qsort(expunge_uidmap, expunge_exists,
	sizeof(struct uiditem), compare_uiditem);
	}

	if (!keepflag && (expunge_exists > 0))
	reconstruct_clear_expunged(&mailbox, expunge_uidmap, expunge_exists);

	/* Create new index/cache/expunge files */
	reconstruct_make_path(fnamebuf, sizeof(fnamebuf), &mailbox,
	IMAP_ENUM_METAPARTITION_FILES_INDEX,
	FNAME_INDEX, ".NEW");
	if ((newindex = fopen(fnamebuf, "w+")) == NULL) {
	r = IMAP_IOERROR;
	goto bail;
	}
	reconstruct_make_path(fnamebuf, sizeof(fnamebuf), &mailbox,
	IMAP_ENUM_METAPARTITION_FILES_CACHE,
	FNAME_CACHE, ".NEW");
	if ((newcache_fd = open(fnamebuf, O_RDWR\|O_TRUNC\|O_CREAT, 0666)) == -1) {
	r = IMAP_IOERROR;
	goto bail;
	}
	reconstruct_make_path(fnamebuf, sizeof(fnamebuf), &mailbox,
	IMAP_ENUM_METAPARTITION_FILES_EXPUNGE,
	FNAME_EXPUNGE_INDEX, ".NEW");
	if ((newexpunge = fopen(fnamebuf, "w+")) == NULL) {
	r = IMAP_IOERROR;
	goto bail;
	}

	/* Create placeholder space for index/cache/expunge headers */
	memset(buf, 0, sizeof(buf));
	((bit32 )(buf+OFFSET_GENERATION_NO)) = htonl(mailbox.generation_no + 1);
	fwrite(buf, 1, INDEX_HEADER_SIZE, newindex);
	fwrite(buf, 1, INDEX_HEADER_SIZE, newexpunge);
	retry_write(newcache_fd, buf, sizeof(bit32));

	/* Find all message files in directory */
	uid = (unsigned long ) xmalloc(UIDGROW sizeof(unsigned long));
	uid_num = 0;
	uid_alloc = UIDGROW;
	dirp = opendir(".");
	if (!dirp) {
	r = IMAP_IOERROR;
	goto bail;
	}
	while ((dirent = readdir(dirp))!=NULL) {
	if (!Uisdigit((dirent->d_name[0])) \|\| dirent->d_name[0] == '0')
	continue;
	p = dirent->d_name;
	msg = 0;
	while (Uisdigit(*p)) {
	msg = msg * 10 + *p++ - '0';
	}
	if (*p++ != '.') continue;
	if (*p) continue;

	if (uid_num == uid_alloc) {
	uid_alloc += UIDGROW;
	uid = (unsigned long *)
	xrealloc((char )uid, uid_alloc sizeof(unsigned long));
	}
	uid[uid_num] = msg;
	uid_num++;
	}
	closedir(dirp);

	qsort((char )uid, uid_num, sizeof(uid), compare_uid);

	/* Put each message file in new index/cache or expunge/cache */
	mailbox.format = format;
	reconstruct_counts_clear(&index_counts);
	reconstruct_counts_clear(&expunge_counts);

	memset(&tmp_index, 0, sizeof(struct index_record));
	memset(&old_index, 0, sizeof(struct index_record));

	old_index.uid = 0; /* Only valid after mailbox_read_index_record() */

	oldmsg = 0;
	expmsg = 0;
	for (msg = 0; msg < uid_num; msg++) {
	char msgfname[MAILBOX_FNAME_LEN+1];

	memset(&message_index, 0, sizeof(struct index_record));
	message_index.uid = uid[msg];

	mailbox_message_get_fname(&mailbox, uid[msg],
	msgfname, sizeof(msgfname));
	msgfile = fopen(msgfname, "r");
	if (!msgfile) {
	fprintf(stderr,
	("reconstruct: fopen() failed for '%s' "
	"[error=%d] -- skipping.\n"), msgfname, errno);
	continue;
	}

	if (fstat(fileno(msgfile), &sbuf)) {
	fclose(msgfile);
	continue;
	}
	if (sbuf.st_size == 0) {
	/* Zero-length message file--blow it away */
	fclose(msgfile);
	unlink(msgfname);
	continue;
	}

	/* Was this message expunged? */
	while ((expmsg < expunge_exists) &&
	(expunge_uidmap[expmsg].uid < uid[msg]))
	expmsg++;

	expunge_found = 0;
	if ((expmsg < expunge_exists) &&
	(expunge_uidmap[expmsg].uid == uid[msg])) {
	expunge_found = 1;
	}

	/* Does this message have index record in cyrus.index or expunge? */
	index_found = 0;
	if (expunge_found) {
	unsigned msgno = expunge_uidmap[expmsg].msgno;

	if ((msgno > 0) && !reconstruct_read_index_record
	(mailbox.name, expunge_base, expunge_len, msgno, &tmp_index))
	index_found = 1;
	expmsg++;
	} else {
	while (oldmsg < mailbox.exists && old_index.uid < uid[msg]) {
	if (mailbox_read_index_record(&mailbox, ++oldmsg,
	&old_index)) {
	old_index.uid = 0;
	}
	}
	if (old_index.uid == uid[msg]) {
	memcpy(&tmp_index, &old_index, sizeof(struct index_record));
	index_found = 1;
	}
	}

	if (index_found) {
	/* Use data in old index file, subject to validity checks */
	message_index.internaldate = tmp_index.internaldate;
	message_index.last_updated = tmp_index.last_updated;

	message_index.modseq = tmp_index.modseq;
	/* This should never happen, but bugs in 2.3.4 and 2.3.5
	* could have left modseq blank. If so, update it */
	if (!message_index.modseq) message_index.modseq = 1;
	message_index.system_flags = tmp_index.system_flags &
	(FLAG_ANSWERED\|FLAG_FLAGGED\|FLAG_DELETED\|FLAG_DRAFT);
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	message_index.user_flags[i] =
	tmp_index.user_flags[i] & valid_user_flags[i];
	}
	/* Copy across MessageGUID if confident that data on disk */
	message_guid_copy(&message_index.guid, &tmp_index.guid);
	} else {
	/* Message file write time is good estimate of internaldate */
	message_index.internaldate = sbuf.st_mtime;
	message_index.last_updated = time(0);
	/* If we are recovering a message, assume new UIDL
	so that stupid clients will retrieve this message */
	mailbox.options \|= OPT_POP3_NEW_UIDL;
	/* Wipe the Message GUID */
	message_guid_set_null(&message_index.guid);
	/* If we are recovering a message, reset MODSEQ */
	message_index.modseq = 1;
	}

	if (message_index.modseq > highestmodseq) {
	highestmodseq = message_index.modseq;
	}

	/* Force rebuild from message_create_record() */
	if (guid_set) message_guid_set_null(&message_index.guid);

	/* NB: message_create_record() will reconstruct GUID if NULL */
	if (((r = message_parse_file(msgfile, NULL, NULL, &body)) != 0) \|\|
	((r = message_create_record(mailbox.name, newcache_fd,
	&message_index, body)) != 0)) {
	r = IMAP_IOERROR;
	goto bail;
	}
	fclose(msgfile);
	if (body) message_free_body(body);

	/* Clear out existing or regenerated GUID */
	if (guid_clear) message_guid_set_null(&message_index.guid);

	if (expunge_found && keepflag) {
	/* Write out new entry in expunge file */
	reconstruct_counts_update(&expunge_counts, &message_index);
	mailbox_index_record_to_buf(&message_index, buf);
	n = fwrite(buf, 1, INDEX_RECORD_SIZE, newexpunge);
	} else {
	/* Write out new entry in index file */
	reconstruct_counts_update(&index_counts, &message_index);
	mailbox_index_record_to_buf(&message_index, buf);
	n = fwrite(buf, 1, INDEX_RECORD_SIZE, newindex);
	}

	if (n != INDEX_RECORD_SIZE) {
	r = IMAP_IOERROR;
	goto bail;
	}
	}

	/* Write out new index and expunge file headers */
	if (uid_num && mailbox.last_uid < uid[uid_num-1]) {
	syslog (LOG_ERR, "Updating last_uid for %s: %lu => %lu",
	mailbox.name, mailbox.last_uid, uid[uid_num-1] + 100);
	mailbox.last_uid = uid[uid_num-1] + 100;
	}

	if (mailbox.last_appenddate == 0 \|\| mailbox.last_appenddate > now) {
	syslog (LOG_ERR, "Updating last_appenddate for %s: %lu => %lu",
	mailbox.name, mailbox.last_appenddate, now);
	mailbox.last_appenddate = now;
	}

	if (mailbox.uidvalidity == 0 \|\| mailbox.uidvalidity > (unsigned)now) {
	syslog (LOG_ERR, "Updating uidvalidity for %s: %lu => %lu",
	mailbox.name, mailbox.uidvalidity, now);
	mailbox.uidvalidity = (unsigned)now;
	}

	if (mailbox.highestmodseq < highestmodseq) {
	syslog (LOG_ERR, "Updating highestmodseq for %s: "
	MODSEQ_FMT " => " MODSEQ_FMT,
	mailbox.name, mailbox.highestmodseq, highestmodseq);
	mailbox.highestmodseq = highestmodseq;
	}

	if (mailbox.quota_mailbox_used != index_counts.newquota_used) {
	syslog (LOG_ERR, "Updating quota_mailbox_used for %s: "
	QUOTA_T_FMT " => " QUOTA_T_FMT,
	mailbox.name, mailbox.quota_mailbox_used, index_counts.newquota_used);
	/* updated by the counts_tobuf below, different in each file */
	}

	rewind(newindex);
	reconstruct_counts_tobuf(buf, &mailbox, &index_counts);
	n = fwrite(buf, 1, INDEX_HEADER_SIZE, newindex);
	if (n != INDEX_HEADER_SIZE) {
	r = IMAP_IOERROR;
	goto bail;
	}
	rewind(newexpunge);
	reconstruct_counts_tobuf(buf, &mailbox, &expunge_counts);
	n = fwrite(buf, 1, INDEX_HEADER_SIZE, newexpunge);
	if (n != INDEX_HEADER_SIZE) {
	r = IMAP_IOERROR;
	goto bail;
	}

	fflush(newindex);
	fflush(newexpunge);
	if (ferror(newindex) \|\| ferror(newexpunge) \|\| fsync(newcache_fd) \|\|
	fsync(fileno(newindex)) \|\| fsync(fileno(newexpunge))) {
	r = IMAP_IOERROR;
	goto bail;
	}

	/* Free temporary resources now that the index/expunge update is done */
	close(newcache_fd);
	fclose(newexpunge);
	fclose(newindex);

	if (expunge_base) map_free(&expunge_base, &expunge_len);
	if (expunge_fd >= 0) close(expunge_fd);
	if (expunge_uidmap) free(expunge_uidmap);
	if (uid) free(uid);
	if (body) free(body);

	expunge_base = NULL;
	expunge_len = 0;
	newexpunge = newindex = NULL;
	uid = NULL;
	body = NULL;
	expunge_uidmap = NULL;
	expunge_fd = -1;
	newcache_fd = -1;

	/* validate uniqueid */
	if (!mailbox.uniqueid) {

	/* this may change uniqueid, but if it does, nothing we can do
	about it */
	mailbox_make_uniqueid(mailbox.name, mailbox.uidvalidity, unique_buf,
	sizeof(unique_buf));
	mailbox.uniqueid = xstrdup(unique_buf);
	} else {
	if (find_uniqid (mailbox.name, mailbox.uniqueid) != NULL ) {

	mailbox_make_uniqueid(mailbox.name, mailbox.uidvalidity,
	unique_buf, sizeof(unique_buf));
	free (mailbox.uniqueid);
	mailbox.uniqueid = xstrdup(unique_buf);
	}
	}
	if (add_uniqid (mailbox.name, mailbox.uniqueid) == NULL) {
	syslog (LOG_ERR, "Failed adding mailbox: %s unique id: %s\n",
	mailbox.name, mailbox.uniqueid );
	}

	/* Write header and commit replacement index/cache files.
	*
	* Do cyrus.index last as this blows away the index lock. In contrast
	* mailbox_write_header() locks the new header file before it commits.
	* That lock is only released on mailbox_close().
	*/
	r = mailbox_write_header(&mailbox);
	if (!r)
	r = reconstruct_rename_single(&mailbox,
	IMAP_ENUM_METAPARTITION_FILES_CACHE,
	FNAME_CACHE);
	if (!r)
	r = reconstruct_rename_single(&mailbox,
	IMAP_ENUM_METAPARTITION_FILES_EXPUNGE,
	FNAME_EXPUNGE_INDEX);
	if (expunge_counts.newexists == 0) {
	reconstruct_delete_single(&mailbox,
	IMAP_ENUM_METAPARTITION_FILES_EXPUNGE,
	FNAME_EXPUNGE_INDEX, NULL);
	}
	if (!r)
	r = reconstruct_rename_single(&mailbox,
	IMAP_ENUM_METAPARTITION_FILES_INDEX,
	FNAME_INDEX);
	if (r) {
	mailbox_close(&mailbox);
	return (r);
	}

	r = seen_reconstruct(&mailbox,
	(time_t)0, (time_t)0, (int ()())0, (void )0);
	if (syncflag) {
	sync_log_mailbox(mailbox.name);
	}
	mailbox_close(&mailbox);

	if (found) {
	if (mympath &&
	(config_metapartition_files &
	IMAP_ENUM_METAPARTITION_FILES_HEADER) &&
	chdir(mympath) == -1) {
	return IMAP_IOERROR;
	}

	/* we recurse down this directory to see if there's any mailboxes
	under this not in the mailboxes database */
	dirp = opendir(".");

	while ((dirent = readdir(dirp)) != NULL) {
	struct discovered *new;

	/* mailbox directories never have a dot in them */
	if (strchr(dirent->d_name, '.')) continue;
	if (stat(dirent->d_name, &sbuf) < 0) continue;
	if (!S_ISDIR(sbuf.st_mode)) continue;

	/* ok, we found a directory that doesn't have a dot in it;
	is there a cyrus.header file? */
	snprintf(fnamebuf, sizeof(fnamebuf), "%s%s",
	dirent->d_name, FNAME_HEADER);
	if (stat(fnamebuf, &sbuf) < 0) continue;

	/* ok, we have a real mailbox directory */
	snprintf(fnamebuf, sizeof(fnamebuf), "%s.%s",
	name, dirent->d_name);

	/* does fnamebuf exist as a mailbox in mboxlist? */
	do {
	r = mboxlist_lookup(fnamebuf, NULL, NULL);
	} while (r == IMAP_AGAIN);
	if (!r) continue; /* mailbox exists; it'll be reconstructed
	with a -r */

	if (r != IMAP_MAILBOX_NONEXISTENT) break; /* erg? */
	else r = 0; /* reset error condition */

	printf("discovered %s\n", fnamebuf);
	new = (struct discovered *) xmalloc(sizeof(struct discovered));
	new->name = strdup(fnamebuf);
	new->next = found->next;
	found->next = new;
	}
	closedir(dirp);
	}

	return r;

	bail:
	if (msgfile) fclose(msgfile);
	if (newindex) fclose(newindex);
	if (newexpunge) fclose(newexpunge);
	if (expunge_fd >= 0) close(expunge_fd);
	if (newcache_fd >= 0) close(newcache_fd);
	if (uid) free(uid);
	if (expunge_uidmap) free(expunge_uidmap);
	if (expunge_base) map_free(&expunge_base, &expunge_len);

	mailbox_close(&mailbox);
	return r;
	}

	/*
	* Reconstruct the mailboxes list.
	*/
	void do_mboxlist(void)
	{
	fprintf(stderr, "reconstructing mailboxes.db currently not supported\n");
	exit(EC_USAGE);
	}

File Metadata

Mime Type: text/x-c
Expires: Mon, Apr 6, 12:37 AM (6 d, 6 h ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 18831725
Default Alt Text: reconstruct.c (41 KB)

reconstruct.cNo OneTemporaryActions

reconstruct.cView Options

File Metadata

Event Timeline

reconstruct.c
No OneTemporary
Actions

reconstruct.c
View Options