No OneTemporary
Actions

Authored By

Unknown

Size

700 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/imap/append.c b/imap/append.c
	index dac46fe95..26fbc8d61 100644
	--- a/imap/append.c
	+++ b/imap/append.c
	@@ -1,975 +1,975 @@
	/* append.c -- Routines for appending messages to a mailbox
	- * $Id: append.c,v 1.82 2001/09/30 12:39:55 ken3 Exp $
	+ * $Id: append.c,v 1.83 2002/02/19 18:50:11 ken3 Exp $
	*
	* Copyright (c)1998, 2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <ctype.h>
	#include <string.h>
	#include <sys/types.h>
	#include <syslog.h>
	#include <sys/stat.h>

	#include "acl.h"
	#include "assert.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "message.h"
	#include "append.h"
	#include "imapconf.h"
	#include "prot.h"
	#include "xmalloc.h"
	#include "mboxlist.h"
	#include "seen.h"
	#include "retry.h"

	struct stagemsg {
	unsigned long size;
	time_t internaldate;
	char fname[1024];

	int num_parts;
	char parts[1][MAX_MAILBOX_PATH];
	};

	static int append_addseen(struct mailbox mailbox, const char userid,
	const char *msgrange);
	static void addme(char *msgrange, int alloced, long uid);

	#define zero_index(i) { memset(&i, 0, sizeof(struct index_record)); }

	/*
	* Check to see if mailbox can be appended to
	*
	* Arguments:
	* name - name of mailbox directory
	* format - mailbox must be of this format
	* aclcheck - user must have these rights on mailbox ACL
	* quotacheck - mailbox must have this much quota left
	* (-1 means don't care about quota)
	*
	*/
	int append_check(const char *name, int format,
	struct auth_state *auth_state,
	long aclcheck, long quotacheck)
	{
	struct mailbox m;
	int r;

	r = mailbox_open_header(name, auth_state, &m);
	if (r) return r;

	if ((m.myrights & aclcheck) != aclcheck) {
	r = (m.myrights & ACL_LOOKUP) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	mailbox_close(&m);
	return r;
	}

	r = mailbox_open_index(&m);
	if (r) {
	mailbox_close(&m);
	return r;
	}

	if (m.format != format) {
	mailbox_close(&m);
	return IMAP_MAILBOX_NOTSUPPORTED;
	}

	r = mailbox_read_quota(&m.quota);
	if (r) {
	mailbox_close(&m);
	return r;
	}

	if (m.quota.limit >= 0 && quotacheck >= 0 &&
	m.quota.used + quotacheck > m.quota.limit * QUOTA_UNITS) {
	mailbox_close(&m);
	return IMAP_QUOTA_EXCEEDED;
	}

	mailbox_close(&m);
	return 0;
	}

	/*
	* Open a mailbox for appending
	*
	* Arguments:
	* name - name of mailbox directory
	* format - mailbox must be of this format
	* aclcheck - user must have these rights on mailbox ACL
	* quotacheck - mailbox must have this much quota left
	* (-1 means don't care about quota)
	*
	* On success, the struct pointed to by 'as' is set up.
	*
	*/
	int append_setup(struct appendstate as, const char name,
	int format,
	const char userid, struct auth_state auth_state,
	long aclcheck, long quotacheck)
	{
	int r;

	r = mailbox_open_header(name, auth_state, &as->m);
	if (r) return r;

	if ((as->m.myrights & aclcheck) != aclcheck) {
	r = (as->m.myrights & ACL_LOOKUP) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	mailbox_close(&as->m);
	return r;
	}

	r = mailbox_lock_header(&as->m);
	if (r) {
	mailbox_close(&as->m);
	return r;
	}

	r = mailbox_open_index(&as->m);
	if (r) {
	mailbox_close(&as->m);
	return r;
	}

	if (as->m.format != format) {
	mailbox_close(&as->m);
	return IMAP_MAILBOX_NOTSUPPORTED;
	}

	r = mailbox_lock_index(&as->m);
	if (r) {
	mailbox_close(&as->m);
	return r;
	}

	r = mailbox_lock_quota(&as->m.quota);
	if (r) {
	mailbox_close(&as->m);
	return r;
	}

	if (as->m.quota.limit >= 0 && quotacheck >= 0 &&
	as->m.quota.used + quotacheck >
	as->m.quota.limit * QUOTA_UNITS) {
	mailbox_close(&as->m);
	return IMAP_QUOTA_EXCEEDED;
	}

	if (userid) {
	strcpy(as->userid, userid);
	} else {
	as->userid[0] = '\0';
	}

	/* zero out metadata */
	as->orig_cache_len = as->m.cache_len;
	as->nummsg = as->numanswered =
	as->numdeleted = as->numflagged = 0;
	as->quota_used = 0;
	as->writeheader = 0;
	as->seen_msgrange = NULL;
	as->seen_alloced = 0;

	as->s = APPEND_READY;

	return 0;
	}

	/* may return non-zero, indicating that the entire append has failed
	and the mailbox is probably in an inconsistent state. */
	int append_commit(struct appendstate *as,
	unsigned long *uidvalidity,
	unsigned long *start,
	unsigned long *num)
	{
	int r = 0;

	if (as->s == APPEND_DONE) return 0;
	as->s = APPEND_DONE;

	if (start) *start = as->m.last_uid + 1;
	if (num) *num = as->nummsg;
	if (uidvalidity) *uidvalidity = as->m.uidvalidity;

	/* write out the header if we created new user flags */
	if (as->writeheader && (r = mailbox_write_header(&as->m))) {
	append_abort(as);
	return r;
	}

	/* flush the new index records */
	if (fsync(as->m.index_fd)) {
	syslog(LOG_ERR, "IOERROR: writing index records for %s: %m",
	as->m.name);
	append_abort(as);
	return IMAP_IOERROR;
	}

	/* Flush out the cache file data */
	if (fsync(as->m.cache_fd)) {
	syslog(LOG_ERR, "IOERROR: writing cache file for %s: %m",
	as->m.name);
	append_abort(as);
	return IMAP_IOERROR;
	}

	/* Calculate new index header information */
	as->m.exists += as->nummsg;
	as->m.last_uid += as->nummsg;

	as->m.answered += as->numanswered;
	as->m.deleted += as->numdeleted;
	as->m.flagged += as->numflagged;

	as->m.last_appenddate = time(0);
	as->m.quota_mailbox_used += as->quota_used;
	if (as->m.minor_version > MAILBOX_MINOR_VERSION) {
	as->m.minor_version = MAILBOX_MINOR_VERSION;
	}

	/* Write out index header & synchronize to disk.
	this writes to acappush too. */
	r = mailbox_write_index_header(&as->m);
	if (r) {
	append_abort(as);
	return r;
	}

	/* Write out quota file */
	as->m.quota.used += as->quota_used;
	r = mailbox_write_quota(&as->m.quota);
	if (r) {
	syslog(LOG_ERR,
	"LOSTQUOTA: unable to record use of %u bytes in quota file %s",
	as->quota_used, as->m.quota.root);
	}

	/* set seen state */
	if (as->seen_msgrange && as->userid[0]) {
	append_addseen(&as->m, as->userid, as->seen_msgrange);
	}
	if (as->seen_msgrange) {
	free(as->seen_msgrange);
	}

	mailbox_unlock_quota(&as->m.quota);
	mailbox_unlock_index(&as->m);
	mailbox_unlock_header(&as->m);
	mailbox_close(&as->m);

	return 0;
	}

	/* may return non-zero, indicating an internal error of some sort. */
	int append_abort(struct appendstate *as)
	{
	int r = 0;
	int uid;

	if (as->s == APPEND_DONE) return 0;
	as->s = APPEND_DONE;

	/* unlink message files that were created */
	for (uid = as->m.last_uid + 1; uid <= as->m.last_uid + as->nummsg; uid++) {
	char fname[MAX_MAILBOX_PATH];

	/* Create message file */
	strcpy(fname, as->m.path);
	strcat(fname, "/");
	mailbox_message_get_fname(&as->m, uid, fname + strlen(fname));
	if (unlink(fname) < 0) {
	/* hmmm, never got appended? */
	/* r = IMAP_IOERROR; */
	}
	}

	/* truncate the cache */
	ftruncate(as->m.cache_fd, as->orig_cache_len);

	/* unlock mailbox */
	mailbox_unlock_quota(&as->m.quota);
	mailbox_unlock_index(&as->m);
	mailbox_unlock_header(&as->m);
	mailbox_close(&as->m);

	if (as->seen_msgrange) {
	free(as->seen_msgrange);
	}

	return r;
	}

	/*
	* Return the number of stage msgs
	*/

	int append_stageparts(struct stagemsg *stagep)
	{
	if (!stagep) return 0;

	return stagep->num_parts;
	}

	/*
	* staging, to allow for single-instance store. the complication here
	* is multiple partitions.
	*/
	int append_fromstage(struct appendstate *as,
	struct protstream *messagefile,
	unsigned long size, time_t internaldate,
	const char **flag, int nflags,
	struct stagemsg **stagep)
	{
	struct mailbox *mailbox = &as->m;
	struct index_record message_index;
	char fname[MAX_MAILBOX_PATH];
	FILE *destfile;
	int i, r;
	int userflag, emptyflag;

	/* for staging */
	struct stagemsg *stage;
	char stagefile[1024];
	FILE *f;
	int sp;

	assert(stagep != NULL);
	assert(mailbox->format == MAILBOX_FORMAT_NORMAL);
	assert(size != 0);

	zero_index(message_index);
	if (!stagep) { / create a new stage */
	stage = xmalloc(sizeof(struct stagemsg) +
	5 * MAX_MAILBOX_PATH * sizeof(char));

	stage->size = size;
	stage->internaldate = internaldate;
	sprintf(stage->fname, "%d-%d",(int) getpid(), (int) internaldate);
	stage->num_parts = 5; /* room for 5 paths */
	stage->parts[0][0] = '\0';
	} else {
	stage = stagep; / reuse existing stage */
	}

	mboxlist_findstage(mailbox->name, stagefile);
	strcat(stagefile, stage->fname);

	sp = 0;
	while (stage->parts[sp][0] != '\0') {
	if (!strcmp(stagefile, stage->parts[sp]))
	break;
	sp++;
	}
	if (stage->parts[sp][0] == '\0') {
	/* ok, create this file and add put it into stage->parts[sp] */
	f = fopen(stagefile, "w+");
	if (!f) {
	char stagedir[1024];

	mboxlist_findstage(mailbox->name, stagedir);
	if (mkdir(stagedir, 0755) != 0) {
	syslog(LOG_ERR, "couldn't create stage directory: %s: %m",
	stagedir);
	} else {
	syslog(LOG_NOTICE, "created stage directory %s",
	stagedir);
	f = fopen(stagefile, "w+");
	}
	}
	if (!f) {
	syslog(LOG_ERR, "IOERROR: creating message file %s: %m",
	stagefile);
	return IMAP_IOERROR;
	}

	r = message_copy_strict(messagefile, f, size);
	fclose(f);
	if (r) {
	unlink(stagefile);
	return r;
	}

	if (sp == stage->num_parts) {
	/* need more room */
	stage->num_parts += 5;
	stage = xrealloc(stage, sizeof(struct stagemsg) +
	stage->num_parts * MAX_MAILBOX_PATH *
	sizeof(char));
	}
	strcpy(stage->parts[sp], stagefile);
	stage->parts[sp+1][0] = '\0';
	}

	/* stage->parts[sp] contains the message and is on the same partition
	as the mailbox we're looking at */

	/* Setup */
	message_index.uid = mailbox->last_uid + as->nummsg + 1;
	message_index.last_updated = time(0);
	message_index.internaldate = internaldate;
	lseek(mailbox->cache_fd, 0L, SEEK_END);

	/* Create message file */
	as->nummsg++;
	strcpy(fname, mailbox->path);
	strcat(fname, "/");
	mailbox_message_get_fname(mailbox, message_index.uid,
	fname + strlen(fname));

	r = mailbox_copyfile(stage->parts[sp], fname);
	destfile = fopen(fname, "r");
	if (!r && destfile) {
	/* ok, we've successfully created the file */
	r = message_parse_file(destfile, mailbox, &message_index);
	}
	if (destfile) {
	/* this will hopefully ensure that the link() actually happened */
	if (APPEND_ULTRA_PARANOID) fsync(fileno(destfile));
	fclose(destfile);
	}
	if (r) {
	append_abort(as);
	return r;
	}

	/* Handle flags the user wants to set in the message */
	for (i = 0; i < nflags; i++) {
	if (!strcmp(flag[i], "\\seen")) {
	addme(&as->seen_msgrange, &as->seen_alloced, message_index.uid);
	}
	else if (!strcmp(flag[i], "\\deleted")) {
	if (mailbox->myrights & ACL_DELETE) {
	message_index.system_flags \|= FLAG_DELETED;
	as->numdeleted++;
	}
	}
	else if (!strcmp(flag[i], "\\draft")) {
	if (mailbox->myrights & ACL_WRITE) {
	message_index.system_flags \|= FLAG_DRAFT;
	}
	}
	else if (!strcmp(flag[i], "\\flagged")) {
	if (mailbox->myrights & ACL_WRITE) {
	message_index.system_flags \|= FLAG_FLAGGED;
	as->numflagged++;
	}
	}
	else if (!strcmp(flag[i], "\\answered")) {
	if (mailbox->myrights & ACL_WRITE) {
	message_index.system_flags \|= FLAG_ANSWERED;
	as->numanswered++;
	}
	}
	else if (mailbox->myrights & ACL_WRITE) {
	/* User flag */
	emptyflag = -1;
	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if (mailbox->flagname[userflag]) {
	if (!strcasecmp(flag[i], mailbox->flagname[userflag]))
	break;
	}
	else if (emptyflag == -1) emptyflag = userflag;
	}

	/* Flag is not defined--create it */
	if (userflag == MAX_USER_FLAGS && emptyflag != -1) {
	userflag = emptyflag;
	mailbox->flagname[userflag] = xstrdup(flag[i]);
	as->writeheader++;
	}

	if (userflag != MAX_USER_FLAGS) {
	message_index.user_flags[userflag/32] \|= 1<<(userflag&31);
	}
	}
	}

	/* Write out index file entry */
	r = mailbox_append_index(mailbox, &message_index,
	mailbox->exists + as->nummsg - 1, 1, 0);
	if (r) {
	append_abort(as);
	return r;
	}

	/* ok, we've successfully added a message */
	as->quota_used += message_index.size;

	*stagep = stage;
	return 0;
	}

	int append_removestage(struct stagemsg *stage)
	{
	int i;

	if (stage == NULL) return 0;

	i = 0;
	while (stage->parts[i][0] != '\0') {
	/* unlink the staging file */
	if (unlink(stage->parts[i]) != 0) {
	syslog(LOG_ERR, "IOERROR: error unlinking file %s: %m",
	stage->parts[i]);
	}
	i++;
	}

	free(stage);
	return 0;
	}

	/*
	* Append to 'mailbox' from the prot stream 'messagefile'.
	* 'mailbox' must have been opened with append_setup().
	* 'size' is the expected size of the message.
	* 'internaldate' specifies the internaldate for the new message.
	* 'flags' contains the names of the 'nflags' flags that the
	* user wants to set in the message. If the '\Seen' flag is
	* in 'flags', then the 'userid' passed to append_setup controls whose
	* \Seen flag gets set.
	*
	* The message is not committed to the mailbox (nor is the mailbox
	* unlocked) until append_commit() is called. multiple
	* append_onefromstream()s can be aborted by calling append_abort().
	*/
	int append_fromstream(struct appendstate *as,
	struct protstream *messagefile,
	unsigned long size,
	time_t internaldate,
	const char **flag,
	int nflags)
	{
	struct mailbox *mailbox = &as->m;
	struct index_record message_index;
	char fname[MAX_MAILBOX_PATH];
	FILE *destfile;
	int i, r;
	int userflag, emptyflag;

	assert(mailbox->format == MAILBOX_FORMAT_NORMAL);
	assert(size != 0);

	zero_index(message_index);
	/* Setup */
	message_index.uid = mailbox->last_uid + as->nummsg + 1;
	message_index.last_updated = time(0);
	message_index.internaldate = internaldate;
	lseek(mailbox->cache_fd, 0L, SEEK_END);

	/* Create message file */
	strcpy(fname, mailbox->path);
	strcat(fname, "/");
	mailbox_message_get_fname(mailbox, message_index.uid,
	fname + strlen(fname));
	as->nummsg++;

	unlink(fname);
	destfile = fopen(fname, "w+");
	if (!destfile) {
	syslog(LOG_ERR, "IOERROR: creating message file %s: %m", fname);
	append_abort(as);
	return IMAP_IOERROR;
	}

	/* Copy and parse message */
	r = message_copy_strict(messagefile, destfile, size);
	if (!r) {
	r = message_parse_file(destfile, mailbox, &message_index);
	}
	fclose(destfile);
	if (r) {
	append_abort(as);
	return r;
	}

	/* Handle flags the user wants to set in the message */
	for (i = 0; i < nflags; i++) {
	if (!strcmp(flag[i], "\\seen")) {
	addme(&as->seen_msgrange, &as->seen_alloced, message_index.uid);
	}
	else if (!strcmp(flag[i], "\\deleted")) {
	if (mailbox->myrights & ACL_DELETE) {
	message_index.system_flags \|= FLAG_DELETED;
	as->numdeleted++;
	}
	}
	else if (!strcmp(flag[i], "\\draft")) {
	if (mailbox->myrights & ACL_WRITE) {
	message_index.system_flags \|= FLAG_DRAFT;
	}
	}
	else if (!strcmp(flag[i], "\\flagged")) {
	if (mailbox->myrights & ACL_WRITE) {
	message_index.system_flags \|= FLAG_FLAGGED;
	as->numflagged++;
	}
	}
	else if (!strcmp(flag[i], "\\answered")) {
	if (mailbox->myrights & ACL_WRITE) {
	message_index.system_flags \|= FLAG_ANSWERED;
	as->numanswered++;
	}
	}
	else if (mailbox->myrights & ACL_WRITE) {
	/* User flag */
	emptyflag = -1;
	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if (mailbox->flagname[userflag]) {
	if (!strcasecmp(flag[i], mailbox->flagname[userflag]))
	break;
	}
	else if (emptyflag == -1) emptyflag = userflag;
	}

	/* Flag is not defined--create it */
	if (userflag == MAX_USER_FLAGS && emptyflag != -1) {
	userflag = emptyflag;
	mailbox->flagname[userflag] = xstrdup(flag[i]);
	as->writeheader++;
	}

	if (userflag != MAX_USER_FLAGS) {
	message_index.user_flags[userflag/32] \|= 1<<(userflag&31);
	}
	}
	}

	/* Write out index file entry; if we abort later, it's not
	important */
	r = mailbox_append_index(mailbox, &message_index,
	mailbox->exists + as->nummsg - 1, 1, 0);
	if (r) {
	append_abort(as);
	return r;
	}

	/* ok, we've successfully added a message */
	as->quota_used += message_index.size;

	return 0;
	}

	/*
	* Append to 'append_mailbox' ('as') the 'nummsg' messages from the
	* mailbox 'mailbox' listed in the array pointed to by 'copymsg'.
	* 'as' must have been opened with append_setup(). If the '\Seen'
	* flag is to be set anywhere then 'userid' passed to append_setup()
	* contains the name of the user whose \Seen flag gets set.
	*/
	int append_copy(struct mailbox *mailbox,
	struct appendstate *as,
	int nummsg,
	struct copymsg *copymsg)
	{
	struct mailbox *append_mailbox = &as->m;
	int msg;
	struct index_record *message_index;
	char fname[MAX_MAILBOX_PATH];
	const char *src_base;
	unsigned long src_size;
	const char startline, endline;
	FILE *destfile;
	int r, n;
	int flag, userflag, emptyflag;

	assert(append_mailbox->format == MAILBOX_FORMAT_NORMAL);

	if (!nummsg) {
	append_abort(as);
	return 0;
	}

	lseek(append_mailbox->cache_fd, 0L, SEEK_END);
	message_index = (struct index_record *)
	xmalloc(nummsg * sizeof(struct index_record));

	/* Copy/link all files and cache info */
	for (msg = 0; msg < nummsg; msg++) {
	zero_index(message_index[msg]);
	message_index[msg].uid = append_mailbox->last_uid + 1 + as->nummsg;
	message_index[msg].last_updated = time(0);
	message_index[msg].internaldate = copymsg[msg].internaldate;
	as->nummsg++;

	strcpy(fname, append_mailbox->path);
	strcat(fname, "/");
	mailbox_message_get_fname(append_mailbox, message_index[msg].uid,
	fname + strlen(fname));

	if (copymsg[msg].cache_len) {
	char fnamebuf[MAILBOX_FNAME_LEN];

	mailbox_message_get_fname(mailbox, copymsg[msg].uid, fnamebuf);
	/* Link/copy message file */
	r = mailbox_copyfile(fnamebuf, fname);
	if (r) goto fail;

	/* Write out cache info, copy other info */
	message_index[msg].cache_offset =
	lseek(append_mailbox->cache_fd, 0L, SEEK_CUR);
	message_index[msg].sentdate = copymsg[msg].sentdate;
	message_index[msg].size = copymsg[msg].size;
	message_index[msg].header_size = copymsg[msg].header_size;
	message_index[msg].content_offset = copymsg[msg].header_size;

	n = retry_write(append_mailbox->cache_fd, copymsg[msg].cache_begin,
	copymsg[msg].cache_len);
	if (n == -1) {
	syslog(LOG_ERR, "IOERROR: writing cache file for %s: %m",
	append_mailbox->name);
	r = IMAP_IOERROR;
	goto fail;
	}
	} else {
	/*
	* Have to copy the message, possibly converting LF to CR LF
	* Then, we have to parse the message.
	*/
	r = 0;
	unlink(fname);
	destfile = fopen(fname, "w+");
	if (!destfile) {
	syslog(LOG_ERR, "IOERROR: writing message file %s: %m", fname);
	r = IMAP_IOERROR;
	goto fail;
	}
	if (mailbox_map_message(mailbox, 0, copymsg[msg].uid,
	&src_base, &src_size) != 0) {
	fclose(destfile);
	- syslog(LOG_ERR, "IOERROR: opening message file %u of %s: %m",
	+ syslog(LOG_ERR, "IOERROR: opening message file %lu of %s: %m",
	copymsg[msg].uid, mailbox->name);
	r = IMAP_IOERROR;
	goto fail;
	}

	startline = src_base;
	while ( (endline = memchr(startline, '\n',
	src_size - (startline - src_base))) ) {
	fwrite(startline, 1, (endline - startline), destfile);
	if (endline == startline \|\| endline[-1] != '\r') {
	putc('\r', destfile);
	}
	putc('\n', destfile);
	startline = endline+1;
	}
	fwrite(startline, 1, src_size - (startline - src_base), destfile);

	fflush(destfile);
	if (ferror(destfile) \|\| fsync(fileno(destfile))) {
	syslog(LOG_ERR, "IOERROR: writing message: %m");
	r = IMAP_IOERROR;
	}

	mailbox_unmap_message(mailbox, copymsg[msg].uid,
	&src_base, &src_size);

	if (!r) r = message_parse_file(destfile, append_mailbox,
	&message_index[msg]);
	fclose(destfile);
	if (r) goto fail;
	}

	as->quota_used += message_index[msg].size;

	/* Handle any flags that need to be copied */
	if (append_mailbox->myrights & ACL_WRITE) {
	message_index[msg].system_flags =
	copymsg[msg].system_flags & ~FLAG_DELETED;

	for (flag = 0; copymsg[msg].flag[flag]; flag++) {
	emptyflag = -1;
	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if (append_mailbox->flagname[userflag]) {
	if (!strcasecmp(copymsg[msg].flag[flag],
	append_mailbox->flagname[userflag]))
	break;
	}
	else if (emptyflag == -1) emptyflag = userflag;
	}

	/* Flag is not defined--create it */
	if (userflag == MAX_USER_FLAGS && emptyflag != -1) {
	userflag = emptyflag;
	append_mailbox->flagname[userflag] =
	xstrdup(copymsg[msg].flag[flag]);
	as->writeheader++;
	}

	if (userflag != MAX_USER_FLAGS) {
	message_index[msg].user_flags[userflag/32] \|=
	1<<(userflag&31);
	}
	}
	}
	if (append_mailbox->myrights & ACL_DELETE) {
	message_index[msg].system_flags \|=
	copymsg[msg].system_flags & FLAG_DELETED;
	}

	if (message_index[msg].system_flags & FLAG_DELETED) as->numdeleted++;
	if (message_index[msg].system_flags & FLAG_ANSWERED) as->numanswered++;
	if (message_index[msg].system_flags & FLAG_FLAGGED) as->numflagged++;

	/* should this message be marked \Seen? */
	if (copymsg[msg].seen) {
	addme(&as->seen_msgrange, &as->seen_alloced,
	message_index[msg].uid);
	}
	}

	/* Write out index file entries */
	r = mailbox_append_index(append_mailbox, message_index,
	append_mailbox->exists + as->nummsg - nummsg,
	nummsg, 0);

	fail:
	if (r) append_abort(as);
	free(message_index);

	return r;
	}

	/* add 'uid' to 'msgrange'. 'uid' should be larger than anything in
	* 'msgrange'.
	*/
	static void addme(char *msgrange, int alloced, long uid)
	{
	char *p;
	int wasrange;
	int len;

	assert(msgrange != NULL);
	len = msgrange ? strlen(msgrange) : 0;
	if (*alloced < len + 40) {
	*alloced += 40;
	msgrange = (char ) xrealloc(msgrange, sizeof(char ) * (*alloced));
	}

	p = *msgrange;

	if (!len) {
	/* first time */
	sprintf(*msgrange, "%ld", uid);
	} else {
	/* this is tricky if this is the second number we're adding */
	wasrange = 0;

	/* see what the last one is */
	p = *msgrange + len - 1;
	while (isdigit((int) p) && p > msgrange) p--;
	/* second time, p == msgrange here */
	if (*p == ':') wasrange = 1;
	p++;
	if (atoi(p) == uid - 1) {
	if (!wasrange) {
	p = *msgrange + len;
	*p++ = ':';
	} else {
	/* we'll just overwrite the current number */
	}
	} else {
	p = *msgrange + len;
	*p++ = ',';
	}
	sprintf(p, "%ld", uid);
	return;
	}
	}

	/*
	* Set the \Seen flag for 'userid' in 'mailbox' for the messages from
	* 'msgrange'. the lowest msgrange must be larger than any previously
	* seen message.
	*/
	static int append_addseen(struct mailbox *mailbox,
	const char *userid,
	const char *msgrange)
	{
	int r;
	struct seen *seendb;
	time_t last_read, last_change;
	unsigned last_uid;
	char *seenuids;
	int last_seen;
	char tail, p;
	int oldlen;
	int start;

	/* what's the first uid in our new list? */
	start = atoi(msgrange);

	r = seen_open(mailbox, userid, &seendb);
	if (r) return r;

	r = seen_lockread(seendb, &last_read, &last_uid, &last_change, &seenuids);
	if (r) return r;

	oldlen = strlen(seenuids);
	seenuids = xrealloc(seenuids, oldlen + strlen(msgrange) + 10);

	tail = seenuids + oldlen;
	/* Scan back to last uid */
	while (tail > seenuids && isdigit((int) tail[-1])) tail--;
	for (p = tail, last_seen=0; p; p++) last_seen = last_seen 10 + *p - '0';
	if (last_seen && last_seen >= start-1) {
	if (tail > seenuids && tail[-1] == ':') p = tail - 1;
	*p++ = ':';
	}
	else {
	if (p > seenuids) *p++ = ',';
	}
	strcpy(p, msgrange);

	r = seen_write(seendb, last_read, last_uid, time(NULL), seenuids);
	seen_close(seendb);
	free(seenuids);
	return r;
	}

	diff --git a/imap/duplicate.c b/imap/duplicate.c
	index 3f080c8e4..8d849728b 100644
	--- a/imap/duplicate.c
	+++ b/imap/duplicate.c
	@@ -1,341 +1,341 @@
	/*
	* Copyright (c) 2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	*/

	#include <config.h>

	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <syslog.h>
	#include <assert.h>
	#include <ctype.h>
	#include <time.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <fcntl.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#if HAVE_DIRENT_H
	# include <dirent.h>
	#else
	# define dirent direct
	# if HAVE_SYS_NDIR_H
	# include <sys/ndir.h>
	# endif
	# if HAVE_SYS_DIR_H
	# include <sys/dir.h>
	# endif
	# if HAVE_NDIR_H
	# include <ndir.h>
	# endif
	#endif
	#include <errno.h>

	#include <db.h>

	#include "xmalloc.h"
	#include "imap_err.h"
	#include "imapconf.h"
	#include "exitcodes.h"
	#include "util.h"
	#include "cyrusdb.h"

	#include "duplicate.h"

	#define DB (CONFIG_DB_DUPLICATE)
	#define FNAME_DELIVERDB "/deliver.db"

	static struct db *dupdb = NULL;
	static int duplicate_dbopen = 0;


	int duplicate_init(char *fname, int myflags)
	{
	char buf[1024];
	int r = 0;
	int flags = 0;

	/* create the name of the db file */
	strcpy(buf, config_dir);
	strcat(buf, FNAME_DBDIR);
	if (myflags & DUPLICATE_RECOVER) flags \|= CYRUSDB_RECOVER;
	r = DB->init(buf, flags);

	if (r != 0)
	syslog(LOG_ERR, "DBERROR: init %s: %s", buf,
	cyrusdb_strerror(r));
	else {
	char *tofree = NULL;

	/* create db file name */
	if (!fname) {
	fname = xmalloc(strlen(config_dir)+sizeof(FNAME_DELIVERDB));
	tofree = fname;
	strcpy(fname, config_dir);
	strcat(fname, FNAME_DELIVERDB);
	}

	r = DB->open(fname, &dupdb);
	if (r != 0)
	syslog(LOG_ERR, "DBERROR: opening %s: %s", fname,
	cyrusdb_strerror(r));
	else
	duplicate_dbopen = 1;

	if (tofree) free(tofree);
	}

	return r;
	}

	time_t duplicate_check(char id, int idlen, char to, int tolen)
	{
	char buf[1024];
	int r;
	const char *data = NULL;
	int len = 0;
	time_t mark = 0;

	if (!duplicate_dbopen) return 0;

	if (idlen + tolen > sizeof(buf) - 30) return 0;
	memcpy(buf, id, idlen);
	buf[idlen] = '\0';
	memcpy(buf + idlen + 1, to, tolen);
	buf[idlen + tolen + 1] = '\0';

	do {
	r = DB->fetch(dupdb, buf,
	idlen + tolen + 2, /* +2 b/c 1 for the center null;
	+1 for the terminating null */
	&data, &len, NULL);
	} while (r == CYRUSDB_AGAIN);

	if (data) {
	assert(len == sizeof(time_t));

	/* found the record */
	memcpy(&mark, data, sizeof(time_t));
	} else if (r != CYRUSDB_OK) {
	syslog(LOG_ERR, "duplicate_check: error looking up %s/%s: %s",
	id, to,
	cyrusdb_strerror(r));
	mark = 0;
	}

	- syslog(LOG_DEBUG, "duplicate_check: %-40s %-20s %d",
	+ syslog(LOG_DEBUG, "duplicate_check: %-40s %-20s %ld",
	buf, buf+idlen+1, mark);

	return mark;
	}

	void duplicate_mark(char id, int idlen, char to, int tolen, time_t mark)
	{
	char buf[1024];
	int r;

	if (!duplicate_dbopen) return;

	if (idlen + tolen > sizeof(buf) - 30) return;
	memcpy(buf, id, idlen);
	buf[idlen] = '\0';
	memcpy(buf + idlen + 1, to, tolen);
	buf[idlen + tolen + 1] = '\0';

	do {
	r = DB->store(dupdb, buf,
	idlen + tolen + 2, /* +2 b/c 1 for the center null;
	+1 for the terminating null */
	(char *) &mark, sizeof(mark), NULL);
	} while (r == CYRUSDB_AGAIN);

	- syslog(LOG_DEBUG, "duplicate_mark: %-40s %-20s %d",
	+ syslog(LOG_DEBUG, "duplicate_mark: %-40s %-20s %ld",
	buf, buf+idlen+1, mark);

	return;
	}

	struct prunerock {
	struct db *db;
	time_t expmark;
	int count;
	int deletions;
	};

	static int prune_p(void rock, const char id, int idlen,
	const char *data, int datalen)
	{
	struct prunerock prock = (struct prunerock ) rock;
	time_t mark;

	prock->count++;

	/* grab the mark */
	memcpy(&mark, data, sizeof(time_t));

	/* check if we should prune this entry */
	return (mark < prock->expmark);
	}

	static int prune_cb(void rock, const char id, int idlen,
	const char *data, int datalen)
	{
	struct prunerock prock = (struct prunerock ) rock;
	int r;

	prock->deletions++;

	do {
	r = DB->delete(prock->db, id, idlen, NULL);
	} while (r == CYRUSDB_AGAIN);


	return 0;
	}

	int duplicate_prune(int days)
	{
	struct prunerock prock;

	if (days < 0) fatal("must specify positive number of days", EC_USAGE);

	prock.count = prock.deletions = 0;
	prock.expmark = time(NULL) - (days * 60 * 60 * 24);
	syslog(LOG_NOTICE, "duplicate_prune: pruning back %d days", days);

	/* check each entry in our database */
	prock.db = dupdb;
	DB->foreach(dupdb, "", 0, &prune_p, &prune_cb, &prock, NULL);

	syslog(LOG_NOTICE, "duplicate_prune: purged %d out of %d entries",
	prock.deletions, prock.count);

	return 0;
	}

	struct dumprock {
	FILE *f;
	int count;
	};

	static int dump_p(void *rock,
	const char *key, int keylen,
	const char *data, int datalen)
	{
	struct dumprock drock = (struct dumprock ) rock;

	drock->count++;

	return 1;
	}

	static const char hexcodes[] = "0123456789ABCDEF";

	static int dump_cb(void *rock,
	const char *key, int keylen,
	const char *data, int datalen)
	{
	struct dumprock drock = (struct dumprock ) rock;
	time_t mark;
	char id, to, *freeme;
	int idlen, i;

	assert(datalen == sizeof(time_t));

	memcpy(&mark, data, sizeof(time_t));
	to = (char*) key + strlen(key) + 1;
	id = (char *) key;
	idlen = strlen(id);

	for (i = 0; i < idlen; i++) {
	if (!isprint((unsigned char) id[i])) break;
	}

	if (i != idlen) {
	/* change to hexadecimal */
	freeme = (char ) xmalloc(sizeof(char) idlen * 2 + 1);
	for (i = 0; i < idlen; i++) {
	freeme[2 * i] = hexcodes[(id[i] >> 4) & 0xf];
	freeme[2 * i + 1] = hexcodes[id[i] & 0xf];
	}
	freeme[2 * idlen] = '\0';
	id = freeme;
	} else {
	freeme = NULL;
	}

	fprintf(drock->f, "id: %-40s\tto: %-20s\tat: %ld\n", id, to, (long) mark);

	if (freeme) free(freeme);

	return 0;
	}

	int duplicate_dump(FILE *f)
	{
	struct dumprock drock;

	drock.f = f;
	drock.count = 0;

	/* check each entry in our database */
	DB->foreach(dupdb, "", 0, &dump_p, &dump_cb, &drock, NULL);

	return drock.count;
	}

	int duplicate_done(void)
	{
	int r;

	if (duplicate_dbopen) {
	r = DB->close(dupdb);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error closing deliverdb: %s",
	cyrusdb_strerror(r));
	}
	duplicate_dbopen = 0;
	}
	r = DB->done();

	return r;
	}
	diff --git a/imap/idled.c b/imap/idled.c
	index 1f4f9bfe0..8bc34ee4f 100644
	--- a/imap/idled.c
	+++ b/imap/idled.c
	@@ -1,407 +1,404 @@
	/*
	* Copyright (c) 2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*/

	-/* $Id: idled.c,v 1.8 2001/03/14 18:26:35 ken3 Exp $ */
	+/* $Id: idled.c,v 1.9 2002/02/19 18:50:11 ken3 Exp $ */

	#ifdef HAVE_CONFIG_H
	#include <config.h>
	#endif

	#include <sys/types.h>
	#include <sys/socket.h>
	#include <sys/un.h>
	#include <syslog.h>
	#include <sys/time.h>
	#include <sys/stat.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <time.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <signal.h>
	#include <fcntl.h>

	#include "idled.h"
	#include "imapconf.h"
	#include "mboxlist.h"
	#include "xmalloc.h"
	#include "hash.h"
	#include "exitcodes.h"

	extern int optind;
	extern char *optarg;

	static int verbose = 0;
	static int debugmode = 0;
	static time_t idle_timeout;

	struct ientry {
	pid_t pid;
	time_t itime;
	struct ientry *next;
	};
	static struct hash_table itable;
	static struct ientry *ifreelist;
	static int itable_inc = 100;
	void idle_done(char *mboxname, pid_t pid);

	void fatal(const char *msg, int err)
	{
	if (debugmode) fprintf(stderr, "dying with %s %d\n",msg,err);
	syslog(LOG_CRIT, "%s", msg);
	syslog(LOG_NOTICE, "exiting");
	exit(err);
	}

	static int mbox_count_p(void *rockp,
	const char *key, int keylen,
	const char *data, int datalen)
	{
	return 1;
	}

	static int mbox_count_cb(void *rockp,
	const char *key, int keylen,
	const char *data, int datalen)
	{
	int ip = (int ) rockp;
	(*ip)++;

	return 0;
	}

	/* return a new 'ientry', either from the freelist or by malloc'ing it */
	static struct ientry *get_ientry(void)
	{
	struct ientry *t;

	if (!ifreelist) {
	/* create child_table_inc more and add them to the freelist */
	struct ientry *n;
	int i;

	n = xmalloc(itable_inc * sizeof(struct ientry));
	ifreelist = n;
	for (i = 0; i < itable_inc - 1; i++) {
	n[i].next = n + (i + 1);
	}
	/* i == child_table_inc - 1, last item in block */
	n[i].next = NULL;
	}

	t = ifreelist;
	ifreelist = ifreelist->next;

	return t;
	}

	/* remove pid from list of those idling on mboxname */
	void idle_done(char *mboxname, pid_t pid)
	{
	struct ientry t, p = NULL;

	t = (struct ientry *) hash_lookup(mboxname, &itable);
	while (t && t->pid != pid) {
	p = t;
	t = t->next;
	}
	if (t) {
	if (!p) {
	/* first pid in the linked list */

	p = t->next; /* remove node */

	/* we just removed the data that the hash entry
	was pointing to, so insert the new data */
	hash_insert(mboxname, p, &itable);
	}
	else {
	/* not the first pid in the linked list */

	p->next = t->next; /* remove node */
	}
	t->next = ifreelist; /* add to freelist */
	ifreelist = t;
	}
	}

	void process_msg(idle_data_t *idledata)
	{
	struct ientry t, n;
	- int s;
	- int fdflags;
	- struct stat sbuf;

	switch (idledata->msg) {
	case IDLE_INIT:
	if (verbose \|\| debugmode)
	- syslog(LOG_DEBUG, "imapd[%d]: IDLE_INIT '%s'\n",
	+ syslog(LOG_DEBUG, "imapd[%ld]: IDLE_INIT '%s'\n",
	idledata->pid, idledata->mboxname);

	/* add pid to list of those idling on mboxname */
	t = (struct ientry *) hash_lookup(idledata->mboxname, &itable);
	n = get_ientry();
	n->pid = idledata->pid;
	n->itime = time(NULL);
	n->next = t;
	hash_insert(idledata->mboxname, n, &itable);
	break;

	case IDLE_NOTIFY:
	if (verbose \|\| debugmode)
	syslog(LOG_DEBUG, "IDLE_NOTIFY '%s'\n", idledata->mboxname);

	/* send a message to all pids idling on mboxname */
	t = (struct ientry *) hash_lookup(idledata->mboxname, &itable);
	while (t) {
	if ((t->itime + idle_timeout) < time(NULL)) {
	/* This process has been idling for longer than the timeout
	* period, so it probably died. Remove it from the list.
	*/
	if (verbose \|\| debugmode)
	syslog(LOG_DEBUG, " TIMEOUT %d\n", t->pid);

	n = t;
	t = t->next;
	idle_done(idledata->mboxname, n->pid);
	}
	else { /* signal process to update */
	if (verbose \|\| debugmode)
	syslog(LOG_DEBUG, " SIGUSR1 %d\n", t->pid);

	kill(t->pid, SIGUSR1);
	t = t->next;
	}
	}
	break;

	case IDLE_DONE:
	if (verbose \|\| debugmode)
	- syslog(LOG_DEBUG, "imapd[%d]: IDLE_DONE '%s'\n",
	+ syslog(LOG_DEBUG, "imapd[%ld]: IDLE_DONE '%s'\n",
	idledata->pid, idledata->mboxname);

	/* remove pid from list of those idling on mboxname */
	idle_done(idledata->mboxname, idledata->pid);
	break;

	default:
	- syslog(LOG_ERR, "unrecognized message: %x", idledata->msg);
	+ syslog(LOG_ERR, "unrecognized message: %lx", idledata->msg);
	break;
	}
	}

	void idle_alert(char key, void data, void *rock)
	{
	struct ientry t = (struct ientry ) data;

	while (t) {
	/* signal process to check ALERTs */
	if (verbose \|\| debugmode)
	syslog(LOG_DEBUG, " SIGUSR2 %d\n", t->pid);
	kill(t->pid, SIGUSR2);

	t = t->next;
	}
	}

	int main(int argc, char **argv)
	{
	char shutdownfilename[1024];
	char *p = NULL;
	int opt;
	int nmbox = 0;
	int s, len;
	struct sockaddr_un local;
	idle_data_t idledata;
	struct sockaddr_un from;
	socklen_t fromlen;
	mode_t oldumask;
	fd_set read_set, rset;
	int nfds;
	struct timeval timeout;
	pid_t pid;
	int fd;
	char *alt_config = NULL;
	const char *idle_sock;

	p = getenv("CYRUS_VERBOSE");
	if (p) verbose = atoi(p) + 1;

	while ((opt = getopt(argc, argv, "C:d")) != EOF) {
	switch (opt) {
	case 'C': /* alt config file */
	alt_config = optarg;
	break;
	case 'd': /* don't fork. debugging mode */
	debugmode = 1;
	break;
	default:
	fprintf(stderr, "invalid argument\n");
	exit(EC_USAGE);
	break;
	}
	}

	if (debugmode) {
	openlog("idled", LOG_PID, LOG_LOCAL6);
	}

	config_init(alt_config, "idled");

	/* get name of shutdown file */
	sprintf(shutdownfilename, "%s/msg/shutdown", config_dir);

	/* Set inactivity timer (convert from minutes to seconds) */
	idle_timeout = config_getint("timeout", 30);
	if (idle_timeout < 30) idle_timeout = 30;
	idle_timeout *= 60;

	/* count the number of mailboxes */
	mboxlist_init(0);
	mboxlist_open(NULL);
	CONFIG_DB_MBOX->foreach(mbdb, "", 0, &mbox_count_p, &mbox_count_cb,
	&nmbox, NULL);
	mboxlist_close();
	mboxlist_done();

	/* create idle table */
	construct_hash_table(&itable, nmbox);
	ifreelist = NULL;

	/* create socket we are going to use for listening */
	if ((s = socket(AF_UNIX, SOCK_DGRAM, 0)) == -1) {
	perror("socket");
	exit(1);
	}

	/* bind it to a local file */
	local.sun_family = AF_UNIX;
	idle_sock = config_getstring("idlesocket", NULL);
	if (idle_sock) {
	strcpy(local.sun_path, idle_sock);
	}
	else {
	strcpy(local.sun_path, config_dir);
	strcat(local.sun_path, FNAME_IDLE_SOCK);
	}
	unlink(local.sun_path);
	len = sizeof(local.sun_family) + strlen(local.sun_path) + 1;

	oldumask = umask((mode_t) 0); /* for Linux */

	if (bind(s, (struct sockaddr *)&local, len) == -1) {
	perror("bind");
	exit(1);
	}
	umask(oldumask); /* for Linux */
	chmod(local.sun_path, 0777); /* for DUX */

	/* fork unless we were given the -d option */
	if (debugmode == 0) {

	pid = fork();

	if (pid == -1) {
	perror("fork");
	exit(1);
	}

	if (pid != 0) { /* parent */
	exit(0);
	}
	}
	/* child */

	/* get ready for select() */
	FD_ZERO(&read_set);
	FD_SET(s, &read_set);
	nfds = s + 1;

	for (;;) {
	int n;

	/* check for shutdown file */
	if ((fd = open(shutdownfilename, O_RDONLY, 0)) != -1) {
	/* signal all processes to shutdown */
	if (verbose \|\| debugmode)
	syslog(LOG_DEBUG, "IDLE_ALERT\n");

	hash_enumerate(&itable, idle_alert, NULL);
	}

	/* timeout for select is 1 second */
	timeout.tv_sec = 1;
	timeout.tv_usec = 0;

	/* check for the next input */
	rset = read_set;
	n = select(nfds, &rset, NULL, NULL, &timeout);
	if (n < 0 && errno == EAGAIN) continue;
	if (n < 0 && errno == EINTR) continue;
	if (n == -1) {
	/* uh oh */
	syslog(LOG_ERR, "select(): %m");
	close(s);
	fatal("select error",-1);
	}

	/* read on unix socket */
	if (FD_ISSET(s, &rset)) {
	fromlen = sizeof(from);
	n = recvfrom(s, (void*) &idledata, sizeof(idle_data_t), 0,
	(struct sockaddr *) &from, &fromlen);

	if (n > 0) {
	if (n <= IDLEDATA_BASE_SIZE \|\|
	idledata.mboxname[n - 1 - IDLEDATA_BASE_SIZE] != '\0')
	syslog(LOG_ERR, "Invalid message received, size=%d\n", n);
	else
	process_msg(&idledata);
	}
	} else {
	/* log some sort of error */
	}

	}

	/* never gets here */
	exit(1);
	}
	diff --git a/imap/imapd.c b/imap/imapd.c
	index 58df0f30d..62665df5d 100644
	--- a/imap/imapd.c
	+++ b/imap/imapd.c
	@@ -1,5864 +1,5868 @@
	/*
	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*/

	-/* $Id: imapd.c,v 1.341 2002/02/19 00:27:36 rjs3 Exp $ */
	+/* $Id: imapd.c,v 1.342 2002/02/19 18:50:11 ken3 Exp $ */

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <string.h>
	#include <ctype.h>
	#include <time.h>
	#include <signal.h>
	#include <fcntl.h>
	#include <sys/types.h>
	#include <sys/param.h>
	#include <sys/stat.h>
	#include <syslog.h>
	#include <com_err.h>
	#include <netdb.h>
	#include <sys/socket.h>
	#include <sys/wait.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>

	#include <sasl/sasl.h>

	#include "acl.h"
	#include "util.h"
	#include "auth.h"
	#include "imapconf.h"
	#include "tls.h"
	#include "version.h"
	#include "charset.h"
	#include "imparse.h"
	#include "mkgmtime.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "imapd.h"
	#include "xmalloc.h"
	#include "mboxname.h"
	#include "append.h"
	#include "iptostring.h"
	#include "mboxlist.h"
	#include "idle.h"
	#include "telemetry.h"
	#include "user.h"

	#include "pushstats.h" /* SNMP interface */

	extern void seen_done(void);

	extern int optind;
	extern char *optarg;
	extern int errno;

	/* global state */
	static char shutdownfilename[1024];
	static int imaps = 0;
	static sasl_ssf_t extprops_ssf = 0;

	/* per-user/session state */
	struct protstream *imapd_out = NULL;
	struct protstream *imapd_in = NULL;
	static char imapd_clienthost[250] = "[local]";
	static time_t imapd_logtime;
	static int imapd_logfd = -1;
	char *imapd_userid;
	struct auth_state *imapd_authstate = 0;
	static int imapd_userisadmin;
	static sasl_conn_t imapd_saslconn; / the sasl connection context */
	static int imapd_starttls_done = 0; /* have we done a successful starttls? */
	#ifdef HAVE_SSL
	/* our tls connection, if any */
	static SSL *tls_conn = NULL;
	#endif /* HAVE_SSL */

	/* current sub-user state */
	static struct mailbox mboxstruct;
	static struct mailbox *imapd_mailbox;
	int imapd_exists;

	/* current namespace */
	static struct namespace imapd_namespace;

	static const char *monthname[] = {
	"jan", "feb", "mar", "apr", "may", "jun",
	"jul", "aug", "sep", "oct", "nov", "dec"
	};

	void shutdown_file(int fd);
	void motd_file(int fd);
	void shut_down(int code);
	void fatal(const char *s, int code);

	void cmdloop(void);
	void cmd_login(char tag, char user, char *passwd);
	void cmd_authenticate(char tag, char authtype);
	void cmd_noop(char tag, char cmd);
	void cmd_capability(char *tag);
	void cmd_append(char tag, char name);
	void cmd_select(char tag, char cmd, char *name);
	void cmd_close(char *tag);
	void cmd_fetch(char tag, char sequence, int usinguid);
	void cmd_partial(char tag, char msgno, char *data,
	char start, char count);
	void cmd_store(char tag, char sequence, char *operation, int usinguid);
	void cmd_search(char *tag, int usinguid);
	void cmd_sort(char *tag, int usinguid);
	void cmd_thread(char *tag, int usinguid);
	void cmd_copy(char tag, char sequence, char *name, int usinguid);
	void cmd_expunge(char tag, char sequence);
	void cmd_create(char tag, char name, char *partition);
	void cmd_delete(char tag, char name);
	void cmd_rename(const char tag, char oldname,
	char newname, char partition);
	void cmd_reconstruct(const char tag, const char name);
	void cmd_find(char tag, char namespace, char *pattern);
	void cmd_list(char tag, int subscribed, char reference, char *pattern);
	void cmd_changesub(char tag, char namespace, char *name, int add);
	void cmd_getacl(char tag, char name, int oldform);
	void cmd_listrights(char tag, char name, char *identifier);
	void cmd_myrights(char tag, char name, int oldform);
	void cmd_setacl(char tag, char name, char identifier, char rights);
	void cmd_getquota(char tag, char name);
	void cmd_getquotaroot(char tag, char name);
	void cmd_setquota(char tag, char quotaroot);
	void cmd_status(char tag, char name);
	void cmd_getuids(char tag, char startuid);
	void cmd_unselect(char* tag);
	void cmd_namespace(char* tag);

	void cmd_id(char* tag);
	struct idparamlist {
	char *field;
	char *value;
	struct idparamlist *next;
	};
	extern void id_getcmdline(int argc, char **argv);
	extern void id_response(struct protstream *pout);
	void id_appendparamlist(struct idparamlist *l, char field, char *value);
	void id_freeparamlist(struct idparamlist *l);

	void cmd_idle(char* tag);
	void idle_update(idle_flags_t flags);

	void cmd_starttls(char *tag, int imaps);

	#ifdef ENABLE_X_NETSCAPE_HACK
	void cmd_netscrape(char* tag);
	#endif

	+#ifdef ENABLE_LISTEXT
	+int getlistopts(char tag, int listopts);
	+#endif
	+
	int getsearchprogram(char tag, struct searchargs searchargs,
	int *charset, int parsecharset);
	int getsearchcriteria(char tag, struct searchargs searchargs,
	int *charset, int parsecharset);
	int getsearchdate(time_t start, time_t end);
	int getsortcriteria(char tag, struct sortcrit *sortcrit);
	int getdatetime(time_t *date);

	void printstring(const char *s);
	void printastring(const char *s);

	void appendfieldlist(struct fieldlist *l, char section,
	struct strlist fields, char trail);
	void appendstrlist(struct strlist *l, char s);
	void appendstrlistpat(struct strlist *l, char s);
	void freefieldlist(struct fieldlist *l);
	void freestrlist(struct strlist *l);
	void appendsearchargs(struct searchargs s, struct searchargs s1,
	struct searchargs *s2);
	void freesearchargs(struct searchargs *s);
	static void freesortcrit(struct sortcrit *s);

	static int mailboxdata(char name, int matchlen, int maycreate, void rock);
	static int listdata(char name, int matchlen, int maycreate, void rock);
	static int lsubdata(char name, int matchlen, int maycreate, void rock);
	static void mstringdata(char cmd, char name, int matchlen, int maycreate,
	int listopts);

	extern void setproctitle_init(int argc, char argv, char envp);
	extern int proc_register(const char progname, const char clienthost,
	const char userid, const char mailbox);
	extern void proc_cleanup(void);

	static int hash_simple (const char *str);

	/* Enable the resetting of a sasl_conn_t */
	static int reset_saslconn(sasl_conn_t **conn);

	static struct
	{
	char *ipremoteport;
	char *iplocalport;
	sasl_ssf_t ssf;
	char *authid;
	} saslprops = {NULL,NULL,0,NULL};

	/*
	* acl_ok() checks to see if the the inbox for 'user' grants the 'a'
	* right to the principal 'auth_identity'. Returns 1 if so, 0 if not.
	*/
	/* xxx is auth_identity really needed here? */
	static int acl_ok(const char *user,
	const char *auth_identity __attribute__((unused)),
	struct auth_state *authstate)
	{
	char *acl;
	char inboxname[1024];
	int r;

	if (strchr(user, '.') \|\| strlen(user)+6 >= sizeof(inboxname)) return 0;

	strcpy(inboxname, "user.");
	strcat(inboxname, user);

	if (!authstate \|\|
	mboxlist_lookup(inboxname, (char **)0, &acl, NULL)) {
	r = 0; /* Failed so assume no proxy access */
	}
	else {
	r = (cyrus_acl_myrights(authstate, acl) & ACL_ADMIN) != 0;
	}
	return r;
	}

	/* should we allow users to proxy? return SASL_OK if yes,
	SASL_BADAUTH otherwise */
	static int mysasl_authproc(sasl_conn_t *conn,
	void *context __attribute__((unused)),
	const char *requested_user, unsigned rlen,
	const char *auth_identity, unsigned alen,
	const char *def_realm __attribute__((unused)),
	unsigned urlen __attribute__((unused)),
	struct propctx *propctx __attribute__((unused)))
	{
	const char *val;
	char *realm;

	/* check if remote realm */
	if ((realm = strchr(auth_identity, '@'))!=NULL) {
	realm++;
	val = config_getstring("loginrealms", "");
	while (*val) {
	if (!strncasecmp(val, realm, strlen(realm)) &&
	(!val[strlen(realm)] \|\| isspace((int) val[strlen(realm)]))) {
	break;
	}
	/* not this realm, try next one */
	while (val && !isspace((int) val)) val++;
	while (val && isspace((int) val)) val++;
	}
	if (!*val) {
	sasl_seterror(conn, 0, "cross-realm login %s denied",
	auth_identity);
	return SASL_BADAUTH;
	}
	}

	imapd_authstate = auth_newstate(auth_identity, NULL);

	/* ok, is auth_identity an admin? */
	imapd_userisadmin = authisa(imapd_authstate, "imap", "admins");

	if (alen != rlen \|\| strncmp(auth_identity, requested_user, alen)) {
	/* we want to authenticate as a different user; we'll allow this
	if we're an admin or if we've allowed ACL proxy logins */
	int use_acl = config_getswitch("loginuseacl", 0);

	if (imapd_userisadmin \|\|
	(use_acl && acl_ok(requested_user, auth_identity, imapd_authstate)) \|\|
	authisa(imapd_authstate, "imap", "proxyservers")) {
	/* proxy ok! */

	imapd_userisadmin = 0; /* no longer admin */
	auth_freestate(imapd_authstate);

	imapd_authstate = auth_newstate(requested_user, NULL);
	} else {
	sasl_seterror(conn, 0, "user %s is not allowed to proxy",
	auth_identity);

	auth_freestate(imapd_authstate);

	return SASL_BADAUTH;
	}
	}

	return SASL_OK;
	}

	static const struct sasl_callback mysasl_cb[] = {
	{ SASL_CB_GETOPT, &mysasl_config, NULL },
	{ SASL_CB_PROXY_POLICY, &mysasl_authproc, NULL },
	{ SASL_CB_CANON_USER, &mysasl_canon_user, NULL },
	{ SASL_CB_LIST_END, NULL, NULL }
	};

	static void imapd_reset(void)
	{
	proc_cleanup();

	if (imapd_mailbox) {
	index_closemailbox(imapd_mailbox);
	mailbox_close(imapd_mailbox);
	imapd_mailbox = 0;
	}

	if (imapd_in) prot_free(imapd_in);
	if (imapd_out) prot_free(imapd_out);
	imapd_in = imapd_out = NULL;
	close(0);
	close(1);
	close(2);

	strcpy(imapd_clienthost, "[local]");
	imapd_logtime = 0;
	if (imapd_logfd != -1) {
	close(imapd_logfd);
	imapd_logfd = -1;
	}
	if (imapd_userid != NULL) {
	free(imapd_userid);
	imapd_userid = NULL;
	}
	if (imapd_authstate) {
	auth_freestate(imapd_authstate);
	imapd_authstate = NULL;
	}
	imapd_userisadmin = 0;
	if (imapd_saslconn) {
	sasl_dispose(&imapd_saslconn);
	imapd_saslconn = NULL;
	}
	imapd_starttls_done = 0;

	if(saslprops.iplocalport) {
	free(saslprops.iplocalport);
	saslprops.iplocalport = NULL;
	}
	if(saslprops.ipremoteport) {
	free(saslprops.ipremoteport);
	saslprops.ipremoteport = NULL;
	}
	if(saslprops.authid) {
	free(saslprops.authid);
	saslprops.authid = NULL;
	}
	saslprops.ssf = 0;

	#ifdef HAVE_SSL
	if (tls_conn) {
	if (tls_reset_servertls(&tls_conn) == -1) {
	fatal("tls_reset() failed", EC_TEMPFAIL);
	}
	tls_conn = NULL;
	}
	#endif

	imapd_exists = -1;
	}

	/*
	* run once when process is forked;
	* MUST NOT exit directly; must return with non-zero error code
	*/
	int service_init(int argc, char argv, char envp)
	{
	int r;
	int opt;

	config_changeident("imapd");

	if (geteuid() == 0) fatal("must run as the Cyrus user", EC_USAGE);
	setproctitle_init(argc, argv, envp);

	/* set signal handlers */
	signals_set_shutdown(&shut_down);
	signals_add_handlers();
	signal(SIGPIPE, SIG_IGN);

	/* set the SASL allocation functions */
	sasl_set_alloc((sasl_malloc_t *) &xmalloc,
	(sasl_calloc_t *) &calloc,
	(sasl_realloc_t *) &xrealloc,
	(sasl_free_t *) &free);

	/* load the SASL plugins */
	if ((r = sasl_server_init(mysasl_cb, "Cyrus")) != SASL_OK) {
	syslog(LOG_ERR, "SASL failed initializing: sasl_server_init(): %s",
	sasl_errstring(r, NULL, NULL));
	return EC_SOFTWARE;
	}

	#ifndef DELAY_SASL_CLIENT_INIT
	if ((r = sasl_client_init(NULL)) != SASL_OK) {
	syslog(LOG_ERR, "SASL failed initializing: sasl_client_init(): %s",
	sasl_errstring(r, NULL, NULL));
	return EC_SOFTWARE;
	}
	#endif

	sprintf(shutdownfilename, "%s/msg/shutdown", config_dir);

	/* open the mboxlist, we'll need it for real work */
	mboxlist_init(0);
	mboxlist_open(NULL);
	mailbox_initialize();

	/* setup for sending IMAP IDLE notifications */
	idle_enabled();

	/* create connection to the SNMP listener, if available. */
	snmp_connect(); /* ignore return code */
	snmp_set_str(SERVER_NAME_VERSION,CYRUS_VERSION);

	while ((opt = getopt(argc, argv, "C:sp:")) != EOF) {
	switch (opt) {
	case 'C': /* alt config file - handled by service::main() */
	break;
	case 's': /* imaps (do starttls right away) */
	imaps = 1;
	if (!tls_enabled("imap")) {
	syslog(LOG_ERR, "imaps: required OpenSSL options not present");
	fatal("imaps: required OpenSSL options not present",
	EC_CONFIG);
	}
	break;
	case 'p': /* external protection */
	extprops_ssf = atoi(optarg);
	break;
	default:
	break;
	}
	}

	return 0;
	}

	/*
	* run for each accepted connection
	*/
	#ifdef ID_SAVE_CMDLINE
	int service_main(int argc, char argv, char envp __attribute__((unused)))
	#else
	int service_main(int argc __attribute__((unused)),
	char **argv __attribute__((unused)),
	char **envp __attribute__((unused)))
	#endif
	{
	socklen_t salen;
	struct hostent *hp;
	int timeout;
	sasl_security_properties_t *secprops = NULL;
	struct sockaddr_in imapd_localaddr, imapd_remoteaddr;
	char localip[60], remoteip[60];
	int imapd_haveaddr = 0;

	signals_poll();

	#ifdef ID_SAVE_CMDLINE
	/* get command line args for use in ID before getopt mangles them */
	id_getcmdline(argc, argv);
	#endif

	imapd_in = prot_new(0, 0);
	imapd_out = prot_new(1, 1);

	/* Find out name of client host */
	salen = sizeof(imapd_remoteaddr);
	if (getpeername(0, (struct sockaddr *)&imapd_remoteaddr, &salen) == 0 &&
	imapd_remoteaddr.sin_family == AF_INET) {
	hp = gethostbyaddr((char *)&imapd_remoteaddr.sin_addr,
	sizeof(imapd_remoteaddr.sin_addr), AF_INET);
	if (hp != NULL) {
	strncpy(imapd_clienthost, hp->h_name, sizeof(imapd_clienthost)-30);
	imapd_clienthost[sizeof(imapd_clienthost)-30] = '\0';
	} else {
	imapd_clienthost[0] = '\0';
	}
	strcat(imapd_clienthost, "[");
	strcat(imapd_clienthost, inet_ntoa(imapd_remoteaddr.sin_addr));
	strcat(imapd_clienthost, "]");
	salen = sizeof(imapd_localaddr);
	if (getsockname(0, (struct sockaddr *)&imapd_localaddr, &salen) == 0) {
	if(iptostring((struct sockaddr *)&imapd_remoteaddr,
	sizeof(struct sockaddr_in),
	remoteip, 60) == 0
	&& iptostring((struct sockaddr *)&imapd_localaddr,
	sizeof(struct sockaddr_in),
	localip, 60) == 0) {
	imapd_haveaddr = 1;
	}
	}
	}

	/* create the SASL connection */
	if (sasl_server_new("imap", config_servername,
	NULL, NULL, NULL, NULL, 0,
	&imapd_saslconn) != SASL_OK) {
	fatal("SASL failed initializing: sasl_server_new()", EC_TEMPFAIL);
	}

	/* never allow plaintext, since IMAP has the LOGIN command */
	secprops = mysasl_secprops(SASL_SEC_NOPLAINTEXT);
	sasl_setprop(imapd_saslconn, SASL_SEC_PROPS, secprops);
	sasl_setprop(imapd_saslconn, SASL_SSF_EXTERNAL, &extprops_ssf);

	if (imapd_haveaddr) {
	sasl_setprop(imapd_saslconn, SASL_IPREMOTEPORT, remoteip);
	saslprops.ipremoteport = xstrdup(remoteip);
	sasl_setprop(imapd_saslconn, SASL_IPLOCALPORT, localip);
	saslprops.iplocalport = xstrdup(localip);
	}

	proc_register("imapd", imapd_clienthost, NULL, NULL);

	/* Set inactivity timer */
	timeout = config_getint("timeout", 30);
	if (timeout < 30) timeout = 30;
	prot_settimeout(imapd_in, timeout*60);
	prot_setflushonread(imapd_in, imapd_out);

	/* we were connected on imaps port so we should do
	TLS negotiation immediately */
	if (imaps == 1) cmd_starttls(NULL, 1);

	snmp_increment(TOTAL_CONNECTIONS, 1);
	snmp_increment(ACTIVE_CONNECTIONS, 1);

	cmdloop();

	/* LOGOUT executed */
	prot_flush(imapd_out);
	snmp_increment(ACTIVE_CONNECTIONS, -1);

	/* cleanup */
	imapd_reset();

	return 0;
	}

	/* called if 'service_init()' was called but not 'service_main()' */
	void service_abort(int error)
	{
	shut_down(error);
	}

	/*
	* found a motd file; spit out message and return
	*/
	void motd_file(fd)
	int fd;
	{
	struct protstream *motd_in;
	char buf[1024];
	char *p;

	motd_in = prot_new(fd, 0);

	prot_fgets(buf, sizeof(buf), motd_in);
	if ((p = strchr(buf, '\r'))!=NULL) *p = 0;
	if ((p = strchr(buf, '\n'))!=NULL) *p = 0;

	for(p = buf; p == '['; p++); / can't have [ be first char, sigh */
	prot_printf(imapd_out, "* OK [ALERT] %s\r\n", p);
	}

	/*
	* Found a shutdown file: Spit out an untagged BYE and shut down
	*/
	void shutdown_file(fd)
	int fd;
	{
	struct protstream *shutdown_in;
	char buf[1024];
	char *p;

	shutdown_in = prot_new(fd, 0);

	prot_fgets(buf, sizeof(buf), shutdown_in);
	if ((p = strchr(buf, '\r'))!=NULL) *p = 0;
	if ((p = strchr(buf, '\n'))!=NULL) *p = 0;

	for(p = buf; p == '['; p++); / can't have [ be first char, sigh */
	prot_printf(imapd_out, "* BYE [ALERT] %s\r\n", p);

	shut_down(0);
	}

	/*
	* Cleanly shut down and exit
	*/
	void shut_down(int code) __attribute__((noreturn));
	void shut_down(int code)
	{
	proc_cleanup();
	if (imapd_mailbox) {
	index_closemailbox(imapd_mailbox);
	mailbox_close(imapd_mailbox);
	}
	seen_done();
	mboxlist_close();
	mboxlist_done();
	#ifdef HAVE_SSL
	tls_shutdown_serverengine();
	#endif
	if (imapd_out) {
	prot_flush(imapd_out);

	/* one less active connection */
	snmp_increment(ACTIVE_CONNECTIONS, -1);
	}

	exit(code);
	}

	void fatal(const char *s, int code)
	{
	static int recurse_code = 0;

	if (recurse_code) {
	/* We were called recursively. Just give up */
	proc_cleanup();
	snmp_increment(ACTIVE_CONNECTIONS, -1);
	exit(recurse_code);
	}
	recurse_code = code;
	if (imapd_out) {
	prot_printf(imapd_out, "* BYE Fatal error: %s\r\n", s);
	prot_flush(imapd_out);
	}
	shut_down(code);

	}

	/*
	* Top-level command loop parsing
	*/
	void
	cmdloop()
	{
	int fd;
	char motdfilename[1024];
	int c;
	int usinguid, havepartition, havenamespace, oldform;
	static struct buf tag, cmd, arg1, arg2, arg3, arg4;
	char *p;
	const char *err;

	prot_printf(imapd_out,
	"* OK %s Cyrus IMAP4 %s server ready\r\n", config_servername,
	CYRUS_VERSION);

	sprintf(motdfilename, "%s/msg/motd", config_dir);
	if ((fd = open(motdfilename, O_RDONLY, 0)) != -1) {
	motd_file(fd);
	close(fd);
	}

	for (;;) {
	if ( !imapd_userisadmin && imapd_userid
	&& (fd = open(shutdownfilename, O_RDONLY, 0)) != -1) {
	shutdown_file(fd);
	}

	signals_poll();

	/* Parse tag */
	c = getword(imapd_in, &tag);
	if (c == EOF) {
	if ((err = prot_error(imapd_in))!=NULL) {
	syslog(LOG_WARNING, "%s, closing connection", err);
	prot_printf(imapd_out, "* BYE %s\r\n", err);
	}
	return;
	}
	if (c != ' ' \|\| !imparse_isatom(tag.s) \|\| (tag.s[0] == '*' && !tag.s[1])) {
	prot_printf(imapd_out, "* BAD Invalid tag\r\n");
	eatline(imapd_in, c);
	continue;
	}

	/* Parse command name */
	c = getword(imapd_in, &cmd);
	if (!cmd.s[0]) {
	prot_printf(imapd_out, "%s BAD Null command\r\n", tag.s);
	eatline(imapd_in, c);
	continue;
	}
	if (islower((unsigned char) cmd.s[0]))
	cmd.s[0] = toupper((unsigned char) cmd.s[0]);
	for (p = &cmd.s[1]; *p; p++) {
	if (isupper((unsigned char) p)) p = tolower((unsigned char) *p);
	}

	/* Only Authenticate/Login/Logout/Noop/Capability/Id/Starttls
	allowed when not logged in */
	if (!imapd_userid && !strchr("ALNCIS", cmd.s[0])) goto nologin;

	/* note that about half the commands (the common ones that don't
	hit the mailboxes file) now close the mailboxes file just in
	case it was open. */
	switch (cmd.s[0]) {
	case 'A':
	if (!strcmp(cmd.s, "Authenticate")) {
	if (c != ' ') goto missingargs;
	c = getword(imapd_in, &arg1);
	if (!imparse_isatom(arg1.s)) {
	prot_printf(imapd_out, "%s BAD Invalid authenticate mechanism\r\n", tag.s);
	eatline(imapd_in, c);
	continue;
	}
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	if (imapd_userid) {
	prot_printf(imapd_out, "%s BAD Already authenticated\r\n", tag.s);
	continue;
	}
	cmd_authenticate(tag.s, arg1.s);

	snmp_increment(AUTHENTICATE_COUNT, 1);
	}
	else if (!imapd_userid) goto nologin;
	else if (!strcmp(cmd.s, "Append")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c != ' ') goto missingargs;

	cmd_append(tag.s, arg1.s);

	snmp_increment(APPEND_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'B':
	if (!strcmp(cmd.s, "Bboard")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_select(tag.s, cmd.s, arg1.s);

	snmp_increment(BBOARD_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'C':
	if (!strcmp(cmd.s, "Capability")) {
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_capability(tag.s);

	snmp_increment(CAPABILITY_COUNT, 1);
	}
	else if (!imapd_userid) goto nologin;
	else if (!strcmp(cmd.s, "Check")) {
	if (!imapd_mailbox) goto nomailbox;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_noop(tag.s, cmd.s);

	snmp_increment(CHECK_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Copy")) {
	if (!imapd_mailbox) goto nomailbox;
	usinguid = 0;
	if (c != ' ') goto missingargs;
	copy:
	c = getword(imapd_in, &arg1);
	if (c == '\r') goto missingargs;
	if (c != ' ' \|\| !imparse_issequence(arg1.s)) goto badsequence;
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_copy(tag.s, arg1.s, arg2.s, usinguid);

	snmp_increment(COPY_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Create")) {
	havepartition = 0;
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == EOF) goto missingargs;
	if (c == ' ') {
	havepartition = 1;
	c = getword(imapd_in, &arg2);
	if (!imparse_isatom(arg2.s)) goto badpartition;
	}
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_create(tag.s, arg1.s, havepartition ? arg2.s : 0);

	snmp_increment(CREATE_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Close")) {
	if (!imapd_mailbox) goto nomailbox;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_close(tag.s);

	snmp_increment(CLOSE_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'D':
	if (!strcmp(cmd.s, "Delete")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_delete(tag.s, arg1.s);

	snmp_increment(DELETE_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Deleteacl")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (!strcasecmp(arg1.s, "mailbox")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	}
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_setacl(tag.s, arg1.s, arg2.s, (char *)0);

	snmp_increment(DELETEACL_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'E':
	if (!strcmp(cmd.s, "Expunge")) {
	if (!imapd_mailbox) goto nomailbox;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_expunge(tag.s, 0);

	snmp_increment(EXPUNGE_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Examine")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_select(tag.s, cmd.s, arg1.s);

	snmp_increment(EXAMINE_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'F':
	if (!strcmp(cmd.s, "Fetch")) {
	if (!imapd_mailbox) goto nomailbox;
	usinguid = 0;
	if (c != ' ') goto missingargs;
	fetch:
	c = getword(imapd_in, &arg1);
	if (c == '\r') goto missingargs;
	if (c != ' ' \|\| !imparse_issequence(arg1.s)) goto badsequence;

	cmd_fetch(tag.s, arg1.s, usinguid);

	snmp_increment(FETCH_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Find")) {
	c = getword(imapd_in, &arg1);
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_find(tag.s, arg1.s, arg2.s);

	snmp_increment(FIND_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'G':
	if (!strcmp(cmd.s, "Getacl")) {
	oldform = 0;
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (!strcasecmp(arg1.s, "mailbox")) {
	oldform = 1;
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	}
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_getacl(tag.s, arg1.s, oldform);

	snmp_increment(GETACL_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Getquota")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_getquota(tag.s, arg1.s);

	snmp_increment(GETQUOTA_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Getquotaroot")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_getquotaroot(tag.s, arg1.s);

	snmp_increment(GETQUOTAROOT_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'I':
	if (!strcmp(cmd.s, "Id")) {
	if (c != ' ') goto missingargs;
	cmd_id(tag.s);

	snmp_increment(ID_COUNT, 1);
	}
	else if (!imapd_userid) goto nologin;
	else if (!strcmp(cmd.s, "Idle")) {
	if (!idle_enabled()) {
	/* we don't support idle */
	goto badcmd;
	}

	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_idle(tag.s);

	snmp_increment(IDLE_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'L':
	if (!strcmp(cmd.s, "Login")) {
	if (c != ' ' \|\| (c = getastring(imapd_in, imapd_out, &arg1)) != ' ') {
	goto missingargs;
	}
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	if (imapd_userid) {
	prot_printf(imapd_out, "%s BAD Already logged in\r\n", tag.s);
	continue;
	}
	cmd_login(tag.s, arg1.s, arg2.s);

	snmp_increment(LOGIN_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Logout")) {
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	snmp_increment(LOGOUT_COUNT, 1);

	prot_printf(imapd_out, "* BYE %s\r\n",
	error_message(IMAP_BYE_LOGOUT));
	prot_printf(imapd_out, "%s OK %s\r\n", tag.s,
	error_message(IMAP_OK_COMPLETED));
	return;
	}
	else if (!imapd_userid) goto nologin;
	else if (!strcmp(cmd.s, "List")) {
	int listopts = LIST_CHILDREN;
	#ifdef ENABLE_LISTEXT
	/* Check for and parse LISTEXT options */
	c = prot_getc(imapd_in);
	if (c == '(') {
	c = getlistopts(tag.s, &listopts);
	if (c == EOF) {
	eatline(imapd_in, c);
	continue;
	}
	}
	else
	prot_ungetc(c, imapd_in);
	#endif /* ENABLE_LISTEXT */
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_list(tag.s, listopts, arg1.s, arg2.s);

	snmp_increment(LIST_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Lsub")) {
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_list(tag.s, LIST_LSUB \| LIST_CHILDREN, arg1.s, arg2.s);

	snmp_increment(LSUB_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Listrights")) {
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_listrights(tag.s, arg1.s, arg2.s);

	snmp_increment(LISTRIGHTS_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'M':
	if (!strcmp(cmd.s, "Myrights")) {
	oldform = 0;
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (!strcasecmp(arg1.s, "mailbox")) {
	oldform = 1;
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	}
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_myrights(tag.s, arg1.s, oldform);

	snmp_increment(MYRIGHTS_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'N':
	if (!strcmp(cmd.s, "Noop")) {
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_noop(tag.s, cmd.s);

	snmp_increment(NOOP_COUNT, 1);
	}
	#ifdef ENABLE_X_NETSCAPE_HACK
	else if (!strcmp(cmd.s, "Netscape")) {
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_netscrape(tag.s);
	}
	#endif
	else if (!imapd_userid) goto nologin;
	else if (!strcmp(cmd.s, "Namespace")) {
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_namespace(tag.s);

	snmp_increment(NAMESPACE_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'P':
	if (!strcmp(cmd.s, "Partial")) {
	if (!imapd_mailbox) goto nomailbox;
	if (c != ' ') goto missingargs;
	c = getword(imapd_in, &arg1);
	if (c != ' ') goto missingargs;
	c = getword(imapd_in, &arg2);
	if (c != ' ') goto missingargs;
	c = getword(imapd_in, &arg3);
	if (c != ' ') goto missingargs;
	c = getword(imapd_in, &arg4);
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_partial(tag.s, arg1.s, arg2.s, arg3.s, arg4.s);

	snmp_increment(PARTIAL_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'R':
	if (!strcmp(cmd.s, "Rename")) {
	havepartition = 0;
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c == EOF) goto missingargs;
	if (c == ' ') {
	havepartition = 1;
	c = getword(imapd_in, &arg3);
	if (!imparse_isatom(arg3.s)) goto badpartition;
	}
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_rename(tag.s, arg1.s, arg2.s, havepartition ? arg3.s : 0);

	snmp_increment(RENAME_COUNT, 1);
	} else if(!strcmp(cmd.s, "Reconstruct")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if(c == '\r') c = prot_getc(imapd_in);
	if(c != '\n') goto extraargs;
	cmd_reconstruct(tag.s, arg1.s);

	/* xxx needed? */
	/* snmp_increment(RECONSTRUCT_COUNT, 1); */
	} else goto badcmd;
	break;

	case 'S':
	if (!strcmp(cmd.s, "Starttls")) {
	if (!tls_enabled("imap")) {
	/* we don't support starttls */
	goto badcmd;
	}

	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	/* if we've already done SASL fail */
	if (imapd_userid != NULL) {
	prot_printf(imapd_out,
	"%s BAD Can't Starttls after authentication\r\n", tag.s);
	continue;
	}

	/* check if already did a successful tls */
	if (imapd_starttls_done == 1) {
	prot_printf(imapd_out,
	"%s BAD Already did a successful Starttls\r\n",
	tag.s);
	continue;
	}
	cmd_starttls(tag.s, 0);

	snmp_increment(STARTTLS_COUNT, 1);
	continue;
	}
	if (!imapd_userid) {
	goto nologin;
	} else if (!strcmp(cmd.s, "Store")) {
	if (!imapd_mailbox) goto nomailbox;
	usinguid = 0;
	if (c != ' ') goto missingargs;
	store:
	c = getword(imapd_in, &arg1);
	if (c != ' ' \|\| !imparse_issequence(arg1.s)) goto badsequence;
	c = getword(imapd_in, &arg2);
	if (c != ' ') goto missingargs;

	cmd_store(tag.s, arg1.s, arg2.s, usinguid);

	snmp_increment(STORE_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Select")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;

	cmd_select(tag.s, cmd.s, arg1.s);

	snmp_increment(SELECT_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Search")) {
	if (!imapd_mailbox) goto nomailbox;
	usinguid = 0;
	if (c != ' ') goto missingargs;
	search:

	cmd_search(tag.s, usinguid);

	snmp_increment(SEARCH_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Subscribe")) {
	if (c != ' ') goto missingargs;
	havenamespace = 0;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == ' ') {
	havenamespace = 1;
	c = getastring(imapd_in, imapd_out, &arg2);
	}
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	if (havenamespace) {
	cmd_changesub(tag.s, arg1.s, arg2.s, 1);
	}
	else {
	cmd_changesub(tag.s, (char *)0, arg1.s, 1);
	}
	snmp_increment(SUBSCRIBE_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Setacl")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (!strcasecmp(arg1.s, "mailbox")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	}
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg2);
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg3);
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_setacl(tag.s, arg1.s, arg2.s, arg3.s);

	snmp_increment(SETACL_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Setquota")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c != ' ') goto missingargs;
	cmd_setquota(tag.s, arg1.s);

	snmp_increment(SETQUOTA_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Sort")) {
	if (!imapd_mailbox) goto nomailbox;
	usinguid = 0;
	if (c != ' ') goto missingargs;
	sort:
	cmd_sort(tag.s, usinguid);

	snmp_increment(SORT_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Status")) {
	if (c != ' ') goto missingargs;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c != ' ') goto missingargs;
	cmd_status(tag.s, arg1.s);

	snmp_increment(STATUS_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'T':
	if (!strcmp(cmd.s, "Thread")) {
	if (!imapd_mailbox) goto nomailbox;
	usinguid = 0;
	if (c != ' ') goto missingargs;
	thread:
	cmd_thread(tag.s, usinguid);

	snmp_increment(THREAD_COUNT, 1);
	}
	else goto badcmd;
	break;

	case 'U':
	if (!strcmp(cmd.s, "Uid")) {
	if (!imapd_mailbox) goto nomailbox;
	usinguid = 1;
	if (c != ' ') goto missingargs;
	c = getword(imapd_in, &arg1);
	if (c != ' ') goto missingargs;
	lcase(arg1.s);
	if (!strcmp(arg1.s, "fetch")) {
	goto fetch;
	}
	else if (!strcmp(arg1.s, "store")) {
	goto store;
	}
	else if (!strcmp(arg1.s, "search")) {
	goto search;
	}
	else if (!strcmp(arg1.s, "sort")) {
	goto sort;
	}
	else if (!strcmp(arg1.s, "thread")) {
	goto thread;
	}
	else if (!strcmp(arg1.s, "copy")) {
	goto copy;
	}
	else if (!strcmp(arg1.s, "expunge")) {
	c = getword(imapd_in, &arg1);
	if (!imparse_issequence(arg1.s)) goto badsequence;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_expunge(tag.s, arg1.s);

	snmp_increment(EXPUNGE_COUNT, 1);
	}
	else {
	prot_printf(imapd_out, "%s BAD Unrecognized UID subcommand\r\n", tag.s);
	eatline(imapd_in, c);
	}
	}
	else if (!strcmp(cmd.s, "Unsubscribe")) {
	if (c != ' ') goto missingargs;
	havenamespace = 0;
	c = getastring(imapd_in, imapd_out, &arg1);
	if (c == ' ') {
	havenamespace = 1;
	c = getastring(imapd_in, imapd_out, &arg2);
	}
	if (c == EOF) goto missingargs;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	if (havenamespace) {
	cmd_changesub(tag.s, arg1.s, arg2.s, 0);
	}
	else {
	cmd_changesub(tag.s, (char *)0, arg1.s, 0);
	}

	snmp_increment(UNSUBSCRIBE_COUNT, 1);
	}
	else if (!strcmp(cmd.s, "Unselect")) {
	if (!imapd_mailbox) goto nomailbox;
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') goto extraargs;
	cmd_unselect(tag.s);

	snmp_increment(UNSELECT_COUNT, 1);
	}
	else goto badcmd;
	break;

	default:
	badcmd:
	prot_printf(imapd_out, "%s BAD Unrecognized command\r\n", tag.s);
	eatline(imapd_in, c);
	}

	continue;

	nologin:
	prot_printf(imapd_out, "%s BAD Please login first\r\n", tag.s);
	eatline(imapd_in, c);
	continue;

	nomailbox:
	prot_printf(imapd_out, "%s BAD Please select a mailbox first\r\n", tag.s);
	eatline(imapd_in, c);
	continue;

	missingargs:
	prot_printf(imapd_out, "%s BAD Missing required argument to %s\r\n", tag.s, cmd.s);
	eatline(imapd_in, c);
	continue;

	extraargs:
	prot_printf(imapd_out, "%s BAD Unexpected extra arguments to %s\r\n", tag.s, cmd.s);
	eatline(imapd_in, c);
	continue;

	badsequence:
	prot_printf(imapd_out, "%s BAD Invalid sequence in %s\r\n", tag.s, cmd.s);
	eatline(imapd_in, c);
	continue;

	badpartition:
	prot_printf(imapd_out, "%s BAD Invalid partition name in %s\r\n",
	tag.s, cmd.s);
	eatline(imapd_in, c);
	continue;
	}
	}

	/*
	* Perform a LOGIN command
	*/
	void
	cmd_login(tag, user, passwd)
	char *tag;
	char *user;
	char *passwd;
	{
	char *canon_user;
	const char *reply = 0;
	const char *val;
	char buf[MAX_MAILBOX_PATH];
	char *p;
	int plaintextloginpause;
	int r;

	canon_user = auth_canonifyid(user, 0);

	/* possibly disallow login */
	if ((imapd_starttls_done == 0) &&
	(config_getswitch("allowplaintext", 1) == 0) &&
	strcmp(canon_user, "anonymous") != 0) {
	prot_printf(imapd_out, "%s NO Login only available under a layer\r\n",
	tag);
	return;
	}

	if (!canon_user) {
	syslog(LOG_NOTICE, "badlogin: %s plaintext %s invalid user",
	imapd_clienthost, beautify_string(user));
	prot_printf(imapd_out, "%s NO %s\r\n", tag,
	error_message(IMAP_INVALID_USER));
	return;
	}

	if (!strcmp(canon_user, "anonymous")) {
	if (config_getswitch("allowanonymouslogin", 0)) {
	passwd = beautify_string(passwd);
	if (strlen(passwd) > 500) passwd[500] = '\0';
	syslog(LOG_NOTICE, "login: %s anonymous %s",
	imapd_clienthost, passwd);
	reply = "Anonymous access granted";
	imapd_userid = xstrdup("anonymous");
	}
	else {
	syslog(LOG_NOTICE, "badlogin: %s anonymous login refused",
	imapd_clienthost);
	prot_printf(imapd_out, "%s NO %s\r\n", tag,
	error_message(IMAP_ANONYMOUS_NOT_PERMITTED));
	return;
	}
	}
	else if ((r = sasl_checkpass(imapd_saslconn,
	canon_user,
	strlen(canon_user),
	passwd,
	strlen(passwd))) != SASL_OK) {
	syslog(LOG_NOTICE, "badlogin: %s plaintext %s %s",
	imapd_clienthost, canon_user, sasl_errdetail(imapd_saslconn));

	sleep(3);

	if (reply) {
	prot_printf(imapd_out, "%s NO Login failed: %s\r\n", tag, reply);
	} else if ((reply = sasl_errstring(r, NULL, NULL)) != NULL) {
	prot_printf(imapd_out, "%s NO Login failed: %s\r\n", tag, reply);
	} else {
	prot_printf(imapd_out, "%s NO Login failed: %d\r\n", tag, r);
	}
	snmp_increment_args(AUTHENTICATION_NO, 1,
	VARIABLE_AUTH, hash_simple("LOGIN"),
	VARIABLE_LISTEND);
	return;
	}
	else {
	imapd_userid = xstrdup(canon_user);
	snmp_increment_args(AUTHENTICATION_YES, 1,
	VARIABLE_AUTH, hash_simple("LOGIN"),
	VARIABLE_LISTEND);
	syslog(LOG_NOTICE, "login: %s %s plaintext%s %s", imapd_clienthost,
	canon_user, imapd_starttls_done ? "+TLS" : "",
	reply ? reply : "");

	plaintextloginpause = config_getint("plaintextloginpause", 0);
	if (plaintextloginpause != 0 && !imapd_starttls_done) {
	/* Apply penalty only if not under layer */
	sleep(plaintextloginpause);
	}
	}

	imapd_authstate = auth_newstate(canon_user, (char *)0);
	val = config_getstring("admins", "");
	while (*val) {
	for (p = (char )val; p && !isspace((int) *p); p++);
	strlcpy(buf, val, p - val);
	buf[p-val] = 0;
	if (auth_memberof(imapd_authstate, buf)) {
	imapd_userisadmin = 1;
	break;
	}
	val = p;
	while (val && isspace((int) val)) val++;
	}

	if (!reply) reply = "User logged in";

	prot_printf(imapd_out, "%s OK %s\r\n", tag, reply);

	/* Create telemetry log */
	imapd_logfd = telemetry_log(imapd_userid, imapd_in, imapd_out);

	/* Set namespace */
	if ((r = mboxname_init_namespace(&imapd_namespace, imapd_userisadmin)) != 0) {
	syslog(LOG_ERR, error_message(r));
	fatal(error_message(r), EC_CONFIG);
	}

	/* Translate any separators in userid */
	mboxname_hiersep_tointernal(&imapd_namespace, imapd_userid);

	return;
	}

	static int hash_simple (const char *str)
	{
	int value = 0;
	int i;

	if (!str)
	return 0;
	for (i = 0; *str; i++)
	{
	value ^= (str++ << ((i & 3)8));
	}
	return value;
	}

	/*
	* Perform an AUTHENTICATE command
	*/
	void
	cmd_authenticate(char tag,char authtype)
	{
	int sasl_result;
	static struct buf clientin;
	int clientinlen=0;

	const char *serverout;
	unsigned int serveroutlen;

	const int *ssfp;
	char *ssfmsg=NULL;

	const char *canon_user;

	int r;

	sasl_result = sasl_server_start(imapd_saslconn, authtype,
	NULL, 0,
	&serverout, &serveroutlen);

	/* sasl_server_start will return SASL_OK or SASL_CONTINUE on success */

	while (sasl_result == SASL_CONTINUE)
	{
	char c;

	/* print the message to the user */
	printauthready(imapd_out, serveroutlen, (unsigned char *)serverout);

	c = prot_getc(imapd_in);
	if(c == '*') {
	eatline(imapd_in,c);
	prot_printf(imapd_out,
	"%s NO Client canceled authentication\r\n", tag);
	reset_saslconn(&imapd_saslconn);
	return;
	} else {
	prot_ungetc(c, imapd_in);
	}

	/* get string from user */
	clientinlen = getbase64string(imapd_in, &clientin);
	if (clientinlen == -1) {
	reset_saslconn(&imapd_saslconn);
	prot_printf(imapd_out, "%s BAD Invalid base64 string\r\n", tag);
	return;
	}

	sasl_result = sasl_server_step(imapd_saslconn,
	clientin.s,
	clientinlen,
	&serverout, &serveroutlen);
	}


	/* failed authentication */
	if (sasl_result != SASL_OK)
	{
	syslog(LOG_NOTICE, "badlogin: %s %s [%s]",
	imapd_clienthost, authtype, sasl_errdetail(imapd_saslconn));

	snmp_increment_args(AUTHENTICATION_NO, 1,
	VARIABLE_AUTH, hash_simple(authtype),
	VARIABLE_LISTEND);
	sleep(3);

	reset_saslconn(&imapd_saslconn);
	prot_printf(imapd_out, "%s NO Error authenticating\r\n", tag);

	return;
	}

	/* successful authentication */

	/* get the userid from SASL --- already canonicalized from
	* mysasl_authproc()
	*/
	sasl_result = sasl_getprop(imapd_saslconn, SASL_USERNAME,
	(const void **) &canon_user);
	imapd_userid = xstrdup(canon_user);
	if (sasl_result != SASL_OK) {
	prot_printf(imapd_out, "%s NO weird SASL error %d SASL_USERNAME\r\n",
	tag, sasl_result);
	syslog(LOG_ERR, "weird SASL error %d getting SASL_USERNAME",
	sasl_result);
	reset_saslconn(&imapd_saslconn);
	return;
	}

	proc_register("imapd", imapd_clienthost, imapd_userid, (char *)0);

	syslog(LOG_NOTICE, "login: %s %s %s%s %s", imapd_clienthost, imapd_userid,
	authtype, imapd_starttls_done ? "+TLS" : "", "User logged in");

	sasl_getprop(imapd_saslconn, SASL_SSF, (const void **) &ssfp);

	/* really, we should be doing a sasl_getprop on SASL_SSF_EXTERNAL,
	but the current libsasl doesn't allow that. */
	if (imapd_starttls_done) {
	switch(*ssfp) {
	case 0: ssfmsg = "tls protection"; break;
	case 1: ssfmsg = "tls plus integrity protection"; break;
	default: ssfmsg = "tls plus privacy protection"; break;
	}
	} else {
	switch(*ssfp) {
	case 0: ssfmsg = "no protection"; break;
	case 1: ssfmsg = "integrity protection"; break;
	default: ssfmsg = "privacy protection"; break;
	}
	}

	snmp_increment_args(AUTHENTICATION_YES, 1,
	VARIABLE_AUTH, hash_simple(authtype),
	VARIABLE_LISTEND);

	prot_printf(imapd_out, "%s OK Success (%s)\r\n", tag, ssfmsg);

	prot_setsasl(imapd_in, imapd_saslconn);
	prot_setsasl(imapd_out, imapd_saslconn);

	/* Create telemetry log */
	imapd_logfd = telemetry_log(imapd_userid, imapd_in, imapd_out);

	/* Set namespace */
	if ((r = mboxname_init_namespace(&imapd_namespace, imapd_userisadmin)) != 0) {
	syslog(LOG_ERR, error_message(r));
	fatal(error_message(r), EC_CONFIG);
	}

	/* Translate any separators in userid */
	mboxname_hiersep_tointernal(&imapd_namespace, imapd_userid);

	return;
	}

	/*
	* Perform a NOOP command
	*/
	void
	cmd_noop(char tag, char cmd __attribute__((unused)))
	{
	if (imapd_mailbox) {
	index_check(imapd_mailbox, 0, 1);
	}
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	/*
	* Parse and perform an ID command.
	*
	* the command has been parsed up to the parameter list.
	*
	* we only allow one ID in non-authenticated state from a given client.
	* we only allow MAXIDFAILED consecutive failed IDs from a given client.
	* we only record MAXIDLOG ID responses from a given client.
	*/
	void cmd_id(char *tag)
	{
	static int did_id = 0;
	static int failed_id = 0;
	static int logged_id = 0;
	int error = 0;
	int c = EOF, npair = 0;
	static struct buf arg, field;
	struct idparamlist *params = 0;

	/* check if we've already had an ID in non-authenticated state */
	if (!imapd_userid && did_id) {
	prot_printf(imapd_out,
	"%s NO Only one Id allowed in non-authenticated state\r\n",
	tag);
	eatline(imapd_in, c);
	return;
	}

	/* check if we've had too many failed IDs in a row */
	if (failed_id >= MAXIDFAILED) {
	prot_printf(imapd_out, "%s NO Too many (%u) invalid Id commands\r\n",
	tag, failed_id);
	eatline(imapd_in, c);
	return;
	}

	/* ok, accept parameter list */
	c = getword(imapd_in, &arg);
	/* check for "NIL" or start of parameter list */
	if (strcasecmp(arg.s, "NIL") && c != '(') {
	prot_printf(imapd_out, "%s BAD Invalid parameter list in Id\r\n", tag);
	eatline(imapd_in, c);
	failed_id++;
	return;
	}

	/* parse parameter list */
	if (c == '(') {
	for (;;) {
	if (c == ')') {
	/* end of string/value pairs */
	break;
	}

	/* get field name */
	c = getstring(imapd_in, imapd_out, &field);
	if (c != ' ') {
	prot_printf(imapd_out,
	"%s BAD Invalid/missing field name in Id\r\n",
	tag);
	error = 1;
	break;
	}

	/* get field value */
	c = getnstring(imapd_in, imapd_out, &arg);
	if (c != ' ' && c != ')') {
	prot_printf(imapd_out,
	"%s BAD Invalid/missing value in Id\r\n",
	tag);
	error = 1;
	break;
	}

	/* ok, we're anal, but we'll still process the ID command */
	if (strlen(field.s) > MAXIDFIELDLEN) {
	prot_printf(imapd_out,
	"%s BAD field longer than %u octets in Id\r\n",
	tag, MAXIDFIELDLEN);
	error = 1;
	break;
	}
	if (strlen(arg.s) > MAXIDVALUELEN) {
	prot_printf(imapd_out,
	"%s BAD value longer than %u octets in Id\r\n",
	tag, MAXIDVALUELEN);
	error = 1;
	break;
	}
	if (++npair > MAXIDPAIRS) {
	prot_printf(imapd_out,
	"%s BAD too many (%u) field-value pairs in ID\r\n",
	tag, MAXIDPAIRS);
	error = 1;
	break;
	}

	/* ok, we're happy enough */
	id_appendparamlist(&params, field.s, arg.s);
	}

	if (error \|\| c != ')') {
	/* erp! */
	eatline(imapd_in, c);
	id_freeparamlist(params);
	failed_id++;
	return;
	}
	c = prot_getc(imapd_in);
	}

	/* check for CRLF */
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	prot_printf(imapd_out,
	"%s BAD Unexpected extra arguments to Id\r\n", tag);
	eatline(imapd_in, c);
	id_freeparamlist(params);
	failed_id++;
	return;
	}

	/* log the client's ID string.
	eventually this should be a callback or something. */
	if (npair && logged_id < MAXIDLOG) {
	char logbuf[MAXIDLOGLEN + 1] = "";
	struct idparamlist *pptr;

	for (pptr = params; pptr; pptr = pptr->next) {
	/* should we check for and format literals here ??? */
	snprintf(logbuf + strlen(logbuf), MAXIDLOGLEN - strlen(logbuf),
	" \"%s\" ", pptr->field);
	if (!strcmp(pptr->value, "NIL"))
	snprintf(logbuf + strlen(logbuf), MAXIDLOGLEN - strlen(logbuf),
	"NIL");
	else
	snprintf(logbuf + strlen(logbuf), MAXIDLOGLEN - strlen(logbuf),
	"\"%s\"", pptr->value);
	}

	syslog(LOG_INFO, "client id:%s", logbuf);

	logged_id++;
	}

	id_freeparamlist(params);

	/* spit out our ID string.
	eventually this might be configurable. */
	if (config_getswitch("imapidresponse", 1)) {
	id_response(imapd_out);
	prot_printf(imapd_out, ")\r\n");
	}
	else
	prot_printf(imapd_out, "* ID NIL\r\n");

	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));

	failed_id = 0;
	did_id = 1;
	}

	/*
	* Append the 'field'/'value' pair to the paramlist 'l'.
	*/
	void id_appendparamlist(struct idparamlist *l, char field, char *value)
	{
	struct idparamlist **tail = l;

	while (tail) tail = &(tail)->next;

	tail = (struct idparamlist )xmalloc(sizeof(struct idparamlist));
	(*tail)->field = xstrdup(field);
	(*tail)->value = xstrdup(value);
	(*tail)->next = 0;
	}

	/*
	* Free the idparamlist 'l'
	*/
	void id_freeparamlist(struct idparamlist *l)
	{
	struct idparamlist *n;

	while (l) {
	n = l->next;
	free(l->field);
	free(l->value);
	l = n;
	}
	}

	/*
	* Perform an IDLE command
	*/
	void cmd_idle(char *tag)
	{
	int c;
	static struct buf arg;

	/* Setup for doing mailbox updates */
	if (!idle_init(imapd_mailbox, idle_update)) {
	prot_printf(imapd_out,
	"%s NO cannot start idling\r\n", tag);
	return;
	}

	/* Tell client we are idling and waiting for end of command */
	prot_printf(imapd_out, "+ go ahead\r\n");
	prot_flush(imapd_out);

	/* Get continuation data */
	c = getword(imapd_in, &arg);
	if (c != EOF) {
	if (!strcasecmp(arg.s, "Done") &&
	(c = (c == '\r') ? prot_getc(imapd_in) : c) == '\n') {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	else {
	prot_printf(imapd_out,
	"%s BAD Invalid Idle continuation\r\n", tag);
	eatline(imapd_in, c);
	}
	}

	/* Do any necessary cleanup */
	idle_done(imapd_mailbox);

	return;
	}

	/* Send unsolicited untagged responses to the client */
	void idle_update(idle_flags_t flags)
	{
	int fd;

	if ((flags & IDLE_MAILBOX) && imapd_mailbox)
	index_check(imapd_mailbox, 0, 1);

	if (flags & IDLE_ALERT) {
	if (! imapd_userisadmin &&
	(fd = open(shutdownfilename, O_RDONLY, 0)) != -1) {
	shutdown_file(fd);
	}
	}

	prot_flush(imapd_out);
	}

	/*
	* Perform a CAPABILITY command
	*/
	void cmd_capability(char *tag)
	{
	const char sasllist; / the list of SASL mechanisms */
	unsigned mechcount;
	const char *mupdate_server;

	if (imapd_mailbox) {
	index_check(imapd_mailbox, 0, 0);
	}
	prot_printf(imapd_out, "* CAPABILITY " CAPABILITY_STRING);

	#ifdef ENABLE_LISTEXT
	prot_printf(imapd_out, " LISTEXT LIST-SUBSCRIBED");
	#endif /* ENABLE_LISTEXT */

	if (idle_enabled()) {
	prot_printf(imapd_out, " IDLE");
	}

	if (tls_enabled("imap")) {
	prot_printf(imapd_out, " STARTTLS");
	}
	if (!imapd_starttls_done && !config_getswitch("allowplaintext", 1)) {
	prot_printf(imapd_out, " LOGINDISABLED");
	}

	mupdate_server = config_getstring("mupdate_server", NULL);
	if(mupdate_server) {
	prot_printf(imapd_out, " MUPDATE=mupdate://%s/", mupdate_server);
	}

	/* add the SASL mechs */
	if (sasl_listmech(imapd_saslconn, NULL,
	"AUTH=", " AUTH=", "",
	&sasllist,
	NULL, &mechcount) == SASL_OK && mechcount > 0) {
	prot_printf(imapd_out, " %s", sasllist);
	} else {
	/* else don't show anything */
	}

	#ifdef ENABLE_X_NETSCAPE_HACK
	prot_printf(imapd_out, " X-NETSCAPE");
	#endif
	prot_printf(imapd_out, "\r\n%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	/*
	* Parse and perform an APPEND command.
	* The command has been parsed up to and including
	* the mailbox name.
	*/
	static int isokflag(char *s)
	{
	if (s[0] == '\\') {
	lcase(s);
	if (!strcmp(s, "\\seen")) return 1;
	if (!strcmp(s, "\\answered")) return 1;
	if (!strcmp(s, "\\flagged")) return 1;
	if (!strcmp(s, "\\draft")) return 1;
	if (!strcmp(s, "\\deleted")) return 1;

	/* uh oh, system flag i don't recognize */
	return 0;
	} else {
	/* valid user flag? */
	return imparse_isatom(s);
	}
	}

	#define FLAGGROW 10
	void
	cmd_append(char tag, char name)

	{
	int c;
	char **flag = NULL;
	int nflags = 0, flagalloc = 0;
	static struct buf arg;
	char *p;
	time_t internaldate;
	unsigned size = 0;
	int sawdigit = 0;
	int isnowait = 0;
	int r;
	char mailboxname[MAX_MAILBOX_NAME+1];
	struct appendstate mailbox;
	unsigned long uidvalidity;
	unsigned long firstuid, num;
	const char *parseerr = NULL;

	/* Set up the append */
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);
	if (!r) {
	r = append_setup(&mailbox, mailboxname, MAILBOX_FORMAT_NORMAL,
	imapd_userid, imapd_authstate, ACL_INSERT, size);
	}
	if (r) {
	eatline(imapd_in, ' ');
	prot_printf(imapd_out, "%s NO %s%s\r\n",
	tag,
	(r == IMAP_MAILBOX_NONEXISTENT &&
	mboxlist_createmailboxcheck(mailboxname, 0, 0,
	imapd_userisadmin,
	imapd_userid, imapd_authstate,
	(char )0, (char )0) == 0)
	? "[TRYCREATE] " : "", error_message(r));
	return;
	}

	c = ' '; /* just parsed a space */
	/* we loop, to support MULTIAPPEND */
	while (!r && c == ' ') {
	/* Parse flags */
	c = getword(imapd_in, &arg);
	if (c == '(' && !arg.s[0]) {
	nflags = 0;
	do {
	c = getword(imapd_in, &arg);
	if (!nflags && !arg.s[0] && c == ')') break; /* empty list */
	if (!isokflag(arg.s)) {
	parseerr = "Invalid flag in Append command";
	r = IMAP_PROTOCOL_ERROR;
	goto done;
	}
	if (nflags == flagalloc) {
	flagalloc += FLAGGROW;
	flag = (char *)xrealloc((char )flag,
	flagalloc * sizeof(char *));
	}
	flag[nflags++] = xstrdup(arg.s);
	} while (c == ' ');
	if (c != ')') {
	parseerr =
	"Missing space or ) after flag name in Append command";
	r = IMAP_PROTOCOL_ERROR;
	goto done;
	}
	c = prot_getc(imapd_in);
	if (c != ' ') {
	parseerr = "Missing space after flag list in Append command";
	r = IMAP_PROTOCOL_ERROR;
	goto done;
	}
	c = getword(imapd_in, &arg);
	}

	/* Parse internaldate */
	if (c == '\"' && !arg.s[0]) {
	prot_ungetc(c, imapd_in);
	c = getdatetime(&internaldate);
	if (c != ' ') {
	parseerr = "Invalid date-time in Append command";
	r = IMAP_PROTOCOL_ERROR;
	goto done;
	}
	c = getword(imapd_in, &arg);
	} else {
	internaldate = time(NULL);
	}

	if (arg.s[0] != '{') {
	parseerr = "Missing required argument to Append command";
	r = IMAP_PROTOCOL_ERROR;
	goto done;
	}

	/* Read size from literal */
	size = 0;
	for (p = arg.s + 1; p && isdigit((int) p); p++) {
	sawdigit++;
	size = size10 + p - '0';
	}
	if (*p == '+') {
	isnowait++;
	p++;
	}

	if (c == '\r') {
	c = prot_getc(imapd_in);
	}
	else {
	prot_ungetc(c, imapd_in);
	c = ' '; /* Force a syntax error */
	}

	if (*p != '}' \|\| p[1] \|\| c != '\n' \|\| !sawdigit) {
	parseerr = "Invalid literal in Append command";
	r = IMAP_PROTOCOL_ERROR;
	goto done;
	}
	if (size < 2) {
	r = IMAP_MESSAGE_NOBLANKLINE;
	goto done;
	}

	if (!isnowait) {
	/* Tell client to send the message */
	prot_printf(imapd_out, "+ go ahead\r\n");
	prot_flush(imapd_out);
	}

	/* Perform the rest of the append */
	if (!r) r = append_fromstream(&mailbox, imapd_in, size, internaldate,
	(const char **) flag, nflags);

	/* if we see a SP, we're trying to append more than one message */

	/* Parse newline terminating command */
	c = prot_getc(imapd_in);
	}

	done:
	if (r) {
	eatline(imapd_in, c);
	} else {
	/* we should be looking at the end of the line */
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	parseerr = "junk after literal";
	r = IMAP_PROTOCOL_ERROR;
	eatline(imapd_in, c);
	}
	}

	if (!r) {
	r = append_commit(&mailbox, &uidvalidity, &firstuid, &num);
	} else {
	append_abort(&mailbox);
	}

	if (imapd_mailbox) {
	index_check(imapd_mailbox, 0, 0);
	}

	if (r == IMAP_PROTOCOL_ERROR && parseerr) {
	prot_printf(imapd_out, "%s BAD %s\r\n", tag, parseerr);
	} else if (r) {
	prot_printf(imapd_out, "%s NO %s%s\r\n",
	tag,
	(r == IMAP_MAILBOX_NONEXISTENT &&
	mboxlist_createmailboxcheck(mailboxname, 0, 0,
	imapd_userisadmin,
	imapd_userid, imapd_authstate,
	(char )0, (char )0) == 0)
	? "[TRYCREATE] " : "", error_message(r));
	} else {
	/* is this a space seperated list or sequence list? */
	- prot_printf(imapd_out, "%s OK [APPENDUID %u", tag, uidvalidity);
	+ prot_printf(imapd_out, "%s OK [APPENDUID %lu", tag, uidvalidity);
	if (num == 1) {
	- prot_printf(imapd_out, " %u", firstuid);
	+ prot_printf(imapd_out, " %lu", firstuid);
	} else {
	- prot_printf(imapd_out, " %u:%u", firstuid, firstuid + num - 1);
	+ prot_printf(imapd_out, " %lu:%lu", firstuid, firstuid + num - 1);
	}
	prot_printf(imapd_out, "] %s\r\n", error_message(IMAP_OK_COMPLETED));
	}

	/* free memory */
	while (nflags--) {
	free(flag[nflags]);
	}
	if (flag) free((char *)flag);
	}

	/*
	* Perform a SELECT/EXAMINE/BBOARD command
	*/
	void cmd_select(char tag, char cmd, char *name)
	{
	struct mailbox mailbox;
	char mailboxname[MAX_MAILBOX_NAME+1];
	int r = 0;
	double usage;
	int doclose = 0;

	if (imapd_mailbox) {
	index_closemailbox(imapd_mailbox);
	mailbox_close(imapd_mailbox);
	imapd_mailbox = 0;
	}

	if (cmd[0] == 'B') {
	/* BBoard namespace is empty */
	r = IMAP_MAILBOX_NONEXISTENT;
	}
	else {
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);
	}

	if (!r) {
	r = mailbox_open_header(mailboxname, imapd_authstate, &mailbox);
	}

	if (!r) {
	doclose = 1;
	r = mailbox_open_index(&mailbox);
	}
	if (!r && !(mailbox.myrights & ACL_READ)) {
	r = (imapd_userisadmin \|\| (mailbox.myrights & ACL_LOOKUP)) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	}
	if (!r && chdir(mailbox.path)) {
	syslog(LOG_ERR, "IOERROR: changing directory to %s: %m", mailbox.path);
	r = IMAP_IOERROR;
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	if (doclose) mailbox_close(&mailbox);
	return;
	}

	mboxstruct = mailbox;
	imapd_mailbox = &mboxstruct;

	index_newmailbox(imapd_mailbox, cmd[0] == 'E');

	/* Examine command puts mailbox in read-only mode */
	if (cmd[0] == 'E') {
	imapd_mailbox->myrights &= ~(ACL_SEEN\|ACL_WRITE\|ACL_DELETE);
	}

	if (imapd_mailbox->myrights & ACL_DELETE) {
	/* Warn if mailbox is close to or over quota */
	mailbox_read_quota(&imapd_mailbox->quota);
	if (imapd_mailbox->quota.limit > 0) {
	usage = ((double) imapd_mailbox->quota.used * 100.0) / (double)
	(imapd_mailbox->quota.limit * QUOTA_UNITS);
	if (usage >= 100.0) {
	prot_printf(imapd_out, "* NO [ALERT] %s\r\n",
	error_message(IMAP_NO_OVERQUOTA));
	}
	else if (usage > config_getint("quotawarn", 90)) {
	int usageint = (int) usage;
	prot_printf(imapd_out, "* NO [ALERT] ");
	prot_printf(imapd_out, error_message(IMAP_NO_CLOSEQUOTA),
	usageint);
	prot_printf(imapd_out, "\r\n");
	}
	}
	}

	prot_printf(imapd_out, "%s OK [READ-%s] %s\r\n", tag,
	(imapd_mailbox->myrights & (ACL_WRITE\|ACL_DELETE)) ?
	"WRITE" : "ONLY", error_message(IMAP_OK_COMPLETED));

	proc_register("imapd", imapd_clienthost, imapd_userid, mailboxname);
	syslog(LOG_DEBUG, "open: user %s opened %s", imapd_userid, name);
	}

	/*
	* Perform a CLOSE command
	*/
	void
	cmd_close(tag)
	char *tag;
	{
	int r;

	if (!(imapd_mailbox->myrights & ACL_DELETE)) r = 0;
	else {
	r = mailbox_expunge(imapd_mailbox, 1, (int ()())0, (char )0);
	}

	index_closemailbox(imapd_mailbox);
	mailbox_close(imapd_mailbox);
	imapd_mailbox = 0;

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	}
	else {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}
	}

	/*
	* Perform an UNSELECT command -- for some support of IMAP proxy.
	* Just like close except no expunge.
	*/
	void
	cmd_unselect(tag)
	char* tag;
	{
	index_closemailbox(imapd_mailbox);
	mailbox_close(imapd_mailbox);
	imapd_mailbox = 0;

	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	/*
	* Parse and perform a FETCH/UID FETCH command
	* The command has been parsed up to and including
	* the sequence
	*/
	void
	cmd_fetch(tag, sequence, usinguid)
	char *tag;
	char *sequence;
	int usinguid;
	{
	char *cmd = usinguid ? "UID Fetch" : "Fetch";
	static struct buf fetchatt, fieldname;
	int c, i;
	int inlist = 0;
	int fetchitems = 0;
	struct fetchargs fetchargs;
	struct strlist *newfields = 0;
	char p, section;
	int fetchedsomething;

	memset(&fetchargs, 0, sizeof(struct fetchargs));

	c = getword(imapd_in, &fetchatt);
	if (c == '(' && !fetchatt.s[0]) {
	inlist = 1;
	c = getword(imapd_in, &fetchatt);
	}
	for (;;) {
	ucase(fetchatt.s);
	switch (fetchatt.s[0]) {
	case 'A':
	if (!inlist && !strcmp(fetchatt.s, "ALL")) {
	fetchitems \|= FETCH_ALL;
	}
	else goto badatt;
	break;

	case 'B':
	if (!strcmp(fetchatt.s, "BODY")) {
	fetchitems \|= FETCH_BODY;
	}
	else if (!strcmp(fetchatt.s, "BODYSTRUCTURE")) {
	fetchitems \|= FETCH_BODYSTRUCTURE;
	}
	else if (!strncmp(fetchatt.s, "BODY[", 5) \|\|
	!strncmp(fetchatt.s, "BODY.PEEK[", 10)) {
	p = section = fetchatt.s + 5;
	if (*p == 'P') {
	p = section += 5;
	}
	else {
	fetchitems \|= FETCH_SETSEEN;
	}
	while (isdigit((int) p) \|\| p == '.') {
	if (*p == '.' && !isdigit((int) p[-1])) break;
	/* Obsolete section 0 can only occur before close brace */
	if (*p == '0' && !isdigit((int) p[-1]) && p[1] != ']') break;
	p++;
	}

	if (*p == 'H' && !strncmp(p, "HEADER.FIELDS", 13) &&
	(p == section \|\| p[-1] == '.') &&
	(p[13] == '\0' \|\| !strcmp(p+13, ".NOT"))) {

	/*
	* If not top-level or a HEADER.FIELDS.NOT, can't pull
	* the headers out of the cache.
	*/
	if (p != section \|\| p[13] != '\0') {
	fetchitems \|= FETCH_UNCACHEDHEADER;
	}

	if (c != ' ') {
	prot_printf(imapd_out,
	"%s BAD Missing required argument to %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}
	c = prot_getc(imapd_in);
	if (c != '(') {
	prot_printf(imapd_out, "%s BAD Missing required open parenthesis in %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}
	do {
	c = getastring(imapd_in, imapd_out, &fieldname);
	for (p = fieldname.s; *p; p++) {
	if (p <= ' ' \|\| p & 0x80 \|\| *p == ':') break;
	}
	if (p \|\| !fieldname.s) {
	prot_printf(imapd_out, "%s BAD Invalid field-name in %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}
	appendstrlist(&newfields, fieldname.s);
	if (!(fetchitems & FETCH_UNCACHEDHEADER)) {
	for (i=0; i<mailbox_num_cache_header; i++) {
	if (!strcasecmp(mailbox_cache_header_name[i],
	fieldname.s)) break;
	}
	if (i == mailbox_num_cache_header) {
	fetchitems \|= FETCH_UNCACHEDHEADER;
	}
	}
	} while (c == ' ');
	if (c != ')') {
	prot_printf(imapd_out, "%s BAD Missing required close parenthesis in %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}

	/* Grab/parse the ]<x.y> part */
	c = getword(imapd_in, &fieldname);
	p = fieldname.s;
	if (*p++ != ']') {
	prot_printf(imapd_out, "%s BAD Missing required close bracket after %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}
	if (*p == '<' && isdigit((int) p[1])) {
	p += 2;
	while (isdigit((int) *p)) p++;

	if (*p == '.' && p[1] >= '1' && p[1] <= '9') {
	p += 2;
	while (isdigit((int) *p)) p++;
	}
	else p--;

	if (*p != '>') {
	prot_printf(imapd_out, "%s BAD Invalid body partial\r\n", tag);
	eatline(imapd_in, c);
	goto freeargs;
	}
	p++;
	}
	if (*p) {
	prot_printf(imapd_out, "%s BAD Junk after body section\r\n", tag);
	eatline(imapd_in, c);
	goto freeargs;
	}
	appendfieldlist(&fetchargs.fsections,
	section, newfields, fieldname.s);
	newfields = 0;
	break;
	}

	switch (*p) {
	case 'H':
	if (p != section && p[-1] != '.') break;
	if (!strncmp(p, "HEADER]", 7)) p += 6;
	break;

	case 'M':
	if (!strncmp(p-1, ".MIME]", 6)) p += 4;
	break;

	case 'T':
	if (p != section && p[-1] != '.') break;
	if (!strncmp(p, "TEXT]", 5)) p += 4;
	break;
	}

	if (*p != ']') {
	prot_printf(imapd_out, "%s BAD Invalid body section\r\n", tag);
	eatline(imapd_in, c);
	goto freeargs;
	}
	p++;
	if (*p == '<' && isdigit((int) p[1])) {
	p += 2;
	while (isdigit((int) *p)) p++;

	if (*p == '.' && p[1] >= '1' && p[1] <= '9') {
	p += 2;
	while (isdigit((int) *p)) p++;
	}
	else p--;

	if (*p != '>') {
	prot_printf(imapd_out, "%s BAD Invalid body partial\r\n", tag);
	eatline(imapd_in, c);
	goto freeargs;
	}
	p++;
	}

	if (*p) {
	prot_printf(imapd_out, "%s BAD Junk after body section\r\n", tag);
	eatline(imapd_in, c);
	goto freeargs;
	}
	appendstrlist(&fetchargs.bodysections, section);
	}
	else goto badatt;
	break;

	case 'E':
	if (!strcmp(fetchatt.s, "ENVELOPE")) {
	fetchitems \|= FETCH_ENVELOPE;
	}
	else goto badatt;
	break;

	case 'F':
	if (!inlist && !strcmp(fetchatt.s, "FAST")) {
	fetchitems \|= FETCH_FAST;
	}
	else if (!inlist && !strcmp(fetchatt.s, "FULL")) {
	fetchitems \|= FETCH_FULL;
	}
	else if (!strcmp(fetchatt.s, "FLAGS")) {
	fetchitems \|= FETCH_FLAGS;
	}
	else goto badatt;
	break;

	case 'I':
	if (!strcmp(fetchatt.s, "INTERNALDATE")) {
	fetchitems \|= FETCH_INTERNALDATE;
	}
	else goto badatt;
	break;

	case 'R':
	if (!strcmp(fetchatt.s, "RFC822")) {
	fetchitems \|= FETCH_RFC822\|FETCH_SETSEEN;
	}
	else if (!strcmp(fetchatt.s, "RFC822.HEADER")) {
	fetchitems \|= FETCH_HEADER;
	}
	else if (!strcmp(fetchatt.s, "RFC822.PEEK")) {
	fetchitems \|= FETCH_RFC822;
	}
	else if (!strcmp(fetchatt.s, "RFC822.SIZE")) {
	fetchitems \|= FETCH_SIZE;
	}
	else if (!strcmp(fetchatt.s, "RFC822.TEXT")) {
	fetchitems \|= FETCH_TEXT\|FETCH_SETSEEN;
	}
	else if (!strcmp(fetchatt.s, "RFC822.TEXT.PEEK")) {
	fetchitems \|= FETCH_TEXT;
	}
	else if (!strcmp(fetchatt.s, "RFC822.HEADER.LINES") \|\|
	!strcmp(fetchatt.s, "RFC822.HEADER.LINES.NOT")) {
	if (c != ' ') {
	prot_printf(imapd_out, "%s BAD Missing required argument to %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}
	c = prot_getc(imapd_in);
	if (c != '(') {
	prot_printf(imapd_out, "%s BAD Missing required open parenthesis in %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}
	do {
	c = getastring(imapd_in, imapd_out, &fieldname);
	for (p = fieldname.s; *p; p++) {
	if (p <= ' ' \|\| p & 0x80 \|\| *p == ':') break;
	}
	if (p \|\| !fieldname.s) {
	prot_printf(imapd_out, "%s BAD Invalid field-name in %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}
	lcase(fieldname.s);;
	appendstrlist(strlen(fetchatt.s) == 19 ?
	&fetchargs.headers : &fetchargs.headers_not,
	fieldname.s);
	if (strlen(fetchatt.s) != 19) {
	fetchitems \|= FETCH_UNCACHEDHEADER;
	}
	if (!(fetchitems & FETCH_UNCACHEDHEADER)) {
	for (i=0; i<mailbox_num_cache_header; i++) {
	if (!strcmp(mailbox_cache_header_name[i],
	fieldname.s)) break;
	}
	if (i == mailbox_num_cache_header) {
	fetchitems \|= FETCH_UNCACHEDHEADER;
	}
	}
	} while (c == ' ');
	if (c != ')') {
	prot_printf(imapd_out, "%s BAD Missing required close parenthesis in %s %s\r\n",
	tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}
	c = prot_getc(imapd_in);
	}
	else goto badatt;
	break;

	case 'U':
	if (!strcmp(fetchatt.s, "UID")) {
	fetchitems \|= FETCH_UID;
	}
	else goto badatt;
	break;

	default:
	badatt:
	prot_printf(imapd_out, "%s BAD Invalid %s attribute %s\r\n", tag, cmd, fetchatt.s);
	eatline(imapd_in, c);
	goto freeargs;
	}

	if (inlist && c == ' ') c = getword(imapd_in, &fetchatt);
	else break;
	}

	if (inlist && c == ')') {
	inlist = 0;
	c = prot_getc(imapd_in);
	}
	if (inlist) {
	prot_printf(imapd_out, "%s BAD Missing close parenthesis in %s\r\n", tag, cmd);
	eatline(imapd_in, c);
	goto freeargs;
	}
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	prot_printf(imapd_out, "%s BAD Unexpected extra arguments to %s\r\n", tag, cmd);
	eatline(imapd_in, c);
	goto freeargs;
	}

	if (!fetchitems && !fetchargs.bodysections && !fetchargs.fsections &&
	!fetchargs.headers && !fetchargs.headers_not) {
	prot_printf(imapd_out, "%s BAD Missing required argument to %s\r\n", tag, cmd);
	goto freeargs;
	}

	if (usinguid) {
	fetchitems \|= FETCH_UID;
	index_check(imapd_mailbox, 1, 0);
	}

	fetchargs.fetchitems = fetchitems;
	index_fetch(imapd_mailbox, sequence, usinguid, &fetchargs,
	&fetchedsomething);

	if (fetchedsomething \|\| usinguid) {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	} else {
	/* normal FETCH, nothing came back */
	prot_printf(imapd_out, "%s NO %s\r\n", tag,
	error_message(IMAP_NO_NOSUCHMSG));
	}

	freeargs:
	freestrlist(newfields);
	freestrlist(fetchargs.bodysections);
	freefieldlist(fetchargs.fsections);
	freestrlist(fetchargs.headers);
	freestrlist(fetchargs.headers_not);
	}

	/*
	* Perform a PARTIAL command
	*/
	void
	cmd_partial(tag, msgno, data, start, count)
	char *tag;
	char *msgno;
	char *data;
	char *start;
	char *count;
	{
	char *p;
	struct fetchargs fetchargs;
	char *section;
	int fetchedsomething;

	memset(&fetchargs, 0, sizeof(struct fetchargs));

	for (p = msgno; *p; p++) {
	if (!isdigit((int) *p)) break;
	}
	if (p \|\| !msgno) {
	prot_printf(imapd_out, "%s BAD Invalid message number\r\n", tag);
	return;
	}

	lcase(data);
	if (!strcmp(data, "rfc822")) {
	fetchargs.fetchitems = FETCH_RFC822\|FETCH_SETSEEN;
	}
	else if (!strcmp(data, "rfc822.header")) {
	fetchargs.fetchitems = FETCH_HEADER;
	}
	else if (!strcmp(data, "rfc822.peek")) {
	fetchargs.fetchitems = FETCH_RFC822;
	}
	else if (!strcmp(data, "rfc822.text")) {
	fetchargs.fetchitems = FETCH_TEXT\|FETCH_SETSEEN;
	}
	else if (!strcmp(data, "rfc822.text.peek")) {
	fetchargs.fetchitems = FETCH_TEXT;
	}
	else if (!strncmp(data, "body[", 5) \|\|
	!strncmp(data, "body.peek[", 10)) {
	p = section = data + 5;
	if (*p == 'p') {
	p = section += 5;
	}
	else {
	fetchargs.fetchitems = FETCH_SETSEEN;
	}
	while (isdigit((int) p) \|\| p == '.') {
	if (*p == '.' && (p == section \|\| !isdigit((int) p[1]))) break;
	p++;
	}
	if (p == section \|\| *p != ']' \|\| p[1]) {
	prot_printf(imapd_out, "%s BAD Invalid body section\r\n", tag);
	freestrlist(fetchargs.bodysections);
	return;
	}
	*p = '\0';
	appendstrlist(&fetchargs.bodysections, section);
	}
	else {
	prot_printf(imapd_out, "%s BAD Invalid Partial item\r\n", tag);
	freestrlist(fetchargs.bodysections);
	return;
	}

	for (p = start; *p; p++) {
	if (!isdigit((int) *p)) break;
	fetchargs.start_octet = fetchargs.start_octet10 + p - '0';
	}
	if (*p \|\| !fetchargs.start_octet) {
	prot_printf(imapd_out, "%s BAD Invalid starting octet\r\n", tag);
	freestrlist(fetchargs.bodysections);
	return;
	}

	for (p = count; *p; p++) {
	if (!isdigit((int) *p)) break;
	fetchargs.octet_count = fetchargs.octet_count10 + p - '0';
	}
	if (p \|\| !count) {
	prot_printf(imapd_out, "%s BAD Invalid octet count\r\n", tag);
	freestrlist(fetchargs.bodysections);
	return;
	}

	index_fetch(imapd_mailbox, msgno, 0, &fetchargs, &fetchedsomething);

	index_check(imapd_mailbox, 0, 0);

	if (fetchedsomething) {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	} else {
	prot_printf(imapd_out,
	"%s BAD Invalid sequence in PARTIAL command\r\n",
	tag);
	}

	freestrlist(fetchargs.bodysections);
	}

	/*
	* Parse and perform a STORE/UID STORE command
	* The command has been parsed up to and including
	* the FLAGS/+FLAGS/-FLAGS
	*/
	void
	cmd_store(tag, sequence, operation, usinguid)
	char *tag;
	char *sequence;
	char *operation;
	int usinguid;
	{
	char *cmd = usinguid ? "UID Store" : "Store";
	struct storeargs storeargs;
	static struct buf flagname;
	int len, c;
	char **flag = 0;
	int nflags = 0, flagalloc = 0;
	int flagsparsed = 0, inlist = 0;
	int r;

	memset(&storeargs, 0, sizeof storeargs);

	lcase(operation);

	len = strlen(operation);
	if (len > 7 && !strcmp(operation+len-7, ".silent")) {
	storeargs.silent = 1;
	operation[len-7] = '\0';
	}

	if (!strcmp(operation, "+flags")) {
	storeargs.operation = STORE_ADD;
	}
	else if (!strcmp(operation, "-flags")) {
	storeargs.operation = STORE_REMOVE;
	}
	else if (!strcmp(operation, "flags")) {
	storeargs.operation = STORE_REPLACE;
	}
	else {
	prot_printf(imapd_out, "%s BAD Invalid %s attribute\r\n", tag, cmd);
	eatline(imapd_in, ' ');
	return;
	}

	for (;;) {
	c = getword(imapd_in, &flagname);
	if (c == '(' && !flagname.s[0] && !flagsparsed && !inlist) {
	inlist = 1;
	continue;
	}

	if (!flagname.s[0]) break;

	if (flagname.s[0] == '\\') {
	lcase(flagname.s);
	if (!strcmp(flagname.s, "\\seen")) {
	storeargs.seen = 1;
	}
	else if (!strcmp(flagname.s, "\\answered")) {
	storeargs.system_flags \|= FLAG_ANSWERED;
	}
	else if (!strcmp(flagname.s, "\\flagged")) {
	storeargs.system_flags \|= FLAG_FLAGGED;
	}
	else if (!strcmp(flagname.s, "\\deleted")) {
	storeargs.system_flags \|= FLAG_DELETED;
	}
	else if (!strcmp(flagname.s, "\\draft")) {
	storeargs.system_flags \|= FLAG_DRAFT;
	}
	else {
	prot_printf(imapd_out, "%s BAD Invalid system flag in %s command\r\n",
	tag, cmd);
	eatline(imapd_in, c);
	goto freeflags;
	}
	}
	else if (!imparse_isatom(flagname.s)) {
	prot_printf(imapd_out, "%s BAD Invalid flag name %s in %s command\r\n",
	tag, flagname.s, cmd);
	eatline(imapd_in, c);
	goto freeflags;
	}
	else {
	if (nflags == flagalloc) {
	flagalloc += FLAGGROW;
	flag = (char *)xrealloc((char )flag,
	flagallocsizeof(char ));
	}
	flag[nflags++] = xstrdup(flagname.s);
	}

	flagsparsed++;
	if (c != ' ') break;
	}

	if (!inlist && !flagsparsed) {
	prot_printf(imapd_out, "%s BAD Missing required argument to %s\r\n", tag, cmd);
	eatline(imapd_in, c);
	return;
	}
	if (inlist && c == ')') {
	inlist = 0;
	c = prot_getc(imapd_in);
	}
	if (inlist) {
	prot_printf(imapd_out, "%s BAD Missing close parenthesis in %s\r\n", tag, cmd);
	eatline(imapd_in, c);
	goto freeflags;
	}
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	prot_printf(imapd_out, "%s BAD Unexpected extra arguments to %s\r\n", tag, cmd);
	eatline(imapd_in, c);
	goto freeflags;
	}

	r = index_store(imapd_mailbox, sequence, usinguid, &storeargs,
	flag, nflags);

	if (usinguid) {
	index_check(imapd_mailbox, 1, 0);
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	}
	else {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	freeflags:
	while (nflags--) {
	free(flag[nflags]);
	}
	if (flag) free((char *)flag);
	}

	void
	cmd_search(tag, usinguid)
	char *tag;
	int usinguid;
	{
	int c;
	int charset = 0;
	struct searchargs *searchargs;
	clock_t start = clock();
	char mytime[100];
	int n;

	searchargs = (struct searchargs *)xzmalloc(sizeof(struct searchargs));

	c = getsearchprogram(tag, searchargs, &charset, 1);
	if (c == EOF) {
	eatline(imapd_in, ' ');
	freesearchargs(searchargs);
	return;
	}

	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	prot_printf(imapd_out, "%s BAD Unexpected extra arguments to Search\r\n", tag);
	eatline(imapd_in, c);
	freesearchargs(searchargs);
	return;
	}

	if (charset == -1) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag,
	error_message(IMAP_UNRECOGNIZED_CHARSET));
	}
	else {
	n = index_search(imapd_mailbox, searchargs, usinguid);
	sprintf(mytime, "%2.3f", (clock() - start) / (double) CLOCKS_PER_SEC);
	prot_printf(imapd_out, "%s OK %s (%d msgs in %s secs)\r\n", tag,
	error_message(IMAP_OK_COMPLETED), n, mytime);
	}

	freesearchargs(searchargs);
	}

	/*
	* Perform a SORT/UID SORT command
	*/
	void
	cmd_sort(tag, usinguid)
	char *tag;
	int usinguid;
	{
	int c;
	struct sortcrit *sortcrit = NULL;
	static struct buf arg;
	int charset = 0;
	struct searchargs *searchargs;
	clock_t start = clock();
	char mytime[100];
	int n;

	c = getsortcriteria(tag, &sortcrit);
	if (c == EOF) {
	eatline(imapd_in, ' ');
	freesortcrit(sortcrit);
	return;
	}

	/* get charset */
	if (c != ' ') {
	prot_printf(imapd_out, "%s BAD Missing charset in Sort\r\n",
	tag);
	eatline(imapd_in, c);
	freesortcrit(sortcrit);
	return;
	}

	c = getword(imapd_in, &arg);
	if (c != ' ') {
	prot_printf(imapd_out, "%s BAD Missing search criteria in Sort\r\n",
	tag);
	eatline(imapd_in, c);
	freesortcrit(sortcrit);
	return;
	}
	lcase(arg.s);
	charset = charset_lookupname(arg.s);

	if (charset == -1) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag,
	error_message(IMAP_UNRECOGNIZED_CHARSET));
	eatline(imapd_in, c);
	freesortcrit(sortcrit);
	return;
	}

	searchargs = (struct searchargs *)xzmalloc(sizeof(struct searchargs));

	c = getsearchprogram(tag, searchargs, &charset, 0);
	if (c == EOF) {
	eatline(imapd_in, ' ');
	freesearchargs(searchargs);
	freesortcrit(sortcrit);
	return;
	}

	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	prot_printf(imapd_out,
	"%s BAD Unexpected extra arguments to Sort\r\n", tag);
	eatline(imapd_in, c);
	freesearchargs(searchargs);
	freesortcrit(sortcrit);
	return;
	}

	n = index_sort(imapd_mailbox, sortcrit, searchargs, usinguid);
	sprintf(mytime, "%2.3f", (clock() - start) / (double) CLOCKS_PER_SEC);
	prot_printf(imapd_out, "%s OK %s (%d msgs in %s secs)\r\n", tag,
	error_message(IMAP_OK_COMPLETED), n, mytime);

	freesortcrit(sortcrit);
	freesearchargs(searchargs);
	return;
	}

	/*
	* Perform a THREAD/UID THREAD command
	*/
	void
	cmd_thread(tag, usinguid)
	char *tag;
	int usinguid;
	{
	static struct buf arg;
	int c;
	int charset = 0;
	int alg;
	struct searchargs *searchargs;
	clock_t start = clock();
	char mytime[100];
	int n;

	/* get algorithm */
	c = getword(imapd_in, &arg);
	if (c != ' ') {
	prot_printf(imapd_out, "%s BAD Missing algorithm in Thread\r\n", tag);
	eatline(imapd_in, c);
	return;
	}

	if ((alg = find_thread_algorithm(arg.s)) == -1) {
	prot_printf(imapd_out, "%s BAD Invalid Thread algorithm %s\r\n",
	tag, arg.s);
	eatline(imapd_in, c);
	return;
	}

	/* get charset */
	c = getword(imapd_in, &arg);
	if (c != ' ') {
	prot_printf(imapd_out, "%s BAD Missing charset in Thread\r\n",
	tag);
	eatline(imapd_in, c);
	return;
	}
	lcase(arg.s);
	charset = charset_lookupname(arg.s);

	if (charset == -1) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag,
	error_message(IMAP_UNRECOGNIZED_CHARSET));
	eatline(imapd_in, c);
	return;
	}

	searchargs = (struct searchargs *)xzmalloc(sizeof(struct searchargs));

	c = getsearchprogram(tag, searchargs, &charset, 0);
	if (c == EOF) {
	eatline(imapd_in, ' ');
	freesearchargs(searchargs);
	return;
	}

	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	prot_printf(imapd_out,
	"%s BAD Unexpected extra arguments to Thread\r\n", tag);
	eatline(imapd_in, c);
	freesearchargs(searchargs);
	return;
	}

	n = index_thread(imapd_mailbox, alg, searchargs, usinguid);
	sprintf(mytime, "%2.3f", (clock() - start) / (double) CLOCKS_PER_SEC);
	prot_printf(imapd_out, "%s OK %s (%d msgs in %s secs)\r\n", tag,
	error_message(IMAP_OK_COMPLETED), n, mytime);

	freesearchargs(searchargs);
	return;
	}

	/*
	* Perform a COPY/UID COPY command
	*/
	void
	cmd_copy(tag, sequence, name, usinguid)
	char *tag;
	char *sequence;
	char *name;
	int usinguid;
	{
	int r;
	char mailboxname[MAX_MAILBOX_NAME+1];
	char *copyuid;

	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);
	if (!r) {
	r = index_copy(imapd_mailbox, sequence, usinguid, mailboxname,
	&copyuid);
	}

	index_check(imapd_mailbox, usinguid, 0);

	if (r) {
	prot_printf(imapd_out, "%s NO %s%s\r\n", tag,
	(r == IMAP_MAILBOX_NONEXISTENT &&
	mboxlist_createmailboxcheck(mailboxname, 0, 0,
	imapd_userisadmin,
	imapd_userid, imapd_authstate,
	(char )0, (char )0) == 0)
	? "[TRYCREATE] " : "", error_message(r));
	}
	else {
	if (copyuid) {
	prot_printf(imapd_out, "%s OK [COPYUID %s] %s\r\n", tag,
	copyuid, error_message(IMAP_OK_COMPLETED));
	free(copyuid);
	}
	else if (usinguid) {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}
	else {
	/* normal COPY, message doesn't exist */
	prot_printf(imapd_out, "%s NO %s\r\n", tag,
	error_message(IMAP_NO_NOSUCHMSG));
	}
	}
	}

	/*
	* Perform an EXPUNGE command
	*/
	void
	cmd_expunge(tag, sequence)
	char *tag;
	char *sequence;
	{
	int r;

	if (!(imapd_mailbox->myrights & ACL_DELETE)) r = IMAP_PERMISSION_DENIED;
	else if (sequence) {
	r = mailbox_expunge(imapd_mailbox, 1, index_expungeuidlist, sequence);
	}
	else {
	r = mailbox_expunge(imapd_mailbox, 1, (mailbox_decideproc_t *)0,
	(void *)0);
	}

	index_check(imapd_mailbox, 0, 0);

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	}
	else {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}
	}

	/*
	* Perform a CREATE command
	*/
	void
	cmd_create(tag, name, partition)
	char *tag;
	char *name;
	char *partition;
	{
	int r = 0;
	char mailboxname[MAX_MAILBOX_NAME+1];
	int autocreatequota;

	if (partition && !imapd_userisadmin) {
	r = IMAP_PERMISSION_DENIED;
	}

	if (name[0] && name[strlen(name)-1] == imapd_namespace.hier_sep) {
	/* We don't care about trailing hierarchy delimiters. */
	name[strlen(name)-1] = '\0';
	}

	if (!r) {
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);
	}

	if (!r) {
	r = mboxlist_createmailbox(mailboxname, 0, partition,
	imapd_userisadmin,
	imapd_userid, imapd_authstate);

	if (r == IMAP_PERMISSION_DENIED && !strcasecmp(name, "INBOX") &&
	(autocreatequota = config_getint("autocreatequota", 0))) {

	/* Auto create */
	r = mboxlist_createmailbox(mailboxname, 0,
	partition, 1, imapd_userid,
	imapd_authstate);

	if (!r && autocreatequota > 0) {
	(void) mboxlist_setquota(mailboxname, autocreatequota);
	}
	}
	}

	if (imapd_mailbox) {
	index_check(imapd_mailbox, 0, 0);
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	}
	else {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}
	}

	/* 'tmplist' is used for recursive means in cmd_delete() and cmd_rename() */
	struct tmplist {
	int alloc;
	int num;
	char mb[1][MAX_MAILBOX_NAME];
	};

	#define TMPLIST_INC 50

	/* Callback for use by cmd_delete */
	static int addmbox(char *name,
	int matchlen __attribute__((unused)),
	int maycreate __attribute__((unused)),
	void *rock)
	{
	struct tmplist lptr = (struct tmplist ) rock;
	struct tmplist l = lptr;

	if (l->alloc == l->num) {
	l->alloc += TMPLIST_INC;
	l = xrealloc(l, sizeof(struct tmplist) +
	l->alloc * MAX_MAILBOX_NAME * (sizeof(char)));
	*lptr = l;
	}

	strcpy(l->mb[l->num++], name);

	return 0;
	}

	/*
	* Perform a DELETE command
	*/
	void
	cmd_delete(tag, name)
	char *tag;
	char *name;
	{
	int r;
	char mailboxname[MAX_MAILBOX_NAME+1];

	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);

	if (!r) {
	r = mboxlist_deletemailbox(mailboxname, imapd_userisadmin,
	imapd_userid, imapd_authstate, 1);
	}

	/* was it a top-level user mailbox? */
	if (!r &&
	!strncmp(mailboxname, "user.", 5) && !strchr(mailboxname+5, '.')) {
	struct tmplist *l = xmalloc(sizeof(struct tmplist));
	char *p;
	int r2, i;

	l->alloc = 0;
	l->num = 0;

	p = mailboxname + strlen(mailboxname); /* end of mailboxname */
	strcpy(p, ".*");
	/* build a list of mailboxes - we're using internal names here */
	mboxlist_findall(NULL, mailboxname, imapd_userisadmin, imapd_userid,
	imapd_authstate, addmbox, &l);

	/* foreach mailbox in list, remove it */
	for (i = 0; i < l->num; i++) {
	r2 = mboxlist_deletemailbox(l->mb[i], imapd_userisadmin,
	imapd_userid, imapd_authstate, 0);
	if (r2) {
	prot_printf(imapd_out, "* NO delete %s: %s\r\n",
	l->mb[i], error_message(r2));
	}
	}

	free(l);

	/* take care of deleting ACLs, subscriptions, seen state and quotas */
	p = '\0'; / clip off pattern */
	user_delete(mailboxname+5, imapd_userid, imapd_authstate);
	}

	if (imapd_mailbox) {
	index_check(imapd_mailbox, 0, 0);
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	}
	else {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}
	}

	/*
	* Perform a RENAME command
	*/
	void cmd_rename(const char *tag,
	char oldname, char newname, char *partition)
	{
	int r = 0;
	char oldmailboxname[MAX_MAILBOX_NAME+3];
	char newmailboxname[MAX_MAILBOX_NAME+2];
	int omlen, nmlen;
	char *p;
	int recursive_rename;

	/* canonicalize names */
	if (partition && !imapd_userisadmin) {
	r = IMAP_PERMISSION_DENIED;
	}

	recursive_rename = 1;

	/* if this is my inbox, don't do recursive renames */
	if (!strcasecmp(oldname, "inbox")) {
	recursive_rename = 0;
	}

	if (!r)
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, oldname,
	imapd_userid, oldmailboxname);
	if (!r)
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, newname,
	imapd_userid, newmailboxname);

	/* if we're renaming something inside of something else,
	don't recursively rename stuff */
	omlen = strlen(oldmailboxname);
	nmlen = strlen(newmailboxname);
	if (strlen(oldmailboxname) < strlen(newmailboxname)) {
	if (!strncmp(oldmailboxname, newmailboxname, omlen) &&
	newmailboxname[omlen] == '.') {
	recursive_rename = 0;
	}
	} else {
	if (!strncmp(oldmailboxname, newmailboxname, nmlen) &&
	oldmailboxname[nmlen] == '.') {
	recursive_rename = 0;
	}
	}

	/* verify that the mailbox doesn't have a wildcard in it */
	for (p = oldmailboxname; !r && *p; p++) {
	if (p == '' \|\| *p == '%') r = IMAP_MAILBOX_BADNAME;
	}

	/* attempt to rename the base mailbox */
	if (!r) {
	r = mboxlist_renamemailbox(oldmailboxname, newmailboxname, partition,
	imapd_userisadmin,
	imapd_userid, imapd_authstate);
	}

	/* rename all mailboxes matching this */
	if (!r && recursive_rename) {
	struct tmplist *l = xmalloc(sizeof(struct tmplist));
	int ol = omlen + 1;
	int nl = nmlen + 1;
	int i;

	l->alloc = 0;
	l->num = 0;

	strcat(oldmailboxname, ".*");
	strcat(newmailboxname, ".");

	/* add submailboxes; we pretend we're an admin since we successfully
	renamed the parent - we're using internal names here */
	mboxlist_findall(NULL, oldmailboxname, 1, imapd_userid,
	imapd_authstate, addmbox, &l);

	/* foreach mailbox in list, rename it, pretending we're admin */
	for (i = 0; i < l->num; i++) {
	int r2 = 0;

	if (nl + strlen(l->mb[i] + ol) > MAX_MAILBOX_NAME) {
	/* this mailbox name is too long */
	continue;
	}
	strcpy(newmailboxname + nl, l->mb[i] + ol);
	r2 = mboxlist_renamemailbox(l->mb[i], newmailboxname,
	partition,
	1, imapd_userid, imapd_authstate);
	if (r2) {
	prot_printf(imapd_out, "* NO rename %s %s: %s\r\n",
	l->mb[i], newmailboxname, error_message(r2));
	if (RENAME_STOP_ON_ERROR) break;
	}
	}

	free(l);
	}

	if (imapd_mailbox) {
	index_check(imapd_mailbox, 0, 0);
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	} else {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}
	}

	/*
	* Perform a RECONSTRUCT command
	*/
	void
	cmd_reconstruct(const char tag, const char name)
	{
	int r = 0;
	char mailboxname[MAX_MAILBOX_NAME+1];

	/* administrators only please */
	if (!imapd_userisadmin) {
	r = IMAP_PERMISSION_DENIED;
	}

	if (!r) {
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);
	}

	if (!r) {
	/* Check for mailbox in mailbox list */
	r = mboxlist_lookup(mailboxname, NULL, NULL, NULL);
	}

	if (!r) {
	int pid;
	/* Reconstruct it */

	pid = fork();
	if(pid == -1) {
	r = IMAP_SYS_ERROR;
	} else if(pid == 0) {
	char buf[4096];
	/* Child - exec reconstruct*/
	syslog(LOG_NOTICE, "Reconstructing '%s' for user '%s'",
	mailboxname, imapd_userid);

	snprintf(buf, sizeof(buf), "%s/reconstruct", SERVICE_PATH);

	fclose(stdin);
	fclose(stdout);
	fclose(stderr);

	execl(buf, buf, mailboxname, NULL);

	/* if we are here, we have a problem */
	exit(-1);
	} else {
	int status;

	/* Parent, wait on child */
	if(waitpid(pid, &status, 0) < 0) r = IMAP_SYS_ERROR;

	/* Did we fail? */
	if(WEXITSTATUS(status) != 0) r = IMAP_SYS_ERROR;
	}
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	} else {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}
	}

	/*
	* Perform a FIND command
	*/
	void
	cmd_find(tag, namespace, pattern)
	char *tag;
	char *namespace;
	char *pattern;
	{
	char *p;
	lcase(namespace);

	for (p = pattern; *p; p++) {
	if (p == '%') p = '?';
	}

	/* Translate any separators in pattern */
	mboxname_hiersep_tointernal(&imapd_namespace, pattern);

	if (!strcmp(namespace, "mailboxes")) {
	(*imapd_namespace.mboxlist_findsub)(&imapd_namespace, pattern,
	imapd_userisadmin, imapd_userid,
	imapd_authstate, mailboxdata,
	NULL, 0);
	}
	else if (!strcmp(namespace, "all.mailboxes")) {
	(*imapd_namespace.mboxlist_findall)(&imapd_namespace, pattern,
	imapd_userisadmin, imapd_userid,
	imapd_authstate, mailboxdata, NULL);
	}
	else if (!strcmp(namespace, "bboards")
	\|\| !strcmp(namespace, "all.bboards")) {
	;
	}
	else {
	prot_printf(imapd_out, "%s BAD Invalid FIND subcommand\r\n", tag);
	return;
	}
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	static int mstringdatacalls;

	/*
	* Perform a LIST or LSUB command
	*/
	void cmd_list(char tag, int listopts, char reference, char *pattern)
	{
	char *buf = NULL;
	int patlen = 0;
	int reflen = 0;
	static int ignorereference = 0;
	clock_t start = clock();
	char mytime[100];

	/* Ignore the reference argument?
	(the behavior in 1.5.10 & older) */
	if (ignorereference == 0) {
	ignorereference = config_getswitch("ignorereference", 0);
	}

	/* Reset state in mstringdata */
	mstringdata(NULL, NULL, 0, 0, 0);

	if (!pattern[0] && !(listopts & LIST_LSUB)) {
	/* Special case: query top-level hierarchy separator */
	prot_printf(imapd_out, "* LIST (\\Noselect) \"%c\" \"\"\r\n",
	imapd_namespace.hier_sep);
	} else {
	/* Do we need to concatenate fields? */
	if (!ignorereference \|\| pattern[0] == imapd_namespace.hier_sep) {
	/* Either
	* - name begins with dot
	* - we're configured to honor the reference argument */

	/* Allocate a buffer, figure out how to stick the arguments
	together, do it, then do that instead of using pattern. */
	patlen = strlen(pattern);
	reflen = strlen(reference);

	buf = xmalloc(patlen + reflen + 1);
	buf[0] = '\0';

	if (*reference) {
	/* check for LIST A. .B, change to LIST "" A.B */
	if (reference[reflen-1] == imapd_namespace.hier_sep &&
	pattern[0] == imapd_namespace.hier_sep) {
	reference[--reflen] = '\0';
	}
	strcpy(buf, reference);
	}
	strcat(buf, pattern);
	pattern = buf;
	}

	/* Translate any separators in pattern */
	mboxname_hiersep_tointernal(&imapd_namespace, pattern);

	if (listopts & LIST_LSUB) {
	int force = config_getswitch("allowallsubscribe", 0);

	(*imapd_namespace.mboxlist_findsub)(&imapd_namespace, pattern,
	imapd_userisadmin, imapd_userid,
	imapd_authstate, lsubdata,
	&listopts, force);
	lsubdata((char *)0, 0, 0, &listopts);
	}
	else if (listopts & LIST_SUBSCRIBED) {
	int force = config_getswitch("allowallsubscribe", 0);

	(*imapd_namespace.mboxlist_findsub)(&imapd_namespace, pattern,
	imapd_userisadmin, imapd_userid,
	imapd_authstate, listdata,
	&listopts, force);
	listdata((char *)0, 0, 0, &listopts);
	}
	else {
	(*imapd_namespace.mboxlist_findall)(&imapd_namespace, pattern,
	imapd_userisadmin, imapd_userid,
	imapd_authstate, listdata,
	&listopts);
	listdata((char *)0, 0, 0, &listopts);
	}

	if (buf) free(buf);
	}
	sprintf(mytime, "%2.3f", (clock() - start) / (double) CLOCKS_PER_SEC);
	prot_printf(imapd_out, "%s OK %s (%s secs %d calls)\r\n", tag,
	error_message(IMAP_OK_COMPLETED), mytime, mstringdatacalls);
	}

	/*
	* Perform a SUBSCRIBE (add is nonzero) or
	* UNSUBSCRIBE (add is zero) command
	*/
	void cmd_changesub(char tag, char namespace,
	char *name, int add)
	{
	int r;
	char mailboxname[MAX_MAILBOX_NAME+1];
	int force = config_getswitch("allowallsubscribe", 0);

	if (namespace) lcase(namespace);
	if (!namespace \|\| !strcmp(namespace, "mailbox")) {
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);
	if (!r) {
	r = mboxlist_changesub(mailboxname, imapd_userid,
	imapd_authstate, add, force);
	}
	}
	else if (!strcmp(namespace, "bboard")) {
	r = add ? IMAP_MAILBOX_NONEXISTENT : 0;
	}
	else {
	prot_printf(imapd_out, "%s BAD Invalid %s subcommand\r\n", tag,
	add ? "Subscribe" : "Unsubscribe");
	return;
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s: %s\r\n", tag,
	add ? "Subscribe" : "Unsubscribe", error_message(r));
	}
	else {
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}
	}

	/*
	* Perform a GETACL command
	*/
	void
	cmd_getacl(tag, name, oldform)
	char *tag;
	char *name;
	int oldform;
	{
	char mailboxname[MAX_MAILBOX_NAME+1];
	int r, access;
	char *acl;
	char rights, nextid;

	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);

	if (!r) {
	r = mboxlist_lookup(mailboxname, NULL, &acl, NULL);
	}

	if (!r) {
	access = cyrus_acl_myrights(imapd_authstate, acl);

	if (!(access & (ACL_READ\|ACL_ADMIN)) &&
	!imapd_userisadmin &&
	!mboxname_userownsmailbox(imapd_userid, mailboxname)) {
	r = (access&ACL_LOOKUP) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	}
	}
	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	return;
	}

	if (oldform) {
	while (acl) {
	rights = strchr(acl, '\t');
	if (!rights) break;
	*rights++ = '\0';

	nextid = strchr(rights, '\t');
	if (!nextid) break;
	*nextid++ = '\0';

	prot_printf(imapd_out, "* ACL MAILBOX ");
	printastring(name);
	prot_printf(imapd_out, " ");
	printastring(acl);
	prot_printf(imapd_out, " ");
	printastring(rights);
	prot_printf(imapd_out, "\r\n");
	acl = nextid;
	}
	}
	else {
	prot_printf(imapd_out, "* ACL ");
	printastring(name);

	while (acl) {
	rights = strchr(acl, '\t');
	if (!rights) break;
	*rights++ = '\0';

	nextid = strchr(rights, '\t');
	if (!nextid) break;
	*nextid++ = '\0';

	prot_printf(imapd_out, " ");
	printastring(acl);
	prot_printf(imapd_out, " ");
	printastring(rights);
	acl = nextid;
	}
	prot_printf(imapd_out, "\r\n");
	}
	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	/*
	* Perform a LISTRIGHTS command
	*/
	void
	cmd_listrights(tag, name, identifier)
	char *tag;
	char *name;
	char *identifier;
	{
	char mailboxname[MAX_MAILBOX_NAME+1];
	int r, rights;
	char *canon_identifier;
	int canonidlen = 0;
	char *acl;
	char *rightsdesc;

	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);

	if (!r) {
	r = mboxlist_lookup(mailboxname, (char **)0, &acl, NULL);
	}

	if (!r) {
	rights = cyrus_acl_myrights(imapd_authstate, acl);

	if (!rights && !imapd_userisadmin &&
	!mboxname_userownsmailbox(imapd_userid, mailboxname)) {
	r = IMAP_MAILBOX_NONEXISTENT;
	}
	}

	if (!r) {
	canon_identifier = auth_canonifyid(identifier, 0);
	if (canon_identifier) canonidlen = strlen(canon_identifier);

	if (!canon_identifier) {
	rightsdesc = "\"\"";
	}
	else if (!strncmp(mailboxname, "user.", 5) &&
	!strchr(canon_identifier, '.') &&
	!strncmp(mailboxname+5, canon_identifier, canonidlen) &&
	(mailboxname[5+canonidlen] == '\0' \|\|
	mailboxname[5+canonidlen] == '.')) {
	rightsdesc = "lca r s w i p d 0 1 2 3 4 5 6 7 8 9";
	}
	else {
	rightsdesc = "\"\" l r s w i p c d a 0 1 2 3 4 5 6 7 8 9";
	}

	prot_printf(imapd_out, "* LISTRIGHTS ");
	printastring(name);
	prot_putc(' ', imapd_out);
	printastring(identifier);
	prot_printf(imapd_out, " %s", rightsdesc);

	prot_printf(imapd_out, "\r\n%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	return;
	}

	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	}

	/*
	* Perform a MYRIGHTS command
	*/
	void
	cmd_myrights(tag, name, oldform)
	char *tag;
	char *name;
	int oldform;
	{
	char mailboxname[MAX_MAILBOX_NAME+1];
	int r, rights = 0;
	char *acl;
	char str[ACL_MAXSTR];

	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);

	if (!r) {
	r = mboxlist_lookup(mailboxname, (char **)0, &acl, NULL);
	}

	if (!r) {
	rights = cyrus_acl_myrights(imapd_authstate, acl);

	/* Add in implicit rights */
	if (imapd_userisadmin \|\|
	mboxname_userownsmailbox(imapd_userid, mailboxname)) {
	rights \|= ACL_LOOKUP\|ACL_ADMIN;
	}

	if (!rights) {
	r = IMAP_MAILBOX_NONEXISTENT;
	}
	}
	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	return;
	}

	prot_printf(imapd_out, "* MYRIGHTS ");
	if (oldform) prot_printf(imapd_out, "MAILBOX ");
	printastring(name);
	prot_printf(imapd_out, " ");
	printastring(cyrus_acl_masktostr(rights, str));
	prot_printf(imapd_out, "\r\n%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	/*
	* Perform a SETACL command
	*/
	void
	cmd_setacl(tag, name, identifier, rights)
	char *tag;
	char *name;
	char *identifier;
	char *rights;
	{
	int r;
	char mailboxname[MAX_MAILBOX_NAME+1];

	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);

	if (!r) {
	r = mboxlist_setacl(mailboxname, identifier, rights,
	imapd_userisadmin, imapd_userid, imapd_authstate);
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	return;
	}

	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	/*
	* Perform a GETQUOTA command
	*/
	void
	cmd_getquota(tag, name)
	char *tag;
	char *name;
	{
	int r;
	struct quota quota;
	char buf[MAX_MAILBOX_PATH];
	char mailboxname[MAX_MAILBOX_NAME+1];

	quota.fd = -1;

	if (!imapd_userisadmin) r = IMAP_PERMISSION_DENIED;
	else {
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);
	if (!r) {
	quota.root = mailboxname;
	mailbox_hash_quota(buf, quota.root);
	quota.fd = open(buf, O_RDWR, 0);
	if (quota.fd == -1) {
	r = IMAP_QUOTAROOT_NONEXISTENT;
	}
	else {
	r = mailbox_read_quota(&quota);
	}
	}
	}

	if (!r) {
	prot_printf(imapd_out, "* QUOTA ");
	printastring(name);
	prot_printf(imapd_out, " (");
	if (quota.limit >= 0) {
	- prot_printf(imapd_out, "STORAGE %u %d",
	+ prot_printf(imapd_out, "STORAGE %lu %d",
	quota.used/QUOTA_UNITS, quota.limit);
	}
	prot_printf(imapd_out, ")\r\n");
	}

	if (quota.fd != -1) {
	close(quota.fd);
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	return;
	}

	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}


	/*
	* Perform a GETQUOTAROOT command
	*/
	void
	cmd_getquotaroot(tag, name)
	char *tag;
	char *name;
	{
	char mailboxname[MAX_MAILBOX_NAME+1];
	struct mailbox mailbox;
	int r;
	int doclose = 0;

	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);

	if (!r) {
	r = mailbox_open_header(mailboxname, imapd_authstate, &mailbox);
	}

	if (!r) {
	doclose = 1;
	if (!imapd_userisadmin && !(mailbox.myrights & ACL_READ)) {
	r = (mailbox.myrights & ACL_LOOKUP) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	}
	}

	if (!r) {
	prot_printf(imapd_out, "* QUOTAROOT ");
	printastring(name);
	if (mailbox.quota.root) {
	(*imapd_namespace.mboxname_toexternal)(&imapd_namespace,
	mailbox.quota.root,
	imapd_userid, mailboxname);
	prot_printf(imapd_out, " ");
	printastring(mailboxname);
	r = mailbox_read_quota(&mailbox.quota);
	if (!r) {
	prot_printf(imapd_out, "\r\n* QUOTA ");
	printastring(mailboxname);
	prot_printf(imapd_out, " (");
	if (mailbox.quota.limit >= 0) {
	- prot_printf(imapd_out, "STORAGE %u %d",
	+ prot_printf(imapd_out, "STORAGE %lu %d",
	mailbox.quota.used/QUOTA_UNITS,
	mailbox.quota.limit);
	}
	prot_putc(')', imapd_out);
	}
	}
	prot_printf(imapd_out, "\r\n");
	}

	if (doclose) mailbox_close(&mailbox);

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	return;
	}

	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}

	/*
	* Parse and perform a SETQUOTA command
	* The command has been parsed up to the resource list
	*/
	void
	cmd_setquota(tag, quotaroot)
	char *tag;
	char *quotaroot;
	{
	int newquota = -1;
	int badresource = 0;
	int c;
	static struct buf arg;
	char *p;
	int r;
	char mailboxname[MAX_MAILBOX_NAME+1];

	c = prot_getc(imapd_in);
	if (c != '(') goto badlist;

	c = getword(imapd_in, &arg);
	if (c != ')' \|\| arg.s[0] != '\0') {
	for (;;) {
	if (c != ' ') goto badlist;
	if (strcasecmp(arg.s, "storage") != 0) badresource = 1;
	c = getword(imapd_in, &arg);
	if (c != ' ' && c != ')') goto badlist;
	if (arg.s[0] == '\0') goto badlist;
	newquota = 0;
	for (p = arg.s; *p; p++) {
	if (!isdigit((int) *p)) goto badlist;
	newquota = newquota * 10 + *p - '0';
	}
	if (c == ')') break;
	}
	}
	c = prot_getc(imapd_in);
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	prot_printf(imapd_out, "%s BAD Unexpected extra arguments to SETQUOTA\r\n", tag);
	eatline(imapd_in, c);
	return;
	}

	if (badresource) r = IMAP_UNSUPPORTED_QUOTA;
	else if (!imapd_userisadmin) r = IMAP_PERMISSION_DENIED;
	else {
	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, quotaroot,
	imapd_userid, mailboxname);

	if (!r) {
	r = mboxlist_setquota(mailboxname, newquota);
	}
	}

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	return;
	}

	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	return;

	badlist:
	prot_printf(imapd_out, "%s BAD Invalid quota list in Setquota\r\n", tag);
	eatline(imapd_in, c);
	}

	#ifdef HAVE_SSL
	/*
	* this implements the STARTTLS command, as described in RFC 2595.
	* one caveat: it assumes that no external layer is currently present.
	* if a client executes this command, information about the external
	* layer that was passed on the command line is disgarded. this should
	* be fixed.
	*/
	/* imaps - whether this is an imaps transaction or not */
	void cmd_starttls(char *tag, int imaps)
	{
	int result;
	int *layerp;

	char *auth_id;
	sasl_ssf_t ssf;

	/* SASL and openssl have different ideas about whether ssf is signed */
	layerp = (int *) &ssf;

	if (imapd_starttls_done == 1)
	{
	prot_printf(imapd_out, "%s NO TLS already active\r\n", tag);
	return;
	}

	result=tls_init_serverengine("imap",
	5, /* depth to verify */
	!imaps, /* can client auth? */
	0, /* require client to auth? */
	!imaps); /* TLS only? */

	if (result == -1) {

	syslog(LOG_ERR, "error initializing TLS");

	if (imaps == 0) {
	prot_printf(imapd_out, "%s NO Error initializing TLS\r\n", tag);
	} else {
	fatal("tls_init() failed", EC_CONFIG);
	}

	return;
	}

	if (imaps == 0)
	{
	prot_printf(imapd_out, "%s OK Begin TLS negotiation now\r\n", tag);
	/* must flush our buffers before starting tls */
	prot_flush(imapd_out);
	}

	result=tls_start_servertls(0, /* read */
	1, /* write */
	layerp,
	&auth_id,
	&tls_conn);

	/* if error */
	if (result==-1) {
	if (imaps == 0) {
	prot_printf(imapd_out, "%s NO Starttls failed\r\n", tag);
	syslog(LOG_NOTICE, "STARTTLS failed: %s", imapd_clienthost);
	return;
	} else {
	syslog(LOG_NOTICE, "imaps failed: %s", imapd_clienthost);
	fatal("tls_start_servertls() failed", EC_TEMPFAIL);
	return;
	}
	}

	/* tell SASL about the negotiated layer */
	result = sasl_setprop(imapd_saslconn, SASL_SSF_EXTERNAL, &ssf);
	if (result != SASL_OK) {
	fatal("sasl_setprop() failed: cmd_starttls()", EC_TEMPFAIL);
	}
	saslprops.ssf = ssf;

	result = sasl_setprop(imapd_saslconn, SASL_AUTH_EXTERNAL, auth_id);
	if (result != SASL_OK) {
	fatal("sasl_setprop() failed: cmd_starttls()", EC_TEMPFAIL);
	}
	if(saslprops.authid) {
	free(saslprops.authid);
	saslprops.authid = NULL;
	}
	if(auth_id)
	saslprops.authid = xstrdup(auth_id);

	/* tell the prot layer about our new layers */
	prot_settls(imapd_in, tls_conn);
	prot_settls(imapd_out, tls_conn);

	imapd_starttls_done = 1;
	}
	#else
	void cmd_starttls(char *tag, int imaps)
	{
	fatal("cmd_starttls() executed, but starttls isn't implemented!",
	EC_SOFTWARE);
	}
	#endif /* HAVE_SSL */

	/*
	* Parse and perform a STATUS command
	* The command has been parsed up to the attribute list
	*/
	void
	cmd_status(tag, name)
	char *tag;
	char *name;
	{
	int c;
	int statusitems = 0;
	static struct buf arg;
	struct mailbox mailbox;
	char mailboxname[MAX_MAILBOX_NAME+1];
	int r = 0;
	int doclose = 0;

	c = prot_getc(imapd_in);
	if (c != '(') goto badlist;

	c = getword(imapd_in, &arg);
	if (arg.s[0] == '\0') goto badlist;
	for (;;) {
	lcase(arg.s);
	if (!strcmp(arg.s, "messages")) {
	statusitems \|= STATUS_MESSAGES;
	}
	else if (!strcmp(arg.s, "recent")) {
	statusitems \|= STATUS_RECENT;
	}
	else if (!strcmp(arg.s, "uidnext")) {
	statusitems \|= STATUS_UIDNEXT;
	}
	else if (!strcmp(arg.s, "uidvalidity")) {
	statusitems \|= STATUS_UIDVALIDITY;
	}
	else if (!strcmp(arg.s, "unseen")) {
	statusitems \|= STATUS_UNSEEN;
	}
	else {
	prot_printf(imapd_out, "%s BAD Invalid Status attribute %s\r\n",
	tag, arg.s);
	eatline(imapd_in, c);
	return;
	}

	if (c == ' ') c = getword(imapd_in, &arg);
	else break;
	}

	if (c != ')') {
	prot_printf(imapd_out,
	"%s BAD Missing close parenthesis in Status\r\n", tag);
	eatline(imapd_in, c);
	return;
	}

	c = prot_getc(imapd_in);
	if (c == '\r') c = prot_getc(imapd_in);
	if (c != '\n') {
	prot_printf(imapd_out,
	"%s BAD Unexpected extra arguments to Status\r\n", tag);
	eatline(imapd_in, c);
	return;
	}

	/*
	* Perform a full checkpoint of any open mailbox, in case we're
	* doing a STATUS check of the current mailbox.
	*/
	if (imapd_mailbox) {
	index_check(imapd_mailbox, 0, 1);
	}

	r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
	imapd_userid, mailboxname);

	if (!r) {
	r = mailbox_open_header(mailboxname, imapd_authstate, &mailbox);
	}

	if (!r) {
	doclose = 1;
	r = mailbox_open_index(&mailbox);
	}
	if (!r && !(mailbox.myrights & ACL_READ)) {
	r = (imapd_userisadmin \|\| (mailbox.myrights & ACL_LOOKUP)) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	}

	if (!r) {
	r = index_status(&mailbox, name, statusitems);
	}

	if (doclose) mailbox_close(&mailbox);

	if (r) {
	prot_printf(imapd_out, "%s NO %s\r\n", tag, error_message(r));
	return;
	}

	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	return;

	badlist:
	prot_printf(imapd_out, "%s BAD Invalid status list in Status\r\n", tag);
	eatline(imapd_in, c);
	}

	#ifdef ENABLE_X_NETSCAPE_HACK
	/*
	* Reply to Netscape's crock with a crock of my own
	*/
	void
	cmd_netscrape(tag)
	char *tag;
	{
	const char *url;
	/* so tempting, and yet ... */
	/* url = "http://random.yahoo.com/ryl/"; */
	url = config_getstring("netscapeurl",
	"http://andrew2.andrew.cmu.edu/cyrus/imapd/netscape-admin.html");

	/* I only know of three things to reply with: */
	prot_printf(imapd_out,
	"* OK [NETSCAPE] Carnegie Mellon Cyrus IMAP\r\n* VERSION %s\r\n",
	CYRUS_VERSION);
	prot_printf(imapd_out,
	"* ACCOUNT-URL %s\r\n%s OK %s\r\n",
	url, tag, error_message(IMAP_OK_COMPLETED));
	}
	#endif /* ENABLE_X_NETSCAPE_HACK */

	/* Callback for cmd_namespace to be passed to mboxlist_findall.
	* For each top-level mailbox found, print a bit of the response
	* if it is a shared namespace. The rock is used as an integer in
	* order to ensure the namespace response is correct on a server with
	* no shared namespace.
	*/
	static int namespacedata(char *name,
	int matchlen __attribute__((unused)),
	int maycreate __attribute__((unused)),
	void *rock)
	{
	int* sawone = (int*) rock;

	if (!name) {
	return 0;
	}

	if (!(strncmp(name, "INBOX.", 6))) {
	/* The user has a "personal" namespace. */
	sawone[NAMESPACE_INBOX] = 1;
	} else if (!(strncmp(name, "user.", 5))) {
	/* The user can see the "other users" namespace. */
	sawone[NAMESPACE_USER] = 1;
	} else {
	/* The user can see the "shared" namespace. */
	sawone[NAMESPACE_SHARED] = 1;
	}

	return 0;
	}

	/*
	* Print out a response to the NAMESPACE command defined by
	* RFC 2342.
	*/
	void cmd_namespace(tag)
	char* tag;
	{
	int sawone[3] = {0, 0, 0};
	char* pattern;

	if (SLEEZY_NAMESPACE) {
	char inboxname[MAX_MAILBOX_NAME+1];

	if (strlen(imapd_userid) + 5 > MAX_MAILBOX_NAME)
	sawone[NAMESPACE_INBOX] = 0;
	else {
	sprintf(inboxname, "user.%s", imapd_userid);
	sawone[NAMESPACE_INBOX] =
	!mboxlist_lookup(inboxname, NULL, NULL, NULL);
	}
	sawone[NAMESPACE_USER] = 1;
	sawone[NAMESPACE_SHARED] = 1;
	} else {
	pattern = xstrdup("%");
	/* now find all the exciting toplevel namespaces -
	* we're using internal names here
	*/
	mboxlist_findall(NULL, pattern, imapd_userisadmin, imapd_userid,
	imapd_authstate, namespacedata, (void*) sawone);
	free(pattern);
	}

	prot_printf(imapd_out, "* NAMESPACE");
	if (sawone[NAMESPACE_INBOX]) {
	prot_printf(imapd_out, " ((\"%s\" \"%c\"))",
	imapd_namespace.prefix[NAMESPACE_INBOX],
	imapd_namespace.hier_sep);
	} else {
	prot_printf(imapd_out, " NIL");
	}
	if (sawone[NAMESPACE_USER]) {
	prot_printf(imapd_out, " ((\"%s\" \"%c\"))",
	imapd_namespace.prefix[NAMESPACE_USER],
	imapd_namespace.hier_sep);
	} else {
	prot_printf(imapd_out, " NIL");
	}
	if (sawone[NAMESPACE_SHARED]) {
	prot_printf(imapd_out, " ((\"%s\" \"%c\"))",
	imapd_namespace.prefix[NAMESPACE_SHARED],
	imapd_namespace.hier_sep);
	} else {
	prot_printf(imapd_out, " NIL");
	}
	prot_printf(imapd_out, "\r\n");

	prot_printf(imapd_out, "%s OK %s\r\n", tag,
	error_message(IMAP_OK_COMPLETED));
	}


	/*
	* Parse a search program
	*/
	int getsearchprogram(tag, searchargs, charset, parsecharset)
	char *tag;
	struct searchargs *searchargs;
	int *charset;
	int parsecharset;
	{
	int c;

	do {
	c = getsearchcriteria(tag, searchargs, charset, parsecharset);
	parsecharset = 0;
	} while (c == ' ');
	return c;
	}

	/*
	* Parse a search criteria
	*/
	int getsearchcriteria(tag, searchargs, charset, parsecharset)
	char *tag;
	struct searchargs *searchargs;
	int *charset;
	int parsecharset;
	{
	static struct buf criteria, arg;
	struct searchargs sub1, sub2;
	char p, str;
	int i, c, flag;
	unsigned size;
	time_t start, end;

	c = getword(imapd_in, &criteria);
	lcase(criteria.s);
	switch (criteria.s[0]) {
	case '\0':
	if (c != '(') goto badcri;
	c = getsearchprogram(tag, searchargs, charset, 0);
	if (c == EOF) return EOF;
	if (c != ')') {
	prot_printf(imapd_out, "%s BAD Missing required close paren in Search command\r\n",
	tag);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;
	}
	c = prot_getc(imapd_in);
	break;

	case '0': case '1': case '2': case '3': case '4':
	case '5': case '6': case '7': case '8': case '9':
	case '*':
	if (imparse_issequence(criteria.s)) {
	appendstrlist(&searchargs->sequence, criteria.s);
	}
	else goto badcri;
	break;

	case 'a':
	if (!strcmp(criteria.s, "answered")) {
	searchargs->system_flags_set \|= FLAG_ANSWERED;
	}
	else if (!strcmp(criteria.s, "all")) {
	break;
	}
	else goto badcri;
	break;

	case 'b':
	if (!strcmp(criteria.s, "before")) {
	if (c != ' ') goto missingarg;
	c = getsearchdate(&start, &end);
	if (c == EOF) goto baddate;
	if (!searchargs->before \|\| searchargs->before > start) {
	searchargs->before = start;
	}
	}
	else if (!strcmp(criteria.s, "bcc")) {
	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c == EOF) goto missingarg;
	str = charset_convert(arg.s, *charset, NULL, 0);
	if (strchr(str, EMPTY)) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	}
	else {
	appendstrlistpat(&searchargs->bcc, str);
	}
	}
	else if (!strcmp(criteria.s, "body")) {
	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c == EOF) goto missingarg;
	str = charset_convert(arg.s, *charset, NULL, 0);
	if (strchr(str, EMPTY)) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	}
	else {
	appendstrlistpat(&searchargs->body, str);
	}
	}
	else goto badcri;
	break;

	case 'c':
	if (!strcmp(criteria.s, "cc")) {
	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c == EOF) goto missingarg;
	str = charset_convert(arg.s, *charset, NULL, 0);
	if (strchr(str, EMPTY)) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	}
	else {
	appendstrlistpat(&searchargs->cc, str);
	}
	}
	else if (parsecharset && !strcmp(criteria.s, "charset")) {
	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c != ' ') goto missingarg;
	lcase(arg.s);
	*charset = charset_lookupname(arg.s);
	}
	else goto badcri;
	break;

	case 'd':
	if (!strcmp(criteria.s, "deleted")) {
	searchargs->system_flags_set \|= FLAG_DELETED;
	}
	else if (!strcmp(criteria.s, "draft")) {
	searchargs->system_flags_set \|= FLAG_DRAFT;
	}
	else goto badcri;
	break;

	case 'f':
	if (!strcmp(criteria.s, "flagged")) {
	searchargs->system_flags_set \|= FLAG_FLAGGED;
	}
	else if (!strcmp(criteria.s, "from")) {
	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c == EOF) goto missingarg;
	str = charset_convert(arg.s, *charset, NULL, 0);
	if (strchr(str, EMPTY)) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	}
	else {
	appendstrlistpat(&searchargs->from, str);
	}
	}
	else goto badcri;
	break;

	case 'h':
	if (!strcmp(criteria.s, "header")) {
	struct strlist **patlist;

	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c != ' ') goto missingarg;
	lcase(arg.s);

	/* some headers can be reduced to search terms */
	if (!strcmp(arg.s, "bcc")) {
	patlist = &searchargs->bcc;
	}
	else if (!strcmp(arg.s, "cc")) {
	patlist = &searchargs->cc;
	}
	else if (!strcmp(arg.s, "to")) {
	patlist = &searchargs->to;
	}
	else if (!strcmp(arg.s, "from")) {
	patlist = &searchargs->from;
	}
	else if (!strcmp(arg.s, "subject")) {
	patlist = &searchargs->subject;
	}

	/* we look message-id up in the envelope */
	else if (!strcmp(arg.s, "message-id")) {
	patlist = &searchargs->messageid;
	}

	/* all other headers we handle normally */
	else {
	if (!(searchargs->flags & SEARCH_UNCACHEDHEADER)) {
	for (i=0; i<mailbox_num_cache_header; i++) {
	if (!strcmp(mailbox_cache_header_name[i], arg.s)) break;
	}
	if (i == mailbox_num_cache_header) {
	searchargs->flags \|= SEARCH_UNCACHEDHEADER;
	}
	}
	appendstrlist(&searchargs->header_name, arg.s);
	patlist = &searchargs->header;
	}

	c = getastring(imapd_in, imapd_out, &arg);
	if (c == EOF) goto missingarg;
	str = charset_convert(arg.s, *charset, NULL, 0);
	if (strchr(str, EMPTY)) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	}
	else {
	appendstrlistpat(patlist, str);
	}
	}
	else goto badcri;
	break;

	case 'k':
	if (!strcmp(criteria.s, "keyword")) {
	if (c != ' ') goto missingarg;
	c = getword(imapd_in, &arg);
	if (!imparse_isatom(arg.s)) goto badflag;
	lcase(arg.s);
	for (flag=0; flag < MAX_USER_FLAGS; flag++) {
	if (imapd_mailbox->flagname[flag] &&
	!strcasecmp(imapd_mailbox->flagname[flag], arg.s)) break;
	}
	if (flag == MAX_USER_FLAGS) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	break;
	}
	searchargs->user_flags_set[flag/32] \|= 1<<(flag&31);
	}
	else goto badcri;
	break;

	case 'l':
	if (!strcmp(criteria.s, "larger")) {
	if (c != ' ') goto missingarg;
	c = getword(imapd_in, &arg);
	size = 0;
	for (p = arg.s; p && isdigit((int) p); p++) {
	size = size * 10 + *p - '0';
	}
	if (!arg.s \|\| *p) goto badnumber;
	if (size > searchargs->larger) searchargs->larger = size;
	}
	else goto badcri;
	break;

	case 'n':
	if (!strcmp(criteria.s, "not")) {
	if (c != ' ') goto missingarg;
	sub1 = (struct searchargs *)xzmalloc(sizeof(struct searchargs));
	c = getsearchcriteria(tag, sub1, charset, 0);
	if (c == EOF) {
	freesearchargs(sub1);
	return EOF;
	}

	appendsearchargs(searchargs, sub1, (struct searchargs *)0);
	}
	else if (!strcmp(criteria.s, "new")) {
	searchargs->flags \|= (SEARCH_SEEN_UNSET\|SEARCH_RECENT_SET);
	}
	else goto badcri;
	break;

	case 'o':
	if (!strcmp(criteria.s, "or")) {
	if (c != ' ') goto missingarg;
	sub1 = (struct searchargs *)xzmalloc(sizeof(struct searchargs));
	c = getsearchcriteria(tag, sub1, charset, 0);
	if (c == EOF) {
	freesearchargs(sub1);
	return EOF;
	}
	if (c != ' ') goto missingarg;
	sub2 = (struct searchargs *)xzmalloc(sizeof(struct searchargs));
	c = getsearchcriteria(tag, sub2, charset, 0);
	if (c == EOF) {
	freesearchargs(sub1);
	freesearchargs(sub2);
	return EOF;
	}
	appendsearchargs(searchargs, sub1, sub2);
	}
	else if (!strcmp(criteria.s, "old")) {
	searchargs->flags \|= SEARCH_RECENT_UNSET;
	}
	else if (!strcmp(criteria.s, "on")) {
	if (c != ' ') goto missingarg;
	c = getsearchdate(&start, &end);
	if (c == EOF) goto baddate;
	if (!searchargs->before \|\| searchargs->before > end) {
	searchargs->before = end;
	}
	if (!searchargs->after \|\| searchargs->after < start) {
	searchargs->after = start;
	}
	}
	else goto badcri;
	break;

	case 'r':
	if (!strcmp(criteria.s, "recent")) {
	searchargs->flags \|= SEARCH_RECENT_SET;
	}
	else goto badcri;
	break;

	case 's':
	if (!strcmp(criteria.s, "seen")) {
	searchargs->flags \|= SEARCH_SEEN_SET;
	}
	else if (!strcmp(criteria.s, "sentbefore")) {
	if (c != ' ') goto missingarg;
	c = getsearchdate(&start, &end);
	if (c == EOF) goto baddate;
	if (!searchargs->sentbefore \|\| searchargs->sentbefore > start) {
	searchargs->sentbefore = start;
	}
	}
	else if (!strcmp(criteria.s, "senton")) {
	if (c != ' ') goto missingarg;
	c = getsearchdate(&start, &end);
	if (c == EOF) goto baddate;
	if (!searchargs->sentbefore \|\| searchargs->sentbefore > end) {
	searchargs->sentbefore = end;
	}
	if (!searchargs->sentafter \|\| searchargs->sentafter < start) {
	searchargs->sentafter = start;
	}
	}
	else if (!strcmp(criteria.s, "sentsince")) {
	if (c != ' ') goto missingarg;
	c = getsearchdate(&start, &end);
	if (c == EOF) goto baddate;
	if (!searchargs->sentafter \|\| searchargs->sentafter < start) {
	searchargs->sentafter = start;
	}
	}
	else if (!strcmp(criteria.s, "since")) {
	if (c != ' ') goto missingarg;
	c = getsearchdate(&start, &end);
	if (c == EOF) goto baddate;
	if (!searchargs->after \|\| searchargs->after < start) {
	searchargs->after = start;
	}
	}
	else if (!strcmp(criteria.s, "smaller")) {
	if (c != ' ') goto missingarg;
	c = getword(imapd_in, &arg);
	size = 0;
	for (p = arg.s; p && isdigit((int) p); p++) {
	size = size * 10 + *p - '0';
	}
	if (!arg.s \|\| *p) goto badnumber;
	if (size == 0) size = 1;
	if (!searchargs->smaller \|\| size < searchargs->smaller)
	searchargs->smaller = size;
	}
	else if (!strcmp(criteria.s, "subject")) {
	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c == EOF) goto missingarg;
	str = charset_convert(arg.s, *charset, NULL, 0);
	if (strchr(str, EMPTY)) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	}
	else {
	appendstrlistpat(&searchargs->subject, str);
	}
	}
	else goto badcri;
	break;

	case 't':
	if (!strcmp(criteria.s, "to")) {
	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c == EOF) goto missingarg;
	str = charset_convert(arg.s, *charset, NULL, 0);
	if (strchr(str, EMPTY)) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	}
	else {
	appendstrlistpat(&searchargs->to, str);
	}
	}
	else if (!strcmp(criteria.s, "text")) {
	if (c != ' ') goto missingarg;
	c = getastring(imapd_in, imapd_out, &arg);
	if (c == EOF) goto missingarg;
	str = charset_convert(arg.s, *charset, NULL, 0);
	if (strchr(str, EMPTY)) {
	/* Force failure */
	searchargs->flags = (SEARCH_RECENT_SET\|SEARCH_RECENT_UNSET);
	}
	else {
	appendstrlistpat(&searchargs->text, str);
	}
	}
	else goto badcri;
	break;

	case 'u':
	if (!strcmp(criteria.s, "uid")) {
	if (c != ' ') goto missingarg;
	c = getword(imapd_in, &arg);
	if (!imparse_issequence(arg.s)) goto badcri;
	appendstrlist(&searchargs->uidsequence, arg.s);
	}
	else if (!strcmp(criteria.s, "unseen")) {
	searchargs->flags \|= SEARCH_SEEN_UNSET;
	}
	else if (!strcmp(criteria.s, "unanswered")) {
	searchargs->system_flags_unset \|= FLAG_ANSWERED;
	}
	else if (!strcmp(criteria.s, "undeleted")) {
	searchargs->system_flags_unset \|= FLAG_DELETED;
	}
	else if (!strcmp(criteria.s, "undraft")) {
	searchargs->system_flags_unset \|= FLAG_DRAFT;
	}
	else if (!strcmp(criteria.s, "unflagged")) {
	searchargs->system_flags_unset \|= FLAG_FLAGGED;
	}
	else if (!strcmp(criteria.s, "unkeyword")) {
	if (c != ' ') goto missingarg;
	c = getword(imapd_in, &arg);
	if (!imparse_isatom(arg.s)) goto badflag;
	lcase(arg.s);
	for (flag=0; flag < MAX_USER_FLAGS; flag++) {
	if (imapd_mailbox->flagname[flag] &&
	!strcasecmp(imapd_mailbox->flagname[flag], arg.s)) break;
	}
	if (flag != MAX_USER_FLAGS) {
	searchargs->user_flags_unset[flag/32] \|= 1<<(flag&31);
	}
	}
	else goto badcri;
	break;

	default:
	badcri:
	prot_printf(imapd_out, "%s BAD Invalid Search criteria\r\n", tag);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;
	}

	return c;

	missingarg:
	prot_printf(imapd_out, "%s BAD Missing required argument to Search %s\r\n",
	tag, criteria.s);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;

	badflag:
	prot_printf(imapd_out, "%s BAD Invalid flag name %s in Search command\r\n",
	tag, arg.s);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;

	baddate:
	prot_printf(imapd_out, "%s BAD Invalid date in Search command\r\n", tag);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;

	badnumber:
	prot_printf(imapd_out, "%s BAD Invalid number in Search command\r\n", tag);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;
	}

	/*
	* Parse a "date", for SEARCH criteria
	* The time_t's pointed to by 'start' and 'end' are set to the
	* times of the start and end of the parsed date.
	*/
	int getsearchdate(start, end)
	time_t start, end;
	{
	int c;
	struct tm tm;
	int quoted = 0;
	char month[4];

	memset(&tm, 0, sizeof tm);

	c = prot_getc(imapd_in);
	if (c == '\"') {
	quoted++;
	c = prot_getc(imapd_in);
	}

	/* Day of month */
	if (!isdigit(c)) goto baddate;
	tm.tm_mday = c - '0';
	c = prot_getc(imapd_in);
	if (isdigit(c)) {
	tm.tm_mday = tm.tm_mday * 10 + c - '0';
	c = prot_getc(imapd_in);
	}

	if (c != '-') goto baddate;
	c = prot_getc(imapd_in);

	/* Month name */
	if (!isalpha(c)) goto baddate;
	month[0] = c;
	c = prot_getc(imapd_in);
	if (!isalpha(c)) goto baddate;
	month[1] = c;
	c = prot_getc(imapd_in);
	if (!isalpha(c)) goto baddate;
	month[2] = c;
	c = prot_getc(imapd_in);
	month[3] = '\0';
	lcase(month);

	for (tm.tm_mon = 0; tm.tm_mon < 12; tm.tm_mon++) {
	if (!strcmp(month, monthname[tm.tm_mon])) break;
	}
	if (tm.tm_mon == 12) goto baddate;

	if (c != '-') goto baddate;
	c = prot_getc(imapd_in);

	/* Year */
	if (!isdigit(c)) goto baddate;
	tm.tm_year = c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_year = tm.tm_year * 10 + c - '0';
	c = prot_getc(imapd_in);
	if (isdigit(c)) {
	if (tm.tm_year < 19) goto baddate;
	tm.tm_year -= 19;
	tm.tm_year = tm.tm_year * 10 + c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_year = tm.tm_year * 10 + c - '0';
	c = prot_getc(imapd_in);
	}

	if (quoted) {
	if (c != '\"') goto baddate;
	c = prot_getc(imapd_in);
	}

	tm.tm_isdst = -1;
	*start = mktime(&tm);

	tm.tm_sec = tm.tm_min = 59;
	tm.tm_hour = 23;
	tm.tm_isdst = -1;
	*end = mktime(&tm);

	return c;

	baddate:
	prot_ungetc(c, imapd_in);
	return EOF;
	}

	#define SORTGROWSIZE 10

	/*
	* Parse sort criteria
	*/
	int getsortcriteria(char tag, struct sortcrit *sortcrit)
	{
	int c;
	static struct buf criteria;
	int nsort, n;

	*sortcrit = NULL;

	c = prot_getc(imapd_in);
	if (c != '(') goto missingcrit;

	c = getword(imapd_in, &criteria);
	if (criteria.s[0] == '\0') goto missingcrit;

	nsort = 0;
	n = 0;
	for (;;) {
	if (n >= nsort - 1) { /* leave room for implicit criterion */
	/* (Re)allocate an array for sort criteria */
	nsort += SORTGROWSIZE;
	*sortcrit =
	(struct sortcrit ) xrealloc(sortcrit,
	nsort * sizeof(struct sortcrit));
	/* Zero out the newly added sortcrit */
	memset((sortcrit)+n, 0, SORTGROWSIZE sizeof(struct sortcrit));
	}

	lcase(criteria.s);
	if (!strcmp(criteria.s, "reverse")) {
	(*sortcrit)[n].flags \|= SORT_REVERSE;
	goto nextcrit;
	}
	else if (!strcmp(criteria.s, "arrival"))
	(*sortcrit)[n].key = SORT_ARRIVAL;
	else if (!strcmp(criteria.s, "cc"))
	(*sortcrit)[n].key = SORT_CC;
	else if (!strcmp(criteria.s, "date"))
	(*sortcrit)[n].key = SORT_DATE;
	else if (!strcmp(criteria.s, "from"))
	(*sortcrit)[n].key = SORT_FROM;
	else if (!strcmp(criteria.s, "size"))
	(*sortcrit)[n].key = SORT_SIZE;
	else if (!strcmp(criteria.s, "subject"))
	(*sortcrit)[n].key = SORT_SUBJECT;
	else if (!strcmp(criteria.s, "to"))
	(*sortcrit)[n].key = SORT_TO;
	#if 0
	else if (!strcmp(criteria.s, "annotation")) {
	(*sortcrit)[n].key = SORT_ANNOTATION;
	if (c != ' ') goto missingarg;
	c = getstring(imapd_in, &arg);
	if (c != ' ') goto missingarg;
	(*sortcrit)[n].args.annot.entry = strdup(arg.s);
	c = getstring(imapd_in, &arg);
	if (c == EOF) goto missingarg;
	(*sortcrit)[n].args.annot.attrib = strdup(arg.s);
	}
	#endif
	else {
	prot_printf(imapd_out, "%s BAD Invalid Sort criterion %s\r\n",
	tag, criteria.s);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;
	}

	n++;

	nextcrit:
	if (c == ' ') c = getword(imapd_in, &criteria);
	else break;
	}

	if ((sortcrit)[n].flags & SORT_REVERSE && !(sortcrit)[n].key) {
	prot_printf(imapd_out,
	"%s BAD Missing Sort criterion to reverse\r\n", tag);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;
	}

	if (c != ')') {
	prot_printf(imapd_out,
	"%s BAD Missing close parenthesis in Sort\r\n", tag);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;
	}

	/* Terminate the list with the implicit sort criterion */
	(*sortcrit)[n++].key = SORT_SEQUENCE;

	c = prot_getc(imapd_in);

	return c;

	missingcrit:
	prot_printf(imapd_out, "%s BAD Missing Sort criteria\r\n", tag);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;
	#if 0 /* For annotations stuff above */
	missingarg:
	prot_printf(imapd_out, "%s BAD Missing argument to Sort criterion %s\r\n",
	tag, criteria.s);
	if (c != EOF) prot_ungetc(c, imapd_in);
	return EOF;
	#endif
	}

	#ifdef ENABLE_LISTEXT
	/*
	* Parse LIST options.
	* The command has been parsed up to and including the opening '('.
	*/
	int getlistopts(char tag, int listopts)
	{
	int c;
	static struct buf arg;

	*listopts = LIST_EXT;

	for (;;) {
	c = getword(imapd_in, &arg);
	if (!arg.s[0]) break;

	lcase(arg.s);
	if (!strcmp(arg.s, "subscribed")) {
	*listopts \|= LIST_SUBSCRIBED;
	}
	else if (!strcmp(arg.s, "children")) {
	*listopts \|= LIST_CHILDREN;
	}
	#if 0
	else if (!strcmp(arg.s, "remote")) {
	*listopts \|= LIST_REMOTE;
	}
	#endif
	else {
	prot_printf(imapd_out, "%s BAD Invalid List option %s\r\n",
	tag, arg.s);
	return EOF;
	}

	if (c != ' ') break;
	}

	if (c != ')') {
	prot_printf(imapd_out,
	"%s BAD Missing close parenthesis in List\r\n", tag);
	return EOF;
	}

	c = prot_getc(imapd_in);

	return c;
	}
	#endif /* ENABLE_LISTEXT */

	/*
	* Parse a date_time, for the APPEND command
	*/
	int getdatetime(date)
	time_t *date;
	{
	int c;
	struct tm tm;
	int old_format = 0;
	char month[4], zone[4], *p;
	int zone_off;

	memset(&tm, 0, sizeof tm);

	c = prot_getc(imapd_in);
	if (c != '\"') goto baddate;

	/* Day of month */
	c = prot_getc(imapd_in);
	if (c == ' ') c = '0';
	if (!isdigit(c)) goto baddate;
	tm.tm_mday = c - '0';
	c = prot_getc(imapd_in);
	if (isdigit(c)) {
	tm.tm_mday = tm.tm_mday * 10 + c - '0';
	c = prot_getc(imapd_in);
	}

	if (c != '-') goto baddate;
	c = prot_getc(imapd_in);

	/* Month name */
	if (!isalpha(c)) goto baddate;
	month[0] = c;
	c = prot_getc(imapd_in);
	if (!isalpha(c)) goto baddate;
	month[1] = c;
	c = prot_getc(imapd_in);
	if (!isalpha(c)) goto baddate;
	month[2] = c;
	c = prot_getc(imapd_in);
	month[3] = '\0';
	lcase(month);

	for (tm.tm_mon = 0; tm.tm_mon < 12; tm.tm_mon++) {
	if (!strcmp(month, monthname[tm.tm_mon])) break;
	}
	if (tm.tm_mon == 12) goto baddate;

	if (c != '-') goto baddate;
	c = prot_getc(imapd_in);

	/* Year */
	if (!isdigit(c)) goto baddate;
	tm.tm_year = c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_year = tm.tm_year * 10 + c - '0';
	c = prot_getc(imapd_in);
	if (isdigit(c)) {
	if (tm.tm_year < 19) goto baddate;
	tm.tm_year -= 19;
	tm.tm_year = tm.tm_year * 10 + c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_year = tm.tm_year * 10 + c - '0';
	c = prot_getc(imapd_in);
	}
	else old_format++;

	/* Hour */
	if (c != ' ') goto baddate;
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_hour = c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_hour = tm.tm_hour * 10 + c - '0';
	c = prot_getc(imapd_in);
	if (tm.tm_hour > 23) goto baddate;

	/* Minute */
	if (c != ':') goto baddate;
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_min = c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_min = tm.tm_min * 10 + c - '0';
	c = prot_getc(imapd_in);
	if (tm.tm_min > 59) goto baddate;

	/* Second */
	if (c != ':') goto baddate;
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_sec = c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	tm.tm_sec = tm.tm_sec * 10 + c - '0';
	c = prot_getc(imapd_in);
	if (tm.tm_min > 60) goto baddate;

	/* Time zone */
	if (old_format) {
	if (c != '-') goto baddate;
	c = prot_getc(imapd_in);

	if (!isalpha(c)) goto baddate;
	zone[0] = c;
	c = prot_getc(imapd_in);

	if (c == '\"') {
	/* Military (single-char) zones */
	zone[1] = '\0';
	lcase(zone);
	if (zone[0] <= 'm') {
	zone_off = (zone[0] - 'a' + 1)*60;
	}
	else if (zone[0] < 'z') {
	zone_off = ('m' - zone[0])*60;
	}
	else zone_off = 0;
	}
	else {
	/* UT (universal time) */
	zone[1] = c;
	c = prot_getc(imapd_in);
	if (c == '\"') {
	zone[2] = '\0';
	lcase(zone);
	if (!strcmp(zone, "ut")) goto baddate;
	zone_off = 0;
	}
	else {
	/* 3-char time zone */
	zone[2] = c;
	c = prot_getc(imapd_in);
	if (c != '\"') goto baddate;
	zone[3] = '\0';
	lcase(zone);
	p = strchr("aecmpyhb", zone[0]);
	if (c != '\"' \|\| zone[2] != 't' \|\| !p) goto baddate;
	zone_off = (strlen(p) - 12)*60;
	if (zone[1] == 'd') zone_off -= 60;
	else if (zone[1] != 's') goto baddate;
	}
	}
	}
	else {
	if (c != ' ') goto baddate;
	c = prot_getc(imapd_in);

	if (c != '+' && c != '-') goto baddate;
	zone[0] = c;

	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	zone_off = c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	zone_off = zone_off * 10 + c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	zone_off = zone_off * 6 + c - '0';
	c = prot_getc(imapd_in);
	if (!isdigit(c)) goto baddate;
	zone_off = zone_off * 10 + c - '0';

	if (zone[0] == '-') zone_off = -zone_off;

	c = prot_getc(imapd_in);
	if (c != '\"') goto baddate;

	}

	c = prot_getc(imapd_in);

	tm.tm_isdst = -1;
	date = mkgmtime(&tm) - zone_off60;

	return c;

	baddate:
	prot_ungetc(c, imapd_in);
	return EOF;
	}

	/*
	* Print 's' as a quoted-string or literal (but not an atom)
	*/
	void
	printstring(s)
	const char *s;
	{
	const char *p;
	int len = 0;

	/* Look for any non-QCHAR characters */
	for (p = s; *p && len < 1024; p++) {
	len++;
	if (p & 0x80 \|\| p == '\r' \|\| *p == '\n'
	\|\| p == '\"' \|\| p == '%' \|\| *p == '\\') break;
	}

	/* if it's too long, literal it */
	if (*p \|\| len >= 1024) {
	prot_printf(imapd_out, "{%u}\r\n%s", strlen(s), s);
	} else {
	prot_printf(imapd_out, "\"%s\"", s);
	}
	}

	/*
	* Print 's' as an atom, quoted-string, or literal
	*/
	void
	printastring(s)
	const char *s;
	{
	const char *p;
	int len = 0;

	if (imparse_isatom(s)) {
	prot_printf(imapd_out, "%s", s);
	return;
	}

	/* Look for any non-QCHAR characters */
	for (p = s; *p && len < 1024; p++) {
	len++;
	if (p & 0x80 \|\| p == '\r' \|\| *p == '\n'
	\|\| p == '\"' \|\| p == '%' \|\| *p == '\\') break;
	}

	/* if it's too long, literal it */
	if (*p \|\| len >= 1024) {
	prot_printf(imapd_out, "{%u}\r\n%s", strlen(s), s);
	} else {
	prot_printf(imapd_out, "\"%s\"", s);
	}
	}

	/*
	* Append 'section', 'fields', 'trail' to the fieldlist 'l'.
	*/
	void
	appendfieldlist(l, section, fields, trail)
	struct fieldlist **l;
	char *section;
	struct strlist *fields;
	char *trail;
	{
	struct fieldlist **tail = l;

	while (tail) tail = &(tail)->next;

	tail = (struct fieldlist )xmalloc(sizeof(struct fieldlist));
	(*tail)->section = xstrdup(section);
	(*tail)->fields = fields;
	(*tail)->trail = xstrdup(trail);
	(*tail)->next = 0;
	}

	/*
	* Append 's' to the strlist 'l'.
	*/
	void
	appendstrlist(l, s)
	struct strlist **l;
	char *s;
	{
	struct strlist **tail = l;

	while (tail) tail = &(tail)->next;

	tail = (struct strlist )xmalloc(sizeof(struct strlist));
	(*tail)->s = xstrdup(s);
	(*tail)->p = 0;
	(*tail)->next = 0;
	}

	/*
	* Append 's' to the strlist 'l', compiling it as a pattern.
	* Caller must pass in memory that is freed when the strlist is freed.
	*/
	void
	appendstrlistpat(l, s)
	struct strlist **l;
	char *s;
	{
	struct strlist **tail = l;

	while (tail) tail = &(tail)->next;

	tail = (struct strlist )xmalloc(sizeof(struct strlist));
	(*tail)->s = s;
	(*tail)->p = charset_compilepat(s);
	(*tail)->next = 0;
	}

	/*
	* Free the fieldlist 'l'
	*/
	void
	freefieldlist(l)
	struct fieldlist *l;
	{
	struct fieldlist *n;

	while (l) {
	n = l->next;
	free(l->section);
	freestrlist(l->fields);
	free(l->trail);
	free((char *)l);
	l = n;
	}
	}

	/*
	* Free the strlist 'l'
	*/
	void
	freestrlist(l)
	struct strlist *l;
	{
	struct strlist *n;

	while (l) {
	n = l->next;
	free(l->s);
	if (l->p) charset_freepat(l->p);
	free((char *)l);
	l = n;
	}
	}

	/*
	* Append the searchargs 's1' and 's2' to the sublist of 's'
	*/
	void
	appendsearchargs(s, s1, s2)
	struct searchargs s, s1, *s2;
	{
	struct searchsub **tail = &s->sublist;

	while (tail) tail = &(tail)->next;

	tail = (struct searchsub )xmalloc(sizeof(struct searchsub));
	(*tail)->sub1 = s1;
	(*tail)->sub2 = s2;
	(*tail)->next = 0;
	}


	/*
	* Free the searchargs 's'
	*/
	void
	freesearchargs(s)
	struct searchargs *s;
	{
	struct searchsub sub, n;

	if (!s) return;

	freestrlist(s->sequence);
	freestrlist(s->uidsequence);
	freestrlist(s->from);
	freestrlist(s->to);
	freestrlist(s->cc);
	freestrlist(s->bcc);
	freestrlist(s->subject);
	freestrlist(s->body);
	freestrlist(s->text);
	freestrlist(s->header_name);
	freestrlist(s->header);

	for (sub = s->sublist; sub; sub = n) {
	n = sub->next;
	freesearchargs(sub->sub1);
	freesearchargs(sub->sub2);
	free(sub);
	}
	free(s);
	}

	/*
	* Free an array of sortcrit
	*/
	static void freesortcrit(struct sortcrit *s)
	{
	int i = 0;

	if (!s) return;
	do {
	switch (s[i].key) {
	case SORT_ANNOTATION:
	free(s[i].args.annot.entry);
	free(s[i].args.annot.attrib);
	break;
	}
	i++;
	} while (s[i].key != SORT_SEQUENCE);
	free(s);
	}

	/*
	* Issue a MAILBOX untagged response
	*/
	static int mailboxdata(char *name,
	int matchlen __attribute__((unused)),
	int maycreate __attribute__((unused)),
	void *rock __attribute__((unused)))
	{
	char mboxname[MAX_MAILBOX_PATH+1];

	(*imapd_namespace.mboxname_toexternal)(&imapd_namespace, name,
	imapd_userid, mboxname);
	prot_printf(imapd_out, "* MAILBOX %s\r\n", mboxname);
	return 0;
	}

	/*
	* Issue a LIST or LSUB untagged response
	*/
	static void mstringdata(char cmd, char name, int matchlen, int maycreate,
	int listopts)
	{
	static char lastname[MAX_MAILBOX_PATH];
	static int lastnamedelayed = 0;
	static int lastnamenoinferiors = 0;
	static int nonexistent = 0;
	static int sawuser = 0;
	int lastnamehassub = 0;
	int c;
	char mboxname[MAX_MAILBOX_PATH+1];

	/* We have to reset the sawuser flag before each list command.
	* Handle it as a dirty hack.
	*/
	if (cmd == NULL) {
	sawuser = 0;
	mstringdatacalls = 0;
	return;
	}
	mstringdatacalls++;

	if (lastnamedelayed) {
	/* Check if lastname has children */
	if (name && strncmp(lastname, name, strlen(lastname)) == 0 &&
	name[strlen(lastname)] == '.') {
	lastnamehassub = 1;
	}
	prot_printf(imapd_out, "* %s (", cmd);
	if (nonexistent) {
	prot_printf(imapd_out, "\\NonExistent");
	}
	if (lastnamenoinferiors) {
	prot_printf(imapd_out, "%s\\Noinferiors", nonexistent ? " " : "");
	}
	else if ((listopts & LIST_CHILDREN) &&
	/* we can't determine \HasNoChildren for subscriptions */
	(lastnamehassub \|\|
	!(listopts & (LIST_LSUB \| LIST_SUBSCRIBED)))) {
	prot_printf(imapd_out, "%s%s", nonexistent ? " " : "",
	lastnamehassub ? "\\HasChildren" : "\\HasNoChildren");
	}
	prot_printf(imapd_out, ") \"%c\" ", imapd_namespace.hier_sep);

	(*imapd_namespace.mboxname_toexternal)(&imapd_namespace, lastname,
	imapd_userid, mboxname);
	printstring(mboxname);
	prot_printf(imapd_out, "\r\n");
	lastnamedelayed = lastnamenoinferiors = 0;
	}

	/* Special-case to flush any final state */
	if (!name) {
	lastname[0] = '\0';
	return;
	}

	/* Suppress any output of a partial match */
	if (name[matchlen] && strncmp(lastname, name, matchlen) == 0) {
	return;
	}

	/*
	* We can get a partial match for "user" multiple times with
	* other matches inbetween. Handle it as a special case
	*/
	if (matchlen == 4 && strncasecmp(name, "user", 4) == 0) {
	if (sawuser) return;
	sawuser = 1;
	}

	strcpy(lastname, name);
	lastname[matchlen] = '\0';
	nonexistent = 0;

	/* See if subscribed mailbox exists */
	if ((listopts & LIST_SUBSCRIBED) &&
	config_getswitch("allowallsubscribe", 0)) {
	/* convert "INBOX" to "user.<userid>" */
	if (!strncasecmp(lastname, "inbox", 5))
	sprintf(mboxname, "user.%s%s", imapd_userid, lastname+5);
	else
	strcpy(mboxname, lastname);
	nonexistent = mboxlist_lookup(mboxname, NULL, NULL, NULL);
	}

	if (!name[matchlen]) {
	lastnamedelayed = 1;
	if (!maycreate) lastnamenoinferiors = 1;
	return;
	}

	c = name[matchlen];
	if (c) name[matchlen] = '\0';
	prot_printf(imapd_out, "* %s (", cmd);
	if (c) {
	/* Handle namespace prefix as a special case */
	if (!strcmp(name, "user") \|\|
	!strcmp(name, imapd_namespace.prefix[NAMESPACE_SHARED])) {
	prot_printf(imapd_out, "\\Noselect");
	if (listopts & LIST_EXT)
	prot_printf(imapd_out, " \\PlaceHolder");
	}
	else {
	if (nonexistent)
	prot_printf(imapd_out, "\\NonExistent");
	/* LISTEXT uses \PlaceHolder instead of \Noselect */
	if (listopts & LIST_EXT)
	prot_printf(imapd_out, "%s\\PlaceHolder", nonexistent ? " " : "");
	else
	prot_printf(imapd_out, "%s\\Noselect", nonexistent ? " " : "");
	}
	if (listopts & LIST_CHILDREN)
	prot_printf(imapd_out, " \\HasChildren");
	}
	prot_printf(imapd_out, ") \"%c\" ", imapd_namespace.hier_sep);

	(*imapd_namespace.mboxname_toexternal)(&imapd_namespace, name,
	imapd_userid, mboxname);
	printstring(mboxname);
	prot_printf(imapd_out, "\r\n");
	if (c) name[matchlen] = c;
	return;
	}

	/*
	* Issue a LIST untagged response
	*/
	static int listdata(char name, int matchlen, int maycreate, void rock)
	{
	mstringdata("LIST", name, matchlen, maycreate, ((int) rock));
	return 0;
	}

	/*
	* Issue a LSUB untagged response
	*/
	static int lsubdata(char name, int matchlen, int maycreate, void rock)
	{
	mstringdata("LSUB", name, matchlen, maycreate, ((int) rock));
	return 0;
	}

	/* Reset the given sasl_conn_t to a sane state */
	static int reset_saslconn(sasl_conn_t **conn)
	{
	int ret;
	sasl_security_properties_t *secprops = NULL;

	sasl_dispose(conn);
	/* do initialization typical of service_main */
	ret = sasl_server_new("imap", config_servername,
	NULL, NULL, NULL,
	NULL, 0, conn);
	if(ret != SASL_OK) return ret;

	if(saslprops.ipremoteport)
	ret = sasl_setprop(*conn, SASL_IPREMOTEPORT,
	saslprops.ipremoteport);
	if(ret != SASL_OK) return ret;

	if(saslprops.iplocalport)
	ret = sasl_setprop(*conn, SASL_IPLOCALPORT,
	saslprops.iplocalport);
	if(ret != SASL_OK) return ret;

	secprops = mysasl_secprops(SASL_SEC_NOPLAINTEXT);
	ret = sasl_setprop(*conn, SASL_SEC_PROPS, secprops);
	if(ret != SASL_OK) return ret;
	/* end of service_main initialization excepting SSF */

	/* If we have TLS/SSL info, set it */
	if(saslprops.ssf) {
	ret = sasl_setprop(*conn, SASL_SSF_EXTERNAL, &saslprops.ssf);
	} else {
	ret = sasl_setprop(*conn, SASL_SSF_EXTERNAL, &extprops_ssf);
	}
	if(ret != SASL_OK) return ret;

	if(saslprops.authid) {
	ret = sasl_setprop(*conn, SASL_AUTH_EXTERNAL, saslprops.authid);
	if(ret != SASL_OK) return ret;
	}
	/* End TLS/SSL Info */

	return SASL_OK;
	}

	diff --git a/imap/imapparse.c b/imap/imapparse.c
	index 553f8d86a..27bda7b2b 100644
	--- a/imap/imapparse.c
	+++ b/imap/imapparse.c
	@@ -1,225 +1,226 @@
	#include <config.h>

	#include <ctype.h>
	+#include <string.h>

	#include "prot.h"
	#include "xmalloc.h"
	#include "imapconf.h"

	void freebuf(struct buf *buf)
	{
	if (buf->s) {
	free(buf->s);
	buf->s = NULL;
	}
	buf->alloc = 0;
	}

	/*
	* Parse a word
	* (token not containing whitespace, parens, or double quotes)
	*/
	#define BUFGROWSIZE 100
	int getword(struct protstream in, struct buf buf)
	{
	int c;
	int len = 0;

	if (buf->alloc == 0) {
	buf->alloc = BUFGROWSIZE;
	buf->s = xmalloc(buf->alloc+1);
	}

	for (;;) {
	c = prot_getc(in);
	if (c == EOF \|\| isspace(c) \|\| c == '(' \|\| c == ')' \|\| c == '\"') {
	buf->s[len] = '\0';
	return c;
	}
	if (len == buf->alloc) {
	buf->alloc += BUFGROWSIZE;
	buf->s = xrealloc(buf->s, buf->alloc+1);
	}
	buf->s[len++] = c;
	}
	}

	/*
	* Parse an xstring
	* (astring, nstring or string based on type)
	*/
	int getxstring(struct protstream pin, struct protstream pout,
	struct buf *buf, int type)
	{
	int c;
	int i, len = 0;
	int sawdigit = 0;
	int isnowait;

	if (buf->alloc == 0) {
	buf->alloc = BUFGROWSIZE;
	buf->s = xmalloc(buf->alloc+1);
	}

	c = prot_getc(pin);
	switch (c) {
	case EOF:
	case ' ':
	case '(':
	case ')':
	case '\r':
	case '\n':
	/* Invalid starting character */
	buf->s[0] = '\0';
	if (c != EOF) prot_ungetc(c, pin);
	return EOF;

	default:
	switch (type) {
	case IMAP_ASTRING: /* atom, quoted-string or literal */
	/*
	* Atom -- server is liberal in accepting specials other
	* than whitespace, parens, or double quotes
	*/
	for (;;) {
	if (c == EOF \|\| isspace(c) \|\| c == '(' \|\|
	c == ')' \|\| c == '\"') {
	buf->s[len] = '\0';
	return c;
	}
	if (len == buf->alloc) {
	buf->alloc += BUFGROWSIZE;
	buf->s = xrealloc(buf->s, buf->alloc+1);
	}
	buf->s[len++] = c;
	c = prot_getc(pin);
	}
	break;

	case IMAP_NSTRING: /* "NIL", quoted-string or literal */
	/*
	* Look for "NIL"
	*/
	if (c == 'N') {
	prot_ungetc(c, pin);
	c = getword(pin, buf);
	if (!strcmp(buf->s, "NIL"))
	return c;
	}
	if (c != EOF) prot_ungetc(c, pin);
	return EOF;
	break;

	case IMAP_STRING: /* quoted-string or literal */
	/*
	* Nothing to do here - fall through.
	*/
	break;
	}

	case '\"':
	/*
	* Quoted-string. Server is liberal in accepting qspecials
	* other than double-quote, CR, and LF.
	*/
	for (;;) {
	c = prot_getc(pin);
	if (c == '\\') {
	c = prot_getc(pin);
	}
	else if (c == '\"') {
	buf->s[len] = '\0';
	return prot_getc(pin);
	}
	else if (c == EOF \|\| c == '\r' \|\| c == '\n') {
	buf->s[len] = '\0';
	if (c != EOF) prot_ungetc(c, pin);
	return EOF;
	}
	if (len == buf->alloc) {
	buf->alloc += BUFGROWSIZE;
	buf->s = xrealloc(buf->s, buf->alloc+1);
	}
	buf->s[len++] = c;
	}
	case '{':
	/* Literal */
	isnowait = 0;
	buf->s[0] = '\0';
	while ((c = prot_getc(pin)) != EOF && isdigit(c)) {
	sawdigit = 1;
	len = len*10 + c - '0';
	}
	if (c == '+') {
	isnowait++;
	c = prot_getc(pin);
	}
	if (!sawdigit \|\| c != '}') {
	if (c != EOF) prot_ungetc(c, pin);
	return EOF;
	}
	c = prot_getc(pin);
	if (c != '\r') {
	if (c != EOF) prot_ungetc(c, pin);
	return EOF;
	}
	c = prot_getc(pin);
	if (c != '\n') {
	if (c != EOF) prot_ungetc(c, pin);
	return EOF;
	}
	if (len >= buf->alloc) {
	buf->alloc = len+1;
	buf->s = xrealloc(buf->s, buf->alloc+1);
	}
	if (!isnowait) {
	prot_printf(pout, "+ go ahead\r\n");
	prot_flush(pout);
	}
	for (i = 0; i < len; i++) {
	c = prot_getc(pin);
	if (c == EOF) {
	buf->s[len] = '\0';
	return EOF;
	}
	buf->s[i] = c;
	}
	buf->s[len] = '\0';
	if (strlen(buf->s) != len) return EOF; /* Disallow imbedded NUL */
	return prot_getc(pin);
	}
	}

	/*
	* Eat characters up to and including the next newline
	* Also look for and eat non-synchronizing literals.
	*/
	void eatline(struct protstream *pin, int c)
	{
	int state = 0;
	char *statediagram = " {+}\r";
	int size = -1;

	for (;;) {
	if (c == '\n') return;
	if (c == statediagram[state+1]) {
	state++;
	if (state == 1) size = 0;
	else if (c == '\r') {
	/* Got a non-synchronizing literal */
	c = prot_getc(pin);/* Eat newline */
	while (size--) {
	c = prot_getc(pin); /* Eat contents */
	}
	state = 0; /* Go back to scanning for eol */
	}
	}
	else if (state == 1 && isdigit(c)) {
	size = size * 10 + c - '0';
	}
	else state = 0;

	c = prot_getc(pin);
	if (c == EOF) return;
	}
	}
	diff --git a/imap/index.c b/imap/index.c
	index 90b8b7912..aa26c12f1 100644
	--- a/imap/index.c
	+++ b/imap/index.c
	@@ -1,4541 +1,4541 @@
	/* index.c -- Routines for dealing with the index file in the imapd
	*
	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	*
	*/
	/*
	- * $Id: index.c,v 1.171 2002/02/13 21:53:35 rjs3 Exp $
	+ * $Id: index.c,v 1.172 2002/02/19 18:50:12 ken3 Exp $
	*/
	#include <config.h>

	#include <stdio.h>
	#include <string.h>
	#include <time.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <fcntl.h>
	#include <netinet/in.h>
	#include <syslog.h>
	#include <com_err.h>
	#include <errno.h>
	#include <ctype.h>
	#include <time.h>

	#include "acl.h"
	#include "util.h"
	#include "map.h"
	#include "assert.h"
	#include "exitcodes.h"
	#include "gmtoff.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "imapd.h"
	#include "append.h"
	#include "charset.h"
	#include "xmalloc.h"
	#include "seen.h"
	#include "lsort.h"
	#include "message.h"
	#include "parseaddr.h"
	#include "hash.h"
	#include "stristr.h"
	#include "search_engines.h"

	extern int errno;

	extern void printastring (const char *s);

	/* The index and cache files, mapped into memory */
	static const char *index_base;
	static unsigned long index_len;
	static const char *cache_base;
	static unsigned long cache_len;
	static unsigned long cache_end;

	/* Attributes of memory-mapped index file */
	static long index_ino;
	static unsigned long start_offset;
	static unsigned long record_size;

	static unsigned recentuid; /* UID of last non-\Recent message */
	static unsigned lastnotrecent; /* Msgno of last non-\Recent message */

	static time_t flagreport; / Array for each msgno of last_updated when
	* FLAGS data reported to client.
	* Zero if FLAGS data never reported */
	static char seenflag; / Array for each msgno, nonzero if \Seen */
	static time_t seen_last_change; /* Last mod time of \Seen state change */
	static int flagalloced = -1; /* Allocated size of above two arrays */
	static int examining; /* Nonzero if opened with EXAMINE command */
	static int keepingseen; /* Nonzero if /Seen is meaningful */
	static unsigned allseen; /* Last UID if all msgs /Seen last checkpoint */
	struct seen seendb; / Seen state database object */
	static char seenuids; / Sequence of UID's from last seen checkpoint */

	/* Access macros for the memory-mapped index file data */
	#define INDEC_OFFSET(msgno) (index_base+start_offset+(((msgno)-1)*record_size))
	#define UID(msgno) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_UID)))
	#define INTERNALDATE(msgno) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_INTERNALDATE)))
	#define SENTDATE(msgno) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_SENTDATE)))
	#define SIZE(msgno) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_SIZE)))
	#define HEADER_SIZE(msgno) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_HEADER_SIZE)))
	#define CONTENT_OFFSET(msgno) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_CONTENT_OFFSET)))
	#define CACHE_OFFSET(msgno) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_CACHE_OFFSET)))
	#define LAST_UPDATED(msgno) ((time_t)ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_LAST_UPDATED))))
	#define SYSTEM_FLAGS(msgno) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_SYSTEM_FLAGS)))
	#define USER_FLAGS(msgno,i) ntohl(((bit32 )(INDEC_OFFSET(msgno)+OFFSET_USER_FLAGS+((i)*4))))

	/* Access assistance macros for memory-mapped cache file data */
	#define CACHE_ITEM_BIT32(ptr) (ntohl(((bit32 )(ptr))))
	#define CACHE_ITEM_LEN(ptr) CACHE_ITEM_BIT32(ptr)
	#define CACHE_ITEM_NEXT(ptr) ((ptr)+4+((3+CACHE_ITEM_LEN(ptr))&~3))

	/* Cached envelope token positions */
	enum {
	ENV_DATE = 0,
	ENV_SUBJECT,
	ENV_FROM,
	ENV_SENDER,
	ENV_REPLYTO,
	ENV_TO,
	ENV_CC,
	ENV_BCC,
	ENV_INREPLYTO,
	ENV_MSGID
	};
	#define NUMENVTOKENS (10)

	/* Special "sort criteria" to load message-id and references/in-reply-to
	* into msgdata array for threaders that need them.
	*/
	#define LOAD_IDS 256

	struct copyargs {
	struct copymsg *copymsg;
	int nummsg;
	int msgalloc;
	};

	struct mapfile {
	const char *base;
	unsigned long size;
	};

	typedef struct msgdata {
	unsigned msgno; /* message number */
	char msgid; / message ID */
	char *ref; / array of references */
	int nref; /* number of references */
	time_t date; /* sent date & time of message
	from Date: header (adjusted by time zone) */
	char cc; / local-part of first "cc" address */
	char from; / local-part of first "from" address */
	char to; / local-part of first "to" address */
	char xsubj; / extracted subject text */
	unsigned xsubj_hash; /* hash of extracted subject text */
	int is_refwd; /* is message a reply or forward? */
	char *annot; / array of annotation attribute values
	(stored in order of sortcrit) */
	int nannot; /* number of annotation values */
	struct msgdata *next;
	} MsgData;

	typedef struct thread {
	MsgData msgdata; / message data */
	struct thread parent; / parent message */
	struct thread child; / first child message */
	struct thread next; / next sibling message */
	} Thread;

	struct rootset {
	Thread *root;
	unsigned nroot;
	};

	struct thread_algorithm {
	char *alg_name;
	void (threader)(unsigned msgno_list, int nmsg, int usinguid);
	};

	/* Forward declarations */
	typedef int index_sequenceproc_t(struct mailbox *mailbox, unsigned msgno,
	void *rock);

	static int index_forsequence(struct mailbox mailbox, char sequence,
	int usinguid,
	index_sequenceproc_t proc, void rock,
	int* fetchedsomething);
	static int index_insequence(int num, char *sequence, int usinguid);

	static void index_fetchmsg(const char *msg_base, unsigned long msg_size,
	int format, unsigned offset, unsigned size,
	unsigned start_octet, unsigned octet_count);
	static void index_fetchsection(const char *msg_base, unsigned long msg_size,
	int format, char *section,
	const char *cacheitem, unsigned size,
	unsigned start_octet, unsigned octet_count);
	static void index_fetchfsection(const char *msg_base,
	unsigned long msg_size,
	int format, struct fieldlist *fsection,
	const char *cacheitem);
	static char index_readheader(const char msg_base, unsigned long msg_size,
	int format, unsigned offset, unsigned size);
	static void index_pruneheader(char buf, struct strlist headers,
	struct strlist *headers_not);
	static void index_fetchheader(const char *msg_base, unsigned long msg_size,
	int format, unsigned size,
	struct strlist *headers,
	struct strlist *headers_not);
	static void index_fetchcacheheader(unsigned msgno, struct strlist *headers,
	char *trail);
	static void index_listflags(struct mailbox *mailbox);
	static void index_fetchflags(struct mailbox *mailbox, unsigned msgno,
	bit32 system_flags, bit32 *user_flags,
	time_t last_updated);
	static index_sequenceproc_t index_fetchreply;
	static index_sequenceproc_t index_storeseen;
	static index_sequenceproc_t index_storeflag;
	static int index_search_evaluate(struct mailbox *mailbox,
	struct searchargs *searchargs,
	unsigned msgno, struct mapfile *msgfile);
	static int index_searchmsg(char substr, comp_pat pat,
	struct mapfile *msgfile, int format,
	int skipheader, const char *cacheitem);
	static int index_searchheader(char name, char substr, comp_pat *pat,
	struct mapfile *msgfile, int format,
	int size);
	static int index_searchcacheheader(unsigned msgno, char name, char substr,
	comp_pat *pat);
	static index_sequenceproc_t index_copysetup;
	static int _index_search(unsigned *msgno_list, struct mailbox mailbox,
	struct searchargs *searchargs);

	static void parse_cached_envelope(char env, char tokens[]);
	static char find_msgid(char str, int *len);
	static char get_localpart_addr(const char header);
	static char index_extract_subject(const char subj, int *is_refwd);
	static char _index_extract_subject(char s, int *is_refwd);
	static void index_get_ids(MsgData *msgdata,
	char envtokens[], const char headers);
	static MsgData index_msgdata_load(unsigned msgno_list, int n,
	struct sortcrit *sortcrit);

	static void index_sort_getnext(MsgData node);
	static void index_sort_setnext(MsgData node, MsgData next);
	static int index_sort_compare(MsgData md1, MsgData md2,
	struct sortcrit *call_data);
	static void index_msgdata_free(MsgData *md);

	static void index_thread_getnext(Thread thread);
	static void index_thread_setnext(Thread thread, Thread next);
	static int index_thread_compare(Thread t1, Thread t2,
	struct sortcrit *call_data);
	static void index_thread_orderedsubj(unsigned *msgno_list, int nmsg,
	int usinguid);
	static void index_thread_sort(Thread root, struct sortcrit sortcrit);
	static void index_thread_print(Thread *threads, int usinguid);
	static void index_thread_ref(unsigned *msgno_list, int nmsg, int usinguid);

	/* NOTE: Make sure these are listed in CAPABILITY_STRING */
	static const struct thread_algorithm thread_algs[] = {
	{ "ORDEREDSUBJECT", index_thread_orderedsubj },
	{ "REFERENCES", index_thread_ref },
	{ NULL, NULL }
	};

	/*
	* A mailbox is about to be closed.
	*/
	void
	index_closemailbox(mailbox)
	struct mailbox *mailbox;
	{
	if (seendb) {
	index_checkseen(mailbox, 1, 0, imapd_exists);
	seen_close(seendb);
	seendb = 0;
	}
	if (index_len) {
	map_free(&index_base, &index_len);
	map_free(&cache_base, &cache_len);
	index_len = cache_end = 0;
	}
	}

	/*
	* A new mailbox has been selected, map it into memory and do the
	* initial CHECK.
	*/
	void
	index_newmailbox(mailbox, examine_mode)
	struct mailbox *mailbox;
	int examine_mode;
	{
	keepingseen = (mailbox->myrights & ACL_SEEN);
	examining = examine_mode;
	allseen = 0;
	recentuid = 0;
	index_listflags(mailbox);
	imapd_exists = -1;
	index_check(mailbox, 0, 1);
	}

	void index_operatemailbox(struct mailbox *mailbox)
	{
	keepingseen = 0;
	examining = 1;
	allseen = 0;
	recentuid = 0;
	index_base = mailbox->index_base;
	index_len = mailbox->index_len;
	cache_base = mailbox->cache_base;
	cache_len = mailbox->cache_len;
	cache_end = mailbox->cache_size;

	index_ino = mailbox->index_ino;
	start_offset = mailbox->start_offset;
	record_size = mailbox->record_size;
	imapd_exists = mailbox->exists;
	}

	#define SLOP 50

	/*
	* Check for and report updates
	*/
	void index_check(struct mailbox *mailbox, int usinguid, int checkseen)
	{
	struct stat sbuf;
	int newexists, oldexists, oldmsgno, msgno, nexpunge, i, r;
	struct index_record record;
	time_t last_read;
	bit32 user_flags[MAX_USER_FLAGS/32];

	oldexists = imapd_exists;

	/* Check for expunge */
	if (index_len) {
	if (stat(FNAME_INDEX+1, &sbuf) != 0) {
	if (errno == ENOENT) {
	/* Mailbox has been deleted */
	while (imapd_exists--) {
	prot_printf(imapd_out, "* 1 EXPUNGE\r\n");
	}
	mailbox->exists = 0;
	imapd_exists = -1;
	if (seendb) {
	seen_close(seendb);
	seendb = 0;
	}
	}
	}
	else if ((sbuf.st_ino != mailbox->index_ino) \|\|
	(index_ino != mailbox->index_ino)) {
	if (mailbox_open_index(mailbox)) {
	fatal("failed to reopen index file", EC_IOERR);
	}

	for (oldmsgno = msgno = 1; oldmsgno <= imapd_exists;
	oldmsgno++, msgno++) {
	if (msgno <= mailbox->exists) {
	mailbox_read_index_record(mailbox, msgno, &record);
	}
	else {
	record.uid = mailbox->last_uid+1;
	}

	nexpunge = 0;
	while (oldmsgno<=imapd_exists && UID(oldmsgno) < record.uid) {
	nexpunge++;
	oldmsgno++;
	}
	if (nexpunge) {
	memmove(flagreport+msgno, flagreport+msgno+nexpunge,
	(oldexists-msgno-nexpunge+1)sizeof(flagreport));
	memmove(seenflag+msgno, seenflag+msgno+nexpunge,
	(oldexists-msgno-nexpunge+1)sizeof(seenflag));
	oldexists -= nexpunge;
	while (nexpunge--) {
	prot_printf(imapd_out, "* %u EXPUNGE\r\n", msgno);
	}
	}
	}

	/* Force re-map of index/cache files */
	map_free(&index_base, &index_len);
	map_free(&cache_base, &cache_len);
	cache_end = 0;

	/* Force a * n EXISTS message */
	imapd_exists = -1;
	}
	else if (sbuf.st_mtime != mailbox->index_mtime
	\|\| sbuf.st_size != mailbox->index_size) {
	mailbox_read_index_header(mailbox);
	}
	}
	index_ino = mailbox->index_ino;

	start_offset = mailbox->start_offset;
	record_size = mailbox->record_size;
	newexists = mailbox->exists;

	/* Refresh the index and cache files */
	map_refresh(mailbox->index_fd, 0, &index_base, &index_len,
	start_offset + newexists * record_size,
	"index", mailbox->name);
	if (fstat(mailbox->cache_fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: stating cache file for %s: %m",
	mailbox->name);
	fatal("failed to stat cache file", EC_IOERR);
	}
	if (cache_end < sbuf.st_size) {
	cache_end = sbuf.st_size;
	map_refresh(mailbox->cache_fd, 0, &cache_base, &cache_len,
	cache_end, "cache", mailbox->name);
	}

	/* If opening mailbox, get \Recent info */
	if (oldexists == -1 && keepingseen) {
	r = seen_open(mailbox, imapd_userid, &seendb);
	if (!r) {
	free(seenuids);
	r = seen_lockread(seendb, &last_read, &recentuid,
	&seen_last_change, &seenuids);
	if (r) seen_close(seendb);
	}
	if (r) {
	seendb = 0;
	prot_printf(imapd_out, "* OK %s: %s\r\n",
	error_message(IMAP_NO_CHECKPRESERVE), error_message(r));
	}
	else {
	/*
	* Empty seenuids so that index_checkseen() will pick up the
	* initial \Seen info. Leave the database locked.
	*/
	*seenuids = '\0';
	}
	}

	/* If opening mailbox or had an EXPUNGE, find where \Recent starts */
	if (imapd_exists == -1) {
	imapd_exists = newexists;
	lastnotrecent = index_finduid(recentuid);
	imapd_exists = -1;
	}

	/* If EXISTS changed, report it */
	if (newexists != imapd_exists) {
	/* Re-size flagreport and seenflag arrays if necessary */
	if (newexists > flagalloced) {
	flagalloced = newexists + SLOP;
	flagreport = (time_t *)
	xrealloc((char )flagreport, (flagalloced+1) sizeof(time_t));
	seenflag = xrealloc(seenflag, flagalloced+1);
	}

	/* Zero out array entry for newly arrived messages */
	for (i = oldexists+1; i <= newexists; i++) {
	flagreport[i] = LAST_UPDATED(i);
	seenflag[i] = 0;
	}

	checkseen = 1;
	imapd_exists = newexists;
	prot_printf(imapd_out, "* %u EXISTS\r\n* %u RECENT\r\n", imapd_exists,
	imapd_exists-lastnotrecent);
	}

	/* Check Flags */
	if (checkseen) index_checkseen(mailbox, 0, usinguid, oldexists);
	else if (oldexists == -1) seen_unlock(seendb);
	for (i = 1; i <= imapd_exists && seenflag[i]; i++);
	if (i == imapd_exists + 1) allseen = mailbox->last_uid;
	if (oldexists == -1) {
	if (imapd_exists && i <= imapd_exists) {
	prot_printf(imapd_out, "* OK [UNSEEN %u] \r\n", i);
	}
	- prot_printf(imapd_out, "* OK [UIDVALIDITY %u] \r\n",
	+ prot_printf(imapd_out, "* OK [UIDVALIDITY %lu] \r\n",
	mailbox->uidvalidity);
	- prot_printf(imapd_out, "* OK [UIDNEXT %u] \r\n",
	+ prot_printf(imapd_out, "* OK [UIDNEXT %lu] \r\n",
	mailbox->last_uid + 1);
	}

	for (msgno = 1; msgno <= oldexists; msgno++) {
	if (flagreport[msgno] < LAST_UPDATED(msgno)) {
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	user_flags[i] = USER_FLAGS(msgno, i);
	}
	index_fetchflags(mailbox, msgno, SYSTEM_FLAGS(msgno), user_flags,
	LAST_UPDATED(msgno));
	if (usinguid) prot_printf(imapd_out, " UID %u", UID(msgno));
	prot_printf(imapd_out, ")\r\n");
	}
	}
	}

	/*
	* Checkpoint the user's \Seen state
	*/
	#define SAVEGROW 200
	void
	index_checkseen(mailbox, quiet, usinguid, oldexists)
	struct mailbox *mailbox;
	int quiet;
	int usinguid;
	int oldexists;
	{
	int r;
	time_t last_read;
	unsigned last_uid;
	char *newseenuids;
	char old, new;
	unsigned oldnext = 0, oldseen = 0;
	unsigned newnext = 0, newseen = 0;
	int neweof = 0;
	unsigned msgno, uid, dirty = 0;
	int i;
	bit32 user_flags[MAX_USER_FLAGS/32];
	char saveseenuids, save;
	int savealloced;
	unsigned start, newallseen, inrange, usecomma;

	if (!keepingseen \|\| !seendb) return;
	if (imapd_exists == 0) {
	seen_unlock(seendb);
	return;
	}

	/* Lock \Seen database and read current values */
	r = seen_lockread(seendb, &last_read, &last_uid, &seen_last_change,
	&newseenuids);
	if (r) {
	prot_printf(imapd_out, "* OK %s: %s\r\n",
	error_message(IMAP_NO_CHECKSEEN), error_message(r));
	seen_close(seendb);
	return;
	}

	/*
	* Propagate changes in the database to the seenflag[] array
	* and possibly to the client.
	*/
	old = seenuids;
	new = newseenuids;
	while (isdigit((int) old)) oldnext = oldnext 10 + *old++ - '0';
	while (isdigit((int) new)) newnext = newnext 10 + *new++ - '0';

	for (msgno = 1; msgno <= imapd_exists; msgno++) {
	uid = UID(msgno);
	while (oldnext <= uid) {
	if (*old != ':' && !oldseen && oldnext == uid) {
	oldseen = 1;
	break;
	}
	else {
	oldseen = (*old == ':');
	oldnext = 0;
	if (!*old) oldnext = mailbox->last_uid+1;
	else old++;
	while (isdigit((int) *old)) {
	oldnext = oldnext * 10 + *old++ - '0';
	}
	oldnext += oldseen;
	}
	}
	while (newnext <= uid) {
	if (*new != ':' && !newseen && newnext == uid) {
	newseen = 1;
	break;
	}
	else {
	newseen = (*new == ':');
	newnext = 0;
	if (!*new) {
	newnext = mailbox->last_uid+1;
	neweof++;
	}
	else new++;
	while (isdigit((int) *new)) {
	newnext = newnext * 10 + *new++ - '0';
	}
	newnext += newseen;
	}
	}

	/* report flags that have changed */
	if (oldseen != newseen) {
	if (seenflag[msgno] != newseen) {
	seenflag[msgno] = newseen;
	if (!quiet && msgno <= oldexists && oldexists != -1) {
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	user_flags[i] = USER_FLAGS(msgno, i);
	}
	index_fetchflags(mailbox, msgno, SYSTEM_FLAGS(msgno),
	user_flags, LAST_UPDATED(msgno));
	if (usinguid) {
	prot_printf(imapd_out, " UID %u", UID(msgno));
	}
	prot_printf(imapd_out, ")\r\n");
	}
	}
	}
	else if (seenflag[msgno] != newseen) {
	dirty++;
	}
	}

	if (dirty) {
	seen_last_change = time((time_t *)0);
	}

	if (!examining && oldexists != imapd_exists) {
	/* If just did a SELECT, record time of our reading the mailbox */
	if (oldexists == -1) last_read = time((time_t *)0);

	/* Update the \Recent high-water mark */
	last_uid = mailbox->last_uid;
	dirty++;
	}

	/* If there's nothing to save back to the database, clean up and return */
	if (!dirty) {
	seen_unlock(seendb);
	free(seenuids);
	seenuids = newseenuids;
	/* We might have deleted our last unseen message */
	if (!allseen) {
	for (msgno = 1; msgno <= imapd_exists; msgno++) {
	if (!seenflag[msgno]) break;
	}
	#if TOIMSP
	if (msgno == imapd_exists + 1) {
	toimsp(mailbox->name, mailbox->uidvalidity,
	"SEENsnn", imapd_userid, mailbox->last_uid,
	seen_last_change, 0);
	}
	#endif
	}
	return;
	}

	/* Build the seenuids string to save to the database */
	start = 1;
	inrange = 1;
	newallseen = mailbox->last_uid;
	usecomma = 0;
	savealloced = SAVEGROW;
	save = saveseenuids = xmalloc(savealloced);
	*save = '\0';
	for (msgno = 1; msgno <= imapd_exists; msgno++) {
	uid = UID(msgno);
	if (seenflag[msgno] != inrange) {
	newallseen = 0;
	if (inrange) {
	if (start == uid-1) {
	if (usecomma++) *save++ = ',';
	sprintf(save, "%u", start);
	save += strlen(save);
	}
	else if (uid > 1) {
	if (usecomma++) *save++ = ',';
	sprintf(save, "%u:", start);
	save += strlen(save);
	sprintf(save, "%u", uid-1);
	save += strlen(save);
	}
	inrange = 0;
	}
	else {
	start = uid;
	inrange = 1;
	}
	}
	if (save - saveseenuids > savealloced - 30) {
	savealloced += SAVEGROW;
	saveseenuids = xrealloc(saveseenuids, savealloced);
	save = saveseenuids + strlen(saveseenuids);
	}
	}

	/* Any messages between uid+1 and mailbox->last_uid get same disposition
	* as uid
	*/
	uid = mailbox->last_uid;
	while (newnext <= uid) {
	if (*new != ':' && !newseen && newnext == uid) {
	newseen = 1;
	break;
	}
	else {
	newseen = (*new == ':');
	newnext = 0;
	if (!*new) {
	newnext = mailbox->last_uid+1;
	neweof++;
	}
	else new++;
	while (isdigit((int) new)) newnext = newnext 10 + *new++ - '0';
	newnext += newseen;
	}
	}

	if (inrange) {
	/* Last message read. */
	if (newseen && newnext > uid+1) {
	/* We parsed a range which went past uid. Include it in output. */
	uid = newnext-1;
	}
	else if (!neweof && !newseen && newnext == uid+1) {
	/* We parsed ",N" where N is one past uid. Include it
	* in the output range */
	if (*new == ':') {
	/* There's a ":M" after the ",N". Parse/include that too. */
	new++;
	newnext = 0;
	while (isdigit((int) new)) newnext = newnext 10 + *new++ - '0';
	}
	uid = newnext;
	newseen++; /* Forget we parsed ",N" */
	}

	if (!start && uid > 1) start = 1;
	if (usecomma++) *save++ = ',';
	if (start && start != uid) {
	sprintf(save, "%u:", start);
	save += strlen(save);
	}
	sprintf(save, "%u", uid);
	save += strlen(save);

	if (!neweof && !newseen) {
	/* Parsed a lone number */
	if (usecomma++) *save++ = ',';
	sprintf(save, "%u", newnext);
	save += strlen(save);
	}
	}
	else if (newseen && newnext > uid+1) {
	/* We parsed a range which went past uid. Include it in output */
	if (usecomma++) *save++ = ',';
	if (newnext > uid+2) {
	sprintf(save, "%u:", uid+1);
	save += strlen(save);
	}
	sprintf(save, "%u", newnext-1);
	save += strlen(save);
	}
	else if (*new == ':') {
	/* Parsed first half of a range. Write it out */
	if (usecomma++) *save++ = ',';
	sprintf(save, "%u", uid+1);
	save += strlen(save);
	}
	else if (!neweof && !newseen) {
	/* Parsed a lone number */
	if (usecomma++) *save++ = ',';
	sprintf(save, "%u", newnext);
	save += strlen(save);
	}

	if (*new) {
	if (save - saveseenuids + strlen(new) >= savealloced) {
	savealloced += strlen(new);
	saveseenuids = xrealloc(saveseenuids, savealloced);
	save = saveseenuids + strlen(saveseenuids);
	}
	strcpy(save, usecomma ? new : new+1);
	}

	/* Write the changes, clean up, and return */
	r = seen_write(seendb, last_read, last_uid, seen_last_change, saveseenuids);
	seen_unlock(seendb);
	free(seenuids);
	if (r) {
	prot_printf(imapd_out, "* OK %s: %s\r\n",
	error_message(IMAP_NO_CHECKSEEN), error_message(r));
	free(saveseenuids);
	seenuids = newseenuids;
	return;
	}

	#if TOIMSP
	if (newallseen) {
	toimsp(mailbox->name, mailbox->uidvalidity, "SEENsnn", imapd_userid,
	mailbox->last_uid, seen_last_change, 0);
	}
	else if (allseen == mailbox->last_uid) {
	toimsp(mailbox->name, mailbox->uidvalidity, "SEENsnn", imapd_userid,
	0, seen_last_change, 0);
	}
	#endif

	free(newseenuids);
	seenuids = saveseenuids;
	}


	/*
	* Perform a FETCH-related command on a sequence.
	* Fetchedsomething argument is 0 if nothing was fetched, 1 if something was
	* fetched. (A fetch command that fetches nothing is not a valid fetch
	* command.)
	*/
	void
	index_fetch(struct mailbox* mailbox,
	char* sequence,
	int usinguid,
	struct fetchargs* fetchargs,
	int* fetchedsomething)
	{
	*fetchedsomething = 0;
	index_forsequence(mailbox, sequence, usinguid,
	index_fetchreply, (char *)fetchargs, fetchedsomething);
	}

	/*
	* Perform a STORE command on a sequence
	*/
	int
	index_store(mailbox, sequence, usinguid, storeargs, flag, nflags)
	struct mailbox *mailbox;
	char *sequence;
	int usinguid;
	struct storeargs *storeargs;
	char **flag;
	int nflags;
	{
	int i, r, userflag, emptyflag;
	int writeheader = 0;
	int newflag[MAX_USER_FLAGS];
	long myrights = mailbox->myrights;

	/* Handle simple case of just changing /Seen */
	if (storeargs->operation != STORE_REPLACE &&
	!storeargs->system_flags && !nflags) {
	if (!storeargs->seen) return 0; /* Nothing to change */
	if (!(myrights & ACL_SEEN)) return IMAP_PERMISSION_DENIED;
	storeargs->usinguid = usinguid;

	index_forsequence(mailbox, sequence, usinguid,
	index_storeseen, (char *)storeargs, NULL);
	return 0;
	}

	mailbox_read_acl(mailbox, imapd_authstate);
	myrights &= mailbox->myrights;

	/* First pass at checking permission */
	if ((storeargs->seen && !(myrights & ACL_SEEN)) \|\|
	((storeargs->system_flags & FLAG_DELETED) &&
	!(myrights & ACL_DELETE)) \|\|
	(((storeargs->system_flags & ~FLAG_DELETED) \|\| nflags) &&
	!(myrights & ACL_WRITE))) {
	mailbox->myrights = myrights;
	return IMAP_PERMISSION_DENIED;
	}

	/* Check to see if we have to add new user flags */
	for (userflag=0; userflag < MAX_USER_FLAGS; userflag++)
	newflag[userflag] = 0;
	for (i=0; i < nflags; i++) {
	emptyflag = -1;
	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if (mailbox->flagname[userflag]) {
	if (!strcasecmp(flag[i], mailbox->flagname[userflag]))
	break;
	}
	else if (!newflag[userflag] && emptyflag == -1) {
	emptyflag = userflag;
	}
	}
	if (userflag == MAX_USER_FLAGS) {
	if (emptyflag == -1) {
	return IMAP_USERFLAG_EXHAUSTED;
	}
	newflag[emptyflag] = 1;
	writeheader++;
	}
	}

	/* Add the new user flags */
	if (writeheader) {
	r = mailbox_lock_header(mailbox);
	if (r) return r;

	/*
	* New flags might have been assigned since we last looked
	* Do the assignment again.
	*/
	for (userflag=0; userflag < MAX_USER_FLAGS; userflag++)
	newflag[userflag] = 0;
	for (i=0; i < nflags; i++) {
	emptyflag = -1;
	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if (mailbox->flagname[userflag]) {
	if (!strcasecmp(flag[i], mailbox->flagname[userflag]))
	break;
	}
	else if (emptyflag == -1) {
	emptyflag = userflag;
	}
	}
	if (userflag == MAX_USER_FLAGS) {
	if (emptyflag == -1) {
	mailbox_unlock_header(mailbox);
	mailbox->myrights = myrights;

	/* Undo the new assignments */
	for (userflag=0; userflag < MAX_USER_FLAGS; userflag++) {
	if (newflag[userflag] && mailbox->flagname[userflag]) {
	free(mailbox->flagname[userflag]);
	mailbox->flagname[userflag] = 0;
	}
	}

	/* Tell client about new flags we read while looking */
	index_listflags(mailbox);

	return IMAP_USERFLAG_EXHAUSTED;
	}
	mailbox->flagname[emptyflag] = xstrdup(flag[i]);
	}
	}

	/* Tell client about new flags */
	index_listflags(mailbox);

	r = mailbox_write_header(mailbox);
	mailbox_unlock_header(mailbox);
	mailbox->myrights = myrights;
	if (r) return r;
	}
	/* Not reading header anymore--can put back our working ACL */
	mailbox->myrights = myrights;

	/* Now we know all user flags are in the mailbox header, find the bits */
	for (i=0; i < nflags; i++) {
	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if (mailbox->flagname[userflag]) {
	if (!strcasecmp(flag[i], mailbox->flagname[userflag]))
	break;
	}
	}
	assert(userflag != MAX_USER_FLAGS);
	storeargs->user_flags[userflag/32] \|= 1<<(userflag&31);
	}

	storeargs->update_time = time((time_t *)0);
	storeargs->usinguid = usinguid;

	r = mailbox_lock_index(mailbox);
	if (r) return r;

	r = index_forsequence(mailbox, sequence, usinguid,
	index_storeflag, (char *)storeargs, NULL);

	/* note that index_forsequence() doesn't sync the index file;
	that's done below in mailbox_write_index_header() */
	if (mailbox->dirty) {
	/* xxx what to do on failure? */
	mailbox_write_index_header(mailbox);
	mailbox->dirty = 0;
	}

	mailbox_unlock_index(mailbox);

	/* Refresh the index file, for systems without mmap() */
	map_refresh(mailbox->index_fd, 0, &index_base, &index_len,
	start_offset + imapd_exists * record_size,
	"index", mailbox->name);

	return r;
	}

	/*
	* Guts of the SEARCH command.
	*
	* Returns message numbers in an array. This function is used by
	* SEARCH, SORT and THREAD.
	*/
	static int
	_index_search(msgno_list, mailbox, searchargs)
	unsigned **msgno_list;
	struct mailbox *mailbox;
	struct searchargs *searchargs;
	{
	unsigned msgno;
	struct mapfile msgfile;
	int n = 0;
	int listindex;
	int listcount;

	if (imapd_exists <= 0) return 0;

	msgno_list = (unsigned ) xmalloc(imapd_exists * sizeof(unsigned));

	/* OK, so I'm being a bit clever here. We fill the msgno list with
	a list of message IDs returned by the search engine. Then we
	scan through the list and store matching message IDs back into the
	list. This is OK because we only overwrite message IDs that we've
	already looked at. */
	listcount = search_prefilter_messages(*msgno_list, mailbox, searchargs);

	for (listindex = 0; listindex < listcount; listindex++) {
	msgno = (*msgno_list)[listindex];
	msgfile.base = 0;
	msgfile.size = 0;

	if (index_search_evaluate(mailbox, searchargs, msgno, &msgfile)) {
	(*msgno_list)[n++] = msgno;
	}
	if (msgfile.base) {
	mailbox_unmap_message(mailbox, UID(msgno),
	&msgfile.base, &msgfile.size);
	}
	}

	/* if we didn't find any matches, free msgno_list */
	if (!n && *msgno_list) {
	free(*msgno_list);
	*msgno_list = NULL;
	}

	return n;
	}

	int index_getuid(unsigned msgno) {
	return UID(msgno);
	}

	/* 'uid_list' is malloc'd string representing the hits from searchargs;
	returns number of hits */
	int index_getuidsequence(struct mailbox *mailbox,
	struct searchargs *searchargs,
	unsigned **uid_list)
	{
	unsigned *msgno_list;
	int i, n;

	n = _index_search(&msgno_list, mailbox, searchargs);
	if (n == 0) {
	*uid_list = NULL;
	return 0;
	}

	for (i = 0; i < n; i++) {
	msgno_list[i] = UID(msgno_list[i]);
	}

	*uid_list = msgno_list;
	return n;
	}

	/*
	* Performs a SEARCH command.
	* This is a wrapper around _index_search() which simply prints the results.
	*/
	int
	index_search(mailbox, searchargs, usinguid)
	struct mailbox *mailbox;
	struct searchargs *searchargs;
	int usinguid;
	{
	unsigned *msgno_list;
	int i, n;

	n = _index_search(&msgno_list, mailbox, searchargs);

	prot_printf(imapd_out, "* SEARCH");

	for (i = 0; i < n; i++)
	prot_printf(imapd_out, " %u",
	usinguid ? UID(msgno_list[i]) : msgno_list[i]);

	if (n) free(msgno_list);

	prot_printf(imapd_out, "\r\n");

	return n;
	}

	/*
	* Performs a SORT command
	*/
	int
	index_sort(struct mailbox *mailbox,
	struct sortcrit *sortcrit,
	struct searchargs *searchargs,
	int usinguid)
	{
	unsigned *msgno_list;
	MsgData msgdata = NULL, freeme = NULL;
	int nmsg;
	clock_t start = clock();

	/* Search for messages based on the given criteria */
	nmsg = _index_search(&msgno_list, mailbox, searchargs);

	prot_printf(imapd_out, "* SORT");

	if (nmsg) {
	/* Create/load the msgdata array */
	freeme = msgdata = index_msgdata_load(msgno_list, nmsg, sortcrit);
	free(msgno_list);

	/* Sort the messages based on the given criteria */
	msgdata = lsort(msgdata,
	(void * ()(void)) index_sort_getnext,
	(void ()(void,void*)) index_sort_setnext,
	(int ()(void,void,void)) index_sort_compare,
	sortcrit);

	/* Output the sorted messages */
	while (msgdata) {
	prot_printf(imapd_out, " %u",
	usinguid ? UID(msgdata->msgno) : msgdata->msgno);

	/* free contents of the current node */
	index_msgdata_free(msgdata);

	msgdata = msgdata->next;
	}

	/* free the msgdata array */
	free(freeme);
	}

	prot_printf(imapd_out, "\r\n");

	/* debug */
	if (CONFIG_TIMING_VERBOSE) {
	char *key_names[] = { "SEQUENCE", "ARRIVAL", "CC", "DATE", "FROM",
	"SIZE", "SUBJECT", "TO", "ANNOTATION" };
	char buf[1024] = "";

	while (sortcrit->key) {
	if (sortcrit->flags & SORT_REVERSE) strcat(buf, "REVERSE ");
	strcat(buf, key_names[sortcrit->key]);
	switch (sortcrit->key) {
	case SORT_ANNOTATION:
	snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf) - 1,
	" \"%s\" \"%s\"",
	sortcrit->args.annot.entry, sortcrit->args.annot.attrib);
	break;
	}
	if ((++sortcrit)->key) strcat(buf, " ");
	}

	syslog(LOG_DEBUG, "SORT (%s) processing time: %d msg in %f sec",
	buf, nmsg, (clock() - start) / (double) CLOCKS_PER_SEC);
	}

	return nmsg;
	}

	/*
	* Performs a THREAD command
	*/
	int index_thread(struct mailbox *mailbox, int algorithm,
	struct searchargs *searchargs, int usinguid)
	{
	unsigned *msgno_list;
	int nmsg;
	clock_t start = clock();

	/* Search for messages based on the given criteria */
	nmsg = _index_search(&msgno_list, mailbox, searchargs);

	if (nmsg) {
	/* Thread messages using given algorithm */
	(*thread_algs[algorithm].threader)(msgno_list, nmsg, usinguid);

	free(msgno_list);
	}

	/* print an empty untagged response */
	else
	index_thread_print(NULL, usinguid);

	if (CONFIG_TIMING_VERBOSE) {
	/* debug */
	syslog(LOG_DEBUG, "THREAD %s processing time: %d msg in %f sec",
	thread_algs[algorithm].alg_name, nmsg,
	(clock() - start) / (double) CLOCKS_PER_SEC);
	}

	return nmsg;
	}

	/*
	* Performs a COPY command
	*/
	int
	index_copy(struct mailbox *mailbox,
	char *sequence,
	int usinguid,
	char *name,
	char **copyuidp)
	{
	static struct copyargs copyargs;
	int i;
	unsigned long totalsize = 0;
	int r;
	struct appendstate append_mailbox;
	char *copyuid;
	int copyuid_len, copyuid_size;
	int sepchar;
	unsigned long uidvalidity;
	unsigned long startuid, num;

	copyargs.nummsg = 0;
	index_forsequence(mailbox, sequence, usinguid, index_copysetup,
	(char *)&copyargs, NULL);

	if (copyargs.nummsg == 0) {
	*copyuidp = 0;
	return 0;
	}

	for (i = 0; i < copyargs.nummsg; i++) {
	totalsize += copyargs.copymsg[i].size;
	}

	r = append_setup(&append_mailbox, name, MAILBOX_FORMAT_NORMAL,
	imapd_userid, imapd_authstate, ACL_INSERT, totalsize);
	if (r) return r;

	r = append_copy(mailbox, &append_mailbox, copyargs.nummsg,
	copyargs.copymsg);
	if (!r) append_commit(&append_mailbox, &uidvalidity, &startuid, &num);
	if (!r) {
	copyuid_size = 1024;
	copyuid = xmalloc(copyuid_size);
	sprintf(copyuid, "%lu", uidvalidity);
	copyuid_len = strlen(copyuid);
	sepchar = ' ';

	for (i = 0; i < copyargs.nummsg; i++) {
	if (copyuid_size < copyuid_len + 50) {
	copyuid_size += 1024;
	copyuid = xrealloc(copyuid, copyuid_size);
	}
	sprintf(copyuid+copyuid_len, "%c%lu", sepchar,
	copyargs.copymsg[i].uid);
	copyuid_len += strlen(copyuid+copyuid_len);
	if (i+1 < copyargs.nummsg &&
	copyargs.copymsg[i+1].uid == copyargs.copymsg[i].uid + 1) {
	do {
	i++;
	} while (i+1 < copyargs.nummsg &&
	copyargs.copymsg[i+1].uid == copyargs.copymsg[i].uid + 1);
	sprintf(copyuid+copyuid_len, ":%lu",
	copyargs.copymsg[i].uid);
	copyuid_len += strlen(copyuid+copyuid_len);
	}
	sepchar = ',';
	}
	if (num == 1) {
	sprintf(copyuid+copyuid_len, " %lu", startuid);
	} else {
	sprintf(copyuid+copyuid_len, " %lu:%lu",
	startuid, startuid + num - 1);
	}
	*copyuidp = copyuid;
	}

	return r;
	}

	/*
	* Performs a STATUS command
	*/
	int
	index_status(mailbox, name, statusitems)
	struct mailbox *mailbox;
	char *name;
	int statusitems;
	{
	int r;
	struct seen *status_seendb;
	time_t last_read, last_change = 0;
	unsigned last_uid;
	char *last_seenuids;
	int num_recent = 0;
	int num_unseen = 0;
	int sepchar;

	if (mailbox->exists != 0 &&
	(statusitems &
	(STATUS_RECENT \| STATUS_UNSEEN))) {
	r = seen_open(mailbox, imapd_userid, &status_seendb);
	if (r) return r;

	r = seen_lockread(status_seendb, &last_read, &last_uid,
	&last_change, &last_seenuids);
	seen_close(status_seendb);
	if (r) return r;

	if (statusitems & (STATUS_RECENT \| STATUS_UNSEEN)) {
	const char *base;
	unsigned long len = 0;
	int msg;
	unsigned uid;

	map_refresh(mailbox->index_fd, 0, &base, &len,
	mailbox->start_offset +
	mailbox->exists * mailbox->record_size,
	"index", mailbox->name);
	for (msg = 0; msg < mailbox->exists; msg++) {
	uid = ntohl(((bit32 )(base + mailbox->start_offset +
	msg * mailbox->record_size +
	OFFSET_UID)));
	if (uid > last_uid) num_recent++;
	if ((statusitems & STATUS_UNSEEN) &&
	!index_insequence(uid, last_seenuids, 0)) num_unseen++;
	/* NB: The value of the third argument to index_insequence()
	* above does not matter.
	*/
	}
	map_free(&base, &len);
	free(last_seenuids);
	}
	}

	prot_printf(imapd_out, "* STATUS ");
	printastring(name);
	prot_printf(imapd_out, " ");
	sepchar = '(';

	if (statusitems & STATUS_MESSAGES) {
	- prot_printf(imapd_out, "%cMESSAGES %u", sepchar, mailbox->exists);
	+ prot_printf(imapd_out, "%cMESSAGES %lu", sepchar, mailbox->exists);
	sepchar = ' ';
	}
	if (statusitems & STATUS_RECENT) {
	prot_printf(imapd_out, "%cRECENT %u", sepchar, num_recent);
	sepchar = ' ';
	}
	if (statusitems & STATUS_UIDNEXT) {
	- prot_printf(imapd_out, "%cUIDNEXT %u", sepchar, mailbox->last_uid+1);
	+ prot_printf(imapd_out, "%cUIDNEXT %lu", sepchar, mailbox->last_uid+1);
	sepchar = ' ';
	}
	if (statusitems & STATUS_UIDVALIDITY) {
	- prot_printf(imapd_out, "%cUIDVALIDITY %u", sepchar,
	+ prot_printf(imapd_out, "%cUIDVALIDITY %lu", sepchar,
	mailbox->uidvalidity);
	sepchar = ' ';
	}
	if (statusitems & STATUS_UNSEEN) {
	prot_printf(imapd_out, "%cUNSEEN %u", sepchar, num_unseen);
	sepchar = ' ';
	}
	prot_printf(imapd_out, ")\r\n");
	return 0;
	}

	/*
	* Performs a GETUIDS command
	*/
	int
	index_getuids(mailbox, lowuid)
	struct mailbox *mailbox;
	unsigned lowuid;
	{
	int msgno;
	unsigned firstuid = 0, lastuid = 0;


	prot_printf(imapd_out, "* GETUIDS");

	for (msgno = 1; msgno <= imapd_exists; msgno++) {
	if (firstuid == 0) {
	if (UID(msgno) >= lowuid) {
	prot_printf(imapd_out, " %u %u", msgno, UID(msgno));
	firstuid = lastuid = UID(msgno);
	}
	}
	else {
	if (UID(msgno) != ++lastuid) {
	if (lastuid-1 != firstuid) {
	prot_printf(imapd_out, ":%u", lastuid-1);
	}
	firstuid = lastuid = UID(msgno);
	prot_printf(imapd_out, ",%u", firstuid);
	}
	}
	}
	if (lastuid != firstuid) {
	prot_printf(imapd_out, ":%u", lastuid);
	}
	prot_printf(imapd_out, "\r\n");

	return 0;
	}

	/*
	* Performs a XGETSTATE command
	*/
	int
	index_getstate(mailbox)
	struct mailbox *mailbox;
	{

	- prot_printf(imapd_out, "* XSTATE %u %u\r\n", mailbox->index_mtime,
	+ prot_printf(imapd_out, "* XSTATE %lu %lu\r\n", mailbox->index_mtime,
	seen_last_change);

	return 0;
	}

	#if 0
	/* What's this for? Might as well keep it around. */
	/*
	* Performs a XCHECKSTATE command
	*/
	int
	index_checkstate(mailbox, indexdate, seendate)
	struct mailbox *mailbox;
	unsigned indexdate;
	unsigned seendate;
	{
	int r;
	int msgno;
	unsigned int startmsgno = 0;
	int sepchar = ' ';

	/* No messages == everything OK */
	if (imapd_exists < 1) {
	prot_printf(imapd_out, "* XCHECKSTATE\r\n");
	return 0;
	}

	/* If \Seen data changed, we don't know anything */
	if (seendate != seen_last_change) {
	if (imapd_exists == 1) {
	prot_printf(imapd_out,
	"* XCHECKSTATE %u\r\n", UID(1));
	}
	else {
	prot_printf(imapd_out,
	"* XCHECKSTATE %u:%u\r\n", UID(1), UID(imapd_exists));
	}
	return 0;
	}

	prot_printf(imapd_out, "* XCHECKSTATE");
	for (msgno = 1; msgno <= imapd_exists; msgno++) {
	/*
	* Below is >= instead of > because we can get
	* two STORE commands within the same second.
	*/
	if (LAST_UPDATED(msgno) >= indexdate) {
	if (startmsgno == 0) {
	prot_printf(imapd_out, "%c%u", sepchar, UID(msgno));
	sepchar = ',';
	startmsgno = msgno;
	}
	}
	else {
	if (startmsgno != 0 && startmsgno < msgno - 1) {
	prot_printf(imapd_out, ":%u", UID(msgno-1));
	}
	startmsgno = 0;
	}
	}

	if (startmsgno != 0 && startmsgno < imapd_exists) {
	prot_printf(imapd_out, ":%u", UID(imapd_exists));
	}

	prot_printf(imapd_out, "\r\n");
	return 0;
	}
	#endif

	/*
	* Returns the msgno of the message with UID 'uid'.
	* If no message with UID 'uid', returns the message with
	* the higest UID not greater than 'uid'.
	*/
	int
	index_finduid(uid)
	unsigned uid;
	{
	int low=1, high=imapd_exists, mid;
	unsigned miduid;

	while (low <= high) {
	mid = (high - low)/2 + low;
	miduid = UID(mid);
	if (miduid == uid) {
	return mid;
	}
	else if (miduid > uid) {
	high = mid - 1;
	}
	else {
	low = mid + 1;
	}
	}
	return high;
	}

	/*
	* Expunge decision procedure to get rid of articles
	* both \Deleted and listed in the sequence under 'rock'.
	*/
	int index_expungeuidlist(struct mailbox mailbox, void rock,
	char *indexbuf)
	{
	char sequence = (char )rock;
	unsigned uid = ntohl(((bit32 )(indexbuf+OFFSET_UID)));

	/* Don't expunge if not \Deleted */
	if (!(ntohl(((bit32 )(indexbuf+OFFSET_SYSTEM_FLAGS))) & FLAG_DELETED))
	return 0;

	return index_insequence(uid, sequence, 1);
	}

	/*
	* Call a function 'proc' on each message in 'sequence'. If 'usinguid'
	* is nonzero, 'sequence' is interpreted as a sequence of UIDs instead
	* of a sequence of msgnos. 'proc' is called with arguments 'mailbox',
	* the msgno, and 'rock'. If any invocation of 'proc' returns nonzero,
	* returns the first such returned value. Otherwise, returns zero.
	*/
	static int
	index_forsequence(struct mailbox* mailbox,
	char* sequence,
	int usinguid,
	index_sequenceproc_t proc,
	void* rock,
	int* fetchedsomething)
	{
	unsigned i, start = 0, end;
	int r, result = 0;

	/* no messages, no calls. dumps core otherwise */
	if (! imapd_exists) {
	return 0;
	}

	for (;;) {
	if (isdigit((int) *sequence)) {
	start = start10 + sequence - '0';
	}
	else if (sequence == '') {
	start = usinguid ? UID(imapd_exists) : imapd_exists;
	}
	else if (*sequence == ':') {
	end = 0;
	sequence++;
	while (isdigit((int) *sequence)) {
	end = end10 + sequence++ - '0';
	}
	if (sequence == '') {
	sequence++;
	end = usinguid ? UID(imapd_exists) : imapd_exists;
	}
	if (start > end) {
	i = end;
	end = start;
	start = i;
	}
	if (usinguid) {
	i = index_finduid(start);
	if (!i \|\| start != UID(i)) i++;
	start = i;
	end = index_finduid(end);
	}
	if (start < 1) start = 1;
	if (end > imapd_exists) end = imapd_exists;
	for (i = start; i <= end; i++) {
	if (fetchedsomething) *fetchedsomething = 1;
	r = (*proc)(mailbox, i, rock);
	if (r && !result) result = r;
	}
	start = 0;
	if (!*sequence) return result;
	}
	else {
	if (start && usinguid) {
	i = index_finduid(start);
	if (!i \|\| start != UID(i)) i = 0;
	start = i;
	}
	if (start > 0 && start <= imapd_exists) {
	if (fetchedsomething) *fetchedsomething = 1;
	r = (*proc)(mailbox, start, rock);
	if (r && !result) result = r;
	}
	start = 0;
	if (!*sequence) return result;
	}
	sequence++;
	}
	}

	/*
	* Return nonzero iff 'num' is included in 'sequence'
	*/
	static int
	index_insequence(num, sequence, usinguid)
	int num;
	char *sequence;
	int usinguid;
	{
	unsigned i, start = 0, end;

	for (;;) {
	if (isdigit((int) *sequence)) {
	start = start10 + sequence - '0';
	}
	else if (sequence == '') {
	sequence++;
	start = usinguid ? UID(imapd_exists) : imapd_exists;
	}
	else if (*sequence == ':') {
	end = 0;
	sequence++;
	while (isdigit((int) *sequence)) {
	end = end10 + sequence++ - '0';
	}
	if (sequence == '') {
	sequence++;
	end = usinguid ? UID(imapd_exists) : imapd_exists;
	}
	if (start > end) {
	i = end;
	end = start;
	start = i;
	}
	if (num >= start && num <= end) return 1;
	start = 0;
	if (!*sequence) return 0;
	}
	else {
	if (num == start) return 1;
	start = 0;
	if (!*sequence) return 0;
	}
	sequence++;
	}
	}

	/*
	* Helper function to fetch data from a message file. Writes a
	* quoted-string or literal containing data from 'msg_base', which is
	* of size 'msg_size' and format 'format', starting at 'offset' and
	* containing 'size' octets. If 'start_octet' is nonzero, the data is
	* further constrained by 'start_octet' and 'octet_count' as per the
	* IMAP command PARTIAL.
	*/
	static void
	index_fetchmsg(msg_base, msg_size, format, offset, size,
	start_octet, octet_count)
	const char *msg_base;
	unsigned long msg_size;
	int format;
	unsigned offset;
	unsigned size; /* this is the correct size for a news message after
	having LF translated to CRLF */
	unsigned start_octet;
	unsigned octet_count;
	{
	int n;

	/* partial fetch: adjust 'size', normalize 'start_octet' to be 0-based */
	if (start_octet) {
	start_octet--;
	if (size <= start_octet) {
	size = 0;
	}
	else {
	size -= start_octet;
	}
	if (size > octet_count) size = octet_count;
	}

	/* If no data, output null quoted string */
	if (!msg_base \|\| size == 0) {
	prot_printf(imapd_out, "\"\"");
	return;
	}

	/* Write size of literal */
	prot_printf(imapd_out, "{%u}\r\n", size);

	/* Seek over PARTIAL constraint */
	offset += start_octet;
	n = size;
	if (offset + size > msg_size) {
	n = msg_size - offset;
	}
	prot_write(imapd_out, msg_base + offset, n);
	while (n++ < size) {
	/* File too short, resynch client */
	prot_putc(' ', imapd_out);
	}
	}

	/*
	* Helper function to fetch a body section
	*/
	static void
	index_fetchsection(msg_base, msg_size, format, section, cacheitem, size,
	start_octet, octet_count)
	const char *msg_base;
	unsigned long msg_size;
	int format;
	char *section;
	const char *cacheitem;
	unsigned size;
	unsigned start_octet;
	unsigned octet_count;
	{
	char *p;
	int skip = 0;
	int fetchmime = 0;

	cacheitem += 4;
	p = section;

	/* Special-case BODY[] */
	if (*p == ']') {
	p++;
	if (*p == '<') {
	p++;
	start_octet = octet_count = 0;
	while (isdigit((int) p)) start_octet = start_octet 10 + *p++ - '0';
	p++; /* Skip over '.' */
	while (isdigit((int) p)) octet_count = octet_count 10 + *p++ - '0';
	start_octet++; /* Make 1-based */
	}

	index_fetchmsg(msg_base, msg_size, format, 0, size,
	start_octet, octet_count);
	return;
	}

	while (p != ']' && p != 'M') {
	skip = 0;
	while (isdigit((int) p)) skip = skip 10 + *p++ - '0';
	if (*p == '.') p++;

	/* section number too large */
	if (skip >= CACHE_ITEM_BIT32(cacheitem)) goto badpart;

	/* Handle .0, .HEADER, and .TEXT */
	if (!skip) {
	switch (*p) {
	case 'H':
	p += 6;
	fetchmime++; /* .HEADER maps internally to .0.MIME */
	break;

	case 'T':
	p += 4;
	break; /* .TEXT maps internally to .0 */

	default:
	fetchmime++; /* .0 maps internally to .0.MIME */
	break;
	}
	}

	if (p != ']' && p != 'M') {
	cacheitem += CACHE_ITEM_BIT32(cacheitem) * 5 * 4 + 4;
	while (--skip) {
	if (CACHE_ITEM_BIT32(cacheitem) > 0) {
	skip += CACHE_ITEM_BIT32(cacheitem)-1;
	cacheitem += CACHE_ITEM_BIT32(cacheitem) * 5 * 4;
	}
	cacheitem += 4;
	}
	}
	}

	if (*p == 'M') {
	p += 4;
	fetchmime++;
	}
	cacheitem += skip * 5 * 4 + 4 + (fetchmime ? 0 : 2 * 4);

	if (CACHE_ITEM_BIT32(cacheitem+4) == -1) goto badpart;

	p++;
	if (*p == '<') {
	p++;
	start_octet = octet_count = 0;
	while (isdigit((int) p)) start_octet = start_octet 10 + *p++ - '0';
	p++; /* Skip over '.' */
	while (isdigit((int) p)) octet_count = octet_count 10 + *p++ - '0';
	start_octet++; /* Make 1-based */
	}

	index_fetchmsg(msg_base, msg_size, format, CACHE_ITEM_BIT32(cacheitem),
	CACHE_ITEM_BIT32(cacheitem+4),
	start_octet, octet_count);
	return;

	badpart:
	prot_printf(imapd_out, "NIL");
	}

	/*
	* Helper function to fetch a HEADER.FIELDS[.NOT] body section
	*/
	static void
	index_fetchfsection(msg_base, msg_size, format, fsection, cacheitem)
	const char *msg_base;
	unsigned long msg_size;
	int format;
	struct fieldlist *fsection;
	const char *cacheitem;
	{
	char *p;
	int skip = 0;
	int fields_not = 0;
	unsigned crlf_start = 0;
	unsigned crlf_size = 2;
	int start_octet = 0;
	int octet_count = 0;
	char *buf;
	unsigned size;

	/* If no data, output null quoted string */
	if (!msg_base) {
	prot_printf(imapd_out, "\"\"");
	return;
	}

	cacheitem += 4;
	p = fsection->section;

	while (*p != 'H') {
	skip = 0;
	while (isdigit((int) p)) skip = skip 10 + *p++ - '0';
	if (*p == '.') p++;

	/* section number too large */
	if (skip >= CACHE_ITEM_BIT32(cacheitem)) goto badpart;

	cacheitem += CACHE_ITEM_BIT32(cacheitem) * 5 * 4 + 4;
	while (--skip) {
	if (CACHE_ITEM_BIT32(cacheitem) > 0) {
	skip += CACHE_ITEM_BIT32(cacheitem)-1;
	cacheitem += CACHE_ITEM_BIT32(cacheitem) * 5 * 4;
	}
	cacheitem += 4;
	}
	}

	/* leaf object */
	if (0 == CACHE_ITEM_BIT32(cacheitem)) goto badpart;

	cacheitem += 4;

	if (CACHE_ITEM_BIT32(cacheitem+4) == -1) goto badpart;

	if (p[13]) fields_not++; /* Check for "." after "HEADER.FIELDS" */

	p = fsection->trail;
	if (p[1] == '<') {
	p += 2;
	start_octet = octet_count = 0;
	while (isdigit((int) p)) start_octet = start_octet 10 + *p++ - '0';
	p++; /* Skip over '.' */
	while (isdigit((int) p)) octet_count = octet_count 10 + *p++ - '0';
	start_octet++; /* Make 1-based */
	}

	buf = index_readheader(msg_base, msg_size, format,
	CACHE_ITEM_BIT32(cacheitem),
	CACHE_ITEM_BIT32(cacheitem+4));

	if (fields_not) {
	index_pruneheader(buf, 0, fsection->fields);
	}
	else {
	index_pruneheader(buf, fsection->fields, 0);
	}
	size = strlen(buf);

	/* partial fetch: adjust 'size', normalize 'start_octet' to be 0-based */
	if (start_octet) {
	start_octet--;
	if (size <= start_octet) {
	crlf_start = start_octet - size;
	size = 0;
	start_octet = 0;
	if (crlf_size <= crlf_start) {
	crlf_size = 0;
	}
	else {
	crlf_size -= crlf_start;
	}
	}
	else {
	size -= start_octet;
	}
	if (size > octet_count) {
	size = octet_count;
	crlf_size = 0;
	}
	else if (size + crlf_size > octet_count) {
	crlf_size = octet_count - size;
	}
	}

	/* If no data, output null quoted string */
	if (size + crlf_size == 0) {
	prot_printf(imapd_out, "\"\"");
	return;
	}

	/* Write literal */
	prot_printf(imapd_out, "{%u}\r\n", size + crlf_size);
	prot_write(imapd_out, buf + start_octet, size);
	prot_write(imapd_out, "\r\n" + crlf_start, crlf_size);

	return;

	badpart:
	prot_printf(imapd_out, "NIL");
	}

	/*
	* Helper function to read a header section into a static buffer
	*/
	static char *
	index_readheader(msg_base, msg_size, format, offset, size)
	const char *msg_base;
	unsigned long msg_size;
	int format;
	unsigned offset;
	unsigned size;
	{
	static char *buf;
	static int bufsize;

	if (offset + size > msg_size) {
	/* Message file is too short, truncate request */
	if (offset < msg_size) {
	size = msg_size - offset;
	}
	else {
	size = 0;
	}
	}

	if (bufsize < size+2) {
	bufsize = size+100;
	buf = xrealloc(buf, bufsize);
	}

	msg_base += offset;

	memcpy(buf, msg_base, size);
	buf[size] = '\0';

	return buf;
	}

	/*
	* Prune the header section in buf to include only those headers
	* listed in headers or (if headers_not is non-empty) those headers
	* not in headers_not.
	*/
	static void
	index_pruneheader(buf, headers, headers_not)
	char *buf;
	struct strlist *headers;
	struct strlist *headers_not;
	{
	char p, colon, *nextheader;
	int goodheader;
	char *endlastgood = buf;
	struct strlist *l;

	p = buf;
	while (p && p != '\r') {
	colon = strchr(p, ':');
	if (colon && headers_not) {
	goodheader = 1;
	for (l = headers_not; l; l = l->next) {
	if (colon - p == strlen(l->s) &&
	!strncasecmp(p, l->s, colon - p)) {
	goodheader = 0;
	break;
	}
	}
	}
	else {
	goodheader = 0;
	}
	if (colon) {
	for (l = headers; l; l = l->next) {
	if (colon - p == strlen(l->s) &&
	!strncasecmp(p, l->s, colon - p)) {
	goodheader = 1;
	break;
	}
	}
	}

	nextheader = p;
	do {
	nextheader = strchr(nextheader, '\n');
	if (nextheader) nextheader++;
	else nextheader = p + strlen(p);
	} while (nextheader == ' ' \|\| nextheader == '\t');

	if (goodheader) {
	if (endlastgood != p) {
	strcpy(endlastgood, p);
	nextheader -= p - endlastgood;
	}
	endlastgood = nextheader;
	}
	p = nextheader;
	}

	*endlastgood = '\0';
	}

	/*
	* Handle a FETCH RFC822.HEADER.LINES or RFC822.HEADER.LINES.NOT
	* that can't use the cacheheaders in cyrus.cache
	*/
	static void
	index_fetchheader(msg_base, msg_size, format, size, headers, headers_not)
	const char *msg_base;
	unsigned long msg_size;
	int format;
	unsigned size;
	struct strlist *headers;
	struct strlist *headers_not;
	{
	char *buf;

	/* If no data, output null quoted string */
	if (!msg_base) {
	prot_printf(imapd_out, "\"\"");
	return;
	}

	buf = index_readheader(msg_base, msg_size, format, 0, size);

	index_pruneheader(buf, headers, headers_not);

	size = strlen(buf);
	prot_printf(imapd_out, "{%u}\r\n%s\r\n", size+2, buf);
	}

	/*
	* Handle a FETCH RFC822.HEADER.LINES that can use the
	* cacheheaders in cyrus.cache
	*/
	static void
	index_fetchcacheheader(msgno, headers, trail)
	unsigned msgno;
	struct strlist *headers;
	char *trail;
	{
	static char *buf;
	static int bufsize;
	const char *cacheitem;
	unsigned size;
	unsigned crlf_start = 0;
	unsigned crlf_size = 2;
	int start_octet = 0;
	int octet_count = 0;

	cacheitem = cache_base + CACHE_OFFSET(msgno);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bodystructure */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip section */

	size = CACHE_ITEM_LEN(cacheitem);
	if (bufsize < size+2) {
	bufsize = size+100;
	buf = xrealloc(buf, bufsize);
	}

	memcpy(buf, cacheitem+4, size);
	buf[size] = '\0';

	index_pruneheader(buf, headers, 0);
	size = strlen(buf);

	if (trail[1]) {
	/* Deal with ]<start.count> */
	trail += 2;
	while (isdigit((int) trail)) start_octet = start_octet 10 + *trail++ - '0';
	trail++; /* Skip over '.' */
	while (isdigit((int) trail)) octet_count = octet_count 10 + *trail++ - '0';

	if (size <= start_octet) {
	crlf_start = start_octet - size;
	size = 0;
	start_octet = 0;
	if (crlf_size <= crlf_start) {
	crlf_size = 0;
	}
	else {
	crlf_size -= crlf_start;
	}
	}
	else {
	size -= start_octet;
	}
	if (size > octet_count) {
	size = octet_count;
	crlf_size = 0;
	}
	else if (size + crlf_size > octet_count) {
	crlf_size = octet_count - size;
	}
	}

	if (size + crlf_size == 0) {
	prot_printf(imapd_out, "\"\"");
	}
	else {
	prot_printf(imapd_out, "{%u}\r\n", size + crlf_size);
	prot_write(imapd_out, buf + start_octet, size);
	prot_write(imapd_out, "\r\n" + crlf_start, crlf_size);
	}
	}

	/*
	* Send a * FLAGS response.
	*/
	static void index_listflags(struct mailbox *mailbox)
	{
	int i;
	int cancreate = 0;
	char sepchar = '(';

	prot_printf(imapd_out, "* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen");
	for (i = 0; i < MAX_USER_FLAGS; i++) {
	if (mailbox->flagname[i]) {
	prot_printf(imapd_out, " %s", mailbox->flagname[i]);
	}
	else cancreate++;
	}
	prot_printf(imapd_out, ")\r\n* OK [PERMANENTFLAGS ");
	if (!examining) {
	if (mailbox->myrights & ACL_WRITE) {
	prot_printf(imapd_out, "%c\\Answered \\Flagged \\Draft", sepchar);
	sepchar = ' ';
	}
	if (mailbox->myrights & ACL_DELETE) {
	prot_printf(imapd_out, "%c\\Deleted", sepchar);
	sepchar = ' ';
	}
	if (mailbox->myrights & ACL_SEEN) {
	prot_printf(imapd_out, "%c\\Seen", sepchar);
	sepchar = ' ';
	}
	if (mailbox->myrights & ACL_WRITE) {
	for (i = 0; i < MAX_USER_FLAGS; i++) {
	if (mailbox->flagname[i]) {
	prot_printf(imapd_out, " %s", mailbox->flagname[i]);
	}
	}
	if (cancreate) {
	prot_printf(imapd_out, " \\*");
	}
	}
	}
	if (sepchar == '(') prot_printf(imapd_out, "(");
	prot_printf(imapd_out, ")] \r\n");
	}

	/*
	* Helper function to send * FETCH (FLAGS data.
	* Does not send the terminating close paren or CRLF.
	* Also sends preceeding * FLAGS if necessary.
	*/
	static void
	index_fetchflags(mailbox, msgno, system_flags, user_flags, last_updated)
	struct mailbox *mailbox;
	unsigned msgno;
	bit32 system_flags;
	bit32 user_flags[MAX_USER_FLAGS/32];
	time_t last_updated;
	{
	int sepchar = '(';
	unsigned flag;
	bit32 flagmask = 0;

	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if ((flag & 31) == 0) {
	flagmask = user_flags[flag/32];
	}
	if (!mailbox->flagname[flag] && (flagmask & (1<<(flag & 31)))) {
	mailbox_read_header(mailbox);
	index_listflags(mailbox);
	break;
	}
	}

	prot_printf(imapd_out, "* %u FETCH (FLAGS ", msgno);

	if (msgno > lastnotrecent) {
	prot_printf(imapd_out, "%c\\Recent", sepchar);
	sepchar = ' ';
	}
	if (system_flags & FLAG_ANSWERED) {
	prot_printf(imapd_out, "%c\\Answered", sepchar);
	sepchar = ' ';
	}
	if (system_flags & FLAG_FLAGGED) {
	prot_printf(imapd_out, "%c\\Flagged", sepchar);
	sepchar = ' ';
	}
	if (system_flags & FLAG_DRAFT) {
	prot_printf(imapd_out, "%c\\Draft", sepchar);
	sepchar = ' ';
	}
	if (system_flags & FLAG_DELETED) {
	prot_printf(imapd_out, "%c\\Deleted", sepchar);
	sepchar = ' ';
	}
	if (seenflag[msgno]) {
	prot_printf(imapd_out, "%c\\Seen", sepchar);
	sepchar = ' ';
	}
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if ((flag & 31) == 0) {
	flagmask = user_flags[flag/32];
	}
	if (mailbox->flagname[flag] && (flagmask & (1<<(flag & 31)))) {
	prot_printf(imapd_out, "%c%s", sepchar, mailbox->flagname[flag]);
	sepchar = ' ';
	}
	}
	if (sepchar == '(') prot_putc('(', imapd_out);
	prot_putc(')', imapd_out);

	flagreport[msgno] = last_updated;
	}

	/*
	* Helper function to send requested * FETCH data for a message
	*/
	static int
	index_fetchreply(mailbox, msgno, rock)
	struct mailbox *mailbox;
	unsigned msgno;
	void *rock;
	{
	struct fetchargs fetchargs = (struct fetchargs )rock;
	int fetchitems = fetchargs->fetchitems;
	const char *msg_base = 0;
	unsigned long msg_size = 0;
	int sepchar;
	int i;
	bit32 user_flags[MAX_USER_FLAGS/32];
	const char *cacheitem;
	struct strlist section, field;
	struct fieldlist *fsection;
	char *partialdot;

	/* Open the message file if we're going to need it */
	if ((fetchitems & (FETCH_HEADER\|FETCH_TEXT\|FETCH_RFC822\|FETCH_UNCACHEDHEADER)) \|\|
	fetchargs->bodysections) {
	if (mailbox_map_message(mailbox, 1, UID(msgno), &msg_base, &msg_size)) {
	prot_printf(imapd_out, "* OK ");
	prot_printf(imapd_out, error_message(IMAP_NO_MSGGONE), msgno);
	prot_printf(imapd_out, "\r\n");
	}
	}

	/* set the \Seen flag if necessary */
	if (fetchitems & FETCH_SETSEEN) {
	if (!seenflag[msgno] && (mailbox->myrights & ACL_SEEN)) {
	seenflag[msgno] = 1;
	fetchitems \|= FETCH_FLAGS;
	}
	}

	if (fetchitems & FETCH_FLAGS) {
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	user_flags[i] = USER_FLAGS(msgno, i);
	}
	index_fetchflags(mailbox, msgno, SYSTEM_FLAGS(msgno), user_flags,
	LAST_UPDATED(msgno));
	sepchar = ' ';
	}
	else {
	prot_printf(imapd_out, "* %u FETCH ", msgno);
	sepchar = '(';
	}
	if (fetchitems & FETCH_UID) {
	prot_printf(imapd_out, "%cUID %u", sepchar, UID(msgno));
	sepchar = ' ';
	}
	if (fetchitems & FETCH_INTERNALDATE) {
	time_t msgdate = INTERNALDATE(msgno);
	struct tm *tm = localtime(&msgdate);
	long gmtoff = gmtoff_of(tm, msgdate);
	int gmtnegative = 0;
	static const char *monthname[] = {
	"Jan", "Feb", "Mar", "Apr", "May", "Jun",
	"Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
	char datebuf[30];

	if (msgdate == 0 \|\| tm->tm_year < 69) {
	abort();
	}

	if (gmtoff < 0) {
	gmtoff = -gmtoff;
	gmtnegative = 1;
	}
	gmtoff /= 60;
	sprintf(datebuf, "%2u-%s-%u %.2u:%.2u:%.2u %c%.2lu%.2lu",
	tm->tm_mday, monthname[tm->tm_mon], tm->tm_year+1900,
	tm->tm_hour, tm->tm_min, tm->tm_sec,
	gmtnegative ? '-' : '+', gmtoff/60, gmtoff%60);
	prot_printf(imapd_out, "%cINTERNALDATE \"%s\"",
	sepchar, datebuf);
	sepchar = ' ';
	}
	if (fetchitems & FETCH_SIZE) {
	prot_printf(imapd_out, "%cRFC822.SIZE %u", sepchar, SIZE(msgno));
	sepchar = ' ';
	}
	if (fetchitems & FETCH_ENVELOPE) {
	prot_printf(imapd_out, "%cENVELOPE ", sepchar);
	sepchar = ' ';
	cacheitem = cache_base + CACHE_OFFSET(msgno);
	prot_write(imapd_out, cacheitem+4, CACHE_ITEM_LEN(cacheitem));
	}
	if (fetchitems & FETCH_BODYSTRUCTURE) {
	prot_printf(imapd_out, "%cBODYSTRUCTURE ", sepchar);
	sepchar = ' ';
	cacheitem = cache_base + CACHE_OFFSET(msgno);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	prot_write(imapd_out, cacheitem+4, CACHE_ITEM_LEN(cacheitem));
	}
	if (fetchitems & FETCH_BODY) {
	prot_printf(imapd_out, "%cBODY ", sepchar);
	sepchar = ' ';
	cacheitem = cache_base + CACHE_OFFSET(msgno);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bodystructure */
	prot_write(imapd_out, cacheitem+4, CACHE_ITEM_LEN(cacheitem));
	}

	if (fetchitems & FETCH_HEADER) {
	prot_printf(imapd_out, "%cRFC822.HEADER ", sepchar);
	sepchar = ' ';
	index_fetchmsg(msg_base, msg_size, mailbox->format, 0,
	HEADER_SIZE(msgno),
	fetchargs->start_octet, fetchargs->octet_count);
	}
	else if (fetchargs->headers \|\| fetchargs->headers_not) {
	prot_printf(imapd_out, "%cRFC822.HEADER ", sepchar);
	sepchar = ' ';
	if (fetchitems & FETCH_UNCACHEDHEADER) {
	index_fetchheader(msg_base, msg_size, mailbox->format,
	HEADER_SIZE(msgno),
	fetchargs->headers, fetchargs->headers_not);
	}
	else {
	index_fetchcacheheader(msgno, fetchargs->headers, "]");
	}
	}

	if (fetchitems & FETCH_TEXT) {
	prot_printf(imapd_out, "%cRFC822.TEXT ", sepchar);
	sepchar = ' ';
	index_fetchmsg(msg_base, msg_size, mailbox->format,
	CONTENT_OFFSET(msgno), SIZE(msgno) - HEADER_SIZE(msgno),
	fetchargs->start_octet, fetchargs->octet_count);
	}
	if (fetchitems & FETCH_RFC822) {
	prot_printf(imapd_out, "%cRFC822 ", sepchar);
	sepchar = ' ';
	index_fetchmsg(msg_base, msg_size, mailbox->format, 0, SIZE(msgno),
	fetchargs->start_octet, fetchargs->octet_count);
	}
	for (fsection = fetchargs->fsections; fsection; fsection = fsection->next) {
	prot_printf(imapd_out, "%cBODY[%s ", sepchar, fsection->section);
	sepchar = '(';
	for (field = fsection->fields; field; field = field->next) {
	prot_putc(sepchar, imapd_out);
	sepchar = ' ';
	printastring(field->s);
	}
	prot_putc(')', imapd_out);
	sepchar = ' ';

	if (fsection->trail[1] == '<') {
	/* Have to trim off the maximum number of octets from the reply */
	partialdot = strrchr(fsection->trail, '.');
	*partialdot = '\0';
	prot_printf(imapd_out, "%s> ", fsection->trail);
	*partialdot = '.';
	}
	else {
	prot_printf(imapd_out, "%s ", fsection->trail);
	}

	if (fetchitems & FETCH_UNCACHEDHEADER) {
	cacheitem = cache_base + CACHE_OFFSET(msgno);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bodystructure */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */

	index_fetchfsection(msg_base, msg_size, mailbox->format, fsection,
	cacheitem);
	}
	else {
	index_fetchcacheheader(msgno, fsection->fields, fsection->trail);
	}
	}
	for (section = fetchargs->bodysections; section; section = section->next) {
	if (section->s[strlen(section->s)-1] == '>') {
	/* Have to trim off the maximum number of octets from the reply */
	partialdot = strrchr(section->s, '.');
	*partialdot = '\0';
	prot_printf(imapd_out, "%cBODY[%s> ", sepchar, section->s);
	*partialdot = '.';
	}
	else {
	prot_printf(imapd_out, "%cBODY[%s ", sepchar, section->s);
	}
	sepchar = ' ';
	cacheitem = cache_base + CACHE_OFFSET(msgno);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bodystructure */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */

	index_fetchsection(msg_base, msg_size, mailbox->format, section->s,
	cacheitem, SIZE(msgno),
	fetchargs->start_octet, fetchargs->octet_count);
	}
	prot_printf(imapd_out, ")\r\n");
	if (msg_base) {
	mailbox_unmap_message(mailbox, UID(msgno), &msg_base, &msg_size);
	}
	return 0;
	}

	/*
	* Helper function to perform a STORE command which only changes the
	* \Seen flag.
	*/
	static int
	index_storeseen(mailbox, msgno, rock)
	struct mailbox *mailbox;
	unsigned msgno;
	void *rock;
	{
	struct storeargs storeargs = (struct storeargs )rock;
	int val = (storeargs->operation == STORE_ADD) ? 1 : 0;
	int i;
	bit32 user_flags[MAX_USER_FLAGS/32];

	if (seenflag[msgno] == val) return 0;
	seenflag[msgno] = val;

	if (storeargs->silent) return 0;

	for (i=0; i < MAX_USER_FLAGS/32; i++) {
	user_flags[i] = USER_FLAGS(msgno, i);
	}
	index_fetchflags(mailbox, msgno, SYSTEM_FLAGS(msgno), user_flags,
	LAST_UPDATED(msgno));
	if (storeargs->usinguid) {
	prot_printf(imapd_out, " UID %u", UID(msgno));
	}
	prot_printf(imapd_out, ")\r\n");

	return 0;
	}

	/*
	* Helper function to perform a generalized STORE command
	*/
	static int
	index_storeflag(mailbox, msgno, rock)
	struct mailbox *mailbox;
	unsigned msgno;
	void *rock;
	{
	struct storeargs storeargs = (struct storeargs )rock;
	int i;
	struct index_record record;
	int uid = UID(msgno);
	int low=1, high=mailbox->exists;
	int mid = 0;
	int r;
	int firsttry = 1;
	int dirty = 0;
	bit32 oldflags;

	/* Change \Seen flag */
	if (storeargs->operation == STORE_REPLACE && (mailbox->myrights&ACL_SEEN))
	{
	seenflag[msgno] = storeargs->seen;
	}
	else if (storeargs->seen) {
	i = (storeargs->operation == STORE_ADD) ? 1 : 0;
	seenflag[msgno] = i;
	}

	/* Find index record */
	while (low <= high) {
	if (firsttry && msgno == storeargs->last_msgno+1) {
	/* Take "good" first guess */
	mid = storeargs->last_found + 1;
	if (mid > high) mid = high;
	}
	else {
	mid = (high - low)/2 + low;
	}
	firsttry = 0;
	r = mailbox_read_index_record(mailbox, mid, &record);
	if (r) return r;
	if (record.uid == uid) {
	break;
	}
	else if (record.uid > uid) {
	high = mid - 1;
	}
	else {
	low = mid + 1;
	}
	}
	storeargs->last_msgno = msgno;
	storeargs->last_found = mid;

	if (low > high) {
	/* Message was expunged. */
	if (storeargs->usinguid) {
	/* We're going to * n EXPUNGE it */
	return 0;
	}

	/* Fake setting the flags */
	mid = 0;
	storeargs->last_found = high;
	record.system_flags = SYSTEM_FLAGS(msgno);
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	record.user_flags[i] = USER_FLAGS(msgno, i);
	}
	}

	/* save old for acapmbox foo */
	oldflags = record.system_flags;

	if (storeargs->operation == STORE_REPLACE) {
	if (!(mailbox->myrights & ACL_WRITE)) {
	record.system_flags = (record.system_flags&~FLAG_DELETED) \|
	(storeargs->system_flags&FLAG_DELETED);
	}
	else {
	if (!(mailbox->myrights & ACL_DELETE)) {
	record.system_flags = (record.system_flags&FLAG_DELETED) \|
	(storeargs->system_flags&~FLAG_DELETED);
	}
	else {
	record.system_flags = storeargs->system_flags;
	}
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	record.user_flags[i] = storeargs->user_flags[i];
	}
	}
	dirty++; /* Don't try to be clever */
	}
	else if (storeargs->operation == STORE_ADD) {
	if (~record.system_flags & storeargs->system_flags) dirty++;

	record.system_flags \|= storeargs->system_flags;
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	if (~record.user_flags[i] & storeargs->user_flags[i]) dirty++;
	record.user_flags[i] \|= storeargs->user_flags[i];
	}
	}
	else { /* STORE_REMOVE */
	if (record.system_flags & storeargs->system_flags) dirty++;

	/* change the individual entry */
	record.system_flags &= ~storeargs->system_flags;
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	if (record.user_flags[i] & storeargs->user_flags[i]) dirty++;
	record.user_flags[i] &= ~storeargs->user_flags[i];
	}
	}

	if (dirty) {
	/* update totals */
	if ( (record.system_flags & FLAG_DELETED) && !(oldflags & FLAG_DELETED))
	mailbox->deleted++;
	if ( !(record.system_flags & FLAG_DELETED) && (oldflags & FLAG_DELETED))
	mailbox->deleted--;

	if ( (record.system_flags & FLAG_ANSWERED) && !(oldflags & FLAG_ANSWERED))
	mailbox->answered++;
	if ( !(record.system_flags & FLAG_ANSWERED) && (oldflags & FLAG_ANSWERED))
	mailbox->answered--;

	if ( (record.system_flags & FLAG_FLAGGED) && !(oldflags & FLAG_FLAGGED))
	mailbox->flagged++;
	if ( !(record.system_flags & FLAG_FLAGGED) && (oldflags & FLAG_FLAGGED))
	mailbox->flagged--;

	/* either a system or user flag changed. need to at least touch acap
	to change the modtime */
	mailbox->dirty = 1;

	/* If .SILENT, assume client has updated their cache */
	if (storeargs->silent &&
	flagreport[msgno] == record.last_updated) {
	flagreport[msgno] =
	(record.last_updated >= storeargs->update_time) ?
	record.last_updated + 1 : storeargs->update_time;
	}

	record.last_updated =
	(record.last_updated >= storeargs->update_time) ?
	record.last_updated + 1 : storeargs->update_time;
	}

	if (!storeargs->silent) {
	index_fetchflags(mailbox, msgno, record.system_flags,
	record.user_flags, record.last_updated);
	if (storeargs->usinguid) {
	prot_printf(imapd_out, " UID %u", UID(msgno));
	}
	prot_printf(imapd_out, ")\r\n");
	}

	if (dirty && mid) {
	r = mailbox_write_index_record(mailbox, mid, &record, 0);
	if (r) return r;
	}

	return 0;
	}

	/*
	* Evaluate a searchargs structure on a msgno
	*
	* Note: msgfile argument must be 0 if msg is not mapped in.
	*/
	static int index_search_evaluate(struct mailbox *mailbox,
	struct searchargs *searchargs,
	unsigned msgno,
	struct mapfile *msgfile)
	{
	int i;
	struct strlist l, h;
	const char *cacheitem;
	int cachelen;
	struct searchsub *s;

	if ((searchargs->flags & SEARCH_RECENT_SET) && msgno <= lastnotrecent)
	return 0;
	if ((searchargs->flags & SEARCH_RECENT_UNSET) && msgno > lastnotrecent)
	return 0;
	if ((searchargs->flags & SEARCH_SEEN_SET) && !seenflag[msgno]) return 0;
	if ((searchargs->flags & SEARCH_SEEN_UNSET) && seenflag[msgno]) return 0;

	if (searchargs->smaller && SIZE(msgno) >= searchargs->smaller) return 0;
	if (searchargs->larger && SIZE(msgno) <= searchargs->larger) return 0;

	if (searchargs->after && INTERNALDATE(msgno) < searchargs->after)
	return 0;
	if (searchargs->before && INTERNALDATE(msgno) > searchargs->before)
	return 0;
	if (searchargs->sentafter && SENTDATE(msgno) < searchargs->sentafter)
	return 0;
	if (searchargs->sentbefore && SENTDATE(msgno) > searchargs->sentbefore)
	return 0;

	if (~SYSTEM_FLAGS(msgno) & searchargs->system_flags_set) return 0;
	if (SYSTEM_FLAGS(msgno) & searchargs->system_flags_unset) return 0;

	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	if (~USER_FLAGS(msgno,i) & searchargs->user_flags_set[i])
	return 0;
	if (USER_FLAGS(msgno,i) & searchargs->user_flags_unset[i])
	return 0;
	}

	for (l = searchargs->sequence; l; l = l->next) {
	if (!index_insequence(msgno, l->s, 0)) return 0;
	}
	for (l = searchargs->uidsequence; l; l = l->next) {
	if (!index_insequence(UID(msgno), l->s, 1)) return 0;
	}

	if (searchargs->from \|\| searchargs->to \|\| searchargs->cc \|\|
	searchargs->bcc \|\| searchargs->subject \|\| searchargs->messageid) {

	cacheitem = cache_base + CACHE_OFFSET(msgno);
	cachelen = CACHE_ITEM_LEN(cacheitem);

	if (searchargs->messageid) {
	char *tmpenv;
	char *envtokens[NUMENVTOKENS];
	char *msgid;
	int msgidlen;

	/* get msgid out of the envelope */

	/* get a working copy; strip outer ()'s */
	tmpenv = xstrndup(cacheitem + 5, cachelen - 2);
	parse_cached_envelope(tmpenv, envtokens);

	if (!envtokens[ENV_MSGID]) {
	/* free stuff */
	free(tmpenv);

	return 0;
	}

	msgid = lcase(envtokens[ENV_MSGID]);
	msgidlen = strlen(msgid);
	for (l = searchargs->messageid; l; l = l->next) {
	if (!charset_searchstring(l->s, l->p, msgid, msgidlen)) {
	break;
	}
	}

	/* free stuff */
	free(tmpenv);

	if (l) return 0;
	}

	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bodystructure */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip section */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip cacheheaders */
	cachelen = CACHE_ITEM_LEN(cacheitem);

	for (l = searchargs->from; l; l = l->next) {
	if (cachelen == 0 \|\|
	!charset_searchstring(l->s, l->p, cacheitem+4, cachelen))
	return 0;
	}

	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip from */
	cachelen = CACHE_ITEM_LEN(cacheitem);

	for (l = searchargs->to; l; l = l->next) {
	if (cachelen == 0 \|\|
	!charset_searchstring(l->s, l->p, cacheitem+4, cachelen))
	return 0;
	}

	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip to */
	cachelen = CACHE_ITEM_LEN(cacheitem);

	for (l = searchargs->cc; l; l = l->next) {
	if (cachelen == 0 \|\|
	!charset_searchstring(l->s, l->p, cacheitem+4, cachelen))
	return 0;
	}

	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip cc */
	cachelen = CACHE_ITEM_LEN(cacheitem);

	for (l = searchargs->bcc; l; l = l->next) {
	if (cachelen == 0 \|\|
	!charset_searchstring(l->s, l->p, cacheitem+4, cachelen))
	return 0;
	}

	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bcc */
	cachelen = CACHE_ITEM_LEN(cacheitem);

	for (l = searchargs->subject; l; l = l->next) {
	if ((cachelen == 3 && !strcmp(cacheitem+4, "NIL")) \|\|
	!charset_searchstring(l->s, l->p, cacheitem+4, cachelen))
	return 0;
	}
	}

	for (s = searchargs->sublist; s; s = s->next) {
	if (index_search_evaluate(mailbox, s->sub1, msgno, msgfile)) {
	if (!s->sub2) return 0;
	}
	else {
	if (s->sub2 &&
	!index_search_evaluate(mailbox, s->sub2, msgno, msgfile))
	return 0;
	}
	}

	if (searchargs->body \|\| searchargs->text \|\|
	(searchargs->flags & SEARCH_UNCACHEDHEADER)) {
	if (! msgfile->size) { /* Map the message in if we haven't before */
	if (mailbox_map_message(mailbox, 1, UID(msgno),
	&msgfile->base, &msgfile->size)) {
	return 0;
	}
	}

	h = searchargs->header_name;
	for (l = searchargs->header; l; (l = l->next), (h = h->next)) {
	if (!index_searchheader(h->s, l->s, l->p, msgfile, mailbox->format,
	HEADER_SIZE(msgno))) return 0;
	}

	cacheitem = cache_base + CACHE_OFFSET(msgno);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bodystructure */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */

	for (l = searchargs->body; l; l = l->next) {
	if (!index_searchmsg(l->s, l->p, msgfile, mailbox->format, 1,
	cacheitem)) return 0;
	}
	for (l = searchargs->text; l; l = l->next) {
	if (!index_searchmsg(l->s, l->p, msgfile, mailbox->format, 0,
	cacheitem)) return 0;
	}
	}
	else if (searchargs->header_name) {
	h = searchargs->header_name;
	for (l = searchargs->header; l; (l = l->next), (h = h->next)) {
	if (!index_searchcacheheader(msgno, h->s, l->s, l->p)) return 0;
	}
	}

	return 1;
	}

	/*
	* Search part of a message for a substring.
	* Keep this in sync with index_getsearchtextmsg!
	*/
	static int
	index_searchmsg(char *substr,
	comp_pat *pat,
	struct mapfile *msgfile,
	int format,
	int skipheader,
	const char *cacheitem)
	{
	int partsleft = 1;
	int subparts;
	int start, len, charset, encoding;
	char p, q;

	/* Won't find anything in a truncated file */
	if (msgfile->size == 0) return 0;

	cacheitem += 4;
	while (partsleft--) {
	subparts = CACHE_ITEM_BIT32(cacheitem);
	cacheitem += 4;
	if (subparts) {
	partsleft += subparts-1;

	if (skipheader) {
	skipheader = 0; /* Only skip top-level message header */
	}
	else {
	len = CACHE_ITEM_BIT32(cacheitem+4);
	if (len > 0) {
	p = index_readheader(msgfile->base, msgfile->size,
	format, CACHE_ITEM_BIT32(cacheitem),
	len);
	q = charset_decode1522(p, NULL, 0);
	if (charset_searchstring(substr, pat, q, strlen(q))) {
	free(q);
	return 1;
	}
	free(q);
	}
	}
	cacheitem += 5*4;

	while (--subparts) {
	start = CACHE_ITEM_BIT32(cacheitem+2*4);
	len = CACHE_ITEM_BIT32(cacheitem+3*4);
	charset = CACHE_ITEM_BIT32(cacheitem+4*4) >> 16;
	encoding = CACHE_ITEM_BIT32(cacheitem+4*4) & 0xff;

	if (start < msgfile->size && len > 0 &&
	charset >= 0 && charset < 0xffff) {
	if (charset_searchfile(substr, pat,
	msgfile->base + start,
	format == MAILBOX_FORMAT_NETNEWS,
	len, charset, encoding)) return 1;
	}
	cacheitem += 5*4;
	}
	}
	}

	return 0;
	}

	/*
	* Search named header of a message for a substring
	*/
	static int
	index_searchheader(name, substr, pat, msgfile, format, size)
	char *name;
	char *substr;
	comp_pat *pat;
	struct mapfile *msgfile;
	int format;
	int size;
	{
	char p, q;
	int r;
	static struct strlist header;

	header.s = name;

	p = index_readheader(msgfile->base, msgfile->size, format, 0, size);
	index_pruneheader(p, &header, 0);
	if (!p) return 0; / Header not present, fail */
	if (!substr) return 1; / Only checking existence, succeed */
	q = charset_decode1522(strchr(p, ':') + 1, NULL, 0);
	r = charset_searchstring(substr, pat, q, strlen(q));
	free(q);
	return r;
	}

	/*
	* Search named cached header of a message for a substring
	*/
	static int
	index_searchcacheheader(msgno, name, substr, pat)
	unsigned msgno;
	char *name;
	char *substr;
	comp_pat *pat;
	{
	char *q;
	static struct strlist header;
	static char *buf;
	static int bufsize;
	const char *cacheitem;
	unsigned size;
	int r;

	cacheitem = cache_base + CACHE_OFFSET(msgno);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bodystructure */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip section */

	size = CACHE_ITEM_LEN(cacheitem);
	if (!size) return 0; /* No cached headers, fail */
	if (bufsize < size+2) {
	bufsize = size+100;
	buf = xrealloc(buf, bufsize);
	}

	memcpy(buf, cacheitem+4, size);
	buf[size] = '\0';

	header.s = name;

	index_pruneheader(buf, &header, 0);
	if (!buf) return 0; / Header not present, fail */
	if (!substr) return 1; / Only checking existence, succeed */
	q = charset_decode1522(strchr(buf, ':') + 1, NULL, 0);
	r = charset_searchstring(substr, pat, q, strlen(q));
	free(q);
	return r;
	}


	/* This code was cribbed from index_searchmsg. Instead of checking for matches,
	we call charset_extractfile to send the entire text out to 'receiver'.
	Keep this in sync with index_searchmsg! */
	static void index_getsearchtextmsg(struct mailbox* mailbox,
	int uid,
	index_search_text_receiver_t receiver,
	void* rock,
	char const* cacheitem) {
	struct mapfile msgfile;
	int partsleft = 1;
	int subparts;
	int start, len, charset, encoding;
	int partcount = 0;
	char p, q;
	int format = mailbox->format;

	if (mailbox_map_message(mailbox, 0, uid, &msgfile.base, &msgfile.size)) {
	return;
	}

	/* Won't find anything in a truncated file */
	if (msgfile.size > 0) {
	cacheitem += 4;
	while (partsleft--) {
	subparts = CACHE_ITEM_BIT32(cacheitem);
	cacheitem += 4;
	if (subparts) {
	partsleft += subparts-1;

	partcount++;

	len = CACHE_ITEM_BIT32(cacheitem+4);
	if (len > 0) {
	p = index_readheader(msgfile.base, msgfile.size,
	format, CACHE_ITEM_BIT32(cacheitem),
	len);
	q = charset_decode1522(p, NULL, 0);
	if (partcount == 1) {
	receiver(uid, SEARCHINDEX_PART_HEADERS,
	SEARCHINDEX_CMD_STUFFPART, q, strlen(q), rock);
	receiver(uid, SEARCHINDEX_PART_BODY,
	SEARCHINDEX_CMD_BEGINPART, NULL, 0, rock);
	} else {
	receiver(uid, SEARCHINDEX_PART_BODY,
	SEARCHINDEX_CMD_APPENDPART, q, strlen(q), rock);
	}
	free(q);
	}
	cacheitem += 5*4;

	while (--subparts) {
	start = CACHE_ITEM_BIT32(cacheitem+2*4);
	len = CACHE_ITEM_BIT32(cacheitem+3*4);
	charset = CACHE_ITEM_BIT32(cacheitem+4*4) >> 16;
	encoding = CACHE_ITEM_BIT32(cacheitem+4*4) & 0xff;

	if (start < msgfile.size && len > 0 &&
	charset >= 0 && charset < 0xffff) {
	charset_extractfile(receiver, rock, uid,
	msgfile.base + start,
	format == MAILBOX_FORMAT_NETNEWS,
	len, charset, encoding);
	}
	cacheitem += 5*4;
	}
	}
	}

	receiver(uid, SEARCHINDEX_PART_BODY,
	SEARCHINDEX_CMD_ENDPART, NULL, 0, rock);
	}

	mailbox_unmap_message(mailbox, uid, &msgfile.base, &msgfile.size);
	}

	void index_getsearchtext(struct mailbox* mailbox,
	index_search_text_receiver_t receiver,
	void* rock) {
	int i;

	/* Send the converted text of every message out to the receiver. */
	for (i = 1; i <= imapd_exists; i++) {
	const char *cacheitem;
	int uid = UID(i);

	cacheitem = cache_base + CACHE_OFFSET(i);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip envelope */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bodystructure */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip body */

	index_getsearchtextmsg(mailbox, uid, receiver, rock, cacheitem);

	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip section */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip cacheheaders */

	receiver(uid, SEARCHINDEX_PART_FROM, SEARCHINDEX_CMD_STUFFPART,
	cacheitem + 4, CACHE_ITEM_LEN(cacheitem), rock);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip from */

	receiver(uid, SEARCHINDEX_PART_TO, SEARCHINDEX_CMD_STUFFPART,
	cacheitem + 4, CACHE_ITEM_LEN(cacheitem), rock);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip to */

	receiver(uid, SEARCHINDEX_PART_CC, SEARCHINDEX_CMD_STUFFPART,
	cacheitem + 4, CACHE_ITEM_LEN(cacheitem), rock);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip cc */

	receiver(uid, SEARCHINDEX_PART_BCC, SEARCHINDEX_CMD_STUFFPART,
	cacheitem + 4, CACHE_ITEM_LEN(cacheitem), rock);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip bcc */

	receiver(uid, SEARCHINDEX_PART_SUBJECT, SEARCHINDEX_CMD_STUFFPART,
	cacheitem + 4, CACHE_ITEM_LEN(cacheitem), rock);
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* skip subject */
	}
	}

	/*
	* Helper function to set up arguments to append_copy()
	*/
	#define COPYARGSGROW 30
	static int
	index_copysetup(mailbox, msgno, rock)
	struct mailbox *mailbox;
	unsigned msgno;
	void *rock;
	{
	struct copyargs copyargs = (struct copyargs )rock;
	int flag = 0;
	unsigned userflag;
	bit32 flagmask = 0;

	if (copyargs->nummsg == copyargs->msgalloc) {
	copyargs->msgalloc += COPYARGSGROW;
	copyargs->copymsg = (struct copymsg *)
	xrealloc((char *)copyargs->copymsg,
	copyargs->msgalloc * sizeof(struct copymsg));
	}

	copyargs->copymsg[copyargs->nummsg].uid = UID(msgno);
	copyargs->copymsg[copyargs->nummsg].internaldate = INTERNALDATE(msgno);
	copyargs->copymsg[copyargs->nummsg].sentdate = SENTDATE(msgno);
	copyargs->copymsg[copyargs->nummsg].size = SIZE(msgno);
	copyargs->copymsg[copyargs->nummsg].header_size = HEADER_SIZE(msgno);
	copyargs->copymsg[copyargs->nummsg].cache_begin = cache_base + CACHE_OFFSET(msgno);
	if (mailbox->format != MAILBOX_FORMAT_NORMAL) {
	/* Force copy and re-parse of message */
	copyargs->copymsg[copyargs->nummsg].cache_len = 0;
	}
	else if (msgno < imapd_exists) {
	copyargs->copymsg[copyargs->nummsg].cache_len =
	CACHE_OFFSET(msgno+1) - CACHE_OFFSET(msgno);
	}
	else {
	copyargs->copymsg[copyargs->nummsg].cache_len =
	cache_end - CACHE_OFFSET(msgno);
	}
	copyargs->copymsg[copyargs->nummsg].seen = seenflag[msgno];
	copyargs->copymsg[copyargs->nummsg].system_flags = SYSTEM_FLAGS(msgno);

	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if ((userflag & 31) == 0) {
	flagmask = USER_FLAGS(msgno,userflag/32);
	}
	if (!mailbox->flagname[userflag] && (flagmask & (1<<(userflag&31)))) {
	mailbox_read_header(mailbox);
	index_listflags(mailbox);
	break;
	}
	}

	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if ((userflag & 31) == 0) {
	flagmask = USER_FLAGS(msgno,userflag/32);
	}
	if (mailbox->flagname[userflag] && (flagmask & (1<<(userflag&31)))) {
	copyargs->copymsg[copyargs->nummsg].flag[flag++] =
	mailbox->flagname[userflag];
	}
	}
	copyargs->copymsg[copyargs->nummsg].flag[flag] = 0;

	copyargs->nummsg++;

	return 0;
	}

	/*
	* Creates a list of msgdata.
	*
	* We fill these structs with the processed info that will be needed
	* by the specified sort criteria.
	*/
	#define ANNOTGROWSIZE 10

	static MsgData index_msgdata_load(unsigned msgno_list, int n,
	struct sortcrit *sortcrit)
	{
	MsgData md, cur;
	const char cacheitem = NULL, env = NULL,
	headers = NULL, from = NULL, to = NULL, cc = NULL, *subj = NULL;
	int i, j;
	char *tmpenv;
	char *envtokens[NUMENVTOKENS];
	int did_cache, did_env;
	int label;
	int annotsize;

	if (!n)
	return NULL;

	/* create an array of MsgData to use as nodes of linked list */
	md = (MsgData ) xmalloc(n sizeof(MsgData));
	memset(md, 0, n * sizeof(MsgData));

	for (i = 0, cur = md; i < n; i++, cur = cur->next) {
	/* set msgno */
	cur->msgno = msgno_list[i];

	/* set pointer to next node */
	cur->next = (i+1 < n ? cur+1 : NULL);

	did_cache = did_env = 0;
	tmpenv = NULL;
	annotsize = 0;

	for (j = 0; sortcrit[j].key; j++) {
	label = sortcrit[j].key;

	if ((label == SORT_CC \|\| label == SORT_DATE \|\|
	label == SORT_FROM \|\| label == SORT_SUBJECT \|\|
	label == SORT_TO \|\| label == LOAD_IDS) &&
	!did_cache) {

	/* fetch cached info */
	env = cache_base + CACHE_OFFSET(cur->msgno);
	cacheitem = CACHE_ITEM_NEXT(env); /* bodystructure */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* body */
	cacheitem = CACHE_ITEM_NEXT(cacheitem); /* section */
	headers = CACHE_ITEM_NEXT(cacheitem);
	from = CACHE_ITEM_NEXT(headers);
	to = CACHE_ITEM_NEXT(from);
	cc = CACHE_ITEM_NEXT(to);
	cacheitem = CACHE_ITEM_NEXT(cc); /* bcc */
	subj = CACHE_ITEM_NEXT(cacheitem);

	did_cache++;
	}

	if ((label == SORT_DATE \|\| label == LOAD_IDS) &&
	!did_env) {

	/* make a working copy of envelope -- strip outer ()'s */
	tmpenv = xstrndup(env+5, strlen(env+4) - 2);

	/* parse envelope into tokens */
	parse_cached_envelope(tmpenv, envtokens);

	did_env++;
	}

	switch (label) {
	case SORT_CC:
	cur->cc = get_localpart_addr(cc+4);
	break;
	case SORT_DATE:
	cur->date = message_parse_date(envtokens[ENV_DATE],
	PARSE_TIME \| PARSE_ZONE);
	break;
	case SORT_FROM:
	cur->from = get_localpart_addr(from+4);
	break;
	case SORT_SUBJECT:
	cur->xsubj = index_extract_subject(subj+4, &cur->is_refwd);
	cur->xsubj_hash = hash(cur->xsubj);
	break;
	case SORT_TO:
	cur->to = get_localpart_addr(to+4);
	break;
	case SORT_ANNOTATION:
	/* reallocate space for the annotation values if necessary */
	if (cur->nannot == annotsize) {
	annotsize += ANNOTGROWSIZE;
	cur->annot = (char **)
	xrealloc(cur->annot, annotsize * sizeof(char *));
	}

	/* fetch attribute value - we fake it for now */
	cur->annot[cur->nannot] = xstrdup(sortcrit[j].args.annot.attrib);
	cur->nannot++;
	break;
	case LOAD_IDS:
	index_get_ids(cur, envtokens, headers+4);
	break;
	}
	}

	if (tmpenv) free(tmpenv);
	}

	return md;
	}

	/*
	* Parse a cached envelope into individual tokens
	*
	* When inside a list (ncom > 0), we parse the individual tokens but don't
	* isolate them -- we return the entire list as a single token.
	*/
	static void parse_cached_envelope(char env, char tokens[])
	{
	char *c;
	int i = 0, ncom = 0, len;

	c = env;
	while (*c != '\0') {
	switch (*c) {
	case ' ': /* end of token */
	if (!ncom) c = '\0'; / mark end of token */
	c++;
	break;
	case 'N': /* "NIL" */
	case 'n':
	if (!ncom)
	tokens[i++] = NULL; /* empty token */
	c += 3; /* skip "NIL" */
	break;
	case '"': /* quoted string */
	c++; /* skip open quote */
	if (!ncom) tokens[i++] = c; /* start of string */
	while (c != '"') { / find close quote */
	if (c == '\\') c++; / skip quoted-specials */
	c++;
	}
	if (!ncom) c = '\0'; / end of string */
	c++; /* skip close quote */
	break;
	case '{': /* literal */
	c++; /* skip open brace */
	len = 0; /* determine length of literal */
	while (isdigit((int) *c)) {
	len = len10 + c - '0';
	c++;
	}
	c += 3; /* skip close brace & CRLF */
	if (!ncom) tokens[i++] = c; /* start of literal */
	c += len; /* skip literal */
	break;
	case '(': /* start of address */
	c++; /* skip open paren */
	if (!ncom) tokens[i++] = c; /* start of address list */
	ncom++; /* new open - inc counter */
	break;
	case ')': /* end of address */
	c++; /* skip close paren */
	if (ncom) { /* paranoia */
	ncom--; /* close - dec counter */
	if (!ncom) /* all open paren are closed */
	(c-1) = '\0'; / end of list - trim close paren */
	}
	break;
	default:
	/* yikes! unparsed junk, just skip it */
	c++;
	break;
	}
	}
	}

	/*
	* Get the 'local-part' of an address from a header
	*/
	static char get_localpart_addr(const char header)
	{
	struct address *addr = NULL;
	char *ret;

	parseaddr_list(header, &addr);
	ret = xstrdup(addr && addr->mailbox ? addr->mailbox : "");
	parseaddr_free(addr);
	return ret;
	}

	/*
	* Extract base subject from subject header
	*
	* This is a wrapper around _index_extract_subject() which preps the
	* subj NSTRING and checks for Netscape "[Fwd: ]".
	*/
	static char index_extract_subject(const char subj, int *is_refwd)
	{
	char buf, s, *base;

	/* parse the subj NSTRING and make a working copy */
	if (!strcmp(subj, "NIL")) /* NIL? */
	return xstrdup(""); /* yes, return empty */

	else
	buf = (subj == '"') ? / quoted? */
	xstrndup(subj + 1, strlen(subj) - 2) : /* yes, strip quotes */
	xstrdup(strchr(subj, '}') + 3); /* literal, skip { } */

	for (s = buf;;) {
	base = _index_extract_subject(s, is_refwd);

	/* If we have a Netscape "[Fwd: ...]", extract the contents */
	if (!strncasecmp(base, "[fwd:", 5) &&
	base[strlen(base) - 1] == ']') {

	/* inc refwd counter */
	*is_refwd += 1;

	/* trim "]" */
	base[strlen(base) - 1] = '\0';

	/* trim "[fwd:" */
	s = base + 5;
	}
	else /* otherwise, we're done */
	break;
	}

	base = xstrdup(base);

	free(buf);

	return base;
	}

	/*
	* Guts if subject extraction.
	*
	* Takes a subject string and returns a pointer to the base.
	*/
	static char _index_extract_subject(char s, int *is_refwd)
	{
	char base, x;

	/* trim trailer
	*
	* start at the end of the string and work towards the front,
	* resetting the end of the string as we go.
	*/
	for (x = s + strlen(s) - 1; x >= s;) {
	if (isspace((int) x)) { / whitespace? */
	x = '\0'; / yes, trim it */
	x--; /* skip past it */
	}
	else if (x - s >= 4 &&
	!strncasecmp(x-4, "(fwd)", 5)) { /* "(fwd)"? */
	(x-4) = '\0'; / yes, trim it */
	x -= 5; /* skip past it */
	is_refwd += 1; / inc refwd counter */
	}
	else
	break; /* we're done */
	}

	/* trim leader
	*
	* start at the head of the string and work towards the end,
	* skipping over stuff we don't care about.
	*/
	for (base = s; base;) {
	if (isspace((int) base)) base++; / whitespace? */

	/* possible refwd */
	else if ((!strncasecmp(base, "re", 2) && /* "re"? */
	(x = base + 2)) \|\| /* yes, skip past it */
	(!strncasecmp(base, "fwd", 3) && /* "fwd"? */
	(x = base + 3)) \|\| /* yes, skip past it */
	(!strncasecmp(base, "fw", 2) && /* "fw"? */
	(x = base + 2))) { /* yes, skip past it */
	int count = 0; /* init counter */

	while (isspace((int) x)) x++; / skip whitespace */

	if (x == '[') { / start of blob? */
	for (x++; x;) { /* yes, get count */
	if (!x) { / end of subj, quit */
	x = NULL;
	break;
	}
	else if (x == ']') / end of blob, done */
	break;
	/* if we have a digit, and we're still
	counting, keep building the count */
	else if (isdigit((int) *x) && count != -1)
	count = count * 10 + *x - '0';
	else /* no digit, */
	count = -1; /* abort counting */
	x++;
	}

	if (x) /* end of blob? */
	x++; /* yes, skip past it */
	else
	break; /* no, we're done */
	}

	while (isspace((int) x)) x++; / skip whitespace */

	if (x == ':') { / ending colon? */
	base = x + 1; /* yes, skip past it */
	is_refwd += (count > 0 ? count : 1); / inc refwd counter
	by count or 1 */
	}
	else
	break; /* no, we're done */
	}

	#if 0 /* do nested blobs - wait for decision on this */
	else if (base == '[') { / start of blob? */
	int count = 1; /* yes, */
	x = base + 1; /* find end of blob */
	while (count) { /* find matching ']' */
	if (!x) { / end of subj, quit */
	x = NULL;
	break;
	}
	else if (x == '[') / new open */
	count++; /* inc counter */
	else if (x == ']') / close */
	count--; /* dec counter */
	x++;
	}

	if (!x) /* blob didn't close */
	break; /* so quit */

	else if (x) / end of subj? */
	base = x; /* no, skip blob */
	#else
	else if (base == '[' && / start of blob? */
	(x = strpbrk(base+1, "[]")) && /* yes, end of blob */
	x == ']') { / (w/o nesting)? */

	if ((x+1)) / yes, end of subj? */
	base = x + 1; /* no, skip blob */
	#endif
	else
	break; /* yes, return blob */
	}
	else
	break; /* we're done */
	}

	return base;
	}

	/* Find a message-id looking thingy in a string. Returns a pointer to the
	* id and the length is returned in the *len parameter.
	*
	* This is a poor-man's way of finding the message-id. We simply look for
	* any string having the format "< ... @ ... >" and assume that the mail
	* client created a properly formatted message-id.
	*/
	static char find_msgid(char str, int *len)
	{
	char s, p;

	*len = 0;
	p = str;
	while (p && (s = strchr(p, '<'))) {
	p = s + 1;
	while (p && p != '@' && p != '<' && p != '>') p++;
	if (*p != '@') continue;
	while (p && p != '<' && *p != '>') p++;
	if (*p == '>') {
	*len = p - s + 1;
	return s;
	}
	}

	return NULL;
	}

	/* Get message-id, and references/in-reply-to */
	#define REFGROWSIZE 10

	void index_get_ids(MsgData msgdata, char envtokens[], const char *headers)
	{
	char msgid, refstr, ref, in_reply_to;
	int len, refsize = REFGROWSIZE;
	char buf[100];

	/* get msgid */
	msgid = find_msgid(envtokens[ENV_MSGID], &len);
	/* if we have one, make a copy of it */
	if (msgid)
	msgdata->msgid = xstrndup(msgid, len);
	/* otherwise, create one */
	else {
	sprintf(buf, "<Empty-ID: %u>", msgdata->msgno);
	msgdata->msgid = xstrdup(buf);
	}

	/* grab the References header */
	if ((refstr = stristr(headers, "references:"))) {
	/* allocate some space for refs */
	msgdata->ref = (char *) xmalloc(refsize sizeof(char *));
	/* find references */
	while ((ref = find_msgid(refstr, &len)) != NULL) {
	/* reallocate space for this msgid if necessary */
	if (msgdata->nref == refsize) {
	refsize += REFGROWSIZE;
	msgdata->ref = (char **)
	xrealloc(msgdata->ref, refsize * sizeof(char *));
	}
	/* store this msgid in the array */
	msgdata->ref[msgdata->nref++] = xstrndup(ref, len);
	/* skip past this msgid */
	refstr = ref + len;
	}
	}

	/* if we have no references, try in-reply-to */
	if (!msgdata->nref) {
	/* get in-reply-to id */
	in_reply_to = find_msgid(envtokens[ENV_INREPLYTO], &len);
	/* if we have an in-reply-to id, make it the ref */
	if (in_reply_to) {
	msgdata->ref = (char *) xmalloc(sizeof(char ));
	msgdata->ref[msgdata->nref++] = xstrndup(in_reply_to, len);
	}
	}
	}

	/*
	* Getnext function for sorting message lists.
	*/
	static void index_sort_getnext(MsgData node)
	{
	return node->next;
	}

	/*
	* Setnext function for sorting message lists.
	*/
	static void index_sort_setnext(MsgData node, MsgData next)
	{
	node->next = next;
	}

	/*
	* Function for comparing two integers.
	*/
	static int numcmp(int i1, int i2)
	{
	return ((i1 < i2) ? -1 : (i1 > i2) ? 1 : 0);
	}

	/*
	* Comparison function for sorting message lists.
	*/
	static int index_sort_compare(MsgData md1, MsgData md2,
	struct sortcrit *sortcrit)
	{
	int reverse, ret = 0, i = 0, ann = 0;

	do {
	/* determine sort order from reverse flag bit */
	reverse = sortcrit[i].flags & SORT_REVERSE;

	switch (sortcrit[i].key) {
	case SORT_SEQUENCE:
	ret = numcmp(md1->msgno, md2->msgno);
	break;
	case SORT_ARRIVAL:
	ret = numcmp(INTERNALDATE(md1->msgno), INTERNALDATE(md2->msgno));
	break;
	case SORT_CC:
	ret = strcmp(md1->cc, md2->cc);
	break;
	case SORT_DATE:
	ret = (md1->date && md2->date) ?
	numcmp(md1->date, md2->date) : numcmp(md1->msgno, md2->msgno);
	break;
	case SORT_FROM:
	ret = strcmp(md1->from, md2->from);
	break;
	case SORT_SIZE:
	ret = numcmp(SIZE(md1->msgno), SIZE(md2->msgno));
	break;
	case SORT_SUBJECT:
	ret = strcmp(md1->xsubj, md2->xsubj);
	break;
	case SORT_TO:
	ret = strcmp(md1->to, md2->to);
	break;
	case SORT_ANNOTATION:
	ret = strcmp(md1->annot[ann], md2->annot[ann]);
	ann++;
	break;
	}
	} while (!ret && sortcrit[i++].key != SORT_SEQUENCE);

	return (reverse ? -ret : ret);
	}

	/*
	* Free a msgdata node.
	*/
	static void index_msgdata_free(MsgData *md)
	{
	#define FREE(x) if (x) free(x)
	int i;

	if (!md)
	return;
	FREE(md->cc);
	FREE(md->from);
	FREE(md->to);
	FREE(md->xsubj);
	FREE(md->msgid);
	for (i = 0; i < md->nref; i++)
	free(md->ref[i]);
	FREE(md->ref);
	for (i = 0; i < md->nannot; i++)
	free(md->annot[i]);
	FREE(md->annot);
	}

	/*
	* Getnext function for sorting thread lists.
	*/
	static void index_thread_getnext(Thread thread)
	{
	return thread->next;
	}

	/*
	* Setnext function for sorting thread lists.
	*/
	static void index_thread_setnext(Thread thread, Thread next)
	{
	thread->next = next;
	}

	/*
	* Comparison function for sorting threads.
	*/
	static int index_thread_compare(Thread t1, Thread t2,
	struct sortcrit *call_data)
	{
	MsgData md1, md2;

	/* if the container is empty, use the first child's container */
	md1 = t1->msgdata ? t1->msgdata : t1->child->msgdata;
	md2 = t2->msgdata ? t2->msgdata : t2->child->msgdata;
	return index_sort_compare(md1, md2, call_data);
	}

	/*
	* Sort a list of threads.
	*/
	static void index_thread_sort(Thread root, struct sortcrit sortcrit)
	{
	Thread *child;

	/* sort the grandchildren */
	child = root->child;
	while (child) {
	/* if the child has children, sort them */
	if (child->child)
	index_thread_sort(child, sortcrit);
	child = child->next;
	}

	/* sort the children */
	root->child = lsort(root->child,
	(void * ()(void)) index_thread_getnext,
	(void ()(void,void*)) index_thread_setnext,
	(int ()(void,void,void)) index_thread_compare,
	sortcrit);
	}

	/*
	* Thread a list of messages using the ORDEREDSUBJECT algorithm.
	*/
	static void index_thread_orderedsubj(unsigned *msgno_list, int nmsg,
	int usinguid)
	{
	MsgData msgdata, freeme;
	struct sortcrit sortcrit[] = {{ SORT_SUBJECT, 0, {{NULL, NULL}} },
	{ SORT_DATE, 0, {{NULL, NULL}} },
	{ SORT_SEQUENCE, 0, {{NULL, NULL}} }};
	unsigned psubj_hash = 0;
	char *psubj;
	Thread head, newnode, cur, parent;

	/* Create/load the msgdata array */
	freeme = msgdata = index_msgdata_load(msgno_list, nmsg, sortcrit);

	/* Sort messages by subject and date */
	msgdata = lsort(msgdata,
	(void * ()(void)) index_sort_getnext,
	(void ()(void,void*)) index_sort_setnext,
	(int ()(void,void,void)) index_sort_compare,
	sortcrit);

	/* create an array of Thread to use as nodes of thread tree
	*
	* we will be building threads under a dummy head,
	* so we need (nmsg + 1) nodes
	*/
	head = (Thread ) xmalloc((nmsg + 1) sizeof(Thread));
	memset(head, 0, (nmsg + 1) * sizeof(Thread));

	newnode = head + 1; /* set next newnode to the second
	one in the array (skip the head) */
	parent = head; /* parent is the head node */
	psubj = NULL; /* no previous subject */
	cur = NULL; /* no current thread */

	while (msgdata) {
	/* if no previous subj, or
	current subj = prev subj (subjs have same hash, and
	the strings are equal), then add message to current thread */
	if (!psubj \|\|
	(msgdata->xsubj_hash == psubj_hash &&
	!strcmp(msgdata->xsubj, psubj))) {
	parent->child = newnode; /* create a new child */
	parent->child->msgdata = msgdata;
	if (!cur)
	cur = parent->child; /* first thread */
	parent = parent->child; /* this'll be the parent
	next time around */
	}
	/* otherwise, create a new thread */
	else {
	cur->next = newnode; /* create and start a new thread */
	cur->next->msgdata = msgdata;
	parent = cur = cur->next; /* now work with the new thread */
	}

	psubj_hash = msgdata->xsubj_hash;
	psubj = msgdata->xsubj;
	msgdata = msgdata->next;
	newnode++;
	}

	/* Sort threads by date */
	index_thread_sort(head, sortcrit+1);

	/* Output the threaded messages */
	index_thread_print(head, usinguid);

	/* free the thread array */
	free(head);

	/* free the msgdata array */
	free(freeme);
	}

	/*
	* Guts of thread printing. Recurses over children when necessary.
	*
	* Frees contents of msgdata as a side effect.
	*/
	static void _index_thread_print(Thread *thread, int usinguid)
	{
	Thread *child;

	/* for each thread... */
	while (thread) {
	/* start the thread */
	prot_printf(imapd_out, "(");

	/* if we have a message, print its identifier
	* (do nothing for empty containers)
	*/
	if (thread->msgdata) {
	prot_printf(imapd_out, "%u",
	usinguid ? UID(thread->msgdata->msgno) :
	thread->msgdata->msgno);

	/* if we have a child, print the parent-child separator */
	if (thread->child) prot_printf(imapd_out, " ");

	/* free contents of the current node */
	index_msgdata_free(thread->msgdata);
	}

	/* for each child, grandchild, etc... */
	child = thread->child;
	while (child) {
	/* if the child has siblings, print new branch and break */
	if (child->next) {
	_index_thread_print(child, usinguid);
	break;
	}
	/* otherwise print the only child */
	else {
	prot_printf(imapd_out, "%u",
	usinguid ? UID(child->msgdata->msgno) :
	child->msgdata->msgno);

	/* if we have a child, print the parent-child separator */
	if (child->child) prot_printf(imapd_out, " ");

	/* free contents of the child node */
	index_msgdata_free(child->msgdata);

	child = child->child;
	}
	}

	/* end the thread */
	prot_printf(imapd_out, ")");

	thread = thread->next;
	}
	}

	/*
	* Print a list of threads.
	*
	* This is a wrapper around _index_thread_print() which simply prints the
	* start and end of the untagged thread response.
	*/
	static void index_thread_print(Thread *thread, int usinguid)
	{
	prot_printf(imapd_out, "* THREAD");

	if (thread) {
	prot_printf(imapd_out, " ");
	_index_thread_print(thread->child, usinguid);
	}

	prot_printf(imapd_out, "\r\n");
	}

	/*
	* Find threading algorithm for given arg.
	* Returns index into thread_algs[], or -1 if not found.
	*/
	int find_thread_algorithm(char *arg)
	{
	int alg;

	ucase(arg);
	for (alg = 0; thread_algs[alg].alg_name; alg++) {
	if (!strcmp(arg, thread_algs[alg].alg_name))
	return alg;
	}
	return -1;
	}

	/*
	* The following code is an interpretation of JWZ's description
	* and pseudo-code in http://www.jwz.org/doc/threading.html.
	*
	* It has been modified to match the THREAD=REFERENCES algorithm.
	*/

	/*
	* Determines if child is a descendent of parent.
	*
	* Returns 1 if yes, 0 otherwise.
	*/
	static int thread_is_descendent(Thread parent, Thread child)
	{
	Thread *kid;

	/* self */
	if (parent == child)
	return 1;

	/* search each child's decendents */
	for (kid = parent->child; kid; kid = kid->next) {
	if (thread_is_descendent(kid, child))
	return 1;
	}
	return 0;
	}

	/*
	* Links child into parent's children.
	*/
	static void thread_adopt_child(Thread parent, Thread child)
	{
	child->parent = parent;
	child->next = parent->child;
	parent->child = child;
	}

	/*
	* Unlinks child from it's parent's children.
	*/
	static void thread_orphan_child(Thread *child)
	{
	Thread prev, cur;

	/* sanity check -- make sure child is actually a child of parent */
	for (prev = NULL, cur = child->parent->child;
	cur != child && cur != NULL; prev = cur, cur = cur->next);

	if (!cur) {
	/* uh oh! couldn't find the child in it's parent's children
	* we should probably return NO to thread command
	*/
	return;
	}

	/* unlink child */
	if (!prev) /* first child */
	child->parent->child = child->next;
	else
	prev->next = child->next;
	child->parent = child->next = NULL;
	}

	/*
	* Link messages together using message-id and references.
	*/
	static void ref_link_messages(MsgData msgdata, Thread *newnode,
	struct hash_table *id_table)
	{
	Thread cur, parent, *ref;
	int dup_count = 0;
	char buf[100];
	int i;

	/* for each message... */
	while (msgdata) {
	/* fill the containers with msgdata
	*
	* if we already have a container, use it
	*/
	if ((cur = (Thread *) hash_lookup(msgdata->msgid, id_table))) {
	/* If this container is not empty, then we have a duplicate
	* Message-ID. Make this one unique so that we don't stomp
	* on the old one.
	*/
	if (cur->msgdata) {
	sprintf(buf, "-dup%d", dup_count++);
	msgdata->msgid =
	(char *) xrealloc(msgdata->msgid,
	strlen(msgdata->msgid) + strlen(buf) + 1);
	strcat(msgdata->msgid, buf);
	/* clear cur so that we create a new container */
	cur = NULL;
	}
	else
	cur->msgdata = msgdata;
	}

	/* otherwise, make and index a new container */
	if (!cur) {
	cur = *newnode;
	cur->msgdata = msgdata;
	hash_insert(msgdata->msgid, cur, id_table);
	(*newnode)++;
	}

	/* Step 1.A */
	for (i = 0, parent = NULL; i < msgdata->nref; i++) {
	/* if we don't already have a container for the reference,
	* make and index a new (empty) container
	*/
	if (!(ref = (Thread *) hash_lookup(msgdata->ref[i], id_table))) {
	ref = *newnode;
	hash_insert(msgdata->ref[i], ref, id_table);
	(*newnode)++;
	}

	/* link the references together as parent-child iff:
	* - we won't change existing links, AND
	* - we won't create a loop
	*/
	if (!ref->parent &&
	parent && !thread_is_descendent(ref, parent)) {
	thread_adopt_child(parent, ref);
	}

	parent = ref;
	}

	/* Step 1.B
	*
	* if we have a parent already, it is probably bogus (the result
	* of a truncated references field), so unlink from it because
	* we now have the actual parent
	*/
	if (cur->parent) thread_orphan_child(cur);

	/* make the last reference the parent of our message iff:
	* - we won't create a loop
	*/
	if (parent && !thread_is_descendent(cur, parent))
	thread_adopt_child(parent, cur);

	msgdata = msgdata->next;
	}
	}

	/*
	* Gather orphan messages under the root node.
	*/
	static void ref_gather_orphans(char *key __attribute__((unused)),
	Thread *node,
	struct rootset *rootset)
	{
	/* we only care about nodes without parents */
	if (!node->parent) {
	if (node->next) {
	/* uh oh! a node without a parent should not have a sibling
	* we should probably return NO to thread command
	*/
	return;
	}

	/* add this node to root's children */
	node->next = rootset->root->child;
	rootset->root->child = node;
	rootset->nroot++;
	}
	}

	/*
	* Prune tree of empty containers.
	*/
	static void ref_prune_tree(Thread *parent)
	{
	Thread cur, prev, next, child;

	for (prev = NULL, cur = parent->child, next = cur->next;
	cur;
	prev = cur, cur = next, next = (cur ? cur->next : NULL)) {

	/* if we have an empty container with no children, delete it */
	if (!cur->msgdata && !cur->child) {
	if (!prev) /* first child */
	parent->child = cur->next;
	else
	prev->next = cur->next;

	/* we just removed cur from our list,
	* so we need to keep the same prev for the next pass
	*/
	cur = prev;
	}

	/* if we have empty container with children, AND
	* we're not at the root OR we only have one child,
	* then remove the container but promote its children to this level
	* (splice them into the current child list)
	*/
	else if (!cur->msgdata && cur->child &&
	(cur->parent \|\| !cur->child->next)) {
	/* move cur's children into cur's place (start the splice) */
	if (!prev) /* first child */
	parent->child = cur->child;
	else
	prev->next = cur->child;

	/* make cur's parent the new parent of cur's children
	* (they're moving in with grandma!)
	*/
	child = cur->child;
	do {
	child->parent = cur->parent;
	} while (child->next && (child = child->next));

	/* make the cur's last child point to cur's next sibling
	* (finish the splice)
	*/
	child->next = cur->next;

	/* we just replaced cur with it's children
	* so make it's first child the next node to process
	*/
	next = cur->child;

	/* make cur childless and siblingless */
	cur->child = cur->next = NULL;

	/* we just removed cur from our list,
	* so we need to keep the same prev for the next pass
	*/
	cur = prev;
	}

	/* if we have a message with children, prune it's children */
	else if (cur->child)
	ref_prune_tree(cur);
	}
	}

	/*
	* Sort the messages in the root set by date.
	*/
	static void ref_sort_root(Thread *root)
	{
	Thread *cur;
	struct sortcrit sortcrit[] = {{ SORT_DATE, 0, {{NULL, NULL}} },
	{ SORT_SEQUENCE, 0, {{NULL, NULL}} }};

	cur = root->child;
	while (cur) {
	/* if the message is a dummy, sort its children */
	if (!cur->msgdata) {
	cur->child = lsort(cur->child,
	(void * ()(void)) index_thread_getnext,
	(void ()(void,void*)) index_thread_setnext,
	(int ()(void,void,void)) index_thread_compare,
	sortcrit);
	}
	cur = cur->next;
	}

	/* sort the root set */
	root->child = lsort(root->child,
	(void * ()(void)) index_thread_getnext,
	(void ()(void,void*)) index_thread_setnext,
	(int ()(void,void,void)) index_thread_compare,
	sortcrit);
	}

	/*
	* Group threads with same subject.
	*/
	static void ref_group_subjects(Thread root, unsigned nroot, Thread *newnode)
	{
	Thread cur, old, prev, next, *child;
	struct hash_table subj_table;
	char *subj;

	/* Step 5.A: create a subj_table with one bucket for every possible
	* subject in the root set
	*/
	construct_hash_table(&subj_table, nroot);

	/* Step 5.B: populate the table with a container for each subject
	* at the root
	*/
	for (cur = root->child; cur; cur = cur->next) {
	/* Step 5.B.i: find subject of the thread
	*
	* if the container is not empty, use it's subject
	*/
	if (cur->msgdata)
	subj = cur->msgdata->xsubj;
	/* otherwise, use the subject of it's first child */
	else
	subj = cur->child->msgdata->xsubj;

	/* Step 5.B.ii: if subject is empty, skip it */
	if (!strlen(subj)) continue;

	/* Step 5.B.iii: lookup this subject in the table */
	old = (Thread *) hash_lookup(subj, &subj_table);

	/* Step 5.B.iv: insert the current container into the table iff:
	* - this subject is not in the table, OR
	* - this container is empty AND the one in the table is not
	* (the empty one is more interesting as a root), OR
	* - the container in the table is a re/fwd AND this one is not
	* (the non-re/fwd is the more interesting of the two)
	*/
	if (!old \|\|
	(!cur->msgdata && old->msgdata) \|\|
	(old->msgdata && old->msgdata->is_refwd &&
	cur->msgdata && !cur->msgdata->is_refwd)) {
	hash_insert(subj, cur, &subj_table);
	}
	}

	/* 5.C - group containers with the same subject together */
	for (prev = NULL, cur = root->child, next = cur->next;
	cur;
	prev = cur, cur = next, next = (next ? next->next : NULL)) {
	/* Step 5.C.i: find subject of the thread
	*
	* if container is not empty, use it's subject
	*/
	if (cur->msgdata)
	subj = cur->msgdata->xsubj;
	/* otherwise, use the subject of it's first child */
	else
	subj = cur->child->msgdata->xsubj;

	/* Step 5.C.ii: if subject is empty, skip it */
	if (!strlen(subj)) continue;

	/* Step 5.C.iii: lookup this subject in the table */
	old = (Thread *) hash_lookup(subj, &subj_table);

	/* Step 5.C.iv: if we found ourselves, skip it */
	if (old == cur) continue;

	/* ok, we already have a container which contains our current subject,
	* so pull this container out of the root set, because we are going to
	* merge this node with another one
	*/
	if (!prev) /* we're at the root */
	root->child = cur->next;
	else
	prev->next = cur->next;
	cur->next = NULL;

	/* if both containers are dummies, append cur's children to old's */
	if (!old->msgdata && !cur->msgdata) {
	/* find old's last child */
	for (child = old->child; child->next; child = child->next);

	/* append cur's children to old's children list */
	child->next = cur->child;

	/* make old the parent of cur's children */
	for (child = cur->child; child; child = child->next)
	child->parent = old;

	/* make cur childless */
	cur->child = NULL;
	}

	/* if:
	* - old container is empty, OR
	* - the current message is a re/fwd AND the old one is not,
	* make the current container a child of the old one
	*
	* Note: we don't have to worry about the reverse cases
	* because step 5.B guarantees that they won't happen
	*/
	else if (!old->msgdata \|\|
	(cur->msgdata && cur->msgdata->is_refwd &&
	!old->msgdata->is_refwd)) {
	thread_adopt_child(old, cur);
	}

	/* if both messages are re/fwds OR neither are re/fwds,
	* then make them both children of a new dummy container
	* (we don't want to assume any parent-child relationship between them)
	*
	* perhaps we can create a parent-child relationship
	* between re/fwds by counting the number of re/fwds
	*
	* Note: we need the hash table to still point to old,
	* so we must make old the dummy and make the contents of the
	* new container a copy of old's original contents
	*/
	else {
	Thread new = (newnode)++;

	/* make new a copy of old (except parent and next) */
	new->msgdata = old->msgdata;
	new->child = old->child;
	new->next = NULL;

	/* make new the parent of it's newly adopted children */
	for (child = new->child; child; child = child->next)
	child->parent = new;

	/* make old the parent of cur and new */
	cur->parent = old;
	new->parent = old;

	/* empty old and make it have two children (cur and new) */
	old->msgdata = NULL;
	old->child = cur;
	cur->next = new;
	}

	/* we just removed cur from our list,
	* so we need to keep the same prev for the next pass
	*/
	cur = prev;
	}

	free_hash_table(&subj_table, NULL);
	}

	/*
	* Free an entire thread.
	*/
	static void index_thread_free(Thread *thread)
	{
	Thread *child;

	/* free the head node */
	if (thread->msgdata) index_msgdata_free(thread->msgdata);

	/* free the children recursively */
	child = thread->child;
	while (child) {
	index_thread_free(child);
	child = child->next;
	}
	}

	/*
	* Guts of thread searching. Recurses over children when necessary.
	*/
	static int _index_thread_search(Thread thread, int (searchproc) (MsgData *))
	{
	Thread *child;

	/* test the head node */
	if (thread->msgdata && searchproc(thread->msgdata)) return 1;

	/* test the children recursively */
	child = thread->child;
	while (child) {
	if (_index_thread_search(child, searchproc)) return 1;
	child = child->next;
	}

	/* if we get here, we struck out */
	return 0;
	}

	/*
	* Search a thread to see if it contains a message which matches searchproc().
	*
	* This is a wrapper around _index_thread_search() which iterates through
	* each thread and removes any which fail the searchproc().
	*/
	static void index_thread_search(Thread root, int (searchproc) (MsgData *))
	{
	Thread cur, prev, *next;

	for (prev = NULL, cur = root->child, next = cur->next;
	cur;
	prev = cur, cur= next, next = (cur ? cur->next : NULL)) {
	if (!_index_thread_search(cur, searchproc)) {
	/* unlink the thread from the list */
	if (!prev) /* first thread */
	root->child = cur->next;
	else
	prev->next = cur->next;

	/* free all nodes in the thread */
	index_thread_free(cur);

	/* we just removed cur from our list,
	* so we need to keep the same prev for the next pass
	*/
	cur = prev;
	}
	}
	}

	/*
	* Guts of the REFERENCES algorithms. Behavior is tweaked with loadcrit[],
	* searchproc() and sortcrit[].
	*/
	static void _index_thread_ref(unsigned *msgno_list, int nmsg,
	struct sortcrit loadcrit[],
	int (searchproc) (MsgData ),
	struct sortcrit sortcrit[], int usinguid)
	{
	MsgData msgdata, freeme, *md;
	int tref, nnode;
	Thread *newnode;
	struct hash_table id_table;
	struct rootset rootset;

	/* Create/load the msgdata array */
	freeme = msgdata = index_msgdata_load(msgno_list, nmsg, loadcrit);

	/* calculate the sum of the number of references for all messages */
	for (md = msgdata, tref = 0; md; md = md->next)
	tref += md->nref;

	/* create an array of Thread to use as nodes of thread tree (including
	* empty containers)
	*
	* - We will be building threads under a dummy root, so we need at least
	* (nmsg + 1) nodes.
	* - We also will need containers for references to non-existent messages.
	* To make sure we have enough, we will take the worst case and
	* use the sum of the number of references for all messages.
	* - Finally, we will might need containers to group threads with the same
	* subject together. To make sure we have enough, we will take the
	* worst case which will be half of the number of messages.
	*
	* This is overkill, but it is the only way to make sure we have enough
	* ahead of time. If we tried to use xrealloc(), the array might be moved,
	* and our parent/child/next pointers will no longer be correct
	* (been there, done that).
	*/
	nnode = (int) (1.5 * nmsg + 1 + tref);
	rootset.root = (Thread ) xmalloc(nnode sizeof(Thread));
	memset(rootset.root, 0, nnode * sizeof(Thread));

	newnode = rootset.root + 1; /* set next newnode to the second
	one in the array (skip the root) */

	/* Step 0: create an id_table with one bucket for every possible
	* message-id and reference (nmsg + tref)
	*/
	construct_hash_table(&id_table, nmsg + tref);

	/* Step 1: link messages together */
	ref_link_messages(msgdata, &newnode, &id_table);

	/* Step 2: find the root set (gather all of the orphan messages) */
	rootset.nroot = 0;
	hash_enumerate(&id_table, (void ()(char,void,void)) ref_gather_orphans,
	&rootset);

	/* discard id_table */
	free_hash_table(&id_table, NULL);

	/* Step 3: prune tree of empty containers - get our deposit back :^) */
	ref_prune_tree(rootset.root);

	/* Step 4: sort the root set */
	ref_sort_root(rootset.root);

	/* Step 5: group root set by subject */
	ref_group_subjects(rootset.root, rootset.nroot, &newnode);

	/* Optionally search threads (to be used by REFERENCES derivatives) */
	if (searchproc) index_thread_search(rootset.root, searchproc);

	/* Step 6: sort threads */
	if (sortcrit) index_thread_sort(rootset.root, sortcrit);

	/* Output the threaded messages */
	index_thread_print(rootset.root, usinguid);

	/* free the thread array */
	free(rootset.root);

	/* free the msgdata array */
	free(freeme);
	}

	/*
	* Thread a list of messages using the REFERENCES algorithm.
	*/
	static void index_thread_ref(unsigned *msgno_list, int nmsg, int usinguid)
	{
	struct sortcrit loadcrit[] = {{ LOAD_IDS, 0, {{NULL,NULL}} },
	{ SORT_SUBJECT, 0, {{NULL,NULL}} },
	{ SORT_DATE, 0, {{NULL,NULL}} },
	{ SORT_SEQUENCE, 0, {{NULL,NULL}} }};
	struct sortcrit sortcrit[] = {{ SORT_DATE, 0, {{NULL,NULL}} },
	{ SORT_SEQUENCE, 0, {{NULL,NULL}} }};

	_index_thread_ref(msgno_list, nmsg, loadcrit, NULL, sortcrit, usinguid);
	}
	diff --git a/imap/mailbox.c b/imap/mailbox.c
	index 16f787527..76e235c1a 100644
	--- a/imap/mailbox.c
	+++ b/imap/mailbox.c
	@@ -1,2553 +1,2553 @@
	/* mailbox.c -- Mailbox manipulation routines
	- $Id: mailbox.c,v 1.118 2002/02/13 22:09:03 rjs3 Exp $
	+ $Id: mailbox.c,v 1.119 2002/02/19 18:50:13 ken3 Exp $

	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*

	*
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <errno.h>
	#include <syslog.h>
	#include <sys/types.h>
	#include <fcntl.h>
	#include <netinet/in.h>
	#include <sys/un.h>
	#include <sys/socket.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <ctype.h>
	#include <time.h>

	#ifdef HAVE_DIRENT_H
	# include <dirent.h>
	# define NAMLEN(dirent) strlen((dirent)->d_name)
	#else
	# define dirent direct
	# define NAMLEN(dirent) (dirent)->d_namlen
	# if HAVE_SYS_NDIR_H
	# include <sys/ndir.h>
	# endif
	# if HAVE_SYS_DIR_H
	# include <sys/dir.h>
	# endif
	# if HAVE_NDIR_H
	# include <ndir.h>
	# endif
	#endif

	#include "assert.h"
	#include "imapconf.h"
	#include "acl.h"
	#include "map.h"
	#include "retry.h"
	#include "util.h"
	#include "lock.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "xmalloc.h"
	#include "mboxlist.h"
	#include "acapmbox.h"
	#include "seen.h"
	#include "acappush.h"

	static int mailbox_doing_reconstruct = 0;
	#define zeromailbox(m) { memset(&m, 0, sizeof(struct mailbox)); \
	(m).header_fd = -1; \
	(m).index_fd = -1; \
	(m).cache_fd = -1; }

	static int mailbox_calculate_flagcounts(struct mailbox *mailbox);

	/*
	* Names of the headers we cache in the cyrus.cache file.
	* Any changes to this list require corresponding changes to
	* message_parse_headers() in message.c
	*/
	char *mailbox_cache_header_name[] = {
	/* "in-reply-to", in ENVELOPE */
	"priority",
	"references",
	"resent-from",
	"newsgroups",
	"followup-to",
	};
	int mailbox_num_cache_header =
	sizeof(mailbox_cache_header_name)/sizeof(char *);

	/* acappush variables */
	static int acappush_sock = -1;
	static struct sockaddr_un acappush_remote;
	static int acappush_remote_len = 0;

	/* function to be used for notification of mailbox changes/updates */
	static mailbox_notifyproc_t *updatenotifier = NULL;

	/*
	* Set the updatenotifier function
	*/
	void mailbox_set_updatenotifier(mailbox_notifyproc_t *notifyproc)
	{
	updatenotifier = notifyproc;
	}

	/*
	* Create connection to acappush
	*/
	int mailbox_initialize(void)
	{
	int s;
	int fdflags;
	struct stat sbuf;

	/* if not configured to do acap do nothing */
	if (config_getstring("acap_server", NULL)==NULL) return 0;

	if ((s = socket(AF_UNIX, SOCK_DGRAM, 0)) == -1) {
	return IMAP_IOERROR;
	}

	acappush_remote.sun_family = AF_UNIX;
	strcpy(acappush_remote.sun_path, config_dir);
	strcat(acappush_remote.sun_path, FNAME_ACAPPUSH_SOCK);
	acappush_remote_len = sizeof(acappush_remote.sun_family) +
	strlen(acappush_remote.sun_path) + 1;

	/* check that the socket exists */
	if (stat(acappush_remote.sun_path, &sbuf) < 0) {
	close(s);
	return 0;
	}

	/* put us in non-blocking mode */
	fdflags = fcntl(s, F_GETFD, 0);
	if (fdflags != -1) fdflags = fcntl(s, F_SETFL, O_NONBLOCK \| fdflags);
	if (fdflags == -1) { close(s); return IMAP_IOERROR; }

	acappush_sock = s;

	return 0;
	}

	/* create the unique identifier for a mailbox named 'name' with
	* uidvalidity 'uidvalidity'. 'uniqueid' should be at least 17 bytes
	* long. the unique identifier is just the mailbox name hashed to 32
	* bits followed by the uid, both converted to hex.
	*/
	#define PRIME (2147484043UL)

	void mailbox_make_uniqueid(char *name, unsigned long uidvalidity,
	char *uniqueid)
	{
	unsigned long hash = 0;

	while (*name) {
	hash *= 251;
	hash += *name++;
	hash %= PRIME;
	}
	sprintf(uniqueid, "%08lx%08lx", hash, uidvalidity);
	}

	/*
	* Calculate relative filename for the message with UID 'uid'
	* in 'mailbox'. 'out' must be at least MAILBOX_FNAME_LEN long.
	*/
	void mailbox_message_get_fname(struct mailbox *mailbox, unsigned long uid,
	char *out)
	{
	assert(mailbox->format != MAILBOX_FORMAT_NETNEWS);

	sprintf(out, "%lu.", uid);
	}

	/*
	* Maps in the content for the message with UID 'uid' in 'mailbox'.
	* Returns map in 'basep' and 'lenp'
	*/
	int mailbox_map_message(struct mailbox *mailbox,
	int iscurrentdir,
	unsigned long uid,
	const char **basep,
	unsigned long *lenp)
	{
	int msgfd;
	char buf[4096];
	char *p = buf;
	struct stat sbuf;

	if (!iscurrentdir) {
	strcpy(buf, mailbox->path);
	p = buf + strlen(buf);
	*p++ = '/';
	}

	sprintf(p, "%lu.", uid);

	msgfd = open(buf, O_RDONLY, 0666);
	if (msgfd == -1) return errno;

	if (fstat(msgfd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat on %s: %m", buf);
	fatal("can't fstat message file", EC_OSFILE);
	}
	*basep = 0;
	*lenp = 0;
	map_refresh(msgfd, 1, basep, lenp, sbuf.st_size, buf, mailbox->name);
	close(msgfd);

	return 0;
	}

	/*
	* Releases the buffer obtained from mailbox_map_message()
	*/
	void mailbox_unmap_message(struct mailbox *mailbox __attribute__((unused)),
	unsigned long uid __attribute__((unused)),
	const char *basep, unsigned long lenp)
	{
	map_free(basep, lenp);
	}

	/*
	* Set the "reconstruct" mode. Causes most errors to be ignored.
	*/
	void
	mailbox_reconstructmode()
	{
	mailbox_doing_reconstruct = 1;
	}

	/*
	* Open and read the header of the mailbox with name 'name'
	* The structure pointed to by 'mailbox' is initialized.
	*/
	int mailbox_open_header(const char *name,
	struct auth_state *auth_state,
	struct mailbox *mailbox)
	{
	char path, acl;
	int r;

	r = mboxlist_lookup(name, &path, &acl, NULL);
	if (r) return r;

	return mailbox_open_header_path(name, path, acl, auth_state, mailbox, 0);
	}

	/*
	* Open and read the header of the mailbox with name 'name'
	* path 'path', and ACL 'acl'.
	* The structure pointed to by 'mailbox' is initialized.
	*/
	int mailbox_open_header_path(const char *name,
	const char *path,
	const char *acl,
	struct auth_state *auth_state,
	struct mailbox *mailbox,
	int suppresslog)
	{
	char fnamebuf[MAX_MAILBOX_PATH];
	struct stat sbuf;
	int r;

	zeromailbox(*mailbox);
	mailbox->quota.fd = -1;

	strcpy(fnamebuf, path);
	strcat(fnamebuf, FNAME_HEADER);
	mailbox->header_fd = open(fnamebuf, O_RDWR, 0);

	if (mailbox->header_fd == -1 && !mailbox_doing_reconstruct) {
	if (!suppresslog) {
	syslog(LOG_ERR, "IOERROR: opening %s: %m", fnamebuf);
	}
	return IMAP_IOERROR;
	}

	if (mailbox->header_fd != -1) {
	if (fstat(mailbox->header_fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstating %s: %m", fnamebuf);
	fatal("can't fstat header file", EC_OSFILE);
	}
	map_refresh(mailbox->header_fd, 1, &mailbox->header_base,
	&mailbox->header_len, sbuf.st_size, "header", name);
	mailbox->header_ino = sbuf.st_ino;
	}

	mailbox->name = xstrdup(name);
	mailbox->path = xstrdup(path);
	mailbox->acl = xstrdup(acl);
	mailbox->myrights = cyrus_acl_myrights(auth_state, mailbox->acl);

	if (mailbox->header_fd == -1) return 0;

	r = mailbox_read_header(mailbox);
	if (r && !mailbox_doing_reconstruct) {
	mailbox_close(mailbox);
	return r;
	}

	return 0;
	}

	#define MAXTRIES 60

	/*
	* Open the index and cache files for 'mailbox'. Also
	* read the index header.
	*/
	int mailbox_open_index(struct mailbox *mailbox)
	{
	char fnamebuf[MAX_MAILBOX_PATH];
	bit32 index_gen = 0, cache_gen = 0;
	int tries = 0;

	if (mailbox->index_fd != -1) {
	close(mailbox->index_fd);
	map_free(&mailbox->index_base, &mailbox->index_len);
	}
	if (mailbox->cache_fd != -1) {
	close(mailbox->cache_fd);
	map_free(&mailbox->cache_base, &mailbox->cache_len);
	}
	do {
	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_INDEX);
	mailbox->index_fd = open(fnamebuf, O_RDWR, 0);
	if (mailbox->index_fd != -1) {
	map_refresh(mailbox->index_fd, 0, &mailbox->index_base,
	&mailbox->index_len, MAP_UNKNOWN_LEN, "index",
	mailbox->name);
	}
	if (mailbox_doing_reconstruct) break;
	if (mailbox->index_fd == -1) {
	syslog(LOG_ERR, "IOERROR: opening %s: %m", fnamebuf);
	return IMAP_IOERROR;
	}

	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_CACHE);
	mailbox->cache_fd = open(fnamebuf, O_RDWR, 0);
	if (mailbox->cache_fd != -1) {
	struct stat sbuf;

	if (fstat(mailbox->cache_fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstating %s: %m", mailbox->name);
	fatal("can't fstat cache file", EC_OSFILE);
	}
	mailbox->cache_size = sbuf.st_size;
	map_refresh(mailbox->cache_fd, 0, &mailbox->cache_base,
	&mailbox->cache_len, mailbox->cache_size, "cache",
	mailbox->name);
	}
	if (mailbox->cache_fd == -1) {
	syslog(LOG_ERR, "IOERROR: opening %s: %m", fnamebuf);
	return IMAP_IOERROR;
	}

	/* Check generation number matches */
	if (mailbox->index_len < 4 \|\| mailbox->cache_len < 4) {
	return IMAP_MAILBOX_BADFORMAT;
	}
	index_gen = (bit32 )mailbox->index_base;
	cache_gen = (bit32 )mailbox->cache_base;

	if (index_gen != cache_gen) {
	close(mailbox->index_fd);
	map_free(&mailbox->index_base, &mailbox->index_len);
	close(mailbox->cache_fd);
	map_free(&mailbox->cache_base, &mailbox->cache_len);
	sleep(1);
	}
	} while (index_gen != cache_gen && tries++ < MAXTRIES);

	if (index_gen != cache_gen) {
	return IMAP_MAILBOX_BADFORMAT;
	}
	mailbox->generation_no = index_gen;

	return mailbox_read_index_header(mailbox);
	}

	/*
	* Close the mailbox 'mailbox', freeing all associated resources.
	*/
	void mailbox_close(struct mailbox *mailbox)
	{
	int flag;

	close(mailbox->header_fd);
	map_free(&mailbox->header_base, &mailbox->header_len);

	if (mailbox->index_fd != -1) {
	close(mailbox->index_fd);
	map_free(&mailbox->index_base, &mailbox->index_len);
	}

	if (mailbox->cache_fd != -1) {
	close(mailbox->cache_fd);
	map_free(&mailbox->cache_base, &mailbox->cache_len);
	}

	if (mailbox->quota.fd != -1) {
	close(mailbox->quota.fd);
	}

	free(mailbox->name);
	free(mailbox->path);
	free(mailbox->acl);
	free(mailbox->uniqueid);
	if (mailbox->quota.root) free(mailbox->quota.root);

	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (mailbox->flagname[flag]) free(mailbox->flagname[flag]);
	}

	zeromailbox(*mailbox);
	mailbox->quota.fd = -1;
	}

	/*
	* Read the header of 'mailbox'
	*/
	int mailbox_read_header(struct mailbox *mailbox)
	{
	int flag;
	const char name, p, tab, eol;
	int oldformat = 0;

	/* Check magic number */
	if (mailbox->header_len < sizeof(MAILBOX_HEADER_MAGIC)-1 \|\|
	strncmp(mailbox->header_base, MAILBOX_HEADER_MAGIC,
	sizeof(MAILBOX_HEADER_MAGIC)-1)) {
	return IMAP_MAILBOX_BADFORMAT;
	}

	/* Read quota file pathname */
	p = mailbox->header_base + sizeof(MAILBOX_HEADER_MAGIC)-1;
	tab = memchr(p, '\t', mailbox->header_len - (p - mailbox->header_base));
	eol = memchr(p, '\n', mailbox->header_len - (p - mailbox->header_base));
	if (!tab \|\| tab > eol \|\| !eol) {
	oldformat = 1;
	if (!eol) return IMAP_MAILBOX_BADFORMAT;
	else {
	syslog(LOG_DEBUG, "mailbox '%s' has old cyrus.header",
	mailbox->name);
	}
	tab = eol;
	}
	if (mailbox->quota.root) {
	/* check if this is the same as what's there */
	if (strlen(mailbox->quota.root) != (size_t)(tab-p) \|\|
	strncmp(mailbox->quota.root, p, tab-p) != 0) {
	assert(mailbox->quota.lock_count == 0);
	if (mailbox->quota.fd != -1) {
	close(mailbox->quota.fd);
	}
	mailbox->quota.fd = -1;
	}
	free(mailbox->quota.root);
	}
	if (p < tab) {
	mailbox->quota.root = xstrndup(p, tab - p);
	} else {
	mailbox->quota.root = NULL;
	}

	if (!oldformat) {
	/* read uniqueid */
	p = tab + 1;
	if (p == eol) return IMAP_MAILBOX_BADFORMAT;
	mailbox->uniqueid = xstrndup(p, eol - p);
	} else {
	/* uniqueid needs to be generated when we know the uidvalidity */
	mailbox->uniqueid = NULL;
	}

	/* Read names of user flags */
	p = eol + 1;
	eol = memchr(p, '\n', mailbox->header_len - (p - mailbox->header_base));
	if (!eol) {
	return IMAP_MAILBOX_BADFORMAT;
	}
	name = p;
	flag = 0;
	while (name <= eol && flag < MAX_USER_FLAGS) {
	p = memchr(name, ' ', eol-name);
	if (!p) p = eol;
	if (mailbox->flagname[flag]) free(mailbox->flagname[flag]);
	if (name != p) {
	mailbox->flagname[flag++] = xstrndup(name, p-name);
	}
	else {
	mailbox->flagname[flag++] = NULL;
	}

	name = p+1;
	}
	while (flag < MAX_USER_FLAGS) {
	if (mailbox->flagname[flag]) free(mailbox->flagname[flag]);
	mailbox->flagname[flag++] = NULL;
	}

	if (!mailbox->uniqueid) {
	char buf[32];

	/* generate uniqueid */
	mailbox_lock_header(mailbox);
	mailbox_open_index(mailbox);
	mailbox_make_uniqueid(mailbox->name, mailbox->uidvalidity, buf);
	mailbox->uniqueid = xstrdup(buf);
	mailbox_write_header(mailbox);
	mailbox_unlock_header(mailbox);
	}

	return 0;
	}

	/*
	* Read the acl out of the header of 'mailbox'
	*/
	int mailbox_read_header_acl(struct mailbox *mailbox)
	{
	const char p, eol;

	/* Check magic number */
	if (mailbox->header_len < sizeof(MAILBOX_HEADER_MAGIC)-1 \|\|
	strncmp(mailbox->header_base, MAILBOX_HEADER_MAGIC,
	sizeof(MAILBOX_HEADER_MAGIC)-1)) {
	return IMAP_MAILBOX_BADFORMAT;
	}

	/* Skip quota file pathname */
	p = mailbox->header_base + sizeof(MAILBOX_HEADER_MAGIC)-1;
	eol = memchr(p, '\n', mailbox->header_len - (p - mailbox->header_base));
	if (!eol) {
	return IMAP_MAILBOX_BADFORMAT;
	}

	/* Skip names of user flags */
	p = eol + 1;
	eol = memchr(p, '\n', mailbox->header_len - (p - mailbox->header_base));
	if (!eol) {
	return IMAP_MAILBOX_BADFORMAT;
	}

	/* Read ACL */
	p = eol + 1;
	eol = memchr(p, '\n', mailbox->header_len - (p - mailbox->header_base));
	if (!eol) {
	return IMAP_MAILBOX_BADFORMAT;
	}

	free(mailbox->acl);
	mailbox->acl = xstrndup(p, eol-p);

	return 0;
	}

	/*
	* Read the the ACL for 'mailbox'.
	*/
	int mailbox_read_acl(struct mailbox *mailbox,
	struct auth_state *auth_state)
	{
	int r;
	char *acl;

	r = mboxlist_lookup(mailbox->name, (char **)0, &acl, NULL);
	if (r) return r;

	free(mailbox->acl);
	mailbox->acl = xstrdup(acl);
	mailbox->myrights = cyrus_acl_myrights(auth_state, mailbox->acl);

	return 0;
	}

	/*
	* Read the header of the index file for mailbox
	*/
	int mailbox_read_index_header(struct mailbox *mailbox)
	{
	struct stat sbuf;

	if (mailbox->index_fd == -1) return IMAP_MAILBOX_BADFORMAT;

	fstat(mailbox->index_fd, &sbuf);
	mailbox->index_ino = sbuf.st_ino;
	mailbox->index_mtime = sbuf.st_mtime;
	mailbox->index_size = sbuf.st_size;
	map_refresh(mailbox->index_fd, 0, &mailbox->index_base,
	&mailbox->index_len, sbuf.st_size, "index",
	mailbox->name);

	if (mailbox->index_len < OFFSET_POP3_LAST_LOGIN \|\|
	(mailbox->index_len <
	ntohl(((bit32 )(mailbox->index_base+OFFSET_START_OFFSET))))) {
	return IMAP_MAILBOX_BADFORMAT;
	}

	if (mailbox_doing_reconstruct) {
	mailbox->generation_no =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_GENERATION_NO)));
	}
	mailbox->format =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_FORMAT)));
	mailbox->minor_version =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_MINOR_VERSION)));
	mailbox->start_offset =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_START_OFFSET)));
	mailbox->record_size =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_RECORD_SIZE)));
	mailbox->exists =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_EXISTS)));
	mailbox->last_appenddate =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_LAST_APPENDDATE)));
	mailbox->last_uid =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_LAST_UID)));
	mailbox->quota_mailbox_used =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_QUOTA_MAILBOX_USED)));

	if (mailbox->start_offset < OFFSET_POP3_LAST_LOGIN+sizeof(bit32)) {
	mailbox->pop3_last_login = 0;
	}
	else {
	mailbox->pop3_last_login =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_POP3_LAST_LOGIN)));
	}

	if (mailbox->start_offset < OFFSET_UIDVALIDITY+sizeof(bit32)) {
	mailbox->uidvalidity = 1;
	}
	else {
	mailbox->uidvalidity =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_UIDVALIDITY)));
	}

	if (mailbox->start_offset < OFFSET_FLAGGED+sizeof(bit32)) {
	/* calculate them now */
	if (mailbox_calculate_flagcounts(mailbox))
	return IMAP_IOERROR;

	/* things might have been changed out from under us. reread */
	mailbox_open_index(mailbox);
	} else {
	mailbox->deleted =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_DELETED)));
	mailbox->answered =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_ANSWERED)));
	mailbox->flagged =
	ntohl(((bit32 )(mailbox->index_base+OFFSET_FLAGGED)));
	mailbox->dirty = 0;
	}

	if (!mailbox_doing_reconstruct &&
	(mailbox->minor_version < MAILBOX_MINOR_VERSION)) {
	return IMAP_MAILBOX_BADFORMAT;
	}

	return 0;
	}

	/*
	* Read an index record from a mailbox
	*/
	int
	mailbox_read_index_record(mailbox, msgno, record)
	struct mailbox *mailbox;
	unsigned msgno;
	struct index_record *record;
	{
	unsigned long offset;
	unsigned const char *buf;
	int n;

	offset = mailbox->start_offset + (msgno-1) * mailbox->record_size;
	if (offset + INDEX_RECORD_SIZE > mailbox->index_len) {
	syslog(LOG_ERR,
	"IOERROR: index record %u for %s past end of file",
	msgno, mailbox->name);
	return IMAP_IOERROR;
	}

	buf = (unsigned char*) mailbox->index_base + offset;

	record->uid = htonl(((bit32 )(buf+OFFSET_UID)));
	record->internaldate = htonl(((bit32 )(buf+OFFSET_INTERNALDATE)));
	record->sentdate = htonl(((bit32 )(buf+OFFSET_SENTDATE)));
	record->size = htonl(((bit32 )(buf+OFFSET_SIZE)));
	record->header_size = htonl(((bit32 )(buf+OFFSET_HEADER_SIZE)));
	record->content_offset = htonl(((bit32 )(buf+OFFSET_CONTENT_OFFSET)));
	record->cache_offset = htonl(((bit32 )(buf+OFFSET_CACHE_OFFSET)));
	record->last_updated = htonl(((bit32 )(buf+OFFSET_LAST_UPDATED)));
	record->system_flags = htonl(((bit32 )(buf+OFFSET_SYSTEM_FLAGS)));
	for (n = 0; n < MAX_USER_FLAGS/32; n++) {
	record->user_flags[n] = htonl(((bit32 )(buf+OFFSET_USER_FLAGS+4*n)));
	}
	return 0;
	}

	/*
	* Open and read the quota file 'quota'
	*/
	int
	mailbox_read_quota(quota)
	struct quota *quota;
	{
	const char p, eol;
	char buf[4096];
	const char *quota_base = 0;
	unsigned long quota_len = 0;


	if (!quota->root) {
	quota->used = 0;
	quota->limit = -1;
	return 0;
	}

	if (quota->fd == -1) {
	mailbox_hash_quota(buf, quota->root);
	quota->fd = open(buf, O_RDWR, 0);
	if (quota->fd == -1) {
	syslog(LOG_ERR, "IOERROR: opening quota file %s: %m", buf);
	return IMAP_IOERROR;
	}
	}

	map_refresh(quota->fd, 1, &quota_base, &quota_len,
	MAP_UNKNOWN_LEN, buf, 0);

	p = quota_base;
	eol = memchr(p, '\n', quota_len - (p - quota_base));
	if (!eol) {
	map_free(&quota_base, &quota_len);
	return IMAP_MAILBOX_BADFORMAT;
	}
	quota->used = atol(p);

	p = eol + 1;
	eol = memchr(p, '\n', quota_len - (p - quota_base));
	if (!eol) {
	map_free(&quota_base, &quota_len);
	return IMAP_MAILBOX_BADFORMAT;
	}
	quota->limit = atoi(p);

	map_free(&quota_base, &quota_len);
	return 0;
	}

	/*
	* Lock the header for 'mailbox'. Reread header if necessary.
	*/
	int
	mailbox_lock_header(mailbox)
	struct mailbox *mailbox;
	{
	char fnamebuf[MAX_MAILBOX_PATH];
	struct stat sbuf;
	const char *lockfailaction;
	int r;

	if (mailbox->header_lock_count++) return 0;

	assert(mailbox->index_lock_count == 0);
	assert(mailbox->quota.lock_count == 0);
	assert(mailbox->seen_lock_count == 0);

	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_HEADER);

	r = lock_reopen(mailbox->header_fd, fnamebuf, &sbuf, &lockfailaction);
	if (r) {
	mailbox->header_lock_count--;
	syslog(LOG_ERR, "IOERROR: %s header for %s: %m",
	lockfailaction, mailbox->name);
	return IMAP_IOERROR;
	}

	if (sbuf.st_ino != mailbox->header_ino) {
	map_free(&mailbox->header_base, &mailbox->header_len);
	map_refresh(mailbox->header_fd, 1, &mailbox->header_base,
	&mailbox->header_len, sbuf.st_size, "header",
	mailbox->name);
	mailbox->header_ino = sbuf.st_ino;
	r = mailbox_read_header(mailbox);
	if (r && !mailbox_doing_reconstruct) {
	mailbox_unlock_header(mailbox);
	return r;
	}
	}

	return 0;
	}

	/*
	* Lock the index file for 'mailbox'. Reread index file header if necessary.
	*/
	int
	mailbox_lock_index(mailbox)
	struct mailbox *mailbox;
	{
	char fnamebuf[MAX_MAILBOX_PATH];
	struct stat sbuffd, sbuffile;
	int r;

	if (mailbox->index_lock_count++) return 0;

	assert(mailbox->quota.lock_count == 0);
	assert(mailbox->seen_lock_count == 0);

	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_INDEX);

	for (;;) {
	r = lock_blocking(mailbox->index_fd);
	if (r == -1) {
	mailbox->index_lock_count--;
	syslog(LOG_ERR, "IOERROR: locking index for %s: %m",
	mailbox->name);
	return IMAP_IOERROR;
	}

	fstat(mailbox->index_fd, &sbuffd);
	r = stat(fnamebuf, &sbuffile);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: stating index for %s: %m",
	mailbox->name);
	mailbox_unlock_index(mailbox);
	return IMAP_IOERROR;
	}

	if (sbuffd.st_ino == sbuffile.st_ino) break;

	if ((r = mailbox_open_index(mailbox))) {
	return r;
	}
	}

	r = mailbox_read_index_header(mailbox);
	if (r && !mailbox_doing_reconstruct) {
	mailbox_unlock_index(mailbox);
	return r;
	}

	return 0;
	}

	/*
	* Place a POP lock on 'mailbox'.
	*/
	int
	mailbox_lock_pop(mailbox)
	struct mailbox *mailbox;
	{
	int r = -1;

	if (mailbox->pop_lock_count++) return 0;

	r = lock_nonblocking(mailbox->cache_fd);
	if (r == -1) {
	mailbox->pop_lock_count--;
	if (errno == EWOULDBLOCK \|\| errno == EAGAIN \|\| errno == EACCES) {
	return IMAP_MAILBOX_POPLOCKED;
	}
	syslog(LOG_ERR, "IOERROR: locking cache for %s: %m", mailbox->name);
	return IMAP_IOERROR;
	}

	return 0;
	}

	/*
	* Lock the quota file 'quota'. Reread quota file if necessary.
	*/
	int
	mailbox_lock_quota(quota)
	struct quota *quota;
	{
	char quota_path[MAX_MAILBOX_PATH];
	struct stat sbuf;
	const char *lockfailaction;
	int r;

	/* assert(mailbox->header_lock_count != 0); */

	if (quota->lock_count++) return 0;

	/* assert(mailbox->seen_lock_count == 0); */

	if (!quota->root) {
	quota->used = 0;
	quota->limit = -1;
	return 0;
	}
	mailbox_hash_quota(quota_path, quota->root);
	if (quota->fd == -1) {
	quota->fd = open(quota_path, O_RDWR, 0);
	if (quota->fd == -1) {
	syslog(LOG_ERR, "IOERROR: opening quota file %s: %m",
	quota_path);
	return IMAP_IOERROR;
	}
	}

	r = lock_reopen(quota->fd, quota_path, &sbuf, &lockfailaction);
	if (r == -1) {
	quota->lock_count--;
	syslog(LOG_ERR, "IOERROR: %s quota %s: %m", lockfailaction,
	quota->root);
	return IMAP_IOERROR;
	}

	return mailbox_read_quota(quota);
	}

	/*
	* Release lock on the header for 'mailbox'
	*/
	void mailbox_unlock_header(struct mailbox *mailbox)
	{
	assert(mailbox->header_lock_count != 0);

	if (--mailbox->header_lock_count == 0) {
	lock_unlock(mailbox->header_fd);
	}
	}

	/*
	* Release lock on the index file for 'mailbox'
	*/
	void
	mailbox_unlock_index(mailbox)
	struct mailbox *mailbox;
	{
	assert(mailbox->index_lock_count != 0);

	if (--mailbox->index_lock_count == 0) {
	lock_unlock(mailbox->index_fd);
	}
	}

	/*
	* Release POP lock for 'mailbox'
	*/
	void
	mailbox_unlock_pop(mailbox)
	struct mailbox *mailbox;
	{
	assert(mailbox->pop_lock_count != 0);

	if (--mailbox->pop_lock_count == 0) {
	lock_unlock(mailbox->cache_fd);
	}
	}

	/*
	* Release lock on the quota file 'quota'
	*/
	void
	mailbox_unlock_quota(quota)
	struct quota *quota;
	{
	assert(quota->lock_count != 0);

	if (--quota->lock_count == 0 && quota->root) {
	lock_unlock(quota->fd);
	}
	}

	/*
	* Write the header file for 'mailbox'
	*/
	int mailbox_write_header(struct mailbox *mailbox)
	{
	int flag;
	FILE *newheader;
	int newheader_fd;
	char fnamebuf[MAX_MAILBOX_PATH];
	char newfnamebuf[MAX_MAILBOX_PATH];
	struct stat sbuf;

	assert(mailbox->header_lock_count != 0);

	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_HEADER);
	strcpy(newfnamebuf, fnamebuf);
	strcat(newfnamebuf, ".NEW");

	newheader = fopen(newfnamebuf, "w+");
	if (!newheader) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", newfnamebuf);
	return IMAP_IOERROR;
	}

	fputs(MAILBOX_HEADER_MAGIC, newheader);
	fprintf(newheader, "%s\t%s\n",
	mailbox->quota.root ? mailbox->quota.root : "",
	mailbox->uniqueid);
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (mailbox->flagname[flag]) {
	fprintf(newheader, "%s ", mailbox->flagname[flag]);
	}
	}
	fprintf(newheader, "\n");
	fprintf(newheader, "%s\n", mailbox->acl);

	fflush(newheader);
	newheader_fd = dup(fileno(newheader));
	if (ferror(newheader) \|\| fsync(fileno(newheader)) \|\|
	lock_blocking(newheader_fd) == -1 \|\|
	rename(newfnamebuf, fnamebuf) == -1) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", newfnamebuf);
	fclose(newheader);
	close(newheader_fd);
	unlink(newfnamebuf);
	return IMAP_IOERROR;
	}

	fclose(newheader);

	if (mailbox->header_fd != -1) {
	close(mailbox->header_fd);
	map_free(&mailbox->header_base, &mailbox->header_len);
	}
	mailbox->header_fd = newheader_fd;

	if (fstat(mailbox->header_fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstating %s: %m", fnamebuf);
	fatal("can't fstat header file", EC_OSFILE);
	}
	map_refresh(mailbox->header_fd, 1, &mailbox->header_base,
	&mailbox->header_len, sbuf.st_size, "header", mailbox->name);
	mailbox->header_ino = sbuf.st_ino;

	return 0;
	}

	/*
	* Write the index header for 'mailbox'
	*/
	int mailbox_write_index_header(struct mailbox *mailbox)
	{
	char buf[INDEX_HEADER_SIZE];
	unsigned long header_size = INDEX_HEADER_SIZE;
	int n;

	assert(mailbox->index_lock_count != 0);

	if (updatenotifier) updatenotifier(mailbox);

	if (acappush_sock != -1) {
	acapmbdata_t acapdata;

	/* fill in structure */
	strcpy(acapdata.name, mailbox->name);
	acapdata.uidvalidity = mailbox->uidvalidity;
	acapdata.exists = mailbox->exists;
	acapdata.deleted = mailbox->deleted;
	acapdata.answered = mailbox->answered;
	acapdata.flagged = mailbox->flagged;

	/* send */
	if (sendto(acappush_sock, &acapdata, 20+strlen(mailbox->name), 0,
	(struct sockaddr *) &acappush_remote,
	acappush_remote_len) == -1) {
	syslog(LOG_ERR, "sending to acappush: %m");
	}
	}

	((bit32 )(buf+OFFSET_GENERATION_NO)) = mailbox->generation_no;
	((bit32 )(buf+OFFSET_FORMAT)) = htonl(mailbox->format);
	((bit32 )(buf+OFFSET_MINOR_VERSION)) = htonl(mailbox->minor_version);
	((bit32 )(buf+OFFSET_START_OFFSET)) = htonl(mailbox->start_offset);
	((bit32 )(buf+OFFSET_RECORD_SIZE)) = htonl(mailbox->record_size);
	((bit32 )(buf+OFFSET_EXISTS)) = htonl(mailbox->exists);
	((bit32 )(buf+OFFSET_LAST_APPENDDATE)) = htonl(mailbox->last_appenddate);
	((bit32 )(buf+OFFSET_LAST_UID)) = htonl(mailbox->last_uid);
	((bit32 )(buf+OFFSET_QUOTA_MAILBOX_USED)) = htonl(mailbox->quota_mailbox_used);
	((bit32 )(buf+OFFSET_POP3_LAST_LOGIN)) = htonl(mailbox->pop3_last_login);
	((bit32 )(buf+OFFSET_UIDVALIDITY)) = htonl(mailbox->uidvalidity);
	((bit32 )(buf+OFFSET_DELETED)) = htonl(mailbox->deleted);
	((bit32 )(buf+OFFSET_ANSWERED)) = htonl(mailbox->answered);
	((bit32 )(buf+OFFSET_FLAGGED)) = htonl(mailbox->flagged);

	if (mailbox->start_offset < header_size)
	header_size = mailbox->start_offset;

	lseek(mailbox->index_fd, 0, SEEK_SET);
	n = retry_write(mailbox->index_fd, buf, header_size);
	if ((unsigned long)n != header_size \|\| fsync(mailbox->index_fd)) {
	syslog(LOG_ERR, "IOERROR: writing index header for %s: %m",
	mailbox->name);
	/* xxx can we unroll the acap send??? */
	return IMAP_IOERROR;
	}

	return 0;
	}

	/*
	* Write an index record to a mailbox
	* call fsync() on index_fd if 'sync' is true
	*/
	int
	mailbox_write_index_record(struct mailbox *mailbox,
	unsigned msgno,
	struct index_record *record,
	int sync)
	{
	int n;
	char buf[INDEX_RECORD_SIZE];

	((bit32 )(buf+OFFSET_UID)) = htonl(record->uid);
	((bit32 )(buf+OFFSET_INTERNALDATE)) = htonl(record->internaldate);
	((bit32 )(buf+OFFSET_SENTDATE)) = htonl(record->sentdate);
	((bit32 )(buf+OFFSET_SIZE)) = htonl(record->size);
	((bit32 )(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size);
	((bit32 )(buf+OFFSET_CONTENT_OFFSET)) = htonl(record->content_offset);
	((bit32 )(buf+OFFSET_CACHE_OFFSET)) = htonl(record->cache_offset);
	((bit32 )(buf+OFFSET_LAST_UPDATED)) = htonl(record->last_updated);
	((bit32 )(buf+OFFSET_SYSTEM_FLAGS)) = htonl(record->system_flags);
	for (n = 0; n < MAX_USER_FLAGS/32; n++) {
	((bit32 )(buf+OFFSET_USER_FLAGS+4*n)) = htonl(record->user_flags[n]);
	}

	n = lseek(mailbox->index_fd,
	mailbox->start_offset + (msgno-1) * mailbox->record_size,
	SEEK_SET);
	if (n == -1) {
	syslog(LOG_ERR, "IOERROR: seeking index record %u for %s: %m",
	msgno, mailbox->name);
	return IMAP_IOERROR;
	}

	n = retry_write(mailbox->index_fd, buf, INDEX_RECORD_SIZE);
	if (n != INDEX_RECORD_SIZE \|\| (sync && fsync(mailbox->index_fd))) {
	syslog(LOG_ERR, "IOERROR: writing index record %u for %s: %m",
	msgno, mailbox->name);
	return IMAP_IOERROR;
	}

	return 0;
	}

	/*
	* Append a new record to the index file
	* call fsync() on index_fd if 'sync' is true
	*/
	int mailbox_append_index(struct mailbox *mailbox,
	struct index_record *record,
	unsigned start,
	unsigned num,
	int sync)
	{
	unsigned i;
	int j, len, n;
	char buf, p;
	long last_offset;

	assert(mailbox->index_lock_count != 0);

	if (mailbox->record_size < INDEX_RECORD_SIZE) {
	return IMAP_MAILBOX_BADFORMAT;
	}

	len = num * mailbox->record_size;
	buf = xmalloc(len);
	memset(buf, 0, len);

	for (i = 0; i < num; i++) {
	p = buf + i*mailbox->record_size;
	((bit32 )(p+OFFSET_UID)) = htonl(record[i].uid);
	((bit32 )(p+OFFSET_INTERNALDATE)) = htonl(record[i].internaldate);
	((bit32 )(p+OFFSET_SENTDATE)) = htonl(record[i].sentdate);
	((bit32 )(p+OFFSET_SIZE)) = htonl(record[i].size);
	((bit32 )(p+OFFSET_HEADER_SIZE)) = htonl(record[i].header_size);
	((bit32 )(p+OFFSET_CONTENT_OFFSET)) = htonl(record[i].content_offset);
	((bit32 )(p+OFFSET_CACHE_OFFSET)) = htonl(record[i].cache_offset);
	((bit32 )(p+OFFSET_LAST_UPDATED)) = htonl(record[i].last_updated);
	((bit32 )(p+OFFSET_SYSTEM_FLAGS)) = htonl(record[i].system_flags);
	p += OFFSET_USER_FLAGS;
	for (j = 0; j < MAX_USER_FLAGS/32; j++, p += 4) {
	((bit32 )p) = htonl(record[i].user_flags[j]);
	}
	}

	last_offset = mailbox->start_offset + start * mailbox->record_size;
	lseek(mailbox->index_fd, last_offset, SEEK_SET);
	n = retry_write(mailbox->index_fd, buf, len);
	free(buf);
	if (n != len \|\| (sync && fsync(mailbox->index_fd))) {
	syslog(LOG_ERR, "IOERROR: appending index records for %s: %m",
	mailbox->name);
	ftruncate(mailbox->index_fd, last_offset);
	return IMAP_IOERROR;
	}

	return 0;
	}

	/*
	* Write out the quota 'quota'
	*/
	int
	mailbox_write_quota(quota)
	struct quota *quota;
	{
	int r;
	char quota_path[MAX_MAILBOX_PATH];
	char new_quota_path[MAX_MAILBOX_PATH];
	FILE *newfile;
	int newfd;

	assert(quota->lock_count != 0);

	if (!quota->root) return 0;

	mailbox_hash_quota(quota_path, quota->root);
	strcpy(new_quota_path, quota_path);
	strcat(new_quota_path, ".NEW");

	newfile = fopen(new_quota_path, "w+");
	if (!newfile) {
	syslog(LOG_ERR, "IOERROR: creating quota file %s: %m", new_quota_path);
	return IMAP_IOERROR;
	}
	newfd = dup(fileno(newfile));
	r = lock_blocking(newfd);
	if (r) {
	syslog(LOG_ERR, "IOERROR: locking quota file %s: %m",
	new_quota_path);
	fclose(newfile);
	close(newfd);
	return IMAP_IOERROR;
	}

	fprintf(newfile, "%lu\n%d\n", quota->used, quota->limit);
	fflush(newfile);
	if (ferror(newfile) \|\| fsync(fileno(newfile))) {
	syslog(LOG_ERR, "IOERROR: writing quota file %s: %m",
	new_quota_path);
	fclose(newfile);
	close(newfd);
	return IMAP_IOERROR;
	}

	if (rename(new_quota_path, quota_path)) {
	syslog(LOG_ERR, "IOERROR: renaming quota file %s: %m",
	quota_path);
	fclose(newfile);
	close(newfd);
	return IMAP_IOERROR;
	}
	fclose(newfile);

	if (quota->fd != -1) {
	close(quota->fd);
	quota->fd = -1;
	}

	quota->fd = newfd;

	return 0;
	}

	/*
	* Remove the quota root 'quota'
	*/
	int
	mailbox_delete_quota(quota)
	struct quota *quota;
	{
	char quota_path[MAX_MAILBOX_PATH];

	assert(quota->lock_count != 0);

	if (!quota->root) return 0;

	mailbox_hash_quota(quota_path, quota->root);

	unlink(quota_path);

	if (quota->fd != -1) {
	close(quota->fd);
	quota->fd = -1;
	}

	free(quota->root);
	quota->root = 0;

	return 0;
	}

	/*
	* Lock the index file for 'mailbox'. DON'T Reread index file header if necessary.
	*/
	int
	mailbox_lock_index_forcalc(mailbox)
	struct mailbox *mailbox;
	{
	char fnamebuf[MAX_MAILBOX_PATH];
	struct stat sbuffd, sbuffile;
	int r;

	if (mailbox->index_lock_count++) return 0;

	assert(mailbox->quota.lock_count == 0);
	assert(mailbox->seen_lock_count == 0);

	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_INDEX);

	for (;;) {
	r = lock_blocking(mailbox->index_fd);
	if (r == -1) {
	mailbox->index_lock_count--;
	syslog(LOG_ERR, "IOERROR: locking index for %s: %m",
	mailbox->name);
	return IMAP_IOERROR;
	}

	fstat(mailbox->index_fd, &sbuffd);
	r = stat(fnamebuf, &sbuffile);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: stating index for %s: %m",
	mailbox->name);
	mailbox_unlock_index(mailbox);
	return IMAP_IOERROR;
	}

	if (sbuffd.st_ino == sbuffile.st_ino) break;

	if ((r = mailbox_open_index(mailbox))) {
	return r;
	}
	}

	return 0;
	}

	/*
	* Calculate the number of messages in the mailbox with answered/deleted/flagged system
	* flags
	*/

	static int mailbox_calculate_flagcounts(struct mailbox *mailbox)
	{
	int r;
	unsigned msgno;
	bit32 numansweredflag = 0;
	bit32 numdeletedflag = 0;
	bit32 numflaggedflag = 0;
	bit32 oldstart_offset, newstart_offset;
	bit32 newversion;
	struct stat sbuf;
	char buf[INDEX_HEADER_SIZE];
	char fnamebuf[MAX_MAILBOX_PATH], fnamebufnew[MAX_MAILBOX_PATH];
	FILE *newindex;
	char *fnametail;
	char *bufp;

	/* Lock files and open new index/cache files */
	r = mailbox_lock_header(mailbox);
	if (r) return r;
	r = mailbox_lock_index_forcalc(mailbox);
	if (r) {
	mailbox_unlock_header(mailbox);
	return r;
	}

	r = mailbox_lock_pop(mailbox);
	if (r) {
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);
	return r;
	}

	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_INDEX);
	strcat(fnamebuf, ".NEW");
	newindex = fopen(fnamebuf, "w+");
	if (!newindex) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fnamebuf);
	mailbox_unlock_pop(mailbox);
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);
	return IMAP_IOERROR;
	}

	/* Copy over old headers to new file */
	memcpy(buf, mailbox->index_base, mailbox->start_offset);

	/* insert new version number */
	newversion = htonl(MAILBOX_MINOR_VERSION);
	memcpy(buf+OFFSET_MINOR_VERSION, &newversion, sizeof(bit32));

	/* save old start_offset; change start_offset */
	oldstart_offset = mailbox->start_offset;
	newstart_offset = htonl(INDEX_HEADER_SIZE);
	memcpy(buf+OFFSET_START_OFFSET, &newstart_offset, sizeof(bit32));

	fwrite(buf, 1, oldstart_offset, newindex);

	if (fstat(mailbox->cache_fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstating %s: %m", mailbox->name);
	fatal("can't fstat cache file", EC_OSFILE);
	}
	mailbox->cache_size = sbuf.st_size;
	map_refresh(mailbox->cache_fd, 0, &mailbox->cache_base,
	&mailbox->cache_len, mailbox->cache_size,
	"cache", mailbox->name);

	/* for each message look at the system flags */
	for (msgno = 1; msgno <= mailbox->exists; msgno++) {
	bit32 sysflags;

	bufp = (char *) (mailbox->index_base + mailbox->start_offset +
	(msgno - 1)*mailbox->record_size);

	/* Sanity check */
	if (((bit32 )(bufp+OFFSET_UID)) == 0) {
	- syslog(LOG_ERR, "IOERROR: %s zero index record %u/%u",
	+ syslog(LOG_ERR, "IOERROR: %s zero index record %u/%lu",
	mailbox->name, msgno, mailbox->exists);
	goto fail;
	}

	sysflags = ntohl(((bit32 )(bufp+OFFSET_SYSTEM_FLAGS)));
	if (sysflags & FLAG_ANSWERED)
	numansweredflag++;
	if (sysflags & FLAG_DELETED)
	numdeletedflag++;
	if (sysflags & FLAG_FLAGGED)
	numflaggedflag++;
	}

	mailbox->answered = numansweredflag;
	mailbox->deleted = numdeletedflag;
	mailbox->flagged = numflaggedflag;

	/* Grow the index header */
	numansweredflag = htonl(numansweredflag);
	fwrite(&numansweredflag, sizeof(bit32),1, newindex);
	numdeletedflag = htonl(numdeletedflag);
	fwrite(&numdeletedflag, sizeof(bit32),1, newindex);
	numflaggedflag = htonl(numflaggedflag);
	fwrite(&numflaggedflag, sizeof(bit32),1, newindex);

	/* Write the rest of new index header same as old */
	for (msgno = 1; msgno <= mailbox->exists; msgno++) {
	bufp = (char *) (mailbox->index_base + oldstart_offset +
	(msgno - 1)*mailbox->record_size);

	fwrite(bufp, mailbox->record_size, 1, newindex);
	}

	/* Ensure everything made it to disk */
	fflush(newindex);
	if (ferror(newindex) \|\|
	fsync(fileno(newindex))) {
	syslog(LOG_ERR, "IOERROR: writing index for %s: %m",
	mailbox->name);
	goto fail;
	}

	strcpy(fnamebuf, mailbox->path);
	fnametail = fnamebuf + strlen(fnamebuf);
	strcpy(fnametail, FNAME_INDEX);
	strcpy(fnamebufnew, fnamebuf);
	strcat(fnamebufnew, ".NEW");
	if (rename(fnamebufnew, fnamebuf)) {
	syslog(LOG_ERR, "IOERROR: renaming index file for %s: %m",
	mailbox->name);
	goto fail;
	}

	mailbox_unlock_pop(mailbox);
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);
	fclose(newindex);

	return 0;

	fail:
	mailbox_unlock_pop(mailbox);
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);

	return IMAP_IOERROR;
	}

	/*
	* Perform an expunge operation on 'mailbox'. If 'iscurrentdir' is nonzero,
	* the current directory is set to the mailbox directory. If nonzero, the
	* function pointed to by 'decideproc' is called (with 'deciderock') to
	* determine which messages to expunge. If 'decideproc' is a null pointer,
	* then messages with the \Deleted flag are expunged.
	*/
	int
	mailbox_expunge(mailbox, iscurrentdir, decideproc, deciderock)
	struct mailbox *mailbox;
	int iscurrentdir;
	mailbox_decideproc_t *decideproc;
	void *deciderock;
	{
	int r, n;
	char fnamebuf[MAX_MAILBOX_PATH], fnamebufnew[MAX_MAILBOX_PATH];
	FILE newindex, newcache;
	unsigned long *deleted;
	unsigned numdeleted = 0, quotadeleted = 0;
	unsigned numansweredflag = 0;
	unsigned numdeletedflag = 0;
	unsigned numflaggedflag = 0;
	unsigned newexists;
	unsigned newdeleted;
	unsigned newanswered;
	unsigned newflagged;
	char *buf;
	unsigned msgno;
	int lastmsgdeleted = 1;
	unsigned long cachediff = 0;
	unsigned long cachestart = sizeof(bit32);
	unsigned long cache_offset;
	struct stat sbuf;
	char *fnametail;

	/* Lock files and open new index/cache files */
	r = mailbox_lock_header(mailbox);
	if (r) return r;
	r = mailbox_lock_index(mailbox);
	if (r) {
	mailbox_unlock_header(mailbox);
	return r;
	}

	r = mailbox_lock_pop(mailbox);
	if (r) {
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);
	return r;
	}

	if (fstat(mailbox->cache_fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstating %s: %m", fnamebuf); /* xxx is fnamebuf initialized??? */
	fatal("can't fstat cache file", EC_OSFILE);
	}
	mailbox->cache_size = sbuf.st_size;
	map_refresh(mailbox->cache_fd, 0, &mailbox->cache_base,
	&mailbox->cache_len, mailbox->cache_size,
	"cache", mailbox->name);

	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_INDEX);
	strcat(fnamebuf, ".NEW");
	newindex = fopen(fnamebuf, "w+");
	if (!newindex) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fnamebuf);
	mailbox_unlock_pop(mailbox);
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);
	return IMAP_IOERROR;
	}

	strcpy(fnamebuf, mailbox->path);
	strcat(fnamebuf, FNAME_CACHE);
	strcat(fnamebuf, ".NEW");
	newcache = fopen(fnamebuf, "w+");
	if (!newcache) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fnamebuf);
	fclose(newindex);
	mailbox_unlock_pop(mailbox);
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);
	return IMAP_IOERROR;
	}

	/* Allocate temporary buffers */
	if (mailbox->exists > 0) {
	/* XXX kludge: not all mallocs return a valid pointer to 0 bytes;
	some have the good sense to return 0 */
	deleted = (unsigned long *)
	xmalloc(mailbox->exists*sizeof(unsigned long));
	} else {
	deleted = 0;
	}
	buf = xmalloc(mailbox->start_offset > mailbox->record_size ?
	mailbox->start_offset : mailbox->record_size);

	/* Copy over headers */
	memcpy(buf, mailbox->index_base, mailbox->start_offset);
	((bit32 )buf)++; /* Increment generation number */
	fwrite(buf, 1, mailbox->start_offset, newindex);
	/* Grow the index header if necessary */
	for (n = mailbox->start_offset; n < INDEX_HEADER_SIZE; n++) {
	if (n == OFFSET_FLAGGED+3) {
	putc(1, newindex);
	}
	else {
	putc(0, newindex);
	}
	}
	fwrite(buf, 1, sizeof(bit32), newcache);

	/* Copy over records for nondeleted messages */
	for (msgno = 1; msgno <= mailbox->exists; msgno++) {
	memcpy(buf,
	mailbox->index_base + mailbox->start_offset +
	(msgno - 1)*mailbox->record_size, mailbox->record_size);

	/* Sanity check */
	if (((bit32 )(buf+OFFSET_UID)) == 0) {
	- syslog(LOG_ERR, "IOERROR: %s zero index record %u/%u",
	+ syslog(LOG_ERR, "IOERROR: %s zero index record %u/%lu",
	mailbox->name, msgno, mailbox->exists);
	goto fail;
	}

	if (decideproc ? decideproc(mailbox, deciderock, buf) :
	(ntohl(((bit32 )(buf+OFFSET_SYSTEM_FLAGS))) & FLAG_DELETED)) {
	bit32 sysflags;

	/* Remember UID and size */
	deleted[numdeleted++] = ntohl(((bit32 )(buf+OFFSET_UID)));
	quotadeleted += ntohl(((bit32 )(buf+OFFSET_SIZE)));

	/* figure out if deleted msg has system flags. update counts accordingly */
	sysflags = ntohl(((bit32 )(buf+OFFSET_SYSTEM_FLAGS)));
	if (sysflags & FLAG_ANSWERED)
	numansweredflag++;
	if (sysflags & FLAG_DELETED)
	numdeletedflag++;
	if (sysflags & FLAG_FLAGGED)
	numflaggedflag++;

	/* Copy over cache file data */
	if (!lastmsgdeleted) {
	cache_offset = ntohl(((bit32 )(buf+OFFSET_CACHE_OFFSET)));

	fwrite(mailbox->cache_base + cachestart,
	1, cache_offset - cachestart, newcache);
	cachestart = cache_offset;
	lastmsgdeleted = 1;
	}
	}
	else {
	cache_offset = ntohl(((bit32 )(buf+OFFSET_CACHE_OFFSET)));

	/* Set up for copying cache file data */
	if (lastmsgdeleted) {
	cachediff += cache_offset - cachestart;
	cachestart = cache_offset;
	lastmsgdeleted = 0;
	}

	/* Fix up cache file offset */
	((bit32 )(buf+OFFSET_CACHE_OFFSET)) = htonl(cache_offset - cachediff);

	fwrite(buf, 1, mailbox->record_size, newindex);
	}
	}

	/* Copy over any remaining cache file data */
	if (!lastmsgdeleted) {
	fwrite(mailbox->cache_base + cachestart, 1,
	mailbox->cache_size - cachestart, newcache);
	}

	/* Fix up information in index header */
	rewind(newindex);
	n = fread(buf, 1, mailbox->start_offset, newindex);
	if ((unsigned long)n != mailbox->start_offset) {
	- syslog(LOG_ERR, "IOERROR: reading index header for %s: got %d of %d",
	+ syslog(LOG_ERR, "IOERROR: reading index header for %s: got %d of %ld",
	mailbox->name, n, mailbox->start_offset);
	goto fail;
	}
	/* Fix up exists */
	newexists = ntohl(((bit32 )(buf+OFFSET_EXISTS)))-numdeleted;
	((bit32 )(buf+OFFSET_EXISTS)) = htonl(newexists);
	/* fix up other counts */
	newanswered = ntohl(((bit32 )(buf+OFFSET_ANSWERED)))-numansweredflag;
	((bit32 )(buf+OFFSET_ANSWERED)) = htonl(newanswered);
	newdeleted = ntohl(((bit32 )(buf+OFFSET_DELETED)))-numdeletedflag;
	((bit32 )(buf+OFFSET_DELETED)) = htonl(newdeleted);
	newflagged = ntohl(((bit32 )(buf+OFFSET_FLAGGED)))-numflaggedflag;
	((bit32 )(buf+OFFSET_FLAGGED)) = htonl(newflagged);

	/* Fix up quota_mailbox_used */
	((bit32 )(buf+OFFSET_QUOTA_MAILBOX_USED)) =
	htonl(ntohl(((bit32 )(buf+OFFSET_QUOTA_MAILBOX_USED)))-quotadeleted);
	/* Fix up start offset if necessary */
	if (mailbox->start_offset < INDEX_HEADER_SIZE) {
	((bit32 )(buf+OFFSET_START_OFFSET)) = htonl(INDEX_HEADER_SIZE);
	}

	rewind(newindex);
	fwrite(buf, 1, mailbox->start_offset, newindex);

	/* Ensure everything made it to disk */
	fflush(newindex);
	fflush(newcache);
	if (ferror(newindex) \|\| ferror(newcache) \|\|
	fsync(fileno(newindex)) \|\| fsync(fileno(newcache))) {
	syslog(LOG_ERR, "IOERROR: writing index/cache for %s: %m",
	mailbox->name);
	goto fail;
	}

	/* Record quota release */
	r = mailbox_lock_quota(&mailbox->quota);
	if (r) goto fail;
	if (mailbox->quota.used >= quotadeleted) {
	mailbox->quota.used -= quotadeleted;
	}
	else {
	mailbox->quota.used = 0;
	}
	r = mailbox_write_quota(&mailbox->quota);
	if (r) {
	syslog(LOG_ERR,
	"LOSTQUOTA: unable to record free of %u bytes in quota %s",
	quotadeleted, mailbox->quota.root);
	}
	mailbox_unlock_quota(&mailbox->quota);

	strcpy(fnamebuf, mailbox->path);
	fnametail = fnamebuf + strlen(fnamebuf);
	strcpy(fnametail, FNAME_INDEX);
	strcpy(fnamebufnew, fnamebuf);
	strcat(fnamebufnew, ".NEW");
	if (rename(fnamebufnew, fnamebuf)) {
	syslog(LOG_ERR, "IOERROR: renaming index file for %s: %m",
	mailbox->name);
	goto fail;
	}

	strcpy(fnametail, FNAME_CACHE);
	strcpy(fnamebufnew, fnamebuf);
	strcat(fnamebufnew, ".NEW");
	if (rename(fnamebufnew, fnamebuf)) {
	syslog(LOG_CRIT,
	"CRITICAL IOERROR: renaming cache file for %s, need to reconstruct: %m",
	mailbox->name);
	/* Fall through and delete message files anyway */
	}

	if (numdeleted) {
	if (updatenotifier) updatenotifier(mailbox);

	if (acappush_sock != -1) {
	acapmbdata_t acapdata;

	/* fill in structure */
	strcpy(acapdata.name, mailbox->name);
	acapdata.uidvalidity = mailbox->uidvalidity;
	acapdata.exists = newexists;
	acapdata.deleted = newdeleted;
	acapdata.answered = newanswered;
	acapdata.flagged = newflagged;

	/* send */
	if (sendto(acappush_sock, &acapdata, 20+strlen(mailbox->name), 0,
	(struct sockaddr *) &acappush_remote,
	acappush_remote_len) == -1) {
	syslog(LOG_ERR, "Error sending to acappush: %m");
	}
	}
	}

	mailbox_unlock_pop(mailbox);
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);
	fclose(newindex);
	fclose(newcache);

	/* Delete message files */
	*fnametail++ = '/';
	for (msgno = 0; msgno < numdeleted; msgno++) {
	if (iscurrentdir) {
	char shortfnamebuf[MAILBOX_FNAME_LEN];
	mailbox_message_get_fname(mailbox, deleted[msgno], shortfnamebuf);
	unlink(shortfnamebuf);
	}
	else {
	mailbox_message_get_fname(mailbox, deleted[msgno], fnametail);
	unlink(fnamebuf);
	}
	}

	free(buf);
	if (deleted) free(deleted);

	return 0;

	fail:
	free(buf);
	free(deleted);
	fclose(newindex);
	fclose(newcache);
	mailbox_unlock_pop(mailbox);
	mailbox_unlock_index(mailbox);
	mailbox_unlock_header(mailbox);
	return IMAP_IOERROR;
	}

	/* find the mailbox 'name' 's quotaroot, and return it in 'start'.
	'start' must be at least MAX_MAILBOX_PATH.

	returns true if a quotaroot is found, 0 otherwise.
	*/
	int mailbox_findquota(char start, const char name)
	{
	char quota_path[MAX_MAILBOX_PATH];
	char *tail;
	struct stat sbuf;

	strcpy(start, name);
	lcase(start);

	mailbox_hash_quota(quota_path, start);
	while (stat(quota_path, &sbuf) == -1) {
	tail = strrchr(start, '.');
	if (!tail) return 0;
	*tail = '\0';
	mailbox_hash_quota(quota_path, start);
	}
	return 1;
	}


	int mailbox_create(const char *name,
	char *path,
	const char *acl,
	const char *uniqueid,
	int format,
	struct mailbox *mailboxp)
	{
	int r;
	char *p=path;
	char quota_root[MAX_MAILBOX_PATH];
	int hasquota;
	char fnamebuf[MAX_MAILBOX_PATH];
	struct mailbox mailbox;
	int save_errno;
	int n;
	struct stat sbuf;

	while ((p = strchr(p+1, '/'))) {
	*p = '\0';
	if (mkdir(path, 0755) == -1 && errno != EEXIST) {
	save_errno = errno;
	if (stat(path, &sbuf) == -1) {
	errno = save_errno;
	syslog(LOG_ERR, "IOERROR: creating directory %s: %m", path);
	return IMAP_IOERROR;
	}
	}
	*p = '/';
	}
	if (mkdir(path, 0755) == -1 && errno != EEXIST) {
	save_errno = errno;
	if (stat(path, &sbuf) == -1) {
	errno = save_errno;
	syslog(LOG_ERR, "IOERROR: creating directory %s: %m", path);
	return IMAP_IOERROR;
	}
	}

	zeromailbox(mailbox);
	mailbox.quota.fd = -1;

	hasquota = mailbox_findquota(quota_root, name);

	strcpy(fnamebuf, path);
	p = fnamebuf + strlen(fnamebuf);
	strcpy(p, FNAME_HEADER);
	mailbox.header_fd = open(fnamebuf, O_WRONLY\|O_TRUNC\|O_CREAT, 0666);
	if (mailbox.header_fd == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fnamebuf);
	return IMAP_IOERROR;
	}

	mailbox.name = xstrdup(name);
	mailbox.path = xstrdup(path);
	mailbox.acl = xstrdup(acl);

	strcpy(p, FNAME_INDEX);
	mailbox.index_fd = open(fnamebuf, O_WRONLY\|O_TRUNC\|O_CREAT, 0666);
	if (mailbox.index_fd == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fnamebuf);
	mailbox_close(&mailbox);
	return IMAP_IOERROR;
	}

	strcpy(p, FNAME_CACHE);
	mailbox.cache_fd = open(fnamebuf, O_WRONLY\|O_TRUNC\|O_CREAT, 0666);
	if (mailbox.cache_fd == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fnamebuf);
	mailbox_close(&mailbox);
	return IMAP_IOERROR;
	}

	mailbox.header_lock_count = 1;
	mailbox.index_lock_count = 1;

	if (hasquota) mailbox.quota.root = xstrdup(quota_root);
	mailbox.generation_no = 0;
	mailbox.format = format;
	mailbox.minor_version = MAILBOX_MINOR_VERSION;
	mailbox.start_offset = INDEX_HEADER_SIZE;
	mailbox.record_size = INDEX_RECORD_SIZE;
	mailbox.exists = 0;
	mailbox.last_appenddate = 0;
	mailbox.last_uid = 0;
	mailbox.quota_mailbox_used = 0;
	mailbox.pop3_last_login = 0;
	mailbox.uidvalidity = time(0);
	mailbox.deleted = 0;
	mailbox.answered = 0;
	mailbox.flagged = 0;

	if (!uniqueid) {
	mailbox.uniqueid = xmalloc(sizeof(char) * 32);
	mailbox_make_uniqueid(mailbox.name, mailbox.uidvalidity,
	mailbox.uniqueid);
	} else {
	mailbox.uniqueid = xstrdup(uniqueid);
	}

	r = mailbox_write_header(&mailbox);
	if (!r) r = mailbox_write_index_header(&mailbox);
	if (!r) {
	n = retry_write(mailbox.cache_fd, (char *)&mailbox.generation_no, 4);
	if (n != 4 \|\| fsync(mailbox.cache_fd)) {
	syslog(LOG_ERR, "IOERROR: writing initial cache for %s: %m",
	mailbox.name);
	r = IMAP_IOERROR;
	}
	}
	if (!r) r = seen_create_mailbox(&mailbox);

	if (mailboxp) {
	*mailboxp = mailbox;
	}
	else {
	mailbox_close(&mailbox);
	}
	return r;
	}

	/*
	* Delete and close the mailbox 'mailbox'. Closes 'mailbox' whether
	* or not the deletion was successful.
	*/
	int mailbox_delete(struct mailbox *mailbox, int delete_quota_root)
	{
	int r;
	DIR *dirp;
	struct dirent *f;
	char buf[MAX_MAILBOX_PATH];
	char *tail;

	/* Lock everything in sight */
	r = mailbox_lock_header(mailbox);
	if (!r && mailbox->index_fd == -1) r = mailbox_open_index(mailbox);
	if (!r) r = mailbox_lock_index(mailbox);
	if (!r) r = mailbox_lock_quota(&mailbox->quota);
	if (r) {
	mailbox_close(mailbox);
	return r;
	}

	seen_delete_mailbox(mailbox);

	if (delete_quota_root) {
	mailbox_delete_quota(&mailbox->quota);
	} else {
	/* Free any quota being used by this mailbox */
	if (mailbox->quota.used >= mailbox->quota_mailbox_used) {
	mailbox->quota.used -= mailbox->quota_mailbox_used;
	}
	else {
	mailbox->quota.used = 0;
	}
	r = mailbox_write_quota(&mailbox->quota);
	if (r) {
	syslog(LOG_ERR,
	- "LOSTQUOTA: unable to record free of %u bytes in quota %s",
	+ "LOSTQUOTA: unable to record free of %lu bytes in quota %s",
	mailbox->quota_mailbox_used, mailbox->quota.root);
	}
	mailbox_unlock_quota(&mailbox->quota);
	}

	/* remove all files in directory */
	strcpy(buf, mailbox->path);
	tail = buf + strlen(buf);
	*tail++ = '/';
	dirp = opendir(mailbox->path);
	if (dirp) {
	while ((f = readdir(dirp))!=NULL) {
	if (f->d_name[0] == '.'
	&& (f->d_name[1] == '\0'
	\|\| (f->d_name[1] == '.' &&
	f->d_name[2] == '\0'))) {
	/* readdir() can return "." or "..", and I got a bug report
	that SCO might blow the file system to smithereens if we
	unlink(".."). Let's not do that. */
	continue;
	}
	strcpy(tail, f->d_name);
	(void) unlink(buf);
	}
	closedir(dirp);
	}

	/* Remove empty directories, going up path */
	tail--;
	do {
	*tail = '\0';
	} while (rmdir(buf) == 0 && (tail = strrchr(buf, '/')));

	mailbox_close(mailbox);
	return 0;
	}

	/*
	* Expunge decision proc used by mailbox_rename() to expunge all messages
	* in INBOX
	*/
	static int expungeall(struct mailbox *mailbox __attribute__((unused)),
	void *rock __attribute__((unused)),
	char *indexbuf __attribute__((unused)))
	{
	return 1;
	}

	int
	mailbox_rename(const char oldname, const char oldpath, const char *oldacl,
	const char newname, char newpath, int isinbox,
	bit32 olduidvalidityp, bit32 newuidvalidityp,
	struct mailbox *mailboxp)
	{
	int r, r2;
	struct mailbox oldmailbox, newmailbox;
	int flag, msgno;
	struct index_record record;
	char oldfname[MAX_MAILBOX_PATH], newfname[MAX_MAILBOX_PATH];
	char oldfnametail, newfnametail;

	/* Open old mailbox and lock */
	mailbox_open_header_path(oldname, oldpath, oldacl, 0, &oldmailbox, 0);

	if (oldmailbox.format == MAILBOX_FORMAT_NETNEWS) {
	mailbox_close(&oldmailbox);
	return IMAP_MAILBOX_NOTSUPPORTED;
	}

	r = mailbox_lock_header(&oldmailbox);
	if (!r) r = mailbox_open_index(&oldmailbox);
	if (!r) r = mailbox_lock_index(&oldmailbox);
	if (r) {
	mailbox_close(&oldmailbox);
	return r;
	}

	/* Create new mailbox */
	r = mailbox_create(newname, newpath,
	oldmailbox.acl, oldmailbox.uniqueid, oldmailbox.format,
	&newmailbox);
	if (r) {
	mailbox_close(&oldmailbox);
	return r;
	}
	if (strcmp(oldname, newname) == 0) {
	/* Just moving mailboxes between partitions */
	newmailbox.uidvalidity = oldmailbox.uidvalidity;
	}

	if (olduidvalidityp) *olduidvalidityp = oldmailbox.uidvalidity;
	if (newuidvalidityp) *newuidvalidityp = newmailbox.uidvalidity;

	/* Copy flag names */
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (oldmailbox.flagname[flag]) {
	newmailbox.flagname[flag] = xstrdup(oldmailbox.flagname[flag]);
	}
	}
	r = mailbox_write_header(&newmailbox);
	if (r) {
	mailbox_close(&newmailbox);
	mailbox_close(&oldmailbox);
	return r;
	}

	/* Check quota if necessary */
	if (newmailbox.quota.root) {
	r = mailbox_lock_quota(&newmailbox.quota);
	if (!oldmailbox.quota.root \|\|
	strcmp(oldmailbox.quota.root, newmailbox.quota.root) != 0) {
	if (!r && newmailbox.quota.limit >= 0 &&
	newmailbox.quota.used + oldmailbox.quota_mailbox_used >
	newmailbox.quota.limit * QUOTA_UNITS) {
	r = IMAP_QUOTA_EXCEEDED;
	}
	}
	if (r) {
	mailbox_close(&newmailbox);
	mailbox_close(&oldmailbox);
	return r;
	}
	}

	strcpy(oldfname, oldmailbox.path);
	oldfnametail = oldfname + strlen(oldfname);
	strcpy(newfname, newmailbox.path);
	newfnametail = newfname + strlen(newfname);

	/* Copy over index/cache files */
	strcpy(oldfnametail, FNAME_INDEX);
	strcpy(newfnametail, FNAME_INDEX);
	unlink(newfname); /* Make link() possible */
	r = mailbox_copyfile(oldfname, newfname);
	strcpy(oldfnametail, FNAME_CACHE);
	strcpy(newfnametail, FNAME_CACHE);
	unlink(newfname);
	if (!r) r = mailbox_copyfile(oldfname, newfname);
	if (r) {
	mailbox_close(&newmailbox);
	mailbox_close(&oldmailbox);
	return r;
	}

	/* Re-open index file and store new uidvalidity */
	close(newmailbox.index_fd);
	newmailbox.index_fd = dup(oldmailbox.index_fd);
	(void) mailbox_read_index_header(&newmailbox);
	newmailbox.generation_no = oldmailbox.generation_no;
	(void) mailbox_write_index_header(&newmailbox);

	/* Copy over message files */
	oldfnametail++;
	newfnametail++;
	for (msgno = 1; msgno <= oldmailbox.exists; msgno++) {
	r = mailbox_read_index_record(&oldmailbox, msgno, &record);
	if (r) break;
	mailbox_message_get_fname(&oldmailbox, record.uid, oldfnametail);
	strcpy(newfnametail, oldfnametail);
	r = mailbox_copyfile(oldfname, newfname);
	if (r) break;
	}
	if (!r) r = seen_copy(&oldmailbox, &newmailbox);

	/* Record new quota usage */
	if (!r && newmailbox.quota.root) {
	newmailbox.quota.used += oldmailbox.quota_mailbox_used;
	r = mailbox_write_quota(&newmailbox.quota);
	mailbox_unlock_quota(&newmailbox.quota);
	}
	if (r) goto fail;

	if (isinbox) {
	/* Expunge old mailbox */
	r = mailbox_expunge(&oldmailbox, 0, expungeall, (char *)0);
	mailbox_close(&oldmailbox);
	} else {
	r = mailbox_delete(&oldmailbox, 0);
	}

	if (r && newmailbox.quota.root) {
	r2 = mailbox_lock_quota(&newmailbox.quota);
	newmailbox.quota.used += newmailbox.quota_mailbox_used;
	if (!r2) {
	r2 = mailbox_write_quota(&newmailbox.quota);
	mailbox_unlock_quota(&newmailbox.quota);
	}
	if (r2) {
	syslog(LOG_ERR,
	- "LOSTQUOTA: unable to record use of %u bytes in quota %s",
	+ "LOSTQUOTA: unable to record use of %lu bytes in quota %s",
	newmailbox.quota_mailbox_used, newmailbox.quota.root);
	}
	}
	if (r) goto fail;

	if (mailboxp) {
	*mailboxp = newmailbox;
	} else {
	mailbox_close(&newmailbox);
	}
	return 0;

	fail:
	for (msgno = 1; msgno <= oldmailbox.exists; msgno++) {
	if (mailbox_read_index_record(&oldmailbox, msgno, &record)) continue;
	mailbox_message_get_fname(&oldmailbox, record.uid, newfnametail);
	(void) unlink(newfname);
	}
	mailbox_close(&newmailbox);
	mailbox_close(&oldmailbox);
	return r;
	}


	/*
	* Synchronize 'new' mailbox to 'old' mailbox.
	*/
	int
	mailbox_sync(const char oldname, const char oldpath, const char *oldacl,
	const char newname, char newpath, int docreate,
	bit32 olduidvalidityp, bit32 newuidvalidityp,
	struct mailbox *mailboxp)
	{
	int r, r2;
	struct mailbox oldmailbox, newmailbox;
	int flag, oldmsgno, newmsgno;
	struct index_record oldrecord, newrecord;
	char oldfname[MAX_MAILBOX_PATH], newfname[MAX_MAILBOX_PATH];
	char oldfnametail, newfnametail;

	/* Open old mailbox and lock */
	mailbox_open_header_path(oldname, oldpath, oldacl, 0, &oldmailbox, 0);

	if (oldmailbox.format == MAILBOX_FORMAT_NETNEWS) {
	mailbox_close(&oldmailbox);
	return IMAP_MAILBOX_NOTSUPPORTED;
	}

	r = mailbox_lock_header(&oldmailbox);
	if (!r) r = mailbox_open_index(&oldmailbox);
	if (!r) r = mailbox_lock_index(&oldmailbox);
	if (r) {
	mailbox_close(&oldmailbox);
	return r;
	}

	if (docreate) {
	/* Create new mailbox */
	r = mailbox_create(newname, newpath,
	oldmailbox.acl, oldmailbox.uniqueid, oldmailbox.format,
	&newmailbox);
	}
	else {
	/* Open new mailbox and lock */
	r = mailbox_open_header_path(newname, newpath, oldacl, 0, &newmailbox, 0);
	r = mailbox_lock_header(&newmailbox);
	if (!r) r = mailbox_open_index(&newmailbox);
	if (!r) r = mailbox_lock_index(&newmailbox);
	if (r) {
	mailbox_close(&newmailbox);
	}
	}
	if (r) {
	mailbox_close(&oldmailbox);
	return r;
	}

	newmailbox.uidvalidity = oldmailbox.uidvalidity;
	if (olduidvalidityp) *olduidvalidityp = oldmailbox.uidvalidity;
	if (newuidvalidityp) *newuidvalidityp = newmailbox.uidvalidity;

	/* Copy flag names */
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (oldmailbox.flagname[flag]) {
	newmailbox.flagname[flag] = xstrdup(oldmailbox.flagname[flag]);
	}
	}
	r = mailbox_write_header(&newmailbox);
	if (r) {
	mailbox_close(&newmailbox);
	mailbox_close(&oldmailbox);
	return r;
	}

	/* Check quota if necessary */
	if (newmailbox.quota.root) {
	r = mailbox_lock_quota(&newmailbox.quota);
	if (!oldmailbox.quota.root \|\|
	strcmp(oldmailbox.quota.root, newmailbox.quota.root) != 0) {
	if (!r && newmailbox.quota.limit >= 0 &&
	newmailbox.quota.used + oldmailbox.quota_mailbox_used >
	newmailbox.quota.limit * QUOTA_UNITS) {
	r = IMAP_QUOTA_EXCEEDED;
	}
	}
	if (r) {
	mailbox_close(&newmailbox);
	mailbox_close(&oldmailbox);
	return r;
	}
	}

	strcpy(oldfname, oldmailbox.path);
	strcat(oldfname, "/");
	oldfnametail = oldfname + strlen(oldfname);
	strcpy(newfname, newmailbox.path);
	strcat(newfname, "/");
	newfnametail = newfname + strlen(newfname);

	/*
	* Copy over new message files and delete expunged ones.
	*
	* We use the fact that UIDs are monotonically increasing to our
	* advantage; we compare the UIDs from each mailbox in order, and:
	*
	* - if UID in "slave" mailbox < UID in "master" mailbox,
	* then the message has been deleted from "master" since last sync,
	* so delete it from "slave" and move on to next "slave" UID
	* - if UID in "slave" mailbox == UID in "master" mailbox,
	* then message is still current and we already have a copy,
	* so move on to next UID in each mailbox
	* - if UID in "master" mailbox > last UID in "slave" mailbox,
	* then this is a new arrival in "master" since last sync,
	* so copy it to "slave" and move on to next "master" UID
	*/
	newmsgno = 1;
	for (oldmsgno = 1; oldmsgno <= oldmailbox.exists; oldmsgno++) {
	r = mailbox_read_index_record(&oldmailbox, oldmsgno, &oldrecord);
	if (r) break;
	if (newmsgno <= newmailbox.exists) {
	do {
	r = mailbox_read_index_record(&newmailbox, newmsgno,
	&newrecord);
	if (r) goto fail;
	newmsgno++;

	if (newrecord.uid < oldrecord.uid) {
	/* message expunged since last sync - delete message file */
	mailbox_message_get_fname(&newmailbox, newrecord.uid,
	newfnametail);
	unlink(newfname);
	}
	} while ((newrecord.uid < oldrecord.uid) &&
	(newmsgno <= newmailbox.exists));
	}
	/* we check 'exists' instead of last UID in case of empty mailbox */
	if (newmsgno > newmailbox.exists) {
	/* message arrived since last sync - copy message file */
	mailbox_message_get_fname(&oldmailbox, oldrecord.uid, oldfnametail);
	strcpy(newfnametail, oldfnametail);
	r = mailbox_copyfile(oldfname, newfname);
	if (r) break;
	}
	}
	if (!r) r = seen_copy(&oldmailbox, &newmailbox);

	if (!r) {
	/* Copy over index/cache files */
	oldfnametail--;
	newfnametail--;
	strcpy(oldfnametail, FNAME_INDEX);
	strcpy(newfnametail, FNAME_INDEX);
	unlink(newfname); /* Make link() possible */
	r = mailbox_copyfile(oldfname, newfname);
	strcpy(oldfnametail, FNAME_CACHE);
	strcpy(newfnametail, FNAME_CACHE);
	unlink(newfname);
	if (!r) r = mailbox_copyfile(oldfname, newfname);
	if (r) {
	mailbox_close(&newmailbox);
	mailbox_close(&oldmailbox);
	return r;
	}

	/* Re-open index file and store new uidvalidity */
	close(newmailbox.index_fd);
	newmailbox.index_fd = dup(oldmailbox.index_fd);
	(void) mailbox_read_index_header(&newmailbox);
	newmailbox.generation_no = oldmailbox.generation_no;
	(void) mailbox_write_index_header(&newmailbox);
	}

	/* Record new quota usage */
	if (!r && newmailbox.quota.root) {
	newmailbox.quota.used += oldmailbox.quota_mailbox_used;
	r = mailbox_write_quota(&newmailbox.quota);
	mailbox_unlock_quota(&newmailbox.quota);
	}
	if (r) goto fail;

	if (r && newmailbox.quota.root) {
	r2 = mailbox_lock_quota(&newmailbox.quota);
	newmailbox.quota.used += newmailbox.quota_mailbox_used;
	if (!r2) {
	r2 = mailbox_write_quota(&newmailbox.quota);
	mailbox_unlock_quota(&newmailbox.quota);
	}
	if (r2) {
	syslog(LOG_ERR,
	- "LOSTQUOTA: unable to record use of %u bytes in quota %s",
	+ "LOSTQUOTA: unable to record use of %lu bytes in quota %s",
	newmailbox.quota_mailbox_used, newmailbox.quota.root);
	}
	}
	if (r) goto fail;

	if (mailboxp) {
	*mailboxp = newmailbox;
	} else {
	mailbox_close(&newmailbox);
	}
	return 0;

	fail:
	#if 0
	for (msgno = 1; msgno <= oldmailbox.exists; msgno++) {
	if (mailbox_read_index_record(&oldmailbox, msgno, &record)) continue;
	mailbox_message_get_fname(&oldmailbox, record.uid, newfnametail);
	(void) unlink(newfname);
	}
	#endif
	mailbox_close(&newmailbox);
	mailbox_close(&oldmailbox);
	return r;
	}


	/*
	* Copy (or link) the file 'from' to the file 'to'
	*/
	int mailbox_copyfile(from, to)
	const char *from;
	const char *to;
	{
	int srcfd, destfd;
	struct stat sbuf;
	const char *src_base = 0;
	unsigned long src_size = 0;
	int n;

	if (link(from, to) == 0) return 0;
	if (errno == EEXIST) {
	if (unlink(to) == -1) {
	syslog(LOG_ERR, "IOERROR: unlinking to recreate %s: %m", to);
	return IMAP_IOERROR;
	}
	if (link(from, to) == 0) return 0;
	}

	destfd = open(to, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	if (destfd == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", to);
	return IMAP_IOERROR;
	}

	srcfd = open(from, O_RDONLY, 0666);
	if (srcfd == -1) {
	syslog(LOG_ERR, "IOERROR: opening %s: %m", from);
	close(destfd);
	return IMAP_IOERROR;
	}


	if (fstat(srcfd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat on %s: %m", from);
	close(srcfd);
	close(destfd);
	return IMAP_IOERROR;
	}
	map_refresh(srcfd, 1, &src_base, &src_size, sbuf.st_size, from, 0);

	n = retry_write(destfd, src_base, src_size);

	if (n == -1 \|\| fsync(destfd)) {
	map_free(&src_base, &src_size);
	close(srcfd);
	close(destfd);
	syslog(LOG_ERR, "IOERROR: writing %s: %m", to);
	return IMAP_IOERROR;
	}
	map_free(&src_base, &src_size);
	close(srcfd);
	close(destfd);
	return 0;
	}

	void mailbox_hash_mbox(char *buf,
	const char *root,
	const char *name)
	{
	const char *idx;
	char c, *p;

	if (config_hashimapspool) {
	idx = strchr(name, '.');
	if (idx == NULL) {
	idx = name;
	} else {
	idx++;
	}
	c = (char) dir_hash_c(idx);

	sprintf(buf, "%s/%c/%s", root, c, name);
	} else {
	/* standard mailbox placement */
	sprintf(buf, "%s/%s", root, name);
	}

	/* change all '.'s to '/' */
	for (p = buf + strlen(root) + 1; *p; p++) {
	if (p == '.') p = '/';
	}
	}

	/* simple hash so it's easy to find these things in the filesystem;
	our human time is worth more than efficiency */
	void mailbox_hash_quota(char buf, const char qr)
	{
	const char *idx;
	char c;

	idx = strchr(qr, '.'); /* skip past user. */
	if (idx == NULL) {
	idx = qr;
	} else {
	idx++;
	}
	c = (char) dir_hash_c(idx);

	sprintf(buf, "%s%s%c/%s", config_dir, FNAME_QUOTADIR, c, qr);
	}
	diff --git a/imap/mboxlist.c b/imap/mboxlist.c
	index ad5b69e9b..1999b0531 100644
	--- a/imap/mboxlist.c
	+++ b/imap/mboxlist.c
	@@ -1,2512 +1,2512 @@
	/* mboxlist.c -- Mailbox list manipulation routines
	*
	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	*/
	/*
	- * $Id: mboxlist.c,v 1.167 2002/02/12 20:35:52 rjs3 Exp $
	+ * $Id: mboxlist.c,v 1.168 2002/02/19 18:50:13 ken3 Exp $
	*/

	#include <config.h>

	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <errno.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/uio.h>
	#include <fcntl.h>
	#include <ctype.h>
	#include <time.h>
	#include <syslog.h>
	#include <com_err.h>

	#include <sys/ipc.h>
	#include <sys/msg.h>

	extern int errno;

	#include "acl.h"
	#include "auth.h"
	#include "glob.h"
	#include "assert.h"
	#include "imapconf.h"
	#include "cyrusdb.h"
	#include "util.h"
	#include "mailbox.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "xmalloc.h"

	#include "mboxname.h"
	#include "mupdate-client.h"

	#include "mboxlist.h"

	#define DB CONFIG_DB_MBOX
	#define SUBDB CONFIG_DB_SUBS

	cyrus_acl_canonproc_t mboxlist_ensureOwnerRights;

	struct db *mbdb;

	static int mboxlist_dbopen = 0;

	#define HOSTNAME_SIZE 512
	static const char *mupdate_server = NULL;

	static int mboxlist_opensubs();
	static void mboxlist_closesubs();

	static int mboxlist_changequota(const char *name, int matchlen, int maycreate,
	void *rock);

	#define FNAME_SUBSSUFFIX ".sub"

	/*
	* Convert a partition into a path
	*/
	static int mboxlist_getpath(const char partition, const char name,
	char **pathp)
	{
	int partitionlen;
	char optionbuf[MAX_MAILBOX_NAME+1];
	static char pathresult[MAX_MAILBOX_PATH];
	const char *root;

	assert(partition && pathp);

	partitionlen = strlen(partition);

	if (partitionlen > sizeof(optionbuf)-11) {
	return IMAP_PARTITION_UNKNOWN;
	}
	strcpy(optionbuf, "partition-");
	strcat(optionbuf, partition);

	root = config_getstring(optionbuf, (char *)0);
	if (!root) {
	return IMAP_PARTITION_UNKNOWN;
	}
	mailbox_hash_mbox(pathresult, root, name);

	*pathp = pathresult;

	return 0;
	}

	char mboxlist_makeentry(int mbtype, const char part, const char *acl)
	{
	char mboxent = (char ) xmalloc(sizeof(char) *
	(30 + strlen(acl) + strlen(part)));
	sprintf(mboxent, "%d %s %s", mbtype, part, acl);
	return mboxent;
	}

	static const int get_deleteright(void)
	{
	const char *r = config_getstring("deleteright", "c");

	return cyrus_acl_strtomask(r);
	}

	/*
	* Lookup 'name' in the mailbox list.
	* The capitalization of 'name' is canonicalized to the way it appears
	* in the mailbox list.
	* If 'path' is non-nil, a pointer to the full pathname of the mailbox
	* is placed in the char * pointed to by it. If 'acl' is non-nil, a pointer
	* to the mailbox ACL is placed in the char * pointed to by it.
	*/
	static int mboxlist_mylookup(const char name, int typep,
	char pathp, char partp,
	char **aclp,
	struct txn **tid, int wrlock)
	{
	int acllen;
	static char partition[MAX_PARTITION_LEN];
	static char *aclresult;
	static int aclresultalloced;
	int r;
	const char *data;
	char p, q;
	int datalen;
	int namelen;
	int mbtype;

	namelen = strlen(name);

	if (namelen == 0) {
	return IMAP_MAILBOX_NONEXISTENT;
	}
	if (wrlock) {
	r = DB->fetchlock(mbdb, name, namelen, &data, &datalen, tid);
	} else {
	r = DB->fetch(mbdb, name, namelen, &data, &datalen, tid);
	}
	switch (r) {
	case CYRUSDB_OK:
	if (data == NULL) {
	return IMAP_MAILBOX_NONEXISTENT;
	break;
	}

	/* copy out interesting parts */
	mbtype = strtol(data, &p, 10);
	if (typep) *typep = mbtype;

	if (*p == ' ') p++;
	q = partition;
	while (p != ' ') { / copy out partition name */
	q++ = p++;
	}
	*q = '\0';
	p++;

	if (partp) {
	*partp = partition;
	}

	/* construct pathname if requested */
	if (pathp) {
	if (mbtype & MBTYPE_REMOTE) {
	*pathp = partition;
	} else {
	r = mboxlist_getpath(partition, name, pathp);
	if (r) {
	return r;
	}
	}
	}

	/* the rest is ACL; return it if requested */
	if (aclp) {
	acllen = datalen - (p - data);
	if (acllen >= aclresultalloced) {
	aclresultalloced = acllen + 100;
	aclresult = xrealloc(aclresult, aclresultalloced);
	}
	memcpy(aclresult, p, acllen);
	aclresult[acllen] = '\0';

	*aclp = aclresult;
	}
	break;

	case CYRUSDB_AGAIN:
	return IMAP_AGAIN;
	break;

	default:
	syslog(LOG_ERR, "DBERROR: error fetching %s: %s",
	name, cyrusdb_strerror(r));
	return IMAP_IOERROR;
	break;
	}

	return 0;
	}

	/*
	* Lookup 'name' in the mailbox list.
	* The capitalization of 'name' is canonicalized to the way it appears
	* in the mailbox list.
	* If 'path' is non-nil, a pointer to the full pathname of the mailbox
	* is placed in the char * pointed to by it. If 'acl' is non-nil, a pointer
	* to the mailbox ACL is placed in the char * pointed to by it.
	*/
	int mboxlist_lookup(const char name, char pathp, char *aclp,
	void *tid __attribute__((unused)))
	{
	return mboxlist_mylookup(name, NULL, pathp, NULL, aclp, NULL, 0);
	}

	int mboxlist_detail(const char name, int typep, char pathp, char partp,
	char *aclp, struct txn tid __attribute__((unused)))
	{
	return mboxlist_mylookup(name, typep, pathp, partp, aclp, NULL, 0);
	}

	int mboxlist_findstage(const char name, char stagedir)
	{
	char optionbuf[MAX_MAILBOX_NAME+1];
	const char *root;
	char *partition;
	int r;

	assert(stagedir != NULL);

	/* Find mailbox */
	r = mboxlist_mylookup(name, NULL, NULL, &partition, NULL, NULL, 0);
	switch (r) {
	case 0:
	break;
	default:
	return r;
	break;
	}

	strcpy(optionbuf, "partition-");
	strcpy(optionbuf + 10, partition);

	root = config_getstring(optionbuf, (char *)0);
	if (!root) {
	return IMAP_PARTITION_UNKNOWN;
	}

	sprintf(stagedir, "%s/stage./", root);

	return 0;
	}

	/*
	* Check/set up for mailbox creation
	*/
	int
	mboxlist_mycreatemailboxcheck(char name, int mbtype, char partition,
	int isadmin, char *userid,
	struct auth_state *auth_state,
	char newacl, char newpartition,
	int RMW, struct txn **tid)
	{
	int r;
	char *p;
	char acl, path;
	char defaultacl, identifier, *rights;
	char parent[MAX_MAILBOX_NAME+1];
	unsigned long parentlen;
	char *parentname = NULL;
	char *parentpartition = NULL;
	char *parentacl = NULL;
	unsigned long parentpartitionlen = 0;
	unsigned long parentacllen = 0;

	/* Check for invalid name/partition */
	if (partition && strlen(partition) > MAX_PARTITION_LEN) {
	return IMAP_PARTITION_UNKNOWN;
	}
	r = mboxname_policycheck(name);
	if (r) return r;

	/* User has admin rights over their own mailbox namespace */
	if (mboxname_userownsmailbox(userid, name)) {
	isadmin = 1;
	}

	/* Check to see if new mailbox exists */
	r = mboxlist_mylookup(name, NULL, &path, NULL, &acl, tid, RMW);
	switch (r) {
	case 0:
	r = IMAP_MAILBOX_EXISTS;

	/* Lie about error if privacy demands */
	if (!isadmin &&
	!(cyrus_acl_myrights(auth_state, acl) & ACL_LOOKUP)) {
	r = IMAP_PERMISSION_DENIED;
	}
	return r;
	break;
	case IMAP_MAILBOX_NONEXISTENT:
	break;

	default:
	return r;
	break;
	}

	/* Search for a parent */
	strcpy(parent, name);
	parentlen = 0;
	while ((parentlen==0) && (p = strrchr(parent, '.'))) {
	*p = '\0';

	r = mboxlist_mylookup(parent, NULL, NULL, &parentpartition,
	&parentacl, tid, 0);
	switch (r) {
	case 0:
	parentlen = strlen(parent);
	parentname = parent;

	parentpartitionlen = strlen(parentpartition);

	parentacllen = strlen(parentacl);
	break;

	case IMAP_MAILBOX_NONEXISTENT:
	break;

	default:
	return r;
	break;
	}
	}
	if (parentlen != 0) {
	/* check acl */
	if (!isadmin &&
	!(cyrus_acl_myrights(auth_state, parentacl) & ACL_CREATE)) {
	return IMAP_PERMISSION_DENIED;
	}

	/* Copy partition, if not specified */
	if (partition == NULL) {
	partition = xmalloc(parentpartitionlen + 1);
	memcpy(partition, parentpartition, parentpartitionlen);
	partition[parentpartitionlen] = '\0';
	} else {
	partition = xstrdup(partition);
	}

	/* Copy ACL */
	acl = xmalloc(parentacllen + 1);
	memcpy(acl, parentacl, parentacllen);
	acl[parentacllen] = '\0';

	/* Canonicalize case of parent prefix */
	strlcpy(name, parent, strlen(parent));
	} else { /* parentlen == 0, no parent mailbox */
	if (!isadmin) {
	return IMAP_PERMISSION_DENIED;
	}

	acl = xstrdup("");
	if (!strncmp(name, "user.", 5)) {
	if (strchr(name+5, '.')) {
	/* Disallow creating user.X.* when no user.X */
	free(acl);
	return IMAP_PERMISSION_DENIED;
	}
	/* disallow wildcards in userids with inboxes. */
	if (strchr(name, '*') \|\| strchr(name, '%') \|\| strchr(name, '?')) {
	return IMAP_MAILBOX_BADNAME;
	}
	/*
	* Users by default have all access to their personal mailbox(es),
	* Nobody else starts with any access to same.
	*/
	identifier = xstrdup(name+5);
	if (config_getswitch("unixhierarchysep", 0)) {
	/*
	* The mailboxname is now in the internal format,
	* so we we need to change DOTCHARs back to '.'
	* in the identifier in order to have the correct ACL.
	*/
	for (p = identifier; *p; p++) {
	if (p == DOTCHAR) p = '.';
	}
	}
	cyrus_acl_set(&acl, identifier, ACL_MODE_SET, ACL_ALL,
	(cyrus_acl_canonproc_t )0, (void )0);
	free(identifier);
	} else {
	defaultacl = identifier =
	xstrdup(config_getstring("defaultacl", "anyone lrs"));
	for (;;) {
	while (identifier && isspace((int) identifier)) identifier++;
	rights = identifier;
	while (rights && !isspace((int) rights)) rights++;
	if (!*rights) break;
	*rights++ = '\0';
	while (rights && isspace((int) rights)) rights++;
	if (!*rights) break;
	p = rights;
	while (p && !isspace((int) p)) p++;
	if (p) p++ = '\0';
	cyrus_acl_set(&acl, identifier, ACL_MODE_SET, cyrus_acl_strtomask(rights),
	(cyrus_acl_canonproc_t )0, (void )0);
	identifier = p;
	}
	free(defaultacl);
	}

	if (!partition) {
	partition = (char *)config_defpartition;
	if (strlen(partition) > MAX_PARTITION_LEN) {
	/* Configuration error */
	fatal("name of default partition is too long", EC_CONFIG);
	}
	}
	partition = xstrdup(partition);
	}

	if (newpartition) *newpartition = partition;
	else free(partition);
	if (newacl) *newacl = acl;
	else free(acl);

	return 0;
	}

	int
	mboxlist_createmailboxcheck(char name, int mbtype, char partition,
	int isadmin, char *userid,
	struct auth_state *auth_state,
	char newacl, char newpartition)
	{
	return mboxlist_mycreatemailboxcheck(name, mbtype, partition, isadmin,
	userid, auth_state, newacl,
	newpartition, 0, NULL);
	}

	/*
	* Create a mailbox
	*
	* 1. start mailboxes transaction
	* 2. verify ACL's to best of ability (CRASH: abort)
	* 3. open mupdate connection if necessary
	* 4. verify parent ACL's if need to
	* 5. create mupdate entry and set as reserved (CRASH: mupdate inconsistant)
	* 6. create on disk (CRASH: mupdate inconsistant, disk inconsistant)
	* 8. commit local transaction (CRASH: mupdate inconsistant)
	* 9. set mupdate entry as commited (CRASH: commited)
	*
	*/

	int mboxlist_createmailbox(char name, int mbtype, char partition,
	int isadmin, char *userid,
	struct auth_state *auth_state)
	{
	int r;
	char *acl = NULL;
	const char *root = NULL;
	char *newpartition = NULL;
	struct mailbox newmailbox;
	struct txn *tid = NULL;
	mupdate_handle *mupdate_h;
	char *mboxent = NULL;

	/* Must be atleast MAX_PARTITION_LEN + 30 for partition, need
	* MAX_PARTITION_LEN + HOSTNAME_SIZE + 2 for mupdate location */
	char buf[MAX_PARTITION_LEN + HOSTNAME_SIZE + 2];
	int madereserved = 0; /* made reserved entry on mupdate server */

	retry:
	tid = NULL;

	/* 2. verify ACL's to best of ability (CRASH: abort) */
	r = mboxlist_mycreatemailboxcheck(name, mbtype, partition, isadmin,
	userid, auth_state,
	&acl, &newpartition, 1, &tid);
	switch (r) {
	case 0:
	break;
	case IMAP_AGAIN:
	goto retry;
	default:
	goto done;
	}

	if (!(mbtype & MBTYPE_REMOTE)) {
	/* Get partition's path */
	sprintf(buf, "partition-%s", newpartition);
	root = config_getstring(buf, (char *)0);
	if (!root) {
	r = IMAP_PARTITION_UNKNOWN;
	goto done;
	}
	if (strlen(root)+strlen(name)+20 > MAX_MAILBOX_PATH) {
	r = IMAP_MAILBOX_BADNAME;
	goto done;
	}
	}

	/* 4. Create mupdate reservation */
	if (mupdate_server) {
	r = mupdate_connect(mupdate_server, NULL, &mupdate_h, NULL);
	if(r) {
	syslog(LOG_ERR,
	"can not connect to mupdate server for reservation on '%s'",
	name);
	goto done;
	}

	sprintf(buf, "%s!%s", config_servername, newpartition);

	/* reserve the mailbox in MUPDATE */
	r = mupdate_reserve(mupdate_h, name, buf);
	if(r) {
	syslog(LOG_ERR,
	"MUPDATE: can't reserve mailbox entry for '%s'", name);
	goto done;
	}
	}

	madereserved = 1; /* so we can roll back on failure */

	/* 5. add the new entry */
	mboxent = mboxlist_makeentry(mbtype, newpartition, acl);

	r = DB->store(mbdb, name, strlen(name), mboxent, strlen(mboxent),
	&tid);
	switch (r) {
	case CYRUSDB_OK:
	break;
	case CYRUSDB_AGAIN:
	goto retry;
	break;
	default:
	syslog(LOG_ERR, "DBERROR: error updating database %s: %s",
	name, cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	goto done;
	}

	done: /* ALL DATABASE OPERATIONS DONE; NEED TO DO FILESYSTEM OPERATIONS */
	if (!r && !(mbtype & MBTYPE_REMOTE)) {
	char mbbuf[MAX_MAILBOX_PATH];

	/* Create new mailbox and move new mailbox list file into place */
	mailbox_hash_mbox(mbbuf, root, name);
	r = mailbox_create(name, mbbuf, acl, NULL,
	((mbtype & MBTYPE_NETNEWS) ?
	MAILBOX_FORMAT_NETNEWS :
	MAILBOX_FORMAT_NORMAL),
	&newmailbox);
	if (!r) {
	mailbox_close(&newmailbox);
	}
	}

	if (r) { /* CREATE failed */
	int r2;

	r2 = 0;
	if (tid) r2 = DB->abort(mbdb, tid);
	switch (r2) {
	case 0:
	break;
	default:
	syslog(LOG_ERR, "DBERROR: failed on abort: %s",
	cyrusdb_strerror(r2));
	}

	/* delete mupdate entry if we made it */
	if (madereserved == 1 && mupdate_server) {
	r2 = mupdate_delete(mupdate_h, name);
	if(r2 > 0) {
	/* Disconnect, reconnect, and retry */
	syslog(LOG_WARNING,
	"MUPDATE: lost connection, retrying");
	mupdate_disconnect(&mupdate_h);
	r2 = mupdate_connect(mupdate_server, NULL,
	&mupdate_h, NULL);
	if(!r2) {
	r2 = mupdate_delete(mupdate_h, name);
	}
	}
	if(r2) {
	syslog(LOG_ERR,
	"MUPDATE: can't unreserve mailbox entry '%s'",
	name);
	}
	}
	} else { /* all is well */
	switch (r = DB->commit(mbdb, tid)) {
	case 0:
	break;
	default:
	syslog(LOG_ERR, "DBERROR: failed on commit: %s",
	cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	}
	}

	/* 9. set MUPDATE entry as commited (CRASH: commited) */
	/* xxx maybe we should roll back if this fails? */
	if (!r && mupdate_server) {
	/* commit the mailbox in MUPDATE */
	sprintf(buf, "%s!%s", config_servername, newpartition);

	r = mupdate_activate(mupdate_h, name, buf, acl);
	if(r > 0) {
	/* Disconnect, reconnect, and retry */
	syslog(LOG_WARNING,
	"MUPDATE: lost connection, retrying");
	mupdate_disconnect(&mupdate_h);
	r = mupdate_connect(mupdate_server, NULL, &mupdate_h, NULL);
	if(!r) {
	r = mupdate_activate(mupdate_h, name, buf, acl);
	}
	}
	if(r) {
	syslog(LOG_ERR,
	"MUPDATE: can't commit mailbox entry for '%s'", name);
	}
	}

	if(mupdate_server) mupdate_disconnect(&mupdate_h);

	if (acl) free(acl);
	if (newpartition) free(newpartition);
	if (mboxent) free(mboxent);

	return r;
	}

	/* insert an entry for the proxy */
	int mboxlist_insertremote(const char name, int mbtype, const char host,
	const char acl, void *rettid)
	{
	char *mboxent;
	int r = 0;

	assert(name != NULL && host != NULL);

	mboxent = mboxlist_makeentry(mbtype \| MBTYPE_REMOTE, host, acl);

	/* database put */
	r = DB->store(mbdb, name, strlen(name), mboxent, strlen(mboxent), NULL);
	switch (r) {
	case CYRUSDB_OK:
	break;
	case CYRUSDB_AGAIN:
	abort(); /* shouldn't happen ! */
	break;
	default:
	syslog(LOG_ERR, "DBERROR: error updating database %s: %s",
	name, cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	break;
	}

	free(mboxent);

	return r;
	}

	/*
	* Delete a mailbox.
	* Deleting the mailbox user.FOO deletes the user "FOO". It may only be
	* performed by an admin. The operation removes the user "FOO"'s
	* subscriptions and all sub-mailboxes of user.FOO
	*
	* 1. Begin transaction
	* 2. Verify ACL's
	* 3. remove from database
	* 4. remove from disk
	* 5. commit transaction
	* 6. Open mupdate connection if necessary
	* 7. delete from mupdate
	*
	*/
	int mboxlist_deletemailbox(const char name, int isadmin, char userid,
	struct auth_state *auth_state, int checkacl)
	{
	int r;
	char *acl;
	long access;
	int deleteuser = 0; /* if we are deleting user.<user> */
	struct mailbox mailbox;
	int deletequotaroot = 0;
	char *path;
	struct txn *tid = NULL;
	int isremote = 0;
	int mbtype;
	int deleteright = get_deleteright();
	mupdate_handle *mupdate_h;

	retry:
	/* Check for request to delete a user:
	user.<x> with no dots after it */
	if (!strncmp(name, "user.", 5) && !strchr(name+5, '.')) {
	/* Can't DELETE INBOX (your own inbox) */
	if (!strcmp(name+5, userid)) {
	r = IMAP_MAILBOX_NOTSUPPORTED;
	goto done;
	}

	/* Only admins may delete user */
	if (!isadmin) { r = IMAP_PERMISSION_DENIED; goto done; }

	r = mboxlist_mylookup(name, NULL, NULL, NULL, &acl, &tid, 1);
	switch (r) {
	case 0:
	break;
	case IMAP_AGAIN:
	goto retry;
	break;
	default:
	DB->abort(mbdb, tid);
	goto done;
	break;
	}

	/* Check ACL before doing anything stupid
	* We don't have to lie about the error code since we know
	* the user is an admin.
	*/
	if (checkacl &&
	(!(cyrus_acl_myrights(auth_state, acl) & deleteright))) {
	r = IMAP_PERMISSION_DENIED;
	DB->abort(mbdb, tid);
	goto done;
	}

	deleteuser = 1;
	}

	r = mboxlist_mylookup(name, &mbtype, &path, NULL, &acl, &tid, 1);
	switch (r) {
	case 0:
	break;

	case IMAP_AGAIN:
	goto retry;
	break;

	default:
	DB->abort(mbdb, tid);
	goto done;
	}

	isremote = mbtype & MBTYPE_REMOTE;

	/* check if user has Delete right */
	if(checkacl) {
	access = cyrus_acl_myrights(auth_state, acl);
	if(!(access & deleteright)) {
	/* User has admin rights over their own mailbox namespace */
	if (mboxname_userownsmailbox(userid, name)) {
	isadmin = 1;
	}

	/* Lie about error if privacy demands */
	r = (isadmin \|\| (access & ACL_LOOKUP)) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	DB->abort(mbdb, tid);
	goto done;
	}
	}

	/* delete entry */
	r = DB->delete(mbdb, name, strlen(name), &tid);
	switch (r) {
	case CYRUSDB_OK: /* success */
	break;
	case CYRUSDB_AGAIN:
	goto retry;
	default:
	syslog(LOG_ERR, "DBERROR: error deleting %s: %s",
	name, cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	goto done;
	}

	/* commit db operations */
	if (!r) {
	r = DB->commit(mbdb, tid);
	if (r) {
	syslog(LOG_ERR, "DBERROR: failed on commit: %s",
	cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	}
	}

	if (r \|\| isremote) goto done;

	if (!r) r = mailbox_open_header_path(name, path, acl, 0, &mailbox, 0);
	if (!r) r = mailbox_delete(&mailbox, deletequotaroot);

	if (!r && mupdate_server) {
	/* delete the mailbox in MUPDATE */
	r = mupdate_connect(mupdate_server, NULL, &mupdate_h, NULL);
	if(r) {
	syslog(LOG_ERR,
	"can not connect to mupdate server for delete of '%s'",
	name);
	goto done;
	}
	r = mupdate_delete(mupdate_h, name);
	if(r) {
	syslog(LOG_ERR,
	"MUPDATE: can't delete mailbox entry '%s'", name);
	}
	mupdate_disconnect(&mupdate_h);
	}

	/*
	* See if we have to remove mailbox's quota root
	*/
	if (!r && mailbox.quota.root != NULL) {
	/* xxx look for any other mailboxes in this quotaroot */
	}

	done:
	return r;
	}

	/*
	* Rename/move a single mailbox (recursive renames are handled at a
	* higher level)
	*
	* 1. start transaction
	* 2. verify acls
	* 3. open mupdate connection if needed
	* 4. Delete entry from berkeley db
	* 5. mupdate make the new entry
	* 7. delete from disk
	* 8. commit transaction
	* 9. set new mupdate entry commited
	* 10. delete old mupdate entry
	* */

	/* note: partition moving should really be moved to another function */
	int mboxlist_renamemailbox(char oldname, char newname, char *partition,
	int isadmin, char *userid,
	struct auth_state *auth_state)
	{
	int r;
	long access;
	int isusermbox = 0;
	int partitionmove = 0;
	int mbtype;
	char *oldpath = NULL;
	char newpath[MAX_MAILBOX_PATH];
	struct mailbox newmailbox;
	char oldacl = NULL, newacl = NULL;
	const char *root = NULL;
	struct txn *tid = NULL;
	char *newpartition = NULL;
	char *mboxent = NULL;
	int deleteright = get_deleteright();

	mupdate_handle *mupdate_h;
	int madenew = 0;

	retry:
	/* lookup the mailbox to make sure it exists and get its acl */
	r = mboxlist_mylookup(oldname, &mbtype, &oldpath, NULL, &oldacl, &tid, 1);
	switch (r) {
	case 0:
	break;
	case IMAP_AGAIN:
	goto retry;
	default:
	goto done;
	}

	/* make a copy of the old ACL so it doesn't get overwritten
	by another call to mboxlist_mylookup() */
	newacl = xstrdup(oldacl);

	/* Check ability to delete old mailbox */
	if (!strcmp(oldname, newname) && !(mbtype & MBTYPE_REMOTE)) {
	/* Attempt to move mailbox across partition */
	if (!isadmin \|\| !partition) {
	r = IMAP_MAILBOX_EXISTS;
	goto done;
	}

	partitionmove = 1;
	root = config_partitiondir(partition);
	if (!root) {
	r = IMAP_PARTITION_UNKNOWN;
	goto done;
	}
	if (!strncmp(root, oldpath, strlen(root)) &&
	oldpath[strlen(root)] == '/') {
	/* partitions are the same or share common prefix */
	r = IMAP_MAILBOX_EXISTS;
	goto done;
	}
	} else if (!strncmp(oldname, "user.", 5) && !strchr(oldname+5, '.')) {
	if (!strcmp(oldname+5, userid)) {
	/* Special case of renaming inbox */
	access = cyrus_acl_myrights(auth_state, oldacl);
	if (!(access & deleteright)) {
	r = IMAP_PERMISSION_DENIED;
	goto done;
	}
	isusermbox = 1;
	} else {
	/* Even admins can't rename users */
	r = IMAP_MAILBOX_NOTSUPPORTED;
	goto done;
	}
	} else {
	access = cyrus_acl_myrights(auth_state, oldacl);
	if (!(access & deleteright) && !isadmin) {
	r = (isadmin \|\| (access & ACL_LOOKUP)) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	goto done;
	}
	}

	/* Check ability to create new mailbox */
	if (!partitionmove) {
	if (!strncmp(newname, "user.", 5) && !strchr(newname+5, '.')) {
	/* Even admins can't rename to user's inboxes */
	r = IMAP_MAILBOX_NOTSUPPORTED;
	goto done;
	}
	r = mboxlist_mycreatemailboxcheck(newname, 0, partition, isadmin,
	userid, auth_state, NULL,
	&newpartition, 1, &tid);
	switch (r) {
	case 0:
	break;
	case IMAP_AGAIN:
	goto retry;
	break;
	default: /* not allowed to create the new mailbox */
	goto done;
	break;
	}
	} else {
	newpartition = xstrdup(partition);
	}

	if (!(mbtype & MBTYPE_REMOTE)) {
	/* Get partition's path */
	root = config_partitiondir(newpartition);
	if (!root) {
	r = IMAP_PARTITION_UNKNOWN;
	goto done;
	}
	}

	if (!r && !partitionmove && mupdate_server) {
	/* Reserve new name in MUPDATE */
	char buf[MAX_PARTITION_LEN + HOSTNAME_SIZE + 2];
	sprintf(buf, "%s!%s", config_servername, newpartition);

	r = mupdate_connect(mupdate_server, NULL, &mupdate_h, NULL);
	if(r) {
	syslog(LOG_ERR,
	"can not connect to mupdate server for reservation on '%s'",
	newname);
	goto done;
	}

	/* reserve the mailbox in MUPDATE */
	r = mupdate_reserve(mupdate_h, newname, buf);
	if(r) {
	syslog(LOG_ERR,
	"MUPDATE: can't reserve mailbox entry for '%s'",
	newname);
	goto done;
	}

	madenew = 1;
	} else if(!r && partitionmove && mupdate_server) {
	/* commit the update to MUPDATE */
	char buf[MAX_PARTITION_LEN + HOSTNAME_SIZE + 2];
	sprintf(buf, "%s!%s", config_servername, newpartition);

	r = mupdate_connect(mupdate_server, NULL, &mupdate_h, NULL);
	if(r) {
	syslog(LOG_ERR,
	"can not connect to mupdate server for reservation on '%s'",
	newname);
	goto done;
	}

	r = mupdate_activate(mupdate_h, oldname, buf, newacl);
	if(r) {
	syslog(LOG_ERR,
	"MUPDATE: can't update mailbox entry for '%s'",
	oldname);
	goto done;
	}
	}

	if (!isusermbox) {
	/* 4. Delete entry from berkeley db */
	r = DB->delete(mbdb, oldname, strlen(oldname), &tid);
	switch (r) {
	case 0: /* success */
	break;
	case CYRUSDB_AGAIN:
	goto retry;
	break;
	default:
	syslog(LOG_ERR, "DBERROR: error deleting %s: %s",
	oldname, cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	goto done;
	break;
	}
	}

	/* create new entry */
	mboxent = mboxlist_makeentry(mbtype, newpartition, newacl);

	/* put it into the db */
	r = DB->store(mbdb, newname, strlen(newname),
	mboxent, strlen(mboxent), &tid);
	switch (r) {
	case 0:
	break;
	case CYRUSDB_AGAIN:
	goto retry;
	default:
	syslog(LOG_ERR, "DBERROR: error renaming %s: %s",
	newname, cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	goto done;
	}

	done: /* ALL DATABASE OPERATIONS DONE; NEED TO DO FILESYSTEM OPERATIONS */
	if (!r && !(mbtype & MBTYPE_REMOTE)) {
	/* Rename the actual mailbox */
	assert(root != NULL); /* from above */
	mailbox_hash_mbox(newpath, root, newname);

	r = mailbox_rename(oldname, oldpath, newacl, newname,
	newpath, isusermbox, NULL, NULL, &newmailbox);
	if (!r) {
	mailbox_close(&newmailbox);
	}
	}

	if (r != 0) {
	int r2 = 0;

	if (tid) r2 = DB->abort(mbdb, tid);
	if (r2) {
	syslog(LOG_ERR, "DBERROR: can't abort: %s", cyrusdb_strerror(r2));
	}

	/* unroll mupdate operations if necessary */
	if (madenew && mupdate_server) {
	r2 = mupdate_delete(mupdate_h, newname);
	if(r2 > 0) {
	/* Disconnect, reconnect, and retry */
	syslog(LOG_WARNING,
	"MUPDATE: lost connection, retrying");
	mupdate_disconnect(&mupdate_h);
	r2 = mupdate_connect(mupdate_server, NULL,
	&mupdate_h, NULL);
	if(!r2) {
	r2 = mupdate_delete(mupdate_h, newname);
	}
	}
	if(r2) {
	syslog(LOG_ERR,
	"MUPDATE: can't unreserve mailbox entry '%s'",
	newname);
	}
	}
	} else {
	/* commit now */
	switch (r = DB->commit(mbdb, tid)) {
	case 0:
	break;

	default:
	syslog(LOG_ERR, "DBERROR: failed on commit: %s",
	cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	break;
	}
	}

	if (!r && !partitionmove && mupdate_server) {
	/* commit the mailbox in MUPDATE */
	char buf[MAX_PARTITION_LEN + HOSTNAME_SIZE + 2];
	sprintf(buf, "%s!%s", config_servername, newpartition);

	r = mupdate_activate(mupdate_h, newname, buf, newacl);
	if(r > 0) {
	/* Disconnect, reconnect, and retry */
	syslog(LOG_WARNING,
	"MUPDATE: lost connection, retrying");
	mupdate_disconnect(&mupdate_h);
	r = mupdate_connect(mupdate_server, NULL, &mupdate_h, NULL);
	if(!r) {
	r = mupdate_activate(mupdate_h, newname, buf, newacl);
	}
	}
	if(r) {
	syslog(LOG_ERR,
	"MUPDATE: can't commit mailbox entry for '%s'",
	newname);
	}
	}

	if (!r && !partitionmove && mupdate_server) {
	/* delete the old mailbox in MUPDATE */
	r = mupdate_delete(mupdate_h, oldname);
	if(r > 0) {
	/* Disconnect, reconnect, and retry */
	syslog(LOG_WARNING,
	"MUPDATE: lost connection, retrying");
	mupdate_disconnect(&mupdate_h);
	r = mupdate_connect(mupdate_server, NULL, &mupdate_h, NULL);
	if(!r) {
	r = mupdate_delete(mupdate_h, oldname);
	}
	}
	if(r) {
	syslog(LOG_ERR,
	"MUPDATE: can't delete mailbox entry '%s'", oldname);
	}
	}

	if(mupdate_server) mupdate_disconnect(&mupdate_h);

	/* free memory */
	if (newacl) free(newacl); /* we're done with the new ACL */
	if (newpartition) free(newpartition);
	if (mboxent) free(mboxent);

	return r;
	}

	/*
	* Change the ACL for mailbox 'name' so that 'identifier' has the
	* rights enumerated in the string 'rights'. If 'rights' is the null
	* pointer, removes the ACL entry for 'identifier'. 'isadmin' is
	* nonzero if user is a mailbox admin. 'userid' is the user's login id.
	*
	*
	* 1. Start transaction
	* 2. Check rights
	* 3. Open mupdate connection if necessary
	* 4. Set db entry
	* 5. Change on disk
	* 6. Commit transaction
	* 7. Change mupdate entry
	*
	*/
	int mboxlist_setacl(char name, char identifier, char *rights,
	int isadmin, char *userid,
	struct auth_state *auth_state)
	{
	int useridlen = strlen(userid);
	int r;
	int access;
	int mode = ACL_MODE_SET;
	int isusermbox = 0;
	struct mailbox mailbox;
	char acl, newacl = NULL;
	char partition, path;
	char *mboxent = NULL;
	int mbtype;
	struct txn *tid = NULL;

	if (!strncmp(name, "user.", 5) &&
	!strchr(userid, '.') &&
	!strncmp(name+5, userid, useridlen) &&
	(name[5+useridlen] == '\0' \|\| name[5+useridlen] == '.')) {
	isusermbox = 1;
	}

	retry:
	/* lookup the mailbox to make sure it exists and get its acl */
	r = mboxlist_mylookup(name, &mbtype, &path, &partition, &acl, &tid, 1);
	switch (r) {
	case 0:
	break;
	case IMAP_AGAIN:
	goto retry;
	default:
	goto done;
	}

	if (!r && !isadmin && !isusermbox) {
	access = cyrus_acl_myrights(auth_state, acl);
	if (!(access & ACL_ADMIN)) {
	r = (access & ACL_LOOKUP) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	goto done;
	}
	}

	/* Make change to ACL */
	newacl = xstrdup(acl);
	if (rights) {
	mode = ACL_MODE_SET;
	if (*rights == '+') {
	rights++;
	mode = ACL_MODE_ADD;
	}
	else if (*rights == '-') {
	rights++;
	mode = ACL_MODE_REMOVE;
	}

	if (cyrus_acl_set(&newacl, identifier, mode, cyrus_acl_strtomask(rights),
	isusermbox ? mboxlist_ensureOwnerRights : 0,
	(void *)userid))
	{
	r = IMAP_INVALID_IDENTIFIER;
	goto done;
	}
	}
	else {
	if (cyrus_acl_remove(&newacl, identifier,
	isusermbox ? mboxlist_ensureOwnerRights : 0,
	(void *)userid)) {
	r = IMAP_INVALID_IDENTIFIER;
	goto done;
	}
	}

	/* ok, make the change */
	mboxent = mboxlist_makeentry(mbtype, partition, newacl);

	r = DB->store(mbdb, name, strlen(name), mboxent, strlen(mboxent), &tid);
	switch (r) {
	case 0:
	break;
	case CYRUSDB_AGAIN:
	goto retry;
	default:
	syslog(LOG_ERR, "DBERROR: error updating acl %s: %s",
	name, cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	goto done;
	}

	if (!(mbtype & MBTYPE_REMOTE)) {
	/* open & lock mailbox header */
	r = mailbox_open_header_path(name, path, acl, NULL, &mailbox, 0);

	if (!r) {
	r = mailbox_lock_header(&mailbox);
	if (!r) {
	/* set it in the /var/spool part */
	(void) mailbox_write_header(&mailbox);
	}
	mailbox_close(&mailbox);
	}
	}

	done:
	if (mboxent) free(mboxent);

	if (r) {
	int r2;

	if ((r2 = DB->abort(mbdb, tid)) != 0) {
	syslog(LOG_ERR, "DBERROR: error aborting txn: %s",
	cyrusdb_strerror(r2));
	r2 = IMAP_IOERROR;
	}
	} else {
	/* commit now */
	switch (r = DB->commit(mbdb, tid)) {
	case 0:
	break;
	default:
	syslog(LOG_ERR, "DBERROR: failed on commit: %s",
	cyrusdb_strerror(r));
	r = IMAP_IOERROR;
	}
	}

	/* 7. Change mupdate entry */
	if (!r && mupdate_server) {
	mupdate_handle *mupdate_h;
	/* commit the update to MUPDATE */
	char buf[MAX_PARTITION_LEN + HOSTNAME_SIZE + 2];
	sprintf(buf, "%s!%s", config_servername, partition);

	r = mupdate_connect(mupdate_server, NULL, &mupdate_h, NULL);
	if(r) {
	syslog(LOG_ERR,
	"can not connect to mupdate server for reservation on '%s'",
	name);
	} else {
	r = mupdate_activate(mupdate_h, name, buf, newacl);
	if(r) {
	syslog(LOG_ERR,
	"MUPDATE: can't update mailbox entry for '%s'",
	name);
	}
	}
	mupdate_disconnect(&mupdate_h);
	}
	if (newacl) free(newacl);

	return r;
	}

	struct find_rock {
	struct glob *g;
	struct namespace *namespace;
	int find_namespace;
	int inboxoffset;
	const char *inboxcase;
	const char *usermboxname;
	int usermboxnamelen;
	int checkmboxlist;
	int checkshared;
	int isadmin;
	struct auth_state *auth_state;
	int (proc)(char , int, int, void *rock);
	void *procrock;
	};

	/* return non-zero if we like this one */
	static int find_p(void *rockp,
	const char *key, int keylen,
	const char *data, int datalen)
	{
	struct find_rock rock = (struct find_rock ) rockp;
	long minmatch;
	struct glob *g = rock->g;
	long matchlen;

	minmatch = 0;
	if (rock->inboxoffset) {
	char namebuf[MAX_MAILBOX_NAME+1];

	memcpy(namebuf, key, keylen);
	namebuf[keylen] = '\0';

	if (rock->inboxoffset) {
	namebuf[rock->inboxoffset] = rock->inboxcase[0];
	namebuf[rock->inboxoffset+1] = rock->inboxcase[1];
	namebuf[rock->inboxoffset+2] = rock->inboxcase[2];
	namebuf[rock->inboxoffset+3] = rock->inboxcase[3];
	namebuf[rock->inboxoffset+4] = rock->inboxcase[4];
	}

	matchlen = glob_test(g, namebuf+rock->inboxoffset,
	keylen-rock->inboxoffset, &minmatch);
	} else {
	matchlen = glob_test(g, key, keylen, &minmatch);
	}
	if (matchlen == -1) return 0;

	if (rock->find_namespace != NAMESPACE_INBOX &&
	rock->usermboxname &&
	keylen >= rock->usermboxnamelen &&
	(keylen == rock->usermboxnamelen \|\|
	key[rock->usermboxnamelen] == '.') &&
	!strncmp(key, rock->usermboxname, rock->usermboxnamelen)) {
	/* this would've been output with the inbox stuff, so skip it */
	return 0;
	}

	if (rock->find_namespace == NAMESPACE_SHARED &&
	rock->namespace && rock->namespace->isalt &&
	!strncmp(key, "user", 4) &&
	(key[4] == '\0' \|\| key[4] == '.')) {
	/* this would've been output with the user stuff, so skip it */
	return 0;
	}

	/* check acl */
	if (!rock->isadmin) {
	/* check the acls */
	const char p, acl;
	int rights;
	int acllen;
	static char *aclbuf = NULL;
	static int aclbufsz = 0;

	p = strchr(data, ' ');
	if (!p) {
	syslog(LOG_ERR, "%s: can't find partition", key);
	return 0;
	}
	p++;
	acl = strchr(p, ' ');
	if (!acl) {
	syslog(LOG_ERR, "%s: can't find acl", key);
	return 0;
	}
	acl++;
	acllen = datalen - (acl - data);
	if (acllen >= aclbufsz) {
	aclbufsz = acllen + 500;
	aclbuf = xrealloc(aclbuf, aclbufsz);
	}
	memcpy(aclbuf, acl, acllen);
	aclbuf[acllen] = '\0';

	rights = cyrus_acl_myrights(rock->auth_state, aclbuf);
	if (!(rights & ACL_LOOKUP)) {
	return 0;
	}
	}

	/* if we get here, close enough for us to spend the time
	acting interested */
	return 1;
	}

	static int find_cb(void *rockp,
	const char *key, int keylen,
	const char *data, int datalen)
	{
	char namebuf[MAX_MAILBOX_NAME+1];
	struct find_rock rock = (struct find_rock ) rockp;
	int r = 0;
	long minmatch;
	struct glob *g = rock->g;

	/* foreach match, do this test */
	minmatch = 0;
	while (minmatch >= 0) {
	long matchlen;

	memcpy(namebuf, key, keylen);
	namebuf[keylen] = '\0';

	if (rock->find_namespace != NAMESPACE_INBOX &&
	rock->usermboxname &&
	!strncmp(namebuf, rock->usermboxname, rock->usermboxnamelen)
	&& (keylen == rock->usermboxnamelen \|\|
	namebuf[rock->usermboxnamelen] == '.')) {
	/* this would've been output with the inbox stuff, so skip it */
	return 0;
	}

	/* make sure it's in the mailboxes db */
	if (rock->checkmboxlist) {
	r = mboxlist_lookup(namebuf, NULL, NULL, NULL);
	} else {
	r = 0; /* don't bother checking */
	}

	if (!r && rock->inboxoffset) {
	namebuf[rock->inboxoffset] = rock->inboxcase[0];
	namebuf[rock->inboxoffset+1] = rock->inboxcase[1];
	namebuf[rock->inboxoffset+2] = rock->inboxcase[2];
	namebuf[rock->inboxoffset+3] = rock->inboxcase[3];
	namebuf[rock->inboxoffset+4] = rock->inboxcase[4];
	}

	matchlen = glob_test(g, namebuf+rock->inboxoffset,
	keylen-rock->inboxoffset, &minmatch);

	if (matchlen == -1) {
	r = 0;
	break;
	}

	switch (r) {
	case 0:
	/* found the entry; output it */
	if (rock->find_namespace == NAMESPACE_SHARED &&
	rock->checkshared && rock->namespace) {
	/* special case: LIST "" % -- output prefix only */
	r = (*rock->proc)(rock->namespace->prefix[NAMESPACE_SHARED],
	strlen(rock->namespace->prefix[NAMESPACE_SHARED])-1,
	1, rock->procrock);
	/* short-circuit the foreach - one mailbox is sufficient */
	r = CYRUSDB_DONE;
	}
	else {
	r = (*rock->proc)(namebuf+rock->inboxoffset, matchlen,
	1, rock->procrock);
	}
	break;

	case IMAP_MAILBOX_NONEXISTENT:
	/* didn't find the entry */
	r = 0;
	break;

	default:
	break;
	}

	if (r) break;
	}

	return r;
	}

	/*
	* Find all mailboxes that match 'pattern'.
	* 'isadmin' is nonzero if user is a mailbox admin. 'userid'
	* is the user's login id. For each matching mailbox, calls
	* 'proc' with the name of the mailbox. If 'proc' ever returns
	* a nonzero value, mboxlist_findall immediately stops searching
	* and returns that value. 'rock' is passed along as an argument to proc in
	* case it wants some persistant storage or extra data.
	*/
	/* Find all mailboxes that match 'pattern'. */
	int mboxlist_findall(struct namespace *namespace __attribute__((unused)),
	char pattern, int isadmin, char userid,
	struct auth_state auth_state, int (proc)(), void *rock)
	{
	struct find_rock cbrock;
	char usermboxname[MAX_MAILBOX_NAME+1];
	int usermboxnamelen = 0;
	const char *data;
	int datalen;
	int r = 0;
	char *p;
	int prefixlen;

	cbrock.g = glob_init(pattern, GLOB_HIERARCHY\|GLOB_INBOXCASE);
	cbrock.namespace = NULL;
	cbrock.inboxcase = glob_inboxcase(cbrock.g);
	cbrock.isadmin = isadmin;
	cbrock.auth_state = auth_state;
	cbrock.checkmboxlist = 0; /* don't duplicate work */
	cbrock.checkshared = 0;
	cbrock.proc = proc;
	cbrock.procrock = rock;

	/* Build usermboxname */
	if (userid && !strchr(userid, '.') &&
	strlen(userid)+5 < MAX_MAILBOX_NAME) {
	strcpy(usermboxname, "user.");
	strcat(usermboxname, userid);
	usermboxnamelen = strlen(usermboxname);
	}
	else {
	userid = NULL;
	}

	/* Check for INBOX first of all */
	if (userid) {
	if (GLOB_TEST(cbrock.g, "INBOX") != -1) {
	r = DB->fetch(mbdb, usermboxname, usermboxnamelen,
	&data, &datalen, NULL);
	if (!r && data) {
	r = (*proc)(cbrock.inboxcase, 5, 1, rock);
	}
	}
	else if (!strncmp(pattern, usermboxname, usermboxnamelen) &&
	GLOB_TEST(cbrock.g, usermboxname) != -1) {
	r = DB->fetch(mbdb, usermboxname, usermboxnamelen,
	&data, &datalen, NULL);
	if (!r && data) {
	r = (*proc)(usermboxname, usermboxnamelen, 1, rock);
	}
	}
	strcpy(usermboxname+usermboxnamelen, ".");
	usermboxnamelen++;

	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	} else {
	cbrock.usermboxname = NULL;
	cbrock.usermboxnamelen = 0;
	}

	if (r) goto done;

	/* Find fixed-string pattern prefix */
	for (p = pattern; *p; p++) {
	if (p == '' \|\| p == '%' \|\| p == '?') break;
	}
	prefixlen = p - pattern;
	*p = '\0';

	/*
	* If user.X.* or INBOX.* can match pattern,
	* search for those mailboxes next
	*/
	if (userid &&
	(!strncmp(usermboxname, pattern, usermboxnamelen-1) \|\|
	!strncasecmp("inbox.", pattern, prefixlen < 6 ? prefixlen : 6))) {

	if (!strncmp(usermboxname, pattern, usermboxnamelen-1)) {
	cbrock.inboxoffset = 0;
	}
	else {
	cbrock.inboxoffset = strlen(userid);
	}

	cbrock.find_namespace = NAMESPACE_INBOX;
	/* iterate through prefixes matching usermboxname */
	r = DB->foreach(mbdb,
	usermboxname, usermboxnamelen,
	&find_p, &find_cb, &cbrock,
	NULL);
	}

	if(!r) {
	cbrock.find_namespace = NAMESPACE_USER;
	cbrock.inboxoffset = 0;
	if (usermboxnamelen) {
	usermboxname[--usermboxnamelen] = '\0';
	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	}
	/* search for all remaining mailboxes.
	just bother looking at the ones that have the same pattern
	prefix. */
	r = DB->foreach(mbdb,
	pattern, prefixlen,
	&find_p, &find_cb, &cbrock,
	NULL);
	}


	done:
	glob_free(&cbrock.g);

	return r;
	}

	int mboxlist_findall_alt(struct namespace *namespace,
	char pattern, int isadmin, char userid,
	struct auth_state auth_state, int (proc)(),
	void *rock)
	{
	struct find_rock cbrock;
	char usermboxname[MAX_MAILBOX_NAME+1], patbuf[MAX_MAILBOX_NAME+1];
	int usermboxnamelen = 0;
	const char *data;
	int datalen;
	int r = 0;
	char *p;
	int prefixlen, len;

	cbrock.g = glob_init(pattern, GLOB_HIERARCHY\|GLOB_INBOXCASE);
	cbrock.namespace = namespace;
	cbrock.inboxcase = glob_inboxcase(cbrock.g);
	cbrock.isadmin = isadmin;
	cbrock.auth_state = auth_state;
	cbrock.checkmboxlist = 0; /* don't duplicate work */
	cbrock.checkshared = 0;
	cbrock.proc = proc;
	cbrock.procrock = rock;

	/* Build usermboxname */
	if (userid && !strchr(userid, '.') &&
	strlen(userid)+5 < MAX_MAILBOX_NAME) {
	strcpy(usermboxname, "user.");
	strcat(usermboxname, userid);
	usermboxnamelen = strlen(usermboxname);
	}
	else {
	userid = 0;
	}

	/* Check for INBOX first of all */
	if (userid) {
	if (GLOB_TEST(cbrock.g, "INBOX") != -1) {
	r = DB->fetch(mbdb, usermboxname, usermboxnamelen,
	&data, &datalen, NULL);
	if (!r && data) {
	r = (*proc)(cbrock.inboxcase, 5, 0, rock);
	}
	}
	strcpy(usermboxname+usermboxnamelen, ".");
	usermboxnamelen++;

	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	} else {
	cbrock.usermboxname = NULL;
	cbrock.usermboxnamelen = 0;
	}

	if (r) goto done;

	glob_free(&cbrock.g);

	/* Find fixed-string pattern prefix */
	for (p = pattern; *p; p++) {
	if (p == '' \|\| p == '%' \|\| p == '?') break;
	}
	prefixlen = p - pattern;

	/*
	* Personal (INBOX) namespace
	*
	* Append pattern to "INBOX.", search for those mailboxes next
	*/
	if (userid) {
	strcpy(patbuf, "INBOX.");
	strcat(patbuf, pattern);
	cbrock.g = glob_init(patbuf, GLOB_HIERARCHY\|GLOB_INBOXCASE);
	cbrock.inboxcase = glob_inboxcase(cbrock.g);
	cbrock.inboxoffset = strlen(userid);
	cbrock.find_namespace = NAMESPACE_INBOX;

	/* iterate through prefixes matching usermboxname */
	DB->foreach(mbdb,
	usermboxname, usermboxnamelen,
	&find_p, &find_cb, &cbrock,
	NULL);

	glob_free(&cbrock.g);
	}

	if (usermboxnamelen) {
	usermboxname[--usermboxnamelen] = '\0';
	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	}

	/*
	* Other Users namespace
	*
	* If "Other Users*" can match pattern, search for those mailboxes next
	*/
	len = strlen(namespace->prefix[NAMESPACE_USER])-1;
	if (!strncmp(namespace->prefix[NAMESPACE_USER], pattern,
	prefixlen < len ? prefixlen : len)) {

	if (prefixlen < len)
	cbrock.g = glob_init(pattern+prefixlen, GLOB_HIERARCHY);
	else {
	strcpy(patbuf, "user");
	strcat(patbuf, pattern+len);
	cbrock.g = glob_init(patbuf, GLOB_HIERARCHY);
	}
	cbrock.find_namespace = NAMESPACE_USER;
	cbrock.inboxoffset = 0;

	/* iterate through prefixes matching usermboxname */
	DB->foreach(mbdb,
	"user", 4,
	&find_p, &find_cb, &cbrock,
	NULL);

	glob_free(&cbrock.g);
	}

	/*
	* Shared namespace
	*
	* search for all remaining mailboxes.
	* just bother looking at the ones that have the same pattern prefix.
	*/
	len = strlen(namespace->prefix[NAMESPACE_SHARED]);
	if (!strncmp(namespace->prefix[NAMESPACE_SHARED], pattern,
	prefixlen < len - 1 ? prefixlen : len - 1)) {

	cbrock.find_namespace = NAMESPACE_SHARED;
	cbrock.inboxoffset = 0;

	if (prefixlen < len) {
	/* Find pattern which matches shared namespace prefix */
	for (p = pattern+prefixlen; *p; p++) {
	if (*p == '%') continue;
	else if (*p == '.') p++;
	break;
	}

	if (!*p) {
	/* special case: LIST "" % -- see if we have a shared mbox */
	cbrock.g = glob_init("*", GLOB_HIERARCHY);
	cbrock.checkshared = 1;
	}
	else {
	cbrock.g = glob_init(p, GLOB_HIERARCHY);
	}

	DB->foreach(mbdb,
	"", 0,
	&find_p, &find_cb, &cbrock,
	NULL);
	}
	else if (pattern[len-1] == '.') {
	strcpy(patbuf, "");
	strcat(patbuf, pattern+len);
	cbrock.g = glob_init(patbuf, GLOB_HIERARCHY);

	pattern[prefixlen] = '\0';
	DB->foreach(mbdb,
	pattern+len, prefixlen-len,
	&find_p, &find_cb, &cbrock,
	NULL);
	}
	}

	done:
	glob_free(&cbrock.g);

	return r;
	}

	/*
	* Set the quota on or create a quota root
	*/
	int mboxlist_setquota(const char *root, int newquota)
	{
	char quota_path[MAX_MAILBOX_PATH];
	char pattern[MAX_MAILBOX_PATH];
	struct quota quota;
	int r, t;

	if (!root[0] \|\| root[0] == '.' \|\| strchr(root, '/')
	\|\| strchr(root, '*') \|\| strchr(root, '%') \|\| strchr(root, '?')) {
	return IMAP_MAILBOX_BADNAME;
	}

	memset(&quota, 0, sizeof(struct quota));

	quota.root = (char *) root;
	mailbox_hash_quota(quota_path, root);

	if ((quota.fd = open(quota_path, O_RDWR, 0)) != -1) {
	/* Just lock and change it */
	r = mailbox_lock_quota(&quota);

	quota.limit = newquota;

	if (!r) r = mailbox_write_quota(&quota);
	if (quota.fd != -1) {
	close(quota.fd);
	}
	return r;
	}

	/*
	* Have to create a new quota root
	*/
	/* look for a top-level mailbox in the proposed quotaroot */
	r = mboxlist_detail(quota.root, &t, NULL, NULL, NULL, NULL);
	if (r) {
	return r;
	}
	/* Can't set quota on a remote mailbox */
	if (t & MBTYPE_REMOTE) {
	return IMAP_MAILBOX_NOTSUPPORTED;
	}

	/* perhaps create .NEW, lock, check if it got recreated, move in place */
	quota.lock_count = 1;
	quota.used = 0;
	quota.limit = newquota;
	r = mailbox_write_quota(&quota);

	if (r) {
	return r;
	}

	strcpy(pattern, quota.root);
	strcat(pattern, ".*");

	/* top level mailbox */
	mboxlist_changequota(quota.root, 0, 0, &quota);
	/* submailboxes - we're using internal names here */
	mboxlist_findall(NULL, pattern, 1, 0, 0, mboxlist_changequota, &quota);

	r = mailbox_write_quota(&quota);
	if (quota.fd != -1) {
	close(quota.fd);
	}

	return r;
	}

	/*
	* Retrieve internal information, for reconstructing mailboxes file
	*/
	void mboxlist_getinternalstuff(const char **listfnamep,
	const char **newlistfnamep,
	const char **basep,
	unsigned long * sizep)
	{
	printf("yikes! don't reconstruct me!\n");
	abort();
	}

	/*
	* ACL access canonicalization routine which ensures that 'owner'
	* retains lookup, administer, and create rights over a mailbox.
	*/
	int mboxlist_ensureOwnerRights(rock, identifier, access)
	void *rock;
	const char *identifier;
	int access;
	{
	char owner = (char )rock;
	if (strcmp(identifier, owner) != 0) return access;
	return access\|ACL_LOOKUP\|ACL_ADMIN\|ACL_CREATE;
	}

	/*
	* Helper function to change the quota root for 'name' to that pointed
	* to by the static global struct pointer 'mboxlist_newquota'.
	*/
	static int mboxlist_changequota(const char *name, int matchlen, int maycreate,
	void *rock)
	{
	int r;
	struct mailbox mailbox;
	struct quota mboxlist_newquota = (struct quota ) rock;

	assert(rock != NULL);

	r = mailbox_open_header(name, 0, &mailbox);
	if (r) goto error_noclose;

	r = mailbox_lock_header(&mailbox);
	if (r) goto error;

	r = mailbox_open_index(&mailbox);
	if (r) goto error;

	r = mailbox_lock_index(&mailbox);
	if (r) goto error;

	if (mailbox.quota.root) {
	if (strlen(mailbox.quota.root) >= strlen(mboxlist_newquota->root)) {
	/* Part of a child quota root */
	mailbox_close(&mailbox);
	return 0;
	}

	r = mailbox_lock_quota(&mailbox.quota);
	if (r) goto error;
	if (mailbox.quota.used >= mailbox.quota_mailbox_used) {
	mailbox.quota.used -= mailbox.quota_mailbox_used;
	}
	else {
	mailbox.quota.used = 0;
	}
	r = mailbox_write_quota(&mailbox.quota);
	if (r) {
	syslog(LOG_ERR,
	- "LOSTQUOTA: unable to record free of %u bytes in quota %s",
	+ "LOSTQUOTA: unable to record free of %lu bytes in quota %s",
	mailbox.quota_mailbox_used, mailbox.quota.root);
	}
	mailbox_unlock_quota(&mailbox.quota);
	free(mailbox.quota.root);
	}

	mailbox.quota.root = xstrdup(mboxlist_newquota->root);
	r = mailbox_write_header(&mailbox);
	if (r) goto error;

	mboxlist_newquota->used += mailbox.quota_mailbox_used;
	mailbox_close(&mailbox);
	return 0;

	error:
	mailbox_close(&mailbox);
	error_noclose:
	syslog(LOG_ERR, "LOSTQUOTA: unable to change quota root for %s to %s: %s",
	name, mboxlist_newquota->root, error_message(r));

	return 0;
	}

	void mboxlist_init(int myflags)
	{
	int r;
	char dbdir[1024];
	int flags = 0;

	/* create the name of the db file */
	strcpy(dbdir, config_dir);
	strcat(dbdir, FNAME_DBDIR);
	if (myflags & MBOXLIST_RECOVER) flags \|= CYRUSDB_RECOVER;
	r = DB->init(dbdir, flags);
	if (r != CYRUSDB_OK) {
	fatal("can't initialize mboxlist environment", EC_TEMPFAIL);
	}

	if (myflags & MBOXLIST_SYNC) {
	r = DB->sync();
	}

	mupdate_server = config_getstring("mupdate_server", NULL);
	if(mupdate_server) {
	/* We're going to need SASL */
	r = sasl_client_init(NULL);
	if(r != 0) {
	syslog(LOG_ERR, "could not initialize SASL library");
	fatal("could not initialize SASL library", EC_TEMPFAIL);
	}
	}
	}

	void mboxlist_open(char *fname)
	{
	int ret;
	char *tofree = NULL;

	/* create db file name */
	if (!fname) {
	fname = xmalloc(strlen(config_dir)+sizeof(FNAME_MBOXLIST));
	tofree = fname;
	strcpy(fname, config_dir);
	strcat(fname, FNAME_MBOXLIST);
	}

	ret = DB->open(fname, &mbdb);
	if (ret != 0) {
	syslog(LOG_ERR, "DBERROR: opening %s: %s", fname,
	cyrusdb_strerror(ret));
	/* Exiting TEMPFAIL because Sendmail thinks this
	EC_OSFILE == permanent failure. */
	fatal("can't read mailboxes file", EC_TEMPFAIL);
	}

	if (tofree) free(tofree);

	mboxlist_dbopen = 1;
	}

	void
	mboxlist_close(void)
	{
	int r;

	if (mboxlist_dbopen) {
	r = DB->close(mbdb);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error closing mailboxes: %s",
	cyrusdb_strerror(r));
	}
	mboxlist_dbopen = 0;
	}
	}

	void mboxlist_done(void)
	{
	int r;

	r = DB->done();
	if (r) {
	syslog(LOG_ERR, "DBERROR: error exiting application: %s",
	cyrusdb_strerror(r));
	}
	}

	/* hash the userid to a file containing the subscriptions for that user */
	char mboxlist_hash_usersubs(const char userid)
	{
	char *fname = xmalloc(strlen(config_dir) + sizeof(FNAME_USERDIR) +
	strlen(userid) + sizeof(FNAME_SUBSSUFFIX) + 10);
	char c;

	c = (char) dir_hash_c(userid);
	sprintf(fname, "%s%s%c/%s%s", config_dir, FNAME_USERDIR, c, userid,
	FNAME_SUBSSUFFIX);

	return fname;
	}

	/*
	* Open the subscription list for 'userid'.
	*
	* On success, returns zero.
	* On failure, returns an error code.
	*/
	static int
	mboxlist_opensubs(const char *userid,
	struct db **ret)
	{
	int r = 0;
	char *subsfname;

	/* Build subscription list filename */
	subsfname = mboxlist_hash_usersubs(userid);
	r = SUBDB->open(subsfname, ret);
	if (r != CYRUSDB_OK) {
	r = IMAP_IOERROR;
	}
	free(subsfname);

	return r;
	}

	/*
	* Close a subscription file
	*/
	static void mboxlist_closesubs(struct db *sub)
	{
	SUBDB->close(sub);
	}

	/*
	* Find subscribed mailboxes that match 'pattern'.
	* 'isadmin' is nonzero if user is a mailbox admin. 'userid'
	* is the user's login id. For each matching mailbox, calls
	* 'proc' with the name of the mailbox.
	*/
	int mboxlist_findsub(struct namespace *namespace __attribute__((unused)),
	char pattern, int isadmin, char userid,
	struct auth_state *auth_state,
	int (proc)(), void rock, int force)
	{
	struct db *subs = NULL;
	struct find_rock cbrock;
	char usermboxname[MAX_MAILBOX_NAME+1];
	int usermboxnamelen = 0;
	const char *data;
	int datalen;
	int r = 0;
	char *p;
	int prefixlen;

	cbrock.g = glob_init(pattern, GLOB_HIERARCHY\|GLOB_INBOXCASE);
	cbrock.namespace = NULL;
	cbrock.inboxcase = glob_inboxcase(cbrock.g);
	cbrock.isadmin = 1; /* user can always see their subs */
	cbrock.auth_state = auth_state;
	cbrock.checkmboxlist = !force;
	cbrock.checkshared = 0;
	cbrock.proc = proc;
	cbrock.procrock = rock;

	/* open the subscription file that contains the mailboxes the
	user is subscribed to */
	if ((r = mboxlist_opensubs(userid, &subs)) != 0) {
	goto done;
	}

	/* Build usermboxname */
	if (userid && !strchr(userid, '.') &&
	strlen(userid)+5 < MAX_MAILBOX_NAME) {
	strcpy(usermboxname, "user.");
	strcat(usermboxname, userid);
	usermboxnamelen = strlen(usermboxname);
	}
	else {
	userid = 0;
	}

	/* Check for INBOX first of all */
	if (userid) {
	if (GLOB_TEST(cbrock.g, "INBOX") != -1) {
	r = SUBDB->fetch(subs, usermboxname, usermboxnamelen,
	&data, &datalen, NULL);
	if (!r && data) {
	r = (*proc)(cbrock.inboxcase, 5, 1, rock);
	}
	}
	else if (!strncmp(pattern, usermboxname, usermboxnamelen) &&
	GLOB_TEST(cbrock.g, usermboxname) != -1) {
	r = SUBDB->fetch(subs, usermboxname, usermboxnamelen,
	&data, &datalen, NULL);
	if (!r && data) {
	r = (*proc)(usermboxname, usermboxnamelen, 1, rock);
	}
	}
	strcpy(usermboxname+usermboxnamelen, ".");
	usermboxnamelen++;

	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	}

	if (r) goto done;

	/* Find fixed-string pattern prefix */
	for (p = pattern; *p; p++) {
	if (p == '' \|\| p == '%' \|\| p == '?') break;
	}
	prefixlen = p - pattern;
	*p = '\0';

	/*
	* If user.X.* or INBOX.* can match pattern,
	* search for those mailboxes next
	*/
	if (userid &&
	(!strncmp(usermboxname, pattern, usermboxnamelen-1) \|\|
	!strncasecmp("inbox.", pattern, prefixlen < 6 ? prefixlen : 6))) {

	if (!strncmp(usermboxname, pattern, usermboxnamelen-1)) {
	cbrock.inboxoffset = 0;
	}
	else {
	cbrock.inboxoffset = strlen(userid);
	}

	cbrock.find_namespace = NAMESPACE_INBOX;
	/* iterate through prefixes matching usermboxname */
	SUBDB->foreach(subs,
	usermboxname, usermboxnamelen,
	&find_p, &find_cb, &cbrock,
	NULL);

	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	} else {
	cbrock.usermboxname = NULL;
	cbrock.usermboxnamelen = 0;
	}

	cbrock.find_namespace = NAMESPACE_USER;
	cbrock.inboxoffset = 0;
	if (usermboxnamelen) {
	usermboxname[--usermboxnamelen] = '\0';
	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	}
	/* search for all remaining mailboxes.
	just bother looking at the ones that have the same pattern prefix. */
	SUBDB->foreach(subs, pattern, prefixlen,
	&find_p, &find_cb, &cbrock, NULL);

	done:
	if (subs) mboxlist_closesubs(subs);
	glob_free(&cbrock.g);

	return r;
	}

	int mboxlist_findsub_alt(struct namespace *namespace,
	char pattern, int isadmin, char userid,
	struct auth_state *auth_state,
	int (proc)(), void rock, int force)
	{
	struct db *subs = NULL;
	struct find_rock cbrock;
	char usermboxname[MAX_MAILBOX_NAME+1], patbuf[MAX_MAILBOX_NAME+1];
	int usermboxnamelen = 0;
	const char *data;
	int datalen;
	int r = 0;
	char *p;
	int prefixlen, len;

	cbrock.g = glob_init(pattern, GLOB_HIERARCHY\|GLOB_INBOXCASE);
	cbrock.namespace = namespace;
	cbrock.inboxcase = glob_inboxcase(cbrock.g);
	cbrock.isadmin = 1; /* user can always see their subs */
	cbrock.auth_state = auth_state;
	cbrock.checkmboxlist = !force;
	cbrock.checkshared = 0;
	cbrock.proc = proc;
	cbrock.procrock = rock;

	/* open the subscription file that contains the mailboxes the
	user is subscribed to */
	if ((r = mboxlist_opensubs(userid, &subs)) != 0) {
	goto done;
	}

	/* Build usermboxname */
	if (userid && !strchr(userid, '.') &&
	strlen(userid)+5 < MAX_MAILBOX_NAME) {
	strcpy(usermboxname, "user.");
	strcat(usermboxname, userid);
	usermboxnamelen = strlen(usermboxname);
	}
	else {
	userid = 0;
	}

	/* Check for INBOX first of all */
	if (userid) {
	if (GLOB_TEST(cbrock.g, "INBOX") != -1) {
	r = SUBDB->fetch(subs, usermboxname, usermboxnamelen,
	&data, &datalen, NULL);
	if (!r && data) {
	r = (*proc)(cbrock.inboxcase, 5, 0, rock);
	}
	}
	strcpy(usermboxname+usermboxnamelen, ".");
	usermboxnamelen++;

	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	}

	if (r) goto done;

	glob_free(&cbrock.g);

	/* Find fixed-string pattern prefix */
	for (p = pattern; *p; p++) {
	if (p == '' \|\| p == '%' \|\| p == '?') break;
	}
	prefixlen = p - pattern;

	/*
	* Personal (INBOX) namespace
	*
	* Append pattern to "INBOX.", search for those subscriptions next
	*/
	if (userid) {
	strcpy(patbuf, "INBOX.");
	strcat(patbuf, pattern);
	cbrock.g = glob_init(patbuf, GLOB_HIERARCHY\|GLOB_INBOXCASE);
	cbrock.inboxcase = glob_inboxcase(cbrock.g);
	cbrock.inboxoffset = strlen(userid);
	cbrock.find_namespace = NAMESPACE_INBOX;

	/* iterate through prefixes matching usermboxname */
	SUBDB->foreach(subs,
	usermboxname, usermboxnamelen,
	&find_p, &find_cb, &cbrock,
	NULL);

	glob_free(&cbrock.g);

	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	} else {
	cbrock.usermboxname = NULL;
	cbrock.usermboxnamelen = 0;
	}

	if (usermboxnamelen) {
	usermboxname[--usermboxnamelen] = '\0';
	cbrock.usermboxname = usermboxname;
	cbrock.usermboxnamelen = usermboxnamelen;
	}

	/*
	* Other Users namespace
	*
	* If "Other Users*" can match pattern, search for those subscriptions next
	*/
	len = strlen(namespace->prefix[NAMESPACE_USER])-1;
	if (!strncmp(namespace->prefix[NAMESPACE_USER], pattern,
	prefixlen < len ? prefixlen : len)) {

	if (prefixlen < len)
	cbrock.g = glob_init(pattern+prefixlen, GLOB_HIERARCHY);
	else {
	strcpy(patbuf, "user");
	strcat(patbuf, pattern+len);
	cbrock.g = glob_init(patbuf, GLOB_HIERARCHY);
	}
	cbrock.find_namespace = NAMESPACE_USER;
	cbrock.inboxoffset = 0;

	/* iterate through prefixes matching usermboxname */
	SUBDB->foreach(subs,
	"user", 4,
	&find_p, &find_cb, &cbrock,
	NULL);

	glob_free(&cbrock.g);
	}

	/*
	* Shared namespace
	*
	* search for all remaining subscriptions.
	* just bother looking at the ones that have the same pattern prefix.
	*/
	len = strlen(namespace->prefix[NAMESPACE_SHARED]);
	if (!strncmp(namespace->prefix[NAMESPACE_SHARED], pattern,
	prefixlen < len - 1 ? prefixlen : len - 1)) {

	cbrock.find_namespace = NAMESPACE_SHARED;
	cbrock.inboxoffset = 0;

	if (prefixlen < len) {
	/* Find pattern which matches shared namespace prefix */
	for (p = pattern+prefixlen; *p; p++) {
	if (*p == '%') continue;
	else if (*p == '.') p++;
	break;
	}

	if (!*p) {
	/* special case: LSUB "" % -- see if we have a shared mbox */
	cbrock.g = glob_init("*", GLOB_HIERARCHY);
	cbrock.checkshared = 1;
	}
	else {
	cbrock.g = glob_init(p, GLOB_HIERARCHY);
	}

	SUBDB->foreach(subs,
	"", 0,
	&find_p, &find_cb, &cbrock,
	NULL);
	}
	else if (pattern[len-1] == '.') {
	strcpy(patbuf, "");
	strcat(patbuf, pattern+len);
	cbrock.g = glob_init(patbuf, GLOB_HIERARCHY);

	pattern[prefixlen] = '\0';
	SUBDB->foreach(subs,
	pattern+len, prefixlen-len,
	&find_p, &find_cb, &cbrock,
	NULL);
	}
	}

	done:
	if (subs) mboxlist_closesubs(subs);
	glob_free(&cbrock.g);

	return r;
	}

	/*
	* Change 'user's subscription status for mailbox 'name'.
	* Subscribes if 'add' is nonzero, unsubscribes otherwise.
	* if 'force' is set, force the subscription through even if
	* we don't know about 'name'.
	*/
	int mboxlist_changesub(const char name, const char userid,
	struct auth_state *auth_state, int add, int force)
	{
	int r;
	char *acl;
	struct db *subs;

	if ((r = mboxlist_opensubs(userid, &subs)) != 0) {
	return r;
	}

	if (add && !force) {
	/* Ensure mailbox exists and can be either seen or read by user */
	if ((r = mboxlist_lookup(name, NULL, &acl, NULL))!=0) {
	mboxlist_closesubs(subs);
	return r;
	}
	if ((cyrus_acl_myrights(auth_state, acl) & (ACL_READ\|ACL_LOOKUP)) == 0) {
	mboxlist_closesubs(subs);
	return IMAP_MAILBOX_NONEXISTENT;
	}
	}

	if (add) {
	r = SUBDB->store(subs, name, strlen(name), "", 0, NULL);
	} else {
	r = SUBDB->delete(subs, name, strlen(name), NULL);
	/* if it didn't exist, that's ok */
	if (r == CYRUSDB_EXISTS) r = CYRUSDB_OK;
	}

	switch (r) {
	case CYRUSDB_OK:
	r = 0;
	break;

	default:
	r = IMAP_IOERROR;
	break;
	}

	mboxlist_closesubs(subs);
	return r;
	}
	diff --git a/imap/pop3proxyd.c b/imap/pop3proxyd.c
	index ffe5eba70..5eff3fdef 100644
	--- a/imap/pop3proxyd.c
	+++ b/imap/pop3proxyd.c
	@@ -1,1343 +1,1343 @@
	/* pop3proxyd.c -- POP3 server protocol parsing (proxy)
	*
	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*/

	/*
	- * $Id: pop3proxyd.c,v 1.31 2002/02/15 17:21:15 rjs3 Exp $
	+ * $Id: pop3proxyd.c,v 1.32 2002/02/19 18:50:13 ken3 Exp $
	*/
	#include <config.h>


	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <string.h>
	#include <fcntl.h>
	#include <time.h>
	#include <signal.h>
	#include <assert.h>
	#include <sys/types.h>
	#include <sys/param.h>
	#include <syslog.h>
	#include <com_err.h>
	#include <netdb.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <ctype.h>
	#include "prot.h"

	#include <sasl/sasl.h>
	#include <sasl/saslutil.h>

	#include "acl.h"
	#include "util.h"
	#include "auth.h"
	#include "imapconf.h"
	#include "tls.h"

	#include "iptostring.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "version.h"
	#include "xmalloc.h"
	#include "mboxlist.h"

	#ifdef HAVE_KRB
	/* kerberos des is purported to conflict with OpenSSL DES */
	#define DES_DEFS
	#include <krb.h>

	/* MIT's kpop authentication kludge */
	char klrealm[REALM_SZ];
	AUTH_DAT kdata;
	#endif /* HAVE_KRB */
	static int kflag = 0;

	extern int optind;
	extern char *optarg;
	extern int opterr;

	extern int errno;



	#ifdef HAVE_SSL
	static SSL *tls_conn;
	#endif /* HAVE_SSL */

	sasl_conn_t popd_saslconn; / the sasl connection context */

	char *popd_userid = 0;
	struct sockaddr_in popd_localaddr, popd_remoteaddr;
	int popd_haveaddr = 0;
	char popd_clienthost[250] = "[local]";
	struct protstream popd_out, popd_in;
	int popd_starttls_done = 0;
	int popd_auth_done = 0;

	struct protstream backend_out, backend_in;
	int backend_sock;
	sasl_conn_t *backend_saslconn;

	/* current namespace */
	static struct namespace popd_namespace;

	static void cmd_apop(char *response);
	static int apop_enabled(void);
	static char popd_apop_chal[45 + MAXHOSTNAMELEN + 1]; /* <rand.time@hostname> */

	static void cmd_auth();
	static void cmd_capa();
	static void cmd_pass();
	static void cmd_user();
	static void cmd_starttls(int pop3s);
	static void cmdloop(void);
	static void kpop(void);
	static void usage(void);
	static void openproxy(void);
	static void bitpipe(void);

	extern void setproctitle_init(int argc, char argv, char envp);
	extern int proc_register(const char progname, const char clienthost,
	const char userid, const char mailbox);
	extern void proc_cleanup(void);
	void shut_down(int code) __attribute__ ((noreturn));

	/* Enable the resetting of a sasl_conn_t */
	static int reset_saslconn(sasl_conn_t **conn);

	static struct
	{
	char *ipremoteport;
	char *iplocalport;
	sasl_ssf_t ssf;
	char *authid;
	} saslprops = {NULL,NULL,0,NULL};

	static struct sasl_callback mysasl_cb[] = {
	{ SASL_CB_GETOPT, &mysasl_config, NULL },
	{ SASL_CB_LIST_END, NULL, NULL }
	};

	/*
	* run once when process is forked;
	* MUST NOT exit directly; must return with non-zero error code
	*/
	int service_init(int argc, char argv, char envp)
	{
	int r;

	config_changeident("pop3d");
	if (geteuid() == 0) fatal("must run as the Cyrus user", EC_USAGE);
	setproctitle_init(argc, argv, envp);

	/* set signal handlers */
	signals_set_shutdown(&shut_down);
	signals_add_handlers();
	signal(SIGPIPE, SIG_IGN);

	/* set the SASL allocation functions */
	sasl_set_alloc((sasl_malloc_t *) &xmalloc,
	(sasl_calloc_t *) &calloc,
	(sasl_realloc_t *) &xrealloc,
	(sasl_free_t *) &free);

	/* load the SASL plugins */
	if ((r = sasl_server_init(mysasl_cb, "Cyrus")) != SASL_OK) {
	syslog(LOG_ERR, "SASL failed initializing: sasl_server_init(): %s",
	sasl_errstring(r, NULL, NULL));
	return 2;
	}

	/* open the mboxlist, we'll need it for real work */
	mboxlist_init(0);
	mboxlist_open(NULL);

	/* Set namespace */
	if ((r = mboxname_init_namespace(&popd_namespace, 0)) != 0) {
	syslog(LOG_ERR, error_message(r));
	fatal(error_message(r), EC_CONFIG);
	}

	return 0;
	}

	/*
	* run for each accepted connection
	*/
	int service_main(int argc, char argv, char envp)
	{
	int pop3s = 0;
	int opt;
	socklen_t salen;
	struct hostent *hp;
	char localip[60], remoteip[60];
	int timeout;
	sasl_security_properties_t *secprops=NULL;

	signals_poll();

	popd_in = prot_new(0, 0);
	popd_out = prot_new(1, 1);

	while ((opt = getopt(argc, argv, "C:sk")) != EOF) {
	switch(opt) {
	case 'C': /* alt config file - handled by service::main() */
	break;

	case 's': /* pop3s (do starttls right away) */
	pop3s = 1;
	if (!tls_enabled("pop3")) {
	syslog(LOG_ERR, "pop3s: required OpenSSL options not present");
	fatal("pop3s: required OpenSSL options not present",
	EC_CONFIG);
	}
	case 'k':
	kflag++;
	break;
	default:
	usage();
	}
	}

	/* Find out name of client host */
	salen = sizeof(popd_remoteaddr);
	if (getpeername(0, (struct sockaddr *)&popd_remoteaddr, &salen) == 0 &&
	popd_remoteaddr.sin_family == AF_INET) {
	hp = gethostbyaddr((char *)&popd_remoteaddr.sin_addr,
	sizeof(popd_remoteaddr.sin_addr), AF_INET);
	if (hp != NULL) {
	strncpy(popd_clienthost, hp->h_name, sizeof(popd_clienthost)-30);
	popd_clienthost[sizeof(popd_clienthost)-30] = '\0';
	} else {
	popd_clienthost[0] = '\0';
	}
	strcat(popd_clienthost, "[");
	strcat(popd_clienthost, inet_ntoa(popd_remoteaddr.sin_addr));
	strcat(popd_clienthost, "]");
	salen = sizeof(popd_localaddr);
	if (getsockname(0, (struct sockaddr *)&popd_localaddr, &salen) == 0) {
	popd_haveaddr = 1;
	}
	}

	/* other params should be filled in */
	if (sasl_server_new("pop", config_servername, NULL, NULL, NULL,
	NULL, 0, &popd_saslconn) != SASL_OK)
	fatal("SASL failed initializing: sasl_server_new()",EC_TEMPFAIL);

	/* will always return something valid */
	secprops = mysasl_secprops(SASL_SEC_NOPLAINTEXT);
	sasl_setprop(popd_saslconn, SASL_SEC_PROPS, secprops);

	if(iptostring((struct sockaddr *)&popd_localaddr,
	sizeof(struct sockaddr_in), localip, 60) == 0) {
	sasl_setprop(popd_saslconn, SASL_IPLOCALPORT, localip);
	saslprops.iplocalport = xstrdup(localip);
	}
	if(iptostring((struct sockaddr *)&popd_remoteaddr,
	sizeof(struct sockaddr_in), remoteip, 60) == 0) {
	sasl_setprop(popd_saslconn, SASL_IPREMOTEPORT, remoteip);
	saslprops.ipremoteport = xstrdup(remoteip);
	}

	proc_register("pop3d", popd_clienthost, NULL, NULL);

	/* Set inactivity timer */
	timeout = config_getint("poptimeout", 10);
	if (timeout < 10) timeout = 10;
	prot_settimeout(popd_in, timeout*60);
	prot_setflushonread(popd_in, popd_out);

	if (kflag) kpop();

	/* we were connected on pop3s port so we should do
	TLS negotiation immediatly */
	if (pop3s == 1) cmd_starttls(1);

	/* Create APOP challenge for banner */
	if (!sasl_mkchal(popd_saslconn, popd_apop_chal, sizeof(popd_apop_chal), 1)) {
	syslog(LOG_ERR, "APOP disabled: can't create challenge");
	*popd_apop_chal = 0;
	}

	prot_printf(popd_out, "+OK %s Cyrus POP3 Murder %s server ready %s\r\n",
	config_servername, CYRUS_VERSION,
	apop_enabled() ? popd_apop_chal : "");
	cmdloop();

	return 0;
	}

	/* called if 'service_init()' was called but not 'service_main()' */
	void service_abort(int error)
	{
	shut_down(error);
	}

	void usage(void)
	{
	prot_printf(popd_out,
	"-ERR usage: pop3proxyd [-C <alt_config>]"
	" [-k] [-s]\r\n");
	prot_flush(popd_out);
	exit(EC_USAGE);
	}

	/*
	* Cleanly shut down and exit
	*/
	void shut_down(int code)
	{
	proc_cleanup();
	mboxlist_close();
	mboxlist_done();
	#ifdef HAVE_SSL
	tls_shutdown_serverengine();
	#endif
	prot_flush(popd_out);
	exit(code);
	}

	void fatal(const char* s, int code)
	{
	static int recurse_code = 0;

	if (recurse_code) {
	/* We were called recursively. Just give up */
	proc_cleanup();
	exit(recurse_code);
	}
	recurse_code = code;
	prot_printf(popd_out, "-ERR [SYS/PERM] Fatal error: %s\r\n", s);
	prot_flush(popd_out);
	shut_down(code);
	}

	/*
	* Found a shutdown file: Spit out an untagged BYE and shut down
	*/
	void shutdown_file(void)
	{
	int fd;
	struct protstream *shutdown_in;
	char buf[1024];
	char *p;
	static char shutdownfilename[1024];

	if (!shutdownfilename[0])
	sprintf(shutdownfilename, "%s/msg/shutdown", config_dir);
	if ((fd = open(shutdownfilename, O_RDONLY, 0)) == -1) return;

	shutdown_in = prot_new(fd, 0);
	prot_fgets(buf, sizeof(buf), shutdown_in);
	if ((p = strchr(buf, '\r')) != NULL) *p = 0;
	if ((p = strchr(buf, '\n')) != NULL) *p = 0;

	for (p = buf; p == '['; p++); / can't have [ be first char, sigh */
	prot_printf(popd_out, "-ERR [SYS/TEMP] %s\r\n", p);

	shut_down(0);
	}

	#ifdef HAVE_KRB
	/*
	* MIT's kludge of a kpop protocol
	* Client does a krb_sendauth() first thing
	*/
	static void kpop(void)
	{
	Key_schedule schedule;
	KTEXT_ST ticket;
	char instance[INST_SZ];
	char version[9];
	const char *srvtab;
	int r;

	if (!popd_haveaddr) {
	fatal("Cannot get client's IP address", EC_OSERR);
	}

	srvtab = config_getstring("srvtab", "");

	strcpy(instance, "*");
	r = krb_recvauth(0L, 0, &ticket, "pop", instance,
	&popd_remoteaddr, (struct sockaddr_in *) NULL,
	&kdata, (char*) srvtab, schedule, version);

	if (r) {
	prot_printf(popd_out, "-ERR [AUTH] Kerberos authentication failure: %s\r\n",
	krb_err_txt[r]);
	syslog(LOG_NOTICE,
	"badlogin: %s kpop ? %s%s%s@%s %s",
	popd_clienthost, kdata.pname,
	kdata.pinst[0] ? "." : "", kdata.pinst,
	kdata.prealm, krb_err_txt[r]);
	shut_down(0);
	}

	r = krb_get_lrealm(klrealm,1);
	if (r) {
	prot_printf(popd_out, "-ERR [AUTH] Kerberos failure: %s\r\n",
	krb_err_txt[r]);
	syslog(LOG_NOTICE,
	"badlogin: %s kpop ? %s%s%s@%s krb_get_lrealm: %s",
	popd_clienthost, kdata.pname,
	kdata.pinst[0] ? "." : "", kdata.pinst,
	kdata.prealm, krb_err_txt[r]);
	shut_down(0);
	}
	}
	#else
	static void kpop(void)
	{
	usage();
	}
	#endif

	/*
	* Top-level command loop parsing
	*/
	static void cmdloop(void)
	{
	char inputbuf[8192];
	char p, arg;

	for (;;) {
	signals_poll();

	if (popd_auth_done) {
	bitpipe();
	return;
	}

	/* check for shutdown file */
	shutdown_file();

	if (!prot_fgets(inputbuf, sizeof(inputbuf), popd_in)) {
	return;
	}

	p = inputbuf + strlen(inputbuf);
	if (p > inputbuf && p[-1] == '\n') *--p = '\0';
	if (p > inputbuf && p[-1] == '\r') *--p = '\0';

	/* Parse into keword and argument */
	for (p = inputbuf; p && !isspace((int) p); p++);
	if (*p) {
	*p++ = '\0';
	arg = p;
	if (strcasecmp(inputbuf, "pass") != 0) {
	while (arg && isspace((int) arg)) {
	arg++;
	}
	}
	if (!*arg) {
	prot_printf(popd_out, "-ERR Syntax error\r\n");
	continue;
	}
	}
	else {
	arg = 0;
	}
	lcase(inputbuf);

	if (!strcmp(inputbuf, "quit")) {
	if (!arg) {
	prot_printf(popd_out, "+OK\r\n");
	shut_down(0);
	}
	else prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
	}
	else if (!strcmp(inputbuf, "capa")) {
	if (arg) {
	prot_printf(popd_out, "-ERR Unexpected extra argument\r\n");
	} else {
	cmd_capa();
	}
	}
	else if (!strcmp(inputbuf, "user")) {
	if (!arg) {
	prot_printf(popd_out, "-ERR Missing argument\r\n");
	}
	else {
	cmd_user(arg);
	}
	}
	else if (!strcmp(inputbuf, "pass")) {
	if (!arg) prot_printf(popd_out, "-ERR Missing argument\r\n");
	else cmd_pass(arg);
	}
	else if (!strcmp(inputbuf, "apop") && apop_enabled()) {
	if (!arg) prot_printf(popd_out, "-ERR Missing argument\r\n");
	else cmd_apop(arg);
	}
	else if (!strcmp(inputbuf, "auth")) {
	cmd_auth(arg);
	}
	else if (!strcmp(inputbuf, "stls") && tls_enabled("pop3")) {
	if (arg) {
	prot_printf(popd_out,
	"-ERR STLS doesn't take any arguements\r\n");
	} else {
	cmd_starttls(0);
	}
	}
	else {
	prot_printf(popd_out, "-ERR Unrecognized command\r\n");
	}
	}
	}

	#ifdef HAVE_SSL
	static void cmd_starttls(int pop3s)
	{
	char tls_cert, tls_key;
	int result;
	int *layerp;
	char *auth_id;
	sasl_ssf_t ssf;

	/* SASL and openssl have different ideas about whether ssf is signed */
	layerp = (int *) &ssf;

	if (popd_starttls_done == 1)
	{
	prot_printf(popd_out, "-ERR %s\r\n",
	"Already successfully executed STLS");
	return;
	}

	tls_cert = (char *)config_getstring("tls_pop3_cert_file",
	config_getstring("tls_cert_file", ""));
	tls_key = (char *)config_getstring("tls_pop3_key_file",
	config_getstring("tls_key_file", ""));

	result=tls_init_serverengine("pop3",
	5, /* depth to verify */
	!pop3s, /* can client auth? */
	0, /* require client to auth? */
	!pop3s); /* TLSv1 only? */

	if (result == -1) {

	syslog(LOG_ERR, "[pop3d] error initializing TLS");

	if (pop3s == 0)
	prot_printf(popd_out, "-ERR [SYS/PERM] %s\r\n", "Error initializing TLS");
	else
	fatal("tls_init() failed",EC_TEMPFAIL);

	return;
	}

	if (pop3s == 0)
	{
	prot_printf(popd_out, "+OK %s\r\n", "Begin TLS negotiation now");
	/* must flush our buffers before starting tls */
	prot_flush(popd_out);
	}

	result=tls_start_servertls(0, /* read */
	1, /* write */
	layerp,
	&auth_id,
	&tls_conn);

	/* if error */
	if (result==-1) {
	if (pop3s == 0) {
	prot_printf(popd_out, "-ERR [SYS/PERM] Starttls failed\r\n");
	syslog(LOG_NOTICE, "[pop3d] STARTTLS failed: %s", popd_clienthost);
	} else {
	syslog(LOG_NOTICE, "pop3s failed: %s", popd_clienthost);
	fatal("tls_start_servertls() failed", EC_TEMPFAIL);
	}
	return;
	}

	/* tell SASL about the negotiated layer */
	result = sasl_setprop(popd_saslconn, SASL_SSF_EXTERNAL, &ssf);
	if (result != SASL_OK) {
	fatal("sasl_setprop() failed: cmd_starttls()", EC_TEMPFAIL);
	}
	saslprops.ssf = ssf;

	result = sasl_setprop(popd_saslconn, SASL_AUTH_EXTERNAL, auth_id);
	if (result != SASL_OK) {
	fatal("sasl_setprop() failed: cmd_starttls()", EC_TEMPFAIL);
	}
	if(saslprops.authid) {
	free(saslprops.authid);
	saslprops.authid = NULL;
	}
	if(auth_id)
	saslprops.authid = xstrdup(auth_id);

	/* tell the prot layer about our new layers */
	prot_settls(popd_in, tls_conn);
	prot_settls(popd_out, tls_conn);

	popd_starttls_done = 1;
	}
	#else
	static void cmd_starttls(int pop3s)
	{
	fatal("cmd_starttls() called, but no OpenSSL", EC_SOFTWARE);
	}
	#endif /* HAVE_SSL */

	static int apop_enabled(void)
	{
	/* Check if pseudo APOP mechanism is enabled (challenge == NULL) */
	if (sasl_checkapop(popd_saslconn, NULL, 0, NULL, 0) != SASL_OK) return 0;
	/* Check if we have a challenge string */
	if (!*popd_apop_chal) return 0;
	return 1;
	}

	static void cmd_apop(char *response)
	{
	int fd;
	struct protstream *shutdown_in;
	char buf[1024];
	char *p;
	char shutdownfilename[1024];
	int sasl_result;
	char *canon_user;

	assert(response != NULL);

	if (popd_userid) {
	prot_printf(popd_out, "-ERR [AUTH] Must give PASS command\r\n");
	return;
	}

	/* Check if it is enabled (challenge == NULL) */
	if(sasl_checkapop(popd_saslconn, NULL, 0, NULL, 0) != SASL_OK)
	fatal("cmd_apop called without working sasl_checkapop", EC_SOFTWARE);

	sprintf(shutdownfilename, "%s/msg/shutdown", config_dir);
	if ((fd = open(shutdownfilename, O_RDONLY, 0)) != -1) {
	shutdown_in = prot_new(fd, 0);
	prot_fgets(buf, sizeof(buf), shutdown_in);
	if ((p = strchr(buf, '\r'))!=NULL) *p = 0;
	if ((p = strchr(buf, '\n'))!=NULL) *p = 0;

	for(p = buf; p == '['; p++); / can't have [ be first char */
	prot_printf(popd_out, "-ERR [SYS/TEMP] %s\r\n", p);
	prot_flush(popd_out);
	shut_down(0);
	}

	sasl_result = sasl_checkapop(popd_saslconn,
	popd_apop_chal,
	strlen(popd_apop_chal),
	response,
	strlen(response));

	/* failed authentication */
	if (sasl_result != SASL_OK)
	{
	sleep(3);

	prot_printf(popd_out, "-ERR [AUTH] authenticating: %s\r\n",
	sasl_errstring(sasl_result, NULL, NULL));

	syslog(LOG_NOTICE, "badlogin: %s APOP (%s) %s",
	popd_clienthost, popd_apop_chal,
	sasl_errdetail(popd_saslconn));

	return;
	}

	/* successful authentication */

	/*
	* get the userid from SASL --- already canonicalized from
	* mysasl_authproc()
	*/
	sasl_result = sasl_getprop(popd_saslconn, SASL_USERNAME,
	(const void **) &canon_user);
	popd_userid = xstrdup(canon_user);
	if (sasl_result != SASL_OK) {
	prot_printf(popd_out,
	"-ERR [AUTH] weird SASL error %d getting SASL_USERNAME\r\n",
	sasl_result);
	return;
	}

	syslog(LOG_NOTICE, "login: %s %s APOP %s",
	popd_clienthost, popd_userid, "User logged in");

	openproxy();
	popd_auth_done = 1;
	}

	void
	cmd_user(user)
	char *user;
	{
	char *p;

	/* possibly disallow USER */
	if (!(kflag \|\| popd_starttls_done \|\|
	config_getswitch("allowplaintext", 1))) {
	prot_printf(popd_out,
	"-ERR [AUTH] USER command only available under a layer\r\n");
	return;
	}

	if (popd_userid) {
	prot_printf(popd_out, "-ERR [AUTH] Must give PASS command\r\n");
	return;
	}

	shutdown_file(); /* check for shutdown file */
	if (!(p = auth_canonifyid(user,0)) \|\|
	/* '.' isn't allowed if '.' is the hierarchy separator */
	(popd_namespace.hier_sep == '.' && strchr(p, '.')) \|\|
	strlen(p) + 6 > MAX_MAILBOX_PATH) {

	prot_printf(popd_out, "-ERR [AUTH] Invalid user\r\n");
	syslog(LOG_NOTICE,
	"badlogin: %s plaintext %s invalid user",
	popd_clienthost, beautify_string(user));
	}
	else {
	popd_userid = xstrdup(p);
	prot_printf(popd_out, "+OK Name is a valid mailbox\r\n");
	}
	}

	void cmd_pass(char *pass)
	{
	char *reply = 0;
	int plaintextloginpause;

	if (!popd_userid) {
	prot_printf(popd_out, "-ERR [AUTH] Must give USER command\r\n");
	return;
	}

	#ifdef HAVE_KRB
	if (kflag) {
	if (strcmp(popd_userid, kdata.pname) != 0 \|\|
	kdata.pinst[0] \|\|
	strcmp(klrealm, kdata.prealm) != 0) {
	prot_printf(popd_out, "-ERR [AUTH] Invalid login\r\n");
	syslog(LOG_NOTICE,
	"badlogin: %s kpop %s %s%s%s@%s access denied",
	popd_clienthost, popd_userid,
	kdata.pname, kdata.pinst[0] ? "." : "",
	kdata.pinst, kdata.prealm);
	return;
	}

	openproxy();
	syslog(LOG_NOTICE, "login: %s %s kpop", popd_clienthost, popd_userid);
	popd_auth_done = 1;
	return;
	}
	#endif

	if (!strcmp(popd_userid, "anonymous")) {
	if (config_getswitch("allowanonymouslogin", 0)) {
	pass = beautify_string(pass);
	if (strlen(pass) > 500) pass[500] = '\0';
	syslog(LOG_NOTICE, "login: %s anonymous %s",
	popd_clienthost, pass);
	}
	else {
	syslog(LOG_NOTICE, "badlogin: %s anonymous login refused",
	popd_clienthost);
	prot_printf(popd_out, "-ERR [AUTH] Invalid login\r\n");
	return;
	}
	}
	else if (sasl_checkpass(popd_saslconn,
	popd_userid,
	strlen(popd_userid),
	pass,
	strlen(pass))!=SASL_OK) {
	if (reply) {
	syslog(LOG_NOTICE, "badlogin: %s plaintext %s %s",
	popd_clienthost, popd_userid, reply);
	}
	sleep(3);
	prot_printf(popd_out, "-ERR [AUTH] Invalid login\r\n");
	free(popd_userid);
	popd_userid = 0;

	return;
	}
	else {
	syslog(LOG_NOTICE, "login: %s %s plaintext %s",
	popd_clienthost, popd_userid, reply ? reply : "");
	plaintextloginpause = config_getint("plaintextloginpause", 0);
	if (plaintextloginpause) sleep(plaintextloginpause);
	}

	openproxy();
	popd_auth_done = 1;
	}

	/* Handle the POP3 Extension extension.
	*/
	void
	cmd_capa()
	{
	int minpoll = config_getint("popminpoll", 0) * 60;
	int expire = config_getint("popexpiretime", -1);
	unsigned mechcount;
	const char *mechlist;

	prot_printf(popd_out, "+OK List of capabilities follows\r\n");

	/* SASL special case: print SASL, then a list of supported capabilities */
	if (sasl_listmech(popd_saslconn,
	NULL, /* should be id string */
	"SASL ", " ", "\r\n",
	&mechlist,
	NULL, &mechcount) == SASL_OK && mechcount > 0) {
	prot_write(popd_out, mechlist, strlen(mechlist));
	}

	if (tls_enabled("pop3")) {
	prot_printf(popd_out, "STLS\r\n");
	}
	if (expire < 0) {
	prot_printf(popd_out, "EXPIRE NEVER\r\n");
	} else {
	prot_printf(popd_out, "EXPIRE %d\r\n", expire);
	}

	prot_printf(popd_out, "LOGIN-DELAY %d\r\n", minpoll);
	prot_printf(popd_out, "TOP\r\n");
	prot_printf(popd_out, "UIDL\r\n");
	prot_printf(popd_out, "PIPELINING\r\n");
	prot_printf(popd_out, "RESP-CODES\r\n");
	prot_printf(popd_out, "AUTH-RESP-CODE\r\n");

	if (kflag \|\| popd_starttls_done \|\| config_getswitch("allowplaintext", 1)) {
	prot_printf(popd_out, "USER\r\n");
	}

	prot_printf(popd_out,
	"IMPLEMENTATION Cyrus POP3 proxy server %s\r\n",
	CYRUS_VERSION);

	prot_printf(popd_out, ".\r\n");
	prot_flush(popd_out);
	}


	/* according to RFC 2449, since we advertise the "SASL" capability, we
	* must accept an optional second argument of the initial client
	* response (base64 encoded!).
	*/
	void cmd_auth(char *arg)
	{
	int sasl_result;
	static struct buf clientin;
	unsigned clientinlen=0;
	char *authtype;
	const char *serverout;
	unsigned int serveroutlen;
	char *cin;

	/* if client didn't specify an argument we give them the list */
	if (!arg) {
	const char *sasllist;
	unsigned int mechnum;

	prot_printf(popd_out, "+OK List of supported mechanisms follows\r\n");

	/* CRLF seperated, dot terminated */
	if (sasl_listmech(popd_saslconn, NULL,
	"", "\r\n", "\r\n",
	&sasllist,
	NULL, &mechnum) == SASL_OK) {
	if (mechnum>0) {
	prot_printf(popd_out,"%s",sasllist);
	}
	}

	prot_printf(popd_out, ".\r\n");
	return;
	}

	authtype = arg;
	while (arg && !isspace((int) arg)) {
	arg++;
	}
	if (isspace((int) *arg)) {
	/* null terminate authtype, get argument */
	*arg++ = '\0';
	} else {
	/* no optional client response */
	arg = NULL;
	}

	/* if arg != NULL, it's an initial client response */
	if (arg) {
	int arglen = strlen(arg);

	clientin.alloc = arglen + 1;
	cin = clientin.s = xmalloc(clientin.alloc);
	sasl_result = sasl_decode64(arg, arglen, clientin.s,
	clientin.alloc, &clientinlen);
	} else {
	sasl_result = SASL_OK;
	cin = NULL;
	clientinlen = 0;
	}

	/* server did specify a command, so let's try to authenticate */
	if (sasl_result == SASL_OK \|\| sasl_result == SASL_CONTINUE)
	sasl_result = sasl_server_start(popd_saslconn, authtype,
	cin, clientinlen,
	&serverout, &serveroutlen);
	/* sasl_server_start will return SASL_OK or SASL_CONTINUE on success */
	while (sasl_result == SASL_CONTINUE)
	{
	char c;

	/* print the message to the user */
	printauthready(popd_out, serveroutlen, (unsigned char *)serverout);

	c = prot_getc(popd_in);
	if(c == '*') {
	eatline(popd_in,c);
	prot_printf(popd_out,
	"-ERR [AUTH] Client canceled authentication\r\n");
	reset_saslconn(&popd_saslconn);
	return;
	} else {
	prot_ungetc(c, popd_in);
	}

	/* get string from user */
	clientinlen = getbase64string(popd_in, &clientin);
	if (clientinlen == -1) {
	prot_printf(popd_out, "-ERR [AUTH] Invalid base64 string\r\n");
	return;
	}

	sasl_result = sasl_server_step(popd_saslconn,
	clientin.s,
	clientinlen,
	&serverout, &serveroutlen);
	}

	/* failed authentication */
	if (sasl_result != SASL_OK)
	{
	sleep(3);

	reset_saslconn(&popd_saslconn);

	/* convert the sasl error code to a string */
	prot_printf(popd_out, "-ERR [AUTH] authenticating: %s\r\n",
	sasl_errstring(sasl_result, NULL, NULL));

	if (authtype) {
	syslog(LOG_NOTICE, "badlogin: %s %s %s",
	popd_clienthost, authtype, sasl_errdetail(popd_saslconn));
	} else {
	syslog(LOG_NOTICE, "badlogin: %s %s",
	popd_clienthost, sasl_errdetail(popd_saslconn));
	}

	return;
	}

	/* successful authentication */

	/* get the userid from SASL --- already canonicalized from
	* mysasl_authproc()
	*/
	/* FIXME XXX: popd_userid is NOT CONST */
	sasl_result = sasl_getprop(popd_saslconn, SASL_USERNAME,
	(const void **) &popd_userid);
	if (sasl_result != SASL_OK) {
	prot_printf(popd_out,
	"-ERR [AUTH] weird SASL error %d getting SASL_USERNAME\r\n",
	sasl_result);
	return;
	}

	proc_register("pop3d", popd_clienthost, popd_userid, NULL);
	syslog(LOG_NOTICE, "login: %s %s %s %s", popd_clienthost, popd_userid,
	authtype, "User logged in");

	prot_setsasl(popd_in, popd_saslconn);
	prot_setsasl(popd_out, popd_saslconn);

	openproxy();
	popd_auth_done = 1;
	}

	static int mysasl_getauthline(struct protstream p, char *line,
	unsigned int *linelen)
	{
	char buf[2096];
	char str = (char ) buf;

	if (!prot_fgets(str, sizeof(buf), p)) {
	return SASL_FAIL;
	}
	if (!strncasecmp(str, "+OK", 3)) { return SASL_OK; }
	if (!strncasecmp(str, "-ERR", 4)) { return SASL_BADAUTH; }
	if (str[0] == '+' && str[1] == ' ') {
	size_t len;
	str += 2; /* jump past the "+ " */

	len = strlen(str) + 1;

	*line = xmalloc(strlen(str) + 1);
	if (str != '\r') { / decode it */
	int r;

	r = sasl_decode64(str, strlen(str), *line, len, linelen);
	if (r != SASL_OK) {
	return r;
	}

	return SASL_CONTINUE;
	} else { /* blank challenge */
	*line = NULL;
	*linelen = 0;

	return SASL_CONTINUE;
	}
	} else {
	/* huh??? */
	return SASL_FAIL;
	}
	}

	extern sasl_callback_t mysasl_callbacks(const char username,
	const char *authname,
	const char *realm,
	const char *password);
	extern void free_callbacks(sasl_callback_t *in);

	static int proxy_authenticate(const char *hostname)
	{
	int r;
	sasl_security_properties_t *secprops = NULL;
	struct sockaddr_in saddr_l;
	struct sockaddr_in saddr_r;
	socklen_t addrsize;
	sasl_callback_t *cb;
	char buf[2048];
	char optstr[128];
	char in, p;
	const char *out;
	unsigned int inlen, outlen;
	const char *mechusing;
	unsigned b64len;
	char localip[60], remoteip[60];
	const char *pass;

	strcpy(optstr, hostname);
	p = strchr(optstr, '.');
	if (p) *p = '\0';
	strcat(optstr, "_password");
	pass = config_getstring(optstr, NULL);
	cb = mysasl_callbacks(popd_userid,
	config_getstring("proxy_authname", "proxy"),
	config_getstring("proxy_realm", NULL),
	pass);

	r = sasl_client_new("pop", hostname, NULL, NULL,
	cb, 0, &backend_saslconn);
	if (r != SASL_OK) {
	return r;
	}

	secprops = mysasl_secprops(0);
	r = sasl_setprop(backend_saslconn, SASL_SEC_PROPS, secprops);
	if (r != SASL_OK) {
	return r;
	}

	/* set the IP addresses */
	addrsize=sizeof(struct sockaddr_in);
	if (getpeername(backend_sock, (struct sockaddr *)&saddr_r, &addrsize) != 0)
	return SASL_FAIL;
	addrsize=sizeof(struct sockaddr_in);
	if (getsockname(backend_sock, (struct sockaddr *)&saddr_l,&addrsize)!=0)
	return SASL_FAIL;

	if (iptostring((struct sockaddr *)&saddr_r,
	sizeof(struct sockaddr_in), remoteip, 60) != 0)
	return SASL_FAIL;
	if (iptostring((struct sockaddr *)&saddr_l,
	sizeof(struct sockaddr_in), localip, 60) != 0)
	return SASL_FAIL;

	r = sasl_setprop(popd_saslconn, SASL_IPLOCALPORT, localip);
	if (r != SASL_OK) return r;
	r = sasl_setprop(popd_saslconn, SASL_IPREMOTEPORT, remoteip);
	if (r != SASL_OK) return r;

	/* read the initial greeting */
	if (!prot_fgets(buf, sizeof(buf), backend_in)) {
	return SASL_FAIL;
	}

	strcpy(buf, hostname);
	p = strchr(buf, '.');
	*p = '\0';
	strcat(buf, "_mechs");

	/* we now do the actual SASL exchange */
	r = sasl_client_start(backend_saslconn,
	config_getstring(buf, "KERBEROS_V4"),
	NULL, &out, &outlen, &mechusing);
	if ((r != SASL_OK) && (r != SASL_CONTINUE)) {
	return r;
	}
	if (out == NULL \|\| outlen == 0) {
	prot_printf(backend_out, "AUTH %s\r\n", mechusing);
	} else {
	/* send initial challenge */
	r = sasl_encode64(out, outlen, buf, sizeof(buf), &b64len);
	if (r != SASL_OK)
	return r;
	prot_printf(backend_out, "AUTH %s %s\r\n", mechusing, buf);
	}

	in = NULL;
	inlen = 0;
	r = mysasl_getauthline(backend_in, &in, &inlen);
	while (r == SASL_CONTINUE) {
	r = sasl_client_step(backend_saslconn, in, inlen, NULL, &out, &outlen);
	if (in) {
	free(in);
	}
	if (r != SASL_OK && r != SASL_CONTINUE) {
	return r;
	}

	r = sasl_encode64(out, outlen, buf, sizeof(buf), &b64len);
	if (r != SASL_OK) {
	return r;
	}

	prot_write(backend_out, buf, b64len);
	prot_printf(backend_out, "\r\n");

	r = mysasl_getauthline(backend_in, &in, &inlen);
	}

	/* Done with callbacks */
	free_callbacks(cb);

	if (r == SASL_OK) {
	prot_setsasl(backend_in, backend_saslconn);
	prot_setsasl(backend_out, backend_saslconn);
	}

	/* r == SASL_OK on success */
	return r;
	}

	static void openproxy(void)
	{
	struct hostent *hp;
	struct sockaddr_in sin;
	char inboxname[MAX_MAILBOX_PATH];
	int r;
	char *server = NULL;

	/* have to figure out what server to connect to */
	strcpy(inboxname, "user.");
	strcat(inboxname, popd_userid);

	/* Translate any separators in userid part of inboxname
	(we need the original userid for AUTH to backend) */
	mboxname_hiersep_tointernal(&popd_namespace, inboxname+5);

	r = mboxlist_lookup(inboxname, &server, NULL, NULL);
	if (r) fatal("couldn't find backend server", EC_CONFIG);

	/* xxx hide the fact that we are storing partitions */
	if(server) {
	char *c;
	c = strchr(server, '!');
	if(c) *c = '\0';
	}

	hp = gethostbyname(server);
	if (!hp) fatal("gethostbyname failed", EC_CONFIG);
	sin.sin_family = AF_INET;
	memcpy(&sin.sin_addr, hp->h_addr, hp->h_length);
	sin.sin_port = htons(110);

	if ((backend_sock = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
	syslog(LOG_ERR, "socket() failed: %m");
	fatal("socket failed", EC_CONFIG);
	}
	if (connect(backend_sock, (struct sockaddr *) &sin, sizeof(sin)) < 0) {
	syslog(LOG_ERR, "connect() failed: %m");
	fatal("connect failed", 1);
	}

	backend_in = prot_new(backend_sock, 0);
	backend_out = prot_new(backend_sock, 1);
	prot_setflushonread(backend_in, backend_out);

	if (proxy_authenticate(server) != SASL_OK) {
	- syslog(LOG_ERR, "couldn't authenticate to backend server", EC_CONFIG);
	+ syslog(LOG_ERR, "couldn't authenticate to backend server");
	fatal("couldn't authenticate to backend server", 1);
	}

	prot_printf(popd_out, "+OK Maildrop locked and ready\r\n");
	return;
	}

	/* we've authenticated the client, we've connected to the backend.
	now it's all up to them */
	static void bitpipe(void)
	{
	fd_set read_set, rset;
	int nfds, r;
	char buf[4096];

	FD_ZERO(&read_set);
	FD_SET(0, &read_set);
	FD_SET(backend_sock, &read_set);
	nfds = backend_sock + 1;

	for (;;) {
	rset = read_set;
	r = select(nfds, &rset, NULL, NULL, NULL);
	/* if select() failed it's not worth trying to figure anything out */
	if (r < 0) goto done;

	if (FD_ISSET(0, &rset)) {
	do {
	int c = prot_read(popd_in, buf, sizeof(buf));
	if (c == 0 \|\| c < 0) goto done;
	prot_write(backend_out, buf, c);
	} while (popd_in->cnt > 0);
	prot_flush(backend_out);
	}

	if (FD_ISSET(backend_sock, &rset)) {
	do {
	int c = prot_read(backend_in, buf, sizeof(buf));
	if (c == 0 \|\| c < 0) goto done;
	prot_write(popd_out, buf, c);
	} while (backend_in->cnt > 0);
	prot_flush(popd_out);
	}
	}
	done:
	/* ok, we're done. close backend connection */
	prot_free(backend_in);
	prot_free(backend_out);
	close(backend_sock);

	/* close the connection to the client */
	close(0);
	close(1);
	prot_free(popd_in);
	prot_free(popd_out);

	return;
	}

	/* Reset the given sasl_conn_t to a sane state */
	static int reset_saslconn(sasl_conn_t **conn)
	{
	int ret;
	sasl_security_properties_t *secprops = NULL;

	sasl_dispose(conn);
	/* do initialization typical of service_main */
	ret = sasl_server_new("pop", config_servername,
	NULL, NULL, NULL,
	NULL, 0, conn);
	if(ret != SASL_OK) return ret;
	if(saslprops.ipremoteport)
	ret = sasl_setprop(*conn, SASL_IPREMOTEPORT,
	saslprops.ipremoteport);
	if(ret != SASL_OK) return ret;

	if(saslprops.iplocalport)
	ret = sasl_setprop(*conn, SASL_IPLOCALPORT,
	saslprops.iplocalport);
	if(ret != SASL_OK) return ret;
	secprops = mysasl_secprops(SASL_SEC_NOPLAINTEXT);
	ret = sasl_setprop(*conn, SASL_SEC_PROPS, secprops);
	if(ret != SASL_OK) return ret;
	/* end of service_main initialization excepting SSF */

	/* If we have TLS/SSL info, set it */
	ret = sasl_setprop(*conn, SASL_IPLOCALPORT,
	saslprops.iplocalport);
	if(ret != SASL_OK) return ret;
	secprops = mysasl_secprops(SASL_SEC_NOPLAINTEXT);
	ret = sasl_setprop(*conn, SASL_SEC_PROPS, secprops);
	if(ret != SASL_OK) return ret;
	/* end of service_main initialization excepting SSF */

	/* If we have TLS/SSL info, set it */
	if(saslprops.ssf) {
	ret = sasl_setprop(*conn, SASL_SSF_EXTERNAL, &saslprops.ssf);
	}

	if(ret != SASL_OK) return ret;

	if(saslprops.authid) {
	ret = sasl_setprop(*conn, SASL_AUTH_EXTERNAL, saslprops.authid);
	if(ret != SASL_OK) return ret;
	}
	/* End TLS/SSL Info */

	return SASL_OK;
	}
	diff --git a/imap/squat.c b/imap/squat.c
	index 07b6fda60..686fd2b9e 100644
	--- a/imap/squat.c
	+++ b/imap/squat.c
	@@ -1,617 +1,618 @@
	/*
	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	- * $Id: squat.c,v 1.1 2001/09/25 16:49:51 ken3 Exp $
	+ * $Id: squat.c,v 1.2 2002/02/19 18:50:14 ken3 Exp $
	*/

	/*
	SQUAT code for searching indexes.
	Robert O'Callahan
	*/

	#include <sys/stat.h>
	#include <sys/mman.h>
	#include <unistd.h>
	#include <stdlib.h>
	+#include <string.h>
	#include <assert.h>

	#include "squat_internal.h"

	struct _SquatSearchIndex {
	int index_fd; /* the index file */
	char const* data; /* where it's mmaped to */
	char const* doc_list; /* where does the doc-list
	sequence start in memory */
	char const* word_list; /* where does the word trie
	offset table start in memory */
	char const* doc_ID_list; /* where does the doc-ID-list
	array start in memory */
	char const* data_end; /* the end of the mmaped file */
	unsigned char valid_char_bits[32]; /* which characters are valid in
	queries according to whoever
	created the index */
	};

	/* For each 0 <= i < 256, bit_counts[i] is the number of bits set in i */
	static char bit_counts[256];

	/* Returns true IFF the 'len' bytes starting at 's' are each equal to 'v' */
	static int memconst(char const* s, int len, char v) {
	while (len > 0 && *s == v) {
	s++;
	len--;
	}
	return len == 0;
	}

	SquatSearchIndex* squat_search_open(int fd) {
	struct stat buf;
	SquatSearchIndex* index;
	SquatDiskHeader const* header;
	SquatInt64 doc_list_offset, doc_ID_list_offset, word_list_offset;
	SquatInt64 data_len;

	squat_set_last_error(SQUAT_ERR_OK);

	/* initialize bit_counts constant array.
	This is so clever, I could die */
	if (bit_counts[1] == 0) {
	int c;
	for (c = 1; c < 256; c++) {
	bit_counts[c] = bit_counts[c >> 1] + (c & 1);
	}
	}

	index = (SquatSearchIndex*)malloc(sizeof(SquatSearchIndex));
	if (index == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return NULL;
	}

	index->index_fd = fd;

	if (fstat(fd, &buf) != 0) { /* fstat64? */
	squat_set_last_error(SQUAT_ERR_SYSERR);
	goto cleanup_index;
	}
	data_len = buf.st_size - SQUAT_SAFETY_ZONE;
	if (data_len < sizeof(SquatDiskHeader)) {
	squat_set_last_error(SQUAT_ERR_INVALID_INDEX_FILE);
	goto cleanup_index;
	}

	index->data = mmap(NULL, data_len, PROT_READ, MAP_SHARED, fd, 0);
	if (index->data == MAP_FAILED) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	goto cleanup_index;
	}

	header = (SquatDiskHeader const*)index->data;
	doc_list_offset = squat_decode_64(header->doc_list_offset);
	word_list_offset = squat_decode_64(header->word_list_offset);
	doc_ID_list_offset = squat_decode_64(header->doc_ID_list_offset);

	/* Do some sanity checking in case the header was corrupted. We wouldn't
	want to dereference any bad pointers... */
	if (memcmp(header->header_text, squat_index_file_header, 8) != 0
	\|\| doc_list_offset < 0 \|\| doc_list_offset >= data_len
	\|\| word_list_offset < 0 \|\| word_list_offset >= data_len
	\|\| doc_ID_list_offset < 0 \|\| doc_ID_list_offset >= data_len
	\|\| !memconst(index->data + data_len, 16, 0)) {
	squat_set_last_error(SQUAT_ERR_INVALID_INDEX_FILE);
	goto cleanup_unmap;
	}

	index->doc_list = index->data + doc_list_offset;
	index->word_list = index->data + word_list_offset;
	index->doc_ID_list = index->data + doc_ID_list_offset;
	index->data_end = index->data + data_len;
	memcpy(index->valid_char_bits, header->valid_char_bits,
	sizeof(index->valid_char_bits));

	return index;

	cleanup_unmap:
	munmap((void*)index->data, data_len + SQUAT_SAFETY_ZONE);

	cleanup_index:
	free(index);
	return NULL;
	}

	int squat_search_list_docs(SquatSearchIndex* index,
	SquatListDocCallback handler, void* closure) {
	char const* s = index->doc_list;

	squat_set_last_error(SQUAT_ERR_OK);

	while (*s != 0) {
	SquatListDoc list_doc;
	int r;

	list_doc.doc_name = s;
	s += strlen(s) + 1;
	list_doc.size = squat_decode_I(&s);
	r = handler(closure, &list_doc);

	if (r == SQUAT_CALLBACK_ABORT) {
	break;
	}
	assert(r == SQUAT_CALLBACK_CONTINUE);
	}

	return SQUAT_OK;
	}

	/* Get a pointer to the index file's list of documents containing the
	word 'data' */
	static char const* lookup_word_docs(SquatSearchIndex* index,
	char const* data, int* invalid_file) {
	int i;
	char const* s = index->word_list;

	for (i = 0; i < SQUAT_WORD_SIZE; i++) {
	char p;
	char ch = data[i];
	char const* branch_start = s;
	int skip;

	/* decode 'present' bits to see if ch is present at this level of
	the tries */
	p = *s++;
	if ((p & 0xE0) != 0) { /* singleton */
	if (ch != p) {
	return NULL;
	}
	skip = 0;
	/* we're done. s is now pointing at the data for the singleton */
	} else { /* list of bits */
	char count;
	char const* base;
	int offset, j;

	if ((unsigned char)ch < 8p) { / before start of list */
	return NULL;
	}

	count = (*s++) + 1;

	if ((unsigned char)ch >= 8(p + count)) { / beyond end of list */
	return NULL;
	}

	offset = (unsigned char)ch/8 - p;
	if ((s[offset] & (1 << (ch & 7))) == 0) { /* not in list */
	return NULL;
	}

	base = s;
	s += count;

	/* figure out how many entries there are before our entry */
	skip = 0;
	for (j = 0; j < offset; j++) {
	skip += bit_counts[(unsigned char)base[j]];
	}
	for (j = 0; j < (ch & 7); j++) {
	if ((base[offset] & (1 << j)) != 0) {
	skip++;
	}
	}
	}

	if (i < SQUAT_WORD_SIZE - 1) {
	int next_offset;

	s = squat_decode_skip_I(s, skip);

	/* find offset to next branch data */
	next_offset = squat_decode_I(&s);
	s = branch_start - next_offset;
	if (next_offset < 0 \|\| s >= index->data_end) {
	*invalid_file = 1;
	return NULL; /* corrupt index */
	}
	} else {
	/* leaf case. We need to scan through the document lists for each
	leaf to skip. */
	while (skip-- > 0) {
	char const* t = s;
	int v = (int)squat_decode_I(&t);

	if ((v & 1) != 0) {
	s = t; /* singleton; no more data to eat for this word */
	} else {
	s = t + (v >> 1); /* run-list; size is in v>>1 */
	}
	}
	}
	/* s now points at the trie branch for the data */
	}

	return s;
	}

	/* Get the pointer to the list of documents containing 'data' into
	'run_start', and return the number of documents in the list. /
	static int count_docs_containing_word(SquatSearchIndex* index,
	char const* data, char const** run_start) {
	int invalid_file = 0;
	char const* raw_doc_list = lookup_word_docs(index, data, &invalid_file);
	int i;

	if (raw_doc_list == NULL) {
	return invalid_file ? -1 : 0;
	}

	*run_start = raw_doc_list;

	i = (int)squat_decode_I(&raw_doc_list);
	if ((i & 1) != 0) {
	return 1; /* singleton */
	} else {
	int size = i >> 1;
	char const* s = raw_doc_list;
	int count = 0;

	if (raw_doc_list + size >= index->data_end) {
	return -1;
	}

	while (s - raw_doc_list < size) {
	i = (int)squat_decode_I(&s);
	if ((i & 1) == 1) {
	count++;
	} else {
	count += i >> 1;
	s = squat_decode_skip_I(s, 1);
	}
	}

	if (raw_doc_list + size != s) {
	return -1;
	}

	return count;
	}
	}

	/* We store a set of documents in this little structure. The set
	also maintains a 'current' document pointer. */
	typedef struct {
	int array_len; /* The length of the array below */
	int* array_data; /* An array of document IDs, sorted by increasing
	document ID. It can also contain elements equal
	to -1, which means "no document".
	*/
	int index; /* The index of the 'current' document within the array. */
	} SquatDocSet;

	/* Extract the list of documents containing the word 'data' into a
	SquatDocSet. The list is extracted from the index file data
	'doc_list' which refers to 'doc_count' documents.
	*/
	static int set_to_docs_containing_word(SquatSearchIndex* index,
	SquatDocSet* set, char const* data, int doc_count, char const* doc_list) {
	int i;

	set->array_len = doc_count;

	set->array_data = (int)malloc(sizeof(int)set->array_len);
	if (set->array_data == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return SQUAT_ERR;
	}

	i = (int)squat_decode_I(&doc_list);
	if ((i & 1) != 0) {
	set->array_data[0] = i >> 1;
	} else {
	int size = i >> 1;
	char const* s = doc_list;
	int last_doc = 0;
	int j = 0;

	while (s - doc_list < size) {
	i = (int)squat_decode_I(&s);
	if ((i & 1) == 1) {
	last_doc = set->array_data[j++] = last_doc + (i >> 1);
	} else {
	int count = i >> 1;
	int delta = squat_decode_I(&s);

	last_doc += delta;
	set->array_data[j++] = last_doc;
	while (--count > 0) {
	last_doc++;
	set->array_data[j++] = last_doc;
	}
	}
	}
	}

	return SQUAT_OK;
	}

	/* Advance the "current document" in the set to the first document
	with ID > 'doc'. Remove any documents found along the way that were
	not 'doc'.
	*/
	static void filter_doc(SquatDocSet* set, int doc) {
	int i = set->index;

	while (i < set->array_len && set->array_data[i] < doc) {
	/* this document is not in the currently filtered set */
	set->array_data[i] = -1;
	i++;
	}

	/* skip over the matched document, if we matched */
	if (i < set->array_len && set->array_data[i] == doc) {
	i++;
	}

	set->index = i;
	}

	/* Remove from a SquatDocSet any documents not in the list of
	documents containing the word 'data'. The list is extracted from
	the index file data 'doc_list'.
	*/
	static void filter_to_docs_containing_word(SquatSearchIndex* index,
	SquatDocSet* set, char const* data, char const* doc_list) {
	int i = (int)squat_decode_I(&doc_list);

	set->index = 0;

	if ((i & 1) != 0) {
	filter_doc(set, i >> 1);
	} else {
	int size = i >> 1;
	char const* s = doc_list;
	int last_doc = 0;

	while (s - doc_list < size) {
	i = (int)squat_decode_I(&s);
	if ((i & 1) == 1) {
	filter_doc(set, last_doc += i >> 1);
	} else {
	int count = i >> 1;
	int delta = squat_decode_I(&s);

	last_doc += delta;
	filter_doc(set, last_doc);
	while (--count > 0) {
	last_doc++;
	filter_doc(set, last_doc);
	}
	}
	}
	}
	}

	/* Advance the "current document" pointer to the first document in the set. */
	static void select_first_doc(SquatDocSet* set) {
	set->index = 0;
	while (set->index < set->array_len && set->array_data[set->index] < 0) {
	set->index++;
	}
	}

	/* Is the "current document" pointer pointing to any real document? */
	static int has_more_docs(SquatDocSet* set) {
	return set->index < set->array_len;
	}

	/* Advance the "current document" pointer to the next document in the set,
	and return its old value */
	static int get_next_doc(SquatDocSet* set) {
	int doc = set->array_data[set->index];

	set->index++;
	while (set->index < set->array_len && set->array_data[set->index] < 0) {
	set->index++;
	}

	return doc;
	}

	static void destroy_docset(SquatDocSet* set) {
	free(set->array_data);
	}

	/* The basic strategy here is pretty simple. We just want to find the
	documents that contain every subword of the search string. The
	index tells us which documents contain each subword so it's just a
	matter of doing O(N) lookups into the index. We construct an
	explicit document list for one of the subwords and then iterate
	through that list for each other subword, throwing out any
	documents that don't contain that subword.

	The only trick is that some subwords may occur in lots of documents
	while others only occur in a few (or no) documents. In that case we
	would rather construct the list with the smallest possible number
	of documents, to save memory and the cost of traversing that list
	several times.
	*/
	int squat_search_execute(SquatSearchIndex* index, char const* data,
	int data_len, SquatSearchResultCallback handler, void* closure) {
	int i;
	int min_doc_count_word; /* The subword of 'data' that appears in
	fewest documents */
	int min_doc_count; /* The number of documents that include that
	subword */
	SquatDocSet set;
	char const** run_starts;

	/* First, do sanity checking on the string. We wouldn't want invalid
	client searches to mysteriously return 'no documents'. */
	if (data_len < SQUAT_WORD_SIZE) {
	squat_set_last_error(SQUAT_ERR_SEARCH_STRING_TOO_SHORT);
	return SQUAT_ERR;
	}

	for (i = 0; i < data_len; i++) {
	int ch = (unsigned char)data[i];

	if ((index->valid_char_bits[ch >> 3] & (1 << (ch & 7))) == 0) {
	squat_set_last_error(SQUAT_ERR_SEARCH_STRING_INVALID_CHAR);
	return SQUAT_ERR;
	}
	}

	/* We search for every subword of the search string. We save a
	pointer to the document list for each subword in this array
	... so we don't have to traverse the trie data structures more
	than once per subword.
	*/
	run_starts = (char const*)malloc(sizeof(char const)*
	(data_len - SQUAT_WORD_SIZE + 1));
	if (run_starts == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return SQUAT_ERR;
	}

	squat_set_last_error(SQUAT_ERR_OK);

	/* Now, for each subword, find its list of documents and how many
	documents are in the list. Remember the word which had minimum
	number of documents.
	*/
	min_doc_count = count_docs_containing_word(index, data, run_starts);
	if (min_doc_count < 0) {
	squat_set_last_error(SQUAT_ERR_INVALID_INDEX_FILE);
	goto cleanup_run_starts;
	} else if (min_doc_count == 0) {
	/* The first word of the substring isn't in any documents, so we
	can just stop now. */
	goto cleanup_run_starts_ok;
	}
	min_doc_count_word = 0;
	for (i = 1; i <= data_len - SQUAT_WORD_SIZE; i++) {
	int doc_count = count_docs_containing_word(index, data + i,
	run_starts + i);
	if (doc_count < 0) {
	squat_set_last_error(SQUAT_ERR_INVALID_INDEX_FILE);
	goto cleanup_run_starts;
	} else if (doc_count == 0) {
	/* This word isn't in any documents, we can stop now. */
	goto cleanup_run_starts_ok;
	} else if (doc_count < min_doc_count) {
	min_doc_count = doc_count;
	min_doc_count_word = i;
	}
	}

	/* Now, extract the shortest document list into an array. By
	starting with the shortest document list we avoid pathological
	situations where one or more of the subwords occurs in zillions
	of documents, and we'd allocate a huge array and have to iterate
	through it all lots of times.
	*/
	if (set_to_docs_containing_word(index, &set, data + min_doc_count_word,
	min_doc_count, run_starts[min_doc_count_word]) == SQUAT_ERR) {
	goto cleanup_run_starts;
	}
	/* Scan through the other document lists and throw out any documents
	that aren't in all those lists. */
	for (i = 0; i <= data_len - SQUAT_WORD_SIZE; i++) {
	if (i != min_doc_count_word) {
	filter_to_docs_containing_word(index, &set, data + i, run_starts[i]);
	}
	}

	/* Now we have the results. Scan through the set and report each
	element to the callback function. */
	select_first_doc(&set);
	while (has_more_docs(&set)) {
	int next_doc;
	char const* next_doc_info;
	char const* next_doc_data;
	int r;

	/* Lookup the document info so we can get the document name to report. */
	next_doc = get_next_doc(&set);
	next_doc_info = index->doc_ID_list + next_doc*4;
	if (next_doc < 0 && next_doc_info >= index->data_end) {
	squat_set_last_error(SQUAT_ERR_INVALID_INDEX_FILE);
	goto cleanup_docset;
	}

	next_doc_data = index->doc_list + squat_decode_32(next_doc_info);
	if (next_doc_data < index->doc_list \|\| next_doc_data >= index->data_end) {
	squat_set_last_error(SQUAT_ERR_INVALID_INDEX_FILE);
	goto cleanup_docset;
	}

	r = handler(closure, next_doc_data);
	if (r == SQUAT_CALLBACK_ABORT) {
	break;
	}
	assert(r == SQUAT_CALLBACK_CONTINUE);
	}

	destroy_docset(&set);

	cleanup_run_starts_ok:
	free(run_starts);
	return SQUAT_OK;

	cleanup_docset:
	destroy_docset(&set);

	cleanup_run_starts:
	free(run_starts);
	return SQUAT_ERR;
	}

	int squat_search_close(SquatSearchIndex* index) {
	int r = SQUAT_OK;

	squat_set_last_error(SQUAT_ERR_OK);

	if (munmap((void*)index->data,
	index->data_end + SQUAT_SAFETY_ZONE - index->data) != 0) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	r = SQUAT_ERR;
	}

	free(index);
	return r;
	}
	diff --git a/imap/squat_build.c b/imap/squat_build.c
	index 2d462e36a..ab7e5b29f 100644
	--- a/imap/squat_build.c
	+++ b/imap/squat_build.c
	@@ -1,1481 +1,1482 @@
	/*
	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	- * $Id: squat_build.c,v 1.1 2001/09/25 16:49:51 ken3 Exp $
	+ * $Id: squat_build.c,v 1.2 2002/02/19 18:50:14 ken3 Exp $
	*/

	/*
	SQUAT code for building indexes.
	Robert O'Callahan

	IMPLEMENTATION NOTES:

	The basic strategy here is pretty simple. During the index build
	process we keep 256 temporary files. Each time we read a source
	document, we add all its words that start with byte i, along with
	the document ID, to file #i. Once we've seen all the source
	documents we proceed through each temporary file #i, one by one,
	constructing a trie of all the words starting with byte i, and which
	stores the IDs of the documents that contain each word. When we get
	to the end of each temporary file, we can write out the trie to the
	index file and start all over again on the next temporary file.

	This is marvellously scalable! During the document reading phase,
	we're just dumping data out into temporary files, and the amount of
	data we dump out is proportional to the total size of the source
	documents. (In the worst case, with large input files of random
	data, we write out 3 bytes per input byte into temporary files.)
	During the trie-building phase, we reread the temporary files and
	output the final index. In this phase we consume a fair bit of
	memory, but in the worst case only 8 bytes per document ID per word
	which starts with the right byte. Even in the very worst case, if
	there were gigabytes of random data, there are only 2^24 possible
	such words, and in practice of course there are far fewer.

	In practice performance is dominated by sequential I/O. On my email,
	I can index half a megabyte of source text per second on a
	single-disk desktop PC.

	The same trie data structures are used to build tries to record the
	words used in a particular document (while the source document is
	being fed in) and to build tries to record the words used in all
	documents that start with a given byte (while we process each
	temporary file).

	Each "per document" trie stores all words occurring in the
	document. We make it a depth 3 trie, and at the leaves we store a
	bit vector recording which words are present in the document, with a
	bit set to 1 if a word occurs with its 4th character set to the
	corresponding byte.

	Each "all document" trie assumes a fixed first word byte, and
	therefore is only of depth 3. The leaves store the list of document
	IDs containing the word.
	*/

	#include <stdlib.h>
	+#include <string.h>
	#include <unistd.h>
	#include <assert.h>
	#include <sys/mman.h>

	#include "squat_internal.h"

	/* A simple write-buffering module which avoids copying of the output data. */

	typedef struct {
	char* buf; /* The malloc'ed buffer, or NULL if there
	isn't one. */
	int buf_size; /* The size of that buffer. */
	int data_len; /* How much data in that buffer is valid. */
	int fd; /* The fd to write to. */
	int total_output_bytes; /* How much data have we written out
	through this buffer in total? */
	} SquatWriteBuffer;

	static int init_write_buffer(SquatWriteBuffer* b, int buf_size, int fd) {
	b->buf_size = buf_size;
	b->buf = malloc(b->buf_size);
	if (b->buf == NULL) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	return SQUAT_ERR;
	}
	b->fd = fd;
	b->data_len = 0;
	b->total_output_bytes = 0;

	return SQUAT_OK;
	}

	/* Make sure that there is enough space in the buffer to write 'len' bytes.
	Return a pointer to where the written data should be placed. */
	static char* prepare_buffered_write(SquatWriteBuffer* b, int len) {
	if (b->data_len + len >= b->buf_size) {
	if (write(b->fd, b->buf, b->data_len) != b->data_len) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	return NULL;
	}
	if (b->buf_size < len) {
	b->buf = (char*)realloc(b->buf, len);
	if (b->buf == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return NULL;
	}
	}
	b->data_len = 0;
	}

	return b->buf + b->data_len;
	}

	/* Signal that data has been written up to the mark 'ptr'.
	Call this after prepare_buffered_write. */
	static void complete_buffered_write(SquatWriteBuffer* b, char* ptr) {
	int old_data_len = b->data_len;

	b->data_len = ptr - b->buf;
	b->total_output_bytes += b->data_len - old_data_len;
	}

	/* Flush the output buffer to the file. Reset the file pointer to the start
	of the file. */
	static int flush_and_reset_buffered_writes(SquatWriteBuffer* b) {
	if (b->data_len > 0) {
	if (write(b->fd, b->buf, b->data_len) != b->data_len) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	return SQUAT_ERR;
	}
	b->data_len = 0;
	}

	if (lseek(b->fd, 0, SEEK_SET) != 0) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	return SQUAT_ERR;
	}

	return SQUAT_OK;
	}

	/* A circular linked list of document IDs, stored in increasing order
	of document ID. */
	typedef struct _WordDocEntry {
	struct _WordDocEntry* next;
	int doc_ID;
	} WordDocEntry;

	/* These form the leaves of the "all documents" tries. For each of the
	256 words with trailing byte 'i', docs[i] is NULL if the word does
	not occur in any document, otherwise it is the head of a linked
	list of document IDs for the documents which contain the word. */
	typedef struct {
	short first_valid_entry; /* We record the first and last valid
	entries in the array below. These could
	be computed by just scanning the array,
	but it turns out that in practice such
	array scanning dominates the CPU
	consumption of the indexer. We get
	major speedup by maintaining these
	entries on the fly. */
	short last_valid_entry;
	WordDocEntry* docs[256]; /* Pointers to the document ID lists for
	each of the 256 words rooted at this
	part of the trie. Each non-NULL pointer
	points to the LAST element of the
	linked list (i.e. the entry with the
	highest document ID). This means we can
	efficiently add to the end of the
	linked list, and also efficiently get
	to the start of the linked list (the
	element with lowest document ID)
	(because it's circular). */
	} SquatWordTableLeafDocs;

	/* These form the leaves of the "per document" tries. For each of the
	256 words with trailing byte 'i', presence[i >> 3] & (1 << (i & 7))
	is 1 if the word occurs in the document, otherwise 0. */
	typedef struct {
	short first_valid_entry; /* We record the first and last valid
	entries in the bit vector below. These
	could be computed by just scanning the
	array, but we get significant speedup
	by maintaining them here. */
	short last_valid_entry;
	char presence[32];
	} SquatWordTableLeafPresence;

	/* This is an entry in a trie. */
	typedef union _SquatWordTableEntry {
	struct _SquatWordTable* table; /* This is a branch node */

	/* These variants are used for leaves of "per document" tries.
	They are distinguished by the value of the low bit. */
	SquatWordTableLeafPresence* leaf_presence; /* low bit is 0 */
	int leaf_presence_singleton; /* low bit is 1 */

	/* This variant is used for leaves of "all document" tries. */
	SquatWordTableLeafDocs* leaf_docs;
	} SquatWordTableEntry;

	/* This is a trie branch node. */
	typedef struct _SquatWordTable {
	short first_valid_entry; /* We record the first and last valid
	entries in the array below, as in the
	above data structures. */
	short last_valid_entry;
	SquatWordTableEntry entries[256];
	} SquatWordTable;

	struct _SquatIndex {
	char* tmp_path; /* Saved tmp_path option, with
	the temporary filename
	pattern appended */
	SquatWriteBuffer out; /* The buffer for the index file itself */
	char* doc_ID_list; /* A buffer where we hold the
	encoded array that maps from
	a document ID to the offset
	of the document record within
	the index file. */
	int doc_ID_list_size; /* The allocated size of the
	above buffer, measured in
	multiples of
	sizeof(SquatInt32) (i.e., 4) */
	int current_doc_ID; /* The current document
	ID. Document IDs are numbered
	starting at zero and
	incremented by 1 every time
	we finish processing a source
	document. */
	int current_doc_len; /* The total number of bytes
	processed in the current
	source document. */
	SquatWordTable doc_word_table; /* The root of the trie being
	built for the current
	document or for the current
	initial byte. */
	char runover_buf[SQUAT_WORD_SIZE]; /* holds the last runover_len
	bytes of the current source
	document */
	int runover_len;
	WordDocEntry* word_doc_allocator; /* A preallocated buffer of
	WordDocEntries; this pointer
	is bumped up one every
	allocation */
	unsigned char valid_char_bits[32]; /* Saved valid_char_bits option */
	SquatStatsCallback stats_callback; /* Saved stats_callback option */
	void* stats_callback_closure;

	/* put the big structures at the end */

	SquatWriteBuffer index_buffers[256]; /* Buffers for the temporary
	files, one for each first
	byte of words occurring in
	the source documents */
	int total_num_words[256]; /* total number of words starting with
	given char */
	int doc_words[256]; /* number of words in current document
	starting with given char */
	};

	/* Initally, before we see a document, there are no words for the document. */
	static void init_doc_word_table(SquatWordTable* t) {
	t->first_valid_entry = 256;
	t->last_valid_entry = 0;
	memset(t->entries, 0, sizeof(t->entries));
	}

	SquatIndex* squat_index_init(int fd, SquatOptions const* options) {
	SquatIndex* index;
	int i;
	int path_len;
	char* buf;
	char const* tmp_path;

	squat_set_last_error(SQUAT_ERR_OK);

	index = (SquatIndex*)malloc(sizeof(SquatIndex));
	if (index == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return NULL;
	}

	/* Copy processed options into the SquatIndex */
	if (options != NULL && (options->option_mask & SQUAT_OPTION_TMP_PATH) != 0) {
	tmp_path = options->tmp_path;
	} else {
	tmp_path = "/tmp";
	}
	path_len = strlen(tmp_path);
	index->tmp_path = malloc(path_len + 1 + 12);
	if (index->tmp_path == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	goto cleanup_index;
	}
	memcpy(index->tmp_path, tmp_path, path_len);
	strcpy(index->tmp_path + path_len, "/squatXXXXXX");

	if (options != NULL &&
	(options->option_mask & SQUAT_OPTION_VALID_CHARS) != 0) {
	int i;

	memset(index->valid_char_bits, 0, sizeof(index->valid_char_bits));
	for (i = 0; options->valid_chars[i] != 0; i++) {
	int ch = (unsigned char)options->valid_chars[i];

	index->valid_char_bits[ch >> 3] \|= 1 << (ch & 7);
	}
	} else {
	memset(index->valid_char_bits, 255, sizeof(index->valid_char_bits));
	}

	if (options != NULL &&
	(options->option_mask & SQUAT_OPTION_STATISTICS) != 0) {
	index->stats_callback = options->stats_callback;
	index->stats_callback_closure = options->stats_callback_closure;
	} else {
	index->stats_callback = NULL;
	}

	/* Finish initializing the SquatIndex */
	for (i = 0; i < 256; i++) {
	index->index_buffers[i].buf = NULL;
	}

	index->doc_ID_list_size = 1000;
	index->doc_ID_list = (char)malloc(index->doc_ID_list_sizesizeof(SquatInt32));
	if (index->doc_ID_list == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	goto cleanup_tmp_path;
	}

	/* Use a 128K write buffer for the main index file */
	if (init_write_buffer(&index->out, 128*1024, fd) != SQUAT_OK) {
	goto cleanup_doc_ID_list;
	}

	/* Write out a dummy header. This will be replaced by the real header at the
	end of the process. */
	buf = prepare_buffered_write(&index->out, sizeof(SquatDiskHeader));
	if (buf == NULL) {
	goto cleanup_out_buffer;
	}
	memset(buf, 0, sizeof(SquatDiskHeader));
	complete_buffered_write(&index->out, buf + sizeof(SquatDiskHeader));

	index->current_doc_ID = 0;
	init_doc_word_table(&index->doc_word_table);

	memset(index->total_num_words, 0, sizeof(index->total_num_words));

	return index;

	cleanup_out_buffer:
	free(index->out.buf);

	cleanup_doc_ID_list:
	free(index->doc_ID_list);

	cleanup_tmp_path:
	free(index->tmp_path);

	cleanup_index:
	free(index);
	return NULL;
	}

	/* Initialize a write buffer for a temporary file. We generate the
	temporary file name here. The file is unlinked right away so if we
	crash, the temporary file doesn't need to be cleaned up. */
	static int init_write_buffer_to_temp(SquatIndex* index, SquatWriteBuffer* b) {
	int fd = mkstemp(index->tmp_path);

	if (fd < 0) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	return SQUAT_ERR;
	}

	if (unlink(index->tmp_path) < 0) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	goto cleanup_fd;
	}

	strcpy(index->tmp_path + strlen(index->tmp_path) - 6, "XXXXXX");

	if (init_write_buffer(b, 64*1024, fd) != SQUAT_OK) {
	goto cleanup_fd;
	}

	return SQUAT_OK;

	cleanup_fd:
	close(b->fd);
	return SQUAT_ERR;
	}

	int squat_index_open_document(SquatIndex* index, char const* name) {
	int name_len;
	char* buf;

	squat_set_last_error(SQUAT_ERR_OK);

	/* Grow the document ID array as necessary */
	if (index->current_doc_ID >= index->doc_ID_list_size) {
	index->doc_ID_list_size *= 2;
	index->doc_ID_list =
	(char)realloc(index->doc_ID_list, index->doc_ID_list_sizesizeof(SquatInt32));

	if (index->doc_ID_list == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return SQUAT_ERR;
	}
	}

	/* Store the offset of the new document record into the array */
	squat_encode_32(index->doc_ID_list + index->current_doc_ID*4,
	index->out.total_output_bytes - sizeof(SquatDiskHeader));

	/* Now write the new document name out to the file. Later we will
	write the document length right after this. Nobody writes to the
	file in the interim. */
	name_len = strlen(name) + 1;
	if ((buf = prepare_buffered_write(&index->out, name_len)) == NULL) {
	return SQUAT_ERR;
	}
	strcpy(buf, name);
	complete_buffered_write(&index->out, buf + name_len);

	index->current_doc_len = 0;
	index->runover_len = 0;
	memset(index->doc_words, 0, sizeof(index->doc_words));

	return SQUAT_OK;
	}

	/* Destroy the SquatWordTable. The leaf data and the internal nodes are free'd. */
	static void delete_doc_word_table(SquatWordTable* t, int depth) {
	if (depth > 2) {
	int i;

	depth--;
	for (i = 0; i < 256; i++) {
	SquatWordTableEntry* e = t->entries + i;

	if (e->table != NULL) {
	delete_doc_word_table(e->table, depth);
	}
	}
	} else {
	int i;

	/* this happens to work whether the leaf entries are leaf_presence
	or leaf_docs. This is ugly but acceptable :-) */
	for (i = 0; i < 256; i++) {
	SquatWordTableEntry* e = t->entries + i;

	if (e->leaf_presence != NULL && ((int)e->leaf_presence & 1) == 0) {
	free(e->leaf_presence);
	}
	}
	}
	free(t);
	}

	#define SQUAT_ADD_NEW_WORD (SQUAT_LAST_BUILTIN + 1)

	/* Add an entry to the compressed presence set. We maintain
	first_valid_entry and last_valid_entry.
	This is faster than scanning to compute them later.
	We return SQUAT_ADD_NEW_WORD if the bit wasn't already set. */
	static int set_presence_bit(SquatWordTableLeafPresence* p, int ch) {
	int mask = 1 << (ch & 7);
	char* ptr = p->presence + (ch >> 3);

	if (ch < p->first_valid_entry) {
	p->first_valid_entry = ch;
	}
	if (ch > p->last_valid_entry) {
	p->last_valid_entry = ch;
	}

	if ((*ptr & mask) == 0) {
	*ptr \|= mask;
	return SQUAT_ADD_NEW_WORD;
	} else {
	return SQUAT_OK;
	}
	}

	/* Add a word to the SquatWordTable trie.
	If word_entry is NULL then we are in "per document" mode and just record
	the presence or absence of a word, not the actual document.
	We return SQUAT_ADD_NEW_WORD if this is the first occurrence of the
	word in the trie. */
	static int add_to_table(SquatIndex* index, char const* data, int data_len,
	WordDocEntry* word_entry) {
	SquatWordTable* t = &index->doc_word_table;
	int ch;
	SquatWordTableEntry* e;

	while (data_len > 2) {
	/* Follow the branch node down to the next level of the trie. */
	ch = (unsigned char)data[0];
	/* Maintain the valid_entry variables so that we don't have to
	perform expensive scans of the 256-element arrays
	later. Surprisingly, this optimization really matters! */
	if (ch < t->first_valid_entry) {
	t->first_valid_entry = ch;
	}
	if (ch > t->last_valid_entry) {
	t->last_valid_entry = ch;
	}

	e = t->entries + ch;
	t = e->table;
	/* Allocate the next branch node if it doesn't already exist. */
	if (t == NULL) {
	t = (SquatWordTable*)malloc(sizeof(SquatWordTable));
	if (t == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return SQUAT_ERR;
	}
	e->table = t;
	/* Initially there are no valid entries. Set things up so that
	the obvious tests will set first_valid_entry and
	last_valid_entry correctly. */
	t->first_valid_entry = 256;
	t->last_valid_entry = 0;
	memset(t->entries, 0, sizeof(t->entries));
	}

	data++;
	data_len--;
	}

	/* Follow the branch node down to the leaf level */
	ch = (unsigned char)data[0];
	if (ch < t->first_valid_entry) {
	t->first_valid_entry = ch;
	}
	if (ch > t->last_valid_entry) {
	t->last_valid_entry = ch;
	}
	e = t->entries + ch;

	ch = (unsigned char)data[1];

	if (word_entry == NULL) {
	/* We are in "per document" mode. */
	if (((int)e->leaf_presence & 1) != 0) {
	/* We currently have a singleton here. */
	int oldch = e->leaf_presence_singleton >> 1;

	/* If the singleton indicates the same word as the current word,
	then we don't have to do anything. */
	if (oldch != ch) {
	/* Otherwise we have to add the new word. This means we have
	to convert the singleton to a bit vector. */
	SquatWordTableLeafPresence* p;

	/* Make an empty bit vector. */
	p = (SquatWordTableLeafPresence*)
	malloc(sizeof(SquatWordTableLeafPresence));
	if (p == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return SQUAT_ERR;
	}
	p->first_valid_entry = 256;
	p->last_valid_entry = 0;
	memset(p->presence, 0, sizeof(p->presence));
	e->leaf_presence = p;

	/* Update the bit vector */
	set_presence_bit(p, ch);
	return set_presence_bit(p, oldch); /* will always be SQUAT_ADD_NEW_WORD */
	}
	} else if (e->leaf_presence == NULL) {
	/* There's nothing here. Let's make a singleton. */
	/* this next step might be necessary if sizeof(void*) >
	sizeof(int). We make sure that the low bit of the pointer in
	leaf_presence is definitely 1. */
	e->leaf_presence = (void*)1;
	e->leaf_presence_singleton = (ch << 1) \| 1;
	return SQUAT_ADD_NEW_WORD;
	} else {
	/* We already have the bit vector, so let's just set another bit in it. */
	return set_presence_bit(e->leaf_presence, ch);
	}
	} else {
	/* We are in "all documents" mode. */
	SquatWordTableLeafDocs* docs = e->leaf_docs;
	WordDocEntry** entry_ptr;

	/* Make a new leaf table if we don't already have one. */
	if (docs == NULL) {
	docs = (SquatWordTableLeafDocs*)
	malloc(sizeof(SquatWordTableLeafDocs));
	if (docs == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return SQUAT_ERR;
	}
	docs->first_valid_entry = 256;
	docs->last_valid_entry = 0;
	memset(docs->docs, 0, sizeof(docs->docs));
	e->leaf_docs = docs;
	}

	entry_ptr = docs->docs + ch;

	if (*entry_ptr == NULL) {
	/* Adding a new word, so may need to update the valid_entry markers */
	if (ch < docs->first_valid_entry) {
	docs->first_valid_entry = ch;
	}
	if (ch > docs->last_valid_entry) {
	docs->last_valid_entry = ch;
	}
	/* Create the linked list with the single element 'word_entry'. */
	word_entry->next = word_entry; /* make it circular */
	*entry_ptr = word_entry;
	return SQUAT_ADD_NEW_WORD;
	} else {
	/* Just add the document to the linked list. word_entry will be
	the new last element since the document IDs are strictly
	increasing as we build the trie from its temporary file. */
	word_entry->next = (entry_ptr)->next; / (*entry_ptr)->next is
	(still) the first
	element of the list */
	(entry_ptr)->next = word_entry; / the old last element's
	next now points to the
	new last element. */
	entry_ptr = word_entry; / save the new last element */
	}
	}

	return SQUAT_OK;
	}

	/* Add 'doc_ID' to the list of document IDs for word 'word_ptr'
	in the "all documents" trie. */
	static int add_word_to_trie(SquatIndex* index, char const* word_ptr,
	int doc_ID) {
	WordDocEntry* word_entry = index->word_doc_allocator++;

	word_entry->doc_ID = doc_ID;
	add_to_table(index, word_ptr, SQUAT_WORD_SIZE - 1, word_entry);

	return SQUAT_OK;
	}

	/* Add the word 'data' to the "per document" trie for the current document. */
	static int add_word_to_table(SquatIndex* index, char const* data) {
	int r;
	int i;

	/* Just ignore the word if it uses an invalid character. */
	for (i = 0; i < SQUAT_WORD_SIZE; i++) {
	int ch = (unsigned char)data[i];

	if ((index->valid_char_bits[ch >> 3] & (1 << (ch & 7))) == 0) {
	/* this word contains an invalid character and need not be indexed,
	since search strings will never contain such a character. */
	return SQUAT_OK;
	}
	}

	r = add_to_table(index, data, SQUAT_WORD_SIZE, NULL);
	if (r == SQUAT_ADD_NEW_WORD) {
	/* Remember how many unique words in this document started with
	the given first character. */
	index->doc_words[(unsigned char)data[0]]++;
	return SQUAT_OK;
	} else {
	return r;
	}
	}

	int squat_index_append_document(SquatIndex* index, char const* data,
	int data_len) {
	int i;
	char buf[SQUAT_WORD_SIZE];
	int new_runover;
	int new_runover_data;

	assert(data_len >= 0);

	squat_set_last_error(SQUAT_ERR_OK);

	if (data_len == 0) {
	return SQUAT_OK;
	}

	/* Scan runover */
	for (i = 0; i < index->runover_len; i++) {
	/* Check if we can make a whole word starting with runover bytes
	from offset i within the runover buffer and with the remaining
	bytes taken from the new text */
	if (index->runover_len - i + data_len >= SQUAT_WORD_SIZE) {
	/* Yep. Build the complete word into 'buf' and then add it. */
	memcpy(buf, index->runover_buf + i, index->runover_len - i);
	memcpy(buf + index->runover_len - i, data,
	SQUAT_WORD_SIZE - (index->runover_len - i));
	if (add_word_to_table(index, buf) != SQUAT_OK) {
	return SQUAT_ERR;
	}
	}
	}

	/* Scan main text */
	for (i = 0; i <= data_len - SQUAT_WORD_SIZE; i++) {
	if (add_word_to_table(index, data + i) != SQUAT_OK) {
	return SQUAT_ERR;
	}
	}

	/* Fill runover. We have to be careful to handle all the cases,
	particularly we just saw less than SQUAT_WORD_SIZE bytes and we
	need to copy some data from the old runover buffer into the new
	runover buffer. */
	new_runover = index->runover_len + data_len;
	if (new_runover > SQUAT_WORD_SIZE) {
	new_runover = SQUAT_WORD_SIZE;
	}
	new_runover_data = data_len;
	if (new_runover_data > new_runover) {
	new_runover_data = new_runover;
	}
	/* Copy data from the old runover buffer into its new position in
	the new runover buffer */
	memcpy(index->runover_buf,
	index->runover_buf + index->runover_len -
	(new_runover - new_runover_data),
	new_runover - new_runover_data);
	/* Copy data from the new text into the new runover buffer */
	memcpy(index->runover_buf + new_runover - new_runover_data,
	data + data_len - new_runover_data, new_runover_data);
	index->runover_len = new_runover;

	/* Tracking how much data we've seen for this document in total */
	index->current_doc_len += data_len;

	return SQUAT_OK;
	}

	/* Write the word to the given temporary file. Since each temporary
	file is dedicated to a given initial byte, the word passed to us
	has the initial byte removed. */
	static int output_word(SquatWriteBuffer* b, char const* word) {
	char* buf = prepare_buffered_write(b, SQUAT_WORD_SIZE - 1);

	if (buf == NULL) {
	return SQUAT_ERR;
	}
	memcpy(buf, word, SQUAT_WORD_SIZE - 1);
	complete_buffered_write(b, buf + SQUAT_WORD_SIZE - 1);

	return SQUAT_OK;
	}

	/* Write the word data from the trie 't' into the temporary file
	accessed through 'b'. Words to write are assembled starting at
	'word'; we assume that 'len' bytes have already been assembled
	leading up to 'word'. This function clears the word data after
	writing it out. This makes it ready to handle the next document
	without reallocating everything. */
	static int write_words(SquatIndex* index, SquatWriteBuffer* b,
	SquatWordTable* t, int len, char* word) {
	if (len == 2) {
	/* Handle a branch node that refers to leaves. */
	int i;

	for (i = t->first_valid_entry; i <= t->last_valid_entry; i++) {
	SquatWordTableEntry* e = t->entries + i;

	word[0] = (char)i;

	if (((int)e->leaf_presence & 1) != 0) {
	/* Got a singleton at this branch point. Just output the single word. */
	word[1] = (char)(e->leaf_presence_singleton >> 1);
	e->leaf_presence = NULL; /* clear the leaf out */
	if (output_word(b, word - (SQUAT_WORD_SIZE - 3)) != SQUAT_OK) {
	return SQUAT_ERR;
	}
	} else if (e->leaf_presence != NULL) {
	/* Got a bit vector array which we have to scan. */
	/* The following code is performance critical. It can dominate
	the performance of the entire indexer. That's why we need
	the valid_entry fields! */
	SquatWordTableLeafPresence* p = e->leaf_presence;
	int i;
	int last_byte = p->last_valid_entry >> 3;

	for (i = p->first_valid_entry >> 3; i <= last_byte; i++) {
	int bits = (unsigned char)p->presence[i];
	int j;

	for (j = 0; bits > 0; j++, bits >>= 1) {
	if ((bits & 1) != 0) {
	/* Output a word for each bit that is set */
	word[1] = (char)(i*8 + j);
	if (output_word(b, word - (SQUAT_WORD_SIZE - 3)) != SQUAT_OK) {
	return SQUAT_ERR;
	}
	}
	}
	}
	free(p);
	e->leaf_presence = NULL;
	}
	}
	} else {
	/* Handle an interior branch node. A simple matter of recursion. */
	int i;

	for (i = t->first_valid_entry; i <= t->last_valid_entry; i++) {
	SquatWordTable* new_t = t->entries[i].table;

	if (new_t != NULL) {
	word[0] = (char)i;
	if (write_words(index, b, new_t, len - 1, word + 1)
	!= SQUAT_OK) {
	return SQUAT_ERR;
	}
	}
	}
	}

	/* This effectively clears the array because we trust these entries. */
	t->first_valid_entry = 256;
	t->last_valid_entry = 0;

	return SQUAT_OK;
	}

	int squat_index_close_document(SquatIndex* index) {
	char* buf;
	int i;

	squat_set_last_error(SQUAT_ERR_OK);

	/* Write out the length of the current document to the index file,
	just after the document's name. */
	if ((buf = prepare_buffered_write(&index->out, 10)) == NULL) {
	return SQUAT_ERR;
	}
	buf = squat_encode_I(buf, index->current_doc_len);
	complete_buffered_write(&index->out, buf);

	if (index->stats_callback != NULL) {
	SquatStatsEvent event;

	event.generic.type = SQUAT_STATS_COMPLETED_DOC;
	event.completed_doc.num_unique_words = index->doc_words;
	index->stats_callback(index->stats_callback_closure, &event);
	}

	/* For each byte that started a word in the source document, we need
	to dump all the words that occurred starting with that byte to
	the corresponding temporary file. */
	for (i = 0; i < 256; i++) {
	if (index->doc_words[i] > 0) {
	char* write_ptr;
	char word_buf[SQUAT_WORD_SIZE - 1];
	int cur_offset;

	if (index->index_buffers[i].buf == NULL) {
	/* This is the first document that used a word starting with this byte.
	We need to create the temporary file. */
	if (init_write_buffer_to_temp(index, index->index_buffers + i)
	!= SQUAT_OK) {
	return SQUAT_ERR;
	}
	}

	index->total_num_words[i] += index->doc_words[i];

	/* Write out the document ID and the number of words in this
	document that start with the initial byte. Then we write out
	the list of words themselves, SQUAT_WORD_SIZE-1 bytes
	each. Very simple format for the temporary files. We could
	compress them more but why bother? */
	write_ptr = prepare_buffered_write(index->index_buffers + i, 20);
	if (write_ptr == NULL) {
	return SQUAT_ERR;
	}
	write_ptr = squat_encode_I(write_ptr, index->current_doc_ID);
	write_ptr = squat_encode_I(write_ptr, index->doc_words[i]);
	complete_buffered_write(index->index_buffers + i, write_ptr);

	cur_offset = index->index_buffers[i].total_output_bytes;
	if (write_words(index, index->index_buffers + i,
	index->doc_word_table.entries[i].table,
	SQUAT_WORD_SIZE - 1, word_buf)
	!= SQUAT_OK) {
	return SQUAT_ERR;
	}
	/* Make sure that we actually output the exact number of words
	we thought we added to the trie. It's really easy to break
	this invariant with bugs in the above code! */
	assert(index->index_buffers[i].total_output_bytes - cur_offset
	== (SQUAT_WORD_SIZE - 1)*index->doc_words[i]);
	}
	}

	index->current_doc_len = -1;

	index->current_doc_ID++;

	return SQUAT_OK;
	}

	/* Dump out a branch node of an "all documents" trie to the index
	file. It's dumped as a presence table (telling us which branches
	are non-NULL) followed by a list of relative file offsets in
	I-format pointing to the subtries for the non-NULL branches. */
	static int dump_word_table_offsets(SquatIndex* index, SquatWordTable* t,
	int *offset_buf) {
	int start_present = t->first_valid_entry;
	int end_present = t->last_valid_entry;
	char* buf;
	int present_count; /* We store here the actual number of present branches */

	if (start_present > end_present) {
	/* There are no non-empty branches so just write an empty presence table */
	if ((buf = prepare_buffered_write(&index->out, 2)) == NULL) {
	return SQUAT_ERR;
	} else {
	buf[0] = buf[1] = 0;
	complete_buffered_write(&index->out, buf + 2);
	return SQUAT_OK;
	}
	}

	/* If there is just one valid entry but its index is < 32, then we
	can't use the one-byte representation for a singleton presence
	because it would be mistaken for the first byte of a (count,
	start) presence vector header. A singleton whose index is >= 32
	can be written out without ambiguity. */
	if (end_present == start_present && end_present >= 32) {
	if ((buf = prepare_buffered_write(&index->out, 1)) == NULL) {
	return SQUAT_ERR;
	} else {
	*buf++ = (char)end_present;
	present_count = 1;
	}
	} else {
	/* We're going to use the presence bit vector format. */
	int first_byte = start_present >> 3;
	int byte_count = (end_present >> 3) - first_byte + 1;

	if ((buf = prepare_buffered_write(&index->out, 2 + byte_count)) == NULL) {
	return SQUAT_ERR;
	} else {
	int i;

	*buf++ = (char)first_byte;
	buf++ = (char)byte_count - 1; / subtract 1 to avoid ambiguity
	over the value '32' (we
	wouldn't use 0 anyway) */
	/* Clear the vector */
	memset(buf, 0, byte_count);
	present_count = 0;
	for (i = start_present; i <= end_present; i++) {
	if (offset_buf[i] > 0) {
	present_count++;
	/* Set the bit in the vector. */
	buf[(i >> 3) - first_byte] \|= 1 << (i & 7);
	}
	}
	buf += byte_count;
	}
	}
	complete_buffered_write(&index->out, buf);

	/* Now we write out the actual offset table in I-format. */
	if ((buf = prepare_buffered_write(&index->out, 10*present_count)) == NULL) {
	return SQUAT_ERR;
	} else {
	int i;

	for (i = start_present; i <= end_present; i++) {
	int off = offset_buf[i];

	if (off > 0) {
	buf = squat_encode_I(buf, off);
	}
	}
	}
	complete_buffered_write(&index->out, buf);

	return SQUAT_OK;
	}

	/* Write out the presence table for an "all documents" trie leaf. */
	static int dump_doc_list_present_bits(SquatIndex* index,
	SquatWordTableLeafDocs* docs) {
	int start_present = docs->first_valid_entry;
	int end_present = docs->last_valid_entry;
	char* buf;
	int present_count;

	/* If the leaf is empty, we should never get here! */
	assert(start_present <= end_present);

	/* if it's a singleton < 32, then we can't use the one-byte
	representation because it would be mistaken for a starting byte */
	if (end_present == start_present && end_present >= 32) {
	if ((buf = prepare_buffered_write(&index->out, 1)) == NULL) {
	return SQUAT_ERR;
	} else {
	*buf++ = (char)end_present;
	present_count = 1;
	}
	} else {
	int first_byte = start_present >> 3;
	int byte_count = (end_present >> 3) - first_byte + 1;

	if ((buf = prepare_buffered_write(&index->out, 2 + byte_count)) == NULL) {
	return SQUAT_ERR;
	} else {
	int i;

	*buf++ = (char)first_byte;
	*buf++ = (char)byte_count - 1;
	memset(buf, 0, byte_count);
	present_count = 0;
	for (i = start_present; i <= end_present; i++) {
	if (docs->docs[i] != NULL) {
	present_count++;
	buf[(i >> 3) - first_byte] \|= 1 << (i & 7);
	}
	}
	buf += byte_count;
	}
	}
	complete_buffered_write(&index->out, buf);

	return SQUAT_OK;
	}

	/* Write out the document lists for an "all documents" trie leaf. */
	static int dump_doc_list_docs(SquatIndex* index,
	SquatWordTableLeafDocs* docs) {
	int i;
	WordDocEntry** doc_list = docs->docs;

	for (i = docs->first_valid_entry; i <= docs->last_valid_entry; i++) {
	if (doc_list[i] != NULL) {
	WordDocEntry* first_doc;
	WordDocEntry* doc;
	int run_size = 0; /* Bytes required to store the doclist for this word */
	int last_doc_ID;
	int run_seq_delta = 0;
	int run_seq_count;
	int doc_count = 0; /* number of documents containing this word */
	char* buf;

	doc = first_doc = doc_list[i]->next;

	last_doc_ID = 0;
	run_seq_count = 0;
	/* First compute the run_size bytes required to store the doclist */
	do {
	if (doc->doc_ID == last_doc_ID + 1 && run_seq_count > 0) {
	run_seq_count++;
	} else {
	if (run_seq_count > 0) {
	if (run_seq_count > 1) {
	run_size += squat_count_encode_I(run_seq_count << 1)
	+ squat_count_encode_I(run_seq_delta);
	} else {
	run_size += squat_count_encode_I((run_seq_delta << 1) \| 1);
	}
	}
	run_seq_count = 1;
	run_seq_delta = doc->doc_ID - last_doc_ID;
	}
	last_doc_ID = doc->doc_ID;
	doc = doc->next;
	doc_count++;
	} while (doc != first_doc);
	if (run_seq_count > 0) {
	if (run_seq_count > 1) {
	run_size += squat_count_encode_I(run_seq_count << 1)
	+ squat_count_encode_I(run_seq_delta);
	} else {
	run_size += squat_count_encode_I((run_seq_delta << 1) \| 1);
	}
	}

	/* reserve more than enough space in the buffer */
	if ((buf = prepare_buffered_write(&index->out, 10 + run_size))
	== NULL) {
	return SQUAT_ERR;
	}

	/* If there's only one document, use singleton document format */
	if (doc_count == 1) {
	buf = squat_encode_I(buf, (doc->doc_ID << 1) \| 1);
	} else {
	/* Store the entire document list, with its size first. */
	buf = squat_encode_I(buf, run_size << 1);

	last_doc_ID = 0;
	run_seq_count = 0;
	/* This logic should mirror the logic above that counts the bytes. */
	do {
	if (doc->doc_ID == last_doc_ID + 1 && run_seq_count > 0) {
	run_seq_count++;
	} else {
	if (run_seq_count > 0) {
	if (run_seq_count > 1) {
	buf = squat_encode_I(buf, run_seq_count << 1);
	buf = squat_encode_I(buf, run_seq_delta);
	} else {
	buf = squat_encode_I(buf, (run_seq_delta << 1) \| 1);
	}
	}
	run_seq_count = 1;
	run_seq_delta = doc->doc_ID - last_doc_ID;
	}
	last_doc_ID = doc->doc_ID;
	doc = doc->next;
	} while (doc != first_doc);
	if (run_seq_count > 0) {
	if (run_seq_count > 1) {
	buf = squat_encode_I(buf, run_seq_count << 1);
	buf = squat_encode_I(buf, run_seq_delta);
	} else {
	buf = squat_encode_I(buf, (run_seq_delta << 1) \| 1);
	}
	}
	}

	complete_buffered_write(&index->out, buf);
	}
	}

	return SQUAT_OK;
	}

	/* Write an "all documents" subtrie to the index file.
	'result_offset' is an absolute offset within the file where this
	subtrie was stored. We free the trie leaves as we go. */
	static int write_trie_word_data(SquatIndex* index, SquatWordTable* t, int len,
	int* result_offset) {
	int i;
	int offsets[256]; /* Collect the offsets of the subtries in this array. */
	int off;
	SquatWordTableEntry* entries = t->entries;
	int r;

	memset(offsets, 0, t->first_valid_entry*sizeof(int));
	if (len > 2) {
	/* interior branch */
	for (i = t->first_valid_entry; i <= t->last_valid_entry; i++) {
	SquatWordTable* new_t = entries[i].table;

	if (new_t != NULL) {
	if (write_trie_word_data(index, new_t, len - 1, offsets + i)
	!= SQUAT_OK) {
	return SQUAT_ERR;
	}
	} else {
	offsets[i] = 0;
	}
	}
	} else {
	/* Leaf case */
	for (i = t->first_valid_entry; i <= t->last_valid_entry; i++) {
	SquatWordTableLeafDocs* leaf_docs = entries[i].leaf_docs;

	if (leaf_docs != NULL) {
	offsets[i] = index->out.total_output_bytes;

	if (dump_doc_list_present_bits(index, leaf_docs) != SQUAT_OK
	\|\| dump_doc_list_docs(index, leaf_docs) != SQUAT_OK) {
	return SQUAT_ERR;
	}
	free(entries[i].leaf_docs);
	entries[i].leaf_docs = NULL;
	} else {
	offsets[i] = 0;
	}
	}
	}
	memset(offsets + i, 0, (256 - i)*sizeof(int));

	/* Now we've written out our subtries, we know where our branch
	table is going to be. */
	*result_offset = off = index->out.total_output_bytes;

	/* Relativize the offsets. This is just to reduce the probable
	magnitude of the numbers so they will pack better into I-format. */
	for (i = t->first_valid_entry; i <= t->last_valid_entry; i++) {
	if (offsets[i] != 0) {
	offsets[i] = off - offsets[i];
	}
	}

	r = dump_word_table_offsets(index, t, offsets);

	/* Mark this subtrie as empty. */
	t->first_valid_entry = 256;
	t->last_valid_entry = 0;

	return r;
	}

	/* Dump out a complete trie for the given initial byte from its temporary file.
	The absolute offset of the trie's root table within the file is
	returned in 'result_offset'. */
	static int dump_index_trie_words(SquatIndex* index, int first_char,
	int* result_offset) {
	SquatWriteBuffer* buf = index->index_buffers + first_char;
	int num_words = index->total_num_words[first_char];
	WordDocEntry* doc_table;
	char const* word_list_ptr;
	int r = SQUAT_OK;
	char const* word_ptr;

	/* Allocate all the necessary document-ID linked list entries at once. */
	doc_table = (WordDocEntry)malloc(sizeof(WordDocEntry)num_words);
	if (doc_table == NULL) {
	squat_set_last_error(SQUAT_ERR_OUT_OF_MEMORY);
	return SQUAT_ERR;
	}
	index->word_doc_allocator = doc_table;

	/* mmap the temporary file. */
	word_list_ptr = mmap(NULL, buf->total_output_bytes, PROT_READ, MAP_SHARED,
	buf->fd, 0);
	if (word_list_ptr == MAP_FAILED) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	r = SQUAT_ERR;
	goto cleanup;
	}
	word_ptr = word_list_ptr;

	/* Scan through the file */
	while (num_words > 0) {
	/* For each document, add all its words to the trie with this document ID */
	int doc_ID = (int)squat_decode_I(&word_ptr);
	int doc_words = (int)squat_decode_I(&word_ptr);

	num_words -= doc_words;

	while (doc_words > 0) {
	if (add_word_to_trie(index, word_ptr, doc_ID)
	!= SQUAT_OK) {
	r = SQUAT_ERR;
	goto cleanup_map;
	}
	word_ptr += SQUAT_WORD_SIZE - 1;
	doc_words--;
	}
	}

	/* Make sure we used exactly as many linked list entries as we
	thought we would. */
	assert(index->word_doc_allocator - doc_table
	== index->total_num_words[first_char]);
	/* Make sure we read all the bytes from the temporary file. */
	assert(word_ptr - word_list_ptr == buf->total_output_bytes);

	/* Now dump the trie to the index file. */
	r = write_trie_word_data(index, &index->doc_word_table,
	SQUAT_WORD_SIZE - 1, result_offset);

	cleanup_map:
	if (munmap((void*)word_list_ptr, buf->total_output_bytes) != 0
	&& r == SQUAT_OK) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	r = SQUAT_ERR;
	}

	cleanup:
	free(doc_table);

	return r;
	}

	/* This does the grunt work of completing the index. If OK is false we
	just take the cleanup path ... this is used by squat_index_destroy. */
	static int index_close_internal(SquatIndex* index, int OK) {
	int r = SQUAT_OK;
	int doc_list_offset;
	int doc_ID_list_offset;
	int word_list_offset;
	char* buf;
	int i;
	SquatDiskHeader* header;
	int offset_buf[256];

	squat_set_last_error(SQUAT_ERR_OK);

	if (!OK) {
	goto cleanup;
	}

	/* Close any open document ... this would really be a client bug. */
	if (index->current_doc_len >= 0) {
	squat_index_close_document(index);
	}

	/* Clear the current trie. We are now going to use it to build
	all-documents tries. */
	delete_doc_word_table(&index->doc_word_table, SQUAT_WORD_SIZE);
	init_doc_word_table(&index->doc_word_table);

	/* Write out the array that maps document IDs to offsets of the
	document records. */
	doc_list_offset = sizeof(SquatDiskHeader);
	doc_ID_list_offset = index->out.total_output_bytes + 1;
	if ((buf = prepare_buffered_write(&index->out,
	SQUAT_SAFETY_ZONE + ((index->current_doc_ID + 1)*4))) == NULL) {
	r = SQUAT_ERR;
	goto cleanup;
	}
	*buf++ = 0;
	memcpy(buf, index->doc_ID_list, index->current_doc_ID*4);
	buf += index->current_doc_ID*4;
	memset(buf, 0, 4);
	complete_buffered_write(&index->out, buf + 4);

	/* Now write out the trie for every initial byte that we saw. The
	offsets are collected in 'offset_buf'. */
	memset(offset_buf, 0, sizeof(offset_buf));
	for (i = 0; i < 256; i++) {
	if (index->stats_callback != NULL) {
	SquatStatsEvent event;

	event.generic.type = SQUAT_STATS_COMPLETED_INITIAL_CHAR;
	event.completed_initial_char.completed_char = i;
	event.completed_initial_char.num_words = index->total_num_words[i];
	if (index->index_buffers[i].buf != NULL) {
	event.completed_initial_char.temp_file_size =
	index->index_buffers[i].total_output_bytes;
	} else {
	event.completed_initial_char.temp_file_size = 0;
	}
	index->stats_callback(index->stats_callback_closure, &event);
	}

	if (index->index_buffers[i].buf != NULL) {
	/* We have to flush the temporary file output buffer before we try to use
	the temporary file. */
	if (flush_and_reset_buffered_writes(index->index_buffers + i) != SQUAT_OK
	\|\| dump_index_trie_words(index, i, offset_buf + i) != SQUAT_OK) {
	r = SQUAT_ERR;
	goto cleanup;
	}
	/* Close files and free memory as we go. This could be important
	if disk space is low and we're generating a huge index. */
	if (close(index->index_buffers[i].fd) < 0) {
	squat_set_last_error(SQUAT_ERR_SYSERR);
	r = SQUAT_ERR;
	}
	free(index->index_buffers[i].buf);
	index->index_buffers[i].buf = NULL;
	} else {
	offset_buf[i] = 0;
	}
	}

	/* Save the offset where the root of the index trie is going to go. */
	word_list_offset = index->out.total_output_bytes;

	/* Relativize the subtrie offsets. */
	for (i = 0; i < 256; i++) {
	if (offset_buf[i] != 0) {
	offset_buf[i] = word_list_offset - offset_buf[i];

	if (i < index->doc_word_table.first_valid_entry) {
	index->doc_word_table.first_valid_entry = i;
	}
	index->doc_word_table.last_valid_entry = i;
	}
	}

	/* Dump out the offset buffer at last. */
	if (dump_word_table_offsets(index, &index->doc_word_table, offset_buf)
	!= SQUAT_OK) {
	r = SQUAT_ERR;
	goto cleanup;
	}

	/* finally, write trailing zeroes and the header ... now that we know
	we initialized the file with no errors */
	if ((buf = prepare_buffered_write(&index->out, SQUAT_SAFETY_ZONE)) == NULL) {
	r = SQUAT_ERR;
	goto cleanup;
	}
	memset(buf, 0, SQUAT_SAFETY_ZONE);
	complete_buffered_write(&index->out, buf + SQUAT_SAFETY_ZONE);

	/* Flush writes before we seek back to the start to write the header */
	if (flush_and_reset_buffered_writes(&index->out) != SQUAT_OK) {
	r = SQUAT_ERR;
	goto cleanup;
	}

	/* Blat out the header */
	if ((header = (SquatDiskHeader*)prepare_buffered_write(&index->out,
	sizeof(SquatDiskHeader))) == NULL) {
	r = SQUAT_ERR;
	goto cleanup;
	}
	memcpy(header->header_text, squat_index_file_header, 8);
	squat_encode_64(header->doc_list_offset, doc_list_offset);
	squat_encode_64(header->doc_ID_list_offset, doc_ID_list_offset);
	squat_encode_64(header->word_list_offset, word_list_offset);
	memcpy(header->valid_char_bits, index->valid_char_bits,
	sizeof(header->valid_char_bits));
	complete_buffered_write(&index->out, (char*)(header + 1));

	/* Flush out the header */
	if (flush_and_reset_buffered_writes(&index->out) != SQUAT_OK) {
	r = SQUAT_ERR;
	goto cleanup;
	}

	/* WOOHOO! It's done! */

	cleanup:
	free(index->out.buf);
	delete_doc_word_table(&index->doc_word_table, SQUAT_WORD_SIZE - 1);
	/* If we're bailing out because of an error, we might not have
	released all the temporary file resources. */
	for (i = 0; i < 256; i++) {
	if (index->index_buffers[i].buf != NULL) {
	close(index->index_buffers[i].fd);
	free(index->index_buffers[i].buf);
	}
	}
	free(index->tmp_path);
	free(index->doc_ID_list);
	free(index);

	return r;
	}

	int squat_index_finish(SquatIndex* index) {
	return index_close_internal(index, 1);
	}

	int squat_index_destroy(SquatIndex* index) {
	return index_close_internal(index, 0);
	}
	diff --git a/lib/auth_unix.c b/lib/auth_unix.c
	index 551630733..c50c49ae3 100644
	--- a/lib/auth_unix.c
	+++ b/lib/auth_unix.c
	@@ -1,268 +1,268 @@
	/* auth_unix.c -- Unix passwd file authorization
	*
	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	*/

	/*
	- * $Id: auth_unix.c,v 1.29 2001/11/27 02:25:02 ken3 Exp $
	+ * $Id: auth_unix.c,v 1.30 2002/02/19 18:50:14 ken3 Exp $
	*/

	#include <config.h>
	#include <pwd.h>
	#include <grp.h>
	#include <ctype.h>
	#include <string.h>

	#include "auth.h"
	#include "xmalloc.h"

	const char *auth_method_desc = "unix";

	struct auth_state {
	char userid[81];
	char **group;
	int ngroups;
	};

	static struct auth_state auth_anonymous = {
	"anonymous", 0, 0
	};

	/*
	* Determine if the user is a member of 'identifier'
	* Returns one of:
	* 0 User does not match identifier
	* 1 identifier matches everybody
	* 2 User is in the group that is identifier
	* 3 User is identifer
	*/
	-auth_memberof(auth_state, identifier)
	+int auth_memberof(auth_state, identifier)
	struct auth_state *auth_state;
	const char *identifier;
	{
	int i;

	if (!auth_state) auth_state = &auth_anonymous;

	if (strcmp(identifier, "anyone") == 0) return 1;

	if (strcmp(identifier, auth_state->userid) == 0) return 3;

	if (strncmp(identifier, "group:", 6) != 0) return 0;

	for (i=0; i<auth_state->ngroups; i++) {
	if (strcmp(identifier+6, auth_state->group[i]) == 0) return 2;
	}
	return 0;
	}

	/* Map of which characters are allowed by auth_canonifyid.
	* Key: 0 -> not allowed (special, ctrl, or would confuse Unix or imapd)
	* 1 -> allowed, but requires an alpha somewhere else in the string
	* 2 -> allowed, and is an alpha
	*
	* At least one character must be an alpha.
	*
	* This may not be restrictive enough.
	* Here are the reasons for the restrictions:
	*
	* & forbidden because of MUTF-7. (This could be fixed.)
	* : forbidden because it's special in /etc/passwd
	* / forbidden because it can't be used in a mailbox name
	* * % forbidden because they're IMAP magic in the LIST/LSUB commands
	* ? it just scares me
	* ctrl chars, DEL
	* can't send them as IMAP characters in plain folder names, I think
	* 80-FF forbidden because you can't send them in IMAP anyway
	* (and they're forbidden as folder names). (This could be fixed.)
	*
	* + and - are allowed although '+' is probably used for userid+detail
	* subaddressing and qmail users use '-' for subaddressing.
	*
	* Identifiers don't require a digit, really, so that should probably be
	* relaxed, too.
	*/
	static char allowedchars[256] = {
	/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 00-0F */
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 10-1F */
	1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* 20-2F */
	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* 30-3F */

	1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 40-4F */
	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, /* 50-5F */
	1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 60-6F */
	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 0, /* 70-7F */

	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,

	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
	};

	/*
	* Convert 'identifier' into canonical form.
	* Returns a pointer to a static buffer containing the canonical form
	* or NULL if 'identifier' is invalid.
	*
	* XXX If any of the characters marked with 0 are valid and are cropping up,
	* the right thing to do is probably to canonicalize the identifier to two
	* representations: one for getpwent calls and one for folder names. The
	* latter canonicalizes to a MUTF7 representation.
	*/
	char *auth_canonifyid(identifier, len)
	const char *identifier;
	size_t len;
	{
	static char retbuf[81];
	struct group *grp;
	char sawalpha;
	char *p;

	if(!len) len = strlen(identifier);
	if(len >= sizeof(retbuf)) return NULL;

	if (strcasecmp(identifier, "anonymous") == 0) {
	return "anonymous";
	}
	if (strcasecmp(identifier, "anybody") == 0 \|\|
	strcasecmp(identifier, "anyone") == 0) {
	return "anyone";
	}

	memcpy(retbuf, identifier, len);
	retbuf[len] = '\0';

	/* This used to be far more restrictive, but many sites seem to ignore the
	* ye olde Unix conventions of username. Specifically, we used to
	* - drop case on the buffer
	* - disallow lots of non-alpha characters ('-', '_', others)
	* Now we do neither of these, but impose a very different policy based on
	* the character map above.
	*/

	if (!strncmp(retbuf, "group:", 6)) {
	grp = getgrnam(retbuf+6);
	if (!grp) return 0;
	strcpy(retbuf+6, grp->gr_name);
	return retbuf;
	}

	/* Copy the string and look up values in the allowedchars array above.
	* If we see any we don't like, reject the string.
	*/
	sawalpha = 0;
	for(p = retbuf; *p; p++) {
	switch (allowedchars[(unsigned char) p]) {
	case 0:
	return NULL;

	case 2:
	sawalpha = 1;
	/* FALL THROUGH */

	default:
	;
	}
	}

	if (!sawalpha) return NULL; /* has to be one alpha char */

	return retbuf;
	}

	/*
	* Set the current user to 'identifier'. 'cacheid', if non-null,
	* points to a 16-byte binary key to cache identifier's information
	* with.
	*/
	struct auth_state *
	auth_newstate(identifier, cacheid)
	const char *identifier;
	const char *cacheid;
	{
	struct auth_state *newstate;
	struct passwd *pwd;
	struct group *grp;
	char **mem;

	identifier = auth_canonifyid(identifier, 0);
	if (!identifier) return 0;
	if (!strncmp(identifier, "group:", 6)) return 0;

	pwd = getpwnam(identifier);

	newstate = (struct auth_state *)xmalloc(sizeof(struct auth_state));

	strcpy(newstate->userid, identifier);
	newstate->ngroups = 0;
	newstate->group = (char **) 0;

	setgrent();
	- while (grp = getgrent()) {
	+ while ((grp = getgrent())) {
	for (mem = grp->gr_mem; *mem; mem++) {
	if (!strcmp(*mem, identifier)) break;
	}

	if (*mem \|\| (pwd && pwd->pw_gid == grp->gr_gid)) {
	newstate->ngroups++;
	newstate->group = (char *)xrealloc((char )newstate->group,
	newstate->ngroups * sizeof(char *));
	newstate->group[newstate->ngroups-1] = xstrdup(grp->gr_name);
	}
	}
	endgrent();
	return newstate;
	}

	void
	auth_freestate(auth_state)
	struct auth_state *auth_state;
	{
	if (auth_state->group) free((char *)auth_state->group);
	free((char *)auth_state);
	}


	diff --git a/lib/cyrusdb_db3.c b/lib/cyrusdb_db3.c
	index dd6c2d143..617c7b718 100644
	--- a/lib/cyrusdb_db3.c
	+++ b/lib/cyrusdb_db3.c
	@@ -1,991 +1,992 @@
	/* cyrusdb_db3: berkeley db backend
	*
	* Copyright (c) 2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*/

	#include <config.h>

	#include <db.h>
	#include <syslog.h>
	#include <assert.h>
	#include <string.h>
	#include <errno.h>
	#include <stdlib.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <fcntl.h>

	#include "cyrusdb.h"
	#include "exitcodes.h"
	#include "xmalloc.h"
	+#include "retry.h"

	/* --- cut here --- */
	/*
	* what berkeley db algorithm should we use for deadlock detection?
	*
	* DB_LOCK_DEFAULT
	* Use the default policy as specified by db_deadlock.
	* DB_LOCK_OLDEST
	* Abort the oldest transaction.
	* DB_LOCK_RANDOM
	* Abort a random transaction involved in the deadlock.
	* DB_LOCK_YOUNGEST
	* Abort the youngest transaction.
	*/

	#define CONFIG_DEADLOCK_DETECTION DB_LOCK_YOUNGEST
	#define FNAME_DBDIR "/db"

	/* --- cut here --- */

	#if DB_VERSION_MAJOR >= 4
	#define txn_checkpoint(xx1,xx2,xx3,xx4) (xx1)->txn_checkpoint(xx1,xx2,xx3,xx4)
	#define txn_id(xx1) (xx1)->id(xx1)
	#define log_archive(xx1,xx2,xx3,xx4) (xx1)->log_archive(xx1,xx2,xx3)
	#elsif DB_VERSION_MINOR == 3
	#define log_archive(xx1,xx2,xx3,xx4) log_archive(xx1,xx2,xx3)
	#endif

	static int dbinit = 0;
	static DB_ENV *dbenv;

	/* other routines call this one when they fail */
	static int commit_txn(struct db db, struct txn tid);
	static int abort_txn(struct db db, struct txn tid);

	static void db_panic(DB_ENV *dbenv, int errno)
	{
	syslog(LOG_CRIT, "DBERROR: critical database situation");
	/* but don't bounce mail */
	exit(EC_TEMPFAIL);
	}

	static void db_err(const char db_prfx, char buffer)
	{
	syslog(LOG_ERR, "DBERROR %s: %s", db_prfx, buffer);
	}

	static int init(const char *dbdir, int myflags)
	{
	int r, do_retry = 1;
	int flags = 0;
	int maj, min, patch;
	char *vstr;

	if (dbinit++) return 0;

	vstr = db_version(&maj, &min, &patch);
	if (maj != DB_VERSION_MAJOR \|\| min != DB_VERSION_MINOR \|\|
	DB_VERSION_PATCH > patch) {
	syslog(LOG_CRIT, "incorrect version of Berkeley db: "
	"compiled against %d.%d.%d, linked against %d.%d.%d",
	DB_VERSION_MAJOR, DB_VERSION_MINOR, DB_VERSION_PATCH,
	maj, min, patch);
	fatal("wrong db version", EC_SOFTWARE);
	}

	if (myflags & CYRUSDB_RECOVER) {
	flags \|= DB_RECOVER \| DB_CREATE;
	}

	if ((r = db_env_create(&dbenv, 0)) != 0) {
	syslog(LOG_ERR, "DBERROR: db_appinit failed: %s", db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	dbenv->set_paniccall(dbenv, (void ()(DB_ENV , int)) &db_panic);
	if (CONFIG_DB_VERBOSE) {
	dbenv->set_verbose(dbenv, DB_VERB_DEADLOCK, 1);
	dbenv->set_verbose(dbenv, DB_VERB_WAITSFOR, 1);
	}
	if (CONFIG_DB_VERBOSE > 1) {
	dbenv->set_verbose(dbenv, DB_VERB_CHKPOINT, 1);
	}
	dbenv->set_lk_detect(dbenv, CONFIG_DEADLOCK_DETECTION);

	/* XXX should make this value runtime configurable */
	r = dbenv->set_lk_max(dbenv, 50000);
	if (r) {
	syslog(LOG_ERR, "DBERROR: set_lk_max(): %s", db_strerror(r));
	abort();
	}

	/* XXX should make this value runtime configurable */
	r = dbenv->set_tx_max(dbenv, 100);
	if (r) {
	syslog(LOG_ERR, "DBERROR: set_tx_max(): %s", db_strerror(r));
	abort();
	}

	dbenv->set_errcall(dbenv, db_err);
	dbenv->set_errpfx(dbenv, "db3");

	#if 0
	/* XXX should make this value runtime configurable */
	if ((r = dbenv->set_cachesize(dbenv, 0, 64 * 1024, 0)) != 0) {
	dbenv->err(dbenv, r, "set_cachesize");
	dbenv->close(dbenv, 0);
	syslog(LOG_ERR, "DBERROR: set_cachesize(): %s", db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	#endif

	/* what directory are we in? */
	retry:
	flags \|= DB_INIT_LOCK \| DB_INIT_MPOOL \|
	DB_INIT_LOG \| DB_INIT_TXN;
	#if (DB_VERSION_MAJOR > 3) \|\| ((DB_VERSION_MAJOR == 3) && (DB_VERSION_MINOR > 0))
	r = dbenv->open(dbenv, dbdir, flags, 0644);
	#else
	r = dbenv->open(dbenv, dbdir, NULL, flags, 0644);
	#endif
	if (r) {
	if (do_retry && (r == ENOENT)) {
	/* Per sleepycat Support Request #3838 reporting a performance problem:

	Berkeley DB only transactionally protects the open if you're
	doing a DB_CREATE. Even if the Cyrus application is opening
	the file read/write, we don't need a transaction. I see
	from their source that they are always specifying DB_CREATE.
	I bet if they changed it to not specifying CREATE and only
	creating if necessary, the problem would probably go away.

	Given that in general the file should exist, we optimize the most
	often case: the file exists. So, we add DB_CREATE only if we fail
	to open the file and thereby avoid doing a stat(2) needlessly. Sure, it
	should be cached by why waste the cycles anyway?
	*/
	flags \|= DB_CREATE;
	do_retry = 0;
	goto retry;
	}

	syslog(LOG_ERR, "DBERROR: dbenv->open '%s' failed: %s", dbdir,
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}

	dbinit = 1;

	return 0;
	}

	static int done(void)
	{
	int r;

	if (--dbinit) return 0;

	r = dbenv->close(dbenv, 0);
	dbinit = 0;
	if (r) {
	syslog(LOG_ERR, "DBERROR: error exiting application: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}

	return 0;
	}

	static int mysync(void)
	{
	int r;

	assert(dbinit);

	do {
	#if (DB_VERSION_MAJOR > 3) \|\| ((DB_VERSION_MAJOR == 3) && (DB_VERSION_MINOR > 0))
	r = txn_checkpoint(dbenv, 0, 0, 0);
	#else
	r = txn_checkpoint(dbenv, 0, 0);
	#endif
	} while (r == DB_INCOMPLETE);
	if (r) {
	syslog(LOG_ERR, "DBERROR: couldn't checkpoint: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}

	return 0;
	}

	static int copyfile(const char srcname, const char dstname)
	{
	int srcfd, dstfd;
	struct stat sbuf;
	char *buf;
	int bufsize, n;

	if ((srcfd = open(srcname, O_RDONLY)) < 0) {
	syslog(LOG_DEBUG, "error opening %s for reading", srcname);
	return -1;
	}

	if (fstat(srcfd, &sbuf) < 0) {
	syslog(LOG_DEBUG, "error fstating %s", srcname);
	close(srcfd);
	return -1;
	}

	if ((dstfd = open(dstname, O_WRONLY \| O_CREAT, sbuf.st_mode)) < 0) {
	syslog(LOG_DEBUG, "error opening %s for writing (%d)",
	dstname, sbuf.st_mode);
	close(srcfd);
	return -1;
	}

	bufsize = sbuf.st_blksize;
	if ((buf = (char*) xmalloc(bufsize)) == NULL) {
	syslog(LOG_DEBUG, "error allocing buf (%d)", bufsize);
	close(srcfd);
	close(dstfd);
	return -1;
	}

	for (;;) {
	n = read(srcfd, buf, bufsize);

	if (n < 0) {
	if (errno == EINTR)
	continue;

	syslog(LOG_DEBUG, "error reading buf (%d)", bufsize);
	close(srcfd);
	close(dstfd);
	unlink(dstname);
	return -1;
	}

	if (n == 0)
	break;

	if (retry_write(dstfd, buf, n) != n) {
	syslog(LOG_DEBUG, "error writing buf (%d)", n);
	close(srcfd);
	close(dstfd);
	unlink(dstname);
	return -1;
	}
	}

	close(srcfd);
	close(dstfd);
	return 0;
	}

	static int myarchive(const char *dirname)
	{
	int r;
	char begin, list;
	char dstname[1024], *dp;

	strcpy(dstname, dirname);
	dp = dstname + strlen(dstname);

	/* Get the list of log files to remove. */
	r = log_archive(dbenv, &list, DB_ARCH_ABS, NULL);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error listing log files: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	if (list != NULL) {
	for (begin = list; *list != NULL; ++list) {
	syslog(LOG_DEBUG, "removing log file: %s", *list);
	r = unlink(*list);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error removing log file: %s",
	*list);
	return CYRUSDB_IOERROR;
	}
	}
	free (begin);
	}

	/* Get the list of database files to archive. */
	r = log_archive(dbenv, &list, DB_ARCH_ABS \| DB_ARCH_DATA, NULL);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error listing database files: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	if (list != NULL) {
	for (begin = list; *list != NULL; ++list) {
	syslog(LOG_DEBUG, "archiving database file: %s", *list);
	strcpy(dp, strrchr(*list, '/'));
	r = copyfile(*list, dstname);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error archiving database file: %s",
	*list);
	return CYRUSDB_IOERROR;
	}
	}
	free (begin);
	}

	/* Get the list of log files to archive. */
	r = log_archive(dbenv, &list, DB_ARCH_ABS \| DB_ARCH_LOG, NULL);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error listing log files: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	if (list != NULL) {
	for (begin = list; *list != NULL; ++list) {
	syslog(LOG_DEBUG, "archiving log file: %s", *list);
	strcpy(dp, strrchr(*list, '/'));
	r = copyfile(*list, dstname);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error archiving log file: %s",
	*list);
	return CYRUSDB_IOERROR;
	}
	}
	free (begin);
	}

	return 0;
	}

	static int myopen(const char fname, struct db *ret)
	{
	DB *db;
	int r;

	assert(dbinit && fname && ret);

	*ret = NULL;

	r = db_create(&db, dbenv, 0);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: opening %s: %s", fname, db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	/* xxx set comparator! */

	r = db->open(db, fname, NULL, DB_BTREE, DB_CREATE, 0664);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: opening %s: %s", fname, db_strerror(r));
	return CYRUSDB_IOERROR;
	}

	ret = (struct db ) db;

	return r;
	}

	static int myclose(struct db *db)
	{
	int r;
	DB a = (DB ) db;

	assert(dbinit && db);

	/* since we're using txns, we can supply DB_NOSYNC */
	r = a->close(a, DB_NOSYNC);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: error closing: %s", db_strerror(r));
	r = CYRUSDB_IOERROR;
	}

	return r;
	}

	static int gettid(struct txn mytid, DB_TXN tid, char *where)
	{
	int r;

	if (mytid) {
	if (*mytid) {
	assert((txn_id((DB_TXN )mytid) != 0));
	tid = (DB_TXN ) *mytid;
	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "%s: reusing txn %lu", where, txn_id(*tid));
	} else {
	r = txn_begin(dbenv, NULL, tid, 0);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: error beginning txn (%s): %s", where,
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "%s: starting txn %lu", where, txn_id(*tid));
	}
	mytid = (struct txn ) *tid;
	}

	return 0;
	}

	static int myfetch(struct db *mydb,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid, int flags)
	{
	int r = 0;
	DBT k, d;
	DB db = (DB ) mydb;
	DB_TXN *tid = NULL;

	assert(dbinit && db);

	r = gettid(mytid, &tid, "myfetch");
	if (r) return r;

	memset(&k, 0, sizeof(k));
	memset(&d, 0, sizeof(d));

	k.data = (char *) key;
	k.size = keylen;

	r = db->get(db, tid, &k, &d, flags);
	switch (r) {
	case 0:
	if (data) *data = d.data;
	if (datalen) *datalen = d.size;
	break;
	case DB_NOTFOUND:
	*data = NULL;
	*datalen = 0;
	r = 0;
	break;
	case DB_LOCK_DEADLOCK:
	if (mytid) {
	abort_txn(mydb, *mytid);
	*mytid = NULL;
	}
	r = CYRUSDB_AGAIN;
	break;
	default:
	syslog(LOG_ERR, "DBERROR: error fetching %s: %s", key,
	db_strerror(r));
	r = CYRUSDB_IOERROR;
	break;
	}

	return r;
	}

	static int fetch(struct db *mydb,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid)
	{
	return myfetch(mydb, key, keylen, data, datalen, mytid, 0);
	}

	static int fetchlock(struct db *mydb,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid)
	{
	return myfetch(mydb, key, keylen, data, datalen, mytid, DB_RMW);
	}

	#define OPENCURSOR() do { \
	r = db->cursor(db, tid, &cursor, 0); \
	if (r != 0) { \
	syslog(LOG_ERR, "DBERROR: unable to create cursor: %s", \
	db_strerror(r)); \
	cursor = NULL; \
	goto done; \
	} \
	} while (0)

	#define CLOSECURSOR() do { \
	int r = cursor->c_close(cursor); \
	if (r) { \
	syslog(LOG_ERR, "DBERROR: error closing cursor: %s", \
	db_strerror(r)); \
	cursor = NULL; \
	goto done; \
	} \
	} while (0)


	/* instead of "DB_DBT_REALLOC", we might want DB_DBT_USERMEM and allocate
	this to the maximum length at the beginning. */
	static int foreach(struct db *mydb,
	char *prefix, int prefixlen,
	foreach_p *goodp,
	foreach_cb cb, void rock,
	struct txn **mytid)
	{
	int r = 0;
	DBT k, d;
	DBC *cursor = NULL;
	DB db = (DB ) mydb;
	DB_TXN *tid = NULL;

	assert(dbinit && db);
	assert(cb);

	memset(&k, 0, sizeof(k));
	memset(&d, 0, sizeof(d));

	/* k.flags \|= DB_DBT_REALLOC;
	d.flags \|= DB_DBT_REALLOC;*/

	r = gettid(mytid, &tid, "foreach");
	if (r) return r;

	if (0) {
	restart:
	CLOSECURSOR();
	}

	/* create cursor */
	OPENCURSOR();

	/* find first record */
	if (prefix && *prefix) {
	/* if (k.data) free(k.data); */
	k.data = prefix;
	k.size = prefixlen;

	r = cursor->c_get(cursor, &k, &d, DB_SET_RANGE);
	} else {
	r = cursor->c_get(cursor, &k, &d, DB_FIRST);
	}
	if (!tid && r == DB_LOCK_DEADLOCK) goto restart;

	/* iterate over all mailboxes matching prefix */
	while (!r) {
	/* does this match our prefix? */
	if (prefixlen && memcmp(k.data, prefix, prefixlen)) break;

	if (goodp(rock, k.data, k.size, d.data, d.size)) {
	/* we have a winner! */

	/* close the cursor, so we're not holding locks
	during a callback */
	CLOSECURSOR(); cursor = NULL;

	r = cb(rock, k.data, k.size, d.data, d.size);
	if (r != 0) {
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: foreach cb() failed");
	}
	/* don't mistake this for a db error */
	r = 0;

	break;
	}

	/* restore the current location & advance */
	OPENCURSOR();

	r = cursor->c_get(cursor, &k, &d, DB_SET);
	switch (r) {
	case 0:
	r = cursor->c_get(cursor, &k, &d, DB_NEXT);
	break;

	case DB_NOTFOUND:
	/* deleted during callback? */
	r = cursor->c_get(cursor, &k, &d, DB_SET_RANGE);
	break;

	default:
	/* handle other cases below */
	break;
	}
	} else {
	/* advance the cursor */
	r = cursor->c_get(cursor, &k, &d, DB_NEXT);
	}

	while (r == DB_LOCK_DEADLOCK) {
	if (tid) {
	break; /* don't autoretry txn-protected */
	}

	/* if we deadlock, close and reopen the cursor, and
	reposition it */
	CLOSECURSOR();
	OPENCURSOR();

	r = cursor->c_get(cursor, &k, &d, DB_SET);
	switch (r) {
	case 0:
	r = cursor->c_get(cursor, &k, &d, DB_NEXT);
	break;
	case DB_LOCK_DEADLOCK:
	continue;
	case DB_NOTFOUND: /* deleted? */
	r = cursor->c_get(cursor, &k, &d, DB_SET_RANGE);
	break;
	}
	}
	}

	done:
	if (cursor) {
	CLOSECURSOR();
	}

	switch (r) {
	case 0: /* ok */
	break;
	case DB_NOTFOUND: /* also ok */
	r = 0;
	break;
	case DB_LOCK_DEADLOCK: /* erg, we're in a txn! */
	if (mytid) {
	abort_txn(mydb, *mytid);
	*mytid = NULL;
	}
	r = CYRUSDB_AGAIN;
	break;
	default:
	if (mytid) {
	abort_txn(mydb, *mytid);
	*mytid = NULL;
	}
	syslog(LOG_ERR, "DBERROR: error advancing: %s", db_strerror(r));
	r = CYRUSDB_IOERROR;
	break;
	}

	/* if (k.data) free(k.data);
	if (d.data) free(d.data);*/

	return r;
	}

	static int mystore(struct db *mydb,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid, int putflags, int txnflags)
	{
	int r = 0;
	DBT k, d;
	DB_TXN *tid;
	DB db = (DB ) mydb;

	assert(dbinit && db);
	assert(key && keylen);

	r = gettid(mytid, &tid, "mystore");
	if (r) return r;

	memset(&k, 0, sizeof(k));
	memset(&d, 0, sizeof(d));

	k.data = (char *) key;
	k.size = keylen;
	d.data = (char *) data;
	d.size = datalen;

	if (!mytid) {
	/* start a transaction for the write */
	restart:
	r = txn_begin(dbenv, NULL, &tid, 0);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: mystore: error beginning txn: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "mystore: starting txn %lu", txn_id(tid));
	}
	r = db->put(db, tid, &k, &d, putflags);
	if (!mytid) {
	/* finish once-off txn */
	if (r) {
	int r2;

	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "mystore: aborting txn %lu", txn_id(tid));
	r2 = txn_abort(tid);
	if (r2) {
	syslog(LOG_ERR, "DBERROR: mystore: error aborting txn: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}

	if (r == DB_LOCK_DEADLOCK) {
	goto restart;
	}
	} else {
	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "mystore: committing txn %lu", txn_id(tid));
	r = txn_commit(tid, txnflags);
	}
	}

	if ( r != 0) {
	if (mytid) {
	abort_txn(mydb, *mytid);
	*mytid = NULL;
	}
	if (r == DB_LOCK_DEADLOCK) {
	r = CYRUSDB_AGAIN;
	} else {
	syslog(LOG_ERR, "DBERROR: mystore: error storing %s: %s",
	key, db_strerror(r));
	r = CYRUSDB_IOERROR;
	}
	}

	return r;
	}

	static int create(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, DB_NOOVERWRITE, 0);
	}

	static int store(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 0, 0);
	}

	static int create_nosync(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, DB_NOOVERWRITE,
	DB_TXN_NOSYNC);
	}

	static int store_nosync(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 0, DB_TXN_NOSYNC);
	}

	static int mydelete(struct db *mydb,
	const char *key, int keylen,
	struct txn **mytid, int txnflags)
	{
	int r = 0;
	DBT k;
	DB_TXN *tid;
	DB db = (DB ) mydb;

	assert(dbinit && db);
	assert(key && keylen);

	r = gettid(mytid, &tid, "delete");
	if (r) return r;

	memset(&k, 0, sizeof(k));

	k.data = (char *) key;
	k.size = keylen;

	if (!mytid) {
	restart:
	/* start txn for the write */
	r = txn_begin(dbenv, NULL, &tid, 0);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: mydelete: error beginning txn: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}
	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "mydelete: starting txn %lu", txn_id(tid));
	}
	r = db->del(db, tid, &k, 0);
	if (!mytid) {
	/* finish txn for the write */
	if (r) {
	int r2;
	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "mydelete: aborting txn %lu", txn_id(tid));
	r2 = txn_abort(tid);
	if (r2) {
	syslog(LOG_ERR, "DBERROR: mydelete: error aborting txn: %s",
	db_strerror(r));
	return CYRUSDB_IOERROR;
	}

	if (r == DB_LOCK_DEADLOCK) {
	goto restart;
	}
	} else {
	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "mydelete: committing txn %lu", txn_id(tid));
	r = txn_commit(tid, txnflags);
	}
	}

	if (r != 0) {
	if (mytid) {
	abort_txn(mydb, *mytid);
	*mytid = NULL;
	}
	if (r == DB_LOCK_DEADLOCK) {
	r = CYRUSDB_AGAIN;
	} else {
	syslog(LOG_ERR, "DBERROR: mydelete: error deleting %s: %s",
	key, db_strerror(r));
	r = CYRUSDB_IOERROR;
	}
	}

	return r;
	}

	static int delete(struct db *db,
	const char *key, int keylen,
	struct txn **tid)
	{
	return mydelete(db, key, keylen, tid, 0);
	}

	static int delete_nosync(struct db *db,
	const char *key, int keylen,
	struct txn **tid)
	{
	return mydelete(db, key, keylen, tid, DB_TXN_NOSYNC);
	}

	static int mycommit(struct db db, struct txn tid, int txnflags)
	{
	int r;
	DB_TXN t = (DB_TXN ) tid;

	assert(dbinit && tid);

	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "mycommit: committing txn %lu", txn_id(t));
	r = txn_commit(t, txnflags);
	switch (r) {
	case 0:
	break;
	case EINVAL:
	syslog(LOG_WARNING, "mycommit: tried to commit an already aborted transaction");
	r = CYRUSDB_IOERROR;
	break;
	default:
	syslog(LOG_ERR, "DBERROR: mycommit failed on commit: %s",
	db_strerror(r));
	r = CYRUSDB_IOERROR;
	break;
	}

	return r;
	}

	static int commit_txn(struct db db, struct txn tid)
	{
	return mycommit(db, tid, 0);
	}

	static int commit_nosync(struct db db, struct txn tid)
	{
	return mycommit(db, tid, DB_TXN_NOSYNC);
	}

	static int abort_txn(struct db db, struct txn tid)
	{
	int r;
	DB_TXN t = (DB_TXN ) tid;

	assert(dbinit && tid);

	if (CONFIG_DB_VERBOSE)
	syslog(LOG_DEBUG, "abort_txn: aborting txn %lu", txn_id(t));
	r = txn_abort(t);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: abort_txn: error aborting txn: %s", db_strerror(r));
	return CYRUSDB_IOERROR;
	}

	return 0;
	}

	struct cyrusdb_backend cyrusdb_db3 =
	{
	"db3", /* name */

	&init,
	&done,
	&mysync,
	&myarchive,

	&myopen,
	&myclose,

	&fetch,
	&fetchlock,
	&foreach,
	&create,
	&store,
	&delete,

	&commit_txn,
	&abort_txn
	};

	struct cyrusdb_backend cyrusdb_db3_nosync =
	{
	"db3-nosync", /* name */

	&init,
	&done,
	&mysync,
	&myarchive,

	&myopen,
	&myclose,

	&fetch,
	&fetchlock,
	&foreach,
	&create_nosync,
	&store_nosync,
	&delete_nosync,

	&commit_nosync,
	&abort_txn,

	NULL
	};
	diff --git a/lib/cyrusdb_skiplist.c b/lib/cyrusdb_skiplist.c
	index f54373401..2d3e6c1af 100644
	--- a/lib/cyrusdb_skiplist.c
	+++ b/lib/cyrusdb_skiplist.c
	@@ -1,1864 +1,1865 @@
	/* skip-list.c -- generic skip list routines
	- * $Id: cyrusdb_skiplist.c,v 1.20 2002/02/12 20:26:20 rjs3 Exp $
	+ * $Id: cyrusdb_skiplist.c,v 1.21 2002/02/19 18:50:14 ken3 Exp $
	*
	* Copyright (c) 1998, 2000, 2002 Carnegie Mellon University.
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*/

	/* xxx check retry_xxx for failure */

	#include <config.h>

	#include <stdlib.h>
	+#include <string.h>
	#include <limits.h>
	#include <assert.h>
	#include <syslog.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <fcntl.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <time.h>
	#include <netinet/in.h>

	#include "cyrusdb.h"
	#include "xmalloc.h"
	#include "map.h"
	#include "lock.h"
	#include "retry.h"

	#define PROB (0.5)

	/*
	*
	* disk format; all numbers in network byte order
	*
	* there's the data file, consisting of the
	* multiple records of "key", "data", and "skip pointers", where skip
	* pointers are the record number of the data pointer.
	*
	* on startup, recovery is performed. the last known good data file
	* is taken and the intent log is replayed on it. the index file is
	* regenerated from scratch.
	*
	* during operation ckecpoints will compress the data. the data file
	* is locked. then a checkpoint rewrites the data file in order,
	* removing any unused records. this is written and fsync'd to
	* dfile.NEW and stored for use during recovery.
	*/

	/*
	header "skiplist file\0\0\0"
	version (4 bytes)
	version_minor (4 bytes)
	maxlevel (4 bytes)
	curlevel (4 bytes)
	listsize (4 bytes)
	in active items
	log start (4 bytes)
	offset where log records start, used mainly to tell when to compress
	last recovery (4 bytes)
	seconds since unix epoch

	1 or more skipnodes, one of:

	record type (4 bytes) [DUMMY, INORDER, ADD]
	key size (4 bytes)
	key string (bit string, rounded to up to 4 byte multiples w/ 0s)
	data size (4 bytes)
	data string (bit string, rounded to up to 4 byte multiples w/ 0s)
	skip pointers (4 bytes each)
	least to most
	padding (4 bytes, must be -1)

	record type (4 bytes) [DELETE]
	record ptr (4 bytes; record to be deleted)

	record type (4 bytes) [COMMIT]


	record type is either
	DUMMY (first node is of this type)
	INORDER
	ADD
	DELETE
	COMMIT (commit the previous records)
	*/

	enum {
	INORDER = 1,
	ADD = 2,
	DELETE = 4,
	COMMIT = 255,
	DUMMY = 257
	};

	struct db {
	/* file data */
	char *fname;
	int fd;

	const char *map_base;
	unsigned long map_len; /* mapped size */
	unsigned long map_size; /* actual size */
	long map_ino;

	/* header info */
	int version;
	int version_minor;
	int maxlevel;
	int curlevel;
	int listsize;
	int logstart; /* where the log starts from last chkpnt */
	time_t last_recovery;

	};

	struct txn {
	int ismalloc;

	int oldcurlevel; /* any updates to curlevel necessary? */

	/* logstart is where we start changes from on commit, where we truncate
	to on abort */
	int logstart;
	int logend; /* where to write to continue this txn */
	};

	static time_t global_recovery = 0;
	static int do_fsync = 1;

	static int myinit(const char *dbdir, int myflags)
	{
	char sfile[1024];
	int fd, r = 0;
	time_t a;

	snprintf(sfile, sizeof(sfile), "%s/skipstamp", dbdir);

	if (myflags & CYRUSDB_RECOVER) {
	/* set the recovery timestamp; all databases earlier than this
	time need recovery run when opened */

	global_recovery = time(NULL);
	fd = open(sfile, O_RDWR \| O_CREAT, 0666);
	if (fd == -1) r = -1;

	if (r != -1) r = ftruncate(fd, 0);
	a = htonl(global_recovery);
	if (r != -1) r = write(fd, &a, 4);
	if (r != -1) r = close(fd);

	if (r == -1) {
	syslog(LOG_ERR, "DBERROR: writing %s: %m", sfile);
	if (fd != -1) close(fd);
	return CYRUSDB_IOERROR;
	}
	} else {
	/* read the global recovery timestamp */

	fd = open(sfile, O_RDONLY, 0666);
	if (fd == -1) r = -1;
	if (r != -1) r = read(fd, &a, 4);
	if (r != -1) r = close(fd);

	if (r == -1) {
	syslog(LOG_ERR, "DBERROR: reading %s, assuming the worst: %m",
	sfile);
	global_recovery = 0;
	} else {
	global_recovery = ntohl(a);
	}
	}

	srand(time(NULL) * getpid());

	if (getenv("CYRUS_SKIPLIST_UNSAFE")) {
	do_fsync = 0;
	}

	return 0;
	}

	static int mydone(void)
	{
	return 0;
	}

	static int mysync(void)
	{
	return 0;
	}

	static int myarchive(const char *dirname)
	{
	return 0;
	}

	enum {
	SKIPLIST_VERSION = 1,
	SKIPLIST_VERSION_MINOR = 2,
	SKIPLIST_MAXLEVEL = 20,
	SKIPLIST_MINREWRITE = 16834 /* don't rewrite logs smaller than this */
	};

	#define BIT32_MAX 4294967295U

	#if UINT_MAX == BIT32_MAX
	typedef unsigned int bit32;
	#elif ULONG_MAX == BIT32_MAX
	typedef unsigned long bit32;
	#elif USHRT_MAX == BIT32_MAX
	typedef unsigned short bit32;
	#else
	#error dont know what to use for bit32
	#endif

	#define HEADER_MAGIC ("\241\002\213\015skiplist file\0\0\0")
	#define HEADER_MAGIC_SIZE (20)

	/* offsets of header files */
	enum {
	OFFSET_HEADER = 0,
	OFFSET_VERSION = 20,
	OFFSET_VERSION_MINOR = 24,
	OFFSET_MAXLEVEL = 28,
	OFFSET_CURLEVEL = 32,
	OFFSET_LISTSIZE = 36,
	OFFSET_LOGSTART = 40,
	OFFSET_LASTRECOVERY = 44
	};

	enum {
	HEADER_SIZE = OFFSET_LASTRECOVERY + 4
	};

	static int mycommit(struct db db, struct txn tid);
	static int myabort(struct db db, struct txn tid);
	static int mycheckpoint(struct db *db, int locked);
	static int recovery(struct db *db);

	/* file looks like:
	struct header {
	...
	}
	struct dummy {
	bit32 t = htonl(DUMMY);
	bit32 ks = 0;
	bit32 ds = 0;
	bit32 forward[db->maxlevel];
	bit32 pad = -1;
	} */
	#define DUMMY_OFFSET(db) (HEADER_SIZE)
	#define DUMMY_PTR(db) ((db)->map_base + HEADER_SIZE)
	#define DUMMY_SIZE(db) (4 * (3 + db->maxlevel + 1))

	/* bump to the next multiple of 4 bytes */
	#define ROUNDUP(num) (((num) + 3) & 0xFFFFFFFC)

	#define TYPE(ptr) (ntohl(((bit32 )(ptr))))
	#define KEY(ptr) ((ptr) + 8)
	#define KEYLEN(ptr) (ntohl(((bit32 )((ptr) + 4))))
	#define DATA(ptr) ((ptr) + 8 + ROUNDUP(KEYLEN(ptr)) + 4)
	#define DATALEN(ptr) (ntohl(((bit32 )((ptr) + 8 + ROUNDUP(KEYLEN(ptr))))))
	#define FIRSTPTR(ptr) ((ptr) + 8 + ROUNDUP(KEYLEN(ptr)) + 4 + ROUNDUP(DATALEN(ptr)))

	/* return a pointer to the pointer */
	#define PTR(ptr, x) (FIRSTPTR(ptr) + 4 * (x))

	/* FORWARD(ptr, x)
	* given a pointer to the start of the record, return the offset
	* corresponding to the xth pointer
	*/
	#define FORWARD(ptr, x) (ntohl(((bit32 )(FIRSTPTR(ptr) + 4 * (x)))))

	/* how many levels does this record have? */
	static int LEVEL(const char *ptr)
	{
	const bit32 p, q;

	assert(TYPE(ptr) == DUMMY \|\| TYPE(ptr) == INORDER \|\| TYPE(ptr) == ADD);
	p = q = (bit32 *) FIRSTPTR(ptr);
	while (*p != -1) p++;
	return (p - q);
	}

	/* how big is this record? */
	static int RECSIZE(const char *ptr)
	{
	int ret = 0;
	switch (TYPE(ptr)) {
	case DUMMY:
	case INORDER:
	case ADD:
	ret += 4; /* tag */
	ret += 4; /* keylen */
	ret += ROUNDUP(KEYLEN(ptr)); /* key */
	ret += 4; /* datalen */
	ret += ROUNDUP(DATALEN(ptr)); /* data */
	ret += 4 * LEVEL(ptr); /* pointers */
	ret += 4; /* padding */
	break;

	case DELETE:
	ret += 8;
	break;

	case COMMIT:
	ret += 4;
	break;
	}

	return ret;
	}

	#define PADDING(ptr) (ntohl(((bit32 )((ptr) + RECSIZE(ptr) - 4))))

	/* given an open, mapped db, read in the header information */
	static int read_header(struct db *db)
	{
	const char *dptr;
	int r;

	assert(db && db->map_len && db->fname && db->map_base);
	if (db->map_len < HEADER_SIZE) {
	syslog(LOG_ERR,
	"skiplist: file not large enough for header: %s", db->fname);
	}

	if (memcmp(db->map_base, HEADER_MAGIC, HEADER_MAGIC_SIZE)) {
	syslog(LOG_ERR, "skiplist: invalid magic header: %s", db->fname);
	return CYRUSDB_IOERROR;
	}

	db->version = ntohl(((bit32 )(db->map_base + OFFSET_VERSION)));
	db->version_minor =
	ntohl(((bit32 )(db->map_base + OFFSET_VERSION_MINOR)));
	if (db->version != SKIPLIST_VERSION) {
	syslog(LOG_ERR, "skiplist: version mismatch: %s has version %d.%d",
	db->fname, db->version, db->version_minor);
	return CYRUSDB_IOERROR;
	}

	db->maxlevel = ntohl(((bit32 )(db->map_base + OFFSET_MAXLEVEL)));
	db->curlevel = ntohl(((bit32 )(db->map_base + OFFSET_CURLEVEL)));
	db->listsize = ntohl(((bit32 )(db->map_base + OFFSET_LISTSIZE)));
	db->logstart = ntohl(((bit32 )(db->map_base + OFFSET_LOGSTART)));
	db->last_recovery =
	ntohl(((bit32 )(db->map_base + OFFSET_LASTRECOVERY)));

	/* verify dummy node */
	dptr = DUMMY_PTR(db);
	r = 0;

	if (!r && TYPE(dptr) != DUMMY) {
	syslog(LOG_ERR, "DBERROR: %s: first node not type DUMMY",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	if (!r && KEYLEN(dptr) != 0) {
	syslog(LOG_ERR, "DBERROR: %s: DUMMY has non-zero KEYLEN",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	if (!r && DATALEN(dptr) != 0) {
	syslog(LOG_ERR, "DBERROR: %s: DUMMY has non-zero DATALEN",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	if (!r && LEVEL(dptr) != db->maxlevel) {
	syslog(LOG_ERR, "DBERROR: %s: DUMMY level(%d) != db->maxlevel(%d)",
	db->fname, LEVEL(dptr), db->maxlevel);
	r = CYRUSDB_IOERROR;
	}

	return r;
	}

	/* given an open, mapped db, locked db,
	write the header information */
	static int write_header(struct db *db)
	{
	char buf[HEADER_SIZE];
	int n;

	memcpy(buf + 0, HEADER_MAGIC, HEADER_MAGIC_SIZE);
	((bit32 )(buf + OFFSET_VERSION)) = htonl(db->version);
	((bit32 )(buf + OFFSET_VERSION_MINOR)) = htonl(db->version_minor);
	((bit32 )(buf + OFFSET_MAXLEVEL)) = htonl(db->maxlevel);
	((bit32 )(buf + OFFSET_CURLEVEL)) = htonl(db->curlevel);
	((bit32 )(buf + OFFSET_LISTSIZE)) = htonl(db->listsize);
	((bit32 )(buf + OFFSET_LOGSTART)) = htonl(db->logstart);
	((bit32 )(buf + OFFSET_LASTRECOVERY)) = htonl(db->last_recovery);

	/* write it out */
	lseek(db->fd, 0, SEEK_SET);
	n = retry_write(db->fd, buf, HEADER_SIZE);
	if (n != HEADER_SIZE) {
	syslog(LOG_ERR, "DBERROR: writing skiplist header for %s: %m",
	db->fname);
	return CYRUSDB_IOERROR;
	}

	return 0;
	}

	static int dispose_db(struct db *db)
	{
	if (!db) return 0;
	if (db->fname) {
	free(db->fname);
	}
	if (db->map_base) {
	map_free(&db->map_base, &db->map_len);
	}
	if (db->fd != -1) {
	close(db->fd);
	}

	free(db);

	return 0;
	}

	static int write_lock(struct db db, const char altname)
	{
	struct stat sbuf;
	const char *lockfailaction;
	const char *fname = altname ? altname : db->fname;

	if (lock_reopen(db->fd, fname, &sbuf, &lockfailaction) < 0) {
	syslog(LOG_ERR, "IOERROR: %s %s: %m", lockfailaction, fname);
	return CYRUSDB_IOERROR;
	}
	if (db->map_ino != sbuf.st_ino) {
	map_free(&db->map_base, &db->map_len);
	}
	db->map_size = sbuf.st_size;
	db->map_ino = sbuf.st_ino;

	map_refresh(db->fd, 0, &db->map_base, &db->map_len, sbuf.st_size,
	fname, 0);

	return 0;
	}

	static int read_lock(struct db *db)
	{
	struct stat sbuf, sbuffile;
	int newfd = -1;

	for (;;) {
	if (lock_shared(db->fd) < 0) {
	syslog(LOG_ERR, "IOERROR: lock_shared %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}

	if (fstat(db->fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat %s: %m", db->fname);
	lock_unlock(db->fd);
	return CYRUSDB_IOERROR;
	}

	if (stat(db->fname, &sbuffile) == -1) {
	syslog(LOG_ERR, "IOERROR: stat %s: %m", db->fname);
	lock_unlock(db->fd);
	return CYRUSDB_IOERROR;
	}
	if (sbuf.st_ino == sbuffile.st_ino) break;

	newfd = open(db->fname, O_RDWR);
	if (newfd == -1) {
	syslog(LOG_ERR, "IOERROR: open %s: %m", db->fname);
	lock_unlock(db->fd);
	return CYRUSDB_IOERROR;
	}

	dup2(newfd, db->fd);
	close(newfd);
	}

	if (db->map_ino != sbuf.st_ino) {
	map_free(&db->map_base, &db->map_len);
	}
	db->map_size = sbuf.st_size;
	db->map_ino = sbuf.st_ino;

	map_refresh(db->fd, 0, &db->map_base, &db->map_len, sbuf.st_size,
	db->fname, 0);

	return 0;
	}

	static int unlock(struct db *db)
	{
	if (lock_unlock(db->fd) < 0) {
	syslog(LOG_ERR, "IOERROR: lock_unlock %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}

	return 0;
	}

	static int myopen(const char fname, struct db *ret)
	{
	struct db db = (struct db ) xzmalloc(sizeof(struct db));
	int r;
	int new = 0;
	struct stat sbuf;

	db->fd = -1;
	db->fname = xstrdup(fname);

	db->fd = open(fname, O_RDWR, 0666);
	if (db->fd == -1) {
	db->fd = open(fname, O_RDWR \| O_CREAT, 0666);
	new = 1;
	}
	if (db->fd == -1) {
	syslog(LOG_ERR, "IOERROR: opening %s: %m", fname);
	dispose_db(db);
	return CYRUSDB_IOERROR;
	}

	if (new) {
	/* lock the db */
	if (write_lock(db, NULL) < 0) {
	dispose_db(db);
	return CYRUSDB_IOERROR;
	}

	/* initialize in memory structure */
	db->version = SKIPLIST_VERSION;
	db->version_minor = SKIPLIST_VERSION_MINOR;
	db->maxlevel = SKIPLIST_MAXLEVEL;
	db->curlevel = 0;
	db->listsize = 0;
	/* where do we start writing new entries? */
	db->logstart = DUMMY_OFFSET(db) + DUMMY_SIZE(db);
	db->last_recovery = time(NULL);

	/* create the header */
	r = write_header(db);

	if (!r) {
	int n;
	int dsize = DUMMY_SIZE(db);
	bit32 buf = (bit32 ) xzmalloc(dsize);

	buf[0] = htonl(DUMMY);
	buf[(dsize / 4) - 1] = htonl(-1);

	lseek(db->fd, DUMMY_OFFSET(db), SEEK_SET);
	n = retry_write(db->fd, (char *) buf, dsize);
	if (n != dsize) {
	syslog(LOG_ERR, "DBERROR: writing dummy node for %s: %m",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	free(buf);
	}

	/* sync the db */
	if (!r && do_fsync && (fsync(db->fd) < 0)) {
	syslog(LOG_ERR, "DBERROR: fsync(%s): %m", db->fname);
	r = CYRUSDB_IOERROR;
	}

	/* unlock the db */
	unlock(db);
	}

	if (fstat(db->fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat %s: %m", fname);
	dispose_db(db);
	return CYRUSDB_IOERROR;
	}
	db->map_ino = sbuf.st_ino;
	db->map_size = sbuf.st_size;

	db->map_base = 0;
	db->map_len = 0;
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, sbuf.st_size,
	fname, 0);

	r = read_header(db);
	if (r) {
	dispose_db(db);
	return r;
	}

	if (!global_recovery \|\| db->last_recovery < global_recovery) {
	/* run recovery; we rebooted since the last time recovery
	was run */
	r = recovery(db);
	if (r) {
	dispose_db(db);
	return r;
	}
	}

	*ret = db;
	return 0;
	}

	int myclose(struct db *db)
	{
	return dispose_db(db);
	}

	static int compare(const char s1, int l1, const char s2, int l2)
	{
	int min = l1 < l2 ? l1 : l2;
	int cmp = 0;

	while (min-- > 0 && (cmp = s1 - s2) == 0) {
	s1++;
	s2++;
	}
	if (min >= 0) {
	return cmp;
	} else {
	if (l1 > l2) return 1;
	else if (l2 > l1) return -1;
	else return 0;
	}
	}

	/* returns the offset to the node asked for, or the node after it
	if it doesn't exist.
	if previous is set, finds the last node < key */
	static const char find_node(struct db db,
	const char *key, int keylen,
	int *updateoffsets)
	{
	const char *ptr = db->map_base + DUMMY_OFFSET(db);
	int i;
	int offset;

	if (updateoffsets) {
	for (i = 0; i < db->maxlevel; i++) {
	updateoffsets[i] = DUMMY_OFFSET(db);
	}
	}

	for (i = db->curlevel - 1; i >= 0; i--) {
	while ((offset = FORWARD(ptr, i)) &&
	compare(KEY(db->map_base + offset), KEYLEN(db->map_base + offset),
	key, keylen) < 0) {
	/* move forward at level 'i' */
	ptr = db->map_base + offset;
	}
	if (updateoffsets) updateoffsets[i] = ptr - db->map_base;
	}

	ptr = db->map_base + FORWARD(ptr, 0);

	return ptr;
	}

	int myfetch(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid)
	{
	const char *ptr;
	struct txn t, *tp;
	int r;

	assert(db != NULL && key != NULL);
	assert(data != NULL && datalen != NULL);

	if (!mytid) {
	/* grab a r lock */
	if ((r = read_lock(db)) < 0) {
	return r;
	}

	tp = NULL;
	} else if (!*mytid) {
	/* grab a r/w lock */
	if ((r = write_lock(db, NULL)) < 0) {
	return r;
	}

	/* fill in t */
	t.ismalloc = 0;
	t.oldcurlevel = db->curlevel;
	t.logstart = lseek(db->fd, 0, SEEK_END);
	assert(t.logstart != -1);
	t.logend = t.logstart;

	tp = &t;
	} else {
	tp = *mytid;
	}

	ptr = find_node(db, key, keylen, 0);

	if (ptr == db->map_base \|\| compare(KEY(ptr), KEYLEN(ptr), key, keylen)) {
	/* failed to find key/keylen */
	*data = NULL;
	*datalen = 0;
	} else {
	*datalen = DATALEN(ptr);
	*data = DATA(ptr);
	}

	if (mytid) {
	if (!*mytid) {
	/* return the txn structure */

	*mytid = xmalloc(sizeof(struct txn));
	memcpy(*mytid, tp, sizeof(struct txn));
	(*mytid)->ismalloc = 1;
	}
	} else {
	/* release read lock */
	if ((r = unlock(db)) < 0) {
	return r;
	}
	}

	return 0;
	}

	static int fetch(struct db *mydb,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid)
	{
	return myfetch(mydb, key, keylen, data, datalen, mytid);
	}

	static int fetchlock(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid)
	{
	return myfetch(db, key, keylen, data, datalen, mytid);
	}

	int myforeach(struct db *db,
	char *prefix, int prefixlen,
	foreach_p *goodp,
	foreach_cb cb, void rock,
	struct txn **tid)
	{
	const char *ptr;
	struct txn t, *tp;
	int r = 0, cb_r = 0;

	assert(db != NULL);

	if (!tid) {
	/* grab a r lock */
	if ((r = read_lock(db)) < 0) {
	return r;
	}

	tp = NULL;
	} else if (!*tid) {
	/* grab a r/w lock */
	if ((r = write_lock(db, NULL)) < 0) {
	return r;
	}

	/* fill in t */
	t.ismalloc = 0;
	t.oldcurlevel = db->curlevel;
	t.logstart = lseek(db->fd, 0, SEEK_END);
	assert(t.logstart != -1);
	t.logend = t.logstart;

	tp = &t;
	} else {
	tp = *tid;
	}

	ptr = find_node(db, prefix, prefixlen, 0);

	while (ptr != db->map_base) {
	int r;

	/* does it match prefix? */
	if (KEYLEN(ptr) < prefixlen) break;
	if (prefixlen && compare(KEY(ptr), prefixlen, prefix, prefixlen)) break;

	if (goodp(rock, KEY(ptr), KEYLEN(ptr), DATA(ptr), DATALEN(ptr))) {
	/* xxx need to unlock */

	/* make callback */
	cb_r = cb(rock, KEY(ptr), KEYLEN(ptr), DATA(ptr), DATALEN(ptr));
	if (cb_r) break;

	/* xxx relock, reposition */
	}

	ptr = db->map_base + FORWARD(ptr, 0);
	}

	if (tid) {
	if (!*tid) {
	/* return the txn structure */

	*tid = xmalloc(sizeof(struct txn));
	memcpy(*tid, tp, sizeof(struct txn));
	(*tid)->ismalloc = 1;
	}
	} else {
	/* release read lock */
	if ((r = unlock(db)) < 0) {
	return r;
	}
	}

	return r ? r : cb_r;
	}

	int randlvl(struct db *db)
	{
	int lvl = 1;

	while ((((float) rand() / (float) (RAND_MAX)) < PROB)
	&& (lvl < db->maxlevel)) {
	lvl++;
	}
	/* syslog(LOG_DEBUG, "picked level %d", lvl); */

	return lvl;
	}

	int mystore(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid, int overwrite)
	{
	const char *ptr;
	bit32 klen, dlen;
	struct iovec iov[50];
	int lvl;
	int num_iov;
	struct txn t, *tp;
	bit32 endpadding = htonl(-1);
	bit32 zeropadding[4] = { 0, 0, 0, 0 };
	int updateoffsets[SKIPLIST_MAXLEVEL];
	int newoffsets[SKIPLIST_MAXLEVEL];
	int addrectype = htonl(ADD);
	int delrectype = htonl(DELETE);
	bit32 todelete;
	bit32 newoffset;
	int r, i;

	assert(db != NULL);
	assert(key && keylen);

	if (!tid \|\| !*tid) {
	/* grab a r/w lock */
	if ((r = write_lock(db, NULL)) < 0) {
	return r;
	}

	/* fill in t */
	t.ismalloc = 0;
	t.oldcurlevel = db->curlevel;
	t.logstart = lseek(db->fd, 0, SEEK_END);
	assert(t.logstart != -1);
	t.logend = t.logstart;

	tp = &t;
	} else {
	tp = *tid;
	}

	num_iov = 0;

	newoffset = tp->logend;
	ptr = find_node(db, key, keylen, updateoffsets);
	if (ptr != db->map_base &&
	!compare(KEY(ptr), KEYLEN(ptr), key, keylen)) {

	if (!overwrite) {
	myabort(db, tp); /* releases lock */
	return CYRUSDB_EXISTS;
	} else {
	/* replace with an equal height node */
	lvl = LEVEL(ptr);

	/* log a removal */
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &delrectype, 4);
	todelete = htonl(ptr - db->map_base);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &todelete, 4);

	/* now we write at newoffset */
	newoffset += 8;

	/* our pointers are whatever the old node pointed to */
	for (i = 0; i < lvl; i++) {
	newoffsets[i] = htonl(FORWARD(ptr, i));
	}
	}
	} else {
	/* pick a size for the new node */
	lvl = randlvl(db);

	/* do we need to update the header ? */
	if (lvl > tp->oldcurlevel) {
	for (i = db->curlevel; i < lvl; i++) {
	updateoffsets[i] = DUMMY_OFFSET(db);
	}
	db->curlevel = lvl;

	/* write out that change */
	write_header(db); /* xxx errors? */
	}

	/* we point to what we're updating used to point to */
	/* newoffsets is written in the iovec later */
	for (i = 0; i < lvl; i++) {
	/* written in the iovec */
	newoffsets[i] =
	htonl(FORWARD(db->map_base + updateoffsets[i], i));
	}
	}

	klen = htonl(keylen);
	dlen = htonl(datalen);

	newoffset = htonl(newoffset);

	/* set pointers appropriately */
	for (i = 0; i < lvl; i++) {
	/* write pointer updates */
	/* FORWARD(updates[i], i) = newoffset; */
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &newoffset, 4);
	}

	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &addrectype, 4);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &klen, 4);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) key, keylen);
	if (ROUNDUP(keylen) - keylen > 0) {
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) zeropadding,
	ROUNDUP(keylen) - keylen);
	}
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &dlen, 4);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) data, datalen);
	if (ROUNDUP(datalen) - datalen > 0) {
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) zeropadding,
	ROUNDUP(datalen) - datalen);
	}
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char ) newoffsets, 4 lvl);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &endpadding, 4);

	lseek(db->fd, tp->logend, SEEK_SET);
	r = retry_writev(db->fd, iov, num_iov);
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: retry_writev(): %m");
	myabort(db, tp);
	return CYRUSDB_IOERROR;
	}
	tp->logend += r; /* update where to write next */

	if (tid) {
	if (!*tid) {
	/* return the txn structure */

	*tid = xmalloc(sizeof(struct txn));
	memcpy(*tid, tp, sizeof(struct txn));
	(*tid)->ismalloc = 1;
	}
	} else {
	/* commit the store, which releases the write lock */
	mycommit(db, tp);
	}

	return 0;
	}

	static int create(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 0);
	}

	static int store(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 1);
	}

	int mydelete(struct db *db,
	const char *key, int keylen,
	struct txn **tid)
	{
	const char *ptr;
	int delrectype = htonl(DELETE);
	int updateoffsets[SKIPLIST_MAXLEVEL];
	int offset;
	bit32 writebuf[2];
	struct txn t, *tp;
	int i;
	int r;

	if (!tid \|\| !*tid) {
	/* grab a r/w lock */
	if ((r = write_lock(db, NULL)) < 0) {
	return r;
	}

	/* fill in t */
	t.ismalloc = 0;
	t.oldcurlevel = db->curlevel;
	t.logstart = lseek(db->fd, 0, SEEK_END);
	assert(t.logstart != -1);
	t.logend = t.logstart;

	tp = &t;
	} else {
	tp = *tid;
	}

	ptr = find_node(db, key, keylen, updateoffsets);
	if (ptr == db->map_base \|\|
	!compare(KEY(ptr), KEYLEN(ptr), key, keylen)) {
	/* gotcha */
	offset = ptr - db->map_base;

	/* update pointers */
	for (i = 0; i < db->curlevel; i++) {
	int newoffset;

	if (FORWARD(db->map_base + updateoffsets[i], i) != offset) {
	break;
	}
	newoffset = htonl(FORWARD(ptr, i));
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &newoffset, 4);
	}

	/* log the deletion */
	lseek(db->fd, tp->logend, SEEK_SET);
	writebuf[0] = delrectype;
	writebuf[1] = htonl(offset);

	/* update end-of-log */
	tp->logend += retry_write(db->fd, (char *) writebuf, 8);
	}

	if (tid) {
	if (!*tid) {
	/* return the txn structure */

	*tid = xmalloc(sizeof(struct txn));
	memcpy(*tid, tp, sizeof(struct txn));
	(*tid)->ismalloc = 1;
	}
	} else {
	/* commit the store, which releases the write lock */
	mycommit(db, tp);
	}

	return 0;
	}

	int mycommit(struct db db, struct txn tid)
	{
	bit32 commitrectype = htonl(COMMIT);
	int r;

	assert(db && tid);

	/* fsync */
	if (do_fsync && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}

	/* write a commit record */
	lseek(db->fd, tid->logend, SEEK_SET);
	retry_write(db->fd, (char *) &commitrectype, 4);

	/* fsync */
	if (do_fsync && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}

	/* consider checkpointing */
	if (tid->logend > (2 * db->logstart + SKIPLIST_MINREWRITE)) {
	r = mycheckpoint(db, 1);
	}

	/* release the write lock */
	if ((r = unlock(db)) < 0) {
	return r;
	}

	/* free tid if needed */
	if (tid->ismalloc) {
	free(tid);
	}

	return 0;
	}

	int myabort(struct db db, struct txn tid) /* xxx */
	{
	const char *ptr;
	int updateoffsets[SKIPLIST_MAXLEVEL];
	int offset;
	int i;
	int r = 0;

	assert(db && tid);

	/* look at the log entries we've written, and undo their effects */
	while (tid->logstart != tid->logend) {
	/* find the last log entry */
	for (offset = tid->logstart, ptr = db->map_base + offset;
	offset + RECSIZE(ptr) != tid->logend;
	offset += RECSIZE(ptr), ptr = db->map_base + offset) ;

	offset = ptr - db->map_base;

	assert(TYPE(ptr) == ADD \|\| TYPE(ptr) == DELETE);
	switch (TYPE(ptr)) {
	case DUMMY:
	case INORDER:
	case COMMIT:
	abort();

	case ADD:
	/* remove this record */
	(void) find_node(db, KEY(ptr), KEYLEN(ptr), updateoffsets);
	for (i = 0; i < db->curlevel; i++) {
	int newoffset;

	if (FORWARD(db->map_base + updateoffsets[i], i) != offset) {
	break;
	}

	newoffset = htonl(FORWARD(ptr, i));
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &newoffset, 4);
	}
	break;
	case DELETE:
	{
	int lvl;
	int newoffset;
	const char *q;

	/* re-add this record. it can't exist right now. */
	newoffset = ((bit32 )(ptr + 4));
	q = db->map_base + ntohl(newoffset);
	lvl = LEVEL(q);
	(void) find_node(db, KEY(q), KEYLEN(q), updateoffsets);
	for (i = 0; i < lvl; i++) {
	/* the current pointers FROM this node are correct,
	so we just have to update 'updateoffsets' */
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &newoffset, 4);
	}
	break;
	}
	}

	/* remove looking at this */
	tid->logend -= RECSIZE(ptr);
	}

	/* truncate the file to remove log entries */
	if (ftruncate(db->fd, tid->logstart) < 0) {
	syslog(LOG_ERR,
	"DBERROR: skiplist abort %s: ftruncate: %m",
	db->fname);
	r = CYRUSDB_IOERROR;
	unlock(db);
	return r;
	}

	/* release the write lock */
	if ((r = unlock(db)) < 0) {
	return r;
	}

	/* free the tid */
	if (tid->ismalloc) {
	free(tid);
	}

	return 0;
	}

	/* compress 'db'. if 'locked != 0', the database is already R/W locked and
	will be returned as such. */
	static int mycheckpoint(struct db *db, int locked)
	{
	char fname[1024];
	int oldfd;
	struct iovec iov[50];
	int num_iov;
	int updateoffsets[SKIPLIST_MAXLEVEL];
	const char *ptr;
	int offset;
	int r = 0;
	int iorectype = htonl(INORDER);
	int i;
	time_t start = time(NULL);

	/* grab write lock (could be read but this prevents multiple checkpoints
	simultaneously) */
	if (!locked) {
	r = write_lock(db, NULL);
	if (r < 0) return r;
	} else {
	/* we need the latest and greatest data */
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, MAP_UNKNOWN_LEN,
	db->fname, 0);
	}

	/* open fname.NEW */
	snprintf(fname, sizeof(fname), "%s.NEW", db->fname);
	oldfd = db->fd;
	db->fd = open(fname, O_RDWR \| O_CREAT, 0666);
	if (db->fd < 0) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint: open(%s): %m", fname);
	if (!locked) unlock(db);
	db->fd = oldfd;
	return CYRUSDB_IOERROR;
	}

	/* write dummy record */
	if (!r) {
	int dsize = DUMMY_SIZE(db);
	bit32 buf = (bit32 ) xzmalloc(dsize);

	buf[0] = htonl(DUMMY);
	buf[(dsize / 4) - 1] = htonl(-1);

	lseek(db->fd, DUMMY_OFFSET(db), SEEK_SET);
	r = retry_write(db->fd, (char *) buf, dsize);
	if (r != dsize) {
	r = CYRUSDB_IOERROR;
	} else {
	r = 0;
	}
	free(buf);

	/* initialize the updateoffsets array so when we append records
	we know where to set the pointers */
	for (i = 0; i < db->maxlevel; i++) {
	/* header_size + 4 (rectype) + 4 (ksize) + 4 (dsize)
	+ 4 * i */
	updateoffsets[i] = DUMMY_OFFSET(db) + 12 + 4 * i;
	}
	}

	/* write records to new file */
	offset = FORWARD(db->map_base + DUMMY_OFFSET(db), 0);
	db->listsize = 0;
	while (!r && offset != 0) {
	int lvl;
	bit32 newoffset, newoffsetnet;

	ptr = db->map_base + offset;
	lvl = LEVEL(ptr);
	db->listsize++;

	num_iov = 0;
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &iorectype, 4);
	/* copy all but the rectype from the record */
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) ptr + 4, RECSIZE(ptr) - 4);

	newoffset = lseek(db->fd, 0, SEEK_END);
	newoffsetnet = htonl(newoffset);
	r = retry_writev(db->fd, iov, num_iov);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	} else {
	r = 0;
	}
	for (i = 0; !r && i < lvl; i++) {
	/* update pointers */
	r = lseek(db->fd, updateoffsets[i], SEEK_SET);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	r = retry_write(db->fd, (char *) &newoffsetnet, 4);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	/* PTR(ptr, i) - ptr is the offset relative to me
	to my ith pointer */
	updateoffsets[i] = newoffset + (PTR(ptr, i) - ptr);
	}

	offset = FORWARD(ptr, 0);
	}

	/* set any dangling pointers to zero */
	for (i = 0; !r && i < db->maxlevel; i++) {
	bit32 newoffset = htonl(0);

	r = lseek(db->fd, updateoffsets[i], SEEK_SET);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	r = retry_write(db->fd, (char *) &newoffset, 4);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}
	}

	/* create the header */
	db->logstart = lseek(db->fd, 0, SEEK_END);
	r = write_header(db);

	/* sync new file */
	if (!r && do_fsync && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint: fdatasync(%s): %m", fname);
	r = CYRUSDB_IOERROR;
	}

	if (!r) {
	/* get new lock */
	r = write_lock(db, fname);
	}

	/* move new file to original file name */
	if (!r && (rename(fname, db->fname) < 0)) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint: rename(%s, %s): %m",
	fname, db->fname);
	r = CYRUSDB_IOERROR;
	}

	/* force the new file name to disk */
	if (!r && do_fsync && (fsync(db->fd) < 0)) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint: fsync(%s): %m", fname);
	r = CYRUSDB_IOERROR;
	}

	if (r) {
	/* clean up */
	close(db->fd);
	db->fd = oldfd;
	unlink(fname);
	}

	/* release old write lock */
	close(oldfd);

	{
	struct stat sbuf;

	/* let's make sure we're up to date */
	map_free(&db->map_base, &db->map_len);
	if (fstat(db->fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}
	db->map_size = sbuf.st_size;
	db->map_ino = sbuf.st_ino;
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, sbuf.st_size,
	db->fname, 0);
	}

	if (!locked) {
	/* unlock the new db files */
	unlock(db);
	}

	{
	int diff = time(NULL) - start;
	syslog(LOG_INFO,
	"skiplist: checkpointed %s (%d record%s, %d bytes) in %d second%s",
	db->fname, db->listsize, db->listsize == 1 ? "" : "s",
	db->logstart, diff, diff == 1 ? "" : "s");
	}

	return r;
	}

	/* dump the database.
	if detail == 1, dump all records.
	if detail == 2, also dump pointers for active records.
	if detail == 3, dump all records/all pointers.
	*/
	static int dump(struct db *db, int detail)
	{
	const char ptr, end;
	int i;

	read_lock(db);

	ptr = db->map_base + DUMMY_OFFSET(db);
	end = db->map_base + db->map_size;
	while (ptr < end) {
	printf("%04X: ", ptr - db->map_base);
	switch (TYPE(ptr)) {
	case DUMMY:
	printf("DUMMY ");
	break;
	case INORDER:
	printf("INORDER ");
	break;
	case ADD:
	printf("ADD ");
	break;
	case DELETE:
	printf("DELETE ");
	break;
	case COMMIT:
	printf("COMMIT ");
	break;
	}

	switch (TYPE(ptr)) {
	case DUMMY:
	case INORDER:
	case ADD:
	printf("kl=%d dl=%d lvl=%d\n",
	KEYLEN(ptr), DATALEN(ptr), LEVEL(ptr));
	printf("\t");
	for (i = 0; i < LEVEL(ptr); i++) {
	printf("%04X ", FORWARD(ptr, i));
	}
	printf("\n");
	break;

	case DELETE:
	printf("offset=%04X\n", ntohl(((bit32 )(ptr + 4))));
	break;

	case COMMIT:
	printf("\n");
	break;
	}

	ptr += RECSIZE(ptr);
	}

	unlock(db);
	return 0;
	}

	/* perform some basic consistency checks */
	static int consistent(struct db db) / xxx */
	{
	const char *ptr;
	int offset;

	read_lock(db);

	offset = FORWARD(db->map_base + DUMMY_OFFSET(db), 0);
	while (offset != 0) {
	ptr = db->map_base + offset;

	offset = FORWARD(ptr, 0);
	if (offset != 0) {
	/* check to see that ptr < ptr -> next */

	}
	}

	return 0;
	}

	/* run recovery on this file */
	static int recovery(struct db *db)
	{
	const char ptr, keyptr;
	int updateoffsets[SKIPLIST_MAXLEVEL];
	int offset, offsetnet, myoff = 0;
	int r = 0;
	time_t start = time(NULL);
	int i;

	if ((r = write_lock(db, NULL)) < 0) {
	return r;
	}

	if (global_recovery && db->last_recovery >= global_recovery) {
	/* someone beat us to it */
	unlock(db);
	return 0;
	}
	db->listsize = 0;

	ptr = DUMMY_PTR(db);
	r = 0;

	/* verify this is DUMMY */
	if (!r && TYPE(ptr) != DUMMY) {
	r = CYRUSDB_IOERROR;
	syslog(LOG_ERR, "DBERROR: skiplist recovery %s: no dummy node?",
	db->fname);
	}

	/* zero key */
	if (!r && KEYLEN(ptr) != 0) {
	r = CYRUSDB_IOERROR;
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: dummy node KEYLEN != 0",
	db->fname);
	}

	/* zero data */
	if (!r && DATALEN(ptr) != 0) {
	r = CYRUSDB_IOERROR;
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: dummy node DATALEN != 0",
	db->fname);
	}

	/* pointers for db->maxlevel */
	if (!r && LEVEL(ptr) != db->maxlevel) {
	r = CYRUSDB_IOERROR;
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: dummy node level: %d != %d",
	db->fname, LEVEL(ptr), db->maxlevel);
	}

	for (i = 0; i < db->maxlevel; i++) {
	/* header_size + 4 (rectype) + 4 (ksize) + 4 (dsize)
	+ 4 * i */
	updateoffsets[i] = DUMMY_OFFSET(db) + 12 + 4 * i;
	}

	/* reset the data that was written INORDER by the last checkpoint */
	offset = DUMMY_OFFSET(db) + DUMMY_SIZE(db);
	while (!r && (offset < db->logstart)) {
	ptr = db->map_base + offset;
	offsetnet = htonl(offset);

	db->listsize++;

	/* make sure this is INORDER */
	if (TYPE(ptr) != INORDER) {
	syslog(LOG_ERR, "DBERROR: skiplist recovery: %04X should be INORDER",
	offset);
	r = CYRUSDB_IOERROR;
	continue;
	}

	/* xxx check \0 fill on key */

	/* xxx check \0 fill on data */

	/* update previous pointers, record these for updating */
	for (i = 0; !r && i < LEVEL(ptr); i++) {
	r = lseek(db->fd, updateoffsets[i], SEEK_SET);
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: lseek %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	r = retry_write(db->fd, (char *) &offsetnet, 4);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	/* PTR(ptr, i) - ptr is the offset relative to me
	to my ith pointer */
	updateoffsets[i] = offset + (PTR(ptr, i) - ptr);
	}

	/* check padding */
	if (!r && PADDING(ptr) != -1) {
	syslog(LOG_ERR, "DBERROR: %s: offset %04X padding not -1",
	db->fname, offset);
	r = CYRUSDB_IOERROR;
	}

	if (!r) {
	offset += RECSIZE(ptr);
	}
	}

	/* zero out the remaining pointers */
	if (!r) {
	for (i = 0; !r && i < db->maxlevel; i++) {
	int zerooffset = 0;

	r = lseek(db->fd, updateoffsets[i], SEEK_SET);
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: lseek %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	r = retry_write(db->fd, (char *) &zerooffset, 4);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}
	}
	}

	/* replay the log */
	while (!r && offset < db->map_size) {
	const char p, q;

	/* refresh map, so we see the writes we've just done */
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, db->map_size,
	db->fname, 0);

	ptr = db->map_base + offset;
	offsetnet = htonl(offset);

	/* if this is a commit, we've processed everything in this txn */
	if (TYPE(ptr) == COMMIT) {
	offset += RECSIZE(ptr);
	continue;
	}

	/* make sure this is ADD or DELETE */
	if (TYPE(ptr) != ADD && TYPE(ptr) != DELETE) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery: %04X should be ADD or DELETE",
	offset);
	r = CYRUSDB_IOERROR;
	break;
	}

	/* look ahead for a commit */
	q = db->map_base + db->map_size;
	p = ptr;
	for (;;) {
	p += RECSIZE(p);
	if (p >= q) break;
	if (TYPE(p) == COMMIT) break;
	}
	if (p >= q) {
	syslog(LOG_NOTICE,
	"skiplist recovery %s: found partial txn, not replaying",
	db->fname);

	/* no commit, we should truncate */
	if (ftruncate(db->fd, offset) < 0) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: ftruncate: %m",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	break;
	}

	keyptr = NULL;
	/* look for the key */
	if (TYPE(ptr) == ADD) {
	keyptr = find_node(db, KEY(ptr), KEYLEN(ptr), updateoffsets);
	if (keyptr == db->map_base \|\|
	compare(KEY(ptr), KEYLEN(ptr), KEY(keyptr), KEYLEN(keyptr))) {
	/* didn't find exactly this node */
	keyptr = NULL;
	}
	} else { /* type == DELETE */
	const char *p;

	myoff = ntohl(((bit32 )(ptr + 4)));
	p = db->map_base + myoff;
	keyptr = find_node(db, KEY(p), KEYLEN(p), updateoffsets);
	if (keyptr == db->map_base) {
	keyptr = NULL;
	}
	}

	/* if DELETE & found key, skip over it */
	if (TYPE(ptr) == DELETE && keyptr) {
	db->listsize--;

	for (i = 0; i < db->curlevel; i++) {
	int newoffset;

	if (FORWARD(db->map_base + updateoffsets[i], i) != myoff) {
	break;
	}
	newoffset = htonl(FORWARD(db->map_base + myoff, i));
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &newoffset, 4);
	}

	/* otherwise if DELETE, throw an error */
	} else if (TYPE(ptr) == DELETE) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: DELETE at %04X doesn't exist",
	db->fname, offset);
	r = CYRUSDB_IOERROR;

	/* otherwise if ADD & found key, throw an error */
	} else if (TYPE(ptr) == ADD && keyptr) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: ADD at %04X exists",
	db->fname, offset);
	r = CYRUSDB_IOERROR;

	/* otherwise insert it */
	} else if (TYPE(ptr) == ADD) {
	int lvl;
	bit32 newoffsets[SKIPLIST_MAXLEVEL];

	db->listsize++;
	offsetnet = htonl(offset);

	lvl = LEVEL(ptr);
	for (i = 0; i < lvl; i++) {
	/* set our next pointers */
	newoffsets[i] =
	htonl(FORWARD(db->map_base + updateoffsets[i], i));

	/* replace 'updateoffsets' to point to me */
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &offsetnet, 4);
	}
	/* write out newoffsets */
	lseek(db->fd, FIRSTPTR(ptr) - db->map_base, SEEK_SET);
	retry_write(db->fd, (char ) newoffsets, 4 lvl);

	/* can't happen */
	} else {
	abort();
	}

	/* move to next record */
	offset += RECSIZE(ptr);
	}

	/* fsync the recovered database */
	if (!r && do_fsync && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: fdatasync: %m", db->fname);
	r = CYRUSDB_IOERROR;
	}

	/* set the last recovery timestamp */
	if (!r) {
	db->last_recovery = time(NULL);
	write_header(db);
	}

	/* fsync the new header */
	if (!r && do_fsync && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: fdatasync: %m", db->fname);
	r = CYRUSDB_IOERROR;
	}

	if (!r) {
	int diff = time(NULL) - start;

	syslog(LOG_NOTICE,
	- "skiplist: recovered %s (%d record%s, %d bytes) in %d second%s",
	+ "skiplist: recovered %s (%d record%s, %ld bytes) in %d second%s",
	db->fname, db->listsize, db->listsize == 1 ? "" : "s",
	db->map_size, diff, diff == 1 ? "" : "s");
	}

	unlock(db);

	return r;
	}

	struct cyrusdb_backend cyrusdb_skiplist =
	{
	"skiplist", /* name */

	&myinit,
	&mydone,
	&mysync,
	&myarchive,

	&myopen,
	&myclose,

	&fetch,
	&fetchlock,
	&myforeach,
	&create,
	&store,
	&mydelete,

	&mycommit,
	&myabort,

	&dump
	};
	diff --git a/lib/iptostring.c b/lib/iptostring.c
	index c567c3a66..74da61b83 100644
	--- a/lib/iptostring.c
	+++ b/lib/iptostring.c
	@@ -1,73 +1,74 @@
	/*
	* Copyright (c) 1998-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*/

	-/* $Id: iptostring.c,v 1.3 2002/02/13 20:57:18 rjs3 Exp $ */
	+/* $Id: iptostring.c,v 1.4 2002/02/19 18:50:15 ken3 Exp $ */

	#include <config.h>
	#include <stdlib.h>
	+#include <string.h>
	#include <stdio.h>
	#include <netdb.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <errno.h>
	#include "iptostring.h"

	int iptostring(const struct sockaddr *addr, socklen_t addrlen,
	char *out, unsigned outlen) {
	char hbuf[NI_MAXHOST], pbuf[NI_MAXSERV];

	if(!addr \|\| !out) {
	errno = EINVAL;
	return -1;
	}

	getnameinfo(addr, addrlen, hbuf, sizeof(hbuf), pbuf, sizeof(pbuf),
	NI_NUMERICHOST \| NI_WITHSCOPEID \| NI_NUMERICSERV);

	if(outlen < strlen(hbuf) + strlen(pbuf) + 2) {
	errno = ENOMEM;
	return -1;
	}

	snprintf(out, outlen, "%s;%s", hbuf, pbuf);

	return 0;
	}
	diff --git a/master/cyrusMasterMIB.c b/master/cyrusMasterMIB.c
	index cd996ac5b..f072f9565 100644
	--- a/master/cyrusMasterMIB.c
	+++ b/master/cyrusMasterMIB.c
	@@ -1,226 +1,228 @@
	#include <config.h>

	#ifdef HAVE_UCDSNMP

	/* This file was generated by mib2c and is intended for use as a mib module
	for the ucd-snmp snmpd agent. */


	#ifdef IN_UCD_SNMP_SOURCE
	/* If we're compiling this file inside the ucd-snmp source tree */


	/* This should always be included first before anything else */
	#include <config.h>


	/* minimal include directives */
	#include "mibincl.h"
	#include "util_funcs.h"


	#else /* !IN_UCD_SNMP_SOURCE */


	#include <ucd-snmp/ucd-snmp-config.h>
	#include <ucd-snmp/ucd-snmp-includes.h>
	#include <ucd-snmp/ucd-snmp-agent-includes.h>
	+#include <ucd-snmp/util_funcs.h>


	#endif /* !IN_UCD_SNMP_SOURCE */

	#include <time.h>
	+#include <string.h>

	#include "cyrusMasterMIB.h"

	#include "master.h"
	#include "../imap/version.h"

	/*
	* cyrusMasterMIB_variables_oid:
	* this is the top level oid that we want to register under. This
	* is essentially a prefix, with the suffix appearing in the
	* variable below.
	*/

	oid cyrusMasterMIB_variables_oid[] = { 1,3,6,1,4,1,3,6,1 };


	/*
	* variable4 cyrusMasterMIB_variables:
	* this variable defines function callbacks and type return information
	* for the cyrusMasterMIB mib section
	*/


	struct variable4 cyrusMasterMIB_variables[] = {
	/* magic number , variable type , ro/rw , callback fn , L, oidsuffix */
	#define CYRUSMASTERINFODESCR 1
	{ CYRUSMASTERINFODESCR, ASN_OCTET_STR , RONLY , var_cyrusMasterMIB, 2, { 1,1 } },
	#define CYRUSMASTERINFOVERS 2
	{ CYRUSMASTERINFOVERS , ASN_OCTET_STR , RONLY , var_cyrusMasterMIB, 2, { 1,2 } },
	#define CYRUSMASTERINFOUPTIME 3
	{ CYRUSMASTERINFOUPTIME , ASN_TIMETICKS , RONLY , var_cyrusMasterMIB, 2, { 1,3 } },
	#define SERVICEFORKS 5
	{ SERVICEFORKS , ASN_COUNTER , RONLY , var_serviceTable, 3, { 2,1,1 } },
	#define SERVICEACTIVE 6
	{ SERVICEACTIVE , ASN_GAUGE , RONLY , var_serviceTable, 3, { 2,1,2 } },
	#define SERVICENAME 7
	{ SERVICENAME , ASN_OCTET_STR , RONLY , var_serviceTable, 3, { 2,1,3 } },
	#define SERVICEID 8
	{ SERVICEID , ASN_INTEGER , NOACCESS , var_serviceTable, 3, { 2,1,4 } },
	#define SERVICECONNS 9
	{ SERVICECONNS , ASN_COUNTER , NOACCESS , var_serviceTable, 3, { 2,1,5 } },
	};
	/* (L = length of the oidsuffix) */


	static time_t startTime = 0;

	/*
	* init_cyrusMasterMIB():
	* Initialization routine. This is called when the agent starts up.
	* At a minimum, registration of your variables should take place here.
	*/
	void init_cyrusMasterMIB(void)
	{
	/* register ourselves with the agent to handle our mib tree */
	REGISTER_MIB("cyrusMasterMIB", cyrusMasterMIB_variables, variable4,
	cyrusMasterMIB_variables_oid);


	/* place any other initialization junk you need here */
	if (!startTime) {
	startTime = time(NULL);
	}
	}


	/*
	* var_cyrusMasterMIB():
	* This function is called every time the agent gets a request for
	* a scalar variable that might be found within your mib section
	* registered above. It is up to you to do the right thing and
	* return the correct value.
	* You should also correct the value of "var_len" if necessary.
	*
	* Please see the documentation for more information about writing
	* module extensions, and check out the examples in the examples
	* and mibII directories.
	*/
	unsigned char *
	var_cyrusMasterMIB(struct variable *vp,
	oid *name,
	size_t *length,
	int exact,
	size_t *var_len,
	WriteMethod **write_method)
	{
	/* variables we may use later */
	static long long_ret;
	static unsigned char string[SPRINT_MAX_LEN];
	/* static oid objid[MAX_OID_LEN]; */
	/* static struct counter64 c64; */

	if (header_generic(vp,name,length,exact,var_len,write_method)
	== MATCH_FAILED )
	return NULL;

	/*
	* this is where we do the value assignments for the mib results.
	*/
	switch(vp->magic) {
	case CYRUSMASTERINFODESCR:
	strcpy(string, "Cyrus IMAP server master process");
	*var_len = strlen(string);
	return (unsigned char *) string;

	case CYRUSMASTERINFOVERS:
	strcpy(string, CYRUS_VERSION);
	*var_len = strlen(string);
	return (unsigned char *) string;

	case CYRUSMASTERINFOUPTIME:
	long_ret = 100 * (time(NULL) - startTime);
	return (unsigned char *) &long_ret;

	default:
	ERROR_MSG("");
	}
	return NULL;
	}


	/*
	* var_serviceTable():
	* Handle this table separately from the scalar value case.
	* The workings of this are basically the same as for var_cyrusMasterMIB above.
	*/
	unsigned char *
	var_serviceTable(struct variable *vp,
	oid *name,
	size_t *length,
	int exact,
	size_t *var_len,
	WriteMethod **write_method)
	{
	/* variables we may use later */
	static long long_ret;
	static unsigned char string[SPRINT_MAX_LEN];
	/* static oid objid[MAX_OID_LEN]; */
	/* static struct counter64 c64; */
	int index;

	/*
	* This assumes that the table is a 'simple' table.
	* See the implementation documentation for the meaning of this.
	* You will need to provide the correct value for the TABLE_SIZE parameter
	*
	* If this table does not meet the requirements for a simple table,
	* you will need to provide the replacement code yourself.
	* Mib2c is not smart enough to write this for you.
	* Again, see the implementation documentation for what is required.
	*/
	if (header_simple_table(vp,name,length,exact,var_len,write_method, nservices)
	== MATCH_FAILED )
	return NULL;


	index = name[*length - 1];

	/*
	* this is where we do the value assignments for the mib results.
	*/
	switch(vp->magic) {
	case SERVICEFORKS:
	long_ret = Services[index - 1].nforks;
	return (unsigned char *) &long_ret;

	case SERVICEACTIVE:
	long_ret = Services[index - 1].nactive;
	return (unsigned char *) &long_ret;

	case SERVICENAME:
	strcpy(string, Services[index - 1].name);
	*var_len = strlen(string);
	return (unsigned char *) string;

	case SERVICEID:
	long_ret = index;
	return (unsigned char *) &long_ret;

	case SERVICECONNS:
	long_ret = Services[index - 1].nconnections;
	return (unsigned char *) &long_ret;

	default:
	ERROR_MSG("");
	}
	return NULL;
	}





	#endif /* HAVE_UCDSNMP */
	diff --git a/master/master.c b/master/master.c
	index 8cbbe2691..1682a4b94 100644
	--- a/master/master.c
	+++ b/master/master.c
	@@ -1,1246 +1,1247 @@
	/* master.c -- IMAP master process to handle recovery, checkpointing, spawning
	*
	* Copyright (c) 2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*/

	-/* $Id: master.c,v 1.59 2002/02/18 20:05:00 rjs3 Exp $ */
	+/* $Id: master.c,v 1.60 2002/02/19 18:50:15 ken3 Exp $ */

	#include <config.h>

	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	+#include <time.h>
	#include <sys/time.h>
	#include <grp.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#ifdef HAVE_SYS_RESOURCE_H
	#include <sys/resource.h>
	#endif
	#include <fcntl.h>
	#include <signal.h>
	#include <pwd.h>
	#include <sys/param.h>
	#include <sys/stat.h>
	#include <syslog.h>
	#include <netdb.h>
	#include <ctype.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <sys/un.h>
	#include <arpa/inet.h>
	#include <sysexits.h>
	#include <errno.h>
	#include <limits.h>

	#ifndef INADDR_NONE
	#define INADDR_NONE 0xffffffff
	#endif

	#ifndef INADDR_ANY
	#define INADDR_ANY 0x00000000
	#endif

	#ifdef HAVE_UCDSNMP
	#include <ucd-snmp/ucd-snmp-config.h>
	#include <ucd-snmp/ucd-snmp-includes.h>
	#include <ucd-snmp/ucd-snmp-agent-includes.h>

	#include "cyrusMasterMIB.h"
	#endif

	#include "masterconf.h"

	#include "master.h"
	#include "service.h"

	enum {
	become_cyrus_early = 1,
	child_table_size = 10000,
	child_table_inc = 100
	};

	static int verbose = 0;
	static int listen_queue_backlog = 32;

	struct service *Services = NULL;
	int allocservices = 0;
	int nservices = 0;

	struct recover {
	char *name;
	char const exec;
	};

	struct event {
	char *name;
	time_t mark;
	time_t period;
	char const exec;
	struct event *next;
	};
	static struct event *schedule = NULL;

	struct centry {
	pid_t pid;
	struct service *s;
	struct centry *next;
	};
	static struct centry *ctable[child_table_size];
	static struct centry *cfreelist;

	void limit_fds(rlim_t);

	static char mystrdup(const char s)
	{
	if (!s) return NULL;
	return strdup(s);
	}

	void fatal(char *msg, int code)
	{
	syslog(LOG_CRIT, "%s", msg);
	syslog(LOG_NOTICE, "exiting");
	exit(code);
	}

	int become_cyrus(void)
	{
	struct passwd *p;
	int newuid, newgid;
	int result;
	static int uid = 0;

	if (uid) return setuid(uid);

	p = getpwnam(CYRUS_USER);
	if (p == NULL) {
	syslog(LOG_ERR, "no entry in /etc/passwd for user %s", CYRUS_USER);
	return -1;
	}

	/* Save these in case initgroups does a getpw() /
	newuid = p->pw_uid;
	newgid = p->pw_gid;

	if (initgroups(CYRUS_USER, newgid)) {
	syslog(LOG_ERR, "unable to initialize groups for user %s: %s",
	CYRUS_USER, strerror(errno));
	return -1;
	}

	if (setgid(newgid)) {
	syslog(LOG_ERR, "unable to set group id to %d for user %s: %s",
	newgid, CYRUS_USER, strerror(errno));
	return -1;
	}

	result = setuid(newuid);

	/* Only set static uid if successful, else future calls won't reset gid */
	if (result == 0)
	uid = newuid;
	return result;
	}

	void get_prog(char path, char const *cmd)
	{
	if (cmd[0][0] == '/') strcpy(path, cmd[0]);
	else sprintf(path, "%s/%s", SERVICE_PATH, cmd[0]);
	}

	void get_statsock(int filedes[2])
	{
	int r, fdflags;

	r = pipe(filedes);
	if (r != 0) {
	fatal("couldn't create status socket: %m", 1);
	}

	/* we don't want the master blocking on reads */
	fdflags = fcntl(filedes[0], F_GETFL, 0);
	if (fdflags != -1) fdflags = fcntl(filedes[0], F_SETFL,
	fdflags \| O_NONBLOCK);
	if (fdflags == -1) {
	fatal("unable to set non-blocking: %m", 1);
	}
	/* we don't want the services to be able to read from it */
	fdflags = fcntl(filedes[0], F_GETFD, 0);
	if (fdflags != -1) fdflags = fcntl(filedes[0], F_SETFD,
	fdflags \| FD_CLOEXEC);
	if (fdflags == -1) {
	fatal("unable to set close-on-exec: %m", 1);
	}
	}

	/* return a new 'centry', either from the freelist or by malloc'ing it */
	static struct centry *get_centry(void)
	{
	struct centry *t;

	if (!cfreelist) {
	/* create child_table_inc more and add them to the freelist */
	struct centry *n;
	int i;

	n = malloc(child_table_inc * sizeof(struct centry));
	cfreelist = n;
	for (i = 0; i < child_table_inc - 1; i++) {
	n[i].next = n + (i + 1);
	}
	/* i == child_table_inc - 1, last item in block */
	n[i].next = NULL;
	}

	t = cfreelist;
	cfreelist = cfreelist->next;

	return t;
	}

	/* see if 'listen' parameter has both hostname and port, or just port */
	char parse_listen(char listen)
	{
	char *cp;
	char *port = NULL;

	if ((cp = strrchr(listen,']')) != NULL) {
	/* ":port" after closing bracket for IP address? */
	if (cp++ != '\0' && cp == ':') {
	*cp++ = '\0';
	if (*cp != '\0') {
	port = cp;
	}
	}
	} else if ((cp = strrchr(listen,':')) != NULL) {
	/* ":port" after hostname? */
	*cp++ = '\0';
	if (*cp != '\0') {
	port = cp;
	}
	}
	return port;
	}

	/* set sin_port accordingly. return of 0 indicates failure. */
	int resolve_port(char port, struct service s, struct sockaddr_in *sin)
	{
	struct servent *serv;

	serv = getservbyname(port, s->proto);
	if (serv) {
	sin->sin_port = serv->s_port;
	} else {
	sin->sin_port = htons(atoi(port));
	if (sin->sin_port == 0) {
	syslog(LOG_INFO, "no service '%s' in /etc/services, "
	"disabling %s", port, s->name);
	s->exec = NULL;
	return 0;
	}
	}
	return 1;
	}

	/* set sin_addr accordingly. return of 0 indicates failure. */
	int resolve_host(char listen, struct sockaddr_in sin)
	{
	struct hostent *hp;
	char *cp;

	/* do we have a hostname, or IP number? */
	/* XXX are brackets necessary, like for IPv6 later? */
	if (*listen == '[') {
	listen++; /* skip first bracket */
	if ((cp = strrchr(listen,']')) != NULL) {
	*cp = '\0';
	}
	}
	sin->sin_addr.s_addr = inet_addr(listen);
	if ((sin->sin_addr.s_addr == INADDR_NONE) \|\| (sin->sin_addr.s_addr == 0)) {
	/* looks like it isn't an IP address, so look up the host */
	if ((hp = gethostbyname(listen)) == 0) {
	syslog(LOG_INFO, "host not found: %s", listen);
	return 0;
	}
	if (hp->h_addrtype != AF_INET) {
	syslog(LOG_INFO, "unexpected address family: %d", hp->h_addrtype);
	return 0;
	}
	if (hp->h_length != sizeof(sin->sin_addr)) {
	syslog(LOG_INFO, "unexpected address length %d", hp->h_length);
	return 0;
	}
	memcpy((char *) &sin->sin_addr, hp->h_addr, hp->h_length);
	}
	return 1;
	}

	void service_create(struct service *s)
	{
	struct sockaddr_in sin;
	struct sockaddr_un sunsock;
	struct sockaddr *sa;
	mode_t oldumask;
	int on = 1, salen;
	int r;

	memset(&sin, 0, sizeof(sin));

	if (s->listen[0] == '/') { /* unix socket */
	sunsock.sun_family = AF_UNIX;
	strcpy(sunsock.sun_path, s->listen);
	unlink(s->listen);
	sa = (struct sockaddr *) &sunsock;
	salen = sizeof(sunsock.sun_family) + strlen(sunsock.sun_path) + 1;

	s->socket = socket(AF_UNIX, SOCK_STREAM, 0);
	} else { /* inet socket */
	char listen, port;

	sin.sin_family = AF_INET;

	/* parse_listen() and resolve_host() are destructive,
	* so make a work copy of s->listen
	*/
	listen = strdup(s->listen);

	if ((port = parse_listen(listen)) == NULL) {
	/* listen IS the port */
	if (!resolve_port(listen, s, &sin)) {
	free(listen);
	return;
	}
	sin.sin_addr.s_addr = INADDR_ANY;
	} else {
	/* listen is now just the address */
	if (!resolve_port(port, s, &sin)) {
	free(listen);
	return;
	}
	if (!resolve_host(listen, &sin)) {
	s->exec = NULL;
	free(listen);
	return;
	}
	}
	sa = (struct sockaddr *) &sin;
	salen = sizeof(sin);

	s->socket = socket(AF_INET, SOCK_STREAM, 0);

	free(listen);
	}

	if (s->socket < 0) {
	syslog(LOG_ERR, "unable to create %s listener socket: %m", s->name);
	s->exec = NULL;
	return;
	}

	/* allow reuse of address */
	r = setsockopt(s->socket, SOL_SOCKET, SO_REUSEADDR,
	(void *) &on, sizeof(on));
	if (r < 0) {
	syslog(LOG_ERR, "unable to setsocketopt(SO_REUSEADDR): %m");
	}

	oldumask = umask((mode_t) 0); /* for linux */
	r = bind(s->socket, sa, salen);
	umask(oldumask);
	if (r < 0) {
	syslog(LOG_ERR, "unable to bind %s socket: %m", s->name);
	close(s->socket);
	s->socket = 0;
	s->exec = NULL;
	return;
	}

	if (s->listen[0] == '/') { /* unix socket */
	/* for DUX, where this isn't the default.
	(harmlessly fails on some systems) */
	chmod(s->listen, (mode_t) 0777);
	}

	if (listen(s->socket, listen_queue_backlog) < 0) {
	syslog(LOG_ERR, "unable to listen to %s socket: %m", s->name);
	close(s->socket);
	s->socket = 0;
	s->exec = NULL;
	return;
	}

	s->ready_workers = 0;

	get_statsock(s->stat);
	}

	void run_startup(char **cmd)
	{
	pid_t pid;
	int status;
	char path[1024];

	switch (pid = fork()) {
	case -1:
	syslog(LOG_CRIT, "can't fork process to run startup");
	fatal("can't run startup", 1);
	break;

	case 0:
	if (become_cyrus() != 0) {
	syslog(LOG_ERR, "can't change to the cyrus user");
	exit(1);
	}

	limit_fds(256);

	get_prog(path, cmd);
	syslog(LOG_DEBUG, "about to exec %s", path);
	execv(path, cmd);
	syslog(LOG_ERR, "can't exec %s for startup: %m", path);
	exit(EX_OSERR);

	default:
	if (waitpid(pid, &status, 0) < 0) {
	syslog(LOG_ERR, "waitpid(): %m");
	} else if (status != 0) {
	if (WIFEXITED(status)) {
	syslog(LOG_ERR, "process %d exited, status %d\n", pid,
	WEXITSTATUS(status));
	}
	if (WIFSIGNALED(status)) {
	syslog(LOG_ERR,
	"process %d exited, signaled to death by %d\n",
	pid, WTERMSIG(status));
	}
	}
	break;
	}
	}

	void fcntl_unset(int fd, int flag)
	{
	int fdflags = fcntl(fd, F_GETFD, 0);
	if (fdflags != -1) fdflags = fcntl(STATUS_FD, F_SETFD,
	fdflags & ~flag);
	if (fdflags == -1) {
	syslog(LOG_ERR, "fcntl(): unable to unset %d: %m", flag);
	}
	}

	void spawn_service(struct service *s)
	{
	pid_t p;
	int i;
	char path[1024];
	static char name_env[100];
	struct centry *c;

	switch (p = fork()) {
	case -1:
	syslog(LOG_ERR, "can't fork process to run service %s: %m", s->name);
	break;

	case 0:
	/* child */
	if (become_cyrus() != 0) {
	syslog(LOG_ERR, "can't change to the cyrus user");
	exit(1);
	}

	get_prog(path, s->exec);
	if (dup2(s->stat[1], STATUS_FD) < 0) {
	syslog(LOG_ERR, "can't duplicate status fd: %m");
	exit(1);
	}
	if (dup2(s->socket, LISTEN_FD) < 0) {
	syslog(LOG_ERR, "can't duplicate listener fd: %m");
	exit(1);
	}

	fcntl_unset(STATUS_FD, FD_CLOEXEC);
	fcntl_unset(LISTEN_FD, FD_CLOEXEC);

	/* close all listeners */
	for (i = 0; i < nservices; i++) {
	if (Services[i].socket > 0) close(Services[i].socket);
	if (Services[i].stat[0] > 0) close(Services[i].stat[0]);
	if (Services[i].stat[1] > 0) close(Services[i].stat[1]);
	}
	limit_fds(256);

	syslog(LOG_DEBUG, "about to exec %s", path);

	/* add service name to environment */
	sprintf(name_env, "CYRUS_SERVICE=%s", s->name);
	putenv(name_env);

	execv(path, s->exec);
	syslog(LOG_ERR, "couldn't exec %s: %m", path);
	exit(EX_OSERR);

	default: /* parent */
	s->ready_workers++;
	s->nforks++;
	s->nactive++;

	/* add to child table */
	c = get_centry();
	c->pid = p;
	c->s = s;
	c->next = ctable[p % child_table_size];
	ctable[p % child_table_size] = c;
	break;
	}

	}

	void schedule_event(struct event *a)
	{
	struct event *ptr;

	if (!schedule \|\| a->mark < schedule->mark) {
	a->next = schedule;
	schedule = a;

	return;
	}
	for (ptr = schedule; ptr->next && ptr->next->mark <= a->mark;
	ptr = ptr->next) ;

	/* insert a */
	a->next = ptr->next;
	ptr->next = a;
	}

	void spawn_schedule(time_t now)
	{
	struct event a, b;
	int i;
	char path[1024];
	pid_t p;
	struct centry *c;

	a = NULL;
	/* update schedule accordingly */
	while (schedule && schedule->mark <= now) {
	/* delete from schedule, insert into a */
	struct event *ptr = schedule;

	/* delete */
	schedule = schedule->next;

	/* insert */
	ptr->next = a;
	a = ptr;
	}

	/* run all events */
	while (a && a != schedule) {
	switch (p = fork()) {
	case -1:
	- syslog(LOG_CRIT, "can't fork process to run event %s");
	+ syslog(LOG_CRIT, "can't fork process to run event %s", a->name);
	break;

	case 0:
	if (become_cyrus() != 0) {
	syslog(LOG_ERR, "can't change to the cyrus user");
	exit(1);
	}

	/* close all listeners */
	for (i = 0; i < nservices; i++) {
	if (Services[i].socket > 0) close(Services[i].socket);
	if (Services[i].stat[0] > 0) close(Services[i].stat[0]);
	if (Services[i].stat[1] > 0) close(Services[i].stat[1]);
	}
	limit_fds(256);

	get_prog(path, a->exec);
	syslog(LOG_DEBUG, "about to exec %s", path);
	execv(path, a->exec);
	syslog(LOG_ERR, "can't exec %s on schedule: %m", path);
	exit(EX_OSERR);
	break;

	default:
	/* we don't wait for it to complete */

	/* add to child table */
	c = get_centry();
	c->pid = p;
	c->s = NULL;
	c->next = ctable[p % child_table_size];
	ctable[p % child_table_size] = c;

	break;
	}

	b = a->next;
	if (a->period) {
	a->mark = now + a->period;
	/* reschedule a */
	schedule_event(a);
	} else {
	free(a);
	}
	/* examine next event */
	a = b;
	}
	}

	void reap_child(void)
	{
	int status;
	pid_t pid;
	struct centry *c;

	while ((pid = waitpid((pid_t) -1, &status, WNOHANG)) > 0) {
	if (WIFEXITED(status)) {
	syslog(LOG_DEBUG, "process %d exited, status %d", pid,
	WEXITSTATUS(status));
	}

	if (WIFSIGNALED(status)) {
	syslog(LOG_ERR, "process %d exited, signaled to death by %d",
	pid, WTERMSIG(status));
	}

	/* account for the child */
	c = ctable[pid % child_table_size];
	if (c && c->pid == pid) {
	/* first thing in the linked list */

	/* decrement active count for service */
	if (c->s) c->s->nactive--;

	ctable[pid % child_table_size] = c->next;
	c->next = cfreelist;
	cfreelist = c;
	} else {
	/* not the first thing in the linked list */

	while (c->next) {
	if (c->next->pid == pid) break;
	c = c->next;
	}
	if (c->next) {
	struct centry *t;

	t = c->next;
	/* decrement active count for service */
	if (t->s) t->s->nactive--;

	c->next = t->next; /* remove node */
	t->next = cfreelist; /* add to freelist */
	cfreelist = t;
	} else {
	/* yikes! don't know about this child! */
	syslog(LOG_ERR, "process %d not recognized", pid);
	}
	}
	}
	}

	static int gotsigchld = 0;

	void sigchld_handler(int sig __attribute__((unused)))
	{
	gotsigchld = 1;
	}

	static int gotsighup = 0;

	void sighup_handler(int sig __attribute__((unused)))
	{
	gotsighup = 1;
	}

	void sigterm_handler(int sig __attribute__((unused)))
	{
	struct sigaction action;

	/* send all the other processes SIGTERM, then exit */
	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;
	action.sa_handler = SIG_IGN;
	if (sigaction(SIGTERM, &action, (struct sigaction *) 0) < 0) {
	syslog(LOG_ERR, "sigaction: %m");
	exit(1);
	}
	/* kill my process group */
	if (kill(0, SIGTERM) < 0) {
	syslog(LOG_ERR, "kill(0, SIGTERM): %m");
	}

	#if HAVE_UCDSNMP
	/* tell master agent we're exiting */
	snmp_shutdown("cyrusMaster");
	#endif

	syslog(LOG_INFO, "exiting on SIGTERM");
	exit(0);
	}

	void sigalrm_handler(int sig __attribute__((unused)))
	{
	return;
	}

	void sighandler_setup(void)
	{
	struct sigaction action;

	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;

	action.sa_handler = sighup_handler;
	#ifdef SA_RESTART
	action.sa_flags \|= SA_RESTART;
	#endif
	if (sigaction(SIGHUP, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGHUP: %m", 1);
	}

	action.sa_handler = sigalrm_handler;
	if (sigaction(SIGALRM, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGALRM: %m", 1);
	}

	action.sa_handler = sigterm_handler;
	if (sigaction(SIGTERM, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGTERM: %m", 1);
	}

	action.sa_flags \|= SA_NOCLDSTOP;
	action.sa_handler = sigchld_handler;
	if (sigaction(SIGCHLD, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGCHLD: %m", 1);
	}
	}

	void process_msg(struct service *s, int msg)
	{
	switch (msg) {
	case MASTER_SERVICE_AVAILABLE:
	s->ready_workers++;
	break;

	case MASTER_SERVICE_UNAVAILABLE:
	s->ready_workers--;
	break;

	case MASTER_SERVICE_CONNECTION:
	s->nconnections++;
	break;

	default:
	syslog(LOG_ERR, "unrecognized message for service '%s': %x",
	s->name, msg);
	break;
	}

	if (verbose)
	syslog(LOG_DEBUG, "service %s now has %d workers\n",
	s->name, s->ready_workers);
	}

	static char *tokenize(char p)
	{
	char *tokens = NULL; / allocated in increments of 10 */
	int i = 0;

	if (!p \|\| !p) return NULL; / sanity check */
	while (*p) {
	while (p && isspace((int) p)) p++; /* skip whitespace */

	if (!(i % 10)) tokens = realloc(tokens, (i+10) * sizeof(char *));
	if (!tokens) return NULL;

	/* got a token */
	tokens[i++] = p;
	while (p && !isspace((int) p)) p++;

	/* p is whitespace or end of cmd */
	if (p) p++ = '\0';
	}
	/* add a NULL on the end */
	if (!(i % 10)) tokens = realloc(tokens, (i+1) * sizeof(char *));
	if (!tokens) return NULL;
	tokens[i] = NULL;

	return tokens;
	}

	void add_start(const char name, struct entry e,
	void *rock __attribute__((unused)))
	{
	char *cmd = mystrdup(masterconf_getstring(e, "cmd", NULL));
	char buf[256];
	char **tok;

	if (!cmd) {
	snprintf(buf, sizeof(buf), "unable to find command for %s", name);
	fatal(buf, EX_CONFIG);
	}

	tok = tokenize(cmd);
	if (!tok) fatal("out of memory", EX_UNAVAILABLE);
	run_startup(tok);
	free(tok);
	free(cmd);
	}

	void add_service(const char name, struct entry e,
	void *rock __attribute__((unused)))
	{
	char *cmd = mystrdup(masterconf_getstring(e, "cmd", NULL));
	int prefork = masterconf_getint(e, "prefork", 0);
	char *listen = mystrdup(masterconf_getstring(e, "listen", NULL));
	char *proto = mystrdup(masterconf_getstring(e, "proto", "tcp"));
	char *max = mystrdup(masterconf_getstring(e, "maxchild", "-1"));
	int i;

	if (!cmd \|\| !listen) {
	char buf[256];
	snprintf(buf, sizeof(buf), "unable to find command or port for %s",
	name);
	fatal(buf, EX_CONFIG);
	}

	/* see if we have an existing entry for this service */
	for (i = 0; i < nservices; i++) {
	if (Services[i].name && !strcmp(Services[i].name, name)) break;
	}

	if ((i < nservices) &&
	!strcmp(Services[i].listen, listen) &&
	!strcmp(Services[i].proto, proto)) {

	/* we found an existing entry and the port paramters are the same */
	Services[i].exec = tokenize(cmd);
	if (!Services[i].exec) fatal("out of memory", EX_UNAVAILABLE);
	Services[i].desired_workers = prefork;
	Services[i].max_workers = atoi(max);
	if (Services[i].max_workers == -1) {
	Services[i].max_workers = INT_MAX;
	}

	if (verbose > 2)
	syslog(LOG_DEBUG, "reconfig: service %s (%s, %s/%s, %d, %d)",
	Services[i].name, cmd,
	Services[i].listen, Services[i].proto,
	Services[i].desired_workers,
	Services[i].max_workers);
	}
	else {
	/* either we don't have an existing entry or we are changing
	* the port parameters, so create a new service
	*/
	if (nservices == allocservices) {
	Services = realloc(Services,
	(allocservices+=5) * sizeof(struct service));
	if (!Services) fatal("out of memory", EX_UNAVAILABLE);
	}

	Services[nservices].name = strdup(name);
	Services[nservices].listen = listen;
	Services[nservices].proto = proto;
	Services[nservices].exec = tokenize(cmd);
	if (!Services[nservices].exec) fatal("out of memory", EX_UNAVAILABLE);

	Services[nservices].socket = 0;
	Services[nservices].saddr = NULL;

	Services[nservices].ready_workers = 0;
	Services[nservices].desired_workers = prefork;
	Services[nservices].max_workers = atoi(max);
	if (Services[i].max_workers == -1) {
	Services[i].max_workers = INT_MAX;
	}
	memset(Services[nservices].stat, 0, sizeof(Services[nservices].stat));

	Services[nservices].nforks = 0;
	Services[nservices].nactive = 0;
	Services[nservices].nconnections = 0;

	if (verbose > 2)
	syslog(LOG_DEBUG, "add: service %s (%s, %s/%s, %d, %d)",
	Services[nservices].name, cmd,
	Services[nservices].listen, Services[nservices].proto,
	Services[nservices].desired_workers,
	Services[nservices].max_workers);

	nservices++;
	}

	free(max);
	}

	void add_event(const char name, struct entry e,
	void *rock __attribute__((unused)))
	{
	char *cmd = mystrdup(masterconf_getstring(e, "cmd", NULL));
	int period = 60 * masterconf_getint(e, "period", 0);
	struct event *evt;

	if (!cmd) {
	char buf[256];
	snprintf(buf, sizeof(buf), "unable to find command or port for %s",
	name);
	fatal(buf, EX_CONFIG);
	}

	evt = (struct event *) malloc(sizeof(struct event));
	if (!evt) fatal("out of memory", EX_UNAVAILABLE);
	evt->name = strdup(name);
	evt->mark = 0;
	evt->period = period;
	evt->exec = tokenize(cmd);
	if (!evt->exec) fatal("out of memory", EX_UNAVAILABLE);
	evt->next = schedule;
	schedule = evt;
	}

	#ifdef HAVE_SETRLIMIT

	#ifdef RLIMIT_NOFILE
	# define RLIMIT_NUMFDS RLIMIT_NOFILE
	#else
	# ifdef RLIMIT_OFILE
	# define RLIMIT_NUMFDS RLIMIT_OFILE
	# endif
	#endif
	void limit_fds(rlim_t x)
	{
	struct rlimit rl;
	int r;

	rl.rlim_cur = x;
	rl.rlim_max = x;
	if (setrlimit(RLIMIT_NUMFDS, &rl) < 0) {
	- syslog(LOG_ERR, "setrlimit: Unable to set file descriptors limit to %d: %m", x);
	+ syslog(LOG_ERR, "setrlimit: Unable to set file descriptors limit to %ld: %m", x);
	}

	if (verbose > 1) {
	r = getrlimit(RLIMIT_NUMFDS, &rl);
	- syslog(LOG_DEBUG, "set maximum file descriptors to %d/%d", rl.rlim_cur,
	+ syslog(LOG_DEBUG, "set maximum file descriptors to %ld/%ld", rl.rlim_cur,
	rl.rlim_max);
	}
	}
	#else
	void limit_fds(rlim_t x)
	{
	}
	#endif

	void reread_conf(void)
	{
	int i;
	struct event *ptr;

	/* disable all services -
	they will be re-enabled if they appear in config file */
	for (i = 0; i < nservices; i++) Services[i].exec = NULL;

	/* read services */
	masterconf_getsection("SERVICES", &add_service, NULL);

	for (i = 0; i < nservices; i++) {
	if (!Services[i].exec && Services[i].socket) {
	/* cleanup newly disabled services */

	if (verbose > 2)
	syslog(LOG_DEBUG, "disable: service %s socket %d pipe %d %d",
	Services[i].name, Services[i].socket,
	Services[i].stat[0], Services[i].stat[1]);
	free(Services[i].name); Services[i].name = NULL;
	free(Services[i].listen);
	free(Services[i].proto);
	Services[i].desired_workers = 0;
	Services[i].nforks = 0;
	Services[i].nactive = 0;
	Services[i].nconnections = 0;

	/* close all listeners */
	if (Services[i].socket > 0) close(Services[i].socket);
	Services[i].socket = 0;
	Services[i].saddr = NULL;

	if (Services[i].stat[0] > 0) close(Services[i].stat[0]);
	if (Services[i].stat[1] > 0) close(Services[i].stat[1]);
	memset(Services[i].stat, 0, sizeof(Services[i].stat));
	}
	else if (Services[i].exec && !Services[i].socket) {
	/* initialize new services */

	service_create(&Services[i]);
	if (verbose > 2)
	syslog(LOG_DEBUG, "init: service %s socket %d pipe %d %d",
	Services[i].name, Services[i].socket,
	Services[i].stat[0], Services[i].stat[1]);
	}
	}


	/* remove existing events */
	while (schedule) {
	ptr = schedule;
	schedule = schedule->next;
	free(ptr->name);
	free((char**) ptr->exec);
	free(ptr);
	}

	/* read events */
	masterconf_getsection("EVENTS", &add_event, NULL);
	}

	int main(int argc, char **argv)
	{
	int i, opt, close_std = 1;
	extern int optind;
	extern char *optarg;
	int fd;
	fd_set rfds;
	char *p = NULL;

	p = getenv("CYRUS_VERBOSE");
	if (p) verbose = atoi(p) + 1;
	while ((opt = getopt(argc, argv, "l:D")) != EOF) {
	switch (opt) {
	case 'l': /* user defined listen queue backlog */
	listen_queue_backlog = atoi(optarg);
	break;
	case 'D':
	close_std = 0;
	break;
	default:
	break;
	}
	}

	/* zero out the children table */
	memset(&ctable, 0, sizeof(struct centry ) child_table_size);

	if (close_std) {
	/* close stdin/out/err */
	for (fd = 0; fd < 3; fd++) {
	close(fd);
	if (open("/dev/null", O_RDWR, 0) != fd)
	fatal("couldn't open /dev/null: %m", 2);
	}
	}

	/* we reserve fds 3 and 4 for children to communicate with us, so they
	better be available. */
	for (fd = 3; fd < 5; fd++) {
	close(fd);
	if (dup(0) != fd) fatal("couldn't dup fd 0: %m", 2);
	}

	limit_fds(RLIM_INFINITY);

	masterconf_init("master");
	syslog(LOG_NOTICE, "process started");

	#ifdef HAVE_UCDSNMP
	/* initialize SNMP agent */

	/* make us a agentx client. */
	ds_set_boolean(DS_APPLICATION_ID, DS_AGENT_ROLE, 1);

	/* initialize the agent library */
	init_agent("cyrusMaster");

	init_cyrusMasterMIB();

	init_snmp("cyrusMaster");
	#endif

	masterconf_getsection("START", &add_start, NULL);
	masterconf_getsection("SERVICES", &add_service, NULL);
	masterconf_getsection("EVENTS", &add_event, NULL);

	/* set signal handlers */
	sighandler_setup();

	/* initialize services */
	for (i = 0; i < nservices; i++) {
	service_create(&Services[i]);
	if (verbose > 2)
	syslog(LOG_DEBUG, "init: service %s socket %d pipe %d %d",
	Services[i].name, Services[i].socket,
	Services[i].stat[0], Services[i].stat[1]);
	}

	if (become_cyrus_early) become_cyrus();

	/* ok, we're going to start spawning like mad now */
	syslog(LOG_NOTICE, "ready for work");

	for (;;) {
	int r, i, msg, maxfd;
	struct timeval tv, *tvptr;
	time_t now = time(NULL);
	#if HAVE_UCDSNMP
	int blockp = 0;
	#endif

	/* run any scheduled processes */
	spawn_schedule(now);

	tvptr = NULL;
	if (schedule) {
	if (now < schedule->mark) tv.tv_sec = schedule->mark - now;
	else tv.tv_sec = 0;
	tv.tv_usec = 0;
	tvptr = &tv;
	}

	/* do we have any services undermanned? */
	for (i = 0; i < nservices; i++) {
	if (Services[i].exec /* enabled */ &&
	(Services[i].nactive < Services[i].max_workers) &&
	(Services[i].ready_workers < Services[i].desired_workers)) {
	spawn_service(&Services[i]);
	}
	}

	if (gotsigchld) {
	/* order matters here */
	gotsigchld = 0;
	reap_child();
	}

	if (gotsighup) {
	syslog(LOG_NOTICE, "got SIGHUP");
	gotsighup = 0;
	reread_conf();
	}

	FD_ZERO(&rfds);
	maxfd = 0;
	for (i = 0; i < nservices; i++) {
	int x = Services[i].stat[0];

	int y = Services[i].socket;

	/* messages */
	if (x > 0) {
	if (verbose > 2)
	syslog(LOG_DEBUG, "listening for messages from %s",
	Services[i].name);
	FD_SET(x, &rfds);
	}
	if (x > maxfd) maxfd = x;

	/* connections */
	if (y > 0 && Services[i].ready_workers == 0 &&
	Services[i].nactive < Services[i].max_workers) {
	if (verbose > 2)
	syslog(LOG_DEBUG, "listening for connections for %s",
	Services[i].name);
	FD_SET(y, &rfds);
	if (y > maxfd) maxfd = y;
	}

	/* paranoia */
	if (Services[i].ready_workers < 0) {
	syslog(LOG_ERR, "%s has %d workers?!?", Services[i].name,
	Services[i].ready_workers);
	}
	}
	maxfd++; /* need 1 greater than maxfd */
	#ifdef HAVE_UCDSNMP
	if (tvptr == NULL) blockp = 1;
	snmp_select_info(&maxfd, &rfds, tvptr, &blockp);
	#endif
	errno = 0;
	r = select(maxfd, &rfds, NULL, NULL, tvptr);
	if (r == -1 && errno == EAGAIN) continue;
	if (r == -1 && errno == EINTR) continue;
	if (r == -1) {
	/* uh oh */
	fatal("select failed: %m", 1);
	}

	#ifdef HAVE_UCDSNMP
	/* check for SNMP queries */
	snmp_read(&rfds);
	snmp_timeout();
	#endif
	for (i = 0; i < nservices; i++) {
	int x = Services[i].stat[0];
	int y = Services[i].socket;
	int j;

	if (FD_ISSET(x, &rfds)) {
	r = read(x, &msg, sizeof(int));
	if (r != sizeof(int)) {
	syslog(LOG_ERR, "got weird response from child: %x", i);
	continue;
	}

	process_msg(&Services[i], msg);
	}

	if (Services[i].nactive < Services[i].max_workers) {
	/* bring us up to desired_workers */
	for (j = Services[i].ready_workers;
	j < Services[i].desired_workers;
	j++)
	{
	spawn_service(&Services[i]);
	}

	if (Services[i].ready_workers == 0 &&
	FD_ISSET(y, &rfds)) {
	/* huh, someone wants to talk to us */
	spawn_service(&Services[i]);
	}
	}
	}
	}
	}
	diff --git a/timsieved/actions.c b/timsieved/actions.c
	index 5ffe61321..078bfd0af 100644
	--- a/timsieved/actions.c
	+++ b/timsieved/actions.c
	@@ -1,586 +1,588 @@
	/* actions.c -- executes the commands for timsieved
	* Tim Martin
	- * $Id: actions.c,v 1.29 2001/11/27 02:25:06 ken3 Exp $
	+ * $Id: actions.c,v 1.30 2002/02/19 18:50:15 ken3 Exp $
	*
	*/
	/*
	* Copyright (c) 1999-2000 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any other legal
	* details, please contact
	* Office of Technology Transfer
	* Carnegie Mellon University
	* 5000 Forbes Avenue
	* Pittsburgh, PA 15213-3890
	* (412) 268-4387, fax: (412) 268-7395
	* tech-transfer@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	*/

	#ifdef HAVE_CONFIG_H
	#include <config.h>
	#endif

	#include <stdlib.h>
	#include <stdio.h>
	#include <sys/stat.h>
	#include <errno.h>
	#include <stdlib.h>
	#include <limits.h>
	#include <sys/param.h>
	#include <syslog.h>
	#include <dirent.h>
	#include <ctype.h>
	#include <unistd.h>
	#include <string.h>

	#include "prot.h"
	#include "tls.h"
	#include "util.h"
	#include "imapconf.h"
	#include "xmalloc.h"
	#include "sieve_interface.h"

	#include "codes.h"
	#include "actions.h"
	#include "scripttest.h"

	/* after a user has authentication, our current directory is their Sieve
	directory! */

	char *sieve_dir = NULL;

	int actions_init(void)
	{
	int sieve_usehomedir = 0;

	sieve_usehomedir = config_getswitch("sieveusehomedir", 0);

	if (!sieve_usehomedir) {
	sieve_dir = (char *) config_getstring("sievedir", "/usr/sieve");
	} else {
	/* can't use home directories with timsieved */
	syslog(LOG_ERR, "can't use home directories");

	return TIMSIEVE_FAIL;
	}

	return TIMSIEVE_OK;
	}

	int actions_setuser(const char *userid)
	{
	char hash;
	char *foo=sieve_dir;
	int result;

	sieve_dir=(char *) xmalloc(1024);

	hash = (char) dir_hash_c(userid);

	snprintf(sieve_dir, 1023, "%s/%c/%s", foo, hash,userid);

	result = chdir(sieve_dir);
	if (result != 0) {
	result = mkdir(sieve_dir, 0755);
	if (!result) result = chdir(sieve_dir);
	if (result) {
	syslog(LOG_ERR, "mkdir %s: %m", sieve_dir);
	return TIMSIEVE_FAIL;
	}
	}

	return TIMSIEVE_OK;
	}

	/*
	*
	* Everything but '/' and '\0' are valid.
	*
	*/

	int scriptname_valid(mystring_t *name)
	{
	int lup;
	char *ptr;

	/* must be at least one character long */
	if (name->len < 1) return TIMSIEVE_FAIL;

	ptr=string_DATAPTR(name);

	for (lup=0;lup<name->len;lup++)
	{
	if ((ptr[lup]=='/') \|\| (ptr[lup]=='\0'))
	return TIMSIEVE_FAIL;
	}

	return TIMSIEVE_OK;
	}

	int capabilities(struct protstream conn, sasl_conn_t saslconn)
	{
	const char *sasllist;
	unsigned mechcount;

	/* implementation */
	prot_printf(conn, "\"IMPLEMENTATION\" \"" SIEVED_IDENT " " SIEVED_VERSION "\"\r\n");

	/* SASL */
	if (sasl_listmech(saslconn, NULL,
	"\"SASL\" \"", " ", "\"\r\n",
	&sasllist,
	NULL, &mechcount) == SASL_OK && mechcount > 0)
	{
	prot_printf(conn,"%s",sasllist);
	}

	/* Sieve capabilities */
	prot_printf(conn,"\"SIEVE\" \"%s\"\r\n",sieve_listextensions());

	if (tls_enabled("sieve")) {
	prot_printf(conn, "\"STARTTLS\"\r\n");
	}

	prot_printf(conn,"OK\r\n");

	return TIMSIEVE_OK;
	}

	int getscript(struct protstream conn, mystring_t name)
	{
	FILE *stream;
	struct stat filestats; /* returned by stat */
	int size; /* size of the file */
	int result;
	int cnt;

	char path[1024];

	result = scriptname_valid(name);
	if (result!=TIMSIEVE_OK)
	{
	prot_printf(conn,"NO \"Invalid script name\"\r\n");
	return result;
	}


	snprintf(path, 1023, "%s.script", string_DATAPTR(name));

	result = stat(path, &filestats);
	if (result != 0) {
	prot_printf(conn,"NO \"Script doesn't exist\"\r\n");
	return TIMSIEVE_NOEXIST;
	}
	size = filestats.st_size;

	stream = fopen(path, "r");

	if (stream == NULL) {
	prot_printf(conn,"NO \"fopen failed\"\r\n");
	return TIMSIEVE_NOEXIST;
	}

	prot_printf(conn, "{%d}\r\n", size);

	cnt = 0;
	while (cnt < size) {
	char buf[BLOCKSIZE];
	int amount=BLOCKSIZE;

	if (size-cnt < BLOCKSIZE)
	amount=size-cnt;

	if (fread(buf, 1, BLOCKSIZE, stream) == 0) {
	if (ferror(stream)) {
	fatal("fatal error (fread)", 0);
	}
	}

	prot_write(conn, buf, amount);

	cnt += amount;
	}

	prot_printf(conn,"\r\n");

	prot_printf(conn, "OK\r\n");

	return TIMSIEVE_OK;
	}

	/* counts the number of scripts user has that are DIFFERENT from name.
	used for enforcing quotas */
	static int countscripts(char *name)
	{
	DIR *dp;
	struct dirent *dir;
	int length;
	int number=0;
	char myname[1024];

	snprintf(myname, 1023, "%s.script", name);

	if ((dp = opendir(".")) == NULL) {
	return -1;
	}

	while ((dir=readdir(dp)) != NULL) {
	length=strlen(dir->d_name);
	if (length >= strlen(".script") &&
	(strcmp(dir->d_name + (length - 7), ".script") == 0)) {
	/* this is a sieve script */
	if (strcmp(myname, dir->d_name) != 0) {
	/* and it's different from me */
	number++;
	}
	}
	}

	return number;
	}


	/* save name as a sieve script */
	int putscript(struct protstream conn, mystring_t name, mystring_t *data,
	int verify_only)
	{
	FILE *stream;
	char *dataptr;
	char *errstr;
	int lup;
	int result;
	char path[1024], p2[1024];
	int maxscripts;

	result = scriptname_valid(name);
	if (result!=TIMSIEVE_OK)
	{
	prot_printf(conn,"NO \"Invalid script name\"\r\n");
	return result;
	}

	- if (!verify_only) {
	+ if (verify_only)
	+ stream = tmpfile();
	+
	+ else {
	/* see if this would put the user over quota */
	maxscripts = config_getint("sieve_maxscripts",5);

	if (countscripts(string_DATAPTR(name))+1 > maxscripts)
	{
	prot_printf(conn,
	"NO (\"QUOTA\") \"You are only allowed %d scripts on this server\"\r\n",
	maxscripts);
	return TIMSIEVE_FAIL;
	}

	snprintf(path, 1023, "%s.script.NEW", string_DATAPTR(name));
	+
	+ stream = fopen(path, "w+");
	}
	- else
	- tmpnam(path);

	- stream = fopen(path, "w+");
	if (stream == NULL) {
	prot_printf(conn, "NO \"Unable to open script for writing (%s)\"\r\n",
	path);
	return TIMSIEVE_NOEXIST;
	}

	dataptr = string_DATAPTR(data);

	for (lup=0;lup<= data->len / BLOCKSIZE; lup++) {
	int amount = BLOCKSIZE;

	if (lup*BLOCKSIZE+BLOCKSIZE > data->len)
	amount=data->len % BLOCKSIZE;

	fwrite(dataptr, 1, amount, stream);

	dataptr += amount;
	}

	/* let's make sure this is a valid script
	(no parse errors)
	*/
	result = is_script_parsable(stream, &errstr);

	if (result != TIMSIEVE_OK) {
	if (errstr && *errstr) {
	prot_printf(conn, "NO {%d}\r\n%s\r\n", strlen(errstr), errstr);
	free(errstr);
	} else {
	if (errstr) free(errstr);
	prot_printf(conn, "NO \"parse failed\"\r\n");
	}
	fclose(stream);
	unlink(path);
	return result;
	}

	fflush(stream);
	fclose(stream);

	if (!verify_only) {
	snprintf(p2, 1023, "%s.script", string_DATAPTR(name));
	rename(path, p2);
	}

	prot_printf(conn, "OK\r\n");

	return TIMSIEVE_OK;
	}

	/* delete the active script */

	static int deleteactive(struct protstream *conn)
	{
	if (unlink("default") != 0) {
	prot_printf(conn,"NO \"Unable to unlink active script\"\r\n");
	return TIMSIEVE_FAIL;
	}

	return TIMSIEVE_OK;
	}


	/* is this the active script? */
	static int isactive(char *name)
	{
	char filename[1024];
	char activelink[1024];

	snprintf(filename, 1023, "%s.script", name);
	memset(activelink, 0, sizeof(activelink));
	if ((readlink("default", activelink, sizeof(activelink)-1) < 0) &&
	(errno != ENOENT))
	{
	syslog(LOG_ERR, "readlink(default): %m");
	return FALSE;
	}

	if (!strcmp(filename, activelink)) {
	return TRUE;
	} else {
	return FALSE;
	}
	}

	/* delete a sieve script */
	int deletescript(struct protstream conn, mystring_t name)
	{
	int result;
	char path[1024];

	result = scriptname_valid(name);
	if (result!=TIMSIEVE_OK)
	{
	prot_printf(conn,"NO \"Invalid script name\"\r\n");
	return result;
	}

	snprintf(path, 1023, "%s.script", string_DATAPTR(name));

	if (isactive(string_DATAPTR(name)) && (deleteactive(conn)!=TIMSIEVE_OK)) {
	return TIMSIEVE_FAIL;
	}

	result = unlink(path);

	if (result != 0) {
	prot_printf(conn,"NO \"Error deleting script\"\r\n");
	return TIMSIEVE_FAIL;
	}

	prot_printf(conn,"OK\r\n");
	return TIMSIEVE_OK;
	}

	/* list the scripts user has available */
	int listscripts(struct protstream *conn)
	{
	DIR *dp;
	struct dirent *dir;
	int length;

	/* open the directory */
	dp=opendir(".");

	if (dp==NULL)
	{
	prot_printf(conn,"NO \"Error opening directory\"\r\n");
	return TIMSIEVE_FAIL;
	}

	while ((dir=readdir(dp)) != NULL) /* while there are files here */
	{
	length=strlen(dir->d_name);
	if (length >= strlen(".script")) /* if ends in .script */
	{
	if (strcmp(dir->d_name + (length - 7), ".script")==0)
	{
	char namewo=(char ) xmalloc(length-6);

	memcpy(namewo, dir->d_name, length-7);
	namewo[length-7]='\0';

	if (isactive(namewo)==TRUE)
	prot_printf(conn,"\"%s\" ACTIVE\r\n", namewo);
	else
	prot_printf(conn,"\"%s\"\r\n", namewo);

	free(namewo);
	}
	}
	}

	prot_printf(conn,"OK\r\n");

	return TIMSIEVE_OK;
	}

	/* does the script 'str' exist
	return TRUE \| FALSE */
	static int exists(char *str)
	{
	char filename[1024];
	struct stat filestats; /* returned by stat */
	int result;

	snprintf(filename, 1023, "%s.script", str);

	result = stat(filename,&filestats);

	if (result != 0) {
	return FALSE;
	}

	return TRUE;
	}

	/* set the sieve script 'name' to be the active script */

	int setactive(struct protstream conn, mystring_t name)
	{
	int result;
	char filename[1024];

	/* if string name is empty, disable active script */
	if (!strlen(string_DATAPTR(name))) {
	if (deleteactive(conn) != TIMSIEVE_OK)
	return TIMSIEVE_FAIL;

	prot_printf(conn,"OK\r\n");
	return TIMSIEVE_OK;
	}

	result = scriptname_valid(name);
	if (result!=TIMSIEVE_OK)
	{
	prot_printf(conn,"NO \"Invalid script name\"\r\n");
	return result;
	}

	if (exists(string_DATAPTR(name))==FALSE)
	{
	prot_printf(conn,"NO \"Script does not exist\"\r\n");
	return TIMSIEVE_NOEXIST;
	}

	/* if script already is the active one just say ok */
	if (isactive(string_DATAPTR(name))==TRUE) {
	prot_printf(conn,"OK\r\n");
	return TIMSIEVE_OK;
	}

	/* get the name of the active sieve script */
	snprintf(filename, sizeof filename, "%s.script", string_DATAPTR(name));

	/* ok we want to do this atomically so let's
	- make <activesieve>.NEW as a hard link
	- rename it to <activesieve>
	*/
	result = symlink(filename, "default.NEW");
	if (result) {
	syslog(LOG_ERR, "symlink(%s, default.NEW): %m", filename);
	prot_printf(conn, "NO \"Can't make link\"\r\n");
	return TIMSIEVE_FAIL;
	}

	result = rename("default.NEW", "default");
	if (result) {
	unlink("default.NEW");
	syslog(LOG_ERR, "rename(default.NEW, default): %m");
	prot_printf(conn,"NO \"Error renaming\"\r\n");
	return TIMSIEVE_FAIL;
	}

	prot_printf(conn,"OK\r\n");
	return TIMSIEVE_OK;
	}

	int cmd_havespace(struct protstream conn, mystring_t sieve_name, unsigned long num)
	{
	int result;
	int maxscripts;
	unsigned long maxscriptsize;

	result = scriptname_valid(sieve_name);
	if (result!=TIMSIEVE_OK)
	{
	prot_printf(conn,"NO \"Invalid script name\"\r\n");
	return result;
	}

	/* see if the size of the script is too big */
	maxscriptsize = config_getint("sieve_maxscriptsize", 32);
	maxscriptsize *= 1024;

	if (num > maxscriptsize)
	{
	prot_printf(conn,
	"NO (\"QUOTA\") \"Script size is too large. "
	"Max script size is %ld bytes\"\r\n",
	maxscriptsize);
	return TIMSIEVE_FAIL;
	}

	/* see if this would put the user over quota */
	maxscripts = config_getint("sieve_maxscripts",5);

	if (countscripts(string_DATAPTR(sieve_name))+1 > maxscripts)
	{
	prot_printf(conn,
	"NO (\"QUOTA\") \"You are only allowed %d scripts on this server\"\r\n",
	maxscripts);
	return TIMSIEVE_FAIL;
	}


	prot_printf(conn,"OK\r\n");
	return TIMSIEVE_OK;
	}

File Metadata

Mime Type: text/x-diff
Expires: Fri, Apr 24, 9:47 AM (1 w, 4 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 18896459
Default Alt Text: (700 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions