No OneTemporary
Actions

Authored By

Unknown

Size

794 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/configure.in b/configure.in
	index beda228f1..42ba28c82 100644
	--- a/configure.in
	+++ b/configure.in
	@@ -1,1439 +1,1445 @@
	dnl Process this file with autoconf to produce a configure script.

	dnl
	dnl Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	dnl
	dnl Redistribution and use in source and binary forms, with or without
	dnl modification, are permitted provided that the following conditions
	dnl are met:
	dnl
	dnl 1. Redistributions of source code must retain the above copyright
	dnl notice, this list of conditions and the following disclaimer.
	dnl
	dnl 2. Redistributions in binary form must reproduce the above copyright
	dnl notice, this list of conditions and the following disclaimer in
	dnl the documentation and/or other materials provided with the
	dnl distribution.
	dnl
	dnl 3. The name "Carnegie Mellon University" must not be used to
	dnl endorse or promote products derived from this software without
	dnl prior written permission. For permission or any legal
	dnl details, please contact
	dnl Carnegie Mellon University
	dnl Center for Technology Transfer and Enterprise Creation
	dnl 4615 Forbes Avenue
	dnl Suite 302
	dnl Pittsburgh, PA 15213
	dnl (412) 268-7393, fax: (412) 268-7395
	dnl innovation@andrew.cmu.edu
	dnl
	dnl 4. Redistributions of any form whatsoever must retain the following
	dnl acknowledgment:
	dnl "This product includes software developed by Computing Services
	dnl at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	dnl
	dnl CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	dnl THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	dnl AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	dnl FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	dnl WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	dnl AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	dnl OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	dnl
	dnl $Id: configure.in,v 1.316 2010/01/06 17:01:26 murch Exp $
	dnl
	dnl configure.in for the Cyrus imapd
	dnl


	AC_INIT(imap/imapd.c)
	AC_PREREQ([2.54])
	AC_CONFIG_HEADER(config.h)
	AC_CANONICAL_SYSTEM

	dnl Useful hook for distributions
	AC_ARG_WITH(extraident,[ --with-extraident=STRING use STRING as extra version information],
	[AC_DEFINE_UNQUOTED(EXTRA_IDENT,"$withval", [Extra version information for imap/version.h])])

	AC_CHECK_PROG(MAKEDEPEND,makedepend,makedepend,[`cd ${srcdir};pwd`/tools/not-mkdep])
	if test "$MAKEDEPEND" != "makedepend"; then
	AC_MSG_WARN([Makedepend is not installed on this system. You should compile and install the version from the makedepend subdirectory.])
	fi

	AC_ARG_WITH(login,,AC_ERROR([--with-login is no longer supported.
	Configure SASL appropriately instead.]))

	AC_ARG_WITH(cyrus-prefix,[ --with-cyrus-prefix=DIR use DIR as cyrus server install directory],
	cyrus_prefix="$withval",cyrus_prefix="/usr/cyrus")
	AC_SUBST(cyrus_prefix)
	AC_DEFINE_UNQUOTED(CYRUS_PATH,"$cyrus_prefix",[Where will we be installed?])
	AC_ARG_WITH(service-path,[ --with-service-path=DIR use DIR as service install directory],
	service_path="$withval",service_path="$cyrus_prefix/bin")
	AC_SUBST(service_path)
	AC_DEFINE_UNQUOTED(SERVICE_PATH,"$service_path",[Directory to use for service binaries])
	AC_ARG_WITH(cyrus-user,[ --with-cyrus-user=USERID use USERID cyrus userid],
	cyrus_user="$withval",cyrus_user="cyrus")
	AC_SUBST(cyrus_user)
	AC_DEFINE_UNQUOTED(CYRUS_USER, "$cyrus_user",[What user will we run as?])
	AC_ARG_WITH(cyrus-group,[ --with-cyrus-group=GROUPID use GROUPID cyrus group],
	cyrus_group="$withval",cyrus_group="mail")
	AC_SUBST(cyrus_group)

	dnl allow users to override $sysconfdir, but retain old default (/etc)
	dnl if not specified
	if test $sysconfdir = '${prefix}/etc'; then
	sysconfdir="/etc"
	fi
	AC_DEFINE_UNQUOTED(SYSCONFDIR,"$sysconfdir",[Config File Location])

	AC_PROG_CC
	AC_PROG_RANLIB
	AC_PROG_MAKE_SET
	AC_PROG_INSTALL
	AC_AIX
	AC_ISC_POSIX
	AC_PROG_AWK
	AC_C_CONST
	AC_SYS_LONG_FILE_NAMES
	if test $ac_cv_sys_long_file_names = no; then
	AC_MSG_ERROR(The Cyrus IMAPD requires support for long file names)
	fi
	AC_C_INLINE

	dnl Check the size of various types
	AC_CHECK_SIZEOF(int)
	AC_CHECK_SIZEOF(long)
	AC_CHECK_SIZEOF(size_t)
	AC_CHECK_SIZEOF(off_t)

	dnl Check if `long long int' is available
	AC_CHECK_SIZEOF(long long int)
	AC_CHECK_SIZEOF(unsigned long long int)
	if test "$ac_cv_sizeof_long_long_int" -eq 8 -a \
	"$ac_cv_sizeof_unsigned_long_long_int" -eq 8; then
	AC_DEFINE(HAVE_LONG_LONG_INT,[],[Does the compiler support long long int?])
	AC_C_BIGENDIAN
	fi

	CMU_C___ATTRIBUTE__
	CMU_C_FPIC

	dnl check for -R, etc. switch
	CMU_GUESS_RUNPATH_SWITCH

	AC_CHECK_HEADERS(unistd.h sys/select.h sys/param.h stdarg.h)
	AC_REPLACE_FUNCS(memmove strcasecmp ftruncate strerror)
	AC_CHECK_FUNCS(strlcat strlcpy getgrouplist)
	AC_HEADER_DIRENT

	dnl do this before Berkeley DB/IPv6 detection
	CMU_SOCKETS
	LIBS="$LIBS ${LIB_SOCKET}"

	dnl check for IPv6 functions (fall back to sasl's if we don't have them)
	cyrus_cv_getaddrinfo=yes
	IPv6_CHECK_FUNC(getaddrinfo, [IPv6_CHECK_FUNC(gai_strerror,
	AC_DEFINE(HAVE_GETADDRINFO,[],[Do we have a getaddrinfo?]),
	cyrus_cv_getaddrinfo=no)], cyrus_cv_getaddrinfo=no)

	if test $cyrus_cv_getaddrinfo = no; then
	IPV6_OBJS="getaddrinfo.o"
	fi

	cyrus_cv_getnameinfo=yes
	IPv6_CHECK_FUNC(getnameinfo,
	AC_DEFINE(HAVE_GETNAMEINFO,[],[Do we have a getnameinfo?]),
	cyrus_cv_getnameinfo=no)

	if test $cyrus_cv_getnameinfo = no; then
	IPV6_OBJS="$IPV6_OBJS getnameinfo.o"
	fi

	IPv6_CHECK_SS_FAMILY()
	IPv6_CHECK_SA_LEN()

	AC_SUBST(IPV6_OBJS)

	dnl this is to check for time things
	AC_CHECK_HEADERS(sys/time.h)
	AC_HEADER_TIME
	AC_STRUCT_TM
	AC_STRUCT_TIMEZONE

	AC_SUBST(CPPFLAGS)
	AC_SUBST(PRE_SUBDIRS)
	AC_SUBST(EXTRA_SUBDIRS)
	AC_SUBST(DEPLIBS)
	AC_SUBST(LOCALDEFS)
	AC_FUNC_VPRINTF

	dnl function for doing each of the database backends
	dnl parameters: backend name, variable to set, withval

	CYRUSDB_OBJS="cyrusdb_flat.o cyrusdb_skiplist.o cyrusdb_quotalegacy.o"

	dnl Berkeley DB Detection

	AC_ARG_WITH(bdb, [ --with-bdb=DIR use Berkeley DB (in DIR) [[yes]] ],
	with_bdb=$withval, with_bdb="yes")

	dnl support old-style
	AC_ARG_WITH(dbdir,, with_bdb=$withval)

	case "$with_bdb" in
	no)
	use_berkeley="no"
	;;
	yes)
	use_berkeley="yes"
	with_bdb_lib=none
	with_bdb_inc=none
	;;

	*)
	use_berkeley="yes"
	with_bdb_lib="$with_bdb/lib"
	with_bdb_inc="$with_bdb/include"
	;;
	esac

	if test "$use_berkeley" != "no"; then
	CYRUS_BERKELEY_DB_CHK()

	if test "$dblib" = "no"; then
	AC_ERROR([Berkeley DB 3.x or later was not found. You may need to
	supply the --with-bdb-libdir or --with-bdb-incdir configure options.])
	fi

	if test "$with_bdb_lib" != "none"; then
	CMU_ADD_LIBPATH($with_bdb_lib)
	fi

	BDB_INC=${BDB_INCADD}
	BDB_LIB=${BDB_LIBADD}
	AC_SUBST(BDB_INC)
	AC_SUBST(BDB_LIB)

	LIBS="${LIBS} ${BDB_LIBADD}"
	CPPFLAGS="${BDB_INCADD} ${CPPFLAGS}"

	CYRUSDB_OBJS="$CYRUSDB_OBJS cyrusdb_berkeley.o"
	AC_DEFINE(HAVE_BDB,[],[Build in Berkeley DB support?])
	fi

	dnl End Berkeley DB Detection


	dnl SQL DB Detection

	HAVE_SQL=0

	dnl MySQL Detection
	AC_ARG_WITH(mysql, [ --with-mysql=DIR use MySQL (in DIR) [[no]] ],
	with_mysql=$withval, with_mysql="no")

	case "$with_mysql" in
	no)
	use_mysql="no"
	;;
	yes)
	use_mysql="yes"
	with_mysql_lib=none
	with_mysql_inc=none
	;;

	*)
	use_mysql="yes"
	with_mysql_lib="$with_mysql/lib"
	with_mysql_inc="$with_mysql/include"
	;;
	esac

	if test "$use_mysql" != "no"; then
	CYRUS_MYSQL_CHK()

	if test "$mysqllib" = "no"; then
	AC_ERROR([MySQL was not found. You may need to supply the
	--with-mysql-libdir or --with-mysql-incdir configure options.])
	else
	HAVE_SQL=1
	fi

	if test "$with_mysql_lib" != "none"; then
	CMU_ADD_LIBPATH($with_mysql_lib)
	fi

	LIBS="${LIBS} ${MYSQL_LIBADD}"
	CPPFLAGS="${MYSQL_INCADD} ${CPPFLAGS}"

	AC_DEFINE(HAVE_MYSQL,[],[Build in MySQL support?])
	fi

	dnl End MySQL Detection


	dnl PgSQL Detection
	AC_ARG_WITH(pgsql, [ --with-pgsql=DIR use PostgreSQL (in DIR) [[no]] ],
	with_pgsql=$withval, with_pgsql="no")

	case "$with_pgsql" in
	no)
	use_pgsql="no"
	;;
	yes)
	use_pgsql="yes"
	with_pgsql_lib=none
	with_pgsql_inc=none
	;;

	*)
	use_pgsql="yes"
	with_pgsql_lib="$with_pgsql/lib"
	with_pgsql_inc="$with_pgsql/include"
	;;
	esac

	if test "$use_pgsql" != "no"; then
	CYRUS_PGSQL_CHK()

	if test "$pgsqllib" = "no"; then
	AC_ERROR([PgSQL was not found. You may need to supply the
	--with-pgsql-libdir or --with-pgsql-incdir configure options.])
	else
	HAVE_SQL=1
	fi

	if test "$with_pgsql_lib" != "none"; then
	CMU_ADD_LIBPATH($with_pgsql_lib)
	fi

	LIBS="${LIBS} ${PGSQL_LIBADD}"
	CPPFLAGS="${PGSQL_INCADD} ${CPPFLAGS}"

	AC_DEFINE(HAVE_PGSQL,[],[Build in PgSQL support?])
	fi

	dnl End PgSQL Detection


	dnl SQLite Detection
	AC_ARG_WITH(sqlite, [ --with-sqlite=DIR use SQLite (in DIR) [[no]] ],
	with_sqlite=$withval, with_sqlite=no)

	case "$with_sqlite" in
	no)
	use_sqlite="no"
	;;
	yes)
	use_sqlite="yes"
	with_sqlite_lib=none
	with_sqlite_inc=none
	;;

	*)
	use_sqlite="yes"
	with_sqlite_lib="$with_sqlite/lib"
	with_sqlite_inc="$with_sqlite/include"
	;;
	esac

	if test "$use_sqlite" != "no"; then
	CYRUS_SQLITE_CHK()

	if test "$sqlitelib" = "no"; then
	AC_ERROR([Sqlite was not found. You may need to supply the
	--with-sqlite-libdir or --with-sqlite-incdir configure options.])
	else
	HAVE_SQL=1
	fi

	if test "$with_sqlite_lib" != "none"; then
	CMU_ADD_LIBPATH($with_sqlite_lib)
	fi

	LIBS="${LIBS} ${SQLITE_LIBADD}"
	CPPFLAGS="${SQLITE_INCADD} ${CPPFLAGS}"

	AC_DEFINE(HAVE_SQLITE,[],[Build in SQLite support?])
	fi

	dnl End SQLite Detection


	if test $HAVE_SQL = 1; then
	CYRUSDB_OBJS="${CYRUSDB_OBJS} cyrusdb_sql.o"
	fi

	dnl End SQL DB Detection

	AC_SUBST(CYRUSDB_OBJS)

	SIEVE_SUBDIRS=""
	sievedir="sieve"
	AC_ARG_ENABLE(sieve,
	[ --disable-sieve disable Sieve support],
	if test "$enableval" = no; then
	sievedir="no"
	fi)

	if test "$sievedir" != "no"; then
	SIEVE_OBJS="lmtp_sieve.o smtpclient.o"
	AC_SUBST(SIEVE_OBJS)
	SIEVE_LIBS="../${sievedir}/libsieve.a"
	AC_SUBST(SIEVE_LIBS)
	SIEVE_CPPFLAGS="-I\$(srcdir)/../$sievedir"
	AC_SUBST(SIEVE_CPPFLAGS)
	AC_DEFINE(USE_SIEVE,[],[Build in Sieve support?])

	dnl Sieve configure stuff
	AC_PROG_YACC
	AC_PROG_LEX
	AC_CHECK_LIB(fl,main)

	AC_CHECK_HEADERS(pcreposix.h rxposix.h)
	if test "$ac_cv_header_pcreposix_h" == "yes"; then
	LIBS="$LIBS -lpcre -lpcreposix";
	AC_DEFINE(ENABLE_REGEX, [], [Do we have a regex library?])
	else
	if test "$ac_cv_header_rxposix_h" == "yes"; then
	LIBS="$LIBS -lrx"
	AC_DEFINE(ENABLE_REGEX, [],
	[Do we have a regex library?])
	else
	AC_SEARCH_LIBS(regcomp, regex,
	AC_DEFINE(ENABLE_REGEX, [],
	[Do we have a regex library?]), [])
	fi
	fi

	SIEVE_SUBDIRS="${SIEVE_SUBDIRS} $sievedir"
	EXTRA_OUTPUT="${EXTRA_OUTPUT} $sievedir/Makefile"
	fi

	AC_SUBST(SIEVE_SUBDIRS)

	dnl for et routines
	AC_FUNC_CHECK(strerror,AC_DEFINE(HAS_STRERROR,[],[Do we have strerror()?]),
	AC_DEFINE(NEED_SYS_ERRLIST,[],[Do we have a sys_errlist?]))

	dnl for master fd limits
	AC_CHECK_HEADERS(sys/resource.h)
	AC_CHECK_FUNCS(setrlimit)
	AC_CHECK_FUNCS(getrlimit)

	dnl for detaching terminal
	AC_CHECK_FUNCS(daemon setsid)

	dnl for turning off sockets
	AC_CHECK_FUNCS(shutdown)

	AC_EGREP_HEADER(socklen_t, sys/socket.h, AC_DEFINE(HAVE_SOCKLEN_T,[],[Do we have a socklen_t?]))
	AC_EGREP_HEADER(sockaddr_storage, sys/socket.h,
	AC_DEFINE(HAVE_STRUCT_SOCKADDR_STORAGE,[],[Do we have a sockaddr_storage?]))
	AC_EGREP_HEADER(rlim_t, sys/resource.h, AC_DEFINE(HAVE_RLIM_T,[],[Do we have an rlim_t?]))

	dnl Bunch of setproctitle stuff
	spt_type=""
	AC_CHECK_FUNC(setproctitle,spt_type=SPT_BUILTIN)
	if test "$spt_type" = ""; then
	dnl BSD/OS and FreeBSD put it in -lutil
	AC_CHECK_LIB(util,setproctitle,spt_type=SPT_BUILTIN
	LIBS="${LIBS} -lutil")
	fi
	if test "$spt_type" = ""; then
	AC_CHECK_HEADER(sys/pstat.h,spt_type=SPT_PSTAT)
	fi
	if test "$spt_type" = ""; then
	AC_CHECK_HEADER(sys/sysnews.h,spt_type=SPT_SYSMIPS)
	fi
	if test "$spt_type" = ""; then
	AC_MSG_CHECKING(for PS_STRINGS)
	AC_CACHE_VAL(cyrus_cv_sys_psstrings, AC_TRY_CPP([
	#include <machine/vmparam.h>
	#include <sys/exec.h>
	#ifndef PS_STRINGS
	#include </nonexistent>
	#endif],cyrus_cv_sys_psstrings=yes,cyrus_cv_sys_psstrings=no))
	if test $cyrus_cv_sys_psstrings = yes; then
	spt_type=SPT_PSSTRINGS
	fi
	AC_MSG_RESULT($cyrus_cv_sys_psstrings)
	fi
	if test "$spt_type" = ""; then
	AC_MSG_CHECKING(for SCO)
	AC_CACHE_VAL(cyrus_cv_sys_sco, AC_TRY_CPP([
	#ifndef _SCO_unix_
	#include </nonexistent>
	#endif],cyrus_cv_sys_sco=yes,cyrus_cv_sys_sco=no))
	if test $cyrus_cv_sys_sco = yes; then
	spt_type=SPT_SCO
	fi
	AC_MSG_RESULT($cyrus_cv_sys_sco)
	fi
	if test "$spt_type" = ""; then
	AC_MSG_CHECKING(for setproctitle usability)
	AC_CACHE_VAL(cyrus_cv_sys_setproctitle, AC_TRY_CPP([
	#if defined(DGUX) \|\| defined(_SEQUENT_) \|\| defined(apollo)
	#include </nonexistent>
	#endif],cyrus_cv_sys_setproctitle=yes,cyrus_cv_sys_setproctitle=no))
	if test $cyrus_cv_sys_setproctitle = no; then
	spt_type=SPT_NONE
	fi
	AC_MSG_RESULT($cyrus_cv_sys_setproctitle)
	fi
	if test "$spt_type" != ""; then
	AC_DEFINE_UNQUOTED(SPT_TYPE,$spt_type,[Do we already have setproctitle?])
	fi

	AC_MSG_CHECKING(nonblocking method)
	AC_CACHE_VAL(cyrus_cv_sys_nonblock,AC_TRY_LINK([#include <sys/types.h>
	#include <sys/file.h>
	#include <fcntl.h>
	#ifndef FNDELAY
	#define FNDELAY O_NDELAY
	#endif],[fcntl(0, F_GETFL, 0)&FNDELAY],
	cyrus_cv_sys_nonblock=fcntl,cyrus_cv_sys_nonblock=ioctl))
	WITH_NONBLOCK=$cyrus_cv_sys_nonblock
	AC_SUBST(WITH_NONBLOCK)
	AC_MSG_RESULT($WITH_NONBLOCK)

	AC_MSG_CHECKING(timezone GMT offset method)
	AC_CACHE_VAL(cyrus_cv_struct_sys_gmtoff,AC_TRY_COMPILE([
	#include <time.h>],[struct tm tm;
	tm.tm_gmtoff = 0;
	],cyrus_cv_struct_sys_gmtoff=tm,cyrus_cv_struct_sys_gmtoff=gmtime))
	WITH_GMTOFF=$cyrus_cv_struct_sys_gmtoff
	AC_SUBST(WITH_GMTOFF)
	AC_MSG_RESULT($WITH_GMTOFF)
	AC_MSG_CHECKING(for shared mmap)
	AC_CACHE_VAL(cyrus_cv_func_mmap_shared,AC_TRY_RUN([
	#include <sys/types.h>
	#include <sys/mman.h>
	#include <fcntl.h>
	main() {
	char *base;
	int fd = open("conftestmmap", O_RDWR\|O_CREAT\|O_TRUNC, 0666);
	if (fd == -1) exit(1);
	if (write(fd, "test", 4) != 4) exit(1);
	fsync(fd);
	base = mmap((caddr_t)0, 100, PROT_READ, MAP_SHARED
	#ifdef MAP_FILE
	\| MAP_FILE
	#endif
	#ifdef MAP_VARIABLE
	\| MAP_VARIABLE
	#endif
	, fd, 0L);
	if (base == (caddr_t)-1) exit(1);
	if (strncmp(base, "test", 4) != 0) exit(1);
	if (write(fd, "test", 4) != 4) exit(1);
	fsync(fd);
	if (strncmp(base+4, "test", 4) != 0) exit(1);
	exit(0);}
	],cyrus_cv_func_mmap_shared=yes,cyrus_cv_func_mmap_shared=no,
	cyrus_cv_func_mmap_shared=no))
	AC_MSG_RESULT($cyrus_cv_func_mmap_shared)
	if test $cyrus_cv_func_mmap_shared = yes; then
	WITH_MAP="shared"
	else
	AC_MSG_CHECKING(for stupid shared mmap)
	AC_CACHE_VAL(cyrus_cv_func_mmap_stupidshared,AC_TRY_RUN([
	#include <sys/types.h>
	#include <sys/mman.h>
	#include <fcntl.h>
	main() {
	char *base;
	int fd = open("conftestmmap", O_RDWR\|O_CREAT\|O_TRUNC, 0666);
	if (fd == -1) exit(1);
	if (write(fd, "test", 4) != 4) exit(1);
	fsync(fd);
	base = mmap((caddr_t)0, 4, PROT_READ, MAP_SHARED
	#ifdef MAP_FILE
	\| MAP_FILE
	#endif
	#ifdef MAP_VARIABLE
	\| MAP_VARIABLE
	#endif
	, fd, 0L);
	if (base == (caddr_t)-1) exit(1);
	if (strncmp(base, "test", 4) != 0) exit(1);
	lseek(fd, 0L, 0);
	if (write(fd, "xyzz", 4) != 4) exit(1);
	fsync(fd);
	if (strncmp(base, "xyzz", 4) != 0) exit(1);
	exit(0);}
	],cyrus_cv_func_mmap_stupidshared=yes,cyrus_cv_func_mmap_stupidshared=no,
	cyrus_cv_func_mmap_stupidshared=no))
	AC_MSG_RESULT($cyrus_cv_func_mmap_stupidshared)
	if test $cyrus_cv_func_mmap_stupidshared = yes; then
	WITH_MAP="stupidshared"
	else
	AC_MSG_WARN([*** This system does not have a working mmap()])
	AC_MSG_WARN(*** Expect a considerable performance penalty)
	WITH_MAP=nommap
	fi
	fi

	AC_SUBST(WITH_MAP)
	AC_ARG_WITH(lock,
	[ --with-lock=METHOD force use of METHOD for locking (flock or fcntl)],
	WITH_LOCK="$withval", [
	AC_CHECK_FUNC(fcntl,WITH_LOCK="fcntl",[
	AC_CHECK_FUNC(flock,WITH_LOCK="flock",[
	AC_ERROR(unable to detect locking method)
	])
	])
	])

	AC_SUBST(WITH_LOCK)

	dnl check for fdatasync (used by cyrusdb_skiplist)
	LIB_RT=""
	AC_CHECK_FUNC(fdatasync, AC_DEFINE(HAVE_FDATASYNC,[],[Do we have fdatasync()?]), [
	AC_CHECK_LIB(rt, fdatasync, [
	LIB_RT="-lrt"
	AC_DEFINE(HAVE_FDATASYNC,[],[Do we have fdatasync()?])
	])
	])

	dnl for makedepend and AFS.
	cant_find_sigvec=no
	AC_CACHE_VAL(cyrus_cv_sigveclib,[
	dnl bsd classic flavor
	AC_CHECK_FUNC(sigvec, [
	cyrus_cv_sigveclib=""
	], [
	dnl hp flavor
	AC_CHECK_LIB(BSD, sigvec, cyrus_cv_sigveclib="-lBSD", [
	dnl not hp flavor
	SAVE_LDFLAGS="$LDFLAGS"
	dnl solaris flavor
	LDFLAGS="-L/usr/ucblib -R/usr/ucblib $LDFLAGS"
	AC_CHECK_LIB(ucb, sigvec, [
	dnl more solaris flavor
	cyrus_cv_sigveclib="-lc -L/usr/ucblib -R/usr/ucblib -lucb"],
	[ cant_find_sigvec=yes ])
	LDFLAGS="$SAVE_LDFLAGS"])
	])
	])
	AC_SUBST(cyrus_cv_sigveclib)

	# ok, we still look for this stuff because of checking groups, but
	# all authentication goes through SASL
	+AC_ARG_ENABLE(afs,[ --enable-afs Enable AFS with ptloader],[with_afs=yes],[with_afs=no])
	+
	+AC_ARG_WITH(afs-libdir,[ --with-afs-libdir=PATH use AFS libraries from PATH [[/usr/lib]]],
	+ afs_libdir="${withval}", afs_libdir="/usr/lib")
	+
	+AC_ARG_WITH(afs-incdir,[ --with-afs-incdir=PATH use AFS headers from PATH [[/usr/include]]],
	+ afs_incdir="${withval}", afs_incdir="/usr/include")

	-AC_ARG_WITH(afs,[ --with-afs=PATH use AFS libraries from PATH],
	- with_afs="${withval}", with_afs="no")
	-
	AC_ARG_WITH(ldap, [ --with-ldap=DIR use LDAP (in DIR) (experimental) [/usr/local] ],
	with_ldap="${withval}", with_ldap="no")

	dnl select mode of afspts
	AC_ARG_ENABLE(krb5afspts,[ --enable-krb5afspts compile afskrb PTS module with krb5 support],
	[SASL_SET_GSSAPI_LIBS
	AC_DEFINE(AFSPTS_USE_KRB5,[],[Should the AFS PTS plugin use krb5?])])

	if test "x$with_afs" != "xno"; then
	- if test ! -d $with_afs; then
	- $with_afs=/usr/local
	- fi
	- CFLAGS="${CFLAGS} -I${with_afs}/include"
	- AFS_LIBS="${with_afs}/lib/afs/libkauth.a ${with_afs}/lib/afs/libprot.a ${with_afs}/lib/afs/libauth.a ${with_afs}/lib/afs/libsys.a ${with_afs}/lib/librxkad.a ${with_afs}/lib/librx.a ${with_afs}/lib/afs/libsys.a ${with_afs}/lib/libubik.a ${with_afs}/lib/liblwp.a ${with_afs}/lib/afs/util.a"
	- if test -f ${with_afs}/lib/afs/libaudit.a; then
	- AFS_LIBS="$AFS_LIBS ${with_afs}/lib/afs/libaudit.a"
	+ CFLAGS="${CFLAGS} -I${with_afs_incdir}/include"
	+ AFS_LIBS="${with_afs_libdir}/afs/libkauth.a ${with_afs_libdir}/afs/libprot.a ${with_afs_libdir}/afs/libauth.a ${with_afs_libdir}/afs/libsys.a ${with_afs_libdir}/librxkad.a ${with_afs_libdir}/librx.a ${with_afs_libdir}/afs/libsys.a ${with_afs_libdir}/libubik.a ${with_afs_libdir}/liblwp.a ${with_afs_libdir}/afs/util.a ${with_afs_libdir}/afs/libcom_err.a"
	+ if test -f ${with_afs_libdir}/afs/libaudit.a; then
	+ AFS_LIBS="$AFS_LIBS ${with_afs_libdir}/afs/libaudit.a"
	fi
	if test -f /usr/ucblib/libucb.a; then
	CMU_ADD_LIBPATH_TO(/usr/ucblib, AFS_LDFLAGS)
	AFS_LIBS="$AFS_LIBS -lc -lucb"
	fi
	+ if test -f ${with_afs_libdir}/afs/libdes.a; then
	+ AFS_LIBS="$AFS_LIBS ${with_afs_libdir}/afs/libdes.a"
	+ else
	+ AFS_LIBS="$AFS_LIBS -ldes"
	+ fi

	AC_CACHE_VAL(cyrus_cv_afs_sigvec,[
	SAVE_LIBS="$LIBS"
	- LIBS="${with_afs}/lib/liblwp.a"
	+ LIBS="${with_afs_libdir}/liblwp.a"
	AC_MSG_CHECKING(if AFS libraries need sigvec)
	dnl Does AFS need sigvec? We have to link against lwp and see
	dnl if IOMGR_Initialize wants it
	AC_TRY_LINK([IOMGR_Initialize();],
	[IOMGR_Initialize()],
	[
	dnl it linked; don't need it
	AC_MSG_RESULT(no)
	cyrus_cv_afs_sigvec="no"
	], [
	dnl didn't link; pick up sigvec
	AC_MSG_RESULT(yes)
	cyrus_cv_afs_sigvec="yes"
	])
	])
	if test "$cyrus_cv_afs_sigvec" = yes; then
	if test "$cant_find_sigvec" = yes; then
	AC_MSG_WARN([Can't find a sigvec for AFS libraries which seem to need one.])
	else
	AFS_LIBS="${AFS_LIBS} $cyrus_cv_sigveclib"
	AC_SUBST(AFS_LIBS)
	AC_SUBST(AFS_LDFLAGS)
	AC_DEFINE(HAVE_AFSKRB,[],[Should we build afskrb pts module?])
	fi
	else
	AFS_LIBS="${AFS_LIBS}"
	AC_SUBST(AFS_LIBS)
	AC_SUBST(AFS_LDFLAGS)
	AC_DEFINE(HAVE_AFSKRB,[],[Should we build afskrb pts module?])
	fi
	LIBS="$SAVE_LIBS"
	fi

	LDAP_CPPFLAGS=""
	LDAP_LDFLAGS=""
	LDAP_LIBS=""

	if test "x$with_ldap" != "xno"; then
	if test ! -d $with_ldap; then
	$with_ldap=/usr/local
	fi

	LDAP_CPPFLAGS="$CPPFLAGS -I${with_ldap}/include"
	LDAP_LDFLAGS="$LDFLAGS -L${with_ldap}/lib"
	LDAP_LIBS=""

	save_CPPFLAGS=$CPPFLAGS
	save_LDFLAGS=$LDFLAGS
	CPPFLAGS=$LDAP_CPPFLAGS
	LDFLAGS=$LDAP_LDFLAGS

	AC_CHECK_LIB(ldap, ldap_initialize,
	[ AC_DEFINE(HAVE_LDAP,[],[Should we build ldap pts module?])
	AC_SUBST(LDAP_CPPFLAGS)
	AC_SUBST(LDAP_LDFLAGS)
	AC_SUBST(LDAP_LIBS)
	LDAP_LIBS="-lldap -llber" ],,-llber)

	CPPFLAGS=$save_CPPFLAGS
	LDFLAGS=$LDAP_LDFLAGS

	AC_CHECK_LIB(ldap, ldap_initialize,
	[ AC_DEFINE(HAVE_LDAP,[],[Should we build ldap pts module?])
	AC_SUBST(LDAP_CPPFLAGS)
	AC_SUBST(LDAP_LDFLAGS)
	AC_SUBST(LDAP_LIBS)
	LDAP_LIBS="-lldap -llber" ],,-llber)

	CPPFLAGS=$save_CPPFLAGS
	LDFLAGS=$save_LDFLAGS
	fi

	if test "x$with_afs" != "xno" -o "x$with_ldap" != "xno"; then
	EXTRA_SUBDIRS="${EXTRA_SUBDIRS} ptclient"
	EXTRA_OUTPUT="${EXTRA_OUTPUT} ptclient/Makefile"

	AC_DEFINE(WITH_PTS,[],[Build in PTS support?])
	fi



	SERVER_SUBDIRS="master imap"
	AC_ARG_ENABLE(server,
	[ --disable-server disable compiling servers],
	if test "$enableval" = no; then
	SERVER_SUBDIRS=""
	fi)
	AC_SUBST(SERVER_SUBDIRS)
	# We always output a server makefile (just because we can)

	dnl this is the new simple check for kerberos; since the person had to
	dnl compile SASL, we might as well use the same checks.
	AC_ARG_WITH(krb,[ --with-krb=PATH use Kerberos from PATH],
	with_krb="$withval", with_krb="no")

	AC_ARG_WITH(krbimpl,[ --with-krbimpl=\[kth\|mit\] assume Kerberos 4 from KTH or MIT],
	with_krbimpl="$withval", with_krbimpl="kth")

	AC_ARG_ENABLE(statickrb,
	[ --enable-statickrb link Kerberos statically],
	with_statickrb="yes", with_statickrb="no")

	dnl In order to compile kerberos4, we need libkrb and libdes.

	dnl we might need -lresolv for kerberos
	AC_CHECK_LIB(resolv,res_search)

	if test "$with_statickrb" = "yes" -a ! -d "$with_krb"; then
	AC_MSG_ERROR([--enable-statickrb specified but --with-krb did not specify a valid directory])
	fi

	if test "$with_krb" != "no"; then
	dnl Do we need DES for kerberos?
	AC_ARG_WITH(krbdes,[ --with-krbdes use Kerberos DES implementation [[yes]]],
	with_krbdes="$withval", with_krbdes="yes")
	if test "$with_krbdes" = "yes"; then
	AC_CHECK_LIB(des,des_ecb_encrypt,
	if test "$with_statickrb" = "yes"; then
	KRB_LIBS="$with_krb/lib/libdes.a"
	else
	KRB_LIBS="-ldes"
	fi,
	AC_MSG_ERROR([The Kerberos DES library is required for Kerberos support. You might want --with-auth=unix.]))
	fi
	fi

	dnl if we were ambitious, we'd look more aggressively for the
	dnl krb4 install
	if test -d ${with_krb}; then
	AC_CACHE_CHECK(for Kerberos includes, cyrus_cv_krbinclude, [
	for krbhloc in include/kerberosIV include
	do
	if test -f ${with_krb}/${krbhloc}/krb.h ; then
	cyrus_cv_krbinclude=${with_krb}/${krbhloc}
	break
	fi
	done
	])

	if test -n "${cyrus_cv_krbinclude}"; then
	CPPFLAGS="$CPPFLAGS -I${cyrus_cv_krbinclude}"
	fi
	CMU_ADD_LIBPATH(${with_krb}/lib)
	fi

	if test "$with_krbimpl" != "kth"; then
	KRBLIB="krb4"
	else
	KRBLIB="krb"
	fi

	if test "$with_des" != no; then
	AC_CHECK_HEADER(krb.h,
	AC_CHECK_LIB(${KRBLIB}, krb_mk_priv,
	if test "$with_statickrb" = "yes"; then
	KRB_LIBS="$KRB_LIBS $with_krb/lib/lib${KRBLIB}.a"
	else
	KRB_LIBS="$KRB_LIBS -l${KRBLIB}"
	fi,
	AC_WARN(No Kerberos V4 found); krb4=no,
	$KRB_LIBS),
	AC_WARN(No Kerberos V4 found); krb4=no)
	else
	AC_WARN(No DES library found for Kerberos V4 support)
	krb4=no
	fi

	if test "${krb4}" != no; then
	AC_DEFINE(HAVE_KRB,[],[Support for Kerberos?])
	fi

	LIBS="$KRB_LIBS $LIBS"

	SASL_SET_GSSAPI_LIBS

	dnl
	dnl Test for OpenSSL
	dnl
	IMAP_PROGS=""
	AC_ARG_WITH(openssl,[ --with-openssl=PATH use OpenSSL from PATH],
	with_openssl="${withval}")

	OPENSSL_INC=
	OPENSSL_LIB=
	case "$with_openssl" in
	no) with_openssl="no";;
	""\|yes)
	dnl if openssl has been compiled with the rsaref2 libraries,
	dnl we need to include the rsaref libraries in the crypto check
	LIB_RSAREF=""
	AC_CHECK_LIB(rsaref, RSAPublicEncrypt,
	LIB_RSAREF="-lRSAglue -lrsaref"; cmu_have_rsaref=yes,
	cmu_have_rsaref=no)

	with_openssl="yes"
	AC_CHECK_LIB(crypto,BIO_accept,
	LIBS="-lcrypto $LIB_RSAREF ${LIBS}",
	with_openssl="no", $LIB_RSAREF)
	AC_CHECK_LIB(ssl, SSL_CTX_new, LIBS="-lssl ${LIBS}",
	with_openssl="no", -lcrypto $LIB_RSAREF)

	;;
	*) OPENSSL_INC="-I${with_openssl}/include"
	OPENSSL_LIBPATH="${with_openssl}/lib"
	OPENSSL_LIB="-L${OPENSSL_LIBPATH}"
	CPPFLAGS="${CPPFLAGS} ${OPENSSL_INC}"
	CMU_ADD_LIBPATH(${OPENSSL_LIBPATH})
	CMU_ADD_LIBPATH_TO(${OPENSSL_LIBPATH}, OPENSSL_LIB)
	LIBS="${LIBS} -lssl -lcrypto";;
	esac

	AC_MSG_CHECKING(for openssl)
	AC_MSG_RESULT($with_openssl)

	if test "$with_openssl" != "no"; then
	AC_DEFINE(HAVE_SSL,[],[Build with SSL support?])
	IMAP_PROGS="$IMAP_PROGS tls_prune"
	if test "${krb4}" != no; then
	AC_DEFINE(OPENSSL_ENABLE_OLD_DES_SUPPORT,[],[Configure OpenSSL to provide legacy des apis])
	AC_DEFINE(OPENSSL_DES_LIBDES_COMPATIBILITY,[],[Configure OpenSSL to provide krb4-compatible legacy des apis])
	fi
	fi
	AC_SUBST(OPENSSL_INC)
	AC_SUBST(OPENSSL_LIB)

	dnl
	dnl Allow for setting EGD socket file on systems without /dev/*random.
	dnl
	AC_ARG_WITH(egd-socket,
	[ --with-egd-socket=FILE Entropy Gathering Daemon socket pathname
	for systems without /dev/urandom],
	[ EGD_SOCKET="$withval" ]
	)
	if test -n "$EGD_SOCKET" ; then
	AC_DEFINE_UNQUOTED(EGD_SOCKET, "$EGD_SOCKET", [Alternative to /dev/urandom?])
	fi

	dnl
	dnl Test for zlib
	dnl
	CMU_HAVE_ZLIB
	AC_MSG_CHECKING(for zlib)
	AC_MSG_RESULT($with_zlib)
	AC_SUBST(ZLIB)

	dnl
	dnl Test for Zephyr
	dnl
	AC_ARG_WITH(zephyr,[ --with-zephyr[=PATH] enable Zephyr notification (installed on PATH)],
	with_zephyr="${withval}")
	if test -z "$with_zephyr"; then
	if test -f /usr/local/lib/libzephyr.a; then
	with_zephyr="/usr/local"
	elif test -f /usr/lib/libzephyr.a; then
	with_zephyr="/usr"
	fi
	fi
	ZEPHYR_LIBS=""
	ZEPHYR_CPPFLAGS=""
	case "$with_zephyr" in
	no) true;;
	""\|yes) AC_CHECK_LIB(zephyr,ZInitialize,ZEPHYR_LIBS="-lzephyr",
	with_zephyr="no",);;
	*) if test -d ${with_zephyr}/include/zephyr; then
	ZEPHYR_CPPFLAGS="-I${with_zephyr}/include/zephyr"
	else
	ZEPHYR_CPPFLAGS="-I${with_zephyr}/include"
	fi
	ZEPHYR_LIBS="-lzephyr";;
	esac
	AC_SUBST(ZEPHYR_LIBS)
	AC_SUBST(ZEPHYR_CPPFLAGS)
	if test "$with_zephyr" != "no"; then
	AC_DEFINE(HAVE_ZEPHYR,[],[Build with Zephyr support?])
	fi

	dnl
	dnl Set pidfile location
	dnl
	AC_ARG_WITH(pidfile,[ --with-pidfile[=PATH] pidfile in PATH (/var/run/cyrus-master.pid)],
	[MASTERPIDFILE="$withval"],
	[MASTERPIDFILE="/var/run/cyrus-master.pid"])
	MASTERPIDFILE="\"$MASTERPIDFILE\""
	AC_DEFINE_UNQUOTED(MASTER_PIDFILE, $MASTERPIDFILE,[Name of the pidfile for master])

	dnl
	dnl see if we're compiling with IMAP idled support
	dnl
	AC_ARG_ENABLE(idled,
	[ --enable-idled enable IMAP idled support],
	if test "$enable_val" != no; then
	IMAP_PROGS="$IMAP_PROGS idled"
	fi)

	dnl
	dnl see if we're compiling with NNTP support
	dnl
	ENABLE_NNTP=no
	AC_ARG_ENABLE(nntp,
	[ --enable-nntp enable NNTP support],
	ENABLE_NNTP=$enableval
	if test "$ENABLE_NNTP" != no; then
	IMAP_PROGS="$IMAP_PROGS nntpd fetchnews"
	fi)

	dnl
	dnl see if we're compiling the Murder support programs
	dnl
	ENABLE_MURDER=no
	AC_ARG_ENABLE(murder,
	[ --enable-murder enable IMAP Murder support],
	ENABLE_MURDER=$enableval)

	if test "$ENABLE_MURDER" != no; then
	IMAP_PROGS="$IMAP_PROGS mupdate"
	# for master/slave auto-selection
	AC_CHECK_HEADERS(sys/sockio.h)
	fi

	dnl
	dnl see if we're compiling replication support programs
	dnl
	ENABLE_REPLICATION=no
	AC_ARG_ENABLE(replication,
	[ --enable-replication enable replication support (experimental)],
	ENABLE_REPLICATION=$enableval
	if test "$ENABLE_REPLICATION" != no; then
	IMAP_PROGS="$IMAP_PROGS sync_client sync_server sync_reset"
	fi)

	AC_SUBST(IMAP_PROGS)

	dnl
	dnl Try and find a system version of com_err.
	dnl If we see something that looks a little wacky, ignore it (there are many
	dnl deficient installs of com_err, unfortunately, which leave out compile_et)
	dnl
	AC_ARG_WITH(com_err,
	[ --with-com_err=PATH use com_err from path -- includes in PATH/include,
	libs in PATH/lib, and compile_et in PATH/bin])
	if test -z "$with_com_err"; then
	# no value supplied
	AC_CHECK_LIB(com_err, com_err, [
	# com_err is already in library path
	# guess we're okay
	# can use system com_err
	with_com_err=""
	AC_CHECK_HEADER(et/com_err.h,
	[AC_DEFINE(HAVE_ET_COM_ERR_H,[],[We need et/com_err.h])],
	[AC_CHECK_HEADER(com_err.h,[],[AC_ERROR([cannot locate com_err.h])])])
	AC_PATH_PROG(COMPILE_ET, compile_et, [no compile et])
	], [
	if test -f /usr/local/include/com_err.h -o -f /usr/local/include/et/com_err.h; then
	with_com_err="/usr/local"
	AC_PATH_PROG(COMPILE_ET, /usr/local/bin/compile_et, [no compile et])
	elif test -f /usr/include/com_err.h -o -f /usr/include/et/com_err.h; then
	with_com_err="/usr"
	AC_PATH_PROG(COMPILE_ET, /usr/bin/compile_et, [no compile et])
	else
	# use ours
	with_com_err=yes
	fi
	])

	if test "${with_com_err}" = "no"; then
	AC_MSG_WARN([com_err is required; included version will be used.])
	with_com_err="yes"
	fi
	if test "${COMPILE_ET}" = "no compile et" -o "${COMPILE_ET}" = ""; then
	AC_MSG_WARN([Parts of com_err distribuion were found, but not compile_et.])
	AC_MSG_WARN([Will build com_err from included sources.])
	with_com_err="yes" # build it ourselves
	fi
	fi

	case "$with_com_err" in
	# built-in et
	yes) # use the com_err we're gonna build
	COM_ERR_LIBS="../com_err/et/libcom_err.a"
	COMPILE_ET="../com_err/et/compile_et"
	COM_ERR_LDFLAGS=""
	COM_ERR_CPPFLAGS="-I\${top_srcdir}/com_err/et"
	PRE_SUBDIRS="com_err/et ${PRE_SUBDIRS}"
	EXTRA_OUTPUT="${EXTRA_OUTPUT} com_err/et/Makefile"
	;;
	"") # no problem, we already have it in the paths
	# we do nothing to pick it up
	COM_ERR_LIBS="-lcom_err" # hope it's not shared
	# we already set COMPILE_ET, or we didn't get here
	COM_ERR_LDFLAGS=""
	COM_ERR_CPPFLAGS=""
	;;
	*) # use whatever they told us, or whatever we found
	COMPILE_ET="${with_com_err}/bin/compile_et"
	COM_ERR_LIBS="${with_com_err}/lib/libcom_err.a"
	COM_ERR_CPPFLAGS="-I${with_com_err}/include"
	# Ever get the feeling people hide this stuff on purpose?
	if test -d "${with_com_err}/include/et" ; then
	COM_ERR_CPPFLAGS="-I${with_com_err}/include/et"
	fi
	dnl CMU_ADD_LIBPATH_TO(${with_com_err}/lib, COM_ERR_LDFLAGS)
	COMPILE_ET="${with_com_err}/bin/compile_et"
	esac
	AC_SUBST(COMPILE_ET)
	AC_SUBST(COM_ERR_LIBS)
	AC_SUBST(COM_ERR_LDFLAGS)
	AC_SUBST(COM_ERR_CPPFLAGS)

	AC_MSG_CHECKING(for modern syslog)
	AC_CACHE_VAL(cyrus_cv_lib_syslog, AC_TRY_CPP([#include <syslog.h>
	#ifndef LOG_LOCAL6
	#include </nonexistent>
	#endif],cyrus_cv_lib_syslog=yes,cyrus_cv_lib_syslog=no))
	if test $cyrus_cv_lib_syslog = no; then
	PRE_SUBDIRS="${PRE_SUBDIRS} syslog"
	EXTRA_OUTPUT="${EXTRA_OUTPUT} syslog/Makefile"
	DEPLIBS="${DEPLIBS} ../syslog/libsyslog.a"
	CPPFLAGS="$CPPFLAGS -I\$(srcdir)/../syslog"
	fi
	AC_MSG_RESULT($cyrus_cv_lib_syslog)

	AC_MSG_CHECKING(which syslog facility to use)
	SYSLOG_FACILITY=LOG_LOCAL6
	AC_ARG_WITH(syslogfacility,[ --with-syslogfacility=FACILITY set the syslog facility to use (default LOCAL6)],
	[ if test "$withval" != "yes" -a "$withval" != "no" ; then
	SYSLOG_FACILITY=LOG_$withval
	fi; ])
	AC_DEFINE_UNQUOTED(SYSLOG_FACILITY, $SYSLOG_FACILITY, [Syslog facility to use.])
	AC_MSG_RESULT($SYSLOG_FACILITY)

	dnl Have to check getdtabalesize after adding ossup, as some ossups define it
	AC_REPLACE_FUNCS(getdtablesize)
	AC_ARG_ENABLE(cmulocal,
	[ --enable-cmulocal enable CMU-specific local support],
	if test "$enableval" = yes; then
	EXTRA_SUBDIRS="${EXTRA_SUBDIRS} netnews depot"
	EXTRA_OUTPUT="${EXTRA_OUTPUT} depot/Makefile"
	fi)

	AC_MSG_CHECKING(to use old sieve service name)
	AC_ARG_ENABLE(oldsievename,
	[ --enable-oldsievename enable the use of 'imap' as the sieve service name],
	if test "$enableval" = yes; then
	AC_MSG_RESULT(yes)
	AC_DEFINE(OLD_SIEVE_SERVICE_NAME,[],[Use "imap" as sieve service name?])
	else
	AC_MSG_RESULT(no)
	fi,
	AC_MSG_RESULT(no))

	AC_ARG_ENABLE(netscapehack,
	[ --enable-netscapehack enable Netscape hack for the menu option
	in Communicator to Administrate Mail],
	if test "$enableval" = yes; then
	AC_DEFINE(ENABLE_X_NETSCAPE_HACK,[],[Enable Netscape Menu Option Hack?])
	fi)

	AC_CHECK_FUNC(dlopen,,[AC_CHECK_LIB(dl, dlopen)])
	CMU_SASL2_REQUIRE_VER(2,1,7)
	CMU_SASL2_CHECKAPOP_REQUIRED

	AC_ARG_WITH(perl, [ --with-perl=PERL use PERL for perl],
	with_perl="$withval", with_perl="perl")

	if test "${with_perl}" = yes; then
	with_perl="perl"
	fi
	if test "${with_perl}" != no; then
	if test ${using_static_sasl} = "staticonly"; then
	AC_MSG_WARN([Cannot compile perl utilities using static libsasl])
	with_perl="no"
	else
	AC_CHECK_PROGS(PERL, ${with_perl} perl, with_perl=notfound)
	fi
	fi
	if test "$with_perl" = "notfound"; then
	AC_MSG_WARN(Perl not found: Administrative tools won't be available)
	elif test "${with_perl}" != "no"; then
	dnl compile perl stuff
	EXTRA_SUBDIRS="${EXTRA_SUBDIRS} perl"
	dnl and compile perl/cyradm
	PERL_SUBDIRS="imap"
	PERL="${with_perl}"
	dnl add perl cccdlflags when building libraries -- this ensures that the
	dnl libraries will be compiled as PIC if perl requires PIC objects
	dnl -- this is needed on NetBSD and Linux, but seems to cause problems on atleast Solaris --
	case "${target_os}" in
	linux\|netbsd)
	AC_MSG_CHECKING(for perl cccdlflags needed on "${target_os}")
	eval `${PERL} -V:cccdlflags`
	PERL_CCCDLFLAGS="$cccdlflags"
	AC_SUBST(PERL_CCCDLFLAGS)
	AC_MSG_RESULT($PERL_CCCDLFLAGS)
	;;
	*)
	AC_MSG_WARN(skipping check for perl cccdlflags on "${target_os}")
	esac
	fi

	dnl for timsieved
	if test "$sievedir" != "no"; then
	EXTRA_SUBDIRS="${EXTRA_SUBDIRS} timsieved notifyd"
	EXTRA_OUTPUT="${EXTRA_OUTPUT} timsieved/Makefile notifyd/Makefile"

	PERL_SUBDIRS="${PERL_SUBDIRS} sieve"
	PERL_DEPSUBDIRS="sieve"
	EXTRA_OUTPUT="${EXTRA_OUTPUT} perl/sieve/Makefile perl/sieve/lib/Makefile"
	else
	PERL_DEPSUBDIRS="none"
	fi

	dnl Check for MD5 functions
	AC_FUNC_CHECK(MD5Init,,
	AC_CHECK_LIB(md, MD5Init, LIBS="${LIBS} -lmd",
	MD5OBJ="md5.o"))
	AC_SUBST(MD5OBJ)

	dnl snmp
	dnl (agentx was depricated, but SNMP_SUBDIRS is conveinent as a placeholder)
	SNMP_SUBDIRS=""
	AC_SUBST(SNMP_SUBDIRS)

	CMU_LIBWRAP
	CMU_UCDSNMP

	# Figure out what directories we're linking against.
	# Lots of fun for the whole family.
	# This probably chokes on anything with spaces in it.
	# All we want is the list of -L directories, and -L may or may not be
	# followed by a space.
	isdir=no
	libpath=""
	#echo "debug ldflags: << ${ldflags} >>"
	#echo "debug default_ldflags: << ${default_ldflags} >>"
	for flag in ${ldflags} ${default_ldflags}; do
	case $flag in
	-L)
	# it's a split -L option, we'll mark the next option as a dir.
	isdir=yes
	;;

	-L*)
	# attached -L option: split off the directory
	larg=`echo $flag \| sed -e 's:-L$..*$:\1:'`
	libpath="${libpath} ${larg}"
	;;

	*)
	if test $isdir = yes ; then
	libpath="${libpath} ${flag}"
	isdir=no
	fi
	esac
	done

	IMAP_COM_ERR_LIBS="${COM_ERR_LIBS}"
	IMAP_LIBS="${LIB_SASL} ${LIBS} ${SQL_LIBS}"

	AC_SUBST(LIB_RT)
	AC_SUBST(IMAP_COM_ERR_LIBS)
	AC_SUBST(IMAP_LIBS)

	dnl AC_OUTPUT_COMMANDS([
	dnl if test "$with_perl" != "no"; then
	dnl (cd perl/sieve/managesieve; $perl Makefile.PL PREFIX=$prefix)
	dnl (cd perl/imap; $perl Makefile.PL PREFIX=$prefix)
	dnl fi
	dnl ], perl=$PERL; with_perl=$with_perl; prefix=$prefix; SASL_LIB="$LIB_SASL"; SASL_INC="$SASLFLAGS"; export SASL_LIB SASL_INC)
	AC_SUBST(PERL_SUBDIRS)
	AC_SUBST(PERL_DEPSUBDIRS)
	AC_SUBST(PERL)

	AH_TOP([
	/*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: configure.in,v 1.316 2010/01/06 17:01:26 murch Exp $
	*/

	#ifndef _CYRUS_IMAPD_CONFIG_H_
	#define _CYRUS_IMAPD_CONFIG_H_
	])

	AH_BOTTOM([
	/* time.h */
	#if TIME_WITH_SYS_TIME
	# include <sys/time.h>
	# include <time.h>
	#else
	# if HAVE_SYS_TIME_H
	# include <sys/time.h>
	# else
	# include <time.h>
	# endif
	#endif

	/* com_err.h, as needed */
	#ifndef IN_COM_ERR
	#ifdef HAVE_ET_COM_ERR_H
	#include <et/com_err.h>
	#else
	#include <com_err.h>
	#endif /* HAVE_ET_COM_ERR_H */
	#endif /* IN_COM_ERR */

	/* This allows us to work even when we don't have an fdatasync */
	#ifndef HAVE_FDATASYNC
	#define fdatasync(fd) fsync(fd)
	#endif

	/* A similar setup for not having O_DSYNC */
	#include <fcntl.h>

	#ifndef O_DSYNC
	# ifdef O_SYNC
	# define O_DSYNC O_SYNC /* POSIX */
	# else
	# define O_DSYNC O_FSYNC /* BSD */
	# endif
	#endif

	#ifndef HAVE___ATTRIBUTE__
	/* Can't use attributes... */
	#define __attribute__(foo)
	#endif

	#ifndef HAVE_SOCKLEN_T
	typedef unsigned int socklen_t;
	#endif

	#ifndef HAVE_RLIM_T
	typedef int rlim_t;
	#endif

	/* some potentially memory saving tradeoffs,
	preconfigured in memory-saving mode */

	/* save the cmdlines for the ID command */
	#undef ID_SAVE_CMDLINE

	/* IPv6 things */
	#include <netdb.h>
	#include <sys/types.h>
	#include <sys/socket.h>
	#include <netinet/in.h>

	#ifndef HAVE_STRUCT_SOCKADDR_STORAGE
	#define _SS_MAXSIZE 128 /* Implementation specific max size */
	#define _SS_PADSIZE (_SS_MAXSIZE - sizeof (struct sockaddr))

	struct sockaddr_storage {
	struct sockaddr ss_sa;
	char __ss_pad2[_SS_PADSIZE];
	};
	# define ss_family ss_sa.sa_family
	# define HAVE_SS_FAMILY
	#endif /* !HAVE_STRUCT_SOCKADDR_STORAGE */

	#ifndef HAVE_SS_FAMILY
	#define ss_family __ss_family
	#endif

	#ifndef AF_INET6
	/* Define it to something that should never appear */
	#define AF_INET6 AF_MAX
	#endif

	#if !defined(HAVE_GETADDRINFO) \|\| !defined(HAVE_GETNAMEINFO)
	#include "gai.h"
	#endif

	/* End IPv6 things */

	#ifdef OLD_SIEVE_SERVICE_NAME
	#define SIEVE_SERVICE_NAME "imap"
	#else
	#define SIEVE_SERVICE_NAME "sieve"
	#endif

	/* filenames */
	#define FNAME_DBDIR "/db"
	#define FNAME_USERDIR "/user/"
	#define FNAME_DOMAINDIR "/domain/"
	#define FNAME_LOGDIR "/log/"
	#define FNAME_PTSDB "/ptclient/ptscache.db"
	#define CONFIG_FILENAME (SYSCONFDIR "/imapd.conf")
	#define DEFAULT_MASTER_CONFIG_FILENAME (SYSCONFDIR "/cyrus.conf")

	#ifndef HAVE_SHUTDOWN
	#define shutdown(fd, mode) 0
	#endif

	/* printf() macros /
	#if (SIZEOF_SIZE_T == SIZEOF_INT)
	#define SIZE_T_FMT "%u"
	#elif (SIZEOF_SIZE_T == SIZEOF_LONG)
	#define SIZE_T_FMT "%lu"
	#elif (SIZEOF_SIZE_T == SIZEOF_LONG_LONG_INT)
	#define SIZE_T_FMT "%llu"
	#else
	#error dont know what to use for SIZE_T_FMT
	#endif

	#if (SIZEOF_OFF_T == SIZEOF_LONG)
	#define OFF_T_FMT "%ld"
	#define strtoofft(nptr, endptr, base) strtol(nptr, endptr, base)
	#elif (SIZEOF_OFF_T == SIZEOF_LONG_LONG_INT)
	#define OFF_T_FMT "%lld"
	#define strtoofft(nptr, endptr, base) strtoll(nptr, endptr, base)
	#else
	#error dont know what to use for OFF_T_FMT
	#endif

	/* compile time options; think carefully before modifying */
	enum {
	/* should we send an UNAVAILABLE message to master when
	* a service is exiting (master is already going to be
	* informed of the exit by the SIGCHLD signal anyway) ? */
	MESSAGE_MASTER_ON_EXIT = 0,

	/* should a hierarchical rename stop on error? */
	RENAME_STOP_ON_ERROR = 1,

	/* should we call fsync() to maybe help with softupdates? (it should) */
	APPEND_ULTRA_PARANOID = 1,

	/* should we log extra information at the DEBUG level for DB stuff?
	* 0 -> nothing; 1 -> some; higher -> even more */
	CONFIG_DB_VERBOSE = 1,

	/* log timing information to LOG_DEBUG */
	CONFIG_TIMING_VERBOSE = 0,

	/* should we be pedantic about namespace or sleezy? */
	SLEEZY_NAMESPACE = 1,

	/* should we do a fast TLS session shutdown? */
	TLS_FAST_SHUTDOWN = 1,

	/* should we use the SQUAT engine to accelerate SEARCH? */
	SQUAT_ENGINE = 1,

	/* should we have long LMTP error messages? */
	LMTP_LONG_ERROR_MSGS = 1
	};

	#endif /* _CYRUS_IMAPD_CONFIG_H_ */
	])

	dnl make sure that Makefile is the last thing output
	AC_OUTPUT(man/Makefile master/Makefile lib/Makefile imap/Makefile imtest/Makefile netnews/Makefile perl/Makefile $EXTRA_OUTPUT Makefile)
	diff --git a/imap/dlist.c b/imap/dlist.c
	index e0f1382b5..af0179db1 100644
	--- a/imap/dlist.c
	+++ b/imap/dlist.c
	@@ -1,593 +1,593 @@
	/* dlist.c - list protocol for dump and sync
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: sync_support.c,v 1.25 2010/01/06 17:01:41 murch Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdlib.h>
	#include <stdio.h>
	#include <time.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <fcntl.h>
	#include <syslog.h>
	#include <string.h>
	#include <sys/wait.h>
	#include <errno.h>
	#include <ctype.h>
	#include <dirent.h>
	#include <utime.h>

	#include "global.h"
	#include "assert.h"
	#include "mboxlist.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "quota.h"
	#include "xmalloc.h"
	#include "xstrlcat.h"
	#include "xstrlcpy.h"
	#include "acl.h"
	#include "seen.h"
	#include "mboxname.h"
	#include "map.h"
	#include "imapd.h"
	#include "imparse.h"
	#include "message.h"
	#include "util.h"
	#include "retry.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "prot.h"

	#include "dlist.h"

	/* Parse routines */

	const char *lastkey = NULL;

	static void printfile(struct protstream out, struct dlist dl)
	{
	char buf[4096];
	struct stat sbuf;
	FILE *f;
	unsigned long size;

	f = fopen(dl->sval, "r");
	if (!f) {
	syslog(LOG_ERR, "IOERROR: Failed to read file %s", dl->sval);
	prot_printf(out, "NIL");
	return;
	}
	if (fstat(fileno(f), &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: Failed to stat file %s", dl->sval);
	prot_printf(out, "NIL");
	fclose(f);
	return;
	}
	size = sbuf.st_size;
	if (size != dl->nval) {
	syslog(LOG_ERR, "IOERROR: Size mismatch %s (%lu != " MODSEQ_FMT ")",
	dl->sval, size, dl->nval);
	prot_printf(out, "NIL");
	fclose(f);
	return;
	}

	prot_printf(out, "%%{");
	prot_printastring(out, dl->part);
	prot_printf(out, " ");
	prot_printastring(out, message_guid_encode(&dl->gval));
	prot_printf(out, " %lu}\r\n", size);
	while (size) {
	int n = fread(buf, 1, (size > 4096 ? 4096 : size), f);
	if (n <= 0) break;
	prot_write(out, buf, n);
	size -= n;
	}
	fclose(f);

	if (size) fatal("failed to finish reading file!", EC_IOERR);
	}

	/* XXX - these two functions should be out in append.c or reserve.c
	* or something more general */
	const char dlist_reserve_path(const char part, struct message_guid *guid)
	{
	static char buf[MAX_MAILBOX_PATH];
	snprintf(buf, MAX_MAILBOX_PATH, "%s/sync./%lu/%s",
	config_partitiondir(part), (unsigned long)getpid(),
	message_guid_encode(guid));
	cyrus_mkdir(buf, 0755);
	return buf;
	}

	static int reservefile(struct protstream in, const char part,
	struct message_guid *guid, unsigned long size,
	const char **fname)
	{
	FILE *file;
	char buf[8192+1];
	int r = 0, n;

	/* XXX - write to a temporary file then move in to place! */
	*fname = dlist_reserve_path(part, guid);

	file = fopen(*fname, "w+");
	if (!file) {
	syslog(LOG_ERR, "Failed to upload file %s", message_guid_encode(guid));
	r = IMAP_IOERROR;
	}

	/* XXX - calculate sha1 on the fly? */
	while (size) {
	n = prot_read(in, buf, size > 8192 ? 8192 : size);
	if (!n) {
	syslog(LOG_ERR,
	"IOERROR: reading message: unexpected end of file");
	r = IMAP_IOERROR;
	break;
	}
	size -= n;
	if (!r) fwrite(buf, 1, n, file);
	}

	if (r) return r;

	/* Make sure that message flushed to disk just incase mmap has problems */
	fflush(file);
	if (ferror(file)) {
	fclose(file);
	return IMAP_IOERROR;
	}

	if (fsync(fileno(file)) < 0) {
	fclose(file);
	return IMAP_IOERROR;
	}

	fclose(file);

	return 0;
	}

	/* DLIST STUFF */

	void dlist_stitch(struct dlist dl, struct dlist child)
	{
	if (dl->tail)
	dl->tail = dl->tail->next = child;
	else
	dl->head = dl->tail = child;
	}

	static struct dlist dlist_child(struct dlist dl, const char *name)
	{
	struct dlist *i = xzmalloc(sizeof(struct dlist));
	i->name = xstrdup(name);
	i->type = DL_NIL;
	if (dl)
	dlist_stitch(dl, i);
	return i;
	}

	struct dlist dlist_atom(struct dlist dl, const char name, const char val)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_ATOM;
	i->sval = xstrdup(val);
	return i;
	}

	struct dlist dlist_flag(struct dlist dl, const char name, const char val)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_FLAG;
	i->sval = xstrdup(val);
	return i;
	}

	struct dlist dlist_num(struct dlist dl, const char *name, unsigned long val)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_NUM;
	i->nval = (modseq_t)val;
	return i;
	}

	struct dlist dlist_date(struct dlist dl, const char *name, time_t val)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_DATE;
	i->nval = (modseq_t)val;
	return i;
	}

	struct dlist dlist_modseq(struct dlist dl, const char *name, modseq_t val)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_MODSEQ;
	i->nval = val;
	return i;
	}

	struct dlist dlist_guid(struct dlist dl, const char *name,
	struct message_guid *guid)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_GUID,
	message_guid_copy(&i->gval, guid);
	return i;
	}

	struct dlist dlist_file(struct dlist dl, const char *name,
	const char *part,
	struct message_guid *guid,
	unsigned long size,
	const char *fname)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_FILE;
	message_guid_copy(&i->gval, guid);
	i->sval = xstrdup(fname);
	i->nval = size;
	i->part = xstrdup(part);
	return i;
	}

	struct dlist dlist_buf(struct dlist dl, const char *name,
	char *val, size_t len)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_BUF;
	i->sval = xmalloc(len);
	memcpy(i->sval, val, len);
	i->nval = len;
	return i;
	}

	struct dlist dlist_kvlist(struct dlist dl, const char *name)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_KVLIST;
	return i;
	}

	struct dlist dlist_list(struct dlist dl, const char *name)
	{
	struct dlist *i = dlist_child(dl, name);
	i->type = DL_ATOMLIST;
	return i;
	}

	struct dlist dlist_new(const char name)
	{
	return dlist_kvlist(NULL, name);
	}

	void dlist_print_helper(struct dlist *dl, int printkeys,
	struct protstream *out, int level)
	{
	struct dlist *di;
	int i;

	if (printkeys)
	prot_printf(out, "%s ", dl->name);

	switch (dl->type) {
	case DL_ATOM:
	prot_printastring(out, dl->sval);
	break;
	case DL_FLAG:
	prot_printf(out, "%s", dl->sval);
	break;
	case DL_NUM:
	case DL_DATE: /* for now, we will format it later */
	case DL_MODSEQ:
	prot_printf(out, MODSEQ_FMT, dl->nval);
	break;
	case DL_FILE:
	printfile(out, dl);
	break;
	case DL_BUF:
	prot_printliteral(out, dl->sval, dl->nval);
	break;
	case DL_KVLIST:
	if (level) {
	prot_printf(out, "\r\n");
	for (i = 0; i <= level; i++)
	prot_printf(out, " ");
	}
	prot_printf(out, "%%(");
	for (di = dl->head; di; di = di->next) {
	dlist_print_helper(di, 1, out, level);
	if (di->next) {
	prot_printf(out, " ");
	}
	}
	prot_printf(out, ")");
	break;
	case DL_ATOMLIST:
	prot_printf(out, "(");
	for (di = dl->head; di; di = di->next) {
	dlist_print_helper(di, 0, out, di->type == DL_KVLIST ? level + 1 : level);
	if (di->next)
	prot_printf(out, " ");
	}
	prot_printf(out, ")");
	break;
	}
	}

	void dlist_print(struct dlist dl, int printkeys, struct protstream out)
	{
	dlist_print_helper(dl, printkeys, out, 0);
	}

	void dlist_free(struct dlist **dlp)
	{
	struct dlist i, next;
	if (!*dlp) return;
	i = (*dlp)->head;
	while (i) {
	free(i->name);
	next = i->next;
	switch (i->type) {
	case DL_KVLIST:
	case DL_ATOMLIST:
	dlist_free(&i);
	break;
	case DL_FILE:
	free(i->part);
	/* drop through */
	default:
	free(i->sval);
	}
	free(i);
	i = next;
	}
	free(*dlp);
	*dlp = NULL;
	}

	static char next_nonspace(struct protstream *in, char c)
	{
	while (Uisspace(c)) {
	c = prot_getc(in);
	}
	return c;
	}

	char dlist_parse(struct dlist *dlp, int parsekey, struct protstream in)
	{
	struct dlist *dl = NULL;
	static struct buf kbuf;
	static struct buf vbuf;
	char c;

	/* handle the key if wanted */
	if (parsekey) {
	c = getword(in, &kbuf);
	c = next_nonspace(in, c);
	}
	else {
	buf_setcstr(&kbuf, "");
	c = prot_getc(in);
	}

	/* connection dropped? */
	if (c == EOF) goto fail;

	/* check what sort of value we have */
	if (c == '(') {
	dl = dlist_list(NULL, kbuf.s);
	c = next_nonspace(in, ' ');
	while (c != ')') {
	struct dlist *di = NULL;
	prot_ungetc(c, in);
	c = dlist_parse(&di, 0, in);
	if (di) dlist_stitch(dl, di);
	c = next_nonspace(in, c);
	if (c == EOF) goto fail;
	}
	c = prot_getc(in);
	}
	else if (c == '%') {
	/* no whitespace allowed here */
	c = prot_getc(in);
	if (c == '(') {
	dl = dlist_list(NULL, kbuf.s);
	c = next_nonspace(in, ' ');
	while (c != ')') {
	struct dlist *di = NULL;
	prot_ungetc(c, in);
	c = dlist_parse(&di, 1, in);
	if (di) dlist_stitch(dl, di);
	c = next_nonspace(in, c);
	if (c == EOF) goto fail;
	}
	}
	else if (c == '{') {
	struct message_guid tmp_guid;
	static struct buf pbuf, gbuf;
	unsigned size = 0;
	const char *fname;
	c = getastring(in, NULL, &pbuf);
	if (c != ' ') goto fail;
	c = getastring(in, NULL, &gbuf);
	if (c != ' ') goto fail;
	c = getuint32(in, &size);
	if (c != '}') goto fail;
	c = prot_getc(in);
	if (c == '\r') c = prot_getc(in);
	if (c != '\n') goto fail;
	if (!message_guid_decode(&tmp_guid, gbuf.s)) goto fail;
	if (reservefile(in, pbuf.s, &tmp_guid, size, &fname)) goto fail;
	dl = dlist_file(NULL, kbuf.s, pbuf.s, &tmp_guid, size, fname);
	/* file literal */
	}
	else {
	/* unknown percent type */
	goto fail;
	}
	c = prot_getc(in);
	}
	else if (c == '{') {
	prot_ungetc(c, in);
	/* could be binary in a literal */
	c = getbastring(in, NULL, &vbuf);
	dl = dlist_buf(NULL, kbuf.s, vbuf.s, vbuf.len);
	}
	else {
	prot_ungetc(c, in);
	c = getastring(in, NULL, &vbuf);
	dl = dlist_atom(NULL, kbuf.s, vbuf.s);
	if (imparse_isnumber(vbuf.s))
	dl->nval = atomodseq_t(vbuf.s);
	}

	/* success */
	*dlp = dl;
	return c;

	fail:
	dlist_free(&dl);
	return '-';
	}

	static struct dlist dlist_getchild(struct dlist dl, const char *name)
	{
	struct dlist *i;
	for (i = dl->head; i; i = i->next) {
	if (!strcmp(name, i->name))
	return i;
	}
	lastkey = name;
	return NULL;
	}

	/* XXX - type coercion logic here */

	int dlist_getatom(struct dlist dl, const char name, const char **val)
	{
	struct dlist *i = dlist_getchild(dl, name);
	if (!i) return 0;
	*val = i->sval;
	return 1;
	}

	int dlist_getbuf(struct dlist dl, const char name, const char *val, size_t len)
	{
	struct dlist *i = dlist_getchild(dl, name);
	if (!i) return 0;
	*val = i->sval;
	*len = (size_t)i->nval;
	return 1;
	}

	int dlist_getnum(struct dlist dl, const char name, uint32_t *val)
	{
	struct dlist *i = dlist_getchild(dl, name);
	if (!i) return 0;
	*val = (uint32_t)i->nval;
	return 1;
	}

	int dlist_getdate(struct dlist dl, const char name, time_t *val)
	{
	struct dlist *i = dlist_getchild(dl, name);
	if (!i) return 0;
	/* XXX: string parse when the date format changes */
	*val = (time_t)i->nval;
	return 1;
	}

	int dlist_getmodseq(struct dlist dl, const char name, modseq_t *val)
	{
	struct dlist *i = dlist_getchild(dl, name);
	if (!i) return 0;
	*val = (modseq_t)i->nval;
	return 1;
	}

	int dlist_getguid(struct dlist dl, const char name, struct message_guid **val)
	{
	struct dlist *i = dlist_getchild(dl, name);
	if (!i) return 0;
	/* XXX - maybe malloc like strings? would save some in the general case */
	if (!message_guid_decode(&i->gval, i->sval)) return 0;
	*val = &i->gval;
	return 1;
	}

	int dlist_getfile(struct dlist dl, const char name,
	const char **part,
	struct message_guid **guid,
	unsigned long *size,
	const char **fname)
	{
	struct dlist *i = dlist_getchild(dl, name);
	if (!i) return 0;
	if (!message_guid_decode(&i->gval, i->sval)) return 0;
	*guid = &i->gval;
	*size = i->nval;
	*fname = i->sval;
	*part = i->part;
	return 1;
	}

	int dlist_getlist(struct dlist dl, const char name, struct dlist **val)
	{
	struct dlist *i = dlist_getchild(dl, name);
	if (!i) return 0;
	*val = i;
	return 1;
	}

	const char *dlist_lastkey()
	{
	return lastkey;
	}
	diff --git a/imap/fetchnews.c b/imap/fetchnews.c
	index a1d43624e..9aa505820 100644
	--- a/imap/fetchnews.c
	+++ b/imap/fetchnews.c
	@@ -1,626 +1,626 @@
	/* fetchnews.c -- Program to pull new articles from a peer and push to server
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: fetchnews.c,v 1.22 2010/01/06 17:01:31 murch Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <string.h>
	#include <syslog.h>
	#include <signal.h>

	#include <netdb.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <sys/un.h>

	#include "cyrusdb.h"
	#include "exitcodes.h"
	#include "global.h"
	#include "gmtoff.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "prot.h"
	#include "util.h"
	#include "xmalloc.h"
	#include "xstrlcat.h"

	/* global state */
	const int config_need_data = 0;

	#define FNAME_NEWSRCDB "/fetchnews.db"
	#define DB (&cyrusdb_flat)

	static struct db *newsrc_db = NULL;
	static int newsrc_dbopen = 0;

	/* must be called after cyrus_init */
	int newsrc_init(char *fname, int myflags __attribute__((unused)))
	{
	char buf[1024];
	int r = 0;

	if (r != 0)
	syslog(LOG_ERR, "DBERROR: init %s: %s", buf,
	cyrusdb_strerror(r));
	else {
	char *tofree = NULL;

	/* create db file name */
	if (!fname) {
	fname = xmalloc(strlen(config_dir)+sizeof(FNAME_NEWSRCDB));
	tofree = fname;
	strcpy(fname, config_dir);
	strcat(fname, FNAME_NEWSRCDB);
	}

	r = (DB->open)(fname, CYRUSDB_CREATE, &newsrc_db);
	if (r != 0)
	syslog(LOG_ERR, "DBERROR: opening %s: %s", fname,
	cyrusdb_strerror(r));
	else
	newsrc_dbopen = 1;

	if (tofree) free(tofree);
	}

	return r;
	}

	int newsrc_done(void)
	{
	int r = 0;

	if (newsrc_dbopen) {
	r = (DB->close)(newsrc_db);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error closing fetchnews.db: %s",
	cyrusdb_strerror(r));
	}
	newsrc_dbopen = 0;
	}

	return r;
	}

	void usage(void)
	{
	fprintf(stderr,
	"fetchnews [-C <altconfig>] [-s <server>] [-n] [-y] [-w <wildmat>] [-f <tstamp file>]\n"
	" [-a <authname> [-p <password>]] <peer>\n");
	exit(-1);
	}

	int init_net(const char host, char port,
	struct protstream in, struct protstream out)
	{
	int sock = -1, err;
	struct addrinfo hints, res, res0;

	memset(&hints, 0, sizeof(hints));
	hints.ai_family = PF_UNSPEC;
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_protocol = 0;
	if ((err = getaddrinfo(host, port, &hints, &res0)) != 0) {
	syslog(LOG_ERR, "getaddrinfo(%s, %s) failed: %m", host, port);
	return -1;
	}

	for (res = res0; res; res = res->ai_next) {
	if ((sock = socket(res->ai_family, res->ai_socktype,
	res->ai_protocol)) < 0)
	continue;
	if (connect(sock, res->ai_addr, res->ai_addrlen) >= 0)
	break;
	close(sock);
	sock = -1;
	}
	freeaddrinfo(res0);
	if(sock < 0) {
	syslog(LOG_ERR, "connect(%s:%s) failed: %m", host, port);
	return -1;
	}

	*in = prot_new(sock, 0);
	*out = prot_new(sock, 1);
	prot_setflushonread(in, out);

	return sock;
	}

	int fetch(char *msgid, int bymsgid,
	struct protstream pin, struct protstream pout,
	struct protstream sin, struct protstream sout,
	int rejected, int accepted, int *failed)
	{
	char buf[4096];

	/* see if we want this article */
	prot_printf(sout, "IHAVE %s\r\n", msgid);
	if (!prot_fgets(buf, sizeof(buf), sin)) {
	syslog(LOG_ERR, "IHAVE terminated abnormally");
	return -1;
	}
	else if (strncmp("335", buf, 3)) {
	/* don't want it */
	(*rejected)++;
	return 0;
	}

	/* fetch the article */
	if (bymsgid)
	prot_printf(pout, "ARTICLE %s\r\n", msgid);
	else
	prot_printf(pout, "ARTICLE\r\n");

	if (!prot_fgets(buf, sizeof(buf), pin)) {
	syslog(LOG_ERR, "ARTICLE terminated abnormally");
	return -1;
	}
	else if (strncmp("220", buf, 3)) {
	/* doh! the article doesn't exist, terminate IHAVE */
	prot_printf(sout, ".\r\n");
	}
	else {
	/* store the article */
	while (prot_fgets(buf, sizeof(buf), pin)) {
	if (buf[0] == '.') {
	if (buf[1] == '\r' && buf[2] == '\n') {
	/* End of message */
	prot_printf(sout, ".\r\n");
	break;
	}
	else if (buf[1] != '.') {
	/* Add missing dot-stuffing */
	prot_putc('.', sout);
	}
	}

	do {
	/* look for malformed lines with NUL CR LF */
	if (buf[strlen(buf)-1] != '\n' &&
	strlen(buf)+2 < sizeof(buf)-1 &&
	buf[strlen(buf)+2] == '\n') {
	strlcat(buf, "\r\n", sizeof(buf));
	}
	prot_printf(sout, "%s", buf);
	} while (buf[strlen(buf)-1] != '\n' &&
	prot_fgets(buf, sizeof(buf), pin));
	}

	if (buf[0] != '.') {
	syslog(LOG_ERR, "ARTICLE terminated abnormally");
	return -1;
	}
	}

	/* see how we did */
	if (!prot_fgets(buf, sizeof(buf), sin)) {
	syslog(LOG_ERR, "IHAVE terminated abnormally");
	return -1;
	}
	else if (!strncmp("235", buf, 3))
	(*accepted)++;
	else
	(*failed)++;

	return 0;
	}

	#define RESP_GROW 100
	#define BUFFERSIZE 4096

	int main(int argc, char *argv[])
	{
	extern char *optarg;
	int opt;
	char alt_config = NULL, port = "119";
	const char peer = NULL, server = "localhost", wildmat = "";
	char authname = NULL, password = NULL;
	int psock = -1, ssock = -1;
	struct protstream pin, pout, sin, sout;
	char buf[BUFFERSIZE];
	char sfile[1024] = "";
	int fd = -1, i, n, offered, rejected, accepted, failed;
	time_t stamp;
	char **resp = NULL;
	int newnews = 1;
	char *datefmt = "%y%m%d %H%M%S";

	if ((geteuid()) == 0 && (become_cyrus() != 0)) {
	fatal("must run as the Cyrus user", EC_USAGE);
	}

	while ((opt = getopt(argc, argv, "C:s:w:f:a:p:ny")) != EOF) {
	switch (opt) {
	case 'C': /* alt config file */
	alt_config = optarg;
	break;

	case 's': /* server */
	server = xstrdup(optarg);
	if ((port = strchr(server, ':')))
	*port++ = '\0';
	else
	port = "119";
	break;

	case 'w': /* wildmat */
	wildmat = optarg;
	break;

	case 'f': /* timestamp file */
	snprintf(sfile, sizeof(sfile), "%s", optarg);
	break;

	case 'a': /* authname */
	authname = optarg;
	break;

	case 'p': /* password */
	password = optarg;
	break;

	case 'n': /* no newnews */
	newnews = 0;
	break;

	case 'y': /* newsserver is y2k compliant */
	datefmt = "%Y%m%d %H%M%S";
	break;

	default:
	usage();
	/* NOTREACHED */
	}
	}
	if (argc - optind < 1) {
	usage();
	/* NOTREACHED */
	}

	peer = argv[optind++];

	cyrus_init(alt_config, "fetchnews", 0);

	/* connect to the peer */
	/* xxx configurable port number? */
	if ((psock = init_net(peer, "119", &pin, &pout)) < 0) {
	fprintf(stderr, "connection to %s failed\n", peer);
	cyrus_done();
	exit(-1);
	}

	/* read the initial greeting */
	if (!prot_fgets(buf, sizeof(buf), pin) \|\| strncmp("20", buf, 2)) {
	syslog(LOG_ERR, "peer not available");
	goto quit;
	}

	if (authname) {
	/* authenticate to peer */
	/* XXX this should be modified to support SASL and STARTTLS */

	prot_printf(pout, "AUTHINFO USER %s\r\n", authname);
	if (!prot_fgets(buf, sizeof(buf), pin)) {
	syslog(LOG_ERR, "AUTHINFO USER terminated abnormally");
	goto quit;
	}
	else if (!strncmp("381", buf, 3)) {
	/* password required */
	if (!password)
	password = getpass("Please enter the password: ");

	if (!password) {
	fprintf(stderr, "failed to get password\n");
	goto quit;
	}

	prot_printf(pout, "AUTHINFO PASS %s\r\n", password);
	if (!prot_fgets(buf, sizeof(buf), pin)) {
	syslog(LOG_ERR, "AUTHINFO PASS terminated abnormally");
	goto quit;
	}
	}

	if (strncmp("281", buf, 3)) {
	/* auth failed */
	goto quit;
	}
	}

	/* change to reader mode - not always necessary, so ignore result */
	prot_printf(pout, "MODE READER\r\n");
	prot_fgets(buf, sizeof(buf), pin);

	if (newnews) {
	struct tm ctime, *ptime;

	/* fetch the server's current time */
	prot_printf(pout, "DATE\r\n");

	if (!prot_fgets(buf, sizeof(buf), pin) \|\| strncmp("111 ", buf, 4)) {
	syslog(LOG_ERR, "error fetching DATE");
	goto quit;
	}

	/* parse and normalize the server time */
	memset(&ctime, 0, sizeof(struct tm));
	sscanf(buf+4, "%4d%02d%02d%02d%02d%02d",
	&ctime.tm_year, &ctime.tm_mon, &ctime.tm_mday,
	&ctime.tm_hour, &ctime.tm_min, &ctime.tm_sec);
	ctime.tm_year -= 1900;
	ctime.tm_mon--;
	ctime.tm_isdst = -1;

	/* read the previous timestamp */
	if (!sfile[0]) {
	char oldfile[1024];

	snprintf(sfile, sizeof(sfile), "%s/fetchnews.stamp", config_dir);

	/* upgrade from the old stamp filename to the new */
	snprintf(oldfile, sizeof(oldfile), "%s/newsstamp", config_dir);
	rename(oldfile, sfile);
	}

	if ((fd = open(sfile, O_RDWR \| O_CREAT, 0644)) == -1) {
	syslog(LOG_ERR, "cannot open %s", sfile);
	goto quit;
	}
	if (lock_nonblocking(fd) == -1) {
	syslog(LOG_ERR, "cannot lock %s: %m", sfile);
	goto quit;
	}

	if (read(fd, &stamp, sizeof(stamp)) < (int) sizeof(stamp)) {
	/* XXX do something better here */
	stamp = 0;
	}

	/* ask for new articles */
	if (stamp) stamp -= 180; /* adjust back 3 minutes */
	ptime = gmtime(&stamp);
	ptime->tm_isdst = -1;
	strftime(buf, sizeof(buf), datefmt, ptime);
	prot_printf(pout, "NEWNEWS %s %s GMT\r\n", wildmat, buf);

	if (!prot_fgets(buf, sizeof(buf), pin) \|\| strncmp("230", buf, 3)) {
	syslog(LOG_ERR, "peer doesn't support NEWNEWS");
	newnews = 0;
	}

	/* prepare server's current time as new timestamp */
	stamp = mktime(&ctime);
	/* adjust for local timezone

	XXX We need to do this because we use gmtime() above.
	We can't change this, otherwise we'd be incompatible
	with an old localtime timestamp.
	*/
	stamp += gmtoff_of(&ctime, stamp);
	}

	if (!newnews) {
	prot_printf(pout, "LIST ACTIVE %s\r\n", wildmat);

	if (!prot_fgets(buf, sizeof(buf), pin) \|\| strncmp("215", buf, 3)) {
	syslog(LOG_ERR, "peer doesn't support LIST ACTIVE");
	goto quit;
	}
	}

	/* process the NEWNEWS/LIST ACTIVE list */
	n = 0;
	while (prot_fgets(buf, sizeof(buf), pin)) {
	if (buf[0] == '.') break;

	if (!(n % RESP_GROW)) { /* time to alloc more */
	resp = (char **)
	xrealloc(resp, (n + RESP_GROW) * sizeof(char *));
	}
	resp[n++] = xstrdup(buf);
	}
	if (buf[0] != '.') {
	syslog(LOG_ERR, "%s terminated abnormally",
	newnews ? "NEWNEWS" : "LIST ACTIVE");
	goto quit;
	}

	if (!n) {
	/* nothing matches our wildmat */
	goto quit;
	}

	/* connect to the server */
	if ((ssock = init_net(server, port, &sin, &sout)) < 0) {
	fprintf(stderr, "connection to %s failed\n", server);
	goto quit;
	}

	/* read the initial greeting */
	if (!prot_fgets(buf, sizeof(buf), sin) \|\| strncmp("20", buf, 2)) {
	syslog(LOG_ERR, "server not available");
	goto quit;
	}

	/* fetch and store articles */
	offered = rejected = accepted = failed = 0;
	if (newnews) {
	/* response is a list of msgids */
	for (i = 0; i < n; i++) {
	/* find the end of the msgid */
	*(strrchr(resp[i], '>') + 1) = '\0';

	offered++;
	if (fetch(resp[i], 1, pin, pout, sin, sout,
	&rejected, &accepted, &failed)) {
	goto quit;
	}
	}

	/* write the current timestamp */
	lseek(fd, 0, SEEK_SET);
	if (write(fd, &stamp, sizeof(stamp)) < (int) sizeof(stamp))
	syslog(LOG_ERR, "error writing %s", sfile);
	lock_unlock(fd);
	close(fd);
	}
	else {
	char group[BUFFERSIZE], msgid[BUFFERSIZE], lastbuf[50];
	const char *data;
	unsigned long low, high, last, cur;
	int start;
	int datalen;
	struct txn *tid = NULL;

	newsrc_init(NULL, 0);

	/*
	* response is a list of groups.
	* select each group, and STAT each article we haven't seen yet.
	*/
	for (i = 0; i < n; i++) {
	/* parse the LIST ACTIVE response */
	sscanf(resp[i], "%s %lu %lu", group, &high, &low);

	last = 0;
	if (!DB->fetchlock(newsrc_db, group, strlen(group),
	&data, &datalen, &tid)) {
	last = strtoul(data, NULL, 10);
	}
	if (high <= last) continue;

	/* select the group */
	prot_printf(pout, "GROUP %s\r\n", group);
	if (!prot_fgets(buf, sizeof(buf), pin)) {
	syslog(LOG_ERR, "GROUP terminated abnormally");
	continue;
	}
	else if (strncmp("211", buf, 3)) break;

	for (start = 1, cur = low > last ? low : ++last;; cur++) {
	if (start) {
	/* STAT the first article we haven't seen */
	prot_printf(pout, "STAT %lu\r\n", cur);
	} else {
	/* continue with the NEXT article */
	prot_printf(pout, "NEXT\r\n");
	}

	if (!prot_fgets(buf, sizeof(buf), pin)) {
	syslog(LOG_ERR, "STAT/NEXT terminated abnormally");
	cur--;
	break;
	}
	if (!strncmp("223", buf, 3)) {
	/* parse the STAT/NEXT response */
	sscanf(buf, "223 %lu %s", &cur, msgid);

	/* find the end of the msgid */
	*(strrchr(msgid, '>') + 1) = '\0';

	if (fetch(msgid, 0, pin, pout, sin, sout,
	&rejected, &accepted, &failed)) {
	cur--;
	break;
	}
	offered++;
	start = 0;
	}

	/* have we reached the highwater mark? */
	if (cur >= high) break;
	}

	snprintf(lastbuf, sizeof(lastbuf), "%lu", cur);
	DB->store(newsrc_db, group, strlen(group),
	lastbuf, strlen(lastbuf)+1, &tid);
	}

	if (tid) DB->commit(newsrc_db, tid);
	newsrc_done();
	}

	syslog(LOG_NOTICE,
	"fetchnews: %s offered %d; %s rejected %d, accepted %d, failed %d",
	peer, offered, server, rejected, accepted, failed);

	quit:
	if (psock >= 0) {
	prot_printf(pout, "QUIT\r\n");
	prot_flush(pout);

	/* Flush the incoming buffer */
	prot_NONBLOCK(pin);
	prot_fill(pin);

	/* close/free socket & prot layer */
	close(psock);

	prot_free(pin);
	prot_free(pout);
	}

	if (ssock >= 0) {
	prot_printf(sout, "QUIT\r\n");
	prot_flush(sout);

	/* Flush the incoming buffer */
	prot_NONBLOCK(sin);
	prot_fill(sin);

	/* close/free socket & prot layer */
	close(psock);

	prot_free(sin);
	prot_free(sout);
	}

	cyrus_done();

	return 0;
	}
	diff --git a/imap/index.c b/imap/index.c
	index 21df13f11..1040ba3c4 100644
	--- a/imap/index.c
	+++ b/imap/index.c
	@@ -1,5131 +1,5131 @@
	/* index.c -- Routines for dealing with the index file in the imapd
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: index.c,v 1.259 2010/06/28 12:04:53 brong Exp $
	*/

	#include <config.h>

	#include <stdio.h>
	#include <string.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <fcntl.h>
	#include <netinet/in.h>
	#include <syslog.h>
	#include <errno.h>
	#include <ctype.h>
	#include <stdlib.h>

	#include "acl.h"
	#include "annotate.h"
	#include "append.h"
	#include "assert.h"
	#include "charset.h"
	#include "exitcodes.h"
	#include "hash.h"
	#include "imap_err.h"
	#include "global.h"
	#include "imapd.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "lsort.h"
	#include "mailbox.h"
	#include "map.h"
	#include "message.h"
	#include "parseaddr.h"
	#include "search_engines.h"
	#include "seen.h"
	#include "statuscache.h"
	#include "strhash.h"
	#include "stristr.h"
	#include "util.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"

	#include "index.h"
	#include "sync_log.h"

	/* Forward declarations */
	static void index_refresh(struct index_state *state);
	static void index_tellexists(struct index_state *state);
	int index_lock(struct index_state *state);
	void index_unlock(struct index_state *state);

	int index_writeseen(struct index_state *state);
	void index_fetchmsg(struct index_state *state,
	const char *msg_base, unsigned long msg_size,
	unsigned offset, unsigned size,
	unsigned start_octet, unsigned octet_count);
	static int index_fetchsection(struct index_state state, const char resp,
	const char *msg_base, unsigned long msg_size,
	char *section,
	const char *cachestr, unsigned size,
	unsigned start_octet, unsigned octet_count);
	static void index_fetchfsection(struct index_state *state,
	const char *msg_base, unsigned long msg_size,
	struct fieldlist *fsection,
	const char *cachestr,
	unsigned start_octet, unsigned octet_count);
	static char index_readheader(const char msg_base, unsigned long msg_size,
	unsigned offset, unsigned size);
	static void index_pruneheader(char buf, struct strlist headers,
	struct strlist *headers_not);
	static void index_fetchheader(struct index_state *state,
	const char *msg_base, unsigned long msg_size,
	unsigned size,
	struct strlist *headers,
	struct strlist *headers_not);
	static void index_fetchcacheheader(struct index_state state, struct index_record record,
	struct strlist *headers, unsigned start_octet,
	unsigned octet_count);
	static void index_listflags(struct index_state *state);
	static void index_fetchflags(struct index_state *state, uint32_t msgno);
	static int index_search_evaluate(struct index_state *state,
	struct searchargs *searchargs,
	uint32_t msgno, struct mapfile *msgfile);
	static int index_searchmsg(char substr, comp_pat pat,
	struct mapfile *msgfile,
	int skipheader, const char *cachestr);
	static int index_searchheader(char name, char substr, comp_pat *pat,
	struct mapfile *msgfile,
	int size);
	static int index_searchcacheheader(struct index_state state, uint32_t msgno, char name, char *substr,
	comp_pat *pat);
	static int _index_search(unsigned *msgno_list, struct index_state state,
	struct searchargs *searchargs,
	modseq_t *highestmodseq);

	static int index_copysetup(struct index_state state, uint32_t msgno, struct copyargs copyargs);
	static int index_storeflag(struct index_state *state, uint32_t msgno,
	struct storeargs *storeargs);
	static int index_fetchreply(struct index_state *state, uint32_t msgno,
	struct fetchargs *fetchargs);
	static void index_printflags(struct index_state *state, uint32_t msgno, int usinguid);
	static void index_checkflags(struct index_state *state, int dirty);
	static char find_msgid(char str, char **rem);
	static char get_localpart_addr(const char header);
	static char index_extract_subject(const char subj, size_t len, int *is_refwd);
	static char _index_extract_subject(char s, int *is_refwd);
	static void index_get_ids(MsgData *msgdata,
	char envtokens[], const char headers, unsigned size);
	static MsgData index_msgdata_load(struct index_state state, unsigned *msgno_list, int n,
	struct sortcrit *sortcrit);

	static void index_sort_getnext(MsgData node);
	static void index_sort_setnext(MsgData node, MsgData next);
	static int index_sort_compare(MsgData md1, MsgData md2,
	struct sortcrit *call_data);
	static void index_msgdata_free(MsgData *md);

	static void index_thread_getnext(Thread thread);
	static void index_thread_setnext(Thread thread, Thread next);
	static int index_thread_compare(Thread t1, Thread t2,
	struct sortcrit *call_data);
	static void index_thread_orderedsubj(struct index_state *state,
	unsigned *msgno_list, int nmsg,
	int usinguid);
	static void index_thread_sort(Thread root, struct sortcrit sortcrit);
	static void index_thread_print(struct index_state *state,
	Thread *threads, int usinguid);
	static void index_thread_ref(struct index_state *state,
	unsigned *msgno_list, int nmsg, int usinguid);

	static void index_select(struct index_state *state);
	static struct seqset _index_vanished(struct index_state state,
	struct vanished_params *params);
	static struct seqset _parse_sequence(struct index_state state,
	const char *sequence, int usinguid);

	/* NOTE: Make sure these are listed in CAPABILITY_STRING */
	static const struct thread_algorithm thread_algs[] = {
	{ "ORDEREDSUBJECT", index_thread_orderedsubj },
	{ "REFERENCES", index_thread_ref },
	{ NULL, NULL }
	};


	/*
	* A mailbox is about to be closed.
	*/
	int index_close(struct index_state **stateptr)
	{
	unsigned i;
	struct index_state state = stateptr;

	free(state->userid);
	free(state->map);
	for (i = 0; i < MAX_USER_FLAGS; i++)
	free(state->flagname[i]);
	mailbox_close(&state->mailbox);
	free(state);

	*stateptr = NULL;

	return 0;
	}

	/*
	* A new mailbox has been selected, map it into memory and do the
	* initial CHECK.
	*/
	int index_open(const char name, struct index_init init,
	struct index_state **stateptr)
	{
	int r;
	struct index_state *state = xzmalloc(sizeof(struct index_state));
	struct seqset *vanishedlist = NULL;

	r = mailbox_open_iwl(name, &state->mailbox);
	if (r) goto fail;

	if (init) {
	state->myrights = cyrus_acl_myrights(init->authstate, state->mailbox->acl);
	if (init->examine_mode)
	state->myrights &= ~ACL_READ_WRITE;

	state->authstate = init->authstate;
	state->userid = init->userid ? xstrdup(init->userid) : NULL;

	state->internalseen = mailbox_internal_seen(state->mailbox, state->userid);
	state->keepingseen = (state->myrights & ACL_SEEN);
	state->examining = init->examine_mode;

	state->out = init->out;
	state->qresync = init->qresync;
	}

	/* initialise the index_state */
	index_refresh(state);

	/* have to get the vanished list while we're still locked */
	if (init && init->vanished.uidvalidity == state->mailbox->i.uidvalidity) {
	vanishedlist = _index_vanished(state, &init->vanished);
	}

	index_unlock(state);

	if (init && init->select && state->myrights & ACL_READ) {
	index_select(state);

	if (init->vanished.uidvalidity == state->mailbox->i.uidvalidity) {
	const char *sequence = init->vanished.sequence;
	struct index_map *im;
	uint32_t msgno;
	struct seqset *seq = _parse_sequence(state, sequence, 1);

	/* QRESYNC response:
	* UID FETCH seq FLAGS (CHANGEDSINCE modseq VANISHED)
	*/

	if (vanishedlist && vanishedlist->len) {
	char *vanished = seqset_cstring(vanishedlist);
	prot_printf(state->out, "* VANISHED (EARLIER) %s\r\n", vanished);
	free(vanished);
	}

	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];
	if (sequence && !seqset_ismember(seq, im->record.uid))
	continue;
	if (im->record.modseq <= init->vanished.modseq)
	continue;
	index_printflags(state, msgno, 1);
	}

	seqset_free(seq);
	}
	}

	seqset_free(vanishedlist);

	*stateptr = state;

	return 0;

	fail:
	free(state->mailbox);
	free(state);
	return r;
	}

	int index_expunge(struct index_state state, char sequence)
	{
	int r;
	uint32_t msgno;
	struct index_map *im;
	struct seqset *seq = NULL;

	r = index_lock(state);
	if (r) return r;

	/* XXX - earlier list if the sequence names UIDs that don't exist? */
	seq = _parse_sequence(state, sequence, 1);

	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];

	if (im->record.system_flags & FLAG_EXPUNGED)
	continue; /* already expunged */

	if (!(im->record.system_flags & FLAG_DELETED))
	continue; /* no \Deleted flag */

	/* if there is a sequence list, check it */
	if (sequence && !seqset_ismember(seq, im->record.uid))
	continue; /* not in the list */

	if (!im->isseen)
	state->numunseen--;

	if (im->isrecent)
	state->numrecent--;

	im->record.system_flags \|= FLAG_EXPUNGED;

	r = mailbox_rewrite_index_record(state->mailbox, &im->record);
	if (r) break;
	}

	seqset_free(seq);

	/* unlock before responding */
	index_unlock(state);

	return r;
	}

	char index_buildseen(struct index_state state, const char *oldseenuids)
	{
	struct seqset *outlist;
	uint32_t msgno;
	unsigned oldmax;
	struct index_map *im;
	char *out;

	outlist = seqset_init(0, SEQ_MERGE);
	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];
	seqset_add(outlist, im->record.uid, im->isseen);
	}

	/* there may be future already seen UIDs that this process isn't
	* allowed to know about, but we can't blat them either! This is
	* a massive pain... */
	oldmax = seq_lastnum(oldseenuids, NULL);
	if (oldmax > state->last_uid) {
	struct seqset *seq = seqset_parse(oldseenuids, NULL, oldmax);
	uint32_t uid;

	/* for each future UID, copy the state in the old seenuids */
	for (uid = state->last_uid + 1; uid <= oldmax; uid++)
	seqset_add(outlist, uid, seqset_ismember(seq, uid));

	seqset_free(seq);
	}

	out = seqset_cstring(outlist);
	seqset_free(outlist);

	return out;
	}

	int index_writeseen(struct index_state *state)
	{
	int r;
	struct seen *seendb = NULL;
	struct seendata oldsd;
	struct seendata sd;
	struct mailbox *mailbox = state->mailbox;

	assert(mailbox->index_locktype == LOCK_EXCLUSIVE);

	if (!state->seen_dirty)
	return 0;

	state->seen_dirty = 0;

	/* only examining, can't write any changes */
	if (state->examining)
	return 0;

	/* already handled! Just update the header fields */
	if (state->internalseen) {
	mailbox_index_dirty(mailbox);
	mailbox->i.recenttime = time(0);
	if (mailbox->i.recentuid < state->last_uid)
	mailbox->i.recentuid = state->last_uid;
	return 0;
	}

	r = seen_open(state->userid, SEEN_CREATE, &seendb);
	if (r) return r;

	r = seen_lockread(seendb, mailbox->uniqueid, &oldsd);
	if (r) {
	oldsd.lastread = 0;
	oldsd.lastuid = 0;
	oldsd.lastchange = 0;
	oldsd.seenuids = xstrdup("");
	}

	/* fields of interest... */
	sd.lastuid = oldsd.lastuid;
	sd.seenuids = index_buildseen(state, oldsd.seenuids);
	if (!sd.seenuids) sd.seenuids = xstrdup("");

	/* make comparison only catch some changes */
	sd.lastread = oldsd.lastread;
	sd.lastchange = oldsd.lastchange;

	/* update \Recent lowmark */
	if (sd.lastuid < state->last_uid)
	sd.lastuid = state->last_uid;

	/* only commit if interesting fields have changed */
	if (!seen_compare(&sd, &oldsd)) {
	sd.lastread = time(NULL);
	sd.lastchange = mailbox->i.last_appenddate;
	r = seen_write(seendb, mailbox->uniqueid, &sd);
	}

	seen_close(seendb);

	seen_freedata(&oldsd);
	seen_freedata(&sd);

	return r;
	}

	/* caller must free the list with seqset_free() when done */
	static struct seqset _readseen(struct index_state state, unsigned *recentuid)
	{
	struct mailbox *mailbox = state->mailbox;
	struct seqset *seenlist = NULL;

	/* Obtain seen information */
	if (state->internalseen) {
	*recentuid = mailbox->i.recentuid;
	}
	else if (state->userid) {
	struct seen *seendb;
	struct seendata sd;
	int r;

	r = seen_open(state->userid, SEEN_CREATE, &seendb);
	if (!r) {
	r = seen_read(seendb, mailbox->uniqueid, &sd);
	seen_close(seendb);
	}

	/* handle no seen DB gracefully */
	if (r) {
	*recentuid = mailbox->i.last_uid;
	prot_printf(state->out, "* OK (seen state failure) %s: %s\r\n",
	error_message(IMAP_NO_CHECKPRESERVE), error_message(r));
	syslog(LOG_ERR, "Could not open seen state for %s (%s)",
	state->userid, error_message(r));
	}
	else {
	*recentuid = sd.lastuid;
	seenlist = seqset_parse(sd.seenuids, NULL, *recentuid);
	free(sd.seenuids);
	}
	}
	else {
	recentuid = mailbox->i.last_uid; / nothing is recent! */
	}

	return seenlist;
	}

	void index_refresh(struct index_state *state)
	{
	struct mailbox *mailbox = state->mailbox;
	uint32_t recno;
	uint32_t msgno = 1;
	uint32_t firstnotseen = 0;
	uint32_t numrecent = 0;
	uint32_t numunseen = 0;
	uint32_t recentuid;
	int havenewrecords = 0;
	struct index_map *im;
	modseq_t delayed_modseq = 0;
	uint32_t need_records;
	struct seqset *seenlist;

	if (state->num_records) {
	need_records = mailbox->i.num_records -
	state->num_records + state->exists;
	}
	else {
	/* init case */
	need_records = mailbox->i.exists;
	}

	/* make sure we have space */
	if (need_records >= state->mapsize) {
	state->mapsize = (need_records \| 0xff) + 1; /* round up 1-256 */
	state->map = xrealloc(state->map,
	state->mapsize * sizeof(struct index_map));
	}

	seenlist = _readseen(state, &recentuid);

	/* already known records - flag updates */
	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];
	if (mailbox_read_index_record(mailbox, im->record.recno, &im->record))
	continue; /* bogus read... should probably be fatal */

	/* ignore expunged messages */
	if (im->record.system_flags & FLAG_EXPUNGED) {
	/* http://www.rfc-editor.org/errata_search.php?rfc=5162
	* Errata ID: 1809 - if there are expunged records we
	* aren't telling about, need to make the highestmodseq
	* be one lower so the client can safely resync */
	if (!delayed_modseq \|\| im->record.modseq < delayed_modseq)
	delayed_modseq = im->record.modseq - 1;
	continue;
	}

	/* re-calculate seen flags */
	if (state->internalseen)
	im->isseen = (im->record.system_flags & FLAG_SEEN) ? 1 : 0;
	else
	im->isseen = seqset_ismember(seenlist, im->record.uid);

	/* track select values */
	if (!im->isseen) {
	numunseen++;
	if (!firstnotseen)
	firstnotseen = msgno;
	}
	if (im->isrecent) {
	/* we don't need to dirty seen here, it's a refresh */
	numrecent++;
	}
	}

	/* new records? */
	for (recno = state->num_records + 1; recno <= mailbox->i.num_records; recno++) {
	im = &state->map[msgno-1];
	if (mailbox_read_index_record(mailbox, recno, &im->record))
	continue; /* bogus read... should probably be fatal */
	if (im->record.system_flags & FLAG_EXPUNGED)
	continue;

	/* make sure we don't overflow the memory we mapped */
	if (msgno >= state->mapsize) {
	char buf[2048];
	sprintf(buf, "Exists wrong %u %u %u %u", msgno,
	state->mapsize, mailbox->i.exists, mailbox->i.num_records);
	fatal(buf, EC_IOERR);
	}

	/* calculate flags */
	if (state->internalseen)
	im->isseen = (im->record.system_flags & FLAG_SEEN) ? 1 : 0;
	else
	im->isseen = seqset_ismember(seenlist, im->record.uid);
	im->isrecent = (im->record.uid > recentuid) ? 1 : 0;

	/* track select values */
	if (!im->isseen) {
	numunseen++;
	if (!firstnotseen)
	firstnotseen = msgno;
	}
	if (im->isrecent) {
	numrecent++;
	state->seen_dirty = 1;
	}

	/* don't auto-tell */
	im->told_modseq = im->record.modseq;

	havenewrecords = 1;

	msgno++;
	}

	seqset_free(seenlist);

	/* update the header tracking data */
	state->exists = msgno - 1; /* we actually got this many */
	state->delayed_modseq = delayed_modseq;
	state->highestmodseq = mailbox->i.highestmodseq;
	state->last_uid = mailbox->i.last_uid;
	state->num_records = mailbox->i.num_records;
	state->firstnotseen = firstnotseen;
	state->numunseen = numunseen;
	state->numrecent = numrecent;
	state->havenewrecords = havenewrecords;
	}

	modseq_t index_highestmodseq(struct index_state *state)
	{
	if (state->delayed_modseq)
	return state->delayed_modseq;
	return state->highestmodseq;
	}

	static void index_select(struct index_state *state)
	{
	index_tellexists(state);

	/* always print flags */
	index_checkflags(state, 1);

	if (state->firstnotseen)
	prot_printf(state->out, "* OK [UNSEEN %u] Ok\r\n",
	state->firstnotseen);
	prot_printf(state->out, "* OK [UIDVALIDITY %u] Ok\r\n",
	state->mailbox->i.uidvalidity);
	prot_printf(state->out, "* OK [UIDNEXT %lu] Ok\r\n",
	state->last_uid + 1);
	prot_printf(state->out, "* OK [HIGHESTMODSEQ " MODSEQ_FMT "] Ok\r\n",
	state->highestmodseq);
	prot_printf(state->out, "* OK [URLMECH INTERNAL] Ok\r\n");
	}

	/*
	* Check for and report updates
	*/
	int index_check(struct index_state *state, int usinguid, int printuid)
	{
	struct mailbox *mailbox = state->mailbox;
	int r;

	r = mailbox_lock_index(mailbox, LOCK_EXCLUSIVE);
	if (r) return r;

	/* Check for deleted mailbox */
	if (mailbox->i.options & OPT_MAILBOX_DELETED) {
	/* Mailbox has been (re)moved */
	if (config_getswitch(IMAPOPT_DISCONNECT_ON_VANISHED_MAILBOX)) {
	syslog(LOG_WARNING,
	"Mailbox %s has been (re)moved out from under client",
	mailbox->name);
	fatal("Mailbox has been (re)moved", EC_IOERR);
	}

	if (state->exists && state->qresync) {
	/* XXX - is it OK to just expand to entire possible range? */
	prot_printf(state->out, "* VANISHED 1:%lu\r\n", state->last_uid);
	}
	else {
	int exists;
	for (exists = state->exists; exists > 0; exists--) {
	prot_printf(state->out, "* 1 EXPUNGE\r\n");
	}
	}
	mailbox_unlock_index(mailbox, NULL);
	state->exists = 0;
	return IMAP_MAILBOX_NONEXISTENT;
	}

	index_refresh(state);

	/* any updates? */
	index_tellchanges(state, usinguid, printuid);

	#if TOIMSP
	if (state->firstnotseen) {
	toimsp(mailbox->name, mailbox->i.uidvalidity, "SEENsnn", state->userid,
	0, mailbox->i.recenttime, 0);
	}
	else {
	toimsp(mailbox->name, mailbox->i.uidvalidity, "SEENsnn", state->userid,
	mailbox->last_uid, mailbox->i.recenttime, 0);
	}
	#endif

	index_unlock(state);

	return r;
	}

	/*
	* Perform UID FETCH (VANISHED) on a sequence.
	*/
	static struct seqset _index_vanished(struct index_state state,
	struct vanished_params *params)
	{
	struct mailbox *mailbox = state->mailbox;
	struct index_record record;
	struct seqset *outlist;
	struct seqset *seq;
	uint32_t recno;

	/* No recently expunged messages */
	if (params->modseq >= state->highestmodseq) return NULL;

	outlist = seqset_init(0, SEQ_SPARSE);
	seq = _parse_sequence(state, params->sequence, 1);

	/* XXX - use match_seq and match_uid */

	if (params->modseq >= mailbox->i.deletedmodseq) {
	/* all records are significant */
	/* List only expunged UIDs with MODSEQ > requested */
	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	if (mailbox_read_index_record(mailbox, recno, &record))
	continue;
	if (!(record.system_flags & FLAG_EXPUNGED))
	continue;
	if (record.modseq <= params->modseq)
	continue;
	if (!params->sequence \|\| seqset_ismember(seq, record.uid))
	seqset_add(outlist, record.uid, 1);
	}
	}
	else {
	unsigned prevuid = 0;
	struct seqset *msgnolist;
	struct seqset *uidlist;
	uint32_t msgno;
	unsigned uid;

	syslog(LOG_NOTICE, "inefficient qresync ("
	MODSEQ_FMT " > " MODSEQ_FMT ") %s",
	mailbox->i.deletedmodseq, params->modseq,
	mailbox->name);

	recno = 1;

	/* use the sequence to uid mapping provided by the client to
	* skip over any initial matches - see RFC 5162 section 3.1 */
	if (params->match_seq && params->match_uid) {
	msgnolist = _parse_sequence(state, params->match_seq, 0);
	uidlist = _parse_sequence(state, params->match_uid, 1);
	while ((msgno = seqset_getnext(msgnolist)) != 0) {
	uid = seqset_getnext(uidlist);
	/* first non-match, we'll start here */
	if (state->map[msgno-1].record.uid != uid)
	break;
	/* ok, they matched - so we can start at the recno and UID
	* first past the match */
	prevuid = uid;
	recno = state->map[msgno-1].record.recno + 1;
	}
	seqset_free(msgnolist);
	seqset_free(uidlist);
	}

	/* possible efficiency improvement - use "seq_getnext" on seq
	* to avoid incrementing through every single number for prevuid.
	* Only really an issue if there's a giant block of thousands of
	* expunged messages. Only likely to be seen in the wild if
	* last_uid winds up being bumped up a few million by a bug... */

	/* for the rest of the mailbox, we're just going to have to assume
	* every record in the requested range which DOESN'T exist has been
	* expunged, so build a complete sequence */
	for (; recno <= mailbox->i.num_records; recno++) {
	if (mailbox_read_index_record(mailbox, recno, &record))
	continue;
	if (record.system_flags & FLAG_EXPUNGED)
	continue;
	while (++prevuid < record.uid) {
	if (!params->sequence \|\| seqset_ismember(seq, prevuid))
	seqset_add(outlist, prevuid, 1);
	}
	prevuid = record.uid;
	}

	/* include the space past the final record up to last_uid as well */
	while (++prevuid <= mailbox->i.last_uid) {
	if (!params->sequence \|\| seqset_ismember(seq, prevuid))
	seqset_add(outlist, prevuid, 1);
	}
	}

	seqset_free(seq);

	return outlist;
	}

	static int _fetch_setseen(struct index_state *state, uint32_t msgno)
	{
	struct index_map *im = &state->map[msgno-1];
	int r;

	/* already seen */
	if (im->isseen)
	return 0;

	/* no rights to change it */
	if (!(state->myrights & ACL_SEEN))
	return 0;

	/* store in the record if it's internal seen */
	if (state->internalseen)
	im->record.system_flags \|= FLAG_SEEN;

	/* need to bump modseq anyway, so always rewrite it */
	r = mailbox_rewrite_index_record(state->mailbox, &im->record);
	if (r) return r;

	/* track changes internally */
	state->numunseen--;
	state->seen_dirty = 1;
	im->isseen = 1;

	/* RFC2060 says:
	* The \Seen flag is implicitly set; if this causes
	* the flags to change they SHOULD be included as part
	* of the FETCH responses. This is handled later by
	* always including flags if the modseq has changed.
	*/

	return 0;
	}

	/*
	* Perform a FETCH-related command on a sequence.
	* Fetchedsomething argument is 0 if nothing was fetched, 1 if something was
	* fetched. (A fetch command that fetches nothing is not a valid fetch
	* command.)
	*/
	int index_fetch(struct index_state *state,
	const char *sequence,
	int usinguid,
	struct fetchargs *fetchargs,
	int *fetchedsomething)
	{
	struct seqset *seq;
	struct seqset *vanishedlist = NULL;
	uint32_t msgno;
	unsigned checkval;
	int r;
	struct index_map *im;
	int fetched = 0;

	r = index_lock(state);
	if (r) return r;

	seq = _parse_sequence(state, sequence, usinguid);

	/* set the \Seen flag if necessary - while we still have the lock */
	if (fetchargs->fetchitems & FETCH_SETSEEN && !state->examining) {
	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];
	checkval = usinguid ? im->record.uid : msgno;
	if (!seqset_ismember(seq, checkval))
	continue;
	r = _fetch_setseen(state, msgno);
	if (r) break;
	}
	}

	if (fetchargs->vanished) {
	struct vanished_params v;
	v.sequence = sequence;;
	v.modseq = fetchargs->changedsince;
	v.match_seq = fetchargs->match_seq;
	v.match_uid = fetchargs->match_uid;
	/* XXX - return error unless usinguid? */
	vanishedlist = _index_vanished(state, &v);
	}

	index_unlock(state);

	index_checkflags(state, 0);

	if (vanishedlist && vanishedlist->len) {
	char *vanished = seqset_cstring(vanishedlist);
	prot_printf(state->out, "* VANISHED (EARLIER) %s\r\n", vanished);
	free(vanished);
	}

	seqset_free(vanishedlist);

	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];
	checkval = usinguid ? im->record.uid : msgno;
	if (!seqset_ismember(seq, checkval))
	continue;
	r = index_fetchreply(state, msgno, fetchargs);
	if (r) break;
	fetched = 1;
	}

	seqset_free(seq);

	index_tellchanges(state, usinguid, usinguid);

	if (fetchedsomething) *fetchedsomething = fetched;

	return r;
	}

	/*
	* Perform a STORE command on a sequence
	*/
	int index_store(struct index_state state, char sequence, int usinguid,
	struct storeargs storeargs, char *flag, int nflags)
	{
	struct mailbox *mailbox = state->mailbox;
	int i, r = 0;
	uint32_t msgno;
	unsigned checkval;
	int userflag;
	struct seqset *seq;
	struct index_map *im;

	/* First pass at checking permission */
	if ((storeargs->seen && !(state->myrights & ACL_SEEN)) \|\|
	((storeargs->system_flags & FLAG_DELETED) &&
	!(state->myrights & ACL_DELETEMSG)) \|\|
	(((storeargs->system_flags & ~FLAG_DELETED) \|\| nflags) &&
	!(state->myrights & ACL_WRITE))) {
	return IMAP_PERMISSION_DENIED;
	}

	r = index_lock(state);
	if (r) return r;

	seq = _parse_sequence(state, sequence, usinguid);

	for (i = 0; i < nflags; i++) {
	r = mailbox_user_flag(mailbox, flag[i], &userflag);
	if (r) goto fail;
	storeargs->user_flags[userflag/32] \|= 1<<(userflag&31);
	}

	storeargs->update_time = time((time_t *)0);
	storeargs->usinguid = usinguid;

	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];
	checkval = usinguid ? im->record.uid : msgno;
	if (!seqset_ismember(seq, checkval))
	continue;
	r = index_storeflag(state, msgno, storeargs);
	if (r) goto fail;
	}

	fail:
	seqset_free(seq);
	index_unlock(state);
	index_tellchanges(state, usinguid, usinguid);

	return r;
	}

	static int index_scan_work(const char *s, unsigned long len,
	const char *match, unsigned long min)
	{
	while (len > min) {
	if (!strncasecmp(s, match, min)) return(1);
	s++;
	len--;
	}
	return(0);
	}

	/*
	* Guts of the SCAN command, lifted from _index_search()
	*
	* Returns 1 if we get a hit, otherwise returns 0.
	*/
	int index_scan(struct index_state state, const char contents)
	{
	unsigned *msgno_list;
	uint32_t msgno;
	struct mapfile msgfile;
	int n = 0;
	int listindex;
	int listcount;
	struct searchargs searchargs;
	struct strlist strlist;
	unsigned long length;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im;

	if (!(contents && contents[0])) return(0);

	if (index_check(state, 0, 0))
	return 0;

	if (state->exists <= 0) return 0;

	length = strlen(contents);

	memset(&searchargs, 0, sizeof(struct searchargs));
	searchargs.text = &strlist;

	/* Use US-ASCII to emulate fgrep */
	strlist.s = charset_convert(contents, charset_lookupname("US-ASCII"),
	NULL, 0);
	strlist.p = charset_compilepat(strlist.s);
	strlist.next = NULL;

	msgno_list = (unsigned ) xmalloc(state->exists sizeof(unsigned));

	listcount = search_prefilter_messages(msgno_list, state, &searchargs);

	for (listindex = 0; !n && listindex < listcount; listindex++) {
	msgno = msgno_list[listindex];
	im = &state->map[msgno-1];

	msgfile.base = 0;
	msgfile.size = 0;

	if (mailbox_map_message(mailbox, im->record.uid,
	&msgfile.base, &msgfile.size))
	continue;

	n += index_scan_work(msgfile.base, msgfile.size, contents, length);

	mailbox_unmap_message(mailbox, im->record.uid,
	&msgfile.base, &msgfile.size);
	}

	free(strlist.s);
	free(strlist.p);
	free(msgno_list);

	return n;
	}

	/*
	* Guts of the SEARCH command.
	*
	* Returns message numbers in an array. This function is used by
	* SEARCH, SORT and THREAD.
	*/
	static int _index_search(unsigned *msgno_list, struct index_state state,
	struct searchargs *searchargs,
	modseq_t *highestmodseq)
	{
	uint32_t msgno;
	struct mapfile msgfile;
	int n = 0;
	int listindex, min;
	int listcount;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im;

	if (state->exists <= 0) return 0;

	msgno_list = (unsigned ) xmalloc(state->exists * sizeof(unsigned));

	/* OK, so I'm being a bit clever here. We fill the msgno list with
	a list of message IDs returned by the search engine. Then we
	scan through the list and store matching message IDs back into the
	list. This is OK because we only overwrite message IDs that we've
	already looked at. */
	listcount = search_prefilter_messages(*msgno_list, state, searchargs);

	if (searchargs->returnopts == SEARCH_RETURN_MAX) {
	/* If we only want MAX, then skip forward search,
	and do complete reverse search */
	listindex = listcount;
	min = 0;
	} else {
	/* Otherwise use forward search, potentially skipping reverse search */
	listindex = 0;
	min = listcount;
	}

	/* Forward search. Used for everything other than MAX-only */
	for (; listindex < listcount; listindex++) {
	msgno = (*msgno_list)[listindex];
	im = &state->map[msgno-1];
	msgfile.base = 0;
	msgfile.size = 0;

	/* expunged messages never match */
	if (im->record.system_flags & FLAG_EXPUNGED)
	continue;

	if (index_search_evaluate(state, searchargs, msgno, &msgfile)) {
	(*msgno_list)[n++] = msgno;
	if (highestmodseq && im->record.modseq > *highestmodseq) {
	*highestmodseq = im->record.modseq;
	}

	/* See if we should short-circuit
	(we want MIN, but NOT COUNT or ALL) */
	if ((searchargs->returnopts & SEARCH_RETURN_MIN) &&
	!(searchargs->returnopts & SEARCH_RETURN_COUNT) &&
	!(searchargs->returnopts & SEARCH_RETURN_ALL)) {

	if (searchargs->returnopts & SEARCH_RETURN_MAX) {
	/* If we want MAX, setup for reverse search */
	min = listindex;
	}
	/* We're done */
	listindex = listcount;
	*highestmodseq = im->record.modseq;
	}
	}
	if (msgfile.base) {
	mailbox_unmap_message(mailbox, im->record.uid,
	&msgfile.base, &msgfile.size);
	}
	}

	/* Reverse search. Stops at previously found MIN (if any) */
	for (listindex = listcount; listindex > min; listindex--) {
	msgno = (*msgno_list)[listindex-1];
	im = &state->map[msgno-1];
	msgfile.base = 0;
	msgfile.size = 0;

	/* expunged messages never match */
	if (im->record.system_flags & FLAG_EXPUNGED)
	continue;

	if (index_search_evaluate(state, searchargs, msgno, &msgfile)) {
	(*msgno_list)[n++] = msgno;
	if (highestmodseq && im->record.modseq > *highestmodseq) {
	*highestmodseq = im->record.modseq;
	}
	/* We only care about MAX, so we're done on first match */
	listindex = 0;
	}
	if (msgfile.base) {
	mailbox_unmap_message(mailbox, im->record.uid,
	&msgfile.base, &msgfile.size);
	}
	}

	/* if we didn't find any matches, free msgno_list */
	if (!n && *msgno_list) {
	free(*msgno_list);
	*msgno_list = NULL;
	}

	return n;
	}

	unsigned index_getuid(struct index_state *state, uint32_t msgno) {
	return state->map[msgno-1].record.uid;
	}

	/* 'uid_list' is malloc'd string representing the hits from searchargs;
	returns number of hits */
	int index_getuidsequence(struct index_state *state,
	struct searchargs *searchargs,
	unsigned **uid_list)
	{
	unsigned *msgno_list;
	int i, n;

	n = _index_search(&msgno_list, state, searchargs, NULL);
	if (n == 0) {
	*uid_list = NULL;
	return 0;
	}

	*uid_list = msgno_list;

	/* filthy in-place replacement */
	for (i = 0; i < n; i++)
	(*uid_list)[i] = index_getuid(state, msgno_list[i]);

	return n;
	}

	int index_lock(struct index_state *state)
	{
	int r = mailbox_lock_index(state->mailbox, LOCK_EXCLUSIVE);
	if (!r) index_refresh(state);
	return r;
	}

	int index_status(struct index_state state, struct statusdata sdata)
	{
	int items = STATUS_MESSAGES \| STATUS_UIDNEXT \| STATUS_UIDVALIDITY \|
	STATUS_HIGHESTMODSEQ \| STATUS_RECENT \| STATUS_UNSEEN;
	statuscache_fill(sdata, state->userid, state->mailbox, items,
	state->numrecent, state->numunseen);
	return 0;
	}

	void index_unlock(struct index_state *state)
	{
	/* XXX - errors */

	index_writeseen(state);
	mailbox_commit(state->mailbox);

	/* grab the latest modseq */
	state->highestmodseq = state->mailbox->i.highestmodseq;

	if (config_getswitch(IMAPOPT_STATUSCACHE)) {
	struct statusdata sdata;
	index_status(state, &sdata);
	/* RECENT is zero for everyone else because we wrote a new
	* recentuid! */
	sdata.recent = 0;
	mailbox_unlock_index(state->mailbox, &sdata);
	}
	else
	mailbox_unlock_index(state->mailbox, NULL);
	}

	/*
	* Performs a SEARCH command.
	* This is a wrapper around _index_search() which simply prints the results.
	*/
	int index_search(struct index_state state, struct searchargs searchargs,
	int usinguid)
	{
	unsigned *list = NULL;
	int i, n;
	modseq_t highestmodseq = 0;

	/* update the index */
	if (index_check(state, 0, 0))
	return 0;

	/* now do the search */
	n = _index_search(&list, state, searchargs,
	searchargs->modseq ? &highestmodseq : NULL);

	/* replace the values now */
	if (usinguid)
	for (i = 0; i < n; i++)
	list[i] = state->map[list[i]-1].record.uid;

	if (searchargs->returnopts) {
	prot_printf(state->out, "* ESEARCH");
	if (searchargs->tag) {
	prot_printf(state->out, " (TAG \"%s\")", searchargs->tag);
	}
	if (n) {
	if (usinguid) prot_printf(state->out, " UID");
	if (searchargs->returnopts & SEARCH_RETURN_MIN)
	prot_printf(state->out, " MIN %u", list[0]);
	if (searchargs->returnopts & SEARCH_RETURN_MAX)
	prot_printf(state->out, " MAX %u", list[n-1]);
	if (highestmodseq)
	prot_printf(state->out, " MODSEQ " MODSEQ_FMT, highestmodseq);
	if (searchargs->returnopts & SEARCH_RETURN_ALL) {
	struct seqset *seq;
	char *str;

	/* Create a sequence-set */
	seq = seqset_init(0, SEQ_SPARSE);
	for (i = 0; i < n; i++)
	seqset_add(seq, list[i], 1);

	if (seq->len) {
	str = seqset_cstring(seq);
	prot_printf(state->out, " ALL %s", str);
	free(str);
	}

	seqset_free(seq);
	}
	}
	if (searchargs->returnopts & SEARCH_RETURN_COUNT) {
	prot_printf(state->out, " COUNT %u", n);
	}
	}
	else {
	prot_printf(state->out, "* SEARCH");

	for (i = 0; i < n; i++)
	prot_printf(state->out, " %u", list[i]);

	if (highestmodseq)
	prot_printf(state->out, " (MODSEQ " MODSEQ_FMT ")", highestmodseq);
	}

	if (n) free(list);

	prot_printf(state->out, "\r\n");

	return n;
	}

	/*
	* Performs a SORT command
	*/
	int index_sort(struct index_state state, struct sortcrit sortcrit,
	struct searchargs *searchargs, int usinguid)
	{
	unsigned *msgno_list;
	MsgData msgdata = NULL, freeme = NULL;
	int nmsg;
	clock_t start;
	modseq_t highestmodseq = 0;
	int i, modseq = 0;

	/* update the index */
	if (index_check(state, 0, 0))
	return 0;

	if(CONFIG_TIMING_VERBOSE)
	start = clock();

	if (searchargs->modseq) modseq = 1;
	else {
	for (i = 0; sortcrit[i].key != SORT_SEQUENCE; i++) {
	if (sortcrit[i].key == SORT_MODSEQ) {
	modseq = 1;
	break;
	}
	}
	}

	/* Search for messages based on the given criteria */
	nmsg = _index_search(&msgno_list, state, searchargs,
	modseq ? &highestmodseq : NULL);

	prot_printf(state->out, "* SORT");

	if (nmsg) {
	/* Create/load the msgdata array */
	freeme = msgdata = index_msgdata_load(state, msgno_list, nmsg, sortcrit);
	free(msgno_list);

	/* Sort the messages based on the given criteria */
	msgdata = lsort(msgdata,
	(void * ()(void)) index_sort_getnext,
	(void ()(void,void*)) index_sort_setnext,
	(int ()(void,void,void)) index_sort_compare,
	sortcrit);

	/* Output the sorted messages */
	while (msgdata) {
	unsigned no = usinguid ? state->map[msgdata->msgno-1].record.uid
	: msgdata->msgno;
	prot_printf(state->out, " %u", no);

	/* free contents of the current node */
	index_msgdata_free(msgdata);

	msgdata = msgdata->next;
	}

	/* free the msgdata array */
	free(freeme);
	}

	if (highestmodseq)
	prot_printf(state->out, " (MODSEQ " MODSEQ_FMT ")", highestmodseq);

	prot_printf(state->out, "\r\n");

	/* debug */
	if (CONFIG_TIMING_VERBOSE) {
	int len;
	char *key_names[] = { "SEQUENCE", "ARRIVAL", "CC", "DATE", "FROM",
	"SIZE", "SUBJECT", "TO", "ANNOTATION", "MODSEQ" };
	char buf[1024] = "";

	while (sortcrit->key && sortcrit->key < VECTOR_SIZE(key_names)) {
	if (sortcrit->flags & SORT_REVERSE)
	strlcat(buf, "REVERSE ", sizeof(buf));

	strlcat(buf, key_names[sortcrit->key], sizeof(buf));

	switch (sortcrit->key) {
	case SORT_ANNOTATION:
	len = strlen(buf);
	snprintf(buf + len, sizeof(buf) - len,
	" \"%s\" \"%s\"",
	sortcrit->args.annot.entry, sortcrit->args.annot.attrib);
	break;
	}
	if ((++sortcrit)->key) strlcat(buf, " ", sizeof(buf));
	}

	syslog(LOG_DEBUG, "SORT (%s) processing time: %d msg in %f sec",
	buf, nmsg, (clock() - start) / (double) CLOCKS_PER_SEC);
	}

	return nmsg;
	}

	/*
	* Performs a THREAD command
	*/
	int index_thread(struct index_state *state, int algorithm,
	struct searchargs *searchargs, int usinguid)
	{
	unsigned *msgno_list;
	int nmsg;
	clock_t start;
	modseq_t highestmodseq = 0;

	/* update the index */
	if (index_check(state, 0, 0))
	return 0;

	if(CONFIG_TIMING_VERBOSE)
	start = clock();

	/* Search for messages based on the given criteria */
	nmsg = _index_search(&msgno_list, state, searchargs,
	searchargs->modseq ? &highestmodseq : NULL);

	if (nmsg) {
	/* Thread messages using given algorithm */
	(*thread_algs[algorithm].threader)(state, msgno_list, nmsg, usinguid);

	free(msgno_list);

	if (highestmodseq)
	prot_printf(state->out, " (MODSEQ " MODSEQ_FMT ")", highestmodseq);
	}

	/* print an empty untagged response */
	else
	index_thread_print(state, NULL, usinguid);

	prot_printf(state->out, "\r\n");

	if (CONFIG_TIMING_VERBOSE) {
	/* debug */
	syslog(LOG_DEBUG, "THREAD %s processing time: %d msg in %f sec",
	thread_algs[algorithm].alg_name, nmsg,
	(clock() - start) / (double) CLOCKS_PER_SEC);
	}

	return nmsg;
	}

	/*
	* Performs a COPY command
	*/
	int
	index_copy(struct index_state *state,
	char *sequence,
	int usinguid,
	char *name,
	char **copyuidp,
	int nolink)
	{
	static struct copyargs copyargs;
	int i;
	uquota_t totalsize = 0;
	int r;
	struct appendstate appendstate;
	uint32_t msgno, checkval;
	unsigned long uidvalidity;
	unsigned long startuid, num;
	unsigned baseuid;
	long docopyuid;
	struct seqset *seq;
	struct mailbox *mailbox = state->mailbox;
	struct mailbox *destmailbox = NULL;
	struct index_map *im;

	*copyuidp = NULL;

	copyargs.nummsg = 0;

	r = index_check(state, usinguid, usinguid);
	if (r) return r;

	seq = _parse_sequence(state, sequence, usinguid);

	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];
	checkval = usinguid ? im->record.uid : msgno;
	if (!seqset_ismember(seq, checkval))
	continue;
	index_copysetup(state, msgno, &copyargs);
	}

	seqset_free(seq);

	if (copyargs.nummsg == 0) return IMAP_NO_NOSUCHMSG;

	for (i = 0; i < copyargs.nummsg; i++)
	totalsize += copyargs.copymsg[i].size;

	r = append_setup(&appendstate, name, state->userid,
	state->authstate, ACL_INSERT, totalsize);
	if (r) return r;

	docopyuid = (appendstate.myrights & ACL_READ);
	baseuid = appendstate.mailbox->i.last_uid + 1;

	r = append_copy(mailbox, &appendstate, copyargs.nummsg,
	copyargs.copymsg, nolink);
	if (!r) {
	r = append_commit(&appendstate, totalsize,
	&uidvalidity, &startuid, &num, &destmailbox);
	}

	if (!r && docopyuid) {
	char *source;
	struct seqset *seq;
	unsigned uidvalidity = destmailbox->i.uidvalidity;

	seq = seqset_init(0, SEQ_SPARSE);

	for (i = 0; i < copyargs.nummsg; i++)
	seqset_add(seq, copyargs.copymsg[i].uid, 1);

	source = seqset_cstring(seq);
	*copyuidp = xmalloc(strlen(source) + 50);

	if (appendstate.nummsg == 1)
	sprintf(*copyuidp, "%u %s %u", uidvalidity, source, baseuid);
	else
	sprintf(*copyuidp, "%u %s %u:%u", uidvalidity, source,
	baseuid, baseuid + appendstate.nummsg - 1);

	free(source);
	seqset_free(seq);
	}

	if (!r) {
	/* we log the first name to get GUID-copy magic */
	mailbox_close(&destmailbox);
	sync_log_mailbox_double(mailbox->name, name);
	}

	return r;
	}

	/*
	* Helper function to multiappend a message to remote mailbox
	*/
	static int index_appendremote(struct index_state *state, uint32_t msgno,
	struct protstream *pout)
	{
	struct mailbox *mailbox = state->mailbox;
	const char *msg_base = 0;
	unsigned long msg_size = 0;
	unsigned flag, flagmask;
	char datebuf[30];
	char sepchar = '(';
	struct index_map *im = &state->map[msgno-1];

	/* Open the message file */
	if (mailbox_map_message(mailbox, im->record.uid, &msg_base, &msg_size))
	return IMAP_NO_MSGGONE;

	/* start the individual append */
	prot_printf(pout, " ");

	/* add system flags */
	if (im->record.system_flags & FLAG_ANSWERED) {
	prot_printf(pout, "%c\\Answered", sepchar);
	sepchar = ' ';
	}
	if (im->record.system_flags & FLAG_FLAGGED) {
	prot_printf(pout, "%c\\Flagged", sepchar);
	sepchar = ' ';
	}
	if (im->record.system_flags & FLAG_DRAFT) {
	prot_printf(pout, "%c\\Draft", sepchar);
	sepchar = ' ';
	}
	if (im->record.system_flags & FLAG_DELETED) {
	prot_printf(pout, "%c\\Deleted", sepchar);
	sepchar = ' ';
	}
	if (im->isseen) {
	prot_printf(pout, "%c\\Seen", sepchar);
	sepchar = ' ';
	}

	/* add user flags */
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if ((flag & 31) == 0) {
	flagmask = im->record.user_flags[flag/32];
	}
	if (state->flagname[flag] && (flagmask & (1<<(flag & 31)))) {
	prot_printf(pout, "%c%s", sepchar, state->flagname[flag]);
	sepchar = ' ';
	}
	}

	/* add internal date */
	cyrus_ctime(im->record.internaldate, datebuf);
	prot_printf(pout, ") \"%s\" ", datebuf);

	/* message literal */
	index_fetchmsg(state, msg_base, msg_size, 0, im->record.size, 0, 0);

	/* close the message file */
	if (msg_base)
	mailbox_unmap_message(mailbox, im->record.uid, &msg_base, &msg_size);

	return 0;
	}

	/*
	* Performs a COPY command from a local mailbox to a remote mailbox
	*/
	int index_copy_remote(struct index_state state, char sequence,
	int usinguid, struct protstream *pout)
	{
	uint32_t msgno, checkval;
	struct seqset *seq;
	struct index_map *im;
	int r;

	r = index_check(state, usinguid, usinguid);
	if (r) return r;

	seq = _parse_sequence(state, sequence, usinguid);

	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];
	checkval = usinguid ? im->record.uid : msgno;
	if (!seqset_ismember(seq, checkval))
	continue;
	index_appendremote(state, msgno, pout);
	}

	seqset_free(seq);

	return 0;
	}

	/*
	* Returns the msgno of the message with UID 'uid'.
	* If no message with UID 'uid', returns the message with
	* the higest UID not greater than 'uid'.
	*/
	unsigned index_finduid(struct index_state *state, unsigned uid)
	{
	unsigned low = 1;
	unsigned high = state->exists;
	unsigned mid;
	unsigned miduid;

	while (low <= high) {
	mid = (high - low)/2 + low;
	miduid = index_getuid(state, mid);
	if (miduid == uid)
	return mid;
	else if (miduid > uid)
	high = mid - 1;
	else
	low = mid + 1;
	}
	return high;
	}

	/* Helper function to determine domain of data */
	enum {
	DOMAIN_7BIT = 0,
	DOMAIN_8BIT,
	DOMAIN_BINARY
	};

	static int data_domain(const char *p, size_t n)
	{
	while (n--) {
	if (!*p) return DOMAIN_BINARY;
	if (*p & 0x80) return DOMAIN_8BIT;
	p++;
	}

	return DOMAIN_7BIT;
	}

	/*
	* Helper function to fetch data from a message file. Writes a
	* quoted-string or literal containing data from 'msg_base', which is
	* of size 'msg_size', starting at 'offset' and containing 'size'
	* octets. If 'octet_count' is nonzero, the data is
	* further constrained by 'start_octet' and 'octet_count' as per the
	* IMAP command PARTIAL.
	*/
	void
	index_fetchmsg(state, msg_base, msg_size, offset, size,
	start_octet, octet_count)
	struct index_state *state;
	const char *msg_base;
	unsigned long msg_size;
	unsigned offset;
	unsigned size; /* this is the correct size for a news message after
	having LF translated to CRLF */
	unsigned start_octet;
	unsigned octet_count;
	{
	unsigned n, domain;

	/* If no data, output NIL */
	if (!msg_base) {
	prot_printf(state->out, "NIL");
	return;
	}

	/* partial fetch: adjust 'size' */
	if (octet_count) {
	if (size <= start_octet) {
	size = 0;
	}
	else {
	size -= start_octet;
	}
	if (size > octet_count) size = octet_count;
	}

	/* If zero-length data, output empty quoted string */
	if (size == 0) {
	prot_printf(state->out, "\"\"");
	return;
	}

	/* Seek over PARTIAL constraint */
	offset += start_octet;
	n = size;
	if (offset + size > msg_size) {
	n = msg_size - offset;
	}

	/* Get domain of the data */
	domain = data_domain(msg_base + offset, n);

	if (domain == DOMAIN_BINARY) {
	/* Write size of literal8 */
	prot_printf(state->out, "~{%u}\r\n", size);
	} else {
	/* Write size of literal */
	prot_printf(state->out, "{%u}\r\n", size);
	}

	/* Non-text literal -- tell the protstream about it */
	if (domain != DOMAIN_7BIT) prot_data_boundary(state->out);

	prot_write(state->out, msg_base + offset, n);
	while (n++ < size) {
	/* File too short, resynch client.
	*
	* This can only happen if the reported size of the part
	* is incorrect and would push us past EOF.
	*/
	(void)prot_putc(' ', state->out);
	}

	/* End of non-text literal -- tell the protstream about it */
	if (domain != DOMAIN_7BIT) prot_data_boundary(state->out);
	}

	/*
	* Helper function to fetch a body section
	*/
	static int index_fetchsection(struct index_state state, const char resp,
	const char *msg_base, unsigned long msg_size,
	char section, const char cachestr, unsigned size,
	unsigned start_octet, unsigned octet_count)
	{
	const char *p;
	int32_t skip = 0;
	int fetchmime = 0;
	unsigned offset = 0;
	char *decbuf = NULL;

	p = section;

	/* Special-case BODY[] */
	if (*p == ']') {
	if (strstr(resp, "BINARY.SIZE")) {
	prot_printf(state->out, "%s%u", resp, size);
	} else {
	prot_printf(state->out, "%s", resp);
	index_fetchmsg(state, msg_base, msg_size, 0, size,
	start_octet, octet_count);
	}
	return 0;
	}

	while (p != ']' && p != 'M') {
	int num_parts = CACHE_ITEM_BIT32(cachestr);
	int r;

	/* Generate the actual part number */
	r = parseint32(p, &p, &skip);
	if (*p == '.') p++;

	/* Handle .0, .HEADER, and .TEXT */
	if (r \|\| skip == 0) {
	skip = 0;
	/* We don't have any digits, so its a string */
	switch (*p) {
	case 'H':
	p += 6;
	fetchmime++; /* .HEADER maps internally to .0.MIME */
	break;

	case 'T':
	p += 4;
	break; /* .TEXT maps internally to .0 */

	default:
	fetchmime++; /* .0 maps internally to .0.MIME */
	break;
	}
	}

	/* section number too large */
	if (skip >= num_parts) goto badpart;

	if (p != ']' && p != 'M') {
	/* We are NOT at the end of a part specification, so there's
	* a subpart being requested. Find the subpart in the tree. */

	/* Skip the headers for this part, along with the number of
	* sub parts */
	cachestr += num_parts * 5 * 4 + CACHE_ITEM_SIZE_SKIP;

	/* Skip to the correct part */
	while (--skip) {
	if (CACHE_ITEM_BIT32(cachestr) > 0) {
	/* Skip each part at this level */
	skip += CACHE_ITEM_BIT32(cachestr)-1;
	cachestr += CACHE_ITEM_BIT32(cachestr) * 5 * 4;
	}
	cachestr += CACHE_ITEM_SIZE_SKIP;
	}
	}
	}

	if (*p == 'M') fetchmime++;

	cachestr += skip * 5 * 4 + CACHE_ITEM_SIZE_SKIP + (fetchmime ? 0 : 2 * 4);

	if (CACHE_ITEM_BIT32(cachestr + CACHE_ITEM_SIZE_SKIP) == (bit32) -1)
	goto badpart;

	offset = CACHE_ITEM_BIT32(cachestr);
	size = CACHE_ITEM_BIT32(cachestr + CACHE_ITEM_SIZE_SKIP);

	if (msg_base && (p = strstr(resp, "BINARY"))) {
	/* BINARY or BINARY.SIZE */
	int encoding = CACHE_ITEM_BIT32(cachestr + 2 * 4) & 0xff;
	size_t newsize;

	/* check that the offset isn't corrupt */
	if (offset + size > msg_size) {
	syslog(LOG_ERR, "invalid part offset in %s", state->mailbox->name);
	return IMAP_IOERROR;
	}

	msg_base = charset_decode_mimebody(msg_base + offset, size, encoding,
	&decbuf, 0, &newsize);

	if (!msg_base) {
	/* failed to decode */
	if (decbuf) free(decbuf);
	return IMAP_NO_UNKNOWN_CTE;
	}
	else if (p[6] == '.') {
	/* BINARY.SIZE */
	prot_printf(state->out, "%s%zd", resp, newsize);
	if (decbuf) free(decbuf);
	return 0;
	}
	else {
	/* BINARY */
	offset = 0;
	size = newsize;
	msg_size = newsize;
	}
	}

	/* Output body part */
	prot_printf(state->out, "%s", resp);
	index_fetchmsg(state, msg_base, msg_size, offset, size,
	start_octet, octet_count);

	if (decbuf) free(decbuf);
	return 0;

	badpart:
	if (strstr(resp, "BINARY.SIZE"))
	prot_printf(state->out, "%s0", resp);
	else
	prot_printf(state->out, "%sNIL", resp);
	return 0;
	}

	/*
	* Helper function to fetch a HEADER.FIELDS[.NOT] body section
	*/
	static void index_fetchfsection(struct index_state *state,
	const char *msg_base,
	unsigned long msg_size,
	struct fieldlist *fsection,
	const char *cachestr,
	unsigned start_octet, unsigned octet_count)
	{
	const char *p;
	int32_t skip = 0;
	int fields_not = 0;
	unsigned crlf_start = 0;
	unsigned crlf_size = 2;
	char *buf;
	unsigned size;
	int r;

	/* If no data, output null quoted string */
	if (!msg_base) {
	prot_printf(state->out, "\"\"");
	return;
	}

	p = fsection->section;

	while (*p != 'H') {
	int num_parts = CACHE_ITEM_BIT32(cachestr);

	r = parseint32(p, &p, &skip);
	if (*p == '.') p++;

	/* section number too large */
	if (r \|\| skip == 0 \|\| skip >= num_parts) goto badpart;

	cachestr += num_parts * 5 * 4 + CACHE_ITEM_SIZE_SKIP;
	while (--skip) {
	if (CACHE_ITEM_BIT32(cachestr) > 0) {
	skip += CACHE_ITEM_BIT32(cachestr)-1;
	cachestr += CACHE_ITEM_BIT32(cachestr) * 5 * 4;
	}
	cachestr += CACHE_ITEM_SIZE_SKIP;
	}
	}

	/* leaf object */
	if (0 == CACHE_ITEM_BIT32(cachestr)) goto badpart;

	cachestr += 4;

	if (CACHE_ITEM_BIT32(cachestr+CACHE_ITEM_SIZE_SKIP) == (bit32) -1)
	goto badpart;

	if (p[13]) fields_not++; /* Check for "." after "HEADER.FIELDS" */

	buf = index_readheader(msg_base, msg_size,
	CACHE_ITEM_BIT32(cachestr),
	CACHE_ITEM_BIT32(cachestr+CACHE_ITEM_SIZE_SKIP));

	if (fields_not) {
	index_pruneheader(buf, 0, fsection->fields);
	}
	else {
	index_pruneheader(buf, fsection->fields, 0);
	}
	size = strlen(buf);

	/* partial fetch: adjust 'size' */
	if (octet_count) {
	if (size <= start_octet) {
	crlf_start = start_octet - size;
	size = 0;
	start_octet = 0;
	if (crlf_size <= crlf_start) {
	crlf_size = 0;
	}
	else {
	crlf_size -= crlf_start;
	}
	}
	else {
	size -= start_octet;
	}
	if (size > octet_count) {
	size = octet_count;
	crlf_size = 0;
	}
	else if (size + crlf_size > octet_count) {
	crlf_size = octet_count - size;
	}
	}

	/* If no data, output null quoted string */
	if (size + crlf_size == 0) {
	prot_printf(state->out, "\"\"");
	return;
	}

	/* Write literal */
	prot_printf(state->out, "{%u}\r\n", size + crlf_size);
	prot_write(state->out, buf + start_octet, size);
	prot_write(state->out, "\r\n" + crlf_start, crlf_size);

	return;

	badpart:
	prot_printf(state->out, "NIL");
	}

	/*
	* Helper function to read a header section into a static buffer
	*/
	static char *
	index_readheader(msg_base, msg_size, offset, size)
	const char *msg_base;
	unsigned long msg_size;
	unsigned offset;
	unsigned size;
	{
	static char *buf;
	static unsigned bufsize;

	if (offset + size > msg_size) {
	/* Message file is too short, truncate request */
	if (offset < msg_size) {
	size = msg_size - offset;
	}
	else {
	size = 0;
	}
	}

	if (bufsize < size+2) {
	bufsize = size+100;
	buf = xrealloc(buf, bufsize);
	}

	msg_base += offset;

	memcpy(buf, msg_base, size);
	buf[size] = '\0';

	return buf;
	}

	/*
	* Prune the header section in buf to include only those headers
	* listed in headers or (if headers_not is non-empty) those headers
	* not in headers_not.
	*/
	static void
	index_pruneheader(char buf, struct strlist headers,
	struct strlist *headers_not)
	{
	char p, colon, *nextheader;
	int goodheader;
	char *endlastgood = buf;
	struct strlist *l;

	p = buf;
	while (p && p != '\r') {
	colon = strchr(p, ':');
	if (colon && headers_not) {
	goodheader = 1;
	for (l = headers_not; l; l = l->next) {
	if ((size_t) (colon - p) == strlen(l->s) &&
	!strncasecmp(p, l->s, colon - p)) {
	goodheader = 0;
	break;
	}
	}
	} else {
	goodheader = 0;
	}
	if (colon) {
	for (l = headers; l; l = l->next) {
	if ((size_t) (colon - p) == strlen(l->s) &&
	!strncasecmp(p, l->s, colon - p)) {
	goodheader = 1;
	break;
	}
	}
	}

	nextheader = p;
	do {
	nextheader = strchr(nextheader, '\n');
	if (nextheader) nextheader++;
	else nextheader = p + strlen(p);
	} while (nextheader == ' ' \|\| nextheader == '\t');

	if (goodheader) {
	if (endlastgood != p) {
	/* memmove and not strcpy since this is all within a
	* single buffer */
	memmove(endlastgood, p, strlen(p) + 1);
	nextheader -= p - endlastgood;
	}
	endlastgood = nextheader;
	}
	p = nextheader;
	}

	*endlastgood = '\0';
	}

	/*
	* Handle a FETCH RFC822.HEADER.LINES or RFC822.HEADER.LINES.NOT
	* that can't use the cacheheaders in cyrus.cache
	*/
	static void index_fetchheader(struct index_state *state,
	const char *msg_base,
	unsigned long msg_size,
	unsigned size,
	struct strlist *headers,
	struct strlist *headers_not)
	{
	char *buf;

	/* If no data, output null quoted string */
	if (!msg_base) {
	prot_printf(state->out, "\"\"");
	return;
	}

	buf = index_readheader(msg_base, msg_size, 0, size);

	index_pruneheader(buf, headers, headers_not);

	size = strlen(buf);
	prot_printf(state->out, "{%u}\r\n%s\r\n", size+2, buf);
	}

	/*
	* Handle a FETCH RFC822.HEADER.LINES that can use the
	* cacheheaders in cyrus.cache
	*/
	static void
	index_fetchcacheheader(struct index_state state, struct index_record record,
	struct strlist *headers, unsigned start_octet,
	unsigned octet_count)
	{
	static char *buf;
	static unsigned bufsize;
	unsigned size;
	unsigned crlf_start = 0;
	unsigned crlf_size = 2;
	struct mailbox *mailbox = state->mailbox;

	if (mailbox_cacherecord(mailbox, record)) {
	/* bogus cache record */
	prot_printf(state->out, "\"\"");
	return;
	}

	size = cacheitem_size(record, CACHE_HEADERS);
	if (bufsize < size+2) {
	bufsize = size+100;
	buf = xrealloc(buf, bufsize);
	}

	memcpy(buf, cacheitem_base(record, CACHE_HEADERS), size);
	buf[size] = '\0';

	index_pruneheader(buf, headers, 0);
	size = strlen(buf);

	/* partial fetch: adjust 'size' */
	if (octet_count) {
	if (size <= start_octet) {
	crlf_start = start_octet - size;
	size = 0;
	start_octet = 0;
	if (crlf_size <= crlf_start) {
	crlf_size = 0;
	}
	else {
	crlf_size -= crlf_start;
	}
	}
	else {
	size -= start_octet;
	}
	if (size > octet_count) {
	size = octet_count;
	crlf_size = 0;
	}
	else if (size + crlf_size > octet_count) {
	crlf_size = octet_count - size;
	}
	}

	if (size + crlf_size == 0) {
	prot_printf(state->out, "\"\"");
	}
	else {
	prot_printf(state->out, "{%u}\r\n", size + crlf_size);
	prot_write(state->out, buf + start_octet, size);
	prot_write(state->out, "\r\n" + crlf_start, crlf_size);
	}
	}

	/*
	* Send a * FLAGS response.
	*/
	static void index_listflags(struct index_state *state)
	{
	unsigned i;
	int cancreate = 0;
	char sepchar = '(';

	prot_printf(state->out, "* FLAGS (\\Answered \\Flagged \\Draft \\Deleted \\Seen");
	for (i = 0; i < MAX_USER_FLAGS; i++) {
	if (state->flagname[i]) {
	prot_printf(state->out, " %s", state->flagname[i]);
	}
	else cancreate++;
	}
	prot_printf(state->out, ")\r\n* OK [PERMANENTFLAGS ");
	if (!state->examining) {
	if (state->myrights & ACL_WRITE) {
	prot_printf(state->out, "%c\\Answered \\Flagged \\Draft", sepchar);
	sepchar = ' ';
	}
	if (state->myrights & ACL_DELETEMSG) {
	prot_printf(state->out, "%c\\Deleted", sepchar);
	sepchar = ' ';
	}
	if (state->myrights & ACL_SEEN) {
	prot_printf(state->out, "%c\\Seen", sepchar);
	sepchar = ' ';
	}
	if (state->myrights & ACL_WRITE) {
	for (i = 0; i < MAX_USER_FLAGS; i++) {
	if (state->flagname[i]) {
	prot_printf(state->out, " %s", state->flagname[i]);
	}
	}
	if (cancreate) {
	prot_printf(state->out, " \\*");
	}
	}
	}
	if (sepchar == '(') prot_printf(state->out, "(");
	prot_printf(state->out, ")] Ok\r\n");
	}

	static void index_checkflags(struct index_state *state, int dirty)
	{
	struct mailbox *mailbox = state->mailbox;
	unsigned i;

	for (i = 0; i < MAX_USER_FLAGS; i++) {
	/* both empty */
	if (!mailbox->flagname[i] && !state->flagname[i])
	continue;

	/* both same */
	if (mailbox->flagname[i] && state->flagname[i] &&
	!strcmp(mailbox->flagname[i], state->flagname[i]))
	continue;

	/* ok, got something to change! */
	if (state->flagname[i])
	free(state->flagname[i]);
	if (mailbox->flagname[i])
	state->flagname[i] = xstrdup(mailbox->flagname[i]);
	else
	state->flagname[i] = NULL;

	dirty = 1;
	}

	if (dirty)
	index_listflags(state);
	}

	static void index_tellexpunge(struct index_state *state)
	{
	unsigned oldmsgno;
	uint32_t msgno = 1;
	struct seqset *vanishedlist;
	struct index_map *im;

	vanishedlist = seqset_init(0, SEQ_SPARSE);

	for (oldmsgno = 1; oldmsgno <= state->exists; oldmsgno++) {
	im = &state->map[oldmsgno-1];

	/* inform about expunges */
	if (im->record.system_flags & FLAG_EXPUNGED) {
	if (state->qresync)
	seqset_add(vanishedlist, im->record.uid, 1);
	else
	prot_printf(state->out, "* %u EXPUNGE\r\n", msgno);
	continue;
	}

	/* copy back if necessary (after first expunge) */
	if (msgno < oldmsgno)
	state->map[msgno-1] = *im;

	msgno++;
	}

	/* report all vanished if we're doing it this way */
	if (vanishedlist->len) {
	char *vanished = seqset_cstring(vanishedlist);
	prot_printf(state->out, "* VANISHED %s\r\n", vanished);
	free(vanished);
	}
	seqset_free(vanishedlist);

	/* exists count has reduced to what was left */
	state->exists = msgno - 1;
	/* and highestmodseq can come forward to real-time */
	state->highestmodseq = state->mailbox->i.highestmodseq;
	}

	static void index_tellexists(struct index_state *state)
	{
	prot_printf(state->out, "* %u EXISTS\r\n", state->exists);
	prot_printf(state->out, "* %u RECENT\r\n", state->numrecent);
	state->havenewrecords = 0;
	}

	void index_tellchanges(struct index_state *state, int canexpunge,
	int printuid)
	{
	uint32_t msgno;
	struct index_map *im;

	if (canexpunge) index_tellexpunge(state);

	if (state->havenewrecords) index_tellexists(state);

	index_checkflags(state, 0);

	/* print any changed message flags */
	for (msgno = 1; msgno <= state->exists; msgno++) {
	im = &state->map[msgno-1];

	/* we don't report flag updates if it's been expunged */
	if (im->record.system_flags & FLAG_EXPUNGED)
	continue;

	/* report if it's changed since last told */
	if (im->record.modseq > im->told_modseq)
	index_printflags(state, msgno, printuid);
	}
	}
	/*
	* Helper function to send * FETCH (FLAGS data.
	* Does not send the terminating close paren or CRLF.
	* Also sends preceeding * FLAGS if necessary.
	*/
	static void index_fetchflags(struct index_state *state,
	uint32_t msgno)
	{
	int sepchar = '(';
	unsigned flag;
	bit32 flagmask = 0;
	struct index_map *im = &state->map[msgno-1];

	prot_printf(state->out, "* %u FETCH (FLAGS ", msgno);

	if (im->isrecent) {
	prot_printf(state->out, "%c\\Recent", sepchar);
	sepchar = ' ';
	}
	if (im->record.system_flags & FLAG_ANSWERED) {
	prot_printf(state->out, "%c\\Answered", sepchar);
	sepchar = ' ';
	}
	if (im->record.system_flags & FLAG_FLAGGED) {
	prot_printf(state->out, "%c\\Flagged", sepchar);
	sepchar = ' ';
	}
	if (im->record.system_flags & FLAG_DRAFT) {
	prot_printf(state->out, "%c\\Draft", sepchar);
	sepchar = ' ';
	}
	if (im->record.system_flags & FLAG_DELETED) {
	prot_printf(state->out, "%c\\Deleted", sepchar);
	sepchar = ' ';
	}
	if (im->isseen) {
	prot_printf(state->out, "%c\\Seen", sepchar);
	sepchar = ' ';
	}
	for (flag = 0; flag < VECTOR_SIZE(state->flagname); flag++) {
	if ((flag & 31) == 0) {
	flagmask = im->record.user_flags[flag/32];
	}
	if (state->flagname[flag] && (flagmask & (1<<(flag & 31)))) {
	prot_printf(state->out, "%c%s", sepchar, state->flagname[flag]);
	sepchar = ' ';
	}
	}
	if (sepchar == '(') (void)prot_putc('(', state->out);
	(void)prot_putc(')', state->out);
	im->told_modseq = im->record.modseq;
	}

	static void index_printflags(struct index_state *state,
	uint32_t msgno, int usinguid)
	{
	struct index_map *im = &state->map[msgno-1];

	index_fetchflags(state, msgno);
	/* http://www.rfc-editor.org/errata_search.php?rfc=5162
	* Errata ID: 1807 - MUST send UID and MODSEQ to all
	* untagged FETCH unsolicited responses */
	if (usinguid \|\| state->qresync)
	prot_printf(state->out, " UID %u", im->record.uid);
	if (state->qresync)
	prot_printf(state->out, " MODSEQ (" MODSEQ_FMT ")", im->record.modseq);
	prot_printf(state->out, ")\r\n");
	}

	/*
	* Helper function to send requested * FETCH data for a message
	*/
	static int index_fetchreply(struct index_state *state, uint32_t msgno,
	struct fetchargs *fetchargs)
	{
	struct mailbox *mailbox = state->mailbox;
	int fetchitems = fetchargs->fetchitems;
	const char *msg_base = NULL;
	unsigned long msg_size = 0;
	struct octetinfo *oi = NULL;
	int sepchar = '(';
	int started = 0;
	struct strlist section, field;
	struct fieldlist *fsection;
	char respbuf[100];
	int r = 0;
	struct index_map *im = &state->map[msgno-1];

	/* Check the modseq against changedsince */
	if (fetchargs->changedsince &&
	im->record.modseq <= fetchargs->changedsince) {
	return 0;
	}

	/* Open the message file if we're going to need it */
	if ((fetchitems & (FETCH_HEADER\|FETCH_TEXT\|FETCH_RFC822)) \|\|
	fetchargs->cache_atleast > im->record.cache_version \|\|
	fetchargs->binsections \|\| fetchargs->sizesections \|\|
	fetchargs->bodysections) {
	if (mailbox_map_message(mailbox, im->record.uid, &msg_base, &msg_size)) {
	prot_printf(state->out, "* OK ");
	prot_printf(state->out, error_message(IMAP_NO_MSGGONE), msgno);
	prot_printf(state->out, "\r\n");
	return 0;
	}
	}

	/* display flags if asked _OR_ if they've changed */
	if (fetchitems & FETCH_FLAGS \|\| im->told_modseq < im->record.modseq) {
	index_fetchflags(state, msgno);
	sepchar = ' ';
	}
	else if ((fetchitems & ~FETCH_SETSEEN) \|\| fetchargs->fsections \|\|
	fetchargs->headers \|\| fetchargs->headers_not) {
	/* these fetch items will always succeed, so start the response */
	prot_printf(state->out, "* %u FETCH ", msgno);
	started = 1;
	}
	if (fetchitems & FETCH_UID) {
	prot_printf(state->out, "%cUID %u", sepchar, im->record.uid);
	sepchar = ' ';
	}
	if (fetchitems & FETCH_INTERNALDATE) {
	time_t msgdate = im->record.internaldate;
	char datebuf[30];

	cyrus_ctime(msgdate, datebuf);

	prot_printf(state->out, "%cINTERNALDATE \"%s\"",
	sepchar, datebuf);
	sepchar = ' ';
	}
	if (fetchitems & FETCH_MODSEQ) {
	prot_printf(state->out, "%cMODSEQ (" MODSEQ_FMT ")",
	sepchar, im->record.modseq);
	sepchar = ' ';
	}
	if (fetchitems & FETCH_SIZE) {
	prot_printf(state->out, "%cRFC822.SIZE %u",
	sepchar, im->record.size);
	sepchar = ' ';
	}
	if (fetchitems & FETCH_ENVELOPE) {
	if (!mailbox_cacherecord(mailbox, &im->record)) {
	prot_printf(state->out, "%cENVELOPE ", sepchar);
	sepchar = ' ';
	prot_putbuf(state->out, cacheitem_buf(&im->record, CACHE_ENVELOPE));
	}
	}
	if (fetchitems & FETCH_BODYSTRUCTURE) {
	if (!mailbox_cacherecord(mailbox, &im->record)) {
	prot_printf(state->out, "%cBODYSTRUCTURE ", sepchar);
	sepchar = ' ';
	prot_putbuf(state->out, cacheitem_buf(&im->record, CACHE_BODYSTRUCTURE));
	}
	}
	if (fetchitems & FETCH_BODY) {
	if (!mailbox_cacherecord(mailbox, &im->record)) {
	prot_printf(state->out, "%cBODY ", sepchar);
	sepchar = ' ';
	prot_putbuf(state->out, cacheitem_buf(&im->record, CACHE_BODY));
	}
	}

	if (fetchitems & FETCH_HEADER) {
	prot_printf(state->out, "%cRFC822.HEADER ", sepchar);
	sepchar = ' ';
	index_fetchmsg(state, msg_base, msg_size, 0,
	im->record.header_size,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->start_octet : 0,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->octet_count : 0);
	}
	else if (fetchargs->headers \|\| fetchargs->headers_not) {
	prot_printf(state->out, "%cRFC822.HEADER ", sepchar);
	sepchar = ' ';
	if (fetchargs->cache_atleast > im->record.cache_version) {
	index_fetchheader(state, msg_base, msg_size,
	im->record.header_size,
	fetchargs->headers, fetchargs->headers_not);
	} else {
	index_fetchcacheheader(state, &im->record, fetchargs->headers, 0, 0);
	}
	}

	if (fetchitems & FETCH_TEXT) {
	prot_printf(state->out, "%cRFC822.TEXT ", sepchar);
	sepchar = ' ';
	index_fetchmsg(state, msg_base, msg_size,
	im->record.header_size, im->record.size - im->record.header_size,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->start_octet : 0,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->octet_count : 0);
	}
	if (fetchitems & FETCH_RFC822) {
	prot_printf(state->out, "%cRFC822 ", sepchar);
	sepchar = ' ';
	index_fetchmsg(state, msg_base, msg_size, 0, im->record.size,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->start_octet : 0,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->octet_count : 0);
	}
	for (fsection = fetchargs->fsections; fsection; fsection = fsection->next) {
	prot_printf(state->out, "%cBODY[%s ", sepchar, fsection->section);
	sepchar = '(';
	for (field = fsection->fields; field; field = field->next) {
	(void)prot_putc(sepchar, state->out);
	sepchar = ' ';
	prot_printastring(state->out, field->s);
	}
	(void)prot_putc(')', state->out);
	sepchar = ' ';

	oi = (struct octetinfo *)fsection->rock;

	prot_printf(state->out, "%s ", fsection->trail);

	if (fetchargs->cache_atleast > im->record.cache_version) {
	if (!mailbox_cacherecord(mailbox, &im->record))
	index_fetchfsection(state, msg_base, msg_size,
	fsection,
	cacheitem_base(&im->record, CACHE_SECTION),
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->start_octet : oi->start_octet,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->octet_count : oi->octet_count);
	else
	prot_printf(state->out, "NIL");

	}
	else {
	index_fetchcacheheader(state, &im->record, fsection->fields,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->start_octet : oi->start_octet,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->octet_count : oi->octet_count);
	}
	}
	for (section = fetchargs->bodysections; section; section = section->next) {
	respbuf[0] = 0;
	if (sepchar == '(' && !started) {
	/* we haven't output a fetch item yet, so start the response */
	snprintf(respbuf, sizeof(respbuf), "* %u FETCH ", msgno);
	}
	snprintf(respbuf+strlen(respbuf), sizeof(respbuf)-strlen(respbuf),
	"%cBODY[%s ", sepchar, section->s);

	oi = section->rock;

	if (!mailbox_cacherecord(mailbox, &im->record)) {
	r = index_fetchsection(state, respbuf,
	msg_base, msg_size,
	section->s, cacheitem_base(&im->record, CACHE_SECTION),
	im->record.size,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->start_octet : oi->start_octet,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->octet_count : oi->octet_count);
	if (!r) sepchar = ' ';
	}
	}
	for (section = fetchargs->binsections; section; section = section->next) {
	respbuf[0] = 0;
	if (sepchar == '(' && !started) {
	/* we haven't output a fetch item yet, so start the response */
	snprintf(respbuf, sizeof(respbuf), "* %u FETCH ", msgno);
	}
	snprintf(respbuf+strlen(respbuf), sizeof(respbuf)-strlen(respbuf),
	"%cBINARY[%s ", sepchar, section->s);

	if (!mailbox_cacherecord(mailbox, &im->record)) {
	oi = section->rock;
	r = index_fetchsection(state, respbuf,
	msg_base, msg_size,
	section->s, cacheitem_base(&im->record, CACHE_SECTION),
	im->record.size,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->start_octet : oi->start_octet,
	(fetchitems & FETCH_IS_PARTIAL) ?
	fetchargs->octet_count : oi->octet_count);
	if (!r) sepchar = ' ';
	}
	}
	for (section = fetchargs->sizesections; section; section = section->next) {
	respbuf[0] = 0;
	if (sepchar == '(' && !started) {
	/* we haven't output a fetch item yet, so start the response */
	snprintf(respbuf, sizeof(respbuf), "* %u FETCH ", msgno);
	}
	snprintf(respbuf+strlen(respbuf), sizeof(respbuf)-strlen(respbuf),
	"%cBINARY.SIZE[%s ", sepchar, section->s);

	if (!mailbox_cacherecord(mailbox, &im->record)) {
	r = index_fetchsection(state, respbuf,
	msg_base, msg_size,
	section->s, cacheitem_base(&im->record, CACHE_SECTION),
	im->record.size,
	fetchargs->start_octet, fetchargs->octet_count);
	if (!r) sepchar = ' ';
	}
	}
	if (sepchar != '(') {
	/* finsh the response if we have one */
	prot_printf(state->out, ")\r\n");
	}
	if (msg_base)
	mailbox_unmap_message(mailbox, im->record.uid, &msg_base, &msg_size);

	return r;
	}

	/*
	* Fetch the text data associated with an IMAP URL.
	*
	* If outsize is NULL, the data will be output as a literal (URLFETCH),
	* otherwise just the data will be output (CATENATE), and its size returned
	* in *outsize.
	*
	* This is an amalgamation of index_fetchreply(), index_fetchsection()
	* and index_fetchmsg().
	*/
	int index_urlfetch(struct index_state *state, uint32_t msgno,
	unsigned params, const char *section,
	unsigned long start_octet, unsigned long octet_count,
	struct protstream pout, unsigned long outsize)
	{
	const char data, msg_base = 0;
	unsigned long msg_size = 0;
	const char *cacheitem;
	int fetchmime = 0, domain = DOMAIN_7BIT;
	unsigned size;
	int32_t skip = 0;
	int n, r = 0;
	char *decbuf = NULL;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];

	if (outsize) *outsize = 0;

	r = mailbox_cacherecord(mailbox, &im->record);
	if (r) return r;

	/* Open the message file */
	if (mailbox_map_message(mailbox, im->record.uid, &msg_base, &msg_size))
	return IMAP_NO_MSGGONE;

	data = msg_base;
	size = im->record.size;

	if (size > msg_size) size = msg_size;

	cacheitem = cacheitem_base(&im->record, CACHE_SECTION);
	cacheitem += CACHE_ITEM_SIZE_SKIP;

	/* Special-case BODY[] */
	if (!section \|\| !*section) {
	/* whole message, no further parsing */
	}
	else {
	const char p = ucase((char ) section);

	while (p && p != 'M') {
	int num_parts = CACHE_ITEM_BIT32(cacheitem);

	/* Generate the actual part number */
	r = parseint32(p, &p, &skip);
	if (*p == '.') p++;

	/* Handle .0, .HEADER, and .TEXT */
	if (r \|\| skip == 0) {
	skip = 0;
	/* We don't have any digits, so its a string */
	switch (*p) {
	case 'H':
	p += 6;
	fetchmime++; /* .HEADER maps internally to .0.MIME */
	break;

	case 'T':
	p += 4;
	break; /* .TEXT maps internally to .0 */

	default:
	fetchmime++; /* .0 maps internally to .0.MIME */
	break;
	}
	}

	/* section number too large */
	if (skip >= num_parts) {
	r = IMAP_BADURL;
	goto done;
	}

	if (p && p != 'M') {
	/* We are NOT at the end of a part specification, so there's
	* a subpart being requested. Find the subpart in the tree. */

	/* Skip the headers for this part, along with the number of
	* sub parts */
	cacheitem += num_parts * 5 * 4 + CACHE_ITEM_SIZE_SKIP;

	/* Skip to the correct part */
	while (--skip) {
	if (CACHE_ITEM_BIT32(cacheitem) > 0) {
	/* Skip each part at this level */
	skip += CACHE_ITEM_BIT32(cacheitem)-1;
	cacheitem += CACHE_ITEM_BIT32(cacheitem) * 5 * 4;
	}
	cacheitem += CACHE_ITEM_SIZE_SKIP;
	}
	}
	}

	if (*p == 'M') fetchmime++;

	cacheitem += skip * 5 * 4 + CACHE_ITEM_SIZE_SKIP +
	(fetchmime ? 0 : 2 * 4);

	if (CACHE_ITEM_BIT32(cacheitem + CACHE_ITEM_SIZE_SKIP) == (bit32) -1) {
	r = IMAP_BADURL;
	goto done;
	}

	data += CACHE_ITEM_BIT32(cacheitem);
	size = CACHE_ITEM_BIT32(cacheitem + CACHE_ITEM_SIZE_SKIP);
	}

	/* Handle extended URLFETCH parameters */
	if (params & URLFETCH_BODYPARTSTRUCTURE) {
	prot_printf(pout, " (BODYPARTSTRUCTURE");
	/* XXX Calculate body part structure */
	prot_printf(pout, " NIL");
	prot_printf(pout, ")");
	}

	if (params & URLFETCH_BODY) {
	prot_printf(pout, " (BODY");
	}
	else if (params & URLFETCH_BINARY) {
	int encoding = CACHE_ITEM_BIT32(cacheitem + 2 * 4) & 0xff;

	prot_printf(pout, " (BINARY");

	data = charset_decode_mimebody(data, size, encoding,
	&decbuf, 0, (size_t *) &size);
	if (!data) {
	/* failed to decode */
	prot_printf(pout, " NIL)");
	goto done;
	}
	}

	/* Handle PARTIAL request */
	n = octet_count ? octet_count : size;

	/* Sanity check the requested size */
	if (start_octet + n > size) n = size - start_octet;

	if (outsize) {
	/* Return size (CATENATE) */
	*outsize = n;
	} else {
	domain = data_domain(data + start_octet, n);

	if (domain == DOMAIN_BINARY) {
	/* Write size of literal8 */
	prot_printf(pout, " ~{%u}\r\n", n);
	} else {
	/* Write size of literal */
	prot_printf(pout, " {%u}\r\n", n);
	}
	}

	/* Non-text literal -- tell the protstream about it */
	if (domain != DOMAIN_7BIT) prot_data_boundary(pout);

	prot_write(pout, data + start_octet, n);

	/* End of non-text literal -- tell the protstream about it */
	if (domain != DOMAIN_7BIT) prot_data_boundary(pout);

	/* Complete extended URLFETCH response */
	if (params & (URLFETCH_BODY \| URLFETCH_BINARY)) prot_printf(pout, ")");

	done:
	/* Close the message file */
	mailbox_unmap_message(mailbox, im->record.uid, &msg_base, &msg_size);

	if (decbuf) free(decbuf);
	return r;
	}

	/*
	* Helper function to perform a generalized STORE command
	*/
	static int index_storeflag(struct index_state *state, uint32_t msgno,
	struct storeargs *storeargs)
	{
	bit32 old, new;
	unsigned i;
	int dirty = 0;
	modseq_t oldmodseq;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];
	int r;

	/* if it's changed already, skip out now */
	if (im->record.modseq > storeargs->unchangedsince) return 0;

	/* if it's expunged already, skip out now */
	if (im->record.system_flags & FLAG_EXPUNGED)
	return 0;

	oldmodseq = im->record.modseq;

	/* Change \Seen flag */
	if (state->myrights & ACL_SEEN) {
	old = im->isseen ? 1 : 0;
	new = old;
	if (storeargs->operation == STORE_REPLACE)
	new = storeargs->seen ? 1 : 0;
	else if (storeargs->seen)
	new = (storeargs->operation == STORE_ADD) ? 1 : 0;

	if (new != old) {
	state->numunseen += (old - new);
	im->isseen = new;
	state->seen_dirty = 1;
	dirty++;
	}
	}

	old = im->record.system_flags;
	new = storeargs->system_flags;

	if (storeargs->operation == STORE_REPLACE) {
	if (!(state->myrights & ACL_WRITE)) {
	/* ACL_DELETE handled in index_store() */
	if ((old & FLAG_DELETED) != (new & FLAG_DELETED)) {
	dirty++;
	im->record.system_flags = (old & ~FLAG_DELETED) \| (new & FLAG_DELETED);
	}
	}
	else {
	if (!(state->myrights & ACL_DELETEMSG)) {
	if ((old & ~FLAG_DELETED) != (new & ~FLAG_DELETED)) {
	dirty++;
	im->record.system_flags = (old & FLAG_DELETED) \| (new & ~FLAG_DELETED);
	}
	}
	else {
	if (old != new) {
	dirty++;
	im->record.system_flags = new;
	}
	}
	for (i = 0; i < (MAX_USER_FLAGS/32); i++) {
	if (im->record.user_flags[i] != storeargs->user_flags[i]) {
	dirty++;
	im->record.user_flags[i] = storeargs->user_flags[i];
	}
	}
	}
	}
	else if (storeargs->operation == STORE_ADD) {
	if (~old & new) {
	dirty++;
	im->record.system_flags = old \| new;
	}
	for (i = 0; i < (MAX_USER_FLAGS/32); i++) {
	if (~im->record.user_flags[i] & storeargs->user_flags[i]) {
	dirty++;
	im->record.user_flags[i] \|= storeargs->user_flags[i];
	}
	}
	}
	else { /* STORE_REMOVE */
	if (old & new) {
	dirty++;
	im->record.system_flags &= ~storeargs->system_flags;
	}
	for (i = 0; i < (MAX_USER_FLAGS/32); i++) {
	if (im->record.user_flags[i] & storeargs->user_flags[i]) {
	dirty++;
	im->record.user_flags[i] &= ~storeargs->user_flags[i];
	}
	}
	}

	/* rfc4551:
	* 3.8. Additional Quality-of-Implementation Issues
	*
	* Server implementations should follow the following rule, which
	* applies to any successfully completed STORE/UID STORE (with and
	* without UNCHANGEDSINCE modifier), as well as to a FETCH command that
	* implicitly sets \Seen flag:
	*
	* Adding the flag when it is already present or removing when it is
	* not present SHOULD NOT change the mod-sequence.
	*
	* This will prevent spurious client synchronization requests.
	*/
	if (!dirty) return 0;

	if (state->internalseen) {
	/* set the seen flag */
	if (im->isseen)
	im->record.system_flags \|= FLAG_SEEN;
	else
	im->record.system_flags &= ~FLAG_SEEN;
	}

	r = mailbox_rewrite_index_record(mailbox, &im->record);
	if (r) return r;

	/* if it's silent and unchanged, update the seen value */
	if (storeargs->silent && im->told_modseq == oldmodseq)
	im->told_modseq = im->record.modseq;

	return 0;
	}

	int _search_searchbuf(char s, comp_pat p, struct buf *b)
	{
	if (!b->len)
	return 0;

	return charset_searchstring(s, p, b->s, b->len);
	}

	/*
	* Evaluate a searchargs structure on a msgno
	*
	* Note: msgfile argument must be 0 if msg is not mapped in.
	*/
	static int index_search_evaluate(struct index_state *state,
	struct searchargs *searchargs,
	uint32_t msgno,
	struct mapfile *msgfile)
	{
	unsigned i;
	struct strlist l, h;
	struct searchsub *s;
	struct seqset *seq;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];

	if ((searchargs->flags & SEARCH_RECENT_SET) && !im->isrecent)
	return 0;
	if ((searchargs->flags & SEARCH_RECENT_UNSET) && im->isrecent)
	return 0;
	if ((searchargs->flags & SEARCH_SEEN_SET) && !im->isseen)
	return 0;
	if ((searchargs->flags & SEARCH_SEEN_UNSET) && im->isseen)
	return 0;

	if (searchargs->smaller && im->record.size >= searchargs->smaller)
	return 0;
	if (searchargs->larger && im->record.size <= searchargs->larger)
	return 0;

	if (searchargs->after && im->record.internaldate < searchargs->after)
	return 0;
	if (searchargs->before && im->record.internaldate >= searchargs->before)
	return 0;
	if (searchargs->sentafter && im->record.gmtime < searchargs->sentafter)
	return 0;
	if (searchargs->sentbefore && im->record.gmtime >= searchargs->sentbefore)
	return 0;

	if (searchargs->modseq && im->record.modseq < searchargs->modseq)
	return 0;

	if (~im->record.system_flags & searchargs->system_flags_set)
	return 0;
	if (im->record.system_flags & searchargs->system_flags_unset)
	return 0;

	for (i = 0; i < (MAX_USER_FLAGS/32); i++) {
	if (~im->record.user_flags[i] & searchargs->user_flags_set[i])
	return 0;
	if (im->record.user_flags[i] & searchargs->user_flags_unset[i])
	return 0;
	}

	for (seq = searchargs->sequence; seq; seq = seq->nextseq) {
	if (!seqset_ismember(seq, msgno)) return 0;
	}
	for (seq = searchargs->uidsequence; seq; seq = seq->nextseq) {
	if (!seqset_ismember(seq, im->record.uid)) return 0;
	}

	if (searchargs->from \|\| searchargs->to \|\| searchargs->cc \|\|
	searchargs->bcc \|\| searchargs->subject \|\| searchargs->messageid) {

	if (mailbox_cacherecord(mailbox, &im->record))
	return 0;

	if (searchargs->messageid) {
	char *tmpenv;
	char *envtokens[NUMENVTOKENS];
	char *msgid;
	int msgidlen;

	/* must be long enough to actually HAVE some contents */
	if (cacheitem_size(&im->record, CACHE_ENVELOPE) <= 2)
	return 0;

	/* get msgid out of the envelope */

	/* get a working copy; strip outer ()'s */
	/* +1 -> skip the leading paren */
	/* -2 -> don't include the size of the outer parens */
	tmpenv = xstrndup(cacheitem_base(&im->record, CACHE_ENVELOPE) + 1,
	cacheitem_size(&im->record, CACHE_ENVELOPE) - 2);
	parse_cached_envelope(tmpenv, envtokens, VECTOR_SIZE(envtokens));

	if (!envtokens[ENV_MSGID]) {
	/* free stuff */
	free(tmpenv);
	return 0;
	}

	msgid = lcase(envtokens[ENV_MSGID]);
	msgidlen = strlen(msgid);
	for (l = searchargs->messageid; l; l = l->next) {
	if (!charset_searchstring(l->s, l->p, msgid, msgidlen))
	break;
	}

	/* free stuff */
	free(tmpenv);

	if (l) return 0;
	}

	for (l = searchargs->from; l; l = l->next) {
	if (!_search_searchbuf(l->s, l->p, cacheitem_buf(&im->record, CACHE_FROM)))
	return 0;
	}

	for (l = searchargs->to; l; l = l->next) {
	if (!_search_searchbuf(l->s, l->p, cacheitem_buf(&im->record, CACHE_TO)))
	return 0;
	}

	for (l = searchargs->cc; l; l = l->next) {
	if (!_search_searchbuf(l->s, l->p, cacheitem_buf(&im->record, CACHE_CC)))
	return 0;
	}

	for (l = searchargs->bcc; l; l = l->next) {
	if (!_search_searchbuf(l->s, l->p, cacheitem_buf(&im->record, CACHE_BCC)))
	return 0;
	}

	for (l = searchargs->subject; l; l = l->next) {
	if ((cacheitem_size(&im->record, CACHE_SUBJECT) == 3 &&
	!strncmp(cacheitem_base(&im->record, CACHE_SUBJECT), "NIL", 3)) \|\|
	!_search_searchbuf(l->s, l->p, cacheitem_buf(&im->record, CACHE_SUBJECT)))
	return 0;
	}
	}

	for (s = searchargs->sublist; s; s = s->next) {
	if (index_search_evaluate(state, s->sub1, msgno, msgfile)) {
	if (!s->sub2) return 0;
	}
	else {
	if (s->sub2 &&
	!index_search_evaluate(state, s->sub2, msgno, msgfile))
	return 0;
	}
	}

	if (searchargs->body \|\| searchargs->text \|\|
	searchargs->cache_atleast > im->record.cache_version) {
	if (!msgfile->size) { /* Map the message in if we haven't before */
	if (mailbox_map_message(mailbox, im->record.uid,
	&msgfile->base, &msgfile->size)) {
	return 0;
	}
	}

	h = searchargs->header_name;
	for (l = searchargs->header; l; (l = l->next), (h = h->next)) {
	if (!index_searchheader(h->s, l->s, l->p, msgfile,
	im->record.header_size)) return 0;
	}

	if (mailbox_cacherecord(mailbox, &im->record))
	return 0;

	for (l = searchargs->body; l; l = l->next) {
	if (!index_searchmsg(l->s, l->p, msgfile, 1,
	cacheitem_base(&im->record, CACHE_SECTION))) return 0;
	}
	for (l = searchargs->text; l; l = l->next) {
	if (!index_searchmsg(l->s, l->p, msgfile, 0,
	cacheitem_base(&im->record, CACHE_SECTION))) return 0;
	}
	}
	else if (searchargs->header_name) {
	h = searchargs->header_name;
	for (l = searchargs->header; l; (l = l->next), (h = h->next)) {
	if (!index_searchcacheheader(state, msgno, h->s, l->s, l->p))
	return 0;
	}
	}

	return 1;
	}

	/*
	* Search part of a message for a substring.
	* Keep this in sync with index_getsearchtextmsg!
	*/
	static int
	index_searchmsg(char *substr,
	comp_pat *pat,
	struct mapfile *msgfile,
	int skipheader,
	const char *cachestr)
	{
	int partsleft = 1;
	int subparts;
	unsigned long start;
	int len, charset, encoding;
	char p, q;

	/* Won't find anything in a truncated file */
	if (msgfile->size == 0) return 0;

	while (partsleft--) {
	subparts = CACHE_ITEM_BIT32(cachestr);
	cachestr += 4;
	if (subparts) {
	partsleft += subparts-1;

	if (skipheader) {
	skipheader = 0; /* Only skip top-level message header */
	}
	else {
	len = CACHE_ITEM_BIT32(cachestr + CACHE_ITEM_SIZE_SKIP);
	if (len > 0) {
	p = index_readheader(msgfile->base, msgfile->size,
	CACHE_ITEM_BIT32(cachestr),
	len);
	q = charset_decode_mimeheader(p, NULL, 0);
	if (charset_searchstring(substr, pat, q, strlen(q))) {
	free(q);
	return 1;
	}
	free(q);
	}
	}
	cachestr += 5*4;

	while (--subparts) {
	start = CACHE_ITEM_BIT32(cachestr+2*4);
	len = CACHE_ITEM_BIT32(cachestr+3*4);
	charset = CACHE_ITEM_BIT32(cachestr+4*4) >> 16;
	encoding = CACHE_ITEM_BIT32(cachestr+4*4) & 0xff;

	if (start < msgfile->size && len > 0 &&
	charset >= 0 && charset < 0xffff) {
	if (charset_searchfile(substr, pat,
	msgfile->base + start,
	len, charset, encoding)) return 1;
	}
	cachestr += 5*4;
	}
	}
	}

	return 0;
	}

	/*
	* Search named header of a message for a substring
	*/
	static int index_searchheader(char *name,
	char *substr,
	comp_pat *pat,
	struct mapfile *msgfile,
	int size)
	{
	char p, q;
	int r;
	static struct strlist header;

	header.s = name;

	p = index_readheader(msgfile->base, msgfile->size, 0, size);
	index_pruneheader(p, &header, 0);
	if (!p) return 0; / Header not present, fail */
	if (!substr) return 1; / Only checking existence, succeed */
	q = charset_decode_mimeheader(strchr(p, ':') + 1, NULL, 0);
	r = charset_searchstring(substr, pat, q, strlen(q));
	free(q);
	return r;
	}

	/*
	* Search named cached header of a message for a substring
	*/
	static int index_searchcacheheader(struct index_state *state, uint32_t msgno,
	char name, char substr, comp_pat *pat)
	{
	char *q;
	static struct strlist header;
	static char *buf;
	static unsigned bufsize;
	unsigned size;
	int r;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];

	r = mailbox_cacherecord(mailbox, &im->record);
	if (r) return 0;

	size = cacheitem_size(&im->record, CACHE_HEADERS);
	if (!size) return 0; /* No cached headers, fail */

	if (bufsize < size+2) {
	bufsize = size+100;
	buf = xrealloc(buf, bufsize);
	}

	/* Copy this item to the buffer */
	memcpy(buf, cacheitem_base(&im->record, CACHE_HEADERS), size);
	buf[size] = '\0';

	header.s = name;

	index_pruneheader(buf, &header, 0);
	if (!buf) return 0; / Header not present, fail */
	if (!substr) return 1; / Only checking existence, succeed */
	/* XXX - we could do this in one pass maybe? charset_search_mimeheader */
	q = charset_decode_mimeheader(strchr(buf, ':') + 1, NULL, 0);
	r = charset_searchstring(substr, pat, q, strlen(q));
	free(q);
	return r;
	}


	/* This code was cribbed from index_searchmsg. Instead of checking for matches,
	we call charset_extractfile to send the entire text out to 'receiver'.
	Keep this in sync with index_searchmsg! */
	static void index_getsearchtextmsg(struct index_state *state,
	int uid,
	index_search_text_receiver_t receiver,
	void *rock,
	char const *cachestr) {
	struct mapfile msgfile;
	int partsleft = 1;
	int subparts;
	unsigned long start;
	int len, charset, encoding;
	int partcount = 0;
	char p, q;
	struct mailbox *mailbox = state->mailbox;

	if (mailbox_map_message(mailbox, uid, &msgfile.base, &msgfile.size)) {
	return;
	}

	/* Won't find anything in a truncated file */
	if (msgfile.size > 0) {
	while (partsleft--) {
	subparts = CACHE_ITEM_BIT32(cachestr);
	cachestr += 4;
	if (subparts) {
	partsleft += subparts-1;

	partcount++;

	len = CACHE_ITEM_BIT32(cachestr+4);
	if (len > 0) {
	p = index_readheader(msgfile.base, msgfile.size,
	CACHE_ITEM_BIT32(cachestr),
	len);
	q = charset_decode_mimeheader(p, NULL, 0);
	if (partcount == 1) {
	receiver(uid, SEARCHINDEX_PART_HEADERS,
	SEARCHINDEX_CMD_STUFFPART, q, strlen(q), rock);
	receiver(uid, SEARCHINDEX_PART_BODY,
	SEARCHINDEX_CMD_BEGINPART, NULL, 0, rock);
	} else {
	receiver(uid, SEARCHINDEX_PART_BODY,
	SEARCHINDEX_CMD_APPENDPART, q, strlen(q), rock);
	}
	free(q);
	}
	cachestr += 5*4;

	while (--subparts) {
	start = CACHE_ITEM_BIT32(cachestr+2*4);
	len = CACHE_ITEM_BIT32(cachestr+3*4);
	charset = CACHE_ITEM_BIT32(cachestr+4*4) >> 16;
	encoding = CACHE_ITEM_BIT32(cachestr+4*4) & 0xff;

	if (start < msgfile.size && len > 0) {
	charset_extractfile(receiver, rock, uid,
	msgfile.base + start,
	len, charset, encoding);
	}
	cachestr += 5*4;
	}
	}
	}

	receiver(uid, SEARCHINDEX_PART_BODY,
	SEARCHINDEX_CMD_ENDPART, NULL, 0, rock);
	}

	mailbox_unmap_message(mailbox, uid, &msgfile.base, &msgfile.size);
	}

	void index_getsearchtext_single(struct index_state *state, uint32_t msgno,
	index_search_text_receiver_t receiver,
	void *rock) {
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];

	if (mailbox_cacherecord(mailbox, &im->record))
	return;

	index_getsearchtextmsg(state, im->record.uid, receiver, rock,
	cacheitem_base(&im->record, CACHE_SECTION));
	receiver(im->record.uid, SEARCHINDEX_PART_FROM, SEARCHINDEX_CMD_STUFFPART,
	cacheitem_base(&im->record, CACHE_FROM),
	cacheitem_size(&im->record, CACHE_FROM), rock);
	receiver(im->record.uid, SEARCHINDEX_PART_TO, SEARCHINDEX_CMD_STUFFPART,
	cacheitem_base(&im->record, CACHE_TO),
	cacheitem_size(&im->record, CACHE_TO), rock);
	receiver(im->record.uid, SEARCHINDEX_PART_CC, SEARCHINDEX_CMD_STUFFPART,
	cacheitem_base(&im->record, CACHE_CC),
	cacheitem_size(&im->record, CACHE_CC), rock);
	receiver(im->record.uid, SEARCHINDEX_PART_BCC, SEARCHINDEX_CMD_STUFFPART,
	cacheitem_base(&im->record, CACHE_BCC),
	cacheitem_size(&im->record, CACHE_BCC), rock);
	receiver(im->record.uid, SEARCHINDEX_PART_SUBJECT, SEARCHINDEX_CMD_STUFFPART,
	cacheitem_base(&im->record, CACHE_SUBJECT),
	cacheitem_size(&im->record, CACHE_SUBJECT), rock);
	}

	void index_getsearchtext(struct index_state *state,
	index_search_text_receiver_t receiver,
	void *rock)
	{
	uint32_t msgno;

	/* Send the converted text of every message out to the receiver. */
	for (msgno = 1; msgno <= state->exists; msgno++)
	index_getsearchtext_single(state, msgno, receiver, rock);
	}

	/*
	* Helper function to set up arguments to append_copy()
	*/
	#define COPYARGSGROW 30
	static int index_copysetup(struct index_state *state, uint32_t msgno,
	struct copyargs *copyargs)
	{
	int flag = 0;
	int userflag;
	bit32 flagmask = 0;
	int r;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];

	r = mailbox_cacherecord(mailbox, &im->record);
	if (r) return r;

	if (copyargs->nummsg == copyargs->msgalloc) {
	copyargs->msgalloc += COPYARGSGROW;
	copyargs->copymsg = (struct copymsg *)
	xrealloc((char *)copyargs->copymsg,
	copyargs->msgalloc * sizeof(struct copymsg));
	}

	copyargs->copymsg[copyargs->nummsg].uid = im->record.uid;
	copyargs->copymsg[copyargs->nummsg].internaldate = im->record.internaldate;
	copyargs->copymsg[copyargs->nummsg].sentdate = im->record.sentdate;
	copyargs->copymsg[copyargs->nummsg].gmtime = im->record.gmtime;
	copyargs->copymsg[copyargs->nummsg].size = im->record.size;
	copyargs->copymsg[copyargs->nummsg].header_size = im->record.header_size;
	copyargs->copymsg[copyargs->nummsg].content_lines = im->record.content_lines;
	copyargs->copymsg[copyargs->nummsg].cache_version = im->record.cache_version;
	copyargs->copymsg[copyargs->nummsg].cache_crc = im->record.cache_crc;
	copyargs->copymsg[copyargs->nummsg].crec = im->record.crec;

	message_guid_copy(&copyargs->copymsg[copyargs->nummsg].guid,
	&im->record.guid);

	copyargs->copymsg[copyargs->nummsg].system_flags = im->record.system_flags;
	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if ((userflag & 31) == 0) {
	flagmask = im->record.user_flags[userflag/32];
	}
	if (mailbox->flagname[userflag] && (flagmask & (1<<(userflag&31)))) {
	copyargs->copymsg[copyargs->nummsg].flag[flag++] =
	mailbox->flagname[userflag];
	}
	}
	copyargs->copymsg[copyargs->nummsg].flag[flag] = 0;

	/* grab seen from our state - it's different for different users */
	copyargs->copymsg[copyargs->nummsg].seen = im->isseen;

	copyargs->nummsg++;

	return 0;
	}

	/*
	* Creates a list of msgdata.
	*
	* We fill these structs with the processed info that will be needed
	* by the specified sort criteria.
	*/
	#define ANNOTGROWSIZE 10

	static MsgData index_msgdata_load(struct index_state state,
	unsigned *msgno_list, int n,
	struct sortcrit *sortcrit)
	{
	MsgData md, cur;
	int i, j;
	char *tmpenv;
	char *envtokens[NUMENVTOKENS];
	int did_cache, did_env;
	int label;
	int annotsize;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im;

	if (!n) return NULL;

	/* create an array of MsgData to use as nodes of linked list */
	md = (MsgData ) xmalloc(n sizeof(MsgData));
	memset(md, 0, n * sizeof(MsgData));

	for (i = 0, cur = md; i < n; i++, cur = cur->next) {
	/* set msgno */
	cur->msgno = msgno_list[i];
	im = &state->map[cur->msgno-1];
	cur->uid = im->record.uid;

	/* set pointer to next node */
	cur->next = (i+1 < n ? cur+1 : NULL);

	did_cache = did_env = 0;
	tmpenv = NULL;
	annotsize = 0;

	for (j = 0; sortcrit[j].key; j++) {
	label = sortcrit[j].key;

	if ((label == SORT_CC \|\|
	label == SORT_FROM \|\| label == SORT_SUBJECT \|\|
	label == SORT_TO \|\| label == LOAD_IDS) &&
	!did_cache) {

	/* fetch cached info */
	if (mailbox_cacherecord(mailbox, &im->record))
	continue; /* can't do this with a broken cache */

	did_cache++;
	}

	if ((label == LOAD_IDS) && !did_env) {
	/* make a working copy of envelope -- strip outer ()'s */
	/* +1 -> skip the leading paren */
	/* -2 -> don't include the size of the outer parens */
	if (cacheitem_size(&im->record, CACHE_ENVELOPE) > 2)
	tmpenv = xstrndup(cacheitem_base(&im->record, CACHE_ENVELOPE) + 1,
	cacheitem_size(&im->record, CACHE_ENVELOPE) - 2);
	else
	tmpenv = xstrdup("");

	/* parse envelope into tokens */
	parse_cached_envelope(tmpenv, envtokens,
	VECTOR_SIZE(envtokens));

	did_env++;
	}

	switch (label) {
	case SORT_CC:
	cur->cc = get_localpart_addr(cacheitem_base(&im->record, CACHE_CC));
	break;
	case SORT_DATE:
	cur->date = im->record.gmtime;
	/* fall through */
	case SORT_ARRIVAL:
	cur->internaldate = im->record.internaldate;
	break;
	case SORT_FROM:
	cur->from = get_localpart_addr(cacheitem_base(&im->record, CACHE_FROM));
	break;
	case SORT_MODSEQ:
	cur->modseq = im->record.modseq;
	break;
	case SORT_SIZE:
	cur->size = im->record.size;
	break;
	case SORT_SUBJECT:
	cur->xsubj = index_extract_subject(cacheitem_base(&im->record, CACHE_SUBJECT),
	cacheitem_size(&im->record, CACHE_SUBJECT),
	&cur->is_refwd);
	cur->xsubj_hash = strhash(cur->xsubj);
	break;
	case SORT_TO:
	cur->to = get_localpart_addr(cacheitem_base(&im->record, CACHE_TO));
	break;
	case SORT_ANNOTATION:
	/* reallocate space for the annotation values if necessary */
	if (cur->nannot == annotsize) {
	annotsize += ANNOTGROWSIZE;
	cur->annot = (char **)
	xrealloc(cur->annot, annotsize * sizeof(char *));
	}

	/* fetch attribute value - we fake it for now */
	cur->annot[cur->nannot] = xstrdup(sortcrit[j].args.annot.attrib);
	cur->nannot++;
	break;
	case LOAD_IDS:
	index_get_ids(cur, envtokens, cacheitem_base(&im->record, CACHE_HEADERS),
	cacheitem_size(&im->record, CACHE_HEADERS));
	break;
	}
	}

	if (tmpenv) free(tmpenv);
	}

	return md;
	}

	/*
	* Get the 'local-part' of an address from a header
	*/
	static char get_localpart_addr(const char header)
	{
	struct address *addr = NULL;
	char *ret;

	parseaddr_list(header, &addr);
	ret = xstrdup(addr && addr->mailbox ? addr->mailbox : "");
	parseaddr_free(addr);
	return ret;
	}

	/*
	* Extract base subject from subject header
	*
	* This is a wrapper around _index_extract_subject() which preps the
	* subj NSTRING and checks for Netscape "[Fwd: ]".
	*/
	static char index_extract_subject(const char subj, size_t len, int *is_refwd)
	{
	char buf, s, *base;

	/* parse the subj NSTRING and make a working copy */
	if (!strcmp(subj, "NIL")) { /* NIL? */
	return xstrdup(""); /* yes, return empty */
	} else if (subj == '"') { / quoted? */
	buf = xstrndup(subj + 1, len - 2); /* yes, strip quotes */
	} else {
	s = strchr(subj, '}') + 3; /* literal, skip { }\r\n */
	buf = xstrndup(s, len - (s - subj));
	}

	for (s = buf;;) {
	base = _index_extract_subject(s, is_refwd);

	/* If we have a Netscape "[Fwd: ...]", extract the contents */
	if (!strncasecmp(base, "[fwd:", 5) &&
	base[strlen(base) - 1] == ']') {

	/* inc refwd counter */
	*is_refwd += 1;

	/* trim "]" */
	base[strlen(base) - 1] = '\0';

	/* trim "[fwd:" */
	s = base + 5;
	}
	else /* otherwise, we're done */
	break;
	}

	base = xstrdup(base);

	free(buf);

	return base;
	}

	/*
	* Guts if subject extraction.
	*
	* Takes a subject string and returns a pointer to the base.
	*/
	static char _index_extract_subject(char s, int *is_refwd)
	{
	char base, x;

	/* trim trailer
	*
	* start at the end of the string and work towards the front,
	* resetting the end of the string as we go.
	*/
	for (x = s + strlen(s) - 1; x >= s;) {
	if (Uisspace(x)) { / whitespace? */
	x = '\0'; / yes, trim it */
	x--; /* skip past it */
	}
	else if (x - s >= 4 &&
	!strncasecmp(x-4, "(fwd)", 5)) { /* "(fwd)"? */
	(x-4) = '\0'; / yes, trim it */
	x -= 5; /* skip past it */
	is_refwd += 1; / inc refwd counter */
	}
	else
	break; /* we're done */
	}

	/* trim leader
	*
	* start at the head of the string and work towards the end,
	* skipping over stuff we don't care about.
	*/
	for (base = s; base;) {
	if (Uisspace(base)) base++; / whitespace? */

	/* possible refwd */
	else if ((!strncasecmp(base, "re", 2) && /* "re"? */
	(x = base + 2)) \|\| /* yes, skip past it */
	(!strncasecmp(base, "fwd", 3) && /* "fwd"? */
	(x = base + 3)) \|\| /* yes, skip past it */
	(!strncasecmp(base, "fw", 2) && /* "fw"? */
	(x = base + 2))) { /* yes, skip past it */
	int count = 0; /* init counter */

	while (Uisspace(x)) x++; / skip whitespace */

	if (x == '[') { / start of blob? */
	for (x++; x;) { /* yes, get count */
	if (!x) { / end of subj, quit */
	x = NULL;
	break;
	}
	else if (x == ']') { / end of blob, done */
	break;
	/* if we have a digit, and we're still
	counting, keep building the count */
	} else if (cyrus_isdigit((int) *x) && count != -1) {
	count = count * 10 + *x - '0';
	if (count < 0) { /* overflow */
	count = -1; /* abort counting */
	}
	} else { /* no digit, */
	count = -1; /* abort counting */
	}
	x++;
	}

	if (x) /* end of blob? */
	x++; /* yes, skip past it */
	else
	break; /* no, we're done */
	}

	while (Uisspace(x)) x++; / skip whitespace */

	if (x == ':') { / ending colon? */
	base = x + 1; /* yes, skip past it */
	is_refwd += (count > 0 ? count : 1); / inc refwd counter
	by count or 1 */
	}
	else
	break; /* no, we're done */
	}

	#if 0 /* do nested blobs - wait for decision on this */
	else if (base == '[') { / start of blob? */
	int count = 1; /* yes, */
	x = base + 1; /* find end of blob */
	while (count) { /* find matching ']' */
	if (!x) { / end of subj, quit */
	x = NULL;
	break;
	}
	else if (x == '[') / new open */
	count++; /* inc counter */
	else if (x == ']') / close */
	count--; /* dec counter */
	x++;
	}

	if (!x) /* blob didn't close */
	break; /* so quit */

	else if (x) / end of subj? */
	base = x; /* no, skip blob */
	#else
	else if (base == '[' && / start of blob? */
	(x = strpbrk(base+1, "[]")) && /* yes, end of blob */
	x == ']') { / (w/o nesting)? */

	if ((x+1)) / yes, end of subj? */
	base = x + 1; /* no, skip blob */
	#endif
	else
	break; /* yes, return blob */
	}
	else
	break; /* we're done */
	}

	return base;
	}

	/* Find a message-id looking thingy in a string. Returns a pointer to the
	* alloc'd id and the remaining string is returned in the **loc parameter.
	*
	* This is a poor-man's way of finding the message-id. We simply look for
	* any string having the format "< ... @ ... >" and assume that the mail
	* client created a properly formatted message-id.
	*/
	#define MSGID_SPECIALS "<> @\\"

	static char find_msgid(char str, char **rem)
	{
	char msgid, src, dst, cp;

	if (!str) return NULL;

	msgid = NULL;
	src = str;

	/* find the start of a msgid (don't go past the end of the header) */
	while ((cp = src = strpbrk(src, "<\r")) != NULL) {

	/* check for fold or end of header
	*
	* Per RFC 2822 section 2.2.3, a long header may be folded by
	* inserting CRLF before any WSP (SP and HTAB, per section 2.2.2).
	* Any other CRLF is the end of the header.
	*/
	if (*cp++ == '\r') {
	if (cp++ == '\n' && !(cp == ' ' \|\| *cp == '\t')) {
	/* end of header, we're done */
	break;
	}

	/* skip fold (or junk) */
	src++;
	continue;
	}

	/* see if we have (and skip) a quoted localpart */
	if (*cp == '\"') {
	/* find the endquote, making sure it isn't escaped */
	do {
	++cp; cp = strchr(cp, '\"');
	} while (cp && *(cp-1) == '\\');

	/* no endquote, so bail */
	if (!cp) {
	src++;
	continue;
	}
	}

	/* find the end of the msgid */
	if ((cp = strchr(cp, '>')) == NULL)
	return NULL;

	/* alloc space for the msgid */
	dst = msgid = (char*) xrealloc(msgid, cp - src + 2);

	dst++ = src++;

	/* quoted string */
	if (*src == '\"') {
	src++;
	while (*src != '\"') {
	if (*src == '\\') {
	src++;
	}
	dst++ = src++;
	}
	src++;
	}
	/* atom */
	else {
	while (!strchr(MSGID_SPECIALS, *src))
	dst++ = src++;
	}

	if (src != '@' \|\| (dst-1) == '<') continue;
	dst++ = src++;

	/* domain atom */
	while (!strchr(MSGID_SPECIALS, *src))
	dst++ = src++;

	if (src != '>' \|\| (dst-1) == '@') continue;
	dst++ = src++;
	*dst = '\0';

	if (rem) *rem = src;
	return msgid;
	}

	if (msgid) free(msgid);
	return NULL;
	}

	/* Get message-id, and references/in-reply-to */
	#define REFGROWSIZE 20

	void index_get_ids(MsgData msgdata, char envtokens[], const char *headers,
	unsigned size)
	{
	static char *buf;
	static unsigned bufsize;
	static struct strlist refhdr;
	char refstr, ref, *in_reply_to;
	int refsize = REFGROWSIZE;

	if (bufsize < size+2) {
	bufsize = size+100;
	buf = xrealloc(buf, bufsize);
	}

	/* get msgid */
	msgdata->msgid = find_msgid(envtokens[ENV_MSGID], NULL);
	/* if we don't have one, create one */
	if (!msgdata->msgid) {
	snprintf(buf, bufsize, "<Empty-ID: %u>", msgdata->msgno);
	msgdata->msgid = xstrdup(buf);
	}

	/* Copy headers to the buffer */
	memcpy(buf, headers, size);
	buf[size] = '\0';

	/* grab the References header */
	refhdr.s = "references";
	index_pruneheader(buf, &refhdr, 0);
	if (*buf) {
	/* allocate some space for refs */
	msgdata->ref = (char *) xmalloc(refsize sizeof(char *));
	/* find references */
	refstr = buf;
	while ((ref = find_msgid(refstr, &refstr)) != NULL) {
	/* reallocate space for this msgid if necessary */
	if (msgdata->nref == refsize) {
	refsize += REFGROWSIZE;
	msgdata->ref = (char **)
	xrealloc(msgdata->ref, refsize * sizeof(char *));
	}
	/* store this msgid in the array */
	msgdata->ref[msgdata->nref++] = ref;
	}
	}

	/* if we have no references, try in-reply-to */
	if (!msgdata->nref) {
	/* get in-reply-to id */
	in_reply_to = find_msgid(envtokens[ENV_INREPLYTO], NULL);
	/* if we have an in-reply-to id, make it the ref */
	if (in_reply_to) {
	msgdata->ref = (char *) xmalloc(sizeof(char ));
	msgdata->ref[msgdata->nref++] = in_reply_to;
	}
	}
	}

	/*
	* Getnext function for sorting message lists.
	*/
	static void index_sort_getnext(MsgData node)
	{
	return node->next;
	}

	/*
	* Setnext function for sorting message lists.
	*/
	static void index_sort_setnext(MsgData node, MsgData next)
	{
	node->next = next;
	}

	/*
	* Function for comparing two integers.
	*/
	static int numcmp(modseq_t n1, modseq_t n2)
	{
	return ((n1 < n2) ? -1 : (n1 > n2) ? 1 : 0);
	}

	/*
	* Comparison function for sorting message lists.
	*/
	static int index_sort_compare(MsgData md1, MsgData md2,
	struct sortcrit *sortcrit)
	{
	int reverse, ret = 0, i = 0, ann = 0;

	do {
	/* determine sort order from reverse flag bit */
	reverse = sortcrit[i].flags & SORT_REVERSE;

	switch (sortcrit[i].key) {
	case SORT_SEQUENCE:
	ret = numcmp(md1->msgno, md2->msgno);
	break;
	case SORT_ARRIVAL:
	ret = numcmp(md1->internaldate, md2->internaldate);
	break;
	case SORT_CC:
	ret = strcmp(md1->cc, md2->cc);
	break;
	case SORT_DATE: {
	time_t d1 = md1->date ? md1->date : md1->internaldate;
	time_t d2 = md2->date ? md2->date : md2->internaldate;
	ret = numcmp(d1, d2);
	break;
	}
	case SORT_FROM:
	ret = strcmp(md1->from, md2->from);
	break;
	case SORT_SIZE:
	ret = numcmp(md1->size, md2->size);
	break;
	case SORT_SUBJECT:
	ret = strcmp(md1->xsubj, md2->xsubj);
	break;
	case SORT_TO:
	ret = strcmp(md1->to, md2->to);
	break;
	case SORT_ANNOTATION:
	ret = strcmp(md1->annot[ann], md2->annot[ann]);
	ann++;
	break;
	case SORT_MODSEQ:
	ret = numcmp(md1->modseq, md2->modseq);
	break;
	}
	} while (!ret && sortcrit[i++].key != SORT_SEQUENCE);

	return (reverse ? -ret : ret);
	}

	/*
	* Free a msgdata node.
	*/
	static void index_msgdata_free(MsgData *md)
	{
	#define FREE(x) if (x) free(x)
	int i;

	if (!md)
	return;
	FREE(md->cc);
	FREE(md->from);
	FREE(md->to);
	FREE(md->xsubj);
	FREE(md->msgid);
	for (i = 0; i < md->nref; i++)
	free(md->ref[i]);
	FREE(md->ref);
	for (i = 0; i < md->nannot; i++)
	free(md->annot[i]);
	FREE(md->annot);
	}

	/*
	* Getnext function for sorting thread lists.
	*/
	static void index_thread_getnext(Thread thread)
	{
	return thread->next;
	}

	/*
	* Setnext function for sorting thread lists.
	*/
	static void index_thread_setnext(Thread thread, Thread next)
	{
	thread->next = next;
	}

	/*
	* Comparison function for sorting threads.
	*/
	static int index_thread_compare(Thread t1, Thread t2,
	struct sortcrit *call_data)
	{
	MsgData md1, md2;

	/* if the container is empty, use the first child's container */
	md1 = t1->msgdata ? t1->msgdata : t1->child->msgdata;
	md2 = t2->msgdata ? t2->msgdata : t2->child->msgdata;
	return index_sort_compare(md1, md2, call_data);
	}

	/*
	* Sort a list of threads.
	*/
	static void index_thread_sort(Thread root, struct sortcrit sortcrit)
	{
	Thread *child;

	/* sort the grandchildren */
	child = root->child;
	while (child) {
	/* if the child has children, sort them */
	if (child->child)
	index_thread_sort(child, sortcrit);
	child = child->next;
	}

	/* sort the children */
	root->child = lsort(root->child,
	(void * ()(void)) index_thread_getnext,
	(void ()(void,void*)) index_thread_setnext,
	(int ()(void,void,void)) index_thread_compare,
	sortcrit);
	}

	/*
	* Thread a list of messages using the ORDEREDSUBJECT algorithm.
	*/
	static void index_thread_orderedsubj(struct index_state *state,
	unsigned *msgno_list, int nmsg,
	int usinguid)
	{
	MsgData msgdata, freeme;
	struct sortcrit sortcrit[] = {{ SORT_SUBJECT, 0, {{NULL, NULL}} },
	{ SORT_DATE, 0, {{NULL, NULL}} },
	{ SORT_SEQUENCE, 0, {{NULL, NULL}} }};
	unsigned psubj_hash = 0;
	char *psubj;
	Thread head, newnode, cur, parent, *last;

	/* Create/load the msgdata array */
	freeme = msgdata = index_msgdata_load(state, msgno_list, nmsg, sortcrit);

	/* Sort messages by subject and date */
	msgdata = lsort(msgdata,
	(void * ()(void)) index_sort_getnext,
	(void ()(void,void*)) index_sort_setnext,
	(int ()(void,void,void)) index_sort_compare,
	sortcrit);

	/* create an array of Thread to use as nodes of thread tree
	*
	* we will be building threads under a dummy head,
	* so we need (nmsg + 1) nodes
	*/
	head = (Thread ) xmalloc((nmsg + 1) sizeof(Thread));
	memset(head, 0, (nmsg + 1) * sizeof(Thread));

	newnode = head + 1; /* set next newnode to the second
	one in the array (skip the head) */
	parent = head; /* parent is the head node */
	psubj = NULL; /* no previous subject */
	cur = NULL; /* no current thread */
	last = NULL; /* no last child */

	while (msgdata) {
	newnode->msgdata = msgdata;

	/* if no previous subj, or
	current subj = prev subj (subjs have same hash, and
	the strings are equal), then add message to current thread */
	if (!psubj \|\|
	(msgdata->xsubj_hash == psubj_hash &&
	!strcmp(msgdata->xsubj, psubj))) {
	/* if no children, create first child */
	if (!parent->child) {
	last = parent->child = newnode;
	if (!cur) /* first thread */
	parent = cur = parent->child;
	}
	/* otherwise, add to siblings */
	else {
	last->next = newnode;
	last = last->next;
	}
	}
	/* otherwise, create a new thread */
	else {
	cur->next = newnode; /* create and start a new thread */
	parent = cur = cur->next; /* now work with the new thread */
	}

	psubj_hash = msgdata->xsubj_hash;
	psubj = msgdata->xsubj;
	msgdata = msgdata->next;
	newnode++;
	}

	/* Sort threads by date */
	index_thread_sort(head, sortcrit+1);

	/* Output the threaded messages */
	index_thread_print(state, head, usinguid);

	/* free the thread array */
	free(head);

	/* free the msgdata array */
	free(freeme);
	}

	/*
	* Guts of thread printing. Recurses over children when necessary.
	*
	* Frees contents of msgdata as a side effect.
	*/
	static void _index_thread_print(struct index_state *state,
	Thread *thread, int usinguid)
	{
	Thread *child;

	/* for each thread... */
	while (thread) {
	/* start the thread */
	prot_printf(state->out, "(");

	/* if we have a message, print its identifier
	* (do nothing for empty containers)
	*/
	if (thread->msgdata) {
	prot_printf(state->out, "%u",
	usinguid ? thread->msgdata->uid :
	thread->msgdata->msgno);

	/* if we have a child, print the parent-child separator */
	if (thread->child) prot_printf(state->out, " ");

	/* free contents of the current node */
	index_msgdata_free(thread->msgdata);
	}

	/* for each child, grandchild, etc... */
	child = thread->child;
	while (child) {
	/* if the child has siblings, print new branch and break */
	if (child->next) {
	_index_thread_print(state, child, usinguid);
	break;
	}
	/* otherwise print the only child */
	else {
	prot_printf(state->out, "%u",
	usinguid ? child->msgdata->uid :
	child->msgdata->msgno);

	/* if we have a child, print the parent-child separator */
	if (child->child) prot_printf(state->out, " ");

	/* free contents of the child node */
	index_msgdata_free(child->msgdata);

	child = child->child;
	}
	}

	/* end the thread */
	prot_printf(state->out, ")");

	thread = thread->next;
	}
	}

	/*
	* Print a list of threads.
	*
	* This is a wrapper around _index_thread_print() which simply prints the
	* start and end of the untagged thread response.
	*/
	static void index_thread_print(struct index_state *state,
	Thread *thread, int usinguid)
	{
	prot_printf(state->out, "* THREAD");

	if (thread) {
	prot_printf(state->out, " ");
	_index_thread_print(state, thread->child, usinguid);
	}
	}

	/*
	* Find threading algorithm for given arg.
	* Returns index into thread_algs[], or -1 if not found.
	*/
	int find_thread_algorithm(char *arg)
	{
	int alg;

	ucase(arg);
	for (alg = 0; thread_algs[alg].alg_name; alg++) {
	if (!strcmp(arg, thread_algs[alg].alg_name))
	return alg;
	}
	return -1;
	}

	/*
	* The following code is an interpretation of JWZ's description
	* and pseudo-code in http://www.jwz.org/doc/threading.html.
	*
	* It has been modified to match the THREAD=REFERENCES algorithm.
	*/

	/*
	* Determines if child is a descendent of parent.
	*
	* Returns 1 if yes, 0 otherwise.
	*/
	static int thread_is_descendent(Thread parent, Thread child)
	{
	Thread *kid;

	/* self */
	if (parent == child)
	return 1;

	/* search each child's decendents */
	for (kid = parent->child; kid; kid = kid->next) {
	if (thread_is_descendent(kid, child))
	return 1;
	}
	return 0;
	}

	/*
	* Links child into parent's children.
	*/
	static void thread_adopt_child(Thread parent, Thread child)
	{
	child->parent = parent;
	child->next = parent->child;
	parent->child = child;
	}

	/*
	* Unlinks child from it's parent's children.
	*/
	static void thread_orphan_child(Thread *child)
	{
	Thread prev, cur;

	/* sanity check -- make sure child is actually a child of parent */
	for (prev = NULL, cur = child->parent->child;
	cur != child && cur != NULL; prev = cur, cur = cur->next);

	if (!cur) {
	/* uh oh! couldn't find the child in it's parent's children
	* we should probably return NO to thread command
	*/
	return;
	}

	/* unlink child */
	if (!prev) /* first child */
	child->parent->child = child->next;
	else
	prev->next = child->next;
	child->parent = child->next = NULL;
	}

	/*
	* Link messages together using message-id and references.
	*/
	static void ref_link_messages(MsgData msgdata, Thread *newnode,
	struct hash_table *id_table)
	{
	Thread cur, parent, *ref;
	int dup_count = 0;
	char buf[100];
	int i;

	/* for each message... */
	while (msgdata) {
	/* fill the containers with msgdata
	*
	* if we already have a container, use it
	*/
	if ((cur = (Thread *) hash_lookup(msgdata->msgid, id_table))) {
	/* If this container is not empty, then we have a duplicate
	* Message-ID. Make this one unique so that we don't stomp
	* on the old one.
	*/
	if (cur->msgdata) {
	snprintf(buf, sizeof(buf), "-dup%d", dup_count++);
	msgdata->msgid =
	(char *) xrealloc(msgdata->msgid,
	strlen(msgdata->msgid) + strlen(buf) + 1);
	strcat(msgdata->msgid, buf);
	/* clear cur so that we create a new container */
	cur = NULL;
	}
	else
	cur->msgdata = msgdata;
	}

	/* otherwise, make and index a new container */
	if (!cur) {
	cur = *newnode;
	cur->msgdata = msgdata;
	hash_insert(msgdata->msgid, cur, id_table);
	(*newnode)++;
	}

	/* Step 1.A */
	for (i = 0, parent = NULL; i < msgdata->nref; i++) {
	/* if we don't already have a container for the reference,
	* make and index a new (empty) container
	*/
	if (!(ref = (Thread *) hash_lookup(msgdata->ref[i], id_table))) {
	ref = *newnode;
	hash_insert(msgdata->ref[i], ref, id_table);
	(*newnode)++;
	}

	/* link the references together as parent-child iff:
	* - we won't change existing links, AND
	* - we won't create a loop
	*/
	if (!ref->parent &&
	parent && !thread_is_descendent(ref, parent)) {
	thread_adopt_child(parent, ref);
	}

	parent = ref;
	}

	/* Step 1.B
	*
	* if we have a parent already, it is probably bogus (the result
	* of a truncated references field), so unlink from it because
	* we now have the actual parent
	*/
	if (cur->parent) thread_orphan_child(cur);

	/* make the last reference the parent of our message iff:
	* - we won't create a loop
	*/
	if (parent && !thread_is_descendent(cur, parent))
	thread_adopt_child(parent, cur);

	msgdata = msgdata->next;
	}
	}

	/*
	* Gather orphan messages under the root node.
	*/
	static void ref_gather_orphans(char *key __attribute__((unused)),
	Thread *node,
	struct rootset *rootset)
	{
	/* we only care about nodes without parents */
	if (!node->parent) {
	if (node->next) {
	/* uh oh! a node without a parent should not have a sibling
	* we should probably return NO to thread command
	*/
	return;
	}

	/* add this node to root's children */
	node->next = rootset->root->child;
	rootset->root->child = node;
	rootset->nroot++;
	}
	}

	/*
	* Prune tree of empty containers.
	*/
	static void ref_prune_tree(Thread *parent)
	{
	Thread cur, prev, next, child;

	for (prev = NULL, cur = parent->child, next = cur->next;
	cur;
	prev = cur, cur = next, next = (cur ? cur->next : NULL)) {

	/* if we have an empty container with no children, delete it */
	if (!cur->msgdata && !cur->child) {
	if (!prev) /* first child */
	parent->child = cur->next;
	else
	prev->next = cur->next;

	/* we just removed cur from our list,
	* so we need to keep the same prev for the next pass
	*/
	cur = prev;
	}

	/* if we have an empty container with children, AND
	* we're not at the root OR we only have one child,
	* then remove the container but promote its children to this level
	* (splice them into the current child list)
	*/
	else if (!cur->msgdata && cur->child &&
	(cur->parent \|\| !cur->child->next)) {
	/* move cur's children into cur's place (start the splice) */
	if (!prev) /* first child */
	parent->child = cur->child;
	else
	prev->next = cur->child;

	/* make cur's parent the new parent of cur's children
	* (they're moving in with grandma!)
	*/
	child = cur->child;
	do {
	child->parent = cur->parent;
	} while (child->next && (child = child->next));

	/* make the cur's last child point to cur's next sibling
	* (finish the splice)
	*/
	child->next = cur->next;

	/* we just replaced cur with it's children
	* so make it's first child the next node to process
	*/
	next = cur->child;

	/* make cur childless and siblingless */
	cur->child = cur->next = NULL;

	/* we just removed cur from our list,
	* so we need to keep the same prev for the next pass
	*/
	cur = prev;
	}

	/* if we have a message with children, prune it's children */
	else if (cur->child)
	ref_prune_tree(cur);
	}
	}

	/*
	* Sort the messages in the root set by date.
	*/
	static void ref_sort_root(Thread *root)
	{
	Thread *cur;
	struct sortcrit sortcrit[] = {{ SORT_DATE, 0, {{NULL, NULL}} },
	{ SORT_SEQUENCE, 0, {{NULL, NULL}} }};

	cur = root->child;
	while (cur) {
	/* if the message is a dummy, sort its children */
	if (!cur->msgdata) {
	cur->child = lsort(cur->child,
	(void * ()(void)) index_thread_getnext,
	(void ()(void,void*)) index_thread_setnext,
	(int ()(void,void,void)) index_thread_compare,
	sortcrit);
	}
	cur = cur->next;
	}

	/* sort the root set */
	root->child = lsort(root->child,
	(void * ()(void)) index_thread_getnext,
	(void ()(void,void*)) index_thread_setnext,
	(int ()(void,void,void)) index_thread_compare,
	sortcrit);
	}

	/*
	* Group threads with same subject.
	*/
	static void ref_group_subjects(Thread root, unsigned nroot, Thread *newnode)
	{
	Thread cur, old, prev, next, *child;
	struct hash_table subj_table;
	char *subj;

	/* Step 5.A: create a subj_table with one bucket for every possible
	* subject in the root set
	*/
	construct_hash_table(&subj_table, nroot, 1);

	/* Step 5.B: populate the table with a container for each subject
	* at the root
	*/
	for (cur = root->child; cur; cur = cur->next) {
	/* Step 5.B.i: find subject of the thread
	*
	* if the container is not empty, use it's subject
	*/
	if (cur->msgdata)
	subj = cur->msgdata->xsubj;
	/* otherwise, use the subject of it's first child */
	else
	subj = cur->child->msgdata->xsubj;

	/* Step 5.B.ii: if subject is empty, skip it */
	if (!strlen(subj)) continue;

	/* Step 5.B.iii: lookup this subject in the table */
	old = (Thread *) hash_lookup(subj, &subj_table);

	/* Step 5.B.iv: insert the current container into the table iff:
	* - this subject is not in the table, OR
	* - this container is empty AND the one in the table is not
	* (the empty one is more interesting as a root), OR
	* - the container in the table is a re/fwd AND this one is not
	* (the non-re/fwd is the more interesting of the two)
	*/
	if (!old \|\|
	(!cur->msgdata && old->msgdata) \|\|
	(old->msgdata && old->msgdata->is_refwd &&
	cur->msgdata && !cur->msgdata->is_refwd)) {
	hash_insert(subj, cur, &subj_table);
	}
	}

	/* 5.C - group containers with the same subject together */
	for (prev = NULL, cur = root->child, next = cur->next;
	cur;
	prev = cur, cur = next, next = (next ? next->next : NULL)) {
	/* Step 5.C.i: find subject of the thread
	*
	* if container is not empty, use it's subject
	*/
	if (cur->msgdata)
	subj = cur->msgdata->xsubj;
	/* otherwise, use the subject of it's first child */
	else
	subj = cur->child->msgdata->xsubj;

	/* Step 5.C.ii: if subject is empty, skip it */
	if (!strlen(subj)) continue;

	/* Step 5.C.iii: lookup this subject in the table */
	old = (Thread *) hash_lookup(subj, &subj_table);

	/* Step 5.C.iv: if we found ourselves, skip it */
	if (!old \|\| old == cur) continue;

	/* ok, we already have a container which contains our current subject,
	* so pull this container out of the root set, because we are going to
	* merge this node with another one
	*/
	if (!prev) /* we're at the root */
	root->child = cur->next;
	else
	prev->next = cur->next;
	cur->next = NULL;

	/* if both containers are dummies, append cur's children to old's */
	if (!old->msgdata && !cur->msgdata) {
	/* find old's last child */
	for (child = old->child; child->next; child = child->next);

	/* append cur's children to old's children list */
	child->next = cur->child;

	/* make old the parent of cur's children */
	for (child = cur->child; child; child = child->next)
	child->parent = old;

	/* make cur childless */
	cur->child = NULL;
	}

	/* if:
	* - old container is empty, OR
	* - the current message is a re/fwd AND the old one is not,
	* make the current container a child of the old one
	*
	* Note: we don't have to worry about the reverse cases
	* because step 5.B guarantees that they won't happen
	*/
	else if (!old->msgdata \|\|
	(cur->msgdata && cur->msgdata->is_refwd &&
	!old->msgdata->is_refwd)) {
	thread_adopt_child(old, cur);
	}

	/* if both messages are re/fwds OR neither are re/fwds,
	* then make them both children of a new dummy container
	* (we don't want to assume any parent-child relationship between them)
	*
	* perhaps we can create a parent-child relationship
	* between re/fwds by counting the number of re/fwds
	*
	* Note: we need the hash table to still point to old,
	* so we must make old the dummy and make the contents of the
	* new container a copy of old's original contents
	*/
	else {
	Thread new = (newnode)++;

	/* make new a copy of old (except parent and next) */
	new->msgdata = old->msgdata;
	new->child = old->child;
	new->next = NULL;

	/* make new the parent of it's newly adopted children */
	for (child = new->child; child; child = child->next)
	child->parent = new;

	/* make old the parent of cur and new */
	cur->parent = old;
	new->parent = old;

	/* empty old and make it have two children (cur and new) */
	old->msgdata = NULL;
	old->child = cur;
	cur->next = new;
	}

	/* we just removed cur from our list,
	* so we need to keep the same prev for the next pass
	*/
	cur = prev;
	}

	free_hash_table(&subj_table, NULL);
	}

	/*
	* Free an entire thread.
	*/
	static void index_thread_free(Thread *thread)
	{
	Thread *child;

	/* free the head node */
	if (thread->msgdata) index_msgdata_free(thread->msgdata);

	/* free the children recursively */
	child = thread->child;
	while (child) {
	index_thread_free(child);
	child = child->next;
	}
	}

	/*
	* Guts of thread searching. Recurses over children when necessary.
	*/
	static int _index_thread_search(struct index_state *state,
	Thread thread, int (searchproc) (MsgData *))
	{
	Thread *child;

	/* test the head node */
	if (thread->msgdata && searchproc(thread->msgdata)) return 1;

	/* test the children recursively */
	child = thread->child;
	while (child) {
	if (_index_thread_search(state, child, searchproc)) return 1;
	child = child->next;
	}

	/* if we get here, we struck out */
	return 0;
	}

	/*
	* Search a thread to see if it contains a message which matches searchproc().
	*
	* This is a wrapper around _index_thread_search() which iterates through
	* each thread and removes any which fail the searchproc().
	*/
	static void index_thread_search(struct index_state *state,
	Thread root, int (searchproc) (MsgData *))
	{
	Thread cur, prev, *next;

	for (prev = NULL, cur = root->child, next = cur->next;
	cur;
	prev = cur, cur= next, next = (cur ? cur->next : NULL)) {
	if (!_index_thread_search(state, cur, searchproc)) {
	/* unlink the thread from the list */
	if (!prev) /* first thread */
	root->child = cur->next;
	else
	prev->next = cur->next;

	/* free all nodes in the thread */
	index_thread_free(cur);

	/* we just removed cur from our list,
	* so we need to keep the same prev for the next pass
	*/
	cur = prev;
	}
	}
	}

	/*
	* Guts of the REFERENCES algorithms. Behavior is tweaked with loadcrit[],
	* searchproc() and sortcrit[].
	*/
	static void _index_thread_ref(struct index_state state, unsigned msgno_list, int nmsg,
	struct sortcrit loadcrit[],
	int (searchproc) (MsgData ),
	struct sortcrit sortcrit[], int usinguid)
	{
	MsgData msgdata, freeme, *md;
	int tref, nnode;
	Thread *newnode;
	struct hash_table id_table;
	struct rootset rootset;

	/* Create/load the msgdata array */
	freeme = msgdata = index_msgdata_load(state, msgno_list, nmsg, loadcrit);

	/* calculate the sum of the number of references for all messages */
	for (md = msgdata, tref = 0; md; md = md->next)
	tref += md->nref;

	/* create an array of Thread to use as nodes of thread tree (including
	* empty containers)
	*
	* - We will be building threads under a dummy root, so we need at least
	* (nmsg + 1) nodes.
	* - We also will need containers for references to non-existent messages.
	* To make sure we have enough, we will take the worst case and
	* use the sum of the number of references for all messages.
	* - Finally, we will need containers to group threads with the same
	* subject together. To make sure we have enough, we will take the
	* worst case which will be half of the number of messages.
	*
	* This is overkill, but it is the only way to make sure we have enough
	* ahead of time. If we tried to use xrealloc(), the array might be moved,
	* and our parent/child/next pointers will no longer be correct
	* (been there, done that).
	*/
	nnode = (int) (1.5 * nmsg + 1 + tref);
	rootset.root = (Thread ) xmalloc(nnode sizeof(Thread));
	memset(rootset.root, 0, nnode * sizeof(Thread));

	newnode = rootset.root + 1; /* set next newnode to the second
	one in the array (skip the root) */

	/* Step 0: create an id_table with one bucket for every possible
	* message-id and reference (nmsg + tref)
	*/
	construct_hash_table(&id_table, nmsg + tref, 1);

	/* Step 1: link messages together */
	ref_link_messages(msgdata, &newnode, &id_table);

	/* Step 2: find the root set (gather all of the orphan messages) */
	rootset.nroot = 0;
	hash_enumerate(&id_table, (void ()(char,void,void)) ref_gather_orphans,
	&rootset);

	/* discard id_table */
	free_hash_table(&id_table, NULL);

	/* Step 3: prune tree of empty containers - get our deposit back :^) */
	ref_prune_tree(rootset.root);

	/* Step 4: sort the root set */
	ref_sort_root(rootset.root);

	/* Step 5: group root set by subject */
	ref_group_subjects(rootset.root, rootset.nroot, &newnode);

	/* Optionally search threads (to be used by REFERENCES derivatives) */
	if (searchproc) index_thread_search(state, rootset.root, searchproc);

	/* Step 6: sort threads */
	if (sortcrit) index_thread_sort(rootset.root, sortcrit);

	/* Output the threaded messages */
	index_thread_print(state, rootset.root, usinguid);

	/* free the thread array */
	free(rootset.root);

	/* free the msgdata array */
	free(freeme);
	}

	/*
	* Thread a list of messages using the REFERENCES algorithm.
	*/
	static void index_thread_ref(struct index_state state, unsigned msgno_list, int nmsg, int usinguid)
	{
	struct sortcrit loadcrit[] = {{ LOAD_IDS, 0, {{NULL,NULL}} },
	{ SORT_SUBJECT, 0, {{NULL,NULL}} },
	{ SORT_DATE, 0, {{NULL,NULL}} },
	{ SORT_SEQUENCE, 0, {{NULL,NULL}} }};
	struct sortcrit sortcrit[] = {{ SORT_DATE, 0, {{NULL,NULL}} },
	{ SORT_SEQUENCE, 0, {{NULL,NULL}} }};

	_index_thread_ref(state, msgno_list, nmsg, loadcrit, NULL, sortcrit, usinguid);
	}

	/*
	* NNTP specific stuff.
	*/
	char index_get_msgid(struct index_state state,
	uint32_t msgno)
	{
	char *env;
	char *envtokens[NUMENVTOKENS];
	char *msgid;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];

	if (mailbox_cacherecord(mailbox, &im->record))
	return NULL;

	if (cacheitem_size(&im->record, CACHE_ENVELOPE) <= 2)
	return NULL;

	/* get msgid out of the envelope
	*
	* get a working copy; strip outer ()'s
	* +1 -> skip the leading paren
	* -2 -> don't include the size of the outer parens
	*/
	env = xstrndup(cacheitem_base(&im->record, CACHE_ENVELOPE) + 1,
	cacheitem_size(&im->record, CACHE_ENVELOPE) - 2);
	parse_cached_envelope(env, envtokens, VECTOR_SIZE(envtokens));

	msgid = envtokens[ENV_MSGID] ? xstrdup(envtokens[ENV_MSGID]) : NULL;

	/* free stuff */
	free(env);

	return msgid;
	}

	static void massage_header(char *hdr)
	{
	int n = 0;
	char *p, c;

	for (p = hdr; *p; p++) {
	if (p == ' ' \|\| p == '\t' \|\| *p == '\r') {
	if (!n \|\| *(p+1) == '\n') {
	/* no leading or trailing whitespace */
	continue;
	}
	/* replace with space */
	c = ' ';
	}
	else if (*p == '\n') {
	if ((p+1) == ' ' \|\| (p+1) == '\t') {
	/* folded header */
	continue;
	}
	/* end of header */
	break;
	}
	else
	c = *p;

	hdr[n++] = c;
	}
	hdr[n] = '\0';
	}

	static char parse_nstring(char *str)
	{
	char cp = str, *val;

	if (cp == '"') { / quoted string */
	val = ++cp; /* skip " */
	do {
	cp = strchr(cp, '"');
	if (!cp) return NULL; /* whole thing is broken */
	} while ((cp-1) == '\\'); / skip escaped " */
	*cp++ = '\0';
	}
	else { /* NIL */
	val = NULL;
	cp += 3;
	}

	*str = cp;
	return val;
	}

	static void parse_env_address(char str, struct address addr)
	{
	str++; /* skip ( */
	addr->name = parse_nstring(&str);
	str++; /* skip SP */
	addr->route = parse_nstring(&str);
	str++; /* skip SP */
	addr->mailbox = parse_nstring(&str);
	str++; /* skip SP */
	addr->domain = parse_nstring(&str);
	}

	extern struct nntp_overview index_overview(struct index_state state,
	uint32_t msgno)
	{
	static struct nntp_overview over;
	static char env = NULL, from = NULL, *hdr = NULL;
	static int envsize = 0, fromsize = 0, hdrsize = 0;
	int size;
	char *envtokens[NUMENVTOKENS];
	struct address addr = { NULL, NULL, NULL, NULL, NULL, NULL };
	static struct strlist refhdr;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];

	if (mailbox_cacherecord(mailbox, &im->record))
	return NULL; /* upper layers can cope! */

	/* make a working copy of envelope; strip outer ()'s */
	/* -2 -> don't include the size of the outer parens */
	/* +1 -> leave space for NUL */
	size = cacheitem_size(&im->record, CACHE_ENVELOPE) - 2 + 1;
	if (envsize < size) {
	envsize = size;
	env = xrealloc(env, envsize);
	}
	/* +1 -> skip the leading paren */
	strlcpy(env, cacheitem_base(&im->record, CACHE_ENVELOPE) + 1, size);

	/* make a working copy of headers */
	size = cacheitem_size(&im->record, CACHE_HEADERS);
	if (hdrsize < size+2) {
	hdrsize = size+100;
	hdr = xrealloc(hdr, hdrsize);
	}
	memcpy(hdr, cacheitem_base(&im->record, CACHE_HEADERS), size);
	hdr[size] = '\0';

	parse_cached_envelope(env, envtokens, VECTOR_SIZE(envtokens));

	over.uid = im->record.uid;
	over.bytes = im->record.size;
	over.lines = index_getlines(state, msgno);
	over.date = envtokens[ENV_DATE];
	over.msgid = envtokens[ENV_MSGID];

	/* massage subject */
	if ((over.subj = envtokens[ENV_SUBJECT]))
	massage_header(over.subj);

	/* build original From: header */
	if (envtokens[ENV_FROM]) /* paranoia */
	parse_env_address(envtokens[ENV_FROM], &addr);

	if (addr.mailbox && addr.domain) { /* paranoia */
	/* +3 -> add space for quotes and space */
	/* +4 -> add space for < @ > NUL */
	size = (addr.name ? strlen(addr.name) + 3 : 0) +
	strlen(addr.mailbox) + strlen(addr.domain) + 4;
	if (fromsize < size) {
	fromsize = size;
	from = xrealloc(from, fromsize);
	}
	from[0] = '\0';
	if (addr.name) sprintf(from, "\"%s\" ", addr.name);
	snprintf(from + strlen(from), fromsize - strlen(from),
	"<%s@%s>", addr.mailbox, addr.domain);

	over.from = from;
	}
	else
	over.from = NULL;

	/* massage references */
	refhdr.s = "references";
	index_pruneheader(hdr, &refhdr, 0);
	if (*hdr) {
	over.ref = hdr + 11; /* skip over header name */
	massage_header(over.ref);
	}

	return &over;
	}

	extern char index_getheader(struct index_state state, uint32_t msgno,
	char *hdr)
	{
	static const char *msg_base = 0;
	static unsigned long msg_size = 0;
	struct strlist headers = { NULL, NULL, NULL, NULL };
	static char *alloc = NULL;
	static unsigned allocsize = 0;
	unsigned size;
	char *buf;
	struct mailbox *mailbox = state->mailbox;
	struct index_map *im = &state->map[msgno-1];

	headers.s = hdr;

	if (msg_base) {
	mailbox_unmap_message(NULL, 0, &msg_base, &msg_size);
	msg_base = 0;
	msg_size = 0;
	}

	/* see if the header is cached */
	if (mailbox_cached_header(hdr) != BIT32_MAX &&
	!mailbox_cacherecord(mailbox, &im->record)) {

	size = cacheitem_size(&im->record, CACHE_HEADERS);
	if (allocsize < size+2) {
	allocsize = size+100;
	alloc = xrealloc(alloc, allocsize);
	}

	memcpy(alloc, cacheitem_base(&im->record, CACHE_HEADERS), size);
	alloc[size] = '\0';

	buf = alloc;
	}
	else {
	/* uncached header */
	if (mailbox_map_message(mailbox, im->record.uid, &msg_base, &msg_size))
	return NULL;

	buf = index_readheader(msg_base, msg_size, 0, im->record.header_size);
	}

	index_pruneheader(buf, &headers, NULL);

	if (*buf) {
	buf += strlen(hdr) + 1; /* skip header: */
	massage_header(buf);
	}

	return buf;
	}

	extern unsigned long index_getsize(struct index_state *state,
	uint32_t msgno)
	{
	struct index_map *im = &state->map[msgno-1];
	return im->record.size;
	}

	extern unsigned long index_getlines(struct index_state *state, uint32_t msgno)
	{
	struct index_map *im = &state->map[msgno-1];
	return im->record.content_lines;
	}

	/*
	* Parse a sequence into an array of sorted & merged ranges.
	*/
	static struct seqset _parse_sequence(struct index_state state,
	const char *sequence, int usinguid)
	{
	unsigned maxval = usinguid ? state->last_uid : state->exists;
	return seqset_parse(sequence, NULL, maxval);
	}

	void appendsequencelist(struct index_state *state,
	struct seqset **l,
	char *sequence, int usinguid)
	{
	unsigned maxval = usinguid ? state->last_uid : state->exists;
	seqset_append(l, sequence, maxval);
	}

	void freesequencelist(struct seqset *l)
	{
	seqset_free(l);
	}
	diff --git a/imap/lmtpd.c b/imap/lmtpd.c
	index c8278dc77..73a07960e 100644
	--- a/imap/lmtpd.c
	+++ b/imap/lmtpd.c
	@@ -1,1148 +1,1148 @@
	/* lmtpd.c -- Program to deliver mail to a mailbox
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: lmtpd.c,v 1.167 2010/05/25 20:59:19 wescraig Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <signal.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <ctype.h>
	#include <fcntl.h>
	#include <sys/stat.h>
	#include <syslog.h>
	#include <errno.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <netdb.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <arpa/inet.h>
	#include <sasl/sasl.h>
	#include <sasl/saslutil.h>

	#include "acl.h"
	#include "annotate.h"
	#include "append.h"
	#include "assert.h"
	#include "auth.h"
	#include "backend.h"
	#include "duplicate.h"
	#include "exitcodes.h"
	#include "global.h"
	#include "idle.h"
	#include "imap_err.h"
	#include "imparse.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "mailbox.h"
	#include "map.h"
	#include "mboxlist.h"
	#include "mboxname.h"
	#include "message.h"
	#include "mupdate.h"
	#include "notify.h"
	#include "prot.h"
	#include "proxy.h"
	#include "statuscache.h"
	#include "telemetry.h"
	#include "tls.h"
	#include "util.h"
	#include "version.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"

	#include "lmtpd.h"
	#include "lmtpengine.h"
	#include "lmtpstats.h"
	#ifdef USE_SIEVE
	#include "lmtp_sieve.h"

	static sieve_interp_t *sieve_interp = NULL;
	#endif

	#include "sync_log.h"

	/* forward declarations */
	static int deliver(message_data_t msgdata, char authuser,
	struct auth_state *authstate);
	static int verify_user(const char user, const char domain, char *mailbox,
	quota_t quotacheck, struct auth_state *authstate);
	static char generate_notify(message_data_t m);

	void shut_down(int code);

	static FILE spoolfile(message_data_t msgdata);
	static void removespool(message_data_t *msgdata);

	/* current namespace */
	static struct namespace lmtpd_namespace;

	struct lmtp_func mylmtp = { &deliver, &verify_user, &shut_down,
	&spoolfile, &removespool, &lmtpd_namespace,
	0, 1, 0 };

	static void usage();

	/* global state */
	const int config_need_data = CONFIG_NEED_PARTITION_DATA;

	extern int optind;
	extern char *optarg;
	static int dupelim = 1; /* eliminate duplicate messages with
	same message-id */
	static int singleinstance = 1; /* attempt single instance store */

	struct stagemsg *stage = NULL;

	/* per-user/session state */
	static struct protstream deliver_out, deliver_in;
	int deliver_logfd = -1; /* used in lmtpengine.c */

	/* our cached connections */
	mupdate_handle *mhandle = NULL;
	struct backend **backend_cached = NULL;

	static struct protocol_t lmtp_protocol =
	{ "lmtp", "lmtp",
	{ 0, "220 " },
	{ "LHLO", "lmtpproxyd", "250 ", NULL,
	{ { "AUTH ", CAPA_AUTH },
	{ "STARTTLS", CAPA_STARTTLS },
	{ "PIPELINING", CAPA_PIPELINING },
	{ "IGNOREQUOTA", CAPA_IGNOREQUOTA },
	{ NULL, 0 } } },
	{ "STARTTLS", "220", "454", 0 },
	{ "AUTH", 512, 0, "235", "5", "334 ", "*", NULL, 0 },
	{ NULL, NULL, NULL },
	{ "NOOP", NULL, "250" },
	{ "QUIT", NULL, "221" }
	};

	static struct sasl_callback mysasl_cb[] = {
	{ SASL_CB_GETOPT, &mysasl_config, NULL },
	{ SASL_CB_PROXY_POLICY, &mysasl_proxy_policy, NULL },
	{ SASL_CB_CANON_USER, &mysasl_canon_user, NULL },
	{ SASL_CB_LIST_END, NULL, NULL }
	};


	int service_init(int argc __attribute__((unused)),
	char **argv __attribute__((unused)),
	char **envp __attribute__((unused)))
	{
	int r;

	if (geteuid() == 0) return 1;

	signals_set_shutdown(&shut_down);
	signal(SIGPIPE, SIG_IGN);

	singleinstance = config_getswitch(IMAPOPT_SINGLEINSTANCESTORE);

	global_sasl_init(1, 1, mysasl_cb);

	if (config_mupdate_server &&
	(config_mupdate_config == IMAP_ENUM_MUPDATE_CONFIG_STANDARD) &&
	!config_getstring(IMAPOPT_PROXYSERVERS)) {
	/* proxy only -- talk directly to mupdate master */
	r = mupdate_connect(config_mupdate_server, NULL, &mhandle, NULL);
	if (r) {
	syslog(LOG_ERR, "couldn't connect to MUPDATE server %s: %s",
	config_mupdate_server, error_message(r));
	fatal("error connecting with MUPDATE server", EC_TEMPFAIL);
	}
	}
	else {
	dupelim = config_getswitch(IMAPOPT_DUPLICATESUPPRESSION);

	#ifdef USE_SIEVE
	mylmtp.addheaders = xzmalloc(2 * sizeof(struct addheader));
	mylmtp.addheaders[0].name = "X-Sieve";
	mylmtp.addheaders[0].body = SIEVE_VERSION;

	/* setup sieve support */
	sieve_interp = setup_sieve();
	#else
	if (dupelim)
	#endif
	{
	/* initialize duplicate delivery database */
	if (duplicate_init(NULL, 0) != 0) {
	fatal("lmtpd: unable to init duplicate delivery database",
	EC_SOFTWARE);
	}
	}

	/* so we can do mboxlist operations */
	mboxlist_init(0);
	mboxlist_open(NULL);

	/* so we can do quota operations */
	quotadb_init(0);
	quotadb_open(NULL);

	/* Initialize the annotatemore db (for sieve on shared mailboxes) */
	annotatemore_init(0, NULL, NULL);
	annotatemore_open(NULL);

	/* setup for statuscache invalidation */
	statuscache_open(NULL);

	/* setup for sending IMAP IDLE notifications */
	idle_enabled();
	}

	/* Set namespace */
	if ((r = mboxname_init_namespace(&lmtpd_namespace, 0)) != 0) {
	syslog(LOG_ERR, "%s", error_message(r));
	fatal(error_message(r), EC_CONFIG);
	}

	/* create connection to the SNMP listener, if available. */
	snmp_connect(); /* ignore return code */
	snmp_set_str(SERVER_NAME_VERSION, cyrus_version());

	return 0;
	}

	static int mupdate_ignore_cb(struct mupdate_mailboxdata *mdata __attribute__((unused)),
	const char *cmd __attribute__((unused)),
	void *context __attribute__((unused)))
	{
	/* If we get called, we've recieved something other than an OK in
	* response to the NOOP, so we want to hang up this connection anyway */
	return MUPDATE_FAIL;
	}

	/*
	* run for each accepted connection
	*/
	int service_main(int argc, char **argv,
	char **envp __attribute__((unused)))
	{
	int opt, r;

	session_new_id();

	sync_log_init();

	deliver_in = prot_new(0, 0);
	deliver_out = prot_new(1, 1);
	prot_setflushonread(deliver_in, deliver_out);
	prot_settimeout(deliver_in, 360);

	while ((opt = getopt(argc, argv, "a")) != EOF) {
	switch(opt) {
	case 'a':
	mylmtp.preauth = 1;
	break;

	default:
	usage();
	}
	}

	snmp_increment(TOTAL_CONNECTIONS, 1);
	snmp_increment(ACTIVE_CONNECTIONS, 1);

	/* get a connection to the mupdate server */
	r = 0;
	if (mhandle) {
	/* we have one already, test it */
	r = mupdate_noop(mhandle, mupdate_ignore_cb, NULL);
	if (r) {
	/* will NULL mhandle for us */
	mupdate_disconnect(&mhandle);

	/* connect to the mupdate server */
	r = mupdate_connect(config_mupdate_server, NULL, &mhandle, NULL);
	}
	}
	if (!r) {
	lmtpmode(&mylmtp, deliver_in, deliver_out, 0);
	} else {
	syslog(LOG_ERR, "couldn't connect to %s: %s", config_mupdate_server,
	error_message(r));
	prot_printf(deliver_out, "451");
	if (config_serverinfo) prot_printf(deliver_out, " %s", config_servername);
	if (config_serverinfo == IMAP_ENUM_SERVERINFO_ON) {
	prot_printf(deliver_out, " Cyrus LMTP%s %s",
	config_mupdate_server ? " Murder" : "", cyrus_version());
	}
	prot_printf(deliver_out, " %s\r\n", error_message(r));
	}

	/* free session state */
	if (deliver_in) prot_free(deliver_in);
	if (deliver_out) prot_free(deliver_out);
	deliver_in = deliver_out = NULL;

	if (deliver_logfd != -1) {
	close(deliver_logfd);
	deliver_logfd = -1;
	}

	cyrus_reset_stdio();

	return 0;
	}

	/* Called by service API to shut down the service */
	void service_abort(int error)
	{
	shut_down(error);
	}

	static void
	usage()
	{
	fprintf(stderr, "421-4.3.0 usage: lmtpd [-C <alt_config>] [-a]\r\n");
	fprintf(stderr, "421 4.3.0 %s\n", cyrus_version());
	exit(EC_USAGE);
	}

	struct fuzz_rock {
	char *mboxname;
	size_t prefixlen;
	char *pat;
	size_t patlen;
	size_t matchlen;
	};

	#define WSP_CHARS "- _"

	static int fuzzy_match_cb(char *name,
	int matchlen __attribute__((unused)),
	int maycreate __attribute__((unused)),
	void *rock)
	{
	struct fuzz_rock frock = (struct fuzz_rock ) rock;
	unsigned i;

	for (i = frock->prefixlen; name[i] && frock->pat[i]; i++) {
	if (tolower((int) name[i]) != frock->pat[i] &&
	!(strchr(WSP_CHARS, name[i]) &&
	strchr(WSP_CHARS, frock->pat[i]))) {
	break;
	}
	}

	/* see if we have a [partial] match */
	if (!name[i] && (!frock->pat[i] \|\| frock->pat[i] == '.') &&
	i > frock->matchlen) {
	frock->matchlen = i;
	strlcpy(frock->mboxname, name, i+1);
	if (i == frock->patlen) return CYRUSDB_DONE;
	}

	return 0;
	}

	int fuzzy_match(char *mboxname)
	{
	char name[MAX_MAILBOX_BUFFER], prefix[MAX_MAILBOX_BUFFER], *p = NULL;
	size_t prefixlen;
	struct fuzz_rock frock;

	/* make a working copy */
	strlcpy(name, mboxname, sizeof(name));

	/* check to see if this is an personal mailbox */
	if (!strncmp(name, "user.", 5) \|\| (p = strstr(name, "!user."))) {
	p = p ? p + 6 : name + 5;

	/* check to see if this is an INBOX (no '.' after the userid) */
	if (!(p = strchr(p, '.'))) return 0;
	}

	if (p) p++; /* skip the trailing '.' */
	else p = name;

	/* copy the prefix */
	prefixlen = p - name;
	strlcpy(prefix, name, prefixlen+1);

	/* normalize the rest of the pattern to lowercase */
	lcase(p);

	frock.mboxname = mboxname;
	frock.prefixlen = prefixlen;
	frock.pat = name;
	frock.patlen = strlen(name);
	frock.matchlen = 0;

	strlcat(prefix, "*", sizeof(prefix));
	mboxlist_findall(NULL, prefix, 1, NULL, NULL, fuzzy_match_cb, &frock);

	return frock.matchlen;
	}

	/* proxy mboxlist_lookup; on misses, it asks the listener for this
	machine to make a roundtrip to the master mailbox server to make
	sure it's up to date */
	static int mlookup(const char name, char server, char aclp, void tid)
	{
	int r;
	char *c;

	if (server) *server = NULL;

	if (mhandle) {
	/* proxy only, so check the mupdate master */
	struct mupdate_mailboxdata *mailboxdata;

	/* find what server we're sending this to */
	r = mupdate_find(mhandle, name, &mailboxdata);

	if (r == MUPDATE_MAILBOX_UNKNOWN) {
	return IMAP_MAILBOX_NONEXISTENT;
	} else if (r) {
	/* xxx -- yuck: our error handling for now will be to exit;
	this txn will be retried later -- to do otherwise means
	that we may have to restart this transaction from scratch */
	fatal("error communicating with MUPDATE server", EC_TEMPFAIL);
	}

	if (aclp) aclp = (char ) mailboxdata->acl;
	if (server) server = (char ) mailboxdata->server;
	c = strchr(*server, '!');
	if (c) *c = '\0';
	}
	else {
	struct mboxlist_entry mbentry;
	/* do a local lookup and kick the slave if necessary */
	r = mboxlist_lookup(name, &mbentry, tid);
	if (r == IMAP_MAILBOX_NONEXISTENT && config_mupdate_server) {
	kick_mupdate();
	r = mboxlist_lookup(name, &mbentry, tid);
	}
	if (r) return r;

	if (aclp) *aclp = mbentry.acl;
	if (server) {
	if (mbentry.mbtype & MBTYPE_REMOTE) {
	/* xxx hide the fact that we are storing partitions */
	*server = mbentry.partition;
	c = strchr(*server, '!');
	if (c) *c = '\0';
	}
	}
	}

	return r;
	}

	/* places msg in mailbox mailboxname.
	* if you wish to use single instance store, pass stage as non-NULL
	* if you want to deliver message regardless of duplicates, pass id as NULL
	* if you want to notify, pass user
	* if you want to force delivery (to force delivery to INBOX, for instance)
	* pass acloverride
	*/
	int deliver_mailbox(FILE *f,
	struct message_content *content,
	struct stagemsg *stage,
	unsigned size,
	char **flag,
	int nflags,
	char *authuser,
	struct auth_state *authstate,
	char *id,
	const char *user,
	char *notifyheader,
	const char *mailboxname,
	int quotaoverride,
	int acloverride)
	{
	int r;
	struct appendstate as;
	time_t now = time(NULL);
	unsigned long uid;
	const char *notifier;

	r = append_setup(&as, mailboxname,
	authuser, authstate, acloverride ? 0 : ACL_POST,
	quotaoverride ? (long) -1 :
	config_getswitch(IMAPOPT_LMTP_STRICT_QUOTA) ?
	(long) size : 0);

	/* check for duplicate message */
	if (!r && id && dupelim && !(as.mailbox->i.options & OPT_IMAP_DUPDELIVER) &&
	duplicate_check(id, strlen(id), mailboxname, strlen(mailboxname))) {
	duplicate_log(id, mailboxname, "delivery");
	append_abort(&as);
	return 0;
	}

	if (!r && !content->body) {
	/* parse the message body if we haven't already,
	and keep the file mmap'ed */
	r = message_parse_file(f, &content->base, &content->len, &content->body);
	}

	if (!r) {
	r = append_fromstage(&as, &content->body, stage, now,
	(const char **) flag, nflags, !singleinstance);

	if (r) {
	append_abort(&as);
	} else {
	struct mailbox *mailbox = NULL;
	r = append_commit(&as, quotaoverride ? -1 : 0, NULL, &uid,
	NULL, &mailbox);
	if (!r) {
	syslog(LOG_INFO, "Delivered: %s to mailbox: %s",
	id, mailboxname);
	if (dupelim && id) {
	duplicate_mark(id, strlen(id), mailboxname,
	strlen(mailboxname), now, uid);
	}
	mailbox_close(&mailbox);
	}
	}
	}

	if (!r && user && (notifier = config_getstring(IMAPOPT_MAILNOTIFIER))) {
	char inbox[MAX_MAILBOX_BUFFER];
	char namebuf[MAX_MAILBOX_BUFFER];
	char userbuf[MAX_MAILBOX_BUFFER];
	const char *notify_mailbox = mailboxname;
	int r2;

	/* translate user.foo to INBOX */
	if (!(*lmtpd_namespace.mboxname_tointernal)(&lmtpd_namespace,
	"INBOX", user, inbox)) {
	size_t inboxlen = strlen(inbox);
	if (strlen(mailboxname) >= inboxlen &&
	!strncmp(mailboxname, inbox, inboxlen) &&
	(!mailboxname[inboxlen] \|\| mailboxname[inboxlen] == '.')) {
	strlcpy(inbox, "INBOX", sizeof(inbox));
	strlcat(inbox, mailboxname+inboxlen, sizeof(inbox));
	notify_mailbox = inbox;
	}
	}

	/* translate mailboxname */
	r2 = (*lmtpd_namespace.mboxname_toexternal)(&lmtpd_namespace,
	notify_mailbox,
	user, namebuf);
	if (!r2) {
	strlcpy(userbuf, user, sizeof(userbuf));
	/* translate any separators in user */
	mboxname_hiersep_toexternal(&lmtpd_namespace, userbuf,
	config_virtdomains ?
	strcspn(userbuf, "@") : 0);
	notify(notifier, "MAIL", NULL, userbuf, namebuf, 0, NULL,
	notifyheader ? notifyheader : "");
	}
	}

	return r;
	}

	enum rcpt_status {
	done = 0,
	nosieve, /* no sieve script */
	s_wait, /* processing sieve requests */
	s_err, /* error in sieve processing/sending */
	s_done, /* sieve script successfully run */
	};

	void deliver_remote(message_data_t *msgdata,
	struct dest dlist, enum rcpt_status status)
	{
	struct dest *d;

	/* run the txns */
	d = dlist;
	while (d) {
	struct lmtp_txn *lt = LMTP_TXN_ALLOC(d->rnum);
	struct rcpt *rc;
	struct backend *remote;
	int i = 0;
	int r = 0;

	lt->from = msgdata->return_path;
	lt->auth = d->authas[0] ? d->authas : NULL;
	lt->isdotstuffed = 0;
	lt->tempfail_unknown_mailbox = 1;

	prot_rewind(msgdata->data);
	lt->data = msgdata->data;
	lt->rcpt_num = d->rnum;
	rc = d->to;
	for (rc = d->to; rc != NULL; rc = rc->next, i++) {
	assert(i < d->rnum);
	lt->rcpt[i].addr = rc->rcpt;
	lt->rcpt[i].ignorequota =
	msg_getrcpt_ignorequota(msgdata, rc->rcpt_num);
	}
	assert(i == d->rnum);

	remote = proxy_findserver(d->server, &lmtp_protocol, "",
	&backend_cached, NULL, NULL, NULL);
	if (remote) {
	r = lmtp_runtxn(remote, lt);
	} else {
	/* remote server not available; tempfail all deliveries */
	for (rc = d->to, i = 0; i < d->rnum; i++) {
	lt->rcpt[i].result = RCPT_TEMPFAIL;
	lt->rcpt[i].r = IMAP_SERVER_UNAVAILABLE;
	}
	}

	/* process results of the txn, propogating error state to the
	recipients */
	for (rc = d->to, i = 0; rc != NULL; rc = rc->next, i++) {
	int j = rc->rcpt_num;
	switch (status[j]) {
	case s_wait:
	/* hmmm, if something fails we'll want to try an
	error delivery */
	if (lt->rcpt[i].result != RCPT_GOOD) {
	status[j] = s_err;
	}
	break;
	case s_err:
	/* we've already detected an error for this recipient,
	and nothing will convince me otherwise */
	break;
	case nosieve:
	/* this is the only delivery we're attempting for this rcpt */
	msg_setrcpt_status(msgdata, j, lt->rcpt[i].r);
	status[j] = done;
	break;
	case done:
	case s_done:
	/* yikes! we shouldn't be getting a notification for this
	person! */
	abort();
	break;
	}
	}

	free(lt);
	d = d->next;
	}
	}

	int deliver_local(deliver_data_t mydata, char *flag, int nflags,
	const char username, const char mailboxname)
	{
	char namebuf[MAX_MAILBOX_BUFFER] = "", *tail;
	message_data_t *md = mydata->m;
	int quotaoverride = msg_getrcpt_ignorequota(md, mydata->cur_rcpt);
	int ret;

	/* case 1: shared mailbox request */
	if (!*username \|\| username[0] == '@') {
	if (*username) snprintf(namebuf, sizeof(namebuf), "%s!", username+1);
	strlcat(namebuf, mailboxname, sizeof(namebuf));

	return deliver_mailbox(md->f, mydata->content, mydata->stage,
	md->size, flag, nflags,
	mydata->authuser, mydata->authstate, md->id,
	NULL, mydata->notifyheader,
	namebuf, quotaoverride, 0);
	}

	/* case 2: ordinary user */
	ret = (*mydata->namespace->mboxname_tointernal)(mydata->namespace,
	"INBOX",
	username, namebuf);

	if (!ret) {
	int ret2 = 1;

	tail = namebuf + strlen(namebuf);
	if (mailboxname) {
	strlcat(namebuf, ".", sizeof(namebuf));
	strlcat(namebuf, mailboxname, sizeof(namebuf));

	ret2 = deliver_mailbox(md->f, mydata->content, mydata->stage,
	md->size, flag, nflags,
	mydata->authuser, mydata->authstate, md->id,
	username, mydata->notifyheader,
	namebuf, quotaoverride, 0);
	}
	if (ret2 == IMAP_MAILBOX_NONEXISTENT && mailboxname &&
	config_getswitch(IMAPOPT_LMTP_FUZZY_MAILBOX_MATCH) &&
	fuzzy_match(namebuf)) {
	/* try delivery to a fuzzy matched mailbox */
	ret2 = deliver_mailbox(md->f, mydata->content, mydata->stage,
	md->size, flag, nflags,
	mydata->authuser, mydata->authstate, md->id,
	username, mydata->notifyheader,
	namebuf, quotaoverride, 0);
	}
	if (ret2) {
	/* normal delivery to INBOX */
	struct auth_state *authstate = auth_newstate(username);

	*tail = '\0';

	ret = deliver_mailbox(md->f, mydata->content, mydata->stage,
	md->size, flag, nflags,
	(char *) username, authstate, md->id,
	username, mydata->notifyheader,
	namebuf, quotaoverride, 1);

	if (authstate) auth_freestate(authstate);
	}
	}

	return ret;
	}

	int deliver(message_data_t msgdata, char authuser,
	struct auth_state *authstate)
	{
	int n, nrcpts;
	struct dest *dlist = NULL;
	enum rcpt_status *status;
	struct message_content content = { NULL, 0, NULL };
	char *notifyheader;
	deliver_data_t mydata;

	assert(msgdata);
	nrcpts = msg_getnumrcpt(msgdata);
	assert(nrcpts);

	notifyheader = generate_notify(msgdata);

	/* create our per-recipient status */
	status = xzmalloc(sizeof(enum rcpt_status) * nrcpts);

	/* create 'mydata', our per-delivery data */
	mydata.m = msgdata;
	mydata.content = &content;
	mydata.stage = stage;
	mydata.notifyheader = notifyheader;
	mydata.namespace = &lmtpd_namespace;
	mydata.authuser = authuser;
	mydata.authstate = authstate;

	/* loop through each recipient, attempting delivery for each */
	for (n = 0; n < nrcpts; n++) {
	char namebuf[MAX_MAILBOX_BUFFER] = "", *server;
	char userbuf[MAX_MAILBOX_BUFFER];
	const char rcpt, user, domain, mailbox;
	int r = 0;

	rcpt = msg_getrcptall(msgdata, n);
	msg_getrcpt(msgdata, n, &user, &domain, &mailbox);

	namebuf[0] = '\0';
	userbuf[0] = '\0';

	if (domain) snprintf(namebuf, sizeof(namebuf), "%s!", domain);

	/* case 1: shared mailbox request */
	if (!user) {
	strlcat(namebuf, mailbox, sizeof(namebuf));
	}
	/* case 2: ordinary user */
	else {
	strlcat(namebuf, "user.", sizeof(namebuf));
	strlcat(namebuf, user, sizeof(namebuf));

	strlcpy(userbuf, user, sizeof(userbuf));
	}
	if (domain) {
	strlcat(userbuf, "@", sizeof(userbuf));
	strlcat(userbuf, domain, sizeof(userbuf));
	}

	r = mlookup(namebuf, &server, NULL, NULL);
	if (!r && server) {
	/* remote mailbox */
	proxy_adddest(&dlist, rcpt, n, server, authuser);
	status[n] = nosieve;
	}
	else if (!r) {
	/* local mailbox */
	mydata.cur_rcpt = n;
	#ifdef USE_SIEVE
	r = run_sieve(user, domain, mailbox, sieve_interp, &mydata);
	/* if there was no sieve script, or an error during execution,
	r is non-zero and we'll do normal delivery */
	#else
	r = 1; /* normal delivery */
	#endif

	if (r) {
	r = deliver_local(&mydata, NULL, 0, userbuf, mailbox);
	}
	}

	telemetry_rusage( user );
	msg_setrcpt_status(msgdata, n, r);
	}

	if (dlist) {
	struct dest *d;

	/* run the txns */
	deliver_remote(msgdata, dlist, status);

	/* free the recipient/destination lists */
	d = dlist;
	while (d) {
	struct dest *nextd = d->next;
	struct rcpt *rc = d->to;

	while (rc) {
	struct rcpt *nextrc = rc->next;
	free(rc);
	rc = nextrc;
	}
	free(d);
	d = nextd;
	}
	dlist = NULL;

	/* do any sieve error recovery, if needed */
	for (n = 0; n < nrcpts; n++) {
	switch (status[n]) {
	case s_wait:
	case s_err:
	case s_done:
	/* yikes, we haven't implemented sieve ! */
	syslog(LOG_CRIT,
	"sieve states reached, but we don't implement sieve");
	abort();
	break;
	case nosieve:
	/* yikes, we never got an answer on this one */
	syslog(LOG_CRIT, "still waiting for response to rcpt %d",
	n);
	abort();
	break;
	case done:
	/* good */
	break;
	}
	}

	/* run the error recovery txns */
	deliver_remote(msgdata, dlist, status);

	/* everything should be in the 'done' state now, verify this */
	for (n = 0; n < nrcpts; n++) {
	assert(status[n] == done \|\| status[n] == s_done);
	}
	}

	/* cleanup */
	free(status);
	if (content.base) map_free(&content.base, &content.len);
	if (content.body) {
	message_free_body(content.body);
	free(content.body);
	}
	append_removestage(stage);
	stage = NULL;
	if (notifyheader) free(notifyheader);

	return 0;
	}

	void fatal(const char* s, int code)
	{
	static int recurse_code = 0;

	if(recurse_code) {
	/* We were called recursively. Just give up */
	snmp_increment(ACTIVE_CONNECTIONS, -1);
	exit(recurse_code);
	}
	recurse_code = code;
	if(deliver_out) {
	prot_printf(deliver_out,"421 4.3.0 lmtpd: %s\r\n", s);
	prot_flush(deliver_out);
	}
	if (stage) append_removestage(stage);

	syslog(LOG_ERR, "FATAL: %s", s);

	/* shouldn't return */
	shut_down(code);

	exit(code);
	}

	/*
	* Cleanly shut down and exit
	*/
	void shut_down(int code) __attribute__((noreturn));
	void shut_down(int code)
	{
	int i;

	/* set flag */
	in_shutdown = 1;

	/* close backend connections */
	i = 0;
	while (backend_cached && backend_cached[i]) {
	proxy_downserver(backend_cached[i]);
	free(backend_cached[i]);
	i++;
	}
	if (backend_cached) free(backend_cached);

	if (mhandle) {
	mupdate_disconnect(&mhandle);
	} else {
	#ifdef USE_SIEVE
	sieve_interp_free(&sieve_interp);
	#else
	if (dupelim)
	#endif
	duplicate_done();

	mboxlist_close();
	mboxlist_done();

	quotadb_close();
	quotadb_done();

	annotatemore_close();
	annotatemore_done();

	statuscache_close();
	statuscache_done();
	}

	#ifdef HAVE_SSL
	tls_shutdown_serverengine();
	#endif
	if (deliver_out) {
	prot_flush(deliver_out);

	/* one less active connection */
	snmp_increment(ACTIVE_CONNECTIONS, -1);
	}

	sync_log_done();

	cyrus_done();

	exit(code);
	}

	static int verify_user(const char user, const char domain, char *mailbox,
	quota_t quotacheck, struct auth_state *authstate)
	{
	char namebuf[MAX_MAILBOX_BUFFER] = "";
	int r = 0;

	if ((!user && !mailbox) \|\|
	(domain && (strlen(domain) + 1 > sizeof(namebuf)))) {
	r = IMAP_MAILBOX_NONEXISTENT;
	} else {
	/* construct the mailbox that we will verify */
	if (domain) snprintf(namebuf, sizeof(namebuf), "%s!", domain);

	if (!user) {
	/* shared folder */
	if (strlen(namebuf) + strlen(mailbox) > sizeof(namebuf)) {
	r = IMAP_MAILBOX_NONEXISTENT;
	} else {
	strlcat(namebuf, mailbox, sizeof(namebuf));
	}
	} else {
	/* ordinary user -- check INBOX */
	if (strlen(namebuf) + 5 + strlen(user) > sizeof(namebuf)) {
	r = IMAP_MAILBOX_NONEXISTENT;
	} else {
	strlcat(namebuf, "user.", sizeof(namebuf));
	strlcat(namebuf, user, sizeof(namebuf));
	}
	}
	}

	if (!r) {
	char server, acl;
	long aclcheck = !user ? ACL_POST : 0;
	/*
	* check to see if mailbox exists and we can append to it:
	*
	* - must have posting privileges on shared folders
	* - don't care about ACL on INBOX (always allow post)
	* - don't care about message size (1 msg over quota allowed)
	*/
	r = mlookup(namebuf, &server, &acl, NULL);

	if (r == IMAP_MAILBOX_NONEXISTENT && !user &&
	config_getswitch(IMAPOPT_LMTP_FUZZY_MAILBOX_MATCH) &&
	/* see if we have a mailbox whose name is close */
	fuzzy_match(namebuf)) {

	/* We are guaranteed that the mailbox returned by fuzzy_match()
	will be no longer than the original, so we can copy over
	the existing mailbox. The keeps us from having to do the
	fuzzy match multiple times. */
	strcpy(mailbox, domain ? namebuf+strlen(domain)+1 : namebuf);

	r = mlookup(namebuf, &server, &acl, NULL);
	}

	if (!r && server) {
	int access = cyrus_acl_myrights(authstate, acl);

	if ((access & aclcheck) != aclcheck) {
	r = (access & ACL_LOOKUP) ?
	IMAP_PERMISSION_DENIED : IMAP_MAILBOX_NONEXISTENT;
	}
	} else if (!r) {
	r = append_check(namebuf, authstate,
	aclcheck, (quotacheck < 0)
	\|\| config_getswitch(IMAPOPT_LMTP_STRICT_QUOTA) ?
	quotacheck : 0);
	}
	}

	if (r) syslog(LOG_DEBUG, "verify_user(%s) failed: %s", namebuf,
	error_message(r));

	return r;
	}

	const char *notifyheaders[] = { "From", "Subject", "To", 0 };
	/* returns a malloc'd string that should be sent to users for successful
	delivery of 'm'. */
	char generate_notify(message_data_t m)
	{
	const char **body;
	char *ret = NULL;
	unsigned int len = 0;
	unsigned int pos = 0;
	int i;

	for (i = 0; notifyheaders[i]; i++) {
	const char *h = notifyheaders[i];
	body = msg_getheader(m, h);
	if (body) {
	int j;

	for (j = 0; body[j] != NULL; j++) {
	/* put the header */
	/* need: length + ": " + '\0'*/
	while (pos + strlen(h) + 3 > len) {
	ret = xrealloc(ret, len += 1024);
	}
	pos += sprintf(ret + pos, "%s: ", h);

	/* put the header body.
	xxx it would be nice to linewrap.*/
	/* need: length + '\n' + '\0' */
	while (pos + strlen(body[j]) + 2 > len) {
	ret = xrealloc(ret, len += 1024);
	}
	pos += sprintf(ret + pos, "%s\n", body[j]);
	}
	}
	}

	return ret;
	}

	FILE spoolfile(message_data_t msgdata)
	{
	int i, n;
	time_t now = time(NULL);
	FILE *f = NULL;

	/* spool to the stage of one of the recipients
	(don't bother if we're only a proxy) */
	n = mhandle ? 0 : msg_getnumrcpt(msgdata);
	for (i = 0; !f && (i < n); i++) {
	char namebuf[MAX_MAILBOX_BUFFER] = "", *server;
	const char user, domain, *mailbox;
	int r;

	/* build the mailboxname from the recipient address */
	msg_getrcpt(msgdata, i, &user, &domain, &mailbox);

	if (domain) snprintf(namebuf, sizeof(namebuf), "%s!", domain);

	/* case 1: shared mailbox request */
	if (!user) {
	strlcat(namebuf, mailbox, sizeof(namebuf));
	}

	/* case 2: ordinary user */
	else {
	/* assume delivery to INBOX for now */
	strlcat(namebuf, "user.", sizeof(namebuf));
	strlcat(namebuf, user, sizeof(namebuf));
	}

	r = mlookup(namebuf, &server, NULL, NULL);
	if (!r && !server) {
	/* local mailbox -- setup stage for later use by deliver() */
	f = append_newstage(namebuf, now, 0, &stage);
	}
	}

	if (!f) {
	/* we only have remote mailboxes, so use a tempfile */
	int fd = create_tempfile(config_getstring(IMAPOPT_TEMP_PATH));

	if (fd != -1) f = fdopen(fd, "w+");
	}

	return f;
	}

	void removespool(message_data_t *msgdata __attribute__((unused)))
	{
	append_removestage(stage);
	stage = NULL;
	}
	diff --git a/imap/mailbox.c b/imap/mailbox.c
	index 541e6e0d2..48eaab580 100644
	--- a/imap/mailbox.c
	+++ b/imap/mailbox.c
	@@ -1,3785 +1,3785 @@
	/* mailbox.c -- Mailbox manipulation routines
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: mailbox.c,v 1.201 2010/06/28 12:04:20 brong Exp $
	*/
	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <ctype.h>
	#include <errno.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <syslog.h>
	#include <utime.h>

	#ifdef HAVE_DIRENT_H
	# include <dirent.h>
	# define NAMLEN(dirent) strlen((dirent)->d_name)
	#else
	# define dirent direct
	# define NAMLEN(dirent) (dirent)->d_namlen
	# if HAVE_SYS_NDIR_H
	# include <sys/ndir.h>
	# endif
	# if HAVE_SYS_DIR_H
	# include <sys/dir.h>
	# endif
	# if HAVE_NDIR_H
	# include <ndir.h>
	# endif
	#endif

	#include "acl.h"
	#include "assert.h"
	#include "crc32.h"
	#include "exitcodes.h"
	#include "global.h"
	#include "imap_err.h"
	#include "imparse.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "mailbox.h"
	#include "message.h"
	#include "map.h"
	#include "mboxlist.h"
	#include "retry.h"
	#include "seen.h"
	#include "upgrade_index.h"
	#include "util.h"
	#include "sequence.h"
	#include "statuscache.h"
	#include "sync_log.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"

	struct mailboxlist {
	struct mailboxlist *next;
	struct mailbox m;
	struct mboxlock *l;
	int nopen;
	};

	static struct mailboxlist *open_mailboxes = NULL;

	#define zeromailbox(m) { memset(&m, 0, sizeof(struct mailbox)); \
	(m).index_fd = -1; \
	(m).cache_fd = -1; \
	(m).header_fd = -1; \
	(m).lock_fd = -1; }

	static int mailbox_delete_cleanup(struct mailbox *mailbox);
	static int mailbox_index_unlink(struct mailbox *mailbox);
	static int mailbox_index_repack(struct mailbox *mailbox);

	static struct mailboxlist create_listitem(const char name)
	{
	struct mailboxlist *item = xmalloc(sizeof(struct mailboxlist));
	item->next = open_mailboxes;
	open_mailboxes = item;

	item->nopen = 1;
	item->l = NULL;
	zeromailbox(item->m);
	item->m.name = xstrdup(name);

	return item;
	}

	static struct mailboxlist find_listitem(const char name)
	{
	struct mailboxlist *item;
	struct mailboxlist *previtem = NULL;

	/* remove from the active list */
	for (item = open_mailboxes; item; item = item->next) {
	if (!strcmp(name, item->m.name))
	return item;
	previtem = item;
	}

	return NULL;
	}

	static void remove_listitem(struct mailboxlist *remitem)
	{
	struct mailboxlist *item;
	struct mailboxlist *previtem = NULL;

	for (item = open_mailboxes; item; item = item->next) {
	if (item == remitem) {
	if (previtem)
	previtem->next = item->next;
	else
	open_mailboxes = item->next;
	free(item);
	return;
	}
	previtem = item;
	}

	fatal("didn't find item in list", EC_SOFTWARE);
	}

	char mailbox_meta_fname(struct mailbox mailbox, int metafile)
	{
	static char fnamebuf[MAX_MAILBOX_PATH];
	const char *src;

	src = mboxname_metapath(mailbox->part, mailbox->name, metafile, 0);
	if (!src) return NULL;

	strncpy(fnamebuf, src, MAX_MAILBOX_PATH);
	return fnamebuf;
	}

	char mailbox_meta_newfname(struct mailbox mailbox, int metafile)
	{
	static char fnamebuf[MAX_MAILBOX_PATH];
	const char *src;

	src = mboxname_metapath(mailbox->part, mailbox->name, metafile, 1);
	if (!src) return NULL;

	strncpy(fnamebuf, src, MAX_MAILBOX_PATH);
	return fnamebuf;
	}

	int mailbox_meta_rename(struct mailbox *mailbox, int metafile)
	{
	char *fname = mailbox_meta_fname(mailbox, metafile);
	char *newfname = mailbox_meta_newfname(mailbox, metafile);

	if (rename(newfname, fname))
	return IMAP_IOERROR;

	return 0;
	}

	char mailbox_message_fname(struct mailbox mailbox, unsigned long uid)
	{
	static char localbuf[MAX_MAILBOX_PATH];
	const char *src;

	src = mboxname_datapath(mailbox->part, mailbox->name, uid);
	if (!src) return NULL;

	strncpy(localbuf, src, MAX_MAILBOX_PATH);
	return localbuf;
	}

	char mailbox_datapath(struct mailbox mailbox)
	{
	static char localbuf[MAX_MAILBOX_PATH];
	const char *src;

	src = mboxname_datapath(mailbox->part, mailbox->name, 0);
	if (!src) return NULL;

	strncpy(localbuf, src, MAX_MAILBOX_PATH);
	return localbuf;
	}

	/*
	* Names of the headers we cache in the cyrus.cache file.
	*
	* Changes to this list probably require bumping the cache version
	* number (obviously)
	*
	* note that header names longer than MAX_CACHED_HEADER_SIZE
	* won't be cached regardless
	*
	* xxx can we get benefits by requireing this list to be sorted?
	* (see is_cached_header())
	*
	*/
	const struct mailbox_header_cache mailbox_cache_headers[] = {
	/* things we have always cached */
	{ "priority", 0 },
	{ "references", 0 },
	{ "resent-from", 0 },
	{ "newsgroups", 0 },
	{ "followup-to", 0 },

	/* x headers that we may want to cache anyway */
	{ "x-mailer", 1 },
	{ "x-trace", 1 },

	/* outlook express seems to want these */
	{ "x-ref", 2 },
	{ "x-priority", 2 },
	{ "x-msmail-priority", 2 },
	{ "x-msoesrec", 2 },

	/* for efficient FastMail interface display */
	{ "x-spam-score", 3 },
	{ "x-spam-hits", 3 },
	{ "x-spam-source", 3 },
	{ "x-resolved-to", 3 },
	{ "x-delivered-to", 3 },
	{ "x-mail-from", 3 },
	{ "x-truedomain", 3 },
	{ "x-truedomain-dkim", 3 },
	{ "x-truedomain-spf", 3 },
	{ "x-truedomain-domain", 3 },

	/* things to never cache */
	{ "bcc", BIT32_MAX },
	{ "cc", BIT32_MAX },
	{ "date", BIT32_MAX },
	{ "delivery-date", BIT32_MAX },
	{ "envelope-to", BIT32_MAX },
	{ "from", BIT32_MAX },
	{ "in-reply-to", BIT32_MAX },
	{ "mime-version", BIT32_MAX },
	{ "reply-to", BIT32_MAX },
	{ "received", BIT32_MAX },
	{ "return-path", BIT32_MAX },
	{ "sender", BIT32_MAX },
	{ "subject", BIT32_MAX },
	{ "to", BIT32_MAX },

	/* signatures tend to be large, and are useless without the body */
	{ "dkim-signature", BIT32_MAX },
	{ "domainkey-signature", BIT32_MAX },
	{ "domainkey-x509", BIT32_MAX },

	/* older versions of PINE (before 4.56) need message-id in the cache too
	* though technically it is a waste of space because it is in
	* ENVELOPE. We should probably uncomment the following at some
	* future point [ken3 notes this may also be useful to have here for
	* threading so we can avoid parsing the envelope] */
	/* { "message-id", BIT32_MAX }, */
	};
	const int MAILBOX_NUM_CACHE_HEADERS =
	sizeof(mailbox_cache_headers)/sizeof(struct mailbox_header_cache);

	/*
	* Function to test if a header is in the cache
	*
	* Assume cache entry version 1, unless other data is found
	* in the table.
	*/
	static inline unsigned is_cached_header(const char *hdr)
	{
	int i;

	/* xxx if we can sort the header list we can do better here */
	for (i=0; i<MAILBOX_NUM_CACHE_HEADERS; i++) {
	if (!strcmp(mailbox_cache_headers[i].name, hdr))
	return mailbox_cache_headers[i].min_cache_version;
	}

	/* Don't Cache X- headers unless explicitly configured to*/
	if ((hdr[0] == 'x') && (hdr[1] == '-')) return BIT32_MAX;

	/* Everything else we cache in version 1 */
	return 1;
	}

	/* External API to is_cached_header that prepares the string
	*
	* Returns minimum version required for lookup to succeed
	* or BIT32_MAX if header not cached
	*/
	unsigned mailbox_cached_header(const char *s)
	{
	char hdr[MAX_CACHED_HEADER_SIZE];
	int i;

	/* Generate lower case copy of string */
	/* xxx sometimes the caller has already generated this ..
	* maybe we can just require callers to do it? */
	for (i=0 ; *s && (i < (MAX_CACHED_HEADER_SIZE - 1)) ; i++)
	hdr[i] = tolower(*s++);

	if (s) return BIT32_MAX; / Input too long for match */
	hdr[i] = '\0';

	return is_cached_header(hdr);
	}

	/* Same as mailbox_cached_header, but for use on a header
	* as it appears in the message (i.e. :-terminated, not NUL-terminated)
	*/
	unsigned mailbox_cached_header_inline(const char *text)
	{
	char buf[MAX_CACHED_HEADER_SIZE];
	int i;

	/* Scan for header */
	for (i=0; i < (MAX_CACHED_HEADER_SIZE - 1); i++) {
	if (!text[i] \|\| text[i] == '\r' \|\| text[i] == '\n') break;

	if (text[i] == ':') {
	buf[i] = '\0';
	return is_cached_header(buf);
	} else {
	buf[i] = tolower(text[i]);
	}
	}

	return BIT32_MAX;
	}

	const char cache_base(struct index_record record)
	{
	const char *base = record->crec.base->s;
	return base + record->crec.offset;
	}

	unsigned cache_size(struct index_record *record)
	{
	return record->crec.len;
	}

	struct buf cache_buf(struct index_record record)
	{
	static struct buf staticbuf;

	staticbuf.s = (char *)cache_base(record);
	staticbuf.len = cache_size(record);

	return &staticbuf;
	}

	const char cacheitem_base(struct index_record record, int field)
	{
	const char *base = record->crec.base->s;
	return base + record->crec.item[field].offset;
	}

	unsigned cacheitem_size(struct index_record *record, int field)
	{
	return record->crec.item[field].len;
	}

	struct buf cacheitem_buf(struct index_record record, int field)
	{
	static struct buf staticbuf;

	staticbuf.s = (char *)cacheitem_base(record, field);
	staticbuf.len = cacheitem_size(record, field);

	return &staticbuf;
	}


	/* parse a single cache record from the mapped file - creates buf
	* records which point into the map, so you can't free it while
	* you still have them around! */
	int cache_parserecord(struct buf *cachebase, unsigned cache_offset,
	struct cacherecord *crec)
	{
	unsigned cache_ent;
	unsigned offset;
	const char cacheitem, next;

	offset = cache_offset;

	if (offset >= cachebase->len) {
	syslog(LOG_ERR, "IOERROR: offset greater than cache size");
	return IMAP_IOERROR;
	}

	for (cache_ent = 0; cache_ent < NUM_CACHE_FIELDS; cache_ent++) {
	cacheitem = cachebase->s + offset;
	/* copy locations */
	crec->item[cache_ent].len = CACHE_ITEM_LEN(cacheitem);
	crec->item[cache_ent].offset = offset + CACHE_ITEM_SIZE_SKIP;

	/* moving on */
	next = CACHE_ITEM_NEXT(cacheitem);
	if (next < cacheitem) {
	syslog(LOG_ERR, "IOERROR: cache offset negative");
	return IMAP_IOERROR;
	}

	offset = next - cachebase->s;
	if (offset > cachebase->len) {
	syslog(LOG_ERR, "IOERROR: offset greater than cache size");
	return IMAP_IOERROR;
	}
	}

	/* all fit within the cache, it's gold as far as we can tell */
	crec->base = cachebase;
	crec->len = offset - cache_offset;
	crec->offset = cache_offset;

	return 0;
	}

	int mailbox_open_cache(struct mailbox *mailbox)
	{
	struct stat sbuf;
	unsigned generation;
	int retry = 0;

	/* already got everything? great */
	if (mailbox->cache_fd != -1 && !mailbox->need_cache_refresh)
	return 0;

	retry:
	/* open the file */
	if (mailbox->cache_fd == -1) {
	char *fname;

	/* it's bogus to be dirty here */
	if (mailbox->cache_dirty)
	abort();

	fname = mailbox_meta_fname(mailbox, META_CACHE);
	mailbox->cache_fd = open(fname, O_RDWR, 0);
	if (mailbox->cache_fd == -1)
	goto fail;
	}

	/* get the size and inode */
	if (fstat(mailbox->cache_fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstating cache %s: %m", mailbox->name);
	goto fail;
	}
	mailbox->cache_buf.len = sbuf.st_size;
	if (mailbox->cache_buf.len < 4)
	goto fail;

	map_refresh(mailbox->cache_fd, 0, (const char **)&mailbox->cache_buf.s,
	&mailbox->cache_len, mailbox->cache_buf.len, "cache",
	mailbox->name);

	generation = ntohl(((bit32 )(mailbox->cache_buf.s)));
	if (generation < mailbox->i.generation_no && !retry) {
	/* try a rename - maybe we got killed between renames in repack */
	map_free((const char **)&mailbox->cache_buf.s, &mailbox->cache_len);
	close(mailbox->cache_fd);
	mailbox->cache_fd = -1;
	syslog(LOG_NOTICE, "WARNING: trying to rename cache file %s (%d < %d)",
	mailbox->name, generation, mailbox->i.generation_no);
	mailbox_meta_rename(mailbox, META_CACHE);
	retry = 1;
	goto retry;
	}
	if (generation != mailbox->i.generation_no) {
	map_free((const char **)&mailbox->cache_buf.s, &mailbox->cache_len);
	goto fail;
	}

	mailbox->need_cache_refresh = 0;

	return 0;

	fail:
	/* rebuild the cache from scratch! */
	syslog(LOG_ERR, "IOERROR: %s failed to open cache - rebuilding",
	mailbox->name);

	{
	struct index_record record;
	const char *fname;
	uint32_t recno;
	uint32_t offset;
	char buf[4];

	/* make sure we have a file */
	if (mailbox->cache_fd == -1) {
	fname = mailbox_meta_fname(mailbox, META_CACHE);
	mailbox->cache_fd = open(fname, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	}

	/* update the generation number */
	((bit32 )(buf)) = htonl(mailbox->i.generation_no);
	retry_write(mailbox->cache_fd, buf, 4);

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	if (mailbox_read_index_record(mailbox, recno, &record))
	continue;
	if (record.system_flags & FLAG_UNLINKED)
	continue;
	fname = mailbox_message_fname(mailbox, record.uid);
	offset = record.cache_offset; /* gets overwritten by parse */
	if (message_parse(fname, &record))
	continue;
	lseek(mailbox->cache_fd, offset, SEEK_SET);
	retry_write(mailbox->cache_fd, cache_base(&record),
	cache_size(&record));
	}
	(void)fsync(mailbox->cache_fd);

	/* get the size and inode */
	fstat(mailbox->cache_fd, &sbuf);
	mailbox->cache_buf.len = sbuf.st_size;

	map_refresh(mailbox->cache_fd, 0, (const char **)&mailbox->cache_buf.s,
	&mailbox->cache_len, mailbox->cache_buf.len, "cache",
	mailbox->name);
	}

	mailbox->need_cache_refresh = 0;
	return 0;
	}

	int mailbox_index_islocked(struct mailbox *mailbox, int write)
	{
	if (mailbox->index_locktype == LOCK_EXCLUSIVE) return 1;
	if (mailbox->index_locktype == LOCK_SHARED && !write) return 1;
	return 0;
	}

	/* return the offset for the start of the record! */
	int mailbox_append_cache(struct mailbox *mailbox,
	struct index_record *record)
	{
	int r;

	/* no cache content */
	if (!record->crec.len)
	return 0;

	/* already been written */
	if (record->cache_offset)
	return 0;

	/* ensure we have a cache fd */
	r = mailbox_open_cache(mailbox);
	if (r) {
	syslog(LOG_ERR, "Failed to open cache to %s for %u",
	mailbox->name, record->uid);
	return r; /* unable to append */
	}

	r = cache_append_record(mailbox->cache_fd, record);
	if (r) {
	syslog(LOG_ERR, "Failed to append cache to %s for %u",
	mailbox->name, record->uid);
	return r;
	}

	mailbox->cache_dirty = 1;
	mailbox->need_cache_refresh = 1;

	return 0;
	}

	int mailbox_cacherecord(struct mailbox *mailbox,
	struct index_record *record)
	{
	uint32_t crc;
	int r = 0;

	/* do we already have a record loaded? */
	if (record->crec.len)
	return 0;

	if (!record->cache_offset)
	r = IMAP_IOERROR;
	if (r) goto done;

	r = mailbox_open_cache(mailbox);
	if (r) goto done;

	/* try to parse the cache record */
	r = cache_parserecord(&mailbox->cache_buf,
	record->cache_offset, &record->crec);

	if (r) goto done;
	crc = crc32_buf(cache_buf(record));
	if (crc != record->cache_crc)
	r = IMAP_MAILBOX_CRC;

	done:
	if (r)
	syslog(LOG_ERR, "IOERROR: invalid cache record for %s uid %u (%s)",
	mailbox->name, record->uid, error_message(r));

	return r;
	}

	int cache_append_record(int fd, struct index_record *record)
	{
	unsigned offset;
	unsigned size = cache_size(record);
	int n;

	/* no parsed cache present */
	if (!record->crec.len)
	return 0;

	/* cache offset already there - probably already been written */
	if (record->cache_offset)
	return 0;

	if (record->cache_crc != crc32_buf(cache_buf(record)))
	return IMAP_MAILBOX_CRC;

	offset = lseek(fd, 0L, SEEK_END);
	n = retry_write(fd, cache_base(record), size);
	if (n < 0) {
	syslog(LOG_ERR, "failed to append %u bytes to cache", size);
	return IMAP_IOERROR;
	}

	record->cache_offset = offset;

	return 0;
	}

	int mailbox_commit_cache(struct mailbox *mailbox)
	{
	if (!mailbox->cache_dirty)
	return 0;

	mailbox->cache_dirty = 0;

	/* not open! That's bad */
	if (mailbox->cache_fd == -1)
	abort();

	/* just fsync is all that's needed to commit */
	(void)fsync(mailbox->cache_fd);

	return 0;
	}

	/* function to be used for notification of mailbox changes/updates */
	static mailbox_notifyproc_t *updatenotifier = NULL;

	/*
	* Set the updatenotifier function
	*/
	void mailbox_set_updatenotifier(mailbox_notifyproc_t *notifyproc)
	{
	updatenotifier = notifyproc;
	}

	/*
	* Get the updatenotifier function
	*/
	mailbox_notifyproc_t *mailbox_get_updatenotifier(void)
	{
	return updatenotifier;
	}

	/* create the unique identifier for a mailbox named 'name' with
	* uidvalidity 'uidvalidity'. 'uniqueid' should be at least 17 bytes
	* long. the unique identifier is just the mailbox name hashed to 32
	* bits followed by the uid, both converted to hex.
	*/
	#define PRIME (2147484043UL)

	void mailbox_make_uniqueid(struct mailbox *mailbox)
	{
	unsigned hash = 0;
	const char *name = mailbox->name;

	while (*name) {
	hash *= 251;
	hash += *name++;
	hash %= PRIME;
	}

	free(mailbox->uniqueid);
	mailbox->uniqueid = xmalloc(32);

	snprintf(mailbox->uniqueid, 32, "%08x%08x",
	hash, mailbox->i.uidvalidity);

	mailbox->header_dirty = 1;
	}

	/*
	* Maps in the content for the message with UID 'uid' in 'mailbox'.
	* Returns map in 'basep' and 'lenp'
	*/
	int mailbox_map_message(struct mailbox *mailbox, unsigned long uid,
	const char *basep, unsigned long lenp)
	{
	int msgfd;
	char *fname;
	struct stat sbuf;

	fname = mailbox_message_fname(mailbox, uid);

	msgfd = open(fname, O_RDONLY, 0666);
	if (msgfd == -1) return errno;

	if (fstat(msgfd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat on %s: %m", fname);
	fatal("can't fstat message file", EC_OSFILE);
	}
	*basep = 0;
	*lenp = 0;
	map_refresh(msgfd, 1, basep, lenp, sbuf.st_size, fname, mailbox->name);
	close(msgfd);

	return 0;
	}

	/*
	* Releases the buffer obtained from mailbox_map_message()
	*/
	void mailbox_unmap_message(struct mailbox *mailbox __attribute__((unused)),
	unsigned long uid __attribute__((unused)),
	const char *basep, unsigned long lenp)
	{
	map_free(basep, lenp);
	}

	int mailbox_mboxlock_upgrade(struct mailboxlist *listitem, int locktype)
	{
	struct mailbox *mailbox = &listitem->m;
	int r;

	if (listitem->l->locktype == LOCK_EXCLUSIVE)
	return 0;

	mboxname_release(&listitem->l);
	r = mboxname_lock(mailbox->name, &listitem->l, locktype);
	if (r) return r;

	if (mailbox->index_fd != -1)
	close(mailbox->index_fd);
	if (mailbox->cache_fd != -1)
	close(mailbox->cache_fd);

	if (mailbox->index_base)
	map_free(&mailbox->index_base, &mailbox->index_len);
	if (mailbox->cache_buf.s)
	map_free((const char **)&mailbox->cache_buf.s, &mailbox->cache_len);

	r = mailbox_open_index(mailbox);

	return r;
	}

	/*
	* Open and read the header of the mailbox with name 'name'
	* The structure pointed to by 'mailbox' is initialized.
	*/
	int mailbox_open_advanced(const char *name,
	struct mailbox **mailboxptr,
	int locktype,
	int index_locktype)
	{
	struct mboxlist_entry mbentry;
	struct mailboxlist *listitem;
	struct mailbox *mailbox = NULL;
	int r = 0;

	assert(*mailboxptr == NULL);

	listitem = find_listitem(name);

	/* already open? just use this one */
	if (listitem) {
	/* can't reuse an exclusive locked mailbox */
	if (listitem->l->locktype == LOCK_EXCLUSIVE)
	return IMAP_MAILBOX_LOCKED;
	if (locktype == LOCK_EXCLUSIVE)
	return IMAP_MAILBOX_LOCKED;
	/* can't reuse an already locked index */
	if (listitem->m.index_locktype)
	return IMAP_MAILBOX_LOCKED;

	r = mailbox_lock_index(&listitem->m, index_locktype);
	if (r) return r;

	listitem->nopen++;
	mailbox = &listitem->m;

	goto done;
	}

	listitem = create_listitem(name);
	mailbox = &listitem->m;

	r = mboxname_lock(name, &listitem->l, locktype);
	if (r) {
	/* locked is not an error - just means we asked for NONBLOCKING */
	if (r != IMAP_MAILBOX_LOCKED)
	syslog(LOG_ERR, "IOERROR: locking %s: %m", mailbox->name);
	goto done;
	}

	r = mboxlist_lookup(name, &mbentry, NULL);
	if (r) goto done;

	mailbox->part = xstrdup(mbentry.partition);
	/* Note that the header does have the ACL information, but it is only
	* a backup, and the mboxlist data is considered authoritative, so
	* we will just use what we were passed */
	mailbox->acl = xstrdup(mbentry.acl);
	mailbox->mbtype = mbentry.mbtype;

	r = mailbox_open_index(mailbox);
	if (r) {
	syslog(LOG_ERR, "IOERROR: opening index %s: %m", mailbox->name);
	goto done;
	}

	/* this will open, map and parse the header file */
	r = mailbox_lock_index(mailbox, index_locktype);
	if (r) {
	syslog(LOG_ERR, "IOERROR: locking index %s: %m", mailbox->name);
	goto done;
	}

	/* we may be in the process of deleting this mailbox */
	if (mailbox->i.options & OPT_MAILBOX_DELETED) {
	r = IMAP_MAILBOX_NONEXISTENT;
	goto done;
	}

	done:
	if (r) mailbox_close(&mailbox);
	else *mailboxptr = mailbox;

	return r;
	}

	int mailbox_open_irl(const char name, struct mailbox *mailboxptr)
	{
	return mailbox_open_advanced(name, mailboxptr, LOCK_SHARED, LOCK_SHARED);
	}

	int mailbox_open_iwl(const char name, struct mailbox *mailboxptr)
	{
	return mailbox_open_advanced(name, mailboxptr, LOCK_SHARED, LOCK_EXCLUSIVE);
	}

	int mailbox_open_exclusive(const char name, struct mailbox *mailboxptr)
	{
	return mailbox_open_advanced(name, mailboxptr, LOCK_EXCLUSIVE,
	LOCK_EXCLUSIVE);
	}

	/*
	* Open the index file for 'mailbox'
	*/
	int mailbox_open_index(struct mailbox *mailbox)
	{
	struct stat sbuf;
	char *fname;

	if (mailbox->i.dirty \|\| mailbox->cache_dirty)
	abort();

	if (mailbox->index_fd != -1) {
	close(mailbox->index_fd);
	mailbox->index_fd = -1;
	}
	if (mailbox->index_base)
	map_free(&mailbox->index_base, &mailbox->index_len);

	if (mailbox->cache_fd != -1) {
	close(mailbox->cache_fd);
	mailbox->cache_fd = -1;
	}
	if (mailbox->cache_buf.s)
	map_free((const char **)&mailbox->cache_buf.s, &mailbox->cache_len);

	/* open and map the index file */
	fname = mailbox_meta_fname(mailbox, META_INDEX);
	if (!fname)
	return IMAP_MAILBOX_BADNAME;

	mailbox->index_fd = open(fname, O_RDWR, 0);
	if (mailbox->index_fd == -1)
	return IMAP_IOERROR;

	/* don't open the cache yet, it will be loaded by lazy-loading
	* later */

	fstat(mailbox->index_fd, &sbuf);
	mailbox->index_ino = sbuf.st_ino;
	mailbox->index_mtime = sbuf.st_mtime;
	mailbox->index_size = sbuf.st_size;
	map_refresh(mailbox->index_fd, 0, &mailbox->index_base,
	&mailbox->index_len, mailbox->index_size,
	"index", mailbox->name);

	return 0;
	}

	void mailbox_index_dirty(struct mailbox *mailbox)
	{
	assert(mailbox_index_islocked(mailbox, 1));
	mailbox->i.dirty = 1;
	}

	void mailbox_modseq_dirty(struct mailbox *mailbox)
	{
	assert(mailbox_index_islocked(mailbox, 1));

	if (mailbox->modseq_dirty)
	return;

	mailbox->i.highestmodseq++;
	mailbox->last_updated = time(0);
	mailbox->modseq_dirty = 1;
	mailbox_index_dirty(mailbox);
	}

	/*
	* Close the mailbox 'mailbox', freeing all associated resources.
	*/
	void mailbox_close(struct mailbox **mailboxptr)
	{
	int flag;
	struct mailbox mailbox = mailboxptr;
	struct mailboxlist *listitem;
	int expunge_days = config_getint(IMAPOPT_EXPUNGE_DAYS);

	listitem = find_listitem(mailbox->name);
	assert(listitem && &listitem->m == mailbox);

	*mailboxptr = NULL;

	/* open multiple times? Just close this one */
	if (listitem->nopen > 1) {
	listitem->nopen--;
	mailbox_unlock_index(mailbox, NULL);
	return;
	}

	/* auto-cleanup */
	if (mailbox->i.first_expunged &&
	(mailbox->index_locktype == LOCK_EXCLUSIVE)) {
	time_t floor = time(NULL) - (expunge_days * 86400);
	/* but only if we're more than a full week older than
	* the expunge time, * so it doesn't turn into lots
	* of bitty rewrites.
	* Also, cyr_expire can get first bite if it's been set
	* to run... */
	if (mailbox->i.first_expunged < floor - (8 * 86400)) {
	mailbox_expunge_cleanup(mailbox, floor, NULL);
	/* XXX - handle error code? */
	}
	}

	/* drop the index lock here because we'll lose our right to it
	* when try to upgrade the mboxlock anyway. */
	mailbox_unlock_index(mailbox, NULL);

	/* do we need to try and clean up? (not if doing a shutdown,
	* speed is probably more important!) */
	if (!in_shutdown && (mailbox->i.options & MAILBOX_CLEANUP_MASK)) {
	int r = mailbox_mboxlock_upgrade(listitem, LOCK_NONBLOCKING);
	if (!r) r = mailbox_lock_index(mailbox, LOCK_EXCLUSIVE);
	if (!r) {
	/* finish cleaning up */
	if (mailbox->i.options & OPT_MAILBOX_DELETED)
	mailbox_delete_cleanup(mailbox);
	else if (mailbox->i.options & OPT_MAILBOX_NEEDS_REPACK)
	mailbox_index_repack(mailbox);
	else if (mailbox->i.options & OPT_MAILBOX_NEEDS_UNLINK)
	mailbox_index_unlink(mailbox);
	/* or we missed out - someone else beat us to it */
	}
	/* otherwise someone else has the mailbox locked
	* already, so they can handle the cleanup in
	* THEIR mailbox_close call */
	}

	if (mailbox->index_base)
	map_free(&mailbox->index_base, &mailbox->index_len);
	if (mailbox->cache_buf.s)
	map_free((const char **)&mailbox->cache_buf.s, &mailbox->cache_len);

	if (mailbox->index_fd != -1)
	close(mailbox->index_fd);
	if (mailbox->cache_fd != -1)
	close(mailbox->cache_fd);
	if (mailbox->header_fd != -1)
	close(mailbox->header_fd);

	/* once we close this we're committed - no consistency
	* guarantees left */
	if (mailbox->lock_fd != -1)
	close(mailbox->lock_fd);

	free(mailbox->name);
	free(mailbox->part);
	free(mailbox->acl);
	free(mailbox->uniqueid);
	free(mailbox->quotaroot);

	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	free(mailbox->flagname[flag]);
	}

	if (listitem->l) mboxname_release(&listitem->l);

	remove_listitem(listitem);
	}

	/*
	* Read the header of 'mailbox'
	*/
	int mailbox_read_header(struct mailbox mailbox, char *aclptr)
	{
	int r = 0;
	int flag;
	const char name, p, tab, eol;
	int oldformat = 0;
	const char *fname;
	struct stat sbuf;
	const char *base = NULL;
	unsigned long len = 0;
	unsigned magic_size = sizeof(MAILBOX_HEADER_MAGIC) - 1;

	/* can't be dirty if we're reading it */
	if (mailbox->header_dirty)
	abort();

	if (mailbox->header_fd != -1)
	close(mailbox->header_fd);

	fname = mailbox_meta_fname(mailbox, META_HEADER);
	mailbox->header_fd = open(fname, O_RDONLY, 0);

	if (mailbox->header_fd == -1) {
	r = IMAP_IOERROR;
	goto done;
	}

	if (fstat(mailbox->header_fd, &sbuf) == -1) {
	close(mailbox->header_fd);
	mailbox->header_fd = 1;
	r = IMAP_IOERROR;
	goto done;
	}

	map_refresh(mailbox->header_fd, 1, &base, &len,
	sbuf.st_size, "header", mailbox->name);
	mailbox->header_file_ino = sbuf.st_ino;
	mailbox->header_file_crc = crc32_map(base, sbuf.st_size);

	/* Check magic number */
	if ((unsigned) sbuf.st_size < magic_size \|\|
	strncmp(base, MAILBOX_HEADER_MAGIC, magic_size)) {
	r = IMAP_MAILBOX_BADFORMAT;
	goto done;
	}

	/* Read quota file pathname */
	p = base + sizeof(MAILBOX_HEADER_MAGIC)-1;
	tab = memchr(p, '\t', sbuf.st_size - (p - base));
	eol = memchr(p, '\n', sbuf.st_size - (p - base));
	if (!tab \|\| tab > eol \|\| !eol) {
	oldformat = 1;
	if (!eol) {
	r = IMAP_MAILBOX_BADFORMAT;
	goto done;
	}
	else {
	syslog(LOG_DEBUG, "mailbox '%s' has old cyrus.header",
	mailbox->name);
	}
	tab = eol;
	}
	free(mailbox->quotaroot);
	mailbox->quotaroot = NULL;
	if (p < tab) {
	mailbox->quotaroot = xstrndup(p, tab - p);
	}

	if (oldformat) {
	/* uniqueid needs to be generated when we know the uidvalidity */
	mailbox->uniqueid = NULL;
	} else {
	/* read uniqueid */
	p = tab + 1;
	if (p == eol) {
	r = IMAP_MAILBOX_BADFORMAT;
	goto done;
	}
	mailbox->uniqueid = xstrndup(p, eol - p);
	}

	/* Read names of user flags */
	p = eol + 1;
	eol = memchr(p, '\n', sbuf.st_size - (p - base));
	if (!eol) {
	r = IMAP_MAILBOX_BADFORMAT;
	goto done;
	}
	name = p;
	/* read the names of flags */
	for (flag = 0; name <= eol && flag < MAX_USER_FLAGS; flag++) {
	free(mailbox->flagname[flag]);
	mailbox->flagname[flag] = NULL;
	p = memchr(name, ' ', eol-name);
	if (!p) p = eol;
	if (name != p)
	mailbox->flagname[flag] = xstrndup(name, p-name);
	name = p+1;
	}
	/* zero out the rest */
	for (; flag < MAX_USER_FLAGS; flag++) {
	free(mailbox->flagname[flag]);
	mailbox->flagname[flag] = NULL;
	}

	/* Read ACL */
	p = eol + 1;
	eol = memchr(p, '\n', sbuf.st_size - (p - base));
	if (!eol) {
	r = IMAP_MAILBOX_BADFORMAT;
	goto done;
	}

	if (aclptr)
	*aclptr = xstrndup(p, eol-p);

	done:
	if (base) map_free(&base, &len);
	return r;
	}

	/* set a new ACL - only dirty if changed */
	int mailbox_set_acl(struct mailbox mailbox, const char acl)
	{
	if (mailbox->acl) {
	if (!strcmp(mailbox->acl, acl))
	return 0; /* no change */
	free(mailbox->acl);
	}
	mailbox->acl = xstrdup(acl);
	mailbox->header_dirty = 1;
	return 0;
	}

	/* set a new QUOTAROOT - only dirty if changed */
	int mailbox_set_quotaroot(struct mailbox mailbox, const char quotaroot)
	{
	if (mailbox->quotaroot) {
	if (quotaroot && !strcmp(mailbox->quotaroot, quotaroot))
	return 0; /* no change */
	free(mailbox->quotaroot);
	mailbox->quotaroot = NULL;
	}
	if (quotaroot)
	mailbox->quotaroot = xstrdup(quotaroot);
	mailbox->header_dirty = 1;
	return 0;
	}

	/* find or create a user flag - dirty header if change needed */
	int mailbox_user_flag(struct mailbox mailbox, const char flag,
	int *flagnum)
	{
	int userflag;
	int emptyflag = -1;

	for (userflag = 0; userflag < MAX_USER_FLAGS; userflag++) {
	if (mailbox->flagname[userflag]) {
	if (!strcasecmp(flag, mailbox->flagname[userflag]))
	break;
	}
	else if (emptyflag == -1) {
	emptyflag = userflag;
	}
	}

	if (userflag == MAX_USER_FLAGS) {
	if (emptyflag == -1)
	return IMAP_USERFLAG_EXHAUSTED;

	/* need to be index locked to make flag changes */
	if (!mailbox_index_islocked(mailbox, 1))
	return IMAP_MAILBOX_LOCKED;

	/* set the flag and mark the header dirty */
	userflag = emptyflag;
	mailbox->flagname[userflag] = xstrdup(flag);
	mailbox->header_dirty = 1;
	}

	*flagnum = userflag;

	return 0;
	}

	int mailbox_buf_to_index_header(struct index_header i, const char buf)
	{
	uint32_t crc;

	i->dirty = 0;

	i->generation_no = ntohl(((bit32 )(buf+OFFSET_GENERATION_NO)));
	i->format = ntohl(((bit32 )(buf+OFFSET_FORMAT)));
	i->minor_version = ntohl(((bit32 )(buf+OFFSET_MINOR_VERSION)));
	i->start_offset = ntohl(((bit32 )(buf+OFFSET_START_OFFSET)));
	i->record_size = ntohl(((bit32 )(buf+OFFSET_RECORD_SIZE)));
	i->num_records = ntohl(((bit32 )(buf+OFFSET_NUM_RECORDS)));
	i->last_appenddate = ntohl(((bit32 )(buf+OFFSET_LAST_APPENDDATE)));
	i->last_uid = ntohl(((bit32 )(buf+OFFSET_LAST_UID)));
	#ifdef HAVE_LONG_LONG_INT
	i->quota_mailbox_used = align_ntohll(buf+OFFSET_QUOTA_MAILBOX_USED64);
	#else
	i->quota_mailbox_used = ntohl(((bit32 )(buf+OFFSET_QUOTA_MAILBOX_USED)));
	#endif
	i->pop3_last_login = ntohl(((bit32 )(buf+OFFSET_POP3_LAST_LOGIN)));
	i->uidvalidity = ntohl(((bit32 )(buf+OFFSET_UIDVALIDITY)));
	i->deleted = ntohl(((bit32 )(buf+OFFSET_DELETED)));
	i->answered = ntohl(((bit32 )(buf+OFFSET_ANSWERED)));
	i->flagged = ntohl(((bit32 )(buf+OFFSET_FLAGGED)));
	i->options = ntohl(((bit32 )(buf+OFFSET_MAILBOX_OPTIONS)));
	i->leaked_cache_records = ntohl(((bit32 )(buf+OFFSET_LEAKED_CACHE)));
	#ifdef HAVE_LONG_LONG_INT
	i->highestmodseq = align_ntohll(buf+OFFSET_HIGHESTMODSEQ_64);
	i->deletedmodseq = align_ntohll(buf+OFFSET_DELETEDMODSEQ_64);
	#else
	i->highestmodseq = ntohl(((bit32 )(buf+OFFSET_HIGHESTMODSEQ)));
	i->deletedmodseq = ntohl(((bit32 )(buf+OFFSET_DELETEDMODSEQ)));
	#endif
	i->exists = ntohl(((bit32 )(buf+OFFSET_EXISTS)));
	i->first_expunged = ntohl(((bit32 )(buf+OFFSET_FIRST_EXPUNGED)));
	i->last_repack_time = ntohl(((bit32 )(buf+OFFSET_LAST_REPACK_TIME)));
	i->header_file_crc = ntohl(((bit32 )(buf+OFFSET_HEADER_FILE_CRC)));
	i->sync_crc = ntohl(((bit32 )(buf+OFFSET_SYNC_CRC)));
	i->recentuid = ntohl(((bit32 )(buf+OFFSET_RECENTUID)));
	i->recenttime = ntohl(((bit32 )(buf+OFFSET_RECENTTIME)));
	i->header_crc = ntohl(((bit32 )(buf+OFFSET_HEADER_CRC)));
	i->pop3_show_after = ntohl(((bit32 )(buf+OFFSET_POP3_SHOW_AFTER)));

	if (!i->exists)
	i->options \|= OPT_POP3_NEW_UIDL;

	crc = crc32_map(buf, OFFSET_HEADER_CRC);
	if (crc != i->header_crc)
	return IMAP_MAILBOX_CRC;

	return 0;
	}

	static int mailbox_refresh_index_map(struct mailbox *mailbox)
	{
	size_t need_size;
	struct stat sbuf;

	/* check if we need to extend the mmaped space for the index file
	* (i.e. new records appended since last read) */
	need_size = mailbox->i.start_offset +
	mailbox->i.num_records * mailbox->i.record_size;
	if (mailbox->index_size < need_size) {
	if (fstat(mailbox->index_fd, &sbuf) == -1)
	return IMAP_IOERROR;

	if (sbuf.st_size < (int)need_size)
	return IMAP_MAILBOX_BADFORMAT;

	mailbox->index_size = sbuf.st_size;

	/* need to map some more space! */
	map_refresh(mailbox->index_fd, 1, &mailbox->index_base,
	&mailbox->index_len, mailbox->index_size,
	"index", mailbox->name);

	/* and the cache will be stale too */
	mailbox->need_cache_refresh = 1;
	}

	return 0;
	}

	static int mailbox_read_index_header(struct mailbox *mailbox)
	{
	int r;

	/* no dirty mailboxes please */
	if (mailbox->i.dirty)
	abort();

	/* need to be locked to ensure a consistent read - otherwise
	* a busy mailbox will get CRC errors due to rewrite happening
	* under our feet! */
	if (!mailbox_index_islocked(mailbox, 0))
	return IMAP_MAILBOX_LOCKED;

	/* and of course it needs to exist and have at least a full
	* sized header */
	if (!mailbox->index_base)
	return IMAP_MAILBOX_BADFORMAT;
	if (mailbox->index_size < OFFSET_HEADER_SIZE)
	return IMAP_MAILBOX_BADFORMAT;

	r = mailbox_buf_to_index_header(&mailbox->i, mailbox->index_base);
	if (r) return r;

	r = mailbox_refresh_index_map(mailbox);
	if (r) return r;

	return 0;
	}

	/*
	* Read an index record from a mapped index file
	*/
	int mailbox_buf_to_index_record(const char *buf,
	struct index_record *record)
	{
	uint32_t crc;
	int n;

	/* tracking fields - initialise */
	memset(record, 0, sizeof(struct index_record));

	/* parse buffer in to structure */
	record->uid = ntohl(((bit32 )(buf+OFFSET_UID)));
	record->internaldate = ntohl(((bit32 )(buf+OFFSET_INTERNALDATE)));
	record->sentdate = ntohl(((bit32 )(buf+OFFSET_SENTDATE)));
	record->size = ntohl(((bit32 )(buf+OFFSET_SIZE)));
	record->header_size = ntohl(((bit32 )(buf+OFFSET_HEADER_SIZE)));
	record->gmtime = ntohl(((bit32 )(buf+OFFSET_GMTIME)));
	record->cache_offset = ntohl(((bit32 )(buf+OFFSET_CACHE_OFFSET)));
	record->last_updated = ntohl(((bit32 )(buf+OFFSET_LAST_UPDATED)));
	record->system_flags = ntohl(((bit32 )(buf+OFFSET_SYSTEM_FLAGS)));
	for (n = 0; n < MAX_USER_FLAGS/32; n++) {
	record->user_flags[n] = ntohl(((bit32 )(buf+OFFSET_USER_FLAGS+4*n)));
	}
	record->content_lines = ntohl(((bit32 )(buf+OFFSET_CONTENT_LINES)));
	record->cache_version = ntohl(((bit32 )(buf+OFFSET_CACHE_VERSION)));
	message_guid_import(&record->guid, (unsigned char *)buf+OFFSET_MESSAGE_GUID);
	#ifdef HAVE_LONG_LONG_INT
	record->modseq = ntohll(((bit64 )(buf+OFFSET_MODSEQ_64)));
	#else
	record->modseq = ntohl(((bit32 )(buf+OFFSET_MODSEQ)));
	#endif
	record->cache_crc = ntohl(((bit32 )(buf+OFFSET_CACHE_CRC)));
	record->record_crc = ntohl(((bit32 )(buf+OFFSET_RECORD_CRC)));

	/* check CRC32 */
	crc = crc32_map(buf, OFFSET_RECORD_CRC);
	if (crc != record->record_crc)
	return IMAP_MAILBOX_CRC;

	return 0;
	}

	/*
	* Read an index record from a mailbox
	*/
	int mailbox_read_index_record(struct mailbox *mailbox,
	uint32_t recno,
	struct index_record *record)
	{
	const char *buf;
	unsigned offset;
	int r;

	offset = mailbox->i.start_offset + (recno-1) * mailbox->i.record_size;

	if (offset + mailbox->i.record_size > mailbox->index_size) {
	syslog(LOG_ERR,
	"IOERROR: index record %u for %s past end of file",
	recno, mailbox->name);
	return IMAP_IOERROR;
	}

	buf = mailbox->index_base + offset;

	r = mailbox_buf_to_index_record(buf, record);

	if (!r) record->recno = recno;

	return r;
	}

	/*
	* Lock the index file for 'mailbox'. Reread index file header if necessary.
	*/
	int mailbox_lock_index(struct mailbox *mailbox, int locktype)
	{
	char *fname;
	struct stat sbuf;
	int r = 0;

	assert(mailbox->index_fd != -1);
	assert(!mailbox->index_locktype);

	restart:

	if (locktype == LOCK_EXCLUSIVE)
	r = lock_blocking(mailbox->index_fd);
	else
	r = lock_shared(mailbox->index_fd);

	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: locking index for %s: %m",
	mailbox->name);
	return IMAP_IOERROR;
	}

	mailbox->index_locktype = locktype;

	fname = mailbox_meta_fname(mailbox, META_HEADER);
	r = stat(fname, &sbuf);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: stating header %s for %s: %m",
	fname, mailbox->name);
	mailbox_unlock_index(mailbox, NULL);
	return IMAP_IOERROR;
	}

	/* has the header file changed? */
	if (sbuf.st_ino != mailbox->header_file_ino) {
	r = mailbox_read_header(mailbox, NULL);
	if (r) {
	syslog(LOG_ERR, "IOERROR: reading header for %s: %m",
	mailbox->name);
	mailbox_unlock_index(mailbox, NULL);
	return r;
	}
	}

	/* make sure the mailbox is up to date if we haven't
	* already had a successful load */
	if (!mailbox->i.minor_version) {
	bit32 minor_version = ntohl(((bit32 )(mailbox->index_base+OFFSET_MINOR_VERSION)));
	if (minor_version != MAILBOX_MINOR_VERSION) {
	struct mailboxlist *listitem = find_listitem(mailbox->name);
	r = mailbox_mboxlock_upgrade(listitem, LOCK_EXCLUSIVE);
	if (r) return r;
	/* lie about our index lock status - the exclusive namelock
	* provides equivalent properties - and we know it won't
	* leak because the 'restart' above will cover up our sins */
	mailbox->index_locktype = LOCK_EXCLUSIVE;
	r = upgrade_index(mailbox);
	if (r) return r;
	goto restart;
	}
	}

	/* note: it's guaranteed by our outer cyrus.lock lock that the
	* cyrus.index and cyrus.cache files are never rewritten, so
	* we're safe to just extend the map if needed */
	r = mailbox_read_index_header(mailbox);
	if (r) {
	syslog(LOG_ERR, "IOERROR: refreshing index for %s: %m",
	mailbox->name);
	mailbox_unlock_index(mailbox, NULL);
	return r;
	}

	/* check the CRC */
	if (mailbox->header_file_crc != mailbox->i.header_file_crc) {
	syslog(LOG_ERR, "IOERROR: header CRC mismatch %s: %08X %08X",
	mailbox->name, (unsigned int)mailbox->header_file_crc,
	(unsigned int)mailbox->i.header_file_crc);
	mailbox_unlock_index(mailbox, NULL);
	return IMAP_MAILBOX_CRC;
	}

	/* fix up 2.4.0 bug breakage */
	if (mailbox->i.uidvalidity == 0) {
	mailbox->i.uidvalidity = time(0);
	if (locktype == LOCK_EXCLUSIVE) {
	mailbox_index_dirty(mailbox);
	mailbox_commit(mailbox);
	syslog(LOG_ERR, "%s: fixing zero uidvalidity", mailbox->name);
	}
	}
	if (mailbox->i.highestmodseq == 0) {
	mailbox->i.highestmodseq = 1;
	if (locktype == LOCK_EXCLUSIVE) {
	mailbox_index_dirty(mailbox);
	mailbox_commit(mailbox);
	syslog(LOG_ERR, "%s: fixing zero highestmodseq", mailbox->name);
	}
	}

	return 0;
	}

	/*
	* Release lock on the index file for 'mailbox'
	*/
	void mailbox_unlock_index(struct mailbox mailbox, struct statusdata sdata)
	{
	/* naughty - you can't unlock a dirty mailbox! */
	if (mailbox->i.dirty \|\| mailbox->header_dirty \|\|
	mailbox->modseq_dirty \|\| mailbox->quota_dirty)
	abort();

	if (mailbox->has_changed) {
	if (updatenotifier) updatenotifier(mailbox->name);
	sync_log_mailbox(mailbox->name);
	if (config_getswitch(IMAPOPT_STATUSCACHE))
	statuscache_invalidate(mailbox->name, sdata);
	mailbox->has_changed = 0;
	}

	if (mailbox->index_locktype) {
	if (lock_unlock(mailbox->index_fd))
	syslog(LOG_ERR, "IOERROR: unlocking index of %s: %m",
	mailbox->name);
	}

	mailbox->index_locktype = 0;
	}

	/*
	* Write the header file for 'mailbox'
	*/
	int mailbox_commit_header(struct mailbox *mailbox)
	{
	int flag;
	int fd;
	int r = 0;
	char *quotaroot;
	const char *newfname;
	struct iovec iov[10];
	int niov;

	if (!mailbox->header_dirty)
	return 0; /* nothing to write! */

	/* we actually do all header actions under an INDEX lock, because
	* we need to write the crc32 to be consistent! */
	assert(mailbox_index_islocked(mailbox, 1));

	newfname = mailbox_meta_newfname(mailbox, META_HEADER);

	fd = open(newfname, O_CREAT \| O_TRUNC \| O_RDWR, 0666);
	if (fd == -1) {
	syslog(LOG_ERR, "IOERROR: opening %s: %m", newfname);
	return IMAP_IOERROR;
	}

	/* Write magic header, do NOT write the trailing NUL */
	r = write(fd, MAILBOX_HEADER_MAGIC,
	sizeof(MAILBOX_HEADER_MAGIC) - 1);

	if (r != -1) {
	niov = 0;
	quotaroot = mailbox->quotaroot ? mailbox->quotaroot : "";
	WRITEV_ADDSTR_TO_IOVEC(iov,niov,quotaroot);
	WRITEV_ADD_TO_IOVEC(iov,niov,"\t",1);
	WRITEV_ADDSTR_TO_IOVEC(iov,niov,mailbox->uniqueid);
	WRITEV_ADD_TO_IOVEC(iov,niov,"\n",1);
	r = retry_writev(fd, iov, niov);
	}

	if (r != -1) {
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (mailbox->flagname[flag]) {
	niov = 0;
	WRITEV_ADDSTR_TO_IOVEC(iov,niov,mailbox->flagname[flag]);
	WRITEV_ADD_TO_IOVEC(iov,niov," ",1);
	r = retry_writev(fd, iov, niov);
	if(r == -1) break;
	}
	}
	}

	if (r != -1) {
	niov = 0;
	WRITEV_ADD_TO_IOVEC(iov,niov,"\n",1);
	WRITEV_ADDSTR_TO_IOVEC(iov,niov,mailbox->acl);
	WRITEV_ADD_TO_IOVEC(iov,niov,"\n",1);
	r = retry_writev(fd, iov, niov);
	}

	if (r == -1 \|\| fsync(fd)) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", newfname);
	close(fd);
	unlink(newfname);
	return IMAP_IOERROR;
	}

	close(fd);

	/* rename the new header file over the old one */
	r = mailbox_meta_rename(mailbox, META_HEADER);
	if (r) return r;
	mailbox->header_dirty = 0; /* we wrote it out, so not dirty any more */

	/* re-read the header */
	r = mailbox_read_header(mailbox, NULL);
	if (r) return r;

	/* copy the new CRC into the index header */
	mailbox->i.header_file_crc = mailbox->header_file_crc;
	mailbox_index_dirty(mailbox);

	return 0;
	}

	bit32 mailbox_index_header_to_buf(struct index_header i, unsigned char buf)
	{
	bit32 crc;

	((bit32 )(buf+OFFSET_GENERATION_NO)) = htonl(i->generation_no);
	((bit32 )(buf+OFFSET_FORMAT)) = htonl(i->format);
	((bit32 )(buf+OFFSET_MINOR_VERSION)) = htonl(i->minor_version);
	((bit32 )(buf+OFFSET_START_OFFSET)) = htonl(i->start_offset);
	((bit32 )(buf+OFFSET_RECORD_SIZE)) = htonl(i->record_size);
	((bit32 )(buf+OFFSET_NUM_RECORDS)) = htonl(i->num_records);
	((bit32 )(buf+OFFSET_LAST_APPENDDATE)) = htonl(i->last_appenddate);
	((bit32 )(buf+OFFSET_LAST_UID)) = htonl(i->last_uid);

	/* quotas may be 64bit now */
	#ifdef HAVE_LONG_LONG_INT
	((bit64 )(buf+OFFSET_QUOTA_MAILBOX_USED64)) = htonll(i->quota_mailbox_used);
	#else
	/* zero the unused 32bits */
	((bit32 )(buf+OFFSET_QUOTA_MAILBOX_USED64)) = htonl(0);
	((bit32 )(buf+OFFSET_QUOTA_MAILBOX_USED)) = htonl(i->quota_mailbox_used);
	#endif

	((bit32 )(buf+OFFSET_POP3_LAST_LOGIN)) = htonl(i->pop3_last_login);
	((bit32 )(buf+OFFSET_UIDVALIDITY)) = htonl(i->uidvalidity);
	((bit32 )(buf+OFFSET_DELETED)) = htonl(i->deleted);
	((bit32 )(buf+OFFSET_ANSWERED)) = htonl(i->answered);
	((bit32 )(buf+OFFSET_FLAGGED)) = htonl(i->flagged);
	((bit32 )(buf+OFFSET_MAILBOX_OPTIONS)) = htonl(i->options & OPT_VALID);
	((bit32 )(buf+OFFSET_LEAKED_CACHE)) = htonl(i->leaked_cache_records);
	#ifdef HAVE_LONG_LONG_INT
	align_htonll(buf+OFFSET_HIGHESTMODSEQ_64, i->highestmodseq);
	align_htonll(buf+OFFSET_DELETEDMODSEQ_64, i->deletedmodseq);
	#else
	/* zero the unused 32bits */
	((bit32 )(buf+OFFSET_HIGHESTMODSEQ_64)) = htonl(0);
	((bit32 )(buf+OFFSET_HIGHESTMODSEQ)) = htonl(i->highestmodseq);
	/* zero the unused 32bits */
	((bit32 )(buf+OFFSET_DELETEDMODSEQ_64)) = htonl(0);
	((bit32 )(buf+OFFSET_DELETEDMODSEQ)) = htonl(i->deletedmodseq);
	#endif
	((bit32 )(buf+OFFSET_EXISTS)) = htonl(i->exists);
	((bit32 )(buf+OFFSET_FIRST_EXPUNGED)) = htonl(i->first_expunged);
	((bit32 )(buf+OFFSET_LAST_REPACK_TIME)) = htonl(i->last_repack_time);
	((bit32 )(buf+OFFSET_HEADER_FILE_CRC)) = htonl(i->header_file_crc);
	((bit32 )(buf+OFFSET_SYNC_CRC)) = htonl(i->sync_crc);
	((bit32 )(buf+OFFSET_RECENTUID)) = htonl(i->recentuid);
	((bit32 )(buf+OFFSET_RECENTTIME)) = htonl(i->recenttime);
	((bit32 )(buf+OFFSET_POP3_SHOW_AFTER)) = htonl(i->pop3_show_after);
	((bit32 )(buf+OFFSET_SPARE1)) = htonl(0); /* RESERVED */
	((bit32 )(buf+OFFSET_SPARE2)) = htonl(0); /* RESERVED */

	/* Update checksum */
	crc = htonl(crc32_map((char *)buf, OFFSET_HEADER_CRC));
	((bit32 )(buf+OFFSET_HEADER_CRC)) = crc;

	return crc;
	}

	int mailbox_commit_quota(struct mailbox *mailbox)
	{
	struct txn *tid = NULL;
	int r;
	struct quota q;
	quota_t qdiff;

	/* not dirty */
	if (!mailbox->quota_dirty)
	return 0;

	mailbox->quota_dirty = 0;

	/* unchanged */
	qdiff = mailbox->i.quota_mailbox_used - mailbox->quota_previously_used;
	if (!qdiff)
	return 0;

	/* no quota root means we don't track quota. That's OK */
	if (!mailbox->quotaroot)
	return 0;

	assert(mailbox_index_islocked(mailbox, 1));

	q.root = mailbox->quotaroot;
	r = quota_read(&q, &tid, 1);
	if (!r) {
	/* check we won't underflow */
	if ((quota_t)-qdiff > (quota_t)q.used)
	q.used = 0;
	else
	q.used += qdiff;
	r = quota_write(&q, &tid);
	}

	if (!r) quota_commit(&tid);
	else {
	quota_abort(&tid);
	/* XXX - fail here? It's tempting */
	syslog(LOG_ERR, "LOSTQUOTA: unable to record quota file %s",
	mailbox->quotaroot);
	}

	return 0;
	}

	/*
	* Write the index header for 'mailbox'
	*/
	int mailbox_commit(struct mailbox *mailbox)
	{
	/* XXX - ibuf for alignment? */
	static unsigned char buf[INDEX_HEADER_SIZE];
	int n, r;

	/* try to commit sub parts first */
	r = mailbox_commit_cache(mailbox);
	if (r) return r;

	r = mailbox_commit_quota(mailbox);
	if (r) return r;

	r = mailbox_commit_header(mailbox);
	if (r) return r;

	if (!mailbox->i.dirty)
	return 0;

	assert(mailbox_index_islocked(mailbox, 1));

	if (mailbox->i.start_offset < INDEX_HEADER_SIZE)
	fatal("Mailbox offset bug", EC_SOFTWARE);

	mailbox_index_header_to_buf(&mailbox->i, buf);

	lseek(mailbox->index_fd, 0, SEEK_SET);
	n = retry_write(mailbox->index_fd, buf, INDEX_HEADER_SIZE);
	if ((unsigned long)n != INDEX_HEADER_SIZE \|\| fsync(mailbox->index_fd)) {
	syslog(LOG_ERR, "IOERROR: writing index header for %s: %m",
	mailbox->name);
	return IMAP_IOERROR;
	}

	/* remove all dirty flags! */
	mailbox->i.dirty = 0;
	mailbox->modseq_dirty = 0;
	mailbox->header_dirty = 0;

	/* label changes for later logging */
	mailbox->has_changed = 1;

	return 0;
	}

	/*
	* Put an index record into a buffer suitable for writing to a file.
	*/
	bit32 mailbox_index_record_to_buf(struct index_record *record,
	unsigned char *buf)
	{
	int n;
	bit32 crc;

	((bit32 )(buf+OFFSET_UID)) = htonl(record->uid);
	((bit32 )(buf+OFFSET_INTERNALDATE)) = htonl(record->internaldate);
	((bit32 )(buf+OFFSET_SENTDATE)) = htonl(record->sentdate);
	((bit32 )(buf+OFFSET_SIZE)) = htonl(record->size);
	((bit32 )(buf+OFFSET_HEADER_SIZE)) = htonl(record->header_size);
	((bit32 )(buf+OFFSET_GMTIME)) = htonl(record->gmtime);
	((bit32 )(buf+OFFSET_CACHE_OFFSET)) = htonl(record->cache_offset);
	((bit32 )(buf+OFFSET_LAST_UPDATED)) = htonl(record->last_updated);
	((bit32 )(buf+OFFSET_SYSTEM_FLAGS)) = htonl(record->system_flags);
	for (n = 0; n < MAX_USER_FLAGS/32; n++) {
	((bit32 )(buf+OFFSET_USER_FLAGS+4*n)) = htonl(record->user_flags[n]);
	}
	((bit32 )(buf+OFFSET_CONTENT_LINES)) = htonl(record->content_lines);
	((bit32 )(buf+OFFSET_CACHE_VERSION)) = htonl(record->cache_version);
	message_guid_export(&record->guid, buf+OFFSET_MESSAGE_GUID);
	#ifdef HAVE_LONG_LONG_INT
	((bit64 )(buf+OFFSET_MODSEQ_64)) = htonll(record->modseq);
	#else
	/* zero the unused 32bits */
	((bit32 )(buf+OFFSET_MODSEQ_64)) = htonl(0);
	((bit32 )(buf+OFFSET_MODSEQ)) = htonl(record->modseq);
	#endif
	((bit32 )(buf+OFFSET_CACHE_CRC)) = htonl(record->cache_crc);

	/* calculate the checksum */
	crc = crc32_map((char *)buf, OFFSET_RECORD_CRC);
	((bit32 )(buf+OFFSET_RECORD_CRC)) = htonl(crc);

	return crc;
	}

	bit32 make_sync_crc(struct mailbox mailbox, struct index_record record)
	{
	char buf[4096];
	bit32 flagcrc = 0;
	int flag;

	/* calculate an XORed CRC32 over all the flags on the message, so no
	* matter what order they are store in the header, the final value
	* is the same */
	if (record->system_flags & FLAG_DELETED)
	flagcrc ^= crc32_cstring("\\deleted");
	if (record->system_flags & FLAG_ANSWERED)
	flagcrc ^= crc32_cstring("\\answered");
	if (record->system_flags & FLAG_FLAGGED)
	flagcrc ^= crc32_cstring("\\flagged");
	if (record->system_flags & FLAG_DRAFT)
	flagcrc ^= crc32_cstring("\\draft");
	if (record->system_flags & FLAG_SEEN)
	flagcrc ^= crc32_cstring("\\seen");

	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (!mailbox->flagname[flag])
	continue;
	if (!(record->user_flags[flag/32] & (1<<(flag&31))))
	continue;
	/* need to compare without case being significant */
	strlcpy(buf, mailbox->flagname[flag], 4096);
	lcase(buf);
	flagcrc ^= crc32_cstring(buf);
	}

	/* NOTE: this format is idential to an UPDATE COPY command except
	* that the string format of the flags has been replaced with a
	* checksum over the flags */
	snprintf(buf, 4096, "%u " MODSEQ_FMT " %lu (%u) %lu %s",
	record->uid, record->modseq, record->last_updated,
	flagcrc,
	record->internaldate,
	message_guid_encode(&record->guid));

	return crc32_cstring(buf);
	}

	static void mailbox_quota_dirty(struct mailbox *mailbox)
	{
	/* track quota use */
	if (!mailbox->quota_dirty) {
	mailbox->quota_dirty = 1;
	mailbox->quota_previously_used = mailbox->i.quota_mailbox_used;
	}
	}

	void mailbox_index_update_counts(struct mailbox *mailbox,
	struct index_record *record,
	int is_add)
	{
	int num = is_add ? 1 : -1;

	/* we don't track counts for EXPUNGED records */
	if (record->system_flags & FLAG_EXPUNGED)
	return;

	mailbox_quota_dirty(mailbox);

	/* update mailbox header fields */
	if (record->system_flags & FLAG_ANSWERED)
	mailbox->i.answered += num;

	if (record->system_flags & FLAG_FLAGGED)
	mailbox->i.flagged += num;

	if (record->system_flags & FLAG_DELETED)
	mailbox->i.deleted += num;

	if (is_add) {
	mailbox->i.exists++;
	mailbox->i.quota_mailbox_used += record->size;
	}
	else {
	if (mailbox->i.exists)
	mailbox->i.exists--;

	/* corruption prevention - check we don't go negative */
	if (mailbox->i.quota_mailbox_used > record->size)
	mailbox->i.quota_mailbox_used -= record->size;
	else
	mailbox->i.quota_mailbox_used = 0;
	}
	mailbox->i.sync_crc ^= make_sync_crc(mailbox, record);

	mailbox_index_dirty(mailbox);
	}

	int mailbox_index_recalc(struct mailbox *mailbox)
	{
	struct index_record record;
	int r = 0;
	uint32_t recno;

	assert(mailbox_index_islocked(mailbox, 1));

	/* cache the old used quota */
	mailbox_quota_dirty(mailbox);
	mailbox_index_dirty(mailbox);

	mailbox->i.answered = 0;
	mailbox->i.flagged = 0;
	mailbox->i.deleted = 0;
	mailbox->i.exists = 0;
	mailbox->i.quota_mailbox_used = 0;
	mailbox->i.sync_crc = 0;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	r = mailbox_read_index_record(mailbox, recno, &record);
	if (r) return r;
	mailbox_index_update_counts(mailbox, &record, 1);
	}

	return 0;
	}

	/*
	* Rewrite an index record in a mailbox - updates all
	* necessary tracking fields automatically.
	*/
	int mailbox_rewrite_index_record(struct mailbox *mailbox,
	struct index_record *record)
	{
	int n;
	int r;
	struct index_record oldrecord;
	indexbuffer_t ibuf;
	unsigned char *buf = ibuf.buf;
	size_t offset;
	int expunge_mode = config_getenum(IMAPOPT_EXPUNGE_MODE);
	int immediate = (expunge_mode == IMAP_ENUM_EXPUNGE_MODE_IMMEDIATE \|\|
	expunge_mode == IMAP_ENUM_EXPUNGE_MODE_DEFAULT);

	assert(mailbox_index_islocked(mailbox, 1));
	assert(record->recno > 0 &&
	record->recno <= mailbox->i.num_records);

	r = mailbox_read_index_record(mailbox, record->recno, &oldrecord);
	if (r) return r;

	/* the UID has to match, of course, for it to be the same
	* record. XXX - possibly test all the other suposedly
	* invarient fields here too? */
	if (record->uid != oldrecord.uid)
	return IMAP_IOERROR;

	if (oldrecord.system_flags & FLAG_EXPUNGED) {
	/* it is a sin to unexpunge a message. unexpunge.c copies
	* the data from the old record and appends it with a new
	* UID, which is righteous in the eyes of the IMAP client */
	if (!(record->system_flags & FLAG_EXPUNGED))
	return IMAP_IOERROR;
	}

	/* handle immediate expunges here... */
	if (immediate && (record->system_flags & FLAG_EXPUNGED))
	record->system_flags \|= FLAG_UNLINKED;

	if (record->system_flags & FLAG_UNLINKED) {
	if (expunge_mode == IMAP_ENUM_EXPUNGE_MODE_IMMEDIATE)
	mailbox->i.options \|= OPT_MAILBOX_NEEDS_REPACK;
	mailbox->i.options \|= OPT_MAILBOX_NEEDS_UNLINK;
	}

	/* make sure highestmodseq gets updated unless we're
	* being silent about it (i.e. marking an already EXPUNGED
	* message as UNLINKED, or just updating the content_lines
	* field or cache_offset) */
	if (record->silent) {
	mailbox_index_dirty(mailbox);
	}
	else {
	mailbox_modseq_dirty(mailbox);
	record->modseq = mailbox->i.highestmodseq;
	record->last_updated = mailbox->last_updated;
	}

	/* remove the counts for the old copy, and add them for
	* the new copy */
	mailbox_index_update_counts(mailbox, &oldrecord, 0);
	mailbox_index_update_counts(mailbox, record, 1);

	mailbox_index_record_to_buf(record, buf);

	offset = mailbox->i.start_offset +
	(record->recno-1) * mailbox->i.record_size;

	n = lseek(mailbox->index_fd, offset, SEEK_SET);
	if (n == -1) {
	syslog(LOG_ERR, "IOERROR: seeking index record %u for %s: %m",
	record->recno, mailbox->name);
	return IMAP_IOERROR;
	}

	n = retry_write(mailbox->index_fd, buf, INDEX_RECORD_SIZE);
	if (n != INDEX_RECORD_SIZE) {
	syslog(LOG_ERR, "IOERROR: writing index record %u for %s: %m",
	record->recno, mailbox->name);
	return IMAP_IOERROR;
	}

	/* expunged tracking */
	if ((record->system_flags & FLAG_EXPUNGED) &&
	!(oldrecord.system_flags & FLAG_EXPUNGED)) {
	if (!mailbox->i.first_expunged \|\|
	mailbox->i.first_expunged > record->last_updated)
	mailbox->i.first_expunged = record->last_updated;

	if (config_auditlog)
	syslog(LOG_NOTICE, "auditlog: expunge sessionid=<%s> "
	"mailbox=<%s> uniqueid=<%s> uid=<%u> guid=<%s>",
	session_id(), mailbox->name, mailbox->uniqueid,
	record->uid, message_guid_encode(&record->guid));
	}

	return 0;
	}

	/* append a single message to a mailbox - also updates everything
	* automatically. These two functions are the ONLY way to modify
	* the contents or tracking fields of a message */
	int mailbox_append_index_record(struct mailbox *mailbox,
	struct index_record *record)
	{
	indexbuffer_t ibuf;
	unsigned char *buf = ibuf.buf;
	size_t offset;
	int r;
	int n;
	struct utimbuf settime;

	assert(mailbox_index_islocked(mailbox, 1));

	/* Append MUST be a higher UID than any we've yet seen */
	if (record->uid <= mailbox->i.last_uid)
	return IMAP_IOERROR; /* XXX - better code */

	if (!(record->system_flags & FLAG_UNLINKED)) {
	/* make the file timestamp correct */
	settime.actime = settime.modtime = record->internaldate;
	if (utime(mailbox_message_fname(mailbox, record->uid), &settime) == -1)
	return IMAP_IOERROR;

	/* write the cache record before buffering the message, it
	* will set the cache_offset field. */
	r = mailbox_append_cache(mailbox, record);
	if (r) return r;
	}

	/* update the highestmodseq if needed */
	if (record->silent) {
	mailbox_index_dirty(mailbox);
	}
	else {
	mailbox_modseq_dirty(mailbox);
	record->modseq = mailbox->i.highestmodseq;
	record->last_updated = mailbox->last_updated;
	}

	/* add counts */
	mailbox_index_update_counts(mailbox, record, 1);

	mailbox_index_record_to_buf(record, buf);

	offset = mailbox->i.start_offset +
	(mailbox->i.num_records * mailbox->i.record_size);

	n = lseek(mailbox->index_fd, offset, SEEK_SET);
	if (n == -1) {
	syslog(LOG_ERR, "IOERROR: seeking to append for %s: %m",
	mailbox->name);
	return IMAP_IOERROR;
	}

	n = retry_write(mailbox->index_fd, buf, INDEX_RECORD_SIZE);
	if (n != INDEX_RECORD_SIZE) {
	syslog(LOG_ERR, "IOERROR: appending index record for %s: %m",
	mailbox->name);
	return IMAP_IOERROR;
	}

	mailbox->i.last_uid = record->uid;
	mailbox->i.num_records++;
	mailbox->index_size += INDEX_RECORD_SIZE;

	/* extend the mmaped space for the index file */
	if (mailbox->index_len < mailbox->index_size) {
	map_refresh(mailbox->index_fd, 1, &mailbox->index_base,
	&mailbox->index_len, mailbox->index_size,
	"index", mailbox->name);
	}

	if (config_auditlog)
	syslog(LOG_NOTICE, "auditlog: append sessionid=<%s> mailbox=<%s> uniqueid=<%s> uid=<%u> guid=<%s>",
	session_id(), mailbox->name, mailbox->uniqueid, record->uid,
	message_guid_encode(&record->guid));

	/* expunged tracking */
	if (record->system_flags & FLAG_EXPUNGED) {
	if (!mailbox->i.first_expunged \|\|
	mailbox->i.first_expunged > record->last_updated)
	mailbox->i.first_expunged = record->last_updated;

	if (config_auditlog)
	syslog(LOG_NOTICE, "auditlog: expunge sessionid=<%s> "
	"mailbox=<%s> uniqueid=<%s> uid=<%u> guid=<%s>",
	session_id(), mailbox->name, mailbox->uniqueid,
	record->uid, message_guid_encode(&record->guid));
	}

	/* yep, it could even be pre-unlinked in 'default' expunge mode, joy */
	if (record->system_flags & FLAG_UNLINKED) {
	if (config_auditlog)
	syslog(LOG_NOTICE, "auditlog: unlink sessionid=<%s> "
	"mailbox=<%s> uniqueid=<%s> uid=<%u>",
	session_id(), mailbox->name, mailbox->uniqueid,
	record->uid);
	}

	return 0;
	}

	static void mailbox_message_unlink(struct mailbox *mailbox, uint32_t uid)
	{
	const char *fname = mailbox_message_fname(mailbox, uid);

	/* no error, we removed a file */
	if (unlink(fname) == 0) {
	if (config_auditlog)
	syslog(LOG_NOTICE, "auditlog: unlink sessionid=<%s> "
	"mailbox=<%s> uniqueid=<%s> uid=<%u>",
	session_id(), mailbox->name, mailbox->uniqueid, uid);
	}
	}

	/* need a mailbox exclusive lock, we're removing files */
	static int mailbox_index_unlink(struct mailbox *mailbox)
	{
	struct index_record record;
	uint32_t recno;
	int r;

	syslog(LOG_INFO, "Unlinking files in mailbox %s", mailbox->name);

	/* note: this may try to unlink the same files more than once,
	* but them's the breaks - the alternative is yet another
	* system flag which gets updated once done! */
	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	r = mailbox_read_index_record(mailbox, recno, &record);
	if (r) return r;

	if (record.system_flags & FLAG_UNLINKED)
	mailbox_message_unlink(mailbox, record.uid);
	}

	/* need to clear the flag, even if nothing needed unlinking! */
	mailbox_index_dirty(mailbox);
	mailbox->i.options &= ~OPT_MAILBOX_NEEDS_UNLINK;
	mailbox_commit(mailbox);

	return 0;
	}

	/* need a mailbox exclusive lock, we're rewriting files */
	static int mailbox_index_repack(struct mailbox *mailbox)
	{
	char *fname;
	uint32_t newrecno = 1;
	indexbuffer_t ibuf;
	unsigned char *buf = ibuf.buf;
	uint32_t recno;
	int newindex_fd = -1, newcache_fd = -1;
	struct index_record record;
	size_t offset;
	int newgeneration;
	int r = IMAP_IOERROR;
	int n;

	syslog(LOG_INFO, "Repacking mailbox %s", mailbox->name);

	fname = mailbox_meta_newfname(mailbox, META_INDEX);
	newindex_fd = open(fname, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	if (newindex_fd == -1) goto fail;

	fname = mailbox_meta_newfname(mailbox, META_CACHE);
	newcache_fd = open(fname, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	if (newcache_fd == -1) goto fail;

	/* update the generation number */
	newgeneration = mailbox->i.generation_no + 1;
	((bit32 )(buf)) = htonl(newgeneration);
	n = retry_write(newcache_fd, buf, 4);
	if (n != 4) goto fail;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	r = mailbox_read_index_record(mailbox, recno, &record);
	if (r) goto fail;

	/* we aren't keeping unlinked files, that's kind of the point */
	if (record.system_flags & FLAG_UNLINKED) {
	/* just in case it was left lying around */
	/* XXX - log error if unlink fails */
	mailbox_message_unlink(mailbox, record.uid);
	if (record.modseq > mailbox->i.deletedmodseq)
	mailbox->i.deletedmodseq = record.modseq;
	continue;
	}

	/* read in the old cache record */
	r = mailbox_cacherecord(mailbox, &record);
	if (r) goto fail;

	/* write out the new cache record - need to clear the cache_offset
	* so it gets reset in the new record */
	record.cache_offset = 0;
	r = cache_append_record(newcache_fd, &record);
	if (r) goto fail;

	offset = mailbox->i.start_offset +
	(newrecno-1) * mailbox->i.record_size;

	/* write the index record out */
	mailbox_index_record_to_buf(&record, buf);
	n = lseek(newindex_fd, offset, SEEK_SET);
	if (n == -1) {
	syslog(LOG_ERR, "IOERROR: seeking index record %u for %s: %m",
	recno, mailbox->name);
	goto fail;
	}
	n = retry_write(newindex_fd, buf, INDEX_RECORD_SIZE);
	if (n != INDEX_RECORD_SIZE) {
	syslog(LOG_ERR, "IOERROR: writing index record %u for %s: %m",
	recno, mailbox->name);
	goto fail;
	}

	newrecno++;
	}

	/* update final header fields */
	mailbox->i.generation_no = newgeneration;
	mailbox->i.first_expunged = 0;
	mailbox->i.last_repack_time = time(0);
	mailbox->i.num_records = newrecno - 1;
	mailbox->i.leaked_cache_records = 0;
	/* we unlinked any "needs unlink" in the process */
	mailbox->i.options &= ~(OPT_MAILBOX_NEEDS_REPACK\|OPT_MAILBOX_NEEDS_UNLINK);

	mailbox_index_header_to_buf(&mailbox->i, buf);
	n = lseek(newindex_fd, 0, SEEK_SET);
	if (n == -1) goto fail;
	n = retry_write(newindex_fd, buf, INDEX_HEADER_SIZE);
	if (n != INDEX_HEADER_SIZE) goto fail;

	if (fsync(newindex_fd) \|\| fsync(newcache_fd))
	goto fail;

	close(newcache_fd);
	close(newindex_fd);

	r = mailbox_meta_rename(mailbox, META_INDEX);
	if (!r) r = mailbox_meta_rename(mailbox, META_CACHE);

	return r;

	fail:
	if (newcache_fd != -1) close(newcache_fd);
	if (newindex_fd != -1) close(newindex_fd);
	return IMAP_IOERROR;
	}

	/*
	* Used by mailbox_rename() to expunge all messages in INBOX
	*/
	static unsigned expungeall(struct mailbox *mailbox __attribute__((unused)),
	struct index_record *record __attribute__((unused)),
	void *rock __attribute__((unused)))
	{
	return 1;
	}

	/*
	* Expunge decision proc used by mailbox_expunge()
	* to expunge \Deleted messages.
	*/
	static unsigned expungedeleted(struct mailbox *mailbox __attribute__((unused)),
	struct index_record *record,
	void *rock __attribute__((unused)))
	{
	if (record->system_flags & FLAG_DELETED)
	return 1;

	return 0;
	}

	/*
	* Perform an expunge operation on 'mailbox'. If nonzero, the
	* function pointed to by 'decideproc' is called (with 'deciderock') to
	* determine which messages to expunge. If 'decideproc' is a null pointer,
	* then messages with the \Deleted flag are expunged.
	*/
	int mailbox_expunge(struct mailbox *mailbox,
	mailbox_decideproc_t decideproc, void deciderock,
	unsigned *nexpunged)
	{
	int r = 0;
	int numexpunged = 0;
	uint32_t recno;
	struct index_record record;

	assert(mailbox_index_islocked(mailbox, 1));

	/* anything to do? */
	if (!mailbox->i.num_records) {
	if (nexpunged) *nexpunged = 0;
	return 0;
	}

	if (!decideproc) decideproc = expungedeleted;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	r = mailbox_read_index_record(mailbox, recno, &record);
	if (r) continue;

	/* skip already expunged records */
	if (record.system_flags & FLAG_EXPUNGED)
	continue;

	if (decideproc(mailbox, &record, deciderock)) {
	numexpunged++;

	/* mark deleted */
	record.system_flags \|= FLAG_EXPUNGED;

	r = mailbox_rewrite_index_record(mailbox, &record);
	if (r) return IMAP_IOERROR;
	}
	}

	if (numexpunged > 0) {
	syslog(LOG_NOTICE, "Expunged %d messages from %s",
	numexpunged, mailbox->name);
	}

	if (nexpunged) *nexpunged = numexpunged;

	return 0;
	}

	int mailbox_expunge_cleanup(struct mailbox *mailbox, time_t expunge_mark,
	unsigned *ndeleted)
	{
	uint32_t recno;
	int dirty = 0;
	unsigned numdeleted = 0;
	struct index_record record;
	time_t first_expunged = 0;
	int r = 0;

	/* run the actual expunge phase */
	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	if (mailbox_read_index_record(mailbox, recno, &record))
	continue;

	/* already unlinked, skip it (but dirty so we mark a repack is needed) */
	if (record.system_flags & FLAG_UNLINKED) {
	dirty = 1;
	continue;
	}

	/* not actually expunged, skip it */
	if (!(record.system_flags & FLAG_EXPUNGED))
	continue;

	/* not stale enough yet, skip it - but track the updated time
	* so we know when to run again */
	if (record.last_updated > expunge_mark) {
	if (!first_expunged \|\| (first_expunged > record.last_updated))
	first_expunged = record.last_updated;
	continue;
	}

	dirty = 1;

	numdeleted++;

	record.system_flags \|= FLAG_UNLINKED;
	record.silent = 1;
	if (mailbox_rewrite_index_record(mailbox, &record)) {
	syslog(LOG_ERR, "failed to write changes to %s recno %d",
	mailbox->name, recno);
	break;
	}
	}

	if (dirty) {
	mailbox_index_dirty(mailbox);
	mailbox->i.options \|= OPT_MAILBOX_NEEDS_REPACK;
	mailbox->i.first_expunged = first_expunged;
	r = mailbox_commit(mailbox);
	}

	if (ndeleted) *ndeleted = numdeleted;

	return r;
	}

	int mailbox_internal_seen(struct mailbox mailbox, const char userid)
	{
	/* shared seen - everyone's state is internal */
	if (mailbox->i.options & OPT_IMAP_SHAREDSEEN)
	return 1;

	/* no username => use internal as well */
	if (!userid)
	return 1;

	/* otherwise the owner's seen state is internal */
	return mboxname_userownsmailbox(userid, mailbox->name);
	}

	/* returns a mailbox locked in MAILBOX EXCLUSIVE mode, so you
	* don't need to lock the index file to work with it :) */
	int mailbox_create(const char *name,
	const char *part,
	const char *acl,
	const char *uniqueid,
	int options,
	unsigned uidvalidity,
	struct mailbox **mailboxptr)

	{
	int r = 0;
	char quotaroot[MAX_MAILBOX_BUFFER];
	int hasquota;
	char *fname;
	struct mailbox *mailbox = NULL;
	int n;
	char generation_buf[4];
	int createfnames[] = { META_INDEX, META_CACHE, META_HEADER, 0 };
	struct mailboxlist *listitem;

	/* if we already have this name open then that's an error too */
	listitem = find_listitem(name);
	if (listitem) return IMAP_MAILBOX_LOCKED;

	listitem = create_listitem(name);
	mailbox = &listitem->m;

	/* if we can't get an exclusive lock first try, there's something
	* racy going on! */
	r = mboxname_lock(name, &listitem->l, LOCK_NONBLOCKING);
	if (r) goto done;

	mailbox->part = xstrdup(part);
	mailbox->acl = xstrdup(acl);

	hasquota = quota_findroot(quotaroot, sizeof(quotaroot), name);

	/* ensure all paths exist */
	for (n = 0; createfnames[n]; n++) {
	fname = mailbox_meta_fname(mailbox, createfnames[n]);
	if (!fname) {
	syslog(LOG_ERR, "IOERROR: Mailbox name too long (%s)", mailbox->name);
	r = IMAP_MAILBOX_BADNAME;
	goto done;
	}
	if (cyrus_mkdir(fname, 0755) == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fname);
	r = IMAP_IOERROR;
	goto done;
	}
	}

	/* ensure we can fit the longest possible file name */
	fname = mailbox_message_fname(mailbox, UINT32_MAX);
	if (!fname) {
	syslog(LOG_ERR, "IOERROR: Mailbox name too long (%s)", mailbox->name);
	r = IMAP_MAILBOX_BADNAME;
	goto done;
	}
	/* and create the directory too :) */
	if (cyrus_mkdir(fname, 0755) == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fname);
	r = IMAP_IOERROR;
	goto done;
	}

	fname = mailbox_meta_fname(mailbox, META_INDEX);
	if (!fname) {
	syslog(LOG_ERR, "IOERROR: Mailbox name too long (%s)", mailbox->name);
	r = IMAP_MAILBOX_BADNAME;
	goto done;
	}
	mailbox->index_fd = open(fname, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	if (mailbox->index_fd == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fname);
	r = IMAP_IOERROR;
	goto done;
	}
	r = lock_blocking(mailbox->index_fd);
	if (r) {
	syslog(LOG_ERR, "IOERROR: locking %s: %m", fname);
	r = IMAP_IOERROR;
	goto done;
	}
	mailbox->index_locktype = LOCK_EXCLUSIVE;

	fname = mailbox_meta_fname(mailbox, META_CACHE);
	if (!fname) {
	syslog(LOG_ERR, "IOERROR: Mailbox name too long (%s)", mailbox->name);
	r = IMAP_MAILBOX_BADNAME;
	goto done;
	}
	mailbox->cache_fd = open(fname, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	if (mailbox->cache_fd == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", fname);
	r = IMAP_IOERROR;
	goto done;
	}

	if (hasquota) mailbox_set_quotaroot(mailbox, quotaroot);

	/* init non-zero fields */
	mailbox_index_dirty(mailbox);
	mailbox->i.minor_version = MAILBOX_MINOR_VERSION;
	mailbox->i.start_offset = INDEX_HEADER_SIZE;
	mailbox->i.record_size = INDEX_RECORD_SIZE;
	mailbox->i.uidvalidity = uidvalidity;
	mailbox->i.options = options;
	mailbox->i.highestmodseq = 1;

	mailbox->header_dirty = 1;
	if (!uniqueid) {
	mailbox_make_uniqueid(mailbox);
	} else {
	mailbox->uniqueid = xstrdup(uniqueid);
	}

	/* write out the initial generation number to the cache file */
	((bit32 )generation_buf) = htonl(mailbox->i.generation_no);
	n = retry_write(mailbox->cache_fd, generation_buf, 4);
	if (n != 4 \|\| fsync(mailbox->cache_fd)) {
	syslog(LOG_ERR, "IOERROR: writing initial cache for %s: %m",
	mailbox->name);
	r = IMAP_IOERROR;
	goto done;
	}

	r = seen_create_mailbox(NULL, mailbox);
	if (r) goto done;
	r = mailbox_commit(mailbox);
	if (r) goto done;

	if (config_auditlog)
	syslog(LOG_NOTICE, "auditlog: create sessionid=<%s> "
	"mailbox=<%s> uniqueid=<%s>",
	session_id(),
	mailbox->name, mailbox->uniqueid);

	done:
	if (!r && mailboxptr)
	*mailboxptr = mailbox;
	else
	mailbox_close(&mailbox);

	return r;
	}

	/*
	* Remove all files in directory
	*/
	static void mailbox_delete_files(char *path)
	{
	DIR *dirp;
	struct dirent *f;
	char buf[MAX_MAILBOX_PATH+1];
	char *tail;

	strlcpy(buf, path, sizeof(buf));

	if(strlen(buf) >= sizeof(buf) - 2) {
	syslog(LOG_ERR, "IOERROR: Path too long (%s)", buf);
	fatal("path too long", EC_OSFILE);
	}

	tail = buf + strlen(buf);
	*tail++ = '/';
	*tail = '\0';
	dirp = opendir(path);
	if (dirp) {
	while ((f = readdir(dirp))!=NULL) {
	if (f->d_name[0] == '.'
	&& (f->d_name[1] == '\0'
	\|\| (f->d_name[1] == '.' &&
	f->d_name[2] == '\0'))) {
	/* readdir() can return "." or "..", and I got a bug report
	that SCO might blow the file system to smithereens if we
	unlink(".."). Let's not do that. */
	continue;
	}

	if(strlen(buf) + strlen(f->d_name) >= sizeof(buf)) {
	syslog(LOG_ERR, "IOERROR: Path too long (%s + %s)",
	buf, f->d_name);
	fatal("Path too long", EC_OSFILE);
	}
	strcpy(tail, f->d_name);
	unlink(buf);
	*tail = '\0';
	}
	closedir(dirp);
	}
	}

	/* Callback for use by cmd_delete */
	static int chkchildren(char *name __attribute__((unused)),
	int matchlen __attribute__((unused)),
	int maycreate __attribute__((unused)),
	void *rock __attribute__((unused)))
	{
	return CYRUSDB_DONE;
	}

	/*
	* Delete and close the mailbox 'mailbox'. Closes 'mailbox' whether
	* or not the deletion was successful. Requires a locked mailbox.
	*/
	int mailbox_delete(struct mailbox **mailboxptr)
	{
	int r = 0;
	struct mailbox mailbox = mailboxptr;

	/* mark the mailbox deleted */
	mailbox_index_dirty(mailbox);
	mailbox->i.options \|= OPT_MAILBOX_DELETED;

	/* mark the quota removed */
	mailbox_quota_dirty(mailbox);
	mailbox->i.quota_mailbox_used = 0;

	/* commit the changes */
	r = mailbox_commit(mailbox);
	if (r) return r;

	/* remove any seen */
	seen_delete_mailbox(NULL, mailbox);

	/* can't unlink any files yet, because our promise to other
	* users of the mailbox applies! Can only unlink with an
	* exclusive lock. mailbox_close will try to get one of
	* those.
	*/

	syslog(LOG_NOTICE, "Deleted mailbox %s", mailbox->name);

	if (config_auditlog)
	syslog(LOG_NOTICE, "auditlog: delete sessionid=<%s> "
	"mailbox=<%s> uniqueid=<%s>",
	session_id(),
	mailbox->name, mailbox->uniqueid);

	mailbox_close(mailboxptr);

	return 0;
	}

	/* XXX - move this part of cleanup into mboxlist. Really
	* needs to be done with mailboxes.db locked so nobody can
	* try to create a mailbox while the delete is underway.
	* VERY tight race condition exists right now... */
	/* we need an exclusive namelock for this */
	int mailbox_delete_cleanup(struct mailbox *mailbox)
	{
	char nbuf[MAX_MAILBOX_BUFFER];
	char pbuf[MAX_MAILBOX_PATH+1], mbuf[MAX_MAILBOX_PATH+1];
	char ntail, ptail, *mtail = NULL;
	char path, mpath;
	int r;

	/* make sure it really is deleted! */
	assert(mailbox->i.options & OPT_MAILBOX_DELETED);

	/* XXX - use explicit paths to each type of file */

	/* Flush data (message file) directory */
	path = mboxname_datapath(mailbox->part, mailbox->name, 0);
	mailbox_delete_files(path);
	strlcpy(pbuf, path, sizeof(pbuf));
	ptail = pbuf + strlen(pbuf);

	/* Flush metadata directory */
	mpath = mboxname_metapath(mailbox->part, mailbox->name, 0, 0);
	if (strcmp(path, mpath)) {
	mailbox_delete_files(mpath);
	strlcpy(mbuf, mpath, sizeof(mbuf));
	mtail = mbuf + strlen(mbuf);
	}

	strlcpy(nbuf, mailbox->name, sizeof(nbuf));
	ntail = nbuf + strlen(nbuf);

	do {
	/* Check if the mailbox has children */
	strcpy(ntail, ".*");
	r = mboxlist_findall(NULL, nbuf, 1, NULL, NULL, chkchildren, NULL);
	if (r != 0) break; /* We short-circuit with CYRUSDB_DONE */

	/* No children, remove mailbox spool dir(s) */
	if (rmdir(pbuf)) {
	syslog(LOG_NOTICE,
	"Remove of supposedly empty directory %s failed: %m",
	pbuf);
	}
	ptail = strrchr(pbuf, '/');
	*ptail ='\0';

	if (mtail) {
	if (rmdir(mbuf)) {
	syslog(LOG_NOTICE,
	"Remove of supposedly empty directory %s failed: %m",
	mbuf);
	}
	mtail = strrchr(mbuf, '/');
	*mtail ='\0';
	}

	/* Check if parent mailbox exists */
	*ntail = '\0';
	ntail = strrchr(nbuf, '.');
	if (!ntail \|\| strchr(ntail, '!')) {
	/* Hit top of hierarchy or domain separator */
	break;
	}
	*ntail = '\0';
	if (!strcmp(nbuf, "user") \|\|
	((ntail - nbuf > 5) && !strcmp(ntail-5, "!user"))) {
	/* Hit top of 'user' hierarchy */
	break;
	}
	r = mboxlist_lookup(nbuf, NULL, NULL);
	} while(r == IMAP_MAILBOX_NONEXISTENT);

	return 0;
	}

	struct meta_file {
	unsigned long metaflag;
	int optional;
	int nolink;
	};

	static struct meta_file meta_files[] = {
	{ META_INDEX, 0, 1 },
	{ META_CACHE, 0, 1 },
	{ META_SQUAT, 1, 0 },
	{ 0, 0, 0 }
	};

	/* if 'userid' is set, we perform the funky RENAME INBOX INBOX.old
	semantics, regardless of whether or not the name of the mailbox is
	'user.foo'.*/
	/* requires a write-locked oldmailbox pointer, since we delete it
	immediately afterwards */
	int mailbox_rename_copy(struct mailbox *oldmailbox,
	const char *newname,
	const char *newpartition,
	const char *userid, int ignorequota,
	struct mailbox **newmailboxptr)
	{
	int r;
	unsigned int flag;
	char oldbuf[MAX_MAILBOX_PATH], newbuf[MAX_MAILBOX_PATH];
	struct meta_file *mf;
	char path, mpath;
	struct mailbox *newmailbox = NULL;
	unsigned uidvalidity;
	uint32_t recno;
	struct index_record record;

	assert(mailbox_index_islocked(oldmailbox, 1));

	if (strcmp(oldmailbox->name, newname) == 0) {
	/* Just moving mailboxes between partitions */
	uidvalidity = oldmailbox->i.uidvalidity;
	}
	else {
	uidvalidity = time(0);
	}

	/* Create new mailbox */
	r = mailbox_create(newname, newpartition,
	oldmailbox->acl, (userid ? NULL : oldmailbox->uniqueid),
	oldmailbox->i.options, uidvalidity, &newmailbox);

	if (r) return r;

	/* Check quota if necessary */
	if (!ignorequota && newmailbox->quotaroot && (!oldmailbox->quotaroot \|\|
	strcmp(oldmailbox->quotaroot, newmailbox->quotaroot))) {
	struct quota q;

	q.root = newmailbox->quotaroot;
	r = quota_read(&q, NULL, 1);

	/* check if the limit is exceeded */
	if (!r && q.limit >= 0 && q.used + oldmailbox->i.quota_mailbox_used >
	(uquota_t) q.limit * QUOTA_UNITS) {
	r = IMAP_QUOTA_EXCEEDED;
	}

	/* then we abort - no space to rename */
	if (r && r != IMAP_QUOTAROOT_NONEXISTENT)
	goto fail;
	}

	/* Copy over meta files */
	for (mf = meta_files; !r && mf->metaflag; mf++) {
	struct stat sbuf;

	strncpy(oldbuf, mailbox_meta_fname(oldmailbox, mf->metaflag), MAX_MAILBOX_PATH);
	strncpy(newbuf, mailbox_meta_fname(newmailbox, mf->metaflag), MAX_MAILBOX_PATH);

	unlink(newbuf); /* Make link() possible */

	if (!mf->optional \|\| stat(oldbuf, &sbuf) != -1) {
	r = mailbox_copyfile(oldbuf, newbuf, mf->nolink);
	if (r) goto fail;
	}
	}

	/* Re-open index file */
	r = mailbox_open_index(newmailbox);
	if (r) goto fail;

	r = mailbox_read_index_header(newmailbox);
	if (r) goto fail;

	for (recno = 1; recno <= newmailbox->i.num_records; recno++) {
	r = mailbox_read_index_record(newmailbox, recno, &record);
	if (r) goto fail;

	if (record.system_flags & FLAG_UNLINKED)
	continue;

	strncpy(oldbuf, mailbox_message_fname(oldmailbox, record.uid), MAX_MAILBOX_PATH);
	strncpy(newbuf, mailbox_message_fname(newmailbox, record.uid), MAX_MAILBOX_PATH);

	r = mailbox_copyfile(oldbuf, newbuf, 0);
	if (r) goto fail;
	}

	/* Copy flag names */
	newmailbox->header_dirty = 1;
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (oldmailbox->flagname[flag]) {
	newmailbox->flagname[flag] = xstrdup(oldmailbox->flagname[flag]);
	}
	}

	r = seen_copy(userid, oldmailbox, newmailbox);
	if (r) goto fail;

	/* mark the "used" back to zero, so it updates the new quota! */
	mailbox_quota_dirty(newmailbox);
	newmailbox->quota_previously_used = 0;

	/* commit the index changes last, once files are in place */
	r = mailbox_commit(newmailbox);
	if (r) goto fail;

	if (config_auditlog)
	syslog(LOG_NOTICE, "auditlog: rename sessionid=<%s> "
	"oldmailbox=<%s> newmailbox=<%s> uniqueid=<%s>",
	session_id(),
	oldmailbox->name, newname, newmailbox->uniqueid);

	if (newmailboxptr) *newmailboxptr = newmailbox;
	else mailbox_close(&newmailbox);

	return 0;

	fail:
	/* failure and back out */
	/* XXX - per file paths, need to clean up individual filenames */
	path = mboxname_datapath(newmailbox->part, newmailbox->name, 0);
	mailbox_delete_files(path);
	mpath = mboxname_metapath(newmailbox->part, newmailbox->name, 0, 0);
	if (strcmp(path, mpath)) mailbox_delete_files(mpath);
	mailbox_close(&newmailbox);

	return r;
	}

	int mailbox_rename_cleanup(struct mailbox **mailboxptr, int isinbox)
	{
	int r = 0;
	struct mailbox oldmailbox = mailboxptr;
	char *name = xstrdup(oldmailbox->name);

	if (isinbox) {
	/* Expunge old mailbox */
	r = mailbox_expunge(oldmailbox, expungeall, (char *)0, NULL);
	if (!r) r = mailbox_commit(oldmailbox);
	mailbox_close(mailboxptr);
	} else {
	r = mailbox_delete(mailboxptr);
	}

	if (r) {
	syslog(LOG_CRIT,
	"Rename Failure during mailbox_rename_cleanup (%s), " \
	"potential leaked space (%s)", name,
	error_message(r));
	}
	free(name);

	return r;
	}

	/*
	* Copy (or link) the file 'from' to the file 'to'
	*/
	int mailbox_copyfile(const char from, const char to, int nolink)
	{
	int srcfd, destfd;
	struct stat sbuf;
	const char *src_base = 0;
	unsigned long src_size = 0;
	int n;

	if (!nolink) {
	if (link(from, to) == 0) return 0;
	if (errno == EEXIST) {
	if (unlink(to) == -1) {
	syslog(LOG_ERR, "IOERROR: unlinking to recreate %s: %m", to);
	return IMAP_IOERROR;
	}
	if (link(from, to) == 0) return 0;
	}
	}

	destfd = open(to, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	if (destfd == -1) {
	syslog(LOG_ERR, "IOERROR: creating %s: %m", to);
	return IMAP_IOERROR;
	}

	srcfd = open(from, O_RDONLY, 0666);
	if (srcfd == -1) {
	syslog(LOG_ERR, "IOERROR: opening %s: %m", from);
	close(destfd);
	return IMAP_IOERROR;
	}


	if (fstat(srcfd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat on %s: %m", from);
	close(srcfd);
	close(destfd);
	return IMAP_IOERROR;
	}
	map_refresh(srcfd, 1, &src_base, &src_size, sbuf.st_size, from, 0);

	n = retry_write(destfd, src_base, src_size);

	if (n == -1 \|\| fsync(destfd)) {
	map_free(&src_base, &src_size);
	close(srcfd);
	close(destfd);
	syslog(LOG_ERR, "IOERROR: writing %s: %m", to);
	return IMAP_IOERROR;
	}
	map_free(&src_base, &src_size);
	close(srcfd);
	close(destfd);
	return 0;
	}

	/* ---------------------------------------------------------------------- */
	/* RECONSTRUCT SUPPORT */
	/* ---------------------------------------------------------------------- */

	#define UIDGROW 300

	struct found_files {
	unsigned long *uids;
	unsigned nalloc;
	unsigned nused;
	};

	static int sort_uid(const void a, const void b)
	{
	return (unsigned long )a - (unsigned long )b;
	}

	static void init_files(struct found_files *ff)
	{
	ff->uids = NULL;
	ff->nused = 0;
	ff->nalloc = 0;
	}

	static void add_files(struct found_files *ff, unsigned long uid)
	{
	/* make sure there's space */
	if (ff->nused >= ff->nalloc) {
	ff->nalloc += UIDGROW;
	ff->uids = xrealloc(ff->uids, ff->nalloc * sizeof(unsigned long));
	}
	ff->uids[ff->nused++] = uid;
	}

	static void free_files(struct found_files *ff)
	{
	if (ff->nalloc)
	free(ff->uids);
	init_files(ff);
	}

	static int find_files(struct mailbox mailbox, struct found_files files,
	int flags)
	{
	const char *dirpath;
	DIR *dirp;
	struct dirent *dirent;
	uint32_t uid;
	const char *p;
	char buf[MAX_MAILBOX_PATH];
	struct stat sbuf;
	int r;

	init_files(files);

	dirpath = mailbox_datapath(mailbox);
	if (!dirpath) return IMAP_MAILBOX_BADNAME;

	dirp = opendir(dirpath);
	if (!dirp) {
	printf("%s data directory is missing %s\n", mailbox->name, dirpath);
	/* need to re-create data directory */
	if (cyrus_mkdir(dirpath, 0755) == -1)
	return IMAP_IOERROR;
	if (mkdir(dirpath, 0755) == -1)
	return IMAP_IOERROR;
	return 0;
	}

	/* data directory is fine */
	while ((dirent = readdir(dirp)) != NULL) {
	p = dirent->d_name;
	if (p == '.') continue; / dot files */
	if (!strncmp(p, "cyrus.", 6)) continue; /* cyrus.* files */

	r = parseuint32(p, &p, &uid);

	/* it has to have a . after the number and nothing else */
	if (r \|\| uid == 0 \|\| p++ != '.' \|\| p) {
	/* check if it's a directory */
	snprintf(buf, MAX_MAILBOX_PATH, "%s/%s", dirpath, dirent->d_name);
	if (stat(buf, &sbuf) == -1) continue; /* ignore emepheral */
	if (!S_ISDIR(sbuf.st_mode)) {
	if (!(flags & RECONSTRUCT_IGNORE_ODDFILES)) {
	printf("%s odd file %s\n", mailbox->name, buf);
	syslog(LOG_ERR, "%s odd file %s", mailbox->name, buf);
	if (flags & RECONSTRUCT_REMOVE_ODDFILES)
	unlink(buf);
	else {
	printf("run reconstruct with -O to remove odd files\n");
	syslog(LOG_ERR, "run reconstruct with -O to "
	"remove odd files");
	}
	}
	}
	}
	else {
	/* it's one of ours :) */
	add_files(files, uid);
	}
	}

	closedir(dirp);

	/* make sure UIDs are sorted for comparison */
	qsort(files->uids, files->nused, sizeof(unsigned long), sort_uid);

	return 0;
	}

	/* this is kind of like mailbox_create, but we try to rescue
	* what we can from the filesystem! */
	static int mailbox_reconstruct_create(const char name, struct mailbox *mbptr)
	{
	struct mailbox *mailbox = NULL;
	int options = config_getint(IMAPOPT_MAILBOX_DEFAULT_OPTIONS)
	\| OPT_POP3_NEW_UIDL;
	struct mboxlist_entry mbentry;
	struct mailboxlist *listitem;
	int r;

	/* make sure it's not already open. Very odd, since we already
	* discovered it's not openable! */
	listitem = find_listitem(name);
	if (listitem) return IMAP_MAILBOX_LOCKED;

	listitem = create_listitem(name);
	mailbox = &listitem->m;

	/* if we can't get an exclusive lock first try, there's something
	* racy going on! */
	r = mboxname_lock(name, &listitem->l, LOCK_NONBLOCKING);
	if (r) goto done;

	/* Start by looking up current data in mailbox list */
	/* XXX - no mboxlist entry? Can we recover? */
	r = mboxlist_lookup(name, &mbentry, NULL);
	if (r) goto done;

	syslog(LOG_NOTICE, "create new mailbox %s", name);

	mailbox->part = xstrdup(mbentry.partition);
	mailbox->acl = xstrdup(mbentry.acl);

	/* Attempt to open index */
	r = mailbox_open_index(mailbox);
	if (!r) r = mailbox_read_index_header(mailbox);
	if (r) {
	printf("%s: failed to read index header\n", mailbox->name);
	syslog(LOG_ERR, "failed to read index header for %s", mailbox->name);
	/* no cyrus.index file at all - well, we're in a pickle!
	* no point trying to rescue anything else... */
	mailbox_close(&mailbox);
	return mailbox_create(name, mbentry.partition, mbentry.acl,
	NULL, options, time(0), mbptr);
	}

	/* read header, if it is not there, we need to create it */
	r = mailbox_read_header(mailbox, NULL);
	if (r) {
	/* Header failed to read - recreate it */
	printf("%s: failed to read header file\n", mailbox->name);
	syslog(LOG_ERR, "failed to read header file for %s", mailbox->name);

	mailbox_make_uniqueid(mailbox);
	r = mailbox_commit(mailbox);
	if (r) goto done;
	}

	if (mailbox->header_file_crc != mailbox->i.header_file_crc) {
	mailbox->i.header_file_crc = mailbox->header_file_crc;
	printf("%s: header file CRC mismatch, correcting\n", mailbox->name);
	syslog(LOG_ERR, "%s: header file CRC mismatch, correcting", mailbox->name);
	mailbox_index_dirty(mailbox);
	r = mailbox_commit(mailbox);
	if (r) goto done;
	}

	done:
	if (r) mailbox_close(&mailbox);
	else *mbptr = mailbox;

	return r;
	}

	static int mailbox_reconstruct_acl(struct mailbox *mailbox, int flags)
	{
	int make_changes = flags & RECONSTRUCT_MAKE_CHANGES;
	char *acl = NULL;
	int r;

	r = mailbox_read_header(mailbox, &acl);
	if (r) return r;

	if (strcmp(mailbox->acl, acl)) {
	printf("%s: update acl from header %s => %s\n", mailbox->name,
	mailbox->acl, acl);
	if (make_changes) {
	r = mboxlist_update(mailbox->name, mailbox->mbtype,
	mailbox->part, acl, 0);
	}
	}

	free(acl);

	return r;
	}

	static int records_match(const char *mboxname,
	struct index_record *old,
	struct index_record *new)
	{
	int i;
	int match = 1;
	int userflags_dirty = 0;

	if (old->internaldate != new->internaldate) {
	printf("%s uid %u mismatch: internaldate\n",
	mboxname, new->uid);
	match = 0;
	}
	if (old->sentdate != new->sentdate) {
	printf("%s uid %u mismatch: sentdate\n",
	mboxname, new->uid);
	match = 0;
	}
	if (old->size != new->size) {
	printf("%s uid %u mismatch: size\n",
	mboxname, new->uid);
	match = 0;
	}
	if (old->header_size != new->header_size) {
	printf("%s uid %u mismatch: header_size\n",
	mboxname, new->uid);
	match = 0;
	}
	if (old->gmtime != new->gmtime) {
	printf("%s uid %u mismatch: gmtime\n",
	mboxname, new->uid);
	match = 0;
	}
	if (old->content_lines != new->content_lines) {
	printf("%s uid %u mismatch: content_lines\n",
	mboxname, new->uid);
	match = 0;
	}
	if (old->system_flags != new->system_flags) {
	printf("%s uid %u mismatch: systemflags\n",
	mboxname, new->uid);
	match = 0;
	}
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	if (old->user_flags[i] != new->user_flags[i])
	userflags_dirty = 1;
	}
	if (userflags_dirty) {
	printf("%s uid %u mismatch: userflags\n",
	mboxname, new->uid);
	match = 0;
	}
	if (!message_guid_compare(&old->guid, &new->guid)) {
	printf("%s uid %u mismatch: guid\n",
	mboxname, new->uid);
	match = 0;
	}

	if (!match) {
	syslog(LOG_ERR, "%s uid %u record mismatch, rewriting",
	mboxname, new->uid);
	}

	/* cache issues - don't print, probably just a version
	* upgrade... */
	if (old->cache_version != new->cache_version) {
	match = 0;
	}
	if (old->cache_crc != new->cache_crc) {
	match = 0;
	}
	if (cache_size(old) != cache_size(new)) {
	match = 0;
	}
	/* only compare cache records if size matches */
	else if (memcmp(cache_base(old), cache_base(new), cache_size(new))) {
	match = 0;
	}

	return match;
	}

	static int mailbox_reconstruct_compare_update(struct mailbox *mailbox,
	struct index_record *record,
	bit32 *valid_user_flags,
	int flags, int have_file,
	struct found_files *discovered)
	{
	char *fname = mailbox_message_fname(mailbox, record->uid);
	int r = 0;
	int i;
	struct index_record copy;
	struct stat sbuf;
	int make_changes = flags & RECONSTRUCT_MAKE_CHANGES;
	int re_parse = flags & RECONSTRUCT_ALWAYS_PARSE;
	int do_stat = flags & RECONSTRUCT_DO_STAT;
	int re_pack = 0;
	int did_stat = 0;

	/* does the file actually exist? */
	if (have_file && do_stat) {
	if (stat(fname, &sbuf) == -1 \|\| (sbuf.st_size == 0)) {
	have_file = 0;
	}
	else if (record->size != (unsigned) sbuf.st_size) {
	re_parse = 1;
	}
	did_stat = 1;
	}

	if (!have_file) {
	/* well, that's OK if it's supposed to be missing! */
	if (record->system_flags & FLAG_UNLINKED)
	return 0;

	printf("%s uid %u not found\n", mailbox->name, record->uid);
	syslog(LOG_ERR, "%s uid %u not found", mailbox->name, record->uid);

	if (!make_changes) return 0;

	/* otherwise we have issues, mark it unlinked */
	unlink(fname);
	record->system_flags \|= FLAG_EXPUNGED \| FLAG_UNLINKED;
	mailbox->i.options \|= OPT_MAILBOX_NEEDS_REPACK;
	return mailbox_rewrite_index_record(mailbox, record);
	}

	if (mailbox_cacherecord(mailbox, record) \|\| record->crec.len == 0) {
	re_parse = 1;
	re_pack = 1; /* cache record will have to be rewritten */
	}

	/* copy once the cache record is read in... */
	copy = *record;

	if (!record->internaldate) {
	re_parse = 1;
	}

	/* re-calculate all the "derived" fields by parsing the file on disk */
	if (re_parse) {
	r = message_parse(fname, record);
	if (r) return r;

	/* it's not the same message! */
	if (!message_guid_compare(&record->guid, &copy.guid)) {
	int do_unlink = 0;

	printf("%s uid %u guid mismatch\n",
	mailbox->name, record->uid);
	syslog(LOG_ERR, "%s uid %u guid mismatch",
	mailbox->name, record->uid);

	if (!make_changes) return 0;

	if (record->system_flags & FLAG_EXPUNGED) {
	/* already expunged, just unlink it */
	printf("%s uid %u already expunged, unlinking\n",
	mailbox->name, record->uid);
	syslog(LOG_ERR, "%s uid %u already expunged, unlinking",
	mailbox->name, record->uid);
	do_unlink = 1;
	}
	else if (flags & RECONSTRUCT_GUID_REWRITE) {
	/* treat this file as discovered */
	add_files(discovered, record->uid);
	printf("%s uid %u marking for uid upgrade\n",
	mailbox->name, record->uid);
	syslog(LOG_ERR, "%s uid %u marking for uid upgrade",
	mailbox->name, record->uid);
	do_unlink = 1;
	}
	else if (flags & RECONSTRUCT_GUID_UNLINK) {
	printf("%s uid %u unlinking as requested with -U\n",
	mailbox->name, record->uid);
	syslog(LOG_ERR, "%s uid %u unlinking as requested with -U",
	mailbox->name, record->uid);
	do_unlink = 1;
	}

	if (do_unlink) {
	record->system_flags \|= FLAG_EXPUNGED \| FLAG_UNLINKED;
	mailbox->i.options \|= OPT_MAILBOX_NEEDS_UNLINK;
	return mailbox_rewrite_index_record(mailbox, record);
	}

	/* otherwise we just report it and move on - hopefully the
	* correct file can be restored from backup or something */
	printf("run reconstruct with -R to fix or -U to remove\n");
	syslog(LOG_ERR, "run reconstruct with -R to fix or -U to remove");
	return 0;
	}
	}

	/* get internaldate from the file if not available from the file */
	if (!record->internaldate) {
	if (did_stat \|\| stat(fname, &sbuf) != -1)
	record->internaldate = sbuf.st_mtime;
	else
	record->internaldate = time(NULL);
	}

	if (did_stat && sbuf.st_mtime != record->internaldate) {
	printf("%s timestamp mismatch %u\n",
	mailbox->name, record->uid);
	syslog(LOG_ERR, "%s timestamp mismatch %u",
	mailbox->name, record->uid);
	if (make_changes) {
	/* make the file timestamp correct */
	struct utimbuf settime;
	settime.actime = settime.modtime = record->internaldate;
	if (utime(fname, &settime) == -1)
	return IMAP_IOERROR;
	}
	}

	/* XXX - conditions under which modseq or uid or internaldate could be bogus? */
	if (record->modseq > mailbox->i.highestmodseq) {
	printf("%s uid %u future modseq " MODSEQ_FMT " found\n",
	mailbox->name, record->uid, record->modseq);
	syslog(LOG_ERR, "%s uid %u future modseq " MODSEQ_FMT " found",
	mailbox->name, record->uid, record->modseq);
	mailbox_index_dirty(mailbox);
	mailbox->i.highestmodseq = record->modseq;
	}

	if (record->uid > mailbox->i.last_uid) {
	printf("%s future uid %u found\n",
	mailbox->name, record->uid);
	syslog(LOG_ERR, "%s future uid %u found",
	mailbox->name, record->uid);
	mailbox_index_dirty(mailbox);
	mailbox->i.last_uid = record->uid;
	}

	/* remove any user_flags that are missing from the header */
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	record->user_flags[i] &= valid_user_flags[i];
	}

	/* after all this - if it still matches in every respect, we don't need
	* to rewrite the record - just return */
	if (records_match(mailbox->name, &copy, record))
	return 0;

	/* XXX - inform of changes */
	if (!make_changes)
	return 0;

	/* rewrite the cache record */
	if (re_pack \|\| record->cache_crc != copy.cache_crc) {
	mailbox->i.options \|= OPT_MAILBOX_NEEDS_REPACK;
	record->cache_offset = 0;
	r = mailbox_append_cache(mailbox, record);
	if (r) return r;
	}

	return mailbox_rewrite_index_record(mailbox, record);
	}

	static int mailbox_reconstruct_append(struct mailbox *mailbox, uint32_t uid,
	int flags)
	{
	char *fname = mailbox_message_fname(mailbox, uid);
	int r = 0;
	struct index_record record;
	struct stat sbuf;
	int make_changes = flags & RECONSTRUCT_MAKE_CHANGES;

	if (stat(fname, &sbuf) == -1) r = IMAP_MAILBOX_NONEXISTENT;
	else if (sbuf.st_size == 0) r = IMAP_MAILBOX_NONEXISTENT;

	/* no file, nothing to do! */
	if (r) {
	syslog(LOG_ERR, "%s uid %u not found", mailbox->name, uid);
	printf("%s uid %u not found", mailbox->name, uid);
	if (!make_changes) return 0;
	unlink(fname);
	return 0;
	}

	memset(&record, 0, sizeof(struct index_record));
	record.internaldate = sbuf.st_mtime;

	r = message_parse(fname, &record);
	if (r) return r;

	if (uid > mailbox->i.last_uid) {
	printf("%s uid %u found - adding\n", mailbox->name, uid);
	syslog(LOG_ERR, "%s uid %u found - adding", mailbox->name, uid);
	record.uid = uid;
	}
	else {
	char *oldfname;
	char *newfname;

	printf("%s uid %u rediscovered - appending\n", mailbox->name, uid);
	syslog(LOG_ERR, "%s uid %u rediscovered - appending", mailbox->name, uid);
	/* XXX - check firstexpunged? */
	record.uid = mailbox->i.last_uid + 1;

	if (!make_changes) return 0;

	oldfname = xstrdup(mailbox_message_fname(mailbox, uid));
	newfname = xstrdup(mailbox_message_fname(mailbox, record.uid));
	r = rename(oldfname, newfname);
	free(oldfname);
	free(newfname);
	if (r) return IMAP_IOERROR;
	}

	/* XXX - inform of changes */
	if (!make_changes)
	return 0;

	return mailbox_append_index_record(mailbox, &record);
	}


	static void reconstruct_compare_headers(struct mailbox *mailbox,
	struct index_header *old,
	struct index_header *new)
	{
	if (old->quota_mailbox_used != new->quota_mailbox_used) {
	printf("%s updating quota_mailbox_used: "
	QUOTA_T_FMT " => " QUOTA_T_FMT "\n", mailbox->name,
	old->quota_mailbox_used, new->quota_mailbox_used);
	syslog(LOG_ERR, "%s updating quota_mailbox_used: "
	QUOTA_T_FMT " => " QUOTA_T_FMT, mailbox->name,
	old->quota_mailbox_used, new->quota_mailbox_used);
	}

	if (old->answered != new->answered) {
	syslog(LOG_ERR, "%s: updating answered %u => %u",
	mailbox->name, old->answered, new->answered);
	printf("%s: updating answered %u => %u\n",
	mailbox->name, old->answered, new->answered);
	}

	if (old->flagged != new->flagged) {
	syslog(LOG_ERR, "%s: updating flagged %u => %u",
	mailbox->name, old->flagged, new->flagged);
	printf("%s: updating flagged %u => %u\n",
	mailbox->name, old->flagged, new->flagged);
	}

	if (old->deleted != new->deleted) {
	syslog(LOG_ERR, "%s: updating deleted %u => %u",
	mailbox->name, old->deleted, new->deleted);
	printf("%s: updating deleted %u => %u\n",
	mailbox->name, old->deleted, new->deleted);
	}

	if (old->exists != new->exists) {
	syslog(LOG_ERR, "%s: updating exists %u => %u",
	mailbox->name, old->exists, new->exists);
	printf("%s: updating exists %u => %u\n",
	mailbox->name, old->exists, new->exists);
	}

	if (old->sync_crc != new->sync_crc) {
	syslog(LOG_ERR, "%s: updating sync_crc %08X => %08X",
	mailbox->name, old->sync_crc, new->sync_crc);
	printf("%s: updating sync_crc %08X => %08X\n",
	mailbox->name, old->sync_crc, new->sync_crc);
	}
	}

	/*
	* Reconstruct the single mailbox named 'name'
	*/
	int mailbox_reconstruct(const char *name, int flags)
	{
	/* settings */
	int make_changes = (flags & RECONSTRUCT_MAKE_CHANGES);

	int r = 0;
	uint32_t msg;
	int i, flag;
	struct index_record record;
	struct mailbox *mailbox = NULL;
	struct found_files files;
	struct found_files discovered;
	struct index_header old_header;
	int have_file;
	uint32_t recno;
	bit32 valid_user_flags[MAX_USER_FLAGS/32];

	if (make_changes && !(flags & RECONSTRUCT_QUIET)) {
	syslog(LOG_NOTICE, "reconstructing %s", name);
	}

	r = mailbox_open_iwl(name, &mailbox);
	if (r) {
	if (!make_changes) return r;
	/* returns a locktype == LOCK_EXCLUSIVE mailbox */
	r = mailbox_reconstruct_create(name, &mailbox);
	}
	if (r) return r;

	r = mailbox_reconstruct_acl(mailbox, flags);
	if (r) goto close;

	/* Validate user flags */
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	valid_user_flags[i] = 0;
	}
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (!mailbox->flagname[flag]) continue;
	if ((flag && !mailbox->flagname[flag-1]) \|\|
	!imparse_isatom(mailbox->flagname[flag])) {
	printf("%s: bogus flag name %d:%s",
	mailbox->name, flag, mailbox->flagname[flag]);
	syslog(LOG_ERR, "%s: bogus flag name %d:%s",
	mailbox->name, flag, mailbox->flagname[flag]);
	mailbox->header_dirty = 1;
	free(mailbox->flagname[flag]);
	mailbox->flagname[flag] = NULL;
	continue;
	}
	valid_user_flags[flag/32] \|= 1<<(flag&31);
	}

	r = mailbox_open_cache(mailbox);
	if (r) {
	const char *fname = mailbox_meta_fname(mailbox, META_CACHE);
	char buf[4];
	int n;

	printf("%s: missing cache file, recreating\n",
	mailbox->name);
	syslog(LOG_ERR, "%s: missing cache file, recreating",
	mailbox->name);

	if (!make_changes) goto close;

	if (cyrus_mkdir(fname, 0755)) goto close;
	mailbox->cache_fd = open(fname, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	if (mailbox->cache_fd == -1) goto close;

	/* set the generation number */
	((bit32 )(buf)) = htonl(mailbox->i.generation_no);
	n = retry_write(mailbox->cache_fd, buf, 4);
	if (n != 4) goto close;

	/* ensure that next user will create the MMAPing */
	mailbox->need_cache_refresh = 1;
	}

	r = find_files(mailbox, &files, flags);
	if (r) goto close;
	init_files(&discovered);
	msg = 0;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	if (mailbox_read_index_record(mailbox, recno, &record)) {
	printf("%s: record corrupted %u (maybe uid %u)\n",
	mailbox->name, recno, record.uid);
	continue;
	}

	/* lower UID file exists */
	while (msg < files.nused && files.uids[msg] < record.uid) {
	add_files(&discovered, files.uids[msg]);
	msg++;
	}

	/* if they match, advance the pointer */
	have_file = 0;
	if (msg < files.nused && files.uids[msg] == record.uid) {
	have_file = 1;
	msg++;
	}

	r = mailbox_reconstruct_compare_update(mailbox, &record,
	valid_user_flags,
	flags, have_file,
	&discovered);
	if (r) goto close;
	}

	/* add discovered messages before last_uid to the list in order */
	while (msg < files.nused && files.uids[msg] <= mailbox->i.last_uid) {
	add_files(&discovered, files.uids[msg]);
	msg++;
	}

	/* messages AFTER last_uid can keep the same UID (see also, restore
	* from list .index file) - so don't bother moving those */
	while (msg < files.nused) {
	r = mailbox_reconstruct_append(mailbox, files.uids[msg], flags);
	if (r) goto close;
	msg++;
	}

	/* handle new list */
	msg = 0;
	while (msg < discovered.nused) {
	r = mailbox_reconstruct_append(mailbox, discovered.uids[msg], flags);
	if (r) goto close;
	msg++;
	}

	/* make sure we have enough index file mmaped */
	r = mailbox_refresh_index_map(mailbox);

	old_header = mailbox->i;

	/* re-calculate derived fields */
	r = mailbox_index_recalc(mailbox);
	if (r) goto close;

	/* inform users of any changed header fields */
	reconstruct_compare_headers(mailbox, &old_header, &mailbox->i);

	/* fix up 2.4.0 bug breakage */
	if (mailbox->i.uidvalidity == 0) {
	if (make_changes) {
	mailbox->i.uidvalidity = time(0);
	mailbox_index_dirty(mailbox);
	}
	syslog(LOG_ERR, "%s: zero uidvalidity", mailbox->name);
	}
	if (mailbox->i.highestmodseq == 0) {
	if (make_changes) {
	mailbox_index_dirty(mailbox);
	mailbox->i.highestmodseq = 1;
	}
	syslog(LOG_ERR, "%s: zero highestmodseq", mailbox->name);
	}

	if (make_changes) {
	r = mailbox_commit(mailbox);
	}
	else {
	/* undo any dirtyness before we close, we didn't actually
	* write any changes */
	mailbox->i.dirty = 0;
	mailbox->quota_dirty = 0;
	mailbox->cache_dirty = 0;
	mailbox->modseq_dirty = 0;
	mailbox->header_dirty = 0;
	}

	close:
	free_files(&files);
	free_files(&discovered);
	mailbox_close(&mailbox);
	return r;
	}

	diff --git a/imap/mboxname.c b/imap/mboxname.c
	index 9477ead21..bfb82211a 100644
	--- a/imap/mboxname.c
	+++ b/imap/mboxname.c
	@@ -1,1193 +1,1193 @@
	/* mboxname.c -- Mailbox list manipulation routines
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: mboxname.c,v 1.51 2010/06/28 12:04:30 brong Exp $
	*/

	#include <config.h>

	#include <stdio.h>
	#include <string.h>
	#include <errno.h>
	#include <syslog.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif

	#include "assert.h"
	#include "exitcodes.h"
	#include "glob.h"
	#include "global.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "util.h"
	#include "xmalloc.h"

	#include "mboxname.h"
	#include "mboxlist.h"
	-#include "lock.h"
	+#include "cyr_lock.h"

	struct mboxlocklist {
	struct mboxlocklist *next;
	struct mboxlock l;
	int nopen;
	};

	static struct mboxlocklist *open_mboxlocks = NULL;

	/* Mailbox patterns which the design of the server prohibits */
	static char *badmboxpatterns[] = {
	"",
	"\t",
	"\n",
	"/",
	".*",
	"*.",
	"..",
	"user",
	".INBOX.INBOX",
	};
	#define NUM_BADMBOXPATTERNS (sizeof(badmboxpatterns)/sizeof(*badmboxpatterns))

	#define XX 127
	/*
	* Table for decoding modified base64 for IMAP UTF-7 mailbox names
	*/
	static const char index_mod64[256] = {
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,62, 63,XX,XX,XX,
	52,53,54,55, 56,57,58,59, 60,61,XX,XX, XX,XX,XX,XX,
	XX, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
	15,16,17,18, 19,20,21,22, 23,24,25,XX, XX,XX,XX,XX,
	XX,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
	41,42,43,44, 45,46,47,48, 49,50,51,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX, XX,XX,XX,XX,
	};
	#define CHARMOD64(c) (index_mod64[(unsigned char)(c)])


	static struct mboxlocklist create_lockitem(const char name)
	{
	struct mboxlocklist *item = xmalloc(sizeof(struct mboxlocklist));
	item->next = open_mboxlocks;
	open_mboxlocks = item;

	item->nopen = 1;
	item->l.name = xstrdup(name);
	item->l.lock_fd = -1;
	item->l.locktype = 0;

	return item;
	}

	struct mboxlocklist find_lockitem(const char name)
	{
	struct mboxlocklist *item;
	struct mboxlocklist *previtem = NULL;

	/* remove from the active list */
	for (item = open_mboxlocks; item; item = item->next) {
	if (!strcmp(name, item->l.name))
	return item;
	previtem = item;
	}

	return NULL;
	}

	void remove_lockitem(struct mboxlocklist *remitem)
	{
	struct mboxlocklist *item;
	struct mboxlocklist *previtem = NULL;

	for (item = open_mboxlocks; item; item = item->next) {
	if (item == remitem) {
	if (previtem)
	previtem->next = item->next;
	else
	open_mboxlocks = item->next;
	if (item->l.lock_fd != -1)
	close(item->l.lock_fd);
	free(item->l.name);
	free(item);
	return;
	}
	previtem = item;
	}

	fatal("didn't find item in list", EC_SOFTWARE);
	}

	/* name locking support */

	int mboxname_lock(const char mboxname, struct mboxlock *mboxlockptr,
	int locktype)
	{
	const char *fname;
	int r = 0;
	struct mboxlocklist *lockitem;

	fname = mboxname_lockpath(mboxname);
	if (!fname)
	return IMAP_MAILBOX_BADNAME;

	lockitem = find_lockitem(mboxname);

	/* already open? just use this one */
	if (lockitem) {
	if (locktype == LOCK_NONBLOCKING)
	locktype = LOCK_EXCLUSIVE;
	/* can't change locktype! */
	if (lockitem->l.locktype != locktype)
	return IMAP_MAILBOX_LOCKED;

	lockitem->nopen++;
	goto done;
	}

	lockitem = create_lockitem(mboxname);

	/* assume success, and only create directory on failure.
	* More efficient on a common codepath */
	lockitem->l.lock_fd = open(fname, O_CREAT \| O_TRUNC \| O_RDWR, 0666);
	if (lockitem->l.lock_fd == -1) {
	if (cyrus_mkdir(fname, 0755) == -1) {
	r = IMAP_IOERROR;
	goto done;
	}
	lockitem->l.lock_fd = open(fname, O_CREAT \| O_TRUNC \| O_RDWR, 0666);
	}
	/* but if it still didn't succeed, we have problems */
	if (lockitem->l.lock_fd == -1) {
	r = IMAP_IOERROR;
	goto done;
	}

	switch (locktype) {
	case LOCK_SHARED:
	r = lock_shared(lockitem->l.lock_fd);
	if (!r) lockitem->l.locktype = LOCK_SHARED;
	break;
	case LOCK_EXCLUSIVE:
	r = lock_blocking(lockitem->l.lock_fd);
	if (!r) lockitem->l.locktype = LOCK_EXCLUSIVE;
	break;
	case LOCK_NONBLOCKING:
	r = lock_nonblocking(lockitem->l.lock_fd);
	if (r == -1) r = IMAP_MAILBOX_LOCKED;
	else if (!r) lockitem->l.locktype = LOCK_EXCLUSIVE;
	break;
	default:
	fatal("unknown lock type", EC_SOFTWARE);
	}

	done:
	if (r) remove_lockitem(lockitem);
	else *mboxlockptr = &lockitem->l;

	return r;
	}

	void mboxname_release(struct mboxlock **mboxlockptr)
	{
	struct mboxlocklist *lockitem;
	struct mboxlock lock = mboxlockptr;

	lockitem = find_lockitem(lock->name);
	assert(lockitem && &lockitem->l == lock);

	*mboxlockptr = NULL;

	if (lockitem->nopen > 1) {
	lockitem->nopen--;
	return;
	}

	remove_lockitem(lockitem);
	}


	/*
	* Convert the external mailbox 'name' to an internal name.
	* If 'userid' is non-null, it is the name of the current user.
	* On success, results are placed in the buffer pointed to by
	* 'result', the buffer must be of size MAX_MAILBOX_BUFFER to
	* allow space for DELETED mailboxes and moving the domain from
	* one end to the other and such. Yay flexibility.
	*/

	/* Handle conversion from the standard namespace to the internal namespace */
	static int mboxname_tointernal(struct namespace namespace, const char name,
	const char userid, char result)
	{
	char *cp;
	char *atp;
	char *mbresult;
	int userlen, domainlen = 0, namelen;

	/* Blank the result, just in case */
	result[0] = '\0';
	result[MAX_MAILBOX_BUFFER-1] = '\0';

	userlen = userid ? strlen(userid) : 0;
	namelen = strlen(name);

	if (config_virtdomains) {
	if (userid && (cp = strrchr(userid, '@'))) {
	/* user logged in as user@domain */
	userlen = cp - userid;
	/* don't prepend default domain */
	if (!(config_defdomain && !strcasecmp(config_defdomain, cp+1))) {
	domainlen = strlen(cp+1)+1;
	snprintf(result, MAX_MAILBOX_BUFFER, "%s!", cp+1);
	}
	}
	if ((cp = strrchr(name, '@'))) {
	/* mailbox specified as mbox@domain */
	namelen = cp - name;

	if (config_defdomain && !strcasecmp(config_defdomain, cp+1)) {
	if (domainlen) {
	/* don't allow cross-domain access */
	return IMAP_MAILBOX_BADNAME;
	}
	/* don't prepend default domain */
	}
	else {
	if ((!domainlen && !namespace->isadmin) \|\|
	(domainlen && strcasecmp(userid+userlen, cp))) {
	/* don't allow cross-domain access
	(except for global admin) */
	return IMAP_MAILBOX_BADNAME;
	}
	domainlen = strlen(cp+1)+1;
	snprintf(result, MAX_MAILBOX_BUFFER, "%s!", cp+1);
	}

	atp = strchr(name, '@');
	if (atp && atp != cp) {
	/* don't allow multiple '@' in name */
	return IMAP_MAILBOX_BADNAME;
	}
	}

	/* if no domain specified, we're in the default domain */
	}

	mbresult = result + domainlen;

	/* Personal (INBOX) namespace */
	if ((name[0] == 'i' \|\| name[0] == 'I') &&
	!strncasecmp(name, "inbox", 5) &&
	(namelen == 5 \|\| name[5] == namespace->hier_sep)) {

	if (!userid \|\| ((cp = strchr(userid, namespace->hier_sep)) &&
	(cp - userid < userlen))) {
	return IMAP_MAILBOX_BADNAME;
	}

	snprintf(mbresult, MAX_MAILBOX_BUFFER - domainlen,
	"user.%.s%.s", userlen, userid, namelen-5, name+5);

	/* Translate any separators in userid+mailbox */
	mboxname_hiersep_tointernal(namespace, mbresult+5+userlen, 0);
	}

	else {
	/* Other Users & Shared namespace */
	snprintf(mbresult, MAX_MAILBOX_BUFFER - domainlen,
	"%.*s", namelen, name);

	/* Translate any separators in mailboxname */
	mboxname_hiersep_tointernal(namespace, mbresult, 0);
	}

	if (result[MAX_MAILBOX_BUFFER-1] != '\0') {
	syslog(LOG_ERR, "IOERROR: long mailbox name attempt: %s", name);
	return IMAP_MAILBOX_BADNAME;
	}
	return 0;
	}

	/* Handle conversion from the alternate namespace to the internal namespace */
	static int mboxname_tointernal_alt(struct namespace *namespace,
	const char *name,
	const char userid, char result)
	{
	char *cp;
	int userlen, domainlen = 0, namelen;
	int prefixlen;
	size_t resultlen;

	/* Blank the result, just in case */
	result[0] = '\0';

	userlen = userid ? strlen(userid) : 0;
	namelen = strlen(name);

	if (config_virtdomains) {
	if (userid && (cp = strchr(userid, '@'))) {
	/* user logged in as user@domain */
	userlen = cp++ - userid;
	if (!(config_defdomain && !strcasecmp(config_defdomain, cp))) {
	/* don't prepend default domain */
	domainlen = strlen(cp)+1;
	if (domainlen > MAX_MAILBOX_NAME)
	return IMAP_MAILBOX_BADNAME;
	sprintf(result, "%s!", cp);
	}
	}
	if ((cp = strrchr(name, '@'))) {
	/* mailbox specified as mbox@domain */
	namelen = cp - name;

	if (config_defdomain && !strcasecmp(config_defdomain, cp+1)) {
	if (domainlen) {
	/* don't allow cross-domain access */
	return IMAP_MAILBOX_BADNAME;
	}
	/* don't prepend default domain */
	}
	else {
	if ((!domainlen && !namespace->isadmin) \|\|
	(domainlen && strcasecmp(userid+userlen, cp))) {
	/* don't allow cross-domain access
	(except for global admin) */
	return IMAP_MAILBOX_BADNAME;
	}
	domainlen = strlen(cp+1)+1;
	if (domainlen > MAX_MAILBOX_NAME)
	return IMAP_MAILBOX_BADNAME;
	sprintf(result, "%s!", cp+1);
	}
	}

	/* if no domain specified, we're in the default domain */
	}

	result += domainlen;

	/* Shared namespace */
	prefixlen = strlen(namespace->prefix[NAMESPACE_SHARED]);
	if(prefixlen == 0) return IMAP_MAILBOX_BADNAME;

	if (!strncmp(name, namespace->prefix[NAMESPACE_SHARED], prefixlen-1) &&
	(namelen == prefixlen-1 \|\| name[prefixlen-1] == namespace->hier_sep)) {

	if (namelen == prefixlen-1) {
	/* can't create folders using undelimited prefix */
	return IMAP_MAILBOX_BADNAME;
	}

	if (domainlen+namelen-prefixlen > MAX_MAILBOX_NAME) {
	return IMAP_MAILBOX_BADNAME;
	}

	sprintf(result, "%.*s", namelen-prefixlen, name+prefixlen);

	/* Translate any separators in mailboxname */
	mboxname_hiersep_tointernal(namespace, result, 0);
	return 0;
	}

	/* Other Users namespace */
	prefixlen = strlen(namespace->prefix[NAMESPACE_USER]);
	if(prefixlen == 0) return IMAP_MAILBOX_BADNAME;

	if (!strncmp(name, namespace->prefix[NAMESPACE_USER], prefixlen-1) &&
	(namelen == prefixlen-1 \|\| name[prefixlen-1] == namespace->hier_sep)) {

	if (namelen == prefixlen-1) {
	/* can't create folders using undelimited prefix */
	return IMAP_MAILBOX_BADNAME;
	}

	if (domainlen+namelen-prefixlen+5 > MAX_MAILBOX_NAME) {
	return IMAP_MAILBOX_BADNAME;
	}

	sprintf(result, "user.%.*s", namelen-prefixlen, name+prefixlen);

	/* Translate any separators in userid+mailbox */
	mboxname_hiersep_tointernal(namespace, result+5, 0);
	return 0;
	}

	/* Personal (INBOX) namespace */
	if (!userid \|\| ((cp = strchr(userid, namespace->hier_sep)) &&
	(cp - userid < userlen))) {
	return IMAP_MAILBOX_BADNAME;
	}

	if (domainlen+userlen+5 > MAX_MAILBOX_NAME) {
	return IMAP_MAILBOX_BADNAME;
	}

	sprintf(result, "user.%.*s", userlen, userid);

	/* INBOX */
	if ((name[0] == 'i' \|\| name[0] == 'I') &&
	!strncasecmp(name, "inbox", 5) &&
	(namelen == 5 \|\| name[5] == namespace->hier_sep)) {

	if (name[5] == namespace->hier_sep) {
	/* can't create folders under INBOX */
	return IMAP_MAILBOX_BADNAME;
	}

	return 0;
	}

	resultlen = strlen(result);

	/* other personal folder */
	if (domainlen+resultlen+6+namelen > MAX_MAILBOX_NAME) {
	return IMAP_MAILBOX_BADNAME;
	}
	snprintf(result+resultlen, MAX_MAILBOX_BUFFER-resultlen, ".%.*s",
	namelen, name);

	/* Translate any separators in mailboxname */
	mboxname_hiersep_tointernal(namespace, result+6+userlen, 0);
	return 0;
	}

	/*
	* Convert the internal mailbox 'name' to an external name.
	* If 'userid' is non-null, it is the name of the current user.
	* On success, results are placed in the buffer pointed to by
	* 'result', the buffer must be of size MAX_MAILBOX_BUFFER.
	*/

	/* Handle conversion from the internal namespace to the standard namespace */
	static int mboxname_toexternal(struct namespace namespace, const char name,
	const char userid, char result)
	{
	char domain = NULL, cp;
	size_t domainlen = 0, resultlen;

	/* Blank the result, just in case */
	result[0] = '\0';

	if(strlen(name) > MAX_MAILBOX_NAME) return IMAP_MAILBOX_BADNAME;

	if (config_virtdomains && (cp = strchr(name, '!'))) {
	domain = (char*) name;
	domainlen = cp++ - name;
	name = cp;

	/* don't use the domain if it matches the user's domain */
	if (userid && (cp = strchr(userid, '@')) &&
	(strlen(++cp) == domainlen) && !strncmp(domain, cp, domainlen))
	domain = NULL;
	}

	strcpy(result, name);

	/* Translate any separators in mailboxname */
	mboxname_hiersep_toexternal(namespace, result, 0);

	resultlen = strlen(result);

	/* Append domain */
	if (domain) {
	if(resultlen+domainlen+1 > MAX_MAILBOX_NAME)
	return IMAP_MAILBOX_BADNAME;

	snprintf(result+resultlen, MAX_MAILBOX_BUFFER-resultlen,
	"@%.*s", (int) domainlen, domain);
	}

	return 0;
	}

	/* Handle conversion from the internal namespace to the alternate namespace */
	static int mboxname_toexternal_alt(struct namespace namespace, const char name,
	const char userid, char result)
	{
	char *domain;
	size_t userlen, resultlen;

	/* Blank the result, just in case */
	result[0] = '\0';

	if(strlen(name) > MAX_MAILBOX_NAME) return IMAP_MAILBOX_BADNAME;

	if (!userid) return IMAP_MAILBOX_BADNAME;

	userlen = strlen(userid);

	if (config_virtdomains && (domain = strchr(userid, '@'))) {
	size_t domainlen = strlen(domain);

	userlen = domain - userid;

	if (!strncmp(name, domain+1, domainlen-1) &&
	name[domainlen-1] == '!') {
	name += domainlen;
	}
	}

	/* Personal (INBOX) namespace */
	if (!strncasecmp(name, "inbox", 5) &&
	(name[5] == '\0' \|\| name[5] == '.')) {
	if (name[5] == '\0')
	strcpy(result, name);
	else
	strcpy(result, name+6);
	}
	/* paranoia - this shouldn't be needed */
	else if (!strncmp(name, "user.", 5) &&
	!strncmp(name+5, userid, userlen) &&
	(name[5+userlen] == '\0' \|\|
	name[5+userlen] == '.')) {
	if (name[5+userlen] == '\0')
	strcpy(result, "INBOX");
	else
	strcpy(result, name+5+userlen+1);
	}

	/* Other Users namespace */
	else if (!strncmp(name, "user", 4) &&
	(name[4] == '\0' \|\| name[4] == '.')) {
	size_t prefixlen = strlen(namespace->prefix[NAMESPACE_USER]);

	if ((prefixlen > MAX_MAILBOX_NAME) \|\|
	((name[4] == '.') &&
	((prefixlen+1+strlen(name+5)) > MAX_MAILBOX_NAME)))
	return IMAP_MAILBOX_BADNAME;

	sprintf(result, "%.*s",
	(int) (prefixlen-1), namespace->prefix[NAMESPACE_USER]);
	resultlen = strlen(result);
	if (name[4] == '.') {
	sprintf(result+resultlen, "%c%s", namespace->hier_sep, name+5);
	}
	}

	/* Shared namespace */
	else {
	/* special case: LIST/LSUB "" % */
	if (!strncmp(name, namespace->prefix[NAMESPACE_SHARED],
	strlen(namespace->prefix[NAMESPACE_SHARED])-1)) {
	strcpy(result, name);
	}
	else {
	strcpy(result, namespace->prefix[NAMESPACE_SHARED]);
	strcat(result, name);
	}
	}

	/* Translate any separators in mailboxname */
	mboxname_hiersep_toexternal(namespace, result, 0);
	return 0;
	}

	/*
	* Create namespace based on config options.
	*/
	int mboxname_init_namespace(struct namespace *namespace, int isadmin)
	{
	const char *prefix;

	assert(namespace != NULL);

	namespace->isadmin = isadmin;

	namespace->hier_sep =
	config_getswitch(IMAPOPT_UNIXHIERARCHYSEP) ? '/' : '.';
	namespace->isalt = !isadmin && config_getswitch(IMAPOPT_ALTNAMESPACE);

	if (namespace->isalt) {
	/* alternate namespace */
	strcpy(namespace->prefix[NAMESPACE_INBOX], "");

	prefix = config_getstring(IMAPOPT_USERPREFIX);
	if (!prefix \|\| strlen(prefix) == 0 \|\|
	strlen(prefix) >= MAX_NAMESPACE_PREFIX \|\|
	strchr(prefix,namespace->hier_sep) != NULL)
	return IMAP_NAMESPACE_BADPREFIX;
	sprintf(namespace->prefix[NAMESPACE_USER], "%.*s%c",
	MAX_NAMESPACE_PREFIX-1, prefix, namespace->hier_sep);

	prefix = config_getstring(IMAPOPT_SHAREDPREFIX);
	if (!prefix \|\| strlen(prefix) == 0 \|\|
	strlen(prefix) >= MAX_NAMESPACE_PREFIX \|\|
	strchr(prefix, namespace->hier_sep) != NULL \|\|
	!strncmp(namespace->prefix[NAMESPACE_USER], prefix, strlen(prefix)))
	return IMAP_NAMESPACE_BADPREFIX;
	sprintf(namespace->prefix[NAMESPACE_SHARED], "%.*s%c",
	MAX_NAMESPACE_PREFIX-1, prefix, namespace->hier_sep);

	namespace->mboxname_tointernal = mboxname_tointernal_alt;
	namespace->mboxname_toexternal = mboxname_toexternal_alt;
	namespace->mboxlist_findall = mboxlist_findall_alt;
	namespace->mboxlist_findsub = mboxlist_findsub_alt;
	}

	else {
	/* standard namespace */
	sprintf(namespace->prefix[NAMESPACE_INBOX], "%s%c",
	"INBOX", namespace->hier_sep);
	sprintf(namespace->prefix[NAMESPACE_USER], "%s%c",
	"user", namespace->hier_sep);
	strcpy(namespace->prefix[NAMESPACE_SHARED], "");

	namespace->mboxname_tointernal = mboxname_tointernal;
	namespace->mboxname_toexternal = mboxname_toexternal;
	namespace->mboxlist_findall = mboxlist_findall;
	namespace->mboxlist_findsub = mboxlist_findsub;
	}

	return 0;
	}

	/*
	* Translate separator charactors in a mailboxname from its external
	* representation to its internal representation '.'.
	* If using the unixhierarchysep '/', all '.'s get translated to DOTCHAR.
	*/
	char mboxname_hiersep_tointernal(struct namespace namespace, char *name,
	int length)
	{
	char *p;

	assert(namespace != NULL);
	assert(namespace->hier_sep == '.' \|\| namespace->hier_sep == '/');

	if (!length) length = strlen(name);

	if (namespace->hier_sep == '/') {
	/* change all '/'s to '.' and all '.'s to DOTCHAR */
	for (p = name; *p && length; p++, length--) {
	if (p == '/') p = '.';
	else if (p == '.') p = DOTCHAR;
	}
	}

	return name;
	}

	/*
	* Translate separator charactors in a mailboxname from its internal
	* representation '.' to its external representation.
	* If using the unixhierarchysep '/', all DOTCHAR get translated to '.'.
	*/
	char mboxname_hiersep_toexternal(struct namespace namespace, char *name,
	int length)
	{
	char *p;

	assert(namespace != NULL);
	assert(namespace->hier_sep == '.' \|\| namespace->hier_sep == '/');

	if (!length) length=strlen(name);

	if (namespace->hier_sep == '/') {
	/* change all '.'s to '/' and all DOTCHARs to '.' */
	for (p = name; *p && length; p++, length--) {
	if (p == '.') p = '/';
	else if (p == DOTCHAR) p = '.';
	}
	}

	return name;
	}

	/*
	* Return nonzero if 'userid' owns the (internal) mailbox 'name'.
	*/
	int mboxname_userownsmailbox(const char userid, const char name)
	{
	struct namespace internal = { '.', 0, 0, { "INBOX.", "user.", "" },
	NULL, NULL, NULL, NULL };
	char inboxname[MAX_MAILBOX_BUFFER];

	if (!mboxname_tointernal(&internal, "INBOX", userid, inboxname) &&
	!strncmp(name, inboxname, strlen(inboxname)) &&
	(name[strlen(inboxname)] == '\0' \|\| name[strlen(inboxname)] == '.')) {
	return 1;
	}
	return 0;
	}

	/*
	* If (internal) mailbox 'name' is a user's mailbox (optionally INBOX),
	* returns a pointer to the userid, otherwise returns NULL.
	*/
	char mboxname_isusermailbox(const char name, int isinbox)
	{
	const char *p;
	const char *start = name;

	/* step past the domain part */
	if (config_virtdomains && (p = strchr(start, '!')))
	start = p + 1;

	/* starts with "user." AND
	* we don't care if it's an inbox OR
	* there's no dots after the username
	*/
	if (!strncmp(start, "user.", 5) && (!isinbox \|\| !strchr(start+5, '.')))
	return (char*) start+5;
	else
	return NULL;
	}

	/*
	* If (internal) mailbox 'name' is a DELETED mailbox
	* returns boolean
	*/
	int mboxname_isdeletedmailbox(const char *name)
	{
	static const char *deletedprefix = NULL;
	static int deletedprefix_len = 0;
	int domainlen = 0;
	char *p;

	if (!deletedprefix) {
	deletedprefix = config_getstring(IMAPOPT_DELETEDPREFIX);
	deletedprefix_len = strlen(deletedprefix);
	}

	if (config_virtdomains && (p = strchr(name, '!')))
	domainlen = p - name + 1;

	return ((!strncmp(name + domainlen, deletedprefix, deletedprefix_len) &&
	name[domainlen + deletedprefix_len] == '.') ? 1 : 0);
	}

	/*
	* Translate (internal) inboxname into corresponding userid.
	*/
	char mboxname_to_userid(const char mboxname)
	{
	static char userid[MAX_MAILBOX_BUFFER];
	const char domain = NULL, cp;
	char *rp;
	int domainlen = 0;

	if (config_virtdomains && (cp = strchr(mboxname, '!'))) {
	/* locate, save, and skip domain */
	domain = mboxname;
	domainlen = cp++ - mboxname;
	} else {
	cp = mboxname;
	}

	/* not a user mailbox? */
	if (strncmp(cp, "user.", 5))
	return NULL;

	/* skip "user." */
	strcpy(userid, cp + 5);

	/* find end of userid */
	rp = strchr(userid, '.');
	if (!rp) rp = userid + strlen(userid);

	if (domain) {
	/* append domain */
	sprintf(rp, "@%.*s", domainlen, domain);
	}
	else {
	/* otherwise close off at end of userid anyway */
	*rp = '\0';
	}

	return(userid);
	}

	/*
	* Apply additional restrictions on netnews mailbox names.
	* Cannot have all-numeric name components.
	*/
	int mboxname_netnewscheck(const char *name)
	{
	int c;
	int sawnonnumeric = 0;

	while ((c = *name++)!=0) {
	switch (c) {
	case '.':
	if (!sawnonnumeric) return IMAP_MAILBOX_BADNAME;
	sawnonnumeric = 0;
	break;

	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;

	default:
	sawnonnumeric = 1;
	break;
	}
	}
	if (!sawnonnumeric) return IMAP_MAILBOX_BADNAME;
	return 0;
	}

	/*
	* Apply site policy restrictions on mailbox names.
	* Restrictions are hardwired for now.
	*/
	#define GOODCHARS " #$'+,-.0123456789:=@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~"
	int mboxname_policycheck(const char *name)
	{
	unsigned i;
	struct glob *g;
	int sawutf7 = 0;
	unsigned c1, c2, c3, c4, c5, c6, c7, c8;
	int ucs4;
	int unixsep;

	unixsep = config_getswitch(IMAPOPT_UNIXHIERARCHYSEP);

	/* Skip policy check on mailbox created in delayed delete namespace
	* assuming the mailbox existed before and was OK then.
	* This should allow mailboxes that are extremely long to be
	* deleted when delayed_delete is enabled.
	* A thorough fix might remove the prefix and timestamp
	* then continue with the check
	*/
	if (!mboxname_isdeletedmailbox(name)) {
	if (strlen(name) > MAX_MAILBOX_NAME)
	return IMAP_MAILBOX_BADNAME;
	}
	for (i = 0; i < NUM_BADMBOXPATTERNS; i++) {
	g = glob_init(badmboxpatterns[i], 0);
	if (GLOB_TEST(g, name) != -1) {
	glob_free(&g);
	return IMAP_MAILBOX_BADNAME;
	}
	glob_free(&g);
	}

	if (*name == '~') return IMAP_MAILBOX_BADNAME;
	while (*name) {
	if (*name == '&') {
	/* Modified UTF-7 */
	name++;
	while (*name != '-') {
	if (sawutf7) {
	/* Two adjacent utf7 sequences */
	return IMAP_MAILBOX_BADNAME;
	}

	if ((c1 = CHARMOD64(*name++)) == XX \|\|
	(c2 = CHARMOD64(*name++)) == XX \|\|
	(c3 = CHARMOD64(*name++)) == XX) {
	/* Non-base64 character */
	return IMAP_MAILBOX_BADNAME;
	}
	ucs4 = (c1 << 10) \| (c2 << 4) \| (c3 >> 2);
	if ((ucs4 & 0xff80) == 0 \|\| (ucs4 & 0xf800) == 0xd800) {
	/* US-ASCII or multi-word character */
	return IMAP_MAILBOX_BADNAME;
	}
	if (*name == '-') {
	/* Trailing bits not zero */
	if (c3 & 0x03) return IMAP_MAILBOX_BADNAME;

	/* End of UTF-7 sequence */
	break;
	}

	if ((c4 = CHARMOD64(*name++)) == XX \|\|
	(c5 = CHARMOD64(*name++)) == XX \|\|
	(c6 = CHARMOD64(*name++)) == XX) {
	/* Non-base64 character */
	return IMAP_MAILBOX_BADNAME;
	}
	ucs4 = ((c3 & 0x03) << 14) \| (c4 << 8) \| (c5 << 2) \| (c6 >> 4);
	if ((ucs4 & 0xff80) == 0 \|\| (ucs4 & 0xf800) == 0xd800) {
	/* US-ASCII or multi-word character */
	return IMAP_MAILBOX_BADNAME;
	}
	if (*name == '-') {
	/* Trailing bits not zero */
	if (c6 & 0x0f) return IMAP_MAILBOX_BADNAME;

	/* End of UTF-7 sequence */
	break;
	}

	if ((c7 = CHARMOD64(*name++)) == XX \|\|
	(c8 = CHARMOD64(*name++)) == XX) {
	/* Non-base64 character */
	return IMAP_MAILBOX_BADNAME;
	}
	ucs4 = ((c6 & 0x0f) << 12) \| (c7 << 6) \| c8;
	if ((ucs4 & 0xff80) == 0 \|\| (ucs4 & 0xf800) == 0xd800) {
	/* US-ASCII or multi-word character */
	return IMAP_MAILBOX_BADNAME;
	}
	}

	if (name[-1] == '&') sawutf7 = 0; /* '&-' is sequence for '&' */
	else sawutf7 = 1;

	name++; /* Skip over terminating '-' */
	}
	else {
	if (!strchr(GOODCHARS, *name) &&
	/* If we're using unixhierarchysep, DOTCHAR is allowed */
	!(unixsep && *name == DOTCHAR))
	return IMAP_MAILBOX_BADNAME;
	name++;
	sawutf7 = 0;
	}
	}
	return 0;
	}

	void mboxname_hash(char *buf, size_t buf_len,
	const char *root,
	const char *name)
	{
	const char *idx;
	char c, *p;

	snprintf(buf, buf_len, "%s", root);
	buf_len -= strlen(buf);
	buf += strlen(buf);

	if (config_virtdomains && (p = strchr(name, '!'))) {
	p = '\0'; / split domain!user */
	if (config_hashimapspool) {
	c = (char) dir_hash_c(name, config_fulldirhash);
	snprintf(buf, buf_len, "%s%c/%s", FNAME_DOMAINDIR, c, name);
	}
	else {
	snprintf(buf, buf_len, "%s%s", FNAME_DOMAINDIR, name);
	}
	p++ = '!'; / reassemble domain!user */
	name = p;
	buf_len -= strlen(buf);
	buf += strlen(buf);
	}

	if (config_hashimapspool) {
	idx = strchr(name, '.');
	if (idx == NULL) {
	idx = name;
	} else {
	idx++;
	}
	c = (char) dir_hash_c(idx, config_fulldirhash);

	snprintf(buf, buf_len, "/%c/%s", c, name);
	} else {
	/* standard mailbox placement */
	snprintf(buf, buf_len, "/%s", name);
	}

	/* change all '.'s to '/' */
	for (p = buf; *p; p++) {
	if (p == '.') p = '/';
	}
	}

	/* note: mboxname must be internal */
	char mboxname_datapath(const char partition, const char *mboxname, unsigned long uid)
	{
	static char pathresult[MAX_MAILBOX_PATH+1];
	const char *root;

	root = config_partitiondir(partition);
	if (!root) return NULL;

	if (!mboxname) {
	strncpy(pathresult, root, MAX_MAILBOX_PATH);
	return pathresult;
	}

	mboxname_hash(pathresult, MAX_MAILBOX_PATH, root, mboxname);

	if (uid) {
	int len = strlen(pathresult);
	snprintf(pathresult + len, MAX_MAILBOX_PATH - len, "/%lu.", uid);
	}
	pathresult[MAX_MAILBOX_PATH] = '\0';

	if (strlen(pathresult) == MAX_MAILBOX_PATH)
	return NULL;

	return pathresult;
	}

	char mboxname_lockpath(const char mboxname)
	{
	static char lockresult[MAX_MAILBOX_PATH+1];
	char basepath[MAX_MAILBOX_PATH+1];
	const char *root = config_getstring(IMAPOPT_MBOXNAME_LOCKPATH);
	int len;

	if (!root) {
	snprintf(basepath, MAX_MAILBOX_PATH, "%s/lock", config_dir);
	root = basepath;
	}

	mboxname_hash(lockresult, MAX_MAILBOX_PATH, root, mboxname);

	len = strlen(lockresult);
	snprintf(lockresult + len, MAX_MAILBOX_PATH - len, "%s", ".lock");
	lockresult[MAX_MAILBOX_PATH] = '\0';

	if (strlen(lockresult) == MAX_MAILBOX_PATH)
	return NULL;

	return lockresult;
	}

	char mboxname_metapath(const char partition, const char *mboxname,
	int metafile, int isnew)
	{
	static char metaresult[MAX_MAILBOX_PATH];
	int metaflag = 0;
	const char *root = NULL;
	const char *filename = NULL;
	char confkey[256];

	*confkey = '\0';

	switch (metafile) {
	case META_HEADER:
	snprintf(confkey, 256, "metadir-header-%s", partition);
	metaflag = IMAP_ENUM_METAPARTITION_FILES_HEADER;
	filename = FNAME_HEADER;
	break;
	case META_INDEX:
	snprintf(confkey, 256, "metadir-index-%s", partition);
	metaflag = IMAP_ENUM_METAPARTITION_FILES_INDEX;
	filename = FNAME_INDEX;
	break;
	case META_CACHE:
	snprintf(confkey, 256, "metadir-cache-%s", partition);
	metaflag = IMAP_ENUM_METAPARTITION_FILES_CACHE;
	filename = FNAME_CACHE;
	break;
	case META_EXPUNGE:
	/* not movable, it's only old */
	metaflag = IMAP_ENUM_METAPARTITION_FILES_EXPUNGE;
	filename = FNAME_EXPUNGE;
	break;
	case META_SQUAT:
	snprintf(confkey, 256, "metadir-squat-%s", partition);
	metaflag = IMAP_ENUM_METAPARTITION_FILES_SQUAT;
	filename = FNAME_SQUAT;
	break;
	case 0:
	break;
	default:
	fatal("Unknown meta file requested", EC_SOFTWARE);
	}

	if (*confkey)
	root = config_getoverflowstring(confkey, NULL);

	if (!root && (!metaflag \|\| (config_metapartition_files & metaflag)))
	root = config_metapartitiondir(partition);

	if (!root)
	root = config_partitiondir(partition);

	if (!root)
	return NULL;

	if (!mboxname) {
	strncpy(metaresult, root, MAX_MAILBOX_PATH);
	return metaresult;
	}

	mboxname_hash(metaresult, MAX_MAILBOX_PATH, root, mboxname);

	if (filename) {
	int len = strlen(metaresult);
	if (isnew)
	snprintf(metaresult + len, MAX_MAILBOX_PATH - len, "%s.NEW", filename);
	else
	snprintf(metaresult + len, MAX_MAILBOX_PATH - len, "%s", filename);
	}

	if (strlen(metaresult) >= MAX_MAILBOX_PATH)
	return NULL;

	return metaresult;
	}

	void mboxname_todeleted(const char name, char result, int withtime)
	{
	int domainlen = 0;
	char *p;
	const char *deletedprefix = config_getstring(IMAPOPT_DELETEDPREFIX);

	strncpy(result, name, MAX_MAILBOX_BUFFER);

	if (config_virtdomains && (p = strchr(name, '!')))
	domainlen = p - name + 1;

	if (withtime) {
	struct timeval tv;
	gettimeofday( &tv, NULL );
	snprintf(result+domainlen, MAX_MAILBOX_BUFFER-domainlen, "%s.%s.%X",
	deletedprefix, name+domainlen, (unsigned) tv.tv_sec);
	} else {
	snprintf(result+domainlen, MAX_MAILBOX_BUFFER-domainlen, "%s.%s",
	deletedprefix, name+domainlen);
	}
	}
	diff --git a/imap/mbpath.c b/imap/mbpath.c
	index bbaae2eb0..45cd696e5 100644
	--- a/imap/mbpath.c
	+++ b/imap/mbpath.c
	@@ -1,162 +1,162 @@
	/* mbpath.c -- help the sysadmin to find the path matching the mailbox
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: mbpath.c,v 1.23 2010/01/06 17:01:37 murch Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <string.h>
	#include <ctype.h>
	#include <fcntl.h>
	#include <sys/stat.h>
	#include <sys/param.h>

	#include "acl.h"
	#include "util.h"
	#include "auth.h"
	#include "prot.h"
	#include "imparse.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "global.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "xmalloc.h"
	#include "mboxlist.h"

	extern int optind;
	extern char *optarg;

	/* current namespace */
	static struct namespace mbpath_namespace;

	/* config.c stuff */
	const int config_need_data = 0;

	static int
	usage(void) {
	fprintf(stderr,"usage: mbpath [-C <alt_config>] [-q] [-s] [-m] <mailbox name>...\n");
	fprintf(stderr,"\t-q\tquietly drop any error messages\n");
	fprintf(stderr,"\t-s\tstop on error\n");
	fprintf(stderr,"\t-m\toutput the path to the metadata files (if different from the message files)\n");
	exit(-1);
	}

	int
	main(int argc, char **argv)
	{
	struct mboxlist_entry mbentry;
	int rc, i, quiet = 0, stop_on_error=0, metadata=0;
	int opt; /* getopt() returns an int */
	char *alt_config = NULL;
	char buf[MAX_MAILBOX_PATH+1];

	if ((geteuid()) == 0 && (become_cyrus() != 0)) {
	fatal("must run as the Cyrus user", EC_USAGE);
	}

	while ((opt = getopt(argc, argv, "C:qsm")) != EOF) {
	switch(opt) {
	case 'C': /* alt config file */
	alt_config = optarg;
	break;
	case 'q':
	quiet = 1;
	break;
	case 's':
	stop_on_error = 1;
	break;
	case 'm':
	metadata = 1;
	break;

	default:
	usage();
	}
	}

	cyrus_init(alt_config, "mbpath", 0);

	if ((rc = mboxname_init_namespace(&mbpath_namespace, 1)) != 0) {
	fatal(error_message(rc), -1);
	}

	mboxlist_init(0);
	mboxlist_open(NULL);

	for (i = optind; i < argc; i++) {
	/* Translate mailboxname */
	(*mbpath_namespace.mboxname_tointernal)(&mbpath_namespace, argv[i], NULL, buf);

	if ((rc = mboxlist_lookup(buf, &mbentry, NULL)) == 0) {
	char *path = mboxname_metapath(mbentry.partition, mbentry.name, 0, 0);
	printf("%s\n", path);
	} else {
	if (!quiet && (rc == IMAP_MAILBOX_NONEXISTENT)) {
	fprintf(stderr, "Invalid mailbox name: %s\n", argv[i]);
	}
	if (stop_on_error) {
	if (quiet) {
	fatal("", -1);
	} else {
	fatal("Error in processing mailbox. Stopping\n", -1);
	}
	}
	}
	}

	mboxlist_close();
	mboxlist_done();

	cyrus_done();

	return 0;
	}

	/* $Header: /mnt/data/cyrus/cvsroot/src/cyrus/imap/mbpath.c,v 1.23 2010/01/06 17:01:37 murch Exp $ */

	diff --git a/imap/reconstruct.c b/imap/reconstruct.c
	index 01c3717b0..65780a0f8 100644
	--- a/imap/reconstruct.c
	+++ b/imap/reconstruct.c
	@@ -1,584 +1,584 @@
	/* reconstruct.c -- program to reconstruct a mailbox
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: reconstruct.c,v 1.112 2010/01/06 17:01:39 murch Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdlib.h>
	#include <stdio.h>
	#include <errno.h>
	#include <string.h>
	#include <fcntl.h>
	#include <ctype.h>
	#include <utime.h>
	#include <syslog.h>
	#include <sys/types.h>
	#include <netinet/in.h>
	#include <sys/stat.h>
	#include <stdlib.h>
	#ifdef HAVE_ZLIB
	#include <zlib.h>
	#endif

	#if HAVE_DIRENT_H
	# include <dirent.h>
	# define NAMLEN(dirent) strlen((dirent)->d_name)
	#else
	# define dirent direct
	# define NAMLEN(dirent) (dirent)->d_namlen
	# if HAVE_SYS_NDIR_H
	# include <sys/ndir.h>
	# endif
	# if HAVE_SYS_DIR_H
	# include <sys/dir.h>
	# endif
	# if HAVE_NDIR_H
	# include <ndir.h>
	# endif
	#endif

	#include "acl.h"
	#include "assert.h"
	#include "bsearch.h"
	#include "crc32.h"
	#include "imparse.h"
	#include "global.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "map.h"
	#include "message.h"
	#include "message_guid.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"
	#include "global.h"
	#include "mboxname.h"
	#include "mboxlist.h"
	#include "quota.h"
	#include "seen.h"
	#include "retry.h"
	#include "convert_code.h"
	#include "util.h"
	#include "sync_log.h"
	-#include "lock.h"
	+#include "cyr_lock.h"

	extern int optind;
	extern char *optarg;

	struct discovered {
	char *name;
	struct discovered *next;
	};
	struct uniqmailid {
	char * uniqmbxid;
	char *uniqname;
	struct uniqmailid *uniqnext;
	};

	struct uniqmailid *uniqmid_head;

	/* current namespace */
	static struct namespace recon_namespace;

	/* config.c stuff */
	const int config_need_data = CONFIG_NEED_PARTITION_DATA;

	/* forward declarations */
	void do_mboxlist(void);
	int do_reconstruct(char name, int matchlen, int maycreate, void rock);
	int reconstruct(char name, struct discovered l);
	void usage(void);
	char * getmailname (char * mailboxname);
	struct uniqmailid * add_uniqid (char * mailboxname, char * mailboxid);
	struct uniqmailid * find_uniqid (char * mailboxname, char * mailboxid);

	extern cyrus_acl_canonproc_t mboxlist_ensureOwnerRights;

	int reconstruct_flags = RECONSTRUCT_MAKE_CHANGES \| RECONSTRUCT_DO_STAT;

	int main(int argc, char **argv)
	{
	int opt, i, r;
	int rflag = 0;
	int mflag = 0;
	int fflag = 0;
	int xflag = 0;
	char buf[MAX_MAILBOX_PATH+1];
	char *fname;
	struct discovered head;
	char *alt_config = NULL;
	char *start_part = NULL;

	memset(&head, 0, sizeof(head));

	if ((geteuid()) == 0 && (become_cyrus() != 0)) {
	fatal("must run as the Cyrus user", EC_USAGE);
	}

	/* Ensure we're up-to-date on the index file format */
	assert(INDEX_HEADER_SIZE == (OFFSET_HEADER_CRC+4));
	assert(INDEX_RECORD_SIZE == (OFFSET_RECORD_CRC+4));

	while ((opt = getopt(argc, argv, "C:kp:rmfsxgGqRoO")) != EOF) {
	switch (opt) {
	case 'C': /* alt config file */
	alt_config = optarg;
	break;

	case 'p':
	start_part = optarg;
	break;

	case 'r':
	rflag = 1;
	break;

	case 'm':
	mflag = 1;
	break;

	case 'n':
	reconstruct_flags &= ~RECONSTRUCT_MAKE_CHANGES;
	break;

	case 'g':
	fprintf(stderr, "deprecated option -g used\n");
	break;

	case 'G':
	reconstruct_flags \|= RECONSTRUCT_ALWAYS_PARSE;
	break;

	case 'f':
	fflag = 1;
	break;

	case 'x':
	xflag = 1;
	break;

	case 'k':
	fprintf(stderr, "deprecated option -k used\n");
	break;

	case 's':
	reconstruct_flags &= ~RECONSTRUCT_DO_STAT;
	break;

	case 'q':
	reconstruct_flags \|= RECONSTRUCT_QUIET;
	break;

	case 'R':
	reconstruct_flags \|= RECONSTRUCT_GUID_REWRITE;
	break;

	case 'U':
	reconstruct_flags \|= RECONSTRUCT_GUID_UNLINK;
	break;

	case 'o':
	reconstruct_flags \|= RECONSTRUCT_IGNORE_ODDFILES;
	break;

	case 'O':
	reconstruct_flags \|= RECONSTRUCT_REMOVE_ODDFILES;
	break;

	default:
	usage();
	}
	}

	cyrus_init(alt_config, "reconstruct", 0);
	global_sasl_init(1,0,NULL);

	/* Set namespace -- force standard (internal) */
	if ((r = mboxname_init_namespace(&recon_namespace, 1)) != 0) {
	syslog(LOG_ERR, "%s", error_message(r));
	fatal(error_message(r), EC_CONFIG);
	}

	sync_log_init();

	if (mflag) {
	if (rflag \|\| fflag \|\| optind != argc) {
	cyrus_done();
	usage();
	}
	do_mboxlist();
	}

	mboxlist_init(0);
	mboxlist_open(NULL);

	quotadb_init(0);
	quotadb_open(NULL);

	/* Deal with nonexistent mailboxes */
	if (start_part) {
	/* We were handed a mailbox that does not exist currently */
	if(optind == argc) {
	fprintf(stderr,
	"When using -p, you must specify a mailbox to attempt to reconstruct.");
	exit(EC_USAGE);
	}

	/* do any of the mailboxes exist in mboxlist already? */
	/* Do they look like mailboxes? */
	for (i = optind; i < argc; i++) {
	struct stat sbuf;

	if(strchr(argv[i],'%') \|\| strchr(argv[i],'*')) {
	fprintf(stderr, "Using wildcards with -p is not supported.\n");
	exit(EC_USAGE);
	}

	/* Translate mailboxname */
	(*recon_namespace.mboxname_tointernal)(&recon_namespace, argv[i],
	NULL, buf);

	/* Does it exist */
	do {
	r = mboxlist_lookup(buf, NULL, NULL);
	} while (r == IMAP_AGAIN);

	if (r != IMAP_MAILBOX_NONEXISTENT) {
	fprintf(stderr,
	"Mailbox %s already exists. Cannot specify -p.\n",
	argv[i]);
	exit(EC_USAGE);
	}

	/* Does the suspected path look like a mailbox? */
	fname = mboxname_metapath(start_part, buf, META_HEADER, 0);
	if (stat(fname, &sbuf) < 0) {
	fprintf(stderr,
	"%s does not appear to be a mailbox (no %s).\n",
	argv[i], fname);
	exit(EC_USAGE);
	}
	}

	/* None of them exist. Create them. */
	for (i = optind; i < argc; i++) {
	/* Translate mailboxname */
	(*recon_namespace.mboxname_tointernal)(&recon_namespace, argv[i],
	NULL, buf);

	r = mboxlist_createmailbox(buf, 0, start_part, 1,
	"cyrus", NULL, 0, 0, !xflag);
	if(r) {
	fprintf(stderr, "could not create %s\n", argv[i]);
	}
	}
	}

	/* Normal Operation */
	if (optind == argc) {
	if (rflag) {
	fprintf(stderr, "please specify a mailbox to recurse from\n");
	cyrus_done();
	exit(EC_USAGE);
	}
	assert(!rflag);
	strlcpy(buf, "*", sizeof(buf));
	(*recon_namespace.mboxlist_findall)(&recon_namespace, buf, 1, 0, 0,
	do_reconstruct, NULL);
	}

	for (i = optind; i < argc; i++) {
	char *domain = NULL;

	/* save domain */
	if (config_virtdomains) domain = strchr(argv[i], '@');

	strlcpy(buf, argv[i], sizeof(buf));
	/* Translate any separators in mailboxname */
	mboxname_hiersep_tointernal(&recon_namespace, buf,
	config_virtdomains ?
	strcspn(buf, "@") : 0);

	/* reconstruct the first mailbox/pattern */
	(*recon_namespace.mboxlist_findall)(&recon_namespace, buf, 1, 0,
	0, do_reconstruct,
	fflag ? &head : NULL);
	if (rflag) {
	/* build a pattern for submailboxes */
	char *p = strchr(buf, '@');
	if (p) *p = '\0';
	strlcat(buf, ".*", sizeof(buf));

	/* append the domain */
	if (domain) strlcat(buf, domain, sizeof(buf));

	/* reconstruct the submailboxes */
	(*recon_namespace.mboxlist_findall)(&recon_namespace, buf, 1, 0,
	0, do_reconstruct,
	fflag ? &head : NULL);
	}
	}

	/* examine our list to see if we discovered anything */
	while (head.next) {
	struct discovered *p;
	int r = 0;

	p = head.next;
	head.next = p->next;

	/* create p (database only) and reconstruct it */
	/* partition is defined by the parent mailbox */
	r = mboxlist_createmailbox(p->name, 0, NULL, 1,
	"cyrus", NULL, 0, 0, !xflag);
	if (r) {
	fprintf(stderr, "createmailbox %s: %s\n",
	p->name, error_message(r));
	} else {
	do_reconstruct(p->name, strlen(p->name), 0, &head);
	}
	/* may have added more things into our list */

	free(p->name);
	free(p);
	}

	sync_log_done();

	mboxlist_close();
	mboxlist_done();

	quotadb_close();
	quotadb_done();

	cyrus_done();

	return 0;
	}

	void usage(void)
	{
	fprintf(stderr,
	"usage: reconstruct [-C <alt_config>] [-p partition] [-ksrfx] mailbox...\n");
	fprintf(stderr, " reconstruct [-C <alt_config>] -m\n");
	exit(EC_USAGE);
	}

	/*
	* mboxlist_findall() callback function to reconstruct a mailbox
	*/
	int
	do_reconstruct(char *name,
	int matchlen,
	int maycreate __attribute__((unused)),
	void *rock)
	{
	struct discovered found = (struct discovered )rock;
	int r;
	char buf[MAX_MAILBOX_NAME];
	static char lastname[MAX_MAILBOX_NAME] = "";
	struct mailbox *mailbox = NULL;
	char outpath[MAX_MAILBOX_PATH];

	signals_poll();

	/* don't repeat */
	if (matchlen == (int) strlen(lastname) &&
	!strncmp(name, lastname, matchlen)) return 0;

	if(matchlen >= (int) sizeof(lastname))
	matchlen = sizeof(lastname) - 1;

	strncpy(lastname, name, matchlen);
	lastname[matchlen] = '\0';

	r = mailbox_reconstruct(lastname, reconstruct_flags);
	if (r) {
	com_err(lastname, r, "%s",
	(r == IMAP_IOERROR) ? error_message(errno) : NULL);
	return 0;
	}

	r = mailbox_open_iwl(lastname, &mailbox);
	if (r) {
	com_err(lastname, r, "Failed to open after reconstruct");
	return 0;
	}

	if (!add_uniqid(lastname, mailbox->uniqueid)) {
	syslog (LOG_ERR, "Failed adding mailbox: %s unique id: %s\n",
	mailbox->name, mailbox->uniqueid );
	}

	/* Convert internal name to external */
	(*recon_namespace.mboxname_toexternal)(&recon_namespace, lastname,
	NULL, buf);
	if (!(reconstruct_flags & RECONSTRUCT_QUIET))
	printf("%s\n", buf);

	strncpy(outpath, mailbox_meta_fname(mailbox, META_HEADER), MAX_MAILBOX_NAME);
	mailbox_close(&mailbox);

	if (found) {
	char fnamebuf[MAX_MAILBOX_PATH];
	char *ptr;
	DIR *dirp;
	struct dirent *dirent;
	struct stat sbuf;

	ptr = strstr(outpath, "cyrus.header");
	if (!ptr) return 0;
	*ptr = 0;

	r = chdir(outpath);
	if (r) return 0;

	/* we recurse down this directory to see if there's any mailboxes
	under this not in the mailboxes database */
	dirp = opendir(".");
	if (!dirp) return 0;

	while ((dirent = readdir(dirp)) != NULL) {
	struct discovered *new;

	/* mailbox directories never have a dot in them */
	if (strchr(dirent->d_name, '.')) continue;
	if (stat(dirent->d_name, &sbuf) < 0) continue;
	if (!S_ISDIR(sbuf.st_mode)) continue;

	/* ok, we found a directory that doesn't have a dot in it;
	is there a cyrus.header file? */
	snprintf(fnamebuf, MAX_MAILBOX_PATH, "%s%s",
	dirent->d_name, FNAME_HEADER);
	if (stat(fnamebuf, &sbuf) < 0) continue;

	/* ok, we have a real mailbox directory */
	snprintf(buf, MAX_MAILBOX_NAME, "%s.%s",
	name, dirent->d_name);

	/* does fnamebuf exist as a mailbox in mboxlist? */
	do {
	r = mboxlist_lookup(buf, NULL, NULL);
	} while (r == IMAP_AGAIN);
	if (!r) continue; /* mailbox exists; it'll be reconstructed
	with a -r */

	if (r != IMAP_MAILBOX_NONEXISTENT) break; /* erg? */
	else r = 0; /* reset error condition */

	printf("discovered %s\n", buf);
	new = (struct discovered *) xmalloc(sizeof(struct discovered));
	new->name = strdup(buf);
	new->next = found->next;
	found->next = new;
	}
	closedir(dirp);
	}

	return 0;
	}

	char getmailname(char mailboxname)
	{
	static char namebuf[MAX_MAILBOX_PATH + 1];

	char * pname;

	strlcpy (namebuf, mailboxname, sizeof (namebuf));
	pname = strchr (namebuf, '.');
	if (pname) {
	pname = strchr(pname + 1, '.');
	if (pname)
	*pname = '\0';
	}
	return (namebuf);
	}

	struct uniqmailid *
	find_uniqid ( char * mailboxname, char * mailboxid)
	{
	struct uniqmailid *puniq;
	char * nameptr;

	nameptr = getmailname (mailboxname);
	for (puniq = uniqmid_head; puniq != NULL; puniq = puniq->uniqnext) {
	if (strcmp (puniq->uniqmbxid, mailboxid) == 0) {
	if (strcmp (puniq->uniqname, nameptr) == 0) {
	return (puniq);
	}
	}
	}

	return NULL;
	}

	struct uniqmailid *
	add_uniqid ( char * mailboxname, char * mailboxid)
	{
	struct uniqmailid *puniq;
	char *pboxname;

	pboxname = getmailname (mailboxname);

	puniq = xmalloc (sizeof (struct uniqmailid));
	puniq->uniqmbxid = xstrdup(mailboxid);
	puniq->uniqname = xstrdup(pboxname);

	puniq->uniqnext = uniqmid_head;
	uniqmid_head = puniq;

	return (puniq);
	}

	/*
	* Reconstruct the mailboxes list.
	*/
	void do_mboxlist(void)
	{
	fprintf(stderr, "reconstructing mailboxes.db currently not supported\n");
	exit(EC_USAGE);
	}
	diff --git a/imap/sync_client.c b/imap/sync_client.c
	index 936593ecc..4579f9f49 100644
	--- a/imap/sync_client.c
	+++ b/imap/sync_client.c
	@@ -1,2927 +1,2927 @@
	/* sync_client.c -- Cyrus synchonization client
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: sync_client.c,v 1.51 2010/06/28 12:04:20 brong Exp $
	*
	* Original version written by David Carter <dpc22@cam.ac.uk>
	* Rewritten and integrated into Cyrus by Ken Murchison <ken@oceana.com>
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdlib.h>
	#include <stdio.h>
	#include <time.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <fcntl.h>
	#include <syslog.h>
	#include <string.h>
	#include <sys/wait.h>
	#include <errno.h>
	#include <ctype.h>
	#include <signal.h>
	#include <utime.h>

	#include <netinet/tcp.h>

	#include "global.h"
	#include "assert.h"
	#include "append.h"
	#include "mboxlist.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "quota.h"
	#include "xmalloc.h"
	#include "acl.h"
	#include "seen.h"
	#include "mboxname.h"
	#include "map.h"
	#include "imapd.h"
	#include "imparse.h"
	#include "util.h"
	#include "prot.h"
	#include "message_guid.h"
	#include "sync_support.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "backend.h"
	#include "xstrlcat.h"
	#include "xstrlcpy.h"
	#include "signals.h"
	#include "cyrusdb.h"

	/* signal to config.c */
	const int config_need_data = 0; /* YYY */

	/* ====================================================================== */

	/* Static global variables and support routines for sync_client */

	extern char *optarg;
	extern int optind;

	static const char *servername = NULL;
	static struct protstream *sync_out = NULL;
	static struct protstream *sync_in = NULL;

	static struct namespace sync_namespace;

	static int verbose = 0;
	static int verbose_logging = 0;
	static int connect_once = 0;
	static int foreground = 0;
	static int do_compress = 0;

	static struct protocol_t csync_protocol =
	{ "csync", "csync",
	{ 1, "* OK" },
	{ NULL, NULL, "* OK", NULL,
	{ { "* SASL ", CAPA_AUTH },
	{ "* STARTTLS", CAPA_STARTTLS },
	{ NULL, 0 } } },
	{ "STARTTLS", "OK", "NO", 0 },
	{ "AUTHENTICATE", USHRT_MAX, 0, "OK", "NO", "+ ", "*", NULL, 0 },
	{ NULL, NULL, NULL },
	{ "NOOP", NULL, "OK" },
	{ "EXIT", NULL, "OK" }
	};

	static int do_meta(char *user);

	static void shut_down(int code) __attribute__((noreturn));
	static void shut_down(int code)
	{
	in_shutdown = 1;

	seen_done();
	annotatemore_close();
	annotatemore_done();
	quotadb_close();
	quotadb_done();
	mboxlist_close();
	mboxlist_done();
	cyrus_done();
	exit(code);
	}

	static int usage(const char *name)
	{
	fprintf(stderr,
	"usage: %s -S <servername> [-C <alt_config>] [-r] [-v] mailbox...\n", name);

	exit(EC_USAGE);
	}

	void fatal(const char *s, int code)
	{
	fprintf(stderr, "Fatal error: %s\n", s);
	syslog(LOG_ERR, "Fatal error: %s", s);
	exit(code);
	}

	/* ====================================================================== */

	/* Routines relevant to reserve operation */

	/* Find the messages that we will want to upload from this mailbox,
	* flag messages that are already available at the server end */

	static int find_reserve_messages(struct mailbox *mailbox,
	unsigned last_uid,
	struct sync_msgid_list *part_list)
	{
	struct index_record record;
	uint32_t recno;
	int r;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	r = mailbox_read_index_record(mailbox, recno, &record);

	if (r) {
	syslog(LOG_ERR,
	"IOERROR: reading index entry for recno %u of %s: %m",
	recno, mailbox->name);
	return IMAP_IOERROR;
	}

	if (record.system_flags & FLAG_UNLINKED)
	continue;

	/* skip over records already on replica */
	if (record.uid <= last_uid)
	continue;

	sync_msgid_add(part_list, &record.guid);
	}

	return(0);
	}

	static int find_reserve_all(struct sync_name_list *mboxname_list,
	struct sync_folder_list *master_folders,
	struct sync_folder_list *replica_folders,
	struct sync_reserve_list *reserve_guids)
	{
	struct sync_name *mbox;
	struct sync_folder *rfolder;
	struct sync_msgid_list *part_list;
	struct mailbox *mailbox = NULL;
	int r = 0;

	/* Find messages we want to upload that are available on server */
	for (mbox = mboxname_list->head; mbox; mbox = mbox->next) {
	r = mailbox_open_irl(mbox->name, &mailbox);

	/* Quietly skip over folders which have been deleted since we
	started working (but record fact in case caller cares) */
	if (r == IMAP_MAILBOX_NONEXISTENT) {
	r = 0;
	continue;
	}

	/* Quietly ignore objects that we don't have access to.
	* Includes directory stubs, which have not underlying cyrus.*
	* files in the filesystem */
	if (r == IMAP_PERMISSION_DENIED) {
	r = 0;
	continue;
	}

	if (r) {
	syslog(LOG_ERR, "IOERROR: Failed to open %s: %s",
	mbox->name, error_message(r));
	goto bail;
	}

	/* mailbox is open from here, no exiting without closing it! */

	part_list = sync_reserve_partlist(reserve_guids, mailbox->part);

	sync_folder_list_add(master_folders, mailbox->uniqueid, mailbox->name,
	mailbox->part, mailbox->acl, mailbox->i.options,
	mailbox->i.uidvalidity, mailbox->i.last_uid,
	mailbox->i.highestmodseq, mailbox->i.sync_crc,
	mailbox->i.recentuid, mailbox->i.recenttime,
	mailbox->i.pop3_last_login);

	rfolder = sync_folder_lookup(replica_folders, mailbox->uniqueid);
	if (rfolder)
	find_reserve_messages(mailbox, rfolder->last_uid, part_list);
	else
	find_reserve_messages(mailbox, 0, part_list);

	mailbox_close(&mailbox);
	}

	bail:
	return r;
	}

	static int mark_missing (struct dlist *kin,
	struct sync_msgid_list *part_list)
	{
	struct dlist *kl = kin->head;
	struct dlist *ki;
	struct message_guid tmp_guid;
	struct sync_msgid *msgid;

	/* no missing at all, good */
	if (!kl) return 0;

	if (strcmp(kl->name, "MISSING")) {
	syslog(LOG_ERR, "Illegal response to RESERVE: %s", kl->name);
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	}

	/* unmark each missing item */
	for (ki = kl->head; ki; ki = ki->next) {
	if (!message_guid_decode(&tmp_guid, ki->sval)) {
	syslog(LOG_ERR, "RESERVE: failed to parse GUID %s", ki->sval);
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	}

	msgid = sync_msgid_lookup(part_list, &tmp_guid);
	if (!msgid) {
	syslog(LOG_ERR, "RESERVE: Got unexpected GUID %s", ki->sval);
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	}

	msgid->mark = 0;
	part_list->marked--;
	}

	return 0;
	}

	static int reserve_partition(char *partition,
	struct sync_folder_list *replica_folders,
	struct sync_msgid_list *part_list)
	{
	const char *cmd = "RESERVE";
	struct sync_msgid *msgid;
	struct sync_folder *folder;
	struct dlist *kl;
	struct dlist *kin = NULL;
	struct dlist *ki;
	int r;

	if (!part_list->count)
	return 0; /* nothing to reserve */

	if (!replica_folders->head)
	return 0; /* nowhere to reserve */

	kl = dlist_new(cmd);
	dlist_atom(kl, "PARTITION", partition);

	ki = dlist_list(kl, "MBOXNAME");
	for (folder = replica_folders->head; folder; folder = folder->next)
	dlist_atom(ki, "MBOXNAME", folder->name);

	ki = dlist_list(kl, "GUID");
	for (msgid = part_list->head; msgid; msgid = msgid->next) {
	dlist_atom(ki, "GUID", message_guid_encode(&msgid->guid));
	msgid->mark = 1;
	part_list->marked++;
	}

	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	r = sync_parse_response(cmd, sync_in, &kin);
	if (r) return r;

	r = mark_missing(kin, part_list);
	dlist_free(&kin);

	return r;
	}

	static int reserve_messages(struct sync_name_list *mboxname_list,
	struct sync_folder_list *master_folders,
	struct sync_folder_list *replica_folders,
	struct sync_reserve_list *reserve_guids)
	{
	struct sync_reserve *reserve;
	int r;

	r = find_reserve_all(mboxname_list, master_folders,
	replica_folders, reserve_guids);
	if (r) return r;

	for (reserve = reserve_guids->head; reserve; reserve = reserve->next) {
	r = reserve_partition(reserve->part, replica_folders, reserve->list);
	if (r) return r;
	}

	return 0;
	}

	/* ====================================================================== */

	static int response_parse(const char *cmd,
	struct sync_folder_list *folder_list,
	struct sync_name_list *sub_list,
	struct sync_sieve_list *sieve_list,
	struct sync_seen_list *seen_list,
	struct sync_quota_list *quota_list)
	{
	struct dlist *kin = NULL;
	struct dlist *kl;
	int r;

	r = sync_parse_response(cmd, sync_in, &kin);

	/* Unpleasant: translate remote access error into "please reset me" */
	if (r == IMAP_MAILBOX_NONEXISTENT)
	return 0;

	for (kl = kin->head; kl; kl = kl->next) {
	if (!strcmp(kl->name, "SIEVE")) {
	const char *filename = NULL;
	time_t modtime = 0;
	uint32_t active = 0;
	if (!sieve_list) goto parse_err;
	if (!dlist_getatom(kl, "FILENAME", &filename)) goto parse_err;
	if (!dlist_getdate(kl, "LAST_UPDATE", &modtime)) goto parse_err;
	dlist_getnum(kl, "ISACTIVE", &active); /* optional */
	sync_sieve_list_add(sieve_list, filename, modtime, active);
	}

	else if (!strcmp(kl->name, "QUOTA")) {
	const char *root = NULL;
	uint32_t limit = 0;
	if (!quota_list) goto parse_err;
	if (!dlist_getatom(kl, "ROOT", &root)) goto parse_err;
	if (!dlist_getnum(kl, "LIMIT", &limit)) goto parse_err;
	sync_quota_list_add(quota_list, root, limit);
	}

	else if (!strcmp(kl->name, "LSUB")) {
	struct dlist *i;
	if (!sub_list) goto parse_err;
	for (i = kl->head; i; i = i->next) {
	sync_name_list_add(sub_list, i->sval);
	}
	}

	else if (!strcmp(kl->name, "SEEN")) {
	const char *uniqueid = NULL;
	time_t lastread = 0;
	uint32_t lastuid = 0;
	time_t lastchange = 0;
	const char *seenuids = NULL;
	if (!seen_list) goto parse_err;
	if (!dlist_getatom(kl, "UNIQUEID", &uniqueid)) goto parse_err;
	if (!dlist_getdate(kl, "LASTREAD", &lastread)) goto parse_err;
	if (!dlist_getnum(kl, "LASTUID", &lastuid)) goto parse_err;
	if (!dlist_getdate(kl, "LASTCHANGE", &lastchange)) goto parse_err;
	if (!dlist_getatom(kl, "SEENUIDS", &seenuids)) goto parse_err;
	sync_seen_list_add(seen_list, uniqueid, lastread,
	lastuid, lastchange, seenuids);
	}

	else if (!strcmp(kl->name, "MAILBOX")) {
	const char *uniqueid = NULL;
	const char *mboxname = NULL;
	const char *part = NULL;
	const char *acl = NULL;
	const char *options = NULL;
	modseq_t highestmodseq = 0;
	uint32_t uidvalidity = 0;
	uint32_t last_uid = 0;
	uint32_t sync_crc = 0;
	uint32_t recentuid = 0;
	time_t recenttime = 0;
	time_t pop3_last_login = 0;
	if (!folder_list) goto parse_err;
	if (!dlist_getatom(kl, "UNIQUEID", &uniqueid)) goto parse_err;
	if (!dlist_getatom(kl, "MBOXNAME", &mboxname)) goto parse_err;
	if (!dlist_getatom(kl, "PARTITION", &part)) goto parse_err;
	if (!dlist_getatom(kl, "ACL", &acl)) goto parse_err;
	if (!dlist_getatom(kl, "OPTIONS", &options)) goto parse_err;
	if (!dlist_getmodseq(kl, "HIGHESTMODSEQ", &highestmodseq)) goto parse_err;
	if (!dlist_getnum(kl, "UIDVALIDITY", &uidvalidity)) goto parse_err;
	if (!dlist_getnum(kl, "LAST_UID", &last_uid)) goto parse_err;
	if (!dlist_getnum(kl, "SYNC_CRC", &sync_crc)) goto parse_err;
	if (!dlist_getnum(kl, "RECENTUID", &recentuid)) goto parse_err;
	if (!dlist_getdate(kl, "RECENTTIME", &recenttime)) goto parse_err;
	if (!dlist_getdate(kl, "POP3_LAST_LOGIN", &pop3_last_login)) goto parse_err;

	sync_folder_list_add(folder_list, uniqueid,
	mboxname, part, acl,
	sync_parse_options(options),
	uidvalidity, last_uid,
	highestmodseq, sync_crc,
	recentuid, recenttime,
	pop3_last_login);
	}
	else
	goto parse_err;
	}
	dlist_free(&kin);

	return r;

	parse_err:
	dlist_free(&kin);
	syslog(LOG_ERR, "%s: Invalid response %s", cmd, dlist_lastkey());
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	}

	static int user_reset(char *userid)
	{
	const char *cmd = "UNUSER";
	struct dlist *kl;

	kl = dlist_atom(NULL, cmd, userid);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int folder_rename(char oldname, char newname, char *partition)
	{
	const char *cmd = "RENAME";
	struct dlist *kl;

	kl = dlist_new(cmd);
	dlist_atom(kl, "OLDMBOXNAME", oldname);
	dlist_atom(kl, "NEWMBOXNAME", newname);
	dlist_atom(kl, "PARTITION", partition);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int folder_delete(char *mboxname)
	{
	const char *cmd = "UNMAILBOX";
	struct dlist *kl;

	kl = dlist_atom(NULL, cmd, mboxname);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int set_sub(const char userid, const char mboxname, int add)
	{
	const char *cmd = add ? "SUB" : "UNSUB";
	struct dlist *kl;

	if (verbose)
	printf("%s %s %s\n", cmd, userid, mboxname);

	if (verbose_logging)
	syslog(LOG_INFO, "%s %s %s", cmd, userid, mboxname);

	kl = dlist_new(cmd);
	dlist_atom(kl, "USERID", userid);
	dlist_atom(kl, "MBOXNAME", mboxname);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int folder_setannotation(const char mboxname, const char entry,
	const char userid, const char value)
	{
	const char *cmd = "ANNOTATION";
	struct dlist *kl;

	kl = dlist_new(cmd);
	dlist_atom(kl, "MBOXNAME", mboxname);
	dlist_atom(kl, "ENTRY", entry);
	dlist_atom(kl, "USERID", userid);
	dlist_atom(kl, "VALUE", value);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int folder_unannotation(const char mboxname, const char entry,
	const char *userid)
	{
	const char *cmd = "UNANNOTATION";
	struct dlist *kl;

	kl = dlist_new(cmd);
	dlist_atom(kl, "MBOXNAME", mboxname);
	dlist_atom(kl, "ENTRY", entry);
	dlist_atom(kl, "USERID", userid);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	/* ====================================================================== */

	static int sieve_upload(const char userid, const char filename,
	unsigned long last_update)
	{
	const char *cmd = "SIEVE";
	struct dlist *kl;
	char *sieve;
	uint32_t size;

	sieve = sync_sieve_read(userid, filename, &size);
	if (!sieve) return IMAP_IOERROR;

	kl = dlist_new(cmd);
	dlist_atom(kl, "USERID", userid);
	dlist_atom(kl, "FILENAME", filename);
	dlist_date(kl, "LAST_UPDATE", last_update);
	dlist_buf(kl, "CONTENT", sieve, size);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);
	free(sieve);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int sieve_delete(const char userid, const char filename)
	{
	const char *cmd = "UNSIEVE";
	struct dlist *kl;

	kl = dlist_new(cmd);
	dlist_atom(kl, "USERID", userid);
	dlist_atom(kl, "FILENAME", filename);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int sieve_activate(const char userid, const char filename)
	{
	const char *cmd = "ACTIVATE_SIEVE";
	struct dlist *kl;

	kl = dlist_new(cmd);
	dlist_atom(kl, "USERID", userid);
	dlist_atom(kl, "FILENAME", filename);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int sieve_deactivate(const char *userid)
	{
	const char *cmd = "UNACTIVATE_SIEVE";
	struct dlist *kl;

	kl = dlist_new(cmd);
	dlist_atom(kl, "USERID", userid);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	/* ====================================================================== */

	static int delete_quota(const char *root)
	{
	const char *cmd = "UNQUOTA";
	struct dlist *kl;

	kl = dlist_atom(NULL, cmd, root);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int update_quota_work(struct quota *client,
	struct sync_quota *server)
	{
	const char *cmd = "QUOTA";
	struct dlist *kl;
	int r;

	r = quota_read(client, NULL, 0);

	/* disappeared? Delete it*/
	if (r == IMAP_QUOTAROOT_NONEXISTENT)
	return delete_quota(client->root);

	if (r) {
	syslog(LOG_INFO, "Warning: failed to read quotaroot %s: %s",
	client->root, error_message(r));
	return r;
	}

	if (server && (client->limit == server->limit))
	return(0);

	kl = dlist_new(cmd);
	dlist_atom(kl, "ROOT", client->root);
	dlist_num(kl, "LIMIT", client->limit);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int user_sub(const char userid, const char mboxname)
	{
	int r;

	r = mboxlist_checksub(mboxname, userid);

	switch (r) {
	case CYRUSDB_OK:
	return set_sub(userid, mboxname, 1);
	case CYRUSDB_NOTFOUND:
	return set_sub(userid, mboxname, 0);
	default:
	return r;
	}
	}

	static int copy_local(struct mailbox *mailbox, unsigned long uid)
	{
	uint32_t recno;
	struct index_record record;
	char oldfname, newfname;
	int r;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	r = mailbox_read_index_record(mailbox, recno, &record);
	if (r) return r;
	if (record.uid == uid) {
	/* store the old record, expunged */
	record.system_flags \|= FLAG_EXPUNGED;
	r = mailbox_rewrite_index_record(mailbox, &record);
	if (r) return r;

	/* create the new record */
	record.system_flags &= ~FLAG_EXPUNGED;
	record.uid = mailbox->i.last_uid + 1;

	/* copy the file in to place */
	oldfname = xstrdup(mailbox_message_fname(mailbox, uid));
	newfname = xstrdup(mailbox_message_fname(mailbox, record.uid));
	r = mailbox_copyfile(oldfname, newfname, 0);
	free(oldfname);
	free(newfname);
	if (r) return r;

	/* and append the new record (a clone apart from the EXPUNGED flag) */
	r = mailbox_append_index_record(mailbox, &record);

	/* done - return */
	return r;
	}
	}

	/* not finding the record is an error! (should never happen) */
	return IMAP_MAILBOX_NONEXISTENT;
	}

	static int fetch_file(struct mailbox *mailbox, unsigned long uid,
	struct index_record *rp)
	{
	const char *cmd = "FETCH";
	struct dlist *kin = NULL;
	struct dlist *kl;
	int r;

	kl = dlist_new(cmd);
	dlist_atom(kl, "MBOXNAME", mailbox->name);
	dlist_atom(kl, "PARTITION", mailbox->part);
	dlist_atom(kl, "GUID", message_guid_encode(&rp->guid));
	dlist_num(kl, "UID", uid);
	sync_send_lookup(kl, sync_out);
	dlist_free(&kl);

	r = sync_parse_response(cmd, sync_in, &kin);
	if (r) return r;

	kl = kin->head;
	if (!kl) {
	r = IMAP_MAILBOX_NONEXISTENT;
	goto done;
	}

	if (!message_guid_compare(&kl->gval, &rp->guid))
	r = IMAP_MAILBOX_CRC;

	done:
	dlist_free(&kin);
	return r;
	}

	static int copy_remote(struct mailbox *mailbox, unsigned long uid,
	struct dlist *kr)
	{
	struct index_record record;
	struct dlist *ki;
	int r;

	for (ki = kr->head; ki; ki = ki->next) {
	r = parse_upload(ki, mailbox, &record);
	if (r) return r;
	if (record.uid == uid) {
	/* find the destination UID */
	record.uid = mailbox->i.last_uid + 1;

	/* upload the file */
	r = fetch_file(mailbox, uid, &record);
	if (r) return r;

	/* append the file */
	r = sync_append_copyfile(mailbox, &record);

	return r;
	}
	}
	/* not finding the record is an error! (should never happen) */
	return IMAP_MAILBOX_NONEXISTENT;
	}

	static int copyback_one_record(struct mailbox *mailbox,
	struct index_record *rp,
	struct dlist *kaction)
	{
	int r;

	/* don't want to copy back expunged records! */
	if (rp->system_flags & FLAG_EXPUNGED)
	return 0;

	/* if the UID is lower than master's last_uid,
	* we'll need to renumber */
	if (rp->uid <= mailbox->i.last_uid) {
	/* Ok, now we need to check if it's just really stale
	* (has been cleaned out locally) or an error.
	* In the error case we copy back, stale
	* we remove from the replica */
	if (rp->modseq < mailbox->i.deletedmodseq)
	dlist_num(kaction, "EXPUNGE", rp->uid);
	else
	dlist_num(kaction, "COPYBACK", rp->uid);
	}

	/* otherwise we can pull it in with the same UID,
	* which saves causing renumbering on the replica
	* end, so is preferable */
	else {
	/* grab the file */
	r = fetch_file(mailbox, rp->uid, rp);
	if (r) return r;
	/* append the file */
	r = sync_append_copyfile(mailbox, rp);
	if (r) return r;
	}

	return 0;
	}

	static int renumber_one_record(struct index_record *mp,
	struct dlist *kaction)
	{
	/* don't want to renumber expunged records */
	if (mp->system_flags & FLAG_EXPUNGED)
	return 0;

	dlist_num(kaction, "RENUMBER", mp->uid);

	return 0;
	}

	static const char make_flags(struct mailbox mailbox, struct index_record *record)
	{
	static char buf[4096];
	const char *sep = "";
	int flag;

	if (record->system_flags & FLAG_DELETED) {
	snprintf(buf, 4096, "%s\\Deleted", sep);
	sep = " ";
	}
	if (record->system_flags & FLAG_ANSWERED) {
	snprintf(buf, 4096, "%s\\Answered", sep);
	sep = " ";
	}
	if (record->system_flags & FLAG_FLAGGED) {
	snprintf(buf, 4096, "%s\\Flagged", sep);
	sep = " ";
	}
	if (record->system_flags & FLAG_DRAFT) {
	snprintf(buf, 4096, "%s\\Draft", sep);
	sep = " ";
	}
	if (record->system_flags & FLAG_EXPUNGED) {
	snprintf(buf, 4096, "%s\\Expunged", sep);
	sep = " ";
	}
	if (record->system_flags & FLAG_SEEN) {
	snprintf(buf, 4096, "%s\\Seen", sep);
	sep = " ";
	}

	/* print user flags in mailbox order */
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (!mailbox->flagname[flag])
	continue;
	if (!(record->user_flags[flag/32] & (1<<(flag&31))))
	continue;
	snprintf(buf, 4096, "%s%s", sep, mailbox->flagname[flag]);
	sep = " ";
	}

	return buf;
	}

	static void log_record(const char name, struct mailbox mailbox,
	struct index_record *record)
	{
	syslog(LOG_ERR, "%s uid:%u modseq:" MODSEQ_FMT " last_updated:%lu internaldate:%lu flags:(%s)", name,
	record->uid, record->modseq, record->last_updated, record->internaldate, make_flags(mailbox, record));
	}

	static void log_mismatch(const char reason, struct mailbox mailbox,
	struct index_record *mp,
	struct index_record *rp)
	{
	syslog(LOG_ERR, "RECORD MISMATCH WITH REPLICA: %s", reason);
	log_record("master", mailbox, mp);
	log_record("replica", mailbox, rp);
	}

	static int compare_one_record(struct mailbox *mailbox,
	struct index_record *mp,
	struct index_record *rp,
	struct dlist *kaction)
	{
	int diff = 0;
	int i;
	int r;

	/* if the GUIDs don't match, then treat as two
	* un-matched records :) */
	if (!message_guid_compare(&mp->guid, &rp->guid)) {
	if (!(rp->system_flags & FLAG_EXPUNGED))
	dlist_num(kaction, "COPYBACK", rp->uid);
	if (!(mp->system_flags & FLAG_EXPUNGED))
	dlist_num(kaction, "RENUMBER", mp->uid);
	return 0;
	}

	/* UIDs match, GUIDs match: look for differences in
	* everything else that's part of sync_crc! */

	if (mp->modseq != rp->modseq)
	diff = 1;
	if (mp->last_updated != rp->last_updated)
	diff = 1;
	if (mp->internaldate != rp->internaldate)
	diff = 1;
	if (mp->system_flags != rp->system_flags)
	diff = 1;
	for (i = 0; i < MAX_USER_FLAGS/32; i++) {
	if (mp->user_flags[i] != rp->user_flags[i])
	diff = 1;
	}

	/* if differences we'll have to rewrite to bump the modseq
	* so that regular replication will cause an update */
	if (diff) {
	/* interesting case - expunged locally */
	if (mp->system_flags & FLAG_EXPUNGED) {
	/* if the remote record is MORE recent, we
	* probably want to keep it */
	if (rp->modseq > mp->modseq \|\|
	rp->last_updated > mp->last_updated)
	return copyback_one_record(mailbox, rp, kaction);
	/* otherwise fall through - the modseq update
	* will cause it to expunge */
	}
	/* evil - expunged remotely, NOT locally */
	else if (rp->system_flags & FLAG_EXPUNGED) {
	/* is the replica "newer"? */
	if (rp->modseq > mp->modseq \|\|
	rp->last_updated > mp->last_updated) {
	syslog(LOG_ERR, "recent expunged on replica %s:%u, expunging locally",
	mailbox->name, mp->uid);
	mp->system_flags \|= FLAG_EXPUNGED;
	}
	else {
	/* will have to move the local record */
	return renumber_one_record(mp, kaction);
	}
	}

	/* general case */
	else {
	/* is the replica "newer"? */
	if (rp->modseq > mp->modseq \|\|
	rp->last_updated > mp->last_updated) {
	log_mismatch("more recent on replica", mailbox, mp, rp);
	mp->system_flags = rp->system_flags;
	for (i = 0; i < MAX_USER_FLAGS/32; i++)
	mp->user_flags[i] = rp->user_flags[i];
	mp->internaldate = rp->internaldate;
	/* no point copying modseq, it will be updated regardless */
	}
	else {
	log_mismatch("more recent on master", mailbox, mp, rp);
	}
	}

	/* this will bump the modseq and force a resync either way :) */
	r = mailbox_rewrite_index_record(mailbox, mp);
	if (r) return r;
	}

	return 0;
	}

	static int mailbox_full_update(const char *mboxname)
	{
	const char *cmd = "FULLMAILBOX";
	uint32_t recno;
	unsigned old_num_records;
	struct index_record mrecord, rrecord;
	struct mailbox *mailbox = NULL;
	int r;
	struct dlist *kin = NULL;
	struct dlist *ki = NULL;
	struct dlist *kr = NULL;
	struct dlist *ka = NULL;
	struct dlist *kuids = NULL;
	struct dlist *kl = NULL;
	struct dlist *kaction = NULL;
	struct dlist *kexpunge = NULL;
	modseq_t highestmodseq;
	uint32_t last_uid;

	kl = dlist_atom(NULL, cmd, mboxname);
	sync_send_lookup(kl, sync_out);
	dlist_free(&kl);

	r = sync_parse_response(cmd, sync_in, &kin);
	if (r) return r;

	kl = kin->head;

	if (!kl) {
	r = IMAP_MAILBOX_NONEXISTENT;
	goto done;
	}

	/* XXX - handle the header. I want to do some ordering on timestamps
	* in particular here - if there's more recent data on the replica then
	* it should be copied back. This depends on having a nice way to
	* parse the mailbox structure back in to a struct index_header rather
	* than the by hand stuff though, because that sucks. NOTE - this
	* doesn't really matter too much, because we'll blat the replica's
	* values anyway! */

	if (!dlist_getmodseq(kl, "HIGHESTMODSEQ", &highestmodseq)) {
	r = IMAP_PROTOCOL_BAD_PARAMETERS;
	goto done;
	}

	if (!dlist_getnum(kl, "LAST_UID", &last_uid)) {
	r = IMAP_PROTOCOL_BAD_PARAMETERS;
	goto done;
	}

	if (!dlist_getlist(kl, "RECORD", &kr)) {
	r = IMAP_PROTOCOL_BAD_PARAMETERS;
	goto done;
	}

	/* we'll be updating it! */
	r = mailbox_open_iwl(mboxname, &mailbox);
	if (r) goto done;

	/* re-calculate our local CRC just in case it's out of sync */
	r = mailbox_index_recalc(mailbox);
	if (r) goto done;

	old_num_records = mailbox->i.num_records;

	if (mailbox->i.highestmodseq < highestmodseq) {
	/* highestmodseq on replica is dirty - we must go to at least one higher! */
	mailbox->i.highestmodseq = highestmodseq;
	mailbox_modseq_dirty(mailbox);
	}

	/* initialise the two loops */
	kaction = dlist_list(NULL, "ACTION");
	recno = 1;
	ki = kr->head;

	/* while there are more records on either master OR replica,
	* work out what to do with them */
	while (ki \|\| recno <= old_num_records) {
	/* most common case - both a master AND a replica record exist */
	if (ki && recno <= old_num_records) {
	r = mailbox_read_index_record(mailbox, recno, &mrecord);
	if (r) goto done;
	r = parse_upload(ki, mailbox, &rrecord);
	if (r) goto done;

	/* same UID - compare the records */
	if (rrecord.uid == mrecord.uid) {
	/* hasn't been changed already, check it */
	if (mrecord.modseq <= highestmodseq) {
	r = compare_one_record(mailbox,
	&mrecord, &rrecord,
	kaction);
	if (r) goto done;
	}
	/* increment both */
	recno++;
	ki = ki->next;
	}
	else if (rrecord.uid > mrecord.uid) {
	/* record only exists on the master */
	r = renumber_one_record(&mrecord, kaction);
	if (r) goto done;
	/* only increment master */
	recno++;
	}
	else {
	/* record only exists on the replica */
	r = copyback_one_record(mailbox, &rrecord, kaction);
	if (r) goto done;
	/* only increment replica */
	ki = ki->next;
	}
	}

	/* no more replica records, but still master records */
	else if (recno <= old_num_records) {
	r = mailbox_read_index_record(mailbox, recno, &mrecord);
	if (r) goto done;
	/* if the replica has seen this UID, we need to renumber.
	* Otherwise it will replicate fine as-is */
	if (mrecord.uid <= last_uid) {
	r = renumber_one_record(&mrecord, kaction);
	if (r) goto done;
	}
	recno++;
	}

	/* record only exists on the replica */
	else {
	r = parse_upload(ki, mailbox, &rrecord);
	if (r) goto done;

	/* going to need this one */
	r = copyback_one_record(mailbox, &rrecord, kaction);
	if (r) goto done;

	ki = ki->next;
	}
	}

	/* if replica still has a higher last_uid, bump our local
	* number to match so future records don't clash */
	if (mailbox->i.last_uid < last_uid)
	mailbox->i.last_uid = last_uid;

	/* blatant reuse 'r' us */
	kexpunge = dlist_new("EXPUNGE");
	dlist_atom(kexpunge, "MBOXNAME", mailbox->name);
	dlist_atom(kexpunge, "UNIQUEID", mailbox->uniqueid); /* just for safety */
	kuids = dlist_list(kexpunge, "UID");
	for (ka = kaction->head; ka; ka = ka->next) {
	if (!strcmp(ka->name, "EXPUNGE")) {
	dlist_num(kuids, "UID", ka->nval);
	}
	else if (!strcmp(ka->name, "COPYBACK")) {
	r = copy_remote(mailbox, ka->nval, kr);
	if (r) goto done;
	dlist_num(kuids, "UID", ka->nval);
	}
	else if (!strcmp(ka->name, "RENUMBER")) {
	r = copy_local(mailbox, ka->nval);
	if (r) goto done;
	}
	}

	/* commit and close the mailbox before sending any expunges
	* to avoid deadlocks */
	r = mailbox_commit(mailbox);
	if (r) goto done;
	mailbox_close(&mailbox);

	/* only send expunge if we have some UIDs to expunge */
	if (kuids->head) {
	sync_send_apply(kexpunge, sync_out);
	r = sync_parse_response("EXPUNGE", sync_in, NULL);
	}

	done:
	if (mailbox) mailbox_close(&mailbox);
	dlist_free(&kin);
	dlist_free(&kaction);
	dlist_free(&kexpunge);
	return r;
	}

	static int is_unchanged(struct mailbox mailbox, struct sync_folder remote)
	{
	/* look for any mismatches */
	if (!remote) return 0;
	if (remote->last_uid != mailbox->i.last_uid) return 0;
	if (remote->highestmodseq != mailbox->i.highestmodseq) return 0;
	if (remote->sync_crc != mailbox->i.sync_crc) return 0;
	if (remote->recentuid != mailbox->i.recentuid) return 0;
	if (remote->recenttime != mailbox->i.recenttime) return 0;
	if (remote->pop3_last_login != mailbox->i.pop3_last_login) return 0;
	if (remote->options != mailbox->i.options) return 0;
	if (strcmp(remote->acl, mailbox->acl)) return 0;

	/* otherwise it's unchanged! */
	return 1;
	}

	static int update_mailbox_once(struct sync_folder *local,
	struct sync_folder *remote,
	struct sync_reserve_list *reserve_guids)
	{
	struct sync_msgid_list *part_list;
	struct mailbox *mailbox = NULL;
	int r = 0;
	struct dlist *kl = dlist_new("MAILBOX");
	struct dlist *kupload = dlist_list(NULL, "MESSAGE");

	r = mailbox_open_irl(local->name, &mailbox);
	if (r == IMAP_MAILBOX_NONEXISTENT) {
	/* been deleted in the meanwhile... */
	r = folder_delete(remote->name);
	goto done;
	}
	else if (r)
	goto done;

	/* definitely bad if these don't match! */
	if (strcmp(mailbox->uniqueid, local->uniqueid) \|\|
	strcmp(mailbox->part, local->part)) {
	r = IMAP_MAILBOX_MOVED;
	goto done;
	}

	/* nothing changed - nothing to send */
	if (is_unchanged(mailbox, remote))
	goto done;

	/* check that replication stands a chance of succeeding */
	if (remote && (mailbox->i.deletedmodseq > remote->highestmodseq)) {
	syslog(LOG_NOTICE, "inefficient replication ("
	MODSEQ_FMT " > " MODSEQ_FMT ") %s",
	mailbox->i.deletedmodseq, remote->highestmodseq,
	local->name);
	/* close the mailbox here before sending */
	mailbox_close(&mailbox);
	r = mailbox_full_update(local->name);
	if (r) goto done;
	/* and open it again afterwards */
	r = mailbox_open_irl(local->name, &mailbox);
	if (r) goto done;
	}

	part_list = sync_reserve_partlist(reserve_guids, mailbox->part);
	r = sync_mailbox(mailbox, remote, part_list, kl, kupload, 1);
	if (r) goto done;

	/* upload any messages required */
	if (kupload->head) {
	/* keep the mailbox locked for shorter time! Unlock the index now
	* but don't close it, because we need to guarantee that message
	* files don't get deleted until we're finished with them... */
	mailbox_unlock_index(mailbox, NULL);
	sync_send_apply(kupload, sync_out);
	r = sync_parse_response("MESSAGE", sync_in, NULL);
	if (!r) {
	/* update our list of reserved messages on the replica */
	struct dlist *ki;
	struct sync_msgid *msgid;
	for (ki = kupload->head; ki; ki = ki->next) {
	msgid = sync_msgid_lookup(part_list, &ki->gval);
	if (!msgid)
	msgid = sync_msgid_add(part_list, &ki->gval);
	msgid->mark = 1;
	part_list->marked++;
	}
	}
	}

	/* close before sending the apply - all data is already read */
	mailbox_close(&mailbox);

	/* update the mailbox */
	sync_send_apply(kl, sync_out);
	r = sync_parse_response("MAILBOX", sync_in, NULL);

	done:
	if (mailbox) mailbox_close(&mailbox);
	dlist_free(&kupload);
	dlist_free(&kl);
	return r;
	}

	static int update_mailbox(struct sync_folder *local,
	struct sync_folder *remote,
	struct sync_reserve_list *reserve_guids)
	{
	int r = update_mailbox_once(local, remote, reserve_guids);

	if (r == IMAP_MAILBOX_CRC) {
	syslog(LOG_ERR, "CRC failure on sync for %s, trying full update",
	local->name);
	r = mailbox_full_update(local->name);
	if (!r)
	r = update_mailbox_once(local, remote, reserve_guids);
	}

	return r;
	}

	/* ====================================================================== */


	static int update_seen_work(const char user, const char uniqueid,
	struct seendata *sd)
	{
	const char *cmd = "SEEN";
	struct dlist *kl;

	/* Update seen list */
	kl = dlist_new(cmd);
	dlist_atom(kl, "USERID", user);
	dlist_atom(kl, "UNIQUEID", uniqueid);
	dlist_date(kl, "LASTREAD", sd->lastread);
	dlist_num(kl, "LASTUID", sd->lastuid);
	dlist_date(kl, "LASTCHANGE", sd->lastchange);
	dlist_atom(kl, "SEENUIDS", sd->seenuids);
	sync_send_apply(kl, sync_out);
	dlist_free(&kl);

	return sync_parse_response(cmd, sync_in, NULL);
	}

	static int do_seen(char user, char uniqueid)
	{
	int r = 0;
	struct seen *seendb;
	struct seendata sd;

	if (verbose)
	printf("SEEN %s %s\n", user, uniqueid);

	if (verbose_logging)
	syslog(LOG_INFO, "SEEN %s %s", user, uniqueid);

	/* ignore read failures */
	r = seen_open(user, SEEN_SILENT, &seendb);
	if (r) return 0;

	r = seen_read(seendb, uniqueid, &sd);
	if (r) {
	seen_close(seendb);
	return 0;
	}

	r = update_seen_work(user, uniqueid, &sd);

	seen_close(seendb);
	seen_freedata(&sd);

	return r;
	}

	/* ====================================================================== */

	static int do_quota(const char *root)
	{
	int r = 0;
	struct quota q;

	if (verbose)
	printf("SETQUOTA %s\n", root);

	if (verbose_logging)
	syslog(LOG_INFO, "SETQUOTA: %s", root);

	q.root = root;
	r = update_quota_work(&q, NULL);

	return r;
	}

	static int getannotation_cb(const char *mailbox __attribute__((unused)),
	const char entry, const char userid,
	struct annotation_data attrib, void rock)
	{
	struct sync_annot_list l = (struct sync_annot_list ) rock;

	sync_annot_list_add(l, entry, userid, attrib->value);

	return 0;
	}

	static int parse_annotation(struct dlist *kin,
	struct sync_annot_list *replica_annot)
	{
	struct dlist *kl;
	const char *entry;
	const char *userid;
	const char *value;

	for (kl = kin->head; kl; kl = kl->next) {
	if (!dlist_getatom(kl, "ENTRY", &entry))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	if (!dlist_getatom(kl, "USERID", &userid))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	if (!dlist_getatom(kl, "VALUE", &value))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	sync_annot_list_add(replica_annot, entry, userid, value);
	}

	return 0;
	}

	static int do_getannotation(char *mboxname,
	struct sync_annot_list *replica_annot)
	{
	const char *cmd = "ANNOTATION";
	struct dlist *kl;
	struct dlist *kin = NULL;
	int r;

	/* Update seen list */
	kl = dlist_atom(NULL, cmd, mboxname);
	sync_send_lookup(kl, sync_out);
	dlist_free(&kl);

	r = sync_parse_response(cmd, sync_in, &kin);
	if (r) return r;

	r = parse_annotation(kin, replica_annot);
	dlist_free(&kin);

	return r;
	}

	static int do_annotation(char *mboxname)
	{
	int r;
	struct sync_annot_list *replica_annot = sync_annot_list_create();
	struct sync_annot_list *master_annot = sync_annot_list_create();
	struct sync_annot ma, ra;
	int n;

	r = do_getannotation(mboxname, replica_annot);
	if (r) goto bail;

	r = annotatemore_findall(mboxname, "*", &getannotation_cb, master_annot, NULL);
	if (r) goto bail;

	/* both lists are sorted, so we work our way through the lists
	top-to-bottom and determine what we need to do based on order */
	ma = master_annot->head;
	ra = replica_annot->head;
	while (ma \|\| ra) {
	if (!ra) n = -1; /* add all master annotations */
	else if (!ma) n = 1; /* remove all replica annotations */
	else if ((n = strcmp(ma->entry, ra->entry)) == 0)
	n = strcmp(ma->userid, ra->userid);

	if (n > 0) {
	/* remove replica annotation */
	r = folder_unannotation(mboxname, ra->entry, ra->userid);
	if (r) goto bail;
	ra = ra->next;
	continue;
	}

	if (n == 0) {
	/* already have the annotation, but is the value different? */
	if (!strcmp(ra->value, ma->value)) {
	ra = ra->next;
	ma = ma->next;
	continue;
	}
	ra = ra->next;
	}

	/* add the current client annotation */
	r = folder_setannotation(mboxname, ma->entry, ma->userid, ma->value);
	if (r) goto bail;

	ma = ma->next;
	}

	bail:
	sync_annot_list_free(&master_annot);
	sync_annot_list_free(&replica_annot);
	return r;
	}

	/* ====================================================================== */

	int do_folders(struct sync_name_list *mboxname_list,
	struct sync_folder_list *replica_folders)
	{
	int r;
	struct sync_folder_list *master_folders;
	struct sync_rename_list *rename_folders;
	struct sync_reserve_list *reserve_guids;
	struct sync_folder mfolder, rfolder;

	master_folders = sync_folder_list_create();
	rename_folders = sync_rename_list_create();
	reserve_guids = sync_reserve_list_create(SYNC_MSGID_LIST_HASH_SIZE);

	r = reserve_messages(mboxname_list, master_folders,
	replica_folders, reserve_guids);
	if (r) goto bail;

	/* Tag folders on server which still exist on the client. Anything
	* on the server which remains untagged can be deleted immediately */
	for (mfolder = master_folders->head; mfolder; mfolder = mfolder->next) {
	rfolder = sync_folder_lookup(replica_folders, mfolder->uniqueid);
	if (!rfolder) continue;
	rfolder->mark = 1;

	/* does it need a rename? */
	if (strcmp(mfolder->name, rfolder->name) \|\| strcmp(mfolder->part, rfolder->part))
	sync_rename_list_add(rename_folders, mfolder->uniqueid, rfolder->name,
	mfolder->name, mfolder->part);
	}

	/* Delete folders on server which no longer exist on client */
	for (rfolder = replica_folders->head; rfolder; rfolder = rfolder->next) {
	if (rfolder->mark) continue;
	r = folder_delete(rfolder->name);
	if (r) goto bail;
	}

	/* Need to rename folders in an order which avoids dependancy conflicts
	* following isn't wildly efficient, but rename_folders will typically be
	* short and contain few dependancies. Algorithm is to simply pick a
	* rename operation which has no dependancy and repeat until done */

	while (rename_folders->done < rename_folders->count) {
	int rename_success = 0;
	struct sync_rename item, item2;

	for (item = rename_folders->head; item; item = item->next) {
	if (item->done) continue;

	item2 = sync_rename_lookup(rename_folders, item->newname);
	if (item2 && !item2->done) continue;

	/* Found unprocessed item which should rename cleanly */
	r = folder_rename(item->oldname, item->newname, item->part);
	if (r) {
	syslog(LOG_ERR, "do_folders(): failed to rename: %s -> %s ",
	item->oldname, item->newname);
	goto bail;
	}

	rename_folders->done++;
	item->done = 1;
	rename_success = 1;
	}

	if (!rename_success) {
	/* Scanned entire list without a match */
	syslog(LOG_ERR,
	"do_folders(): failed to order folders correctly");
	r = IMAP_AGAIN;
	goto bail;
	}
	}

	for (mfolder = master_folders->head; mfolder; mfolder = mfolder->next) {
	/* NOTE: rfolder->name may now be wrong, but we're guaranteed that
	* it was successfully renamed above, so just use mfolder->name for
	* all commands */
	rfolder = sync_folder_lookup(replica_folders, mfolder->uniqueid);
	r = update_mailbox(mfolder, rfolder, reserve_guids);
	if (r) {
	syslog(LOG_ERR, "do_folders(): update failed: %s '%s'",
	mfolder->name, error_message(r));
	goto bail;
	}
	}

	bail:
	sync_folder_list_free(&master_folders);
	sync_rename_list_free(&rename_folders);
	sync_reserve_list_free(&reserve_guids);
	return r;
	}

	static int do_mailboxes(struct sync_name_list *mboxname_list)
	{
	struct sync_name *mbox;
	struct sync_folder_list *replica_folders = sync_folder_list_create();
	struct dlist *kl = NULL;
	int r;

	if (verbose) {
	printf("MAILBOXES");
	for (mbox = mboxname_list->head; mbox; mbox = mbox->next) {
	printf(" %s", mbox->name);
	}
	printf("\n");
	}

	if (verbose_logging) {
	for (mbox = mboxname_list->head; mbox; mbox = mbox->next)
	syslog(LOG_INFO, "MAILBOX %s", mbox->name);
	}

	kl = dlist_list(NULL, "MAILBOXES");
	for (mbox = mboxname_list->head; mbox; mbox = mbox->next)
	dlist_atom(kl, "MBOXNAME", mbox->name);
	sync_send_lookup(kl, sync_out);
	dlist_free(&kl);

	r = response_parse("MAILBOXES", replica_folders, NULL, NULL, NULL, NULL);

	if (!r)
	r = do_folders(mboxname_list, replica_folders);

	sync_folder_list_free(&replica_folders);

	return r;
	}

	/* ====================================================================== */

	struct mboxinfo {
	struct sync_name_list *mboxlist;
	struct sync_name_list *quotalist;
	};

	static int do_mailbox_info(char *name,
	int matchlen __attribute__((unused)),
	int maycreate __attribute__((unused)),
	void *rock)
	{
	int r;
	struct mailbox *mailbox = NULL;
	struct mboxinfo info = (struct mboxinfo )rock;

	r = mailbox_open_irl(name, &mailbox);
	/* doesn't exist? Probably not finished creating or removing yet */
	if (r == IMAP_MAILBOX_NONEXISTENT) return 0;
	if (r == IMAP_MAILBOX_RESERVED) return 0;
	if (r) return r;

	if (info->quotalist && mailbox->quotaroot) {
	if (!sync_name_lookup(info->quotalist, mailbox->quotaroot))
	sync_name_list_add(info->quotalist, mailbox->quotaroot);
	}

	mailbox_close(&mailbox);

	addmbox(name, 0, 0, info->mboxlist);

	return 0;
	}

	static int do_user_quota(struct sync_name_list *master_quotaroots,
	struct sync_quota_list *replica_quota)
	{
	int r;
	struct sync_name *mitem;
	struct sync_quota *rquota;
	struct quota q;

	/* set any new or changed quotas */
	for (mitem = master_quotaroots->head; mitem; mitem = mitem->next) {
	rquota = sync_quota_lookup(replica_quota, mitem->name);
	q.root = mitem->name;
	if (rquota)
	rquota->done = 1;
	r = update_quota_work(&q, rquota);
	if (r) return r;
	}

	/* delete any quotas no longer on the master */
	for (rquota = replica_quota->head; rquota; rquota = rquota->next) {
	if (rquota->done) continue;
	r = delete_quota(rquota->root);
	if (r) return r;
	}

	return 0;
	}

	int do_user_main(char user, struct sync_folder_list replica_folders,
	struct sync_quota_list *replica_quota)
	{
	char buf[MAX_MAILBOX_BUFFER];
	int r = 0;
	struct sync_name_list *mboxname_list = sync_name_list_create();
	struct sync_name_list *master_quotaroots = sync_name_list_create();
	struct mboxinfo info;

	info.mboxlist = mboxname_list;
	info.quotalist = master_quotaroots;

	/* Generate full list of folders on client side */
	(sync_namespace.mboxname_tointernal)(&sync_namespace, "INBOX",
	user, buf);
	do_mailbox_info(buf, 0, 0, &info);

	/* deleted namespace items if enabled */
	if (mboxlist_delayed_delete_isenabled()) {
	char deletedname[MAX_MAILBOX_BUFFER];
	mboxname_todeleted(buf, deletedname, 0);
	strlcat(deletedname, ".*", sizeof(deletedname));
	r = (sync_namespace.mboxlist_findall)(&sync_namespace, deletedname, 1,
	user, NULL, do_mailbox_info,
	&info);
	}

	/* subfolders */
	if (!r) {
	strlcat(buf, ".*", sizeof(buf));
	r = (sync_namespace.mboxlist_findall)(&sync_namespace, buf, 1,
	user, NULL, do_mailbox_info,
	&info);
	}

	if (!r) r = do_folders(mboxname_list, replica_folders);
	if (!r) r = do_user_quota(master_quotaroots, replica_quota);

	sync_name_list_free(&mboxname_list);
	sync_name_list_free(&master_quotaroots);

	if (r) syslog(LOG_ERR, "IOERROR: %s", error_message(r));

	return r;
	}

	int do_user_sub(const char userid, struct sync_name_list replica_subs)
	{
	char buf[MAX_MAILBOX_BUFFER];
	struct sync_name_list *master_subs = sync_name_list_create();
	struct sync_name msubs, rsubs;
	int r;

	/* Includes subsiduary nodes automatically */
	r = (sync_namespace.mboxlist_findsub)(&sync_namespace, "*", 1,
	userid, NULL, addmbox_sub,
	master_subs, 1);
	if (r) {
	syslog(LOG_ERR, "IOERROR: %s", error_message(r));
	goto bail;
	}

	/* add any folders that need adding, and mark any which
	* still exist */
	for (msubs = master_subs->head; msubs; msubs = msubs->next) {
	r = (sync_namespace.mboxname_tointernal)(&sync_namespace, msubs->name,
	userid, buf);
	if (r) continue;
	rsubs = sync_name_lookup(replica_subs, buf);
	if (rsubs) {
	rsubs->mark = 1;
	continue;
	}
	r = set_sub(userid, buf, 1);
	if (r) goto bail;
	}

	/* remove any no-longer-subscribed folders */
	for (rsubs = replica_subs->head; rsubs; rsubs = rsubs->next) {
	if (rsubs->mark)
	continue;
	r = set_sub(userid, rsubs->name, 0);
	if (r) goto bail;
	}

	bail:
	sync_name_list_free(&master_subs);
	return(r);
	}

	static int get_seen(const char uniqueid, struct seendata sd, void *rock)
	{
	struct sync_seen_list list = (struct sync_seen_list )rock;

	sync_seen_list_add(list, uniqueid, sd->lastread, sd->lastuid,
	sd->lastchange, sd->seenuids);

	return 0;
	}

	static int do_user_seen(char user, struct sync_seen_list replica_seen)
	{
	int r;
	struct sync_seen mseen, rseen;
	struct seen *seendb;
	struct sync_seen_list *list;

	/* silently ignore errors */
	r = seen_open(user, SEEN_SILENT, &seendb);
	if (r) return 0;

	list = sync_seen_list_create();

	seen_foreach(seendb, get_seen, list);
	seen_close(seendb);

	for (mseen = list->head; mseen; mseen = mseen->next) {
	rseen = sync_seen_list_lookup(replica_seen, mseen->uniqueid);
	if (rseen) {
	rseen->mark = 1;
	if (seen_compare(&rseen->sd, &mseen->sd))
	continue; /* nothing changed */
	}
	r = update_seen_work(user, mseen->uniqueid, &mseen->sd);
	}

	/* XXX - delete seen on the replica for records that don't exist? */

	sync_seen_list_free(&list);

	return 0;
	}

	int do_user_sieve(char userid, struct sync_sieve_list replica_sieve)
	{
	int r = 0;
	struct sync_sieve_list *master_sieve;
	struct sync_sieve mitem, ritem;
	int master_active = 0;
	int replica_active = 0;

	master_sieve = sync_sieve_list_generate(userid);
	if (!master_sieve) {
	syslog(LOG_ERR, "Unable to list sieve scripts for %s", userid);
	return IMAP_IOERROR;
	}

	/* Upload missing and out of date scripts */
	for (mitem = master_sieve->head; mitem; mitem = mitem->next) {
	ritem = sync_sieve_lookup(replica_sieve, mitem->name);
	if (ritem) {
	ritem->mark = 1;
	if (ritem->last_update >= mitem->last_update)
	continue; /* doesn't need updating */
	}
	r = sieve_upload(userid, mitem->name, mitem->last_update);
	if (r) goto bail;
	}

	/* Delete scripts which no longer exist on the master */
	replica_active = 0;
	for (ritem = replica_sieve->head; ritem; ritem = ritem->next) {
	if (ritem->mark) {
	if (ritem->active)
	replica_active = 1;
	} else {
	r = sieve_delete(userid, ritem->name);
	if (r) goto bail;
	}
	}

	/* Change active script if necessary */
	master_active = 0;
	for (mitem = master_sieve->head; mitem; mitem = mitem->next) {
	if (!mitem->active)
	continue;

	master_active = 1;
	ritem = sync_sieve_lookup(replica_sieve, mitem->name);
	if (ritem && ritem->active)
	break;

	r = sieve_activate(userid, mitem->name);
	if (r) goto bail;

	replica_active = 1;
	break;
	}

	if (!master_active && replica_active)
	r = sieve_deactivate(userid);

	bail:
	sync_sieve_list_free(&master_sieve);
	return(r);
	}

	int do_user(char *userid)
	{
	char buf[MAX_MAILBOX_BUFFER];
	int r = 0;
	struct sync_folder_list *replica_folders = sync_folder_list_create();
	struct sync_name_list *replica_subs = sync_name_list_create();
	struct sync_sieve_list *replica_sieve = sync_sieve_list_create();
	struct sync_seen_list *replica_seen = sync_seen_list_create();
	struct sync_quota_list *replica_quota = sync_quota_list_create();
	struct dlist *kl = NULL;
	struct mailbox *mailbox = NULL;

	if (verbose)
	printf("USER %s\n", userid);

	if (verbose_logging)
	syslog(LOG_INFO, "USER %s", userid);

	kl = dlist_atom(NULL, "USER", userid);
	sync_send_lookup(kl, sync_out);
	dlist_free(&kl);

	r = response_parse("USER",
	replica_folders, replica_subs,
	replica_sieve, replica_seen, replica_quota);
	/* can happen! */
	if (r == IMAP_MAILBOX_NONEXISTENT) r = 0;
	if (r) goto done;

	(sync_namespace.mboxname_tointernal)(&sync_namespace, "INBOX",
	userid, buf);
	r = mailbox_open_irl(buf, &mailbox);
	if (r == IMAP_MAILBOX_NONEXISTENT) {
	/* user has been removed, RESET server */
	syslog(LOG_ERR, "Inbox missing on master for %s", userid);
	r = user_reset(userid);
	goto done;
	}
	if (r) goto done;

	/* we don't hold locks while sending commands */
	mailbox_close(&mailbox);
	r = do_user_main(userid, replica_folders, replica_quota);
	if (r) goto done;
	r = do_user_sub(userid, replica_subs);
	if (r) goto done;
	r = do_user_sieve(userid, replica_sieve);
	if (r) goto done;
	r = do_user_seen(userid, replica_seen);

	done:
	sync_folder_list_free(&replica_folders);
	sync_name_list_free(&replica_subs);
	sync_sieve_list_free(&replica_sieve);
	sync_seen_list_free(&replica_seen);
	sync_quota_list_free(&replica_quota);

	return r;
	}

	/* ====================================================================== */

	static int do_meta(char *userid)
	{
	struct sync_name_list *replica_subs = sync_name_list_create();
	struct sync_sieve_list *replica_sieve = sync_sieve_list_create();
	struct sync_seen_list *replica_seen = sync_seen_list_create();
	struct dlist *kl = NULL;
	int r = 0;

	if (verbose)
	printf("META %s\n", userid);

	if (verbose_logging)
	syslog(LOG_INFO, "META %s", userid);

	kl = dlist_atom(NULL, "META", userid);
	sync_send_lookup(kl, sync_out);
	dlist_free(&kl);

	r = response_parse("META", NULL, replica_subs, replica_sieve, replica_seen, NULL);
	if (!r) r = do_user_seen(userid, replica_seen);
	if (!r) r = do_user_sub(userid, replica_subs);
	if (!r) r = do_user_sieve(userid, replica_sieve);
	sync_seen_list_free(&replica_seen);
	sync_name_list_free(&replica_subs);
	sync_sieve_list_free(&replica_sieve);

	return r;
	}

	/* ====================================================================== */

	static void remove_meta(char user, struct sync_action_list list)
	{
	struct sync_action *action;

	for (action = list->head ; action ; action = action->next) {
	if (!strcmp(user, action->user)) {
	action->active = 0;
	}
	}
	}

	static void remove_folder(char name, struct sync_action_list list,
	int chk_child)
	{
	struct sync_action *action;
	size_t len = strlen(name);

	for (action = list->head ; action ; action = action->next) {
	if (!strncmp(name, action->name, len) &&
	((action->name[len] == '\0') \|\|
	(chk_child && (action->name[len] == '.')))) {
	action->active = 0;
	}
	}
	}

	/* ====================================================================== */

	static int do_sync(const char *filename)
	{
	struct sync_action_list *user_list = sync_action_list_create();
	struct sync_action_list *meta_list = sync_action_list_create();
	struct sync_action_list *mailbox_list = sync_action_list_create();
	struct sync_action_list *quota_list = sync_action_list_create();
	struct sync_action_list *annot_list = sync_action_list_create();
	struct sync_action_list *seen_list = sync_action_list_create();
	struct sync_action_list *sub_list = sync_action_list_create();
	struct sync_name_list *mboxname_list = sync_name_list_create();
	static struct buf type, arg1, arg2;
	char arg1s, arg2s;
	struct sync_action *action;
	int c;
	int fd = -1;
	int doclose = 0;
	struct protstream *input;
	int r = 0;

	if ((filename == NULL) \|\| !strcmp(filename, "-"))
	fd = 0; /* STDIN */
	else {
	fd = open(filename, O_RDWR);
	if (fd < 0) {
	syslog(LOG_ERR, "Failed to open %s: %m", filename);
	r = IMAP_IOERROR;
	goto cleanup;
	}

	doclose = 1;

	if (lock_blocking(fd) < 0) {
	syslog(LOG_ERR, "Failed to lock %s: %m", filename);
	r = IMAP_IOERROR;
	goto cleanup;
	}
	}

	input = prot_new(fd, 0);

	while (1) {
	if ((c = getword(input, &type)) == EOF)
	break;

	/* Ignore blank lines */
	if (c == '\r') c = prot_getc(input);
	if (c == '\n')
	continue;

	if (c != ' ') {
	syslog(LOG_ERR, "Invalid input");
	eatline(input, c);
	continue;
	}

	if ((c = getastring(input, 0, &arg1)) == EOF) break;
	arg1s = arg1.s;

	if (c == ' ') {
	if ((c = getastring(input, 0, &arg2)) == EOF) break;
	arg2s = arg2.s;

	} else
	arg2s = NULL;

	if (c == '\r') c = prot_getc(input);
	if (c != '\n') {
	syslog(LOG_ERR, "Garbage at end of input line");
	eatline(input, c);
	continue;
	}

	ucase(type.s);

	if (!strcmp(type.s, "USER"))
	sync_action_list_add(user_list, NULL, arg1s);
	else if (!strcmp(type.s, "META"))
	sync_action_list_add(meta_list, NULL, arg1s);
	else if (!strcmp(type.s, "SIEVE"))
	sync_action_list_add(meta_list, NULL, arg1s);
	else if (!strcmp(type.s, "MAILBOX"))
	sync_action_list_add(mailbox_list, arg1s, NULL);
	else if (!strcmp(type.s, "QUOTA"))
	sync_action_list_add(quota_list, arg1s, NULL);
	else if (!strcmp(type.s, "ANNOTATION"))
	sync_action_list_add(annot_list, arg1s, NULL);
	else if (!strcmp(type.s, "SEEN"))
	sync_action_list_add(seen_list, arg2s, arg1s);
	else if (!strcmp(type.s, "SUB"))
	sync_action_list_add(sub_list, arg2s, arg1s);
	else if (!strcmp(type.s, "UNSUB"))
	sync_action_list_add(sub_list, arg2s, arg1s);
	else
	syslog(LOG_ERR, "Unknown action type: %s", type.s);
	}

	prot_free(input);
	if (doclose) {
	close(fd);
	doclose = 0;
	}

	/* Optimise out redundant clauses */

	for (action = user_list->head; action; action = action->next) {
	char inboxname[MAX_MAILBOX_BUFFER];
	char deletedname[MAX_MAILBOX_BUFFER];

	/* USER action overrides any MAILBOX action on any of the
	* user's mailboxes or any META, SEEN or SUB/UNSUB
	* action for same user */
	(sync_namespace.mboxname_tointernal)(&sync_namespace, "INBOX",
	action->user, inboxname);
	remove_folder(inboxname, mailbox_list, 1);

	/* remove deleted namespace items as well */
	if (mboxlist_delayed_delete_isenabled()) {
	mboxname_todeleted(inboxname, deletedname, 0);
	remove_folder(deletedname, mailbox_list, 1);
	}

	/* remove per-user items */
	remove_meta(action->user, meta_list);
	remove_meta(action->user, seen_list);
	remove_meta(action->user, sub_list);
	}

	for (action = meta_list->head; action; action = action->next) {
	/* META action overrides any user SEEN or SUB/UNSUB action
	for same user */
	remove_meta(action->user, seen_list);
	remove_meta(action->user, sub_list);
	}

	/* And then run tasks. */
	for (action = quota_list->head; action; action = action->next) {
	if (!action->active)
	continue;

	if (do_quota(action->name)) {
	/* XXX - bogus handling, should be user */
	sync_action_list_add(mailbox_list, action->name, NULL);
	if (verbose) {
	printf(" Promoting: QUOTA %s -> MAILBOX %s\n",
	action->name, action->name);
	}
	if (verbose_logging) {
	syslog(LOG_INFO, " Promoting: QUOTA %s -> MAILBOX %s",
	action->name, action->name);
	}
	}
	}

	for (action = annot_list->head; action; action = action->next) {
	if (!action->active)
	continue;

	/* NOTE: ANNOTATION "" is a special case - it's a server
	* annotation, hence the check for a character at the
	* start of the name */
	if (do_annotation(action->name) && *action->name) {
	/* XXX - bogus handling, should be ... er, something */
	sync_action_list_add(mailbox_list, action->name, NULL);
	if (verbose) {
	printf(" Promoting: ANNOTATION %s -> MAILBOX %s\n",
	action->name, action->name);
	}
	if (verbose_logging) {
	syslog(LOG_INFO, " Promoting: ANNOTATION %s -> MAILBOX %s",
	action->name, action->name);
	}
	}
	}

	for (action = seen_list->head; action; action = action->next) {
	if (!action->active)
	continue;

	if (do_seen(action->user, action->name)) {
	char *userid = mboxname_isusermailbox(action->name, 1);
	if (userid && !strcmp(userid, action->user)) {
	sync_action_list_add(user_list, NULL, action->user);
	if (verbose) {
	printf(" Promoting: SEEN %s %s -> USER %s\n",
	action->user, action->name, action->user);
	}
	if (verbose_logging) {
	syslog(LOG_INFO, " Promoting: SEEN %s %s -> USER %s",
	action->user, action->name, action->user);
	}
	} else {
	sync_action_list_add(meta_list, NULL, action->user);
	if (verbose) {
	printf(" Promoting: SEEN %s %s -> META %s\n",
	action->user, action->name, action->user);
	}
	if (verbose_logging) {
	syslog(LOG_INFO, " Promoting: SEEN %s %s -> META %s",
	action->user, action->name, action->user);
	}
	}
	}
	}

	for (action = sub_list->head; action; action = action->next) {
	if (!action->active)
	continue;

	if (user_sub(action->user, action->name)) {
	sync_action_list_add(meta_list, NULL, action->user);
	if (verbose) {
	printf(" Promoting: SUB %s %s -> META %s\n",
	action->user, action->name, action->user);
	}
	if (verbose_logging) {
	syslog(LOG_INFO, " Promoting: SUB %s %s -> META %s",
	action->user, action->name, action->name);
	}
	}
	}

	for (action = mailbox_list->head; action; action = action->next) {
	if (!action->active)
	continue;

	sync_name_list_add(mboxname_list, action->name);
	}

	if (mboxname_list->count) {
	int nonuser = 0;
	r = do_mailboxes(mboxname_list);
	if (r) {
	/* promote failed personal mailboxes to USER */
	struct sync_name *mbox;
	char userid, p, *useridp;

	for (mbox = mboxname_list->head; mbox; mbox = mbox->next) {
	/* done OK? Good :) */
	if (mbox->mark)
	continue;

	useridp = mboxname_isusermailbox(mbox->name, 0);
	if (useridp) {
	userid = xstrdup(useridp);
	if ((p = strchr(userid, '.'))) *p = '\0';
	mbox->mark = 1;

	sync_action_list_add(user_list, NULL, userid);
	if (verbose) {
	printf(" Promoting: MAILBOX %s -> USER %s\n",
	mbox->name, userid);
	}
	if (verbose_logging) {
	syslog(LOG_INFO, " Promoting: MAILBOX %s -> USER %s",
	mbox->name, userid);
	}
	free(userid);
	}
	else
	nonuser = 1;
	}
	}

	if (r && nonuser) goto cleanup;
	}

	for (action = meta_list->head; action; action = action->next) {
	if (!action->active)
	continue;

	r = do_meta(action->user);
	if (r) {
	if (r == IMAP_INVALID_USER) goto cleanup;

	sync_action_list_add(user_list, NULL, action->user);
	if (verbose) {
	printf(" Promoting: META %s -> USER %s\n",
	action->user, action->user);
	}
	if (verbose_logging) {
	syslog(LOG_INFO, " Promoting: META %s -> USER %s",
	action->user, action->user);
	}
	}
	}

	for (action = user_list->head; action; action = action->next) {
	r = do_user(action->user);
	if (r) goto cleanup;
	}

	cleanup:
	if (doclose) close(fd);

	if (r) {
	if (verbose)
	fprintf(stderr, "Error in do_sync(): bailing out! %s\n", error_message(r));

	syslog(LOG_ERR, "Error in do_sync(): bailing out! %s", error_message(r));
	}

	sync_action_list_free(&user_list);
	sync_action_list_free(&meta_list);
	sync_action_list_free(&mailbox_list);
	sync_action_list_free(&quota_list);
	sync_action_list_free(&annot_list);
	sync_action_list_free(&seen_list);
	sync_action_list_free(&sub_list);
	sync_name_list_free(&mboxname_list);

	return r;
	}

	/* ====================================================================== */

	enum {
	RESTART_NONE = 0,
	RESTART_NORMAL,
	RESTART_RECONNECT
	};

	int do_daemon_work(const char sync_log_file, const char sync_shutdown_file,
	unsigned long timeout, unsigned long min_delta,
	int *restartp)
	{
	int r = 0;
	char *work_file_name;
	time_t session_start;
	time_t single_start;
	int delta;
	struct stat sbuf;

	*restartp = RESTART_NONE;

	/* Create a work log filename. Use the parent PID so we can
	* try to reprocess it if the child fails.
	*/
	work_file_name = xmalloc(strlen(sync_log_file)+20);
	snprintf(work_file_name, strlen(sync_log_file)+20,
	"%s-%d", sync_log_file, getppid());

	session_start = time(NULL);

	while (1) {
	single_start = time(NULL);

	signals_poll();

	/* Check for shutdown file */
	if (sync_shutdown_file && !stat(sync_shutdown_file, &sbuf)) {
	unlink(sync_shutdown_file);
	break;
	}

	/* See if its time to RESTART */
	if ((timeout > 0) &&
	((single_start - session_start) > (time_t) timeout)) {
	*restartp = RESTART_NORMAL;
	break;
	}

	if ((stat(work_file_name, &sbuf) == 0) &&
	(sbuf.st_mtime - single_start < 3600)) {
	/* Existing work log file from our parent < 1 hour old */
	/* XXX Is 60 minutes a resonable timeframe? */
	syslog(LOG_NOTICE,
	"Reprocessing sync log file %s", work_file_name);
	}
	else {
	/* Check for sync_log file */
	if (stat(sync_log_file, &sbuf) < 0) {
	if (min_delta > 0) {
	sleep(min_delta);
	} else {
	usleep(100000); /* 1/10th second */
	}
	continue;
	}

	/* Move sync_log to our work file */
	if (rename(sync_log_file, work_file_name) < 0) {
	syslog(LOG_ERR, "Rename %s -> %s failed: %m",
	sync_log_file, work_file_name);
	r = IMAP_IOERROR;
	break;
	}
	}

	/* Process the work log */
	if ((r=do_sync(work_file_name))) {
	syslog(LOG_ERR,
	"Processing sync log file %s failed: %s",
	work_file_name, error_message(r));
	break;
	}

	/* Remove the work log */
	if (unlink(work_file_name) < 0) {
	syslog(LOG_ERR, "Unlink %s failed: %m", work_file_name);
	r = IMAP_IOERROR;
	break;
	}
	delta = time(NULL) - single_start;

	if (((unsigned) delta < min_delta) && ((min_delta-delta) > 0))
	sleep(min_delta-delta);
	}
	free(work_file_name);

	if (*restartp == RESTART_NORMAL) {
	prot_printf(sync_out, "RESTART\r\n");
	prot_flush(sync_out);

	r = sync_parse_response("RESTART", sync_in, NULL);

	if (r) {
	syslog(LOG_ERR, "sync_client RESTART failed: %s",
	error_message(r));
	} else {
	syslog(LOG_INFO, "sync_client RESTART succeeded");
	}
	r = 0;
	}

	return(r);
	}

	struct backend replica_connect(struct backend be, const char *servername,
	sasl_callback_t *cb)
	{
	int wait;
	struct protoent *proto;

	/* get the right port */
	csync_protocol.service = config_getstring(IMAPOPT_SYNC_PORT);

	for (wait = 15;; wait *= 2) {
	be = backend_connect(be, servername, &csync_protocol,
	"", cb, NULL);

	if (be \|\| connect_once \|\| wait > 1000) break;

	fprintf(stderr,
	"Can not connect to server '%s', retrying in %d seconds\n",
	servername, wait);
	sleep(wait);
	}

	if (!be) {
	fprintf(stderr, "Can not connect to server '%s'\n",
	servername);
	syslog(LOG_ERR, "Can not connect to server '%s'", servername);
	_exit(1);
	}

	/* Disable Nagle's Algorithm => increase throughput
	*
	* http://en.wikipedia.org/wiki/Nagle's_algorithm
	*/
	if (servername[0] != '/') {
	if (be->sock >= 0 && (proto = getprotobyname("tcp")) != NULL) {
	int on = 1;

	if (setsockopt(be->sock, proto->p_proto, TCP_NODELAY,
	(void *) &on, sizeof(on)) != 0) {
	syslog(LOG_ERR, "unable to setsocketopt(TCP_NODELAY): %m");
	}

	/* turn on TCP keepalive if set */
	if (config_getswitch(IMAPOPT_TCP_KEEPALIVE)) {
	int r;
	int optval = 1;
	socklen_t optlen = sizeof(optval);

	r = setsockopt(be->sock, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen);
	if (r < 0) {
	syslog(LOG_ERR, "unable to setsocketopt(SO_KEEPALIVE): %m");
	}
	#ifdef TCP_KEEPCNT
	if (config_getint(IMAPOPT_TCP_KEEPALIVE_CNT)) {
	r = setsockopt(be->sock, SOL_TCP, TCP_KEEPCNT, &optval, optlen);
	if (r < 0) {
	syslog(LOG_ERR, "unable to setsocketopt(TCP_KEEPCNT): %m");
	}
	}
	#endif
	#ifdef TCP_KEEPIDLE
	if (config_getint(IMAPOPT_TCP_KEEPALIVE_IDLE)) {
	r = setsockopt(be->sock, SOL_TCP, TCP_KEEPIDLE, &optval, optlen);
	if (r < 0) {
	syslog(LOG_ERR, "unable to setsocketopt(TCP_KEEPIDLE): %m");
	}
	}
	#endif
	#ifdef TCP_KEEPINTVL
	if (config_getint(IMAPOPT_TCP_KEEPALIVE_INTVL)) {
	r = setsockopt(be->sock, SOL_TCP, TCP_KEEPINTVL, &optval, optlen);
	if (r < 0) {
	syslog(LOG_ERR, "unable to setsocketopt(TCP_KEEPINTVL): %m");
	}
	}
	#endif
	}
	} else {
	syslog(LOG_ERR, "unable to getprotobyname(\"tcp\"): %m");
	}
	}

	#ifdef HAVE_ZLIB
	/* check if we should compress */
	if (do_compress \|\| config_getswitch(IMAPOPT_SYNC_COMPRESS)) {
	prot_printf(be->out, "COMPRESS DEFLATE\r\n");
	prot_flush(be->out);

	if (sync_parse_response("COMPRESS", be->in, NULL)) {
	syslog(LOG_ERR, "Failed to enable compression, continuing uncompressed");
	}
	else {
	prot_setcompress(be->in);
	prot_setcompress(be->out);
	}
	}
	#endif

	return be;
	}

	void do_daemon(const char sync_log_file, const char sync_shutdown_file,
	unsigned long timeout, unsigned long min_delta,
	struct backend be, sasl_callback_t cb)
	{
	int r = 0;
	pid_t pid;
	int status;
	int restart;

	if (!foreground) {
	/* fork a child so we can release from master */
	if ((pid=fork()) < 0) fatal("fork failed", EC_SOFTWARE);

	if (pid != 0) { /* parent */
	cyrus_done();
	exit(0);
	}
	/* child */
	}

	if (foreground \|\| timeout == 0) {
	do_daemon_work(sync_log_file, sync_shutdown_file,
	timeout, min_delta, &restart);
	return;
	}

	signal(SIGPIPE, SIG_IGN); /* don't fail on server disconnects */

	do {
	/* fork a child so we can RESTART (flush memory) */
	if ((pid=fork()) < 0) fatal("fork failed", EC_SOFTWARE);

	if (pid == 0) { /* child */

	if (be->sock == -1) {
	/* Reopen up connection to server */
	be = replica_connect(be, servername, cb);

	if (!be) {
	fprintf(stderr, "Can not connect to server '%s'\n",
	be->hostname);
	syslog(LOG_ERR, "Can not connect to server '%s'",
	be->hostname);
	_exit(1);
	}

	sync_in = be->in;
	sync_out = be->out;
	}

	r = do_daemon_work(sync_log_file, sync_shutdown_file,
	timeout, min_delta, &restart);

	if (r) {
	/* See if we're still connected to the server.
	* If we are, we had some type of error, so we exit.
	* Otherwise, try reconnecting.
	*/
	if (!backend_ping(be)) _exit(1);

	syslog(LOG_WARNING, "Lost connection to server. Reconnecting");
	restart = 1;
	}

	if (restart) _exit(EX_TEMPFAIL);
	_exit(0);
	}

	/* parent */
	if (waitpid(pid, &status, 0) < 0) fatal("waitpid failed", EC_SOFTWARE);

	backend_disconnect(be);
	} while (WIFEXITED(status) && (WEXITSTATUS(status) == EX_TEMPFAIL));

	if (WIFEXITED(status)) {
	syslog(LOG_ERR, "process %d exited, status %d\n", pid,
	WEXITSTATUS(status));
	}
	if (WIFSIGNALED(status)) {
	syslog(LOG_ERR,
	"process %d exited, signaled to death by %d\n",
	pid, WTERMSIG(status));
	}
	}

	/* ====================================================================== */

	static struct sasl_callback mysasl_cb[] = {
	{ SASL_CB_GETOPT, &mysasl_config, NULL },
	{ SASL_CB_CANON_USER, &mysasl_canon_user, NULL },
	{ SASL_CB_LIST_END, NULL, NULL }
	};

	enum {
	MODE_UNKNOWN = -1,
	MODE_REPEAT,
	MODE_USER,
	MODE_MAILBOX,
	MODE_META
	};

	int main(int argc, char **argv)
	{
	int opt, i = 0;
	char *alt_config = NULL;
	char *input_filename = NULL;
	int r = 0;
	int exit_rc = 0;
	int mode = MODE_UNKNOWN;
	int wait = 0;
	int timeout = 600;
	int min_delta = 0;
	char sync_log_file[MAX_MAILBOX_PATH+1];
	char *sync_log_name = NULL;
	const char *sync_shutdown_file = NULL;
	char buf[512];
	FILE *file;
	int len;
	struct backend *be = NULL;
	sasl_callback_t *cb;
	int config_virtdomains;

	if ((geteuid()) == 0 && (become_cyrus() != 0)) {
	fatal("must run as the Cyrus user", EC_USAGE);
	}

	setbuf(stdout, NULL);

	while ((opt = getopt(argc, argv, "C:vlS:F:f:w:t:d:n:rRumsoz")) != EOF) {
	switch (opt) {
	case 'C': /* alt config file */
	alt_config = optarg;
	break;

	case 'o': /* only try to connect once */
	connect_once = 1;
	break;

	case 'v': /* verbose */
	verbose++;
	break;

	case 'l': /* verbose Logging */
	verbose_logging++;
	break;

	case 'S': /* Socket descriptor for server */
	servername = optarg;
	break;

	case 'F': /* Shutdown file */
	sync_shutdown_file = optarg;
	break;

	case 'f': /* input_filename used by user and mailbox modes; OR
	alternate sync_log_file used by single-run repeat mode */
	input_filename = optarg;
	break;

	case 'n':
	sync_log_name = optarg;
	break;

	case 'w':
	wait = atoi(optarg);
	break;

	case 't':
	timeout = atoi(optarg);
	break;

	case 'd':
	min_delta = atoi(optarg);
	break;

	case 'R':
	foreground = 1;
	case 'r':
	if (mode != MODE_UNKNOWN)
	fatal("Mutually exclusive options defined", EC_USAGE);
	mode = MODE_REPEAT;
	break;

	case 'u':
	if (mode != MODE_UNKNOWN)
	fatal("Mutually exclusive options defined", EC_USAGE);
	mode = MODE_USER;
	break;

	case 'm':
	if (mode != MODE_UNKNOWN)
	fatal("Mutually exclusive options defined", EC_USAGE);
	mode = MODE_MAILBOX;
	break;

	case 's':
	if (mode != MODE_UNKNOWN)
	fatal("Mutually exclusive options defined", EC_USAGE);
	mode = MODE_META;
	break;

	case 'z':
	#ifdef HAVE_ZLIB
	do_compress = 1;
	#else
	fatal("Compress not available without zlib compiled in", EC_SOFTWARE);
	#endif
	break;

	default:
	usage("sync_client");
	}
	}

	if (mode == MODE_UNKNOWN)
	fatal("No replication mode specified", EC_USAGE);

	cyrus_init(alt_config, "sync_client", 0);

	if (!servername &&
	!(servername = config_getstring(IMAPOPT_SYNC_HOST))) {
	fatal("sync_host not defined", EC_SOFTWARE);
	}

	/* Just to help with debugging, so we have time to attach debugger */
	if (wait > 0) {
	fprintf(stderr, "Waiting for %d seconds for gdb attach...\n", wait);
	sleep(wait);
	}

	/* Set namespace -- force standard (internal) */
	config_virtdomains = config_getenum(IMAPOPT_VIRTDOMAINS);
	if ((r = mboxname_init_namespace(&sync_namespace, 1)) != 0) {
	fatal(error_message(r), EC_CONFIG);
	}

	/* open the mboxlist, we'll need it for real work */
	mboxlist_init(0);
	mboxlist_open(NULL);

	/* open the quota db, we'll need it for real work */
	quotadb_init(0);
	quotadb_open(NULL);

	/* open the annotation db */
	annotatemore_init(0, NULL, NULL);
	annotatemore_open(NULL);

	signals_set_shutdown(&shut_down);
	signals_add_handlers(0);

	/* load the SASL plugins */
	global_sasl_init(1, 0, mysasl_cb);

	cb = mysasl_callbacks(NULL,
	config_getstring(IMAPOPT_SYNC_AUTHNAME),
	config_getstring(IMAPOPT_SYNC_REALM),
	config_getstring(IMAPOPT_SYNC_PASSWORD));

	/* Open up connection to server */
	be = replica_connect(NULL, servername, cb);

	if (!be) {
	fprintf(stderr, "Can not connect to server '%s'\n", servername);
	exit(1);
	}

	sync_in = be->in;
	sync_out = be->out;

	switch (mode) {
	case MODE_USER:
	if (input_filename) {
	if ((file=fopen(input_filename, "r")) == NULL) {
	syslog(LOG_NOTICE, "Unable to open %s: %m", input_filename);
	shut_down(1);
	}
	while (fgets(buf, sizeof(buf), file)) {
	/* Chomp, then ignore empty/comment lines. */
	if (((len=strlen(buf)) > 0) && (buf[len-1] == '\n'))
	buf[--len] = '\0';

	if ((len == 0) \|\| (buf[0] == '#'))
	continue;

	mboxname_hiersep_tointernal(&sync_namespace, buf,
	config_virtdomains ?
	strcspn(buf, "@") : 0);
	if (do_user(buf)) {
	if (verbose)
	fprintf(stderr,
	"Error from do_user(%s): bailing out!\n",
	buf);
	syslog(LOG_ERR, "Error in do_user(%s): bailing out!",
	buf);
	exit_rc = 1;
	}
	}
	fclose(file);
	} else for (i = optind; !r && i < argc; i++) {
	mboxname_hiersep_tointernal(&sync_namespace, argv[i],
	config_virtdomains ?
	strcspn(argv[i], "@") : 0);
	if (do_user(argv[i])) {
	if (verbose)
	fprintf(stderr, "Error from do_user(%s): bailing out!\n",
	argv[i]);
	syslog(LOG_ERR, "Error in do_user(%s): bailing out!", argv[i]);
	exit_rc = 1;
	}
	}
	break;

	case MODE_MAILBOX:
	{
	struct sync_name_list *mboxname_list = sync_name_list_create();
	char mailboxname[MAX_MAILBOX_BUFFER];

	if (input_filename) {
	if ((file=fopen(input_filename, "r")) == NULL) {
	syslog(LOG_NOTICE, "Unable to open %s: %m", input_filename);
	shut_down(1);
	}
	while (fgets(buf, sizeof(buf), file)) {
	/* Chomp, then ignore empty/comment lines. */
	if (((len=strlen(buf)) > 0) && (buf[len-1] == '\n'))
	buf[--len] = '\0';

	if ((len == 0) \|\| (buf[0] == '#'))
	continue;

	(*sync_namespace.mboxname_tointernal)(&sync_namespace, buf,
	NULL, mailboxname);
	if (!sync_name_lookup(mboxname_list, mailboxname))
	sync_name_list_add(mboxname_list, mailboxname);
	}
	fclose(file);
	} else for (i = optind; i < argc; i++) {
	(*sync_namespace.mboxname_tointernal)(&sync_namespace, argv[i],
	NULL, mailboxname);
	if (!sync_name_lookup(mboxname_list, mailboxname))
	sync_name_list_add(mboxname_list, mailboxname);
	}

	if (do_mailboxes(mboxname_list)) {
	if (verbose) {
	fprintf(stderr,
	"Error from do_mailboxes(): bailing out!\n");
	}
	syslog(LOG_ERR, "Error in do_mailboxes(): bailing out!");
	exit_rc = 1;
	}

	sync_name_list_free(&mboxname_list);
	}
	break;

	case MODE_META:
	for (i = optind; i < argc; i++) {
	mboxname_hiersep_tointernal(&sync_namespace, argv[i],
	config_virtdomains ?
	strcspn(argv[i], "@") : 0);
	if (do_meta(argv[i])) {
	if (verbose) {
	fprintf(stderr,
	"Error from do_meta(%s): bailing out!\n",
	argv[i]);
	}
	syslog(LOG_ERR, "Error in do_meta(%s): bailing out!",
	argv[i]);
	exit_rc = 1;
	}
	}
	break;

	case MODE_REPEAT:
	if (input_filename) {
	exit_rc = do_sync(input_filename);
	}
	else {
	if (sync_log_name) {
	strlcpy(sync_log_file, config_dir, sizeof(sync_log_file));
	strlcat(sync_log_file, "/sync/", sizeof(sync_log_file));
	strlcat(sync_log_file, sync_log_name, sizeof(sync_log_file));
	strlcat(sync_log_file, "/log", sizeof(sync_log_file));
	} else {
	strlcpy(sync_log_file, config_dir, sizeof(sync_log_file));
	strlcat(sync_log_file, "/sync/log", sizeof(sync_log_file));
	}

	if (!sync_shutdown_file)
	sync_shutdown_file = config_getstring(IMAPOPT_SYNC_SHUTDOWN_FILE);

	if (!min_delta)
	min_delta = config_getint(IMAPOPT_SYNC_REPEAT_INTERVAL);

	do_daemon(sync_log_file, sync_shutdown_file, timeout, min_delta,
	be, cb);
	}
	break;

	default:
	if (verbose) fprintf(stderr, "Nothing to do!\n");
	break;
	}

	backend_disconnect(be);

	shut_down(exit_rc);
	}
	diff --git a/imap/sync_log.c b/imap/sync_log.c
	index fb4252b05..9d1e013b3 100644
	--- a/imap/sync_log.c
	+++ b/imap/sync_log.c
	@@ -1,285 +1,285 @@
	/* sync_log.c -- Cyrus synchonization logging functions
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: sync_log.c,v 1.7 2010/01/06 17:01:41 murch Exp $
	*
	* Original version written by David Carter <dpc22@cam.ac.uk>
	* Rewritten and integrated into Cyrus by Ken Murchison <ken@oceana.com>
	*/

	/* YYY Need better quoting for obscure filenames: use literals? */

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#ifdef HAVE_STDARG_H
	#include <stdarg.h>
	#else
	#include <varargs.h>
	#endif
	#include <stdlib.h>
	#include <stdio.h>
	#include <string.h>
	#include <syslog.h>
	#include <errno.h>

	#include "sync_log.h"
	#include "global.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "mailbox.h"
	#include "retry.h"
	#include "util.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"

	static int sync_log_enabled = 0;

	struct sync_log_target {
	char file[MAX_MAILBOX_PATH+1];
	struct sync_log_target *next;
	};
	static struct sync_log_target *sync_target = NULL;

	void sync_log_init(void)
	{
	struct sync_log_target *item = NULL;
	const char *names;
	char *copy;
	char *start;
	char *end;

	sync_log_enabled = config_getswitch(IMAPOPT_SYNC_LOG);
	names = config_getstring(IMAPOPT_SYNC_LOG_NAMES);

	if (names) {
	copy = start = xstrdup(names);
	while (start[0]) {
	end = strchr(start, ' ');
	if (end) {
	*end = '\0';
	}
	item = sync_target;
	sync_target = (struct sync_log_target *) xmalloc(sizeof(struct sync_log_target));
	sync_target->next = item;
	snprintf(sync_target->file, MAX_MAILBOX_PATH,
	"%s/sync/%s/log", config_dir, start);
	if (!end) break;
	start = end + 1;
	}
	free(copy);
	} else {
	sync_target = (struct sync_log_target *) xmalloc(sizeof(struct sync_log_target));
	sync_target->next = NULL;
	snprintf(sync_target->file, MAX_MAILBOX_PATH,
	"%s/sync/log", config_dir);
	}
	}

	void sync_log_suppress(void)
	{
	sync_log_enabled = 0;
	}

	void sync_log_done(void)
	{
	struct sync_log_target *item = NULL;
	while (sync_target) {
	item = sync_target->next;
	free(sync_target);
	sync_target = item;
	}
	}

	static void sync_log_base(const char *string, int len)
	{
	int fd, rc;
	struct stat sbuffile, sbuffd;
	int retries = 0;
	struct sync_log_target *item = sync_target;

	if (!sync_log_enabled) return;

	while (item) {
	while (retries++ < SYNC_LOG_RETRIES) {
	fd = open(item->file, O_WRONLY\|O_APPEND\|O_CREAT, 0640);
	if (fd < 0 && errno == ENOENT) {
	if (!cyrus_mkdir(item->file, 0755)) {
	fd = open(item->file, O_WRONLY\|O_APPEND\|O_CREAT, 0640);
	}
	}
	if (fd < 0) {
	syslog(LOG_ERR, "sync_log(): Unable to write to log file %s: %s",
	item->file, strerror(errno));
	return;
	}

	if (lock_blocking(fd) == -1) {
	syslog(LOG_ERR, "sync_log(): Failed to lock %s for %s: %m",
	item->file, string);
	close(fd);
	return;
	}

	/* Check that the file wasn't renamed after it was opened above */
	if ((fstat(fd, &sbuffd) == 0) &&
	(stat(item->file, &sbuffile) == 0) &&
	(sbuffd.st_ino == sbuffile.st_ino))
	break;

	close(fd);
	}
	if (retries >= SYNC_LOG_RETRIES) {
	close(fd);
	syslog(LOG_ERR,
	"sync_log(): Failed to lock %s for %s after %d attempts",
	item->file, string, retries);
	return;
	}

	if ((rc = retry_write(fd, string, len)) < 0)
	syslog(LOG_ERR, "write() to %s failed: %s",
	item->file, strerror(errno));

	if (rc < len)
	syslog(LOG_ERR, "Partial write to %s: %d out of %d only written",
	item->file, rc, len);

	(void)fsync(fd); /* paranoia */
	close(fd);
	item = item->next;
	}
	}

	static const char sync_quote_name(const char name)
	{
	static char buf[MAX_MAILBOX_BUFFER+3]; /* "x2 plus \0 */
	char c;
	int src;
	int dst = 0;
	int need_quote = 0;

	/* initial quote */
	buf[dst++] = '"';

	/* degenerate case - no name is the empty string, quote it */
	if (!name \|\| !*name) {
	need_quote = 1;
	goto end;
	}

	for (src = 0; name[src]; src++) {
	c = name[src];
	if ((c == '\r') \|\| (c == '\n'))
	fatal("Illegal line break in folder name", EC_IOERR);

	/* quoteable characters */
	if ((c == '\\') \|\| (c == '\"') \|\| (c == '{') \|\| (c == '}')) {
	need_quote = 1;
	buf[dst++] = '\\';
	}

	/* non-atom characters */
	else if ((c == ' ') \|\| (c == '\t') \|\| (c == '(') \|\| (c == ')')) {
	need_quote = 1;
	}

	buf[dst++] = c;

	if (dst > MAX_MAILBOX_BUFFER)
	fatal("word too long", EC_IOERR);
	}

	end:
	if (need_quote) {
	buf[dst++] = '\"';
	buf[dst] = '\0';
	return buf;
	}
	else {
	buf[dst] = '\0';
	return buf + 1; /* skip initial quote */
	}
	}

	#define BUFSIZE 4096

	void sync_log(char *fmt, ...)
	{
	va_list ap;
	char buf[BUFSIZE+1], *p;
	size_t len;
	int ival;
	const char *sval;

	if (!sync_log_enabled) return;

	va_start(ap, fmt);
	for (len = 0, p = fmt; *p && len < BUFSIZE; p++) {
	if (*p != '%') {
	buf[len++] = *p;
	continue;
	}
	switch (*++p) {
	case 'd':
	ival = va_arg(ap, int);
	len += snprintf(buf+len, BUFSIZE-len, "%d", ival);
	break;
	case 's':
	sval = va_arg(ap, const char *);
	sval = sync_quote_name(sval);
	strlcpy(buf+len, sval, BUFSIZE-len);
	len += strlen(sval);
	break;
	default:
	buf[len++] = *p;
	break;
	}
	}
	va_end(ap);

	if (buf[len-1] != '\n') buf[len++] = '\n';
	buf[len] = '\0';

	sync_log_base(buf, len);
	}
	diff --git a/imap/sync_support.c b/imap/sync_support.c
	index c35d0c1e2..2ec2dd03f 100644
	--- a/imap/sync_support.c
	+++ b/imap/sync_support.c
	@@ -1,1546 +1,1546 @@
	/* sync_support.c -- Cyrus synchonization support functions
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: sync_support.c,v 1.25 2010/01/06 17:01:41 murch Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdlib.h>
	#include <stdio.h>
	#include <time.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <fcntl.h>
	#include <syslog.h>
	#include <string.h>
	#include <sys/wait.h>
	#include <errno.h>
	#include <ctype.h>
	#include <dirent.h>
	#include <utime.h>

	#include "global.h"
	#include "assert.h"
	#include "mboxlist.h"
	#include "exitcodes.h"
	#include "imap_err.h"
	#include "mailbox.h"
	#include "quota.h"
	#include "xmalloc.h"
	#include "xstrlcat.h"
	#include "xstrlcpy.h"
	#include "acl.h"
	#include "seen.h"
	#include "mboxname.h"
	#include "map.h"
	#include "imapd.h"
	#include "imparse.h"
	#include "message.h"
	#include "util.h"
	#include "user.h"
	#include "retry.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "prot.h"
	#include "dlist.h"

	#include "message_guid.h"
	#include "sync_support.h"
	#include "sync_log.h"

	/* Parse routines */

	char *sync_encode_options(int options)
	{
	static char buf[4];
	int i = 0;

	if (options & OPT_POP3_NEW_UIDL)
	buf[i++] = 'P';
	if (options & OPT_IMAP_SHAREDSEEN)
	buf[i++] = 'S';
	if (options & OPT_IMAP_DUPDELIVER)
	buf[i++] = 'D';
	buf[i] = '\0';

	return buf;
	}

	int sync_parse_options(const char *options)
	{
	int res = 0;
	const char *p = options;

	if (!options) return 0;

	while (*p) {
	switch(*p) {
	case 'P':
	res \|= OPT_POP3_NEW_UIDL;
	break;
	case 'S':
	res \|= OPT_IMAP_SHAREDSEEN;
	break;
	case 'D':
	res \|= OPT_IMAP_DUPDELIVER;
	break;
	}
	p++;
	}

	return res;
	}

	/* Get a simple line (typically error text) */
	#define BUFGROWSIZE 100
	int sync_getline(struct protstream in, struct buf buf)
	{
	unsigned len = 0;
	int c;

	if (buf->alloc == 0) {
	buf->alloc = BUFGROWSIZE;
	buf->s = xmalloc(buf->alloc+1);
	}

	for (;;) {
	c = prot_getc(in);

	if (c == EOF \|\| (c == '\r') \|\| (c == '\n')) {
	/* Munch optional LF after CR */
	if (c == '\r' && ((c = prot_getc(in)) != EOF && c != '\n')) {
	prot_ungetc(c, in);
	c = '\r';
	}
	buf->s[len] = '\0';
	buf->len = len;
	return c;
	}
	if (len == buf->alloc) {
	buf->alloc += BUFGROWSIZE;
	buf->s = xrealloc(buf->s, buf->alloc+1);
	if (len > config_maxword) {
	fatal("word too long", EC_IOERR);
	}
	}
	buf->s[len++] = c;
	}
	return(c);
	}

	/*
	* Eat lines up to next OK/NO/BAD response line
	*
	*/

	int sync_eatlines_unsolicited(struct protstream *in, int c)
	{
	static struct buf response; /* BSS */
	static struct buf line; /* BSS */

	if (c != '\n') {
	sync_getline(in, &line); /* Partial line */
	syslog(LOG_ERR, "Discarding: %s", line.s);
	}

	do {
	if ((c = getword(in, &response)) == EOF)
	return(IMAP_PROTOCOL_ERROR);

	sync_getline(in, &line);
	syslog(LOG_ERR, "Discarding: %s", line.s);
	} while (response.s[0] == '*');

	if (!strcmp(response.s, "OK") \|\|
	!strcmp(response.s, "NO") \|\|
	!strcmp(response.s, "BAD")) {
	syslog(LOG_ERR, "sync_eatlines_unsolicited(): resynchronised okay");
	return(0);
	}

	syslog(LOG_ERR, "sync_eatlines_unsolicited(): failed to resynchronise!");
	return(IMAP_PROTOCOL_ERROR);
	}

	/* ====================================================================== */

	void sync_print_flags(struct dlist *kl,
	struct mailbox *mailbox,
	struct index_record *record)
	{
	int flag;
	struct dlist *fl = dlist_list(kl, "FLAGS");

	if (record->system_flags & FLAG_DELETED)
	dlist_flag(fl, "FLAG", "\\Deleted");
	if (record->system_flags & FLAG_ANSWERED)
	dlist_flag(fl, "FLAG", "\\Answered");
	if (record->system_flags & FLAG_FLAGGED)
	dlist_flag(fl, "FLAG", "\\Flagged");
	if (record->system_flags & FLAG_DRAFT)
	dlist_flag(fl, "FLAG", "\\Draft");
	if (record->system_flags & FLAG_EXPUNGED)
	dlist_flag(fl, "FLAG", "\\Expunged");
	if (record->system_flags & FLAG_SEEN)
	dlist_flag(fl, "FLAG", "\\Seen");

	/* print user flags in mailbox order */
	for (flag = 0; flag < MAX_USER_FLAGS; flag++) {
	if (!mailbox->flagname[flag])
	continue;
	if (!(record->user_flags[flag/32] & (1<<(flag&31))))
	continue;
	dlist_flag(fl, "FLAG", mailbox->flagname[flag]);
	}
	}

	int sync_getflags(struct dlist *kl,
	struct mailbox *mailbox,
	struct index_record *record)
	{
	struct dlist *ki;
	int userflag;

	for (ki = kl->head; ki; ki = ki->next) {
	char *s = xstrdup(ki->sval);

	if (s[0] == '\\') {
	/* System flags */
	lcase(s);
	if (!strcmp(s, "\\seen")) {
	record->system_flags \|= FLAG_SEEN;
	} else if (!strcmp(s, "\\expunged")) {
	record->system_flags \|= FLAG_EXPUNGED;
	} else if (!strcmp(s, "\\answered")) {
	record->system_flags \|= FLAG_ANSWERED;
	} else if (!strcmp(s, "\\flagged")) {
	record->system_flags \|= FLAG_FLAGGED;
	} else if (!strcmp(s, "\\deleted")) {
	record->system_flags \|= FLAG_DELETED;
	} else if (!strcmp(s, "\\draft")) {
	record->system_flags \|= FLAG_DRAFT;
	} else {
	syslog(LOG_ERR, "Unknown system flag: %s", s);
	}
	}
	else {
	if (mailbox_user_flag(mailbox, s, &userflag)) {
	syslog(LOG_ERR, "Unable to record user flag: %s", s);
	return IMAP_IOERROR;
	}
	record->user_flags[userflag/32] \|= 1<<(userflag&31);
	}

	free(s);
	}

	return 0;
	}

	int parse_upload(struct dlist kr, struct mailbox mailbox,
	struct index_record *record)
	{
	struct dlist *fl;
	const char *guid;
	int r;

	memset(record, 0, sizeof(struct index_record));

	if (!dlist_getnum(kr, "UID", &record->uid))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	if (!dlist_getmodseq(kr, "MODSEQ", &record->modseq))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	if (!dlist_getdate(kr, "LAST_UPDATED", &record->last_updated))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	if (!dlist_getlist(kr, "FLAGS", &fl))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	if (!dlist_getdate(kr, "INTERNALDATE", &record->internaldate))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	if (!dlist_getnum(kr, "SIZE", &record->size))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	if (!dlist_getatom(kr, "GUID", &guid))
	return IMAP_PROTOCOL_BAD_PARAMETERS;

	/* parse the flags */
	r = sync_getflags(fl, mailbox, record);
	if (r) return r;

	/* check the GUID format */
	if (!message_guid_decode(&record->guid, guid))
	return IMAP_PROTOCOL_BAD_PARAMETERS;

	return 0;
	}


	/* ====================================================================== */

	struct sync_msgid_list *sync_msgid_list_create(int hash_size)
	{
	struct sync_msgid_list *l = xzmalloc(sizeof (struct sync_msgid_list));

	/* Pick a sensible default if no size given */
	if (hash_size == 0)
	hash_size = 256;

	l->head = NULL;
	l->tail = NULL;
	l->hash_size = hash_size;
	l->hash = xzmalloc(hash_size * sizeof(struct sync_msgid *));
	l->count = 0;
	l->marked = 0;

	return(l);
	}

	struct sync_msgid sync_msgid_add(struct sync_msgid_list l,
	struct message_guid *guid)
	{
	struct sync_msgid *result;
	int offset;

	if (message_guid_isnull(guid))
	return(NULL);

	result = xzmalloc(sizeof(struct sync_msgid));
	offset = message_guid_hash(guid, l->hash_size);

	message_guid_copy(&result->guid, guid);

	l->count++;
	if (l->tail)
	l->tail = l->tail->next = result;
	else
	l->head = l->tail = result;

	/* Insert at start of list */
	result->hash_next = l->hash[offset];
	l->hash[offset] = result;

	return(result);
	}

	void sync_msgid_remove(struct sync_msgid_list *l,
	struct message_guid *guid)
	{
	int offset = message_guid_hash(guid, l->hash_size);
	struct sync_msgid *msgid;

	if (message_guid_isnull(guid)) return;

	for (msgid = l->hash[offset] ; msgid ; msgid = msgid->hash_next) {
	if (message_guid_compare(&msgid->guid, guid)) {
	message_guid_set_null(&msgid->guid);
	return;
	}
	}
	}

	void sync_msgid_list_free(struct sync_msgid_list **lp)
	{
	struct sync_msgid_list l = lp;
	struct sync_msgid current, next;

	current = l->head;
	while (current) {
	next = current->next;
	free(current);
	current = next;
	}
	free(l->hash);
	free(l);

	*lp = NULL;
	}

	struct sync_msgid sync_msgid_lookup(struct sync_msgid_list l,
	struct message_guid *guid)
	{
	int offset = message_guid_hash(guid, l->hash_size);
	struct sync_msgid *msgid;

	if (message_guid_isnull(guid))
	return(NULL);

	for (msgid = l->hash[offset] ; msgid ; msgid = msgid->hash_next) {
	if (message_guid_compare(&msgid->guid, guid))
	return(msgid);
	}
	return(NULL);
	}

	struct sync_reserve_list *sync_reserve_list_create(int hash_size)
	{
	struct sync_reserve_list *l = xmalloc(sizeof(struct sync_reserve_list));

	l->head = NULL;
	l->tail = NULL;
	l->hash_size = hash_size;

	return l;
	}

	struct sync_msgid_list sync_reserve_partlist(struct sync_reserve_list l,
	const char *part)
	{
	struct sync_reserve *item;

	for (item = l->head; item; item = item->next) {
	if (!strcmp(item->part, part))
	return item->list;
	}

	/* not found, create it */
	item = xmalloc(sizeof(struct sync_reserve));
	item->part = xstrdup(part);
	item->next = NULL;
	item->list = sync_msgid_list_create(l->hash_size);

	if (l->tail)
	l->tail = l->tail->next = item;
	else
	l->tail = l->head = item;

	return item->list;
	}

	void sync_reserve_list_free(struct sync_reserve_list **lp)
	{
	struct sync_reserve_list l = lp;
	struct sync_reserve current, next;

	current = l->head;
	while (current) {
	next = current->next;
	sync_msgid_list_free(&current->list);
	free(current->part);
	free(current);
	current = next;
	}
	free(l);

	*lp = NULL;
	}

	/* ====================================================================== */

	struct sync_folder_list *sync_folder_list_create(void)
	{
	struct sync_folder_list *l = xzmalloc(sizeof (struct sync_folder_list));

	l->head = NULL;
	l->tail = NULL;
	l->count = 0;

	return(l);
	}

	struct sync_folder sync_folder_list_add(struct sync_folder_list l,
	const char uniqueid, const char name,
	const char part, const char acl,
	uint32_t options,
	uint32_t uidvalidity,
	uint32_t last_uid,
	modseq_t highestmodseq,
	uint32_t crc,
	uint32_t recentuid,
	time_t recenttime,
	time_t pop3_last_login)
	{
	struct sync_folder *result = xzmalloc(sizeof(struct sync_folder));

	if (l->tail)
	l->tail = l->tail->next = result;
	else
	l->head = l->tail = result;

	l->count++;

	result->next = NULL;

	result->uniqueid = (uniqueid) ? xstrdup(uniqueid) : NULL;
	result->name = (name) ? xstrdup(name) : NULL;
	result->part = (part) ? xstrdup(part) : NULL;
	result->acl = (acl) ? xstrdup(acl) : NULL;
	result->uidvalidity = uidvalidity;
	result->last_uid = last_uid;
	result->highestmodseq = highestmodseq;
	result->options = options;
	result->sync_crc = crc;
	result->recentuid = recentuid;
	result->recenttime = recenttime;
	result->pop3_last_login = pop3_last_login;

	result->mark = 0;
	result->reserve = 0;

	return(result);
	}

	struct sync_folder sync_folder_lookup(struct sync_folder_list l,
	const char *uniqueid)
	{
	struct sync_folder *p;

	for (p = l->head; p; p = p->next) {
	if (!strcmp(p->uniqueid, uniqueid))
	return p;
	}
	return NULL;
	}

	struct sync_folder sync_folder_lookup_byname(struct sync_folder_list l,
	const char *name)
	{
	struct sync_folder *p;

	for (p = l->head; p; p = p->next) {
	if (!strcmp(p->name, name))
	return p;
	}
	return NULL;
	}

	int sync_folder_mark(struct sync_folder_list l, const char uniqueid)
	{
	struct sync_folder *p = sync_folder_lookup(l, uniqueid);

	if (p) {
	p->mark = 1;
	return 1;
	}

	return 0;
	}

	void sync_folder_list_free(struct sync_folder_list **lp)
	{
	struct sync_folder_list l = lp;
	struct sync_folder current, next;

	if (!l) return;

	current = l->head;
	while (current) {
	next = current->next;
	free(current->uniqueid);
	free(current->name);
	free(current->part);
	free(current->acl);
	free(current);
	current = next;
	}
	free(l);
	*lp = NULL;
	}

	/* ====================================================================== */

	struct sync_rename_list *sync_rename_list_create(void)
	{
	struct sync_rename_list *l = xzmalloc(sizeof(struct sync_rename_list));

	l->head = NULL;
	l->tail = NULL;
	l->count = 0;
	l->done = 0;

	return(l);
	}

	struct sync_rename sync_rename_list_add(struct sync_rename_list l,
	const char uniqueid, const char oldname,
	const char newname, const char partition)
	{
	struct sync_rename *result
	= xzmalloc(sizeof(struct sync_rename));

	if (l->tail)
	l->tail = l->tail->next = result;
	else
	l->head = l->tail = result;

	l->count++;

	result->next = NULL;
	result->uniqueid = xstrdup(uniqueid);
	result->oldname = xstrdup(oldname);
	result->newname = xstrdup(newname);
	result->part = xstrdup(partition);
	result->done = 0;

	return result;
	}

	struct sync_rename sync_rename_lookup(struct sync_rename_list l,
	const char *oldname)
	{
	struct sync_rename *p;

	for (p = l->head; p; p = p->next) {
	if (!strcmp(p->oldname, oldname))
	return p;
	}

	return NULL;
	}

	void sync_rename_list_free(struct sync_rename_list **lp)
	{
	struct sync_rename_list l = lp;
	struct sync_rename current, next;

	if (!l) return;

	current = l->head;
	while (current) {
	next = current->next;
	free(current->uniqueid);
	free(current->oldname);
	free(current->newname);
	free(current->part);
	free(current);
	current = next;
	}
	free(l);
	*lp = NULL;
	}

	/* ====================================================================== */

	struct sync_quota_list *sync_quota_list_create(void)
	{
	struct sync_quota_list *l = xzmalloc(sizeof(struct sync_quota_list));

	l->head = NULL;
	l->tail = NULL;
	l->count = 0;
	l->done = 0;

	return(l);
	}

	struct sync_quota sync_quota_list_add(struct sync_quota_list l,
	const char *root, int limit)
	{
	struct sync_quota *result
	= xzmalloc(sizeof(struct sync_quota));

	if (l->tail)
	l->tail = l->tail->next = result;
	else
	l->head = l->tail = result;

	l->count++;

	result->next = NULL;
	result->root = xstrdup(root);
	result->limit = limit;
	result->done = 0;

	return result;
	}

	struct sync_quota sync_quota_lookup(struct sync_quota_list l,
	const char *name)
	{
	struct sync_quota *p;

	for (p = l->head; p; p = p->next) {
	if (!strcmp(p->root, name))
	return p;
	}

	return NULL;
	}

	void sync_quota_list_free(struct sync_quota_list **lp)
	{
	struct sync_quota_list l = lp;
	struct sync_quota current, next;

	if (!l) return;

	current = l->head;
	while (current) {
	next = current->next;
	free(current->root);
	free(current);
	current = next;
	}
	free(l);
	*lp = NULL;
	}

	/* ====================================================================== */

	struct sync_sieve_list *sync_sieve_list_create()
	{
	struct sync_sieve_list *l = xzmalloc(sizeof (struct sync_sieve_list));

	l->head = NULL;
	l->tail = NULL;
	l->count = 0;

	return l;
	}

	void sync_sieve_list_add(struct sync_sieve_list l, const char name,
	time_t last_update, int active)
	{
	struct sync_sieve *item = xzmalloc(sizeof(struct sync_sieve));

	item->name = xstrdup(name);
	item->last_update = last_update;
	item->active = active;
	item->mark = 0;

	if (l->tail)
	l->tail = l->tail->next = item;
	else
	l->head = l->tail = item;

	l->count++;
	}

	struct sync_sieve sync_sieve_lookup(struct sync_sieve_list l, char *name)
	{
	struct sync_sieve *p;

	for (p = l->head; p; p = p->next) {
	if (!strcmp(p->name, name))
	return p;
	}

	return NULL;
	}

	void sync_sieve_list_set_active(struct sync_sieve_list l, char name)
	{
	struct sync_sieve *item;

	for (item = l->head; item; item = item->next) {
	if (!strcmp(item->name, name)) {
	item->active = 1;
	break;
	}
	}
	}

	void sync_sieve_list_free(struct sync_sieve_list **lp)
	{
	struct sync_sieve_list l = lp;
	struct sync_sieve current, next;

	current = l->head;
	while (current) {
	next = current->next;
	free(current->name);
	free(current);
	current = next;
	}
	free(l);
	*lp = NULL;
	}

	struct sync_sieve_list sync_sieve_list_generate(const char userid)
	{
	struct sync_sieve_list *list = sync_sieve_list_create();
	const char *sieve_path = user_sieve_path(userid);
	char filename[2048];
	char active[2048];
	DIR *mbdir;
	struct dirent *next = NULL;
	struct stat sbuf;
	int count;


	if (!(mbdir = opendir(sieve_path)))
	return(list);

	active[0] = '\0';
	while((next = readdir(mbdir)) != NULL) {
	if(!strcmp(next->d_name, ".") \|\| !strcmp(next->d_name, ".."))
	continue;

	snprintf(filename, sizeof(filename), "%s/%s",
	sieve_path, next->d_name);

	if (stat(filename, &sbuf) < 0)
	continue;

	if (!strcmp(next->d_name, "defaultbc")) {
	if (sbuf.st_mode & S_IFLNK) {
	count = readlink(filename, active, 2047);

	if (count >= 0) {
	active[count] = '\0';
	} else {
	/* XXX Report problem? */
	}
	}
	continue;
	}
	sync_sieve_list_add(list, next->d_name, sbuf.st_mtime, 0);
	}
	closedir(mbdir);

	if (active[0])
	sync_sieve_list_set_active(list, active);

	return list;
	}

	char sync_sieve_read(const char userid, const char name, uint32_t sizep)
	{
	const char *sieve_path = user_sieve_path(userid);
	char filename[2048];
	FILE *file;
	struct stat sbuf;
	char result, s;
	uint32_t count;
	int c;

	if (sizep)
	*sizep = 0;

	snprintf(filename, sizeof(filename), "%s/%s", sieve_path, name);

	file = fopen(filename, "r");
	if (!file) return NULL;

	if (fstat(fileno(file), &sbuf) < 0) {
	fclose(file);
	return(NULL);
	}

	count = sbuf.st_size;
	s = result = xmalloc(count+1);

	if (sizep)
	*sizep = count;

	while (count > 0) {
	if ((c=fgetc(file)) == EOF)
	break;
	*s++ = c;
	count--;
	}
	fclose(file);
	*s = '\0';

	return(result);
	}

	int sync_sieve_upload(const char userid, const char name,
	time_t last_update, const char *content,
	size_t len)
	{
	const char *sieve_path = user_sieve_path(userid);
	char tmpname[2048];
	char newname[2048];
	FILE *file;
	int r = 0;
	struct stat sbuf;
	struct utimbuf utimbuf;

	if (stat(sieve_path, &sbuf) == -1 && errno == ENOENT) {
	if (cyrus_mkdir(sieve_path, 0755) == -1) return IMAP_IOERROR;
	if (mkdir(sieve_path, 0755) == -1 && errno != EEXIST) {
	syslog(LOG_ERR, "Failed to create %s:%m", sieve_path);
	return IMAP_IOERROR;
	}
	}

	snprintf(tmpname, sizeof(tmpname), "%s/sync_tmp-%lu",
	sieve_path, (unsigned long)getpid());
	snprintf(newname, sizeof(newname), "%s/%s", sieve_path, name);

	if ((file=fopen(tmpname, "w")) == NULL) {
	return(IMAP_IOERROR);
	}

	/* XXX - error handling */
	fwrite(content, 1, len, file);

	if ((fflush(file) != 0) \|\| (fsync(fileno(file)) < 0))
	r = IMAP_IOERROR;

	fclose(file);

	utimbuf.actime = time(NULL);
	utimbuf.modtime = last_update;

	if (!r && (utime(tmpname, &utimbuf) < 0))
	r = IMAP_IOERROR;

	if (!r && (rename(tmpname, newname) < 0))
	r = IMAP_IOERROR;

	sync_log_sieve(userid);

	return r;
	}

	int sync_sieve_activate(const char userid, const char name)
	{
	const char *sieve_path = user_sieve_path(userid);
	char target[2048];
	char active[2048];

	snprintf(target, sizeof(target), "%s", name);
	snprintf(active, sizeof(active), "%s/%s", sieve_path, "defaultbc");
	unlink(active);

	if (symlink(target, active) < 0)
	return(IMAP_IOERROR);

	sync_log_sieve(userid);

	return(0);
	}

	int sync_sieve_deactivate(const char *userid)
	{
	const char *sieve_path = user_sieve_path(userid);
	char active[2048];

	snprintf(active, sizeof(active), "%s/%s", sieve_path, "defaultbc");
	unlink(active);

	sync_log_sieve(userid);

	return(0);
	}

	int sync_sieve_delete(const char userid, const char name)
	{
	const char *sieve_path = user_sieve_path(userid);
	char filename[2048];
	char active[2048];
	DIR *mbdir;
	struct dirent *next = NULL;
	struct stat sbuf;
	int is_default = 0;
	int count;

	if (!(mbdir = opendir(sieve_path)))
	return(IMAP_IOERROR);

	while((next = readdir(mbdir)) != NULL) {
	if(!strcmp(next->d_name, ".") \|\| !strcmp(next->d_name, ".."))
	continue;

	snprintf(filename, sizeof(filename), "%s/%s",
	sieve_path, next->d_name);

	if (stat(filename, &sbuf) < 0)
	continue;

	if (!strcmp(next->d_name, "defaultbc")) {
	if (sbuf.st_mode & S_IFLNK) {
	count = readlink(filename, active, 2047);

	if (count >= 0) {
	active[count] = '\0';
	if (!strcmp(active, name))
	is_default = 1;
	}
	}
	continue;
	}
	}
	closedir(mbdir);

	if (is_default) {
	snprintf(filename, sizeof(filename), "%s/defaultbc", sieve_path);
	unlink(filename);
	}

	snprintf(filename, sizeof(filename), "%s/%s", sieve_path, name);
	unlink(filename);

	sync_log_sieve(userid);

	return(0);
	}

	/* ====================================================================== */

	struct sync_name_list *sync_name_list_create()
	{
	struct sync_name_list *l = xzmalloc(sizeof (struct sync_name_list));
	l->head = NULL;
	l->tail = NULL;
	l->count = 0;
	l->marked = 0;
	return l;
	}

	struct sync_name sync_name_list_add(struct sync_name_list l,
	const char *name)
	{
	struct sync_name *item = xzmalloc(sizeof(struct sync_name));

	if (l->tail)
	l->tail = l->tail->next = item;
	else
	l->head = l->tail = item;

	l->count++;

	item->next = NULL;
	item->name = xstrdup(name);
	item->mark = 0;

	return item;
	}

	struct sync_name sync_name_lookup(struct sync_name_list l,
	const char *name)
	{
	struct sync_name *p;

	for (p = l->head; p; p = p->next)
	if (!strcmp(p->name, name))
	return p;

	return NULL;
	}

	void sync_name_list_free(struct sync_name_list **lp)
	{
	struct sync_name current, next;

	current = (*lp)->head;
	while (current) {
	next = current->next;
	free(current->name);
	free(current);
	current = next;
	}
	free(*lp);
	*lp = NULL;
	}

	/* ====================================================================== */

	struct sync_seen_list *sync_seen_list_create()
	{
	struct sync_seen_list *l = xzmalloc(sizeof (struct sync_seen_list));
	l->head = NULL;
	l->tail = NULL;
	l->count = 0;
	return l;
	}

	struct sync_seen sync_seen_list_add(struct sync_seen_list l,
	const char *uniqueid, time_t lastread,
	unsigned lastuid, time_t lastchange,
	const char *seenuids)
	{
	struct sync_seen *item = xzmalloc(sizeof(struct sync_seen));

	if (l->tail)
	l->tail = l->tail->next = item;
	else
	l->head = l->tail = item;

	l->count++;

	item->next = NULL;
	item->uniqueid = xstrdup(uniqueid);
	item->sd.lastread = lastread;
	item->sd.lastuid = lastuid;
	item->sd.lastchange = lastchange;
	item->sd.seenuids = xstrdup(seenuids);
	item->mark = 0;

	return item;
	}

	struct sync_seen sync_seen_list_lookup(struct sync_seen_list l,
	const char *uniqueid)
	{
	struct sync_seen *p;

	for (p = l->head; p; p = p->next)
	if (!strcmp(p->uniqueid, uniqueid))
	return p;

	return NULL;
	}

	void sync_seen_list_free(struct sync_seen_list **lp)
	{
	struct sync_seen current, next;

	current = (*lp)->head;
	while (current) {
	next = current->next;
	free(current->uniqueid);
	seen_freedata(&current->sd);
	free(current);
	current = next;
	}
	free(*lp);
	*lp = NULL;
	}

	/* ====================================================================== */

	struct sync_annot_list *sync_annot_list_create()
	{
	struct sync_annot_list *l = xzmalloc(sizeof (struct sync_annot_list));

	l->head = NULL;
	l->tail = NULL;
	l->count = 0;
	return(l);
	}

	void sync_annot_list_add(struct sync_annot_list *l,
	const char entry, const char userid,
	const char *value)
	{
	struct sync_annot *item = xzmalloc(sizeof(struct sync_annot));

	item->entry = xstrdup(entry);
	item->userid = xstrdup(userid);
	item->value = xstrdup(value);
	item->mark = 0;

	if (l->tail)
	l->tail = l->tail->next = item;
	else
	l->head = l->tail = item;

	l->count++;
	}

	void sync_annot_list_free(struct sync_annot_list **lp)
	{
	struct sync_annot_list l = lp;
	struct sync_annot current, next;

	current = l->head;
	while (current) {
	next = current->next;
	if (current->entry) free(current->entry);
	if (current->userid) free(current->userid);
	if (current->value) free(current->value);
	free(current);
	current = next;
	}
	free(l);
	*lp = NULL;
	}

	/* ====================================================================== */

	struct sync_action_list *sync_action_list_create(void)
	{
	struct sync_action_list *l = xzmalloc(sizeof (struct sync_action_list));

	l->head = NULL;
	l->tail = NULL;
	l->count = 0;

	return(l);
	}

	void sync_action_list_add(struct sync_action_list l, char name, char *user)
	{
	struct sync_action *current;

	if (!name && !user) return;

	for (current = l->head ; current ; current = current->next) {
	if ((!name \|\| (current->name && !strcmp(current->name, name))) &&
	(!user \|\| (current->user && !strcmp(current->user, user)))) {
	current->active = 1; /* Make sure active */
	return;
	} else {
	/* name and/or user don't match current: no match possible */
	}
	}

	current = xzmalloc(sizeof(struct sync_action));
	current->next = NULL;
	current->name = (name) ? xstrdup(name) : NULL;
	current->user = (user) ? xstrdup(user) : NULL;
	current->active = 1;

	if (l->tail)
	l->tail = l->tail->next = current;
	else
	l->head = l->tail = current;

	l->count++;

	}

	void sync_action_list_free(struct sync_action_list **lp)
	{
	struct sync_action_list l = lp;
	struct sync_action current, next;

	current = l->head;
	while (current) {
	next = current->next;

	if (current->name) free(current->name);
	if (current->user) free(current->user);

	free(current);
	current = next;
	}
	free(l);
	*lp = NULL;
	}

	/* simple binary search */
	unsigned sync_mailbox_finduid(struct mailbox *mailbox, unsigned uid)
	{
	unsigned low=1, high=mailbox->i.num_records, mid;
	struct index_record record;

	while (low <= high) {
	mid = (high - low)/2 + low;
	if (mailbox_read_index_record(mailbox, mid, &record))
	return 0;

	if (record.uid == uid)
	return mid;
	else if (record.uid > uid)
	high = mid - 1;
	else
	low = mid + 1;
	}
	return 0;
	}

	int addmbox(char *name,
	int matchlen __attribute__((unused)),
	int maycreate __attribute__((unused)),
	void *rock)
	{
	struct sync_name_list list = (struct sync_name_list ) rock;
	struct mboxlist_entry mbentry;

	if (mboxlist_lookup(name, &mbentry, NULL))
	return 0;

	/* only want normal mailboxes... */
	if (!(mbentry.mbtype & (MBTYPE_RESERVE \| MBTYPE_MOVING \| MBTYPE_REMOTE)))
	sync_name_list_add(list, name);

	return 0;
	}

	int addmbox_sub(char *name,
	int matchlen __attribute__((unused)),
	int maycreate __attribute__((unused)),
	void *rock)
	{
	struct sync_name_list list = (struct sync_name_list ) rock;

	sync_name_list_add(list, name);

	return 0;
	}

	/* NOTE - we don't prot_flush here, as we always send an OK at the
	* end of a response anyway */
	void sync_send_response(struct dlist kl, struct protstream out)
	{
	prot_printf(out, "* ");
	dlist_print(kl, 1, out);
	prot_printf(out, "\r\n");
	}

	/* these are one-shot commands for get and apply, so flush the stream
	* after sending */
	void sync_send_apply(struct dlist kl, struct protstream out)
	{
	prot_printf(out, "APPLY ");
	dlist_print(kl, 1, out);
	prot_printf(out, "\r\n");
	prot_flush(out);
	}

	void sync_send_lookup(struct dlist kl, struct protstream out)
	{
	prot_printf(out, "GET ");
	dlist_print(kl, 1, out);
	prot_printf(out, "\r\n");
	prot_flush(out);
	}

	struct dlist sync_parseline(struct protstream in)
	{
	struct dlist *dl = NULL;
	char c;

	c = dlist_parse(&dl, 1, in);

	/* end line - or fail */
	if (c == '\r') c = prot_getc(in);
	if (c == '\n') return dl;

	dlist_free(&dl);
	eatline(in, c);
	return NULL;
	}

	int sync_mailbox(struct mailbox *mailbox,
	struct sync_folder *remote,
	struct sync_msgid_list *part_list,
	struct dlist kl, struct dlist kupload,
	int printrecords)
	{

	dlist_atom(kl, "UNIQUEID", mailbox->uniqueid);
	dlist_atom(kl, "MBOXNAME", mailbox->name);
	dlist_num(kl, "LAST_UID", mailbox->i.last_uid);
	dlist_modseq(kl, "HIGHESTMODSEQ", mailbox->i.highestmodseq);
	dlist_num(kl, "RECENTUID", mailbox->i.recentuid);
	dlist_date(kl, "RECENTTIME", mailbox->i.recenttime);
	dlist_date(kl, "LAST_APPENDDATE", mailbox->i.last_appenddate);
	dlist_date(kl, "POP3_LAST_LOGIN", mailbox->i.pop3_last_login);
	dlist_num(kl, "UIDVALIDITY", mailbox->i.uidvalidity);
	dlist_atom(kl, "PARTITION", mailbox->part);
	dlist_atom(kl, "ACL", mailbox->acl);
	dlist_atom(kl, "OPTIONS", sync_encode_options(mailbox->i.options));
	dlist_num(kl, "SYNC_CRC", mailbox->i.sync_crc);
	if (mailbox->quotaroot)
	dlist_atom(kl, "QUOTAROOT", mailbox->quotaroot);

	if (printrecords) {
	struct index_record record;
	struct dlist *il;
	struct dlist *rl = dlist_list(kl, "RECORD");
	struct stat sbuf;
	const char *fname;
	struct sync_msgid *msgid;
	uint32_t recno;
	int send_file;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	/* we can't send bogus records, just skip them! */
	if (mailbox_read_index_record(mailbox, recno, &record))
	continue;

	/* start off thinking we're sending the file too */
	send_file = 1;

	/* seen it already! SKIP */
	if (remote && record.modseq <= remote->highestmodseq)
	continue;

	/* skip expunged if we're not updating something */
	if (!remote && (record.system_flags & FLAG_EXPUNGED))
	continue;

	/* does it exist at the other end? Don't send it */
	if (remote && record.uid <= remote->last_uid)
	send_file = 0;

	/* if we're not uploading messages... don't send file */
	if (!part_list \|\| !kupload)
	send_file = 0;

	/* if we don't HAVE the file we can't send it */
	if (record.system_flags & FLAG_UNLINKED)
	send_file = 0;

	if (send_file) {
	/* is it already reserved? */
	msgid = sync_msgid_lookup(part_list, &record.guid);
	if (!msgid \|\| !msgid->mark) {
	/* have to make sure the file exists */
	fname = mailbox_message_fname(mailbox, record.uid);
	if (!fname) return IMAP_MAILBOX_BADNAME;
	if (stat(fname, &sbuf) < 0) {
	syslog(LOG_ERR, "IOERROR: failed to stat file %s", fname);
	return IMAP_IOERROR;
	}
	if ((unsigned) sbuf.st_size != record.size) {
	syslog(LOG_ERR, "IOERROR: size mismatch %s %u (%lu != %u)",
	mailbox->name, record.uid, sbuf.st_size, record.size);
	return IMAP_IOERROR;
	}
	dlist_file(kupload, "MESSAGE", mailbox->part, &record.guid,
	record.size, fname);
	}
	}

	il = dlist_kvlist(rl, "RECORD");
	dlist_num(il, "UID", record.uid);
	dlist_modseq(il, "MODSEQ", record.modseq);
	dlist_date(il, "LAST_UPDATED", record.last_updated);
	sync_print_flags(il, mailbox, &record);
	dlist_date(il, "INTERNALDATE", record.internaldate);
	dlist_num(il, "SIZE", record.size);
	dlist_atom(il, "GUID", message_guid_encode(&record.guid));
	}
	}

	return 0;
	}

	int sync_parse_response(const char cmd, struct protstream in,
	struct dlist **klp)
	{
	static struct buf response; /* BSS */
	static struct buf errmsg;
	struct dlist *kl = NULL;
	int c;

	if ((c = getword(in, &response)) == EOF)
	return IMAP_PROTOCOL_ERROR;

	if (c != ' ') goto parse_err;

	kl = dlist_new(cmd);
	while (!strcmp(response.s, "*")) {
	struct dlist *item = sync_parseline(in);
	if (!item) goto parse_err;
	dlist_stitch(kl, item);
	if ((c = getword(in, &response)) == EOF)
	goto parse_err;
	}

	if (!strcmp(response.s, "OK")) {
	if (klp) *klp = kl;
	else dlist_free(&kl);
	eatline(in, c);
	return 0;
	}
	if (!strcmp(response.s, "NO")) {
	dlist_free(&kl);
	sync_getline(in, &errmsg);
	syslog(LOG_ERR, "%s received NO response: %s", cmd, errmsg.s);

	/* Slight hack to transform certain error strings into equivalent
	* imap_err value so that caller has some idea of cause. Match
	* this to the logic at print_response in sync_server */
	if (!strncmp(errmsg.s, "IMAP_INVALID_USER ",
	strlen("IMAP_INVALID_USER ")))
	return IMAP_INVALID_USER;
	else if (!strncmp(errmsg.s, "IMAP_MAILBOX_NONEXISTENT ",
	strlen("IMAP_MAILBOX_NONEXISTENT ")))
	return IMAP_MAILBOX_NONEXISTENT;
	else if (!strncmp(errmsg.s, "IMAP_MAILBOX_CRC ",
	strlen("IMAP_MAILBOX_CRC ")))
	return IMAP_MAILBOX_CRC;
	else if (!strncmp(errmsg.s, "IMAP_PROTOCOL_ERROR ",
	strlen("IMAP_PROTOCOL_ERROR ")))
	return IMAP_PROTOCOL_ERROR;
	else if (!strncmp(errmsg.s, "IMAP_PROTOCOL_BAD_PARAMETERS ",
	strlen("IMAP_PROTOCOL_BAD_PARAMETERS ")))
	return IMAP_PROTOCOL_BAD_PARAMETERS;
	else
	return IMAP_REMOTE_DENIED;
	}

	parse_err:
	dlist_free(&kl);
	sync_getline(in, &errmsg);
	syslog(LOG_ERR, "%s received %s response: %s",
	cmd, response.s, errmsg.s);
	return IMAP_PROTOCOL_ERROR;
	}

	int sync_append_copyfile(struct mailbox *mailbox,
	struct index_record *record)
	{
	const char fname, destname;
	struct message_guid tmp_guid;
	int internaldate = record->internaldate;
	int r;

	message_guid_copy(&tmp_guid, &record->guid);

	fname = dlist_reserve_path(mailbox->part, &tmp_guid);
	if (!fname) {
	r = IMAP_IOERROR;
	syslog(LOG_ERR, "IOERROR: Failed to reserve file %s",
	message_guid_encode(&tmp_guid));
	return r;
	}

	r = message_parse(fname, record);
	if (r) {
	/* deal with unlinked master records */
	if (record->system_flags & FLAG_EXPUNGED) {
	record->system_flags \|= FLAG_UNLINKED;
	goto just_write;
	}
	syslog(LOG_ERR, "IOERROR: failed to parse %s", fname);
	return r;
	}

	if (!message_guid_compare(&tmp_guid, &record->guid)) {
	syslog(LOG_ERR, "IOERROR: guid mismatch on parse %s", fname);
	return IMAP_MAILBOX_CRC;
	}

	destname = mailbox_message_fname(mailbox, record->uid);
	cyrus_mkdir(destname, 0755);
	r = mailbox_copyfile(fname, destname, 0);
	if (r) {
	syslog(LOG_ERR, "IOERROR: Failed to copy %s to %s",
	fname, destname);
	return r;
	}

	/* repair broken internaldate if requried */
	if (!record->internaldate)
	record->internaldate = internaldate;

	just_write:
	record->silent = 1;
	return mailbox_append_index_record(mailbox, record);
	}
	diff --git a/imap/unexpunge.c b/imap/unexpunge.c
	index e504f3de0..c04179940 100644
	--- a/imap/unexpunge.c
	+++ b/imap/unexpunge.c
	@@ -1,396 +1,396 @@
	/* unexpunge.c -- Program to unexpunge messages
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: unexpunge.c,v 1.15 2010/01/06 17:01:42 murch Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdlib.h>
	#include <stdio.h>
	#include <string.h>
	#include <ctype.h>
	#include <fcntl.h>
	#include <sys/stat.h>
	#include <syslog.h>
	#include <errno.h>
	#include <signal.h>

	#include "annotate.h"
	#include "cyrusdb.h"
	#include "duplicate.h"
	#include "exitcodes.h"
	#include "global.h"
	#include "hash.h"
	#include "imap_err.h"
	#include "index.h"
	#include "libcyr_cfg.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "map.h"
	#include "mailbox.h"
	#include "mboxlist.h"
	#include "util.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"
	#include "sync_log.h"

	/* global state */
	const int config_need_data = 0;

	/* current namespace */
	static struct namespace unex_namespace;

	int verbose = 0;
	int unsetdeleted = 0;

	void usage(void)
	{
	fprintf(stderr,
	"unexpunge [-C <altconfig>] -l <mailbox>\n"
	"unexpunge [-C <altconfig>] -t time-interval [ -d ] [ -v ] mailbox\n"
	"unexpunge [-C <altconfig>] -a [-d] [-v] <mailbox>\n"
	"unexpunge [-C <altconfig>] -u [-d] [-v] <mailbox> <uid>...\n");
	exit(-1);
	}

	int compare_uid(const void a, const void b)
	{
	return ((unsigned long ) a) - ((unsigned long ) b);
	}

	enum {
	MODE_UNKNOWN = -1,
	MODE_LIST,
	MODE_ALL,
	MODE_TIME,
	MODE_UID
	};

	void list_expunged(struct mailbox *mailbox)
	{
	struct index_record record;
	uint32_t recno;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	if (mailbox_read_index_record(mailbox, recno, &record))
	continue;

	/* still active */
	if (!(record.system_flags & FLAG_EXPUNGED))
	continue;
	/* no file, unrescuable */
	if (record.system_flags & FLAG_UNLINKED)
	continue;

	printf("UID: %u\n", record.uid);
	printf("\tSize: %u\n", record.size);
	printf("\tSent: %s", ctime(&record.sentdate));
	printf("\tRecv: %s", ctime(&record.internaldate));
	printf("\tExpg: %s", ctime(&record.last_updated));

	if (mailbox_cacherecord(mailbox, &record)) {
	printf("\tERROR: cache record missing or corrupt, "
	"not printing cache details\n\n");
	continue;
	}

	printf("\tFrom: %.*s\n", cacheitem_size(&record, CACHE_FROM),
	cacheitem_base(&record, CACHE_FROM));
	printf("\tTo : %.*s\n", cacheitem_size(&record, CACHE_TO),
	cacheitem_base(&record, CACHE_TO));
	printf("\tCc : %.*s\n", cacheitem_size(&record, CACHE_CC),
	cacheitem_base(&record, CACHE_CC));
	printf("\tBcc : %.*s\n", cacheitem_size(&record, CACHE_BCC),
	cacheitem_base(&record, CACHE_BCC));
	printf("\tSubj: %.*s\n\n", cacheitem_size(&record, CACHE_SUBJECT),
	cacheitem_base(&record, CACHE_SUBJECT));
	}
	}

	int restore_expunged(struct mailbox mailbox, int mode, unsigned long uids,
	unsigned nuids, time_t time_since, unsigned *numrestored)
	{
	uint32_t recno;
	struct index_record record;
	unsigned uidnum = 0;
	char oldfname[MAX_MAILBOX_PATH];
	char *fname;
	uint32_t *deleteduids;
	int r = 0;

	deleteduids = (uint32_t *)
	xmalloc(mailbox->i.num_records * sizeof(uint32_t));
	*numrestored = 0;

	for (recno = 1; recno <= mailbox->i.num_records; recno++) {
	r = mailbox_read_index_record(mailbox, recno, &record);
	if (r) goto done;

	/* still active */
	if (!(record.system_flags & FLAG_EXPUNGED))
	continue;
	/* no file, unrescuable */
	if (record.system_flags & FLAG_UNLINKED)
	continue;

	if (mode == MODE_UID) {
	while (uidnum < nuids && record.uid > uids[uidnum])
	uidnum++;
	if (uidnum >= nuids)
	continue;
	if (record.uid != uids[uidnum])
	continue;
	/* otherwise we want this one */
	}
	else if (mode == MODE_TIME) {
	if (record.last_updated < time_since)
	continue;
	/* otherwise we want this one */
	}

	/* mark the old one unlinked so we don't see it again */
	deleteduids[*numrestored] = record.uid;
	fname = mailbox_message_fname(mailbox, record.uid);
	record.system_flags \|= FLAG_UNLINKED;
	r = mailbox_rewrite_index_record(mailbox, &record);
	if (r) goto done;

	/* duplicate the old filename */
	strncpy(oldfname, fname, MAX_MAILBOX_PATH);

	/* bump the UID, strip the flags */
	record.uid = mailbox->i.last_uid + 1;
	record.system_flags &= ~(FLAG_UNLINKED\|FLAG_EXPUNGED);
	if (unsetdeleted)
	record.system_flags &= ~FLAG_DELETED;

	/* copy the message file */
	fname = mailbox_message_fname(mailbox, record.uid);
	r = mailbox_copyfile(oldfname, fname, 0);
	if (r) goto done;

	/* and append the new record */
	mailbox_append_index_record(mailbox, &record);

	if (verbose)
	printf("Unexpunged %s: %u => %u\n", mailbox->name,
	deleteduids[*numrestored], record.uid);

	(*numrestored)++;
	}

	if (*numrestored) {
	unsigned i;
	/* commit first */
	mailbox_commit(mailbox);

	/* then complete the unlinks once safe to do so */
	for (i = 0; i < *numrestored; i++) {
	fname = mailbox_message_fname(mailbox, deleteduids[i]);
	unlink(fname);
	}
	}

	done:
	free(deleteduids);

	return r;
	}

	int main(int argc, char *argv[])
	{
	extern char *optarg;
	int opt, r = 0;
	char *alt_config = NULL;
	char buf[MAX_MAILBOX_PATH+1];
	struct mailbox *mailbox = NULL;
	int mode = MODE_UNKNOWN;
	unsigned numrestored = 0;
	time_t time_since = time(NULL);
	int len, secs = 0;
	unsigned long *uids = NULL;
	unsigned nuids = 0;

	if ((geteuid()) == 0 && (become_cyrus() != 0)) {
	fatal("must run as the Cyrus user", EC_USAGE);
	}

	while ((opt = getopt(argc, argv, "C:laudt:v")) != EOF) {
	switch (opt) {
	case 'C': /* alt config file */
	alt_config = optarg;
	break;

	case 'l':
	if (mode != MODE_UNKNOWN) usage();
	mode = MODE_LIST;
	break;

	case 'a':
	if (mode != MODE_UNKNOWN) usage();
	mode = MODE_ALL;
	break;

	case 't':
	if (mode != MODE_UNKNOWN) usage();

	mode = MODE_TIME;
	secs = atoi(optarg);
	len = strlen(optarg);

	if ((secs > 0) && (len > 1)) {
	switch (optarg[len-1]) {
	case 'm':
	secs *= 60;
	break;
	case 'h':
	secs = (6060);
	break;
	case 'd':
	secs = (2460*60);
	break;
	case 'w':
	secs = (7246060);
	break;
	}
	}
	time_since = time(NULL) - secs;
	break;

	case 'u':
	if (mode != MODE_UNKNOWN) usage();
	mode = MODE_UID;
	break;

	case 'd':
	unsetdeleted = 1;
	break;

	case 'v':
	verbose = 1;
	break;

	default:
	usage();
	break;
	}
	}

	/* sanity check */
	if (mode == MODE_UNKNOWN \|\|
	(optind + (mode == MODE_UID ? 1 : 0)) >= argc) usage();

	cyrus_init(alt_config, "unexpunge", 0);

	mboxlist_init(0);
	mboxlist_open(NULL);

	quotadb_init(0);
	quotadb_open(NULL);

	sync_log_init();

	/* Set namespace -- force standard (internal) */
	if ((r = mboxname_init_namespace(&unex_namespace, 1)) != 0) {
	syslog(LOG_ERR, "%s", error_message(r));
	fatal(error_message(r), EC_CONFIG);
	}

	/* Translate mailboxname */
	(*unex_namespace.mboxname_tointernal)(&unex_namespace, argv[optind],
	NULL, buf);

	/* Open/lock header */
	r = mailbox_open_iwl(buf, &mailbox);
	if (r) {
	printf("Failed to open mailbox '%s'\n", buf);
	goto done;
	}

	if (mode == MODE_UID) {
	unsigned i;

	nuids = argc - ++optind;
	uids = (unsigned long ) xmalloc(nuids sizeof(unsigned long));

	for (i = 0; i < nuids; i++)
	uids[i] = strtoul(argv[optind+i], NULL, 10);

	/* Sort the UIDs so we can binary search */
	qsort(uids, nuids, sizeof(unsigned long), compare_uid);
	}

	if (mode == MODE_LIST)
	list_expunged(mailbox);
	else {
	printf("restoring %sexpunged messages in mailbox '%s'\n",
	mode == MODE_ALL ? "all " : "", mailbox->name);

	r = restore_expunged(mailbox, mode, uids, nuids, time_since, &numrestored);

	if (!r) {
	printf("restored %u expunged messages\n",
	numrestored);
	syslog(LOG_NOTICE,
	"restored %u expunged messages in mailbox '%s'",
	numrestored, mailbox->name);
	}
	}

	mailbox_close(&mailbox);

	done:
	sync_log_done();

	quotadb_close();
	quotadb_done();

	mboxlist_close();
	mboxlist_done();

	cyrus_done();

	exit(r);
	}
	diff --git a/imap/upgrade_index.c b/imap/upgrade_index.c
	index aac4c5ee9..516e9fbbe 100644
	--- a/imap/upgrade_index.c
	+++ b/imap/upgrade_index.c
	@@ -1,497 +1,497 @@
	/* upgrade_index.c -- Mailbox upgrade routines
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: mailbox.c,v 1.199 2010/01/06 17:01:36 murch Exp $
	*/
	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <errno.h>
	#include <syslog.h>
	#include <utime.h>

	#ifdef HAVE_DIRENT_H
	# include <dirent.h>
	# define NAMLEN(dirent) strlen((dirent)->d_name)
	#else
	# define dirent direct
	# define NAMLEN(dirent) (dirent)->d_namlen
	# if HAVE_SYS_NDIR_H
	# include <sys/ndir.h>
	# endif
	# if HAVE_SYS_DIR_H
	# include <sys/dir.h>
	# endif
	# if HAVE_NDIR_H
	# include <ndir.h>
	# endif
	#endif

	#include "assert.h"
	#include "crc32.h"
	#include "exitcodes.h"
	#include "global.h"
	#include "imap_err.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "mailbox.h"
	#include "message.h"
	#include "map.h"
	#include "retry.h"
	#include "seen.h"
	#include "util.h"
	#include "sequence.h"
	#include "xmalloc.h"

	static int sort_record(const void a, const void b)
	{
	struct index_record ra = (struct index_record )a;
	struct index_record rb = (struct index_record )b;
	return ra->uid - rb->uid;
	}

	static int update_record_from_cache(struct mailbox *mailbox,
	struct index_record *record)
	{
	int r;
	bit32 crc;

	r = mailbox_open_cache(mailbox);
	if (r) return r;

	if (!record->cache_offset)
	return IMAP_IOERROR;

	r = cache_parserecord(&mailbox->cache_buf,
	record->cache_offset, &record->crec);
	if (r) return r;

	crc = crc32_buf(cache_buf(record));
	if (record->cache_crc) {
	if (crc != record->cache_crc)
	return IMAP_MAILBOX_CRC;
	}
	else {
	record->cache_crc = crc;
	}

	/* extract the date for GMTIME field */
	if (cacheitem_size(record, CACHE_ENVELOPE) > 2) {
	char *envtokens[NUMENVTOKENS];
	char *tmpenv = xstrndup(cacheitem_base(record, CACHE_ENVELOPE) + 1,
	cacheitem_size(record, CACHE_ENVELOPE) - 2);
	parse_cached_envelope(tmpenv, envtokens, VECTOR_SIZE(envtokens));
	record->gmtime = message_parse_date(envtokens[ENV_DATE],
	PARSE_TIME\|PARSE_ZONE\|PARSE_NOCREATE\|PARSE_GMT);
	free(tmpenv);
	}
	else {
	/* better than nothing! */
	record->gmtime = record->internaldate;
	}

	return 0;
	}

	static int upgrade_index_record(struct mailbox *mailbox,
	const char *buf,
	int old_version,
	struct index_record *record,
	int record_size)
	{
	indexbuffer_t rbuf;
	char recordbuf = (char )rbuf.buf;
	int recalc = 0;
	struct utimbuf settime;
	const char *fname;

	memset(recordbuf, 0, INDEX_RECORD_SIZE);
	if (INDEX_RECORD_SIZE < record_size)
	memcpy(recordbuf, buf, INDEX_RECORD_SIZE);
	else
	memcpy(recordbuf, buf, record_size);

	/* CONTENT_LINES added with minor version 5 */
	/* CACHE_VERSION added with minor version 6 */

	/* 12-byte GUIDs added with minor version 7 */
	/* GUIDs extended from 12 to 20 bytes with minor version 10 */
	if (old_version < 10)
	recalc = 1;
	else {
	/* if it's all zeros for the final 8 bits, it was probably upgraded
	* and also needs recalculation */
	if (ntohl(((bit32 )(recordbuf+OFFSET_MESSAGE_GUID+12))) == 0 &&
	ntohl(((bit32 )(recordbuf+OFFSET_MESSAGE_GUID+16))) == 0)
	recalc = 1;
	}

	/* do the initial parse. Ignore the result, crc32 will mismatch
	* for sure */
	mailbox_buf_to_index_record(recordbuf, record);

	if (!recalc && old_version < 12) {
	/* let's try a cheaper upgrade option - just reading the
	cache record for details */
	if (update_record_from_cache(mailbox, record))
	recalc = 1;
	}

	fname = mailbox_message_fname(mailbox, record->uid);

	if (recalc) {
	int r = message_parse(fname, record);
	if (r) return r;
	}

	/* update the mtime to match the internaldate */
	settime.actime = settime.modtime = record->internaldate;
	if (utime(mailbox_message_fname(mailbox, record->uid), &settime) == -1)
	return IMAP_IOERROR;

	return 0;
	}

	/*
	* Upgrade an index/expunge file for 'mailbox'
	*/
	int upgrade_index(struct mailbox *mailbox)
	{
	uint32_t recno, erecno;
	unsigned long oldmapnum;
	unsigned long oldnum_records;
	unsigned long expunge_num = 0;
	unsigned uid = 0;
	bit32 oldminor_version, oldstart_offset, oldrecord_size;
	indexbuffer_t headerbuf;
	indexbuffer_t recordbuf;
	const char *bufp = NULL;
	char hbuf = (char )headerbuf.buf;
	char rbuf = (char )recordbuf.buf;
	int newindex_fd = -1;
	char *fname;
	struct seqset *seq = NULL;
	struct index_record record;
	struct index_record *expunge_records = NULL;
	struct index_record *recordptr;
	int r, n;

	if (mailbox->index_size < OFFSET_NUM_RECORDS)
	return IMAP_MAILBOX_BADFORMAT;

	oldminor_version = ntohl(((bit32 )(mailbox->index_base+OFFSET_MINOR_VERSION)));
	oldstart_offset = ntohl(((bit32 )(mailbox->index_base+OFFSET_START_OFFSET)));
	oldrecord_size = ntohl(((bit32 )(mailbox->index_base+OFFSET_RECORD_SIZE)));
	oldnum_records = ntohl(((bit32 )(mailbox->index_base+OFFSET_NUM_RECORDS)));
	oldmapnum = (mailbox->index_size - oldstart_offset) / oldrecord_size;
	if (oldmapnum < oldnum_records) {
	syslog(LOG_ERR, "upgrade: %s map doesn't fit, shrinking index %lu to %lu",
	mailbox->name, oldnum_records, oldmapnum);
	oldnum_records = oldmapnum;
	}

	/* check if someone else already upgraded the index! */
	if (oldminor_version == MAILBOX_MINOR_VERSION)
	goto done;

	/* Copy existing header so we can upgrade it */
	memset(hbuf, 0, INDEX_HEADER_SIZE);
	if (oldstart_offset > INDEX_HEADER_SIZE)
	memcpy(hbuf, mailbox->index_base, INDEX_HEADER_SIZE);
	else
	memcpy(hbuf, mailbox->index_base, oldstart_offset);

	/* QUOTA_MAILBOX_USED64 added with minor version 6 */
	if (oldminor_version < 6) {
	/* upgrade quota to 64-bits (bump existing fields) */
	memmove(hbuf+OFFSET_QUOTA_MAILBOX_USED, hbuf+OFFSET_QUOTA_MAILBOX_USED64,
	INDEX_HEADER_SIZE - OFFSET_QUOTA_MAILBOX_USED);
	/* zero the unused 32-bits */
	((bit32 )(hbuf+OFFSET_QUOTA_MAILBOX_USED64)) = htonl(0);
	}

	/* ignore the result - we EXPECT a CRC32 mismatch */
	mailbox_buf_to_index_header(&mailbox->i, hbuf);

	/* HIGHESTMODSEQ[_64] added with minor version 8 */
	if (oldminor_version < 8)
	mailbox->i.highestmodseq = 1;

	/* new version fields */
	mailbox->i.minor_version = MAILBOX_MINOR_VERSION;
	mailbox->i.start_offset = INDEX_HEADER_SIZE;
	mailbox->i.record_size = INDEX_RECORD_SIZE;

	/* upgrade other fields as necessary
	*
	* minor version wasn't updated religiously in the early days,
	* so we need to use the old offset instead */
	if (oldstart_offset < OFFSET_POP3_LAST_LOGIN)
	mailbox->i.pop3_last_login = 0;
	if (oldstart_offset < OFFSET_UIDVALIDITY)
	mailbox->i.uidvalidity = 1;
	if (oldstart_offset < OFFSET_MAILBOX_OPTIONS)
	mailbox->i.options = config_getint(IMAPOPT_MAILBOX_DEFAULT_OPTIONS);

	if (oldminor_version < 12) {
	struct seen *seendb;
	struct seendata sd;
	unsigned long erecno;
	unsigned long emapnum;
	bit32 eversion, eoffset, esize;
	char *owner_userid;
	struct stat sbuf;
	int expunge_fd = -1;
	const char *expunge_base = NULL;
	unsigned long expunge_len = 0; /* mapped size */

	/* remove the CONDSTORE option - it's implicit now */
	mailbox->i.options &= ~OPT_IMAP_CONDSTORE;

	if (mailbox->i.options & OPT_IMAP_SHAREDSEEN)
	owner_userid = "anyone";
	else
	owner_userid = mboxname_to_userid(mailbox->name);

	r = mailbox_read_header(mailbox, NULL);
	if (r) goto fail;

	/* NEW HEADER FIELDS */

	/* we'll set this if there are expunged records */
	mailbox->i.first_expunged = 0;
	/* we can't know about deletions before the current modseq */
	mailbox->i.deletedmodseq = mailbox->i.highestmodseq;
	/* we're repacking now! */
	mailbox->i.last_repack_time = time(NULL);
	/* bootstrap CRC matching */
	mailbox->i.header_file_crc = mailbox->header_file_crc;

	/* set up seen tracking for user inside the mailbox */
	if (!owner_userid) {
	r = IMAP_MAILBOX_NONEXISTENT;
	} else {
	r = seen_open(owner_userid, SEEN_SILENT, &seendb);
	if (!r) {
	r = seen_read(seendb, mailbox->uniqueid, &sd);
	seen_close(seendb);
	}
	}
	if (r) { /* no seen data? */
	mailbox->i.recentuid = mailbox->i.last_uid;
	mailbox->i.recenttime = time(NULL);
	}
	else {
	mailbox->i.recentuid = sd.lastuid;
	mailbox->i.recenttime = sd.lastchange;
	seq = seqset_parse(sd.seenuids, NULL, sd.lastuid);
	seen_freedata(&sd);
	}

	/* check for expunge */
	fname = mailbox_meta_fname(mailbox, META_EXPUNGE);
	expunge_fd = open(fname, O_RDWR, 0666);
	if (expunge_fd == -1) goto no_expunge;

	r = fstat(expunge_fd, &sbuf);
	if (r == -1) goto no_expunge;

	if (sbuf.st_size < INDEX_HEADER_SIZE) goto no_expunge;
	map_refresh(expunge_fd, 1, &expunge_base,
	&expunge_len, sbuf.st_size, "expunge",
	mailbox->name);

	/* use the expunge file's header information just in case
	* versions are skewed for some reason */
	eversion = ntohl(((bit32 )(expunge_base+OFFSET_MINOR_VERSION)));
	eoffset = ntohl(((bit32 )(expunge_base+OFFSET_START_OFFSET)));
	esize = ntohl(((bit32 )(expunge_base+OFFSET_RECORD_SIZE)));
	expunge_num = ntohl(((bit32 )(expunge_base+OFFSET_NUM_RECORDS)));
	expunge_records = xmalloc(expunge_num * sizeof(struct index_record));
	emapnum = (sbuf.st_size - eoffset) / esize;
	if (emapnum < expunge_num) {
	syslog(LOG_ERR, "IOERROR: %s map doesn't fit, shrinking expunge %lu to %lu",
	mailbox->name, expunge_num, emapnum);
	expunge_num = emapnum;
	}

	for (erecno = 1; erecno <= expunge_num; erecno++) {
	struct index_record *record = &expunge_records[erecno-1];
	bufp = expunge_base + eoffset + (erecno-1)*esize;
	upgrade_index_record(mailbox, bufp, eversion, record, esize);
	record->system_flags \|= FLAG_EXPUNGED;
	if (!mailbox->i.first_expunged \|\|
	mailbox->i.first_expunged > record->last_updated)
	mailbox->i.first_expunged = record->last_updated;
	}

	/* expunge files were not sorted. So sort them now for easier
	* interleaving */
	qsort(expunge_records, expunge_num,
	sizeof(struct index_record), &sort_record);

	no_expunge:
	if (expunge_fd != -1) close(expunge_fd);
	if (expunge_base) map_free(&expunge_base, &expunge_len);
	}

	/* update buffer with upgraded values */
	mailbox_index_header_to_buf(&mailbox->i, (unsigned char *)hbuf);

	/* open the new index file */
	fname = mailbox_meta_newfname(mailbox, META_INDEX);
	newindex_fd = open(fname, O_RDWR\|O_TRUNC\|O_CREAT, 0666);
	if (newindex_fd == -1) goto fail;

	/* Write new header - first pass only */
	n = retry_write(newindex_fd, hbuf, INDEX_HEADER_SIZE);
	if (n == -1) goto fail;

	/* initialise counters */
	mailbox->i.quota_mailbox_used = 0;
	mailbox->i.num_records = 0;
	mailbox->i.sync_crc = 0; /* no records is blank */
	mailbox->i.answered = 0;
	mailbox->i.deleted = 0;
	mailbox->i.flagged = 0;
	mailbox->i.exists = 0;

	/* Write the rest of new index */
	recno = 1;
	erecno = 1;
	while (recno <= oldnum_records \|\| erecno <= expunge_num) {
	/* read the uid */
	if (recno <= oldnum_records) {
	bufp = mailbox->index_base + oldstart_offset + (recno-1)*oldrecord_size;
	uid = ntohl(((bit32 )(bufp+OFFSET_UID)));
	}

	/* case: only expunge records left */
	if (recno > oldnum_records) {
	recordptr = &expunge_records[erecno-1];
	erecno++;
	}

	/* case: index record is lower uid */
	else if (erecno > expunge_num \|\| uid <= expunge_records[erecno-1].uid) {
	upgrade_index_record(mailbox, bufp, oldminor_version, &record,
	oldrecord_size);
	recno++;
	if (erecno <= expunge_num && uid == expunge_records[erecno-1].uid)
	erecno++; /* duplicate UID - skip expunge record */
	recordptr = &record;
	}

	/* case: expunge record is lower uid */
	else {
	recordptr = &expunge_records[erecno-1];
	erecno++;
	}

	if (oldminor_version < 12 && seqset_ismember(seq, recordptr->uid))
	recordptr->system_flags \|= FLAG_SEEN;

	/* write the cache record if necessary */
	r = mailbox_append_cache(mailbox, recordptr);
	if (r) goto fail;

	mailbox_index_update_counts(mailbox, recordptr, 1);
	mailbox_index_record_to_buf(recordptr, (unsigned char *)rbuf);

	n = retry_write(newindex_fd, rbuf, INDEX_RECORD_SIZE);
	if (n == -1) goto fail;

	mailbox->i.num_records++;
	}

	mailbox_index_header_to_buf(&mailbox->i, (unsigned char *)hbuf);

	lseek(newindex_fd, 0L, SEEK_SET);
	n = retry_write(newindex_fd, hbuf, INDEX_HEADER_SIZE);
	if (n == -1) goto fail;

	r = fsync(newindex_fd);
	if (r == -1) goto fail;
	close(newindex_fd);

	r = mailbox_meta_rename(mailbox, META_INDEX);
	if (r == -1) goto fail;

	/* don't need this file any more! */
	unlink(mailbox_meta_fname(mailbox, META_EXPUNGE));

	/* XXX - remove seen record */

	syslog(LOG_INFO, "Index upgrade: %s (%d -> %d)", mailbox->name,
	oldminor_version, MAILBOX_MINOR_VERSION);

	done:
	seqset_free(seq);
	free(expunge_records);

	/* commit the cache first so it doesn't stay "dirty" */
	mailbox_commit_cache(mailbox);

	/* special case, completely forgiven from being clean... */
	mailbox->i.dirty = 0;
	mailbox->quota_dirty = 0;
	mailbox->modseq_dirty = 0;

	/* it's definitely changed! */
	r = mailbox_open_index(mailbox);
	if (r) return r;

	return 0;

	fail:
	if (newindex_fd != -1) close(newindex_fd);
	seqset_free(seq);
	free(expunge_records);

	syslog(LOG_ERR, "Index upgrade failed: %s", mailbox->name);

	return IMAP_IOERROR;
	}

	diff --git a/imap/version.c b/imap/version.c
	index 50195414b..203fda59a 100644
	--- a/imap/version.c
	+++ b/imap/version.c
	@@ -1,205 +1,205 @@
	/* version.c: versioning functions
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: version.c,v 1.25 2010/02/03 16:37:42 murch Exp $
	*/

	#include <config.h>

	#include <sasl/sasl.h>
	#include <sys/utsname.h>
	#ifdef HAVE_BDB
	#include <db.h>
	#endif
	#ifdef HAVE_KRB
	#include <krb.h>
	#endif
	#ifdef HAVE_UCDSNMP
	#include <ucd-snmp/version.h>
	#endif

	#include <string.h>
	#include "../xversion.h"
	#include "version.h"
	#include "prot.h"
	#include "cyrusdb.h"
	#include "map.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "nonblock.h"
	#include "idle.h"

	#ifdef USE_SIEVE
	#include "sieve_interface.h"
	#endif

	static char id_resp_command[MAXIDVALUELEN];
	static char id_resp_arguments[MAXIDVALUELEN] = "";

	/* EXTRA_IDENT is a hack to add some version information for which compile
	* was used to build this version (at CMU, but we don't care what you do with
	* it).
	*/

	#ifdef EXTRA_IDENT
	#define CYRUS_VERSION _CYRUS_VERSION "-" EXTRA_IDENT
	#else
	#define CYRUS_VERSION _CYRUS_VERSION
	#endif

	const char *cyrus_version()
	{
	return CYRUS_VERSION;
	}

	/*
	* Grab the command line args for the ID response.
	*/
	void id_getcmdline(int argc, char **argv)
	{
	snprintf(id_resp_command, MAXIDVALUELEN, "%s", *argv);
	while (--argc > 0) {
	snprintf(id_resp_arguments + strlen(id_resp_arguments),
	MAXIDVALUELEN - strlen(id_resp_arguments),
	"%s%s", *++argv, (argc > 1) ? " " : "");
	}
	}

	/*
	* Output the ID response.
	* We do NOT close the parameter list so other stuff can be added later.
	*/
	void id_response(struct protstream *pout)
	{
	struct utsname os;
	const char *sasl_imp;
	int sasl_ver;
	char env_buf[MAXIDVALUELEN+1];

	prot_printf(pout, "* ID ("
	"\"name\" \"Cyrus IMAPD\""
	" \"version\" \"%s %s\""
	" \"vendor\" \"Project Cyrus\""
	" \"support-url\" \"http://cyrusimap.web.cmu.edu\"",
	CYRUS_VERSION, CYRUS_GITVERSION);

	/* add the os info */
	if (uname(&os) != -1)
	prot_printf(pout,
	" \"os\" \"%s\""
	" \"os-version\" \"%s\"",
	os.sysname, os.release);

	#ifdef ID_SAVE_CMDLINE
	/* add the command line info */
	prot_printf(pout, " \"command\" \"%s\"", id_resp_command);
	if (strlen(id_resp_arguments)) {
	prot_printf(pout, " \"arguments\" \"%s\"", id_resp_arguments);
	} else {
	prot_printf(pout, " \"arguments\" NIL");
	}
	#endif

	/* SASL information */
	snprintf(env_buf, MAXIDVALUELEN,"Built w/Cyrus SASL %d.%d.%d",
	SASL_VERSION_MAJOR, SASL_VERSION_MINOR, SASL_VERSION_STEP);

	sasl_version(&sasl_imp, &sasl_ver);
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; Running w/%s %d.%d.%d", sasl_imp,
	(sasl_ver & 0xFF000000) >> 24,
	(sasl_ver & 0x00FF0000) >> 16,
	(sasl_ver & 0x0000FFFF));

	/* add the environment info */
	#ifdef DB_VERSION_STRING
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; Built w/%s", DB_VERSION_STRING);
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; Running w/%s", db_version(NULL, NULL, NULL));
	#endif
	#ifdef HAVE_SSL
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; Built w/%s", OPENSSL_VERSION_TEXT);
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; Running w/%s", SSLeay_version(SSLEAY_VERSION));
	#ifdef EGD_SOCKET
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	" (with EGD)");
	#endif
	#endif
	#ifdef HAVE_ZLIB
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; Built w/zlib %s", ZLIB_VERSION);
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; Running w/zlib %s", zlibVersion());
	#endif
	#ifdef USE_SIEVE
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; %s", SIEVE_VERSION);
	#endif
	#ifdef HAVE_LIBWRAP
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; TCP Wrappers");
	#endif
	#ifdef HAVE_UCDSNMP
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; UCD-SNMP %s", VersionInfo);
	#endif
	#ifdef HAVE_NETSNMP
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; NET-SNMP");
	#endif
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; mmap = %s", map_method_desc);
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; lock = %s", lock_method_desc);
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; nonblock = %s", nonblock_method_desc);
	#ifdef HAVE_KRB
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	" (%s)", krb4_version);
	#endif
	if (idle_method_desc)
	snprintf(env_buf + strlen(env_buf), MAXIDVALUELEN - strlen(env_buf),
	"; idle = %s", idle_method_desc);

	prot_printf(pout, " \"environment\" \"%s\"", env_buf);
	}
	diff --git a/lib/Makefile.in b/lib/Makefile.in
	index ccf55ac01..e06acc419 100644
	--- a/lib/Makefile.in
	+++ b/lib/Makefile.in
	@@ -1,165 +1,165 @@
	# Makefile for cyrus library
	#
	# @configure_input@
	#
	# Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions
	# are met:
	#
	# 1. Redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer.
	#
	# 2. Redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in
	# the documentation and/or other materials provided with the
	# distribution.
	#
	# 3. The name "Carnegie Mellon University" must not be used to
	# endorse or promote products derived from this software without
	# prior written permission. For permission or any legal
	# details, please contact
	# Carnegie Mellon University
	# Center for Technology Transfer and Enterprise Creation
	# 4615 Forbes Avenue
	# Suite 302
	# Pittsburgh, PA 15213
	# (412) 268-7393, fax: (412) 268-7395
	# innovation@andrew.cmu.edu
	#
	# 4. Redistributions of any form whatsoever must retain the following
	# acknowledgment:
	# "This product includes software developed by Computing Services
	# at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	#
	# CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	# AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	# FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	# AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	# OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	#
	# $Id: Makefile.in,v 1.77 2010/06/28 12:06:31 brong Exp $

	# Authorization namespace.
	AUTH=auth.o auth_krb.o auth_unix.o auth_krb5.o auth_pts.o

	# ACL interpretation. Only one choice for now:
	ACL=acl_afs.o

	srcdir = @srcdir@
	top_srcdir = @top_srcdir@
	VPATH = @srcdir@

	CC = @CC@

	INSTALL = @INSTALL@
	RANLIB = @RANLIB@

	DEFS = @DEFS@ @LOCALDEFS@
	CPPFLAGS = -I.. @CPPFLAGS@ @COM_ERR_CPPFLAGS@ @SASLFLAGS@
	LIBS = @LIBS@

	MAKEDEPEND_CFLAGS = @CFLAGS@
	CFLAGS = @CFLAGS@ @PERL_CCCDLFLAGS@
	LDFLAGS = @LDFLAGS@

	SHELL = /bin/sh
	MAKEDEPEND = @MAKEDEPEND@

	prefix = @prefix@
	exec_prefix = @exec_prefix@
	cyrus_prefix = @cyrus_prefix@

	libdir = @libdir@

	BUILTSOURCES = imapopts.h imapopts.c

	LIBCYR_HDRS = $(srcdir)/acl.h $(srcdir)/assert.h $(srcdir)/auth.h \
	$(srcdir)/bsearch.h $(srcdir)/charset.h $(srcdir)/glob.h \
	$(srcdir)/gmtoff.h $(srcdir)/imclient.h $(srcdir)/imparse.h \
	- $(srcdir)/lock.h $(srcdir)/map.h $(srcdir)/mkgmtime.h \
	+ $(srcdir)/cyr_lock.h $(srcdir)/map.h $(srcdir)/mkgmtime.h \
	$(srcdir)/nonblock.h $(srcdir)/parseaddr.h $(srcdir)/prot.h \
	$(srcdir)/retry.h $(srcdir)/sysexits.h $(srcdir)/strhash.h \
	$(srcdir)/lsort.h $(srcdir)/stristr.h $(srcdir)/signals.h \
	$(srcdir)/util.h $(srcdir)/xstrlcpy.h $(srcdir)/xstrlcat.h \
	$(srcdir)/xmalloc.h $(srcdir)/imapurl.h \
	$(srcdir)/cyrusdb.h $(srcdir)/iptostring.h $(srcdir)/rfc822date.h \
	$(srcdir)/libcyr_cfg.h $(srcdir)/byteorder64.h \
	$(srcdir)/md5.h $(srcdir)/hmac-md5.h $(srcdir)/crc32.h

	LIBCYR_OBJS = acl.o bsearch.o charset.o glob.o retry.o util.o \
	libcyr_cfg.o mkgmtime.o prot.o parseaddr.o imclient.o imparse.o \
	lsort.o stristr.o rfc822date.o signals.o cyrusdb.o strhash.o \
	chartable.o imapurl.o nonblock_@WITH_NONBLOCK@.o lock_@WITH_LOCK@.o \
	gmtoff_@WITH_GMTOFF@.o map_@WITH_MAP@.o $(ACL) $(AUTH) \
	@LIBOBJS@ @CYRUSDB_OBJS@ @MD5OBJ@ \
	iptostring.o xmalloc.o wildmat.o byteorder64.o \
	xstrlcat.o xstrlcpy.o crc32.o

	LIBCYRM_HDRS = $(srcdir)/hash.h $(srcdir)/mpool.h $(srcdir)/xmalloc.h \
	$(srcdir)/xstrlcat.h $(srcdir)/xstrlcpy.h $(srcdir)/util.h \
	$(srcdir)/strhash.h $(srcdir)/libconfig.h $(srcdir)/assert.h \
	imapopts.h $(srcdir)/crc32.h
	LIBCYRM_OBJS = libconfig.o imapopts.o hash.o mpool.o xmalloc.o strhash.o \
	xstrlcat.o xstrlcpy.o assert.o util.o signals.o crc32.o @IPV6_OBJS@

	all: $(BUILTSOURCES) libcyrus_min.a libcyrus.a

	install:
	$(srcdir)/../install-sh -d $(DESTDIR)$(libdir)
	$(INSTALL) -m 644 libcyrus.a $(DESTDIR)$(libdir)
	$(INSTALL) -m 644 libcyrus_min.a $(DESTDIR)$(libdir)
	$(RANLIB) $(DESTDIR)$(libdir)/libcyrus.a
	for file in $(LIBCYR_HDRS); \
	do \
	$(INSTALL) -m 644 $$file $(DESTDIR)$(prefix)/include/cyrus \|\| exit 1; \
	done
	for file in $(LIBCYRM_HDRS); \
	do \
	$(INSTALL) -m 644 $$file $(DESTDIR)$(prefix)/include/cyrus \|\| exit 1; \
	done

	.c.o:
	$(CC) -c $(CPPFLAGS) $(DEFS) $(CFLAGS) \
	$<

	libcyrus.a: $(LIBCYR_OBJS)
	rm -f libcyrus.a
	ar cr libcyrus.a $(LIBCYR_OBJS)
	$(RANLIB) libcyrus.a

	libcyrus_min.a: $(LIBCYRM_OBJS)
	rm -f libcyrus_min.a
	ar cr libcyrus_min.a $(LIBCYRM_OBJS)
	$(RANLIB) libcyrus_min.a

	imapopts.c: imapoptions $(srcdir)/../tools/config2header
	$(srcdir)/../tools/config2header CC="$(CC)" $(srcdir)/imapopts.c $(srcdir)/imapopts.h < $(srcdir)/imapoptions

	imapopts.h: imapopts.c

	chartable.c: mkchartable.pl charset/*.t charset/unifix.txt charset/unidata5_2.txt
	@echo "### Building chartables..."
	rm -f chartable.c
	perl ./mkchartable.pl \
	-m $(srcdir)/charset/unifix.txt \
	-m $(srcdir)/charset/unidata5_2.txt \
	$(srcdir)/charset/*.t \
	> chartable.c \
	\|\| (rm -f chartable.c && exit 1)
	@echo "### Done building chartables."

	clean:
	rm -f .o .a chartable.c Makefile.bak makedepend.log \
	$(BUILTSOURCES)

	distclean: clean
	rm -f Makefile

	depend:
	${MAKEDEPEND} $(CPPFLAGS) $(DEFS) -I$(srcdir) $(MAKEDEPEND_CFLAGS) .c $(srcdir)/.c 1>makedepend.log 2>&1

	# DO NOT DELETE THIS LINE -- make depend depends on it.
	diff --git a/lib/auth_pts.c b/lib/auth_pts.c
	index 2e9ed6f7b..d6d587b16 100644
	--- a/lib/auth_pts.c
	+++ b/lib/auth_pts.c
	@@ -1,523 +1,523 @@
	/* auth_pts.c -- PTLOADER authorization
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: auth_pts.c,v 1.16 2010/01/06 17:01:44 murch Exp $
	*/

	#include <config.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <errno.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <fcntl.h>
	#include <sys/socket.h>
	#include <sys/un.h>
	#include <sys/uio.h>

	#include "auth.h"
	#include "auth_pts.h"
	#include "cyrusdb.h"
	#include "exitcodes.h"
	#include "libcyr_cfg.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "retry.h"
	#include "strhash.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"

	static char *canonuser_id = NULL;
	static struct auth_state *canonuser_cache = NULL;

	/* XXX should make this an imap option */
	#define PT_TIMEOUT_SEC 30

	#define TS_READ 1
	#define TS_WRITE 2
	#define TS_RW 3

	static int
	timeout_select (int sock, int op, int sec) {
	struct timeval tv;
	int r;
	fd_set rfds, wfds, rp, wp;

	FD_ZERO(&rfds);
	FD_ZERO(&wfds);
	rp = NULL;
	wp = NULL;

	switch (op) {
	case TS_READ:
	FD_SET(sock, &rfds);
	rp = &rfds;
	break;
	case TS_WRITE:
	FD_SET(sock, &wfds);
	wp = &wfds;
	break;
	case TS_RW:
	FD_SET(sock, &rfds);
	FD_SET(sock, &wfds);
	rp = &rfds;
	wp = &wfds;
	default: /* no action */
	break;
	}

	tv.tv_sec = sec;
	tv.tv_usec = 0;

	syslog(LOG_DEBUG, "timeout_select: sock = %d, rp = 0x%lx, wp = 0x%lx, sec = %d",
	sock, (unsigned long)rp, (unsigned long)wp, sec);

	if ((r = select(sock+1, rp, wp, NULL, &tv)) == 0) {
	/* r == 0 then timed out. we change this into an error */
	errno = ETIMEDOUT;
	r = -1;
	}

	syslog(LOG_DEBUG, "timeout_select exiting. r = %d; errno = %d", r, errno);
	return r;
	}


	static int
	nb_connect(int s, struct sockaddr *sa, socklen_t slen, int sec) {
	int flags, r, rc=0;

	if ((flags = fcntl(s, F_GETFL,0)) == -1) {
	syslog(LOG_ERR, "unable to get socket flags");
	return -1;
	}

	if (fcntl(s, F_SETFL, flags\|O_NONBLOCK) == -1) {
	syslog(LOG_ERR, "unable to set socket to NON_BLOCK");
	return -1;
	}

	if ((r = connect(s, sa, slen)) < 0) {
	if (errno != EINPROGRESS) {
	rc = -1;
	goto done;
	}
	} else {
	/* yay, it got through on the first shot. */
	syslog(LOG_DEBUG, "connected with no delay");
	rc = 0;
	goto done;
	}

	syslog(LOG_DEBUG, "didn't immediately connect. waiting...");

	if (timeout_select(s, TS_RW, sec) < 0) {
	syslog(LOG_ERR, "timeoutselect: %m");
	rc = -1;
	goto done;
	}

	syslog(LOG_DEBUG, "connect: connected in time.");
	rc = 0;

	done:
	/* set back to blocking so the reads/writes don't screw up), but why bother on an error... */
	if (!rc && (fcntl(s, F_SETFL, flags) == -1)) {
	syslog(LOG_ERR, "unable to set socket back to nonblocking: %m");
	rc = -1;
	}

	return rc;
	}

	/* Returns 0 on successful connection to ptloader/valid cache entry,
	* complete with allocated & filled in struct auth_state.
	*
	* state must be a NULL pointer when passed in */
	static int ptload(const char identifier,struct auth_state *state);
	static void myfreestate(struct auth_state *auth_state);


	/*
	* Determine if the user is a member of 'identifier'
	* Returns one of:
	* 0 User does not match identifier
	* 1 identifier matches everybody
	* 2 User is in the group that is identifier
	* 3 User is identifer
	*/
	static int mymemberof(struct auth_state *auth_state,
	const char *identifier)
	{
	int i;
	unsigned idhash = strhash(identifier);
	static unsigned anyonehash = 0;

	anyonehash = !anyonehash ? strhash("anyone") : anyonehash;

	if (!auth_state) {
	/* special case anonymous */
	if (!strcmp(identifier, "anyone")) return 1;
	else if (!strcmp(identifier, "anonymous")) return 3;

	/* "anonymous" is not a member of any group */
	else return 0;
	}

	/* is 'identifier' "anyone"? */
	if (idhash == anyonehash &&
	!strcmp(identifier, "anyone")) return 1;

	/* is 'identifier' me? */
	if (idhash == auth_state->userid.hash &&
	!strcmp(identifier, auth_state->userid.id)) return 3;

	/* is it a group i'm a member of ? */
	for (i=0; i < auth_state->ngroups; i++)
	if (idhash == auth_state->groups[i].hash &&
	!strcmp(identifier, auth_state->groups[i].id))
	return 2;

	return 0;
	}

	/*
	* Convert 'identifier' into canonical form.
	* Returns a pointer to a static buffer containing the canonical form
	* or NULL if 'identifier' is invalid.
	*/
	static char mycanonifyid(const char identifier,
	size_t len __attribute__((unused)))
	{
	static char retbuf[PTS_DB_KEYSIZE];

	if(canonuser_id &&
	(!strcmp(identifier, canonuser_id) \|\| !strcmp(identifier, retbuf))) {
	/* It's the currently cached user, return the previous result */
	return retbuf;
	} else if(canonuser_id) {
	/* We've got a new one, invalidate our cache */
	free(canonuser_id);
	myfreestate(canonuser_cache);

	canonuser_id = NULL;
	canonuser_cache = NULL;
	}

	if(!strcmp(identifier, "anyone") \|\|
	!strcmp(identifier, "anonymous")) {
	/* we can fill this in ourselves - no cacheing */
	strlcpy(retbuf, identifier, sizeof(retbuf));
	return retbuf;
	}

	if (!strcmp(identifier, "")) {
	syslog(LOG_ERR, "unable to canonify empty identifier");
	return NULL;
	}


	canonuser_cache = NULL;
	if(ptload(identifier, &canonuser_cache) < 0) {
	if (canonuser_cache == NULL) {
	syslog(LOG_ERR, "ptload completely failed: unable to canonify identifier: %s",
	identifier);
	return NULL;
	} else {
	syslog(LOG_ERR, "ptload failed: but canonified %s -> %s", identifier,
	canonuser_cache->userid.id);
	}
	}

	canonuser_id = xstrdup(identifier);
	strlcpy(retbuf, canonuser_cache->userid.id, sizeof(retbuf));
	syslog(LOG_DEBUG, "canonified %s -> %s", identifier, retbuf);
	return retbuf;
	}

	/*
	* Produce an auth_state structure for the given identifier
	*/
	static struct auth_state mynewstate(const char identifier)
	{
	struct auth_state *output = NULL;

	if(canonuser_id &&
	(!strcmp(identifier, canonuser_id) \|\|
	!strcmp(identifier, canonuser_cache->userid.id))) {
	/* It's the currently cached user, return the previous result */
	free(canonuser_id);
	canonuser_id = NULL;

	output = canonuser_cache;
	canonuser_cache = NULL;
	return output;
	}

	/*
	* If anyone or anonymous, just pass through. Otherwise, try to load the
	* groups the user is in
	*/
	if(strcmp(identifier, "anyone") &&
	strcmp(identifier, "anonymous")) {

	if(ptload(identifier, &output) < 0) {
	syslog(LOG_ERR, "ptload failed for %s", identifier);
	/* Allowing this to go through is a problem if negative group access is
	* used significantly. Allowing this to go through is a feature when
	* the ptserver is having problems and the user wants to get to his
	* inbox.
	*
	* note that even on a failure, output should either be NULL or a
	* correct (enough) value.
	*/
	}
	}

	if (output == NULL) {
	output =
	(struct auth_state *)xzmalloc(sizeof(struct auth_state));
	strlcpy(output->userid.id, identifier,
	sizeof(output->userid.id));
	output->userid.hash = strhash(identifier);
	syslog(LOG_DEBUG, "creating empty auth_state for %s", identifier);
	} else {
	syslog(LOG_DEBUG, "using ptloaded value of: %s", output->userid.id);
	}

	return output;
	}

	static struct cyrusdb_backend *the_ptscache_db = NULL;

	/* Returns 0 on success */
	static int ptload(const char identifier, struct auth_state *state)
	{
	struct auth_state *fetched = NULL;
	size_t id_len;
	const char *data = NULL;
	int dsize;
	char fnamebuf[1024];
	struct db *ptdb;
	int s;
	struct sockaddr_un srvaddr;
	int r, rc=0;
	static char response[1024];
	struct iovec iov[10];
	int niov, n;
	unsigned int start;
	const char *config_dir =
	libcyrus_config_getstring(CYRUSOPT_CONFIG_DIR);

	/* xxx this sucks, but it seems to be the only way to satisfy the linker */
	if(the_ptscache_db == NULL) {
	the_ptscache_db =
	cyrusdb_fromname(libcyrus_config_getstring(CYRUSOPT_PTSCACHE_DB));
	}

	if(!state \|\| *state) {
	fatal("bad state pointer passed to ptload()", EC_TEMPFAIL);
	}

	strcpy(fnamebuf, config_dir);
	strcat(fnamebuf, PTS_DBFIL);
	r = (the_ptscache_db->open)(fnamebuf, CYRUSDB_CREATE, &ptdb);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: opening %s: %s", fnamebuf,
	cyrusdb_strerror(ret));
	*state = NULL;
	return -1;
	}

	id_len = strlen(identifier);
	if(id_len > PTS_DB_KEYSIZE) {
	syslog(LOG_ERR, "identifier too long in auth_newstate");
	*state = NULL;
	return -1;
	}

	/* fetch the current record for the user */
	r = the_ptscache_db->fetch(ptdb, identifier, id_len,
	&data, &dsize, NULL);
	if (r && r != CYRUSDB_NOTFOUND) {
	syslog(LOG_ERR, "auth_newstate: error fetching record: %s",
	cyrusdb_strerror(r));

	rc = -1;
	goto done;
	}

	/* if it's expired (or nonexistant),
	* ask the ptloader to reload it and reread it */
	fetched = (struct auth_state *) data;

	if(fetched) {
	time_t now = time(NULL);
	int timeout = libcyrus_config_getint(CYRUSOPT_PTS_CACHE_TIMEOUT);

	syslog(LOG_DEBUG,
	"ptload(): fetched cache record (%s)" \
	"(mark %ld, current %ld, limit %ld)", identifier,
	fetched->mark, now, now - timeout);

	if (fetched->mark > (now - timeout)) {
	/* not expired; let's return it */
	goto done;
	}
	}

	syslog(LOG_DEBUG, "ptload(): pinging ptloader");

	s = socket(AF_UNIX, SOCK_STREAM, 0);
	if (s == -1) {
	syslog(LOG_ERR,
	"ptload(): unable to create socket for ptloader: %m");
	rc = -1;
	goto done;
	}

	if (libcyrus_config_getstring(CYRUSOPT_PTLOADER_SOCK))
	strcpy(fnamebuf, libcyrus_config_getstring(CYRUSOPT_PTLOADER_SOCK));
	else {
	strcpy(fnamebuf, config_dir);
	strcat(fnamebuf, PTS_DBSOCKET);
	}

	memset((char *)&srvaddr, 0, sizeof(srvaddr));
	srvaddr.sun_family = AF_UNIX;
	strcpy(srvaddr.sun_path, fnamebuf);
	r = nb_connect(s, (struct sockaddr *)&srvaddr, sizeof(srvaddr), PT_TIMEOUT_SEC);

	if (r == -1) {
	syslog(LOG_ERR, "ptload(): can't connect to ptloader server: %m");
	close(s);
	rc = -1;
	goto done;
	}

	syslog(LOG_DEBUG, "ptload(): connected");
	niov = 0;
	WRITEV_ADD_TO_IOVEC(iov, niov, (char *) &id_len, sizeof(id_len));
	WRITEV_ADD_TO_IOVEC(iov, niov, (char *) identifier, id_len);

	if (timeout_select(s, TS_WRITE, PT_TIMEOUT_SEC) < 0) {
	syslog(LOG_ERR, "timeoutselect: writing to ptloader %m");
	rc = -1;
	goto done;
	}
	retry_writev(s, iov, niov);
	syslog(LOG_DEBUG, "ptload sent data");

	start = 0;
	while (start < sizeof(response) - 1) {
	if (timeout_select(s, TS_READ, PT_TIMEOUT_SEC) < 0) {
	syslog(LOG_ERR, "timeout_select: reading from ptloader: %m");
	rc = -1;
	goto done;
	}
	n = read(s, response+start, sizeof(response) - 1 - start);
	if (n < 1) break;
	start += n;
	}

	close(s);
	syslog(LOG_DEBUG, "ptload read data back");

	if (start <= 1 \|\| strncmp(response, "OK", 2)) {
	if(start > 1) {
	syslog(LOG_ERR,
	"ptload(): bad response from ptloader server: %s", response);
	} else {
	syslog(LOG_ERR, "ptload(): empty response from ptloader server");
	}
	rc = -1;
	goto done;
	}

	/* fetch the current record for the user */
	r = the_ptscache_db->fetch(ptdb, identifier, id_len,
	&data, &dsize, NULL);
	if (r != 0 \|\| !data) {
	syslog(LOG_ERR, "ptload(): error fetching record: %s"
	"(did ptloader add the record?)",
	cyrusdb_strerror(r));
	data = NULL;
	rc = -1;
	goto done;
	}

	done:
	/* ok, we got real data, let's use it */
	if (data != NULL) {
	fetched = (struct auth_state *) data;
	}

	if (fetched == NULL) {
	*state = NULL;
	syslog(LOG_DEBUG, "No data available at all from ptload()");
	} else {
	/* copy it into our structure */
	state = (struct auth_state )xmalloc(dsize);
	memcpy(*state, fetched, dsize);
	syslog(LOG_DEBUG, "ptload returning data");
	}

	/* close and unlock the database */
	(the_ptscache_db->close)(ptdb);

	return rc;
	}

	static void myfreestate(struct auth_state *auth_state)
	{
	free(auth_state);
	}

	struct auth_mech auth_pts =
	{
	"pts", /* name */

	&mycanonifyid,
	&mymemberof,
	&mynewstate,
	&myfreestate,
	};
	diff --git a/lib/lock.h b/lib/cyr_lock.h
	similarity index 98%
	rename from lib/lock.h
	rename to lib/cyr_lock.h
	index d817353d1..127280976 100644
	--- a/lib/lock.h
	+++ b/lib/cyr_lock.h
	@@ -1,68 +1,68 @@
	-/* lock.h -- file locking primitives
	+/* cyr_lock.h -- file locking primitives
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: lock.h,v 1.9 2010/01/06 17:01:46 murch Exp $
	*/

	#ifndef INCLUDED_LOCK_H
	#define INCLUDED_LOCK_H

	#ifndef P
	#ifdef __STDC__
	#define P(x) x
	#else
	#define P(x) ()
	#endif
	#endif

	#include <sys/stat.h>

	extern const char *lock_method_desc;

	extern int lock_reopen P((int fd, const char *filename,
	struct stat sbuf, const char *failaction));

	extern int lock_blocking P((int fd));
	extern int lock_shared P((int fd));
	extern int lock_nonblocking P((int fd));
	extern int lock_unlock P((int fd));

	#endif /* INCLUDED_LOCK_H */
	diff --git a/lib/cyrusdb_flat.c b/lib/cyrusdb_flat.c
	index d33400691..0608ae8a8 100644
	--- a/lib/cyrusdb_flat.c
	+++ b/lib/cyrusdb_flat.c
	@@ -1,746 +1,746 @@
	/* cyrusdb_flat: a sorted flat textfile backend
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: cyrusdb_flat.c,v 1.40 2010/01/06 17:01:45 murch Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <stdlib.h>
	#include <syslog.h>
	#include <string.h>
	#include <errno.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/uio.h>
	#include <fcntl.h>

	#include "assert.h"
	#include "cyrusdb.h"
	#include "exitcodes.h"
	#include "map.h"
	#include "bsearch.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "retry.h"
	#include "util.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"

	/* we have the file locked iff we have an outstanding transaction */

	struct db {
	char *fname;

	int fd; /* current file open */
	ino_t ino;

	const char base; / contents of file */
	unsigned long size; /* actual size */
	unsigned long len; /* mapped size */
	};

	struct txn {
	char *fnamenew;
	int fd;
	};

	/* other routines call this one when they fail */
	static int abort_txn(struct db db, struct txn tid)
	{
	int r = CYRUSDB_OK;
	int rw = 0;
	struct stat sbuf;

	assert(db && tid);

	/* cleanup done while lock is held */
	if (tid->fnamenew) {
	unlink(tid->fnamenew);
	free(tid->fnamenew);
	rw = 1;
	}

	/* release lock */
	r = lock_unlock(db->fd);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: unlocking db %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	}

	if (rw) {
	/* return to our normally scheduled fd */
	if (!r && fstat(db->fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat on %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	}
	if (!r) {
	map_free(&db->base, &db->len);
	map_refresh(db->fd, 0, &db->base, &db->len, sbuf.st_size,
	db->fname, 0);
	db->size = sbuf.st_size;
	}
	}

	free(tid);

	return 0;
	}

	static void free_db(struct db *db)
	{
	if (db) {
	if (db->fname) free(db->fname);
	free(db);
	}
	}

	static struct txn *new_txn(void)
	{
	struct txn ret = (struct txn ) xmalloc(sizeof(struct txn));
	ret->fnamenew = NULL;
	ret->fd = 0;
	return ret;
	}

	static int init(const char *dbdir __attribute__((unused)),
	int myflags __attribute__((unused)))
	{
	return 0;
	}

	static int done(void)
	{
	return 0;
	}

	static int mysync(void)
	{
	return 0;
	}

	static int myarchive(const char *fnames, const char dirname)
	{
	int r, d_length, d_remain;
	const char **fname;
	char dstname[1024], *dp;

	strlcpy(dstname, dirname, sizeof(dstname));
	d_length = strlen(dstname);
	dp = dstname + d_length;
	d_remain = sizeof(dstname)-d_length;

	/* archive those files specified by the app */
	for (fname = fnames; *fname != NULL; ++fname) {
	syslog(LOG_DEBUG, "archiving database file: %s", *fname);
	strlcpy(dp, strrchr(*fname, '/'), d_remain);
	r = cyrusdb_copyfile(*fname, dstname);
	if (r) {
	syslog(LOG_ERR,
	"DBERROR: error archiving database file: %s", *fname);
	return CYRUSDB_IOERROR;
	}
	}

	return 0;
	}

	static int myopen(const char fname, int flags, struct db *ret)
	{
	struct db db = (struct db ) xzmalloc(sizeof(struct db));
	struct stat sbuf;

	assert(fname && ret);

	db->fd = open(fname, O_RDWR, 0644);
	if (db->fd == -1 && errno == ENOENT && (flags & CYRUSDB_CREATE)) {
	if (cyrus_mkdir(fname, 0755) == -1) return CYRUSDB_IOERROR;

	db->fd = open(fname, O_RDWR \| O_CREAT, 0644);
	}

	if (db->fd == -1) {
	int level = (flags & CYRUSDB_CREATE) ? LOG_ERR : LOG_DEBUG;
	syslog(level, "IOERROR: opening %s: %m", fname);
	free_db(db);
	return CYRUSDB_IOERROR;
	}

	if (fstat(db->fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat on %s: %m", fname);
	close(db->fd);
	free_db(db);
	return CYRUSDB_IOERROR;
	}
	db->ino = sbuf.st_ino;

	map_refresh(db->fd, 0, &db->base, &db->len, sbuf.st_size,
	fname, 0);
	db->size = sbuf.st_size;

	db->fname = xstrdup(fname);

	*ret = db;
	return 0;
	}

	static int myclose(struct db *db)
	{
	assert(db);

	map_free(&db->base, &db->len);
	close(db->fd);
	free_db(db);

	return 0;
	}

	static int starttxn_or_refetch(struct db db, struct txn *mytid)
	{
	int r = 0;
	struct stat sbuf;

	assert(db);

	if (mytid && !*mytid) {
	const char *lockfailaction;

	/* start txn; grab lock */

	r = lock_reopen(db->fd, db->fname, &sbuf, &lockfailaction);
	if (r < 0) {
	syslog(LOG_ERR, "IOERROR: %s %s: %m", lockfailaction, db->fname);
	return CYRUSDB_IOERROR;
	}
	*mytid = new_txn();

	if (db->ino != sbuf.st_ino) {
	map_free(&db->base, &db->len);
	}
	map_refresh(db->fd, 0, &db->base, &db->len, sbuf.st_size,
	db->fname, 0);

	/* we now have the latest & greatest open */
	db->size = sbuf.st_size;
	db->ino = sbuf.st_ino;
	}

	if (!mytid) {
	/* no txn, but let's try to be reasonably up-to-date */

	if (stat(db->fname, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: stating flat %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}

	if (sbuf.st_ino != db->ino) {
	/* reopen */
	int newfd = open(db->fname, O_RDWR);

	if (newfd == -1) {
	/* fail! */
	syslog(LOG_ERR, "couldn't reopen %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}
	dup2(newfd, db->fd);
	close(newfd);
	if (stat(db->fname, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: stating flat %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}

	db->ino = sbuf.st_ino;
	map_free(&db->base, &db->len);
	}
	map_refresh(db->fd, 0, &db->base, &db->len,
	sbuf.st_size, db->fname, 0);
	db->size = sbuf.st_size;
	}

	return 0;
	}

	static int myfetch(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid)
	{
	int r = 0;
	int offset;
	unsigned long len;

	assert(db);

	if (data) *data = NULL;
	if (datalen) *datalen = 0;

	r = starttxn_or_refetch(db, mytid);
	if (r) return r;

	offset = bsearch_mem(key, 1, db->base, db->size, 0, &len);
	if (len) {
	if (data) *data = db->base + offset + keylen + 1;
	/* subtract one for \t, and one for the \n */
	if (data) *datalen = len - keylen - 2;
	} else {
	r = CYRUSDB_NOTFOUND;
	}

	return r;
	}

	static int fetch(struct db *mydb,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid)
	{
	return myfetch(mydb, key, keylen, data, datalen, mytid);
	}

	static int fetchlock(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid)
	{
	return myfetch(db, key, keylen, data, datalen, mytid);
	}

	#define GETENTRY(p) \
	key = p; \
	data = strchr(key, '\t'); \
	\
	if (!data) { \
	/* huh, might be corrupted? */ \
	r = CYRUSDB_IOERROR; \
	break; \
	} \
	keylen = data - key; \
	data++; /* skip of the \t */ \
	\
	dataend = strchr(data, '\n'); \
	if (!dataend) { \
	/* huh, might be corrupted? */ \
	r = CYRUSDB_IOERROR; \
	break; \
	} \
	datalen = dataend - data;

	static int foreach(struct db *db,
	char *prefix, int prefixlen,
	foreach_p *goodp,
	foreach_cb cb, void rock,
	struct txn **mytid)
	{
	int r = CYRUSDB_OK;
	int offset;
	unsigned long len;
	const char p, pend;

	/* for use inside the loop, but we need the values to be retained
	* from loop to loop */
	const char *key = NULL;
	size_t keylen = 0;
	const char data = NULL, dataend = NULL;
	size_t datalen = 0;
	int dontmove = 0;

	/* For when we have a transaction running */
	char *savebuf = NULL;
	size_t savebuflen = 0;
	size_t savebufsize = 0;

	/* for the local iteration so that the db can change out from under us */
	const char *dbbase = NULL;
	unsigned long dblen = 0;
	int dbfd = -1;

	r = starttxn_or_refetch(db, mytid);
	if (r) return r;

	if(!mytid) {
	/* No transaction, use the fast method to avoid stomping on our
	* memory map if changes happen */
	dbfd = dup(db->fd);
	if(dbfd == -1) return CYRUSDB_IOERROR;

	map_refresh(dbfd, 1, &dbbase, &dblen, db->size, db->fname, 0);

	/* drop our read lock on the file, since we don't really care
	* if it gets replaced out from under us, our mmap stays on the
	* old version */
	lock_unlock(db->fd);
	} else {
	/* use the same variables as in the no transaction case, just to
	* get things set up */
	dbbase = db->base;
	dblen = db->len;
	}

	if (prefix) {
	char *realprefix;
	if(prefix[prefixlen] != '\0') {
	realprefix = xmalloc(prefixlen+1);
	memcpy(realprefix, prefix, prefixlen);
	realprefix[prefixlen] = '\0';
	} else {
	realprefix = prefix;
	}
	offset = bsearch_mem(realprefix, 1, dbbase, db->size, 0, &len);

	if(prefix[prefixlen] != '\0') free(realprefix);
	} else {
	offset = 0;
	}

	p = dbbase + offset;
	pend = dbbase + db->size;

	while (p < pend) {
	if(!dontmove) {
	GETENTRY(p)
	}
	else dontmove = 0;

	/* does it still match prefix? */
	if (keylen < (size_t) prefixlen) break;
	if (prefixlen && memcmp(key, prefix, prefixlen)) break;

	if (!goodp \|\| goodp(rock, key, keylen, data, datalen)) {
	unsigned long ino = db->ino;
	unsigned long sz = db->size;

	if(mytid) {
	/* transaction present, this means we do the slow way */
	if (!savebuf \|\| keylen > savebuflen) {
	int dblsize = 2 * savebuflen;
	int addsize = keylen + 32;

	savebuflen = (dblsize > addsize) ? dblsize : addsize;
	savebuf = xrealloc(savebuf, savebuflen);
	}
	memcpy(savebuf, key, keylen);
	savebuf[keylen] = '\0';
	savebufsize = keylen;
	}

	/* make callback */
	r = cb(rock, key, keylen, data, datalen);
	if (r) break;

	if(mytid) {
	/* reposition? (we made a change) */
	if (!(ino == db->ino && sz == db->size)) {
	/* something changed in the file; reseek */
	offset = bsearch_mem(savebuf, 1, db->base, db->size,
	0, &len);
	p = db->base + offset;

	GETENTRY(p);

	/* 'key' might not equal 'savebuf'. if it's different,
	we want to stay where we are. if it's the same, we
	should move on to the next one */
	if (savebufsize == keylen &&
	!memcmp(savebuf, key, savebufsize)) {
	p = dataend + 1;
	} else {
	/* 'savebuf' got deleted, so we're now pointing at the
	right thing */
	dontmove = 1;
	}
	}
	}
	}

	p = dataend + 1;
	}

	if(!mytid) {
	/* cleanup the fast method */
	map_free(&dbbase, &dblen);
	close(dbfd);
	} else if(savebuf) {
	free(savebuf);
	}

	return r;
	}

	#undef GETENTRY

	static int mystore(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **mytid, int overwrite)
	{
	int r = 0;
	char fnamebuf[1024];
	int offset;
	unsigned long len;
	const char *lockfailaction;
	int writefd;
	struct iovec iov[10];
	int niov;
	struct stat sbuf;
	char *tmpkey = NULL;

	/* lock file, if needed */
	if (!mytid \|\| !*mytid) {
	r = lock_reopen(db->fd, db->fname, &sbuf, &lockfailaction);
	if (r < 0) {
	syslog(LOG_ERR, "IOERROR: %s %s: %m", lockfailaction, db->fname);
	return CYRUSDB_IOERROR;
	}

	if (sbuf.st_ino != db->ino) {
	db->ino = sbuf.st_ino;
	map_free(&db->base, &db->len);
	map_refresh(db->fd, 0, &db->base, &db->len,
	sbuf.st_size, db->fname, 0);
	db->size = sbuf.st_size;
	}

	if (mytid) {
	*mytid = new_txn();
	}
	}

	/* if we need to truncate the key, do so */
	if(key[keylen] != '\0') {
	tmpkey = xmalloc(keylen + 1);
	memcpy(tmpkey, key, keylen);
	tmpkey[keylen] = '\0';
	key = tmpkey;
	}

	/* find entry, if it exists */
	offset = bsearch_mem(key, 1, db->base, db->size, 0, &len);

	/* overwrite? */
	if (len && !overwrite) {
	if (mytid) abort_txn(db, *mytid);
	if (tmpkey) free(tmpkey);
	return CYRUSDB_EXISTS;
	}

	/* write new file */
	if (mytid && (*mytid)->fnamenew) {
	strlcpy(fnamebuf, (*mytid)->fnamenew, sizeof(fnamebuf));
	} else {
	strlcpy(fnamebuf, db->fname, sizeof(fnamebuf));
	strlcat(fnamebuf, ".NEW", sizeof(fnamebuf));
	}

	unlink(fnamebuf);
	r = writefd = open(fnamebuf, O_RDWR \| O_CREAT, 0666);
	if (r < 0) {
	syslog(LOG_ERR, "opening %s for writing failed: %m", fnamebuf);
	if (mytid) abort_txn(db, *mytid);
	if (tmpkey) free(tmpkey);
	return CYRUSDB_IOERROR;
	}

	niov = 0;
	if (offset) {
	WRITEV_ADD_TO_IOVEC(iov, niov, (char *) db->base, offset);
	}

	if (data) {
	/* new entry */
	WRITEV_ADD_TO_IOVEC(iov, niov, (char *) key, keylen);
	WRITEV_ADD_TO_IOVEC(iov, niov, "\t", 1);
	WRITEV_ADD_TO_IOVEC(iov, niov, (char *) data, datalen);
	WRITEV_ADD_TO_IOVEC(iov, niov, "\n", 1);
	}

	if (db->size - (offset + len) > 0) {
	WRITEV_ADD_TO_IOVEC(iov, niov, (char *) db->base + offset + len,
	db->size - (offset + len));
	}

	/* do the write */
	r = retry_writev(writefd, iov, niov);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", fnamebuf);
	close(writefd);
	if (mytid) abort_txn(db, *mytid);
	return CYRUSDB_IOERROR;
	}
	r = 0;

	if (mytid) {
	/* setup so further accesses will be against fname.NEW */
	if (fstat(writefd, &sbuf) == -1) {
	/* xxx ? */
	}

	if (!(mytid)->fnamenew) (mytid)->fnamenew = xstrdup(fnamebuf);
	if ((mytid)->fd) close((mytid)->fd);
	(*mytid)->fd = writefd;
	map_free(&db->base, &db->len);
	map_refresh(writefd, 0, &db->base, &db->len, sbuf.st_size,
	fnamebuf, 0);
	db->size = sbuf.st_size;
	} else {
	/* commit immediately */
	if (fsync(writefd) \|\|
	fstat(writefd, &sbuf) == -1 \|\|
	rename(fnamebuf, db->fname) == -1) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", fnamebuf);
	close(writefd);
	if (tmpkey) free(tmpkey);
	return CYRUSDB_IOERROR;
	}

	close(db->fd);
	db->fd = writefd;

	/* release lock */
	r = lock_unlock(db->fd);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: unlocking db %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	}

	db->ino = sbuf.st_ino;
	map_free(&db->base, &db->len);
	map_refresh(writefd, 0, &db->base, &db->len, sbuf.st_size,
	db->fname, 0);
	db->size = sbuf.st_size;
	}

	if(tmpkey) free(tmpkey);

	return r;
	}

	static int create(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 0);
	}

	static int store(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 1);
	}

	static int delete(struct db *db,
	const char *key, int keylen,
	struct txn **mytid, int force __attribute__((unused)))
	{
	return mystore(db, key, keylen, NULL, 0, mytid, 1);
	}

	static int commit_txn(struct db db, struct txn tid)
	{
	int writefd;
	int r = 0;
	struct stat sbuf;

	assert(db && tid);

	if (tid->fnamenew) {
	/* we wrote something */

	writefd = tid->fd;
	if (fsync(writefd) \|\|
	fstat(writefd, &sbuf) == -1 \|\|
	rename(tid->fnamenew, db->fname) == -1) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", tid->fnamenew);
	close(writefd);
	r = CYRUSDB_IOERROR;
	} else {
	/* successful */
	/* we now deal exclusively with our new fd */
	close(db->fd);
	db->fd = writefd;
	db->ino = sbuf.st_ino;
	}
	free(tid->fnamenew);
	} else {
	/* read-only txn */
	/* release lock */
	r = lock_unlock(db->fd);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: unlocking db %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	}
	}

	free(tid);
	return r;
	}

	struct cyrusdb_backend cyrusdb_flat =
	{
	"flat", /* name */

	&init,
	&done,
	&mysync,
	&myarchive,

	&myopen,
	&myclose,

	&fetch,
	&fetchlock,
	&foreach,
	&create,
	&store,
	&delete,

	&commit_txn,
	&abort_txn,

	NULL,
	NULL
	};
	diff --git a/lib/cyrusdb_quotalegacy.c b/lib/cyrusdb_quotalegacy.c
	index 504df7a18..c03ea49b7 100644
	--- a/lib/cyrusdb_quotalegacy.c
	+++ b/lib/cyrusdb_quotalegacy.c
	@@ -1,882 +1,882 @@
	/* cyrusdb_quotalegacy: cyrusdb backend for accessing legacy quota files
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: cyrusdb_quotalegacy.c,v 1.21 2010/01/06 17:01:45 murch Exp $
	*/

	#include <config.h>

	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <stdio.h>
	#include <stdlib.h>
	#include <syslog.h>
	#include <string.h>
	#include <errno.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/uio.h>
	#include <fcntl.h>

	#if HAVE_DIRENT_H
	# include <dirent.h>
	# define NAMLEN(dirent) strlen((dirent)->d_name)
	#else
	# define dirent direct
	# define NAMLEN(dirent) (dirent)->d_namlen
	# if HAVE_SYS_NDIR_H
	# include <sys/ndir.h>
	# endif
	# if HAVE_SYS_DIR_H
	# include <sys/dir.h>
	# endif
	# if HAVE_NDIR_H
	# include <ndir.h>
	# endif
	#endif

	#include "assert.h"
	#include "cyrusdb.h"
	#include "exitcodes.h"
	#include "hash.h"
	#include "map.h"
	#include "libcyr_cfg.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "retry.h"
	#include "util.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"

	#define FNAME_QUOTADIR "/quota/"
	#define MAX_QUOTA_PATH 4096

	/* we have the file locked iff we have an outstanding transaction */

	struct subtxn {
	int fd;

	char *fnamenew;
	int fdnew;

	int delete;
	};

	struct txn {
	hash_table table; /* hash table of sub-transactions */

	int (proc)(char , struct subtxn ); / commit/abort procedure */

	int result; /* final result of the commit/abort */
	};

	struct db {
	char *path;

	char data; / allocated buffer for fetched data */

	struct txn txn; /* transaction associated with this db handle */
	};

	static int abort_txn(struct db db __attribute__((unused)), struct txn tid);


	/* simple hash so it's easy to find these things in the filesystem;
	our human time is worth more than efficiency */
	static void hash_quota(char buf, size_t size, const char qr, char *path)
	{
	int config_virtdomains = libcyrus_config_getswitch(CYRUSOPT_VIRTDOMAINS);
	int config_fulldirhash = libcyrus_config_getswitch(CYRUSOPT_FULLDIRHASH);
	const char *idx;
	char c, *p;
	unsigned len;

	if ((len = snprintf(buf, size, "%s", path)) >= size) {
	fatal("insufficient buffer size in hash_quota", EC_TEMPFAIL);
	}
	buf += len;
	size -= len;

	if (config_virtdomains && (p = strchr(qr, '!'))) {
	p = '\0'; / split domain!qr */
	c = (char) dir_hash_c(qr, config_fulldirhash);
	if ((len = snprintf(buf, size, "%s%c/%s",
	FNAME_DOMAINDIR, c, qr)) >= size) {
	fatal("insufficient buffer size in hash_quota", EC_TEMPFAIL);
	}
	p++ = '!'; / reassemble domain!qr */
	qr = p;
	buf += len;
	size -= len;

	if (!*qr) {
	/* quota for entire domain */
	if (snprintf(buf, size, "%sroot", FNAME_QUOTADIR) >= (int) size) {
	fatal("insufficient buffer size in hash_quota",
	EC_TEMPFAIL);
	}
	return;
	}
	}

	idx = strchr(qr, '.'); /* skip past user. */
	if (idx == NULL) {
	idx = qr;
	} else {
	idx++;
	}
	c = (char) dir_hash_c(idx, config_fulldirhash);

	if (snprintf(buf, size, "%s%c/%s", FNAME_QUOTADIR, c, qr) >= (int) size) {
	fatal("insufficient buffer size in hash_quota", EC_TEMPFAIL);
	}
	}

	/* other routines call this one when they fail */
	static int abort_subtxn(char fname, struct subtxn tid)
	{
	int r = CYRUSDB_OK;

	assert(fname && tid);

	/* cleanup done while lock is held */
	if (tid->fnamenew) {
	unlink(tid->fnamenew);
	free(tid->fnamenew);
	}

	if (tid->fdnew != -1) {
	r = close(tid->fdnew);
	}

	if (tid->fd != -1) {
	/* release lock */
	r = lock_unlock(tid->fd);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: unlocking %s: %m", fname);
	r = CYRUSDB_IOERROR;
	}

	/* close */
	r = close(tid->fd);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: closing %s: %m", fname);
	r = CYRUSDB_IOERROR;
	}
	}

	free(tid);

	return r;
	}

	static int commit_subtxn(char fname, struct subtxn tid)
	{
	int writefd;
	int r = 0;
	struct stat sbuf;

	assert(fname && tid);

	if ((writefd = tid->fdnew) != -1) {
	/* we wrote something */

	if (fsync(writefd) \|\|
	fstat(writefd, &sbuf) == -1 \|\|
	rename(tid->fnamenew, fname) == -1 \|\|
	lock_unlock(writefd) == -1) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", tid->fnamenew);
	r = CYRUSDB_IOERROR;
	}
	close(writefd);
	free(tid->fnamenew);
	} else if (tid->delete) {
	/* delete file */
	r = unlink(fname);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: unlinking %s: %m", fname);
	r = CYRUSDB_IOERROR;
	}
	} else {
	/* read-only txn */
	}

	/* release lock */
	if (tid->fd != -1) {
	r = lock_unlock(tid->fd);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: unlocking %s: %m", fname);
	r = CYRUSDB_IOERROR;
	}

	r = close(tid->fd);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: closing %s: %m", fname);
	r = CYRUSDB_IOERROR;
	}
	}

	free(tid);

	return r;
	}

	static void free_db(struct db *db)
	{
	if (db) {
	if (db->path) free(db->path);
	if (db->data) free(db->data);
	free_hash_table(&db->txn.table, NULL);
	free(db);
	}
	}

	static struct subtxn new_subtxn(const char fname __attribute__((unused)),
	int fd)
	{
	struct subtxn ret = (struct subtxn ) xmalloc(sizeof(struct subtxn));

	ret->fd = fd;
	ret->fnamenew = NULL;
	ret->fdnew = -1;
	ret->delete = 0;
	return ret;
	}

	static int init(const char *dbdir __attribute__((unused)),
	int myflags __attribute__((unused)))
	{
	return 0;
	}

	static int done(void)
	{
	return 0;
	}

	static int mysync(void)
	{
	return 0;
	}

	static int myarchive(const char **fnames __attribute__((unused)),
	const char *dirname __attribute__((unused)))
	{
	return 0;
	}

	static int myopen(const char fname, int flags, struct db *ret)
	{
	struct db db = (struct db ) xzmalloc(sizeof(struct db));
	struct stat sbuf;
	char *p;
	int r;

	assert(fname && ret);

	db->path = xstrdup(fname);
	construct_hash_table(&db->txn.table, 200, 0);

	/* strip any filename from the path */
	if ((p = strrchr(db->path, '/'))) *p = '\0';

	r = stat(db->path, &sbuf);
	if (r == -1 && errno == ENOENT && (flags & CYRUSDB_CREATE)) {
	if (cyrus_mkdir(fname, 0755) != -1) {
	r = stat(db->path, &sbuf);
	}
	}

	if (r == -1) {
	int level = (flags & CYRUSDB_CREATE) ? LOG_ERR : LOG_DEBUG;
	syslog(level, "IOERROR: stating quota %s: %m", db->path);
	free_db(db);
	return CYRUSDB_IOERROR;
	}

	*ret = db;
	return 0;
	}

	static int myclose(struct db *db)
	{
	assert(db);

	free_db(db);

	return 0;
	}

	static int myfetch(struct db db, char quota_path,
	const char *data, int datalen,
	struct txn **tid)
	{
	struct subtxn *mytid = NULL;
	int quota_fd;
	const char *quota_base = 0;
	unsigned long quota_len = 0;

	assert(db);

	if (data) *data = NULL;
	if (datalen) *datalen = 0;

	if (!data \|\| !datalen) {
	/* just check if the key exists */
	struct stat sbuf;

	if (stat(quota_path, &sbuf) == -1)
	return CYRUSDB_NOTFOUND;

	return 0;
	}

	if (tid) {
	if (!*tid)
	*tid = &db->txn;
	else
	mytid = (struct subtxn *) hash_lookup(quota_path, &db->txn.table);
	}

	/* open and lock file, if needed */
	if (!mytid) {
	quota_fd = open(quota_path, O_RDWR, 0);
	if (quota_fd == -1) {
	if (errno == ENOENT) {
	/* key doesn't exist */
	return CYRUSDB_NOTFOUND;
	}

	syslog(LOG_ERR, "IOERROR: opening quota file %s: %m", quota_path);
	return CYRUSDB_IOERROR;
	}

	if (tid) {
	int r;
	struct stat sbuf;
	const char *lockfailaction;

	r = lock_reopen(quota_fd, quota_path, &sbuf, &lockfailaction);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: %s quota %s: %m", lockfailaction,
	quota_path);
	return CYRUSDB_IOERROR;
	}

	mytid = new_subtxn(quota_path, quota_fd);
	hash_insert(quota_path, mytid, &db->txn.table);
	}
	}
	else
	quota_fd = mytid->fd;

	map_refresh(quota_fd, 1, &quota_base, &quota_len,
	MAP_UNKNOWN_LEN, quota_path, 0);

	if (quota_len) {
	char p, eol;

	db->data = xrealloc(db->data, quota_len);
	memcpy(db->data, quota_base, quota_len);

	p = db->data;
	eol = memchr(p, '\n', quota_len - (p - db->data));
	if (!eol) {
	map_free(&quota_base, &quota_len);
	return CYRUSDB_IOERROR;
	}
	/* convert the separating \n to SP */
	*eol = ' ';

	p = eol + 1;
	eol = memchr(p, '\n', quota_len - (p - db->data));
	if (!eol) {
	map_free(&quota_base, &quota_len);
	return CYRUSDB_IOERROR;
	}
	/* convert the terminating \n to \0 */
	*eol = '\0';

	*data = db->data;
	*datalen = strlen(db->data);
	}

	map_free(&quota_base, &quota_len);
	if (!tid) close(quota_fd);

	return 0;
	}

	static int fetch(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	char quota_path[MAX_QUOTA_PATH+1], *tmpkey = NULL;

	/* if we need to truncate the key, do so */
	if (key[keylen] != '\0') {
	tmpkey = xmalloc(keylen + 1);
	memcpy(tmpkey, key, keylen);
	tmpkey[keylen] = '\0';
	key = tmpkey;
	}

	hash_quota(quota_path, sizeof(quota_path), key, db->path);
	if (tmpkey) free(tmpkey);

	return myfetch(db, quota_path, data, datalen, tid);
	}

	static const char path_to_qr(const char path, char *buf)
	{
	const char *qr;
	char *p;

	qr = strrchr(path, '/') + 1;
	if ((p = strstr(path, FNAME_DOMAINDIR))) {
	/* use the quota_path as a buffer to construct virtdomain qr */
	p += strlen(FNAME_DOMAINDIR) + 2; /* +2 for hashdir */
	sprintf(buf, "%.*s!%s", (int) strcspn(p, "/"), p,
	strcmp(qr, "root") ? qr : "");
	qr = buf;
	}

	return qr;
	}

	static int compar_qr(const void v1, const void v2)
	{
	const char qr1, qr2;
	char qrbuf1[MAX_QUOTA_PATH+1], qrbuf2[MAX_QUOTA_PATH+1];

	qr1 = path_to_qr(((const char *) v1), qrbuf1);
	qr2 = path_to_qr(((const char *) v2), qrbuf2);

	return strcmp(qr1, qr2);
	}

	#define PATH_ALLOC 100
	struct qr_path {
	char **path;
	size_t count;
	size_t alloc;
	};

	static void scan_qr_dir(char quota_path, char prefix, struct qr_path *pathbuf)
	{
	int config_fulldirhash = libcyrus_config_getswitch(CYRUSOPT_FULLDIRHASH);
	int config_virtdomains = libcyrus_config_getswitch(CYRUSOPT_VIRTDOMAINS);
	char *endp;
	int c, i;
	DIR *qrdir;
	struct dirent *next = NULL;

	/* strip off the qr specific path */
	endp = strstr(quota_path, FNAME_QUOTADIR) + strlen(FNAME_QUOTADIR);
	strcpy(endp, "?/");

	c = config_fulldirhash ? 'A' : 'a';
	for (i = 0; i < 26; i++, c++) {
	*endp = c;

	qrdir = opendir(quota_path);

	if (qrdir) {
	while ((next = readdir(qrdir)) != NULL) {
	if (!strcmp(next->d_name, ".")
	\|\| !strcmp(next->d_name, "..")) continue;

	if (!strncmp(next->d_name, prefix, strlen(prefix))) {
	if (pathbuf->count == pathbuf->alloc) {
	pathbuf->alloc += PATH_ALLOC;
	pathbuf->path = xrealloc(pathbuf->path,
	pathbuf->alloc * sizeof(char *));
	}
	pathbuf->path[pathbuf->count] = xmalloc(MAX_QUOTA_PATH+1);
	sprintf(pathbuf->path[pathbuf->count++],
	"%s%s", quota_path, next->d_name);
	}
	}

	closedir(qrdir);
	}
	}

	if (config_virtdomains && !strlen(prefix) &&
	strstr(quota_path, FNAME_DOMAINDIR)) {
	/* search for a domain quota */
	struct stat buf;

	strcpy(endp, "root");

	if (!stat(quota_path, &buf)) {
	if (pathbuf->count == pathbuf->alloc) {
	pathbuf->alloc += PATH_ALLOC;
	pathbuf->path = xrealloc(pathbuf->path,
	pathbuf->alloc * sizeof(char *));
	}
	pathbuf->path[pathbuf->count] = xmalloc(MAX_QUOTA_PATH+1);
	sprintf(pathbuf->path[pathbuf->count++],
	"%s", quota_path);
	}
	}
	}

	static int foreach(struct db *db,
	char *prefix, int prefixlen,
	foreach_p *goodp,
	foreach_cb cb, void rock,
	struct txn **tid)
	{
	int r = CYRUSDB_OK;
	int config_fulldirhash = libcyrus_config_getswitch(CYRUSOPT_FULLDIRHASH);
	int config_virtdomains = libcyrus_config_getswitch(CYRUSOPT_VIRTDOMAINS);
	char quota_path[MAX_QUOTA_PATH+1];
	struct qr_path pathbuf;
	size_t i;
	char tmpprefix = NULL, p = NULL;

	/* if we need to truncate the prefix, do so */
	if (prefix[prefixlen] != '\0') {
	tmpprefix = xmalloc(prefixlen + 1);
	memcpy(tmpprefix, prefix, prefixlen);
	tmpprefix[prefixlen] = '\0';
	prefix = tmpprefix;
	}

	hash_quota(quota_path, sizeof(quota_path), prefix, db->path);
	if (config_virtdomains && (p = strchr(prefix, '!')))
	prefix = p + 1;

	/* search for the quotaroots */
	memset(&pathbuf, 0, sizeof(struct qr_path));
	scan_qr_dir(quota_path, prefix, &pathbuf);

	if (config_virtdomains && !prefixlen) {
	/* search for all virtdomain quotaroots */
	char *endp;
	int c, i, n;
	DIR *qrdir;
	struct dirent *next = NULL;

	n = snprintf(quota_path, sizeof(quota_path), "%s%s",
	db->path, FNAME_DOMAINDIR);

	endp = quota_path + n;
	strcpy(endp, "?/");

	c = config_fulldirhash ? 'A' : 'a';
	for (i = 0; i < 26; i++, c++) {
	*endp = c;

	qrdir = opendir(quota_path);

	if (qrdir) {
	while ((next = readdir(qrdir)) != NULL) {
	if (!strcmp(next->d_name, ".")
	\|\| !strcmp(next->d_name, "..")) continue;

	snprintf(endp+2, sizeof(quota_path) - (n+2),
	"%s%s", next->d_name, FNAME_QUOTADIR);
	scan_qr_dir(quota_path, "", &pathbuf);
	}

	closedir(qrdir);
	}
	}
	}

	if (tmpprefix) free(tmpprefix);

	if (tid && !tid) tid = &db->txn;

	/* sort the quotaroots (ignoring paths) */
	qsort(pathbuf.path, pathbuf.count, sizeof(char *), &compar_qr);

	for (i = 0; i < pathbuf.count; i++) {
	const char data, key;
	int keylen, datalen;

	r = myfetch(db, pathbuf.path[i], &data, &datalen, tid);
	if (r) break;

	key = path_to_qr(pathbuf.path[i], quota_path);
	keylen = strlen(key);

	free(pathbuf.path[i]);

	if (!goodp \|\| goodp(rock, key, keylen, data, datalen)) {
	/* make callback */
	r = cb(rock, key, keylen, data, datalen);
	if (r) break;
	}
	}

	free(pathbuf.path);

	return r;
	}

	static int mystore(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid, int overwrite)
	{
	char quota_path[MAX_QUOTA_PATH+1], *tmpkey = NULL;
	struct subtxn *mytid = NULL;
	int r = 0;

	/* if we need to truncate the key, do so */
	tmpkey = xmalloc(keylen + 1);
	memcpy(tmpkey, key, keylen);
	tmpkey[keylen] = '\0';

	hash_quota(quota_path, sizeof(quota_path), tmpkey, db->path);
	if (tmpkey) free(tmpkey);

	if (tid) {
	if (!*tid)
	*tid = &db->txn;
	else
	mytid = (struct subtxn *) hash_lookup(quota_path, &db->txn.table);
	}

	/* open and lock file, if needed */
	if (!mytid) {
	int fd;
	struct stat sbuf;
	const char *lockfailaction;

	fd = open(quota_path, O_RDWR, 0644);
	if (fd == -1 && errno == ENOENT && data) {
	if (cyrus_mkdir(quota_path, 0755) != -1) {
	fd = open(quota_path, O_RDWR \| O_CREAT, 0644);
	}
	}
	if (fd == -1 && (errno != ENOENT \|\| data)) {
	syslog(LOG_ERR, "IOERROR: opening quota file %s: %m", quota_path);
	return CYRUSDB_IOERROR;
	}

	if (fd != -1) {
	r = lock_reopen(fd, quota_path, &sbuf, &lockfailaction);
	if (r == -1) {
	syslog(LOG_ERR, "IOERROR: %s quota %s: %m", lockfailaction,
	quota_path);
	return CYRUSDB_IOERROR;
	}
	}

	mytid = new_subtxn(quota_path, fd);

	if (tid) hash_insert(quota_path, mytid, &db->txn.table);
	}

	if (!data) {
	mytid->delete = 1;
	}
	else {
	char new_quota_path[MAX_QUOTA_PATH+1], buf, p;
	int newfd = -1, r1 = 0;
	ssize_t n;

	if (mytid->fd != -1 && !overwrite) {
	if (tid)
	abort_txn(db, *tid);
	else
	abort_subtxn(quota_path, mytid);
	return CYRUSDB_EXISTS;
	}

	if (mytid->fdnew == -1) {
	strlcpy(new_quota_path, quota_path, sizeof(new_quota_path));
	strlcat(new_quota_path, ".NEW", sizeof(new_quota_path));

	unlink(new_quota_path);
	newfd = open(new_quota_path, O_CREAT \| O_TRUNC \| O_RDWR, 0666);
	if (newfd == -1 && errno == ENOENT) {
	if (cyrus_mkdir(new_quota_path, 0755) != -1)
	newfd = open(new_quota_path, O_CREAT \| O_TRUNC \| O_RDWR, 0666);
	}
	if (newfd == -1) {
	syslog(LOG_ERR, "IOERROR: creating quota file %s: %m",
	new_quota_path);
	if (tid)
	abort_txn(db, *tid);
	else
	abort_subtxn(quota_path, mytid);
	return CYRUSDB_IOERROR;
	}

	mytid->fdnew = newfd;
	r = lock_blocking(newfd);
	if (r) {
	syslog(LOG_ERR, "IOERROR: locking quota file %s: %m",
	new_quota_path);
	if (tid)
	abort_txn(db, *tid);
	else
	abort_subtxn(quota_path, mytid);
	return CYRUSDB_IOERROR;
	}
	}

	buf = xmalloc(datalen+1);
	memcpy(buf, data, datalen);
	/* convert separating SP to \n */
	p = memchr(buf, ' ', datalen);
	*p = '\n';
	/* add a terminating \n */
	buf[datalen] = '\n';

	lseek(mytid->fdnew, 0, SEEK_SET);
	n = write(mytid->fdnew, buf, datalen+1);
	if (n == datalen+1) r1 = ftruncate(mytid->fdnew, datalen+1);
	free(buf);

	if (n != datalen+1 \|\| r1 == -1) {
	if (n == -1 \|\| r1 == -1)
	syslog(LOG_ERR, "IOERROR: writing quota file %s: %m",
	new_quota_path);
	else
	syslog(LOG_ERR,
	"IOERROR: writing quota file %s: failed to write %d bytes",
	new_quota_path, datalen+1);
	if (tid)
	abort_txn(db, *tid);
	else
	abort_subtxn(quota_path, mytid);
	return CYRUSDB_IOERROR;
	}

	if (!mytid->fnamenew)
	mytid->fnamenew = xstrdup(new_quota_path);
	}

	if (!tid) {
	/* commit immediately */
	r = commit_subtxn(quota_path, mytid);
	}

	return r;
	}

	static int create(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 0);
	}

	static int store(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 1);
	}

	static int delete(struct db *db,
	const char *key, int keylen,
	struct txn **mytid, int force __attribute__((unused)))
	{
	return mystore(db, key, keylen, NULL, 0, mytid, 1);
	}

	static void txn_proc(char fname, void data, void *rock)
	{
	struct txn tid = (struct txn ) rock;
	int r;

	r = tid->proc(fname, (struct subtxn *) data);
	hash_del(fname, &tid->table);

	if (r && !tid->result) tid->result = r;
	}

	static int commit_txn(struct db db __attribute__((unused)), struct txn tid)
	{
	tid->proc = commit_subtxn;
	tid->result = 0;

	hash_enumerate(&tid->table, txn_proc, tid);

	return tid->result;
	}

	static int abort_txn(struct db db __attribute__((unused)), struct txn tid)
	{
	tid->proc = abort_subtxn;
	tid->result = 0;

	hash_enumerate(&tid->table, txn_proc, tid);

	return tid->result;
	}

	struct cyrusdb_backend cyrusdb_quotalegacy =
	{
	"quotalegacy", /* name */

	&init,
	&done,
	&mysync,
	&myarchive,

	&myopen,
	&myclose,

	&fetch,
	&fetch,
	&foreach,
	&create,
	&store,
	&delete,

	&commit_txn,
	&abort_txn,

	NULL,
	NULL
	};
	diff --git a/lib/cyrusdb_skiplist.c b/lib/cyrusdb_skiplist.c
	index f73f0ce02..345621276 100644
	--- a/lib/cyrusdb_skiplist.c
	+++ b/lib/cyrusdb_skiplist.c
	@@ -1,2412 +1,2412 @@
	/* cyrusdb_skiplist.c -- cyrusdb skiplist implementation
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: cyrusdb_skiplist.c,v 1.71 2010/01/06 17:01:45 murch Exp $
	*/

	/* xxx check retry_xxx for failure */

	/* xxx all offsets should be uint32_ts i think */

	#include <config.h>

	#include <stdlib.h>
	#include <string.h>
	#include <limits.h>
	#include <errno.h>
	#include <syslog.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <fcntl.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#include <netinet/in.h>

	#include "assert.h"
	#include "bsearch.h"
	#include "cyrusdb.h"
	#include "libcyr_cfg.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "map.h"
	#include "retry.h"
	#include "util.h"
	#include "xmalloc.h"
	#include "xstrlcpy.h"
	#include "xstrlcat.h"

	#define PROB (0.5)

	/*
	*
	* disk format; all numbers in network byte order
	*
	* there's the data file, consisting of the
	* multiple records of "key", "data", and "skip pointers", where skip
	* pointers are the record number of the data pointer.
	*
	* on startup, recovery is performed. the last known good data file
	* is taken and the intent log is replayed on it. the index file is
	* regenerated from scratch.
	*
	* during operation ckecpoints will compress the data. the data file
	* is locked. then a checkpoint rewrites the data file in order,
	* removing any unused records. this is written and fsync'd to
	* dfile.NEW and stored for use during recovery.
	*/

	/*
	header "skiplist file\0\0\0"
	version (4 bytes)
	version_minor (4 bytes)
	maxlevel (4 bytes)
	curlevel (4 bytes)
	listsize (4 bytes)
	in active items
	log start (4 bytes)
	offset where log records start, used mainly to tell when to compress
	last recovery (4 bytes)
	seconds since unix epoch

	1 or more skipnodes, one of:

	record type (4 bytes) [DUMMY, INORDER, ADD]
	key size (4 bytes)
	key string (bit string, rounded to up to 4 byte multiples w/ 0s)
	data size (4 bytes)
	data string (bit string, rounded to up to 4 byte multiples w/ 0s)
	skip pointers (4 bytes each)
	least to most
	padding (4 bytes, must be -1)

	record type (4 bytes) [DELETE]
	record ptr (4 bytes; record to be deleted)

	record type (4 bytes) [COMMIT]


	record type is either
	DUMMY (first node is of this type)
	INORDER
	ADD
	DELETE
	COMMIT (commit the previous records)
	*/

	enum {
	INORDER = 1,
	ADD = 2,
	DELETE = 4,
	COMMIT = 255,
	DUMMY = 257
	};

	enum {
	UNLOCKED = 0,
	READLOCKED = 1,
	WRITELOCKED = 2,
	};

	struct txn {
	int syncfd;

	/* logstart is where we start changes from on commit, where we truncate
	to on abort */
	unsigned logstart;
	unsigned logend; /* where to write to continue this txn */
	};

	struct db {
	/* file data */
	char *fname;
	int fd;

	const char *map_base;
	unsigned long map_len; /* mapped size */
	unsigned long map_size; /* actual size */
	ino_t map_ino;

	/* header info */
	uint32_t version;
	uint32_t version_minor;
	uint32_t maxlevel;
	uint32_t curlevel;
	uint32_t listsize;
	uint32_t logstart; /* where the log starts from last chkpnt */
	time_t last_recovery;

	/* tracking info */
	int lock_status;
	int is_open;
	struct txn *current_txn;

	/* comparator function to use for sorting */
	int (compar) (const char s1, int l1, const char *s2, int l2);
	};

	struct db_list {
	struct db *db;
	struct db_list *next;
	int refcount;
	};

	static time_t global_recovery = 0;
	static struct db_list *open_db = NULL;

	/* Perform an FSYNC/FDATASYNC if we are not operating in UNSAFE mode */
	#define DO_FSYNC (!libcyrus_config_getswitch(CYRUSOPT_SKIPLIST_UNSAFE))

	enum {
	be_paranoid = 0,
	use_osync = 0
	};

	static int compare(const char s1, int l1, const char s2, int l2);

	static void getsyncfd(struct db db, struct txn t)
	{
	if (!use_osync) {
	t->syncfd = db->fd;
	} else if (t->syncfd == -1) {
	t->syncfd = open(db->fname, O_RDWR \| O_DSYNC, 0666);
	assert(t->syncfd != -1); /* xxx do better error recovery */
	}
	}

	static void closesyncfd(struct db *db __attribute__((unused)),
	struct txn *t)
	{
	/* if we're using fsync, then we don't want to close the file */
	if (use_osync && (t->syncfd != -1)) {
	close(t->syncfd);
	}
	t->syncfd = -1;
	}

	static int myinit(const char *dbdir, int myflags)
	{
	char sfile[1024];
	int fd, r = 0;
	uint32_t net32_time;

	snprintf(sfile, sizeof(sfile), "%s/skipstamp", dbdir);

	if (myflags & CYRUSDB_RECOVER) {
	/* set the recovery timestamp; all databases earlier than this
	time need recovery run when opened */

	global_recovery = time(NULL);
	fd = open(sfile, O_RDWR \| O_CREAT, 0644);
	if (fd == -1) r = -1;

	if (r != -1) r = ftruncate(fd, 0);
	net32_time = htonl(global_recovery);
	if (r != -1) r = write(fd, &net32_time, 4);
	if (r != -1) r = close(fd);

	if (r == -1) {
	syslog(LOG_ERR, "DBERROR: writing %s: %m", sfile);
	if (fd != -1) close(fd);
	return CYRUSDB_IOERROR;
	}
	} else {
	/* read the global recovery timestamp */

	fd = open(sfile, O_RDONLY, 0644);
	if (fd == -1) r = -1;
	if (r != -1) r = read(fd, &net32_time, 4);
	if (r != -1) r = close(fd);

	if (r == -1) {
	syslog(LOG_ERR, "DBERROR: reading %s, assuming the worst: %m",
	sfile);
	global_recovery = 0;
	} else {
	global_recovery = ntohl(net32_time);
	}
	}

	srand(time(NULL) * getpid());

	open_db = NULL;

	return 0;
	}

	static int mydone(void)
	{
	return 0;
	}

	static int mysync(void)
	{
	return 0;
	}

	static int myarchive(const char *fnames, const char dirname)
	{
	int r;
	const char **fname;
	char dstname[1024], *dp;
	int length, rest;

	strlcpy(dstname, dirname, sizeof(dstname));
	length = strlen(dstname);
	dp = dstname + length;
	rest = sizeof(dstname) - length;

	/* archive those files specified by the app */
	for (fname = fnames; *fname != NULL; ++fname) {
	syslog(LOG_DEBUG, "archiving database file: %s", *fname);
	strlcpy(dp, strrchr(*fname, '/'), rest);
	r = cyrusdb_copyfile(*fname, dstname);
	if (r) {
	syslog(LOG_ERR,
	"DBERROR: error archiving database file: %s", *fname);
	return CYRUSDB_IOERROR;
	}
	}

	return 0;
	}

	enum {
	SKIPLIST_VERSION = 1,
	SKIPLIST_VERSION_MINOR = 2,
	SKIPLIST_MAXLEVEL = 20,
	SKIPLIST_MINREWRITE = 16834 /* don't rewrite logs smaller than this */
	};

	#define HEADER_MAGIC ("\241\002\213\015skiplist file\0\0\0")
	#define HEADER_MAGIC_SIZE (20)

	/* offsets of header files */
	enum {
	OFFSET_HEADER = 0,
	OFFSET_VERSION = 20,
	OFFSET_VERSION_MINOR = 24,
	OFFSET_MAXLEVEL = 28,
	OFFSET_CURLEVEL = 32,
	OFFSET_LISTSIZE = 36,
	OFFSET_LOGSTART = 40,
	OFFSET_LASTRECOVERY = 44
	};

	enum {
	HEADER_SIZE = OFFSET_LASTRECOVERY + 4
	};

	static int mycommit(struct db db, struct txn tid);
	static int myabort(struct db db, struct txn tid);
	static int mycheckpoint(struct db *db, int locked);
	static int myconsistent(struct db db, struct txn tid, int locked);
	static int recovery(struct db *db, int flags);

	enum {
	/* Force recovery regardless of timestamp on database */
	RECOVERY_FORCE = 1,
	/* Caller already has a write lock on the database. In the case
	* of successful recovery, the database will still be locked on return.
	*
	* If the recovery fails, then the database will be unlocked an an
	* error will be returned */
	RECOVERY_CALLER_LOCKED = 2
	};

	/* file looks like:
	struct header {
	...
	}
	struct dummy {
	uint32_t t = htonl(DUMMY);
	uint32_t ks = 0;
	uint32_t ds = 0;
	uint32_t forward[db->maxlevel];
	uint32_t pad = -1;
	} */
	#define DUMMY_OFFSET(db) (HEADER_SIZE)
	#define DUMMY_PTR(db) ((db)->map_base + HEADER_SIZE)
	#define DUMMY_SIZE(db) (4 * (3 + db->maxlevel + 1))

	/* bump to the next multiple of 4 bytes */
	#define ROUNDUP(num) (((num) + 3) & 0xFFFFFFFC)

	#define TYPE(ptr) (ntohl(((uint32_t )(ptr))))
	#define KEY(ptr) ((ptr) + 8)
	#define KEYLEN(ptr) (ntohl(((uint32_t )((ptr) + 4))))
	#define DATA(ptr) ((ptr) + 8 + ROUNDUP(KEYLEN(ptr)) + 4)
	#define DATALEN(ptr) (ntohl(((uint32_t )((ptr) + 8 + ROUNDUP(KEYLEN(ptr))))))
	#define FIRSTPTR(ptr) ((ptr) + 8 + ROUNDUP(KEYLEN(ptr)) + 4 + ROUNDUP(DATALEN(ptr)))

	/* return a pointer to the pointer */
	#define PTR(ptr, x) (FIRSTPTR(ptr) + 4 * (x))

	/* FORWARD(ptr, x)
	* given a pointer to the start of the record, return the offset
	* corresponding to the xth pointer
	*/
	#define FORWARD(ptr, x) (ntohl(((uint32_t )(FIRSTPTR(ptr) + 4 * (x)))))

	/* how many levels does this record have? */
	static unsigned LEVEL(const char *ptr)
	{
	const uint32_t p, q;

	assert(TYPE(ptr) == DUMMY \|\| TYPE(ptr) == INORDER \|\| TYPE(ptr) == ADD);
	p = q = (uint32_t *) FIRSTPTR(ptr);
	while (*p != (uint32_t)-1) p++;
	return (p - q);
	}

	/* how big is this record? */
	static unsigned RECSIZE(const char *ptr)
	{
	int ret = 0;
	switch (TYPE(ptr)) {
	case DUMMY:
	case INORDER:
	case ADD:
	ret += 4; /* tag */
	ret += 4; /* keylen */
	ret += ROUNDUP(KEYLEN(ptr)); /* key */
	ret += 4; /* datalen */
	ret += ROUNDUP(DATALEN(ptr)); /* data */
	ret += 4 * LEVEL(ptr); /* pointers */
	ret += 4; /* padding */
	break;

	case DELETE:
	ret += 8;
	break;

	case COMMIT:
	ret += 4;
	break;
	}

	return ret;
	}

	/* Determine if it is safe to append to this skiplist database.
	* e.g. does it end in 4 bytes of -1 followed by a commit record?
	* or does it end with 'DELETE' + 4 bytes + a commit record?
	* or is this the beginning of the log, in which case we only need
	* the padding from the last INORDER (or DUMMY) record
	*/
	static int SAFE_TO_APPEND(struct db *db)
	{
	/* check it's a multiple of 4 */
	if (db->map_size % 4) return 1;

	/* is it the beginning of the log? */
	if (db->map_size == db->logstart) {
	if (((uint32_t )(db->map_base + db->map_size - 4)) != htonl(-1)) {
	return 1;
	}
	}

	/* in the middle of the log somewhere */
	else {
	if (((uint32_t )(db->map_base + db->map_size - 4)) != htonl(COMMIT)) {
	return 1;
	}

	/* if it's not an end of a record or a delete */
	if (!((((uint32_t )(db->map_base + db->map_size - 8)) == htonl(-1)) \|\|
	(((uint32_t )(db->map_base + db->map_size -12)) == htonl(DELETE)))) {
	return 1;
	}
	}

	return 0;
	}

	static int newtxn(struct db db, struct txn *tidptr)
	{
	struct txn *tid;
	/* is this file safe to append to?
	*
	* If it isn't, we need to run recovery. */
	if (SAFE_TO_APPEND(db)) {
	int r = recovery(db, RECOVERY_FORCE \| RECOVERY_CALLER_LOCKED);
	if (r) return r;
	}

	/* create the transaction */
	tid = xmalloc(sizeof(struct txn));
	tid->syncfd = -1;
	tid->logstart = db->map_size;
	/* assert(t->logstart != -1);*/
	tid->logend = tid->logstart;
	db->current_txn = tid;

	/* pass it back out */
	*tidptr = tid;

	return 0;
	}


	#define PADDING(ptr) (ntohl(((uint32_t )((ptr) + RECSIZE(ptr) - 4))))

	/* given an open, mapped db, read in the header information */
	static int read_header(struct db *db)
	{
	const char *dptr;
	int r;

	assert(db && db->map_len && db->fname && db->map_base
	&& db->is_open && db->lock_status);
	if (db->map_len < HEADER_SIZE) {
	syslog(LOG_ERR,
	"skiplist: file not large enough for header: %s", db->fname);
	}

	if (memcmp(db->map_base, HEADER_MAGIC, HEADER_MAGIC_SIZE)) {
	syslog(LOG_ERR, "skiplist: invalid magic header: %s", db->fname);
	return CYRUSDB_IOERROR;
	}

	db->version = ntohl(((uint32_t )(db->map_base + OFFSET_VERSION)));
	db->version_minor =
	ntohl(((uint32_t )(db->map_base + OFFSET_VERSION_MINOR)));
	if (db->version != SKIPLIST_VERSION) {
	syslog(LOG_ERR, "skiplist: version mismatch: %s has version %d.%d",
	db->fname, db->version, db->version_minor);
	return CYRUSDB_IOERROR;
	}

	db->maxlevel = ntohl(((uint32_t )(db->map_base + OFFSET_MAXLEVEL)));

	if (db->maxlevel > SKIPLIST_MAXLEVEL) {
	syslog(LOG_ERR,
	"skiplist %s: MAXLEVEL %d in database beyond maximum %d\n",
	db->fname, db->maxlevel, SKIPLIST_MAXLEVEL);
	return CYRUSDB_IOERROR;
	}

	db->curlevel = ntohl(((uint32_t )(db->map_base + OFFSET_CURLEVEL)));

	if (db->curlevel > db->maxlevel) {
	syslog(LOG_ERR,
	"skiplist %s: CURLEVEL %d in database beyond maximum %d\n",
	db->fname, db->curlevel, db->maxlevel);
	return CYRUSDB_IOERROR;
	}

	db->listsize = ntohl(((uint32_t )(db->map_base + OFFSET_LISTSIZE)));
	db->logstart = ntohl(((uint32_t )(db->map_base + OFFSET_LOGSTART)));
	db->last_recovery =
	ntohl(((uint32_t )(db->map_base + OFFSET_LASTRECOVERY)));

	/* verify dummy node */
	dptr = DUMMY_PTR(db);
	r = 0;

	if (!r && TYPE(dptr) != DUMMY) {
	syslog(LOG_ERR, "DBERROR: %s: first node not type DUMMY",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	if (!r && KEYLEN(dptr) != 0) {
	syslog(LOG_ERR, "DBERROR: %s: DUMMY has non-zero KEYLEN",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	if (!r && DATALEN(dptr) != 0) {
	syslog(LOG_ERR, "DBERROR: %s: DUMMY has non-zero DATALEN",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	if (!r && LEVEL(dptr) != db->maxlevel) {
	syslog(LOG_ERR, "DBERROR: %s: DUMMY level(%d) != db->maxlevel(%d)",
	db->fname, LEVEL(dptr), db->maxlevel);
	r = CYRUSDB_IOERROR;
	}

	return r;
	}

	/* given an open, mapped db, locked db,
	write the header information */
	static int write_header(struct db *db)
	{
	char buf[HEADER_SIZE];
	int n;

	assert (db->lock_status == WRITELOCKED);
	memcpy(buf + 0, HEADER_MAGIC, HEADER_MAGIC_SIZE);
	((uint32_t )(buf + OFFSET_VERSION)) = htonl(db->version);
	((uint32_t )(buf + OFFSET_VERSION_MINOR)) = htonl(db->version_minor);
	((uint32_t )(buf + OFFSET_MAXLEVEL)) = htonl(db->maxlevel);
	((uint32_t )(buf + OFFSET_CURLEVEL)) = htonl(db->curlevel);
	((uint32_t )(buf + OFFSET_LISTSIZE)) = htonl(db->listsize);
	((uint32_t )(buf + OFFSET_LOGSTART)) = htonl(db->logstart);
	((uint32_t )(buf + OFFSET_LASTRECOVERY)) = htonl(db->last_recovery);

	/* write it out */
	lseek(db->fd, 0, SEEK_SET);
	n = retry_write(db->fd, buf, HEADER_SIZE);
	if (n != HEADER_SIZE) {
	syslog(LOG_ERR, "DBERROR: writing skiplist header for %s: %m",
	db->fname);
	return CYRUSDB_IOERROR;
	}

	return 0;
	}

	/* make sure our mmap() is big enough */
	static int update_lock(struct db db, struct txn txn)
	{
	/* txn->logend is the current size of the file */
	assert (db->is_open && db->lock_status == WRITELOCKED);
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, txn->logend,
	db->fname, 0);
	db->map_size = txn->logend;

	return 0;
	}

	static int write_lock(struct db db, const char altname)
	{
	struct stat sbuf;
	const char *lockfailaction;
	const char *fname = altname ? altname : db->fname;

	assert(db->lock_status == UNLOCKED);
	if (lock_reopen(db->fd, fname, &sbuf, &lockfailaction) < 0) {
	syslog(LOG_ERR, "IOERROR: %s %s: %m", lockfailaction, fname);
	return CYRUSDB_IOERROR;
	}
	if (db->map_ino != sbuf.st_ino) {
	map_free(&db->map_base, &db->map_len);
	}
	db->map_size = sbuf.st_size;
	db->map_ino = sbuf.st_ino;
	db->lock_status = WRITELOCKED;

	map_refresh(db->fd, 0, &db->map_base, &db->map_len, sbuf.st_size,
	fname, 0);

	if (db->is_open) {
	/* reread header */
	read_header(db);
	}

	/* printf("%d: write lock: %d\n", getpid(), db->map_ino); */

	return 0;
	}

	static int read_lock(struct db *db)
	{
	struct stat sbuf, sbuffile;
	int newfd = -1;

	assert(db->lock_status == UNLOCKED);
	for (;;) {
	if (lock_shared(db->fd) < 0) {
	syslog(LOG_ERR, "IOERROR: lock_shared %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}

	if (fstat(db->fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat %s: %m", db->fname);
	lock_unlock(db->fd);
	return CYRUSDB_IOERROR;
	}

	if (stat(db->fname, &sbuffile) == -1) {
	syslog(LOG_ERR, "IOERROR: stat %s: %m", db->fname);
	lock_unlock(db->fd);
	return CYRUSDB_IOERROR;
	}
	if (sbuf.st_ino == sbuffile.st_ino) break;

	newfd = open(db->fname, O_RDWR, 0644);
	if (newfd == -1) {
	syslog(LOG_ERR, "IOERROR: open %s: %m", db->fname);
	lock_unlock(db->fd);
	return CYRUSDB_IOERROR;
	}

	dup2(newfd, db->fd);
	close(newfd);
	}

	if (db->map_ino != sbuf.st_ino) {
	map_free(&db->map_base, &db->map_len);
	}
	db->map_size = sbuf.st_size;
	db->map_ino = sbuf.st_ino;
	db->lock_status = READLOCKED;

	/* printf("%d: read lock: %d\n", getpid(), db->map_ino); */

	map_refresh(db->fd, 0, &db->map_base, &db->map_len, sbuf.st_size,
	db->fname, 0);

	if (db->is_open) {
	/* reread header */
	read_header(db);
	}

	return 0;
	}

	static int unlock(struct db *db)
	{
	if (db->lock_status == UNLOCKED) {
	syslog(LOG_NOTICE, "skiplist: unlock while not locked");
	}
	if (lock_unlock(db->fd) < 0) {
	syslog(LOG_ERR, "IOERROR: lock_unlock %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}
	db->lock_status = UNLOCKED;

	/* printf("%d: unlock: %d\n", getpid(), db->map_ino); */

	return 0;
	}

	static int lock_or_refresh(struct db db, struct txn *tidptr)
	{
	int r;

	assert(db != NULL && tidptr != NULL);

	if (*tidptr) {
	/* check that the DB agrees that we're in this transaction */
	assert(db->current_txn == *tidptr);

	/* just update the active transaction */
	update_lock(db, *tidptr);

	} else {
	/* check that the DB isn't in a transaction */
	assert(db->current_txn == NULL);

	/* grab a r/w lock */
	if ((r = write_lock(db, NULL)) < 0) {
	return r;
	}

	/* start the transaction */
	if ((r = newtxn(db, tidptr))) {
	return r;
	}
	}

	return 0;
	}

	static int dispose_db(struct db *db)
	{
	if (!db) return 0;

	if (db->lock_status) {
	syslog(LOG_ERR, "skiplist: closed while still locked");
	unlock(db);
	}
	if (db->fname) {
	free(db->fname);
	}
	if (db->map_base) {
	map_free(&db->map_base, &db->map_len);
	}
	if (db->fd != -1) {
	close(db->fd);
	}

	free(db);

	return 0;
	}

	static int myopen(const char fname, int flags, struct db *ret)
	{
	struct db *db;
	struct db_list *list_ent = open_db;
	int r;
	int new = 0;

	while (list_ent && strcmp(list_ent->db->fname, fname)) {
	list_ent = list_ent->next;
	}
	if (list_ent) {
	/* we already have this DB open! */
	syslog(LOG_NOTICE, "skiplist: %s is already open %d time%s, returning object",
	fname, list_ent->refcount, list_ent->refcount == 1 ? "" : "s");
	*ret = list_ent->db;
	++list_ent->refcount;
	return 0;
	}

	db = (struct db *) xzmalloc(sizeof(struct db));
	db->fd = -1;
	db->fname = xstrdup(fname);
	db->compar = (flags & CYRUSDB_MBOXSORT) ? bsearch_ncompare : compare;

	db->fd = open(fname, O_RDWR, 0644);
	if (db->fd == -1 && errno == ENOENT) {
	if (!(flags & CYRUSDB_CREATE)) {
	dispose_db(db);
	return CYRUSDB_NOTFOUND;
	}
	if (cyrus_mkdir(fname, 0755) == -1) {
	dispose_db(db);
	return CYRUSDB_IOERROR;
	}
	db->fd = open(fname, O_RDWR \| O_CREAT, 0644);
	new = 1;
	}

	if (db->fd == -1) {
	syslog(LOG_ERR, "IOERROR: opening %s: %m", fname);
	dispose_db(db);
	return CYRUSDB_IOERROR;
	}

	db->curlevel = 0;
	db->is_open = 0;
	db->lock_status = UNLOCKED;

	/* grab a read lock, only reading the header */
	r = read_lock(db);
	if (r < 0) {
	dispose_db(db);
	return r;
	}

	/* if the file is empty, then the header needs to be created first */
	if (db->map_size == 0) {
	unlock(db);
	r = write_lock(db, NULL);
	if (r < 0) {
	dispose_db(db);
	return r;
	}
	}

	/* race condition. Another process may have already got the write
	* lock and created the header. Only go ahead if the map_size is
	* still zero (read/write_lock updates map_size). */
	if (db->map_size == 0) {
	/* initialize in memory structure */
	db->version = SKIPLIST_VERSION;
	db->version_minor = SKIPLIST_VERSION_MINOR;
	db->maxlevel = SKIPLIST_MAXLEVEL;
	db->curlevel = 1;
	db->listsize = 0;
	/* where do we start writing new entries? */
	db->logstart = DUMMY_OFFSET(db) + DUMMY_SIZE(db);
	db->last_recovery = time(NULL);

	/* create the header */
	r = write_header(db);

	if (!r) {
	int n;
	int dsize = DUMMY_SIZE(db);
	uint32_t buf = (uint32_t ) xzmalloc(dsize);

	buf[0] = htonl(DUMMY);
	buf[(dsize / 4) - 1] = htonl(-1);

	lseek(db->fd, DUMMY_OFFSET(db), SEEK_SET);
	n = retry_write(db->fd, (char *) buf, dsize);
	if (n != dsize) {
	syslog(LOG_ERR, "DBERROR: writing dummy node for %s: %m",
	db->fname);
	r = CYRUSDB_IOERROR;
	}
	free(buf);
	}

	/* sync the db */
	if (!r && DO_FSYNC && (fsync(db->fd) < 0)) {
	syslog(LOG_ERR, "DBERROR: fsync(%s): %m", db->fname);
	r = CYRUSDB_IOERROR;
	}

	/* map the new file */
	db->map_size = db->logstart;
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, db->logstart,
	db->fname, 0);
	}

	db->is_open = 1;

	r = read_header(db);
	if (r) {
	dispose_db(db);
	return r;
	}

	/* unlock the db */
	unlock(db);

	if (!global_recovery \|\| db->last_recovery < global_recovery) {
	/* run recovery; we rebooted since the last time recovery
	was run */
	r = recovery(db, 0);
	if (r) {
	dispose_db(db);
	return r;
	}
	}

	*ret = db;

	/* track this database in the open list */
	list_ent = (struct db_list *) xzmalloc(sizeof(struct db_list));
	list_ent->db = db;
	list_ent->next = open_db;
	list_ent->refcount = 1;
	open_db = list_ent;

	return 0;
	}

	int myclose(struct db *db)
	{
	struct db_list *list_ent = open_db;
	struct db_list *prev = NULL;

	/* remove this DB from the open list */
	while (list_ent && list_ent->db != db) {
	prev = list_ent;
	list_ent = list_ent->next;
	}
	assert(list_ent);
	if (--list_ent->refcount <= 0) {
	if (prev) prev->next = list_ent->next;
	else open_db = list_ent->next;
	free(list_ent);
	return dispose_db(db);
	}

	return 0;
	}

	static int compare(const char s1, int l1, const char s2, int l2)
	{
	int min = l1 < l2 ? l1 : l2;
	int cmp = 0;

	while (min-- > 0 && (cmp = s1 - s2) == 0) {
	s1++;
	s2++;
	}
	if (min >= 0) {
	return cmp;
	} else {
	if (l1 > l2) return 1;
	else if (l2 > l1) return -1;
	else return 0;
	}
	}

	/* returns the offset to the node asked for, or the node after it
	if it doesn't exist.
	if previous is set, finds the last node < key */
	static const char find_node(struct db db,
	const char *key, int keylen,
	unsigned *updateoffsets)
	{
	const char *ptr = db->map_base + DUMMY_OFFSET(db);
	int i;
	unsigned offset;

	if (updateoffsets) {
	for (i = 0; (unsigned) i < db->maxlevel; i++) {
	updateoffsets[i] = DUMMY_OFFSET(db);
	}
	}

	for (i = db->curlevel - 1; i >= 0; i--) {
	while ((offset = FORWARD(ptr, i)) &&
	db->compar(KEY(db->map_base + offset), KEYLEN(db->map_base + offset),
	key, keylen) < 0) {
	/* move forward at level 'i' */
	ptr = db->map_base + offset;
	}
	if (updateoffsets) updateoffsets[i] = ptr - db->map_base;
	}

	ptr = db->map_base + FORWARD(ptr, 0);

	return ptr;
	}

	int myfetch(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tidptr)
	{
	const char *ptr;
	int r = 0;

	assert(db != NULL && key != NULL);

	if (data) *data = NULL;
	if (datalen) *datalen = 0;

	/* Hacky workaround:
	*
	* If no transaction was passed, but we're in a transaction,
	* then just do the read within that transaction.
	*/
	if (!tidptr && db->current_txn != NULL) {
	tidptr = &(db->current_txn);
	}

	if (tidptr) {
	/* make sure we're write locked and up to date */
	if ((r = lock_or_refresh(db, tidptr)) < 0) {
	return r;
	}
	} else {
	/* grab a r lock */
	if ((r = read_lock(db)) < 0) {
	return r;
	}
	}

	ptr = find_node(db, key, keylen, 0);

	if (ptr == db->map_base \|\| db->compar(KEY(ptr), KEYLEN(ptr), key, keylen)) {
	/* failed to find key/keylen */
	r = CYRUSDB_NOTFOUND;
	} else {
	if (datalen) *datalen = DATALEN(ptr);
	if (data) *data = DATA(ptr);
	}

	if (!tidptr) {
	/* release read lock */
	int r1;
	if ((r1 = unlock(db)) < 0) {
	return r1;
	}
	}

	return r;
	}

	static int fetch(struct db *mydb,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tidptr)
	{
	return myfetch(mydb, key, keylen, data, datalen, tidptr);
	}
	static int fetchlock(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tidptr)
	{
	return myfetch(db, key, keylen, data, datalen, tidptr);
	}

	/* foreach allows for subsidary mailbox operations in 'cb'.
	if there is a txn, 'cb' must make use of it.
	*/
	int myforeach(struct db *db,
	char *prefix, int prefixlen,
	foreach_p *goodp,
	foreach_cb cb, void rock,
	struct txn **tidptr)
	{
	const char *ptr;
	char *savebuf = NULL;
	size_t savebuflen = 0;
	size_t savebufsize;
	int r = 0, cb_r = 0;
	int need_unlock = 0;

	assert(db != NULL);
	assert(prefixlen >= 0);

	/* Hacky workaround:
	*
	* If no transaction was passed, but we're in a transaction,
	* then just do the read within that transaction.
	*/
	if (!tidptr && db->current_txn != NULL) {
	tidptr = &(db->current_txn);
	}

	if (tidptr) {
	/* make sure we're write locked and up to date */
	if ((r = lock_or_refresh(db, tidptr)) < 0) {
	return r;
	}
	} else {
	/* grab a r lock */
	if ((r = read_lock(db)) < 0) {
	return r;
	}
	need_unlock = 1;
	}

	ptr = find_node(db, prefix, prefixlen, 0);

	while (ptr != db->map_base) {
	/* does it match prefix? */
	if (KEYLEN(ptr) < (uint32_t) prefixlen) break;
	if (prefixlen && db->compar(KEY(ptr), prefixlen, prefix, prefixlen)) break;

	if (!goodp \|\|
	goodp(rock, KEY(ptr), KEYLEN(ptr), DATA(ptr), DATALEN(ptr))) {
	ino_t ino = db->map_ino;
	unsigned long sz = db->map_size;

	if (!tidptr) {
	/* release read lock */
	if ((r = unlock(db)) < 0) {
	return r;
	}
	need_unlock = 0;
	}

	/* save KEY, KEYLEN */
	if (!savebuf \|\| KEYLEN(ptr) > savebuflen) {
	savebuflen = KEYLEN(ptr) + 1024;
	savebuf = xrealloc(savebuf, savebuflen);
	}
	memcpy(savebuf, KEY(ptr), KEYLEN(ptr));
	savebufsize = KEYLEN(ptr);

	/* make callback */
	cb_r = cb(rock, KEY(ptr), KEYLEN(ptr), DATA(ptr), DATALEN(ptr));
	if (cb_r) break;

	if (!tidptr) {
	/* grab a r lock */
	if ((r = read_lock(db)) < 0) {
	free(savebuf);
	return r;
	}
	need_unlock = 1;
	} else {
	/* make sure we're up to date */
	update_lock(db, *tidptr);
	}

	/* reposition */
	if (!(ino == db->map_ino && sz == db->map_size)) {
	/* something changed in the file; reseek */
	ptr = find_node(db, savebuf, savebufsize, 0);

	/* 'ptr' might not equal 'savebuf'. if it's different,
	we want to stay where we are. if it's the same, we
	should move on to the next one */
	if (savebufsize == KEYLEN(ptr) &&
	!memcmp(savebuf, KEY(ptr), savebufsize)) {
	ptr = db->map_base + FORWARD(ptr, 0);
	} else {
	/* 'savebuf' got deleted, so we're now pointing at the
	right thing */
	}
	} else {
	/* move to the next one */
	ptr = db->map_base + FORWARD(ptr, 0);
	}
	} else {
	/* we didn't make the callback; keep going */
	ptr = db->map_base + FORWARD(ptr, 0);
	}
	}

	free(savebuf);

	if (need_unlock) {
	/* release read lock */
	if ((r = unlock(db)) < 0) {
	return r;
	}
	}

	return r ? r : cb_r;
	}

	unsigned int randlvl(struct db *db)
	{
	unsigned int lvl = 1;

	while ((((float) rand() / (float) (RAND_MAX)) < PROB)
	&& (lvl < db->maxlevel)) {
	lvl++;
	}
	/* syslog(LOG_DEBUG, "picked level %d", lvl); */

	return lvl;
	}

	int mystore(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tidptr, int overwrite)
	{
	const char *ptr;
	uint32_t klen;
	uint32_t dlen;
	struct iovec iov[50];
	unsigned lvl;
	unsigned i;
	unsigned num_iov;
	struct txn *tid;
	struct txn *localtid = NULL;
	uint32_t endpadding = htonl(-1);
	uint32_t zeropadding[4] = { 0, 0, 0, 0 };
	unsigned updateoffsets[SKIPLIST_MAXLEVEL+1];
	unsigned newoffsets[SKIPLIST_MAXLEVEL+1];
	uint32_t addrectype = htonl(ADD);
	uint32_t delrectype = htonl(DELETE);
	uint32_t todelete;
	unsigned newoffset;
	uint32_t netnewoffset;
	int r;

	assert(db != NULL);
	assert(key && keylen);

	/* not keeping the transaction, just create one local to
	* this function */
	if (!tidptr) {
	tidptr = &localtid;
	}

	/* make sure we're write locked and up to date */
	if ((r = lock_or_refresh(db, tidptr)) < 0) {
	return r;
	}

	tid = tidptr; / consistent naming is nice */

	if (be_paranoid) {
	assert(myconsistent(db, tid, 1) == 0);
	}

	num_iov = 0;

	newoffset = tid->logend;
	ptr = find_node(db, key, keylen, updateoffsets);
	if (ptr != db->map_base &&
	!db->compar(KEY(ptr), KEYLEN(ptr), key, keylen)) {

	if (!overwrite) {
	myabort(db, tid); /* releases lock */
	return CYRUSDB_EXISTS;
	} else {
	/* replace with an equal height node */
	lvl = LEVEL(ptr);

	/* log a removal */
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &delrectype, 4);
	todelete = htonl(ptr - db->map_base);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &todelete, 4);

	/* now we write at newoffset */
	newoffset += 8;

	/* our pointers are whatever the old node pointed to */
	for (i = 0; i < lvl; i++) {
	newoffsets[i] = htonl(FORWARD(ptr, i));
	}
	}
	} else {
	/* pick a size for the new node */
	lvl = randlvl(db);

	/* do we need to update the header ? */
	if (lvl > db->curlevel) {
	for (i = db->curlevel; i < lvl; i++) {
	updateoffsets[i] = DUMMY_OFFSET(db);
	}
	db->curlevel = lvl;

	/* write out that change */
	write_header(db); /* xxx errors? */
	}

	/* we point to what we're updating used to point to */
	/* newoffsets is written in the iovec later */
	for (i = 0; i < lvl; i++) {
	/* written in the iovec */
	newoffsets[i] =
	htonl(FORWARD(db->map_base + updateoffsets[i], i));
	}
	}

	klen = htonl(keylen);
	dlen = htonl(datalen);

	netnewoffset = htonl(newoffset);

	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &addrectype, 4);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &klen, 4);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) key, keylen);
	if (ROUNDUP(keylen) - keylen > 0) {
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) zeropadding,
	ROUNDUP(keylen) - keylen);
	}
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &dlen, 4);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) data, datalen);
	if (ROUNDUP(datalen) - datalen > 0) {
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) zeropadding,
	ROUNDUP(datalen) - datalen);
	}
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char ) newoffsets, 4 lvl);
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &endpadding, 4);

	getsyncfd(db, tid);
	lseek(tid->syncfd, tid->logend, SEEK_SET);
	r = retry_writev(tid->syncfd, iov, num_iov);
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: retry_writev(): %m");
	myabort(db, tid);
	return CYRUSDB_IOERROR;
	}
	tid->logend += r; /* update where to write next */

	/* update pointers after writing record so abort is guaranteed to
	* see which records need reverting */
	for (i = 0; i < lvl; i++) {
	/* write pointer updates */
	/* FORWARD(updates[i], i) = newoffset; */
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &netnewoffset, 4);
	}

	if (be_paranoid) {
	assert(myconsistent(db, tid, 1) == 0);
	}

	if (localtid) {
	/* commit the store, which releases the write lock */
	r = mycommit(db, tid);
	if (r) return r;
	}

	return 0;
	}

	static int create(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 0);
	}

	static int store(struct db *db,
	const char *key, int keylen,
	const char *data, int datalen,
	struct txn **tid)
	{
	return mystore(db, key, keylen, data, datalen, tid, 1);
	}

	int mydelete(struct db *db,
	const char *key, int keylen,
	struct txn **tidptr, int force __attribute__((unused)))
	{
	const char *ptr;
	uint32_t delrectype = htonl(DELETE);
	unsigned updateoffsets[SKIPLIST_MAXLEVEL+1];
	uint32_t offset;
	uint32_t writebuf[2];
	struct txn tid, localtid = NULL;
	unsigned i;
	int r;

	/* not keeping the transaction, just create one local to
	* this function */
	if (!tidptr) {
	tidptr = &localtid;
	}

	/* make sure we're write locked and up to date */
	if ((r = lock_or_refresh(db, tidptr)) < 0) {
	return r;
	}

	tid = tidptr; / consistent naming is nice */

	if (be_paranoid) {
	assert(myconsistent(db, tid, 1) == 0);
	}

	ptr = find_node(db, key, keylen, updateoffsets);
	if (ptr != db->map_base &&
	!db->compar(KEY(ptr), KEYLEN(ptr), key, keylen)) {
	/* gotcha */
	offset = ptr - db->map_base;

	/* log the deletion */
	getsyncfd(db, tid);
	lseek(tid->syncfd, tid->logend, SEEK_SET);
	writebuf[0] = delrectype;
	writebuf[1] = htonl(offset);

	/* update end-of-log */
	r = retry_write(tid->syncfd, (char *) writebuf, 8);
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: retry_write(): %m");
	myabort(db, tid);
	return CYRUSDB_IOERROR;
	}
	tid->logend += r;

	/* update pointers after writing record so abort is guaranteed to
	* see which records need reverting */
	for (i = 0; i < db->curlevel; i++) {
	uint32_t netnewoffset;

	if (FORWARD(db->map_base + updateoffsets[i], i) != offset) {
	break;
	}
	netnewoffset = htonl(FORWARD(ptr, i));
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &netnewoffset, 4);
	}
	}

	if (be_paranoid) {
	assert(myconsistent(db, tid, 1) == 0);
	}

	if (localtid) {
	/* commit the store, which releases the write lock */
	mycommit(db, tid);
	}

	return 0;
	}

	int mycommit(struct db db, struct txn tid)
	{
	uint32_t commitrectype = htonl(COMMIT);
	int r = 0;

	assert(db && tid);

	assert(db->current_txn == tid);

	update_lock(db, tid);

	if (be_paranoid) {
	assert(myconsistent(db, tid, 1) == 0);
	}

	/* verify that we did something this txn */
	if (tid->logstart == tid->logend) {
	/* empty txn, done */
	r = 0;
	goto done;
	}

	/* fsync if we're not using O_SYNC writes */
	if (!use_osync && DO_FSYNC && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	goto done;
	}

	/* xxx consider unlocking the database here: the transaction isn't
	yet durable but the file is in a form that is consistent for
	other transactions to use. releasing the lock here would give
	ACI properties. */

	/* write a commit record */
	assert(tid->syncfd != -1);
	lseek(tid->syncfd, tid->logend, SEEK_SET);
	retry_write(tid->syncfd, (char *) &commitrectype, 4);

	/* fsync if we're not using O_SYNC writes */
	if (!use_osync && DO_FSYNC && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR, "IOERROR: writing %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	goto done;
	}

	done:
	if (!r)
	db->current_txn = NULL;

	/* consider checkpointing */
	if (!r && tid->logend > (2 * db->logstart + SKIPLIST_MINREWRITE)) {
	r = mycheckpoint(db, 1);
	}

	if (be_paranoid) {
	assert(myconsistent(db, db->current_txn, 1) == 0);
	}

	if (r) {
	int r2;

	/* error during commit; we must abort */
	r2 = myabort(db, tid);
	if (r2) {
	syslog(LOG_ERR, "DBERROR: skiplist %s: commit AND abort failed",
	db->fname);
	}
	} else {
	/* release the write lock */
	if ((r = unlock(db)) < 0) {
	return r;
	}

	/* must close this after releasing the lock */
	closesyncfd(db, tid);

	/* free tid */
	free(tid);
	}

	return r;
	}

	int myabort(struct db db, struct txn tid)
	{
	const char *ptr;
	unsigned updateoffsets[SKIPLIST_MAXLEVEL+1];
	unsigned offset;
	unsigned i;
	int r = 0;

	assert(db && tid);

	assert(db->current_txn == tid);

	/* update the mmap so we can see the log entries we need to remove */
	update_lock(db, tid);

	/* look at the log entries we've written, and undo their effects */
	while (tid->logstart != tid->logend) {
	/* find the last log entry */
	for (offset = tid->logstart, ptr = db->map_base + offset;
	offset + RECSIZE(ptr) != (uint32_t) tid->logend;
	offset += RECSIZE(ptr), ptr = db->map_base + offset) ;

	offset = ptr - db->map_base;

	assert(TYPE(ptr) == ADD \|\| TYPE(ptr) == DELETE);
	switch (TYPE(ptr)) {
	case DUMMY:
	case INORDER:
	case COMMIT:
	abort();

	case ADD:
	/* remove this record */
	(void) find_node(db, KEY(ptr), KEYLEN(ptr), updateoffsets);
	for (i = 0; i < db->curlevel; i++) {
	uint32_t netnewoffset;

	if (FORWARD(db->map_base + updateoffsets[i], i) != offset) {
	break;
	}

	netnewoffset = htonl(FORWARD(ptr, i));
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &netnewoffset, 4);
	}
	break;
	case DELETE:
	{
	unsigned lvl;
	uint32_t netnewoffset;
	const char *q;

	/* re-add this record. it can't exist right now. */
	netnewoffset = ((uint32_t )(ptr + 4));
	q = db->map_base + ntohl(netnewoffset);
	lvl = LEVEL(q);
	(void) find_node(db, KEY(q), KEYLEN(q), updateoffsets);
	for (i = 0; i < lvl; i++) {
	/* the current pointers FROM this node are correct,
	so we just have to update 'updateoffsets' */
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &netnewoffset, 4);
	}
	break;
	}
	}

	/* remove looking at this */
	tid->logend -= RECSIZE(ptr);
	}

	/* truncate the file to remove log entries */
	if (ftruncate(db->fd, tid->logstart) < 0) {
	syslog(LOG_ERR,
	"DBERROR: skiplist abort %s: ftruncate: %m",
	db->fname);
	r = CYRUSDB_IOERROR;
	unlock(db);
	return r;
	}

	db->map_size = tid->logstart;

	/* release the write lock */
	if ((r = unlock(db)) < 0) {
	return r;
	}

	/* must close this after releasing the lock */
	closesyncfd(db, tid);

	/* free the tid */
	free(tid);

	db->current_txn = NULL;

	return 0;
	}

	/* compress 'db'. if 'locked != 0', the database is already R/W locked and
	will be returned as such. */
	static int mycheckpoint(struct db *db, int locked)
	{
	char fname[1024];
	int oldfd;
	struct iovec iov[50];
	unsigned num_iov;
	unsigned updateoffsets[SKIPLIST_MAXLEVEL+1];
	const char *ptr;
	unsigned offset;
	int r = 0;
	uint32_t iorectype = htonl(INORDER);
	unsigned i;
	time_t start = time(NULL);

	/* grab write lock (could be read but this prevents multiple checkpoints
	simultaneously) */
	if (!locked) {
	r = write_lock(db, NULL);
	if (r < 0) return r;
	} else {
	/* we need the latest and greatest data */
	assert(db->is_open && db->lock_status == WRITELOCKED);
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, MAP_UNKNOWN_LEN,
	db->fname, 0);
	}

	/* can't be in a transaction */
	assert(db->current_txn == NULL);

	if ((r = myconsistent(db, NULL, 1)) < 0) {
	syslog(LOG_ERR, "db %s, inconsistent pre-checkpoint, bailing out",
	db->fname);
	return r;
	}

	/* open fname.NEW */
	snprintf(fname, sizeof(fname), "%s.NEW", db->fname);
	oldfd = db->fd;
	db->fd = open(fname, O_RDWR \| O_CREAT, 0644);
	if (db->fd < 0) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint: open(%s): %m", fname);
	if (!locked) unlock(db);
	db->fd = oldfd;
	return CYRUSDB_IOERROR;
	}

	/* truncate it just in case! */
	r = ftruncate(db->fd, 0);
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint %s: ftruncate %m", fname);
	if (!locked) unlock(db);
	db->fd = oldfd;
	return CYRUSDB_IOERROR;
	}

	/* write dummy record */
	if (!r) {
	int dsize = DUMMY_SIZE(db);
	uint32_t buf = (uint32_t ) xzmalloc(dsize);

	buf[0] = htonl(DUMMY);
	buf[(dsize / 4) - 1] = htonl(-1);

	lseek(db->fd, DUMMY_OFFSET(db), SEEK_SET);
	r = retry_write(db->fd, (char *) buf, dsize);
	if (r != dsize) {
	r = CYRUSDB_IOERROR;
	} else {
	r = 0;
	}
	free(buf);

	/* initialize the updateoffsets array so when we append records
	we know where to set the pointers */
	for (i = 0; i < db->maxlevel; i++) {
	/* header_size + 4 (rectype) + 4 (ksize) + 4 (dsize)
	+ 4 * i */
	updateoffsets[i] = DUMMY_OFFSET(db) + 12 + 4 * i;
	}
	}

	/* write records to new file */
	offset = FORWARD(db->map_base + DUMMY_OFFSET(db), 0);
	db->listsize = 0;
	while (!r && offset != 0) {
	unsigned int lvl;
	unsigned newoffset;
	uint32_t netnewoffset;

	ptr = db->map_base + offset;
	lvl = LEVEL(ptr);
	db->listsize++;

	num_iov = 0;
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &iorectype, 4);
	/* copy all but the rectype from the record */
	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) ptr + 4, RECSIZE(ptr) - 4);

	newoffset = lseek(db->fd, 0, SEEK_END);
	netnewoffset = htonl(newoffset);
	r = retry_writev(db->fd, iov, num_iov);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	} else {
	r = 0;
	}
	for (i = 0; !r && i < lvl; i++) {
	/* update pointers */
	r = lseek(db->fd, updateoffsets[i], SEEK_SET);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	r = retry_write(db->fd, (char *) &netnewoffset, 4);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	/* PTR(ptr, i) - ptr is the offset relative to me
	to my ith pointer */
	updateoffsets[i] = newoffset + (PTR(ptr, i) - ptr);
	}

	offset = FORWARD(ptr, 0);
	}

	/* set any dangling pointers to zero */
	for (i = 0; !r && i < db->maxlevel; i++) {
	uint32_t netnewoffset = htonl(0);

	r = lseek(db->fd, updateoffsets[i], SEEK_SET);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	r = retry_write(db->fd, (char *) &netnewoffset, 4);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}
	}

	/* create the header */
	db->logstart = lseek(db->fd, 0, SEEK_END);
	db->last_recovery = time(NULL);
	r = write_header(db);

	/* sync new file */
	if (!r && DO_FSYNC && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint: fdatasync(%s): %m", fname);
	r = CYRUSDB_IOERROR;
	}

	if (!r) {
	/* get new lock */
	db->lock_status = UNLOCKED; /* well, the new file is... */
	r = write_lock(db, fname);
	}

	/* move new file to original file name */
	if (!r && (rename(fname, db->fname) < 0)) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint: rename(%s, %s): %m",
	fname, db->fname);
	r = CYRUSDB_IOERROR;
	}

	/* force the new file name to disk */
	if (!r && DO_FSYNC && (fsync(db->fd) < 0)) {
	syslog(LOG_ERR, "DBERROR: skiplist checkpoint: fsync(%s): %m", fname);
	r = CYRUSDB_IOERROR;
	}

	if (r) {
	/* clean up */
	close(db->fd);
	db->fd = oldfd;
	unlink(fname);
	}

	/* release old write lock */
	close(oldfd);

	{
	struct stat sbuf;

	/* let's make sure we're up to date */
	map_free(&db->map_base, &db->map_len);
	if (fstat(db->fd, &sbuf) == -1) {
	syslog(LOG_ERR, "IOERROR: fstat %s: %m", db->fname);
	return CYRUSDB_IOERROR;
	}
	db->map_size = sbuf.st_size;
	db->map_ino = sbuf.st_ino;
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, sbuf.st_size,
	db->fname, 0);
	}

	if ((r = myconsistent(db, NULL, 1)) < 0) {
	syslog(LOG_ERR, "db %s, inconsistent post-checkpoint, bailing out",
	db->fname);
	return r;
	}

	if (!locked) {
	/* unlock the new db files */
	unlock(db);
	}

	{
	int diff = time(NULL) - start;
	syslog(LOG_INFO,
	"skiplist: checkpointed %s (%d record%s, %d bytes) in %d second%s",
	db->fname, db->listsize, db->listsize == 1 ? "" : "s",
	db->logstart, diff, diff == 1 ? "" : "s");
	}

	return r;
	}

	/* dump the database.
	if detail == 1, dump all records.
	if detail == 2, also dump pointers for active records.
	if detail == 3, dump all records/all pointers.
	*/
	static int dump(struct db *db, int detail __attribute__((unused)))
	{
	const char ptr, end;
	unsigned i;

	read_lock(db);

	ptr = db->map_base + DUMMY_OFFSET(db);
	end = db->map_base + db->map_size;
	while (ptr < end) {
	printf("%04lX: ", (unsigned long) (ptr - db->map_base));
	switch (TYPE(ptr)) {
	case DUMMY:
	printf("DUMMY ");
	break;
	case INORDER:
	printf("INORDER ");
	break;
	case ADD:
	printf("ADD ");
	break;
	case DELETE:
	printf("DELETE ");
	break;
	case COMMIT:
	printf("COMMIT ");
	break;
	}

	switch (TYPE(ptr)) {
	case DUMMY:
	case INORDER:
	case ADD:
	printf("kl=%d dl=%d lvl=%d\n",
	KEYLEN(ptr), DATALEN(ptr), LEVEL(ptr));
	printf("\t");
	for (i = 0; i < LEVEL(ptr); i++) {
	printf("%04X ", FORWARD(ptr, i));
	}
	printf("\n");
	break;

	case DELETE:
	printf("offset=%04X\n", ntohl(((uint32_t )(ptr + 4))));
	break;

	case COMMIT:
	printf("\n");
	break;
	}

	ptr += RECSIZE(ptr);
	}

	unlock(db);
	return 0;
	}

	static int consistent(struct db *db)
	{
	return myconsistent(db, NULL, 0);
	}

	/* perform some basic consistency checks */
	static int myconsistent(struct db db, struct txn tid, int locked)
	{
	const char *ptr;
	uint32_t offset;

	assert(db->current_txn == tid); /* could both be null */

	if (!locked) read_lock(db);
	else if (tid) update_lock(db, tid);

	offset = FORWARD(db->map_base + DUMMY_OFFSET(db), 0);
	while (offset != 0) {
	unsigned i;

	ptr = db->map_base + offset;

	for (i = 0; i < LEVEL(ptr); i++) {
	offset = FORWARD(ptr, i);

	if (offset > db->map_size) {
	fprintf(stdout,
	"skiplist inconsistent: %04X: ptr %d is %04X; "
	"eof is %04X\n",
	(unsigned int) (ptr - db->map_base),
	i, offset, (unsigned int) db->map_size);
	if (!locked) unlock(db);
	return CYRUSDB_INTERNAL;
	}

	if (offset != 0) {
	/* check to see that ptr < ptr -> next */
	const char *q = db->map_base + offset;
	int cmp;

	cmp = db->compar(KEY(ptr), KEYLEN(ptr), KEY(q), KEYLEN(q));
	if (cmp >= 0) {
	fprintf(stdout,
	"skiplist inconsistent: %04X: ptr %d is %04X; "
	"db->compar() = %d\n",
	(unsigned int) (ptr - db->map_base),
	i,
	offset, cmp);
	if (!locked) unlock(db);
	return CYRUSDB_INTERNAL;
	}
	}
	}

	offset = FORWARD(ptr, 0);
	}

	if (!locked) unlock(db);

	return 0;
	}

	/* run recovery on this file */
	static int recovery(struct db *db, int flags)
	{
	const char ptr, keyptr;
	unsigned updateoffsets[SKIPLIST_MAXLEVEL+1];
	uint32_t offset, offsetnet, myoff = 0;
	int r = 0, need_checkpoint = 0;
	time_t start = time(NULL);
	unsigned i;

	if (!(flags & RECOVERY_CALLER_LOCKED) && (r = write_lock(db, NULL)) < 0) {
	return r;
	}
	assert(db->is_open && db->lock_status == WRITELOCKED);

	if ((r = read_header(db)) < 0) {
	unlock(db);
	return r;
	}

	if (!(flags & RECOVERY_FORCE)
	&& global_recovery
	&& db->last_recovery >= global_recovery) {
	/* someone beat us to it */
	unlock(db);
	return 0;
	}

	/* can't run recovery inside a txn */
	assert(db->current_txn == NULL);

	db->listsize = 0;

	ptr = DUMMY_PTR(db);
	r = 0;

	/* verify this is DUMMY */
	if (!r && TYPE(ptr) != DUMMY) {
	r = CYRUSDB_IOERROR;
	syslog(LOG_ERR, "DBERROR: skiplist recovery %s: no dummy node?",
	db->fname);
	}

	/* zero key */
	if (!r && KEYLEN(ptr) != 0) {
	r = CYRUSDB_IOERROR;
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: dummy node KEYLEN != 0",
	db->fname);
	}

	/* zero data */
	if (!r && DATALEN(ptr) != 0) {
	r = CYRUSDB_IOERROR;
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: dummy node DATALEN != 0",
	db->fname);
	}

	/* pointers for db->maxlevel */
	if (!r && LEVEL(ptr) != db->maxlevel) {
	r = CYRUSDB_IOERROR;
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: dummy node level: %d != %d",
	db->fname, LEVEL(ptr), db->maxlevel);
	}

	for (i = 0; i < db->maxlevel; i++) {
	/* header_size + 4 (rectype) + 4 (ksize) + 4 (dsize)
	+ 4 * i */
	updateoffsets[i] = DUMMY_OFFSET(db) + 12 + 4 * i;
	}

	/* reset the data that was written INORDER by the last checkpoint */
	offset = DUMMY_OFFSET(db) + DUMMY_SIZE(db);
	while (!r && (offset < db->map_size)
	&& TYPE(db->map_base + offset) == INORDER) {
	ptr = db->map_base + offset;
	offsetnet = htonl(offset);

	db->listsize++;

	/* xxx check \0 fill on key */

	/* xxx check \0 fill on data */

	/* update previous pointers, record these for updating */
	for (i = 0; !r && i < LEVEL(ptr); i++) {
	r = lseek(db->fd, updateoffsets[i], SEEK_SET);
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: lseek %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	r = retry_write(db->fd, (char *) &offsetnet, 4);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	/* PTR(ptr, i) - ptr is the offset relative to me
	to my ith pointer */
	updateoffsets[i] = offset + (PTR(ptr, i) - ptr);
	}

	/* check padding */
	if (!r && PADDING(ptr) != (uint32_t) -1) {
	syslog(LOG_ERR, "DBERROR: %s: offset %04X padding not -1",
	db->fname, offset);
	r = CYRUSDB_IOERROR;
	}

	if (!r) {
	offset += RECSIZE(ptr);
	}
	}

	if (offset != db->logstart) {
	syslog(LOG_NOTICE, "skiplist recovery %s: incorrect logstart %04X changed to %04X",
	db->fname, db->logstart, offset);
	db->logstart = offset; /* header will be committed later */
	}

	/* zero out the remaining pointers */
	if (!r) {
	for (i = 0; !r && i < db->maxlevel; i++) {
	int zerooffset = 0;

	r = lseek(db->fd, updateoffsets[i], SEEK_SET);
	if (r < 0) {
	syslog(LOG_ERR, "DBERROR: lseek %s: %m", db->fname);
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}

	r = retry_write(db->fd, (char *) &zerooffset, 4);
	if (r < 0) {
	r = CYRUSDB_IOERROR;
	break;
	} else {
	r = 0;
	}
	}
	}

	/* replay the log */
	while (!r && offset < db->map_size) {
	const char p, q;

	/* refresh map, so we see the writes we've just done */
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, db->map_size,
	db->fname, 0);

	ptr = db->map_base + offset;

	/* bugs in recovery truncates could have left some bogus zeros here */
	if (TYPE(ptr) == 0) {
	int orig = offset;
	while (TYPE(ptr) == 0 && offset < db->map_size) {
	offset += 4;
	ptr = db->map_base + offset;
	}
	syslog(LOG_ERR, "skiplist recovery %s: skipped %d bytes of zeros at %04X",
	db->fname, offset - orig, orig);
	need_checkpoint = 1;
	}

	offsetnet = htonl(offset);

	/* if this is a commit, we've processed everything in this txn */
	if (TYPE(ptr) == COMMIT) {
	offset += RECSIZE(ptr);
	continue;
	}

	/* make sure this is ADD or DELETE */
	if (TYPE(ptr) != ADD && TYPE(ptr) != DELETE) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: %04X should be ADD or DELETE",
	db->fname, offset);
	r = CYRUSDB_IOERROR;
	break;
	}

	/* look ahead for a commit */
	q = db->map_base + db->map_size;
	p = ptr;
	for (;;) {
	if (RECSIZE(p) <= 0) {
	/* hmm, we can't trust this transaction */
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: found a RECSIZE of 0, "
	"truncating corrupted file instead of looping forever...",
	db->fname);
	p = q;
	break;
	}
	p += RECSIZE(p);
	if (p >= q) break;
	if (TYPE(p) == COMMIT) break;
	}
	if (p >= q) {
	syslog(LOG_NOTICE,
	"skiplist recovery %s: found partial txn, not replaying",
	db->fname);

	/* no commit, we should truncate */
	if (ftruncate(db->fd, offset) < 0) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: ftruncate: %m",
	db->fname);
	r = CYRUSDB_IOERROR;
	}

	/* set the map size back as well */
	db->map_size = offset;

	break;
	}

	keyptr = NULL;
	/* look for the key */
	if (TYPE(ptr) == ADD) {
	keyptr = find_node(db, KEY(ptr), KEYLEN(ptr), updateoffsets);
	if (keyptr == db->map_base \|\|
	db->compar(KEY(ptr), KEYLEN(ptr), KEY(keyptr), KEYLEN(keyptr))) {
	/* didn't find exactly this node */
	keyptr = NULL;
	}
	} else { /* type == DELETE */
	const char *p;

	myoff = ntohl(((uint32_t )(ptr + 4)));
	p = db->map_base + myoff;
	keyptr = find_node(db, KEY(p), KEYLEN(p), updateoffsets);
	if (keyptr == db->map_base \|\|
	db->compar(KEY(p), KEYLEN(p), KEY(keyptr), KEYLEN(keyptr))) {
	/* didn't find exactly this node */
	keyptr = NULL;
	}
	}

	/* if DELETE & found key, skip over it */
	if (TYPE(ptr) == DELETE && keyptr) {
	db->listsize--;

	for (i = 0; i < db->curlevel; i++) {
	int newoffset;

	if (FORWARD(db->map_base + updateoffsets[i], i) != myoff) {
	break;
	}
	newoffset = htonl(FORWARD(db->map_base + myoff, i));
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &newoffset, 4);
	}

	/* otherwise if DELETE, throw an error */
	} else if (TYPE(ptr) == DELETE) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: DELETE at %04X doesn't exist, skipping",
	db->fname, offset);
	need_checkpoint = 1;

	/* otherwise insert it */
	} else if (TYPE(ptr) == ADD) {
	unsigned int lvl;
	uint32_t newoffsets[SKIPLIST_MAXLEVEL+1];

	if (keyptr) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: ADD at %04X exists, replacing",
	db->fname, offset);
	need_checkpoint = 1;
	} else {
	db->listsize++;
	}
	offsetnet = htonl(offset);

	lvl = LEVEL(ptr);
	if (lvl > SKIPLIST_MAXLEVEL) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: node claims level %d (greater than max %d)",
	db->fname, lvl, SKIPLIST_MAXLEVEL);
	r = CYRUSDB_IOERROR;
	} else {
	/* NOTE - in the bogus case where a record with the same key already
	* exists, there are three possible cases:
	* lvl == LEVEL(keyptr)
	* * trivial: all to me, all mine to keyptr's FORWARD
	* lvl > LEVEL(keyptr) -
	* * all updateoffsets values should point to me
	* * up until LEVEL(keyptr) set to keyptr's next values
	* (updateoffsets[i] should be keyptr in these cases)
	* then point all my higher pointers are updateoffsets[i]'s
	* FORWARD instead.
	* lvl < LEVEL(keyptr)
	* * updateoffsets values up to lvl should point to me
	* * all mine should point to keyptr's next values
	* * from lvl up, all updateoffsets[i] should point to
	* FORWARD(keyptr, i) instead.
	*
	* All of this fully unstitches keyptr from the chain and stitches
	* the current node in, regardless of height difference. Man what
	* a pain!
	*/
	for (i = 0; i < lvl; i++) {
	/* set our next pointers */
	if (keyptr && i < LEVEL(keyptr)) {
	/* need to replace the matching record key */
	newoffsets[i] =
	htonl(FORWARD(keyptr, i));
	} else {
	newoffsets[i] =
	htonl(FORWARD(db->map_base + updateoffsets[i], i));
	}

	/* replace 'updateoffsets' to point to me */
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &offsetnet, 4);
	}
	/* write out newoffsets */
	lseek(db->fd, FIRSTPTR(ptr) - db->map_base, SEEK_SET);
	retry_write(db->fd, (char ) newoffsets, 4 lvl);

	if (keyptr && lvl < LEVEL(keyptr)) {
	uint32_t newoffsetnet;
	for (i = lvl; i < LEVEL(keyptr); i++) {
	newoffsetnet = htonl(FORWARD(keyptr, i));
	/* replace 'updateoffsets' to point onwards */
	lseek(db->fd,
	PTR(db->map_base + updateoffsets[i], i) - db->map_base,
	SEEK_SET);
	retry_write(db->fd, (char *) &newoffsetnet, 4);
	}
	}
	}
	/* can't happen */
	} else {
	abort();
	}

	/* move to next record */
	offset += RECSIZE(ptr);
	}

	if (libcyrus_config_getswitch(CYRUSOPT_SKIPLIST_ALWAYS_CHECKPOINT)) {
	/* refresh map, so we see the writes we've just done */
	map_refresh(db->fd, 0, &db->map_base, &db->map_len, db->map_size,
	db->fname, 0);

	r = mycheckpoint(db, 1);

	if (r \|\| !(flags & RECOVERY_CALLER_LOCKED)) {
	unlock(db);
	}

	return r;
	}

	/* fsync the recovered database */
	if (!r && DO_FSYNC && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: fdatasync: %m", db->fname);
	r = CYRUSDB_IOERROR;
	}

	/* set the last recovery timestamp */
	if (!r) {
	db->last_recovery = time(NULL);
	write_header(db);
	}

	/* fsync the new header */
	if (!r && DO_FSYNC && (fdatasync(db->fd) < 0)) {
	syslog(LOG_ERR,
	"DBERROR: skiplist recovery %s: fdatasync: %m", db->fname);
	r = CYRUSDB_IOERROR;
	}

	if (!r) {
	int diff = time(NULL) - start;

	syslog(LOG_NOTICE,
	"skiplist: recovered %s (%d record%s, %ld bytes) in %d second%s",
	db->fname, db->listsize, db->listsize == 1 ? "" : "s",
	db->map_size, diff, diff == 1 ? "" : "s");
	}

	if (!r && need_checkpoint) {
	r = mycheckpoint(db, 1);
	}

	if(r \|\| !(flags & RECOVERY_CALLER_LOCKED)) {
	unlock(db);
	}

	return r;
	}

	struct cyrusdb_backend cyrusdb_skiplist =
	{
	"skiplist", /* name */

	&myinit,
	&mydone,
	&mysync,
	&myarchive,

	&myopen,
	&myclose,

	&fetch,
	&fetchlock,
	&myforeach,
	&create,
	&store,
	&mydelete,

	&mycommit,
	&myabort,

	&dump,
	&consistent
	};
	diff --git a/lib/lock_fcntl.c b/lib/lock_fcntl.c
	index 6ed99ae63..e22992371 100644
	--- a/lib/lock_fcntl.c
	+++ b/lib/lock_fcntl.c
	@@ -1,214 +1,214 @@
	/* lock_fcntl.c -- Lock files using fcntl()
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: lock_fcntl.c,v 1.18 2010/01/06 17:01:46 murch Exp $
	*/

	#include <config.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <fcntl.h>
	#include <errno.h>

	-#include "lock.h"
	+#include "cyr_lock.h"

	const char *lock_method_desc = "fcntl";

	/*
	* Block until we obtain an exclusive lock on the file descriptor 'fd',
	* opened for reading and writing on the file named 'filename'. If
	* 'filename' is replaced, will re-open it as 'fd' and acquire a lock
	* on the new file.
	*
	* On success, returns 0. If a pointer to a struct stat is given as
	* 'sbuf', it is filled in.
	*
	* On failure, returns -1 with an error code in errno. If
	* 'failaction' is provided, it is filled in with a pointer to a fixed
	* string naming the action that failed.
	*
	*/
	int lock_reopen(fd, filename, sbuf, failaction)
	int fd;
	const char *filename;
	struct stat *sbuf;
	const char **failaction;
	{
	int r;
	struct flock fl;
	struct stat sbuffile, sbufspare;
	int newfd;

	if (!sbuf) sbuf = &sbufspare;

	for (;;) {
	fl.l_type= F_WRLCK;
	fl.l_whence = SEEK_SET;
	fl.l_start = 0;
	fl.l_len = 0;
	r = fcntl(fd, F_SETLKW, &fl);
	if (r == -1) {
	if (errno == EINTR) continue;
	if (failaction) *failaction = "locking";
	return -1;
	}

	r = fstat(fd, sbuf);
	if (!r) r = stat(filename, &sbuffile);
	if (r == -1) {
	if (failaction) *failaction = "stating";
	fl.l_type= F_UNLCK;
	fl.l_whence = SEEK_SET;
	fl.l_start = 0;
	fl.l_len = 0;
	r = fcntl(fd, F_SETLKW, &fl);
	return -1;
	}

	if (sbuf->st_ino == sbuffile.st_ino) return 0;

	newfd = open(filename, O_RDWR);
	if (newfd == -1) {
	if (failaction) *failaction = "opening";
	fl.l_type= F_UNLCK;
	fl.l_whence = SEEK_SET;
	fl.l_start = 0;
	fl.l_len = 0;
	r = fcntl(fd, F_SETLKW, &fl);
	return -1;
	}
	dup2(newfd, fd);
	close(newfd);
	}
	}

	/*
	* Obtain an exclusive lock on 'fd'.
	* Returns 0 for success, -1 for failure, with errno set to an
	* appropriate error code.
	*/
	int lock_blocking(fd)
	int fd;
	{
	int r;
	struct flock fl;

	for (;;) {
	fl.l_type= F_WRLCK;
	fl.l_whence = SEEK_SET;
	fl.l_start = 0;
	fl.l_len = 0;
	r = fcntl(fd, F_SETLKW, &fl);
	if (r != -1) return 0;
	if (errno == EINTR) continue;
	return -1;
	}
	}

	/*
	* Obtain a shared lock on 'fd'.
	* Returns 0 for success, -1 for failure, with errno set to an
	* appropriate error code.
	*/
	int lock_shared(fd)
	int fd;
	{
	int r;
	struct flock fl;

	for (;;) {
	fl.l_type= F_RDLCK;
	fl.l_whence = SEEK_SET;
	fl.l_start = 0;
	fl.l_len = 0;
	r = fcntl(fd, F_SETLKW, &fl);
	if (r != -1) return 0;
	if (errno == EINTR) continue;
	return -1;
	}
	}

	/*
	* Attempt to get an exclusive lock on 'fd' without blocking.
	* Returns 0 for success, -1 for failure, with errno set to an
	* appropriate error code.
	*/
	int lock_nonblocking(fd)
	int fd;
	{
	int r;
	struct flock fl;

	for (;;) {
	fl.l_type= F_WRLCK;
	fl.l_whence = SEEK_SET;
	fl.l_start = 0;
	fl.l_len = 0;
	r = fcntl(fd, F_SETLK, &fl);
	if (r != -1) return 0;
	if (errno == EINTR) continue;
	return -1;
	}
	}

	/*
	* Release any lock on 'fd'. Always returns success.
	*/
	int lock_unlock(int fd)
	{
	struct flock fl;
	int r;

	fl.l_type= F_UNLCK;
	fl.l_whence = SEEK_SET;
	fl.l_start = 0;
	fl.l_len = 0;

	for (;;) {
	r = fcntl(fd, F_SETLKW, &fl);
	if (r != -1) return 0;
	if (errno == EINTR) continue;
	/* xxx help! */
	return -1;
	}
	}

	diff --git a/lib/lock_flock.c b/lib/lock_flock.c
	index 0bbac4bb9..7d78f05ea 100644
	--- a/lib/lock_flock.c
	+++ b/lib/lock_flock.c
	@@ -1,182 +1,182 @@
	/* lock_flock.c -- Lock files using flock()
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: lock_flock.c,v 1.17 2010/01/06 17:01:46 murch Exp $
	*/

	#include <config.h>
	#include <sys/file.h>
	#include <sys/stat.h>
	#include <fcntl.h>
	#include <errno.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif

	-#include "lock.h"
	+#include "cyr_lock.h"

	const char *lock_method_desc = "flock";

	/*
	* Block until we obtain an exclusive lock on the file descriptor 'fd',
	* opened for reading and writing on the file named 'filename'. If
	* 'filename' is replaced, will re-open it as 'fd' and acquire a lock
	* on the new file.
	*
	* On success, returns 0. If a pointer to a struct stat is given as
	* 'sbuf', it is filled in.
	*
	* On failure, returns -1 with an error code in errno. If
	* 'failaction' is provided, it is filled in with a pointer to a fixed
	* string naming the action that failed.
	*
	*/
	int lock_reopen(fd, filename, sbuf, failaction)
	int fd;
	const char *filename;
	struct stat *sbuf;
	const char **failaction;
	{
	int r;
	struct stat sbuffile, sbufspare;
	int newfd;

	if (!sbuf) sbuf = &sbufspare;

	for (;;) {
	r = flock(fd, LOCK_EX);
	if (r == -1) {
	if (errno == EINTR) continue;
	if (failaction) *failaction = "locking";
	return -1;
	}

	fstat(fd, sbuf);
	r = stat(filename, &sbuffile);
	if (r == -1) {
	if (failaction) *failaction = "stating";
	flock(fd, LOCK_UN);
	return -1;
	}

	if (sbuf->st_ino == sbuffile.st_ino) return 0;

	newfd = open(filename, O_RDWR);
	if (newfd == -1) {
	if (failaction) *failaction = "opening";
	flock(fd, LOCK_UN);
	return -1;
	}
	dup2(newfd, fd);
	close(newfd);
	}
	}

	/*
	* Obtain an exclusive lock on 'fd'.
	* Returns 0 for success, -1 for failure, with errno set to an
	* appropriate error code.
	*/
	int lock_blocking(fd)
	int fd;
	{
	int r;

	for (;;) {
	r = flock(fd, LOCK_EX);
	if (r != -1) return 0;
	if (errno == EINTR) continue;
	return -1;
	}
	}

	/*
	* Obtain a shared lock on 'fd'.
	* Returns 0 for success, -1 for failure, with errno set to an
	* appropriate error code.
	*/
	int lock_shared(fd)
	int fd;
	{
	int r;

	for (;;) {
	r = flock(fd, LOCK_SH);
	if (r != -1) return 0;
	if (errno == EINTR) continue;
	return -1;
	}
	}

	/*
	* Attempt to get an exclusive lock on 'fd' without blocking.
	* Returns 0 for success, -1 for failure, with errno set to an
	* appropriate error code.
	*/
	int lock_nonblocking(fd)
	int fd;
	{
	int r;

	for (;;) {
	r = flock(fd, LOCK_EX\|LOCK_NB);
	if (r != -1) return 0;
	if (errno == EINTR) continue;
	return -1;
	}
	}

	/*
	* Release any lock on 'fd'. Always returns success.
	*/
	int lock_unlock(int fd)
	{
	int r;

	for (;;) {
	r = flock(fd, LOCK_UN);
	if (r != -1) return 0;
	if (errno == EINTR) continue;
	/* xxx help! */
	return -1;
	}
	}

	diff --git a/master/master.c b/master/master.c
	index 49607938a..823be0638 100644
	--- a/master/master.c
	+++ b/master/master.c
	@@ -1,2224 +1,2224 @@
	/* master.c -- IMAP master process to handle recovery, checkpointing, spawning
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: master.c,v 1.117 2010/04/19 19:54:26 murch Exp $
	*/

	#include <config.h>

	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <time.h>
	#include <sys/time.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#ifdef HAVE_UNISTD_H
	#include <unistd.h>
	#endif
	#ifdef HAVE_SYS_RESOURCE_H
	#include <sys/resource.h>
	#endif
	#include <fcntl.h>
	#include <signal.h>
	#include <sys/param.h>
	#include <sys/stat.h>
	#include <syslog.h>
	#include <netdb.h>
	#include <ctype.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <sys/un.h>
	#include <arpa/inet.h>
	#include <sysexits.h>
	#include <errno.h>
	#include <limits.h>

	#ifndef INADDR_NONE
	#define INADDR_NONE 0xffffffff
	#endif

	#ifndef INADDR_ANY
	#define INADDR_ANY 0x00000000
	#endif

	#if !defined(IPV6_V6ONLY) && defined(IPV6_BINDV6ONLY)
	#define IPV6_V6ONLY IPV6_BINDV6ONLY
	#endif

	#if defined(HAVE_NETSNMP)
	#include <net-snmp/net-snmp-config.h>
	#include <net-snmp/net-snmp-includes.h>
	#include <net-snmp/agent/net-snmp-agent-includes.h>
	#if defined(HAVE_NET_SNMP_AGENT_AGENT_MODULE_CONFIG_H)
	#include <net-snmp/agent/agent_module_config.h>
	#endif

	#include "cyrusMasterMIB.h"


	/* Use our own definitions for these */
	#undef TOUPPER
	#undef TOLOWER

	#elif defined(HAVE_UCDSNMP)
	#include <ucd-snmp/ucd-snmp-config.h>
	#include <ucd-snmp/ucd-snmp-includes.h>
	#include <ucd-snmp/ucd-snmp-agent-includes.h>

	#include "cyrusMasterMIB.h"

	int allow_severity = LOG_DEBUG;
	int deny_severity = LOG_ERR;
	#endif

	#include "masterconf.h"

	#include "master.h"
	#include "service.h"

	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "util.h"
	#include "xmalloc.h"

	enum {
	become_cyrus_early = 1,
	child_table_size = 10000,
	child_table_inc = 100
	};

	static int verbose = 0;
	static int listen_queue_backlog = 32;
	static int pidfd = -1;

	static volatile int in_shutdown = 0;

	const char *MASTER_CONFIG_FILENAME = DEFAULT_MASTER_CONFIG_FILENAME;

	#define SERVICE_NONE -1
	#define SERVICE_MAX INT_MAX-10
	#define SERVICENAME(x) ((x) ? x : "unknown")

	struct service *Services = NULL;
	int allocservices = 0;
	int nservices = 0;

	/* make libcyrus_min happy */
	int config_need_data = 0;

	struct event {
	char *name;
	time_t mark;
	time_t period;
	time_t hour;
	time_t min;
	int periodic;
	char const exec;
	struct event *next;
	};
	static struct event *schedule = NULL;

	enum sstate {
	SERVICE_STATE_UNKNOWN = 0, /* duh */
	SERVICE_STATE_INIT = 1, /* Service forked - UNUSED */
	SERVICE_STATE_READY = 2, /* Service told us it is ready */
	/* or it just forked and has not
	* talked to us yet */
	SERVICE_STATE_BUSY = 3, /* Service told us it is not ready */
	SERVICE_STATE_DEAD = 4 /* We received a sigchld from this service */
	};

	struct centry {
	pid_t pid;
	enum sstate service_state; /* SERVICE_STATE_* */
	time_t janitor_deadline; /* cleanup deadline */
	int si; /* Services[] index */
	struct centry *next;
	};
	static struct centry *ctable[child_table_size];
	static struct centry *cfreelist;

	static int janitor_frequency = 1; /* Janitor sweeps per second */
	static int janitor_position; /* Entry to begin at in next sweep */
	static struct timeval janitor_mark; /* Last time janitor did a sweep */

	void limit_fds(rlim_t);
	void schedule_event(struct event *a);

	void fatal(const char *msg, int code)
	{
	syslog(LOG_CRIT, "%s", msg);
	syslog(LOG_NOTICE, "exiting");
	exit(code);
	}

	void event_free(struct event *a)
	{
	if(a->exec) free((char**)a->exec);
	if(a->name) free((char*)a->name);
	free(a);
	}

	void get_prog(char path, unsigned size, char const *cmd)
	{
	if (cmd[0][0] == '/') {
	/* master lacks strlcpy, due to no libcyrus */
	snprintf(path, size, "%s", cmd[0]);
	}
	else snprintf(path, size, "%s/%s", SERVICE_PATH, cmd[0]);
	}

	void get_statsock(int filedes[2])
	{
	int r, fdflags;

	r = pipe(filedes);
	if (r != 0) {
	fatal("couldn't create status socket: %m", 1);
	}

	/* we don't want the master blocking on reads */
	fdflags = fcntl(filedes[0], F_GETFL, 0);
	if (fdflags != -1) fdflags = fcntl(filedes[0], F_SETFL,
	fdflags \| O_NONBLOCK);
	if (fdflags == -1) {
	fatal("unable to set non-blocking: %m", 1);
	}
	/* we don't want the services to be able to read from it */
	fdflags = fcntl(filedes[0], F_GETFD, 0);
	if (fdflags != -1) fdflags = fcntl(filedes[0], F_SETFD,
	fdflags \| FD_CLOEXEC);
	if (fdflags == -1) {
	fatal("unable to set close-on-exec: %m", 1);
	}
	}

	/* return a new 'centry', either from the freelist or by malloc'ing it */
	static struct centry *get_centry(void)
	{
	struct centry *t;

	if (!cfreelist) {
	/* create child_table_inc more and add them to the freelist */
	struct centry *n;
	int i;

	n = xmalloc(child_table_inc * sizeof(struct centry));
	cfreelist = n;
	for (i = 0; i < child_table_inc - 1; i++) {
	n[i].next = n + (i + 1);
	}
	/* i == child_table_inc - 1, last item in block */
	n[i].next = NULL;
	}

	t = cfreelist;
	cfreelist = cfreelist->next;

	t->janitor_deadline = 0;

	return t;
	}

	/* see if 'listen' parameter has both hostname and port, or just port */
	char parse_listen(char listen)
	{
	char *cp;
	char *port = NULL;

	if ((cp = strrchr(listen,']')) != NULL) {
	/* ":port" after closing bracket for IP address? */
	if (cp++ != '\0' && cp == ':') {
	*cp++ = '\0';
	if (*cp != '\0') {
	port = cp;
	}
	}
	} else if ((cp = strrchr(listen,':')) != NULL) {
	/* ":port" after hostname? */
	*cp++ = '\0';
	if (*cp != '\0') {
	port = cp;
	}
	}
	return port;
	}

	char parse_host(char listen)
	{
	char *cp;

	/* do we have a hostname, or IP number? */
	/* XXX are brackets necessary */
	if (*listen == '[') {
	listen++; /* skip first bracket */
	if ((cp = strrchr(listen,']')) != NULL) {
	*cp = '\0';
	}
	}
	return listen;
	}

	int verify_service_file(char const filename)
	{
	char path[PATH_MAX];
	struct stat statbuf;

	get_prog(path, sizeof(path), filename);
	if (stat(path, &statbuf)) return 0;
	if (! S_ISREG(statbuf.st_mode)) return 0;
	return statbuf.st_mode & S_IXUSR;
	}

	void service_create(struct service *s)
	{
	struct service service0, service;
	struct addrinfo hints, res0, res;
	int error, nsocket = 0;
	struct sockaddr_un sunsock;
	mode_t oldumask;
	int on = 1;
	int res0_is_local = 0;
	int r;

	if (s->associate > 0)
	return; /* service is already activated */

	if (!s->name)
	fatal("Serious software bug found: service_create() called on unnamed service!",
	EX_SOFTWARE);

	if (s->listen[0] == '/') { /* unix socket */
	res0_is_local = 1;
	res0 = (struct addrinfo *)malloc(sizeof(struct addrinfo));
	if (!res0)
	fatal("out of memory", EX_UNAVAILABLE);
	memset(res0, 0, sizeof(struct addrinfo));
	res0->ai_flags = AI_PASSIVE;
	res0->ai_family = PF_UNIX;
	if(!strcmp(s->proto, "tcp")) {
	res0->ai_socktype = SOCK_STREAM;
	} else {
	/* udp */
	res0->ai_socktype = SOCK_DGRAM;
	}
	res0->ai_addr = (struct sockaddr *)&sunsock;
	res0->ai_addrlen = sizeof(sunsock.sun_family) + strlen(s->listen) + 1;
	#ifdef SIN6_LEN
	res0->ai_addrlen += sizeof(sunsock.sun_len);
	sunsock.sun_len = res0->ai_addrlen;
	#endif
	sunsock.sun_family = AF_UNIX;
	strcpy(sunsock.sun_path, s->listen);
	unlink(s->listen);
	} else { /* inet socket */
	char listen, port;
	char *listen_addr;

	memset(&hints, 0, sizeof(hints));
	hints.ai_flags = AI_PASSIVE;
	if (!strcmp(s->proto, "tcp")) {
	hints.ai_family = PF_UNSPEC;
	hints.ai_socktype = SOCK_STREAM;
	} else if (!strcmp(s->proto, "tcp4")) {
	hints.ai_family = PF_INET;
	hints.ai_socktype = SOCK_STREAM;
	#ifdef PF_INET6
	} else if (!strcmp(s->proto, "tcp6")) {
	hints.ai_family = PF_INET6;
	hints.ai_socktype = SOCK_STREAM;
	#endif
	} else if (!strcmp(s->proto, "udp")) {
	hints.ai_family = PF_UNSPEC;
	hints.ai_socktype = SOCK_DGRAM;
	} else if (!strcmp(s->proto, "udp4")) {
	hints.ai_family = PF_INET;
	hints.ai_socktype = SOCK_DGRAM;
	#ifdef PF_INET6
	} else if (!strcmp(s->proto, "udp6")) {
	hints.ai_family = PF_INET6;
	hints.ai_socktype = SOCK_DGRAM;
	#endif
	} else {
	syslog(LOG_INFO, "invalid proto '%s', disabling %s",
	s->proto, s->name);
	s->exec = NULL;
	return;
	}

	/* parse_listen() and resolve_host() are destructive,
	* so make a work copy of s->listen
	*/
	listen = xstrdup(s->listen);

	if ((port = parse_listen(listen)) == NULL) {
	/* listen IS the port */
	port = listen;
	listen_addr = NULL;
	} else {
	/* listen is now just the address */
	listen_addr = parse_host(listen);
	if (*listen_addr == '\0')
	listen_addr = NULL;
	}

	error = getaddrinfo(listen_addr, port, &hints, &res0);

	free(listen);

	if (error) {
	syslog(LOG_INFO, "%s, disabling %s", gai_strerror(error), s->name);
	s->exec = NULL;
	return;
	}
	}

	memcpy(&service0, s, sizeof(struct service));

	for (res = res0; res; res = res->ai_next) {
	if (s->socket > 0) {
	memcpy(&service, &service0, sizeof(struct service));
	s = &service;
	}

	s->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
	if (s->socket < 0) {
	s->socket = 0;
	if (verbose > 2)
	syslog(LOG_ERR, "unable to open %s socket: %m", s->name);
	continue;
	}

	/* allow reuse of address */
	r = setsockopt(s->socket, SOL_SOCKET, SO_REUSEADDR,
	(void *) &on, sizeof(on));
	if (r < 0) {
	syslog(LOG_ERR, "unable to setsocketopt(SO_REUSEADDR): %m");
	}
	#if defined(IPV6_V6ONLY) && !(defined(__FreeBSD__) && __FreeBSD__ < 3)
	if (res->ai_family == AF_INET6) {
	r = setsockopt(s->socket, IPPROTO_IPV6, IPV6_V6ONLY,
	(void *) &on, sizeof(on));
	if (r < 0) {
	syslog(LOG_ERR, "unable to setsocketopt(IPV6_V6ONLY): %m");
	}
	}
	#endif

	/* set IP ToS if supported */
	#if defined(SOL_IP) && defined(IP_TOS)
	r = setsockopt(s->socket, SOL_IP, IP_TOS,
	(void *) &config_qosmarking, sizeof(config_qosmarking));
	if (r < 0) {
	syslog(LOG_WARNING, "unable to setsocketopt(IP_TOS): %m");
	}
	#endif

	oldumask = umask((mode_t) 0); /* for linux */
	r = bind(s->socket, res->ai_addr, res->ai_addrlen);
	umask(oldumask);
	if (r < 0) {
	close(s->socket);
	s->socket = 0;
	if (verbose > 2)
	syslog(LOG_ERR, "unable to bind to %s socket: %m", s->name);
	continue;
	}

	if (s->listen[0] == '/') { /* unix socket */
	/* for DUX, where this isn't the default.
	(harmlessly fails on some systems) */
	chmod(s->listen, (mode_t) 0777);
	}

	if ((!strcmp(s->proto, "tcp") \|\| !strcmp(s->proto, "tcp4")
	\|\| !strcmp(s->proto, "tcp6"))
	&& listen(s->socket, listen_queue_backlog) < 0) {
	syslog(LOG_ERR, "unable to listen to %s socket: %m", s->name);
	close(s->socket);
	s->socket = 0;
	continue;
	}

	s->ready_workers = 0;
	s->associate = nsocket;
	s->family = res->ai_family;

	get_statsock(s->stat);

	if (s == &service) {
	if (nservices == allocservices) {
	if (allocservices > SERVICE_MAX - 5)
	fatal("out of service structures, please restart", EX_UNAVAILABLE);
	Services = xrealloc(Services,
	(allocservices+=5) * sizeof(struct service));
	if (!Services) fatal("out of memory", EX_UNAVAILABLE);
	}
	memcpy(&Services[nservices++], s, sizeof(struct service));
	}
	nsocket++;
	}
	if (res0) {
	if(res0_is_local)
	free(res0);
	else
	freeaddrinfo(res0);
	}
	if (nsocket <= 0) {
	syslog(LOG_ERR, "unable to create %s listener socket: %m", s->name);
	s->exec = NULL;
	return;
	}
	}

	void run_startup(char **cmd)
	{
	pid_t pid;
	int status;
	char path[PATH_MAX];

	switch (pid = fork()) {
	case -1:
	syslog(LOG_CRIT, "can't fork process to run startup: %m");
	fatal("can't run startup", 1);
	break;

	case 0:
	/* Child - Release our pidfile lock. */
	if(pidfd != -1) close(pidfd);

	if (become_cyrus() != 0) {
	syslog(LOG_ERR, "can't change to the cyrus user: %m");
	exit(1);
	}

	limit_fds(256);

	get_prog(path, sizeof(path), cmd);
	syslog(LOG_DEBUG, "about to exec %s", path);
	execv(path, cmd);
	syslog(LOG_ERR, "can't exec %s for startup: %m", path);
	exit(EX_OSERR);

	default: /* parent */
	if (waitpid(pid, &status, 0) < 0) {
	syslog(LOG_ERR, "waitpid(): %m");
	} else if (status != 0) {
	if (WIFEXITED(status)) {
	syslog(LOG_ERR, "process %d exited, status %d\n", pid,
	WEXITSTATUS(status));
	}
	if (WIFSIGNALED(status)) {
	syslog(LOG_ERR,
	"process %d exited, signaled to death by %d\n",
	pid, WTERMSIG(status));
	}
	}
	break;
	}
	}

	void fcntl_unset(int fd, int flag)
	{
	int fdflags = fcntl(fd, F_GETFD, 0);
	if (fdflags != -1) fdflags = fcntl(STATUS_FD, F_SETFD,
	fdflags & ~flag);
	if (fdflags == -1) {
	syslog(LOG_ERR, "fcntl(): unable to unset %d: %m", flag);
	}
	}

	void spawn_service(const int si)
	{
	/* Note that there is logic that depends on this being 2 */
	const int FORKRATE_INTERVAL = 2;

	pid_t p;
	int i;
	char path[PATH_MAX];
	static char name_env[100], name_env2[100];
	struct centry *c;
	struct service * const s = &Services[si];
	time_t now = time(NULL);

	if (!s->name) {
	fatal("Serious software bug found: spawn_service() called on unnamed service!",
	EX_SOFTWARE);
	}

	/* update our fork rate */
	if(now - s->last_interval_start >= FORKRATE_INTERVAL) {
	int interval;

	s->forkrate = (s->interval_forks/2) + (s->forkrate/2);
	s->interval_forks = 0;
	s->last_interval_start += FORKRATE_INTERVAL;

	/* if there is an even wider window, however, we need
	* to account for a good deal of zeros, we can do this at once */
	interval = now - s->last_interval_start;

	if(interval > 2) {
	int skipped_intervals = interval / FORKRATE_INTERVAL;
	/* avoid a > 30 bit right shift) */
	if(skipped_intervals > 30) s->forkrate = 0;
	else {
	/* divide by 2^(skipped_intervals).
	* this is the logic mentioned in the comment above */
	s->forkrate >>= skipped_intervals;
	s->last_interval_start = now;
	}
	}
	}

	/* If we've been busy lately, we will refuse to fork! */
	/* (We schedule a wakeup call for sometime soon though to be
	* sure that we don't wait to do the fork that is required forever! */
	if(s->maxforkrate && s->forkrate >= s->maxforkrate) {
	struct event evt = (struct event ) xmalloc(sizeof(struct event));

	memset(evt, 0, sizeof(struct event));

	evt->name = xstrdup("forkrate wakeup call");
	evt->mark = time(NULL) + FORKRATE_INTERVAL;
	schedule_event(evt);

	return;
	}

	switch (p = fork()) {
	case -1:
	syslog(LOG_ERR, "can't fork process to run service %s: %m", s->name);
	break;

	case 0:
	/* Child - Release our pidfile lock. */
	if(pidfd != -1) close(pidfd);

	if (become_cyrus() != 0) {
	syslog(LOG_ERR, "can't change to the cyrus user");
	exit(1);
	}

	get_prog(path, sizeof(path), s->exec);
	if (dup2(s->stat[1], STATUS_FD) < 0) {
	syslog(LOG_ERR, "can't duplicate status fd: %m");
	exit(1);
	}
	if (dup2(s->socket, LISTEN_FD) < 0) {
	syslog(LOG_ERR, "can't duplicate listener fd: %m");
	exit(1);
	}

	fcntl_unset(STATUS_FD, FD_CLOEXEC);
	fcntl_unset(LISTEN_FD, FD_CLOEXEC);

	/* close all listeners */
	for (i = 0; i < nservices; i++) {
	if (Services[i].socket > 0) close(Services[i].socket);
	if (Services[i].stat[0] > 0) close(Services[i].stat[0]);
	if (Services[i].stat[1] > 0) close(Services[i].stat[1]);
	}
	limit_fds(s->maxfds);

	syslog(LOG_DEBUG, "about to exec %s", path);

	/* add service name to environment */
	snprintf(name_env, sizeof(name_env), "CYRUS_SERVICE=%s", s->name);
	putenv(name_env);
	snprintf(name_env2, sizeof(name_env2), "CYRUS_ID=%d", s->associate);
	putenv(name_env2);

	execv(path, s->exec);
	syslog(LOG_ERR, "couldn't exec %s: %m", path);
	exit(EX_OSERR);

	default: /* parent */
	s->ready_workers++;
	s->interval_forks++;
	s->nforks++;
	s->nactive++;

	/* add to child table */
	c = get_centry();
	c->pid = p;
	c->service_state = SERVICE_STATE_READY;
	c->si = si;
	c->next = ctable[p % child_table_size];
	ctable[p % child_table_size] = c;
	break;
	}

	}

	void schedule_event(struct event *a)
	{
	struct event *ptr;

	if (! a->name)
	fatal("Serious software bug found: schedule_event() called on unnamed event!",
	EX_SOFTWARE);

	if (!schedule \|\| a->mark < schedule->mark) {
	a->next = schedule;
	schedule = a;

	return;
	}
	for (ptr = schedule; ptr->next && ptr->next->mark <= a->mark;
	ptr = ptr->next) ;

	/* insert a */
	a->next = ptr->next;
	ptr->next = a;
	}

	void spawn_schedule(time_t now)
	{
	struct event a, b;
	int i;
	char path[PATH_MAX];
	pid_t p;
	struct centry *c;

	a = NULL;
	/* update schedule accordingly */
	while (schedule && schedule->mark <= now) {
	/* delete from schedule, insert into a */
	struct event *ptr = schedule;

	/* delete */
	schedule = schedule->next;

	/* insert */
	ptr->next = a;
	a = ptr;
	}

	/* run all events */
	while (a && a != schedule) {
	/* if a->exec is NULL, we just used the event to wake up,
	* so we actually don't need to exec anything at the moment */
	if(a->exec) {
	switch (p = fork()) {
	case -1:
	syslog(LOG_CRIT,
	"can't fork process to run event %s", a->name);
	break;

	case 0:
	/* Child - Release our pidfile lock. */
	if(pidfd != -1) close(pidfd);

	if (become_cyrus() != 0) {
	syslog(LOG_ERR, "can't change to the cyrus user");
	exit(1);
	}

	/* close all listeners */
	for (i = 0; i < nservices; i++) {
	if (Services[i].socket > 0) close(Services[i].socket);
	if (Services[i].stat[0] > 0) close(Services[i].stat[0]);
	if (Services[i].stat[1] > 0) close(Services[i].stat[1]);
	}
	limit_fds(256);

	get_prog(path, sizeof(path), a->exec);
	syslog(LOG_DEBUG, "about to exec %s", path);
	execv(path, a->exec);
	syslog(LOG_ERR, "can't exec %s on schedule: %m", path);
	exit(EX_OSERR);
	break;

	default:
	/* we don't wait for it to complete */

	/* add to child table */
	c = get_centry();
	c->pid = p;
	c->service_state = SERVICE_STATE_READY;
	c->si = SERVICE_NONE;
	c->next = ctable[p % child_table_size];
	ctable[p % child_table_size] = c;

	break;
	}
	} /* a->exec */

	/* reschedule as needed */
	b = a->next;
	if (a->period) {
	if(a->periodic) {
	a->mark = now + a->period;
	} else {
	struct tm *tm;
	int delta;
	/* Daily Event */
	while(a->mark <= now) {
	a->mark += a->period;
	}
	/* check for daylight savings fuzz... */
	tm = localtime(&(a->mark));
	if (tm->tm_hour != a->hour \|\| tm->tm_min != a->min) {
	/* calculate the same time on the new day */
	tm->tm_hour = a->hour;
	tm->tm_min = a->min;
	delta = mktime(tm) - a->mark;
	/* bring it within half a period either way */
	while (delta > (a->period/2)) delta -= a->period;
	while (delta < -(a->period/2)) delta += a->period;
	/* update the time */
	a->mark += delta;
	/* and let us know about the change */
	syslog(LOG_NOTICE, "timezone shift for %s - altering schedule by %d seconds", a->name, delta);
	}
	}
	/* reschedule a */
	schedule_event(a);
	} else {
	event_free(a);
	}
	/* examine next event */
	a = b;
	}
	}

	void reap_child(void)
	{
	int status;
	pid_t pid;
	struct centry *c;
	struct service *s;

	while ((pid = waitpid((pid_t) -1, &status, WNOHANG)) > 0) {
	if (WIFEXITED(status)) {
	syslog(LOG_DEBUG, "process %d exited, status %d", pid,
	WEXITSTATUS(status));
	}

	if (WIFSIGNALED(status)) {
	syslog(LOG_ERR, "process %d exited, signaled to death by %d",
	pid, WTERMSIG(status));
	}

	/* account for the child */
	c = ctable[pid % child_table_size];
	while(c && c->pid != pid) c = c->next;

	if (c && c->pid == pid) {
	s = ((c->si) != SERVICE_NONE) ? &Services[c->si] : NULL;

	/* paranoia */
	switch (c->service_state) {
	case SERVICE_STATE_READY:
	case SERVICE_STATE_BUSY:
	case SERVICE_STATE_UNKNOWN:
	case SERVICE_STATE_DEAD:
	break;
	default:
	syslog(LOG_CRIT,
	"service %s pid %d in ILLEGAL STATE: exited. Serious software bug or memory corruption detected!",
	s ? SERVICENAME(s->name) : "unknown", pid);
	syslog(LOG_DEBUG,
	"service %s pid %d in ILLEGAL state: forced to valid UNKNOWN state",
	s ? SERVICENAME(s->name) : "unknown", pid);
	c->service_state = SERVICE_STATE_UNKNOWN;
	}
	if (s) {
	/* update counters for known services */
	switch (c->service_state) {
	case SERVICE_STATE_READY:
	s->nactive--;
	s->ready_workers--;
	if (!in_shutdown && (WIFSIGNALED(status) \|\|
	(WIFEXITED(status) && WEXITSTATUS(status)))) {
	syslog(LOG_WARNING,
	"service %s pid %d in READY state: terminated abnormally",
	SERVICENAME(s->name), pid);
	}
	break;

	case SERVICE_STATE_DEAD:
	/* uh? either we got duplicate signals, or we are now MT */
	syslog(LOG_WARNING,
	"service %s pid %d in DEAD state: receiving duplicate signals",
	SERVICENAME(s->name), pid);
	break;

	case SERVICE_STATE_BUSY:
	s->nactive--;
	if (!in_shutdown && (WIFSIGNALED(status) \|\|
	(WIFEXITED(status) && WEXITSTATUS(status)))) {
	syslog(LOG_DEBUG,
	"service %s pid %d in BUSY state: terminated abnormally",
	SERVICENAME(s->name), pid);
	}
	break;

	case SERVICE_STATE_UNKNOWN:
	s->nactive--;
	syslog(LOG_WARNING,
	"service %s pid %d in UNKNOWN state: exited",
	SERVICENAME(s->name), pid);
	break;
	default:
	/* Shouldn't get here */
	break;
	}
	} else {
	/* children from spawn_schedule (events) or
	* children of services removed by reread_conf() */
	if (c->service_state != SERVICE_STATE_READY) {
	syslog(LOG_WARNING,
	"unknown service pid %d in state %d: exited (maybe using a service as an event,"
	" or a service was removed by SIGHUP?)",
	pid, c->service_state);
	}
	}
	c->service_state = SERVICE_STATE_DEAD;
	c->janitor_deadline = time(NULL) + 2;
	} else {
	/* Are we multithreaded now? we don't know this child */
	syslog(LOG_ERR,
	"received SIGCHLD from unknown child pid %d, ignoring",
	pid);
	/* FIXME: is this something we should take lightly? */
	}
	if (verbose && c && (c->si != SERVICE_NONE))
	syslog(LOG_DEBUG, "service %s now has %d ready workers\n",
	SERVICENAME(Services[c->si].name),
	Services[c->si].ready_workers);
	}
	}

	void init_janitor(void)
	{
	struct event evt = (struct event ) malloc(sizeof(struct event));

	if (!evt) fatal("out of memory", EX_UNAVAILABLE);
	memset(evt, 0, sizeof(struct event));

	gettimeofday(&janitor_mark, NULL);
	janitor_position = 0;

	evt->name = xstrdup("janitor periodic wakeup call");
	evt->period = 10;
	evt->periodic = 1;
	evt->mark = time(NULL) + 2;
	schedule_event(evt);
	}

	void child_janitor(time_t now)
	{
	int i;
	struct centry **p;
	struct centry *c;
	struct timeval rightnow;

	/* Estimate the number of entries to clean up in this sweep */
	gettimeofday(&rightnow, NULL);
	if (rightnow.tv_sec > janitor_mark.tv_sec + 1) {
	/* overflow protection */
	i = child_table_size;
	} else {
	double n;

	n = child_table_size * janitor_frequency *
	(double) ((rightnow.tv_sec - janitor_mark.tv_sec) * 1000000 +
	rightnow.tv_usec - janitor_mark.tv_usec ) / 1000000;
	if (n < child_table_size) {
	i = n;
	} else {
	i = child_table_size;
	}
	}

	while (i-- > 0) {
	p = &ctable[janitor_position++];
	janitor_position = janitor_position % child_table_size;
	while (*p) {
	c = *p;
	if (c->service_state == SERVICE_STATE_DEAD) {
	if (c->janitor_deadline < now) {
	*p = c->next;
	c->next = cfreelist;
	cfreelist = c;
	} else {
	p = &((*p)->next);
	}
	} else {
	p = &((*p)->next);
	}
	}
	}
	}

	/* Allow a clean shutdown on SIGQUIT */
	void sigquit_handler(int sig __attribute__((unused)))
	{
	struct sigaction action;

	/* Ignore SIGQUIT ourselves */
	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;
	action.sa_handler = SIG_IGN;
	if (sigaction(SIGQUIT, &action, (struct sigaction *) 0) < 0) {
	syslog(LOG_ERR, "sigaction: %m");
	}

	/* send our process group a SIGQUIT */
	if (kill(0, SIGQUIT) < 0) {
	syslog(LOG_ERR, "sigquit_handler: kill(0, SIGQUIT): %m");
	}

	/* Set a flag so main loop knows to shut down when
	all children have exited */
	in_shutdown = 1;

	syslog(LOG_INFO, "attempting clean shutdown on SIGQUIT");
	}

	static volatile sig_atomic_t gotsigchld = 0;

	void sigchld_handler(int sig __attribute__((unused)))
	{
	gotsigchld = 1;
	}

	static volatile int gotsighup = 0;

	void sighup_handler(int sig __attribute__((unused)))
	{
	gotsighup = 1;
	}

	void sigterm_handler(int sig __attribute__((unused)))
	{
	struct sigaction action;

	/* send all the other processes SIGTERM, then exit */
	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;
	action.sa_handler = SIG_IGN;
	if (sigaction(SIGTERM, &action, (struct sigaction *) 0) < 0) {
	syslog(LOG_ERR, "sigaction: %m");
	exit(1);
	}
	/* kill my process group */
	if (kill(0, SIGTERM) < 0) {
	syslog(LOG_ERR, "sigterm_handler: kill(0, SIGTERM): %m");
	}

	#if defined(HAVE_UCDSNMP) \|\| defined(HAVE_NETSNMP)
	/* tell master agent we're exiting */
	snmp_shutdown("cyrusMaster");
	#endif

	syslog(LOG_INFO, "exiting on SIGTERM/SIGINT");
	exit(0);
	}

	void sigalrm_handler(int sig __attribute__((unused)))
	{
	return;
	}

	void sighandler_setup(void)
	{
	struct sigaction action;

	sigemptyset(&action.sa_mask);
	action.sa_flags = 0;

	action.sa_handler = sighup_handler;
	#ifdef SA_RESTART
	action.sa_flags \|= SA_RESTART;
	#endif
	if (sigaction(SIGHUP, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGHUP: %m", 1);
	}

	action.sa_handler = sigalrm_handler;
	if (sigaction(SIGALRM, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGALRM: %m", 1);
	}

	/* Allow a clean shutdown on SIGQUIT */
	action.sa_handler = sigquit_handler;
	if (sigaction(SIGQUIT, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGQUIT: %m", 1);
	}

	/* Handle SIGTERM and SIGINT the same way -- kill
	* off our children! */
	action.sa_handler = sigterm_handler;
	if (sigaction(SIGTERM, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGTERM: %m", 1);
	}
	if (sigaction(SIGINT, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGINT: %m", 1);
	}

	action.sa_flags \|= SA_NOCLDSTOP;
	action.sa_handler = sigchld_handler;
	if (sigaction(SIGCHLD, &action, NULL) < 0) {
	fatal("unable to install signal handler for SIGCHLD: %m", 1);
	}
	}

	/*
	* Receives a message from a service.
	*
	* Returns zero if all goes well
	* 1 if no msg available
	* 2 if bad message received (incorrectly sized)
	* -1 on error (errno set)
	*/
	int read_msg(int fd, struct notify_message *msg)
	{
	ssize_t r;
	size_t off = 0;
	int s = sizeof(struct notify_message);

	while (s > 0) {
	do
	r = read(fd, msg + off, s);
	while ((r == -1) && (errno == EINTR));
	if (r <= 0) break;
	s -= r;
	off += r;
	}
	if ( ((r == 0) && (off == 0)) \|\|
	((r == -1) && (errno == EAGAIN)) )
	return 1;
	if (r == -1) return -1;
	if (s != 0) return 2;
	return 0;
	}

	void process_msg(const int si, struct notify_message *msg)
	{
	struct centry *c;
	/* si must NOT point to an invalid service */
	struct service * const s = &Services[si];;

	/* Search hash table with linked list for pid */
	c = ctable[msg->service_pid % child_table_size];
	while (c && c->pid != msg->service_pid) c = c->next;

	/* Did we find it? */
	if (!c \|\| c->pid != msg->service_pid) {
	/* If we don't know about the child, that means it has expired from
	* the child list, due to large message delivery delays. This is
	* indeed possible, although it is rare (Debian bug report).
	*
	* Note that this analysis depends on master's single-threaded
	* nature */
	syslog(LOG_WARNING,
	"service %s pid %d: receiving messages from long dead children",
	SERVICENAME(s->name), msg->service_pid);
	/* re-add child to list */
	c = get_centry();
	c->si = si;
	c->pid = msg->service_pid;
	c->service_state = SERVICE_STATE_DEAD;
	c->janitor_deadline = time(NULL) + 2;
	c->next = ctable[c->pid % child_table_size];
	ctable[c->pid % child_table_size] = c;
	}

	/* paranoia */
	if (si != c->si) {
	syslog(LOG_ERR,
	"service %s pid %d: changing from service %s due to received message",
	SERVICENAME(s->name), c->pid,
	((c->si != SERVICE_NONE && Services[c->si].name) ? Services[c->si].name : "unknown"));
	c->si = si;
	}
	switch (c->service_state) {
	case SERVICE_STATE_UNKNOWN:
	syslog(LOG_WARNING,
	"service %s pid %d in UNKNOWN state: processing message 0x%x",
	SERVICENAME(s->name), c->pid, msg->message);
	break;
	case SERVICE_STATE_READY:
	case SERVICE_STATE_BUSY:
	case SERVICE_STATE_DEAD:
	break;
	default:
	syslog(LOG_CRIT,
	"service %s pid %d in ILLEGAL state: detected. Serious software bug or memory corruption uncloaked while processing message 0x%x from child!",
	SERVICENAME(s->name), c->pid, msg->message);
	syslog(LOG_DEBUG,
	"service %s pid %d in ILLEGAL state: forced to valid UNKNOWN state",
	SERVICENAME(s->name), c->pid);
	c->service_state = SERVICE_STATE_UNKNOWN;
	break;
	}

	/* process message, according to state machine */
	switch (msg->message) {
	case MASTER_SERVICE_AVAILABLE:
	switch (c->service_state) {
	case SERVICE_STATE_READY:
	/* duplicate message? */
	syslog(LOG_WARNING,
	"service %s pid %d in READY state: sent available message but it is already ready",
	SERVICENAME(s->name), c->pid);
	break;

	case SERVICE_STATE_UNKNOWN:
	/* since state is unknwon, error in non-DoS way, i.e.
	* we don't increment ready_workers */
	syslog(LOG_DEBUG,
	"service %s pid %d in UNKNOWN state: now available and in READY state",
	SERVICENAME(s->name), c->pid);
	c->service_state = SERVICE_STATE_READY;
	break;

	case SERVICE_STATE_BUSY:
	if (verbose)
	syslog(LOG_DEBUG,
	"service %s pid %d in BUSY state: now available and in READY state",
	SERVICENAME(s->name), c->pid);
	c->service_state = SERVICE_STATE_READY;
	s->ready_workers++;
	break;

	case SERVICE_STATE_DEAD:
	/* echoes from the past... just ignore */
	break;

	default:
	/* Shouldn't get here */
	break;
	}
	break;

	case MASTER_SERVICE_UNAVAILABLE:
	switch (c->service_state) {
	case SERVICE_STATE_BUSY:
	/* duplicate message? */
	syslog(LOG_WARNING,
	"service %s pid %d in BUSY state: sent unavailable message but it is already busy",
	SERVICENAME(s->name), c->pid);
	break;

	case SERVICE_STATE_UNKNOWN:
	syslog(LOG_DEBUG,
	"service %s pid %d in UNKNOWN state: now unavailable and in BUSY state",
	SERVICENAME(s->name), c->pid);
	c->service_state = SERVICE_STATE_BUSY;
	break;

	case SERVICE_STATE_READY:
	if (verbose)
	syslog(LOG_DEBUG,
	"service %s pid %d in READY state: now unavailable and in BUSY state",
	SERVICENAME(s->name), c->pid);
	c->service_state = SERVICE_STATE_BUSY;
	s->ready_workers--;
	break;

	case SERVICE_STATE_DEAD:
	/* echoes from the past... just ignore */
	break;

	default:
	/* Shouldn't get here */
	break;
	}
	break;

	case MASTER_SERVICE_CONNECTION:
	switch (c->service_state) {
	case SERVICE_STATE_BUSY:
	s->nconnections++;
	if (verbose)
	syslog(LOG_DEBUG,
	"service %s pid %d in BUSY state: now serving connection",
	SERVICENAME(s->name), c->pid);
	break;

	case SERVICE_STATE_UNKNOWN:
	s->nconnections++;
	c->service_state = SERVICE_STATE_BUSY;
	syslog(LOG_DEBUG,
	"service %s pid %d in UNKNOWN state: now in BUSY state and serving connection",
	SERVICENAME(s->name), c->pid);
	break;

	case SERVICE_STATE_READY:
	syslog(LOG_ERR,
	"service %s pid %d in READY state: reported new connection, forced to BUSY state",
	SERVICENAME(s->name), c->pid);
	/* be resilient on face of a bogon source, so lets err to the side
	* of non-denial-of-service */
	c->service_state = SERVICE_STATE_BUSY;
	s->nconnections++;
	s->ready_workers--;

	case SERVICE_STATE_DEAD:
	/* echoes from the past... do the accounting */
	s->nconnections++;
	break;

	default:
	/* Shouldn't get here */
	break;
	}
	break;

	case MASTER_SERVICE_CONNECTION_MULTI:
	switch (c->service_state) {
	case SERVICE_STATE_READY:
	s->nconnections++;
	if (verbose)
	syslog(LOG_DEBUG,
	"service %s pid %d in READY state: serving one more multi-threaded connection",
	SERVICENAME(s->name), c->pid);
	break;

	case SERVICE_STATE_BUSY:
	syslog(LOG_ERR,
	"service %s pid %d in BUSY state: serving one more multi-threaded connection, forced to READY state",
	SERVICENAME(s->name), c->pid);
	/* be resilient on face of a bogon source, so lets err to the side
	* of non-denial-of-service */
	c->service_state = SERVICE_STATE_READY;
	s->nconnections++;
	s->ready_workers++;
	break;

	case SERVICE_STATE_UNKNOWN:
	s->nconnections++;
	c->service_state = SERVICE_STATE_READY;
	syslog(LOG_ERR,
	"service %s pid %d in UNKNOWN state: serving one more multi-threaded connection, forced to READY state",
	SERVICENAME(s->name), c->pid);
	break;

	case SERVICE_STATE_DEAD:
	/* echoes from the past... do the accounting */
	s->nconnections++;
	break;

	default:
	/* Shouldn't get here */
	break;
	}
	break;

	default:
	syslog(LOG_CRIT, "service %s pid %d: Software bug: unrecognized message 0x%x",
	SERVICENAME(s->name), c->pid, msg->message);
	break;
	}

	if (verbose)
	syslog(LOG_DEBUG, "service %s now has %d ready workers\n",
	SERVICENAME(s->name), s->ready_workers);
	}

	static char *tokenize(char p)
	{
	char *tokens = NULL; / allocated in increments of 10 */
	int i = 0;

	if (!p \|\| !p) return NULL; / sanity check */
	while (*p) {
	while (p && Uisspace(p)) p++; /* skip whitespace */

	if (!(i % 10)) tokens = xrealloc(tokens, (i+10) * sizeof(char *));

	/* got a token */
	tokens[i++] = p;
	while (p && !Uisspace(p)) p++;

	/* p is whitespace or end of cmd */
	if (p) p++ = '\0';
	}
	/* add a NULL on the end */
	if (!(i % 10)) tokens = xrealloc(tokens, (i+1) * sizeof(char *));
	if (!tokens) return NULL;
	tokens[i] = NULL;

	return tokens;
	}

	void add_start(const char name, struct entry e,
	void *rock __attribute__((unused)))
	{
	char *cmd = xstrdup(masterconf_getstring(e, "cmd", ""));
	char buf[256];
	char **tok;

	if (!strcmp(cmd,"")) {
	snprintf(buf, sizeof(buf), "unable to find command for %s", name);
	fatal(buf, EX_CONFIG);
	}

	tok = tokenize(cmd);
	if (!tok) fatal("out of memory", EX_UNAVAILABLE);
	run_startup(tok);
	free(tok);
	free(cmd);
	}

	void add_service(const char name, struct entry e, void *rock)
	{
	int ignore_err = rock ? 1 : 0;
	char *cmd = xstrdup(masterconf_getstring(e, "cmd", ""));
	int prefork = masterconf_getint(e, "prefork", 0);
	int babysit = masterconf_getswitch(e, "babysit", 0);
	int maxforkrate = masterconf_getint(e, "maxforkrate", 0);
	char *listen = xstrdup(masterconf_getstring(e, "listen", ""));
	char *proto = xstrdup(masterconf_getstring(e, "proto", "tcp"));
	char *max = xstrdup(masterconf_getstring(e, "maxchild", "-1"));
	rlim_t maxfds = (rlim_t) masterconf_getint(e, "maxfds", 256);
	int reconfig = 0;
	int i, j;

	if(babysit && prefork == 0) prefork = 1;
	if(babysit && maxforkrate == 0) maxforkrate = 10; /* reasonable safety */

	if (!strcmp(cmd,"") \|\| !strcmp(listen,"")) {
	char buf[256];
	snprintf(buf, sizeof(buf),
	"unable to find command or port for service '%s'", name);

	if (ignore_err) {
	syslog(LOG_WARNING, "WARNING: %s -- ignored", buf);
	goto done;
	}

	fatal(buf, EX_CONFIG);
	}

	/* see if we have an existing entry that can be reused */
	for (i = 0; i < nservices; i++) {
	/* skip non-primary instances */
	if (Services[i].associate > 0)
	continue;
	/* must have empty/same service name, listen and proto */
	if ((!Services[i].name \|\| !strcmp(Services[i].name, name)) &&
	(!Services[i].listen \|\| !strcmp(Services[i].listen, listen)) &&
	(!Services[i].proto \|\| !strcmp(Services[i].proto, proto)))
	break;
	}

	/* we have duplicate service names in the config file */
	if ((i < nservices) && Services[i].exec) {
	char buf[256];
	snprintf(buf, sizeof(buf), "multiple entries for service '%s'", name);

	if (ignore_err) {
	syslog(LOG_WARNING, "WARNING: %s -- ignored", buf);
	goto done;
	}

	fatal(buf, EX_CONFIG);
	}

	if (i == nservices) {
	/* either we don't have an existing entry or we are changing
	* the port parameters, so create a new service
	*/
	if (nservices == allocservices) {
	if (allocservices > SERVICE_MAX - 5)
	fatal("out of service structures, please restart", EX_UNAVAILABLE);
	Services = xrealloc(Services,
	(allocservices+=5) * sizeof(struct service));
	}
	memset(&Services[nservices++], 0, sizeof(struct service));

	Services[i].last_interval_start = time(NULL);
	}
	else if (Services[i].listen) reconfig = 1;

	if (!Services[i].name) Services[i].name = xstrdup(name);
	if (Services[i].listen) free(Services[i].listen);
	Services[i].listen = listen;
	listen = NULL; /* avoid freeing it */
	if (Services[i].proto) free(Services[i].proto);
	Services[i].proto = proto;
	proto = NULL; /* avoid freeing it */

	Services[i].exec = tokenize(cmd);
	cmd = NULL; /* avoid freeing it */
	if (!Services[i].exec) fatal("out of memory", EX_UNAVAILABLE);

	/* is this service actually there? */
	if (!verify_service_file(Services[i].exec)) {
	char buf[1024];
	snprintf(buf, sizeof(buf),
	"cannot find executable for service '%s'", name);
	/* if it is not, we're misconfigured, die. */
	fatal(buf, EX_CONFIG);
	}

	Services[i].maxforkrate = maxforkrate;
	Services[i].maxfds = maxfds;

	if (!strcmp(Services[i].proto, "tcp") \|\|
	!strcmp(Services[i].proto, "tcp4") \|\|
	!strcmp(Services[i].proto, "tcp6")) {
	Services[i].desired_workers = prefork;
	Services[i].babysit = babysit;
	Services[i].max_workers = atoi(max);
	if (Services[i].max_workers < 0) {
	Services[i].max_workers = INT_MAX;
	}
	} else {
	/* udp */
	if (prefork > 1) prefork = 1;
	Services[i].desired_workers = prefork;
	Services[i].max_workers = 1;
	}

	if (reconfig) {
	/* reconfiguring an existing service, update any other instances */
	for (j = 0; j < nservices; j++) {
	if (Services[j].associate > 0 && Services[j].listen &&
	Services[j].name && !strcmp(Services[j].name, name)) {
	Services[j].maxforkrate = Services[i].maxforkrate;
	Services[j].exec = Services[i].exec;
	Services[j].desired_workers = Services[i].desired_workers;
	Services[j].babysit = Services[i].babysit;
	Services[j].max_workers = Services[i].max_workers;
	}
	}
	}

	if (verbose > 2)
	syslog(LOG_DEBUG, "%s: service '%s' (%s, %s:%s, %d, %d, %d)",
	reconfig ? "reconfig" : "add",
	Services[i].name, cmd,
	Services[i].proto, Services[i].listen,
	Services[i].desired_workers,
	Services[i].max_workers,
	(int) Services[i].maxfds);

	done:
	free(cmd);
	free(listen);
	free(proto);
	free(max);
	return;
	}

	void add_event(const char name, struct entry e, void *rock)
	{
	int ignore_err = rock ? 1 : 0;
	char *cmd = xstrdup(masterconf_getstring(e, "cmd", ""));
	int period = 60 * masterconf_getint(e, "period", 0);
	int at = masterconf_getint(e, "at", -1), hour, min;
	time_t now = time(NULL);
	struct event *evt;

	if (!strcmp(cmd,"")) {
	char buf[256];
	snprintf(buf, sizeof(buf),
	"unable to find command or port for event '%s'", name);

	if (ignore_err) {
	syslog(LOG_WARNING, "WARNING: %s -- ignored", buf);
	return;
	}

	fatal(buf, EX_CONFIG);
	}

	evt = (struct event *) xmalloc(sizeof(struct event));
	evt->name = xstrdup(name);

	if (at >= 0 && ((hour = at / 100) <= 23) && ((min = at % 100) <= 59)) {
	struct tm *tm = localtime(&now);

	period = 86400; /* 24 hours */
	evt->periodic = 0;
	evt->hour = hour;
	evt->min = min;
	tm->tm_hour = hour;
	tm->tm_min = min;
	tm->tm_sec = 0;
	if ((evt->mark = mktime(tm)) < now) {
	/* already missed it, so schedule for next day */
	evt->mark += period;
	}
	}
	else {
	evt->periodic = 1;
	evt->mark = now;
	}
	evt->period = period;

	evt->exec = tokenize(cmd);
	if (!evt->exec) fatal("out of memory", EX_UNAVAILABLE);

	schedule_event(evt);
	}

	#ifdef HAVE_SETRLIMIT

	#ifdef RLIMIT_NOFILE
	# define RLIMIT_NUMFDS RLIMIT_NOFILE
	#else
	# ifdef RLIMIT_OFILE
	# define RLIMIT_NUMFDS RLIMIT_OFILE
	# endif
	#endif
	void limit_fds(rlim_t x)
	{
	struct rlimit rl;
	int r;

	rl.rlim_cur = x;
	rl.rlim_max = x;
	if (setrlimit(RLIMIT_NUMFDS, &rl) < 0) {
	syslog(LOG_ERR, "setrlimit: Unable to set file descriptors limit to %ld: %m", x);

	#ifdef HAVE_GETRLIMIT

	if (!getrlimit(RLIMIT_NUMFDS, &rl)) {
	syslog(LOG_ERR, "retrying with %ld (current max)", rl.rlim_max);
	rl.rlim_cur = rl.rlim_max;
	if (setrlimit(RLIMIT_NUMFDS, &rl) < 0) {
	syslog(LOG_ERR, "setrlimit: Unable to set file descriptors limit to %ld: %m", x);
	}
	}
	}


	if (verbose > 1) {
	r = getrlimit(RLIMIT_NUMFDS, &rl);
	syslog(LOG_DEBUG, "set maximum file descriptors to %ld/%ld", rl.rlim_cur,
	rl.rlim_max);
	}
	#else
	}
	#endif /* HAVE_GETRLIMIT */
	}
	#else
	void limit_fds(rlim_t x)
	{
	}
	#endif /* HAVE_SETRLIMIT */

	void reread_conf(void)
	{
	int i,j;
	struct event *ptr;
	struct centry *c;

	/* disable all services -
	they will be re-enabled if they appear in config file */
	for (i = 0; i < nservices; i++) Services[i].exec = NULL;

	/* read services */
	masterconf_getsection("SERVICES", &add_service, (void*) 1);

	for (i = 0; i < nservices; i++) {
	if (!Services[i].exec && Services[i].socket) {
	/* cleanup newly disabled services */

	if (verbose > 2)
	syslog(LOG_DEBUG, "disable: service %s socket %d pipe %d %d",
	Services[i].name, Services[i].socket,
	Services[i].stat[0], Services[i].stat[1]);

	/* Only free the service info on the primary */
	if(Services[i].associate == 0) {
	free(Services[i].listen);
	free(Services[i].proto);
	}
	Services[i].listen = NULL;
	Services[i].proto = NULL;
	Services[i].desired_workers = 0;

	/* send SIGHUP to all children */
	for (j = 0 ; j < child_table_size ; j++ ) {
	c = ctable[j];
	while (c != NULL) {
	if ((c->si == i) &&
	(c->service_state != SERVICE_STATE_DEAD)) {
	kill(c->pid, SIGHUP);
	}
	c = c->next;
	}
	}

	/* close all listeners */
	if (Services[i].socket > 0) {
	shutdown(Services[i].socket, SHUT_RDWR);
	close(Services[i].socket);
	}
	Services[i].socket = 0;
	}
	else if (Services[i].exec && !Services[i].socket) {
	/* initialize new services */

	service_create(&Services[i]);
	if (verbose > 2)
	syslog(LOG_DEBUG, "init: service %s socket %d pipe %d %d",
	Services[i].name, Services[i].socket,
	Services[i].stat[0], Services[i].stat[1]);
	}
	}

	/* remove existing events */
	while (schedule) {
	ptr = schedule;
	schedule = schedule->next;
	event_free(ptr);
	}
	schedule = NULL;

	/* read events */
	masterconf_getsection("EVENTS", &add_event, (void*) 1);

	/* reinit child janitor */
	init_janitor();

	/* send some feedback to admin */
	syslog(LOG_NOTICE,
	"Services reconfigured. %d out of %d (max %d) services structures are now in use",
	nservices, allocservices, SERVICE_MAX);
	}

	int main(int argc, char **argv)
	{
	const char *default_pidfile = MASTER_PIDFILE;
	const char *lock_suffix = ".lock";

	const char *pidfile = default_pidfile;
	char *pidfile_lock = NULL;

	int startup_pipe[2] = { -1, -1 };
	int pidlock_fd = -1;

	int i, opt, close_std = 1, daemon_mode = 0;
	extern int optind;
	extern char *optarg;

	char *alt_config = NULL;

	int fd;
	fd_set rfds;
	char *p = NULL;

	#ifdef HAVE_NETSNMP
	char *agentxsocket = NULL;
	int agentxpinginterval = -1;
	#endif

	time_t now;

	p = getenv("CYRUS_VERBOSE");
	if (p) verbose = atoi(p) + 1;
	#ifdef HAVE_NETSNMP
	while ((opt = getopt(argc, argv, "C:M:p:l:Ddj:P:x:")) != EOF) {
	#else
	while ((opt = getopt(argc, argv, "C:M:p:l:Ddj:")) != EOF) {
	#endif
	switch (opt) {
	case 'C': /* alt imapd.conf file */
	alt_config = optarg;
	break;
	case 'M': /* alt cyrus.conf file */
	MASTER_CONFIG_FILENAME = optarg;
	break;
	case 'l':
	/* user defined listen queue backlog */
	listen_queue_backlog = atoi(optarg);
	break;
	case 'p':
	/* Set the pidfile name */
	pidfile = optarg;
	break;
	case 'd':
	/* Daemon Mode */
	if(!close_std)
	fatal("Unable to both be debug and daemon mode", EX_CONFIG);
	daemon_mode = 1;
	break;
	case 'D':
	/* Debug Mode */
	if(daemon_mode)
	fatal("Unable to be both debug and daemon mode", EX_CONFIG);
	close_std = 0;
	break;
	case 'j':
	/* Janitor frequency */
	janitor_frequency = atoi(optarg);
	if(janitor_frequency < 1)
	fatal("The janitor period must be at least 1 second", EX_CONFIG);
	break;
	#ifdef HAVE_NETSNMP
	case 'P': /* snmp AgentXPingInterval */
	agentxpinginterval = atoi(optarg);
	break;
	case 'x': /* snmp AgentXSocket */
	agentxsocket = optarg;
	break;
	#endif
	default:
	break;
	}
	}

	masterconf_init("master", alt_config);

	/* zero out the children table */
	memset(&ctable, 0, sizeof(struct centry ) child_table_size);

	if (close_std) {
	/* close stdin/out/err */
	for (fd = 0; fd < 3; fd++) {
	close(fd);
	if (open("/dev/null", O_RDWR, 0) != fd)
	fatal("couldn't open /dev/null: %m", 2);
	}
	}

	/* we reserve fds 3 and 4 for children to communicate with us, so they
	better be available. */
	for (fd = 3; fd < 5; fd++) {
	close(fd);
	if (dup(0) != fd) fatal("couldn't dup fd 0: %m", 2);
	}

	/* Pidfile Algorithm in Daemon Mode. This is a little subtle because
	* we want to ensure that we can report an error to our parent if the
	* child fails to lock the pidfile.
	*
	* [A] Create/lock pidfile.lock. If locked, exit(failure).
	* [A] Create a pipe
	* [A] Fork [B]
	* [A] Block on reading exit code from pipe
	* [B] Create/lock pidfile. If locked, write failure code to pipe and
	* exit(failure)
	* [B] write pid to pidfile
	* [B] write success code to pipe & finish starting up
	* [A] unlink pidfile.lock and exit(code read from pipe)
	*
	*/
	if(daemon_mode) {
	/* Daemonize */
	pid_t pid = -1;

	pidfile_lock = xmalloc(strlen(pidfile) + strlen(lock_suffix) + 1);

	strcpy(pidfile_lock, pidfile);
	strcat(pidfile_lock, lock_suffix);

	pidlock_fd = open(pidfile_lock, O_CREAT\|O_TRUNC\|O_RDWR, 0644);
	if(pidlock_fd == -1) {
	syslog(LOG_ERR, "can't open pidfile lock: %s (%m)", pidfile_lock);
	exit(EX_OSERR);
	} else {
	if(lock_nonblocking(pidlock_fd)) {
	syslog(LOG_ERR, "can't get exclusive lock on %s",
	pidfile_lock);
	exit(EX_TEMPFAIL);
	}
	}

	if(pipe(startup_pipe) == -1) {
	syslog(LOG_ERR, "can't create startup pipe (%m)");
	exit(EX_OSERR);
	}

	do {
	pid = fork();

	if ((pid == -1) && (errno == EAGAIN)) {
	syslog(LOG_WARNING, "master fork failed (sleeping): %m");
	sleep(5);
	}
	} while ((pid == -1) && (errno == EAGAIN));

	if (pid == -1) {
	fatal("fork error", EX_OSERR);
	} else if (pid != 0) {
	int exit_code;

	/* Parent, wait for child */
	if(read(startup_pipe[0], &exit_code, sizeof(exit_code)) == -1) {
	syslog(LOG_ERR, "could not read from startup_pipe (%m)");
	unlink(pidfile_lock);
	exit(EX_OSERR);
	} else {
	unlink(pidfile_lock);
	exit(exit_code);
	}
	}

	/* Child! */
	close(startup_pipe[0]);

	free(pidfile_lock);

	/*
	* We're now running in the child. Lose our controlling terminal
	* and obtain a new process group.
	*/
	if (setsid() == -1) {
	int r;
	int exit_result = EX_OSERR;

	/* Tell our parent that we failed. */
	r = write(startup_pipe[1], &exit_result, sizeof(exit_result));

	fatal("setsid failure", EX_OSERR);
	}
	}

	limit_fds(RLIM_INFINITY);

	/* Write out the pidfile */
	pidfd = open(pidfile, O_CREAT\|O_RDWR, 0644);
	if(pidfd == -1) {
	int exit_result = EX_OSERR;
	int r;

	/* Tell our parent that we failed. */
	r = write(startup_pipe[1], &exit_result, sizeof(exit_result));

	syslog(LOG_ERR, "can't open pidfile: %m");
	exit(EX_OSERR);
	} else {
	char buf[100];

	if(lock_nonblocking(pidfd)) {
	int exit_result = EX_OSERR;
	int r;

	/* Tell our parent that we failed. */
	r = write(startup_pipe[1], &exit_result, sizeof(exit_result));

	fatal("cannot get exclusive lock on pidfile (is another master still running?)", EX_OSERR);
	} else {
	int pidfd_flags = fcntl(pidfd, F_GETFD, 0);
	if (pidfd_flags != -1)
	pidfd_flags = fcntl(pidfd, F_SETFD,
	pidfd_flags \| FD_CLOEXEC);
	if (pidfd_flags == -1) {
	int exit_result = EX_OSERR;
	int r;

	/* Tell our parent that we failed. */
	r = write(startup_pipe[1], &exit_result, sizeof(exit_result));

	fatal("unable to set close-on-exec for pidfile: %m", EX_OSERR);
	}

	/* Write PID */
	snprintf(buf, sizeof(buf), "%lu\n", (unsigned long int)getpid());
	if(lseek(pidfd, 0, SEEK_SET) == -1 \|\|
	ftruncate(pidfd, 0) == -1 \|\|
	write(pidfd, buf, strlen(buf)) == -1) {
	int exit_result = EX_OSERR;
	int r;

	/* Tell our parent that we failed. */
	r = write(startup_pipe[1], &exit_result, sizeof(exit_result));

	fatal("unable to write to pidfile: %m", EX_OSERR);
	}
	if (fsync(pidfd))
	fatal("unable to sync pidfile: %m", EX_OSERR);
	}
	}

	if(daemon_mode) {
	int exit_result = 0;

	/* success! */
	if(write(startup_pipe[1], &exit_result, sizeof(exit_result)) == -1) {
	syslog(LOG_ERR,
	"could not write success result to startup pipe (%m)");
	exit(EX_OSERR);
	}

	close(startup_pipe[1]);
	if(pidlock_fd != -1) close(pidlock_fd);
	}

	syslog(LOG_NOTICE, "process started");

	#if defined(HAVE_UCDSNMP) \|\| defined(HAVE_NETSNMP)
	/* initialize SNMP agent */

	/* make us a agentx client. */
	#ifdef HAVE_NETSNMP
	netsnmp_enable_subagent();

	netsnmp_ds_set_boolean(NETSNMP_DS_LIBRARY_ID,
	NETSNMP_DS_LIB_ALARM_DONT_USE_SIG, 1);
	if (agentxpinginterval >= 0)
	netsnmp_ds_set_int(NETSNMP_DS_APPLICATION_ID,
	NETSNMP_DS_AGENT_AGENTX_PING_INTERVAL, agentxpinginterval);

	if (agentxsocket != NULL)
	netsnmp_ds_set_string(NETSNMP_DS_APPLICATION_ID,
	NETSNMP_DS_AGENT_X_SOCKET, agentxsocket);
	#else
	ds_set_boolean(DS_APPLICATION_ID, DS_AGENT_ROLE, 1);
	#endif

	/* initialize the agent library */
	init_agent("cyrusMaster");

	init_cyrusMasterMIB();

	init_snmp("cyrusMaster");
	#endif

	masterconf_getsection("START", &add_start, NULL);
	masterconf_getsection("SERVICES", &add_service, NULL);
	masterconf_getsection("EVENTS", &add_event, NULL);

	/* set signal handlers */
	sighandler_setup();

	/* initialize services */
	for (i = 0; i < nservices; i++) {
	service_create(&Services[i]);
	if (verbose > 2)
	syslog(LOG_DEBUG, "init: service %s socket %d pipe %d %d",
	Services[i].name, Services[i].socket,
	Services[i].stat[0], Services[i].stat[1]);
	}

	if (become_cyrus_early) {
	if (become_cyrus() != 0) {
	syslog(LOG_ERR, "can't change to the cyrus user: %m");
	exit(1);
	}
	}

	/* init ctable janitor */
	init_janitor();

	/* ok, we're going to start spawning like mad now */
	syslog(LOG_NOTICE, "ready for work");

	now = time(NULL);
	for (;;) {
	int r, i, maxfd, total_children = 0;
	struct timeval tv, *tvptr;
	struct notify_message msg;
	#if defined(HAVE_UCDSNMP) \|\| defined(HAVE_NETSNMP)
	int blockp = 0;
	#endif

	/* run any scheduled processes */
	if (!in_shutdown)
	spawn_schedule(now);

	/* reap first, that way if we need to babysit we will */
	if (gotsigchld) {
	/* order matters here */
	gotsigchld = 0;
	reap_child();
	}

	/* do we have any services undermanned? */
	for (i = 0; i < nservices; i++) {
	total_children += Services[i].nactive;
	if (!in_shutdown) {
	if (Services[i].exec /* enabled */ &&
	(Services[i].nactive < Services[i].max_workers) &&
	(Services[i].ready_workers < Services[i].desired_workers)) {
	spawn_service(i);
	} else if (Services[i].exec
	&& Services[i].babysit
	&& Services[i].nactive == 0) {
	syslog(LOG_ERR,
	"lost all children for service: %s. " \
	"Applying babysitter.",
	Services[i].name);
	spawn_service(i);
	} else if (!Services[i].exec /* disabled */ &&
	Services[i].name /* not yet removed */ &&
	Services[i].nactive == 0) {
	if (verbose > 2)
	syslog(LOG_DEBUG, "remove: service %s pipe %d %d",
	Services[i].name,
	Services[i].stat[0], Services[i].stat[1]);

	/* Only free the service info on the primary */
	if (Services[i].associate == 0) {
	free(Services[i].name);
	}
	Services[i].name = NULL;
	Services[i].nforks = 0;
	Services[i].nactive = 0;
	Services[i].nconnections = 0;
	Services[i].associate = 0;

	if (Services[i].stat[0] > 0) close(Services[i].stat[0]);
	if (Services[i].stat[1] > 0) close(Services[i].stat[1]);
	memset(Services[i].stat, 0, sizeof(Services[i].stat));
	}
	}
	}

	if (in_shutdown && total_children == 0) {
	syslog(LOG_NOTICE, "All children have exited, closing down");
	exit(0);
	}

	if (gotsighup) {
	syslog(LOG_NOTICE, "got SIGHUP");
	gotsighup = 0;
	reread_conf();
	}

	FD_ZERO(&rfds);
	maxfd = 0;
	for (i = 0; i < nservices; i++) {
	int x = Services[i].stat[0];

	int y = Services[i].socket;

	/* messages */
	if (x > 0) {
	if (verbose > 2)
	syslog(LOG_DEBUG, "listening for messages from %s",
	Services[i].name);
	FD_SET(x, &rfds);
	}
	if (x > maxfd) maxfd = x;

	/* connections */
	if (y > 0 && Services[i].ready_workers == 0 &&
	Services[i].nactive < Services[i].max_workers) {
	if (verbose > 2)
	syslog(LOG_DEBUG, "listening for connections for %s",
	Services[i].name);
	FD_SET(y, &rfds);
	if (y > maxfd) maxfd = y;
	}

	/* paranoia */
	if (Services[i].ready_workers < 0) {
	syslog(LOG_ERR, "%s has %d workers?!?", Services[i].name,
	Services[i].ready_workers);
	}
	}
	maxfd++; /* need 1 greater than maxfd */

	/* how long to wait? - do now so that any scheduled wakeup
	* calls get accounted for*/
	tvptr = NULL;
	if (schedule) {
	if (now < schedule->mark) tv.tv_sec = schedule->mark - now;
	else tv.tv_sec = 0;
	tv.tv_usec = 0;
	tvptr = &tv;
	}

	#if defined(HAVE_UCDSNMP) \|\| defined(HAVE_NETSNMP)
	if (tvptr == NULL) blockp = 1;
	snmp_select_info(&maxfd, &rfds, tvptr, &blockp);
	#endif
	errno = 0;
	r = select(maxfd, &rfds, NULL, NULL, tvptr);
	if (r == -1 && errno == EAGAIN) continue;
	if (r == -1 && errno == EINTR) continue;
	if (r == -1) {
	/* uh oh */
	fatal("select failed: %m", 1);
	}

	#if defined(HAVE_UCDSNMP) \|\| defined(HAVE_NETSNMP)
	/* check for SNMP queries */
	snmp_read(&rfds);
	snmp_timeout();
	#endif
	for (i = 0; i < nservices; i++) {
	int x = Services[i].stat[0];
	int y = Services[i].socket;
	int j;

	if (FD_ISSET(x, &rfds)) {
	while ((r = read_msg(x, &msg)) == 0)
	process_msg(i, &msg);

	if (r == 2) {
	syslog(LOG_ERR,
	"got incorrectly sized response from child: %x", i);
	continue;
	}
	if (r < 0) {
	syslog(LOG_ERR,
	"error while receiving message from child %x: %m", i);
	continue;
	}
	}

	if (!in_shutdown && Services[i].exec &&
	Services[i].nactive < Services[i].max_workers) {
	/* bring us up to desired_workers */
	for (j = Services[i].ready_workers;
	j < Services[i].desired_workers;
	j++)
	{
	spawn_service(i);
	}

	if (Services[i].ready_workers == 0 &&
	FD_ISSET(y, &rfds)) {
	/* huh, someone wants to talk to us */
	spawn_service(i);
	}
	}
	}
	now = time(NULL);
	child_janitor(now);

	#ifdef HAVE_NETSNMP
	run_alarms();
	#endif
	}
	}
	diff --git a/ptclient/afskrb.c b/ptclient/afskrb.c
	index e6b1b9c30..43d796288 100644
	--- a/ptclient/afskrb.c
	+++ b/ptclient/afskrb.c
	@@ -1,561 +1,559 @@
	/* afskrb.c - AFS PTS Group (Kerberos Canonicalization) Backend to ptloader
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: afskrb.c,v 1.19 2010/01/06 17:01:57 murch Exp $
	*/
	-
	#include <config.h>
	#include "ptloader.h"
	#include "exitcodes.h"
	#include "util.h"
	#include "xmalloc.h"

	#ifdef HAVE_AFSKRB
	-
	#include <string.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <sys/param.h>
	#include <fcntl.h>
	#include <sys/socket.h>
	#include <sys/un.h>
	#include <sys/uio.h>

	#ifdef AFSPTS_USE_KRB5
	#include <krb5.h>
	#else
	#include <krb.h>
	#endif

	#include "auth_pts.h"
	#include "libconfig.h"
	#include "strhash.h"

	/* AFS stuff */
	-#include <des.h> /* for int32, necessary for the AFS includes below */
	+#include <lock.h>
	#include <afs/ptserver.h>
	#include <afs/pterror.h>
	#include <afs/cellconfig.h>
	#include <rx/rxkad.h>
	#include <afs/auth.h>

	/* blame transarc i've been told */
	#ifndef AFSCONF_CLIENTNAME
	#include <afs/dirpath.h>
	#define AFSCONF_CLIENTNAME AFSDIR_CLIENT_ETC_DIRPATH
	#endif

	/* Sanity Check */
	#if PTS_DB_KEYSIZE < PR_MAXNAMELEN
	#error PTS_DB_KEYSIZE is smaller than PR_MAXNAMELEN
	#endif

	static const char *localrealms = NULL;

	int is_local_realm(const char *realm)
	{
	const char *val = localrealms;

	if(!val \|\| !realm) return 0;

	while (*val) {
	char buf[1024];
	size_t len;
	char *p;

	for (p = (char ) val; p && !Uisspace(*p); p++);
	len = p-val;
	if(len >= sizeof(buf))
	len = sizeof(buf) - 1;
	memcpy(buf, val, len);
	buf[len] = '\0';

	if (!strcasecmp(realm,buf)) {
	return 1;
	}
	val = p;
	while (val && Uisspace(val)) val++;
	}

	return 0;
	}

	#ifdef AFSPTS_USE_KRB5

	/*
	* Convert 'identifier' into canonical form.
	* Returns a pointer to a static buffer containing the canonical form
	* or NULL if 'identifier' is invalid.
	*/
	static char afspts_canonifyid(const char identifier, size_t len)
	{
	static char *retbuf = NULL;
	char *tmp = NULL;
	krb5_context context;
	krb5_principal princ, princ_dummy;
	char *realm;
	char *realmbegin;
	int striprealm = 0;
	char *identifier2;

	if(retbuf) free(retbuf);
	retbuf = NULL;

	if(!identifier) return NULL;
	if(!len) len = strlen(identifier);

	if (strcasecmp(identifier, "anonymous") == 0)
	return "anonymous";

	if (strcasecmp(identifier, "anyone") == 0)
	return "anyone";

	identifier2 = strdup(identifier);
	if (tmp = strchr(identifier2, '+')) {
	syslog(LOG_DEBUG, "afspts_canonifyid stripping: %s", identifier2);
	tmp[0] = 0;
	syslog(LOG_DEBUG, "afspts_canonifyid stripped: %s", identifier2);
	}

	if (krb5_init_context(&context)) {
	syslog(LOG_ERR, "afspts_canonifyid krb5_init_context failed");
	return NULL;
	}

	if (krb5_parse_name(context,identifier2,&princ))
	{
	krb5_free_context(context);
	free(identifier2);
	syslog(LOG_ERR, "afspts_canonifyid krb5_parse_name failed");
	return NULL;
	}
	free(identifier2);

	if(config_getswitch(IMAPOPT_PTSKRB5_STRIP_DEFAULT_REALM)) {
	/* get local realm */
	if (krb5_get_default_realm(context,&realm))
	{
	krb5_free_principal(context,princ);
	krb5_free_context(context);
	syslog(LOG_ERR, "afspts_canonifyid krb5_get_default_realm failed");
	return NULL;
	}

	/* build dummy princ to compare realms */
	if (krb5_build_principal(context,&princ_dummy,
	strlen(realm),realm,"dummy",0))
	{
	krb5_free_principal(context,princ);
	krb5_free_context(context);
	free(realm);
	syslog(LOG_ERR, "afspts_canonifyid krb5_build_principal failed");
	return NULL;
	}

	/* is this principal local ? */
	if (krb5_realm_compare(context,princ,princ_dummy))
	{
	striprealm = 1;
	}

	/* done w/ dummy princ free it & realm */
	krb5_free_principal(context,princ_dummy);
	free(realm);
	}

	if (config_getswitch(IMAPOPT_PTSKRB5_CONVERT524)) {
	char nbuf[64], ibuf[64], rbuf[64];

	if (krb5_524_conv_principal(context, princ, nbuf, ibuf, rbuf)) {
	krb5_free_principal(context,princ);
	krb5_free_context(context);
	return NULL;
	}

	retbuf = xmalloc(3*64 + 3);
	sprintf(retbuf, "%s%s%s%s%s", nbuf,
	ibuf[0] ? "." : "", ibuf,
	rbuf[0] ? "@" : "", rbuf);
	} else {
	/* get the text version of princ */
	if (krb5_unparse_name(context,princ,&retbuf))
	{
	krb5_free_principal(context,princ);
	krb5_free_context(context);
	syslog(LOG_ERR, "afspts_canonifyid krb5_unparse_name failed");
	return NULL;
	}
	}

	/* we have the canonical name pointed to by p -- strip realm if local */
	realmbegin = strrchr(retbuf, '@');
	if(realmbegin) {
	if(!striprealm) {
	realm = realmbegin+1;
	if(is_local_realm(realm))
	striprealm = 1;
	}

	if(striprealm) {
	*realmbegin = '\0';
	} else {
	/* Force realm to uppercase */
	while(*(++realmbegin)) {
	realmbegin = toupper(realmbegin);
	}
	}
	}

	krb5_free_principal(context,princ);
	krb5_free_context(context);
	return retbuf;
	}


	#else /* AFSPTS_USE_KRB5 not defined */

	/* Sanity Check */
	# if PTS_DB_KEYSIZE < MAX_K_NAME_SZ
	# error PTS_DB_KEYSIZE is smaller than MAX_K_NAME_SZ
	# endif

	/* where is krb.equiv? */
	# ifndef KRB_MAPNAME
	# define KRB_MAPNAME (SYSCONFDIR "/krb.equiv")
	# endif

	/*
	* Parse a line 'src' from an /etc/krb.equiv file.
	* Sets the buffer pointed to by 'principal' to be the kerberos
	* identity and sets the buffer pointed to by 'localuser' to
	* be the local user. Both buffers must be of size one larger than
	* MAX_K_NAME_SZ. Returns 1 on success, 0 on failure.
	*/
	static int parse_krbequiv_line(const char *src,
	char *principal,
	char *localuser)
	{
	int i;

	while (Uisspace(*src)) src++;
	if (!*src) return 0;

	for (i = 0; src && !Uisspace(src); i++) {
	if (i >= MAX_K_NAME_SZ) return 0;
	principal++ = src++;
	}
	*principal = 0;

	if (!Uisspace(src)) return 0; / Need at least one separator */
	while (Uisspace(*src)) src++;
	if (!*src) return 0;

	for (i = 0; src && !Uisspace(src); i++) {
	if (i >= MAX_K_NAME_SZ) return 0;
	localuser++ = src++;
	}
	*localuser = 0;
	return 1;
	}

	/*
	* Map a remote kerberos principal to a local username. If a mapping
	* is found, a pointer to the local username is returned. Otherwise,
	* a NULL pointer is returned.
	* Eventually, this may be more sophisticated than a simple file scan.
	*/
	static char auth_map_krbid(const char real_aname,
	const char *real_inst,
	const char *real_realm)
	{
	static char localuser[MAX_K_NAME_SZ + 1];
	char principal[MAX_K_NAME_SZ + 1];
	char aname[ANAME_SZ];
	char inst[INST_SZ];
	char realm[REALM_SZ];
	char lrealm[REALM_SZ];
	char krbhst[MAX_HSTNM];
	char *p;
	char buf[1024];
	FILE *mapfile;

	if (!(mapfile = fopen(KRB_MAPNAME, "r"))) {
	/* If the file can't be opened, don't do mappings */
	return 0;
	}

	for (;;) {
	if (!fgets(buf, sizeof(buf), mapfile)) break;
	if (parse_krbequiv_line(buf, principal, localuser) == 0 \|\|
	kname_parse(aname, inst, realm, principal) != 0) {
	/* Ignore badly formed lines */
	continue;
	}
	if (!strcmp(aname, real_aname) && !strcmp(inst, real_inst) &&
	!strcmp(realm, real_realm)) {
	fclose(mapfile);

	aname[0] = inst[0] = realm[0] = '\0';
	if (kname_parse(aname, inst, realm, localuser) != 0) {
	return 0;
	}

	/* Upcase realm name */
	for (p = realm; *p; p++) {
	if (Uislower(p)) p = toupper(*p);
	}

	if (*realm) {
	if (krb_get_lrealm(lrealm,1) == 0 &&
	strcmp(lrealm, realm) == 0) {
	*realm = 0;
	}
	else if (krb_get_krbhst(krbhst, realm, 1)) {
	return 0; /* Unknown realm */
	}
	}

	strcpy(localuser, aname);
	if (*inst) {
	strcat(localuser, ".");
	strcat(localuser, inst);
	}
	if (*realm) {
	strcat(localuser, "@");
	strcat(localuser, realm);
	}

	return localuser;
	}
	}

	fclose(mapfile);
	return 0;
	}

	/*
	* Convert 'identifier' into canonical form.
	* Returns a pointer to a static buffer containing the canonical form
	* or NULL if 'identifier' is invalid.
	*/
	static char afspts_canonifyid(const char identifier, size_t len)
	{
	static char retbuf[MAX_K_NAME_SZ+1];
	char aname[ANAME_SZ];
	char inst[INST_SZ];
	char realm[REALM_SZ];
	char lrealm[REALM_SZ];
	char krbhst[MAX_HSTNM];
	char *canon_buf;
	char *p;

	if(!len) len = strlen(identifier);

	canon_buf = xmalloc(len + 1);
	memcpy(canon_buf, identifier, len);
	canon_buf[len] = '\0';

	aname[0] = inst[0] = realm[0] = '\0';
	if (kname_parse(aname, inst, realm, canon_buf) != 0) {
	free(canon_buf);
	return 0;
	}

	free(canon_buf);

	/* Upcase realm name */
	for (p = realm; *p; p++) {
	if (Uislower(p)) p = toupper(*p);
	}

	if (*realm) {
	if (krb_get_lrealm(lrealm,1) == 0 &&
	strcmp(lrealm, realm) == 0) {
	*realm = 0;
	}
	else if (krb_get_krbhst(krbhst, realm, 1)) {
	return 0; /* Unknown realm */
	}
	}

	/* Check for krb.equiv remappings. */
	p = auth_map_krbid(aname, inst, realm);
	if (p) {
	strcpy(retbuf, p);
	return retbuf;
	}

	strcpy(retbuf, aname);
	if (*inst) {
	strcat(retbuf, ".");
	strcat(retbuf, inst);
	}
	if (*realm && !is_local_realm(realm)) {
	strcat(retbuf, "@");
	strcat(retbuf, realm);
	}

	return retbuf;
	}
	#endif /* AFSPTS_USE_KRB5 */

	/* API */

	static void myinit(void)
	{
	int r = pr_Initialize (1L, AFSCONF_CLIENTNAME, config_getstring(IMAPOPT_AFSPTS_MYCELL));
	if (r) {
	syslog(LOG_DEBUG, "pr_Initialize failed: %d", r);
	fatal("pr_initialize failed", EC_TEMPFAIL);
	}

	localrealms = config_getstring(IMAPOPT_AFSPTS_LOCALREALMS);

	return;
	}

	static struct auth_state myauthstate(const char identifier,
	size_t size,
	const char *reply, int dsize)
	{
	const char *canon_id = afspts_canonifyid(identifier, size);
	char canon_id_tmp[PTS_DB_KEYSIZE+1];
	namelist groups;
	int i, rc;
	struct auth_state *newstate;

	if (canon_id == NULL) {
	syslog(LOG_ERR, "afspts_canonifyid failed for %s", identifier);
	return NULL;
	}

	*reply = NULL;
	size = strlen(canon_id);

	memset(&groups, 0, sizeof(groups));
	groups.namelist_len = 0;
	groups.namelist_val = NULL;

	/* canon_id_tmp is used because AFS would otherwise trample
	* on our nice canonical user id */
	strlcpy(canon_id_tmp,canon_id,sizeof(canon_id_tmp));
	if ((rc = pr_ListMembers(canon_id_tmp, &groups))) {
	/* Failure may indicate that we need new tokens */
	pr_End();
	rc = pr_Initialize (1L, AFSCONF_CLIENTNAME, 0);
	if (rc) {
	syslog(LOG_DEBUG, "pr_Initialize failed: %d", rc);
	fatal("pr_Initialize failed", EC_TEMPFAIL);
	}
	/* Okay, rerun it now */
	rc = pr_ListMembers(canon_id_tmp, &groups);
	}

	/* Don't die because of afs, but log the error */
	if(rc) {
	syslog(LOG_ERR, "pr_ListMembers %s: %s", canon_id, error_message(rc));
	}

	/* fill in our new state structure */
	*dsize = sizeof(struct auth_state) +
	(groups.namelist_len * sizeof(struct auth_ident));
	newstate = (struct auth_state ) xzmalloc(dsize);

	strcpy(newstate->userid.id, canon_id);
	newstate->userid.hash = strhash(canon_id);

	/* If we get a permission error, assume it may be temporary
	authentication problem, and cache only for a minute.
	Should negative cache time be configurable? */
	if (rc == PRPERM) {
	newstate->mark = time(0) + 60 -
	(config_getint(IMAPOPT_PTSCACHE_TIMEOUT) > 60)?
	config_getint(IMAPOPT_PTSCACHE_TIMEOUT) : 60;
	} else
	newstate->mark = time(0);

	newstate->ngroups = groups.namelist_len;
	/* store group list in contiguous array for easy storage in the database */
	memset(newstate->groups, 0, newstate->ngroups * sizeof(struct auth_ident));
	for (i = 0; i < newstate->ngroups; i++) {
	strlcpy(newstate->groups[i].id, groups.namelist_val[i],
	sizeof(newstate->groups[i].id));
	newstate->groups[i].hash = strhash(groups.namelist_val[i]);
	/* don't free groups.namelist_val[i]. Something else currently
	* takes care of that data.
	*/
	}
	if (groups.namelist_val != NULL) {
	free(groups.namelist_val);
	}

	return newstate;
	}

	#else /* HAVE_AFSKRB */

	static void myinit(void)
	{
	fatal("PTS module (afskrb) not compiled in", EC_CONFIG);
	}

	static struct auth_state *myauthstate(
	const char *identifier __attribute__((unused)),
	size_t size __attribute__((unused)),
	const char **reply __attribute__((unused)),
	int *dsize __attribute__((unused)))
	{
	fatal("PTS module (afskrb) not compiled in", EC_CONFIG);
	return NULL;
	}

	#endif /* HAVE_AFSKRB */

	struct pts_module pts_afskrb =
	{
	"afskrb", /* name */

	&myinit,
	&myauthstate,
	};
	diff --git a/ptclient/ptexpire.c b/ptclient/ptexpire.c
	index a0a2e64bf..a4e9f69a5 100644
	--- a/ptclient/ptexpire.c
	+++ b/ptclient/ptexpire.c
	@@ -1,169 +1,169 @@
	/*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: ptexpire.c,v 1.23 2010/01/06 17:01:57 murch Exp $
	*/

	/* This program purges old entries from the database. It holds an exclusive
	* lock throughout the process.
	*
	* NOTE: by adding the alt_file flag, we let exit() handle the cleanup of
	* the lock file's fd. That's bad in principal but not in practice. We do
	* to make the code easier to read.
	*/

	#include <config.h>

	#include <sys/param.h>
	#ifndef MAXPATHLEN
	#define MAXPATHLEN MAXPATHNAMELEN
	#endif

	#include <stdlib.h>
	#include <stdio.h>
	#include <string.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <fcntl.h>

	#include "auth_pts.h"
	#include "cyrusdb.h"
	#include "exitcodes.h"
	#include "global.h"
	#include "libconfig.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "xmalloc.h"

	static char rcsid[] = "$Id: ptexpire.c,v 1.23 2010/01/06 17:01:57 murch Exp $";

	/* global */
	time_t timenow;
	time_t expire_time = (36060); /* 3 Hours */

	int config_need_data = 0;

	static int expire_p(void *rockp __attribute__((unused)),
	const char *key __attribute__((unused)),
	int keylen __attribute__((unused)),
	const char *data,
	int datalen __attribute__((unused)))
	{
	struct auth_state authstate = (struct auth_state )data;
	if (authstate->mark + expire_time < timenow) {
	return 1;
	}
	return 0; /* skip this one */
	}

	static int expire_cb(void *rockp,
	const char *key, int keylen,
	const char *data __attribute__((unused)),
	int datalen __attribute__((unused)))
	{
	/* We only get called when we want to delete it */
	syslog(LOG_DEBUG, "deleteing entry for %s", key);

	/* xxx maybe we should use transactions for this */
	config_ptscache_db->delete((struct db *)rockp, key, keylen, NULL, 0);
	return 0;
	}

	int main(int argc, char *argv[])
	{
	struct db *ptdb;
	char fnamebuf[1024];
	extern char *optarg;
	int opt;
	int r;
	char *alt_config = NULL;

	if ((geteuid()) == 0 && (become_cyrus() != 0)) {
	fatal("must run as the Cyrus user", EC_USAGE);
	}

	openlog("ptexpire", LOG_PID, SYSLOG_FACILITY);

	while ((opt = getopt(argc, argv, "C:E:")) != EOF) {
	switch (opt) {
	case 'C': /* alt config file */
	alt_config = optarg;
	break;
	case 'E':
	expire_time = atoi(optarg);
	break;
	default:
	fprintf(stderr,"usage: [-C filename] [-E time]"
	"\n\t-C <filename>\tAlternate Config File"
	"\n\t-E <seconds>\tExpiration time"
	"\n");
	syslog(LOG_ERR, "Invalid command line option");
	exit(-1);
	break;
	/* just pass through */
	}
	}

	cyrus_init(alt_config, "ptexpire", 0);

	timenow = time(0);
	syslog(LOG_INFO, "Expiring entries older than %d seconds (currently %d)",
	expire_time, timenow);
	syslog(LOG_DEBUG, "%s", rcsid);

	/* open database */
	strcpy(fnamebuf, config_dir);
	strcat(fnamebuf, PTS_DBFIL);
	r = (config_ptscache_db->open)(fnamebuf, CYRUSDB_CREATE, &ptdb);
	if(r != CYRUSDB_OK) {
	syslog(LOG_ERR, "error opening %s (%s)", fnamebuf,
	cyrusdb_strerror(r));
	exit(1);
	}

	/* iterate through db, wiping expired entries */
	config_ptscache_db->foreach(ptdb, "", 0, expire_p, expire_cb, ptdb, NULL);

	(config_ptscache_db->close)(ptdb);

	cyrus_done();

	syslog(LOG_INFO, "finished");
	return 0;
	}
	diff --git a/ptclient/ptloader.c b/ptclient/ptloader.c
	index 47ca9e4c0..6d84ca198 100644
	--- a/ptclient/ptloader.c
	+++ b/ptclient/ptloader.c
	@@ -1,282 +1,277 @@
	/* ptloader.c -- group loader daemon
	*
	* Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	*
	* 1. Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* 2. Redistributions in binary form must reproduce the above copyright
	* notice, this list of conditions and the following disclaimer in
	* the documentation and/or other materials provided with the
	* distribution.
	*
	* 3. The name "Carnegie Mellon University" must not be used to
	* endorse or promote products derived from this software without
	* prior written permission. For permission or any legal
	* details, please contact
	* Carnegie Mellon University
	* Center for Technology Transfer and Enterprise Creation
	* 4615 Forbes Avenue
	* Suite 302
	* Pittsburgh, PA 15213
	* (412) 268-7393, fax: (412) 268-7395
	* innovation@andrew.cmu.edu
	*
	* 4. Redistributions of any form whatsoever must retain the following
	* acknowledgment:
	* "This product includes software developed by Computing Services
	* at Carnegie Mellon University (http://www.cmu.edu/computing/)."
	*
	* CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
	* THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
	* AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
	* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
	* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
	* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
	* OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
	*
	* $Id: ptloader.c,v 1.50 2010/01/06 17:01:58 murch Exp $
	*/

	#include <config.h>

	#include <string.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <sys/param.h>
	#include <fcntl.h>
	#include <sys/socket.h>
	#include <sys/un.h>
	#include <sys/uio.h>

	#include "auth_pts.h"
	#include "cyrusdb.h"
	#include "exitcodes.h"
	#include "hash.h"
	#include "global.h"
	#include "libconfig.h"
	-#include "lock.h"
	+#include "cyr_lock.h"
	#include "retry.h"
	#include "xmalloc.h"
	#include "ptloader.h"

	struct pts_module *pts_modules[] = {
	#ifdef HAVE_LDAP
	&pts_ldap,
	#endif
	#ifdef HAVE_AFSKRB
	&pts_afskrb,
	#endif
	NULL };

	extern void setproctitle_init(int argc, char argv, char envp);

	static struct pts_module *pts_fromname()
	{
	int i;
	const char *name = config_getstring(IMAPOPT_PTS_MODULE);
	static struct pts_module *pts = NULL;

	if (pts)
	return pts;

	for (i = 0; pts_modules[i]; i++) {
	if (!strcmp(pts_modules[i]->name, name)) {
	pts = pts_modules[i]; break;
	}
	}

	if (!pts) {
	char errbuf[1024];
	snprintf(errbuf, sizeof(errbuf),
	"PTS module %s not supported", name);
	fatal(errbuf, EC_CONFIG);
	}

	return pts;
	}

	void ptsmodule_init(void)
	{
	struct pts_module *pts = pts_fromname();

	pts->init();
	}

	struct auth_state ptsmodule_make_authstate(const char identifier,
	size_t size,
	const char *reply, int dsize)
	{
	struct pts_module *pts = pts_fromname();

	return pts->make_authstate(identifier, size, reply, dsize);
	}

	/* config.c info (libimap) */
	const int config_need_data = 0;

	/* Globals */
	#define DB (config_ptscache_db)

	-/* XXXXXXXXX */
	-void des_init_random_number_generator() {
	- return;
	-}
	-
	static char ptclient_debug = 0;
	struct db *ptsdb = NULL;

	int service_init(int argc, char argv[], char *envp __attribute__((unused)))
	{
	int r;
	int opt;
	char fnamebuf[1024];
	extern char *optarg;

	if (geteuid() == 0) fatal("must run as the Cyrus user", EC_USAGE);
	setproctitle_init(argc, argv, envp);

	/* set signal handlers */
	signal(SIGPIPE, SIG_IGN);

	syslog(LOG_NOTICE, "starting: $Id: ptloader.c,v 1.50 2010/01/06 17:01:58 murch Exp $");

	while ((opt = getopt(argc, argv, "d:")) != EOF) {
	switch (opt) {
	case 'd':
	ptclient_debug = atoi(optarg);
	if (ptclient_debug < 1) {
	ptclient_debug = 1;
	}
	break;
	default:
	syslog(LOG_ERR, "invalid command line option specified");
	break;
	/* just pass through */
	}
	}

	strcpy(fnamebuf, config_dir);
	strcat(fnamebuf, PTS_DBFIL);
	r = (DB->open)(fnamebuf, CYRUSDB_CREATE, &ptsdb);
	if (r != 0) {
	syslog(LOG_ERR, "DBERROR: opening %s: %s", fnamebuf,
	cyrusdb_strerror(ret));
	fatal("can't read pts database", EC_TEMPFAIL);
	}

	ptsmodule_init();

	return 0;
	}

	/* Called by service API to shut down the service */
	void service_abort(int error)
	{
	int r;

	r = (DB->close)(ptsdb);
	if (r) {
	syslog(LOG_ERR, "DBERROR: error closing ptsdb: %s",
	cyrusdb_strerror(r));
	}

	r = DB->done();
	if (r) {
	syslog(LOG_ERR, "DBERROR: error exiting application: %s",
	cyrusdb_strerror(r));
	}

	exit(error);
	}

	/* we're a 'threaded' service, but since we never fork or create any
	threads, we're just one-person-at-a-time based */
	int service_main_fd(int c, int argc __attribute__((unused)),
	char **argv __attribute__((unused)),
	char **envp __attribute__((unused)))
	{
	const char *reply = NULL;
	char user[PTS_DB_KEYSIZE];
	int rc, dsize;
	size_t size;
	struct auth_state *newstate;

	(void)memset(&size, 0, sizeof(size));
	if (read(c, &size, sizeof(size_t)) < 0) {
	syslog(LOG_ERR, "socket (size): %m");
	reply = "Error reading request (size)";
	goto sendreply;
	}

	if (size > PTS_DB_KEYSIZE) {
	syslog(LOG_ERR, "size sent %d is greater than buffer size %d",
	size, PTS_DB_KEYSIZE);
	reply = "Error: invalid request size";
	goto sendreply;
	}

	if (size == 0) {
	syslog(LOG_ERR, "size sent is 0");
	reply = "Error: zero request size";
	goto sendreply;
	}

	memset(&user, 0, sizeof(user));
	if (read(c, &user, size) < 0) {
	syslog(LOG_ERR, "socket(user; size = %d): %m", size);
	reply = "Error reading request (user)";
	goto sendreply;
	}

	if (ptclient_debug) {
	syslog(LOG_DEBUG, "user %s", user);
	}

	newstate = ptsmodule_make_authstate(user, size, &reply, &dsize);

	if(newstate) {
	/* Success! */
	rc = DB->store(ptsdb, user, size, (void *)newstate, dsize, NULL);
	free(newstate);

	/* and we're done */
	reply = "OK";
	} else {
	/* Failure */
	if ( reply == NULL ) {
	reply = "Error making authstate";
	}
	}

	sendreply:
	if (retry_write(c, reply, strlen(reply) + 1) <0) {
	syslog(LOG_WARNING, "retry_write: %m");
	}
	close(c);

	return 0;
	}

	/* we need to have this function here 'cause libcyrus.a
	* makes calls to this function.
	*/
	void fatal(const char *msg, int exitcode)
	{
	syslog(LOG_ERR, "%s", msg);
	exit(exitcode);
	}

	void printstring(const char *s __attribute__((unused)))
	{
	/* needed to link against annotate.o */
	fatal("printstring() executed, but its not used for PTLOADER!",
	EC_SOFTWARE);
	}

File Metadata

Mime Type: text/x-diff
Expires: Apr 5 2026, 9:12 PM (4 w, 2 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 18831189
Default Alt Text: (794 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions