diff --git a/lib/cyrusdb.c b/lib/cyrusdb.c index e718b723c..a7ee9d331 100644 --- a/lib/cyrusdb.c +++ b/lib/cyrusdb.c @@ -1,773 +1,779 @@ /* * Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "assert.h" #include "bsearch.h" #include "cyrusdb.h" #include "util.h" #include "libcyr_cfg.h" #include "xmalloc.h" #include "xstrlcpy.h" //#define DEBUGDB 1 /* Note that some of these may be undefined symbols * if libcyrus was not built with support for them */ extern struct cyrusdb_backend cyrusdb_flat; extern struct cyrusdb_backend cyrusdb_skiplist; extern struct cyrusdb_backend cyrusdb_quotalegacy; extern struct cyrusdb_backend cyrusdb_sql; extern struct cyrusdb_backend cyrusdb_twoskip; extern struct cyrusdb_backend cyrusdb_zeroskip; static struct cyrusdb_backend *_backends[] = { &cyrusdb_flat, &cyrusdb_skiplist, &cyrusdb_quotalegacy, #if defined USE_CYRUSDB_SQL &cyrusdb_sql, #endif &cyrusdb_twoskip, #if defined HAVE_ZEROSKIP &cyrusdb_zeroskip, #endif NULL }; #define DEFAULT_BACKEND "twoskip" struct db { struct dbengine *engine; struct cyrusdb_backend *backend; }; static struct cyrusdb_backend *cyrusdb_fromname(const char *name) { int i; struct cyrusdb_backend *db = NULL; for (i = 0; _backends[i]; i++) { if (!strcmp(_backends[i]->name, name)) { db = _backends[i]; break; } } if (!db) { char errbuf[1024]; snprintf(errbuf, sizeof(errbuf), "cyrusdb backend %s not supported", name); fatal(errbuf, EX_CONFIG); } return db; } static int _myopen(const char *backend, const char *fname, int flags, struct db **ret, struct txn **tid) { const char *realname; struct db *db = xzmalloc(sizeof(struct db)); int r; if (!backend) backend = DEFAULT_BACKEND; /* not used yet, later */ db->backend = cyrusdb_fromname(backend); /* Check if shared lock is requested */ if (flags & CYRUSDB_SHARED) { assert(tid && *tid == NULL); if (flags & CYRUSDB_CONVERT) { - syslog(LOG_ERR, "DBERROR: CONVERT and SHARED are mutually exclusive, " - "won't open db %s (backend %s)", fname, backend); + xsyslog(LOG_ERR, + "DBERROR: CONVERT and SHARED are mutually exclusive," + " won't open db", + "fname=<%s> backend=<%s>", + fname, backend); r = CYRUSDB_INTERNAL; goto done; } } /* This whole thing is a fricking critical section. We don't have the API * in place for a safe rename of a locked database, so the choices are * basically: * a) convert each DB layer to support locked database renames while still * in the transaction. Best, but lots of work. * b) rename and hope... unreliable * c) global lock around this block of code. Safest and least efficient. */ /* check if it opens normally. Horray */ r = db->backend->open(fname, flags, &db->engine, tid); if (r == CYRUSDB_NOTFOUND) goto done; /* no open flags */ if (!r) goto done; /* magic time - we need to work out if the file was created by a different * backend and convert if possible */ realname = cyrusdb_detect(fname); if (!realname) { - syslog(LOG_ERR, "DBERROR: failed to detect DB type for %s (backend %s) (r was %d)", - fname, backend, r); + xsyslog(LOG_ERR, "DBERROR: failed to detect DB type", + "fname=<%s> backend=<%s> r=<%d>", + fname, backend, r); /* r is still set */ goto done; } /* different type */ if (strcmp(realname, backend)) { if (flags & CYRUSDB_CONVERT) { r = cyrusdb_convert(fname, fname, realname, backend); if (r) { - syslog(LOG_ERR, "DBERROR: failed to convert %s from %s to %s, maybe someone beat us", - fname, realname, backend); + xsyslog(LOG_ERR, "DBERROR: failed to convert, maybe someone beat us", + "fname=<%s> from=<%s> to=<%s>", + fname, realname, backend); } else { syslog(LOG_NOTICE, "cyrusdb: converted %s from %s to %s", fname, realname, backend); } } else { syslog(LOG_NOTICE, "cyrusdb: opening %s with backend %s (requested %s)", fname, realname, backend); db->backend = cyrusdb_fromname(realname); } } r = db->backend->open(fname, flags, &db->engine, tid); #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB open(%s, %d) => %llx\n", fname, flags, (long long unsigned)db->engine); #endif done: if (r) free(db); else *ret = db; return r; } EXPORTED int cyrusdb_open(const char *backend, const char *fname, int flags, struct db **ret) { return _myopen(backend, fname, flags, ret, NULL); } EXPORTED int cyrusdb_lockopen(const char *backend, const char *fname, int flags, struct db **ret, struct txn **tid) { return _myopen(backend, fname, flags, ret, tid); } EXPORTED int cyrusdb_close(struct db *db) { #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB close(%llx)\n", (long long unsigned)db->engine); #endif int r = db->backend->close(db->engine); free(db); return r; } EXPORTED int cyrusdb_fetch(struct db *db, const char *key, size_t keylen, const char **data, size_t *datalen, struct txn **mytid) { if (!db->backend->fetch) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB fetch(%llx, %.*s)\n", (long long unsigned)db->engine, (int)keylen, key); #endif return db->backend->fetch(db->engine, key, keylen, data, datalen, mytid); } EXPORTED int cyrusdb_fetchlock(struct db *db, const char *key, size_t keylen, const char **data, size_t *datalen, struct txn **mytid) { if (!db->backend->fetchlock) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB fetchlock(%llx, %.*s)\n", (long long unsigned)db->engine, (int)keylen, key); #endif return db->backend->fetchlock(db->engine, key, keylen, data, datalen, mytid); } EXPORTED int cyrusdb_fetchnext(struct db *db, const char *key, size_t keylen, const char **found, size_t *foundlen, const char **data, size_t *datalen, struct txn **mytid) { if (!db->backend->fetchnext) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB fetchnext(%llx, %.*s)\n", (long long unsigned)db->engine, (int)keylen, key); #endif return db->backend->fetchnext(db->engine, key, keylen, found, foundlen, data, datalen, mytid); } EXPORTED int cyrusdb_foreach(struct db *db, const char *prefix, size_t prefixlen, foreach_p *p, foreach_cb *cb, void *rock, struct txn **tid) { if (!db->backend->foreach) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB foreach(%llx, %.*s)\n", (long long unsigned)db->engine, (int)prefixlen, prefix); #endif return db->backend->foreach(db->engine, prefix, prefixlen, p, cb, rock, tid); } EXPORTED int cyrusdb_forone(struct db *db, const char *key, size_t keylen, foreach_p *p, foreach_cb *cb, void *rock, struct txn **tid) { const char *data; size_t datalen; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB forone(%llx, %.*s)\n", (long long unsigned)db->engine, (int)keylen, key); #endif int r = cyrusdb_fetch(db, key, keylen, &data, &datalen, tid); if (r == CYRUSDB_NOTFOUND) return 0; if (r) return r; if (!p || p(rock, key, keylen, data, datalen)) r = cb(rock, key, keylen, data, datalen); return r; } EXPORTED int cyrusdb_create(struct db *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid) { if (!db->backend->create) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB create(%llx, %.*s)\n", (long long unsigned)db->engine, (int)keylen, key); #endif return db->backend->create(db->engine, key, keylen, data, datalen, tid); } EXPORTED int cyrusdb_store(struct db *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid) { if (!db->backend->store) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB store(%llx, %.*s)\n", (long long unsigned)db->engine, (int)keylen, key); #endif return db->backend->store(db->engine, key, keylen, data, datalen, tid); } EXPORTED int cyrusdb_delete(struct db *db, const char *key, size_t keylen, struct txn **tid, int force) { if (!db->backend->delete_) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB delete(%llx, %.*s)\n", (long long unsigned)db->engine, (int)keylen, key); #endif return db->backend->delete_(db->engine, key, keylen, tid, force); } EXPORTED int cyrusdb_commit(struct db *db, struct txn *tid) { if (!db->backend->commit) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB commit(%llx)\n", (long long unsigned)db->engine); #endif return db->backend->commit(db->engine, tid); } EXPORTED int cyrusdb_abort(struct db *db, struct txn *tid) { if (!db->backend->abort) return CYRUSDB_NOTIMPLEMENTED; #ifdef DEBUGDB syslog(LOG_NOTICE, "DEBUGDB abort(%llx)\n", (long long unsigned)db->engine); #endif return db->backend->abort(db->engine, tid); } EXPORTED int cyrusdb_dump(struct db *db, int detail) { if (!db->backend->dump) return 0; return db->backend->dump(db->engine, detail); } EXPORTED int cyrusdb_consistent(struct db *db) { if (!db->backend->consistent) return 0; return db->backend->consistent(db->engine); } EXPORTED int cyrusdb_repack(struct db *db) { if (!db->backend->repack) return 0; return db->backend->repack(db->engine); } EXPORTED int cyrusdb_compar(struct db *db, const char *a, int alen, const char *b, int blen) { if (!db->backend->compar) return bsearch_ncompare_raw(a, alen, b, blen); return db->backend->compar(db->engine, a, alen, b, blen); } /**********************************************/ EXPORTED void cyrusdb_init(void) { int i, r; char dbdir[1024]; const char *confdir = libcyrus_config_getstring(CYRUSOPT_CONFIG_DIR); int initflags = libcyrus_config_getint(CYRUSOPT_DB_INIT_FLAGS); strcpy(dbdir, confdir); strcat(dbdir, FNAME_DBDIR); - for(i=0; _backends[i]; i++) { + for (i=0; _backends[i]; i++) { r = (_backends[i])->init(dbdir, initflags); - if(r) { - syslog(LOG_ERR, "DBERROR: init() on %s", - _backends[i]->name); + if (r) { + xsyslog(LOG_ERR, "DBERROR: backend init failed", + "backend=<%s>", + _backends[i]->name); } } } EXPORTED void cyrusdb_done(void) { int i; for(i=0; _backends[i]; i++) { (_backends[i])->done(); } } EXPORTED int cyrusdb_copyfile(const char *srcname, const char *dstname) { return cyrus_copyfile(srcname, dstname, COPYFILE_NOLINK); } struct db_rock { struct db *db; struct txn **tid; }; static int delete_cb(void *rock, const char *key, size_t keylen, const char *data __attribute__((unused)), size_t datalen __attribute__((unused))) { struct db_rock *cr = (struct db_rock *)rock; return cyrusdb_delete(cr->db, key, keylen, cr->tid, 1); } static int print_cb(void *rock, const char *key, size_t keylen, const char *data, size_t datalen) { FILE *f = (FILE *)rock; /* XXX: improve binary safety */ fprintf(f, "%.*s\t%.*s\n", (int)keylen, key, (int)datalen, data); return 0; } EXPORTED int cyrusdb_dumpfile(struct db *db, const char *prefix, size_t prefixlen, FILE *f, struct txn **tid) { return cyrusdb_foreach(db, prefix, prefixlen, NULL, print_cb, f, tid); } EXPORTED int cyrusdb_truncate(struct db *db, struct txn **tid) { struct db_rock tr; tr.db = db; tr.tid = tid; return cyrusdb_foreach(db, "", 0, NULL, delete_cb, &tr, tid); } EXPORTED int cyrusdb_undumpfile(struct db *db, FILE *f, struct txn **tid) { struct buf line = BUF_INITIALIZER; const char *tab; const char *str; int r = 0; while (buf_getline(&line, f)) { /* skip blank lines */ if (!line.len) continue; str = buf_cstring(&line); /* skip comments */ if (str[0] == '#') continue; tab = strchr(str, '\t'); /* deletion (no value) */ if (!tab) { r = cyrusdb_delete(db, str, line.len, tid, 1); if (r) goto out; } /* store */ else { unsigned klen = (tab - str); unsigned vlen = line.len - klen - 1; /* TAB */ r = cyrusdb_store(db, str, klen, tab + 1, vlen, tid); if (r) goto out; } } out: buf_free(&line); return r; } static int converter_cb(void *rock, const char *key, size_t keylen, const char *data, size_t datalen) { struct db_rock *cr = (struct db_rock *)rock; return cyrusdb_store(cr->db, key, keylen, data, datalen, cr->tid); } /* convert (just copy every record) from one database to another in possibly a different format. It's up to the surrounding code to copy the new database over the original if it wants to */ EXPORTED int cyrusdb_convert(const char *fromfname, const char *tofname, const char *frombackend, const char *tobackend) { char *newfname = NULL; struct db *fromdb = NULL; struct db *todb = NULL; struct db_rock cr; struct txn *fromtid = NULL; struct txn *totid = NULL; int r; /* open source database */ r = cyrusdb_open(frombackend, fromfname, 0, &fromdb); if (r) goto err; /* use a bogus fetch to lock source DB before touching the destination */ r = cyrusdb_fetch(fromdb, "_", 1, NULL, NULL, &fromtid); if (r == CYRUSDB_NOTFOUND) r = 0; if (r) goto err; /* same file? Create with a new name */ if (!strcmp(tofname, fromfname)) tofname = newfname = strconcat(fromfname, ".NEW", NULL); /* remove any rubbish lying around */ unlink(tofname); r = cyrusdb_open(tobackend, tofname, CYRUSDB_CREATE, &todb); if (r) goto err; /* set up the copy rock */ cr.db = todb; cr.tid = &totid; /* copy each record to the destination DB */ cyrusdb_foreach(fromdb, "", 0, NULL, converter_cb, &cr, &fromtid); /* commit destination transaction */ if (totid) cyrusdb_commit(todb, totid); r = cyrusdb_close(todb); totid = NULL; todb = NULL; if (r) goto err; /* created a new filename - so it's a replace-in-place */ if (newfname) { r = rename(newfname, fromfname); if (r) goto err; } /* and close the source database - nothing should have * written here, so an abort is fine */ if (fromtid) cyrusdb_abort(fromdb, fromtid); cyrusdb_close(fromdb); free(newfname); return 0; err: if (totid) cyrusdb_abort(todb, totid); if (todb) cyrusdb_close(todb); if (fromtid) cyrusdb_abort(fromdb, fromtid); if (fromdb) cyrusdb_close(fromdb); unlink(tofname); free(newfname); return r; } EXPORTED const char *cyrusdb_detect(const char *fname) { FILE *f; char buf[32]; int n; f = fopen(fname, "r"); if (!f) return NULL; /* empty file? */ n = fread(buf, 32, 1, f); fclose(f); if (n != 1) return NULL; /* only compare first 16 bytes, that's OK */ if (!strncmp(buf, "\241\002\213\015skiplist file\0\0\0", 16)) return "skiplist"; if (!strncmp(buf, "\241\002\213\015twoskip file\0\0\0\0", 16)) return "twoskip"; /* unable to detect SQLite databases or flat files explicitly here */ return NULL; } EXPORTED int cyrusdb_sync(const char *backend) { struct cyrusdb_backend *db = cyrusdb_fromname(backend); return db->sync(); } EXPORTED int cyrusdb_unlink(const char *backend, const char *fname, int flags) { struct cyrusdb_backend *db = cyrusdb_fromname(backend); if (!db->unlink) return 0; return db->unlink(fname, flags); } EXPORTED cyrusdb_archiver *cyrusdb_getarchiver(const char *backend) { struct cyrusdb_backend *db = cyrusdb_fromname(backend); return db->archive; /* the function used for archiving */ } EXPORTED int cyrusdb_canfetchnext(const char *backend) { struct cyrusdb_backend *db = cyrusdb_fromname(backend); return db->fetchnext ? 1 : 0; } /* caller is responsible for calling strarray_free() */ EXPORTED strarray_t *cyrusdb_backends(void) { strarray_t *ret = strarray_new(); int i; for (i = 0; _backends[i]; i++) { strarray_add(ret, _backends[i]->name); } return ret; } /* generic backend implementations */ HIDDEN int cyrusdb_generic_init(const char *dbdir __attribute__((unused)), int myflags __attribute__((unused))) { return 0; } HIDDEN int cyrusdb_generic_done(void) { return 0; } HIDDEN int cyrusdb_generic_sync(void) { return 0; } HIDDEN int cyrusdb_generic_archive(const strarray_t *fnames, const char *dirname) { char dstname[1024], *dp; int length, rest; int i; int r; strlcpy(dstname, dirname, sizeof(dstname)); length = strlen(dstname); dp = dstname + length; rest = sizeof(dstname) - length; /* archive those files specified by the app */ for (i = 0; i < fnames->count; i++) { const char *fname = strarray_nth(fnames, i); struct stat sbuf; if (stat(fname, &sbuf) < 0) { syslog(LOG_DEBUG, "not archiving database file: %s: %m", fname); continue; } syslog(LOG_DEBUG, "archiving database file: %s", fname); strlcpy(dp, strrchr(fname, '/'), rest); r = cyrusdb_copyfile(fname, dstname); if (r) { syslog(LOG_ERR, "DBERROR: error archiving database file: %s", fname); return CYRUSDB_IOERROR; } } return 0; } HIDDEN int cyrusdb_generic_noarchive(const strarray_t *fnames __attribute__((unused)), const char *dirname __attribute__((unused))) { return 0; } HIDDEN int cyrusdb_generic_unlink(const char *fname, int flags __attribute__((unused))) { if (fname) unlink(fname); /* XXX - check that it exists unless FORCE flag? */ return 0; } EXPORTED const char *cyrusdb_strerror(int r) { const char *err = "unknown error"; switch (r) { case CYRUSDB_OK: err = "not an error"; break; case CYRUSDB_DONE: err = "done"; break; case CYRUSDB_IOERROR: err = "IO error"; break; case CYRUSDB_AGAIN: err = "again"; break; case CYRUSDB_EXISTS: err = "item exists"; break; case CYRUSDB_INTERNAL: err = "internal error"; break; case CYRUSDB_NOTFOUND: err = "item not found"; break; case CYRUSDB_LOCKED: err = "locked"; break; case CYRUSDB_NOTIMPLEMENTED: err = "action not implemented"; break; case CYRUSDB_FULL: err = "no space available"; break; case CYRUSDB_READONLY: err = "database is readonly"; break; default: err = "not a cyrusdb error"; break; } return err; } diff --git a/lib/cyrusdb_flat.c b/lib/cyrusdb_flat.c index c5945c165..d01bb81ca 100644 --- a/lib/cyrusdb_flat.c +++ b/lib/cyrusdb_flat.c @@ -1,863 +1,891 @@ /* cyrusdb_flat: a sorted flat textfile backend * * Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #ifdef HAVE_UNISTD_H #include #endif #include #include #include #include #include #include #include #include #include #include "assert.h" #include "cyrusdb.h" #include "map.h" #include "bsearch.h" #include "cyr_lock.h" #include "retry.h" #include "util.h" #include "xmalloc.h" #include "xstrlcpy.h" #include "xstrlcat.h" /* we have the file locked iff we have an outstanding transaction */ struct dbengine { char *fname; struct dbengine *next; int refcount; int fd; /* current file open */ ino_t ino; const char *base; /* contents of file */ size_t size; /* actual size */ size_t len; /* mapped size */ struct buf data; /* returned storage for fetch */ }; #define DATA(db) ((db)->data.s ? (db)->data.s : "") #define DATALEN(db) ((db)->data.len) struct txn { char *fnamenew; int fd; }; static struct dbengine *alldbs; /* * We choose an escape character which is an invalid UTF-8 encoding and * thus unlikely to appear in the key or data unless they are completely * non-textual. */ #define ESCAPE 0xff static void encode(const char *ps, int len, struct buf *buf) { const unsigned char *p = (const unsigned char *)ps; buf_reset(buf); /* allocate enough space plus a little slop to cover * escaping a few characters */ buf_ensure(buf, len+10); for ( ; len > 0 ; len--, p++) { switch (*p) { case '\0': case '\t': case '\r': case '\n': buf_putc(buf, ESCAPE); buf_putc(buf, 0x80|(*p)); break; case ESCAPE: buf_putc(buf, ESCAPE); buf_putc(buf, ESCAPE); break; default: buf_putc(buf, *p); break; } } /* ensure the buf is NUL-terminated; we pass the buf's data to * bsearch_mem(), which expects a C string, several times */ buf_cstring(buf); } static void decode(const char *ps, int len, struct buf *buf) { const unsigned char *p = (const unsigned char *)ps; buf_reset(buf); /* allocate enough space; we don't need slop because * decoding can only shrink the result */ buf_ensure(buf, len); for ( ; len > 0 ; len--, p++) { if (*p == ESCAPE) { if (len < 2) { /* invalid encoding, silently ignore */ continue; } len--; p++; if (*p == ESCAPE) buf_putc(buf, ESCAPE); else buf_putc(buf, (*p) & ~0x80); } else buf_putc(buf, *p); } /* Note: buf is not NUL-terminated. It happens that neither * skiplist nor berkeley backends guarantee any such thing, * and so code that depends on it is quite broken anyway */ } /* other routines call this one when they fail */ static int abort_txn(struct dbengine *db, struct txn *tid) { int r = CYRUSDB_OK; int rw = 0; struct stat sbuf; assert(db && tid); /* cleanup done while lock is held */ if (tid->fnamenew) { unlink(tid->fnamenew); free(tid->fnamenew); rw = 1; } /* release lock */ r = lock_unlock(db->fd, db->fname); if (r == -1) { - syslog(LOG_ERR, "IOERROR: unlocking db %s: %m", db->fname); + xsyslog(LOG_ERR, "IOERROR: unlocking db failed", + "fname=<%s>", + db->fname); r = CYRUSDB_IOERROR; } if (rw) { /* return to our normally scheduled fd */ if (!r && fstat(db->fd, &sbuf) == -1) { - syslog(LOG_ERR, "IOERROR: fstat on %s: %m", db->fname); + xsyslog(LOG_ERR, "IOERROR: fstat failed", + "fname=<%s>", + db->fname); r = CYRUSDB_IOERROR; } if (!r) { map_free(&db->base, &db->len); map_refresh(db->fd, 0, &db->base, &db->len, sbuf.st_size, db->fname, 0); db->size = sbuf.st_size; } } free(tid); return 0; } static void free_db(struct dbengine *db) { if (db) { free(db->fname); buf_free(&db->data); free(db); } } static struct dbengine *find_db(const char *fname) { struct dbengine *db; for (db = alldbs ; db ; db = db->next) { if (!strcmp(fname, db->fname)) { db->refcount++; return db; } } return NULL; } static struct txn *new_txn(void) { struct txn *ret = (struct txn *) xmalloc(sizeof(struct txn)); ret->fnamenew = NULL; ret->fd = 0; return ret; } static int starttxn_or_refetch(struct dbengine *db, struct txn **mytid) { int r = 0; struct stat sbuf; assert(db); if (mytid && !*mytid) { const char *lockfailaction; /* start txn; grab lock */ r = lock_reopen(db->fd, db->fname, &sbuf, &lockfailaction); if (r < 0) { - syslog(LOG_ERR, "IOERROR: %s %s: %m", lockfailaction, db->fname); + xsyslog(LOG_ERR, "IOERROR: lock_reopen failed", + "action=<%s> fname=<%s>", + lockfailaction, db->fname); return CYRUSDB_IOERROR; } *mytid = new_txn(); if (db->ino != sbuf.st_ino) { map_free(&db->base, &db->len); } map_refresh(db->fd, 0, &db->base, &db->len, sbuf.st_size, db->fname, 0); /* we now have the latest & greatest open */ db->size = sbuf.st_size; db->ino = sbuf.st_ino; } if (!mytid) { /* no txn, but let's try to be reasonably up-to-date */ if (stat(db->fname, &sbuf) == -1) { - syslog(LOG_ERR, "IOERROR: stating flat %s: %m", db->fname); + xsyslog(LOG_ERR, "IOERROR: stat failed", + "fname=<%s>", + db->fname); return CYRUSDB_IOERROR; } if (sbuf.st_ino != db->ino) { /* reopen */ int newfd = open(db->fname, O_RDWR); if (newfd == -1) { /* fail! */ - syslog(LOG_ERR, "couldn't reopen %s: %m", db->fname); + xsyslog(LOG_ERR, "IOERROR: reopen failed", + "fname=<%s>", + db->fname); return CYRUSDB_IOERROR; } dup2(newfd, db->fd); close(newfd); if (stat(db->fname, &sbuf) == -1) { - syslog(LOG_ERR, "IOERROR: stating flat %s: %m", db->fname); + xsyslog(LOG_ERR, "IOERROR: stat failed", + "fname=<%s>", + db->fname); return CYRUSDB_IOERROR; } db->ino = sbuf.st_ino; map_free(&db->base, &db->len); } map_refresh(db->fd, 0, &db->base, &db->len, sbuf.st_size, db->fname, 0); db->size = sbuf.st_size; } return 0; } static int myopen(const char *fname, int flags, struct dbengine **ret, struct txn **mytid) { struct dbengine *db; struct stat sbuf; assert(fname && ret); db = find_db(fname); if (db) goto out; /* new reference to existing db */ db = (struct dbengine *) xzmalloc(sizeof(struct dbengine)); db->fd = open(fname, O_RDWR, 0644); if (db->fd == -1 && errno == ENOENT) { if (!(flags & CYRUSDB_CREATE)) { free_db(db); return CYRUSDB_NOTFOUND; } if (cyrus_mkdir(fname, 0755) == -1) { free_db(db); return CYRUSDB_IOERROR; } db->fd = open(fname, O_RDWR | O_CREAT, 0644); } if (db->fd == -1) { - syslog(LOG_ERR, "IOERROR: opening %s: %m", fname); + xsyslog(LOG_ERR, "IOERROR: open failed", + "fname=<%s>", + fname); free_db(db); return CYRUSDB_IOERROR; } if (fstat(db->fd, &sbuf) == -1) { - syslog(LOG_ERR, "IOERROR: fstat on %s: %m", fname); + xsyslog(LOG_ERR, "IOERROR: fstat failed", + "fname=<%s>", + fname); close(db->fd); free_db(db); return CYRUSDB_IOERROR; } db->ino = sbuf.st_ino; map_refresh(db->fd, 0, &db->base, &db->len, sbuf.st_size, fname, 0); db->size = sbuf.st_size; db->fname = xstrdup(fname); db->refcount = 1; /* prepend to the list */ db->next = alldbs; alldbs = db; if (mytid) { int r = starttxn_or_refetch(db, mytid); if (r) return r; } out: *ret = db; return 0; } static int myclose(struct dbengine *db) { struct dbengine **prevp; assert(db); if (--db->refcount > 0) return 0; /* now we are dropping the last reference */ /* detach from the list of all dbs */ for (prevp = &alldbs ; *prevp && *prevp != db ; prevp = &(*prevp)->next) ; assert(*prevp == db); /* this struct must be in the list */ *prevp = db->next; /* clean up the internals */ map_free(&db->base, &db->len); close(db->fd); free_db(db); return 0; } static int myfetch(struct dbengine *db, const char *key, size_t keylen, const char **data, size_t *datalen, struct txn **mytid) { int r = 0; int offset; unsigned long len; struct buf keybuf = BUF_INITIALIZER; assert(db); if (data) *data = NULL; if (datalen) *datalen = 0; r = starttxn_or_refetch(db, mytid); if (r) return r; encode(key, keylen, &keybuf); offset = bsearch_mem_mbox(keybuf.s, db->base, db->size, 0, &len); if (len) { if (data) { decode(db->base + offset + keybuf.len + 1, /* subtract one for \t, and one for the \n */ len - keybuf.len - 2, &db->data); if (data) *data = DATA(db); if (datalen) *datalen = DATALEN(db); } } else { r = CYRUSDB_NOTFOUND; } buf_free(&keybuf); return r; } static int fetch(struct dbengine *mydb, const char *key, size_t keylen, const char **data, size_t *datalen, struct txn **mytid) { return myfetch(mydb, key, keylen, data, datalen, mytid); } static int fetchlock(struct dbengine *db, const char *key, size_t keylen, const char **data, size_t *datalen, struct txn **mytid) { return myfetch(db, key, keylen, data, datalen, mytid); } static int getentry(struct dbengine *db, const char *p, struct buf *keybuf, const char **dataendp) { const char *key; int keylen; const char *data; const char *dataend; int datalen; key = p; data = strchr(p, '\t'); if (!data) { /* huh, might be corrupted? */ return CYRUSDB_IOERROR; } keylen = data - key; data++; /* skip the \t */ dataend = strchr(data, '\n'); if (!dataend) { /* huh, might be corrupted? */ return CYRUSDB_IOERROR; } datalen = dataend - data; decode(data, datalen, &db->data); decode(key, keylen, keybuf); *dataendp = dataend; return 0; } #define GETENTRY(p) \ r = getentry(db, p, &keybuf, &dataend); \ if (r) break; static int foreach(struct dbengine *db, const char *prefix, size_t prefixlen, foreach_p *goodp, foreach_cb *cb, void *rock, struct txn **mytid) { int r = CYRUSDB_OK; int offset; unsigned long len; const char *p, *pend; const char *dataend; /* for use inside the loop, but we need the values to be retained * from loop to loop */ struct buf keybuf = BUF_INITIALIZER; int dontmove = 0; /* For when we have a transaction running */ struct buf savebuf = BUF_INITIALIZER; /* for the local iteration so that the db can change out from under us */ const char *dbbase = NULL; size_t dblen = 0; int dbfd = -1; struct buf prefixbuf = BUF_INITIALIZER; assert(cb); r = starttxn_or_refetch(db, mytid); if (r) return r; if (!mytid) { /* No transaction, use the fast method to avoid stomping on our * memory map if changes happen */ dbfd = dup(db->fd); if(dbfd == -1) return CYRUSDB_IOERROR; map_refresh(dbfd, 1, &dbbase, &dblen, db->size, db->fname, 0); /* drop our read lock on the file, since we don't really care * if it gets replaced out from under us, our mmap stays on the * old version */ lock_unlock(db->fd, db->fname); } else { /* use the same variables as in the no transaction case, just to * get things set up */ dbbase = db->base; dblen = db->len; } if (prefix) { encode(prefix, prefixlen, &prefixbuf); offset = bsearch_mem_mbox(prefixbuf.s, dbbase, db->size, 0, &len); } else { offset = 0; } p = dbbase + offset; pend = dbbase + db->size; while (p < pend) { if (!dontmove) { GETENTRY(p) } else dontmove = 0; /* does it still match prefix? */ if (keybuf.len < (size_t) prefixbuf.len) break; if (prefixbuf.len && memcmp(keybuf.s, prefixbuf.s, prefixbuf.len)) break; if (!goodp || goodp(rock, keybuf.s, keybuf.len, DATA(db), DATALEN(db))) { unsigned long ino = db->ino; unsigned long sz = db->size; if(mytid) { /* transaction present, this means we do the slow way */ buf_copy(&savebuf, &keybuf); } /* make callback */ r = cb(rock, keybuf.s, keybuf.len, DATA(db), DATALEN(db)); if (r) break; if (mytid) { /* reposition? (we made a change) */ if (!(ino == db->ino && sz == db->size)) { /* something changed in the file; reseek */ buf_cstring(&savebuf); offset = bsearch_mem_mbox(savebuf.s, db->base, db->size, 0, &len); p = db->base + offset; GETENTRY(p); /* 'key' might not equal 'savebuf'. if it's different, we want to stay where we are. if it's the same, we should move on to the next one */ if (!buf_cmp(&savebuf, &keybuf)) { p = dataend + 1; } else { /* 'savebuf' got deleted, so we're now pointing at the right thing */ dontmove = 1; } } } } p = dataend + 1; } if (!mytid) { /* cleanup the fast method */ map_free(&dbbase, &dblen); close(dbfd); } buf_free(&savebuf); buf_free(&keybuf); buf_free(&prefixbuf); return r; } #undef GETENTRY static int mystore(struct dbengine *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **mytid, int overwrite) { int r = 0; char fnamebuf[1024]; int offset; unsigned long len; const char *lockfailaction; int writefd; struct iovec iov[10]; int niov; struct stat sbuf; struct buf keybuf = BUF_INITIALIZER; struct buf databuf = BUF_INITIALIZER; /* lock file, if needed */ if (!mytid || !*mytid) { r = lock_reopen(db->fd, db->fname, &sbuf, &lockfailaction); if (r < 0) { - syslog(LOG_ERR, "IOERROR: %s %s: %m", lockfailaction, db->fname); + xsyslog(LOG_ERR, "IOERROR: lock_reopen failed", + "action=<%s> fname=<%s>", + lockfailaction, db->fname); return CYRUSDB_IOERROR; } if (sbuf.st_ino != db->ino) { db->ino = sbuf.st_ino; map_free(&db->base, &db->len); map_refresh(db->fd, 0, &db->base, &db->len, sbuf.st_size, db->fname, 0); db->size = sbuf.st_size; } if (mytid) { *mytid = new_txn(); } } encode(key, keylen, &keybuf); /* find entry, if it exists */ offset = bsearch_mem_mbox(keybuf.s, db->base, db->size, 0, &len); /* overwrite? */ if (len && !overwrite) { if (mytid) abort_txn(db, *mytid); buf_free(&keybuf); buf_free(&databuf); return CYRUSDB_EXISTS; } /* write new file */ if (mytid && (*mytid)->fnamenew) { strlcpy(fnamebuf, (*mytid)->fnamenew, sizeof(fnamebuf)); } else { strlcpy(fnamebuf, db->fname, sizeof(fnamebuf)); strlcat(fnamebuf, ".NEW", sizeof(fnamebuf)); } unlink(fnamebuf); r = writefd = open(fnamebuf, O_RDWR | O_CREAT, 0666); if (r < 0) { syslog(LOG_ERR, "opening %s for writing failed: %m", fnamebuf); if (mytid) abort_txn(db, *mytid); buf_free(&keybuf); buf_free(&databuf); return CYRUSDB_IOERROR; } niov = 0; if (offset) { WRITEV_ADD_TO_IOVEC(iov, niov, (char *) db->base, offset); } if (data) { /* new entry */ encode(data, datalen, &databuf); WRITEV_ADD_TO_IOVEC(iov, niov, keybuf.s, keybuf.len); WRITEV_ADD_TO_IOVEC(iov, niov, "\t", 1); WRITEV_ADD_TO_IOVEC(iov, niov, databuf.s, databuf.len); WRITEV_ADD_TO_IOVEC(iov, niov, "\n", 1); } if (db->size - (offset + len) > 0) { WRITEV_ADD_TO_IOVEC(iov, niov, (char *) db->base + offset + len, db->size - (offset + len)); } /* do the write */ r = retry_writev(writefd, iov, niov); if (r == -1) { - syslog(LOG_ERR, "IOERROR: writing %s: %m", fnamebuf); + xsyslog(LOG_ERR, "IOERROR: write failed", + "fname=<%s>", + fnamebuf); close(writefd); if (mytid) abort_txn(db, *mytid); buf_free(&keybuf); buf_free(&databuf); return CYRUSDB_IOERROR; } r = 0; if (mytid) { /* setup so further accesses will be against fname.NEW */ if (fstat(writefd, &sbuf) == -1) { /* xxx ? */ } if (!(*mytid)->fnamenew) (*mytid)->fnamenew = xstrdup(fnamebuf); if ((*mytid)->fd) close((*mytid)->fd); (*mytid)->fd = writefd; map_free(&db->base, &db->len); map_refresh(writefd, 0, &db->base, &db->len, sbuf.st_size, fnamebuf, 0); db->size = sbuf.st_size; } else { /* commit immediately */ if (fsync(writefd) || fstat(writefd, &sbuf) == -1 || rename(fnamebuf, db->fname) == -1) { - syslog(LOG_ERR, "IOERROR: writing %s: %m", fnamebuf); + xsyslog(LOG_ERR, "IOERROR: commit failed", + "fname=<%s>", + fnamebuf); close(writefd); buf_free(&keybuf); buf_free(&databuf); return CYRUSDB_IOERROR; } close(db->fd); db->fd = writefd; /* release lock */ r = lock_unlock(db->fd, db->fname); if (r == -1) { - syslog(LOG_ERR, "IOERROR: unlocking db %s: %m", db->fname); + xsyslog(LOG_ERR, "IOERROR: lock_unlock failed", + "fname=<%s>", + db->fname); r = CYRUSDB_IOERROR; } db->ino = sbuf.st_ino; map_free(&db->base, &db->len); map_refresh(writefd, 0, &db->base, &db->len, sbuf.st_size, db->fname, 0); db->size = sbuf.st_size; } buf_free(&keybuf); buf_free(&databuf); return r; } static int create(struct dbengine *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid) { if (!data) { data = ""; datalen = 0; } return mystore(db, key, keylen, data, datalen, tid, 0); } static int store(struct dbengine *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid) { if (!data) { data = ""; datalen = 0; } return mystore(db, key, keylen, data, datalen, tid, 1); } static int delete(struct dbengine *db, const char *key, size_t keylen, struct txn **mytid, int force __attribute__((unused))) { return mystore(db, key, keylen, NULL, 0, mytid, 1); } static int commit_txn(struct dbengine *db, struct txn *tid) { int writefd; int r = 0; struct stat sbuf; assert(db && tid); if (tid->fnamenew) { /* we wrote something */ writefd = tid->fd; if (fsync(writefd) || fstat(writefd, &sbuf) == -1 || rename(tid->fnamenew, db->fname) == -1) { - syslog(LOG_ERR, "IOERROR: writing %s: %m", tid->fnamenew); + xsyslog(LOG_ERR, "IOERROR: commit failed", + "fname=<%s>", + tid->fnamenew); close(writefd); r = CYRUSDB_IOERROR; } else { /* successful */ /* we now deal exclusively with our new fd */ close(db->fd); db->fd = writefd; db->ino = sbuf.st_ino; } free(tid->fnamenew); } else { /* read-only txn */ /* release lock */ r = lock_unlock(db->fd, db->fname); if (r == -1) { - syslog(LOG_ERR, "IOERROR: unlocking db %s: %m", db->fname); + xsyslog(LOG_ERR, "IOERROR: lock_unlock failed", + "fname=<%s>", + db->fname); r = CYRUSDB_IOERROR; } } free(tid); return r; } /* flat database is always mbox sort order */ static int mycompar(struct dbengine *db __attribute__((unused)), const char *a, int alen, const char *b, int blen) { return bsearch_ncompare_mbox(a, alen, b, blen); } EXPORTED struct cyrusdb_backend cyrusdb_flat = { "flat", /* name */ &cyrusdb_generic_init, &cyrusdb_generic_done, &cyrusdb_generic_sync, &cyrusdb_generic_archive, &cyrusdb_generic_unlink, &myopen, &myclose, &fetch, &fetchlock, NULL, &foreach, &create, &store, &delete, &commit_txn, &abort_txn, NULL, NULL, NULL, &mycompar }; diff --git a/lib/cyrusdb_quotalegacy.c b/lib/cyrusdb_quotalegacy.c index eb7efc3b9..9d5224d61 100644 --- a/lib/cyrusdb_quotalegacy.c +++ b/lib/cyrusdb_quotalegacy.c @@ -1,907 +1,928 @@ /* cyrusdb_quotalegacy: cyrusdb backend for accessing legacy quota files * * Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* FILE FORMAT: * used\nlimit\n * where used is a 64 bit unsigned value in bytes * limit is a 32 bit signed value in kilobytes (1024 bytes) * special value: -1 means unlimited quota * then there's some extra rubbish for multiple quotas * and finally there's the dlist file format, which just plain sucks less */ #include #ifdef HAVE_UNISTD_H #include #endif #include #include #include #include #include #include #include #include #include #include #if HAVE_DIRENT_H # include # define NAMLEN(dirent) strlen((dirent)->d_name) #else # define dirent direct # define NAMLEN(dirent) (dirent)->d_namlen # if HAVE_SYS_NDIR_H # include # endif # if HAVE_SYS_DIR_H # include # endif # if HAVE_NDIR_H # include # endif #endif #include "assert.h" #include "bsearch.h" #include "cyrusdb.h" #include "hash.h" #include "map.h" #include "libcyr_cfg.h" #include "cyr_lock.h" #include "util.h" #include "xmalloc.h" #include "xstrlcpy.h" #include "xstrlcat.h" #include "strarray.h" #define FNAME_QUOTADIR "/quota/" #define MAX_QUOTA_PATH 4096 /* we have the file locked iff we have an outstanding transaction */ struct subtxn { int fd; char *fnamenew; int fdnew; int delete; }; struct txn { hash_table table; /* hash table of sub-transactions */ int (*proc)(const char *, struct subtxn *); /* commit/abort procedure */ int result; /* final result of the commit/abort */ }; struct dbengine { char *path; char *data; /* allocated buffer for fetched data */ struct txn txn; /* transaction associated with this db handle */ /* sorting function */ int (*compar) (const void *s1, const void *s2); }; static int abort_txn(struct dbengine *db __attribute__((unused)), struct txn *tid); static int compar_qr(const void *v1, const void *v2); static int compar_qr_mbox(const void *v1, const void *v2); /* hash the prefix - either with or without 'user.' part */ static char name_to_hashchar(const char *name, int isprefix) { int config_fulldirhash = libcyrus_config_getswitch(CYRUSOPT_FULLDIRHASH); const char *idx; if (!*name) return '\0'; /* you can't actually prefix with a fulldirhash character! (Bug 3735) */ if (config_fulldirhash && isprefix) return '\0'; idx = strchr(name, '.'); /* skip past user. */ if (idx == NULL) { idx = name; } else { idx++; } return (char) dir_hash_c(idx, config_fulldirhash); } /* simple hash so it's easy to find these things in the filesystem; our human time is worth more than efficiency */ static void hash_quota(char *buf, size_t size, const char *qr, char *path) { int config_virtdomains = libcyrus_config_getswitch(CYRUSOPT_VIRTDOMAINS); int config_fulldirhash = libcyrus_config_getswitch(CYRUSOPT_FULLDIRHASH); char c, *p; unsigned len; if ((len = snprintf(buf, size, "%s", path)) >= size) { fatal("insufficient buffer size in hash_quota", EX_TEMPFAIL); } buf += len; size -= len; if (config_virtdomains && (p = strchr(qr, '!'))) { *p = '\0'; /* split domain!qr */ c = (char) dir_hash_c(qr, config_fulldirhash); if ((len = snprintf(buf, size, "%s%c/%s", FNAME_DOMAINDIR, c, qr)) >= size) { fatal("insufficient buffer size in hash_quota", EX_TEMPFAIL); } *p++ = '!'; /* reassemble domain!qr */ qr = p; buf += len; size -= len; if (!*qr) { /* quota for entire domain */ if (snprintf(buf, size, "%sroot", FNAME_QUOTADIR) >= (int) size) { fatal("insufficient buffer size in hash_quota", EX_TEMPFAIL); } return; } } c = name_to_hashchar(qr, 0); if (snprintf(buf, size, "%s%c/%s", FNAME_QUOTADIR, c, qr) >= (int) size) { fatal("insufficient buffer size in hash_quota", EX_TEMPFAIL); } } /* other routines call this one when they fail */ static int abort_subtxn(const char *fname, struct subtxn *tid) { int r = CYRUSDB_OK; assert(fname && tid); /* cleanup done while lock is held */ if (tid->fnamenew) { unlink(tid->fnamenew); free(tid->fnamenew); } if (tid->fdnew != -1) { r = close(tid->fdnew); } if (tid->fd != -1) { /* release lock */ r = lock_unlock(tid->fd, fname); if (r == -1) { - syslog(LOG_ERR, "IOERROR: unlocking %s: %m", fname); + xsyslog(LOG_ERR, "IOERROR: lock_unlock failed", + "fname=<%s>", + fname); r = CYRUSDB_IOERROR; } /* close */ r = close(tid->fd); if (r == -1) { - syslog(LOG_ERR, "IOERROR: closing %s: %m", fname); + xsyslog(LOG_ERR, "IOERROR: close failed", + "fname=<%s>", + fname); r = CYRUSDB_IOERROR; } } free(tid); return r; } static int commit_subtxn(const char *fname, struct subtxn *tid) { int writefd; int r = 0; struct stat sbuf; assert(fname && tid); if ((writefd = tid->fdnew) != -1) { /* we wrote something */ if (fsync(writefd) || fstat(writefd, &sbuf) == -1 || rename(tid->fnamenew, fname) == -1 || lock_unlock(writefd, fname) == -1) { - syslog(LOG_ERR, "IOERROR: writing %s: %m", tid->fnamenew); + xsyslog(LOG_ERR, "IOERROR: commit failed", + "fname=<%s>", + tid->fnamenew); r = CYRUSDB_IOERROR; } close(writefd); free(tid->fnamenew); } else if (tid->delete) { /* delete file */ r = unlink(fname); if (r == -1) { - syslog(LOG_ERR, "IOERROR: unlinking %s: %m", fname); + xsyslog(LOG_ERR, "IOERROR: unlink failed", + "fname=<%s>", + fname); r = CYRUSDB_IOERROR; } } else { /* read-only txn */ } /* release lock */ if (tid->fd != -1) { r = lock_unlock(tid->fd, fname); if (r == -1) { - syslog(LOG_ERR, "IOERROR: unlocking %s: %m", fname); + xsyslog(LOG_ERR, "IOERROR: lock_unlock failed", + "fname=<%s>", + fname); r = CYRUSDB_IOERROR; } r = close(tid->fd); if (r == -1) { - syslog(LOG_ERR, "IOERROR: closing %s: %m", fname); + xsyslog(LOG_ERR, "IOERROR: close failed", + "fname=<%s>", + fname); r = CYRUSDB_IOERROR; } } free(tid); return r; } static void free_db(struct dbengine *db) { if (db) { if (db->path) free(db->path); if (db->data) free(db->data); free_hash_table(&db->txn.table, NULL); free(db); } } static struct subtxn *new_subtxn(const char *fname __attribute__((unused)), int fd) { struct subtxn *ret = (struct subtxn *) xmalloc(sizeof(struct subtxn)); ret->fd = fd; ret->fnamenew = NULL; ret->fdnew = -1; ret->delete = 0; return ret; } static int myopen(const char *fname, int flags, struct dbengine **ret, struct txn **mytid) { struct dbengine *db = (struct dbengine *) xzmalloc(sizeof(struct dbengine)); struct stat sbuf; char *p; int r; assert(fname && ret); db->path = xstrdup(fname); construct_hash_table(&db->txn.table, 200, 0); /* strip any filename from the path */ if ((p = strrchr(db->path, '/'))) *p = '\0'; r = stat(db->path, &sbuf); if (r == -1 && errno == ENOENT && (flags & CYRUSDB_CREATE)) { if (cyrus_mkdir(fname, 0755) != -1) { r = stat(db->path, &sbuf); } } if (r == -1) { int level = (flags & CYRUSDB_CREATE) ? LOG_ERR : LOG_DEBUG; syslog(level, "IOERROR: stating quota %s: %m", db->path); free_db(db); return CYRUSDB_IOERROR; } db->compar = (flags & CYRUSDB_MBOXSORT) ? compar_qr_mbox : compar_qr; *ret = db; if (mytid) { *mytid = &db->txn; } return 0; } static int myclose(struct dbengine *db) { assert(db); free_db(db); return 0; } static int myfetch(struct dbengine *db, char *quota_path, const char **data, size_t *datalen, struct txn **tid) { struct subtxn *mytid = NULL; int quota_fd; const char *quota_base = 0; size_t quota_len = 0; int r = 0; assert(db); if (data) *data = NULL; if (datalen) *datalen = 0; if (!data || !datalen) { /* just check if the key exists */ struct stat sbuf; if (stat(quota_path, &sbuf) == -1) return CYRUSDB_NOTFOUND; return 0; } if (tid) { if (!*tid) *tid = &db->txn; else mytid = (struct subtxn *) hash_lookup(quota_path, &db->txn.table); } /* open and lock file, if needed */ if (!mytid) { quota_fd = open(quota_path, O_RDWR, 0); if (quota_fd == -1) { if (errno == ENOENT) { /* key doesn't exist */ return CYRUSDB_NOTFOUND; } - syslog(LOG_ERR, "IOERROR: opening quota file %s: %m", quota_path); + xsyslog(LOG_ERR, "IOERROR: open quota file failed", + "fname=<%s>", quota_path); return CYRUSDB_IOERROR; } if (tid) { int r; struct stat sbuf; const char *lockfailaction; r = lock_reopen(quota_fd, quota_path, &sbuf, &lockfailaction); if (r == -1) { - syslog(LOG_ERR, "IOERROR: %s quota %s: %m", lockfailaction, - quota_path); + xsyslog(LOG_ERR, "IOERROR: lock_reopen failed", + "action=<%s> fname=<%s>", + lockfailaction, quota_path); xclose(quota_fd); return CYRUSDB_IOERROR; } mytid = new_subtxn(quota_path, quota_fd); hash_insert(quota_path, mytid, &db->txn.table); } } else quota_fd = mytid->fd; free(db->data); db->data = NULL; map_refresh(quota_fd, 1, "a_base, "a_len, MAP_UNKNOWN_LEN, quota_path, 0); if (!quota_len) { *data = db->data = xstrdup(""); *datalen = 0; } else if (quota_base[quota_len-1] != '\n') { r = CYRUSDB_IOERROR; } else { *data = db->data = xstrndup(quota_base, quota_len); *datalen = quota_len - 1; db->data[*datalen] = '\0'; } map_free("a_base, "a_len); if (!tid) close(quota_fd); if (r) return r; /* magic old-format conversion */ if (db->data[0] != '%') { char *eol = strchr(db->data, '\n'); /* convert the separating \n to SP */ if (eol) *eol = ' '; } return 0; } static int fetch(struct dbengine *db, const char *key, size_t keylen, const char **data, size_t *datalen, struct txn **tid) { char quota_path[MAX_QUOTA_PATH+1], *tmpkey = NULL; /* if we need to truncate the key, do so */ if (key[keylen] != '\0') { tmpkey = xmalloc(keylen + 1); memcpy(tmpkey, key, keylen); tmpkey[keylen] = '\0'; key = tmpkey; } hash_quota(quota_path, sizeof(quota_path), key, db->path); if (tmpkey) free(tmpkey); return myfetch(db, quota_path, data, datalen, tid); } static const char *path_to_qr(const char *path, char *buf) { const char *qr; char *p; qr = strrchr(path, '/') + 1; if ((p = strstr(path, FNAME_DOMAINDIR))) { /* use the quota_path as a buffer to construct virtdomain qr */ p += strlen(FNAME_DOMAINDIR) + 2; /* +2 for hashdir */ sprintf(buf, "%.*s!%s", (int) strcspn(p, "/"), p, strcmp(qr, "root") ? qr : ""); qr = buf; } return qr; } static int compar_qr(const void *v1, const void *v2) { const char *qr1, *qr2; char qrbuf1[MAX_QUOTA_PATH+1], qrbuf2[MAX_QUOTA_PATH+1]; qr1 = path_to_qr(*((const char **) v1), qrbuf1); qr2 = path_to_qr(*((const char **) v2), qrbuf2); return strcmp(qr1, qr2); } static int compar_qr_mbox(const void *v1, const void *v2) { const char *qr1, *qr2; char qrbuf1[MAX_QUOTA_PATH+1], qrbuf2[MAX_QUOTA_PATH+1]; qr1 = path_to_qr(*((const char **) v1), qrbuf1); qr2 = path_to_qr(*((const char **) v2), qrbuf2); return bsearch_compare_mbox(qr1, qr2); } static void scan_qr_dir(char *quota_path, const char *prefix, strarray_t *pathbuf) { int config_fulldirhash = libcyrus_config_getswitch(CYRUSOPT_FULLDIRHASH); int config_virtdomains = libcyrus_config_getswitch(CYRUSOPT_VIRTDOMAINS); char *endp; char onlyc = '\0'; int c, i; DIR *qrdir; struct dirent *next = NULL; /* strip off the qr specific path */ endp = strstr(quota_path, FNAME_QUOTADIR) + strlen(FNAME_QUOTADIR); strcpy(endp, "?/"); /* check for path restriction - if there's a prefix we only * need to scan a single directory */ onlyc = name_to_hashchar(prefix, 1); c = config_fulldirhash ? 'A' : 'a'; for (i = 0; i < 26; i++, c++) { if (onlyc && c != onlyc) continue; *endp = c; qrdir = opendir(quota_path); if (qrdir) { while ((next = readdir(qrdir)) != NULL) { if (!strcmp(next->d_name, ".") || !strcmp(next->d_name, "..")) continue; if (!strncmp(next->d_name, prefix, strlen(prefix))) strarray_appendm(pathbuf, strconcat(quota_path, next->d_name, (char *)NULL)); } closedir(qrdir); } } if (config_virtdomains && !strlen(prefix) && strstr(quota_path, FNAME_DOMAINDIR)) { /* search for a domain quota */ struct stat buf; strcpy(endp, "root"); if (!stat(quota_path, &buf)) strarray_append(pathbuf, quota_path); } } static int foreach(struct dbengine *db, const char *prefix, size_t prefixlen, foreach_p *goodp, foreach_cb *cb, void *rock, struct txn **tid) { int r = CYRUSDB_OK; int config_fulldirhash = libcyrus_config_getswitch(CYRUSOPT_FULLDIRHASH); int config_virtdomains = libcyrus_config_getswitch(CYRUSOPT_VIRTDOMAINS); char quota_path[MAX_QUOTA_PATH+1]; strarray_t pathbuf = STRARRAY_INITIALIZER; int i; char *tmpprefix = NULL, *p = NULL; assert(cb); /* if we need to truncate the prefix, do so */ if (prefix[prefixlen] != '\0') { tmpprefix = xmalloc(prefixlen + 1); memcpy(tmpprefix, prefix, prefixlen); tmpprefix[prefixlen] = '\0'; prefix = tmpprefix; } hash_quota(quota_path, sizeof(quota_path), prefix, db->path); if (config_virtdomains && (p = strchr(prefix, '!'))) prefix = p + 1; /* search for the quotaroots */ scan_qr_dir(quota_path, prefix, &pathbuf); if (config_virtdomains && !prefixlen) { /* search for all virtdomain quotaroots */ char *endp; int c, i, n; DIR *qrdir; struct dirent *next = NULL; n = snprintf(quota_path, sizeof(quota_path)-3, "%s%s", db->path, FNAME_DOMAINDIR); endp = quota_path + n; c = config_fulldirhash ? 'A' : 'a'; for (i = 0; i < 26; i++, c++) { endp[0] = c; endp[1] = '/'; endp[2] = '\0'; qrdir = opendir(quota_path); if (qrdir) { while ((next = readdir(qrdir)) != NULL) { if (!strcmp(next->d_name, ".") || !strcmp(next->d_name, "..")) continue; snprintf(endp+2, sizeof(quota_path) - (n+2), "%s%s", next->d_name, FNAME_QUOTADIR); scan_qr_dir(quota_path, "", &pathbuf); } closedir(qrdir); } } } if (tmpprefix) free(tmpprefix); if (tid && !*tid) *tid = &db->txn; /* sort the quotaroots (ignoring paths) */ if (pathbuf.data) qsort(pathbuf.data, pathbuf.count, sizeof(char *), db->compar); for (i = 0; i < pathbuf.count; i++) { const char *data, *key; size_t keylen, datalen; r = myfetch(db, pathbuf.data[i], &data, &datalen, tid); if (r) break; key = path_to_qr(pathbuf.data[i], quota_path); keylen = strlen(key); if (!goodp || goodp(rock, key, keylen, data, datalen)) { /* make callback */ r = cb(rock, key, keylen, data, datalen); if (r) break; } } strarray_fini(&pathbuf); return r; } static int mystore(struct dbengine *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid, int overwrite) { char quota_path[MAX_QUOTA_PATH+1], *tmpkey = NULL; struct subtxn *mytid = NULL; int r = 0; /* if we need to truncate the key, do so */ tmpkey = xmalloc(keylen + 1); memcpy(tmpkey, key, keylen); tmpkey[keylen] = '\0'; hash_quota(quota_path, sizeof(quota_path), tmpkey, db->path); if (tmpkey) free(tmpkey); if (tid) { if (!*tid) *tid = &db->txn; else mytid = (struct subtxn *) hash_lookup(quota_path, &db->txn.table); } /* open and lock file, if needed */ if (!mytid) { int fd; struct stat sbuf; const char *lockfailaction; fd = open(quota_path, O_RDWR, 0644); if (fd == -1 && errno == ENOENT && data) { if (cyrus_mkdir(quota_path, 0755) != -1) { fd = open(quota_path, O_RDWR | O_CREAT, 0644); } } if (fd == -1 && (errno != ENOENT || data)) { - syslog(LOG_ERR, "IOERROR: opening quota file %s: %m", quota_path); + xsyslog(LOG_ERR, "IOERROR: open quota file failed", + "fname=<%s>", + quota_path); return CYRUSDB_IOERROR; } if (fd != -1) { r = lock_reopen(fd, quota_path, &sbuf, &lockfailaction); if (r == -1) { - syslog(LOG_ERR, "IOERROR: %s quota %s: %m", lockfailaction, - quota_path); + xsyslog(LOG_ERR, "IOERROR: lock_reopen failed", + "action=<%s> fname=<%s>", + lockfailaction, quota_path); xclose(fd); return CYRUSDB_IOERROR; } } mytid = new_subtxn(quota_path, fd); if (tid) hash_insert(quota_path, mytid, &db->txn.table); } if (!data) { mytid->delete = 1; } else { char new_quota_path[MAX_QUOTA_PATH+1], *buf, *p; int newfd = -1, r1 = 0; ssize_t n; if (mytid->fd != -1 && !overwrite) { if (tid) abort_txn(db, *tid); else abort_subtxn(quota_path, mytid); return CYRUSDB_EXISTS; } if (mytid->fdnew == -1) { strlcpy(new_quota_path, quota_path, sizeof(new_quota_path)); strlcat(new_quota_path, ".NEW", sizeof(new_quota_path)); unlink(new_quota_path); newfd = open(new_quota_path, O_CREAT | O_TRUNC | O_RDWR, 0666); if (newfd == -1 && errno == ENOENT) { if (cyrus_mkdir(new_quota_path, 0755) != -1) newfd = open(new_quota_path, O_CREAT | O_TRUNC | O_RDWR, 0666); } if (newfd == -1) { - syslog(LOG_ERR, "IOERROR: creating quota file %s: %m", - new_quota_path); + xsyslog(LOG_ERR, "IOERROR: creating quota file failed", + "fname=<%s>", + new_quota_path); if (tid) abort_txn(db, *tid); else abort_subtxn(quota_path, mytid); return CYRUSDB_IOERROR; } mytid->fdnew = newfd; r = lock_blocking(newfd, new_quota_path); if (r) { - syslog(LOG_ERR, "IOERROR: locking quota file %s: %m", - new_quota_path); + xsyslog(LOG_ERR, "IOERROR: lock_blocking failed", + "fname=<%s>", + new_quota_path); if (tid) abort_txn(db, *tid); else abort_subtxn(quota_path, mytid); return CYRUSDB_IOERROR; } } buf = xmalloc(datalen+1); memcpy(buf, data, datalen); if (buf[0] != '%') { /* convert separating SP to \n */ p = memchr(buf, ' ', datalen); if (p) *p = '\n'; } /* add a terminating \n */ buf[datalen] = '\n'; lseek(mytid->fdnew, 0, SEEK_SET); + /* XXX this should maybe use retry_write... */ n = write(mytid->fdnew, buf, datalen+1); if (n == (ssize_t)datalen+1) r1 = ftruncate(mytid->fdnew, datalen+1); free(buf); if (n != (ssize_t)datalen+1 || r1 == -1) { if (n == -1 || r1 == -1) - syslog(LOG_ERR, "IOERROR: writing quota file %s: %m", - new_quota_path); + xsyslog(LOG_ERR, "IOERROR: write failed", + "fname=<%s>", + new_quota_path); else - syslog(LOG_ERR, - "IOERROR: writing quota file %s: failed to write %d bytes", - new_quota_path, (int)datalen+1); + xsyslog(LOG_ERR, "IOERROR: partial write", + "fname=<%s> wanted=<%d>", + new_quota_path, (int)datalen+1); if (tid) abort_txn(db, *tid); else abort_subtxn(quota_path, mytid); return CYRUSDB_IOERROR; } if (!mytid->fnamenew) mytid->fnamenew = xstrdup(new_quota_path); } if (!tid) { /* commit immediately */ r = commit_subtxn(quota_path, mytid); } return r; } static int create(struct dbengine *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid) { return mystore(db, key, keylen, data, datalen, tid, 0); } static int store(struct dbengine *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid) { return mystore(db, key, keylen, data, datalen, tid, 1); } static int delete(struct dbengine *db, const char *key, size_t keylen, struct txn **mytid, int force __attribute__((unused))) { return mystore(db, key, keylen, NULL, 0, mytid, 1); } static void txn_proc(const char *fname, void *data, void *rock) { struct txn *tid = (struct txn *) rock; int r; r = tid->proc(fname, (struct subtxn *) data); hash_del(fname, &tid->table); if (r && !tid->result) tid->result = r; } static int commit_txn(struct dbengine *db __attribute__((unused)), struct txn *tid) { tid->proc = commit_subtxn; tid->result = 0; hash_enumerate(&tid->table, txn_proc, tid); return tid->result; } static int abort_txn(struct dbengine *db __attribute__((unused)), struct txn *tid) { tid->proc = abort_subtxn; tid->result = 0; hash_enumerate(&tid->table, txn_proc, tid); return tid->result; } /* quotalegacy gets compar set at startup, but it's not the same */ static int mycompar(struct dbengine *db, const char *a, int alen, const char*b, int blen) { if (db->compar == compar_qr_mbox) return bsearch_ncompare_mbox(a, alen, b, blen); else return bsearch_ncompare_raw(a, alen, b, blen); } HIDDEN struct cyrusdb_backend cyrusdb_quotalegacy = { "quotalegacy", /* name */ &cyrusdb_generic_init, &cyrusdb_generic_done, &cyrusdb_generic_sync, &cyrusdb_generic_noarchive, &cyrusdb_generic_unlink, &myopen, &myclose, &fetch, &fetch, NULL, &foreach, &create, &store, &delete, &commit_txn, &abort_txn, NULL, NULL, NULL, &mycompar }; diff --git a/lib/cyrusdb_sql.c b/lib/cyrusdb_sql.c index 669811f0c..6c1acef3e 100644 --- a/lib/cyrusdb_sql.c +++ b/lib/cyrusdb_sql.c @@ -1,924 +1,949 @@ /* cyrusdb_sql: SQL db backends * * Copyright (c) 1998-2004 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any other legal * details, please contact * Office of Technology Transfer * Carnegie Mellon University * 5000 Forbes Avenue * Pittsburgh, PA 15213-3890 * (412) 268-4387, fax: (412) 268-7395 * tech-transfer@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include "assert.h" #include "bsearch.h" #include "cyrusdb.h" #include "libcyr_cfg.h" #include "xmalloc.h" #include "util.h" extern void fatal(const char *, int); typedef int exec_cb(void *rock, const char *key, size_t keylen, const char *data, size_t datalen); typedef struct sql_engine { const char *name; const char *binary_type; void *(*sql_open)(char *host, char *port, int usessl, const char *user, const char *password, const char *database); char *(*sql_escape)(void *conn, char **to, const char *from, size_t fromlen); int (*sql_begin_txn)(void *conn); int (*sql_commit_txn)(void *conn); int (*sql_rollback_txn)(void *conn); int (*sql_exec)(void *conn, const char *cmd, exec_cb *cb, void *rock); void (*sql_close)(void *conn); } sql_engine_t; struct dbengine { void *conn; /* connection to database */ char *table; /* table that we are operating on */ char *esc_key; /* allocated buffer for escaped key */ char *esc_data; /* allocated buffer for escaped data */ char *data; /* allocated buffer for fetched data */ }; struct txn { char *lastkey; /* allocated buffer for last SELECTed key */ size_t keylen; }; static int dbinit = 0; static const sql_engine_t *dbengine = NULL; #ifdef HAVE_MYSQL #include static void *_mysql_open(char *host, char *port, int usessl, const char *user, const char *password, const char *database) { MYSQL *mysql; if (!(mysql = mysql_init(NULL))) { - syslog(LOG_ERR, "DBERROR: SQL backend could not execute mysql_init()"); + xsyslog(LOG_ERR, "DBERROR: SQL backend could not execute mysql_init", NULL); return NULL; } return mysql_real_connect(mysql, host, user, password, database, port ? strtoul(port, NULL, 10) : 0, NULL, usessl ? CLIENT_SSL : 0); } static char *_mysql_escape(void *conn, char **to, const char *from, size_t fromlen) { size_t tolen; *to = xrealloc(*to, 2 * fromlen + 1); /* +1 for NUL */ tolen = mysql_real_escape_string(conn, *to, from, fromlen); (void)tolen; return *to; } static int _mysql_exec(void *conn, const char *cmd, exec_cb *cb, void *rock) { MYSQL_RES *result; MYSQL_ROW row; int len, r = 0; syslog(LOG_DEBUG, "executing SQL cmd: %s", cmd); len = strlen(cmd); /* mysql_real_query() doesn't want a terminating ';' */ if (cmd[len-1] == ';') len--; /* run the query */ if ((mysql_real_query(conn, cmd, len) < 0) || *mysql_error(conn)) { - syslog(LOG_ERR, "DBERROR: SQL query failed: %s", mysql_error(conn)); + xsyslog(LOG_ERR, "DBERROR: SQL query failed", + "mysql_error=<%s>", + mysql_error(conn)); return CYRUSDB_INTERNAL; } /* see if we should expect some results */ if (!mysql_field_count(conn)) { /* no results (BEGIN, COMMIT, ROLLBACK, CREATE, INSERT, UPDATE, DELETE) */ syslog(LOG_DEBUG, "no results from SQL cmd"); return 0; } /* get the results */ result = mysql_store_result(conn); /* process the results */ while (!r && (row = mysql_fetch_row(result))) { unsigned long *length = mysql_fetch_lengths(result); r = cb(rock, row[0], length[0], row[1], length[1]); } /* free result */ mysql_free_result(result); return r; } static int _mysql_begin_txn(void *conn) { return _mysql_exec(conn, #if MYSQL_VERSION_ID >= 40011 "START TRANSACTION", #else "BEGIN", #endif NULL, NULL); } static int _mysql_commit_txn(void *conn) { return _mysql_exec(conn, "COMMIT", NULL, NULL); } static int _mysql_rollback_txn(void *conn) { return _mysql_exec(conn, "ROLLBACK", NULL, NULL); } static void _mysql_close(void *conn) { mysql_close(conn); } #endif /* HAVE_MYSQL */ #ifdef HAVE_PGSQL #include #define sql_max(a, b) ((a) > (b) ? (a) : (b)) #define sql_len(input) ((input) ? strlen(input) : 0) #define sql_exists(input) ((input) && (*input)) static void *_pgsql_open(char *host, char *port, int usessl, const char *user, const char *password, const char *database) { PGconn *conn = NULL; struct buf conninfo = BUF_INITIALIZER; /* create the connection info string */ /* add each term that exists */ if (sql_exists(host)) buf_printf(&conninfo, " host='%s'", host); if (sql_exists(port)) buf_printf(&conninfo, " port='%s'", port); if (sql_exists(user)) buf_printf(&conninfo, " user='%s'", user); if (sql_exists(password)) buf_printf(&conninfo, " password='%s'", password); if (sql_exists(database)) buf_printf(&conninfo, " dbname='%s'", database); buf_printf(&conninfo, " requiressl='%d'", usessl); conn = PQconnectdb(buf_cstring(&conninfo)); buf_free(&conninfo); if ((PQstatus(conn) != CONNECTION_OK)) { - syslog(LOG_ERR, "DBERROR: SQL backend: %s", PQerrorMessage(conn)); + xsyslog(LOG_ERR, "DBERROR: connection failed", + "pgsql_error=<%s>", + PQerrorMessage(conn)); return NULL; } return conn; } static char *_pgsql_escape(void *conn __attribute__((unused)), char **to __attribute__((unused)), const char *from, size_t fromlen) { size_t tolen; /* returned buffer MUST be freed by caller */ return (char *) PQescapeBytea((unsigned char *) from, fromlen, &tolen); } static int _pgsql_exec(void *conn, const char *cmd, exec_cb *cb, void *rock) { PGresult *result; int row_count, i, r = 0; ExecStatusType status; syslog(LOG_DEBUG, "executing SQL cmd: %s", cmd); /* run the query */ result = PQexec(conn, cmd); /* check the status */ status = PQresultStatus(result); if (status == PGRES_COMMAND_OK) { /* no results (BEGIN, COMMIT, ROLLBACK, CREATE, INSERT, UPDATE, DELETE) */ PQclear(result); return 0; } else if (status != PGRES_TUPLES_OK) { /* error */ syslog(LOG_DEBUG, "SQL backend: %s ", PQresStatus(status)); PQclear(result); return CYRUSDB_INTERNAL; } row_count = PQntuples(result); for (i = 0; !r && i < row_count; i++) { char *key, *data; size_t keylen, datalen; key = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(result, i, 0), &keylen); data = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(result, i, 1), &datalen); r = cb(rock, key, keylen, data, datalen); free(key); free(data); } /* free result */ PQclear(result); return r; } static int _pgsql_begin_txn(void *conn) { return _pgsql_exec(conn, "BEGIN;", NULL, NULL); } static int _pgsql_commit_txn(void *conn) { return _pgsql_exec(conn, "COMMIT;", NULL, NULL); } static int _pgsql_rollback_txn(void *conn) { return _pgsql_exec(conn, "ROLLBACK;", NULL, NULL); } static void _pgsql_close(void *conn) { PQfinish(conn); } #endif /* HAVE_PGSQL */ #ifdef HAVE_SQLITE #include static void *_sqlite_open(char *host __attribute__((unused)), char *port __attribute__((unused)), int usessl __attribute__((unused)), const char *user __attribute__((unused)), const char *password __attribute__((unused)), const char *database) { int rc; sqlite3 *db; rc = sqlite3_open(database, &db); if (rc != SQLITE_OK) { - syslog(LOG_ERR, "DBERROR: SQL backend: %s", sqlite3_errmsg(db)); + xsyslog(LOG_ERR, "DBERROR: SQL backend", + "sqlite3_error=<%s>", + sqlite3_errmsg(db)); sqlite3_close(db); } return db; } static char *_sqlite_escape(void *conn __attribute__((unused)), char **to, const char *from, size_t fromlen) { size_t tolen; #if 0 *to = xrealloc(*to, 2 + (257 * fromlen) / 254); tolen = sqlite3_encode_binary(from, fromlen, *to); #else *to = xrealloc(*to, fromlen + 1); memcpy(*to, from, fromlen); tolen = fromlen; (*to)[tolen] = '\0'; #endif return *to; } static int _sqlite_exec(void *conn, const char *cmd, exec_cb *cb, void *rock) { int rc, r = 0; sqlite3_stmt *stmt = NULL; const char *tail; syslog(LOG_DEBUG, "executing SQL cmd: %s", cmd); /* compile the SQL cmd */ rc = sqlite3_prepare(conn, cmd, strlen(cmd), &stmt, &tail); if (rc != SQLITE_OK) { syslog(LOG_DEBUG, "SQL backend: %s ", sqlite3_errmsg(conn)); return CYRUSDB_INTERNAL; } /* process the results */ while (!r && (rc = sqlite3_step(stmt)) == SQLITE_ROW) { const unsigned char *key = sqlite3_column_text(stmt, 0); int keylen = sqlite3_column_bytes(stmt, 0); const unsigned char *data = sqlite3_column_text(stmt, 1); int datalen = sqlite3_column_bytes(stmt, 1); r = cb(rock, (char *) key, keylen, (char *) data, datalen); } /* cleanup */ rc = sqlite3_finalize(stmt); if (rc != SQLITE_OK) { syslog(LOG_DEBUG, "SQL backend: %s ", sqlite3_errmsg(conn)); return CYRUSDB_INTERNAL; } return r; } static int _sqlite_begin_txn(void *conn) { return _sqlite_exec(conn, "BEGIN TRANSACTION", NULL, NULL); } static int _sqlite_commit_txn(void *conn) { return _sqlite_exec(conn, "COMMIT TRANSACTION", NULL, NULL); } static int _sqlite_rollback_txn(void *conn) { return _sqlite_exec(conn, "ROLLBACK TRANSACTION", NULL, NULL); } static void _sqlite_close(void *conn) { sqlite3_close(conn); } #endif /* HAVE_SQLITE */ static const sql_engine_t sql_engines[] = { #ifdef HAVE_MYSQL { "mysql", "BLOB", &_mysql_open, &_mysql_escape, &_mysql_begin_txn, &_mysql_commit_txn, &_mysql_rollback_txn, &_mysql_exec, &_mysql_close }, #endif /* HAVE_MYSQL */ #ifdef HAVE_PGSQL { "pgsql", "BYTEA", &_pgsql_open, &_pgsql_escape, &_pgsql_begin_txn, &_pgsql_commit_txn, &_pgsql_rollback_txn, &_pgsql_exec, &_pgsql_close }, #endif #ifdef HAVE_SQLITE { "sqlite", "BLOB", &_sqlite_open, &_sqlite_escape, &_sqlite_begin_txn, &_sqlite_commit_txn, &_sqlite_rollback_txn, &_sqlite_exec, &_sqlite_close }, #endif { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL } }; static int init(const char *dbdir __attribute__((unused)), int flags __attribute__((unused))) { const char *engine_name; int r = 0; if (dbinit++) return 0; engine_name = libcyrus_config_getstring(CYRUSOPT_SQL_ENGINE); /* find the correct engine */ dbengine = sql_engines; while (dbengine->name) { if (!engine_name || !strcasecmp(engine_name, dbengine->name)) break; dbengine++; } if (!dbengine->name) { char errbuf[1024]; snprintf(errbuf, sizeof(errbuf), "SQL engine %s not supported", engine_name); fatal(errbuf, EX_CONFIG); } if (!engine_name) { syslog(LOG_DEBUG, "SQL backend defaulting to engine '%s'", dbengine->name); } dbinit = 1; return r; } static int done(void) { --dbinit; return 0; } static struct txn *start_txn(struct dbengine *db) { /* start a transaction */ if (dbengine->sql_begin_txn(db->conn)) { - syslog(LOG_ERR, "DBERROR: failed to start txn on %s", - db->table); + xsyslog(LOG_ERR, "DBERROR: failed to start transation", + "table=<%s>", + db->table); return NULL; } return xzmalloc(sizeof(struct txn)); } static int myopen(const char *fname, int flags, struct dbengine **ret, struct txn **mytid) { const char *database, *hostnames, *user, *passwd; char *host_ptr, *host, *cur_host, *cur_port; int usessl; void *conn = NULL; char *p, *table, cmd[1024]; assert(fname); assert(ret); /* make a connection to the database */ database = libcyrus_config_getstring(CYRUSOPT_SQL_DATABASE); hostnames = libcyrus_config_getstring(CYRUSOPT_SQL_HOSTNAMES); user = libcyrus_config_getstring(CYRUSOPT_SQL_USER); passwd = libcyrus_config_getstring(CYRUSOPT_SQL_PASSWD); usessl = libcyrus_config_getswitch(CYRUSOPT_SQL_USESSL); /* loop around hostnames until we get a connection */ syslog(LOG_DEBUG, "SQL backend trying to connect to a host"); /* create a working version of the hostnames */ host_ptr = hostnames ? xstrdup(hostnames) : NULL; /* make sqlite clever */ if (!database) database = fname; cur_host = host = host_ptr; while (cur_host != NULL) { host = strchr(host,','); if (host != NULL) { host[0] = '\0'; /* loop till we find some text */ while (!Uisalnum(host[0])) host++; } syslog(LOG_DEBUG, "SQL backend trying to open db '%s' on host '%s'%s", database, cur_host, usessl ? " using SSL" : ""); /* set the optional port */ if ((cur_port = strchr(cur_host, ':'))) *cur_port++ = '\0'; conn = dbengine->sql_open(cur_host, cur_port, usessl, user, passwd, database); if (conn) break; syslog(LOG_WARNING, "DBERROR: SQL backend could not connect to host %s", cur_host); cur_host = host; } if (host_ptr) free(host_ptr); if (!conn) { - syslog(LOG_ERR, "DBERROR: could not open SQL database '%s'", database); + xsyslog(LOG_ERR, "DBERROR: could not open SQL database", + "database=<%s>", + database); return CYRUSDB_IOERROR; } /* get the name of the table and CREATE it if necessary */ /* strip any path from the fname */ p = strrchr(fname, '/'); table = xstrdup(p ? ++p : fname); /* convert '.' to '_' */ if ((p = strrchr(table, '.'))) *p = '_'; /* check if the table exists */ /* XXX is this the best way to do this? */ snprintf(cmd, sizeof(cmd), "SELECT * FROM %s LIMIT 0;", table); if (dbengine->sql_exec(conn, cmd, NULL, NULL)) { if (flags & CYRUSDB_CREATE) { /* create the table */ snprintf(cmd, sizeof(cmd), "CREATE TABLE %s (dbkey %s NOT NULL, data %s);", table, dbengine->binary_type, dbengine->binary_type); if (dbengine->sql_exec(conn, cmd, NULL, NULL)) { - syslog(LOG_ERR, "DBERROR: SQL failed: %s", cmd); + xsyslog(LOG_ERR, "DBERROR: SQL failed", + "command=<%s>", + cmd); dbengine->sql_close(conn); return CYRUSDB_INTERNAL; } } else { return CYRUSDB_NOTFOUND; } } *ret = (struct dbengine *) xzmalloc(sizeof(struct dbengine)); (*ret)->conn = conn; (*ret)->table = table; if (mytid) { *mytid = start_txn(*ret); } return 0; } static int myclose(struct dbengine *db) { assert(db); dbengine->sql_close(db->conn); free(db->table); if (db->esc_key) free(db->esc_key); if (db->esc_data) free(db->esc_data); if (db->data) free(db->data); free(db); return 0; } struct select_rock { int found; struct txn *tid; foreach_cb *goodp; foreach_cb *cb; void *rock; }; static int select_cb(void *rock, const char *key, size_t keylen, const char *data, size_t datalen) { struct select_rock *srock = (struct select_rock *) rock; int r = CYRUSDB_OK; /* if we're in a transaction, save this key */ if (srock->tid) { srock->tid->lastkey = xrealloc(srock->tid->lastkey, keylen); memcpy(srock->tid->lastkey, key, keylen); srock->tid->keylen = keylen; } /* see if we want this entry */ if (!srock->goodp || srock->goodp(srock->rock, key, keylen, data, datalen)) { srock->found = 1; /* make callback */ if (srock->cb) r = srock->cb(srock->rock, key, keylen, data, datalen); } return r; } struct fetch_rock { char **data; size_t *datalen; }; static int fetch_cb(void *rock, const char *key __attribute__((unused)), size_t keylen __attribute__((unused)), const char *data, size_t datalen) { struct fetch_rock *frock = (struct fetch_rock *) rock; if (frock->data) { *(frock->data) = xrealloc(*(frock->data), datalen); memcpy(*(frock->data), data, datalen); } if (frock->datalen) *(frock->datalen) = datalen; return 0; } static int fetch(struct dbengine *db, const char *key, size_t keylen, const char **data, size_t *datalen, struct txn **tid) { char cmd[1024], *esc_key; size_t len = 0; struct fetch_rock frock = { &db->data, &len }; struct select_rock srock = { 0, NULL, NULL, &fetch_cb, &frock }; int r; assert(db); assert(key); assert(keylen); if (datalen) assert(data); if (data) *data = NULL; if (datalen) *datalen = 0; if (tid) { if (!*tid && !(*tid = start_txn(db))) return CYRUSDB_INTERNAL; srock.tid = *tid; } /* fetch the data */ esc_key = dbengine->sql_escape(db->conn, &db->esc_key, key, keylen); snprintf(cmd, sizeof(cmd), "SELECT * FROM %s WHERE dbkey = '%s';", db->table, esc_key); if (esc_key != db->esc_key) free(esc_key); r = dbengine->sql_exec(db->conn, cmd, &select_cb, &srock); if (r) { - syslog(LOG_ERR, "DBERROR: SQL failed %s", cmd); + xsyslog(LOG_ERR, "DBERROR: SQL failed", + "command=<%s>", + cmd); if (tid) dbengine->sql_rollback_txn(db->conn); return CYRUSDB_INTERNAL; } if (!srock.found) return CYRUSDB_NOTFOUND; if (data) *data = db->data; if (datalen) *datalen = len; return 0; } static int foreach(struct dbengine *db, const char *prefix, size_t prefixlen, foreach_p *goodp, foreach_cb *cb, void *rock, struct txn **tid) { char cmd[1024], *esc_key = NULL; struct select_rock srock = { 0, NULL, goodp, cb, rock }; int r; assert(db); assert(cb); if (prefixlen) assert(prefix); if (tid) { if (!*tid && !(*tid = start_txn(db))) return CYRUSDB_INTERNAL; srock.tid = *tid; } /* fetch the data */ if (prefixlen) /* XXX hack for SQLite */ esc_key = dbengine->sql_escape(db->conn, &db->esc_key, prefix, prefixlen); snprintf(cmd, sizeof(cmd), "SELECT * FROM %s WHERE dbkey LIKE '%s%%' ORDER BY dbkey;", db->table, esc_key ? esc_key : ""); if (esc_key && (esc_key != db->esc_key)) free(esc_key); r = dbengine->sql_exec(db->conn, cmd, &select_cb, &srock); if (r) { - syslog(LOG_ERR, "DBERROR: SQL failed %s", cmd); + xsyslog(LOG_ERR, "DBERROR: SQL failed", + "command=<%s>", + cmd); if (tid) dbengine->sql_rollback_txn(db->conn); return CYRUSDB_INTERNAL; } return 0; } static int mystore(struct dbengine *db, const char *key, int keylen, const char *data, int datalen, struct txn **tid, int overwrite, int isdelete) { char cmd[1024], *esc_key; int free_esc_key = 0; const char dummy = 0; int r = 0; assert(db); assert(key); assert(keylen); if (datalen) assert(data); if (!data) data = &dummy; if (tid && !*tid && !(*tid = start_txn(db))) return CYRUSDB_INTERNAL; esc_key = dbengine->sql_escape(db->conn, &db->esc_key, key, keylen); free_esc_key = (esc_key != db->esc_key); if (isdelete) { /* DELETE the entry */ snprintf(cmd, sizeof(cmd), "DELETE FROM %s WHERE dbkey = '%s';", db->table, esc_key); r = dbengine->sql_exec(db->conn, cmd, NULL, NULL); /* see if we just removed the previously SELECTed key */ if (!r && tid && *tid && (*tid)->keylen == strlen(esc_key) && !memcmp((*tid)->lastkey, esc_key, (*tid)->keylen)) { (*tid)->keylen = 0; } } else { /* INSERT/UPDATE the entry */ struct select_rock srock = { 0, NULL, NULL, NULL, NULL }; char *esc_data = dbengine->sql_escape(db->conn, &db->esc_data, data, datalen); int free_esc_data = (esc_data != db->esc_data); /* see if we just SELECTed this key in this transaction */ if (tid && *tid) { if ((*tid)->keylen == strlen(esc_key) && !memcmp((*tid)->lastkey, esc_key, (*tid)->keylen)) { srock.found = 1; } srock.tid = *tid; } /* check if the entry exists */ if (!srock.found) { snprintf(cmd, sizeof(cmd), "SELECT * FROM %s WHERE dbkey = '%s';", db->table, esc_key); r = dbengine->sql_exec(db->conn, cmd, &select_cb, &srock); } if (!r && srock.found) { if (overwrite) { /* already have this entry, UPDATE it */ snprintf(cmd, sizeof(cmd), "UPDATE %s SET data = '%s' WHERE dbkey = '%s';", db->table, esc_data, esc_key); r = dbengine->sql_exec(db->conn, cmd, NULL, NULL); } else { if (tid) dbengine->sql_rollback_txn(db->conn); return CYRUSDB_EXISTS; } } else if (!r && !srock.found) { /* INSERT the new entry */ snprintf(cmd, sizeof(cmd), "INSERT INTO %s VALUES ('%s', '%s');", db->table, esc_key, esc_data); r = dbengine->sql_exec(db->conn, cmd, NULL, NULL); } if (free_esc_data) free(esc_data); } if (free_esc_key) free(esc_key); if (r) { - syslog(LOG_ERR, "DBERROR: SQL failed: %s", cmd); + xsyslog(LOG_ERR, "DBERROR: SQL failed", + "command=<%s>", + cmd); if (tid) dbengine->sql_rollback_txn(db->conn); return CYRUSDB_INTERNAL; } return 0; } static int create(struct dbengine *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid) { return mystore(db, key, keylen, data, datalen, tid, 0, 0); } static int store(struct dbengine *db, const char *key, size_t keylen, const char *data, size_t datalen, struct txn **tid) { return mystore(db, key, keylen, data, datalen, tid, 1, 0); } static int delete(struct dbengine *db, const char *key, size_t keylen, struct txn **tid, int force __attribute__((unused))) { return mystore(db, key, keylen, NULL, 0, tid, 1, 1); } static int finish_txn(struct dbengine *db, struct txn *tid, int commit) { if (tid) { int rc = commit ? dbengine->sql_commit_txn(db->conn) : dbengine->sql_rollback_txn(db->conn); if (tid->lastkey) free(tid->lastkey); free(tid); if (rc) { - syslog(LOG_ERR, "DBERROR: failed to %s txn on %s", - commit ? "commit" : "abort", db->table); + if (commit) { + xsyslog(LOG_ERR, "DBERROR: failed to commit transaction" + "table=<%s>", + db->table); + } + else { + xsyslog(LOG_ERR, "DBERROR: failed to abort transaction", + "table=<%s>", + db->table); + } return CYRUSDB_INTERNAL; } } return 0; } static int commit_txn(struct dbengine *db, struct txn *tid) { assert(db); assert(tid); return finish_txn(db, tid, 1); } static int abort_txn(struct dbengine *db, struct txn *tid) { assert(db); assert(tid); return finish_txn(db, tid, 0); } /* SQL databases have all sorts of evil collations - we can't * make any assumptions though, so just assume raw */ static int mycompar(struct dbengine *db __attribute__((unused)), const char *a, int alen, const char *b, int blen) { (void)db; return bsearch_ncompare_raw(a, alen, b, blen); } HIDDEN struct cyrusdb_backend cyrusdb_sql = { "sql", /* name */ &init, &done, &cyrusdb_generic_sync, &cyrusdb_generic_noarchive, NULL, &myopen, &myclose, &fetch, &fetch, NULL, &foreach, &create, &store, &delete, &commit_txn, &abort_txn, NULL, NULL, NULL, &mycompar };