diff --git a/Makefile.am b/Makefile.am index a3310041e..050f4dfad 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,2090 +1,2090 @@ # # @configure_input@ # # Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. The name "Carnegie Mellon University" must not be used to # endorse or promote products derived from this software without # prior written permission. For permission or any legal # details, please contact # Carnegie Mellon University # Center for Technology Transfer and Enterprise Creation # 4615 Forbes Avenue # Suite 302 # Pittsburgh, PA 15213 # (412) 268-7393, fax: (412) 268-7395 # innovation@andrew.cmu.edu # # 4. Redistributions of any form whatsoever must retain the following # acknowledgment: # "This product includes software developed by Computing Services # at Carnegie Mellon University (http://www.cmu.edu/computing/)." # # CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE # FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN # AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING # OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ACLOCAL_AMFLAGS = -I cmulocal AM_CFLAGS = @PERL_CCCDLFLAGS@ $(GCOV_CFLAGS) AM_CXXFLAGS = $(GCOV_CXXFLAGS) AM_CPPFLAGS = \ $(COM_ERR_CPPFLAGS) \ -I${top_builddir} \ -I${top_builddir}/lib \ -I${top_srcdir} \ -I${top_srcdir}/lib \ -DLIBEXEC_DIR=\"$(libexecdir)\" \ -DSBIN_DIR=\"$(sbindir)\" \ -DSYSCONF_DIR=\"$(sysconfdir)\" \ ${DEFS} \ ${LOCALDEFS} \ $(SASLFLAGS) \ $(SSL_CPPFLAGS) \ $(ICU_CFLAGS) if HAVE_LDAP AM_CPPFLAGS += $(LDAP_CPPFLAGS) endif # HAVE_LDAP AM_LDFLAGS = $(COV_LDFLAGS) $(ICU_LIBS) # have distcheck try to build with all optional components enabled, to aid # detection of missing files for these components AM_DISTCHECK_CONFIGURE_FLAGS = \ --enable-http \ --enable-calalarmd \ --enable-replication \ --with-openssl=yes \ --enable-nntp \ --enable-murder \ --enable-idled \ --enable-sieve \ --enable-autocreate \ --enable-backup \ --enable-xapian \ --enable-jmap \ --with-ldap BUILT_SOURCES = \ lib/imapopts.c \ lib/imapopts.h CLEANFILES = \ lib/chartable.c \ lib/imapopts.c \ lib/imapopts.h DISTCLEANFILES = \ com_err/et/compile_et \ imap/http_caldav_js.h \ imap/http_carddav_js.h \ imap/http_err.c \ imap/http_err.h \ imap/imap_err.c \ imap/imap_err.h \ imap/jmap_err.c \ imap/jmap_err.h \ imap/lmtp_err.c \ imap/lmtp_err.h \ imap/mupdate_err.c \ imap/mupdate_err.h \ imap/nntp_err.c \ imap/nntp_err.h \ imap/promdata.c \ imap/promdata.h \ imap/tz_err.c \ imap/tz_err.h \ perl/sieve/scripts/installsieve \ perl/sieve/scripts/sieveshell \ sieve/sieve_err.c \ sieve/sieve_err.h MAINTAINERCLEANFILES = \ doc/legacy/murder.png \ doc/legacy/netnews.png \ man/imapd.conf.5 \ man/sieveshell.1 \ sieve/addr.h \ sieve/sieve.h SUBDIRS = . DIST_SUBDIRS = . perl/annotator perl/imap perl/sieve/managesieve dist_sysconf_DATA = lib_LTLIBRARIES = lib/libcyrus_min.la lib/libcyrus.la check_PROGRAMS = libexec_PROGRAMS = sbin_PROGRAMS = noinst_HEADERS = noinst_LTLIBRARIES = noinst_PROGRAMS = if COM_ERR COMPILE_ET_DEP = com_err/et/compile_et BUILT_SOURCES += com_err/et/compile_et com_err/et/libcyrus_com_err.la lib_LTLIBRARIES += com_err/et/libcyrus_com_err.la endif # COM_ERR bin_PROGRAMS = imtest/imtest if SERVER BUILT_SOURCES += \ imap/http_err.c \ imap/http_err.h \ imap/imap_err.c \ imap/imap_err.h \ imap/lmtp_err.c \ imap/lmtp_err.h \ imap/mupdate_err.c \ imap/mupdate_err.h \ imap/promdata.c \ imap/promdata.h \ lib/htmlchar.c \ lib/htmlchar.h \ imap/rfc822_header.c \ imap/rfc822_header.h \ imap/mailbox_header_cache.h lib_LTLIBRARIES += imap/libcyrus_imap.la libexec_PROGRAMS += \ master/master \ imap/imapd \ imap/lmtpd \ imap/pop3d \ imap/promstatsd \ imap/smmapd sbin_PROGRAMS += \ imap/arbitron \ imap/chk_cyrus \ imap/ctl_conversationsdb \ imap/ctl_cyrusdb \ imap/ctl_deliver \ imap/ctl_mboxlist \ imap/cvt_cyrusdb \ imap/cyr_df \ imap/cyrdump \ imap/cyr_dbtool \ imap/cyr_deny \ imap/cyr_expire \ imap/cyr_info \ imap/cyr_buildinfo \ imap/cyr_sequence \ imap/cyr_synclog \ imap/cyr_userseen \ imap/cyr_virusscan \ imap/deliver \ imap/ipurge \ imap/mbexamine \ imap/mbpath \ imap/mbtool \ imap/quota \ imap/reconstruct \ imap/cvt_xlist_specialuse noinst_PROGRAMS += \ imap/message_test \ imap/search_test if USE_SQUAT noinst_PROGRAMS += imap/squat_dump endif if SQUATTER # Despite the name, the squatter program handles any search engine sbin_PROGRAMS += imap/squatter endif if NNTPD libexec_PROGRAMS += imap/nntpd sbin_PROGRAMS += imap/fetchnews BUILT_SOURCES += imap/nntp_err.c endif # NNTPD libexec_PROGRAMS += imap/fud if IDLED libexec_PROGRAMS += imap/idled endif # IDLED if MURDER libexec_PROGRAMS += imap/mupdate endif # MURDER if CALALARMD libexec_PROGRAMS += imap/calalarmd endif if HTTPD AM_CPPFLAGS += $(HTTP_CPPFLAGS) AM_LDFLAGS += $(HTTP_LIBS) BUILT_SOURCES += \ imap/http_caldav_js.h \ imap/http_carddav_js.h \ imap/tz_err.c \ imap/tz_err.h libexec_PROGRAMS += imap/httpd check_PROGRAMS += imap/ical_apply_patch sbin_PROGRAMS += \ imap/ctl_zoneinfo \ imap/dav_reconstruct endif # HTTPD if REPLICATION libexec_PROGRAMS += imap/sync_server sbin_PROGRAMS += imap/sync_client imap/sync_reset endif # REPLICATION if BACKUP noinst_LTLIBRARIES += backup/libcyrus_backup.la noinst_HEADERS += backup/backup.h libexec_PROGRAMS += backup/backupd sbin_PROGRAMS += backup/ctl_backups backup/cyr_backup backup/restore endif # BACKUP if HAVE_SSL sbin_PROGRAMS += imap/tls_prune endif # HAVE_SSL sbin_PROGRAMS += imap/unexpunge if SIEVE check_PROGRAMS += notifyd/notifytest libexec_PROGRAMS += notifyd/notifyd endif # SIEVE endif # SERVER if CMULOCAL dist_sysconf_DATA += depot/rc.local.imap depot/rc.local.ptclient sbin_PROGRAMS += netnews/remotepurge endif # CMULOCAL if PTCLIENT sbin_PROGRAMS += ptclient/ptdump ptclient/ptexpire libexec_PROGRAMS += \ ptclient/ptloader endif # PTCLIENT if PERL SUBDIRS += perl/annotator perl/imap noinst_LTLIBRARIES += perl/libcyrus.la perl/libcyrus_min.la endif # PERL if SIEVE if PERL SUBDIRS += perl/sieve/managesieve noinst_LTLIBRARIES += perl/sieve/lib/libisieve.la endif # PERL BUILT_SOURCES += sieve/addr.c sieve/sieve.c sieve/sieve_err.c noinst_LTLIBRARIES += sieve/libcyrus_sieve_lex.la lib_LTLIBRARIES += sieve/libcyrus_sieve.la check_PROGRAMS += sieve/test sieve/test_mailbox sbin_PROGRAMS += sieve/sievec sieve/sieved if SERVER libexec_PROGRAMS += timsieved/timsieved endif # SERVER endif # SIEVE EXTRA_DIST = \ COPYING \ README.md \ VERSION \ com_err/et/et_c.awk \ com_err/et/et_h.awk \ com_err/et/test1.et \ com_err/et/test2.et \ com_err/et/test_et.c \ contrib/sieve-spamassassin \ contrib/fud-client.c \ contrib/deliver-notify-zephyr.patch \ contrib/add-cyrus-user \ contrib/README \ contrib/cyrusv2.mc \ contrib/dkim_canon_ischedule.patch \ contrib/notify_unix/notify \ contrib/notify_unix/net-server-prefork-0.01.tgz \ contrib/notify_unix/README \ contrib/notify_unix/sql_notify.pl \ contrib/notify_unix/simple_notify.pl \ contrib/squatrunner.pl \ contrib/mupdate-test.pl \ contrib/squatrunner.txt \ cunit/cacert.pem \ cunit/cert.pem \ cunit/cunit.pl \ cunit/cunit-to-junit.pl \ cunit/key.pem \ cunit/vg.supp \ doc/legacy/murder.png \ doc/legacy/netnews.png \ doc \ docsrc \ imap/http_caldav.js \ imap/http_carddav.js \ imap/http_err.et \ imap/imap_err.et \ imap/jmap_err.et \ imap/lmtp_err.et \ imap/mailbox_header_cache.gperf \ imap/mupdate_err.et \ imap/nntp_err.et \ imap/promdata.p \ imap/rfc822_header.st \ imap/tz_err.et \ lib/charset/aliases.txt \ lib/charset/UnicodeData.txt \ lib/charset/unifix.txt \ lib/charset/us-ascii.t \ lib/htmlchar.st \ lib/imapoptions \ lib/test/cyrusdb.c \ lib/test/cyrusdb.INPUT \ lib/test/cyrusdblong.INPUT \ lib/test/cyrusdblong.OUTPUT \ lib/test/cyrusdb.OUTPUT \ lib/test/cyrusdbtxn.INPUT \ lib/test/cyrusdbtxn.OUTPUT \ lib/test/pool.c \ lib/test/rnddb.c \ master/README \ netnews/inn.diffs \ perl/annotator/Daemon.pm \ perl/annotator/Makefile.PL \ perl/annotator/Makefile.PL.in \ perl/annotator/MANIFEST \ perl/annotator/MANIFEST.in \ perl/annotator/Message.pm \ perl/annotator/README \ perl/imap/Changes \ perl/imap/cyradm.sh \ perl/imap/cyrperl.h \ perl/imap/examples/auditmbox.pl \ perl/imap/examples/imapcollate.pl \ perl/imap/examples/imapdu.pl \ perl/imap/examples/test-imsp.pl \ perl/imap/IMAP/Admin.pm \ perl/imap/IMAP/IMSP.pm \ perl/imap/IMAP/Shell.pm \ perl/imap/IMAP.pm \ perl/imap/IMAP.xs \ perl/imap/Makefile.PL \ perl/imap/Makefile.PL.in \ perl/imap/MANIFEST \ perl/imap/MANIFEST.in \ perl/imap/README \ perl/imap/t/01-imclient.t \ perl/imap/t/02-admin.t \ perl/imap/typemap \ perl/imap/xsutil.c \ perl/sieve/managesieve/Makefile.PL \ perl/sieve/managesieve/Makefile.PL.in \ perl/sieve/managesieve/managesieve.h \ perl/sieve/managesieve/managesieve.pm \ perl/sieve/managesieve/managesieve.xs \ perl/sieve/managesieve/MANIFEST \ perl/sieve/managesieve/MANIFEST.in \ perl/sieve/managesieve/typemap \ ptclient/README \ ptclient/test.c \ ptclient/test2.c \ sieve/addr.h \ sieve/sieve.h \ sieve/sieve_err.et \ sieve/tests/testExtension \ sieve/tests/testExtension/uberExtensionTestScript.key \ sieve/tests/testExtension/testm \ sieve/tests/testExtension/testm/uetest-envelope \ sieve/tests/testExtension/testm/uetest-asub \ sieve/tests/testExtension/testm/uetest-areg \ sieve/tests/testExtension/testm/uetest-count \ sieve/tests/testExtension/testm/uetest-value \ sieve/tests/testExtension/testm/uetest-hreg \ sieve/tests/testExtension/serverm \ sieve/tests/testExtension/serverm/uetmail-hreg \ sieve/tests/testExtension/serverm/uetmail-value \ sieve/tests/testExtension/serverm/uetmail-count2 \ sieve/tests/testExtension/serverm/uetmail-envelope \ sieve/tests/testExtension/serverm/uetmail-asub \ sieve/tests/testExtension/serverm/uetmail-value2 \ sieve/tests/testExtension/serverm/uetmail-areg \ sieve/tests/testExtension/serverm/uetmail-count \ sieve/tests/testExtension/uberExtensionTestScript.s \ sieve/tests/README \ sieve/tests/action \ sieve/tests/action/testm \ sieve/tests/action/testm/uatest-keep \ sieve/tests/action/testm/uatest-redirect \ sieve/tests/action/testm/uatest-discard \ sieve/tests/action/testm/uatest-stop2 \ sieve/tests/action/testm/uatest-stop \ sieve/tests/action/serverm \ sieve/tests/action/serverm/uamail-stop2 \ sieve/tests/action/serverm/uamail-redirect \ sieve/tests/action/serverm/uamail-stop \ sieve/tests/action/serverm/uamail-keep \ sieve/tests/action/serverm/uamail-discard \ sieve/tests/action/uberActionScript.key \ sieve/tests/action/uberActionScript.s \ sieve/tests/test \ sieve/tests/test/uberTestScript.key \ sieve/tests/test/testm \ sieve/tests/test/testm/utest-header \ sieve/tests/test/testm/utest-address \ sieve/tests/test/serverm \ sieve/tests/test/serverm/utmail-address \ sieve/tests/test/serverm/utmail-header \ sieve/tests/test/uberTestScript.s \ sieve/tests/actionExtensions \ sieve/tests/actionExtensions/uberExtensionActionScript.s \ sieve/tests/actionExtensions/testm \ sieve/tests/actionExtensions/testm/ueatest-flag4 \ sieve/tests/actionExtensions/testm/ueatest-flag2 \ sieve/tests/actionExtensions/testm/ueatest-fileinto \ sieve/tests/actionExtensions/testm/ueatest-denotify \ sieve/tests/actionExtensions/testm/ueatest-vacation \ sieve/tests/actionExtensions/testm/ueatest-reject \ sieve/tests/actionExtensions/testm/ueatest-mark \ sieve/tests/actionExtensions/testm/ueatest-denotify2 \ sieve/tests/actionExtensions/testm/ueatest-flag5 \ sieve/tests/actionExtensions/testm/ueatest-notify2 \ sieve/tests/actionExtensions/testm/ueatest-notify \ sieve/tests/actionExtensions/testm/ueatest-flag1 \ sieve/tests/actionExtensions/testm/ueatest-flag3 \ sieve/tests/actionExtensions/testm/ueatest-unmark \ sieve/tests/actionExtensions/uberExtensionActionScript.key \ sieve/tests/actionExtensions/serverm \ sieve/tests/actionExtensions/serverm/ueamail-flag4 \ sieve/tests/actionExtensions/serverm/ueamail-denotify \ sieve/tests/actionExtensions/serverm/ueamail-mark \ sieve/tests/actionExtensions/serverm/ueamail-denotify2 \ sieve/tests/actionExtensions/serverm/ueamail-flag2 \ sieve/tests/actionExtensions/serverm/ueamail-unmark \ sieve/tests/actionExtensions/serverm/ueamail-reject \ sieve/tests/actionExtensions/serverm/ueamail-flag3 \ sieve/tests/actionExtensions/serverm/ueamail-fileinto \ sieve/tests/actionExtensions/serverm/ueamail-flag1 \ sieve/tests/actionExtensions/serverm/ueamail-notify \ sieve/tests/actionExtensions/serverm/ueamail-flag5 \ sieve/tests/actionExtensions/serverm/ueamail-notify2 \ sieve/tests/actionExtensions/serverm/ueamail-vacation \ timsieved/TODO TEXINFO_TEX = com_err/et/texinfo.tex dist_noinst_SCRIPTS = \ com_err/et/compile_et.sh \ com_err/et/config_script \ imap/promdatagen \ lib/mkchartable.pl \ lib/test/run \ perl/sieve/scripts/installsieve.pl \ perl/sieve/scripts/sieveshell.pl \ tools/arbitronsort.pl \ tools/compile_st.pl \ tools/config2header \ tools/config2rst \ tools/config2sample \ tools/fixsearchpath.pl \ tools/git-version.sh \ tools/jenkins-build.sh \ tools/masssievec \ tools/mkimap \ tools/mknewsgroups \ tools/perl2rst \ tools/rehash \ tools/translatesieve noinst_MAN = \ com_err/et/com_err.3 \ com_err/et/compile_et.1 noinst_TEXINFOS = com_err/et/com_err.texinfo pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = libcyrus_min.pc libcyrus.pc libcyrus_sieve.pc libcyrus_imap.pc com_err_et_libcyrus_com_err_la_SOURCES = \ com_err/et/com_err.c \ com_err/et/com_err.h \ com_err/et/error_message.c \ com_err/et/error_table.h \ com_err/et/et_name.c \ com_err/et/init_et.c \ com_err/et/internal.h \ com_err/et/mit-sipb-copyright.h com_err_et_libcyrus_com_err_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) com_err/et/compile_et: com_err/et/compile_et.sh com_err/et/config_script \ config.h @${top_srcdir}/com_err/et/config_script ${top_srcdir}/com_err/et/compile_et.sh ${AWK} ${SED} > $@ @chmod 755 $@ # ---- Libraries ---- # SIEVE is the libraries that sieve-using components need to link with # # This is empty if sieve is not enabled, so it can be used unconditionally # elsewhere. if SIEVE LD_SIEVE_ADD = sieve/libcyrus_sieve.la $(LD_BASIC_ADD) else LD_SIEVE_ADD = endif # BASIC is the libraries that every Cyrus program (except master) will # need to link with. # # Note that several places in the code use -lcrypto, e.g. for SHA1 or # MD5 algorithms, without needing SSL. Currently we have no way of # minimally linking such code. LD_BASIC_ADD = lib/libcyrus.la lib/libcyrus_min.la ${LIBS} \ ${LIB_SASL} $(SSL_LIBS) $(GCOV_LIBS) # UTILITY is the libraries that utility programs which use Cyrus' # mailbox and message handling code need to link with. LD_UTILITY_ADD = imap/libcyrus_imap.la $(LD_BASIC_ADD) $(LD_SIEVE_ADD) $(COM_ERR_LIBS) # SERVER is the libraries that network-facing servers need to link with # # Note that the code is horribly intertwingled e.g. in imap/global.c # so that even utilities which never open a socket need to link against # the SASL library. LD_SERVER_ADD = $(LD_UTILITY_ADD) $(LIB_WRAP) # ---- if USE_LIBCHARDET AM_LDFLAGS += $(LIBCHARDET_LIBS) AM_CPPFLAGS += $(LIBCHARDET_CFLAGS) LD_SERVER_ADD += $(LIBCHARDET_LIBS) endif if USE_JANSSON AM_LDFLAGS += $(JANSSON_LIBS) AM_CPPFLAGS += $(JANSSON_CFLAGS) LD_SERVER_ADD += $(JANSSON_LIBS) endif if CUNIT CUNIT_PROJECT = cunit/default.cunit BUILT_SOURCES += cunit/registers.h $(CUNIT_PROJECT) CLEANFILES += cunit/registers.h $(CUNIT_PROJECT) check_PROGRAMS += cunit/unit cunit_FRAMEWORK = \ cunit/unit.c \ cunit/cyrunit.h \ cunit/syslog.c \ cunit/cunit-syslog.h \ cunit/timeout.c \ cunit/timeout.h \ cunit/timezones.c \ cunit/timezones.h \ cunit/timeofday.c \ cunit/timeofday.h cunit_TESTS = \ cunit/aaa-db.testc \ cunit/annotate.testc \ cunit/backend.testc \ cunit/binhex.testc \ cunit/bitvector.testc \ cunit/buf.testc \ cunit/byteorder.testc \ cunit/charset.testc \ cunit/command.testc \ cunit/conversations.testc \ cunit/cyr_qsort_r.testc \ cunit/crc32.testc \ cunit/dlist.testc \ cunit/duplicate.testc \ cunit/getxstring.testc \ cunit/glob.testc \ cunit/guid.testc \ cunit/hash.testc \ cunit/hashset.testc \ cunit/imapurl.testc \ cunit/imparse.testc \ cunit/libconfig.testc \ cunit/mboxname.testc \ cunit/md5.testc \ cunit/message.testc \ cunit/msgid.testc \ cunit/parseaddr.testc \ cunit/parse.testc \ cunit/procinfo.testc \ cunit/prot.testc \ cunit/ptrarray.testc \ cunit/quota.testc \ cunit/rfc822tok.testc \ cunit/search_expr.testc \ cunit/seqset.testc if SIEVE cunit_TESTS += cunit/sieve.testc endif cunit_TESTS += \ cunit/spool.testc \ cunit/squat.testc \ cunit/strarray.testc \ cunit/strconcat.testc \ cunit/times.testc \ cunit/tok.testc \ cunit/vparse.testc \ cunit/dynarray.testc cunit_unit_SOURCES = $(cunit_FRAMEWORK) $(cunit_TESTS) \ imap/mutex_fake.c imap/spool.c cunit_unit_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) -lcunit CUNIT_PL = $(top_srcdir)/cunit/cunit.pl --project $(CUNIT_PROJECT) # don't discard the temporary source file if building with coverage if HAVE_COVERAGE CUNIT_RM = true else CUNIT_RM = $(RM) endif .testc.o: $(AM_V_at)$(CUNIT_PL) --generate-wrapper $< $(AM_V_CC)$(COMPILE) -c -o $@ $<-cunit.c $(AM_V_at)$(CUNIT_RM) $<-cunit.c $(CUNIT_PROJECT): $(AM_V_at)$(RM) $@ $(AM_V_GEN)$(CUNIT_PL) --add-sources $(addprefix $(top_srcdir)/,$(cunit_TESTS)) cunit/registers.h: $(CUNIT_PROJECT) $(AM_V_GEN)$(CUNIT_PL) --generate-register-function $@ # To run under Valgrind, do: make VG=1 check VALGRIND = libtool --mode=execute valgrind --tool=memcheck --leak-check=full --suppressions=vg.supp check-local: @echo "Running unit tests" @vg= ; \ test -z "$$VG" || vg="$(VALGRIND)" ; \ f="-v" ; \ test "x$$CUFORMAT" = xjunit && f="-x" ; \ cd cunit ; \ $$vg ./unit $$f ; \ retval=$$? ; \ if [ "x$$CUFORMAT" = xjunit ] ; then \ $(RM) -rf reports ; mkdir reports ; ./cunit-to-junit.pl ; \ fi ; \ exit $$retval # Run every test individually, as if you had run them one at a time as # `make check-suitename:testname` manually. Mainly useful for detecting # abstraction failures within our testing infrastructure -- if tests # succeed with `make check` but fail with `make check-discrete`, or # vice-versa, some state is leaking between tests! # # This assumes your sh supports arithmetic and process substitutions. # Sorry <3 check-discrete: cunit/unit @echo "Running unit tests, one at a time" @vg= ; \ test -z "$$VG" || vg="$(VALGRIND)" ; \ cd cunit ; \ fails=0 ; \ tests=0 ; \ while read t ; do \ tests=$$(($$tests + 1)) ; \ if $$vg ./unit -v $$t >/dev/null 2>/dev/null ; then \ echo "[ OK ] $$t" ; \ else \ fails=$$(($$fails + 1)) ; \ echo "[FAILED] $$t" ; \ fi ; \ done < <( ./unit -l ) ; \ echo "$$fails/$$tests tests failed" ; \ exit $$fails check-%: cunit/unit @echo "Running unit tests for $*" @vg= ; \ test -z "$$VG" || vg="$(VALGRIND)" ; \ cd cunit ; \ $$vg ./unit -v $* ; \ retval=$$? ; \ exit $$retval endif includedir=@includedir@/cyrus include_HEADERS = \ lib/acl.h \ lib/arrayu64.h \ lib/assert.h \ lib/auth.h \ lib/auth_pts.h \ lib/bitvector.h \ lib/bloom.h \ lib/bsearch.h \ lib/bufarray.h \ lib/charset.h \ lib/chartable.h \ lib/command.h \ lib/crc32.h \ lib/cyr_lock.h \ lib/cyr_qsort_r.h \ lib/cyrusdb.h \ lib/dynarray.h \ lib/glob.h \ lib/gmtoff.h \ lib/hash.h \ lib/hashset.h \ lib/hashu64.h \ lib/imapurl.h \ lib/imclient.h \ lib/imparse.h \ lib/iostat.h \ lib/iptostring.h \ lib/libcyr_cfg.h \ lib/lsort.h \ lib/map.h \ lib/mappedfile.h \ lib/mkgmtime.h \ lib/mpool.h \ lib/murmurhash2.h \ lib/nonblock.h \ lib/parseaddr.h \ lib/procinfo.h \ lib/retry.h \ lib/rfc822tok.h \ lib/signals.h \ lib/sqldb.h \ lib/strarray.h \ lib/strhash.h \ lib/stristr.h \ lib/times.h \ lib/tok.h \ lib/vparse.h \ lib/wildmat.h \ lib/xmalloc.h nodist_include_HEADERS = \ lib/imapopts.h nobase_include_HEADERS = sieve/sieve_interface.h nobase_nodist_include_HEADERS = sieve/sieve_err.h noinst_HEADERS += \ lib/byteorder.h \ lib/gai.h \ lib/libconfig.h \ lib/md5.h \ lib/prot.h \ lib/ptrarray.h \ lib/util.h \ lib/xsha1.h \ lib/xstrlcat.h \ lib/xstrlcpy.h \ lib/xstrnchr.h backup_libcyrus_backup_la_SOURCES = \ lib/gzuncat.c \ lib/gzuncat.h \ backup/lcb.c \ backup/lcb_append.c \ backup/lcb_backupdb.c \ backup/lcb_compact.c \ backup/lcb_indexr.c \ backup/lcb_indexw.c \ backup/lcb_internal.c \ backup/lcb_internal.h \ backup/lcb_partlist.c \ backup/lcb_read.c \ backup/lcb_sqlconsts.c \ backup/lcb_sqlconsts.h \ backup/lcb_verify.c backup_libcyrus_backup_la_LIBADD = $(LD_BASIC_ADD) backup_backupd_SOURCES = \ imap/mutex_fake.c \ imap/sync_support.c \ imap/sync_support.h \ master/service.c \ backup/backupd.c backup_backupd_LDADD = backup/libcyrus_backup.la $(LD_SIEVE_ADD) $(LD_SERVER_ADD) backup_ctl_backups_SOURCES = \ imap/mutex_fake.c \ imap/sync_support.c \ imap/sync_support.h \ backup/ctl_backups.c backup_ctl_backups_LDADD = backup/libcyrus_backup.la $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) backup_cyr_backup_SOURCES = \ imap/json_support.h \ imap/mutex_fake.c \ backup/cyr_backup.c backup_cyr_backup_LDADD = backup/libcyrus_backup.la $(LD_UTILITY_ADD) backup_restore_SOURCES = \ imap/mutex_fake.c \ imap/sync_support.c \ imap/sync_support.h \ backup/restore.c backup_restore_LDADD = backup/libcyrus_backup.la $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) imap_arbitron_SOURCES = imap/arbitron.c imap/cli_fatal.c imap/mutex_fake.c imap_arbitron_LDADD = $(LD_UTILITY_ADD) imap_chk_cyrus_SOURCES = imap/chk_cyrus.c imap/cli_fatal.c imap/mutex_fake.c imap_chk_cyrus_LDADD = $(LD_UTILITY_ADD) imap_ctl_conversationsdb_SOURCES = imap/ctl_conversationsdb.c imap/mutex_fake.c imap_ctl_conversationsdb_LDADD = $(LD_UTILITY_ADD) imap_ctl_cyrusdb_SOURCES = imap/cli_fatal.c imap/ctl_cyrusdb.c imap/mutex_fake.c imap_ctl_cyrusdb_LDADD = $(LD_UTILITY_ADD) imap_ctl_deliver_SOURCES = imap/cli_fatal.c imap/ctl_deliver.c imap/mutex_fake.c imap_ctl_deliver_LDADD = $(LD_UTILITY_ADD) imap_ctl_mboxlist_SOURCES = imap/cli_fatal.c imap/ctl_mboxlist.c imap/mutex_fake.c imap_ctl_mboxlist_LDADD = $(LD_UTILITY_ADD) imap_ctl_zoneinfo_SOURCES = imap/cli_fatal.c imap/ctl_zoneinfo.c imap/mutex_fake.c imap/zoneinfo_db.c imap/xml_support.c imap_ctl_zoneinfo_LDADD = $(LD_UTILITY_ADD) imap_cvt_cyrusdb_SOURCES = imap/cli_fatal.c imap/cvt_cyrusdb.c imap/mutex_fake.c imap_cvt_cyrusdb_LDADD = $(LD_UTILITY_ADD) imap_cyrdump_SOURCES = imap/cli_fatal.c imap/cyrdump.c imap/mutex_fake.c imap_cyrdump_LDADD = $(LD_UTILITY_ADD) imap_cyr_dbtool_SOURCES = imap/cli_fatal.c imap/cyr_dbtool.c imap/mutex_fake.c imap_cyr_dbtool_LDADD = $(LD_UTILITY_ADD) imap_cyr_deny_SOURCES = imap/cli_fatal.c imap/cyr_deny.c imap/mutex_fake.c imap_cyr_deny_LDADD = $(LD_UTILITY_ADD) imap_cyr_df_SOURCES = imap/cli_fatal.c imap/cyr_df.c imap/mutex_fake.c imap_cyr_df_LDADD = $(LD_UTILITY_ADD) imap_cyr_expire_SOURCES = imap/cli_fatal.c imap/cyr_expire.c imap/mutex_fake.c imap_cyr_expire_LDADD = $(LD_UTILITY_ADD) imap_cyr_info_SOURCES = imap/cli_fatal.c imap/cyr_info.c imap/mutex_fake.c master/masterconf.c imap_cyr_info_LDADD = $(LD_UTILITY_ADD) imap_cyr_buildinfo_SOURCES = imap/cli_fatal.c imap/cyr_buildinfo.c imap/mutex_fake.c master/masterconf.c imap_cyr_buildinfo_LDADD = $(LD_UTILITY_ADD) imap_cyr_sequence_SOURCES = imap/cli_fatal.c imap/cyr_sequence.c imap/mutex_fake.c imap_cyr_sequence_LDADD = $(LD_UTILITY_ADD) imap_cyr_synclog_SOURCES = imap/cli_fatal.c imap/cyr_synclog.c imap/mutex_fake.c imap_cyr_synclog_LDADD = $(LD_UTILITY_ADD) imap_cyr_userseen_SOURCES = imap/cli_fatal.c imap/cyr_userseen.c imap/mutex_fake.c imap_cyr_userseen_LDADD = $(LD_UTILITY_ADD) imap_cyr_virusscan_SOURCES = imap/cli_fatal.c imap/cyr_virusscan.c imap/mutex_fake.c imap_cyr_virusscan_CFLAGS = $(AM_CFLAGS) $(CLAMAV_CFLAGS) $(CFLAG_VISIBILITY) imap_cyr_virusscan_LDADD = $(LD_UTILITY_ADD) $(CLAMAV_LIBS) imap_deliver_SOURCES = \ imap/deliver.c \ imap/lmtpengine.c \ imap/mutex_fake.c \ imap/proxy.c \ imap/spool.c nodist_imap_deliver_SOURCES = imap/lmtp_err.c imap_deliver_LDADD = $(LD_UTILITY_ADD) imap_fetchnews_SOURCES = imap/cli_fatal.c imap/fetchnews.c imap/mutex_fake.c imap_fetchnews_LDADD = $(LD_UTILITY_ADD) imap_fud_SOURCES = imap/fud.c imap/mutex_fake.c master/service.c imap_fud_LDADD = $(LD_SERVER_ADD) imap_idled_SOURCES = imap/idled.c imap/mutex_fake.c imap_idled_LDADD = $(LD_UTILITY_ADD) imap_calalarmd_SOURCES = imap/calalarmd.c imap/mutex_fake.c imap_calalarmd_LDADD = $(LD_SERVER_ADD) imap_imapd_SOURCES = \ imap/imap_proxy.c \ imap/imap_proxy.h \ imap/imapd.c \ imap/imapd.h \ imap/mutex_fake.c \ imap/proxy.c \ imap/proxy.h \ imap/sync_support.c \ imap/sync_support.h \ master/service.c if AUTOCREATE imap_imapd_SOURCES += \ imap/autocreate.h \ imap/autocreate.c endif imap_imapd_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) imap_ipurge_SOURCES = imap/cli_fatal.c imap/ipurge.c imap/mutex_fake.c imap_ipurge_LDADD = $(LD_UTILITY_ADD) nodist_imap_libcyrus_imap_la_SOURCES = \ imap/http_err.c \ imap/http_err.h \ imap/imap_err.c \ imap/imap_err.h \ imap/mupdate_err.c \ imap/mupdate_err.h \ imap/promdata.c \ imap/promdata.h imap_libcyrus_imap_la_LIBADD = \ $(COM_ERR_LIBS) \ $(LIB_UUID) \ $(GCOV_LIBS) \ lib/libcyrus_min.la \ lib/libcyrus.la imap_libcyrus_imap_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) -imap_libcyrus_imap_la_CXXFLAGS = $(AM_CXXFLAGS) +imap_libcyrus_imap_la_CXXFLAGS = $(AM_CXXFLAGS) $(CFLAG_VISIBILITY) imap_libcyrus_imap_la_SOURCES = \ imap/annotate.c \ imap/annotate.h \ imap/append.c \ imap/append.h \ imap/backend.c \ imap/backend.h \ imap/conversations.c \ imap/conversations.h \ imap/convert_code.c \ imap/convert_code.h \ imap/dlist.c \ imap/dlist.h \ imap/duplicate.c \ imap/duplicate.h \ imap/global.c \ imap/global.h \ imap/http_client.c \ imap/http_client.h \ imap/idle.c \ imap/idle.h \ imap/idlemsg.c \ imap/idlemsg.h \ imap/imapparse.c \ imap/imapparse.h \ imap/index.c \ imap/index.h \ imap/json_support.h \ imap/mailbox.c \ imap/mailbox.h \ imap/mbdump.c \ imap/mbdump.h \ imap/mboxkey.c \ imap/mboxkey.h \ imap/mboxlist.c \ imap/mboxlist.h \ imap/mboxevent.c \ imap/mboxevent.h \ imap/mboxname.c \ imap/mboxname.h \ imap/message_guid.c \ imap/message_guid.h \ imap/message.c \ imap/message.h \ imap/message_priv.h \ imap/msgrecord.c \ imap/msgrecord.h \ imap/mupdate-client.c \ imap/mupdate-client.h \ imap/mutex.h \ imap/notify.c \ imap/notify.h \ imap/partlist.c \ imap/partlist.h \ imap/proc.c \ imap/proc.h \ imap/prometheus.c \ imap/prometheus.h \ imap/protocol.h \ imap/quota_db.c \ imap/rfc822_header.c \ imap/rfc822_header.h \ imap/mailbox_header_cache.h \ imap/saslclient.c \ imap/saslclient.h \ imap/saslserver.c \ imap/search_engines.c \ imap/search_engines.h \ imap/search_expr.c \ imap/search_expr.h \ imap/search_query.c \ imap/search_query.h \ imap/search_part.h \ imap/search_sort.h \ imap/seen.h \ imap/seen_db.c \ imap/sequence.c \ imap/sequence.h \ imap/setproctitle.c \ imap/sievedir.c \ imap/sievedir.h \ imap/spool.c \ imap/spool.h \ imap/statuscache.h \ imap/statuscache_db.c \ imap/sync_log.c \ imap/sync_log.h \ imap/telemetry.c \ imap/telemetry.h \ imap/tls.c \ imap/tls.h \ imap/tls_th-lock.c \ imap/tls_th-lock.h \ imap/user.c \ imap/user.h \ imap/userdeny_db.c \ imap/userdeny.h \ imap/version.c \ imap/version.h \ imap/xstats.c \ imap/xstats.h \ imap/xstats_metrics.h if OBJECTSTORE imap_libcyrus_imap_la_SOURCES += \ imap/objectstore_db.c \ imap/objectstore_db.h \ imap/objectstore.h if WITH_OPENIO imap_libcyrus_imap_la_SOURCES += imap/objectstore_openio.c else if WITH_CARINGO imap_libcyrus_imap_la_SOURCES += imap/objectstore_caringo.c else if WITH_OBJSTR_DUMMY imap_libcyrus_imap_la_SOURCES += imap/objectstore_dummy.c endif endif endif endif if USE_SQUAT imap_libcyrus_imap_la_SOURCES += \ imap/search_squat.c \ imap/squat.c \ imap/squat.h \ imap/squat_build.c \ imap/squat_internal.c \ imap/squat_internal.h endif if HTTPD imap_libcyrus_imap_la_SOURCES += \ imap/caldav_alarm.c \ imap/caldav_alarm.h \ imap/caldav_db.c \ imap/caldav_db.h \ imap/carddav_db.c \ imap/carddav_db.h \ imap/dav_db.c \ imap/dav_db.h \ imap/dav_util.c \ imap/dav_util.h \ imap/ical_support.c \ imap/ical_support.h \ imap/icu_wrap.h \ imap/icu_wrap.cpp \ imap/vcard_support.c \ imap/vcard_support.h \ imap/webdav_db.c \ imap/webdav_db.h endif # HTTPD if JMAP imap_libcyrus_imap_la_SOURCES += \ imap/smtpclient.c \ imap/smtpclient.h endif if USE_XAPIAN imap_libcyrus_imap_la_SOURCES += \ imap/search_xapian.c \ imap/xapian_wrap.h \ imap/xapian_wrap.cpp imap_libcyrus_imap_la_LIBADD += $(XAPIAN_LIBS) imap_libcyrus_imap_la_CXXFLAGS += $(XAPIAN_CXXFLAGS) if HAVE_CLD2 imap_libcyrus_imap_la_LIBADD += $(CLD2_LIBS) imap_libcyrus_imap_la_CXXFLAGS += $(CLD2_CFLAGS) endif endif imap_lmtpd_SOURCES = \ imap/lmtpd.c \ imap/lmtpd.h \ imap/lmtpengine.c \ imap/lmtpengine.h \ imap/mutex_fake.c \ imap/proxy.c \ imap/spool.c \ imap/sync_support.c \ imap/sync_support.h \ master/service.c nodist_imap_lmtpd_SOURCES = imap/lmtp_err.c if AUTOCREATE imap_lmtpd_SOURCES += \ imap/autocreate.c \ imap/autocreate.h endif # AUTOCREATE if SIEVE imap_lmtpd_SOURCES += imap/lmtp_sieve.c imap/lmtp_sieve.h imap/smtpclient.c if JMAP imap_lmtpd_SOURCES += \ imap/jmap_util.c imap/jmap_mail_query.c imap/jmap_mail_query_parse.c endif # JMAP endif # SIEVE imap_lmtpd_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) imap_mbexamine_SOURCES = imap/cli_fatal.c imap/mbexamine.c imap/mutex_fake.c imap_mbexamine_LDADD = $(LD_UTILITY_ADD) imap_mbpath_SOURCES = imap/cli_fatal.c imap/mbpath.c imap/mutex_fake.c imap_mbpath_LDADD = $(LD_UTILITY_ADD) imap_mbtool_SOURCES = imap/cli_fatal.c imap/mbtool.c imap/mutex_fake.c imap_mbtool_LDADD = $(LD_UTILITY_ADD) imap_message_test_SOURCES = imap/message_test.c imap/mutex_fake.c imap_message_test_LDADD = $(LD_UTILITY_ADD) imap_mupdate_SOURCES = \ imap/mupdate.c \ imap/mupdate.h \ imap/mupdate-slave.c \ imap/mutex_pthread.c \ imap/tls_th-lock.c \ master/service-thread.c imap_mupdate_LDADD = $(LD_SERVER_ADD) -lpthread imap_mupdate_CFLAGS = $(AM_CFLAGS) -pthread nodist_imap_nntpd_SOURCES = imap/nntp_err.c imap_nntpd_SOURCES = \ imap/mutex_fake.c \ imap/nntpd.c \ imap/proxy.c \ imap/smtpclient.c \ imap/smtpclient.h \ imap/spool.c \ imap/spool.h \ imap/sync_support.c \ imap/sync_support.h \ master/service.c imap_nntpd_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) nodist_imap_httpd_SOURCES = \ imap/http_caldav_js.h \ imap/http_carddav_js.h \ imap/tz_err.c \ imap/tz_err.h imap_httpd_SOURCES = \ imap/css3_color.c \ imap/css3_color.h \ imap/http_admin.c \ imap/http_applepush.c \ imap/http_caldav.c \ imap/http_caldav.h \ imap/http_caldav_js.h \ imap/http_caldav_sched.c \ imap/http_caldav_sched.h \ imap/http_carddav.c \ imap/http_carddav.h \ imap/http_carddav_js.h \ imap/http_cgi.c \ imap/http_dav.c \ imap/http_dav.h \ imap/http_dav_sharing.c \ imap/http_dav_sharing.h \ imap/http_dblookup.c \ imap/http_h2.c \ imap/http_h2.h \ imap/http_ischedule.c \ imap/http_prometheus.c \ imap/http_proxy.c \ imap/http_proxy.h \ imap/http_rss.c \ imap/http_tzdist.c \ imap/http_webdav.c \ imap/http_ws.c \ imap/http_ws.h \ imap/httpd.c \ imap/httpd.h \ imap/jcal.c \ imap/jcal.h \ imap/json_support.h \ imap/mutex_fake.c \ imap/proxy.c \ imap/smtpclient.c \ imap/spool.c \ imap/sync_support.c \ imap/sync_support.h \ imap/xcal.c \ imap/xcal.h \ imap/xml_support.c \ imap/xml_support.h \ imap/zoneinfo_db.c \ imap/zoneinfo_db.h \ master/masterconf.c \ master/service.c imap_httpd_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) if JMAP BUILT_SOURCES += \ imap/jmap_err.c \ imap/jmap_err.h imap_httpd_SOURCES += \ imap/http_jmap.c \ imap/http_jmap.h \ imap/jmap_api.c \ imap/jmap_api.h \ imap/jmap_backup.c \ imap/jmap_calendar.c \ imap/jmap_contact.c \ imap/jmap_core.c \ imap/jmap_ical.c \ imap/jmap_ical.h \ imap/jmap_mail.c \ imap/jmap_mail.h \ imap/jmap_mail_query.c \ imap/jmap_mail_query.h \ imap/jmap_mail_query_parse.c \ imap/jmap_mail_query_parse.h \ imap/jmap_mail_submission.c \ imap/jmap_mailbox.c \ imap/jmap_mdn.c \ imap/jmap_notes.c \ imap/jmap_util.c \ imap/jmap_util.h \ imap/jmap_vacation.c if SIEVE imap_httpd_SOURCES += \ imap/jmap_sieve.c endif nodist_imap_httpd_SOURCES += \ imap/jmap_err.c \ imap/jmap_err.h imap_libcyrus_imap_la_SOURCES += \ imap/jmap_util.c \ imap/jmap_util.h \ imap/json_support.c \ imap/json_support.h imap_httpd_LDADD += $(LD_SIEVE_ADD) if CUNIT cunit_TESTS += cunit/jmap_util.testc endif # CUNIT endif # JMAP imap_pop3d_SOURCES = \ imap/mutex_fake.c \ imap/pop3d.c \ imap/proxy.c \ imap/sync_support.c \ imap/sync_support.h \ master/service.c if AUTOCREATE imap_pop3d_SOURCES += \ imap/autocreate.c \ imap/autocreate.h endif # AUTOCREATE imap_pop3d_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) PROMDATAGEN = $(abs_top_srcdir)/imap/promdatagen imap/promdata.c: imap/promdata.p $(PROMDATAGEN) $(AM_V_GEN)$(PROMDATAGEN) -c $@.NEW $< && mv $@.NEW $@ imap/promdata.h: imap/promdata.p $(PROMDATAGEN) $(AM_V_GEN)$(PROMDATAGEN) -h $@.NEW $< && mv $@.NEW $@ imap_promstatsd_SOURCES = imap/promstatsd.c imap/mutex_fake.c imap_promstatsd_LDADD = $(LD_UTILITY_ADD) imap_quota_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/quota.c imap/quota.h imap_quota_LDADD = $(LD_UTILITY_ADD) imap_reconstruct_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/reconstruct.c imap_reconstruct_LDADD = $(LD_UTILITY_ADD) imap_dav_reconstruct_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/dav_reconstruct.c imap_dav_reconstruct_LDADD = $(LD_UTILITY_ADD) imap_ical_apply_patch_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/ical_apply_patch.c imap_ical_apply_patch_LDADD = $(LD_UTILITY_ADD) imap_search_test_SOURCES = imap/search_test.c imap/mutex_fake.c imap_search_test_LDADD = $(LD_UTILITY_ADD) imap_smmapd_SOURCES = imap/mutex_fake.c imap/proxy.c imap/smmapd.c master/service.c imap_smmapd_LDADD = $(LD_SERVER_ADD) imap_squatter_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/squatter.c imap_squatter_LDADD = $(LD_UTILITY_ADD) imap_squat_dump_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/squat_dump.c imap_squat_dump_LDADD = $(LD_UTILITY_ADD) imap_sync_client_SOURCES = imap/mutex_fake.c imap/sync_client.c imap/sync_support.c imap/sync_support.h imap_sync_client_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) imap_sync_reset_SOURCES = imap/mutex_fake.c imap/sync_reset.c imap/sync_support.c imap/sync_support.h imap_sync_reset_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) imap_sync_server_SOURCES = imap/mutex_fake.c imap/sync_server.c imap/sync_support.c imap/sync_support.h master/service.c imap_sync_server_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) imap_tls_prune_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/tls_prune.c imap_tls_prune_LDADD = $(LD_UTILITY_ADD) imap_unexpunge_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/unexpunge.c imap_unexpunge_LDADD = $(LD_UTILITY_ADD) # $(COMPILE_ET) creates its output files in the directory it's run from, # so do a tricky directory change %_err.h %_err.c: %_err.et $(COMPILE_ET_DEP) $(AM_V_GEN)(cd $(@D) && $(COMPILE_ET) $(realpath $<)) # xxd cannot have path details in its input filename, otherwise it junks up # the variable names in the output file. so do a tricky directory change. # the /dev/null redirection on cd is to prevent shell environments with # CDPATH echoing the path change on stdout and consequently into the .h file %_js.h: %.js $(AM_V_GEN)(cd $( /dev/null && xxd -i $( $@.NEW && mv $@.NEW $@ if MAINTAINER_MODE imap/rfc822_header.c: imap/rfc822_header.st $(AM_V_GEN)${top_srcdir}/tools/compile_st.pl -c $< > $@.NEW && mv $@.NEW $@ imap/rfc822_header.h: imap/rfc822_header.st $(AM_V_GEN)${top_srcdir}/tools/compile_st.pl -h $< > $@.NEW && mv $@.NEW $@ lib/htmlchar.c: lib/htmlchar.st $(AM_V_GEN)${top_srcdir}/tools/compile_st.pl -c $< > $@.NEW && mv $@.NEW $@ lib/htmlchar.h: lib/htmlchar.st $(AM_V_GEN)${top_srcdir}/tools/compile_st.pl -h $< > $@.NEW && mv $@.NEW $@ imap/mailbox_header_cache.h: imap/mailbox_header_cache.gperf $(AM_V_GEN)gperf --ignore-case --initializer-suffix=,0 -p -j1 -i 1 -g -o -t -H mailbox_header_cache_hash -N mailbox_header_cache_lookup -k1,3,$$ $< > $@.NEW && mv $@.NEW $@ endif imtest_imtest_SOURCES = imtest/imtest.c imtest_imtest_LDADD = $(LD_BASIC_ADD) imtest_imtest_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) nodist_lib_libcyrus_la_SOURCES = lib/chartable.c lib_libcyrus_la_SOURCES = \ lib/acl.c \ lib/acl_afs.c \ lib/auth.c \ lib/auth_krb.c \ lib/auth_krb5.c \ lib/auth_pts.c \ lib/auth_unix.c \ lib/bitvector.c \ lib/bloom.c \ lib/bsearch.c \ lib/charset.c \ lib/command.c \ lib/cyr_qsort_r.c \ lib/cyrusdb.c \ lib/cyrusdb_flat.c \ lib/cyrusdb_quotalegacy.c \ lib/cyrusdb_skiplist.c \ lib/cyrusdb_twoskip.c \ lib/glob.c \ lib/htmlchar.c \ lib/htmlchar.h \ lib/imapurl.c \ lib/imclient.c \ lib/imparse.c \ lib/iostat.c \ lib/iptostring.c \ lib/libcyr_cfg.c \ lib/lsort.c \ lib/mappedfile.c \ lib/murmurhash.c \ lib/mkgmtime.c \ lib/parseaddr.c \ lib/procinfo.c \ lib/prot.c \ lib/ptrarray.c \ lib/rfc822tok.c \ lib/signals.c \ lib/stristr.c \ lib/times.c \ lib/tok.c \ lib/wildmat.c if USE_CYRUSDB_SQL lib_libcyrus_la_SOURCES += lib/cyrusdb_sql.c endif if USE_SQLITE lib_libcyrus_la_SOURCES += lib/sqldb.c endif if GMTOFF_TM lib_libcyrus_la_SOURCES += lib/gmtoff_tm.c else lib_libcyrus_la_SOURCES += lib/gmtoff_gmtime.c endif if NONBLOCK_FCNTL lib_libcyrus_la_SOURCES += lib/nonblock_fcntl.c else lib_libcyrus_la_SOURCES += lib/nonblock_ioctl.c endif lib_libcyrus_la_LIBADD = libcrc32.la ${LIB_SASL} $(SSL_LIBS) $(GCOV_LIBS) lib_libcyrus_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) if USE_ZEROSKIP lib_libcyrus_la_SOURCES += lib/cyrusdb_zeroskip.c lib_libcyrus_la_LIBADD += $(ZEROSKIP_LIBS) lib_libcyrus_la_CFLAGS += $(ZEROSKIP_CFLAGS) AM_CPPFLAGS += $(ZEROSKIP_CFLAGS) endif noinst_LTLIBRARIES += libcrc32.la libcrc32_la_SOURCES = lib/crc32.c libcrc32_la_CFLAGS = -O3 $(AM_CFLAGS) $(CFLAG_VISIBILITY) nodist_lib_libcyrus_min_la_SOURCES = \ lib/imapopts.c lib_libcyrus_min_la_SOURCES = \ lib/arrayu64.c \ lib/assert.c \ lib/bufarray.c \ lib/byteorder.c \ lib/dynarray.c \ lib/hash.c \ lib/hashset.c \ lib/hashu64.c \ lib/libconfig.c \ lib/mpool.c \ lib/retry.c \ lib/strarray.c \ lib/strhash.c \ lib/util.c \ lib/vparse.c \ lib/xmalloc.c \ lib/xstrlcat.c \ lib/xstrlcpy.c \ lib/xstrnchr.c if !HAVE_SSL lib_libcyrus_min_la_SOURCES += lib/xsha1.c endif if IPV6_noGETADDRINFO lib_libcyrus_min_la_SOURCES += lib/getaddrinfo.c endif if IPV6_noGETNAMEINFO lib_libcyrus_min_la_SOURCES += lib/getnameinfo.c endif if LOCK_FCNTL lib_libcyrus_min_la_SOURCES += lib/lock_fcntl.c else lib_libcyrus_min_la_SOURCES += lib/lock_flock.c endif if MAP_SHARED lib_libcyrus_min_la_SOURCES += lib/map_shared.c else if MAP_STUPIDSHARED lib_libcyrus_min_la_SOURCES += lib/map_stupidshared.c else lib_libcyrus_min_la_SOURCES += lib/map_nommap.c endif endif lib_libcyrus_min_la_LIBADD = $(LTLIBOBJS) $(LIB_UUID) $(GCOV_LIBS) $(LIBCAP_LIBS) lib_libcyrus_min_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) # n.b. the order of the -m arguments to mkchartable.pl is important, # in particular ellie believes unifix.txt must be before UnicodeData.txt lib/chartable.c: lib/mkchartable.pl lib/charset/unifix.txt \ $(top_srcdir)/lib/charset/*.t \ lib/charset/UnicodeData.txt lib/charset/aliases.txt @echo "### Building chartables..." $(AM_V_GEN)perl $(top_srcdir)/lib/mkchartable.pl -m $(top_srcdir)/lib/charset/unifix.txt -m $(top_srcdir)/lib/charset/UnicodeData.txt -a $(top_srcdir)/lib/charset/aliases.txt -o $@ $(top_srcdir)/lib/charset/*.t || (rm -f $@ && exit 1) @echo "### Done building chartables." lib/imapopts.h: lib/imapopts.c lib/imapopts.c: lib/imapoptions tools/config2header $(AM_V_GEN)$(top_srcdir)/tools/config2header CC="$(CC)" $(top_builddir)/lib/imapopts.c $(top_builddir)/lib/imapopts.h < $(top_srcdir)/lib/imapoptions imap_cvt_xlist_specialuse_SOURCES = imap/mutex_fake.c imap/cvt_xlist_specialuse.c imap_cvt_xlist_specialuse_LDADD = $(LD_UTILITY_ADD) @SET_MAKE@ dist_man1_MANS = \ man/imtest.1 \ man/installsieve.1 \ man/httptest.1 \ man/lmtptest.1 \ man/mupdatetest.1 \ man/nntptest.1 \ man/pop3test.1 \ man/sieveshell.1 \ man/sivtest.1 \ man/smtptest.1 dist_man3_MANS = \ man/imclient.3 dist_man5_MANS = \ man/cyrus.conf.5 \ man/imapd.conf.5 \ man/krb.equiv.5 dist_man8_MANS = \ man/arbitron.8 \ man/backupd.8 \ man/chk_cyrus.8 \ man/ctl_backups.8 \ man/ctl_conversationsdb.8 \ man/ctl_cyrusdb.8 \ man/ctl_deliver.8 \ man/ctl_mboxlist.8 \ man/cvt_cyrusdb.8 \ man/cyr_backup.8 \ man/cyr_buildinfo.8 \ man/cyr_dbtool.8 \ man/cyr_deny.8 \ man/cyr_df.8 \ man/cyr_expire.8 \ man/cyr_info.8 \ man/cyr_synclog.8 \ man/cyr_virusscan.8 \ man/deliver.8 \ man/fud.8 \ man/idled.8 \ man/imapd.8 \ man/ipurge.8 \ man/lmtpd.8 \ man/master.8 \ man/mbexamine.8 \ man/mbpath.8 \ man/mbtool.8 \ man/notifyd.8 \ man/pop3d.8 \ man/quota.8 \ man/reconstruct.8 \ man/restore.8 \ man/smmapd.8 \ man/timsieved.8 \ man/tls_prune.8 \ man/unexpunge.8 if SQUATTER dist_man8_MANS += \ man/squatter.8 endif # SQUATTER if NNTPD dist_man8_MANS += \ man/fetchnews.8 \ man/nntpd.8 endif # NNTPD if HTTPD dist_man8_MANS += \ man/ctl_zoneinfo.8 \ man/httpd.8 endif # HTTPD if BENCH check_PROGRAMS += bench/cyrdbbench bench_cyrdbbench_SOURCES = bench/cyrdbbench.c imap/mutex_fake.c bench_cyrdbbench_LDADD = $(LD_BASIC_ADD) endif # BENCH if REPLICATION dist_man8_MANS += \ man/sync_client.8 \ man/sync_reset.8 \ man/sync_server.8 endif master_master_SOURCES = \ master/master.c \ master/master.h \ master/masterconf.c \ master/masterconf.h \ master/service.h master_master_LDADD = lib/libcyrus_min.la $(LIBS) $(GCOV_LIBS) -lm netnews_remotepurge_SOURCES = \ netnews/macros.h \ netnews/readconfig.c \ netnews/readconfig.h \ netnews/remotepurge.c netnews_remotepurge_LDADD = $(LD_BASIC_ADD) notifyd_notifyd_SOURCES = \ imap/mutex_fake.c \ master/service.c \ notifyd/notify_external.c \ notifyd/notify_external.h \ notifyd/notify_log.c \ notifyd/notify_log.h \ notifyd/notify_mailto.c \ notifyd/notify_mailto.h \ notifyd/notify_null.c \ notifyd/notify_null.h \ notifyd/notifyd.c \ notifyd/notifyd.h if ZEPHYR notifyd_notifyd_SOURCES += notifyd/notify_zephyr.c notifyd/notify_zephyr.h endif notifyd_notifyd_LDADD = $(LD_SERVER_ADD) $(ZEPHYR_LIBS) notifyd_notifytest_SOURCES = notifyd/notifytest.c imap/mutex_fake.c notifyd_notifytest_LDADD = $(LD_BASIC_ADD) nodist_perl_libcyrus_la_SOURCES = $(nodist_lib_libcyrus_la_SOURCES) perl_libcyrus_la_SOURCES = $(lib_libcyrus_la_SOURCES) perl_libcyrus_la_LIBADD = $(lib_libcyrus_la_LIBADD) nodist_perl_libcyrus_min_la_SOURCES = $(nodist_lib_libcyrus_min_la_SOURCES) perl_libcyrus_min_la_SOURCES = $(lib_libcyrus_min_la_SOURCES) perl_libcyrus_min_la_LIBADD = $(lib_libcyrus_min_la_LIBADD) perl_sieve_lib_libisieve_la_SOURCES = \ perl/sieve/lib/codes.h \ perl/sieve/lib/isieve.c \ perl/sieve/lib/isieve.h \ perl/sieve/lib/lex.c \ perl/sieve/lib/lex.h \ perl/sieve/lib/request.c \ perl/sieve/lib/request.h ptclient_ptdump_SOURCES = imap/cli_fatal.c imap/mutex_fake.c ptclient/ptdump.c ptclient_ptdump_LDADD = $(LD_UTILITY_ADD) ptclient_ptexpire_SOURCES = imap/cli_fatal.c imap/mutex_fake.c ptclient/ptexpire.c ptclient_ptexpire_LDADD = $(LD_UTILITY_ADD) ptclient_ptloader_SOURCES = \ imap/mutex_fake.c \ ptclient/ptloader.c \ ptclient/ptloader.h \ master/service-thread.c ptclient_ptloader_LDFLAGS = ptclient_ptloader_LDADD = $(LD_SERVER_ADD) if HAVE_LDAP ptclient_ptloader_SOURCES += ptclient/ldap.c ptclient_ptloader_LDADD += $(LDAP_LIBS) ptclient_ptloader_LDFLAGS += $(LDAP_LDFLAGS) endif if USE_AFSKRB ptclient_ptloader_SOURCES += ptclient/afskrb.c ptclient_ptloader_LDADD += $(AFS_LIBS) ptclient_ptloader_LDFLAGS += $(AFS_LDFLAGS) endif # Each new version of flex seems to generate new sign-comparison # warnings, while they switch code around between int/yy_size_t. # # This is a pain because most of us Cyrus devs build with -Werror. # # Instead of continuing to maintain an ever-growing, increasingly # precarious set of post-hoc fixes (see: the former lex-fix rule), # let's just ignore sign-comparison warnings from flex-generated # sources only. sieve_libcyrus_sieve_lex_la_SOURCES = \ sieve/addr-lex.l \ sieve/sieve-lex.l sieve_libcyrus_sieve_lex_la_LIBADD = sieve_libcyrus_sieve_lex_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) $(NOWARN_SIGN_COMPARE) nodist_sieve_libcyrus_sieve_la_SOURCES = \ sieve/sieve_err.c \ sieve/sieve_err.h sieve_libcyrus_sieve_la_SOURCES = \ sieve/bytecode.h \ sieve/addr.y \ sieve/bc_emit.c \ sieve/bc_eval.c \ sieve/bc_generate.c \ sieve/bc_parse.c \ sieve/bc_parse.h \ sieve/comparator.c \ sieve/comparator.h \ sieve/flags.c \ sieve/flags.h \ sieve/grammar.c \ sieve/grammar.h \ sieve/interp.c \ sieve/interp.h \ sieve/message.c \ sieve/message.h \ sieve/rebuild.c \ sieve/script.c \ sieve/script.h \ sieve/sieve.y \ sieve/tree.c \ sieve/tree.h \ sieve/variables.c \ sieve/variables.h \ sieve/varlist.c \ sieve/varlist.h if JMAP sieve_libcyrus_sieve_la_SOURCES += \ imap/jmap_mail_query_parse.c \ imap/jmap_mail_query_parse.h \ imap/json_support.c \ imap/json_support.h endif sieve_libcyrus_sieve_la_LIBADD = \ sieve/libcyrus_sieve_lex.la \ $(COM_ERR_LIBS) \ lib/libcyrus_min.la \ lib/libcyrus.la sieve_libcyrus_sieve_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) sieve_sievec_LDADD = $(LD_SIEVE_ADD) sieve_sieved_LDADD = $(LD_SIEVE_ADD) sieve_test_SOURCES = \ sieve/test.c \ imap/mutex_fake.c \ sieve/sieve_interface.h if JMAP sieve_test_SOURCES += \ imap/jmap_util.c imap/jmap_mail_query.c imap/jmap_mail_query_parse.c endif sieve_test_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) sieve_test_mailbox_SOURCES = \ sieve/test_mailbox.c \ imap/mutex_fake.c \ sieve/sieve_interface.h sieve_test_mailbox_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) timsieved_timsieved_SOURCES = \ imap/mutex_fake.c \ imap/proxy.c \ master/service.c \ timsieved/actions.c \ timsieved/actions.h \ timsieved/codes.h \ timsieved/lex.c \ timsieved/lex.h \ timsieved/parser.c \ timsieved/parser.h \ timsieved/timsieved.c timsieved_timsieved_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) # The lex-fix rule is now obsolete, but let's leave a crumb here for a # while so that custom build scripts that expect it to exist don't choke. lex-fix: snapshot:: @echo "Creating snapshot $(PACKAGE_NAME)-$(PACKAGE_VERSION)" @$(MKDIR_P) snapshot @git archive --format=tar --prefix=$(PACKAGE_NAME)-$(PACKAGE_VERSION)/ HEAD | tar -C snapshot/ -x -f - @echo "creating tarfile" tar -C snapshot -c -f - $(PACKAGE_NAME)-$(PACKAGE_VERSION) | gzip -9 > $(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz @rm -rf snapshot docsrc/imap/reference/manpages/systemcommands/cyradm.rst: $(top_srcdir)/perl/imap/IMAP/Shell.pm @$(MKDIR_P) $(@D) $(AM_V_GEN)$(top_srcdir)/tools/perl2rst cyradm < $< > $@.NEW && mv $@.NEW $@ docsrc/imap/reference/manpages/configs/imapd.conf.rst: $(top_srcdir)/tools/config2rst $(top_srcdir)/lib/imapoptions @$(MKDIR_P) $(@D) $(AM_V_GEN)$(top_srcdir)/tools/config2rst $(top_srcdir)/lib/imapoptions > $@ doc/examples/imapd_conf/imapd.conf.sample: $(top_srcdir)/tools/config2sample $(top_srcdir)/lib/imapoptions @$(MKDIR_P) $(@D) $(AM_V_GEN)$(top_srcdir)/tools/config2sample $(top_srcdir)/lib/imapoptions > $@ docsrc/imap/reference/manpages/usercommands/sieveshell.rst: $(top_srcdir)/perl/sieve/scripts/sieveshell.pl @$(MKDIR_P) $(@D) $(AM_V_GEN)$(top_srcdir)/tools/perl2rst sieveshell < $< > $@.NEW && mv $@.NEW $@ %.1: man/.sphinx-build.stamp @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ %.3: man/.sphinx-build.stamp @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ %.5: man/.sphinx-build.stamp @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ %.8: man/.sphinx-build.stamp @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ ## define this unconditionally because dist-hook references it SPHINX_CACHE = docsrc/.doctrees .PHONY: clean-sphinx-cache clean-docsrc clean-sphinx-cache: @$(RM) -r $(SPHINX_CACHE) if HAVE_SPHINX_BUILD SPHINX_OPTS = -d $(SPHINX_CACHE) -n -q ## detect when source directory is not build directory (i.e. VPATH ## build), and clone the docsrc tree into the build directory, so ## that we have a single "source directory" for sphinx-build to use. docsrc/.sphinx-build.stamp: docsrc/imap/reference/manpages/configs/imapd.conf.rst docsrc/imap/reference/manpages/systemcommands/cyradm.rst docsrc/imap/reference/manpages/usercommands/sieveshell.rst $(AM_V_GEN)test x"$(top_srcdir)" = x"$(top_builddir)" || \ (cd $(top_srcdir) && tar cf - --mode=gu+w docsrc doc/examples) | tar xf - @touch $@ clean-docsrc: test x"$(top_srcdir)" = x"$(top_builddir)" || \ $(RM) -r $(top_builddir)/docsrc $(top_builddir)/doc/examples ## XXX doesn't detect if other rst sources are updated... man/.sphinx-build.stamp: docsrc/.sphinx-build.stamp $(AM_V_GEN)DOCSRC=$(top_builddir)/docsrc $(SPHINX_BUILD) $(SPHINX_OPTS) -b cyrman $(top_builddir)/docsrc $(top_builddir)/man @touch $@ clean-man: @$(RM) -r man ## XXX doesn't detect if other rst sources are updated... doc/html/.sphinx-build.stamp: docsrc/.sphinx-build.stamp $(AM_V_GEN)DOCSRC=$(top_builddir)/docsrc $(SPHINX_BUILD) $(SPHINX_OPTS) -b html $(top_builddir)/docsrc $(top_builddir)/doc/html @touch $@ ## XXX doesn't detect if other rst sources are updated... doc/text/.sphinx-build.stamp: docsrc/.sphinx-build.stamp $(AM_V_GEN)DOCSRC=$(top_builddir)/docsrc $(SPHINX_BUILD) $(SPHINX_OPTS) -b text $(top_builddir)/docsrc $(top_builddir)/doc/text @touch $@ else clean-docsrc: man/.sphinx-build.stamp: @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ @echo "warning: missing documentation dependencies. man pages will be empty" 1>&2 clean-man: doc/html/.sphinx-build.stamp: @echo "error: missing documentation dependencies. cannot build html documentation" 1>&2 @false doc/text/.sphinx-build.stamp: @echo "error: missing documentation dependencies. cannot build text documentation" 1>&2 @false endif .PHONY: man man-force clean-man man-force: ## trash the timestamps, so we always re-run sphinx-build. ## sphinx-build tracks source modifications itself, so forced ## rebuilds will still be a lot quicker once it has run once. @$(RM) docsrc/.sphinx-build.stamp man/.sphinx-build.stamp man: man-force $(dist_man1_MANS) $(dist_man3_MANS) $(dist_man5_MANS) $(dist_man8_MANS) # clean-man is conditionally defined above. We do not want to remove # man pages if the user is unable to rebuild them. .PHONY: doc-html doc-html-force clean-doc-html doc-html-force: @$(RM) docsrc/.sphinx-build.stamp doc/html/.sphinx-build.stamp doc-html: doc-html-force doc/html/.sphinx-build.stamp clean-doc-html: @$(RM) -r doc/html .PHONY: doc-text doc-text-force clean-doc-text doc-text-force: @$(RM) docsrc/.sphinx-build.stamp doc/text/.sphinx-build.stamp doc-text: doc-text-force doc/text/.sphinx-build.stamp clean-doc-text: @$(RM) -r doc/text GENERATED_EXAMPLES = doc/examples/imapd_conf/imapd.conf.sample .PHONY: doc-examples doc-examples-force clean-doc-examples doc-examples-force: doc-examples: doc-examples-force $(GENERATED_EXAMPLES) clean-doc-examples: @$(RM) $(GENERATED_EXAMPLES) .PHONY: doc clean-doc doc: man doc-html doc-text doc-examples clean-doc: clean-man clean-doc-html clean-doc-text clean-doc-examples .PHONY: clean-local clean-local: clean-doc clean-sphinx-cache clean-docsrc distgit: $(MKDIR_P) dist @echo "checking out the distribution from tag $(PACKAGE_NAME)-$(PACKAGE_VERSION)" git archive --format=tar --prefix=$(PACKAGE_NAME)-$(PACKAGE_VERSION)/ $(PACKAGE_NAME)-$(PACKAGE_VERSION) | tar -C dist -x -f - touch distgit dist-hook: find $(top_distdir) -type f -name .sphinx-build.stamp -delete find $(top_distdir) -type f -name .gitignore -delete rm -rf $(top_distdir)/$(SPHINX_CACHE) VERSION: tools/git-version.sh $(AM_V_GEN)$< > $@.NEW && mv $@.NEW $@ install-data-hook: if CMULOCAL $(INSTALL) -m 644 $(top_srcdir)/depot/depot.conf $(DESTDIR)/ endif install-exec-hook: if PERL for s in installsieve sieveshell; \ do \ $(MKDIR_P) $(top_builddir)/perl/sieve/scripts ; \ $(PERL_PREINSTALL) < $(top_srcdir)/perl/sieve/scripts/$$s.pl > $(top_builddir)/perl/sieve/scripts/$$s ;\ $(INSTALL) -m 755 $(top_builddir)/perl/sieve/scripts/$$s $(DESTDIR)$(bindir)/$$s ; \ done endif ## The @$(MKDIR_P) line is added due to a bug in Automake 1.10 and can be removed if using Automake 1.12. @$(MKDIR_P) $(DESTDIR)$(libexecdir) if SERVER cd $(DESTDIR)$(libexecdir) && \ $(LN_S) -f pop3d pop3proxyd && \ $(LN_S) -f imapd proxyd && \ $(LN_S) -f lmtpd lmtpproxyd endif ## The @$(MKDIR_P) line is added due to a bug in Automake 1.10 and can be removed if using Automake 1.12. @$(MKDIR_P) $(DESTDIR)$(bindir) cd $(DESTDIR)$(bindir) && \ $(LN_S) -f imtest httptest && \ $(LN_S) -f imtest lmtptest && \ $(LN_S) -f imtest mupdatetest && \ $(LN_S) -f imtest nntptest && \ $(LN_S) -f imtest pop3test && \ $(LN_S) -f imtest sivtest && \ $(LN_S) -f imtest smtptest && \ $(LN_S) -f imtest synctest uninstall-hook: cyrus-makemaker-uninstall-workaround if PERL for s in installsieve sieveshell; do \ rm -f $(DESTDIR)$(bindir)/$$s; \ done endif .PHONY: cyrus-makemaker-uninstall-workaround # Workaround for deprecated no-op MakeMaker uninstall. # Don't remove the packlists themselves, otherwise MakeMaker uninstall will # crash. Just remove the files MakeMaker uninstall won't remove itself. # # We have another layer of workaround in each module's Makefile that # removes the packlist after MakeMaker uninstall has finished ignoring it. cyrus-makemaker-uninstall-workaround: if PERL ( packlists=`find $(DESTDIR)$(libdir) -type f -name .packlist` && \ echo "uninstalling files from packlists: $$packlists" && \ rm -f `cat $$packlists` ) ( podfiles=`find $(DESTDIR)$(libdir) -type f -name perllocal.pod` && \ rm -f `echo $$podfiles` ) endif install-binsymlinks: ## Let's symlink everything else for prog in $(sbin_PROGRAMS); do $(LN_S) -f $(sbindir)/`basename $$prog` $(DESTDIR)$(bindir)/; done for prog in $(libexec_PROGRAMS); do $(LN_S) -f $(libexecdir)/`basename $$prog` $(DESTDIR)$(bindir)/; done SUFFIXES = .fig.png .fig.png: mkdir -p $(@D) fig2dev -L png $< $@ valgrind: $(MAKE) VG=yes check libtool: $(LIBTOOL_DEPS) $(SHELL) ./config.status libtool # Install for Cassandane cassandane: $(MAKE) DESTDIR=`cd ../inst ; /bin/pwd` install diff --git a/cunit/syslog.c b/cunit/syslog.c index abd467272..80322a603 100644 --- a/cunit/syslog.c +++ b/cunit/syslog.c @@ -1,284 +1,284 @@ /* * Copyright (c) 1994-2011 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include "cunit-syslog.h" extern int verbose; struct slmatch { const char *re; /* NULL => disabled */ unsigned int count; regex_t cre; /* compiled regex */ }; #define MAX_SLMATCH 32 static unsigned int nslmatches = 0; static struct slmatch slmatches[MAX_SLMATCH]; #if !defined(va_copy) # if defined(__va_copy) # define va_copy __va_copy # else # define va_copy(d,s) (d) = (s) # endif #endif static char *match_error(struct slmatch *sl, int r) { static char buf[2048]; int n; buf[0] = '\0'; if (sl->re) snprintf(buf, sizeof(buf)-100, "/%s/: ", sl->re); n = strlen(buf); regerror(r, &sl->cre, buf+n, sizeof(buf)-n-1); strcat(buf, "\n"); return buf; } static void __attribute__((format(printf, 2, 0))) vlog(int prio, const char *fmt, va_list args) { if (nslmatches) { int e = errno; /* save errno Just In Case */ va_list args2; unsigned int i; int r; char line[2048]; /* This only works for all cases because of the glibc * extension which supports %m in printf() */ va_copy(args2, args); vsnprintf(line, sizeof(line), fmt, args2); va_end(args2); for (i = 0 ; i < MAX_SLMATCH ; i++) { if (!slmatches[i].re) continue; /* empty slot */ r = regexec(&slmatches[i].cre, line, 0, NULL, 0); if (!r) { /* found */ if (verbose >= 2) fprintf(stderr, "\nSYSLOG matched /%s/\n", slmatches[i].re); slmatches[i].count++; break; } else { /* don't naively report mismatches when we're looking for multiple patterns */ if (nslmatches == 1 || verbose >= 2) fprintf(stderr, "\nSYSLOG didn't match '%s' against '%s'\n", line, slmatches[i].re); } if (r != REG_NOMATCH) { /* error */ const char *msg = match_error(&slmatches[i], r); CU_assertImplementation(0, __LINE__, msg, __FILE__, NULL, CU_TRUE); /* NOTREACHED */ break; } } errno = e; } /* glibc handles %m in vfprintf() so we don't need to do * anything special to simulate that feature of syslog() */ /* TODO: find and expand %m on non-glibc platforms */ if (verbose < 2) return; fprintf(stderr, "\nSYSLOG %d[", prio & LOG_PRIMASK); vfprintf(stderr, fmt, args); fprintf(stderr, "]\n"); fflush(stderr); } #if defined(__GLIBC__) /* Under some but not all combinations of options, glibc * defines syslog() as an inline that calls this function */ -void +EXPORTED void __attribute__((format(printf, 3, 4))) __syslog_chk(int prio, int whatever __attribute__((unused)), const char *fmt, ...) { va_list args; va_start(args, fmt); vlog(prio, fmt, args); va_end(args); } #endif -void syslog(int prio, const char *fmt, ...) +EXPORTED void syslog(int prio, const char *fmt, ...) { va_list args; va_start(args, fmt); vlog(prio, fmt, args); va_end(args); } unsigned int CU_syslogMatchBegin(const char *re, const char *filename, unsigned int lineno) { unsigned int i; int r; /* find an empty slot */ for (i = 0 ; i < MAX_SLMATCH ; i++) { if (!slmatches[i].re) { /* found */ slmatches[i].re = re; slmatches[i].count = 0; r = regcomp(&slmatches[i].cre, re, REG_EXTENDED|REG_ICASE|REG_NOSUB); if (r) { const char *msg = match_error(&slmatches[i], r); memset(&slmatches[i], 0, sizeof(slmatches[i])); CU_assertImplementation(0, lineno, msg, filename, NULL, CU_TRUE); /* NOTREACHED */ return 0; } nslmatches++; return i+1; } } CU_assertImplementation(0, lineno, "No free syslog match slots", filename, NULL, CU_TRUE); /* NOTREACHED */ return 0; } unsigned int CU_syslogMatchEnd(unsigned int match, const char **sp) { unsigned int i; unsigned int count = 0; const char *s = NULL; for (i = 0 ; i < MAX_SLMATCH ; i++) { if (!slmatches[i].re) continue; /* empty slot */ if (match && match != i+1) continue; /* not the slot for @match */ if (!s) s = slmatches[i].re; else s = "(multiple matches)"; count += slmatches[i].count; regfree(&slmatches[i].cre); memset(&slmatches[i], 0, sizeof(slmatches[i])); nslmatches--; if (match) break; /* only looking for a single slot */ } if (match && !s) { s = "invalid match number"; count = ~0U; } if (sp) *sp = s; return count; } /* Meta-test code for CU_*SYSLOG* macros. */ #if 0 static void test_syslog(void) { int m1, m2; // /* invalid regular expression, 1st macro fails */ // CU_SYSLOG_MATCH("[foo"); // syslog(LOG_ERR, "fnarp"); // CU_ASSERT_SYSLOG(/*all*/0, 0); // /* no syslog messages => count is 0 */ // CU_SYSLOG_MATCH("foo.*baz"); // CU_ASSERT_SYSLOG(/*all*/0, 0); // /* one syslog message which doesn't match => count is 0, // * both macros succeed */ // CU_SYSLOG_MATCH("foo.*baz"); // syslog(LOG_ERR, "fnarp"); // CU_ASSERT_SYSLOG(/*all*/0, 0); // /* one syslog message which does match => count is 1, // * both macros succeed */ // CU_SYSLOG_MATCH("foo.*baz"); // syslog(LOG_ERR, "foo bar baz"); // CU_ASSERT_SYSLOG(/*all*/0, 1); // /* one syslog message which does match => count is 1, // * we check for 5, 2nd macro fails */ // CU_SYSLOG_MATCH("foo.*baz"); // syslog(LOG_ERR, "foo bar baz"); // CU_ASSERT_SYSLOG(/*all*/0, 5); // /* one syslog message with multiple matches => count is 1, // * all 3 macros succeed */ // CU_SYSLOG_MATCH("fuu.*bas"); // CU_SYSLOG_MATCH("bleah"); // syslog(LOG_ERR, "fuu bleah bas"); // CU_ASSERT_SYSLOG(/*all*/0, 1); // /* one syslog message with multiple matches which are tracked // * separately => count is 1, all 4 macros succeed */ // m1 = CU_SYSLOG_MATCH("fuu.*bas"); // m2 = CU_SYSLOG_MATCH("bleah"); // syslog(LOG_ERR, "fuu bleah bas"); // CU_ASSERT_SYSLOG(m1, 1); // CU_ASSERT_SYSLOG(m2, 0); } #endif diff --git a/cunit/timeofday.c b/cunit/timeofday.c index d23772f48..50e121bf1 100644 --- a/cunit/timeofday.c +++ b/cunit/timeofday.c @@ -1,318 +1,325 @@ /* * Copyright (c) 1994-2012 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include "timeofday.h" extern int verbose; #define MICROSEC_PER_SEC (1000000) struct trans { /* * We transform time using the formula * * reported_time = (actual_time - start) * factor + epoch; * * where 'factor' is actually fractional. Yes, I realise that * 'start' is mathematically redundant but it makes the code * marginally easier to write and read. */ int64_t start; int64_t epoch; long factor_num; long factor_den; }; static int real_gettimeofday(struct timeval *, ...); #define MAX_TRANS_STACK 5 static int n_trans_stack = 0; static struct trans trans_stack[MAX_TRANS_STACK]; static const struct trans identity = { 0, 0, 1, 1 }; /* * Basic time manipulation. * * Internal time format is microseconds since the Unix epoch * in a signed 64b integer which is convenient to use and * allows some headroom for scaling. */ static int64_t from_timeval(const struct timeval *tv) { return (int64_t)tv->tv_usec + (int64_t)tv->tv_sec * MICROSEC_PER_SEC; } static void to_timeval(int64_t t, struct timeval *tv) { tv->tv_sec = t / MICROSEC_PER_SEC; tv->tv_usec = t % MICROSEC_PER_SEC; } static int64_t from_time_t(time_t tt) { return (int64_t)tt * MICROSEC_PER_SEC; } static time_t to_time_t(int64_t t) { return t / MICROSEC_PER_SEC; } static int64_t now(void) { struct timeval tv = { 0xffffffff, 0xffffffff }; int r = real_gettimeofday(&tv, NULL); assert(r == 0); assert(tv.tv_sec != 0xffffffff); assert(tv.tv_usec != 0xffffffff); return from_timeval(&tv); } /* * Time transform stack handling. */ static const struct trans *trans_top(void) { return (n_trans_stack ? &trans_stack[n_trans_stack-1] : &identity); } static int64_t transform(int64_t t) { const struct trans *tr = trans_top(); int64_t tt = ((t - tr->start) * tr->factor_num) / tr->factor_den + tr->epoch; return tt; } static void trans_push(const struct trans *tr) { assert(n_trans_stack < MAX_TRANS_STACK); trans_stack[n_trans_stack++] = *tr; } /* * Make the reported time go faster or slower from now on. * Reported times are continuous across this function. */ void time_push_rate(long n, long d) { struct trans tr = *trans_top(); tr.start = now(); tr.epoch = transform(tr.start); tr.factor_num *= n; tr.factor_den *= d; trans_push(&tr); } /* * Stop the flow of reported time */ void time_push_stop(void) { time_push_rate(0, 1); } /* * Report a given fixed time */ void time_push_fixed(time_t fixed) { struct trans tr = *trans_top(); tr.start = 0; tr.epoch = from_time_t(fixed); tr.factor_num = 0; tr.factor_den = 1; trans_push(&tr); } void time_pop(void) { assert(n_trans_stack > 0); --n_trans_stack; } void time_restore(void) { n_trans_stack = 0; } /* * Platform-specific libc interception code */ #if defined(__GLIBC__) +/* XXX Annoyingly, we can't just include in this file, + * XXX because for whatever reason it breaks the gettimeofday + * XXX replacement. Assume we just have this flag for now, and + * XXX define EXPORTED ourselves + */ +#define EXPORTED __attribute__((__visibility__("default"))) + /* call the real libc function */ static int real_gettimeofday(struct timeval *tv, ...) { extern int __gettimeofday(struct timeval *, ...); return __gettimeofday(tv, NULL); } /* provide a function to hide the libc weak alias */ -int gettimeofday(struct timeval *tv, ...) +EXPORTED int gettimeofday(struct timeval *tv, ...) { to_timeval(transform(now()), tv); return 0; } -time_t time(time_t *tp) +EXPORTED time_t time(time_t *tp) { time_t tt = to_time_t(transform(now())); if (tp) *tp = tt; return tt; } #else #error "Don't know how to intercept gettimeofday for this libc" #endif /* * Tests - not usefully runnable, sorry. */ #if 0 static void test_time_speedup(void) { time_t clock; time(&clock); fputs(ctime(&clock), stderr); fputs("time_push_rate(5, 1)\n", stderr); time_push_rate(10, 1); time(&clock); fputs(ctime(&clock), stderr); sleep(1); time(&clock); fputs(ctime(&clock), stderr); sleep(1); time(&clock); fputs(ctime(&clock), stderr); sleep(1); time(&clock); fputs(ctime(&clock), stderr); sleep(1); time(&clock); fputs(ctime(&clock), stderr); fputs("time_pop()\n", stderr); time_pop(); time(&clock); fputs(ctime(&clock), stderr); } static void test_time_slowdown(void) { time_t clock; int i; time(&clock); fputs(ctime(&clock), stderr); fputs("time_push_rate(1, 10)\n", stderr); time_push_rate(1, 5); for (i = 0 ; i < 20 ; i++) { time(&clock); fputs(ctime(&clock), stderr); sleep(1); } time(&clock); fputs(ctime(&clock), stderr); fputs("time_pop()\n", stderr); time_pop(); time(&clock); fputs(ctime(&clock), stderr); } static void test_time_fixed(void) { time_t clock; time(&clock); fputs(ctime(&clock), stderr); fputs("time_push_fixed(1354928400)\n", stderr); time_push_fixed(1354928400); time(&clock); fputs(ctime(&clock), stderr); sleep(1); time(&clock); fputs(ctime(&clock), stderr); sleep(1); time(&clock); fputs(ctime(&clock), stderr); sleep(1); time(&clock); fputs(ctime(&clock), stderr); sleep(1); time(&clock); fputs(ctime(&clock), stderr); fputs("time_pop()\n", stderr); time_pop(); time(&clock); fputs(ctime(&clock), stderr); } static void test_time_fixed2(void) { struct timeval tv; gettimeofday(&tv, NULL); fputs(ctime(&tv.tv_sec), stderr); fputs("time_push_fixed(1354928400)\n", stderr); time_push_fixed(1354928400); gettimeofday(&tv, NULL); fputs(ctime(&tv.tv_sec), stderr); sleep(1); gettimeofday(&tv, NULL); fputs(ctime(&tv.tv_sec), stderr); sleep(1); gettimeofday(&tv, NULL); fputs(ctime(&tv.tv_sec), stderr); sleep(1); gettimeofday(&tv, NULL); fputs(ctime(&tv.tv_sec), stderr); sleep(1); gettimeofday(&tv, NULL); fputs(ctime(&tv.tv_sec), stderr); fputs("time_pop()\n", stderr); time_pop(); gettimeofday(&tv, NULL); fputs(ctime(&tv.tv_sec), stderr); } #endif diff --git a/imap/ctl_conversationsdb.c b/imap/ctl_conversationsdb.c index bc07df9d3..2f9c6e400 100644 --- a/imap/ctl_conversationsdb.c +++ b/imap/ctl_conversationsdb.c @@ -1,945 +1,945 @@ /* * Copyright (c) 1994-2011 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #ifdef HAVE_UNISTD_H #include #endif #include #include #include #include #include #include /* cyrus includes */ #include "assert.h" #include "bsearch.h" #include "global.h" #include "index.h" #include "conversations.h" #include "mailbox.h" #include "mboxlist.h" #include "message.h" #include "util.h" #include "xmalloc.h" /* generated headers are not necessarily in current directory */ #include "imap/imap_err.h" /* config.c stuff */ const int config_need_data = CONFIG_NEED_PARTITION_DATA; enum { UNKNOWN, DUMP, UNDUMP, ZERO, BUILD, RECALC, AUDIT, CHECKFOLDERS }; int verbose = 0; int mode = UNKNOWN; static const char *audit_temp_directory; int recalc_silent = 1; static int do_dump(const char *fname, const char *userid) { struct conversations_state *state = NULL; struct stat sb; int r; /* What we really want here is read-only database access without * the create-if-nonexistent semantics. However, the cyrusdb * interface makes it difficult to do that properly. In the * meantime, we can just check if the file exists here. */ r = stat(fname, &sb); if (r < 0) { perror(fname); return -1; } r = conversations_open_path(fname, userid, 0/*shared*/, &state); if (r) { fprintf(stderr, "Failed to open conversations database %s: %s\n", fname, error_message(r)); return -1; } conversations_dump(state, stdout); conversations_commit(&state); return 0; } static int do_undump(const char *fname, const char *userid) { struct conversations_state *state; int r; r = conversations_open_path(fname, userid, 0/*shared*/, &state); if (r) { fprintf(stderr, "Failed to open conversations database %s: %s\n", fname, error_message(r)); return -1; } r = conversations_truncate(state); if (r) { fprintf(stderr, "Failed to truncate conversations database %s: %s\n", fname, error_message(r)); goto out; } r = conversations_undump(state, stdin); if (r) { fprintf(stderr, "Failed to undump to conversations database %s: %s\n", fname, error_message(r)); goto out; } r = conversations_commit(&state); if (r) fprintf(stderr, "Failed to commit conversations database %s: %s\n", fname, error_message(r)); out: conversations_abort(&state); return r; } static int zero_cid_cb(const mbentry_t *mbentry, void *rock __attribute__((unused))) { struct mailbox *mailbox = NULL; int r; r = mailbox_open_iwl(mbentry->name, &mailbox); if (r) { fprintf(stderr, "Failed to open mailbox %s, skipping\n", mbentry->name); return 0; } struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); const message_t *msg; while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); /* already zero, fine */ if (record->cid == NULLCONVERSATION) continue; struct index_record oldrecord = *record; oldrecord.cid = NULLCONVERSATION; oldrecord.basecid = NULLCONVERSATION; oldrecord.internal_flags &= ~FLAG_INTERNAL_SPLITCONVERSATION; r = mailbox_rewrite_index_record(mailbox, &oldrecord); if (r) break; } mailbox_iter_done(&iter); mailbox_close(&mailbox); return r; } static int do_zero(const char *userid) { struct conversations_state *state = NULL; int r; r = conversations_open_user(userid, 0/*shared*/, &state); if (r) return r; r = mboxlist_usermboxtree(userid, NULL, zero_cid_cb, NULL, 0); if (r) goto done; /* XXX: * annotatemore_findall(state->annotmboxname, IMAP_ANNOT_NS "basecid/%", &deleteannot); * remove all the basecid mappings so they don't create bad splits on rebuild */ done: conversations_commit(&state); return r; } static int build_cid_cb(const mbentry_t *mbentry, void *rock __attribute__((unused))) { struct mailbox *mailbox = NULL; int r = 0; int count = 1; struct conversations_state *cstate = conversations_get_mbox(mbentry->name); if (!cstate) return IMAP_CONVERSATIONS_NOT_OPEN; while (!r && count) { r = mailbox_open_iwl(mbentry->name, &mailbox); if (r) { fprintf(stderr, "Failed to open mailbox %s, skipping\n", mbentry->name); return 0; } count = 0; struct mailbox_iter *iter = mailbox_iter_init(mailbox, 0, ITER_SKIP_UNLINKED); const message_t *msg; while ((msg = mailbox_iter_step(iter))) { const struct index_record *record = msg_record(msg); /* already assigned, fine */ if (record->cid != NULLCONVERSATION) continue; struct index_record oldrecord = *record; r = mailbox_cacherecord(mailbox, &oldrecord); if (r) goto done; r = message_update_conversations(cstate, mailbox, &oldrecord, NULL); if (r) goto done; r = mailbox_rewrite_index_record(mailbox, &oldrecord); if (r) goto done; count++; /* batch so we don't lock for ages */ if (count > 8192) break; } mailbox_iter_done(&iter); done: mailbox_close(&mailbox); } return r; } static int do_build(const char *userid) { struct conversations_state *state = NULL; int r; r = conversations_open_user(userid, 0/*shared*/, &state); if (r) return r; r = mboxlist_usermboxtree(userid, NULL, build_cid_cb, NULL, 0); conversations_commit(&state); return r; } static int recalc_counts_cb(const mbentry_t *mbentry, void *rock __attribute__((unused))) { struct mailbox *mailbox = NULL; int r; r = mailbox_open_iwl(mbentry->name, &mailbox); if (r) return r; if (verbose) printf("%s\n", mbentry->name); r = mailbox_add_conversations(mailbox, recalc_silent); mailbox_close(&mailbox); return r; } static int audit_counts_cb(const mbentry_t *mbentry, void *rock __attribute__((unused))) { struct mailbox *mailbox = NULL; int r; r = mailbox_open_irl(mbentry->name, &mailbox); if (r) return r; if (verbose) printf("%s\n", mbentry->name); r = mailbox_add_conversations(mailbox, /*silent*/1); mailbox_close(&mailbox); return r; } static int do_recalc(const char *userid) { struct conversations_state *state = NULL; int r; r = conversations_open_user(userid, 0/*shared*/, &state); if (r) return r; r = conversations_zero_counts(state); if (r) goto err; r = mboxlist_usermboxtree(userid, NULL, recalc_counts_cb, NULL, 0); if (r) goto err; r = conversations_cleanup_zero(state); if (r) goto err; conversations_commit(&state); return 0; err: conversations_abort(&state); return r; } struct cursor { struct db *db; struct txn **txnp; const char *key; size_t keylen; const char *data; size_t datalen; int err; }; static void cursor_init(struct cursor *c, struct db *db, struct txn **txnp) { memset(c, 0, sizeof(*c)); c->db = db; c->txnp = txnp; } static int cursor_next(struct cursor *c) { if (!c->err) c->err = cyrusdb_fetchnext(c->db, c->key, c->keylen, &c->key, &c->keylen, &c->data, &c->datalen, c->txnp); return c->err; } static int blob_compare(const char *a, size_t alen, const char *b, size_t blen) { int d = memcmp(a, b, MIN(alen, blen)); if (!d) d = alen - blen; return d; } static int next_diffable_record(struct cursor *c) { for (;;) { int r = cursor_next(c); if (r) return r; /* skip < records, they won't be in the * temp database and we don't care so much */ if (c->key[0] == '<') continue; /* Subject, not re-calculated */ if (c->key[0] == 'S') continue; return 0; } } static unsigned int diff_records(struct conversations_state *a, struct conversations_state *b) { unsigned int ndiffs = 0; int ra, rb; struct cursor ca, cb; int keydelta; int delta; cursor_init(&ca, a->db, &a->txn); ra = cursor_next(&ca); cursor_init(&cb, b->db, &b->txn); rb = cursor_next(&cb); while (!ra || !rb) { keydelta = blob_compare(ca.key, ca.keylen, cb.key, cb.keylen); if (rb || keydelta < 0) { if (ra) break; ndiffs++; if (verbose) printf("REALONLY: \"%.*s\" data \"%.*s\"\n", (int)ca.keylen, ca.key, (int)ca.datalen, ca.data); ra = next_diffable_record(&ca); continue; } if (ra || keydelta > 0) { if (rb) break; ndiffs++; if (verbose) printf("TEMPONLY: \"%.*s\" data \"%.*s\"\n", (int)cb.keylen, cb.key, (int)cb.datalen, cb.data); rb = next_diffable_record(&cb); continue; } /* both exist an are the same key */ delta = blob_compare(ca.data, ca.datalen, cb.data, cb.datalen); if (delta) { ndiffs++; if (verbose) printf("REAL: \"%.*s\" data \"%.*s\"\n" "TEMP: \"%.*s\" data \"%.*s\"\n", (int)ca.keylen, ca.key, (int)ca.datalen, ca.data, (int)cb.keylen, cb.key, (int)cb.datalen, cb.data); } ra = next_diffable_record(&ca); rb = next_diffable_record(&cb); } return ndiffs; } static int fix_modseqs(struct conversations_state *a, struct conversations_state *b) { int ra, rb; struct cursor ca, cb; int keydelta; int r; cursor_init(&ca, a->db, &a->txn); ra = cursor_next(&ca); cursor_init(&cb, b->db, &b->txn); rb = cursor_next(&cb); while (!ra || !rb) { keydelta = blob_compare(ca.key, ca.keylen, cb.key, cb.keylen); if (rb || keydelta < 0) { if (ra) break; if (ca.key[0] == 'F') { conv_status_t status = CONV_STATUS_INIT; /* need to add record if it's zero */ r = conversation_parsestatus(ca.data, ca.datalen, &status); if (r) return r; if (status.threadexists == 0) { r = conversation_storestatus(b, ca.key, ca.keylen, &status); if (r) { fprintf(stderr, "Failed to store conversations " "record \"%.*s\" to %s: %s, giving up\n", (int)ca.keylen, ca.key, b->path, error_message(r)); return r; } } /* otherwise it's a bug, so leave it in for reporting */ } ra = cursor_next(&ca); continue; } if (ra || keydelta > 0) { if (rb) break; rb = cursor_next(&cb); continue; } /* folders? Just modseq check */ if (ca.key[0] == 'F') { /* check if modseq is higher for real */ conv_status_t statusa = CONV_STATUS_INIT; conv_status_t statusb = CONV_STATUS_INIT; /* need to add record if it's zero */ r = conversation_parsestatus(ca.data, ca.datalen, &statusa); if (r) { fprintf(stderr, "Failed to parse conversations " "record \"%.*s\" in %s: %s\n", (int)ca.keylen, ca.key, a->path, error_message(r)); /* There's no need to report failure to the caller - the * record diffing passing that occurs after this will * also pick up the same problem */ goto next; } r = conversation_parsestatus(cb.data, cb.datalen, &statusb); if (r) { fprintf(stderr, "Failed to parse conversations " "record \"%.*s\" in %s: %s\n", (int)cb.keylen, cb.key, b->path, error_message(r)); goto next; } if (statusa.threadmodseq > statusb.threadmodseq) { statusb.threadmodseq = statusa.threadmodseq; r = conversation_storestatus(b, cb.key, cb.keylen, &statusb); if (r) { fprintf(stderr, "Failed to store conversations " "record \"%.*s\" to %s: %s, giving up\n", (int)cb.keylen, cb.key, b->path, error_message(r)); /* If we cannot write to the temp DB, something is * drastically wrong and we need to report a failure */ return r; } } } if (ca.key[0] == 'B') { /* B keys - check all the modseqs, both top level and per folder */ conversation_t conva = CONVERSATION_INIT; conversation_t convb = CONVERSATION_INIT; conv_folder_t *foldera; conv_folder_t *folderb; conv_sender_t *sendera; r = conversation_parse(ca.data, ca.datalen, &conva, CONV_WITHALL); if (r) { fprintf(stderr, "Failed to parse conversations " "record \"%.*s\" in %s: %s\n", (int)ca.keylen, ca.key, a->path, error_message(r)); goto next; } r = conversation_parse(cb.data, cb.datalen, &convb, CONV_WITHALL); if (r) { fprintf(stderr, "Failed to parse conversations " "record \"%.*s\" in %s: %s\n", (int)cb.keylen, cb.key, b->path, error_message(r)); conversation_fini(&conva); goto next; } /* because expunged messages could have had higher modseqs, * we need to re-copy any higher modseqs in */ if (conva.modseq > convb.modseq) convb.modseq = conva.modseq; for (foldera = conva.folders; foldera; foldera = foldera->next) { folderb = conversation_get_folder(&convb, foldera->number, 1); if (folderb->modseq < foldera->modseq) folderb->modseq = foldera->modseq; } /* senders are timestamped, and the timestamp might be for a * deleted message! */ for (sendera = conva.senders; sendera; sendera = sendera->next) { /* always update! The delta logic will ensure we don't add * the record if it's not already at least present in the * other conversation */ conversation_update_sender(&convb, sendera->name, sendera->route, sendera->mailbox, sendera->domain, sendera->lastseen, /*delta_count*/0); } /* be nice to know if this is needed, but at least twoskip * will dedup for us */ r = conversation_store(b, cb.key, cb.keylen, &convb); /* free first before checking for errors */ conversation_fini(&conva); conversation_fini(&convb); if (r) { fprintf(stderr, "Failed to store conversations " "record \"%.*s\" to %s: %s, giving up\n", (int)cb.keylen, cb.key, b->path, error_message(r)); return r; } } next: ra = cursor_next(&ca); rb = cursor_next(&cb); } return 0; } int do_checkfolders(const char *userid) { int r; struct conversations_state *state = NULL; strarray_t *copy1, *copy2; /* open the DB */ r = conversations_open_user(userid, 0/*shared*/, &state); if (r) { fprintf(stderr, "Cannot open conversations db %s: %s\n", userid, error_message(r)); goto out; } /* don't mess with the original */ copy1 = strarray_dup(state->folder_names); /* remove empty folders first, they will duplicate for sure */ strarray_remove_all(copy1, "-"); copy2 = strarray_dup(copy1); strarray_sort(copy2, cmpstringp_raw); strarray_uniq(copy2); if (copy1->count != copy2->count) { printf("DUPLICATE %s\n", userid); } else { printf("OK %s\n", userid); } strarray_free(copy1); strarray_free(copy2); out: conversations_abort(&state); return r; } static int do_audit(const char *userid) { int r; char temp_suffix[64]; char *filename_temp = NULL; char *filename_real = NULL; struct conversations_state *state_temp = NULL; struct conversations_state *state_real = NULL; unsigned int ndiffs = 0; if (verbose) printf("User %s\n", userid); if (verbose) printf("Pass 1: recalculate counts into temporary db\n"); /* Generate a unique suffix for the temp db */ snprintf(temp_suffix, sizeof(temp_suffix), "conversations.audit.%d", (int)getpid()); /* Get the filenames */ filename_real = conversations_getuserpath(userid); conversations_set_suffix(temp_suffix); conversations_set_directory(audit_temp_directory); filename_temp = conversations_getuserpath(userid); conversations_set_suffix(NULL); conversations_set_directory(NULL); assert(strcmp(filename_temp, filename_real)); /* Initialise the temp copy of the database */ unlink(filename_temp); r = cyrusdb_copyfile(filename_real, filename_temp); if (r) { fprintf(stderr, "Cannot make temp copy of conversations db %s: %s\n", filename_real, error_message(r)); goto out; } /* Begin recalculating in the temp db */ r = conversations_open_path(filename_temp, userid, 0/*shared*/, &state_temp); if (r) { fprintf(stderr, "Cannot open conversations db %s: %s\n", filename_temp, error_message(r)); goto out; } r = conversations_zero_counts(state_temp); if (r) { fprintf(stderr, "Failed to zero counts in %s: %s\n", filename_temp, error_message(r)); goto out; } /* * Set the conversations db suffix during the recalc pass, so that * calls to conversations_open_mbox() from the mailbox code get * redirected to the temporary db. */ conversations_set_suffix(temp_suffix); conversations_set_directory(audit_temp_directory); r = mboxlist_usermboxtree(userid, NULL, audit_counts_cb, NULL, 0); if (r) { fprintf(stderr, "Failed to recalculate counts in %s: %s\n", filename_temp, error_message(r)); goto out; } r = conversations_cleanup_zero(state_temp); if (r) { fprintf(stderr, "Failed to cleanup zero counts in %s: %s\n", filename_temp, error_message(r)); goto out; } conversations_set_suffix(NULL); conversations_set_directory(NULL); r = conversations_commit(&state_temp); if (r) { fprintf(stderr, "Cannot commit conversations db %s: %s\n", filename_temp, error_message(r)); goto out; } if (verbose) printf("Pass 2: find differences from recalculated to live dbs\n"); r = conversations_open_path(filename_temp, userid, 0/*shared*/, &state_temp); if (r) { fprintf(stderr, "Cannot open conversations db %s: %s\n", filename_temp, error_message(r)); goto out; } r = conversations_open_path(filename_real, userid, 0/*shared*/, &state_real); if (r) { fprintf(stderr, "Cannot open conversations db %s: %s\n", filename_real, error_message(r)); goto out; } r = fix_modseqs(state_real, state_temp); if (r) { /* Error reported in fix_modseqs() */ goto out; } ndiffs += diff_records(state_real, state_temp); if (ndiffs) printf("%s is BROKEN (%u differences)\n", userid, ndiffs); else if (verbose) printf("%s is OK\n", userid); out: if (state_temp) conversations_abort(&state_temp); if (state_real) conversations_abort(&state_real); conversations_set_suffix(NULL); conversations_set_directory(NULL); cyrusdb_unlink(config_conversations_db, filename_temp, 0); free(filename_temp); free(filename_real); return r; } static int usage(const char *name) __attribute__((noreturn)); static int do_user(const char *userid, void *rock __attribute__((unused))) { char *fname; int r = 0; fname = conversations_getuserpath(userid); if (fname == NULL) { fprintf(stderr, "Unable to get conversations database " "filename for userid \"%s\"\n", userid); return EX_USAGE; } switch (mode) { case DUMP: if (do_dump(fname, userid)) r = EX_NOINPUT; break; case UNDUMP: if (do_undump(fname, userid)) r = EX_NOINPUT; break; case ZERO: if (do_zero(userid)) r = EX_NOINPUT; break; case BUILD: if (do_build(userid)) r = EX_NOINPUT; break; case RECALC: if (do_recalc(userid)) r = EX_NOINPUT; break; case AUDIT: if (do_audit(userid)) r = EX_NOINPUT; break; case CHECKFOLDERS: if (do_checkfolders(userid)) r = EX_NOINPUT; break; case UNKNOWN: fatal("UNKNOWN MODE", EX_SOFTWARE); } free(fname); return r; } int main(int argc, char **argv) { int c; const char *alt_config = NULL; const char *userid = NULL; int r = 0; int recursive = 0; while ((c = getopt(argc, argv, "durzSAbvRFC:T:")) != EOF) { switch (c) { case 'd': if (mode != UNKNOWN) usage(argv[0]); mode = DUMP; break; case 'r': recursive = 1; break; case 'u': if (mode != UNKNOWN) usage(argv[0]); mode = UNDUMP; break; case 'z': if (mode != UNKNOWN) usage(argv[0]); mode = ZERO; break; case 'b': if (mode != UNKNOWN) usage(argv[0]); mode = BUILD; break; case 'R': if (mode != UNKNOWN) usage(argv[0]); mode = RECALC; break; case 'A': if (mode != UNKNOWN) usage(argv[0]); mode = AUDIT; break; case 'F': if (mode != UNKNOWN) usage(argv[0]); mode = CHECKFOLDERS; break; case 'v': verbose++; break; case 'C': /* alt config file */ alt_config = optarg; break; case 'T': /* tmpfs directory for audit */ audit_temp_directory = optarg; break; case 'S': recalc_silent = 0; break; default: usage(argv[0]); break; } } if (mode == UNKNOWN) usage(argv[0]); if (optind == argc-1) userid = argv[optind]; else if (recursive) userid = ""; else usage(argv[0]); cyrus_init(alt_config, "ctl_conversationsdb", 0, 0); if (recursive) { mboxlist_alluser(do_user, NULL); } else do_user(userid, NULL); cyrus_done(); return r; } static int usage(const char *name) { fprintf(stderr, "usage: %s [options] [-u|-d|-z|-f] [-r] username\n", name); fprintf(stderr, "\n"); fprintf(stderr, "options are:\n"); fprintf(stderr, " -v be more verbose\n"); fprintf(stderr, " -C altconfig use altconfig instead of imapd.conf\n"); fprintf(stderr, " -u undump the conversations database from stdin\n"); fprintf(stderr, " -d dump the conversations database to stdout\n"); fprintf(stderr, " -z zero the conversations DB (make all NULLs)\n"); fprintf(stderr, " -b build conversations entries for any NULL records\n"); fprintf(stderr, " -R recalculate all counts\n"); fprintf(stderr, " -A audit conversations DB counts\n"); fprintf(stderr, " -F check folder names\n"); fprintf(stderr, " -T dir store temporary data for audit in dir\n"); fprintf(stderr, "\n"); fprintf(stderr, " -r recursive mode: username is a prefix\n"); exit(EX_USAGE); } -void fatal(const char* s, int code) +EXPORTED void fatal(const char* s, int code) { fprintf(stderr, "ctl_conversationsdb: %s\n", s); cyrus_done(); exit(code); } diff --git a/imap/httpd.c b/imap/httpd.c index ad7afa871..1e56a818e 100644 --- a/imap/httpd.c +++ b/imap/httpd.c @@ -1,4974 +1,4974 @@ /* httpd.c -- HTTP/RSS/xDAV/JMAP/TZdist/iSchedule server protocol parsing * * Copyright (c) 1994-2017 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. * */ #include #ifdef HAVE_UNISTD_H #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "prot.h" #include #include #include #include "httpd.h" #include "http_h2.h" #include "http_proxy.h" #include "http_ws.h" #include "acl.h" #include "assert.h" #include "util.h" #include "iptostring.h" #include "global.h" #include "tls.h" #include "map.h" #include "imapd.h" #include "proc.h" #include "version.h" #include "stristr.h" #include "xstrlcpy.h" #include "xstrlcat.h" #include "telemetry.h" #include "backend.h" #include "prometheus.h" #include "proxy.h" #include "sync_support.h" #include "userdeny.h" #include "message.h" #include "idle.h" #include "times.h" #include "tok.h" #include "wildmat.h" #include "md5.h" /* generated headers are not necessarily in current directory */ #include "imap/http_err.h" #ifdef WITH_DAV #include "http_dav.h" #endif #include #include #include static unsigned accept_encodings = 0; #ifdef HAVE_ZLIB #include HIDDEN void *zlib_init() { z_stream *zstrm = xzmalloc(sizeof(z_stream)); /* Always use gzip format because IE incorrectly uses raw deflate */ if (deflateInit2(zstrm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16+MAX_WBITS /* gzip */, MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY) != Z_OK) { free(zstrm); return NULL; } else { accept_encodings |= CE_DEFLATE | CE_GZIP; return zstrm; } } HIDDEN int zlib_compress(struct transaction_t *txn, unsigned flags, const char *buf, unsigned len) { z_stream *zstrm = txn->zstrm; unsigned flush, pending; if (flags & COMPRESS_START) deflateReset(zstrm); if (txn->ws_ctx) flush = Z_SYNC_FLUSH; else { /* Only flush for static content or on last (zero-length) chunk */ if (flags & COMPRESS_END) flush = Z_FINISH; else flush = Z_NO_FLUSH; } zstrm->next_in = (Bytef *) buf; zstrm->avail_in = len; buf_reset(&txn->zbuf); buf_ensure(&txn->zbuf, deflateBound(zstrm, zstrm->avail_in)); do { int zr; zstrm->next_out = (Bytef *) txn->zbuf.s + txn->zbuf.len; zstrm->avail_out = txn->zbuf.alloc - txn->zbuf.len; zr = deflate(zstrm, flush); if (!(zr == Z_OK || zr == Z_STREAM_END || zr == Z_BUF_ERROR)) { /* something went wrong */ syslog(LOG_ERR, "zlib deflate error: %d %s", zr, zstrm->msg); return -1; } txn->zbuf.len = txn->zbuf.alloc - zstrm->avail_out; if (zstrm->avail_out) { pending = 0; } else { /* http://www.zlib.net/manual.html says: * If deflate returns with avail_out == 0, this function must be * called again with the same value of the flush parameter and * more output space (updated avail_out), until the flush is * complete (deflate returns with non-zero avail_out). * In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure * that avail_out is greater than six to avoid repeated * flush markers due to avail_out == 0 on return. */ #ifdef HAVE_DEFLATE_PENDING zr = deflatePending(zstrm, &pending, Z_NULL); if (zr != Z_OK) { /* something went wrong */ syslog(LOG_ERR, "zlib deflate error: %d %s", zr, zstrm->msg); return -1; } #else /* Even if we have used all input, this will return non-zero */ pending = deflateBound(zstrm, zstrm->avail_in); #endif buf_ensure(&txn->zbuf, pending); } } while (pending); return 0; } static void zlib_done(z_stream *zstrm) { if (zstrm) { deflateEnd(zstrm); free(zstrm); } } #else /* !HAVE_ZLIB */ HIDDEN void *zlib_init() { return NULL; } HIDDEN int zlib_compress(struct transaction_t *txn __attribute__((unused)), unsigned flags __attribute__((unused)), const char *buf __attribute__((unused)), unsigned len __attribute__((unused))) { fatal("Compression requested, but no zlib", EX_SOFTWARE); } static void zlib_done(void *zstrm __attribute__((unused))) { } #endif /* HAVE_ZLIB */ #ifdef HAVE_BROTLI #include HIDDEN void *brotli_init() { BrotliEncoderState *brotli = BrotliEncoderCreateInstance(NULL, NULL, NULL); if (brotli) { BrotliEncoderSetParameter(brotli, BROTLI_PARAM_MODE, BROTLI_DEFAULT_MODE); BrotliEncoderSetParameter(brotli, BROTLI_PARAM_QUALITY, BROTLI_DEFAULT_QUALITY); BrotliEncoderSetParameter(brotli, BROTLI_PARAM_LGWIN, BROTLI_DEFAULT_WINDOW); BrotliEncoderSetParameter(brotli, BROTLI_PARAM_LGBLOCK, BROTLI_MAX_INPUT_BLOCK_BITS); } return brotli; } static int brotli_compress(struct transaction_t *txn, unsigned flags, const char *buf, unsigned len) { /* Only flush for static content or on last (zero-length) chunk */ unsigned op = (flags & COMPRESS_END) ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_FLUSH; BrotliEncoderState *brotli = txn->brotli; const uint8_t *next_in = (const uint8_t *) buf; size_t avail_in = (size_t) len; buf_reset(&txn->zbuf); buf_ensure(&txn->zbuf, BrotliEncoderMaxCompressedSize(avail_in)); do { uint8_t *next_out = (uint8_t *) txn->zbuf.s + txn->zbuf.len; size_t avail_out = txn->zbuf.alloc - txn->zbuf.len; if (!BrotliEncoderCompressStream(brotli, op, &avail_in, &next_in, &avail_out, &next_out, NULL)) { syslog(LOG_ERR, "Brotli: Error while compressing data"); return -1; } txn->zbuf.len = txn->zbuf.alloc - avail_out; } while (avail_in || BrotliEncoderHasMoreOutput(brotli)); if (BrotliEncoderIsFinished(brotli)) { BrotliEncoderDestroyInstance(brotli); txn->brotli = brotli_init(); } return 0; } static void brotli_done(BrotliEncoderState *brotli) { if (brotli) BrotliEncoderDestroyInstance(brotli); } #else /* !HAVE_BROTLI */ HIDDEN void *brotli_init() { return NULL; } static int brotli_compress(struct transaction_t *txn __attribute__((unused)), unsigned flags __attribute__((unused)), const char *buf __attribute__((unused)), unsigned len __attribute__((unused))) { fatal("Brotli Compression requested, but not available", EX_SOFTWARE); } static void brotli_done(void *brotli __attribute__((unused))) {} #endif /* HAVE_BROTLI */ #ifdef HAVE_ZSTD #include #include HIDDEN void *zstd_init() { ZSTD_CCtx *cctx = ZSTD_createCCtx(); if (cctx) { ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT); ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1); } return cctx; } static int zstd_compress(struct transaction_t *txn, unsigned flags, const char *buf, unsigned len) { /* Only flush for static content or on last (zero-length) chunk */ ZSTD_EndDirective mode = (flags & COMPRESS_END) ? ZSTD_e_end : ZSTD_e_flush; ZSTD_inBuffer input = { buf, len, 0 }; ZSTD_CCtx *cctx = txn->zstd; size_t remaining; if (flags & COMPRESS_START) ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); buf_reset(&txn->zbuf); buf_ensure(&txn->zbuf, ZSTD_compressBound(len)); ZSTD_outBuffer output = { txn->zbuf.s, txn->zbuf.alloc, 0 }; do { remaining = ZSTD_compressStream2(cctx, &output, &input, mode); if (ZSTD_isError(remaining)) { syslog(LOG_ERR, "Zstandard: %s", ZSTD_getErrorString(ZSTD_getErrorCode(remaining))); return -1; } } while (remaining || (input.pos != input.size)); buf_truncate(&txn->zbuf, output.pos); return 0; } static void zstd_done(ZSTD_CCtx *cctx) { if (cctx) ZSTD_freeCCtx(cctx); } #else /* !HAVE_ZSTD */ HIDDEN void *zstd_init() { return NULL; } static int zstd_compress(struct transaction_t *txn __attribute__((unused)), unsigned flags __attribute__((unused)), const char *buf __attribute__((unused)), unsigned len __attribute__((unused))) { fatal("Zstandard Compression requested, but not available", EX_SOFTWARE); } static void zstd_done(void *brotli __attribute__((unused))) {} #endif /* HAVE_ZSTD */ static const char tls_message[] = HTML_DOCTYPE "\n\nTLS Required\n\n" \ "\n

TLS is required prior to authentication

\n" \ "Use %s instead.\n" \ "\n\n"; extern int optind; extern char *optarg; extern int opterr; sasl_conn_t *httpd_saslconn; /* the sasl connection context */ static struct wildmat *allow_cors = NULL; int httpd_timeout, httpd_keepalive; char *httpd_authid = NULL; char *httpd_userid = NULL; char *httpd_extrafolder = NULL; char *httpd_extradomain = NULL; struct auth_state *httpd_authstate = 0; int httpd_userisadmin = 0; int httpd_userisproxyadmin = 0; int httpd_userisanonymous = 1; const char *httpd_localip = NULL, *httpd_remoteip = NULL; struct protstream *httpd_out = NULL; struct protstream *httpd_in = NULL; struct protgroup *protin = NULL; strarray_t *httpd_log_headers = NULL; static sasl_ssf_t extprops_ssf = 0; int https = 0; int httpd_tls_required = 0; unsigned avail_auth_schemes = 0; /* bitmask of available auth schemes */ unsigned long config_httpmodules; int config_httpprettytelemetry; static time_t compile_time; struct buf serverinfo = BUF_INITIALIZER; int ignorequota = 0; int apns_enabled = 0; /* List of HTTP auth schemes that we support - in descending order of security properties */ struct auth_scheme_t auth_schemes[] = { { AUTH_SPNEGO, "Negotiate", "GSS-SPNEGO", AUTH_BASE64 | AUTH_SUCCESS_WWW }, { AUTH_SCRAM_SHA256, "SCRAM-SHA-256", "SCRAM-SHA-256", AUTH_NEED_PERSIST | AUTH_SERVER_FIRST | AUTH_BASE64 | AUTH_REALM_PARAM | AUTH_DATA_PARAM }, { AUTH_SCRAM_SHA1, "SCRAM-SHA-1", "SCRAM-SHA-1", AUTH_NEED_PERSIST | AUTH_SERVER_FIRST | AUTH_BASE64 | AUTH_REALM_PARAM | AUTH_DATA_PARAM }, { AUTH_DIGEST, "Digest", HTTP_DIGEST_MECH, AUTH_NEED_REQUEST | AUTH_SERVER_FIRST }, { AUTH_NTLM, "NTLM", "NTLM", AUTH_NEED_PERSIST | AUTH_BASE64 }, { AUTH_BEARER, "Bearer", NULL, AUTH_SERVER_FIRST | AUTH_REALM_PARAM }, AUTH_SCHEME_BASIC, { 0, NULL, NULL, 0 } }; /* the sasl proxy policy context */ static struct proxy_context httpd_proxyctx = { 0, 1, &httpd_authstate, &httpd_userisadmin, &httpd_userisproxyadmin }; /* signal to config.c */ const int config_need_data = CONFIG_NEED_PARTITION_DATA; /* current namespace */ HIDDEN struct namespace httpd_namespace; /* PROXY STUFF */ /* we want a list of our outgoing connections here and which one we're currently piping */ /* the current server most commands go to */ struct backend *backend_current = NULL; /* our cached connections */ struct backend **backend_cached = NULL; /* end PROXY stuff */ static int starttls(struct transaction_t *txn, struct http_connection *conn); void usage(void); void shut_down(int code) __attribute__ ((noreturn)); /* Enable the resetting of a sasl_conn_t */ static int reset_saslconn(sasl_conn_t **conn); static void cmdloop(struct http_connection *conn); static int parse_expect(struct transaction_t *txn); static int parse_connection(struct transaction_t *txn); static int parse_ranges(const char *hdr, unsigned long len, struct range **ranges); static int proxy_authz(const char **authzid, struct transaction_t *txn); static int auth_success(struct transaction_t *txn, const char *userid); static int http_auth(const char *creds, struct transaction_t *txn); static int meth_get(struct transaction_t *txn, void *params); static int meth_propfind_root(struct transaction_t *txn, void *params); static struct saslprops_t saslprops = SASLPROPS_INITIALIZER; static struct sasl_callback mysasl_cb[] = { { SASL_CB_GETOPT, (mysasl_cb_ft *) &mysasl_config, NULL }, { SASL_CB_PROXY_POLICY, (mysasl_cb_ft *) &mysasl_proxy_policy, (void*) &httpd_proxyctx }, { SASL_CB_CANON_USER, (mysasl_cb_ft *) &mysasl_canon_user, NULL }, { SASL_CB_LIST_END, NULL, NULL } }; /* Array of HTTP methods known by our server. */ const struct known_meth_t http_methods[] = { { "ACL", 0, CYRUS_HTTP_ACL_TOTAL }, { "BIND", 0, CYRUS_HTTP_BIND_TOTAL }, { "CONNECT", METH_NOBODY, CYRUS_HTTP_CONNECT_TOTAL }, { "COPY", METH_NOBODY, CYRUS_HTTP_COPY_TOTAL }, { "DELETE", METH_NOBODY, CYRUS_HTTP_DELETE_TOTAL }, { "GET", METH_NOBODY | METH_SAFE, CYRUS_HTTP_GET_TOTAL }, { "HEAD", METH_NOBODY | METH_SAFE, CYRUS_HTTP_HEAD_TOTAL }, { "LOCK", 0, CYRUS_HTTP_LOCK_TOTAL }, { "MKCALENDAR", 0, CYRUS_HTTP_MKCALENDAR_TOTAL }, { "MKCOL", 0, CYRUS_HTTP_MKCOL_TOTAL }, { "MOVE", METH_NOBODY, CYRUS_HTTP_MOVE_TOTAL }, { "OPTIONS", METH_NOBODY | METH_SAFE, CYRUS_HTTP_OPTIONS_TOTAL }, { "PATCH", 0, CYRUS_HTTP_PATCH_TOTAL }, { "POST", 0, CYRUS_HTTP_POST_TOTAL }, { "PROPFIND", METH_SAFE, CYRUS_HTTP_PROPFIND_TOTAL }, { "PROPPATCH", 0, CYRUS_HTTP_PROPPATCH_TOTAL }, { "PUT", 0, CYRUS_HTTP_PUT_TOTAL }, { "REPORT", METH_SAFE, CYRUS_HTTP_REPORT_TOTAL }, { "SEARCH", METH_SAFE, CYRUS_HTTP_SEARCH_TOTAL }, { "TRACE", METH_NOBODY | METH_SAFE, CYRUS_HTTP_TRACE_TOTAL }, { "UNBIND", 0, CYRUS_HTTP_UNBIND_TOTAL }, { "UNLOCK", METH_NOBODY, CYRUS_HTTP_UNLOCK_TOTAL }, { NULL, 0, 0 } }; /* WebSocket handler */ static int ws_echo(struct buf *inbuf, struct buf *outbuf, struct buf *logbuf, void **rock); static struct connect_params ws_params = { "/", NULL /* sub-protocol */, &ws_echo }; /* Namespace to fetch static content from filesystem */ struct namespace_t namespace_default = { URL_NS_DEFAULT, 1, "default", "", NULL, http_allow_noauth, /*authschemes*/0, /*mbtype*/0, ALLOW_READ, NULL, NULL, NULL, NULL, NULL, NULL, { { NULL, NULL }, /* ACL */ { NULL, NULL }, /* BIND */ { &meth_connect, &ws_params }, /* CONNECT */ { NULL, NULL }, /* COPY */ { NULL, NULL }, /* DELETE */ { &meth_get, NULL }, /* GET */ { &meth_get, NULL }, /* HEAD */ { NULL, NULL }, /* LOCK */ { NULL, NULL }, /* MKCALENDAR */ { NULL, NULL }, /* MKCOL */ { NULL, NULL }, /* MOVE */ { &meth_options, NULL }, /* OPTIONS */ { NULL, NULL }, /* PATCH */ { NULL, NULL }, /* POST */ { &meth_propfind_root, NULL }, /* PROPFIND */ { NULL, NULL }, /* PROPPATCH */ { NULL, NULL }, /* PUT */ { NULL, NULL }, /* REPORT */ { &meth_trace, NULL }, /* TRACE */ { NULL, NULL }, /* UNBIND */ { NULL, NULL }, /* UNLOCK */ } }; /* Array of different namespaces and features supported by the server */ struct namespace_t *http_namespaces[] = { #ifdef WITH_JMAP &namespace_jmap, #endif &namespace_tzdist, /* MUST be before namespace_calendar!! */ #ifdef WITH_DAV &namespace_calendar, &namespace_freebusy, &namespace_addressbook, &namespace_drive, &namespace_principal, /* MUST be after namespace_cal & addr & drive */ &namespace_notify, /* MUST be after namespace_principal */ &namespace_applepush, /* MUST be after namespace_cal & addr */ &namespace_ischedule, &namespace_domainkey, #endif /* WITH_DAV */ &namespace_rss, &namespace_dblookup, &namespace_admin, &namespace_prometheus, &namespace_cgi, &namespace_default, /* MUST be present and be last!! */ NULL, }; static void httpd_reset(struct http_connection *conn) { int i; int bytes_in = 0; int bytes_out = 0; /* Do any namespace specific cleanup */ for (i = 0; http_namespaces[i]; i++) { if (http_namespaces[i]->enabled && http_namespaces[i]->reset) http_namespaces[i]->reset(); } /* Reset available authentication schemes */ avail_auth_schemes = 0; proc_cleanup(); /* close backend connections */ i = 0; while (backend_cached && backend_cached[i]) { proxy_downserver(backend_cached[i]); free(backend_cached[i]->context); free(backend_cached[i]); i++; } if (backend_cached) free(backend_cached); backend_cached = NULL; backend_current = NULL; index_text_extractor_destroy(); if (httpd_in) { prot_NONBLOCK(httpd_in); prot_fill(httpd_in); bytes_in = prot_bytes_in(httpd_in); prot_free(httpd_in); } if (httpd_out) { prot_flush(httpd_out); bytes_out = prot_bytes_out(httpd_out); prot_free(httpd_out); } if (config_auditlog) { syslog(LOG_NOTICE, "auditlog: traffic sessionid=<%s> bytes_in=<%d> bytes_out=<%d>", session_id(), bytes_in, bytes_out); } httpd_in = httpd_out = NULL; if (protin) protgroup_reset(protin); #ifdef HAVE_SSL if (conn->tls_ctx) { tls_reset_servertls((SSL **) &conn->tls_ctx); conn->tls_ctx = NULL; } #endif xmlFreeParserCtxt(conn->xml); http2_end_session(conn->sess_ctx); cyrus_reset_stdio(); conn->clienthost = "[local]"; buf_free(&conn->logbuf); if (conn->logfd != -1) { close(conn->logfd); conn->logfd = -1; } if (httpd_authid != NULL) { free(httpd_authid); httpd_authid = NULL; } if (httpd_userid != NULL) { free(httpd_userid); httpd_userid = NULL; } httpd_userisanonymous = 1; if (httpd_extrafolder != NULL) { free(httpd_extrafolder); httpd_extrafolder = NULL; } if (httpd_extradomain != NULL) { free(httpd_extradomain); httpd_extradomain = NULL; } if (httpd_authstate) { auth_freestate(httpd_authstate); httpd_authstate = NULL; } if (httpd_saslconn) { sasl_dispose(&httpd_saslconn); httpd_saslconn = NULL; } saslprops_reset(&saslprops); session_new_id(); } /* * run once when process is forked; * MUST NOT exit directly; must return with non-zero error code */ int service_init(int argc __attribute__((unused)), char **argv __attribute__((unused)), char **envp __attribute__((unused))) { int r, events, opt, i; int allow_trace = config_getswitch(IMAPOPT_HTTPALLOWTRACE); unsigned long version; unsigned int status, patch, fix, minor, major; LIBXML_TEST_VERSION if (geteuid() == 0) fatal("must run as the Cyrus user", EX_USAGE); setproctitle_init(argc, argv, envp); /* set signal handlers */ signals_set_shutdown(&shut_down); signal(SIGPIPE, SIG_IGN); /* load the SASL plugins */ global_sasl_init(1, 1, mysasl_cb); /* setup for sending IMAP IDLE notifications */ idle_enabled(); /* Set namespace */ if ((r = mboxname_init_namespace(&httpd_namespace, 1)) != 0) { fatal(error_message(r), EX_CONFIG); } /* open the mboxevent system */ events = mboxevent_init(); apns_enabled = (events & EVENT_APPLEPUSHSERVICE_DAV); mboxevent_setnamespace(&httpd_namespace); while ((opt = getopt(argc, argv, "sp:q")) != EOF) { switch(opt) { case 's': /* https (do TLS right away) */ https = 1; if (!tls_enabled()) { fatal("https: required OpenSSL options not present", EX_CONFIG); } break; case 'q': ignorequota = 1; break; case 'p': /* external protection */ extprops_ssf = atoi(optarg); break; default: usage(); } } /* Create a protgroup for input from the client and selected backend */ protin = protgroup_new(2); config_httpprettytelemetry = config_getswitch(IMAPOPT_HTTPPRETTYTELEMETRY); httpd_log_headers = strarray_split(config_getstring(IMAPOPT_HTTPLOGHEADERS), " ", STRARRAY_TRIM | STRARRAY_LCASE); if (config_getstring(IMAPOPT_HTTPALLOWCORS)) { allow_cors = split_wildmats((char *) config_getstring(IMAPOPT_HTTPALLOWCORS), NULL); } /* Construct serverinfo string */ buf_printf(&serverinfo, "Cyrus-HTTP/%s Cyrus-SASL/%u.%u.%u Lib/XML%s Jansson/%s", CYRUS_VERSION, SASL_VERSION_MAJOR, SASL_VERSION_MINOR, SASL_VERSION_STEP, LIBXML_DOTTED_VERSION, JANSSON_VERSION); http2_init(&serverinfo); ws_init(&serverinfo); #ifdef HAVE_SSL version = OPENSSL_VERSION_NUMBER; status = version & 0x0f; version >>= 4; patch = version & 0xff; version >>= 8; fix = version & 0xff; version >>= 8; minor = version & 0xff; version >>= 8; major = version & 0xff; buf_printf(&serverinfo, " OpenSSL/%u.%u.%u", major, minor, fix); if (status == 0) buf_appendcstr(&serverinfo, "-dev"); else if (status < 15) buf_printf(&serverinfo, "-beta%u", status); else if (patch) buf_putc(&serverinfo, patch + 'a' - 1); #endif #ifdef HAVE_ZLIB buf_printf(&serverinfo, " Zlib/%s", ZLIB_VERSION); #endif #ifdef HAVE_BROTLI version = BrotliEncoderVersion(); fix = version & 0xfff; version >>= 12; minor = version & 0xfff; version >>= 12; major = version & 0xfff; buf_printf(&serverinfo, " Brotli/%u.%u.%u", major, minor, fix); #endif #ifdef HAVE_ZSTD buf_printf(&serverinfo, " Zstd/%s", ZSTD_versionString()); #endif /* Initialize libical */ ical_support_init(); /* Do any namespace specific initialization */ config_httpmodules = config_getbitfield(IMAPOPT_HTTPMODULES); for (i = 0; http_namespaces[i]; i++) { if (allow_trace) http_namespaces[i]->allow |= ALLOW_TRACE; if (http_namespaces[i]->init) http_namespaces[i]->init(&serverinfo); } compile_time = calc_compile_time(__TIME__, __DATE__); prometheus_increment(CYRUS_HTTP_READY_LISTENERS); return 0; } static volatile sig_atomic_t gotsigalrm = 0; static void sigalrm_handler(int sig __attribute__((unused))) { gotsigalrm = 1; } /* * run for each accepted connection */ int service_main(int argc __attribute__((unused)), char **argv __attribute__((unused)), char **envp __attribute__((unused))) { sasl_security_properties_t *secprops=NULL; const char *mechlist, *mech; int mechcount = 0; size_t mechlen; struct auth_scheme_t *scheme; struct http_connection http_conn; /* fatal/shut_down will adjust these, so we need to set them early */ prometheus_decrement(CYRUS_HTTP_READY_LISTENERS); prometheus_increment(CYRUS_HTTP_ACTIVE_CONNECTIONS); session_new_id(); signals_poll(); httpd_in = prot_new(0, 0); httpd_out = prot_new(1, 1); protgroup_insert(protin, httpd_in); /* Setup HTTP connection */ memset(&http_conn, 0, sizeof(struct http_connection)); http_conn.pin = httpd_in; http_conn.pout = httpd_out; http_conn.logfd = -1; /* Create XML parser context */ if (!(http_conn.xml = xmlNewParserCtxt())) { fatal("Unable to create XML parser", EX_TEMPFAIL); } /* Find out name of client host */ http_conn.clienthost = get_clienthost(0, &httpd_localip, &httpd_remoteip); if (httpd_localip && httpd_remoteip) { buf_setcstr(&saslprops.ipremoteport, httpd_remoteip); buf_setcstr(&saslprops.iplocalport, httpd_localip); } /* other params should be filled in */ if (sasl_server_new("HTTP", config_servername, NULL, buf_cstringnull_ifempty(&saslprops.iplocalport), buf_cstringnull_ifempty(&saslprops.ipremoteport), NULL, SASL_USAGE_FLAGS, &httpd_saslconn) != SASL_OK) fatal("SASL failed initializing: sasl_server_new()",EX_TEMPFAIL); /* will always return something valid */ secprops = mysasl_secprops(0); /* no HTTP clients seem to use "auth-int" */ secprops->max_ssf = 0; /* "auth" only */ secprops->maxbufsize = 0; /* don't need maxbuf */ if (sasl_setprop(httpd_saslconn, SASL_SEC_PROPS, secprops) != SASL_OK) fatal("Failed to set SASL property", EX_TEMPFAIL); if (sasl_setprop(httpd_saslconn, SASL_SSF_EXTERNAL, &extprops_ssf) != SASL_OK) fatal("Failed to set SASL property", EX_TEMPFAIL); if (httpd_remoteip) { char hbuf[NI_MAXHOST], *p; /* Create pre-authentication telemetry log based on client IP */ strlcpy(hbuf, httpd_remoteip, NI_MAXHOST); if ((p = strchr(hbuf, ';'))) *p = '\0'; http_conn.logfd = telemetry_log(hbuf, httpd_in, httpd_out, 0); } /* See which auth schemes are available to us */ avail_auth_schemes = 0; /* Reset auth schemes for each connection */ if ((extprops_ssf >= 2) || config_getswitch(IMAPOPT_ALLOWPLAINTEXT)) { avail_auth_schemes |= AUTH_BASIC; } sasl_listmech(httpd_saslconn, NULL, NULL, " ", NULL, &mechlist, NULL, &mechcount); for (mech = mechlist; mechcount--; mech += ++mechlen) { mechlen = strcspn(mech, " \0"); for (scheme = auth_schemes; scheme->name; scheme++) { if (scheme->saslmech && !strncmp(mech, scheme->saslmech, mechlen)) { avail_auth_schemes |= scheme->id; break; } } } httpd_tls_required = config_getswitch(IMAPOPT_TLS_REQUIRED) || !avail_auth_schemes; proc_register(config_ident, http_conn.clienthost, NULL, NULL, NULL); /* Set inactivity timer */ httpd_timeout = config_getduration(IMAPOPT_HTTPTIMEOUT, 'm'); if (httpd_timeout < 0) httpd_timeout = 0; prot_settimeout(httpd_in, httpd_timeout); prot_setflushonread(httpd_in, httpd_out); /* we were connected on https port so we should do TLS negotiation immediately */ if (https == 1) { if (starttls(NULL, &http_conn) != 0) shut_down(0); } else if (http2_preface(&http_conn)) { /* HTTP/2 client connection preface */ if (http2_start_session(NULL, &http_conn) != 0) fatal("Failed initializing HTTP/2 session", EX_TEMPFAIL); } /* Setup the signal handler for keepalive heartbeat */ httpd_keepalive = config_getduration(IMAPOPT_HTTPKEEPALIVE, 's'); if (httpd_keepalive < 0) httpd_keepalive = 0; if (httpd_keepalive) { struct sigaction action; sigemptyset(&action.sa_mask); action.sa_flags = 0; #ifdef SA_RESTART action.sa_flags |= SA_RESTART; #endif action.sa_handler = sigalrm_handler; if (sigaction(SIGALRM, &action, NULL) < 0) { syslog(LOG_ERR, "unable to install signal handler for %d: %m", SIGALRM); httpd_keepalive = 0; } } index_text_extractor_init(httpd_in); /* count the connection, now that it's established */ prometheus_increment(CYRUS_HTTP_CONNECTIONS_TOTAL); cmdloop(&http_conn); prometheus_decrement(CYRUS_HTTP_ACTIVE_CONNECTIONS); /* Closing connection */ /* cleanup */ signal(SIGALRM, SIG_IGN); httpd_reset(&http_conn); prometheus_increment(CYRUS_HTTP_READY_LISTENERS); return 0; } /* Called by service API to shut down the service */ void service_abort(int error) { shut_down(error); } void usage(void) { prot_printf(httpd_out, "%s: usage: httpd [-C ] [-s]\r\n", error_message(HTTP_SERVER_ERROR)); prot_flush(httpd_out); exit(EX_USAGE); } /* * Cleanly shut down and exit */ void shut_down(int code) { int i; int bytes_in = 0; int bytes_out = 0; in_shutdown = 1; if (allow_cors) free_wildmats(allow_cors); strarray_free(httpd_log_headers); /* Do any namespace specific cleanup */ for (i = 0; http_namespaces[i]; i++) { if (http_namespaces[i]->enabled && http_namespaces[i]->shutdown) http_namespaces[i]->shutdown(); } xmlCleanupParser(); proc_cleanup(); /* close backend connections */ i = 0; while (backend_cached && backend_cached[i]) { proxy_downserver(backend_cached[i]); free(backend_cached[i]->context); free(backend_cached[i]); i++; } if (backend_cached) free(backend_cached); index_text_extractor_destroy(); annotatemore_close(); if (httpd_in) { prot_NONBLOCK(httpd_in); prot_fill(httpd_in); bytes_in = prot_bytes_in(httpd_in); prot_free(httpd_in); } if (httpd_out) { prot_flush(httpd_out); bytes_out = prot_bytes_out(httpd_out); prot_free(httpd_out); /* one less active connection */ prometheus_decrement(CYRUS_HTTP_ACTIVE_CONNECTIONS); } else { /* one less ready listener */ prometheus_decrement(CYRUS_HTTP_READY_LISTENERS); } prometheus_increment(code ? CYRUS_HTTP_SHUTDOWN_TOTAL_STATUS_ERROR : CYRUS_HTTP_SHUTDOWN_TOTAL_STATUS_OK); if (protin) protgroup_free(protin); if (config_auditlog) syslog(LOG_NOTICE, "auditlog: traffic sessionid=<%s> bytes_in=<%d> bytes_out=<%d>", session_id(), bytes_in, bytes_out); #ifdef HAVE_SSL tls_shutdown_serverengine(); #endif saslprops_free(&saslprops); http2_done(); cyrus_done(); exit(code); } -void fatal(const char* s, int code) +EXPORTED void fatal(const char* s, int code) { static int recurse_code = 0; const char *fatal = "Fatal error: "; if (recurse_code) { /* We were called recursively. Just give up */ proc_cleanup(); if (httpd_out) { /* one less active connection */ prometheus_decrement(CYRUS_HTTP_ACTIVE_CONNECTIONS); } else { /* one less ready listener */ prometheus_decrement(CYRUS_HTTP_READY_LISTENERS); } prometheus_increment(CYRUS_HTTP_SHUTDOWN_TOTAL_STATUS_ERROR); exit(recurse_code); } recurse_code = code; if (httpd_out) { prot_printf(httpd_out, "HTTP/1.1 %s\r\n" "Content-Type: text/plain\r\n" "Content-Length: %zu\r\n" "Connection: close\r\n" "\r\n" "%s%s\r\n", error_message(HTTP_SERVER_ERROR), strlen(fatal) + strlen(s) + 2, fatal, s); prot_flush(httpd_out); } syslog(LOG_ERR, "%s%s", fatal, s); shut_down(code); } #ifdef HAVE_SSL static int starttls(struct transaction_t *txn, struct http_connection *conn) { int https = (txn == NULL); int result; SSL_CTX *ctx = NULL; if (!conn) conn = txn->conn; result=tls_init_serverengine("http", 5, /* depth to verify */ !https, /* can client auth? */ &ctx); if (result == -1) { syslog(LOG_ERR, "error initializing TLS"); if (txn) txn->error.desc = "Error initializing TLS"; return HTTP_SERVER_ERROR; } if (http2_enabled()) { #ifdef HAVE_TLS_ALPN /* enable TLS ALPN extension */ SSL_CTX_set_alpn_select_cb(ctx, alpn_select_cb, conn); #endif } if (!https) { /* tell client to start TLS upgrade (RFC 2817) */ response_header(HTTP_SWITCH_PROT, txn); } result=tls_start_servertls(0, /* read */ 1, /* write */ https ? 180 : httpd_timeout, &saslprops, (SSL **) &conn->tls_ctx); /* if error */ if (result == -1) { syslog(LOG_NOTICE, "starttls failed: %s", conn->clienthost); if (txn) txn->error.desc = "Error negotiating TLS"; return HTTP_BAD_REQUEST; } /* tell SASL about the negotiated layer */ result = saslprops_set_tls(&saslprops, httpd_saslconn); if (result != SASL_OK) { syslog(LOG_NOTICE, "saslprops_set_tls() failed: cmd_starttls()"); if (https == 0) { fatal("saslprops_set_tls() failed: cmd_starttls()", EX_TEMPFAIL); } else { shut_down(0); } } /* tell the prot layer about our new layers */ prot_settls(httpd_in, conn->tls_ctx); prot_settls(httpd_out, conn->tls_ctx); httpd_tls_required = 0; avail_auth_schemes |= AUTH_BASIC; return 0; } #else static int starttls(struct transaction_t *txn __attribute__((unused)), struct http_connection *conn __attribute__((unused))) { fatal("starttls() called, but no OpenSSL", EX_SOFTWARE); } #endif /* HAVE_SSL */ /* Reset the given sasl_conn_t to a sane state */ static int reset_saslconn(sasl_conn_t **conn) { int ret; sasl_security_properties_t *secprops = NULL; sasl_dispose(conn); /* do initialization typical of service_main */ ret = sasl_server_new("HTTP", config_servername, NULL, buf_cstringnull_ifempty(&saslprops.iplocalport), buf_cstringnull_ifempty(&saslprops.ipremoteport), NULL, SASL_USAGE_FLAGS, conn); if(ret != SASL_OK) return ret; secprops = mysasl_secprops(0); /* no HTTP clients seem to use "auth-int" */ secprops->max_ssf = 0; /* "auth" only */ secprops->maxbufsize = 0; /* don't need maxbuf */ ret = sasl_setprop(*conn, SASL_SEC_PROPS, secprops); if(ret != SASL_OK) return ret; /* end of service_main initialization excepting SSF */ /* If we have TLS/SSL info, set it */ if(saslprops.ssf) { ret = saslprops_set_tls(&saslprops, *conn); } else { ret = sasl_setprop(*conn, SASL_SSF_EXTERNAL, &extprops_ssf); } if(ret != SASL_OK) return ret; /* End TLS/SSL Info */ return SASL_OK; } static int parse_request_line(struct transaction_t *txn) { struct request_line_t *req_line = &txn->req_line; char *p; tok_t tok; int ret = 0; /* Trim CRLF from request-line */ p = req_line->buf + strlen(req_line->buf); if (p[-1] == '\n') *--p = '\0'; if (p[-1] == '\r') *--p = '\0'; /* Parse request-line = method SP request-target SP HTTP-version CRLF */ tok_initm(&tok, req_line->buf, " ", 0); if (!(req_line->meth = tok_next(&tok))) { ret = HTTP_BAD_REQUEST; txn->error.desc = "Missing method in request-line"; } else if (!(req_line->uri = tok_next(&tok))) { ret = HTTP_BAD_REQUEST; txn->error.desc = "Missing request-target in request-line"; } else if ((size_t) (p - req_line->buf) > MAX_REQ_LINE - 2) { /* request-line overran the size of our buffer */ ret = HTTP_URI_TOO_LONG; buf_printf(&txn->buf, "Length of request-line MUST be less than %u octets", MAX_REQ_LINE); txn->error.desc = buf_cstring(&txn->buf); } else if (!(req_line->ver = tok_next(&tok))) { ret = HTTP_BAD_REQUEST; txn->error.desc = "Missing HTTP-version in request-line"; } else if (tok_next(&tok)) { ret = HTTP_BAD_REQUEST; txn->error.desc = "Unexpected extra argument(s) in request-line"; } /* Check HTTP-Version - MUST be HTTP/1.x */ else if (strlen(req_line->ver) != HTTP_VERSION_LEN || strncmp(req_line->ver, HTTP_VERSION, HTTP_VERSION_LEN-1) || !isdigit(req_line->ver[HTTP_VERSION_LEN-1])) { ret = HTTP_BAD_VERSION; buf_printf(&txn->buf, "This server only speaks %.*sx", HTTP_VERSION_LEN-1, HTTP_VERSION); txn->error.desc = buf_cstring(&txn->buf); } else if (req_line->ver[HTTP_VERSION_LEN-1] == '0') { /* HTTP/1.0 connection */ txn->flags.ver = VER_1_0; } tok_fini(&tok); return ret; } static int client_need_auth(struct transaction_t *txn, int sasl_result) { if (httpd_tls_required) { /* We only support TLS+Basic, so tell client to use TLS */ const char **hdr; /* Check which response is required */ if ((hdr = spool_getheader(txn->req_hdrs, "Upgrade")) && stristr(hdr[0], TLS_VERSION)) { /* Client (Murder proxy) supports RFC 2817 (TLS upgrade) */ txn->flags.conn |= CONN_UPGRADE; txn->flags.upgrade = UPGRADE_TLS; return HTTP_UPGRADE; } else { /* All other clients use RFC 2818 (HTTPS) */ const char *path = txn->req_uri->path; const char *query = URI_QUERY(txn->req_uri); struct buf *html = &txn->resp_body.payload; /* Create https URL */ hdr = spool_getheader(txn->req_hdrs, ":authority"); buf_printf(&txn->buf, "https://%s", hdr[0]); if (strcmp(path, "*")) { buf_appendcstr(&txn->buf, path); if (query) buf_printf(&txn->buf, "?%s", query); } txn->location = buf_cstring(&txn->buf); /* Create HTML body */ buf_reset(html); buf_printf(html, tls_message, buf_cstring(&txn->buf), buf_cstring(&txn->buf)); /* Output our HTML response */ txn->resp_body.type = "text/html; charset=utf-8"; return HTTP_MOVED; } } else { /* Tell client to authenticate */ if (sasl_result == SASL_CONTINUE) txn->error.desc = "Continue authentication exchange"; else if (sasl_result) txn->error.desc = "Authentication failed"; else txn->error.desc = "Must authenticate to access the specified target"; return HTTP_UNAUTHORIZED; } } static int check_method(struct transaction_t *txn) { const char **hdr; struct request_line_t *req_line = &txn->req_line; if (txn->flags.redirect) return 0; /* Check for HTTP method override */ if (!strcmp(req_line->meth, "POST") && (hdr = spool_getheader(txn->req_hdrs, "X-HTTP-Method-Override"))) { txn->flags.override = 1; req_line->meth = (char *) hdr[0]; } /* Check Method against our list of known methods */ for (txn->meth = 0; (txn->meth < METH_UNKNOWN) && strcmp(http_methods[txn->meth].name, req_line->meth); txn->meth++); if (txn->meth == METH_UNKNOWN) return HTTP_NOT_IMPLEMENTED; return 0; } static int preauth_check_hdrs(struct transaction_t *txn) { int ret = 0; const char **hdr; if (txn->flags.redirect) return 0; /* Check for mandatory Host header (HTTP/1.1+ only) */ if ((hdr = spool_getheader(txn->req_hdrs, "Host"))) { if (hdr[1]) { txn->error.desc = "Too many Host headers"; return HTTP_BAD_REQUEST; } /* Create an :authority pseudo header from Host */ spool_cache_header(xstrdup(":authority"), xstrdup(hdr[0]), txn->req_hdrs); } else { switch (txn->flags.ver) { case VER_2: /* HTTP/2 - check for :authority pseudo header */ if (spool_getheader(txn->req_hdrs, ":authority")) break; /* Fall through and create an :authority pseudo header */ GCC_FALLTHROUGH case VER_1_0: /* HTTP/1.0 - create an :authority pseudo header from URI */ if (txn->req_uri->server) { buf_setcstr(&txn->buf, txn->req_uri->server); if (txn->req_uri->port) buf_printf(&txn->buf, ":%d", txn->req_uri->port); } else buf_setcstr(&txn->buf, config_servername); spool_cache_header(xstrdup(":authority"), buf_release(&txn->buf), txn->req_hdrs); break; case VER_1_1: default: txn->error.desc = "Missing Host header"; return HTTP_BAD_REQUEST; } } /* Check message framing */ if ((ret = http_parse_framing(txn->flags.ver == VER_2, txn->req_hdrs, &txn->req_body, &txn->error.desc))) return ret; /* Check for Expectations */ if ((ret = parse_expect(txn))) return ret; /* Check for Connection options */ if ((ret = parse_connection(txn))) return ret; syslog(LOG_DEBUG, "conn flags: %#x upgrade flags: %#x tls req: %d", txn->flags.conn, txn->flags.upgrade, httpd_tls_required); if (txn->flags.conn & CONN_UPGRADE) { /* Read any request body (can't upgrade in middle of request) */ txn->req_body.flags |= BODY_DECODE; ret = http_read_req_body(txn); if (ret) { txn->flags.conn = CONN_CLOSE; return ret; } if (txn->flags.upgrade & UPGRADE_TLS) { if ((ret = starttls(txn, NULL))) { txn->flags.conn = CONN_CLOSE; return ret; } /* Don't advertise TLS Upgrade anymore */ txn->flags.upgrade &= ~UPGRADE_TLS; } syslog(LOG_DEBUG, "upgrade flags: %#x tls req: %d", txn->flags.upgrade, httpd_tls_required); if ((txn->flags.upgrade & UPGRADE_HTTP2) && !httpd_tls_required) { if ((ret = http2_start_session(txn, NULL))) { txn->flags.conn = CONN_CLOSE; return ret; } /* Upgrade header field mechanism not available under HTTP/2 */ txn->flags.upgrade = 0; } } else if (!txn->conn->tls_ctx && txn->flags.ver == VER_1_1) { /* Advertise available upgrade protocols */ if (tls_enabled() && config_mupdate_server && config_getstring(IMAPOPT_PROXYSERVERS)) { txn->flags.upgrade |= UPGRADE_TLS; } if (http2_enabled()) txn->flags.upgrade |= UPGRADE_HTTP2; } if (txn->flags.upgrade) txn->flags.conn |= CONN_UPGRADE; else txn->flags.conn &= ~CONN_UPGRADE; return 0; } static int check_namespace(struct transaction_t *txn) { int i; const char **hdr, *query = URI_QUERY(txn->req_uri); const struct namespace_t *namespace; const struct method_t *meth_t; /* Find the namespace of the requested resource */ for (i = 0; http_namespaces[i]; i++) { const char *path = txn->req_uri->path; size_t len; /* Skip disabled namespaces */ if (!http_namespaces[i]->enabled) continue; /* Handle any /.well-known/ bootstrapping */ if (http_namespaces[i]->well_known) { len = strlen(http_namespaces[i]->well_known); if (!strncmp(path, http_namespaces[i]->well_known, len) && (!path[len] || path[len] == '/')) { hdr = spool_getheader(txn->req_hdrs, ":authority"); buf_reset(&txn->buf); buf_printf(&txn->buf, "%s://%s", https ? "https" : "http", hdr[0]); buf_appendcstr(&txn->buf, http_namespaces[i]->prefix); buf_appendcstr(&txn->buf, path + len); if (query) buf_printf(&txn->buf, "?%s", query); txn->location = buf_cstring(&txn->buf); return HTTP_MOVED; } } /* See if the prefix matches - terminated with NUL or '/' */ len = strlen(http_namespaces[i]->prefix); if (!strncmp(path, http_namespaces[i]->prefix, len) && (!path[len] || (path[len] == '/') || !strcmp(path, "*"))) { break; } } if ((namespace = http_namespaces[i])) { txn->req_tgt.namespace = namespace; txn->req_tgt.allow = namespace->allow; /* Check if method is supported in this namespace */ meth_t = &namespace->methods[txn->meth]; if (!meth_t->proc) return HTTP_NOT_ALLOWED; if (config_getswitch(IMAPOPT_READONLY) && !(http_methods[txn->meth].flags & METH_SAFE) && !(namespace->allow & ALLOW_READONLY)) { return HTTP_NOT_ALLOWED; } /* Check if method expects a body */ else if ((http_methods[txn->meth].flags & METH_NOBODY) && (txn->req_body.framing != FRAMING_LENGTH || /* XXX Will break if client sends just a last-chunk */ txn->req_body.len)) { return HTTP_BAD_MEDIATYPE; } } else { /* XXX Should never get here */ return HTTP_SERVER_ERROR; } /* See if this namespace whitelists auth schemes */ if (namespace->auth_schemes) { avail_auth_schemes = (namespace->auth_schemes & avail_auth_schemes); /* Bearer auth must be advertised and supported by the namespace */ if ((namespace->auth_schemes & AUTH_BEARER) && namespace->bearer) { avail_auth_schemes |= AUTH_BEARER; } } return 0; } static int auth_check_hdrs(struct transaction_t *txn, int *sasl_result) { int ret = 0, r = 0; const char **hdr; if (txn->flags.redirect) return 0; /* Perform authentication, if necessary */ if ((hdr = spool_getheader(txn->req_hdrs, "Authorization"))) { if (httpd_userid) { /* Reauth - reinitialize */ syslog(LOG_DEBUG, "reauth - reinit"); reset_saslconn(&httpd_saslconn); txn->auth_chal.scheme = NULL; } if (httpd_tls_required) { /* TLS required - redirect handled below */ ret = HTTP_UNAUTHORIZED; } else { /* Check the auth credentials */ r = http_auth(hdr[0], txn); if ((r < 0) || !txn->auth_chal.scheme) { /* Auth failed - reinitialize */ syslog(LOG_DEBUG, "auth failed - reinit"); reset_saslconn(&httpd_saslconn); txn->auth_chal.scheme = NULL; if (r == SASL_UNAVAIL) { /* The namespace to authenticate to is unavailable. * There could be any reason for this, e.g. the DAV * handler could have run into a timeout for the * user's dabatase. In any case, there's no sense * to challenge the client for authentication. */ ret = HTTP_UNAVAILABLE; } else if (r == SASL_FAIL) { ret = HTTP_SERVER_ERROR; } else { ret = HTTP_UNAUTHORIZED; } } else if (r == SASL_CONTINUE) { /* Continue with multi-step authentication */ ret = HTTP_UNAUTHORIZED; } } } else if (!httpd_userid && txn->auth_chal.scheme) { /* Started auth exchange, but client didn't engage - reinit */ syslog(LOG_DEBUG, "client didn't complete auth - reinit"); reset_saslconn(&httpd_saslconn); txn->auth_chal.scheme = NULL; } /* Drop auth credentials, if not a backend in a Murder */ else if (!config_mupdate_server || !config_getstring(IMAPOPT_PROXYSERVERS)) { syslog(LOG_DEBUG, "drop auth creds"); free(httpd_userid); httpd_userid = NULL; free(httpd_extrafolder); httpd_extrafolder = NULL; free(httpd_extradomain); httpd_extradomain = NULL; if (httpd_authstate) { auth_freestate(httpd_authstate); httpd_authstate = NULL; } } /* Perform proxy authorization, if necessary */ else if (httpd_authid && (hdr = spool_getheader(txn->req_hdrs, "Authorize-As")) && *hdr[0]) { const char *authzid = hdr[0]; r = proxy_authz(&authzid, txn); if (r) { /* Proxy authz failed - reinitialize */ syslog(LOG_DEBUG, "proxy authz failed - reinit"); reset_saslconn(&httpd_saslconn); txn->auth_chal.scheme = NULL; ret = HTTP_UNAUTHORIZED; } else { ret = auth_success(txn, authzid); } } *sasl_result = r; return ret; } static void postauth_check_hdrs(struct transaction_t *txn) { const char **hdr; if (txn->flags.redirect) return; /* Check if this is a Cross-Origin Resource Sharing request */ if (allow_cors && (hdr = spool_getheader(txn->req_hdrs, "Origin"))) { const char *err = NULL; xmlURIPtr uri = parse_uri(METH_UNKNOWN, hdr[0], 0, &err); if (uri && uri->scheme && uri->server) { int o_https = !strcasecmp(uri->scheme, "https"); if ((https == o_https) && !strcasecmp(uri->server, *spool_getheader(txn->req_hdrs, ":authority"))) { txn->flags.cors = CORS_SIMPLE; } else { struct wildmat *wild; /* Create URI w/o path or default port */ assert(!buf_len(&txn->buf)); buf_printf(&txn->buf, "%s://%s", lcase(uri->scheme), lcase(uri->server)); if (uri->port && ((o_https && uri->port != 443) || (!o_https && uri->port != 80))) { buf_printf(&txn->buf, ":%d", uri->port); } /* Check Origin against the 'httpallowcors' wildmat */ for (wild = allow_cors; wild->pat; wild++) { if (wildmat(buf_cstring(&txn->buf), wild->pat)) { /* If we have a non-negative match, allow request */ if (!wild->not) txn->flags.cors = CORS_SIMPLE; break; } } buf_reset(&txn->buf); } } xmlFreeURI(uri); } /* Check if we should compress response body XXX Do we want to support deflate even though M$ doesn't implement it correctly (raw deflate vs. zlib)? */ if (txn->zstrm && txn->flags.ver == VER_1_1 && (hdr = spool_getheader(txn->req_hdrs, "TE"))) { struct accept *e, *enc = parse_accept(hdr); for (e = enc; e && e->token; e++) { if (e->qual > 0.0 && (!strcasecmp(e->token, "gzip") || !strcasecmp(e->token, "x-gzip"))) { txn->flags.te = TE_GZIP; } free(e->token); } if (enc) free(enc); } else if ((txn->zstrm || txn->brotli || txn->zstd) && (hdr = spool_getheader(txn->req_hdrs, "Accept-Encoding"))) { struct accept *e, *enc = parse_accept(hdr); float qual = 0.0; for (e = enc; e && e->token; e++) { if (e->qual > 0.0 && e->qual >= qual) { unsigned ce = CE_IDENTITY; encode_proc_t proc = NULL; if (txn->zstd && !strcasecmp(e->token, "zstd")) { ce = CE_ZSTD; proc = &zstd_compress; } else if (txn->brotli && !strcasecmp(e->token, "br")) { ce = CE_BR; proc = &brotli_compress; } else if (txn->zstrm && (!strcasecmp(e->token, "gzip") || !strcasecmp(e->token, "x-gzip"))) { ce = CE_GZIP; proc = &zlib_compress; } else { /* Unknown/unsupported */ e->qual = 0.0; } /* Favor Zstandard over Brotli over GZIP if q values are equal */ if (e->qual > qual || txn->resp_body.enc.type < ce) { txn->resp_body.enc.type = ce; txn->resp_body.enc.proc = proc; qual = e->qual; } } free(e->token); } if (enc) free(enc); } } EXPORTED int examine_request(struct transaction_t *txn, const char *uri) { int ret = 0, sasl_result = 0; const char *query; const struct namespace_t *namespace; struct request_line_t *req_line = &txn->req_line; if (!uri) uri = req_line->uri; /* Check method */ if ((ret = check_method(txn))) return ret; /* Parse request-target URI */ if (!(txn->req_uri = parse_uri(txn->meth, uri, 1, &txn->error.desc))) { return HTTP_BAD_REQUEST; } /* Perform pre-authentication check of headers */ if ((ret = preauth_check_hdrs(txn))) return ret; /* Find the namespace of the requested resource */ if ((ret = check_namespace(txn))) return ret; /* Perform check of authentication headers */ ret = auth_check_hdrs(txn, &sasl_result); if (ret && ret != HTTP_UNAUTHORIZED) return ret; /* Register service/module and method */ namespace = txn->req_tgt.namespace; buf_printf(&txn->buf, "%s%s", config_ident, namespace->well_known ? strrchr(namespace->well_known, '/') : namespace->prefix); proc_register(buf_cstring(&txn->buf), txn->conn->clienthost, httpd_userid, txn->req_tgt.path, txn->req_line.meth); buf_reset(&txn->buf); /* Request authentication, if necessary */ if (!httpd_userid && namespace->need_auth(txn)) { ret = HTTP_UNAUTHORIZED; } if (ret) return client_need_auth(txn, sasl_result); /* Parse any query parameters */ construct_hash_table(&txn->req_qparams, 10, 1); query = URI_QUERY(txn->req_uri); if (query) parse_query_params(txn, query); /* Perform post-authentication check of headers */ postauth_check_hdrs(txn); return 0; } EXPORTED int process_request(struct transaction_t *txn) { int ret = 0; if (txn->req_tgt.namespace->premethod) { ret = txn->req_tgt.namespace->premethod(txn); } if (!ret) { const struct method_t *meth_t = &txn->req_tgt.namespace->methods[txn->meth]; ret = (*meth_t->proc)(txn, meth_t->params); prometheus_increment( prometheus_lookup_label(http_methods[txn->meth].metric, txn->req_tgt.namespace->name)); } if (ret == HTTP_UNAUTHORIZED) { /* User must authenticate */ ret = client_need_auth(txn, 0); } return ret; } static int http1_input(struct transaction_t *txn) { struct request_line_t *req_line = &txn->req_line; int ignore_empty = 1, ret = 0; do { /* Read request-line */ syslog(LOG_DEBUG, "read & parse request-line"); if (!prot_fgets(req_line->buf, MAX_REQ_LINE+1, httpd_in)) { txn->error.desc = prot_error(httpd_in); if (txn->error.desc && strcmp(txn->error.desc, PROT_EOF_STRING)) { /* client timed out */ syslog(LOG_WARNING, "%s, closing connection", txn->error.desc); ret = HTTP_TIMEOUT; } else { /* client closed connection */ syslog(LOG_DEBUG, "client closed connection"); } txn->flags.conn = CONN_CLOSE; return ret; } /* Ignore 1 empty line before request-line per RFC 7230 Sec 3.5 */ } while (ignore_empty-- && (strcspn(req_line->buf, "\r\n") == 0)); /* Parse request-line = method SP request-target SP HTTP-version CRLF */ ret = parse_request_line(txn); /* Parse headers */ if (!ret) { ret = http_read_headers(httpd_in, 1 /* read_sep */, &txn->req_hdrs, &txn->error.desc); } if (ret) { txn->flags.conn = CONN_CLOSE; goto done; } /* Examine request */ ret = examine_request(txn, NULL); if (ret) goto done; /* Start method processing alarm (HTTP/1.1 only) */ if (txn->flags.ver == VER_1_1) alarm(httpd_keepalive); /* Process the requested method */ ret = process_request(txn); done: /* Handle errors (success responses handled by method functions) */ if (ret) error_response(ret, txn); /* Read and discard any unread request body */ if (!(txn->flags.conn & CONN_CLOSE)) { txn->req_body.flags |= BODY_DISCARD; if (http_read_req_body(txn)) { txn->flags.conn = CONN_CLOSE; } } return 0; } static void transaction_reset(struct transaction_t *txn) { txn->meth = METH_UNKNOWN; memset(&txn->flags, 0, sizeof(struct txn_flags_t)); txn->flags.ver = VER_1_1; txn->flags.vary = VARY_AE; memset(&txn->req_line, 0, sizeof(struct request_line_t)); /* Reset Bearer auth scheme for each transaction */ avail_auth_schemes &= ~AUTH_BEARER; if (txn->req_uri) xmlFreeURI(txn->req_uri); txn->req_uri = NULL; /* XXX - split this into a req_tgt cleanup */ free(txn->req_tgt.userid); mboxlist_entry_free(&txn->req_tgt.mbentry); memset(&txn->req_tgt, 0, sizeof(struct request_target_t)); free_hash_table(&txn->req_qparams, (void (*)(void *)) &freestrlist); if (txn->req_hdrs) spool_free_hdrcache(txn->req_hdrs); txn->req_hdrs = NULL; txn->req_body.flags = 0; buf_reset(&txn->req_body.payload); txn->auth_chal.param = NULL; txn->location = NULL; memset(&txn->error, 0, sizeof(struct error_t)); strarray_fini(&txn->resp_body.links); memset(&txn->resp_body, 0, /* Don't zero the response payload buffer */ sizeof(struct resp_body_t) - sizeof(struct buf)); buf_reset(&txn->resp_body.payload); /* Pre-allocate our working buffer */ buf_reset(&txn->buf); buf_ensure(&txn->buf, 1024); } EXPORTED void transaction_free(struct transaction_t *txn) { transaction_reset(txn); ws_end_channel(txn->ws_ctx); http2_end_stream(txn->strm_ctx); zlib_done(txn->zstrm); zstd_done(txn->zstd); brotli_done(txn->brotli); buf_free(&txn->req_body.payload); buf_free(&txn->resp_body.payload); buf_free(&txn->zbuf); buf_free(&txn->buf); } /* * Top-level command loop parsing */ static void cmdloop(struct http_connection *conn) { struct transaction_t txn; /* Start with an empty (clean) transaction */ memset(&txn, 0, sizeof(struct transaction_t)); txn.conn = conn; if (config_getswitch(IMAPOPT_HTTPALLOWCOMPRESS)) { txn.zstrm = zlib_init(); txn.zstd = zstd_init(); txn.brotli = brotli_init(); } /* Enable command timer */ cmdtime_settimer(1); /* Enable provisional responses for long-running mailbox ops */ mailbox_set_wait_cb((mailbox_wait_cb_t *) &keepalive_response, &txn); do { int ret = 0; /* Reset txn state */ transaction_reset(&txn); /* make sure nothing leaked */ assert(!open_mailboxes_exist()); assert(!open_mboxlocks_exist()); sync_log_reset(); /* Check for input from client */ do { /* Flush any buffered output */ prot_flush(httpd_out); if (backend_current) prot_flush(backend_current->out); /* Check for shutdown file */ if (shutdown_file(txn.buf.s, txn.buf.alloc) || (httpd_userid && userdeny(httpd_userid, config_ident, txn.buf.s, txn.buf.alloc))) { txn.error.desc = txn.buf.s; txn.flags.conn = CONN_CLOSE; ret = HTTP_SHUTDOWN; break; } signals_poll(); syslog(LOG_DEBUG, "proxy_check_input()"); } while (!proxy_check_input(protin, httpd_in, httpd_out, backend_current ? backend_current->in : NULL, NULL, 30)); /* Start command timer */ cmdtime_starttimer(); if (txn.conn->sess_ctx) { /* HTTP/2 input */ http2_input(&txn); } else if (txn.ws_ctx) { /* WebSocket over HTTP/1.1 input */ ws_input(&txn); } else if (!ret) { /* HTTP/1.x request */ http1_input(&txn); } if (ret == HTTP_SHUTDOWN) { syslog(LOG_WARNING, "Shutdown file: \"%s\", closing connection", txn.error.desc); protgroup_free(protin); shut_down(0); } } while (!(txn.flags.conn & CONN_CLOSE)); /* Memory cleanup */ transaction_free(&txn); } /**************************** Parsing Routines ******************************/ /* Parse URI, returning the path */ EXPORTED xmlURIPtr parse_uri(unsigned meth, const char *uri, unsigned path_reqd, const char **errstr) { xmlURIPtr p_uri; /* parsed URI */ /* Parse entire URI */ if ((p_uri = xmlParseURI(uri)) == NULL) { *errstr = "Illegal request target URI"; goto bad_request; } if (p_uri->scheme) { /* Check sanity of scheme */ if (strcasecmp(p_uri->scheme, "http") && strcasecmp(p_uri->scheme, "https")) { *errstr = "Unsupported URI scheme"; goto bad_request; } } /* Check sanity of path */ if (path_reqd && (!p_uri->path || !*p_uri->path)) { *errstr = "Empty path in target URI"; goto bad_request; } else if (p_uri->path) { size_t pathlen = strlen(p_uri->path); if ((p_uri->path[0] != '/') && (strcmp(p_uri->path, "*") || (meth != METH_OPTIONS))) { /* No special URLs except for "OPTIONS * HTTP/1.1" */ *errstr = "Illegal request target URI"; goto bad_request; } else if (strstr(p_uri->path, "/../")) { /* Don't allow access up directory tree */ *errstr = "Illegal request target URI"; goto bad_request; } else if (pathlen >= 3 && !strcmp("/..", p_uri->path + pathlen - 3)) { /* Don't allow access up directory tree */ *errstr = "Illegal request target URI"; goto bad_request; } else if (pathlen > MAX_MAILBOX_PATH) { *errstr = "Request target URI too long"; goto bad_request; } } return p_uri; bad_request: if (p_uri) xmlFreeURI(p_uri); return NULL; } /* Calculate compile time of a file for use as Last-Modified and/or ETag */ EXPORTED time_t calc_compile_time(const char *time, const char *date) { struct tm tm; char month[4]; memset(&tm, 0, sizeof(struct tm)); tm.tm_isdst = -1; sscanf(time, "%02d:%02d:%02d", &tm.tm_hour, &tm.tm_min, &tm.tm_sec); sscanf(date, "%3s %2d %4d", month, &tm.tm_mday, &tm.tm_year); tm.tm_year -= 1900; for (tm.tm_mon = 0; tm.tm_mon < 12; tm.tm_mon++) { if (!strcmp(month, monthname[tm.tm_mon])) break; } return mktime(&tm); } /* Parse Expect header(s) for interesting expectations */ static int parse_expect(struct transaction_t *txn) { const char **exp = spool_getheader(txn->req_hdrs, "Expect"); int i, ret = 0; /* Expect not supported by HTTP/1.0 clients */ if (exp && txn->flags.ver == VER_1_0) return HTTP_EXPECT_FAILED; /* Look for interesting expectations. Unknown == error */ for (i = 0; !ret && exp && exp[i]; i++) { tok_t tok = TOK_INITIALIZER(exp[i], ",", TOK_TRIMLEFT|TOK_TRIMRIGHT); char *token; while (!ret && (token = tok_next(&tok))) { /* Check if client wants acknowledgment before sending body */ if (!strcasecmp(token, "100-continue")) { syslog(LOG_DEBUG, "Expect: 100-continue"); txn->req_body.flags |= BODY_CONTINUE; } else { txn->error.desc = "Unsupported Expectation"; ret = HTTP_EXPECT_FAILED; } } tok_fini(&tok); } return ret; } /* Parse Connection header(s) for interesting options */ static int parse_connection(struct transaction_t *txn) { const char **conn = spool_getheader(txn->req_hdrs, "Connection"); int i; if (conn && txn->flags.ver == VER_2) { txn->error.desc = "Connection not allowed in HTTP/2"; return HTTP_BAD_REQUEST; } if (!httpd_timeout || txn->flags.ver == VER_1_0) { /* Non-persistent connection by default */ txn->flags.conn |= CONN_CLOSE; } /* Look for interesting connection tokens */ for (i = 0; conn && conn[i]; i++) { tok_t tok = TOK_INITIALIZER(conn[i], ",", TOK_TRIMLEFT|TOK_TRIMRIGHT); char *token; while ((token = tok_next(&tok))) { switch (txn->flags.ver) { case VER_1_1: if (!strcasecmp(token, "Upgrade")) { /* Client wants to upgrade */ const char **upgrade = spool_getheader(txn->req_hdrs, "Upgrade"); if (upgrade && upgrade[0]) { if (!txn->conn->tls_ctx && tls_enabled() && !strncasecmp(upgrade[0], TLS_VERSION, strcspn(upgrade[0], " ,"))) { /* Upgrade to TLS */ txn->flags.conn |= CONN_UPGRADE; txn->flags.upgrade |= UPGRADE_TLS; } else if (http2_enabled() && !strncasecmp(upgrade[0], NGHTTP2_CLEARTEXT_PROTO_VERSION_ID, strcspn(upgrade[0], " ,"))) { /* Upgrade to HTTP/2 */ txn->flags.conn |= CONN_UPGRADE; txn->flags.upgrade |= UPGRADE_HTTP2; } else if (ws_enabled() && !strncasecmp(upgrade[0], WS_TOKEN, strcspn(upgrade[0], " ,"))) { /* Upgrade to WebSockets */ txn->flags.conn |= CONN_UPGRADE; txn->flags.upgrade |= UPGRADE_WS; } else { /* Unknown/unsupported protocol - no upgrade */ } } } else if (!strcasecmp(token, "close")) { /* Non-persistent connection */ txn->flags.conn |= CONN_CLOSE; } break; case VER_1_0: if (httpd_timeout && !strcasecmp(token, "keep-alive")) { /* Persistent connection */ txn->flags.conn = CONN_KEEPALIVE; } break; } } tok_fini(&tok); } return 0; } /* Compare accept quality values so that they sort in descending order */ static int compare_accept(const struct accept *a1, const struct accept *a2) { if (a2->qual < a1->qual) return -1; if (a2->qual > a1->qual) return 1; return 0; } struct accept *parse_accept(const char **hdr) { int i, n = 0, alloc = 0; struct accept *ret = NULL; #define GROW_ACCEPT 10; for (i = 0; hdr && hdr[i]; i++) { tok_t tok = TOK_INITIALIZER(hdr[i], ";,", TOK_TRIMLEFT|TOK_TRIMRIGHT); char *token; while ((token = tok_next(&tok))) { if (!strncmp(token, "q=", 2)) { if (!ret) break; ret[n-1].qual = strtof(token+2, NULL); } else { if (n + 1 >= alloc) { alloc += GROW_ACCEPT; ret = xrealloc(ret, alloc * sizeof(struct accept)); } ret[n].token = xstrdup(token); ret[n].qual = 1.0; ret[++n].token = NULL; } } tok_fini(&tok); } qsort(ret, n, sizeof(struct accept), (int (*)(const void *, const void *)) &compare_accept); return ret; } /* Parse the query string and add key/value pairs to hash table */ void parse_query_params(struct transaction_t *txn, const char *query) { tok_t tok; char *param; assert(!buf_len(&txn->buf)); /* Unescape buffer */ tok_init(&tok, query, "&", TOK_TRIMLEFT|TOK_TRIMRIGHT|TOK_EMPTY); while ((param = tok_next(&tok))) { struct strlist *vals; char *key, *value; size_t len; /* Split param into key and optional value */ key = param; value = strchr(param, '='); if (!value) value = ""; else *value++ = '\0'; len = strlen(value); buf_ensure(&txn->buf, len+1); vals = hash_lookup(key, &txn->req_qparams); appendstrlist(&vals, xmlURIUnescapeString(value, len, txn->buf.s)); hash_insert(key, vals, &txn->req_qparams); } tok_fini(&tok); buf_reset(&txn->buf); } /**************************** Response Routines *****************************/ /* Create HTTP-date ('buf' must be at least 30 characters) */ EXPORTED char *httpdate_gen(char *buf, size_t len, time_t t) { struct tm *tm = gmtime(&t); snprintf(buf, len, "%3s, %02d %3s %4d %02d:%02d:%02d GMT", wday[tm->tm_wday], tm->tm_mday, monthname[tm->tm_mon], tm->tm_year + 1900, tm->tm_hour, tm->tm_min, tm->tm_sec); return buf; } /* Create an HTTP Status-Line given response code */ EXPORTED const char *http_statusline(unsigned ver, long code) { static struct buf statline = BUF_INITIALIZER; if (ver == VER_2) buf_setcstr(&statline, HTTP2_VERSION); else { buf_setmap(&statline, HTTP_VERSION, HTTP_VERSION_LEN-1); buf_putc(&statline, ver + '0'); } buf_putc(&statline, ' '); buf_appendcstr(&statline, error_message(code)); return buf_cstring(&statline); } /* Output an HTTP response header. * 'code' specifies the HTTP Status-Code and Reason-Phrase. * 'txn' contains the transaction context */ EXPORTED void simple_hdr(struct transaction_t *txn, const char *name, const char *value, ...) { struct buf buf = BUF_INITIALIZER; va_list args; va_start(args, value); buf_vprintf(&buf, value, args); va_end(args); syslog(LOG_DEBUG, "simple_hdr(%s: %s)", name, buf_cstring(&buf)); if (txn->flags.ver == VER_2) { http2_add_header(txn, name, &buf); } else { prot_printf(txn->conn->pout, "%c%s: ", toupper(name[0]), name+1); prot_puts(txn->conn->pout, buf_cstring(&buf)); prot_puts(txn->conn->pout, "\r\n"); buf_free(&buf); } } #define WWW_Authenticate(name, param) \ simple_hdr(txn, "WWW-Authenticate", param ? "%s %s" : "%s", name, param) #define Access_Control_Expose(hdr) \ simple_hdr(txn, "Access-Control-Expose-Headers", hdr) static void comma_list_body(struct buf *buf, const char *vals[], unsigned flags, int has_args, va_list args) { const char *sep = ""; int i; for (i = 0; vals[i]; i++) { if (flags & (1 << i)) { buf_appendcstr(buf, sep); if (has_args) buf_vprintf(buf, vals[i], args); else buf_appendcstr(buf, vals[i]); sep = ", "; } else if (has_args) { /* discard any unused args */ vsnprintf(NULL, 0, vals[i], args); } } } EXPORTED void comma_list_hdr(struct transaction_t *txn, const char *name, const char *vals[], unsigned flags, ...) { struct buf buf = BUF_INITIALIZER; va_list args; va_start(args, flags); comma_list_body(&buf, vals, flags, 1, args); va_end(args); simple_hdr(txn, name, "%s", buf_cstring(&buf)); buf_free(&buf); } EXPORTED void list_auth_schemes(struct transaction_t *txn) { struct auth_challenge_t *auth_chal = &txn->auth_chal; unsigned conn_close = (txn->flags.conn & CONN_CLOSE); struct auth_scheme_t *scheme; /* Advertise available schemes that can work with the type of connection */ for (scheme = auth_schemes; scheme->name; scheme++) { if ((avail_auth_schemes & scheme->id) && !(conn_close && (scheme->flags & AUTH_NEED_PERSIST))) { auth_chal->param = NULL; if (scheme->flags & AUTH_SERVER_FIRST) { /* Generate the initial challenge */ http_auth(scheme->name, txn); if (!auth_chal->param) continue; /* If fail, skip it */ } WWW_Authenticate(scheme->name, auth_chal->param); } } } EXPORTED void allow_hdr(struct transaction_t *txn, const char *name, unsigned allow) { const char *meths[] = { "OPTIONS, GET, HEAD", "POST", "PUT", "PATCH", "DELETE", "TRACE", "CONNECT", NULL }; comma_list_hdr(txn, name, meths, allow); if (allow & ALLOW_DAV) { simple_hdr(txn, name, "PROPFIND, REPORT, COPY%s%s%s%s%s", (allow & ALLOW_DELETE) ? ", MOVE" : "", (allow & ALLOW_PROPPATCH) ? ", PROPPATCH" : "", (allow & ALLOW_MKCOL) ? ", MKCOL" : "", (allow & ALLOW_WRITE) ? ", LOCK, UNLOCK" : "", (allow & ALLOW_ACL) ? ", ACL" : ""); if ((allow & ALLOW_CAL) && (allow & ALLOW_MKCOL)) simple_hdr(txn, name, "MKCALENDAR"); } } EXPORTED void accept_patch_hdr(struct transaction_t *txn, const struct patch_doc_t *patch) { struct buf buf = BUF_INITIALIZER; const char *sep = ""; int i; for (i = 0; patch[i].format; i++) { buf_appendcstr(&buf, sep); buf_appendcstr(&buf, patch[i].format); sep = ", "; } simple_hdr(txn, "Accept-Patch", "%s", buf_cstring(&buf)); buf_free(&buf); } #define MD5_BASE64_LEN 25 /* ((MD5_DIGEST_LENGTH / 3) + 1) * 4 */ EXPORTED void content_md5_hdr(struct transaction_t *txn, const unsigned char *md5) { char base64[MD5_BASE64_LEN+1]; sasl_encode64((char *) md5, MD5_DIGEST_LENGTH, base64, MD5_BASE64_LEN, NULL); simple_hdr(txn, "Content-MD5", "%s", base64); } EXPORTED void begin_resp_headers(struct transaction_t *txn, long code) { if (txn->flags.ver == VER_2) { http2_begin_headers(txn); if (code) simple_hdr(txn, ":status", "%.3s", error_message(code)); } else if (code) prot_printf(txn->conn->pout, "%s\r\n", http_statusline(txn->flags.ver, code)); } EXPORTED int end_resp_headers(struct transaction_t *txn, long code) { int r = 0; if (txn->flags.ver == VER_2) { r = http2_end_headers(txn, code); } else { /* CRLF terminating the header block */ prot_puts(txn->conn->pout, "\r\n"); } return r; } /* Write end-to-end header (ignoring hop-by-hop) from cache to protstream. */ static void write_cachehdr(const char *name, const char *contents, const char *raw __attribute__((unused)), void *rock) { struct transaction_t *txn = (struct transaction_t *) rock; const char **hdr, *hop_by_hop[] = { "connection", "content-length", "content-type", "date", "forwarded", "keep-alive", "location", "status", "strict-transport-security", "upgrade", "via", NULL }; /* Ignore private headers in our cache */ if (name[0] == ':') return; for (hdr = hop_by_hop; *hdr; hdr++) { if (!strcasecmp(name, *hdr)) return; } simple_hdr(txn, name, "%s", contents); } EXPORTED void response_header(long code, struct transaction_t *txn) { int i, size; time_t now; char datestr[30]; va_list noargs; double cmdtime, nettime; const char **hdr, *sep; struct auth_challenge_t *auth_chal = &txn->auth_chal; struct resp_body_t *resp_body = &txn->resp_body; struct buf *logbuf = &txn->conn->logbuf; const char *upgrd_tokens[] = { TLS_VERSION, NGHTTP2_CLEARTEXT_PROTO_VERSION_ID, WS_TOKEN, NULL }; const char *te[] = { "deflate", "gzip", "chunked", NULL }; const char *ce[] = { "deflate", "gzip", "br", "zstd", NULL }; /* Stop method processing alarm */ alarm(0); gotsigalrm = 0; /* Status-Line */ begin_resp_headers(txn, code); switch (code) { default: /* Final response */ now = time(0); httpdate_gen(datestr, sizeof(datestr), now); simple_hdr(txn, "Date", "%s", datestr); /* Fall through and specify connection options and/or links */ GCC_FALLTHROUGH case HTTP_SWITCH_PROT: if (txn->flags.conn && (txn->flags.ver < VER_2)) { /* Construct Connection header */ const char *conn_tokens[] = { "close", "Upgrade", "Keep-Alive", NULL }; comma_list_hdr(txn, "Connection", conn_tokens, txn->flags.conn); if (txn->flags.upgrade) { /* Construct Upgrade header */ comma_list_hdr(txn, "Upgrade", upgrd_tokens, txn->flags.upgrade); if (txn->flags.upgrade & UPGRADE_WS) { /* Add WebSocket headers */ ws_add_resp_hdrs(txn); } } if (txn->flags.conn & CONN_KEEPALIVE) { simple_hdr(txn, "Keep-Alive", "timeout=%d", httpd_timeout); } } /* Fall through and specify links */ GCC_FALLTHROUGH case HTTP_EARLY_HINTS: size = strarray_size(&resp_body->links); for (i = 0; i < size; i++) { simple_hdr(txn, "Link", "%s", strarray_nth(&resp_body->links, i)); } if (code >= HTTP_OK) break; /* Fall through as provisional response */ GCC_FALLTHROUGH case HTTP_CONTINUE: case HTTP_PROCESSING: /* Provisional response - nothing else needed */ end_resp_headers(txn, code); /* Force the response to the client immediately */ prot_flush(httpd_out); /* Restart method processing alarm (HTTP/1.1 only) */ if (!txn->ws_ctx && (txn->flags.ver == VER_1_1)) alarm(httpd_keepalive); goto log; } /* Control Data */ if (txn->conn->tls_ctx) { simple_hdr(txn, "Strict-Transport-Security", "max-age=600"); } if (txn->location) { simple_hdr(txn, "Location", "%s", txn->location); } if (txn->flags.mime) { simple_hdr(txn, "MIME-Version", "1.0"); } if (txn->flags.cc) { /* Construct Cache-Control header */ const char *cc_dirs[] = { "must-revalidate", "no-cache", "no-store", "no-transform", "public", "private", "max-age=%d", "immutable", NULL }; comma_list_hdr(txn, "Cache-Control", cc_dirs, txn->flags.cc, resp_body->maxage); if (txn->flags.cc & CC_MAXAGE) { httpdate_gen(datestr, sizeof(datestr), now + resp_body->maxage); simple_hdr(txn, "Expires", "%s", datestr); } } if (txn->flags.cors) { /* Construct Cross-Origin Resource Sharing headers */ simple_hdr(txn, "Access-Control-Allow-Origin", "%s", *spool_getheader(txn->req_hdrs, "Origin")); simple_hdr(txn, "Access-Control-Allow-Credentials", "true"); if (txn->flags.cors == CORS_PREFLIGHT) { allow_hdr(txn, "Access-Control-Allow-Methods", txn->req_tgt.allow); for (hdr = spool_getheader(txn->req_hdrs, "Access-Control-Request-Headers"); hdr && *hdr; hdr++) { simple_hdr(txn, "Access-Control-Allow-Headers", "%s", *hdr); } simple_hdr(txn, "Access-Control-Max-Age", "3600"); } } if (txn->flags.vary && !(txn->flags.cc & CC_NOCACHE)) { /* Construct Vary header */ const char *vary_hdrs[] = { "Accept", "Accept-Encoding", "Brief", "Prefer", "If-None-Match", "CalDAV-Timezones", NULL }; comma_list_hdr(txn, "Vary", vary_hdrs, txn->flags.vary); } /* Authentication Challenges */ if (code == HTTP_UNAUTHORIZED) { if (!auth_chal->scheme) { /* Require authentication by advertising all available schemes */ list_auth_schemes(txn); } else { /* Continue with current authentication exchange */ WWW_Authenticate(auth_chal->scheme->name, auth_chal->param); } } else if (auth_chal->param) { /* Authentication completed with success data */ if (auth_chal->scheme->flags & AUTH_SUCCESS_WWW) { /* Special handling of success data for this scheme */ WWW_Authenticate(auth_chal->scheme->name, auth_chal->param); } else { /* Default handling of success data */ simple_hdr(txn, "Authentication-Info", "%s", auth_chal->param); } } /* Response Context */ if (txn->req_tgt.allow & ALLOW_ISCHEDULE) { simple_hdr(txn, "iSchedule-Version", "1.0"); if (resp_body->iserial) { simple_hdr(txn, "iSchedule-Capabilities", TIME_T_FMT, resp_body->iserial); } } if (resp_body->patch) { accept_patch_hdr(txn, resp_body->patch); } switch (code) { case HTTP_OK: switch (txn->meth) { case METH_CONNECT: if (txn->ws_ctx) { /* Add WebSocket headers */ ws_add_resp_hdrs(txn); } break; case METH_GET: case METH_HEAD: /* Construct Accept-Ranges header for GET and HEAD responses */ simple_hdr(txn, "Accept-Ranges", txn->flags.ranges ? "bytes" : "none"); break; case METH_OPTIONS: if (config_serverinfo == IMAP_ENUM_SERVERINFO_ON) { simple_hdr(txn, "Server", "%s", buf_cstring(&serverinfo)); } if (!httpd_userid && !auth_chal->scheme) { /* Advertise all available auth schemes */ list_auth_schemes(txn); } if (txn->req_tgt.allow & ALLOW_DAV) { /* Construct DAV header(s) based on namespace of request URL */ simple_hdr(txn, "DAV", "1, 2, 3, access-control," " extended-mkcol, resource-sharing"); if (txn->req_tgt.allow & ALLOW_CAL) { simple_hdr(txn, "DAV", "calendar-access%s%s", (txn->req_tgt.allow & ALLOW_CAL_SCHED) ? ", calendar-auto-schedule" : "", (txn->req_tgt.allow & ALLOW_CAL_NOTZ) ? ", calendar-no-timezone" : ""); simple_hdr(txn, "DAV", "calendar-query-extended%s%s", (txn->req_tgt.allow & ALLOW_CAL_AVAIL) ? ", calendar-availability" : "", (txn->req_tgt.allow & ALLOW_CAL_ATTACH) ? ", calendar-managed-attachments" : ""); /* Backwards compatibility with older Apple clients */ simple_hdr(txn, "DAV", "calendarserver-sharing%s", (txn->req_tgt.allow & (ALLOW_CAL_AVAIL | ALLOW_CAL_SCHED)) == (ALLOW_CAL_AVAIL | ALLOW_CAL_SCHED) ? ", inbox-availability" : ""); } if (txn->req_tgt.allow & ALLOW_CARD) { simple_hdr(txn, "DAV", "addressbook"); } } /* Access-Control-Allow-Methods supersedes Allow */ if (txn->flags.cors != CORS_PREFLIGHT) { /* Construct Allow header(s) */ allow_hdr(txn, "Allow", txn->req_tgt.allow); } break; } /* Fall through and specify supported content codings */ GCC_FALLTHROUGH case HTTP_CREATED: case HTTP_ACCEPTED: case HTTP_NO_CONTENT: case HTTP_RESET_CONTENT: case HTTP_PARTIAL: case HTTP_MULTI_STATUS: if (accept_encodings && buf_len(&txn->req_body.payload)) { comma_list_hdr(txn, "Accept-Encoding", ce, accept_encodings); } break; case HTTP_NOT_ALLOWED: /* Construct Allow header(s) for 405 response */ allow_hdr(txn, "Allow", txn->req_tgt.allow); break; case HTTP_BAD_CE: /* Construct Accept-Encoding header for 415 response */ if (accept_encodings) { comma_list_hdr(txn, "Accept-Encoding", ce, accept_encodings); } else simple_hdr(txn, "Accept-Encoding", "identity"); break; } /* Validators */ if (resp_body->lock) { simple_hdr(txn, "Lock-Token", "<%s>", resp_body->lock); if (txn->flags.cors) Access_Control_Expose("Lock-Token"); } if (resp_body->ctag) { simple_hdr(txn, "CTag", "%s", resp_body->ctag); if (txn->flags.cors) Access_Control_Expose("CTag"); } if (resp_body->stag) { simple_hdr(txn, "Schedule-Tag", "\"%s\"", resp_body->stag); if (txn->flags.cors) Access_Control_Expose("Schedule-Tag"); } if (resp_body->etag) { simple_hdr(txn, "ETag", "%s\"%s\"", resp_body->enc.proc ? "W/" : "", resp_body->etag); if (txn->flags.cors) Access_Control_Expose("ETag"); } if (resp_body->lastmod) { /* Last-Modified MUST NOT be in the future */ resp_body->lastmod = MIN(resp_body->lastmod, now); httpdate_gen(datestr, sizeof(datestr), resp_body->lastmod); simple_hdr(txn, "Last-Modified", "%s", datestr); } /* Representation Metadata */ if (resp_body->prefs) { /* Construct Preference-Applied header */ const char *prefs[] = { "return=minimal", "return=representation", "depth-noroot", NULL }; comma_list_hdr(txn, "Preference-Applied", prefs, resp_body->prefs); if (txn->flags.cors) Access_Control_Expose("Preference-Applied"); } if (resp_body->cmid) { simple_hdr(txn, "Cal-Managed-ID", "%s", resp_body->cmid); if (txn->flags.cors) Access_Control_Expose("Cal-Managed-ID"); } if (resp_body->type) { simple_hdr(txn, "Content-Type", "%s", resp_body->type); if (resp_body->dispo.fname) { /* Construct Content-Disposition header */ const unsigned char *p = (const unsigned char *)resp_body->dispo.fname; char *encfname = NULL; for (p = (unsigned char *)resp_body->dispo.fname; p && *p; p++) { if (*p >= 0x80) { encfname = charset_encode_mimexvalue(resp_body->dispo.fname, NULL); break; } } if (encfname) { simple_hdr(txn, "Content-Disposition", "%s; filename*=%s", resp_body->dispo.attach ? "attachment" : "inline", encfname); } else { simple_hdr(txn, "Content-Disposition", "%s; filename=\"%s\"", resp_body->dispo.attach ? "attachment" : "inline", resp_body->dispo.fname); } free(encfname); } if (txn->resp_body.enc.proc) { /* Construct Content-Encoding header */ comma_list_hdr(txn, "Content-Encoding", ce, txn->resp_body.enc.type); } if (resp_body->lang) { simple_hdr(txn, "Content-Language", "%s", resp_body->lang); } if (resp_body->loc) { xmlChar *uri = xmlURIEscapeStr(BAD_CAST resp_body->loc, BAD_CAST ":/?="); simple_hdr(txn, "Content-Location", "%s", (const char *) uri); free(uri); if (txn->flags.cors) Access_Control_Expose("Content-Location"); } if (resp_body->md5) { content_md5_hdr(txn, resp_body->md5); } } /* Payload */ switch (code) { case HTTP_NO_CONTENT: case HTTP_NOT_MODIFIED: /* MUST NOT include a body */ resp_body->len = 0; break; case HTTP_PARTIAL: case HTTP_BAD_RANGE: if (resp_body->range) { simple_hdr(txn, "Content-Range", "bytes %lu-%lu/%lu", resp_body->range->first, resp_body->range->last, resp_body->len); /* Set actual content length of range */ resp_body->len = resp_body->range->last - resp_body->range->first + 1; free(resp_body->range); } else { simple_hdr(txn, "Content-Range", "bytes */%lu", resp_body->len); resp_body->len = 0; /* No content */ } /* Fall through and specify framing */ GCC_FALLTHROUGH default: if (txn->flags.te) { /* HTTP/1.1 only - we use close-delimiting for HTTP/1.0 */ if (txn->flags.ver == VER_1_1) { /* Construct Transfer-Encoding header */ comma_list_hdr(txn, "Transfer-Encoding", te, txn->flags.te); } if (txn->flags.trailer & ~TRAILER_PROXY) { /* Construct Trailer header */ const char *trailer_hdrs[] = { "Content-MD5", "CTag", NULL }; comma_list_hdr(txn, "Trailer", trailer_hdrs, txn->flags.trailer); } } else { /* Content-Length */ switch (txn->meth) { case METH_CONNECT: /* MUST NOT include per Section 4.3.6 of RFC 7231 */ break; case METH_HEAD: if (!resp_body->len) { /* We don't know if the length is zero or if it wasn't set - MUST NOT include if it doesn't match what would be returned for GET, per Section 3.3.2 of RFC 7231 */ break; } GCC_FALLTHROUGH default: simple_hdr(txn, "Content-Length", "%lu", resp_body->len); break; } } } /* Extra headers */ if (resp_body->extra_hdrs) { spool_enum_hdrcache(resp_body->extra_hdrs, &write_cachehdr, txn); } /* End of headers */ end_resp_headers(txn, code); log: /* Log the client request and our response */ buf_reset(logbuf); /* Add client data */ buf_printf(logbuf, "%s", txn->conn->clienthost); if (httpd_userid) buf_printf(logbuf, " as \"%s\"", httpd_userid); if (txn->req_hdrs && (hdr = spool_getheader(txn->req_hdrs, "User-Agent"))) { buf_printf(logbuf, " with \"%s\"", hdr[0]); if ((hdr = spool_getheader(txn->req_hdrs, "X-Client"))) buf_printf(logbuf, " by \"%s\"", hdr[0]); else if ((hdr = spool_getheader(txn->req_hdrs, "X-Requested-With"))) buf_printf(logbuf, " by \"%s\"", hdr[0]); } /* Add session id */ buf_printf(logbuf, " via SESSIONID=<%s>", session_id()); /* Add request-line */ buf_appendcstr(logbuf, "; \""); if (txn->req_line.meth) { buf_printf(logbuf, "%s", txn->flags.override ? "POST" : txn->req_line.meth); if (txn->req_line.uri) { buf_printf(logbuf, " %s", txn->req_line.uri); if (txn->req_line.ver) { buf_printf(logbuf, " %s", txn->req_line.ver); if (code != HTTP_URI_TOO_LONG && *txn->req_line.buf) { const char *p = txn->req_line.ver + strlen(txn->req_line.ver) + 1; if (*p) buf_printf(logbuf, " %s", p); } } } } buf_appendcstr(logbuf, "\""); if (txn->req_hdrs) { /* Add any request modifying headers */ sep = " ("; if (txn->flags.override) { buf_printf(logbuf, "%smethod-override=%s", sep, txn->req_line.meth); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "Origin"))) { buf_printf(logbuf, "%sorigin=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "Referer"))) { buf_printf(logbuf, "%sreferer=%s", sep, hdr[0]); sep = "; "; } if (txn->flags.upgrade && (hdr = spool_getheader(txn->req_hdrs, "Upgrade"))) { buf_printf(logbuf, "%supgrade=%s", sep, hdr[0]); sep = "; "; } if (code == HTTP_CONTINUE || code == HTTP_EXPECT_FAILED) { hdr = spool_getheader(txn->req_hdrs, "Expect"); buf_printf(logbuf, "%sexpect=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "Transfer-Encoding"))) { buf_printf(logbuf, "%stx-encoding=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "Content-Encoding"))) { buf_printf(logbuf, "%scnt-encoding=%s", sep, hdr[0]); sep = "; "; } if (txn->auth_chal.scheme) { buf_printf(logbuf, "%sauth=%s", sep, txn->auth_chal.scheme->name); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "Destination"))) { buf_printf(logbuf, "%sdestination=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "Lock-Token"))) { buf_printf(logbuf, "%slock-token=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "If"))) { buf_printf(logbuf, "%sif=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "If-Schedule-Tag-Match"))) { buf_printf(logbuf, "%sif-schedule-tag-match=%s", sep, hdr[0]); sep = "; "; } else if ((hdr = spool_getheader(txn->req_hdrs, "If-Match"))) { buf_printf(logbuf, "%sif-match=%s", sep, hdr[0]); sep = "; "; } else if ((hdr = spool_getheader(txn->req_hdrs, "If-Unmodified-Since"))) { buf_printf(logbuf, "%sif-unmodified-since=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "If-None-Match"))) { buf_printf(logbuf, "%sif-none-match=%s", sep, hdr[0]); sep = "; "; } else if ((hdr = spool_getheader(txn->req_hdrs, "If-Modified-Since"))) { buf_printf(logbuf, "%sif-modified-since=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, ":type"))) { buf_printf(logbuf, "%stype=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, ":token"))) { buf_printf(logbuf, "%stoken=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, ":jmap"))) { buf_printf(logbuf, "%sjmap=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, ":dblookup"))) { buf_printf(logbuf, "%slookup=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "Depth"))) { buf_printf(logbuf, "%sdepth=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "Prefer"))) { buf_printf(logbuf, "%sprefer=%s", sep, hdr[0]); sep = "; "; } else if ((hdr = spool_getheader(txn->req_hdrs, "Brief"))) { buf_printf(logbuf, "%sbrief=%s", sep, hdr[0]); sep = "; "; } if ((hdr = spool_getheader(txn->req_hdrs, "CalDAV-Timezones"))) { buf_printf(logbuf, "%scaldav-timezones=%s", sep, hdr[0]); sep = "; "; } /* Add httplogheaders */ size = strarray_size(httpd_log_headers); for (i = 0; i < size; i++) { const char *name = strarray_nth(httpd_log_headers, i); if ((hdr = spool_getheader(txn->req_hdrs, name))) { buf_printf(logbuf, "%s%s=\"%s\"", sep, name, hdr[0]); sep = "; "; } } if (*sep == ';') buf_appendcstr(logbuf, ")"); } if (txn->flags.redirect) { /* Add CGI local redirect */ buf_printf(logbuf, " => \"%s %s %s\"", txn->req_line.meth, txn->req_tgt.path, txn->req_line.ver); } /* Add response */ buf_printf(logbuf, " => \"%s\"", http_statusline(txn->flags.ver, code)); /* Add any auxiliary response data */ sep = " ("; if (txn->strm_ctx) { buf_printf(logbuf, "%sstream-id=%d", sep, http2_get_streamid(txn->strm_ctx)); sep = "; "; } if (code == HTTP_SWITCH_PROT || code == HTTP_UPGRADE) { buf_printf(logbuf, "%supgrade=", sep); comma_list_body(logbuf, upgrd_tokens, txn->flags.upgrade, 0, noargs); sep = "; "; } if (txn->flags.te) { buf_printf(logbuf, "%stx-encoding=", sep); comma_list_body(logbuf, te, txn->flags.te, 0, noargs); sep = "; "; } if (resp_body->enc.proc && (resp_body->len || txn->flags.te)) { buf_printf(logbuf, "%scnt-encoding=", sep); comma_list_body(logbuf, ce, resp_body->enc.type, 0, noargs); sep = "; "; } if (txn->location) { buf_printf(logbuf, "%slocation=%s", sep, txn->location); sep = "; "; } else if (txn->flags.cors) { buf_printf(logbuf, "%sallow-origin", sep); sep = "; "; } else if (txn->error.desc) { buf_printf(logbuf, "%serror=%s", sep, txn->error.desc); sep = "; "; } if (*sep == ';') buf_appendcstr(logbuf, ")"); /* Add timing stats */ cmdtime_endtimer(&cmdtime, &nettime); buf_printf(logbuf, " [timing: cmd=%f net=%f total=%f]", cmdtime, nettime, cmdtime + nettime); syslog(LOG_INFO, "%s", buf_cstring(logbuf)); } #ifdef HAVE_DECLARE_OPTIMIZE EXPORTED inline void keepalive_response(struct transaction_t *txn) __attribute__((always_inline, optimize("-O3"))); #endif EXPORTED void keepalive_response(struct transaction_t *txn) { if (gotsigalrm) { response_header(HTTP_PROCESSING, txn); } } /* * Output an HTTP response with multipart body data. * * An initial call with 'code' != 0 will output a response header * and the preamble. * * All subsequent calls should have 'code' = 0 to output just a body part. * A body part may include custom headers (exluding Content-Type and Length), * which must be properly folded and must end with CRLF. * * A final call with 'len' = 0 ends the multipart body. */ EXPORTED void write_multipart_body(long code, struct transaction_t *txn, const char *buf, unsigned len, const char *part_headers) { static char boundary[100]; struct buf *body = &txn->resp_body.payload; if (code) { const char *preamble = "This is a message with multiple parts in MIME format.\r\n"; txn->flags.mime = 1; /* Create multipart boundary */ snprintf(boundary, sizeof(boundary), "%s-%ld-%ld-%ld", *spool_getheader(txn->req_hdrs, ":authority"), (long) getpid(), (long) time(0), (long) rand()); /* Create Content-Type w/ boundary */ assert(!buf_len(&txn->buf)); buf_printf(&txn->buf, "%s; boundary=\"%s\"", txn->resp_body.type, boundary); txn->resp_body.type = buf_cstring(&txn->buf); /* Setup for chunked response and begin multipart */ txn->flags.te |= TE_CHUNKED; if (!buf) { buf = preamble; len = strlen(preamble); } write_body(code, txn, buf, len); } else if (len) { /* Output delimiter and MIME part-headers */ buf_reset(body); buf_printf(body, "\r\n--%s\r\n", boundary); buf_printf(body, "Content-Type: %s\r\n", txn->resp_body.type); if (txn->resp_body.range) { buf_printf(body, "Content-Range: bytes %lu-%lu/%lu\r\n", txn->resp_body.range->first, txn->resp_body.range->last, txn->resp_body.len); } buf_printf(body, "Content-Length: %d\r\n", len); if (part_headers) { buf_appendcstr(body, part_headers); } buf_appendcstr(body, "\r\n"); write_body(0, txn, buf_cstring(body), buf_len(body)); /* Output body-part data */ write_body(0, txn, buf, len); } else { const char *epilogue = "\r\nEnd of MIME multipart body.\r\n"; /* Output close-delimiter and epilogue */ buf_reset(body); buf_printf(body, "\r\n--%s--\r\n%s", boundary, epilogue); write_body(0, txn, buf_cstring(body), buf_len(body)); /* End of output */ write_body(0, txn, NULL, 0); } } /* Output multipart/byteranges */ static void multipart_byteranges(struct transaction_t *txn, const char *msg_base) { /* Save Content-Range and Content-Type pointers */ struct range *range = txn->resp_body.range; const char *type = txn->resp_body.type; /* Start multipart response */ txn->resp_body.range = NULL; txn->resp_body.type = "multipart/byteranges"; write_multipart_body(HTTP_PARTIAL, txn, NULL, 0, NULL); txn->resp_body.type = type; while (range) { unsigned long offset = range->first; unsigned long datalen = range->last - range->first + 1; struct range *next = range->next; /* Output range as body part */ txn->resp_body.range = range; write_multipart_body(0, txn, msg_base + offset, datalen, NULL); /* Cleanup */ free(range); range = next; } /* End of multipart body */ write_multipart_body(0, txn, NULL, 0, NULL); } /* * Output an HTTP response with body data, compressed as necessary. * * For chunked body data, an initial call with 'code' != 0 will output * a response header and the first body chunk. * All subsequent calls should have 'code' = 0 to output just the body chunk. * A final call with 'len' = 0 ends the chunked body. * * NOTE: HTTP/1.0 clients can't handle chunked encoding, * so we use bare chunks and close the connection when done. */ EXPORTED void write_body(long code, struct transaction_t *txn, const char *buf, unsigned len) { unsigned outlen = len, offset = 0, last_chunk; int do_md5 = (txn->meth == METH_HEAD) ? 0 : config_getswitch(IMAPOPT_HTTPCONTENTMD5); static MD5_CTX ctx; static unsigned char md5[MD5_DIGEST_LENGTH]; syslog(LOG_DEBUG, "write_body(code = %ld, flags.te = %#x, len = %u)", code, txn->flags.te, len); if (txn->flags.te & TE_CHUNKED) last_chunk = !(code || len); else { /* Handle static content as last chunk */ last_chunk = 1; if (len < GZIP_MIN_LEN) { /* Don't compress small static content */ txn->resp_body.enc.type = CE_IDENTITY; txn->resp_body.enc.proc = NULL; txn->flags.te = TE_NONE; } } /* Compress data */ if (txn->resp_body.enc.proc || txn->flags.te & ~TE_CHUNKED) { unsigned flags = 0; if (code) flags |= COMPRESS_START; if (last_chunk) flags |= COMPRESS_END; if (txn->resp_body.enc.proc(txn, flags, buf, len) < 0) { fatal("Error while compressing data", EX_SOFTWARE); } buf = txn->zbuf.s; outlen = txn->zbuf.len; } if (code) { /* Initial call - prepare response header based on CE, TE and version */ if (do_md5) MD5Init(&ctx); if (txn->flags.te & ~TE_CHUNKED) { /* Transfer-Encoded content MUST be chunked */ txn->flags.te |= TE_CHUNKED; } if (!txn->flags.te) { /* Full/partial body (no encoding). * * In all cases, 'resp_body.len' is used to specify complete-length * In the case of a 206 or 416 response, Content-Length will be * set accordingly in response_header(). */ txn->resp_body.len = outlen; if (code == HTTP_PARTIAL) { /* check_precond() tells us that this is a range request */ code = parse_ranges(*spool_getheader(txn->req_hdrs, "Range"), outlen, &txn->resp_body.range); switch (code) { case HTTP_OK: /* Full body (unknown range-unit) */ break; case HTTP_PARTIAL: /* One or more range request(s) */ txn->resp_body.len = outlen; if (txn->resp_body.range->next) { /* Multiple ranges */ multipart_byteranges(txn, buf); return; } else { /* Single range - set data parameters accordingly */ offset += txn->resp_body.range->first; outlen = txn->resp_body.range->last - txn->resp_body.range->first + 1; } break; case HTTP_BAD_RANGE: /* No valid ranges */ outlen = 0; break; } } if (outlen && do_md5) { MD5Update(&ctx, buf+offset, outlen); MD5Final(md5, &ctx); txn->resp_body.md5 = md5; } } else if (txn->flags.ver == VER_1_0) { /* HTTP/1.0 doesn't support chunked - close-delimit the body */ txn->flags.conn = CONN_CLOSE; } else if (do_md5) txn->flags.trailer |= TRAILER_CMD5; response_header(code, txn); /* MUST NOT send a body for 1xx/204/304 response or any HEAD response */ switch (code) { case HTTP_CONTINUE: case HTTP_SWITCH_PROT: case HTTP_PROCESSING: case HTTP_NO_CONTENT: case HTTP_NOT_MODIFIED: return; default: if (txn->meth == METH_HEAD) return; } } /* Output data */ if (txn->flags.ver == VER_2) { /* HTTP/2 chunk */ if (outlen || txn->flags.te) { http2_data_chunk(txn, buf + offset, outlen, last_chunk, &ctx); } } else if (txn->flags.te && txn->flags.ver == VER_1_1) { /* HTTP/1.1 chunk */ if (outlen) { syslog(LOG_DEBUG, "write_body: chunk(%d)", outlen); prot_printf(httpd_out, "%x\r\n", outlen); prot_write(httpd_out, buf, outlen); prot_puts(httpd_out, "\r\n"); if (txn->flags.trailer & TRAILER_CMD5) MD5Update(&ctx, buf, outlen); } if (last_chunk) { /* Terminate the HTTP/1.1 body with a zero-length chunk */ syslog(LOG_DEBUG, "write_body: last chunk"); prot_puts(httpd_out, "0\r\n"); /* Trailer */ if (txn->flags.trailer & TRAILER_CMD5) { syslog(LOG_DEBUG, "write_body: trailer Content-MD5"); MD5Final(md5, &ctx); content_md5_hdr(txn, md5); } if ((txn->flags.trailer & TRAILER_CTAG) && txn->resp_body.ctag) { syslog(LOG_DEBUG, "write_body: trailer CTag"); simple_hdr(txn, "CTag", "%s", txn->resp_body.ctag); } if (txn->flags.trailer != TRAILER_PROXY) { syslog(LOG_DEBUG, "write_body: CRLF"); prot_puts(httpd_out, "\r\n"); } } } else { /* Full body or HTTP/1.0 close-delimited body */ prot_write(httpd_out, buf + offset, outlen); } } /* Output an HTTP response with application/xml body */ EXPORTED void xml_response(long code, struct transaction_t *txn, xmlDocPtr xml) { xmlChar *buf; int bufsiz; switch (code) { case HTTP_OK: case HTTP_CREATED: case HTTP_NO_CONTENT: case HTTP_MULTI_STATUS: break; default: /* Neither Brief nor Prefer affect error response bodies */ txn->flags.vary &= ~(VARY_BRIEF | VARY_PREFER); txn->resp_body.prefs = 0; } /* Dump XML response tree into a text buffer */ xmlDocDumpFormatMemoryEnc(xml, &buf, &bufsiz, "utf-8", config_httpprettytelemetry); if (buf) { if (txn->flags.te & TE_CHUNKED) { /* Start of XML chunked response */ xmlChar *cp; int n; /* Leave root element open */ for (cp = buf + --bufsiz, n = 0; *cp != '/'; cp--, n++); if (*(cp+1) == '>') memmove(cp, cp+1, n); /* */ else bufsiz -= n+1; /* */ } /* Output the XML response */ txn->resp_body.type = "application/xml; charset=utf-8"; write_body(code, txn, (char *) buf, bufsiz); /* Cleanup */ xmlFree(buf); } else { txn->error.precond = 0; txn->error.desc = "Error dumping XML tree\r\n"; error_response(HTTP_SERVER_ERROR, txn); } } /* Output a chunk of an XML response */ EXPORTED void xml_partial_response(struct transaction_t *txn, xmlDocPtr doc, xmlNodePtr node, unsigned level, xmlBufferPtr *buf) { const char *eol = "\n"; unsigned n; if (!config_httpprettytelemetry) { level = 0; eol = ""; } /* Start with clean buffer */ if (!*buf) *buf = xmlBufferCreate(); if (node) { /* Add leading indent to buffer */ for (n = 0; n < level * MARKUP_INDENT; n++) xmlBufferCCat(*buf, " "); /* Dump XML node into buffer */ xmlNodeDump(*buf, doc, node, level, config_httpprettytelemetry); /* Add trailing EOL to buffer */ xmlBufferCCat(*buf, eol); } else { /* End of chunked XML response */ xmlNodePtr root = xmlDocGetRootElement(doc); /* Add close of root element to buffer */ xmlBufferCCat(*buf, "ns->prefix) { xmlBufferCat(*buf, root->ns->prefix); xmlBufferCCat(*buf, ":"); } xmlBufferCat(*buf, root->name); xmlBufferCCat(*buf, ">"); /* Add trailing EOL to buffer */ xmlBufferCCat(*buf, eol); } if (txn) { /* Output the XML buffer */ write_body(0, txn, (char *) xmlBufferContent(*buf), xmlBufferLength(*buf)); /* Reset the buffer for next chunk */ xmlBufferEmpty(*buf); } } EXPORTED void buf_printf_markup(struct buf *buf, unsigned level, const char *fmt, ...) { va_list args; const char *eol = "\n"; if (!config_httpprettytelemetry) { level = 0; eol = ""; } va_start(args, fmt); buf_printf(buf, "%*s", level * MARKUP_INDENT, ""); buf_vprintf(buf, fmt, args); buf_appendcstr(buf, eol); va_end(args); } /* Output an HTTP error response with optional XML or HTML body */ EXPORTED void error_response(long code, struct transaction_t *txn) { struct buf *html = &txn->resp_body.payload; /* Neither Brief nor Prefer affect error response bodies */ txn->flags.vary &= ~(VARY_BRIEF | VARY_PREFER); txn->resp_body.prefs = 0; #ifdef WITH_DAV if (code != HTTP_UNAUTHORIZED && txn->error.precond) { xmlNodePtr root = xml_add_error(NULL, &txn->error, NULL); if (root) { xml_response(code, txn, root->doc); xmlFreeDoc(root->doc); return; } } #endif /* WITH_DAV */ if (!txn->error.desc) { switch (code) { /* 4xx codes */ case HTTP_BAD_REQUEST: txn->error.desc = "The request was not understood by this server."; break; case HTTP_NOT_FOUND: txn->error.desc = "The requested URL was not found on this server."; break; case HTTP_NOT_ALLOWED: txn->error.desc = "The requested method is not allowed for the URL."; break; case HTTP_GONE: txn->error.desc = "The requested URL has been removed from this server."; break; /* 5xx codes */ case HTTP_SERVER_ERROR: if (!txn->error.desc) txn->error.desc = "The server encountered an internal error."; break; case HTTP_NOT_IMPLEMENTED: txn->error.desc = "The requested method is not implemented by this server."; break; case HTTP_UNAVAILABLE: txn->error.desc = "The server is unable to process the request at this time."; break; } } if (txn->error.desc) { const char **hdr, *host = config_servername; char *port = NULL; unsigned level = 0; if (txn->req_hdrs && (hdr = spool_getheader(txn->req_hdrs, ":authority")) && hdr[0] && *hdr[0]) { host = (char *) hdr[0]; if ((port = strchr(host, ':'))) *port++ = '\0'; } if (!port) { port = (buf_len(&saslprops.iplocalport)) ? strchr(buf_cstring(&saslprops.iplocalport), ';')+1 : ""; } buf_printf_markup(html, level, HTML_DOCTYPE); buf_printf_markup(html, level++, ""); buf_printf_markup(html, level++, ""); buf_printf_markup(html, level, "%s", error_message(code)); buf_printf_markup(html, --level, ""); buf_printf_markup(html, level++, ""); buf_printf_markup(html, level, "

%s

", error_message(code)+4); buf_printf_markup(html, level, "

%s

", txn->error.desc); if (config_serverinfo) { buf_printf_markup(html, level, "
"); buf_printf_markup(html, level, "
%s Server at %s Port %s
", (config_serverinfo == IMAP_ENUM_SERVERINFO_ON) ? buf_cstring(&serverinfo) : "HTTP", host, port); } buf_printf_markup(html, --level, ""); buf_printf_markup(html, --level, ""); txn->resp_body.type = "text/html; charset=utf-8"; } write_body(code, txn, buf_cstring(html), buf_len(html)); } static int proxy_authz(const char **authzid, struct transaction_t *txn) { static char authzbuf[MAX_MAILBOX_BUFFER]; unsigned authzlen; int status; syslog(LOG_DEBUG, "proxy_auth: authzid='%s'", *authzid); /* Free userid & authstate previously allocated for auth'd user */ if (httpd_userid) { free(httpd_userid); httpd_userid = NULL; } if (httpd_extrafolder) { free(httpd_extrafolder); httpd_extrafolder = NULL; } if (httpd_extradomain) { free(httpd_extradomain); httpd_extradomain = NULL; } if (httpd_authstate) { auth_freestate(httpd_authstate); httpd_authstate = NULL; } if (!(config_mupdate_server && config_getstring(IMAPOPT_PROXYSERVERS))) { /* Not a backend in a Murder - proxy authz is not allowed */ syslog(LOG_NOTICE, "badlogin: %s %s %s %s", txn->conn->clienthost, txn->auth_chal.scheme->name, httpd_authid, "proxy authz attempted on non-Murder backend"); return SASL_NOAUTHZ; } /* Canonify the authzid */ status = mysasl_canon_user(httpd_saslconn, NULL, *authzid, strlen(*authzid), SASL_CU_AUTHZID, NULL, authzbuf, sizeof(authzbuf), &authzlen); if (status) { syslog(LOG_NOTICE, "badlogin: %s %s %s invalid user", txn->conn->clienthost, txn->auth_chal.scheme->name, beautify_string(*authzid)); return status; } /* See if auth'd user is allowed to proxy */ status = mysasl_proxy_policy(httpd_saslconn, &httpd_proxyctx, authzbuf, authzlen, httpd_authid, strlen(httpd_authid), NULL, 0, NULL); if (status) { syslog(LOG_NOTICE, "badlogin: %s %s %s %s", txn->conn->clienthost, txn->auth_chal.scheme->name, httpd_authid, sasl_errdetail(httpd_saslconn)); return status; } *authzid = authzbuf; return status; } /* Write cached header (redacting authorization credentials) to buffer. */ HIDDEN void log_cachehdr(const char *name, const char *contents, const char *raw, void *rock) { struct buf *buf = (struct buf *) rock; /* Ignore private headers in our cache */ if (name[0] == ':') return; if (!strcasecmp(name, "authorization") && strchr(contents, ' ')) { /* Replace authorization credentials with an ellipsis */ const char *creds = strchr(contents, ' ') + 1; buf_printf(buf, "%c%s: %.*s%-*s\r\n", toupper(name[0]), name+1, (int) (creds - contents), contents, (int) strlen(creds), "..."); } else if (raw) buf_appendcstr(buf, raw); else buf_printf(buf, "%c%s: %s\r\n", toupper(name[0]), name+1, contents); } static int auth_success(struct transaction_t *txn, const char *userid) { struct auth_scheme_t *scheme = txn->auth_chal.scheme; int logfd = txn->conn->logfd; int i; httpd_userid = xstrdup(userid); httpd_userisanonymous = is_userid_anonymous(httpd_userid); syslog(LOG_NOTICE, "login: %s %s %s%s %s SESSIONID=<%s>", txn->conn->clienthost, httpd_userid, scheme->name, txn->conn->tls_ctx ? "+TLS" : "", "User logged in", session_id()); /* Recreate telemetry log entry for request (w/ credentials redacted) */ assert(!buf_len(&txn->buf)); buf_printf(&txn->buf, "<" TIME_T_FMT "<", time(NULL)); /* timestamp */ buf_printf(&txn->buf, "%s %s %s\r\n", /* request-line*/ txn->req_line.meth, txn->req_line.uri, txn->req_line.ver); spool_enum_hdrcache(txn->req_hdrs, /* header fields */ &log_cachehdr, &txn->buf); buf_appendcstr(&txn->buf, "\r\n"); /* CRLF */ buf_append(&txn->buf, &txn->req_body.payload); /* message body */ buf_appendmap(&txn->buf, /* buffered input */ (const char *) httpd_in->ptr, httpd_in->cnt); if (logfd != -1) { /* Rewind log to current request and truncate it */ off_t end = lseek(logfd, 0, SEEK_END); if (ftruncate(logfd, end - buf_len(&txn->buf))) syslog(LOG_ERR, "IOERROR: failed to truncate http log"); /* Close existing telemetry log */ close(logfd); } prot_setlog(httpd_in, PROT_NO_FD); prot_setlog(httpd_out, PROT_NO_FD); /* Create telemetry log based on new userid */ if (txn->conn->sess_ctx) txn->conn->logfd = logfd = telemetry_log(userid, NULL, NULL, 0); else txn->conn->logfd = logfd = telemetry_log(userid, httpd_in, httpd_out, 0); if (logfd != -1) { /* Log credential-redacted request */ if (write(logfd, buf_cstring(&txn->buf), buf_len(&txn->buf)) < 0) syslog(LOG_ERR, "IOERROR: failed to write to http log"); } buf_reset(&txn->buf); /* Do any namespace specific post-auth processing */ for (i = 0; http_namespaces[i]; i++) { if (http_namespaces[i]->enabled && http_namespaces[i]->auth) { int ret = http_namespaces[i]->auth(httpd_userid); if (ret) return ret; } } return 0; } /* Perform HTTP Authentication based on the given credentials ('creds'). * Returns the selected auth scheme and any server challenge in 'chal'. * May be called multiple times if auth scheme requires multiple steps. * SASL status between steps is maintained in 'status'. */ #define MAX_AUTHPARAM_SIZE 10 /* "sid=,data=" */ #define MAX_BASE64_SIZE 21848 /* per RFC 4422: ((16K / 3) + 1) * 4 */ #define BASE64_BUF_SIZE (MAX_AUTHPARAM_SIZE +MAX_SESSIONID_SIZE +MAX_BASE64_SIZE) static int http_auth(const char *creds, struct transaction_t *txn) { struct auth_challenge_t *chal = &txn->auth_chal; static int status = SASL_OK; int slen, r; const char *clientin = NULL, *realm = NULL, *user, **authzid; unsigned int clientinlen = 0; struct auth_scheme_t *scheme; static char base64[BASE64_BUF_SIZE+1]; const void *canon_user = NULL; /* Split credentials into auth scheme and response */ slen = strcspn(creds, " "); if ((clientin = strchr(creds + slen, ' '))) { while (strchr(" ", *++clientin)); /* Trim leading 1*SP */ clientinlen = strlen(clientin); } syslog(LOG_DEBUG, "http_auth: status=%d scheme='%s' creds='%.*s%s'", status, chal->scheme ? chal->scheme->name : "", slen, creds, clientin ? " " : ""); /* Free userid & authstate previously allocated for auth'd user */ if (httpd_userid) { free(httpd_userid); httpd_userid = NULL; } if (httpd_extrafolder) { free(httpd_extrafolder); httpd_extrafolder = NULL; } if (httpd_extradomain) { free(httpd_extradomain); httpd_extradomain = NULL; } if (httpd_authstate) { auth_freestate(httpd_authstate); httpd_authstate = NULL; } chal->param = NULL; if (chal->scheme) { /* Use current scheme, if possible */ scheme = chal->scheme; if (strncasecmp(scheme->name, creds, slen)) { /* Changing auth scheme -> reset state */ syslog(LOG_DEBUG, "http_auth: changing scheme"); reset_saslconn(&httpd_saslconn); chal->scheme = NULL; status = SASL_OK; } } if (!chal->scheme) { /* Find the client-specified auth scheme */ syslog(LOG_DEBUG, "http_auth: find client scheme"); for (scheme = auth_schemes; scheme->name; scheme++) { if (slen && !strncasecmp(scheme->name, creds, slen)) { /* Found a supported scheme, see if its available */ if (!(avail_auth_schemes & scheme->id)) scheme = NULL; break; } } if (!scheme || !scheme->name) { /* Didn't find a matching scheme that is available */ syslog(LOG_DEBUG, "Unknown auth scheme '%.*s'", slen, creds); return SASL_NOMECH; } /* We found it! */ syslog(LOG_DEBUG, "http_auth: found matching scheme: %s", scheme->name); chal->scheme = scheme; status = SASL_OK; if (!clientin && (scheme->flags & AUTH_REALM_PARAM)) { /* Get realm - based on namespace of URL */ switch (txn->req_tgt.namespace->id) { case URL_NS_DEFAULT: case URL_NS_PRINCIPAL: realm = config_getstring(IMAPOPT_DAV_REALM); break; case URL_NS_CALENDAR: realm = config_getstring(IMAPOPT_CALDAV_REALM); break; case URL_NS_ADDRESSBOOK: realm = config_getstring(IMAPOPT_CARDDAV_REALM); break; case URL_NS_RSS: realm = config_getstring(IMAPOPT_RSS_REALM); break; } if (!realm) realm = config_servername; /* Create initial challenge (base64 buffer is static) */ snprintf(base64, BASE64_BUF_SIZE, "realm=\"%s\"", realm); chal->param = base64; chal->scheme = NULL; /* make sure we don't reset the SASL ctx */ return status; } } /* Parse any auth parameters, if necessary */ if (clientin && (scheme->flags & AUTH_DATA_PARAM)) { const char *sid = NULL; unsigned int sid_len; r = http_parse_auth_params(clientin, NULL /* realm */, NULL, &sid, &sid_len, &clientin, &clientinlen); if (r != SASL_OK) return r; if (sid) { const char *mysid = session_id(); if (sid_len != strlen(mysid) || strncmp(mysid, sid, sid_len)) { syslog(LOG_ERR, "%s: Incorrect 'sid' parameter in credentials", scheme->name); return SASL_BADAUTH; } } } /* Base64 decode any client response, if necessary */ if (clientin && (scheme->flags & AUTH_BASE64)) { r = sasl_decode64(clientin, clientinlen, base64, BASE64_BUF_SIZE, &clientinlen); if (r != SASL_OK) { syslog(LOG_ERR, "Base64 decode failed: %s", sasl_errstring(r, NULL, NULL)); return r; } clientin = base64; } if (scheme->id == AUTH_BASIC) { /* Basic (plaintext) authentication */ char *pass; char *extra; char *plus; char *domain; /* Split credentials into ':' . * We are working with base64 buffer, so we can modify it. */ user = base64; pass = strchr(base64, ':'); if (!pass) { syslog(LOG_ERR, "Basic auth: Missing password"); return SASL_BADPARAM; } *pass++ = '\0'; domain = strchr(user, '@'); if (domain) *domain++ = '\0'; extra = strchr(user, '%'); if (extra) *extra++ = '\0'; plus = strchr(user, '+'); if (plus) *plus++ = '\0'; /* Verify the password */ char *realuser = domain ? strconcat(user, "@", domain, (char *) NULL) : xstrdup(user); status = sasl_checkpass(httpd_saslconn, realuser, strlen(realuser), pass, strlen(pass)); memset(pass, 0, strlen(pass)); /* erase plaintext password */ if (status) { if (*user == '\0') // TB can send "Authorization: Basic Og==" txn->error.desc = "All-whitespace username."; syslog(LOG_NOTICE, "badlogin: %s Basic %s %s", txn->conn->clienthost, realuser, sasl_errdetail(httpd_saslconn)); free(realuser); /* Don't allow user probing */ if (status == SASL_NOUSER) status = SASL_BADAUTH; return status; } free(realuser); /* Successful authentication - fall through */ httpd_extrafolder = xstrdupnull(plus); httpd_extradomain = xstrdupnull(extra); } else if (scheme->id == AUTH_BEARER) { /* Bearer authentication */ assert(txn->req_tgt.namespace->bearer); /* Call namespace bearer authentication. * We are working with base64 buffer, so the namespace can * write the canonicalized userid into the buffer */ base64[0] = 0; status = txn->req_tgt.namespace->bearer(clientin, base64, BASE64_BUF_SIZE); if (status) return status; canon_user = user = base64; /* Successful authentication - fall through */ httpd_extrafolder = NULL; httpd_extradomain = NULL; httpd_authstate = auth_newstate(user); } else { /* SASL-based authentication (SCRAM_*, Digest, Negotiate, NTLM) */ const char *serverout = NULL; unsigned int serveroutlen = 0; unsigned int auth_params_len = 0; #ifdef SASL_HTTP_REQUEST /* Setup SASL HTTP request, if necessary */ sasl_http_request_t sasl_http_req; if (scheme->flags & AUTH_NEED_REQUEST) { sasl_http_req.method = txn->req_line.meth; sasl_http_req.uri = txn->req_line.uri; sasl_http_req.entity = NULL; sasl_http_req.elen = 0; sasl_http_req.non_persist = txn->flags.conn & CONN_CLOSE; sasl_setprop(httpd_saslconn, SASL_HTTP_REQUEST, &sasl_http_req); } #endif /* SASL_HTTP_REQUEST */ if (status == SASL_CONTINUE) { /* Continue current authentication exchange */ syslog(LOG_DEBUG, "http_auth: continue %s", scheme->saslmech); status = sasl_server_step(httpd_saslconn, clientin, clientinlen, &serverout, &serveroutlen); } else { /* Start new authentication exchange */ syslog(LOG_DEBUG, "http_auth: start %s", scheme->saslmech); status = sasl_server_start(httpd_saslconn, scheme->saslmech, clientin, clientinlen, &serverout, &serveroutlen); } /* Failure - probably bad client response */ if ((status != SASL_OK) && (status != SASL_CONTINUE)) { syslog(LOG_ERR, "SASL failed: %s", sasl_errstring(status, NULL, NULL)); return status; } /* Prepend any auth parameters, if necessary */ if (scheme->flags & AUTH_DATA_PARAM) { auth_params_len = snprintf(base64, MAX_AUTHPARAM_SIZE + MAX_SESSIONID_SIZE, "sid=%s%s", session_id(), serverout ? ",data=" : ""); } /* Base64 encode any server challenge, if necessary */ if (serverout && (scheme->flags & AUTH_BASE64)) { r = sasl_encode64(serverout, serveroutlen, base64 + auth_params_len, MAX_BASE64_SIZE, NULL); if (r != SASL_OK) { syslog(LOG_ERR, "Base64 encode failed: %s", sasl_errstring(r, NULL, NULL)); return r; } serverout = base64; } chal->param = serverout; if (status == SASL_CONTINUE) { /* Need another step to complete authentication */ return status; } /* Successful authentication * * HTTP doesn't support security layers, * so don't attach SASL context to prot layer. */ } if (!canon_user) { /* Get the userid from SASL - already canonicalized */ status = sasl_getprop(httpd_saslconn, SASL_USERNAME, &canon_user); if (status != SASL_OK) { syslog(LOG_ERR, "weird SASL error %d getting SASL_USERNAME", status); return status; } user = (const char *) canon_user; } if (httpd_authid) free(httpd_authid); httpd_authid = xstrdup(user); authzid = spool_getheader(txn->req_hdrs, "Authorize-As"); if (authzid && *authzid[0]) { /* Trying to proxy as another user */ user = authzid[0]; status = proxy_authz(&user, txn); if (status) return status; } /* Post-process the successful authentication. */ r = auth_success(txn, user); if (r == HTTP_UNAVAILABLE) { status = SASL_UNAVAIL; } else if (r) { /* Any error here comes after the user already logged in, * so avoid to return SASL_BADAUTH. It would trigger the * HTTP handler to send UNAUTHORIZED, and might confuse * users that provided their correct credentials. */ syslog(LOG_ERR, "auth_success returned error: %s", error_message(r)); status = SASL_FAIL; } return status; } /************************* Method Execution Routines ************************/ /* Compare an etag in a header to a resource etag. * Returns 0 if a match, non-zero otherwise. */ EXPORTED int etagcmp(const char *hdr, const char *etag) { size_t len; if (!etag) return -1; /* no representation */ if (!strcmp(hdr, "*")) return 0; /* any representation */ len = strlen(etag); if (!strncmp(hdr, "W/", 2)) hdr+=2; /* skip weak prefix */ if (*hdr++ != '\"') return 1; /* match/skip open DQUOTE */ if (strlen(hdr) != len+1) return 1; /* make sure lengths match */ if (hdr[len] != '\"') return 1; /* match close DQUOTE */ return strncmp(hdr, etag, len); } /* Compare a resource etag to a comma-separated list and/or multiple headers * looking for a match. Returns 1 if a match is found, 0 otherwise. */ static unsigned etag_match(const char *hdr[], const char *etag) { unsigned i, match = 0; tok_t tok; char *token; for (i = 0; !match && hdr[i]; i++) { tok_init(&tok, hdr[i], ",", TOK_TRIMLEFT|TOK_TRIMRIGHT); while (!match && (token = tok_next(&tok))) { if (!etagcmp(token, etag)) match = 1; } tok_fini(&tok); } return match; } static int parse_ranges(const char *hdr, unsigned long len, struct range **ranges) { int ret = HTTP_BAD_RANGE; struct range *new, *tail = *ranges = NULL; tok_t tok; char *token; if (!len) return HTTP_OK; /* need to know length of representation */ /* we only handle byte-unit */ if (!hdr || strncmp(hdr, "bytes=", 6)) return HTTP_OK; tok_init(&tok, hdr+6, ",", TOK_TRIMLEFT|TOK_TRIMRIGHT); while ((token = tok_next(&tok))) { /* default to entire representation */ unsigned long first = 0; unsigned long last = len - 1; char *p, *endp; if (!(p = strchr(token, '-'))) continue; /* bad byte-range-set */ if (p == token) { /* suffix-byte-range-spec */ unsigned long suffix = strtoul(++p, &endp, 10); if (endp == p || *endp) continue; /* bad suffix-length */ if (!suffix) continue; /* unsatisfiable suffix-length */ /* don't start before byte zero */ if (suffix < len) first = len - suffix; } else { /* byte-range-spec */ first = strtoul(token, &endp, 10); if (endp != p) continue; /* bad first-byte-pos */ if (first >= len) continue; /* unsatisfiable first-byte-pos */ if (*++p) { /* last-byte-pos */ last = strtoul(p, &endp, 10); if (*endp || last < first) continue; /* bad last-byte-pos */ /* don't go past end of representation */ if (last >= len) last = len - 1; } } ret = HTTP_PARTIAL; /* Coalesce overlapping ranges, or those with a gap < 80 bytes */ if (tail && first >= tail->first && (long) (first - tail->last) < 80) { tail->last = MAX(last, tail->last); continue; } /* Create a new range and append it to linked list */ new = xzmalloc(sizeof(struct range)); new->first = first; new->last = last; if (tail) tail->next = new; else *ranges = new; tail = new; } tok_fini(&tok); return ret; } /* Check headers for any preconditions. * * Interaction is complex and is documented in RFC 7232 */ EXPORTED int check_precond(struct transaction_t *txn, const char *etag, time_t lastmod) { hdrcache_t hdrcache = txn->req_hdrs; const char **hdr; time_t since = 0; /* Step 1 */ if ((hdr = spool_getheader(hdrcache, "If-Match"))) { if (!etag_match(hdr, etag)) return HTTP_PRECOND_FAILED; /* Continue to step 3 */ } /* Step 2 */ else if ((hdr = spool_getheader(hdrcache, "If-Unmodified-Since"))) { if (time_from_rfc5322(hdr[0], &since, DATETIME_FULL) < 0) return HTTP_BAD_REQUEST; if (lastmod > since) return HTTP_PRECOND_FAILED; /* Continue to step 3 */ } /* Step 3 */ if ((hdr = spool_getheader(hdrcache, "If-None-Match"))) { if (etag_match(hdr, etag)) { if (txn->meth == METH_GET || txn->meth == METH_HEAD) return HTTP_NOT_MODIFIED; else return HTTP_PRECOND_FAILED; } /* Continue to step 5 */ } /* Step 4 */ else if ((txn->meth == METH_GET || txn->meth == METH_HEAD) && (hdr = spool_getheader(hdrcache, "If-Modified-Since"))) { if (time_from_rfc5322(hdr[0], &since, DATETIME_FULL) < 0) return HTTP_BAD_REQUEST; if (lastmod <= since) return HTTP_NOT_MODIFIED; /* Continue to step 5 */ } /* Step 5 */ if (txn->flags.ranges && /* Only if we support Range requests */ txn->meth == METH_GET && (hdr = spool_getheader(hdrcache, "Range"))) { if ((hdr = spool_getheader(hdrcache, "If-Range"))) { time_from_rfc5322(hdr[0], &since, DATETIME_FULL); /* error OK here, could be an etag */ } /* Only process Range if If-Range isn't present or validator matches */ if (!hdr || (since && (lastmod <= since)) || !etagcmp(hdr[0], etag)) return HTTP_PARTIAL; } /* Step 6 */ return HTTP_OK; } const struct mimetype { const char *ext; const char *type; unsigned int compressible; } mimetypes[] = { { ".css", "text/css", 1 }, { ".htm", "text/html", 1 }, { ".html", "text/html", 1 }, { ".ics", "text/calendar", 1 }, { ".ifb", "text/calendar", 1 }, { ".text", "text/plain", 1 }, { ".txt", "text/plain", 1 }, { ".cgm", "image/cgm", 1 }, { ".gif", "image/gif", 0 }, { ".jpg", "image/jpeg", 0 }, { ".jpeg", "image/jpeg", 0 }, { ".png", "image/png", 0 }, { ".svg", "image/svg+xml", 1 }, { ".tif", "image/tiff", 1 }, { ".tiff", "image/tiff", 1 }, { ".aac", "audio/aac", 0 }, { ".m4a", "audio/mp4", 0 }, { ".mp3", "audio/mpeg", 0 }, { ".mpeg", "audio/mpeg", 0 }, { ".oga", "audio/ogg", 0 }, { ".ogg", "audio/ogg", 0 }, { ".wav", "audio/wav", 0 }, { ".avi", "video/x-msvideo", 0 }, { ".mov", "video/quicktime", 0 }, { ".m4v", "video/mp4", 0 }, { ".ogv", "video/ogg", 0 }, { ".qt", "video/quicktime", 0 }, { ".wmv", "video/x-ms-wmv", 0 }, { ".bz", "application/x-bzip", 0 }, { ".bz2", "application/x-bzip2", 0 }, { ".gz", "application/gzip", 0 }, { ".gzip", "application/gzip", 0 }, { ".tgz", "application/gzip", 0 }, { ".zip", "application/zip", 0 }, { ".doc", "application/msword", 1 }, { ".jcs", "application/calendar+json", 1 }, { ".jfb", "application/calendar+json", 1 }, { ".js", "application/javascript", 1 }, { ".json", "application/json", 1 }, { ".pdf", "application/pdf", 1 }, { ".ppt", "application/vnd.ms-powerpoint", 1 }, { ".sh", "application/x-sh", 1 }, { ".tar", "application/x-tar", 1 }, { ".xcs", "application/calendar+xml", 1 }, { ".xfb", "application/calendar+xml", 1 }, { ".xls", "application/vnd.ms-excel", 1 }, { ".xml", "application/xml", 1 }, { NULL, NULL, 0 } }; static int list_well_known(struct transaction_t *txn) { static struct buf body = BUF_INITIALIZER; static time_t lastmod = 0; struct stat sbuf; int precond; /* stat() imapd.conf for Last-Modified and ETag */ stat(config_filename, &sbuf); assert(!buf_len(&txn->buf)); buf_printf(&txn->buf, TIME_T_FMT "-" TIME_T_FMT "-" OFF_T_FMT, compile_time, sbuf.st_mtime, sbuf.st_size); sbuf.st_mtime = MAX(compile_time, sbuf.st_mtime); /* Check any preconditions, including range request */ txn->flags.ranges = 1; precond = check_precond(txn, buf_cstring(&txn->buf), sbuf.st_mtime); switch (precond) { case HTTP_OK: case HTTP_NOT_MODIFIED: /* Fill in ETag, Last-Modified, and Expires */ txn->resp_body.etag = buf_cstring(&txn->buf); txn->resp_body.lastmod = sbuf.st_mtime; txn->resp_body.maxage = 86400; /* 24 hrs */ txn->flags.cc |= CC_MAXAGE; if (precond != HTTP_NOT_MODIFIED) break; GCC_FALLTHROUGH default: /* We failed a precondition - don't perform the request */ return precond; } if (txn->resp_body.lastmod > lastmod) { const char *proto = NULL, *host = NULL; unsigned i, level = 0; /* Start HTML */ buf_reset(&body); buf_printf_markup(&body, level, HTML_DOCTYPE); buf_printf_markup(&body, level++, ""); buf_printf_markup(&body, level++, ""); buf_printf_markup(&body, level, "%s", "Well-Known Locations"); buf_printf_markup(&body, --level, ""); buf_printf_markup(&body, level++, ""); buf_printf_markup(&body, level, "

%s

", "Well-Known Locations"); buf_printf_markup(&body, level++, "
    "); /* Add the list of enabled /.well-known/ URLs */ http_proto_host(txn->req_hdrs, &proto, &host); for (i = 0; http_namespaces[i]; i++) { if (http_namespaces[i]->enabled && http_namespaces[i]->well_known) { buf_printf_markup(&body, level, "
  • %s
    • ", proto, host, http_namespaces[i]->prefix, http_namespaces[i]->well_known); } } /* Finish HTML */ buf_printf_markup(&body, --level, "
"); buf_printf_markup(&body, --level, ""); buf_printf_markup(&body, --level, ""); lastmod = txn->resp_body.lastmod; } /* Output the HTML response */ txn->resp_body.type = "text/html; charset=utf-8"; write_body(precond, txn, buf_cstring(&body), buf_len(&body)); return 0; } /* * WebSockets data callback (no sub-protocol): Echo back non-control messages. * * Can be tested with: * https://github.com/websockets/wscat * https://addons.mozilla.org/en-US/firefox/addon/simple-websocket-client/ * https://chrome.google.com/webstore/detail/simple-websocket-client/gobngblklhkgmjhbpbdlkglbhhlafjnh * https://chrome.google.com/webstore/detail/web-socket-client/lifhekgaodigcpmnakfhaaaboididbdn * * WebSockets over HTTP/2 currently only available in: * https://www.google.com/chrome/browser/canary.html */ static int ws_echo(struct buf *inbuf, struct buf *outbuf, struct buf *logbuf __attribute__((unused)), void **rock __attribute__((unused))) { buf_init_ro(outbuf, buf_base(inbuf), buf_len(inbuf)); return 0; } HIDDEN int meth_connect(struct transaction_t *txn, void *params) { struct connect_params *cparams = (struct connect_params *) params; /* Bootstrap WebSockets over HTTP/2, if requested */ if ((txn->flags.ver != VER_2) || !ws_enabled() || !cparams || !cparams->endpoint) { return HTTP_NOT_IMPLEMENTED; } if (strcmp(txn->req_uri->path, cparams->endpoint)) return HTTP_NOT_ALLOWED; if (!(txn->flags.upgrade & UPGRADE_WS)) { txn->error.desc = "Missing/unsupported :protocol value "; return HTTP_BAD_REQUEST; } int ret = ws_start_channel(txn, cparams->subprotocol, cparams->data_cb); return (ret == HTTP_UPGRADE) ? HTTP_BAD_REQUEST : ret; } #define WELL_KNOWN_PREFIX "/.well-known" /* Perform a GET/HEAD request */ static int meth_get(struct transaction_t *txn, void *params __attribute__((unused))) { int r, fd = -1, precond, len; const char *prefix, *urls, *path, *ext; static struct buf pathbuf = BUF_INITIALIZER; struct stat sbuf; const char *msg_base = NULL; size_t msg_size = 0; struct resp_body_t *resp_body = &txn->resp_body; /* Upgrade to WebSockets over HTTP/1.1 on root, if requested */ if (!strcmp(txn->req_uri->path, "/")) { if (txn->flags.upgrade & UPGRADE_WS) { return ws_start_channel(txn, NULL, &ws_echo); } if (ws_enabled()) { txn->flags.upgrade |= UPGRADE_WS; txn->flags.conn |= CONN_UPGRADE; } } /* Check if this is a request for /.well-known/ listing */ len = strlen(WELL_KNOWN_PREFIX); if (!strncmp(txn->req_uri->path, WELL_KNOWN_PREFIX, len)) { if (txn->req_uri->path[len] == '/') len++; if (txn->req_uri->path[len] == '\0') { return list_well_known(txn); } return HTTP_NOT_FOUND; } /* Serve up static pages */ prefix = config_getstring(IMAPOPT_HTTPDOCROOT); if (!prefix) return HTTP_NOT_FOUND; if (*prefix != '/') { /* Remote content */ struct backend *be; be = proxy_findserver(prefix, &http_protocol, httpd_userid, &backend_cached, NULL, NULL, httpd_in); if (!be) return HTTP_UNAVAILABLE; return http_pipe_req_resp(be, txn); } /* Local content */ if ((urls = config_getstring(IMAPOPT_HTTPALLOWEDURLS))) { tok_t tok = TOK_INITIALIZER(urls, " \t", TOK_TRIMLEFT|TOK_TRIMRIGHT); char *token; while ((token = tok_next(&tok)) && strcmp(token, txn->req_uri->path)); tok_fini(&tok); if (!token) return HTTP_NOT_FOUND; } buf_setcstr(&pathbuf, prefix); buf_appendcstr(&pathbuf, txn->req_uri->path); path = buf_cstring(&pathbuf); /* See if path is a directory and look for index.html */ if (!(r = stat(path, &sbuf)) && S_ISDIR(sbuf.st_mode)) { buf_appendcstr(&pathbuf, "/index.html"); path = buf_cstring(&pathbuf); r = stat(path, &sbuf); } /* See if file exists and get Content-Length & Last-Modified time */ if (r || !S_ISREG(sbuf.st_mode)) return HTTP_NOT_FOUND; if (!resp_body->type) { /* Caller hasn't specified the Content-Type */ resp_body->type = "application/octet-stream"; if ((ext = strrchr(path, '.'))) { /* Try to use filename extension to identity Content-Type */ const struct mimetype *mtype; for (mtype = mimetypes; mtype->ext; mtype++) { if (!strcasecmp(ext, mtype->ext)) { resp_body->type = mtype->type; if (!mtype->compressible) { /* Never compress non-compressible resources */ txn->resp_body.enc.type = CE_IDENTITY; txn->resp_body.enc.proc = NULL; txn->flags.te = TE_NONE; txn->flags.vary &= ~VARY_AE; } break; } } } } /* Generate Etag */ assert(!buf_len(&txn->buf)); buf_printf(&txn->buf, "%ld-%ld", (long) sbuf.st_mtime, (long) sbuf.st_size); /* Check any preconditions, including range request */ txn->flags.ranges = 1; precond = check_precond(txn, buf_cstring(&txn->buf), sbuf.st_mtime); switch (precond) { case HTTP_OK: case HTTP_PARTIAL: case HTTP_NOT_MODIFIED: /* Fill in ETag, Last-Modified, and Expires */ resp_body->etag = buf_cstring(&txn->buf); resp_body->lastmod = sbuf.st_mtime; resp_body->maxage = 86400; /* 24 hrs */ txn->flags.cc |= CC_MAXAGE; if (!httpd_userisanonymous) txn->flags.cc |= CC_PUBLIC; if (precond != HTTP_NOT_MODIFIED) break; GCC_FALLTHROUGH default: /* We failed a precondition - don't perform the request */ resp_body->type = NULL; return precond; } if (txn->meth == METH_GET) { /* Open and mmap the file */ if ((fd = open(path, O_RDONLY)) == -1) return HTTP_SERVER_ERROR; map_refresh(fd, 1, &msg_base, &msg_size, sbuf.st_size, path, NULL); } write_body(precond, txn, msg_base, sbuf.st_size); if (fd != -1) { map_free(&msg_base, &msg_size); close(fd); } return 0; } /* Perform an OPTIONS request */ EXPORTED int meth_options(struct transaction_t *txn, void *params) { parse_path_t parse_path = (parse_path_t) params; int r, i; /* Response should not be cached */ txn->flags.cc |= CC_NOCACHE; /* Response doesn't have a body, so no Vary */ txn->flags.vary = 0; /* Special case "*" - show all features/methods available on server */ if (!strcmp(txn->req_uri->path, "*")) { for (i = 0; http_namespaces[i]; i++) { if (http_namespaces[i]->enabled) txn->req_tgt.allow |= http_namespaces[i]->allow; } if (ws_enabled() && (txn->flags.ver == VER_2)) { /* CONNECT allowed for bootstrapping WebSocket over HTTP/2 */ txn->req_tgt.allow |= ALLOW_CONNECT; } } else { if (parse_path) { /* Parse the path */ r = parse_path(txn->req_uri->path, &txn->req_tgt, &txn->error.desc); if (r) return r; } else if (!strcmp(txn->req_uri->path, "/") && ws_enabled() && (txn->flags.ver == VER_2)) { /* WS 'echo' endpoint */ txn->req_tgt.allow |= ALLOW_CONNECT; } if (txn->flags.cors) { const char **hdr = spool_getheader(txn->req_hdrs, "Access-Control-Request-Method"); if (hdr) { /* CORS preflight request */ unsigned meth; txn->flags.cors = CORS_PREFLIGHT; /* Check Method against our list of known methods */ for (meth = 0; (meth < METH_UNKNOWN) && strcmp(http_methods[meth].name, hdr[0]); meth++); if (meth == METH_UNKNOWN) txn->flags.cors = 0; else { /* Check Method against those supported by the resource */ if (!txn->req_tgt.namespace->methods[meth].proc) txn->flags.cors = 0; } } } } response_header(HTTP_OK, txn); return 0; } /* Perform an PROPFIND request on "/" iff we support CalDAV */ static int meth_propfind_root(struct transaction_t *txn, void *params __attribute__((unused))) { assert(txn); #ifdef WITH_DAV /* Apple iCal and Evolution both check "/" */ if (!strcmp(txn->req_uri->path, "/") || !strcmp(txn->req_uri->path, "/dav/")) { /* Array of known "live" properties */ const struct prop_entry root_props[] = { /* WebDAV ACL (RFC 3744) properties */ { "principal-collection-set", NS_DAV, PROP_COLLECTION, propfind_princolset, NULL, NULL }, /* WebDAV Current Principal (RFC 5397) properties */ { "current-user-principal", NS_DAV, PROP_COLLECTION, propfind_curprin, NULL, NULL }, { NULL, 0, 0, NULL, NULL, NULL } }; struct meth_params root_params = { .propfind = { DAV_FINITE_DEPTH, root_props } }; /* Make a working copy of target path */ strlcpy(txn->req_tgt.path, txn->req_uri->path, sizeof(txn->req_tgt.path)); txn->req_tgt.tail = txn->req_tgt.path + strlen(txn->req_tgt.path); txn->req_tgt.allow |= ALLOW_DAV; return meth_propfind(txn, &root_params); } #endif /* WITH_DAV */ return HTTP_NOT_ALLOWED; } /* Write cached header to buf, excluding any that might have sensitive data. */ static void trace_cachehdr(const char *name, const char *contents, const char *raw, void *rock) { struct buf *buf = (struct buf *) rock; const char **hdr, *sensitive[] = { "authorization", "cookie", "proxy-authorization", NULL }; /* Ignore private headers in our cache */ if (name[0] == ':') return; for (hdr = sensitive; *hdr && strcmp(name, *hdr); hdr++); if (!*hdr) { if (raw) buf_appendcstr(buf, raw); else buf_printf(buf, "%c%s: %s\r\n", toupper(name[0]), name+1, contents); } } /* Perform an TRACE request */ EXPORTED int meth_trace(struct transaction_t *txn, void *params) { parse_path_t parse_path = (parse_path_t) params; const char **hdr; unsigned long max_fwd = -1; struct buf *msg = &txn->resp_body.payload; /* Response should not be cached */ txn->flags.cc |= CC_NOCACHE; /* Make sure method is allowed */ if (!(txn->req_tgt.allow & ALLOW_TRACE)) return HTTP_NOT_ALLOWED; if ((hdr = spool_getheader(txn->req_hdrs, "Max-Forwards"))) { max_fwd = strtoul(hdr[0], NULL, 10); } if (max_fwd && parse_path) { /* Parse the path */ int r; if ((r = parse_path(txn->req_uri->path, &txn->req_tgt, &txn->error.desc))) return r; if (txn->req_tgt.mbentry && txn->req_tgt.mbentry->server) { /* Remote mailbox */ struct backend *be; be = proxy_findserver(txn->req_tgt.mbentry->server, &http_protocol, httpd_userid, &backend_cached, NULL, NULL, httpd_in); if (!be) return HTTP_UNAVAILABLE; return http_pipe_req_resp(be, txn); } /* Local mailbox */ } /* Echo the request back to the client as a message/http: * * - Piece the Request-line back together * - Use all non-sensitive cached headers from client */ buf_reset(msg); buf_printf(msg, "TRACE %s %s\r\n", txn->req_line.uri, txn->req_line.ver); spool_enum_hdrcache(txn->req_hdrs, &trace_cachehdr, msg); buf_appendcstr(msg, "\r\n"); txn->resp_body.type = "message/http"; txn->resp_body.len = buf_len(msg); write_body(HTTP_OK, txn, buf_cstring(msg), buf_len(msg)); return 0; } /* simple wrapper to implicity add READFB if we have the READ ACL */ EXPORTED int httpd_myrights(struct auth_state *authstate, const mbentry_t *mbentry) { int rights = 0; if (mbentry && mbentry->acl) { rights = cyrus_acl_myrights(authstate, mbentry->acl); if (mbentry->mbtype == MBTYPE_CALENDAR && (rights & DACL_READ) == DACL_READ) { rights |= DACL_READFB; } } return rights; } /* Allow unauthenticated GET/HEAD, deny all other unauthenticated requests */ EXPORTED int http_allow_noauth_get(struct transaction_t *txn) { /* Inverse logic: True means we *require* authentication */ switch (txn->meth) { case METH_GET: case METH_HEAD: /* Let method processing function decide if auth is needed */ return 0; default: return 1; } } /* Allow unauthenticated requests */ EXPORTED int http_allow_noauth(struct transaction_t *txn __attribute__((unused))) { return 0; } /* Read the body of a request */ EXPORTED int http_read_req_body(struct transaction_t *txn) { struct body_t *body = &txn->req_body; syslog(LOG_DEBUG, "http_read_req_body(flags=%#x, framing=%d)", body->flags, body->framing); if (body->flags & BODY_DONE) return 0; body->flags |= BODY_DONE; if (body->flags & BODY_CONTINUE) { body->flags &= ~BODY_CONTINUE; if (body->flags & BODY_DISCARD) { /* Don't care about the body and client hasn't sent it, we're done */ return 0; } /* Tell client to send the body */ response_header(HTTP_CONTINUE, txn); } /* Read body from client */ return http_read_body(txn->conn->pin, txn->req_hdrs, body, &txn->error.desc); } diff --git a/imap/icu_wrap.cpp b/imap/icu_wrap.cpp index 6ea805bcc..60b4c92a0 100644 --- a/imap/icu_wrap.cpp +++ b/imap/icu_wrap.cpp @@ -1,76 +1,76 @@ /* icu_wrap.c -- C++ hiding wrapper API for ICU * * Copyright (c) 1994-2019 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ extern "C" { #include "config.h" #include "string.h" #include "xmalloc.h" }; #include #include #include -extern "C" char *icu_getIDForWindowsID(const char *id) +extern "C" EXPORTED char *icu_getIDForWindowsID(const char *id) { UErrorCode status = U_ZERO_ERROR; UConverter *utf8cnv = ucnv_open("utf-8", &status); if (U_FAILURE(status)) return NULL; icu::UnicodeString uWinID {id, -1, utf8cnv, status}; ucnv_close(utf8cnv); icu::UnicodeString uID; icu::TimeZone::getIDForWindowsID(uWinID, NULL, uID, status); std::string str; uID.toUTF8String(str); if (!str.empty()) return xstrdup(str.c_str()); if (!strcasecmp(id, "Mid-Atlantic Standard Time")) { /* ICU doesn't map this ID */ return xstrdup("Atlantic/South_Georgia"); } return NULL; } diff --git a/imap/message_test.c b/imap/message_test.c index f0c1eed36..e5479812c 100644 --- a/imap/message_test.c +++ b/imap/message_test.c @@ -1,317 +1,317 @@ /* * Copyright (c) 1994-2012 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #ifdef HAVE_UNISTD_H #include #endif #include #include #include #include #include #include /* cyrus includes */ #include "assert.h" #include "bsearch.h" #include "global.h" #include "index.h" #include "search_engines.h" #include "conversations.h" #include "mailbox.h" #include "mboxlist.h" #include "message.h" #include "util.h" #include "xmalloc.h" /* generated headers are not necessarily in current directory */ #include "imap/imap_err.h" static int usage(const char *name); int verbose = 0; enum { PART_TREE, TEXT_SECTIONS, TEXT_RECEIVER } dump_mode = PART_TREE; /*-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-*/ static void dump_octets(FILE *fp, const char *base, unsigned int len) { unsigned int i; while (len > 0) { fputs(" ", fp); for (i = 0 ; i < 16 && i < len ; i++) fprintf(fp, "%02x ", ((unsigned char *)base)[i]); for (; i < 16 ; i++) fputs(" ", fp); fputs(" ", fp); for (i = 0 ; i < 16 && i < len ; i++) fputc((isprint(base[i]) && !isspace(base[i]) ? base[i] : '.'), fp); fputc('\n', fp); i = (len > 16 ? 16 : len); len -= i; base += i; } } static void dump_buf(FILE *fp, const struct buf *data) { #define MAX_TEXT 512 if (verbose || data->len <= MAX_TEXT) { dump_octets(fp, data->s, data->len); } else { dump_octets(fp, data->s, MAX_TEXT/2); fputs(" ...\n", fp); dump_octets(fp, data->s + data->len - MAX_TEXT/2, MAX_TEXT/2); } #undef MAX_TEXT } static int dump_one_section(int partno, charset_t charset, int encoding, const char *type __attribute__((unused)), const char *subtype, const struct param *type_params __attribute__((unused)), const char *disposition __attribute__((unused)), const struct param *disposition_params __attribute__((unused)), const struct message_guid *content_guid __attribute__((unused)), const char *part __attribute__((unused)), struct buf *data, void *rock __attribute__((unused))) { #define MAX_TEXT 512 printf("SECTION partno=%d length=%llu subtype=%s charset=%s encoding=%s\n", partno, (unsigned long long)data->len, subtype, charset_alias_name(charset), encoding_name(encoding)); dump_buf(stdout, data); return 0; #undef MAX_TEXT } static int dump_text_sections(message_t *message) { return message_foreach_section(message, dump_one_section, NULL); } /*-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-*/ static int dump_message(message_t *message) { return dump_text_sections(message); } int main(int argc, char **argv) { int c; const char *alt_config = NULL; const char *filename = NULL; const char *mboxname = NULL; int recno = 1; int record_flag = 0; int r = 0; while ((c = getopt(argc, argv, "Rf:m:pr:stvC:")) != EOF) { switch (c) { case 'f': filename = optarg; break; case 'm': mboxname = optarg; break; case 'p': dump_mode = PART_TREE; break; case 'r': recno = atoi(optarg); if (recno <= 0) usage(argv[0]); break; case 's': dump_mode = TEXT_SECTIONS; break; case 't': dump_mode = TEXT_RECEIVER; break; case 'v': verbose++; break; case 'C': /* alt config file */ alt_config = optarg; break; case 'R': record_flag = 1; break; default: usage(argv[0]); break; } } if (optind != argc) usage(argv[0]); if (mboxname && filename) usage(argv[0]); cyrus_init(alt_config, "message_test", 0, CONFIG_NEED_PARTITION_DATA); if (mboxname && record_flag) { struct mailbox *mailbox = NULL; struct index_record record; message_t *message = NULL; r = mailbox_open_irl(mboxname, &mailbox); if (r) { fprintf(stderr, "Failed to open mailbox %s: %s\n", mboxname, error_message(r)); return 1; } memset(&record, 0, sizeof(struct index_record)); record.recno = recno; r = mailbox_reload_index_record(mailbox, &record); if (r) { fprintf(stderr, "Failed to read index record %u of %s: %s\n", recno, mboxname, error_message(r)); return 1; } message = message_new_from_record(mailbox, &record); r = dump_message(message); if (r) { fprintf(stderr, "Error dumping message: %s\n", error_message(r)); return 1; } message_unref(&message); mailbox_close(&mailbox); } else if (mboxname) { struct mailbox *mailbox = NULL; message_t *message = NULL; r = mailbox_open_irl(mboxname, &mailbox); if (r) { fprintf(stderr, "Failed to open mailbox %s: %s\n", mboxname, error_message(r)); return 1; } message = message_new_from_mailbox(mailbox, recno); r = dump_message(message); if (r) { fprintf(stderr, "Error dumping message: %s\n", error_message(r)); return 1; } message_unref(&message); mailbox_close(&mailbox); } else if (filename) { message_t *message = NULL; message = message_new_from_filename(filename); r = dump_message(message); if (r) { fprintf(stderr, "Error dumping message: %s\n", error_message(r)); return 1; } message_unref(&message); } else { message_t *message = NULL; int c; struct buf buf = BUF_INITIALIZER; while ((c = fgetc(stdin)) != EOF) buf_putc(&buf, c); message = message_new_from_data(buf.s, buf.len); dump_message(message); if (r) { fprintf(stderr, "Error dumping message: %s\n", error_message(r)); return 1; } message_unref(&message); buf_free(&buf); } cyrus_done(); return r; } static int usage(const char *name) { fprintf(stderr, "usage: %s [format-options] -m mailbox [-r recno] [-R]\n", name); fprintf(stderr, " %s [format-options] -f filename\n", name); fprintf(stderr, " %s [format-options] < message\n", name); fprintf(stderr, "format-options :=\n"); fprintf(stderr, "-p dump message part tree\n"); fprintf(stderr, "-s dump text sections\n"); fprintf(stderr, "-t dump output from search text receiver\n"); exit(EX_USAGE); } -void fatal(const char* s, int code) +EXPORTED void fatal(const char* s, int code) { fprintf(stderr, "message_test: %s\n", s); cyrus_done(); exit(code); } diff --git a/imap/search_test.c b/imap/search_test.c index 1899da300..0558981e1 100644 --- a/imap/search_test.c +++ b/imap/search_test.c @@ -1,338 +1,338 @@ /* * Copyright (c) 1994-2012 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #ifdef HAVE_UNISTD_H #include #endif #include #include #include #include #include #include /* cyrus includes */ #include "assert.h" #include "bsearch.h" #include "global.h" #include "index.h" #include "search_engines.h" #include "search_expr.h" #include "search_query.h" #include "message.h" #include "util.h" #include "xmalloc.h" /* generated headers are not necessarily in current directory */ #include "imap/imap_err.h" static int usage(const char *name); int verbose = 0; /* ====================================================================== */ static void dump_one_folder(const char *key __attribute__((unused)), void *data, void *rock __attribute__((unused))) { search_folder_t *folder = data; int uid; printf("mailbox %s\n", folder->mboxname); printf("min %u\n", search_folder_get_min(folder)); printf("max %u\n", search_folder_get_max(folder)); printf("count %u\n", search_folder_get_count(folder)); printf("highestmodseq %llu\n", (unsigned long long)search_folder_get_highest_modseq(folder)); search_folder_foreach(folder, uid) { printf("uid %u\n", uid); } } static int do_search(const char *mboxname, int multiple, const char *userid, char **words, int nwords) { struct buf querytext = BUF_INITIALIZER; struct namespace ns; struct index_init init; struct index_state *state = NULL; struct protstream *pin = NULL; struct protstream *pout = NULL; struct searchargs *searchargs = NULL; search_query_t *query = NULL; int i; int r; struct timeval start_time, end_time; memset(&init, 0, sizeof(struct index_init)); for (i = 0 ; i < nwords ; i++) { if (i) buf_putc(&querytext, ' '); buf_appendcstr(&querytext, words[i]); } if (verbose) fprintf(stderr, "search_test: IMAP query is \"%s\"\n", buf_cstring(&querytext)); buf_putc(&querytext, '\r'); buf_cstring(&querytext); r = mboxname_init_namespace(&ns, /*isadmin*/0); if (r) { fprintf(stderr, "Failed to initialise namespace: %s\n", error_message(r)); goto out; } pin = prot_readmap(querytext.s, querytext.len); pout = prot_new(/*fd*/0, /*write*/1); init.userid = userid; init.authstate = auth_newstate(userid); init.out = pout; r = index_open(mboxname, &init, &state); if (r) { fprintf(stderr, "%s: %s\n", mboxname, error_message(r)); goto out; } index_checkflags(state, 0, 0); searchargs = new_searchargs(".", GETSEARCH_CHARSET_KEYWORD, &ns, userid, init.authstate, /*isadmin*/0); gettimeofday(&start_time, NULL); r = get_search_program(pin, pout, searchargs); if (r != '\r') { fprintf(stderr, "Couldn't parse IMAP search program\n"); goto out; } query = search_query_new(state, searchargs); query->multiple = multiple; query->verbose = verbose; r = search_query_run(query); if (r) { fprintf(stderr, "Failed to run query: %s\n", error_message(r)); goto out; } gettimeofday(&end_time, NULL); hash_enumerate(&query->folders_by_name, dump_one_folder, query); if (verbose) fprintf(stderr, "search_test: ran query in %.6f sec\n", timesub(&start_time, &end_time)); out: if (pin) prot_free(pin); if (pout) prot_free(pout); if (searchargs) freesearchargs(searchargs); search_query_free(query); index_close(&state); buf_free(&querytext); if (init.authstate) auth_freestate(init.authstate); return !!r; } /* ====================================================================== */ static int do_serialise(char **words, int nwords) { const char *userid = "cassandane"; struct buf querytext = BUF_INITIALIZER; struct namespace ns; struct protstream *pin = NULL; struct protstream *pout = NULL; struct searchargs *searchargs = NULL; char *str = NULL; search_expr_t *e = NULL; int i; int r; struct timeval start_time, end_time; for (i = 0 ; i < nwords ; i++) { if (i) buf_putc(&querytext, ' '); buf_appendcstr(&querytext, words[i]); } if (verbose) fprintf(stderr, "search_test: IMAP query is \"%s\"\n", buf_cstring(&querytext)); buf_putc(&querytext, '\r'); buf_cstring(&querytext); r = mboxname_init_namespace(&ns, /*isadmin*/0); if (r) { fprintf(stderr, "Failed to initialise namespace: %s\n", error_message(r)); goto out; } pin = prot_readmap(querytext.s, querytext.len); pout = prot_new(/*fd*/0, /*write*/1); searchargs = new_searchargs(".", GETSEARCH_CHARSET_KEYWORD, &ns, userid, auth_newstate(userid), /*isadmin*/0); r = get_search_program(pin, pout, searchargs); if (r != '\r') { fprintf(stderr, "Couldn't parse IMAP search program\n"); goto out; } gettimeofday(&start_time, NULL); str = search_expr_serialise(searchargs->root); gettimeofday(&end_time, NULL); if (verbose) fprintf(stderr, "search_test: serialised query in %.6f sec\n", timesub(&start_time, &end_time)); gettimeofday(&start_time, NULL); e = search_expr_unserialise(str); gettimeofday(&end_time, NULL); if (verbose) fprintf(stderr, "search_test: unserialised query in %.6f sec\n", timesub(&start_time, &end_time)); out: if (pin) prot_free(pin); if (pout) prot_free(pout); if (searchargs) freesearchargs(searchargs); if (e) search_expr_free(e); free(str); return !!r; } /* ====================================================================== */ int main(int argc, char **argv) { int c; const char *alt_config = NULL; const char *userid = NULL; const char *mboxname = NULL; enum { SEARCH, SERIALISE } mode = SEARCH; int multiple = 0; int r = 0; while ((c = getopt(argc, argv, "C:LMSm:u:v")) != EOF) { switch (c) { case 'C': /* alt config file */ alt_config = optarg; break; case 'L': mode = SERIALISE; break; case 'M': multiple = 1; break; case 'S': multiple = 0; break; case 'm': mboxname = optarg; break; case 'u': userid = optarg; break; case 'v': verbose++; break; default: usage(argv[0]); break; } } if (optind == argc) usage(argv[0]); if (mode == SEARCH && !mboxname) usage(argv[0]); cyrus_init(alt_config, "search_test", CYRUSINIT_PERROR, CONFIG_NEED_PARTITION_DATA); char *freeme = NULL; switch (mode) { case SEARCH: if (!userid) { userid = freeme = mboxname_to_userid(mboxname); if (!userid) usage(argv[0]); } r = do_search(mboxname, multiple, userid, argv+optind, argc-optind); free(freeme); break; case SERIALISE: r = do_serialise(argv+optind, argc-optind); break; } cyrus_done(); return r; } static int usage(const char *name) { fprintf(stderr, "usage: %s [format-options] -m mailbox -u userid searchprogram...\n", name); exit(EX_USAGE); } -void fatal(const char* s, int code) +EXPORTED void fatal(const char* s, int code) { fprintf(stderr, "search_test: %s\n", s); cyrus_done(); exit(code); } diff --git a/imap/xapian_wrap.cpp b/imap/xapian_wrap.cpp index 367812bbd..c656bd8f5 100644 --- a/imap/xapian_wrap.cpp +++ b/imap/xapian_wrap.cpp @@ -1,2498 +1,2518 @@ #include #include #include #include #include #include #include #include #include extern "C" { #include #include "libconfig.h" #include "util.h" #include "search_engines.h" #include "search_part.h" #include "xmalloc.h" #include "xapian_wrap.h" #include "charset.h" #include "ptrarray.h" #include "parseaddr.h" /* generated headers are not necessarily in current directory */ #include "imap/imap_err.h" }; #include #include #include #ifdef HAVE_CLD2 #include #endif // from global.h extern int charset_flags; #define SLOT_CYRUSID 0 #define SLOT_DOCLANGS 1 #define SLOT_INDEXLEVEL 2 #define SLOT_INDEXVERSION 3 static const unsigned XAPIAN_MAX_TERM_LENGTH = 200; /* in UTF-8 bytes */ /* ====================================================================== */ static void make_cyrusid(struct buf *dst, const struct message_guid *guid, char doctype) { buf_reset(dst); buf_putc(dst, '*'); buf_putc(dst, doctype); buf_putc(dst, '*'); buf_appendcstr(dst, message_guid_encode(guid)); buf_cstring(dst); } /* ====================================================================== */ /* * A brief history of Xapian db versions: * Version 0: uses STEM_ALL for all terms, term prefixes don't start with 'X' * Version 1: term prefixes start with 'X' * Version 2: uses STEM_SOME for some terms * Version 3: removes all use of STEM_ALL * Version 4: indexes headers and bodies in separate documents * Version 5: indexes headers and bodies together and stems by language * Version 6: stores all detected languages of a document in slot SLOT_DOCLANGS (deprecated) * Version 7: indexes new DELIVEREDTO search part * Version 8: reintroduces language indexing for non-English text * Version 9: introduces index levels as keys to cyrusid metadata * Version 10: indexes new PRIORITY search part * Version 11: indexes LIST-ID as single value * Version 12: indexes email domains as single values. Supports subdomain search. * Version 13: indexes content-type and subtype separately * Version 14: adds SLOT_INDEXVERSION to documents * Version 15: receives indexed header fields and text in original format (rather than search form) * Version 16: indexes entire addr-spec as a single value. Prevents cross-matching localparts and domains */ #define XAPIAN_DB_CURRENT_VERSION 16 #define XAPIAN_DB_MIN_SUPPORTED_VERSION 5 static std::set read_db_versions(const Xapian::Database &database) { std::set versions; // db_version is a comma-separated list of version numbers std::string val = database.get_metadata("cyrus.db_version"); if (!val.empty()) { strarray_t *vstr = strarray_split(val.c_str(), ",", 0); for (int i = 0; i < strarray_size(vstr); i++) { int version = std::atoi(strarray_nth(vstr, i)); if (version) versions.insert(version); } strarray_free(vstr); } // Up to version 3 this was named stem version. val = database.get_metadata("cyrus.stem-version"); if (!val.empty()) { versions.insert(std::stoi(val)); } return versions; } static void write_db_versions(Xapian::WritableDatabase &database, std::set &versions) { std::ostringstream val; for (std::set::iterator it = versions.begin(); it != versions.end(); ++it) { if (it != versions.begin()) val << ","; val << *it; } database.set_metadata("cyrus.db_version", val.str()); database.set_metadata("cyrus.stem-version", ""); } /* ====================================================================== */ #define XAPIAN_LANG_COUNT_KEYPREFIX "lang.count" #define XAPIAN_LANG_DOC_KEYPREFIX "lang.doc" static std::string lang_prefix(const std::string& iso_lang, const char *prefix) { std::string ustr = std::string(prefix) + "XI" + iso_lang; std::transform(ustr.begin(), ustr.end(), ustr.begin(), ::toupper); return ustr; } static std::string lang_doc_key(const char *cyrusid) { std::string key(XAPIAN_LANG_DOC_KEYPREFIX "."); key += cyrusid; return key; } static std::string lang_count_key(const std::string& iso_lang) { std::string key(XAPIAN_LANG_COUNT_KEYPREFIX "."); key += iso_lang; return key; } static int calculate_language_counts(const Xapian::Database& db, std::map& lang_counts) { std::set db_versions = read_db_versions(db); if (db_versions.lower_bound(8) == db_versions.begin()) { // count all indexed body parts size_t nparts = 0; for (Xapian::TermIterator it = db.metadata_keys_begin("cyrusid.*P*"); it != db.metadata_keys_end("cyrusid.*P*"); ++it) { nparts++; } // count body parts with language metadata const std::string prefix{XAPIAN_LANG_DOC_KEYPREFIX ".*P*"}; size_t nlangparts = 0; for (Xapian::TermIterator it = db.metadata_keys_begin(prefix); it != db.metadata_keys_end(prefix); ++it) { lang_counts[db.get_metadata(*it)] += 1; nlangparts++; } // English or unknown language body parts have no metadata. lang_counts["en"] += nparts - nlangparts; // Sanity check data if (nparts < nlangparts) { return IMAP_IOERROR; } } return 0; } static void remove_legacy_metadata(Xapian::WritableDatabase& db) { const std::string prefix{XAPIAN_LANG_DOC_KEYPREFIX "."}; for (Xapian::TermIterator key = db.metadata_keys_begin(prefix); key != db.metadata_keys_end(prefix); ++key) { const std::string& val = db.get_metadata(*key); // Remove legacy keys and values. if ((*key).find('.') != std::string::npos || (!val.empty() && !isalpha(val[0]))) { db.set_metadata(*key, ""); } } for (Xapian::docid docid = 1; docid <= db.get_lastdocid(); ++docid) { try { Xapian::Document doc = db.get_document(docid); const std::string& val = doc.get_value(SLOT_DOCLANGS); // Remove legacy doclang slot values. if (!val.empty() && !isalpha(val[0])) { doc.remove_value(SLOT_DOCLANGS); } } catch (Xapian::DocNotFoundError e) { // ignore } } } static void write_language_counts(Xapian::WritableDatabase& db, const std::map& lang_counts) { for (Xapian::TermIterator it = db.metadata_keys_begin(XAPIAN_LANG_COUNT_KEYPREFIX); it != db.metadata_keys_end(XAPIAN_LANG_COUNT_KEYPREFIX); ++it) { db.set_metadata(*it, ""); } for (const std::pair& it : lang_counts) { db.set_metadata(lang_count_key(it.first), std::to_string(it.second)); } } static void read_language_counts(const Xapian::Database& db, std::map& lang_counts) { std::set db_versions = read_db_versions(db); if (db_versions.lower_bound(8) == db_versions.begin()) { const std::string prefix(XAPIAN_LANG_COUNT_KEYPREFIX "."); for (Xapian::TermIterator it = db.metadata_keys_begin(prefix); it != db.metadata_keys_end(prefix); ++it) { std::string iso_lang = (*it).substr(prefix.length()); unsigned count = std::stol(db.get_metadata(*it)); lang_counts[iso_lang] += count; } } } static void parse_doclangs(const std::string& val, std::set& doclangs) { if (val.empty() || !isalpha(val[0])) return; size_t base = 0, pos; while ((pos = val.find(',', base)) != std::string::npos) { doclangs.insert(val.substr(base, pos - base)); base = pos + 1; } doclangs.insert(val.substr(base)); } static std::string format_doclangs(const std::set& doclangs) { std::ostringstream val; for (std::set::iterator it = doclangs.begin(); it != doclangs.end(); ++it) { if (it != doclangs.begin()) val << ","; val << *it; } std::string s = val.str(); return s; } static std::string parse_langcode(const char *str) { std::string lstr(str); std::transform(lstr.begin(), lstr.end(), lstr.begin(), ::tolower); // accept syntax for two and three letter ISO 639 codes if (!(isalpha(lstr[0]) && isalpha(lstr[1]) && (lstr[2] == '\0' || (isalpha(lstr[2]) && lstr[3] == '\0')))) { return std::string(); } return lstr; } // Process-scoped, thread-unsafe cache of stoppers by ISO 639 code. static std::map> stoppers; static const Xapian::Stopper* get_stopper(const std::string& iso) { // Lookup cached entry. try { return stoppers.at(iso).get(); } catch (const std::out_of_range&) {}; // Lookup language name by ISO code. icu::Locale loc(iso.c_str()); if (loc.isBogus()) return NULL; // Read stopper file and add to cache. const char *swpath = config_getstring(IMAPOPT_SEARCH_STOPWORD_PATH); if (!swpath) return NULL; std::string lang_name; icu::UnicodeString ulang_name; loc.getDisplayLanguage(icu::Locale("en"), ulang_name); ulang_name.toLower(); ulang_name.toUTF8String(lang_name); // Open stopword file // XXX doesn't play nice with WIN32 paths std::string fname(std::string(swpath) + "/" + lang_name + ".txt"); errno = 0; std::ifstream inFile (fname); if (inFile.fail()) { syslog(LOG_DEBUG, "Xapian: could not open stopword file %s: %s", fname.c_str(), errno ? strerror(errno) : "unknown error"); return NULL; } // Create and store the Xapian stopper stoppers[iso].reset(new Xapian::SimpleStopper( std::istream_iterator(inFile), std::istream_iterator())); return stoppers[iso].get(); } class CyrusSearchStemmer : public Xapian::StemImplementation { charset_t utf8 {charset_lookupname("utf-8")}; std::map cache; Xapian::Stem stem {"en"}; public: virtual ~CyrusSearchStemmer() { charset_free(&utf8); } virtual std::string operator() (const std::string &word) override { // Is this word already in the cache? try { return cache.at(word); } catch (const std::out_of_range&) {} // Convert the word to search form std::unique_ptr q {charset_convert(word.c_str(), utf8, charset_flags), std::free}; std::string s = q ? stem(Xapian::Unicode::tolower(q.get())) : stem(word); if (s.size() > XAPIAN_MAX_TERM_LENGTH) return std::string{}; // Store the normalized word in the cache return cache[word] = s; } virtual std::string get_description () const override { return "Cyrus"; } }; class FrenchContractionStemmer : public Xapian::StemImplementation { Xapian::Stem stem {"fr"}; public: virtual std::string operator() (const std::string &word) override { size_t pos = 0; switch (word[0]) { case 'q': if (word.length() <= 3 || word[1] != 'u') { break; } pos++; // fall through case 'c': case 'd': case 'j': case 'l': case 'm': case 'n': case 's': case 't': // APOSTROPHE (U+0027) if (word.length() > pos + 2 && word[pos+1] == 0x27) { return stem(word.substr(pos + 2)); } // RIGHT SINGLE QUOTATION MARK (U+2019) // FULLWIDTH APOSTROPHE (U+FF07) else if (!word.compare(pos + 1, 3, "\xe2\x80\x99") || !word.compare(pos + 1, 3, "\xef\xbc\x87")) { return stem(word.substr(pos + 4)); } // fall through } // not a contraction return stem(word); } virtual std::string get_description () const override { return "fr-contraction"; } }; static Xapian::Stem get_stemmer(const std::string& iso_lang) { return iso_lang == "fr" ? Xapian::Stem{new FrenchContractionStemmer} : Xapian::Stem{iso_lang}; } #ifdef HAVE_CLD2 static std::string detect_language(const struct buf *part) { std::string iso_lang; bool reliable = false; CLD2::Language lang = CLD2::DetectLanguage(part->s, part->len, 1, &reliable); if (reliable && lang != CLD2::UNKNOWN_LANGUAGE) { std::string code(CLD2::LanguageCode(lang)); std::transform(code.begin(), code.end(), code.begin(), ::tolower); // Map CLD2 special codes to ISO 639. if (!code.compare("zh-Hant")) { code = "zh"; } else if (!code.compare("sr-ME" )) { code = "sr"; // not a political statement! } else if (!code.compare("xxx")) { code = ""; } iso_lang = parse_langcode(code.c_str()); } return iso_lang; } #endif /* HAVE_CLD2 */ /* ====================================================================== */ static uint8_t better_indexlevel(uint8_t levela, uint8_t levelb) { uint8_t a = levela & ~SEARCH_INDEXLEVEL_PARTIAL; uint8_t b = levelb & ~SEARCH_INDEXLEVEL_PARTIAL; if (a > b) return levela; if (a < b) return levelb; return (levela & SEARCH_INDEXLEVEL_PARTIAL) ? levelb : levela; } static uint8_t parse_indexlevel(const std::string& s) { uint8_t level = 0; if (hex_to_bin(s.c_str(), s.length(), &level) != 1) { return 0; } return level; } static std::string format_indexlevel(uint8_t level) { char hex[4]; bin_to_lchex(&level, 1, hex); return std::string(hex, 2); } /* ====================================================================== */ class CyrusMetadataCompactor : public Xapian::Compactor { public: CyrusMetadataCompactor() { } std::string resolve_duplicate_metadata(const std::string &key, size_t num_tags, const std::string tags[]) { if (key.rfind("cyrusid.", 0) == 0) { uint8_t indexlevel = parse_indexlevel(tags[0]); size_t bestpos = 0; for (size_t i = 1; i < num_tags; i++) { uint8_t level = parse_indexlevel(tags[i]); if (better_indexlevel(indexlevel, level) == level) { indexlevel = level; bestpos = i; } } return tags[bestpos]; } return tags[0]; } }; -int xapian_compact_dbs(const char *dest, const char **sources) +EXPORTED int xapian_compact_dbs(const char *dest, const char **sources) { int r = 0; Xapian::Database db; const char *thispath = "(unknown path)"; try { std::set db_versions; std::map lang_counts; std::vector subdbs; while (*sources) { thispath = *sources; Xapian::Database subdb(*sources++); db.add_database(subdb); subdbs.push_back(subdb); // Aggregate db versions. bool need_metadata = false; for (Xapian::docid docid = 1; docid <= subdb.get_lastdocid(); ++docid) { try { Xapian::Document doc = subdb.get_document(docid); const std::string& val = doc.get_value(SLOT_INDEXVERSION); if (!val.empty()) { int version = std::atoi(val.c_str()); if (version) db_versions.insert(version); } else need_metadata = true; } catch (Xapian::DocNotFoundError e) { // ignore } } if (need_metadata) { /* At least one document didn't have its index version set. * Read the legacy version from the metadata. */ std::set md_versions = read_db_versions(subdb); db_versions.insert(md_versions.begin(), md_versions.lower_bound(14)); } // Aggregate language counts. r = calculate_language_counts(subdb, lang_counts); if (r) { xsyslog(LOG_ERR, "IOERROR: corrupt language metadata", "path=<%s>", thispath); return r; } } thispath = "(unknown path)"; // Compact database. static CyrusMetadataCompactor comp; // FULLER because we never write to compression targets again. db.compact(dest, Xapian::Compactor::FULLER | Xapian::DBCOMPACT_MULTIPASS, 0, comp); Xapian::WritableDatabase newdb(dest); write_db_versions(newdb, db_versions); // Clean metadata. remove_legacy_metadata(newdb); // Reset language counts. write_language_counts(newdb, lang_counts); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); r = IMAP_IOERROR; } return r; } /* ====================================================================== */ static const char *get_term_prefix(int db_version, int partnum) { /* * We use term prefixes to store terms per search part. * In addition, each Xapian document contains a "XE" * prefix to indicate its document type, listed in * the XAPIAN_WRAP_DOCTYPE definitions. The "XE" prefix * MUST not be used for any search part. * */ static const char * const term_prefixes[SEARCH_NUM_PARTS] = { NULL, /* ANY */ "XF", /* FROM */ "XT", /* TO */ "XC", /* CC */ "XB", /* BCC */ "XS", /* SUBJECT */ "XL", /* LISTID */ "XY", /* TYPE */ "XH", /* HEADERS */ "", /* BODY */ "XO", /* LOCATION */ "XA", /* ATTACHMENTNAME */ "XAB", /* ATTACHMENTBODY */ "XDT", /* DELIVEREDTO */ "XI", /* LANGUAGE */ "XP" /* PRIORITY */ }; static const char * const term_prefixes_v0[SEARCH_NUM_PARTS] = { NULL, /* ANY */ "F", /* FROM */ "T", /* TO */ "C", /* CC */ "B", /* BCC */ "S", /* SUBJECT */ "L", /* LISTID */ "Y", /* TYPE */ "H", /* HEADERS */ "D", /* BODY */ "O", /* LOCATION */ "A", /* ATTACHMENTNAME */ "AB", /* ATTACHMENTBODY */ "E", /* DELIVEREDTO */ NULL, /* LANGUAGE */ NULL /* PRIORITY */ }; return db_version > 0 ? term_prefixes[partnum] : term_prefixes_v0[partnum]; } static Xapian::TermGenerator::stem_strategy get_stem_strategy(int db_version, int partnum) { static Xapian::TermGenerator::stem_strategy stem_strategy[SEARCH_NUM_PARTS] = { // Version 2 and higher Xapian::TermGenerator::STEM_NONE, /* ANY */ Xapian::TermGenerator::STEM_NONE, /* FROM */ Xapian::TermGenerator::STEM_NONE, /* TO */ Xapian::TermGenerator::STEM_NONE, /* CC */ Xapian::TermGenerator::STEM_NONE, /* BCC */ Xapian::TermGenerator::STEM_SOME, /* SUBJECT */ Xapian::TermGenerator::STEM_NONE, /* LISTID */ Xapian::TermGenerator::STEM_NONE, /* TYPE */ Xapian::TermGenerator::STEM_NONE, /* HEADERS */ Xapian::TermGenerator::STEM_SOME, /* BODY */ Xapian::TermGenerator::STEM_SOME, /* LOCATION */ Xapian::TermGenerator::STEM_NONE, /* ATTACHMENTNAME */ Xapian::TermGenerator::STEM_SOME, /* ATTACHMENTBODY */ Xapian::TermGenerator::STEM_NONE, /* DELIVEREDTO */ Xapian::TermGenerator::STEM_NONE, /* LANGUAGE */ Xapian::TermGenerator::STEM_NONE /* PRIORITY */ }; static Xapian::TermGenerator::stem_strategy stem_strategy_v1[SEARCH_NUM_PARTS] = { // Version 1: Stem bodies using STEM_SOME with stopwords Xapian::TermGenerator::STEM_NONE, /* ANY */ Xapian::TermGenerator::STEM_ALL, /* FROM */ Xapian::TermGenerator::STEM_ALL, /* TO */ Xapian::TermGenerator::STEM_ALL, /* CC */ Xapian::TermGenerator::STEM_ALL, /* BCC */ Xapian::TermGenerator::STEM_ALL, /* SUBJECT */ Xapian::TermGenerator::STEM_ALL, /* LISTID */ Xapian::TermGenerator::STEM_ALL, /* TYPE */ Xapian::TermGenerator::STEM_ALL, /* HEADERS */ Xapian::TermGenerator::STEM_SOME, /* BODY */ Xapian::TermGenerator::STEM_SOME, /* LOCATION */ Xapian::TermGenerator::STEM_NONE, /* ATTACHMENTNAME */ Xapian::TermGenerator::STEM_SOME, /* ATTACHMENTBODY */ Xapian::TermGenerator::STEM_ALL, /* DELIVEREDTO */ Xapian::TermGenerator::STEM_NONE, /* LANGUAGE */ Xapian::TermGenerator::STEM_NONE /* PRIORITY */ }; static Xapian::TermGenerator::stem_strategy stem_strategy_v0[SEARCH_NUM_PARTS] = { // Version 0: Initial version Xapian::TermGenerator::STEM_NONE, /* ANY */ Xapian::TermGenerator::STEM_ALL, /* FROM */ Xapian::TermGenerator::STEM_ALL, /* TO */ Xapian::TermGenerator::STEM_ALL, /* CC */ Xapian::TermGenerator::STEM_ALL, /* BCC */ Xapian::TermGenerator::STEM_ALL, /* SUBJECT */ Xapian::TermGenerator::STEM_ALL, /* LISTID */ Xapian::TermGenerator::STEM_ALL, /* TYPE */ Xapian::TermGenerator::STEM_ALL, /* HEADERS */ Xapian::TermGenerator::STEM_ALL, /* BODY */ Xapian::TermGenerator::STEM_ALL, /* LOCATION */ Xapian::TermGenerator::STEM_ALL, /* ATTACHMENTNAME */ Xapian::TermGenerator::STEM_ALL, /* ATTACHMENTBODY */ Xapian::TermGenerator::STEM_ALL, /* DELIVEREDTO */ Xapian::TermGenerator::STEM_NONE, /* LANGUAGE */ Xapian::TermGenerator::STEM_NONE /* PRIORITY */ }; switch (db_version) { case 0: return stem_strategy_v0[partnum]; case 1: return stem_strategy_v1[partnum]; default: return stem_strategy[partnum]; } } /* For all db paths in sources that are not using the latest database * version or not readable, report their paths in toreindex */ -void xapian_check_if_needs_reindex(const strarray_t *sources, strarray_t *toreindex, int always_upgrade) +EXPORTED void xapian_check_if_needs_reindex(const strarray_t *sources, + strarray_t *toreindex, + int always_upgrade) { // Check the version of all dbs in sources for (int i = 0; i < sources->count; i++) { const char *thispath = strarray_nth(sources, i); try { for (const int& it: read_db_versions(Xapian::Database{thispath})) { if (it < XAPIAN_DB_MIN_SUPPORTED_VERSION || (always_upgrade && (it != XAPIAN_DB_CURRENT_VERSION))) { strarray_add(toreindex, thispath); } } } catch (const Xapian::Error &err) { strarray_add(toreindex, thispath); } } } /* ====================================================================== */ static inline void add_boolean_nterm(Xapian::Document& doc, const std::string& term) { if (term.size() && term.size() < XAPIAN_MAX_TERM_LENGTH) { doc.add_boolean_term(term); } } struct xapian_dbw { // Database context. Xapian::WritableDatabase *database; ptrarray_t otherdbs; Xapian::TermGenerator *term_generator; Xapian::Stem *default_stemmer; const Xapian::Stopper* default_stopper; // Document context. Xapian::Document *document; char doctype; char *cyrusid; std::set *doclangs; std::vector *subjects; }; static int xapian_dbw_init(xapian_dbw_t *dbw) { dbw->default_stemmer = new Xapian::Stem(new CyrusSearchStemmer); dbw->default_stopper = get_stopper("en"); dbw->term_generator = new Xapian::TermGenerator; dbw->term_generator->set_max_word_length(XAPIAN_MAX_TERM_LENGTH); /* Always enable CJK word tokenization */ #ifdef USE_XAPIAN_CJK_WORDS dbw->term_generator->set_flags(Xapian::TermGenerator::FLAG_CJK_WORDS, ~Xapian::TermGenerator::FLAG_CJK_WORDS); #else dbw->term_generator->set_flags(Xapian::TermGenerator::FLAG_CJK_NGRAM, ~Xapian::TermGenerator::FLAG_CJK_NGRAM); #endif dbw->doclangs = new std::set; dbw->subjects = new std::vector; return 0; } -int xapian_dbw_open(const char **paths, xapian_dbw_t **dbwp, int mode, int nosync) +EXPORTED int xapian_dbw_open(const char **paths, xapian_dbw_t **dbwp, + int mode, int nosync) { xapian_dbw_t *dbw = (xapian_dbw_t *)xzmalloc(sizeof(xapian_dbw_t)); int r = 0; const char *thispath = *paths++; std::set db_versions; try { int flags = Xapian::DB_BACKEND_GLASS|Xapian::DB_RETRY_LOCK; if (nosync) flags |= Xapian::DB_DANGEROUS|Xapian::DB_NO_SYNC; try { dbw->database = new Xapian::WritableDatabase{thispath, flags|Xapian::DB_OPEN}; db_versions = read_db_versions(*dbw->database); } catch (Xapian::DatabaseOpeningError &e) { /* It's OK not to atomically create or open, since we can assume * the xapianactive file items to be locked. */ dbw->database = new Xapian::WritableDatabase{thispath, flags|Xapian::DB_CREATE}; } if (db_versions.find(XAPIAN_DB_CURRENT_VERSION) == db_versions.end()) { // Always index using latest database version. db_versions.insert(XAPIAN_DB_CURRENT_VERSION); write_db_versions(*dbw->database, db_versions); } r = xapian_dbw_init(dbw); } catch (const Xapian::DatabaseLockError &err) { /* somebody else is already indexing this user. They may be doing a different * mailbox, so we need to re-insert this mailbox into the queue! */ r = IMAP_MAILBOX_LOCKED; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); r = IMAP_IOERROR; } if (r) { xapian_dbw_close(dbw); return r; } /* open the read-only databases */ if (mode == XAPIAN_DBW_XAPINDEXED) { while (*paths) { try { thispath = *paths; ptrarray_append(&dbw->otherdbs, new Xapian::Database{*paths++}); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: reading database", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); } } } *dbwp = dbw; return 0; } -void xapian_dbw_close(xapian_dbw_t *dbw) +EXPORTED void xapian_dbw_close(xapian_dbw_t *dbw) { if (!dbw) return; try { delete dbw->database; delete dbw->term_generator; delete dbw->document; delete dbw->default_stemmer; delete dbw->doclangs; delete dbw->subjects; for (int i = 0; i < dbw->otherdbs.count; i++) { delete (Xapian::Database *)ptrarray_nth(&dbw->otherdbs, i); } ptrarray_fini(&dbw->otherdbs); free(dbw->cyrusid); free(dbw); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); } } -int xapian_dbw_begin_txn(xapian_dbw_t *dbw) +EXPORTED int xapian_dbw_begin_txn(xapian_dbw_t *dbw) { int r = 0; try { dbw->database->begin_transaction(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_dbw_commit_txn(xapian_dbw_t *dbw) +EXPORTED int xapian_dbw_commit_txn(xapian_dbw_t *dbw) { int r = 0; try { dbw->database->commit_transaction(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_dbw_cancel_txn(xapian_dbw_t *dbw) +EXPORTED int xapian_dbw_cancel_txn(xapian_dbw_t *dbw) { int r = 0; try { dbw->database->cancel_transaction(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_dbw_begin_doc(xapian_dbw_t *dbw, const struct message_guid *guid, char doctype) +EXPORTED int xapian_dbw_begin_doc(xapian_dbw_t *dbw, + const struct message_guid *guid, + char doctype) { int r = 0; try { delete dbw->document; dbw->document = new Xapian::Document; dbw->doctype = doctype; /* Set document id and type */ struct buf buf = BUF_INITIALIZER; make_cyrusid(&buf, guid, doctype); dbw->document->add_value(SLOT_CYRUSID, buf_cstring(&buf)); dbw->cyrusid = buf_release(&buf); add_boolean_nterm(*dbw->document, std::string("XE") + doctype); /* Initialize term generator */ dbw->term_generator->set_document(*dbw->document); dbw->term_generator->set_termpos(1); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } static int add_language_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); std::string val = parse_langcode(buf_cstring(part)); if (val.empty()) { syslog(LOG_INFO, "Xapian: not a valid ISO 639 code: %s", buf_cstring(part)); return 0; } add_boolean_nterm(*dbw->document, prefix + val); return 0; } static std::string parse_priority(const char *str) { const char *err; uint32_t u; if (parseuint32(str, &err, &u) == -1 || *err || u == 0) { return std::string(); } return std::to_string(u); } static int add_priority_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); if (buf_len(part)) { std::string val = parse_priority(buf_cstring(part)); if (val.empty()) { syslog(LOG_DEBUG, "Xapian: not a valid priority: %s", buf_cstring(part)); return 0; } add_boolean_nterm(*dbw->document, prefix + val); } return 0; } static std::string parse_listid(const char *str) { std::string val; /* Extract list-id */ const char *start = strrchr(str, '<'); if (start) { /* RFC2919 list-id header (with optional closing bracket) */ const char *end = strchr(++start, '>'); if (end) val = std::string(start, end - start); else val = std::string(start); } else { /* Groups-style header: 'list list-id[; contact list-contact]' * As seen at Google Group, Yahoo, et al. */ for (start = str; isspace(*start); start++) {} if (!strncasecmp("list", start, 4) && isspace(start[4])) { for (start = start + 4; isspace(*start); start++) {} if (*start) { const char *end = strchr(start, ';'); if (!end || end - start) { val = end ? std::string(start, end - start) : std::string{start}; } } } /* just raw value, that's OK too, like sentry creates. Parse up to first whitespace */ else { const char *end; for (end = start; *end && !isspace(*end); end++) {} val = std::string(start, end - start); } } /* Normalize list-id */ val.erase(std::remove_if(val.begin(), val.end(), isspace), val.end()); std::transform(val.begin(), val.end(), val.begin(), ::tolower); return val; } static int add_listid_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); /* Normalize list-id */ std::string val = parse_listid(buf_cstring(part)); val.erase(std::remove_if(val.begin(), val.end(), isspace), val.end()); std::transform(val.begin(), val.end(), val.begin(), ::tolower); if (val.empty()) { syslog(LOG_WARNING, "Xapian: not a valid list-id: %s", buf_cstring(part)); return 0; } add_boolean_nterm(*dbw->document, prefix + val); return 0; } static int add_email_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); std::string lpart = Xapian::Unicode::tolower(buf_cstring(part)); struct address_itr itr; address_itr_init(&itr, lpart.c_str(), 0); const struct address *addr; while ((addr = address_itr_next(&itr))) { if (addr->invalid) { continue; } if (addr->name) { dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); dbw->term_generator->index_text(Xapian::Utf8Iterator(addr->name), 1, prefix + 'N'); dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); dbw->term_generator->index_text(Xapian::Utf8Iterator(addr->name), 1, prefix); } if (addr->mailbox) { // index mailbox as single value std::string val(addr->mailbox); // ignore whitespace (as seen in quoted mailboxes) val.erase(std::remove_if(val.begin(), val.end(), isspace), val.end()); add_boolean_nterm(*dbw->document, prefix + 'L' + val); // index individual terms dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); dbw->term_generator->index_text(Xapian::Utf8Iterator(val), 1, prefix); } if (addr->domain && strcmp(addr->domain, "unspecified-domain")) { // index reversed domain std::string val; strarray_t *sa = strarray_split(addr->domain, ".", 0); val.reserve(buf_len(part)); for (int i = strarray_size(sa) - 1; i >= 0; i--) { val.append(strarray_nth(sa, i)); if (i > 0) { val.append(1, '.'); } } strarray_free(sa); add_boolean_nterm(*dbw->document, prefix + "D" + val); // index individual terms dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); dbw->term_generator->index_text(Xapian::Utf8Iterator(addr->domain, strlen(addr->domain)), 1, prefix); } // index entire addr-spec char *a = address_get_all(addr, /*canon_domain*/1); if (a) { add_boolean_nterm(*dbw->document, prefix + 'A' + std::string(a)); free(a); } } address_itr_fini(&itr); return 0; } static std::pair parse_content_type(const char *str) { std::pair ret; struct buf buf = BUF_INITIALIZER; const char *sep = strchr(str, '/'); if (sep) { /* type */ buf_setmap(&buf, str, sep - str); buf_lcase(&buf); buf_trim(&buf); ret.first = std::string(buf_cstring(&buf)); /* subtype */ buf_setcstr(&buf, sep + 1); buf_lcase(&buf); buf_trim(&buf); ret.second = std::string(buf_cstring(&buf)); } else { /* type or subtype */ buf_setcstr(&buf, str); buf_lcase(&buf); buf_trim(&buf); ret.first = std::string(buf_cstring(&buf)); } buf_free(&buf); return ret; } static int add_type_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); std::pair ct = parse_content_type(buf_cstring(part)); if (!ct.first.empty()) { add_boolean_nterm(*dbw->document, prefix + "T" + ct.first); } if (!ct.second.empty()) { add_boolean_nterm(*dbw->document, prefix + "S" + ct.second); } if (!ct.first.empty() && !ct.second.empty()) { add_boolean_nterm(*dbw->document, prefix + ct.first + '/' + ct.second); } return 0; } -int add_text_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) +static int add_text_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { const char *prefix = get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum); int r = 0; // Index text. Xapian::TermGenerator::stem_strategy stem_strategy = get_stem_strategy(XAPIAN_DB_CURRENT_VERSION, partnum); dbw->term_generator->set_stemming_strategy(stem_strategy); if (stem_strategy != Xapian::TermGenerator::STEM_NONE) { if (config_getswitch(IMAPOPT_SEARCH_INDEX_LANGUAGE)){ // Index by language. #ifndef HAVE_CLD2 // XXX is this really an "IOERROR"? xsyslog(LOG_ERR, "IOERROR: language indexing requires CLD2 library", NULL); return IMAP_IOERROR; #else if (search_part_is_body(partnum)) { const std::string iso_lang = detect_language(part); if (!iso_lang.empty()) { if (iso_lang.compare("en")) { // Stem and index by non-default language. try { dbw->term_generator->set_stemmer(get_stemmer(iso_lang)); dbw->term_generator->set_stopper(get_stopper(iso_lang)); dbw->term_generator->index_text(Xapian::Utf8Iterator(part->s, part->len), 1, lang_prefix(iso_lang, prefix)); } catch (const Xapian::InvalidArgumentError &err) { syslog(LOG_DEBUG, "Xapian: no stemmer for language %s", iso_lang.c_str()); } } if (dbw->doctype == 'P') { // Keep track of stemmer language. std::string key = lang_doc_key(dbw->cyrusid); dbw->database->set_metadata(key, iso_lang); dbw->document->add_value(SLOT_DOCLANGS, iso_lang); // Update language counts for body parts. key = lang_count_key(iso_lang); const std::string val = dbw->database->get_metadata(key); dbw->database->set_metadata(key, val.empty() ? "1" : std::to_string(std::stoi(val) + 1)); } // Store detected languages in document. dbw->doclangs->insert(iso_lang.c_str()); add_boolean_nterm(*dbw->document, std::string("XI") + iso_lang); } } else if (partnum == SEARCH_PART_SUBJECT) { // Keep subject text to index by language later. dbw->subjects->push_back(buf_cstring(part)); } #endif /* HAVE_CLD2 */ } // Index with default stemmer. dbw->term_generator->set_stemmer(*dbw->default_stemmer); dbw->term_generator->set_stopper(dbw->default_stopper); } else { // Index with no stemming. dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); } dbw->term_generator->index_text(Xapian::Utf8Iterator(part->s, part->len), 1, prefix); return r; } -int xapian_dbw_doc_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) +EXPORTED int xapian_dbw_doc_part(xapian_dbw_t *dbw, + const struct buf *part, + int partnum) { int r = 0; if (!get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)) { syslog(LOG_ERR, "xapian_wrapper: no prefix for partnum %d", partnum); return IMAP_INTERNAL; } try { // Handle search parts. switch (partnum) { case SEARCH_PART_PRIORITY: r = add_priority_part(dbw, part, partnum); break; case SEARCH_PART_LISTID: r = add_listid_part(dbw, part, partnum); break; case SEARCH_PART_LANGUAGE: r = add_language_part(dbw, part, partnum); break; case SEARCH_PART_FROM: case SEARCH_PART_TO: case SEARCH_PART_CC: case SEARCH_PART_BCC: case SEARCH_PART_DELIVEREDTO: r = add_email_part(dbw, part, partnum); break; case SEARCH_PART_TYPE: r = add_type_part(dbw, part, partnum); break; default: r = add_text_part(dbw, part, partnum); } // Finalize index. dbw->term_generator->increase_termpos(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_dbw_end_doc(xapian_dbw_t *dbw, uint8_t indexlevel) +EXPORTED int xapian_dbw_end_doc(xapian_dbw_t *dbw, uint8_t indexlevel) { int r = 0; assert(indexlevel > 0); try { if (config_getswitch(IMAPOPT_SEARCH_INDEX_LANGUAGE)){ // Keep track of languages used in this message. if (dbw->doctype == 'G') { std::string val = format_doclangs(*dbw->doclangs); dbw->database->set_metadata(lang_doc_key(dbw->cyrusid), val); dbw->document->add_value(SLOT_DOCLANGS, val); } // Index subjects by detected document languages. for (std::set::iterator it = dbw->doclangs->begin(); it != dbw->doclangs->end(); ++it) { std::string iso_lang = *it; if (iso_lang.compare("en")) { try { const char *tp = get_term_prefix(XAPIAN_DB_CURRENT_VERSION, SEARCH_PART_SUBJECT); std::string prefix = lang_prefix(iso_lang, tp); dbw->term_generator->set_stemmer(get_stemmer(iso_lang)); dbw->term_generator->set_stopper(get_stopper(iso_lang)); for (const std::string& subject : *dbw->subjects) dbw->term_generator->index_text(Xapian::Utf8Iterator(subject), 1, prefix); } catch (const Xapian::InvalidArgumentError &err) { // ignore unknown stemmer } } } } dbw->document->add_value(SLOT_INDEXLEVEL, format_indexlevel(indexlevel)); dbw->document->add_value(SLOT_INDEXVERSION, std::to_string(XAPIAN_DB_CURRENT_VERSION)); dbw->database->add_document(*dbw->document); dbw->database->set_metadata("cyrusid." + std::string(dbw->cyrusid), format_indexlevel(indexlevel)); delete dbw->document; dbw->document = 0; dbw->doctype = 0; free(dbw->cyrusid); dbw->cyrusid = NULL; dbw->doclangs->clear(); dbw->subjects->clear(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -unsigned long xapian_dbw_total_length(xapian_dbw_t *dbw) +EXPORTED unsigned long xapian_dbw_total_length(xapian_dbw_t *dbw) { unsigned long res = 0; try { res = dbw->database->get_total_length(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); } return res; } -uint8_t xapian_dbw_is_indexed(xapian_dbw_t *dbw, const struct message_guid *guid, char doctype) +EXPORTED uint8_t xapian_dbw_is_indexed(xapian_dbw_t *dbw, + const struct message_guid *guid, + char doctype) { struct buf buf = BUF_INITIALIZER; make_cyrusid(&buf, guid, doctype); std::string key = "cyrusid." + std::string(buf_cstring(&buf)); buf_free(&buf); /* indexed in the current DB? */ uint8_t indexlevel = parse_indexlevel(dbw->database->get_metadata(key)); if (indexlevel == SEARCH_INDEXLEVEL_BEST || (indexlevel && doctype == XAPIAN_WRAP_DOCTYPE_PART)) { return indexlevel; } /* indexed in other DBs? */ for (int i = 0; i < dbw->otherdbs.count; i++) { Xapian::Database *database = (Xapian::Database *)ptrarray_nth(&dbw->otherdbs, i); uint8_t level = parse_indexlevel(database->get_metadata(key)); if (level == SEARCH_INDEXLEVEL_BEST || (level && doctype == XAPIAN_WRAP_DOCTYPE_PART)) { return level; } else indexlevel = better_indexlevel(indexlevel, level); } return indexlevel; } /* ====================================================================== */ struct xapian_db { std::string *paths; Xapian::Database *database; // all but version 4 databases std::vector *subdbs; // all database subdbs Xapian::Stem *default_stemmer; const Xapian::Stopper* default_stopper; std::set *stem_languages; Xapian::QueryParser *parser; std::set *db_versions; xapian_dbw_t *dbw; }; static int xapian_db_init(xapian_db_t *db) { int r = 0; try { db->parser = new Xapian::QueryParser; db->parser->set_default_op(Xapian::Query::OP_AND); db->default_stemmer = new Xapian::Stem(new CyrusSearchStemmer); db->default_stopper = get_stopper("en"); // Determine stemmer languages (in addition to English). db->stem_languages = new std::set; std::map lang_counts; size_t total_doccount = 0; for (const Xapian::Database& subdb : *db->subdbs) { read_language_counts(subdb, lang_counts); total_doccount += subdb.get_doccount(); } total_doccount /= 2; // Crude estimate. for (std::pair& it : lang_counts) { if (it.first.compare("en") && ((double) it.second / total_doccount) >= 0.05) { db->stem_languages->insert(it.first); } } } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_db_open(const char **paths, xapian_db_t **dbp) +EXPORTED int xapian_db_open(const char **paths, xapian_db_t **dbp) { xapian_db_t *db = (xapian_db_t *)xzmalloc(sizeof(xapian_db_t)); const char *thispath = "(unknown)"; int r = 0; try { db->paths = new std::string; while (paths && *paths) { thispath = *paths++; Xapian::Database subdb {thispath}; std::set db_versions = read_db_versions(subdb); if (db_versions.empty()) { syslog(LOG_ERR, "xapian_wrapper: invalid db version in %s", thispath); r = IMAP_INTERNAL; goto done; } if (!db->db_versions) db->db_versions = new std::set; db->db_versions->insert(db_versions.begin(), db_versions.end()); // Check for experimental v4 indexes, they were bogus. if (db_versions.find(4) != db_versions.end()) { xsyslog(LOG_WARNING, "deprecated v4 index detected, " "search may return wrong results", "db=<%s>", thispath); } // Add database. if (!db->database) db->database = new Xapian::Database; db->database->add_database(subdb); // Xapian database has no API to access subdbs. if (!db->subdbs) db->subdbs = new std::vector; db->subdbs->push_back(subdb); db->paths->append(thispath).push_back(' '); } thispath = "(unknown)"; if (!db->database) { r = IMAP_NOTFOUND; goto done; } r = xapian_db_init(db); if (r) goto done; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); r = IMAP_IOERROR; } done: if (r) xapian_db_close(db); else *dbp = db; return r; } -int xapian_db_opendbw(struct xapian_dbw *dbw, xapian_db_t **dbp) +EXPORTED int xapian_db_opendbw(struct xapian_dbw *dbw, xapian_db_t **dbp) { xapian_db_t *db = (xapian_db_t *)xzmalloc(sizeof(xapian_db_t)); db->dbw = dbw; db->database = dbw->database; db->db_versions = new std::set(); std::set dbw_versions = read_db_versions(*dbw->database); db->db_versions->insert(dbw_versions.begin(), dbw_versions.end()); db->subdbs = new std::vector; db->subdbs->push_back(*dbw->database); int r = xapian_db_init(db); if (r) { xapian_db_close(db); db = NULL; } *dbp = db; return r; } -void xapian_db_close(xapian_db_t *db) +EXPORTED void xapian_db_close(xapian_db_t *db) { if (!db) return; try { if (!db->dbw) delete db->database; delete db->parser; delete db->paths; delete db->db_versions; delete db->default_stemmer; delete db->stem_languages; delete db->subdbs; free(db); } catch (const Xapian::Error &err) { /* XXX - memory leak? */ xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); } } -int xapian_db_langstats(xapian_db_t *db, ptrarray_t* lstats, size_t *nolang) +EXPORTED int xapian_db_langstats(xapian_db_t *db, ptrarray_t* lstats, + size_t *nolang) { std::map lang_counts; size_t total_part = 0; size_t total_lang = 0; for (const Xapian::Database& subdb : *db->subdbs) { // count body parts for (Xapian::TermIterator it = subdb.metadata_keys_begin("cyrusid.*P*"); it != subdb.metadata_keys_end("cyrusid.*P*"); ++it) { total_part++; } // cummulate language counts read_language_counts(subdb, lang_counts); } for (const std::pair& counts : lang_counts) { struct search_langstat *stat = (struct search_langstat*) xzmalloc(sizeof(struct search_langstat)); stat->iso_lang = xstrdup(counts.first.c_str()); stat->count = counts.second; ptrarray_append(lstats, stat); total_lang += counts.second; } *nolang = total_part > total_lang ? total_part - total_lang : 0; return 0; } -void xapian_query_add_stemmer(xapian_db_t *db, const char *iso_lang) +EXPORTED void xapian_query_add_stemmer(xapian_db_t *db, const char *iso_lang) { if (strcmp(iso_lang, "en")) db->stem_languages->insert(iso_lang); } static Xapian::Query* query_new_textmatch(const xapian_db_t *db, const char *match, const char *prefix, Xapian::TermGenerator::stem_strategy tg_stem_strategy) { unsigned flags = Xapian::QueryParser::FLAG_PHRASE | Xapian::QueryParser::FLAG_WILDCARD; std::string lmatch = Xapian::Unicode::tolower(match); if (tg_stem_strategy != Xapian::TermGenerator::STEM_NONE) { // Query without any stemmer. db->parser->set_stemmer(Xapian::Stem()); db->parser->set_stopper(NULL); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_NONE); Xapian::Query q = db->parser->parse_query(lmatch, flags, prefix); // Query with default stemmer. But don't stem stopwords. if (!db->default_stopper || !(*db->default_stopper)(lmatch)) { db->parser->set_stemmer(*db->default_stemmer); db->parser->set_stopper(db->default_stopper); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_SOME); q |= db->parser->parse_query(lmatch, flags, prefix); } // Stem query for each language detected in the index. for (const std::string& iso_lang : *db->stem_languages) { try { const Xapian::Stopper *stopper = get_stopper(iso_lang); db->parser->set_stemmer(get_stemmer(iso_lang)); db->parser->set_stopper(stopper); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_SOME); if (!stopper || !(*stopper)(lmatch)) { q |= db->parser->parse_query(lmatch, flags, lang_prefix(iso_lang, prefix)); } } catch (const Xapian::InvalidArgumentError &err) { syslog(LOG_INFO, "Xapian: no stemmer for language %s", iso_lang.c_str()); } } return new Xapian::Query(q); } else { db->parser->set_stemmer(Xapian::Stem()); db->parser->set_stopper(NULL); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_NONE); return new Xapian::Query {db->parser->parse_query(lmatch, flags, prefix)}; } } static Xapian::Query *query_new_language(const xapian_db_t *db __attribute__((unused)), const char *prefix, const char *str) { std::string val = parse_langcode(str); if (val.empty()) { syslog(LOG_DEBUG, "Xapian: invalid language in query: %s", str); return new Xapian::Query(Xapian::Query::MatchNothing); } return new Xapian::Query(std::string(prefix) + val); } static Xapian::Query *query_new_priority(const xapian_db_t *db __attribute__((unused)), const char *prefix, const char *str) { std::string val = parse_priority(str); if (val.empty()) { syslog(LOG_DEBUG, "Xapian: invalid priority in query: %s", str); return new Xapian::Query(Xapian::Query::MatchNothing); } return new Xapian::Query(std::string(prefix) + val); } static Xapian::Query *query_new_listid(const xapian_db_t *db, const char *prefix, const char *str) { Xapian::Query *q = NULL; std::string val = parse_listid(str); if (!val.empty()) { q = new Xapian::Query(std::string(prefix) + val); } else { syslog(LOG_DEBUG, "Xapian: invalid listid in query: %s", str); q = new Xapian::Query(Xapian::Query::MatchNothing); } if (db->db_versions->lower_bound(11) != db->db_versions->begin()) { // query in legacy format db->parser->set_stemmer(Xapian::Stem()); db->parser->set_stopper(NULL); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_NONE); q = new Xapian::Query(Xapian::Query::OP_OR, *q, db->parser->parse_query(str, 0, prefix)); } return q; } static Xapian::Query *query_new_email(const xapian_db_t *db, const char *_prefix, const char *str) { std::string prefix(_prefix); unsigned qpflags = Xapian::QueryParser::FLAG_PHRASE | Xapian::QueryParser::FLAG_WILDCARD; db->parser->set_stemmer(Xapian::Stem()); db->parser->set_stopper(NULL); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_NONE); std::string mystr = Xapian::Unicode::tolower(str); str = mystr.c_str(); const char *atsign = strchr(str, '@'); if (!atsign) { // query free text return new Xapian::Query{db->parser->parse_query(str, qpflags, prefix)}; } Xapian::Query q = Xapian::Query::MatchNothing; // query name and mailbox (unless just searching for '@domain') if (atsign > str) { struct address *addr = NULL; parseaddr_list(str, &addr); if (addr && addr->name) { Xapian::Query qq = db->parser->parse_query(addr->name, qpflags, prefix + 'N'); if (q.get_type() != q.LEAF_MATCH_NOTHING) { q &= qq; } else q = qq; } if (addr && addr->mailbox) { // strip the domain from the mailbox std::string mail(addr->mailbox); mail.erase(std::remove_if(mail.begin(), mail.end(), isspace), mail.end()); int wildcard = mail[mail.size()-1] == '*'; if (wildcard) { mail.resize(mail.size()-1); } if (!mail.empty()) { std::string term(prefix + 'L' + mail); Xapian::Query qq = wildcard ? Xapian::Query(Xapian::Query::OP_WILDCARD, term) : Xapian::Query(term); if (q.get_type() != q.LEAF_MATCH_NOTHING) { q &= qq; } else q = qq; } } // ignore @domain - it's being handled below if (addr) parseaddr_free(addr); } // query domain if (atsign[1]) { std::string domain; const char *dstart = atsign + 1; bool wildcard = *dstart == '*'; if (wildcard) dstart++; const char *dend; for (dend = dstart; *dend; dend++) { char c = *dend; if (Uisalnum(c) || c == '-' || c == '[' || c == ']' || c == ':') { continue; } else if (c == '.' && (dend-1 == dstart || dend[-2] != '.')) { continue; } else { break; } } if (dend > dstart) { strarray_t *sa = strarray_nsplit(dstart, dend - dstart, ".", 0); for (int i = strarray_size(sa) - 1; i >= 0; i--) { domain.append(strarray_nth(sa, i)); if (i > 0) { domain.append(1, '.'); } } strarray_free(sa); if (*dstart == '.') { domain.append(1, '.'); } } if (!domain.empty()) { std::string term(prefix + 'D' + domain); Xapian::Query qq = wildcard ? Xapian::Query(Xapian::Query::OP_WILDCARD, term) : Xapian::Query(term); { // FIXME - temporarily also search for '@' prefix std::string term2(prefix + '@' + domain); Xapian::Query qq2 = wildcard ? Xapian::Query(Xapian::Query::OP_WILDCARD, term2) : Xapian::Query(term2); qq |= qq2; } if (q.get_type() != q.LEAF_MATCH_NOTHING) { q &= qq; } else q = qq; } } if (q.get_type() == q.LEAF_MATCH_ALL) { q = Xapian::Query::MatchNothing; } // query in legacy format as well! if (db->db_versions->lower_bound(12) != db->db_versions->begin()) { q |= db->parser->parse_query(str, qpflags, prefix); } // query localpart@domain (ONLY if no wildcards) if ((atsign > str) && atsign[1] && !strchr(str, '*')) { struct address *addr = NULL; parseaddr_list(str, &addr); if (addr) { char *a = address_get_all(addr, /*canon_domain*/1); if (a) { // query 'A' term for index >= 16 std::string term(prefix + 'A' + std::string(a)); Xapian::Query qq = Xapian::Query(Xapian::Query::OP_AND, Xapian::Query(Xapian::Query::OP_VALUE_GE, Xapian::valueno(SLOT_INDEXVERSION), std::string("16")), Xapian::Query(term)); if (q.get_type() != q.LEAF_MATCH_NOTHING) { // otherwise, query 'L' + 'D' terms (as per above) Xapian::Query qq2 = Xapian::Query(Xapian::Query::OP_AND, Xapian::Query(Xapian::Query::OP_VALUE_LE, Xapian::valueno(SLOT_INDEXVERSION), std::string("15")), q); qq |= qq2; } q = qq; } parseaddr_free(addr); free(a); } } return new Xapian::Query(q); } static void append_alnum(struct buf *buf, const char *ss) { const unsigned char *s = (const unsigned char *)ss; for ( ; *s ; ++s) { if (Uisalnum(*s)) buf_putc(buf, *s); } } static Xapian::Query *query_new_type(const xapian_db_t *db __attribute__((unused)), const char *_prefix, const char *str) { std::pair ct = parse_content_type(str); std::string prefix(_prefix); Xapian::Query q = Xapian::Query::MatchNothing; bool query_legacy = db->db_versions->lower_bound(13) != db->db_versions->begin(); struct buf buf = BUF_INITIALIZER; unsigned qpflags = Xapian::QueryParser::FLAG_PHRASE | Xapian::QueryParser::FLAG_WILDCARD; if (!ct.first.empty() && ct.second.empty()) { /* Match either type or subtype */ if (ct.first != "*") { q = Xapian::Query(Xapian::Query::OP_OR, Xapian::Query(prefix + 'T' + ct.first), Xapian::Query(prefix + 'S' + ct.first)); if (query_legacy) { append_alnum(&buf, ct.first.c_str()); q |= db->parser->parse_query(buf_cstring(&buf), qpflags, prefix); } } } else if (ct.first == "*" || ct.second == "*") { /* Wildcard query */ if (!ct.first.empty() && ct.first != "*") { /* Match type */ q = Xapian::Query(prefix + 'T' + ct.first); if (query_legacy) { append_alnum(&buf, ct.first.c_str()); q |= db->parser->parse_query(buf_cstring(&buf), qpflags, prefix); } } if (!ct.second.empty() && ct.second != "*") { /* Match subtype */ q = Xapian::Query(prefix + 'S' + ct.second); if (query_legacy) { append_alnum(&buf, ct.second.c_str()); q |= db->parser->parse_query(buf_cstring(&buf), qpflags, prefix); } } } else if (!ct.first.empty() && !ct.second.empty()) { /* Verbatim search */ q = Xapian::Query(prefix + ct.first + '/' + ct.second); if (query_legacy) { append_alnum(&buf, ct.first.c_str()); buf_putc(&buf, '_'); append_alnum(&buf, ct.second.c_str()); q |= db->parser->parse_query(buf_cstring(&buf), qpflags, prefix); } } buf_free(&buf); return new Xapian::Query(q); } -Xapian::Query* +EXPORTED Xapian::Query * xapian_query_new_match_internal(const xapian_db_t *db, int partnum, const char *str) { const char *prefix = get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum); try { // Handle special value search parts. if (partnum == SEARCH_PART_LANGUAGE) { return query_new_language(db, prefix, str); } else if (partnum == SEARCH_PART_PRIORITY) { return query_new_priority(db, prefix, str); } else if (partnum == SEARCH_PART_LISTID) { return query_new_listid(db, prefix, str); } else if (partnum == SEARCH_PART_FROM || partnum == SEARCH_PART_TO || partnum == SEARCH_PART_CC || partnum == SEARCH_PART_BCC || partnum == SEARCH_PART_DELIVEREDTO) { return query_new_email(db, prefix, str); } else if (partnum == SEARCH_PART_TYPE) { return query_new_type(db, prefix, str); } // Don't stem queries for Thaana codepage (0780) or higher. for (const unsigned char *p = (const unsigned char *)str; *p; p++) { if (*p > 221) //has highbit return new Xapian::Query {db->parser->parse_query( str, #ifdef USE_XAPIAN_CJK_WORDS Xapian::QueryParser::FLAG_CJK_WORDS, #else Xapian::QueryParser::FLAG_CJK_NGRAM, #endif prefix)}; } // Stemable codepage. Xapian::TermGenerator::stem_strategy stem_strategy = get_stem_strategy(XAPIAN_DB_CURRENT_VERSION, partnum); Xapian::Query *qq = query_new_textmatch(db, str, prefix, stem_strategy); if (qq->get_type() == Xapian::Query::LEAF_MATCH_NOTHING) { delete qq; qq = NULL; } return qq; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); return 0; } } -xapian_query_t * +EXPORTED xapian_query_t * xapian_query_new_match(const xapian_db_t *db, int partnum, const char *str) { if (db->subdbs->empty()) { // no database to query return NULL; } const char *prefix = get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum); if (!prefix) { return NULL; } int min_version = *db->db_versions->begin(); if (min_version < XAPIAN_DB_MIN_SUPPORTED_VERSION) { xsyslog(LOG_WARNING, "deprecated database version, reindex required", "version=<%d> min_supported_version=<%d> paths=<%s>", min_version, XAPIAN_DB_MIN_SUPPORTED_VERSION, db->paths->c_str()); } Xapian::Query *q = xapian_query_new_match_internal(db, partnum, str); if (min_version < 15) { /* Older versions indexed header fields in Cyrus search form */ charset_t utf8 = charset_lookupname("utf-8"); char *mystr = charset_convert(str, utf8, charset_flags); if (mystr) { Xapian::Query *qq = xapian_query_new_match_internal(db, partnum, mystr); if (qq && q) { *q |= *qq; } else if (!q) q = qq; } free(mystr); charset_free(&utf8); } return (xapian_query_t*) q; } -xapian_query_t *xapian_query_new_compound(const xapian_db_t *db __attribute__((unused)), - int is_or, xapian_query_t **children, int n) +EXPORTED xapian_query_t * +xapian_query_new_compound(const xapian_db_t *db __attribute__((unused)), + int is_or, xapian_query_t **children, int n) { try { // I want to use std::initializer_list here // but that requires "experimental" gcc C++0x support :( // 'compound' owns a refcount on each child. We need to // drop the one we got when we allocated the children Xapian::Query* compound = new Xapian::Query; if (is_or) for (int i = 0 ; i < n ; i++) { *compound |= *(Xapian::Query*)children[i]; delete (Xapian::Query*)children[i]; } else for (int i = 0 ; i < n ; i++) { if (compound->empty()) *compound = *(Xapian::Query*)children[i]; else *compound &= *(Xapian::Query*)children[i]; delete (Xapian::Query*)children[i]; } return (xapian_query_t *)compound; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); return 0; } } /* Xapian does not have an OP_NOT. WTF? We fake it with * OP_AND_NOT where the left child is MatchAll */ -xapian_query_t *xapian_query_new_not(const xapian_db_t *db __attribute__((unused)), - xapian_query_t *child) +EXPORTED xapian_query_t * +xapian_query_new_not(const xapian_db_t *db __attribute__((unused)), + xapian_query_t *child) { if (!child) return (xapian_query_t*) new Xapian::Query(Xapian::Query::MatchAll); try { Xapian::Query *qq = new Xapian::Query( Xapian::Query::OP_AND_NOT, Xapian::Query::MatchAll, *(Xapian::Query *)child); // 'compound' owns a refcount on each child. We need to // drop the one we got when we allocated the children delete (Xapian::Query *)child; return (xapian_query_t *)qq; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); return 0; } } -xapian_query_t *xapian_query_new_matchall(const xapian_db_t *db __attribute__((unused))) +EXPORTED xapian_query_t * +xapian_query_new_matchall(const xapian_db_t *db __attribute__((unused))) { return (xapian_query_t *) new Xapian::Query(Xapian::Query::MatchAll); } -xapian_query_t *xapian_query_new_has_doctype(const xapian_db_t *db __attribute__((unused)), - char doctype, xapian_query_t *child) +EXPORTED xapian_query_t * +xapian_query_new_has_doctype(const xapian_db_t *db __attribute__((unused)), + char doctype, xapian_query_t *child) { try { Xapian::Query *qq = new Xapian::Query( Xapian::Query::OP_FILTER, child ? *(Xapian::Query *)child : Xapian::Query::MatchAll, std::string("XE") + doctype); // 'compound' owns a refcount on each child. We need to // drop the one we got when we allocated the children delete (Xapian::Query *)child; return (xapian_query_t *)qq; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); return 0; } } -void xapian_query_free(xapian_query_t *qq) +EXPORTED void xapian_query_free(xapian_query_t *qq) { try { delete (Xapian::Query *)qq; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); } } -int xapian_query_run(const xapian_db_t *db, const xapian_query_t *qq, - int (*cb)(void *data, size_t n, void *rock), void *rock) +EXPORTED int xapian_query_run(const xapian_db_t *db, const xapian_query_t *qq, + int (*cb)(void *data, size_t n, void *rock), void *rock) { const Xapian::Query *query = (const Xapian::Query *)qq; void *data = NULL; size_t n = 0; if (!db->database) return 0; try { Xapian::Database *database = db->database; Xapian::Enquire enquire(*database); enquire.set_query(*query); enquire.set_sort_by_value(0, false); // sort by cyrusid ascending Xapian::MSet matches = enquire.get_mset(0, database->get_doccount()); size_t size = matches.size(); if (size) data = xzmalloc(size * 41); for (Xapian::MSetIterator i = matches.begin() ; i != matches.end() ; ++i) { const Xapian::Document& d = i.get_document(); const std::string& cyrusid = d.get_value(SLOT_CYRUSID); /* ignore documents with no cyrusid. Shouldn't happen, but has been seen */ if (cyrusid.length() != 43) { xsyslog(LOG_ERR, "IOERROR: skipping document with zero-length cyrusid", "documentid=<%u> paths=<%s>", d.get_docid(), db->paths->c_str()); continue; } const char *cstr = cyrusid.c_str(); if (cstr[0] != '*' || !isalpha(cstr[1]) || cstr[2] != '*') { xsyslog(LOG_ERR, "IOERROR: skipping document with invalid cyrusid", "cyrusid=<%s> documentid=<%u> paths=<%s>", cstr, d.get_docid(), db->paths->c_str()); continue; } if (n >= size) throw Xapian::DatabaseError("Too many records in MSet"); char *entry = (char *) data + (41*n); memcpy(entry, cstr+3, 40); entry[40] = '\0'; ++n; } } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> query=<%s>", err.get_description().c_str(), query ? query->get_description().c_str() : ""); free(data); return IMAP_IOERROR; } if (!n) { free(data); return 0; } int r = cb(data, n, rock); free(data); return r; } /* ====================================================================== */ struct xapian_snipgen { Xapian::Stem *default_stemmer; xapian_db_t *db; Xapian::Database *memdb; std::vector *loose_terms; std::vector *queries; char *cyrusid; char doctype; struct buf *buf; const char *hi_start; const char *hi_end; const char *omit; size_t max_len; }; -xapian_snipgen_t *xapian_snipgen_new(xapian_db_t *db, - const char *hi_start, - const char *hi_end, - const char *omit) +EXPORTED xapian_snipgen_t * +xapian_snipgen_new(xapian_db_t *db, + const char *hi_start, + const char *hi_end, + const char *omit) { xapian_snipgen_t *snipgen = (xapian_snipgen_t *)xzmalloc(sizeof(xapian_snipgen_t)); snipgen->default_stemmer = new Xapian::Stem(new CyrusSearchStemmer); snipgen->db = db; snipgen->memdb = new Xapian::WritableDatabase(std::string(), Xapian::DB_BACKEND_INMEMORY); snipgen->buf = buf_new(); snipgen->hi_start = hi_start; snipgen->hi_end = hi_end; snipgen->omit = omit; snipgen->max_len = (size_t) config_getint(IMAPOPT_SEARCH_SNIPPET_LENGTH); return snipgen; } -void xapian_snipgen_free(xapian_snipgen_t *snipgen) +EXPORTED void xapian_snipgen_free(xapian_snipgen_t *snipgen) { if (!snipgen) return; delete snipgen->default_stemmer; delete snipgen->loose_terms; delete snipgen->queries; delete snipgen->memdb; free(snipgen->cyrusid); buf_destroy(snipgen->buf); free(snipgen); } static Xapian::Query xapian_snipgen_build_query(xapian_snipgen_t *snipgen, Xapian::Stem& stemmer) { Xapian::TermGenerator term_generator; Xapian::Query q; if (snipgen->loose_terms) { /* Add loose query terms */ term_generator.set_stemmer(stemmer); #ifdef USE_XAPIAN_CJK_WORDS term_generator.set_flags(Xapian::TermGenerator::FLAG_CJK_WORDS, ~Xapian::TermGenerator::FLAG_CJK_WORDS); #else term_generator.set_flags(Xapian::TermGenerator::FLAG_CJK_NGRAM, ~Xapian::TermGenerator::FLAG_CJK_NGRAM); #endif for(size_t i = 0; i < snipgen->loose_terms->size(); ++i) { term_generator.index_text(Xapian::Utf8Iterator((*snipgen->loose_terms)[i])); } const Xapian::Document& doc = term_generator.get_document(); q = Xapian::Query(Xapian::Query::OP_OR, doc.termlist_begin(), doc.termlist_end()); } if (snipgen->queries) { /* Add phrase queries */ unsigned flags = Xapian::QueryParser::FLAG_PHRASE| Xapian::QueryParser::FLAG_WILDCARD| #ifdef USE_XAPIAN_CJK_WORDS Xapian::QueryParser::FLAG_CJK_WORDS; #else Xapian::QueryParser::FLAG_CJK_NGRAM; #endif Xapian::QueryParser queryparser; queryparser.set_stemmer(stemmer); for(size_t i = 0; i < snipgen->queries->size(); ++i) { q |= queryparser.parse_query((*snipgen->queries)[i], flags);; } } return q; } -int xapian_snipgen_add_match(xapian_snipgen_t *snipgen, const char *match) +EXPORTED int xapian_snipgen_add_match(xapian_snipgen_t *snipgen, + const char *match) { size_t len = strlen(match); bool is_query = len > 1 && ((match[0] == '"' && match[len-1] == '"') || (strchr(match, '*') != NULL)); if (is_query) { if (!snipgen->queries) { snipgen->queries = new std::vector; } snipgen->queries->push_back(match); } else { if (!snipgen->loose_terms) { snipgen->loose_terms = new std::vector; } snipgen->loose_terms->push_back(match); } return 0; } -int xapian_snipgen_begin_doc(xapian_snipgen_t *snipgen, - const struct message_guid *guid, char doctype) +EXPORTED int xapian_snipgen_begin_doc(xapian_snipgen_t *snipgen, + const struct message_guid *guid, + char doctype) { struct buf buf = BUF_INITIALIZER; make_cyrusid(&buf, guid, doctype); snipgen->cyrusid = buf_release(&buf); snipgen->doctype = doctype; buf_reset(snipgen->buf); return 0; } -int xapian_snipgen_make_snippet(xapian_snipgen_t *snipgen, - const struct buf *part, - Xapian::Stem* stemmer) +EXPORTED int xapian_snipgen_make_snippet(xapian_snipgen_t *snipgen, + const struct buf *part, + Xapian::Stem* stemmer) { int r = 0; try { std::string text {buf_base(part), buf_len(part)}; Xapian::Enquire enquire(*snipgen->memdb); Xapian::Query qq = xapian_snipgen_build_query(snipgen, *stemmer); if (qq.empty()) return 0; enquire.set_query(qq); unsigned flags = Xapian::MSet::SNIPPET_EXHAUSTIVE | Xapian::MSet::SNIPPET_EMPTY_WITHOUT_MATCH; #ifdef USE_XAPIAN_CJK_WORDS flags |= Xapian::MSet::SNIPPET_CJK_WORDS; #endif const std::string snippet = enquire.get_mset(0, 0).snippet(text, snipgen->max_len - buf_len(snipgen->buf), *stemmer, flags, snipgen->hi_start, snipgen->hi_end, snipgen->omit); if (!snippet.empty()) { if (buf_len(snipgen->buf)) { buf_appendoverlap(snipgen->buf, snipgen->omit); } buf_appendcstr(snipgen->buf, snippet.c_str()); } } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_snipgen_doc_part(xapian_snipgen_t *snipgen, const struct buf *part, - int partnum __attribute__((unused))) +EXPORTED int xapian_snipgen_doc_part(xapian_snipgen_t *snipgen, + const struct buf *part, + int partnum __attribute__((unused))) { // Ignore empty queries. if (!snipgen->loose_terms && !snipgen->queries) return 0; // Don't exceed allowed snippet length. if (buf_len(snipgen->buf) >= snipgen->max_len) return 0; if (config_getswitch(IMAPOPT_SEARCH_INDEX_LANGUAGE) && snipgen->db->database && snipgen->cyrusid) { std::set doclangs; // Lookup stemmer language for this document part, if any. std::string key = lang_doc_key(snipgen->cyrusid); for (const Xapian::Database& subdb : *snipgen->db->subdbs) { std::string val = subdb.get_metadata(key); if (!val.empty()) parse_doclangs(val, doclangs); break; } // Generate snippets for each detected message language. // The first non-empty snippet wins. size_t prev_size = buf_len(snipgen->buf); for (std::set::iterator it = doclangs.begin(); it != doclangs.end(); ++it) { const std::string& iso_lang = *it; if (iso_lang.compare("en")) { try { Xapian::Stem stemmer = get_stemmer(iso_lang); int r = xapian_snipgen_make_snippet(snipgen, part, &stemmer); if (!r && prev_size != buf_len(snipgen->buf)) { return 0; } } catch (const Xapian::InvalidArgumentError &err) { // ignore unknown stemmer } } } } /* Using a custom stemmer did not generate a snippet. * This could be because the query matched using the * default stemmer, so try generating a snippet with * that stemmer instead.*/ return xapian_snipgen_make_snippet(snipgen, part, snipgen->default_stemmer); } -int xapian_snipgen_end_doc(xapian_snipgen_t *snipgen, struct buf *buf) +EXPORTED int xapian_snipgen_end_doc(xapian_snipgen_t *snipgen, struct buf *buf) { buf_reset(buf); buf_copy(buf, snipgen->buf); buf_cstring(buf); buf_reset(snipgen->buf); delete snipgen->loose_terms; snipgen->loose_terms = NULL; delete snipgen->queries; snipgen->queries = NULL; free(snipgen->cyrusid); snipgen->cyrusid = NULL; snipgen->doctype = 0; return 0; } /* cb returns true if document should be copied, false if not */ -int xapian_filter(const char *dest, const char **sources, - int (*cb)(const char *cyrusid, void *rock), - void *rock) +EXPORTED int xapian_filter(const char *dest, const char **sources, + int (*cb)(const char *cyrusid, void *rock), + void *rock) { int r = 0; const char *thispath = "(unknown path)"; try { /* create a destination database */ Xapian::WritableDatabase destdb {dest, Xapian::DB_CREATE|Xapian::DB_BACKEND_GLASS}; /* With multiple databases as above, the docids are interleaved, so it * might be worth trying to open each source and copy its documents to * destdb in turn for better locality of reference, and so better cache * use. -- Olly on the mailing list */ std::vector srcdbs; // Open databases and aggregate database-level metadata. while (*sources) { thispath = *sources++; const Xapian::Database srcdb {thispath}; srcdbs.push_back(srcdb); } // Copy all matching documents. std::set db_versions; for (size_t i = 0; i < srcdbs.size(); ++i) { const Xapian::Database& srcdb = srcdbs.at(i); bool need_md_versions = false; std::set md_versions = read_db_versions(srcdb); /* copy all matching documents to the new DB */ for (Xapian::ValueIterator it = srcdb.valuestream_begin(SLOT_CYRUSID); it != srcdb.valuestream_end(SLOT_CYRUSID); ++it) { const std::string& cyrusid = *it; const std::string idkey {"cyrusid." + cyrusid}; // check if caller wants this cyrusid if (!cb(cyrusid.c_str(), rock)) { continue; } // is it already indexed? if (!destdb.get_metadata(idkey).empty()) { continue; } // is there a subsequent db with a better index level? (only for G docs) uint8_t indexlevel = parse_indexlevel(srcdb.get_metadata(idkey)); if (cyrusid[1] == XAPIAN_WRAP_DOCTYPE_MSG) { int found_better = 0; for (size_t j = i + 1; !found_better && j < srcdbs.size(); ++j) { uint8_t level = parse_indexlevel(srcdbs[j].get_metadata(idkey)); found_better = better_indexlevel(indexlevel, level) != indexlevel; } if (found_better) { continue; } } // add document Xapian::Document srcdoc = srcdb.get_document(it.get_docid()); Xapian::docid docid = destdb.add_document(srcdoc); destdb.set_metadata(idkey, format_indexlevel(indexlevel)); // copy document language metadata const std::string& langkey = lang_doc_key(cyrusid.c_str()); if (destdb.get_metadata(langkey).empty()) { std::string val = srcdb.get_metadata(langkey); if (!val.empty() && isalpha(val[0])) { destdb.set_metadata(langkey, val); } } const std::string& langval = srcdoc.get_value(SLOT_DOCLANGS); if (!langval.empty() && !isalpha(langval[0])) { destdb.get_document(docid).remove_value(SLOT_DOCLANGS); } // add document index version const std::string& verval = srcdoc.get_value(SLOT_INDEXVERSION); if (!verval.empty()) { int version = std::atoi(verval.c_str()); if (version) db_versions.insert(version); } else need_md_versions = true; } if (need_md_versions) { /* At least one added document didn't have its index * version slot set in this subdb. Read legacy versions. */ std::set md_versions = read_db_versions(srcdb); db_versions.insert(md_versions.begin(), md_versions.lower_bound(14)); } } thispath = "(unknown path)"; // set the versions write_db_versions(destdb, db_versions); // recalculate language counts std::map lang_counts; r = calculate_language_counts(destdb, lang_counts); if (r) { xsyslog(LOG_ERR, "IOERROR: corrupt metadata", "filter=<%s>", dest); return r; } write_language_counts(destdb, lang_counts); /* commit all changes explicitly */ destdb.commit(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); r = IMAP_IOERROR; } return r; } -const char *xapian_version_string() +EXPORTED const char *xapian_version_string() { return Xapian::version_string(); } struct xapian_doc { Xapian::TermGenerator *termgen; Xapian::Document *doc; }; -xapian_doc_t *xapian_doc_new(void) +EXPORTED xapian_doc_t *xapian_doc_new(void) { xapian_doc_t *doc = (xapian_doc_t *) xzmalloc(sizeof(struct xapian_doc)); doc->doc = new Xapian::Document; doc->termgen = new Xapian::TermGenerator; doc->termgen->set_document(*doc->doc); return doc; } -void xapian_doc_index_text(xapian_doc_t *doc, const char *text, size_t len) +EXPORTED void xapian_doc_index_text(xapian_doc_t *doc, + const char *text, size_t len) { doc->termgen->index_text(Xapian::Utf8Iterator(text, len)); } -size_t xapian_doc_termcount(xapian_doc_t *doc) +EXPORTED size_t xapian_doc_termcount(xapian_doc_t *doc) { return doc->doc->termlist_count(); } -int xapian_doc_foreach_term(xapian_doc_t *doc, int(*cb)(const char*, void*), void *rock) +EXPORTED int xapian_doc_foreach_term(xapian_doc_t *doc, + int(*cb)(const char*, void*), + void *rock) { for (Xapian::TermIterator ti = doc->doc->termlist_begin(); ti != doc->doc->termlist_end(); ++ti) { int r = cb((*ti).c_str(), rock); if (r) return r; } return 0; } -void xapian_doc_reset(xapian_doc_t *doc) +EXPORTED void xapian_doc_reset(xapian_doc_t *doc) { doc->doc->clear_values(); } -extern void xapian_doc_close(xapian_doc_t *termgen); -void xapian_doc_close(xapian_doc_t *doc) +EXPORTED void xapian_doc_close(xapian_doc_t *doc) { delete doc->termgen; delete doc->doc; free(doc); }