diff --git a/Makefile.am b/Makefile.am index 57bfee21e..8e919cbe5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,2087 +1,2087 @@ # # @configure_input@ # # Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. The name "Carnegie Mellon University" must not be used to # endorse or promote products derived from this software without # prior written permission. For permission or any legal # details, please contact # Carnegie Mellon University # Center for Technology Transfer and Enterprise Creation # 4615 Forbes Avenue # Suite 302 # Pittsburgh, PA 15213 # (412) 268-7393, fax: (412) 268-7395 # innovation@andrew.cmu.edu # # 4. Redistributions of any form whatsoever must retain the following # acknowledgment: # "This product includes software developed by Computing Services # at Carnegie Mellon University (http://www.cmu.edu/computing/)." # # CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO # THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY # AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE # FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN # AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING # OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ACLOCAL_AMFLAGS = -I cmulocal AM_CFLAGS = @PERL_CCCDLFLAGS@ $(GCOV_CFLAGS) AM_CXXFLAGS = $(GCOV_CXXFLAGS) AM_CPPFLAGS = \ $(COM_ERR_CPPFLAGS) \ -I${top_builddir} \ -I${top_builddir}/lib \ -I${top_srcdir} \ -I${top_srcdir}/lib \ -DLIBEXEC_DIR=\"$(libexecdir)\" \ -DSBIN_DIR=\"$(sbindir)\" \ -DSYSCONF_DIR=\"$(sysconfdir)\" \ ${DEFS} \ ${LOCALDEFS} \ $(SASLFLAGS) \ $(SSL_CPPFLAGS) \ $(ICU_CFLAGS) if HAVE_LDAP AM_CPPFLAGS += $(LDAP_CPPFLAGS) endif # HAVE_LDAP AM_LDFLAGS = $(COV_LDFLAGS) $(ICU_LIBS) # have distcheck try to build with all optional components enabled, to aid # detection of missing files for these components AM_DISTCHECK_CONFIGURE_FLAGS = \ --enable-http \ --enable-calalarmd \ --enable-replication \ --with-openssl=yes \ --enable-nntp \ --enable-murder \ --enable-idled \ --enable-sieve \ --enable-autocreate \ --enable-backup \ --enable-xapian \ --enable-jmap \ --with-ldap BUILT_SOURCES = \ lib/imapopts.c \ lib/imapopts.h CLEANFILES = \ lib/chartable.c \ lib/imapopts.c \ lib/imapopts.h DISTCLEANFILES = \ com_err/et/compile_et \ imap/http_caldav_js.h \ imap/http_carddav_js.h \ imap/http_err.c \ imap/http_err.h \ imap/imap_err.c \ imap/imap_err.h \ imap/jmap_err.c \ imap/jmap_err.h \ imap/lmtp_err.c \ imap/lmtp_err.h \ imap/mupdate_err.c \ imap/mupdate_err.h \ imap/nntp_err.c \ imap/nntp_err.h \ imap/promdata.c \ imap/promdata.h \ imap/tz_err.c \ imap/tz_err.h \ perl/sieve/scripts/installsieve \ perl/sieve/scripts/sieveshell \ sieve/sieve_err.c \ sieve/sieve_err.h MAINTAINERCLEANFILES = \ doc/legacy/murder.png \ doc/legacy/netnews.png \ man/imapd.conf.5 \ man/sieveshell.1 \ sieve/addr.h \ sieve/sieve.h SUBDIRS = . DIST_SUBDIRS = . perl/annotator perl/imap perl/sieve/managesieve dist_sysconf_DATA = lib_LTLIBRARIES = lib/libcyrus_min.la lib/libcyrus.la check_PROGRAMS = libexec_PROGRAMS = sbin_PROGRAMS = noinst_HEADERS = noinst_LTLIBRARIES = noinst_PROGRAMS = if COM_ERR COMPILE_ET_DEP = com_err/et/compile_et BUILT_SOURCES += com_err/et/compile_et com_err/et/libcyrus_com_err.la lib_LTLIBRARIES += com_err/et/libcyrus_com_err.la endif # COM_ERR bin_PROGRAMS = imtest/imtest if SERVER BUILT_SOURCES += \ imap/http_err.c \ imap/http_err.h \ imap/imap_err.c \ imap/imap_err.h \ imap/lmtp_err.c \ imap/lmtp_err.h \ imap/mupdate_err.c \ imap/mupdate_err.h \ imap/promdata.c \ imap/promdata.h \ lib/htmlchar.c \ lib/htmlchar.h \ imap/rfc822_header.c \ imap/rfc822_header.h \ imap/mailbox_header_cache.h lib_LTLIBRARIES += imap/libcyrus_imap.la libexec_PROGRAMS += \ master/master \ imap/imapd \ imap/lmtpd \ imap/pop3d \ imap/promstatsd \ imap/smmapd sbin_PROGRAMS += \ imap/arbitron \ imap/chk_cyrus \ imap/ctl_conversationsdb \ imap/ctl_cyrusdb \ imap/ctl_deliver \ imap/ctl_mboxlist \ imap/cvt_cyrusdb \ imap/cyr_df \ imap/cyrdump \ imap/cyr_dbtool \ imap/cyr_deny \ imap/cyr_expire \ imap/cyr_info \ imap/cyr_buildinfo \ imap/cyr_sequence \ imap/cyr_synclog \ imap/cyr_userseen \ imap/cyr_virusscan \ imap/deliver \ imap/ipurge \ imap/mbexamine \ imap/mbpath \ imap/mbtool \ imap/quota \ imap/reconstruct \ imap/cvt_xlist_specialuse noinst_PROGRAMS += \ imap/message_test \ imap/search_test if USE_SQUAT noinst_PROGRAMS += imap/squat_dump endif if SQUATTER # Despite the name, the squatter program handles any search engine sbin_PROGRAMS += imap/squatter endif if NNTPD libexec_PROGRAMS += imap/nntpd sbin_PROGRAMS += imap/fetchnews BUILT_SOURCES += imap/nntp_err.c endif # NNTPD libexec_PROGRAMS += imap/fud if IDLED libexec_PROGRAMS += imap/idled endif # IDLED if MURDER libexec_PROGRAMS += imap/mupdate endif # MURDER if CALALARMD libexec_PROGRAMS += imap/calalarmd endif if HTTPD AM_CPPFLAGS += $(HTTP_CPPFLAGS) AM_LDFLAGS += $(HTTP_LIBS) BUILT_SOURCES += \ imap/http_caldav_js.h \ imap/http_carddav_js.h \ imap/tz_err.c \ imap/tz_err.h libexec_PROGRAMS += imap/httpd check_PROGRAMS += imap/ical_apply_patch sbin_PROGRAMS += \ imap/ctl_zoneinfo \ imap/dav_reconstruct endif # HTTPD if REPLICATION libexec_PROGRAMS += imap/sync_server sbin_PROGRAMS += imap/sync_client imap/sync_reset endif # REPLICATION if BACKUP noinst_LTLIBRARIES += backup/libcyrus_backup.la noinst_HEADERS += backup/backup.h libexec_PROGRAMS += backup/backupd sbin_PROGRAMS += backup/ctl_backups backup/cyr_backup backup/restore endif # BACKUP if HAVE_SSL sbin_PROGRAMS += imap/tls_prune endif # HAVE_SSL sbin_PROGRAMS += imap/unexpunge if SIEVE check_PROGRAMS += notifyd/notifytest libexec_PROGRAMS += notifyd/notifyd endif # SIEVE endif # SERVER if CMULOCAL dist_sysconf_DATA += depot/rc.local.imap depot/rc.local.ptclient sbin_PROGRAMS += netnews/remotepurge endif # CMULOCAL if PTCLIENT sbin_PROGRAMS += ptclient/ptdump ptclient/ptexpire libexec_PROGRAMS += \ ptclient/ptloader endif # PTCLIENT if PERL SUBDIRS += perl/annotator perl/imap noinst_LTLIBRARIES += perl/libcyrus.la perl/libcyrus_min.la endif # PERL if SIEVE if PERL SUBDIRS += perl/sieve/managesieve noinst_LTLIBRARIES += perl/sieve/lib/libisieve.la endif # PERL BUILT_SOURCES += sieve/addr.c sieve/sieve.c sieve/sieve_err.c noinst_LTLIBRARIES += sieve/libcyrus_sieve_lex.la lib_LTLIBRARIES += sieve/libcyrus_sieve.la check_PROGRAMS += sieve/test sieve/test_mailbox sbin_PROGRAMS += sieve/sievec sieve/sieved if SERVER libexec_PROGRAMS += timsieved/timsieved endif # SERVER endif # SIEVE EXTRA_DIST = \ COPYING \ README.md \ VERSION \ com_err/et/et_c.awk \ com_err/et/et_h.awk \ com_err/et/test1.et \ com_err/et/test2.et \ com_err/et/test_et.c \ contrib/sieve-spamassassin \ contrib/fud-client.c \ contrib/deliver-notify-zephyr.patch \ contrib/add-cyrus-user \ contrib/README \ contrib/cyrusv2.mc \ contrib/dkim_canon_ischedule.patch \ contrib/notify_unix/notify \ contrib/notify_unix/net-server-prefork-0.01.tgz \ contrib/notify_unix/README \ contrib/notify_unix/sql_notify.pl \ contrib/notify_unix/simple_notify.pl \ contrib/squatrunner.pl \ contrib/mupdate-test.pl \ contrib/squatrunner.txt \ cunit/cacert.pem \ cunit/cert.pem \ cunit/cunit.pl \ cunit/cunit-to-junit.pl \ cunit/key.pem \ cunit/vg.supp \ doc/legacy/murder.png \ doc/legacy/netnews.png \ doc \ docsrc \ imap/http_caldav.js \ imap/http_carddav.js \ imap/http_err.et \ imap/imap_err.et \ imap/jmap_err.et \ imap/lmtp_err.et \ imap/mailbox_header_cache.gperf \ imap/mupdate_err.et \ imap/nntp_err.et \ imap/promdata.p \ imap/rfc822_header.st \ imap/tz_err.et \ lib/charset/aliases.txt \ lib/charset/UnicodeData.txt \ lib/charset/unifix.txt \ lib/charset/us-ascii.t \ lib/htmlchar.st \ lib/imapoptions \ lib/test/cyrusdb.c \ lib/test/cyrusdb.INPUT \ lib/test/cyrusdblong.INPUT \ lib/test/cyrusdblong.OUTPUT \ lib/test/cyrusdb.OUTPUT \ lib/test/cyrusdbtxn.INPUT \ lib/test/cyrusdbtxn.OUTPUT \ lib/test/pool.c \ lib/test/rnddb.c \ master/README \ netnews/inn.diffs \ perl/annotator/Daemon.pm \ perl/annotator/Makefile.PL \ perl/annotator/Makefile.PL.in \ perl/annotator/MANIFEST \ perl/annotator/MANIFEST.in \ perl/annotator/Message.pm \ perl/annotator/README \ perl/imap/Changes \ perl/imap/cyradm.sh \ perl/imap/cyrperl.h \ perl/imap/examples/auditmbox.pl \ perl/imap/examples/imapcollate.pl \ perl/imap/examples/imapdu.pl \ perl/imap/examples/test-imsp.pl \ perl/imap/IMAP/Admin.pm \ perl/imap/IMAP/IMSP.pm \ perl/imap/IMAP/Shell.pm \ perl/imap/IMAP.pm \ perl/imap/IMAP.xs \ perl/imap/Makefile.PL \ perl/imap/Makefile.PL.in \ perl/imap/MANIFEST \ perl/imap/MANIFEST.in \ perl/imap/README \ perl/imap/t/01-imclient.t \ perl/imap/t/02-admin.t \ perl/imap/typemap \ perl/imap/xsutil.c \ perl/sieve/managesieve/Makefile.PL \ perl/sieve/managesieve/Makefile.PL.in \ perl/sieve/managesieve/managesieve.h \ perl/sieve/managesieve/managesieve.pm \ perl/sieve/managesieve/managesieve.xs \ perl/sieve/managesieve/MANIFEST \ perl/sieve/managesieve/MANIFEST.in \ perl/sieve/managesieve/typemap \ ptclient/README \ ptclient/test.c \ ptclient/test2.c \ sieve/addr.h \ sieve/sieve.h \ sieve/sieve_err.et \ sieve/tests/testExtension \ sieve/tests/testExtension/uberExtensionTestScript.key \ sieve/tests/testExtension/testm \ sieve/tests/testExtension/testm/uetest-envelope \ sieve/tests/testExtension/testm/uetest-asub \ sieve/tests/testExtension/testm/uetest-areg \ sieve/tests/testExtension/testm/uetest-count \ sieve/tests/testExtension/testm/uetest-value \ sieve/tests/testExtension/testm/uetest-hreg \ sieve/tests/testExtension/serverm \ sieve/tests/testExtension/serverm/uetmail-hreg \ sieve/tests/testExtension/serverm/uetmail-value \ sieve/tests/testExtension/serverm/uetmail-count2 \ sieve/tests/testExtension/serverm/uetmail-envelope \ sieve/tests/testExtension/serverm/uetmail-asub \ sieve/tests/testExtension/serverm/uetmail-value2 \ sieve/tests/testExtension/serverm/uetmail-areg \ sieve/tests/testExtension/serverm/uetmail-count \ sieve/tests/testExtension/uberExtensionTestScript.s \ sieve/tests/README \ sieve/tests/action \ sieve/tests/action/testm \ sieve/tests/action/testm/uatest-keep \ sieve/tests/action/testm/uatest-redirect \ sieve/tests/action/testm/uatest-discard \ sieve/tests/action/testm/uatest-stop2 \ sieve/tests/action/testm/uatest-stop \ sieve/tests/action/serverm \ sieve/tests/action/serverm/uamail-stop2 \ sieve/tests/action/serverm/uamail-redirect \ sieve/tests/action/serverm/uamail-stop \ sieve/tests/action/serverm/uamail-keep \ sieve/tests/action/serverm/uamail-discard \ sieve/tests/action/uberActionScript.key \ sieve/tests/action/uberActionScript.s \ sieve/tests/test \ sieve/tests/test/uberTestScript.key \ sieve/tests/test/testm \ sieve/tests/test/testm/utest-header \ sieve/tests/test/testm/utest-address \ sieve/tests/test/serverm \ sieve/tests/test/serverm/utmail-address \ sieve/tests/test/serverm/utmail-header \ sieve/tests/test/uberTestScript.s \ sieve/tests/actionExtensions \ sieve/tests/actionExtensions/uberExtensionActionScript.s \ sieve/tests/actionExtensions/testm \ sieve/tests/actionExtensions/testm/ueatest-flag4 \ sieve/tests/actionExtensions/testm/ueatest-flag2 \ sieve/tests/actionExtensions/testm/ueatest-fileinto \ sieve/tests/actionExtensions/testm/ueatest-denotify \ sieve/tests/actionExtensions/testm/ueatest-vacation \ sieve/tests/actionExtensions/testm/ueatest-reject \ sieve/tests/actionExtensions/testm/ueatest-mark \ sieve/tests/actionExtensions/testm/ueatest-denotify2 \ sieve/tests/actionExtensions/testm/ueatest-flag5 \ sieve/tests/actionExtensions/testm/ueatest-notify2 \ sieve/tests/actionExtensions/testm/ueatest-notify \ sieve/tests/actionExtensions/testm/ueatest-flag1 \ sieve/tests/actionExtensions/testm/ueatest-flag3 \ sieve/tests/actionExtensions/testm/ueatest-unmark \ sieve/tests/actionExtensions/uberExtensionActionScript.key \ sieve/tests/actionExtensions/serverm \ sieve/tests/actionExtensions/serverm/ueamail-flag4 \ sieve/tests/actionExtensions/serverm/ueamail-denotify \ sieve/tests/actionExtensions/serverm/ueamail-mark \ sieve/tests/actionExtensions/serverm/ueamail-denotify2 \ sieve/tests/actionExtensions/serverm/ueamail-flag2 \ sieve/tests/actionExtensions/serverm/ueamail-unmark \ sieve/tests/actionExtensions/serverm/ueamail-reject \ sieve/tests/actionExtensions/serverm/ueamail-flag3 \ sieve/tests/actionExtensions/serverm/ueamail-fileinto \ sieve/tests/actionExtensions/serverm/ueamail-flag1 \ sieve/tests/actionExtensions/serverm/ueamail-notify \ sieve/tests/actionExtensions/serverm/ueamail-flag5 \ sieve/tests/actionExtensions/serverm/ueamail-notify2 \ sieve/tests/actionExtensions/serverm/ueamail-vacation \ timsieved/TODO TEXINFO_TEX = com_err/et/texinfo.tex dist_noinst_SCRIPTS = \ com_err/et/compile_et.sh \ com_err/et/config_script \ imap/promdatagen \ lib/mkchartable.pl \ lib/test/run \ perl/sieve/scripts/installsieve.pl \ perl/sieve/scripts/sieveshell.pl \ tools/arbitronsort.pl \ tools/compile_st.pl \ tools/config2header \ tools/config2rst \ tools/config2sample \ tools/fixsearchpath.pl \ tools/git-version.sh \ tools/jenkins-build.sh \ tools/masssievec \ tools/mkimap \ tools/mknewsgroups \ tools/perl2rst \ tools/rehash \ tools/translatesieve noinst_MAN = \ com_err/et/com_err.3 \ com_err/et/compile_et.1 noinst_TEXINFOS = com_err/et/com_err.texinfo pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = libcyrus_min.pc libcyrus.pc libcyrus_sieve.pc libcyrus_imap.pc com_err_et_libcyrus_com_err_la_SOURCES = \ com_err/et/com_err.c \ com_err/et/com_err.h \ com_err/et/error_message.c \ com_err/et/error_table.h \ com_err/et/et_name.c \ com_err/et/init_et.c \ com_err/et/internal.h \ com_err/et/mit-sipb-copyright.h com_err_et_libcyrus_com_err_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) com_err/et/compile_et: com_err/et/compile_et.sh com_err/et/config_script \ config.h @${top_srcdir}/com_err/et/config_script ${top_srcdir}/com_err/et/compile_et.sh ${AWK} ${SED} > $@ @chmod 755 $@ # ---- Libraries ---- # SIEVE is the libraries that sieve-using components need to link with # # This is empty if sieve is not enabled, so it can be used unconditionally # elsewhere. if SIEVE LD_SIEVE_ADD = sieve/libcyrus_sieve.la $(LD_BASIC_ADD) else LD_SIEVE_ADD = endif # BASIC is the libraries that every Cyrus program (except master) will # need to link with. # # Note that several places in the code use -lcrypto, e.g. for SHA1 or # MD5 algorithms, without needing SSL. Currently we have no way of # minimally linking such code. LD_BASIC_ADD = lib/libcyrus.la lib/libcyrus_min.la ${LIBS} \ ${LIB_SASL} $(SSL_LIBS) $(GCOV_LIBS) # UTILITY is the libraries that utility programs which use Cyrus' # mailbox and message handling code need to link with. LD_UTILITY_ADD = imap/libcyrus_imap.la $(LD_BASIC_ADD) $(LD_SIEVE_ADD) $(COM_ERR_LIBS) # SERVER is the libraries that network-facing servers need to link with # # Note that the code is horribly intertwingled e.g. in imap/global.c # so that even utilities which never open a socket need to link against # the SASL library. LD_SERVER_ADD = $(LD_UTILITY_ADD) $(LIB_WRAP) # ---- if USE_LIBCHARDET AM_LDFLAGS += $(LIBCHARDET_LIBS) AM_CPPFLAGS += $(LIBCHARDET_CFLAGS) LD_SERVER_ADD += $(LIBCHARDET_LIBS) endif if USE_JANSSON AM_LDFLAGS += $(JANSSON_LIBS) AM_CPPFLAGS += $(JANSSON_CFLAGS) LD_SERVER_ADD += $(JANSSON_LIBS) endif if CUNIT CUNIT_PROJECT = cunit/default.cunit BUILT_SOURCES += cunit/registers.h $(CUNIT_PROJECT) CLEANFILES += cunit/registers.h $(CUNIT_PROJECT) check_PROGRAMS += cunit/unit cunit_FRAMEWORK = \ cunit/unit.c \ cunit/cyrunit.h \ cunit/syslog.c \ cunit/cunit-syslog.h \ cunit/timeout.c \ cunit/timeout.h \ cunit/timezones.c \ cunit/timezones.h \ cunit/timeofday.c \ cunit/timeofday.h cunit_TESTS = \ cunit/aaa-db.testc \ cunit/annotate.testc \ cunit/backend.testc \ cunit/binhex.testc \ cunit/bitvector.testc \ cunit/buf.testc \ cunit/byteorder.testc \ cunit/charset.testc \ cunit/command.testc \ cunit/conversations.testc \ cunit/cyr_qsort_r.testc \ cunit/crc32.testc \ cunit/dlist.testc \ cunit/duplicate.testc \ cunit/getxstring.testc \ cunit/glob.testc \ cunit/guid.testc \ cunit/hash.testc \ cunit/hashset.testc \ cunit/imapurl.testc \ cunit/imparse.testc \ cunit/libconfig.testc \ cunit/mboxname.testc \ cunit/md5.testc \ cunit/message.testc \ cunit/msgid.testc \ cunit/parseaddr.testc \ cunit/parse.testc \ cunit/procinfo.testc \ cunit/prot.testc \ cunit/ptrarray.testc \ cunit/quota.testc \ cunit/rfc822tok.testc \ cunit/search_expr.testc \ cunit/seqset.testc if SIEVE cunit_TESTS += cunit/sieve.testc endif cunit_TESTS += \ cunit/spool.testc \ cunit/squat.testc \ cunit/strarray.testc \ cunit/strconcat.testc \ cunit/times.testc \ cunit/tok.testc \ cunit/vparse.testc cunit_unit_SOURCES = $(cunit_FRAMEWORK) $(cunit_TESTS) \ imap/mutex_fake.c imap/spool.c cunit_unit_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) -lcunit CUNIT_PL = $(top_srcdir)/cunit/cunit.pl --project $(CUNIT_PROJECT) # don't discard the temporary source file if building with coverage if HAVE_COVERAGE CUNIT_RM = true else CUNIT_RM = $(RM) endif .testc.o: $(AM_V_at)$(CUNIT_PL) --generate-wrapper $< $(AM_V_CC)$(COMPILE) -c -o $@ $<-cunit.c $(AM_V_at)$(CUNIT_RM) $<-cunit.c $(CUNIT_PROJECT): $(AM_V_at)$(RM) $@ $(AM_V_GEN)$(CUNIT_PL) --add-sources $(addprefix $(top_srcdir)/,$(cunit_TESTS)) cunit/registers.h: $(CUNIT_PROJECT) $(AM_V_GEN)$(CUNIT_PL) --generate-register-function $@ # To run under Valgrind, do: make VG=1 check VALGRIND = libtool --mode=execute valgrind --tool=memcheck --leak-check=full --suppressions=vg.supp check-local: @echo "Running unit tests" @vg= ; \ test -z "$$VG" || vg="$(VALGRIND)" ; \ f="-v" ; \ test "x$$CUFORMAT" = xjunit && f="-x" ; \ cd cunit ; \ $$vg ./unit $$f ; \ retval=$$? ; \ if [ "x$$CUFORMAT" = xjunit ] ; then \ $(RM) -rf reports ; mkdir reports ; ./cunit-to-junit.pl ; \ fi ; \ exit $$retval # Run every test individually, as if you had run them one at a time as # `make check-suitename:testname` manually. Mainly useful for detecting # abstraction failures within our testing infrastructure -- if tests # succeed with `make check` but fail with `make check-discrete`, or # vice-versa, some state is leaking between tests! # # This assumes your sh supports arithmetic and process substitutions. # Sorry <3 check-discrete: cunit/unit @echo "Running unit tests, one at a time" @vg= ; \ test -z "$$VG" || vg="$(VALGRIND)" ; \ cd cunit ; \ fails=0 ; \ tests=0 ; \ while read t ; do \ tests=$$(($$tests + 1)) ; \ if $$vg ./unit -v $$t >/dev/null 2>/dev/null ; then \ echo "[ OK ] $$t" ; \ else \ fails=$$(($$fails + 1)) ; \ echo "[FAILED] $$t" ; \ fi ; \ done < <( ./unit -l ) ; \ echo "$$fails/$$tests tests failed" ; \ exit $$fails check-%: cunit/unit @echo "Running unit tests for $*" @vg= ; \ test -z "$$VG" || vg="$(VALGRIND)" ; \ cd cunit ; \ $$vg ./unit -v $* ; \ retval=$$? ; \ exit $$retval endif includedir=@includedir@/cyrus include_HEADERS = \ lib/acl.h \ lib/arrayu64.h \ lib/assert.h \ lib/auth.h \ lib/auth_pts.h \ lib/bitvector.h \ lib/bloom.h \ lib/bsearch.h \ lib/bufarray.h \ lib/charset.h \ lib/chartable.h \ lib/command.h \ lib/crc32.h \ lib/cyr_lock.h \ lib/cyr_qsort_r.h \ lib/cyrusdb.h \ lib/glob.h \ lib/gmtoff.h \ lib/hash.h \ lib/hashset.h \ lib/hashu64.h \ lib/imapurl.h \ lib/imclient.h \ lib/imparse.h \ lib/iostat.h \ lib/iptostring.h \ lib/libcyr_cfg.h \ lib/lsort.h \ lib/map.h \ lib/mappedfile.h \ lib/mkgmtime.h \ lib/mpool.h \ lib/murmurhash2.h \ lib/nonblock.h \ lib/parseaddr.h \ lib/procinfo.h \ lib/retry.h \ lib/rfc822tok.h \ lib/signals.h \ lib/sqldb.h \ lib/strarray.h \ lib/strhash.h \ lib/stristr.h \ lib/times.h \ lib/tok.h \ lib/vparse.h \ lib/wildmat.h \ lib/xmalloc.h nodist_include_HEADERS = \ lib/imapopts.h nobase_include_HEADERS = sieve/sieve_interface.h nobase_nodist_include_HEADERS = sieve/sieve_err.h noinst_HEADERS += \ lib/byteorder.h \ lib/gai.h \ lib/libconfig.h \ lib/md5.h \ lib/prot.h \ lib/ptrarray.h \ lib/util.h \ lib/xsha1.h \ lib/xstrlcat.h \ lib/xstrlcpy.h \ lib/xstrnchr.h backup_libcyrus_backup_la_SOURCES = \ lib/gzuncat.c \ lib/gzuncat.h \ backup/lcb.c \ backup/lcb_append.c \ backup/lcb_backupdb.c \ backup/lcb_compact.c \ backup/lcb_indexr.c \ backup/lcb_indexw.c \ backup/lcb_internal.c \ backup/lcb_internal.h \ backup/lcb_partlist.c \ backup/lcb_read.c \ backup/lcb_sqlconsts.c \ backup/lcb_sqlconsts.h \ backup/lcb_verify.c backup_libcyrus_backup_la_LIBADD = $(LD_BASIC_ADD) backup_backupd_SOURCES = \ imap/mutex_fake.c \ imap/sync_support.c \ imap/sync_support.h \ master/service.c \ backup/backupd.c backup_backupd_LDADD = backup/libcyrus_backup.la $(LD_SIEVE_ADD) $(LD_SERVER_ADD) backup_ctl_backups_SOURCES = \ imap/mutex_fake.c \ imap/sync_support.c \ imap/sync_support.h \ backup/ctl_backups.c backup_ctl_backups_LDADD = backup/libcyrus_backup.la $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) backup_cyr_backup_SOURCES = \ imap/json_support.h \ imap/mutex_fake.c \ backup/cyr_backup.c backup_cyr_backup_LDADD = backup/libcyrus_backup.la $(LD_UTILITY_ADD) backup_restore_SOURCES = \ imap/mutex_fake.c \ imap/sync_support.c \ imap/sync_support.h \ backup/restore.c backup_restore_LDADD = backup/libcyrus_backup.la $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) imap_arbitron_SOURCES = imap/arbitron.c imap/cli_fatal.c imap/mutex_fake.c imap_arbitron_LDADD = $(LD_UTILITY_ADD) imap_chk_cyrus_SOURCES = imap/chk_cyrus.c imap/cli_fatal.c imap/mutex_fake.c imap_chk_cyrus_LDADD = $(LD_UTILITY_ADD) imap_ctl_conversationsdb_SOURCES = imap/ctl_conversationsdb.c imap/mutex_fake.c imap_ctl_conversationsdb_LDADD = $(LD_UTILITY_ADD) imap_ctl_cyrusdb_SOURCES = imap/cli_fatal.c imap/ctl_cyrusdb.c imap/mutex_fake.c imap_ctl_cyrusdb_LDADD = $(LD_UTILITY_ADD) imap_ctl_deliver_SOURCES = imap/cli_fatal.c imap/ctl_deliver.c imap/mutex_fake.c imap_ctl_deliver_LDADD = $(LD_UTILITY_ADD) imap_ctl_mboxlist_SOURCES = imap/cli_fatal.c imap/ctl_mboxlist.c imap/mutex_fake.c imap_ctl_mboxlist_LDADD = $(LD_UTILITY_ADD) imap_ctl_zoneinfo_SOURCES = imap/cli_fatal.c imap/ctl_zoneinfo.c imap/mutex_fake.c imap/zoneinfo_db.c imap/xml_support.c imap_ctl_zoneinfo_LDADD = $(LD_UTILITY_ADD) imap_cvt_cyrusdb_SOURCES = imap/cli_fatal.c imap/cvt_cyrusdb.c imap/mutex_fake.c imap_cvt_cyrusdb_LDADD = $(LD_UTILITY_ADD) imap_cyrdump_SOURCES = imap/cli_fatal.c imap/cyrdump.c imap/mutex_fake.c imap_cyrdump_LDADD = $(LD_UTILITY_ADD) imap_cyr_dbtool_SOURCES = imap/cli_fatal.c imap/cyr_dbtool.c imap/mutex_fake.c imap_cyr_dbtool_LDADD = $(LD_UTILITY_ADD) imap_cyr_deny_SOURCES = imap/cli_fatal.c imap/cyr_deny.c imap/mutex_fake.c imap_cyr_deny_LDADD = $(LD_UTILITY_ADD) imap_cyr_df_SOURCES = imap/cli_fatal.c imap/cyr_df.c imap/mutex_fake.c imap_cyr_df_LDADD = $(LD_UTILITY_ADD) imap_cyr_expire_SOURCES = imap/cli_fatal.c imap/cyr_expire.c imap/mutex_fake.c imap_cyr_expire_LDADD = $(LD_UTILITY_ADD) imap_cyr_info_SOURCES = imap/cli_fatal.c imap/cyr_info.c imap/mutex_fake.c master/masterconf.c imap_cyr_info_LDADD = $(LD_UTILITY_ADD) imap_cyr_buildinfo_SOURCES = imap/cli_fatal.c imap/cyr_buildinfo.c imap/mutex_fake.c master/masterconf.c imap_cyr_buildinfo_LDADD = $(LD_UTILITY_ADD) imap_cyr_sequence_SOURCES = imap/cli_fatal.c imap/cyr_sequence.c imap/mutex_fake.c imap_cyr_sequence_LDADD = $(LD_UTILITY_ADD) imap_cyr_synclog_SOURCES = imap/cli_fatal.c imap/cyr_synclog.c imap/mutex_fake.c imap_cyr_synclog_LDADD = $(LD_UTILITY_ADD) imap_cyr_userseen_SOURCES = imap/cli_fatal.c imap/cyr_userseen.c imap/mutex_fake.c imap_cyr_userseen_LDADD = $(LD_UTILITY_ADD) imap_cyr_virusscan_SOURCES = imap/cli_fatal.c imap/cyr_virusscan.c imap/mutex_fake.c imap_cyr_virusscan_CFLAGS = $(AM_CFLAGS) $(CLAMAV_CFLAGS) $(CFLAG_VISIBILITY) imap_cyr_virusscan_LDADD = $(LD_UTILITY_ADD) $(CLAMAV_LIBS) imap_deliver_SOURCES = \ imap/deliver.c \ imap/lmtpengine.c \ imap/mutex_fake.c \ imap/proxy.c \ imap/spool.c nodist_imap_deliver_SOURCES = imap/lmtp_err.c imap_deliver_LDADD = $(LD_UTILITY_ADD) imap_fetchnews_SOURCES = imap/cli_fatal.c imap/fetchnews.c imap/mutex_fake.c imap_fetchnews_LDADD = $(LD_UTILITY_ADD) imap_fud_SOURCES = imap/fud.c imap/mutex_fake.c master/service.c imap_fud_LDADD = $(LD_SERVER_ADD) imap_idled_SOURCES = imap/idled.c imap/mutex_fake.c imap_idled_LDADD = $(LD_UTILITY_ADD) imap_calalarmd_SOURCES = imap/calalarmd.c imap/mutex_fake.c imap_calalarmd_LDADD = $(LD_SERVER_ADD) imap_imapd_SOURCES = \ imap/imap_proxy.c \ imap/imap_proxy.h \ imap/imapd.c \ imap/imapd.h \ imap/mutex_fake.c \ imap/proxy.c \ imap/proxy.h \ imap/sync_support.c \ imap/sync_support.h \ master/service.c if AUTOCREATE imap_imapd_SOURCES += \ imap/autocreate.h \ imap/autocreate.c endif imap_imapd_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) imap_ipurge_SOURCES = imap/cli_fatal.c imap/ipurge.c imap/mutex_fake.c imap_ipurge_LDADD = $(LD_UTILITY_ADD) nodist_imap_libcyrus_imap_la_SOURCES = \ imap/http_err.c \ imap/http_err.h \ imap/imap_err.c \ imap/imap_err.h \ imap/mupdate_err.c \ imap/mupdate_err.h \ imap/promdata.c \ imap/promdata.h imap_libcyrus_imap_la_LIBADD = \ $(COM_ERR_LIBS) \ $(LIB_UUID) \ $(GCOV_LIBS) \ lib/libcyrus_min.la \ lib/libcyrus.la imap_libcyrus_imap_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) -imap_libcyrus_imap_la_CXXFLAGS = $(AM_CXXFLAGS) +imap_libcyrus_imap_la_CXXFLAGS = $(AM_CXXFLAGS) $(CFLAG_VISIBILITY) imap_libcyrus_imap_la_SOURCES = \ imap/annotate.c \ imap/annotate.h \ imap/append.c \ imap/append.h \ imap/backend.c \ imap/backend.h \ imap/conversations.c \ imap/conversations.h \ imap/convert_code.c \ imap/convert_code.h \ imap/dlist.c \ imap/dlist.h \ imap/duplicate.c \ imap/duplicate.h \ imap/global.c \ imap/global.h \ imap/http_client.c \ imap/http_client.h \ imap/idle.c \ imap/idle.h \ imap/idlemsg.c \ imap/idlemsg.h \ imap/imapparse.c \ imap/imapparse.h \ imap/index.c \ imap/index.h \ imap/json_support.h \ imap/mailbox.c \ imap/mailbox.h \ imap/mbdump.c \ imap/mbdump.h \ imap/mboxkey.c \ imap/mboxkey.h \ imap/mboxlist.c \ imap/mboxlist.h \ imap/mboxevent.c \ imap/mboxevent.h \ imap/mboxname.c \ imap/mboxname.h \ imap/message_guid.c \ imap/message_guid.h \ imap/message.c \ imap/message.h \ imap/message_priv.h \ imap/msgrecord.c \ imap/msgrecord.h \ imap/mupdate-client.c \ imap/mupdate-client.h \ imap/mutex.h \ imap/notify.c \ imap/notify.h \ imap/partlist.c \ imap/partlist.h \ imap/proc.c \ imap/proc.h \ imap/prometheus.c \ imap/prometheus.h \ imap/protocol.h \ imap/quota_db.c \ imap/rfc822_header.c \ imap/rfc822_header.h \ imap/mailbox_header_cache.h \ imap/saslclient.c \ imap/saslclient.h \ imap/saslserver.c \ imap/search_engines.c \ imap/search_engines.h \ imap/search_expr.c \ imap/search_expr.h \ imap/search_query.c \ imap/search_query.h \ imap/search_part.h \ imap/search_sort.h \ imap/seen.h \ imap/seen_db.c \ imap/sequence.c \ imap/sequence.h \ imap/setproctitle.c \ imap/sievedir.c \ imap/sievedir.h \ imap/spool.c \ imap/spool.h \ imap/statuscache.h \ imap/statuscache_db.c \ imap/sync_log.c \ imap/sync_log.h \ imap/telemetry.c \ imap/telemetry.h \ imap/tls.c \ imap/tls.h \ imap/tls_th-lock.c \ imap/tls_th-lock.h \ imap/user.c \ imap/user.h \ imap/userdeny_db.c \ imap/userdeny.h \ imap/version.c \ imap/version.h \ imap/xstats.c \ imap/xstats.h \ imap/xstats_metrics.h if OBJECTSTORE imap_libcyrus_imap_la_SOURCES += \ imap/objectstore_db.c \ imap/objectstore_db.h \ imap/objectstore.h if WITH_OPENIO imap_libcyrus_imap_la_SOURCES += imap/objectstore_openio.c else if WITH_CARINGO imap_libcyrus_imap_la_SOURCES += imap/objectstore_caringo.c else if WITH_OBJSTR_DUMMY imap_libcyrus_imap_la_SOURCES += imap/objectstore_dummy.c endif endif endif endif if USE_SQUAT imap_libcyrus_imap_la_SOURCES += \ imap/search_squat.c \ imap/squat.c \ imap/squat.h \ imap/squat_build.c \ imap/squat_internal.c \ imap/squat_internal.h endif if HTTPD imap_libcyrus_imap_la_SOURCES += \ imap/caldav_alarm.c \ imap/caldav_alarm.h \ imap/caldav_db.c \ imap/caldav_db.h \ imap/carddav_db.c \ imap/carddav_db.h \ imap/dav_db.c \ imap/dav_db.h \ imap/dav_util.c \ imap/dav_util.h \ imap/ical_support.c \ imap/ical_support.h \ imap/icu_wrap.h \ imap/icu_wrap.cpp \ imap/vcard_support.c \ imap/vcard_support.h \ imap/webdav_db.c \ imap/webdav_db.h endif # HTTPD if JMAP imap_libcyrus_imap_la_SOURCES += \ imap/smtpclient.c \ imap/smtpclient.h endif if USE_XAPIAN imap_libcyrus_imap_la_SOURCES += \ imap/search_xapian.c \ imap/xapian_wrap.h \ imap/xapian_wrap.cpp imap_libcyrus_imap_la_LIBADD += $(XAPIAN_LIBS) imap_libcyrus_imap_la_CXXFLAGS += $(XAPIAN_CXXFLAGS) if HAVE_CLD2 imap_libcyrus_imap_la_LIBADD += $(CLD2_LIBS) imap_libcyrus_imap_la_CXXFLAGS += $(CLD2_CFLAGS) endif endif imap_lmtpd_SOURCES = \ imap/lmtpd.c \ imap/lmtpd.h \ imap/lmtpengine.c \ imap/lmtpengine.h \ imap/mutex_fake.c \ imap/proxy.c \ imap/spool.c \ imap/sync_support.c \ imap/sync_support.h \ master/service.c nodist_imap_lmtpd_SOURCES = imap/lmtp_err.c if AUTOCREATE imap_lmtpd_SOURCES += \ imap/autocreate.c \ imap/autocreate.h endif # AUTOCREATE if SIEVE imap_lmtpd_SOURCES += imap/lmtp_sieve.c imap/lmtp_sieve.h imap/smtpclient.c if JMAP imap_lmtpd_SOURCES += \ imap/jmap_util.c imap/jmap_mail_query.c imap/jmap_mail_query_parse.c endif # JMAP endif # SIEVE imap_lmtpd_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) imap_mbexamine_SOURCES = imap/cli_fatal.c imap/mbexamine.c imap/mutex_fake.c imap_mbexamine_LDADD = $(LD_UTILITY_ADD) imap_mbpath_SOURCES = imap/cli_fatal.c imap/mbpath.c imap/mutex_fake.c imap_mbpath_LDADD = $(LD_UTILITY_ADD) imap_mbtool_SOURCES = imap/cli_fatal.c imap/mbtool.c imap/mutex_fake.c imap_mbtool_LDADD = $(LD_UTILITY_ADD) imap_message_test_SOURCES = imap/message_test.c imap/mutex_fake.c imap_message_test_LDADD = $(LD_UTILITY_ADD) imap_mupdate_SOURCES = \ imap/mupdate.c \ imap/mupdate.h \ imap/mupdate-slave.c \ imap/mutex_pthread.c \ imap/tls_th-lock.c \ master/service-thread.c imap_mupdate_LDADD = $(LD_SERVER_ADD) -lpthread imap_mupdate_CFLAGS = $(AM_CFLAGS) -pthread nodist_imap_nntpd_SOURCES = imap/nntp_err.c imap_nntpd_SOURCES = \ imap/mutex_fake.c \ imap/nntpd.c \ imap/proxy.c \ imap/smtpclient.c \ imap/smtpclient.h \ imap/spool.c \ imap/spool.h \ imap/sync_support.c \ imap/sync_support.h \ master/service.c imap_nntpd_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) nodist_imap_httpd_SOURCES = \ imap/http_caldav_js.h \ imap/http_carddav_js.h \ imap/tz_err.c \ imap/tz_err.h imap_httpd_SOURCES = \ imap/css3_color.c \ imap/css3_color.h \ imap/http_admin.c \ imap/http_applepush.c \ imap/http_caldav.c \ imap/http_caldav.h \ imap/http_caldav_js.h \ imap/http_caldav_sched.c \ imap/http_caldav_sched.h \ imap/http_carddav.c \ imap/http_carddav.h \ imap/http_carddav_js.h \ imap/http_cgi.c \ imap/http_dav.c \ imap/http_dav.h \ imap/http_dav_sharing.c \ imap/http_dav_sharing.h \ imap/http_dblookup.c \ imap/http_h2.c \ imap/http_h2.h \ imap/http_ischedule.c \ imap/http_prometheus.c \ imap/http_proxy.c \ imap/http_proxy.h \ imap/http_rss.c \ imap/http_tzdist.c \ imap/http_webdav.c \ imap/http_ws.c \ imap/http_ws.h \ imap/httpd.c \ imap/httpd.h \ imap/jcal.c \ imap/jcal.h \ imap/json_support.h \ imap/mutex_fake.c \ imap/proxy.c \ imap/smtpclient.c \ imap/spool.c \ imap/sync_support.c \ imap/sync_support.h \ imap/xcal.c \ imap/xcal.h \ imap/xml_support.c \ imap/xml_support.h \ imap/zoneinfo_db.c \ imap/zoneinfo_db.h \ master/masterconf.c \ master/service.c imap_httpd_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) if JMAP BUILT_SOURCES += \ imap/jmap_err.c \ imap/jmap_err.h imap_httpd_SOURCES += \ imap/http_jmap.c \ imap/http_jmap.h \ imap/jmap_api.c \ imap/jmap_api.h \ imap/jmap_backup.c \ imap/jmap_calendar.c \ imap/jmap_contact.c \ imap/jmap_core.c \ imap/jmap_ical.c \ imap/jmap_ical.h \ imap/jmap_mail.c \ imap/jmap_mail.h \ imap/jmap_mail_query.c \ imap/jmap_mail_query.h \ imap/jmap_mail_query_parse.c \ imap/jmap_mail_query_parse.h \ imap/jmap_mail_submission.c \ imap/jmap_mailbox.c \ imap/jmap_mdn.c \ imap/jmap_notes.c \ imap/jmap_util.c \ imap/jmap_util.h \ imap/jmap_vacation.c if SIEVE imap_httpd_SOURCES += \ imap/jmap_sieve.c endif nodist_imap_httpd_SOURCES += \ imap/jmap_err.c \ imap/jmap_err.h imap_libcyrus_imap_la_SOURCES += \ imap/jmap_util.c \ imap/jmap_util.h \ imap/json_support.c \ imap/json_support.h imap_httpd_LDADD += $(LD_SIEVE_ADD) if CUNIT cunit_TESTS += cunit/jmap_util.testc endif # CUNIT endif # JMAP imap_pop3d_SOURCES = \ imap/mutex_fake.c \ imap/pop3d.c \ imap/proxy.c \ imap/sync_support.c \ imap/sync_support.h \ master/service.c if AUTOCREATE imap_pop3d_SOURCES += \ imap/autocreate.c \ imap/autocreate.h endif # AUTOCREATE imap_pop3d_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) PROMDATAGEN = $(abs_top_srcdir)/imap/promdatagen imap/promdata.c: imap/promdata.p $(PROMDATAGEN) $(AM_V_GEN)$(PROMDATAGEN) -c $@.NEW $< && mv $@.NEW $@ imap/promdata.h: imap/promdata.p $(PROMDATAGEN) $(AM_V_GEN)$(PROMDATAGEN) -h $@.NEW $< && mv $@.NEW $@ imap_promstatsd_SOURCES = imap/promstatsd.c imap/mutex_fake.c imap_promstatsd_LDADD = $(LD_UTILITY_ADD) imap_quota_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/quota.c imap/quota.h imap_quota_LDADD = $(LD_UTILITY_ADD) imap_reconstruct_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/reconstruct.c imap_reconstruct_LDADD = $(LD_UTILITY_ADD) imap_dav_reconstruct_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/dav_reconstruct.c imap_dav_reconstruct_LDADD = $(LD_UTILITY_ADD) imap_ical_apply_patch_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/ical_apply_patch.c imap_ical_apply_patch_LDADD = $(LD_UTILITY_ADD) imap_search_test_SOURCES = imap/search_test.c imap/mutex_fake.c imap_search_test_LDADD = $(LD_UTILITY_ADD) imap_smmapd_SOURCES = imap/mutex_fake.c imap/proxy.c imap/smmapd.c master/service.c imap_smmapd_LDADD = $(LD_SERVER_ADD) imap_squatter_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/squatter.c imap_squatter_LDADD = $(LD_UTILITY_ADD) imap_squat_dump_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/squat_dump.c imap_squat_dump_LDADD = $(LD_UTILITY_ADD) imap_sync_client_SOURCES = imap/mutex_fake.c imap/sync_client.c imap/sync_support.c imap/sync_support.h imap_sync_client_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) imap_sync_reset_SOURCES = imap/mutex_fake.c imap/sync_reset.c imap/sync_support.c imap/sync_support.h imap_sync_reset_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) imap_sync_server_SOURCES = imap/mutex_fake.c imap/sync_server.c imap/sync_support.c imap/sync_support.h master/service.c imap_sync_server_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) imap_tls_prune_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/tls_prune.c imap_tls_prune_LDADD = $(LD_UTILITY_ADD) imap_unexpunge_SOURCES = imap/cli_fatal.c imap/mutex_fake.c imap/unexpunge.c imap_unexpunge_LDADD = $(LD_UTILITY_ADD) # $(COMPILE_ET) creates its output files in the directory it's run from, # so do a tricky directory change %_err.h %_err.c: %_err.et $(COMPILE_ET_DEP) $(AM_V_GEN)(cd $(@D) && $(COMPILE_ET) $(realpath $<)) # xxd cannot have path details in its input filename, otherwise it junks up # the variable names in the output file. so do a tricky directory change. # the /dev/null redirection on cd is to prevent shell environments with # CDPATH echoing the path change on stdout and consequently into the .h file %_js.h: %.js $(AM_V_GEN)(cd $( /dev/null && xxd -i $( $@.NEW && mv $@.NEW $@ if MAINTAINER_MODE imap/rfc822_header.c: imap/rfc822_header.st $(AM_V_GEN)${top_srcdir}/tools/compile_st.pl -c $< > $@.NEW && mv $@.NEW $@ imap/rfc822_header.h: imap/rfc822_header.st $(AM_V_GEN)${top_srcdir}/tools/compile_st.pl -h $< > $@.NEW && mv $@.NEW $@ lib/htmlchar.c: lib/htmlchar.st $(AM_V_GEN)${top_srcdir}/tools/compile_st.pl -c $< > $@.NEW && mv $@.NEW $@ lib/htmlchar.h: lib/htmlchar.st $(AM_V_GEN)${top_srcdir}/tools/compile_st.pl -h $< > $@.NEW && mv $@.NEW $@ imap/mailbox_header_cache.h: imap/mailbox_header_cache.gperf $(AM_V_GEN)gperf --ignore-case --initializer-suffix=,0 -p -j1 -i 1 -g -o -t -H mailbox_header_cache_hash -N mailbox_header_cache_lookup -k1,3,$$ $< > $@.NEW && mv $@.NEW $@ endif imtest_imtest_SOURCES = imtest/imtest.c imtest_imtest_LDADD = $(LD_BASIC_ADD) imtest_imtest_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) nodist_lib_libcyrus_la_SOURCES = lib/chartable.c lib_libcyrus_la_SOURCES = \ lib/acl.c \ lib/acl_afs.c \ lib/auth.c \ lib/auth_krb.c \ lib/auth_krb5.c \ lib/auth_pts.c \ lib/auth_unix.c \ lib/bitvector.c \ lib/bloom.c \ lib/bsearch.c \ lib/charset.c \ lib/command.c \ lib/cyr_qsort_r.c \ lib/cyrusdb.c \ lib/cyrusdb_flat.c \ lib/cyrusdb_quotalegacy.c \ lib/cyrusdb_skiplist.c \ lib/cyrusdb_twoskip.c \ lib/glob.c \ lib/htmlchar.c \ lib/htmlchar.h \ lib/imapurl.c \ lib/imclient.c \ lib/imparse.c \ lib/iostat.c \ lib/iptostring.c \ lib/libcyr_cfg.c \ lib/lsort.c \ lib/mappedfile.c \ lib/murmurhash.c \ lib/mkgmtime.c \ lib/parseaddr.c \ lib/procinfo.c \ lib/prot.c \ lib/ptrarray.c \ lib/rfc822tok.c \ lib/signals.c \ lib/stristr.c \ lib/times.c \ lib/tok.c \ lib/wildmat.c if USE_CYRUSDB_SQL lib_libcyrus_la_SOURCES += lib/cyrusdb_sql.c endif if USE_SQLITE lib_libcyrus_la_SOURCES += lib/sqldb.c endif if GMTOFF_TM lib_libcyrus_la_SOURCES += lib/gmtoff_tm.c else lib_libcyrus_la_SOURCES += lib/gmtoff_gmtime.c endif if NONBLOCK_FCNTL lib_libcyrus_la_SOURCES += lib/nonblock_fcntl.c else lib_libcyrus_la_SOURCES += lib/nonblock_ioctl.c endif lib_libcyrus_la_LIBADD = libcrc32.la ${LIB_SASL} $(SSL_LIBS) $(GCOV_LIBS) lib_libcyrus_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) if USE_ZEROSKIP lib_libcyrus_la_SOURCES += lib/cyrusdb_zeroskip.c lib_libcyrus_la_LIBADD += $(ZEROSKIP_LIBS) lib_libcyrus_la_CFLAGS += $(ZEROSKIP_CFLAGS) AM_CPPFLAGS += $(ZEROSKIP_CFLAGS) endif noinst_LTLIBRARIES += libcrc32.la libcrc32_la_SOURCES = lib/crc32.c libcrc32_la_CFLAGS = -O3 $(AM_CFLAGS) $(CFLAG_VISIBILITY) nodist_lib_libcyrus_min_la_SOURCES = \ lib/imapopts.c lib_libcyrus_min_la_SOURCES = \ lib/arrayu64.c \ lib/assert.c \ lib/bufarray.c \ lib/byteorder.c \ lib/hash.c \ lib/hashset.c \ lib/hashu64.c \ lib/libconfig.c \ lib/mpool.c \ lib/retry.c \ lib/strarray.c \ lib/strhash.c \ lib/util.c \ lib/vparse.c \ lib/xmalloc.c \ lib/xstrlcat.c \ lib/xstrlcpy.c \ lib/xstrnchr.c if !HAVE_SSL lib_libcyrus_min_la_SOURCES += lib/xsha1.c endif if IPV6_noGETADDRINFO lib_libcyrus_min_la_SOURCES += lib/getaddrinfo.c endif if IPV6_noGETNAMEINFO lib_libcyrus_min_la_SOURCES += lib/getnameinfo.c endif if LOCK_FCNTL lib_libcyrus_min_la_SOURCES += lib/lock_fcntl.c else lib_libcyrus_min_la_SOURCES += lib/lock_flock.c endif if MAP_SHARED lib_libcyrus_min_la_SOURCES += lib/map_shared.c else if MAP_STUPIDSHARED lib_libcyrus_min_la_SOURCES += lib/map_stupidshared.c else lib_libcyrus_min_la_SOURCES += lib/map_nommap.c endif endif lib_libcyrus_min_la_LIBADD = $(LTLIBOBJS) $(LIB_UUID) $(GCOV_LIBS) $(LIBCAP_LIBS) lib_libcyrus_min_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) # n.b. the order of the -m arguments to mkchartable.pl is important, # in particular ellie believes unifix.txt must be before UnicodeData.txt lib/chartable.c: lib/mkchartable.pl lib/charset/unifix.txt \ $(top_srcdir)/lib/charset/*.t \ lib/charset/UnicodeData.txt lib/charset/aliases.txt @echo "### Building chartables..." $(AM_V_GEN)perl $(top_srcdir)/lib/mkchartable.pl -m $(top_srcdir)/lib/charset/unifix.txt -m $(top_srcdir)/lib/charset/UnicodeData.txt -a $(top_srcdir)/lib/charset/aliases.txt -o $@ $(top_srcdir)/lib/charset/*.t || (rm -f $@ && exit 1) @echo "### Done building chartables." lib/imapopts.h: lib/imapopts.c lib/imapopts.c: lib/imapoptions tools/config2header $(AM_V_GEN)$(top_srcdir)/tools/config2header CC="$(CC)" $(top_builddir)/lib/imapopts.c $(top_builddir)/lib/imapopts.h < $(top_srcdir)/lib/imapoptions imap_cvt_xlist_specialuse_SOURCES = imap/mutex_fake.c imap/cvt_xlist_specialuse.c imap_cvt_xlist_specialuse_LDADD = $(LD_UTILITY_ADD) @SET_MAKE@ dist_man1_MANS = \ man/imtest.1 \ man/installsieve.1 \ man/httptest.1 \ man/lmtptest.1 \ man/mupdatetest.1 \ man/nntptest.1 \ man/pop3test.1 \ man/sieveshell.1 \ man/sivtest.1 \ man/smtptest.1 dist_man3_MANS = \ man/imclient.3 dist_man5_MANS = \ man/cyrus.conf.5 \ man/imapd.conf.5 \ man/krb.equiv.5 dist_man8_MANS = \ man/arbitron.8 \ man/backupd.8 \ man/chk_cyrus.8 \ man/ctl_backups.8 \ man/ctl_conversationsdb.8 \ man/ctl_cyrusdb.8 \ man/ctl_deliver.8 \ man/ctl_mboxlist.8 \ man/cvt_cyrusdb.8 \ man/cyr_backup.8 \ man/cyr_buildinfo.8 \ man/cyr_dbtool.8 \ man/cyr_deny.8 \ man/cyr_df.8 \ man/cyr_expire.8 \ man/cyr_info.8 \ man/cyr_synclog.8 \ man/cyr_virusscan.8 \ man/deliver.8 \ man/fud.8 \ man/idled.8 \ man/imapd.8 \ man/ipurge.8 \ man/lmtpd.8 \ man/master.8 \ man/mbexamine.8 \ man/mbpath.8 \ man/mbtool.8 \ man/notifyd.8 \ man/pop3d.8 \ man/quota.8 \ man/reconstruct.8 \ man/restore.8 \ man/smmapd.8 \ man/timsieved.8 \ man/tls_prune.8 \ man/unexpunge.8 if SQUATTER dist_man8_MANS += \ man/squatter.8 endif # SQUATTER if NNTPD dist_man8_MANS += \ man/fetchnews.8 \ man/nntpd.8 endif # NNTPD if HTTPD dist_man8_MANS += \ man/ctl_zoneinfo.8 \ man/httpd.8 endif # HTTPD if BENCH check_PROGRAMS += bench/cyrdbbench bench_cyrdbbench_SOURCES = bench/cyrdbbench.c imap/mutex_fake.c bench_cyrdbbench_LDADD = $(LD_BASIC_ADD) endif # BENCH if REPLICATION dist_man8_MANS += \ man/sync_client.8 \ man/sync_reset.8 \ man/sync_server.8 endif master_master_SOURCES = \ master/master.c \ master/master.h \ master/masterconf.c \ master/masterconf.h \ master/service.h master_master_LDADD = lib/libcyrus_min.la $(LIBS) $(GCOV_LIBS) -lm netnews_remotepurge_SOURCES = \ netnews/macros.h \ netnews/readconfig.c \ netnews/readconfig.h \ netnews/remotepurge.c netnews_remotepurge_LDADD = $(LD_BASIC_ADD) notifyd_notifyd_SOURCES = \ imap/mutex_fake.c \ master/service.c \ notifyd/notify_external.c \ notifyd/notify_external.h \ notifyd/notify_log.c \ notifyd/notify_log.h \ notifyd/notify_mailto.c \ notifyd/notify_mailto.h \ notifyd/notify_null.c \ notifyd/notify_null.h \ notifyd/notifyd.c \ notifyd/notifyd.h if ZEPHYR notifyd_notifyd_SOURCES += notifyd/notify_zephyr.c notifyd/notify_zephyr.h endif notifyd_notifyd_LDADD = $(LD_SERVER_ADD) $(ZEPHYR_LIBS) notifyd_notifytest_SOURCES = notifyd/notifytest.c imap/mutex_fake.c notifyd_notifytest_LDADD = $(LD_BASIC_ADD) nodist_perl_libcyrus_la_SOURCES = $(nodist_lib_libcyrus_la_SOURCES) perl_libcyrus_la_SOURCES = $(lib_libcyrus_la_SOURCES) perl_libcyrus_la_LIBADD = $(lib_libcyrus_la_LIBADD) nodist_perl_libcyrus_min_la_SOURCES = $(nodist_lib_libcyrus_min_la_SOURCES) perl_libcyrus_min_la_SOURCES = $(lib_libcyrus_min_la_SOURCES) perl_libcyrus_min_la_LIBADD = $(lib_libcyrus_min_la_LIBADD) perl_sieve_lib_libisieve_la_SOURCES = \ perl/sieve/lib/codes.h \ perl/sieve/lib/isieve.c \ perl/sieve/lib/isieve.h \ perl/sieve/lib/lex.c \ perl/sieve/lib/lex.h \ perl/sieve/lib/request.c \ perl/sieve/lib/request.h ptclient_ptdump_SOURCES = imap/cli_fatal.c imap/mutex_fake.c ptclient/ptdump.c ptclient_ptdump_LDADD = $(LD_UTILITY_ADD) ptclient_ptexpire_SOURCES = imap/cli_fatal.c imap/mutex_fake.c ptclient/ptexpire.c ptclient_ptexpire_LDADD = $(LD_UTILITY_ADD) ptclient_ptloader_SOURCES = \ imap/mutex_fake.c \ ptclient/ptloader.c \ ptclient/ptloader.h \ master/service-thread.c ptclient_ptloader_LDFLAGS = ptclient_ptloader_LDADD = $(LD_SERVER_ADD) if HAVE_LDAP ptclient_ptloader_SOURCES += ptclient/ldap.c ptclient_ptloader_LDADD += $(LDAP_LIBS) ptclient_ptloader_LDFLAGS += $(LDAP_LDFLAGS) endif if USE_AFSKRB ptclient_ptloader_SOURCES += ptclient/afskrb.c ptclient_ptloader_LDADD += $(AFS_LIBS) ptclient_ptloader_LDFLAGS += $(AFS_LDFLAGS) endif # Each new version of flex seems to generate new sign-comparison # warnings, while they switch code around between int/yy_size_t. # # This is a pain because most of us Cyrus devs build with -Werror. # # Instead of continuing to maintain an ever-growing, increasingly # precarious set of post-hoc fixes (see: the former lex-fix rule), # let's just ignore sign-comparison warnings from flex-generated # sources only. sieve_libcyrus_sieve_lex_la_SOURCES = \ sieve/addr-lex.l \ sieve/sieve-lex.l sieve_libcyrus_sieve_lex_la_LIBADD = sieve_libcyrus_sieve_lex_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) $(NOWARN_SIGN_COMPARE) nodist_sieve_libcyrus_sieve_la_SOURCES = \ sieve/sieve_err.c \ sieve/sieve_err.h sieve_libcyrus_sieve_la_SOURCES = \ sieve/bytecode.h \ sieve/addr.y \ sieve/bc_emit.c \ sieve/bc_eval.c \ sieve/bc_generate.c \ sieve/bc_parse.c \ sieve/bc_parse.h \ sieve/comparator.c \ sieve/comparator.h \ sieve/flags.c \ sieve/flags.h \ sieve/grammar.c \ sieve/grammar.h \ sieve/interp.c \ sieve/interp.h \ sieve/message.c \ sieve/message.h \ sieve/rebuild.c \ sieve/script.c \ sieve/script.h \ sieve/sieve.y \ sieve/tree.c \ sieve/tree.h \ sieve/variables.c \ sieve/variables.h \ sieve/varlist.c \ sieve/varlist.h if JMAP sieve_libcyrus_sieve_la_SOURCES += \ imap/jmap_mail_query_parse.c \ imap/jmap_mail_query_parse.h \ imap/json_support.c \ imap/json_support.h endif sieve_libcyrus_sieve_la_LIBADD = \ sieve/libcyrus_sieve_lex.la \ $(COM_ERR_LIBS) \ lib/libcyrus_min.la \ lib/libcyrus.la sieve_libcyrus_sieve_la_CFLAGS = $(AM_CFLAGS) $(CFLAG_VISIBILITY) sieve_sievec_LDADD = $(LD_SIEVE_ADD) sieve_sieved_LDADD = $(LD_SIEVE_ADD) sieve_test_SOURCES = \ sieve/test.c \ imap/mutex_fake.c \ sieve/sieve_interface.h if JMAP sieve_test_SOURCES += \ imap/jmap_util.c imap/jmap_mail_query.c imap/jmap_mail_query_parse.c endif sieve_test_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) sieve_test_mailbox_SOURCES = \ sieve/test_mailbox.c \ imap/mutex_fake.c \ sieve/sieve_interface.h sieve_test_mailbox_LDADD = $(LD_SIEVE_ADD) $(LD_UTILITY_ADD) timsieved_timsieved_SOURCES = \ imap/mutex_fake.c \ imap/proxy.c \ master/service.c \ timsieved/actions.c \ timsieved/actions.h \ timsieved/codes.h \ timsieved/lex.c \ timsieved/lex.h \ timsieved/parser.c \ timsieved/parser.h \ timsieved/timsieved.c timsieved_timsieved_LDADD = $(LD_SIEVE_ADD) $(LD_SERVER_ADD) # The lex-fix rule is now obsolete, but let's leave a crumb here for a # while so that custom build scripts that expect it to exist don't choke. lex-fix: snapshot:: @echo "Creating snapshot $(PACKAGE_NAME)-$(PACKAGE_VERSION)" @$(MKDIR_P) snapshot @git archive --format=tar --prefix=$(PACKAGE_NAME)-$(PACKAGE_VERSION)/ HEAD | tar -C snapshot/ -x -f - @echo "creating tarfile" tar -C snapshot -c -f - $(PACKAGE_NAME)-$(PACKAGE_VERSION) | gzip -9 > $(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz @rm -rf snapshot docsrc/imap/reference/manpages/systemcommands/cyradm.rst: $(top_srcdir)/perl/imap/IMAP/Shell.pm @$(MKDIR_P) $(@D) $(AM_V_GEN)$(top_srcdir)/tools/perl2rst cyradm < $< > $@.NEW && mv $@.NEW $@ docsrc/imap/reference/manpages/configs/imapd.conf.rst: $(top_srcdir)/tools/config2rst $(top_srcdir)/lib/imapoptions @$(MKDIR_P) $(@D) $(AM_V_GEN)$(top_srcdir)/tools/config2rst $(top_srcdir)/lib/imapoptions > $@ doc/examples/imapd_conf/imapd.conf.sample: $(top_srcdir)/tools/config2sample $(top_srcdir)/lib/imapoptions @$(MKDIR_P) $(@D) $(AM_V_GEN)$(top_srcdir)/tools/config2sample $(top_srcdir)/lib/imapoptions > $@ docsrc/imap/reference/manpages/usercommands/sieveshell.rst: $(top_srcdir)/perl/sieve/scripts/sieveshell.pl @$(MKDIR_P) $(@D) $(AM_V_GEN)$(top_srcdir)/tools/perl2rst sieveshell < $< > $@.NEW && mv $@.NEW $@ %.1: man/.sphinx-build.stamp @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ %.3: man/.sphinx-build.stamp @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ %.5: man/.sphinx-build.stamp @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ %.8: man/.sphinx-build.stamp @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ ## define this unconditionally because dist-hook references it SPHINX_CACHE = docsrc/.doctrees .PHONY: clean-sphinx-cache clean-docsrc clean-sphinx-cache: @$(RM) -r $(SPHINX_CACHE) if HAVE_SPHINX_BUILD SPHINX_OPTS = -d $(SPHINX_CACHE) -n -q ## detect when source directory is not build directory (i.e. VPATH ## build), and clone the docsrc tree into the build directory, so ## that we have a single "source directory" for sphinx-build to use. docsrc/.sphinx-build.stamp: docsrc/imap/reference/manpages/configs/imapd.conf.rst docsrc/imap/reference/manpages/systemcommands/cyradm.rst docsrc/imap/reference/manpages/usercommands/sieveshell.rst $(AM_V_GEN)test x"$(top_srcdir)" = x"$(top_builddir)" || \ (cd $(top_srcdir) && tar cf - --mode=gu+w docsrc doc/examples) | tar xf - @touch $@ clean-docsrc: test x"$(top_srcdir)" = x"$(top_builddir)" || \ $(RM) -r $(top_builddir)/docsrc $(top_builddir)/doc/examples ## XXX doesn't detect if other rst sources are updated... man/.sphinx-build.stamp: docsrc/.sphinx-build.stamp $(AM_V_GEN)DOCSRC=$(top_builddir)/docsrc $(SPHINX_BUILD) $(SPHINX_OPTS) -b cyrman $(top_builddir)/docsrc $(top_builddir)/man @touch $@ clean-man: @$(RM) -r man ## XXX doesn't detect if other rst sources are updated... doc/html/.sphinx-build.stamp: docsrc/.sphinx-build.stamp $(AM_V_GEN)DOCSRC=$(top_builddir)/docsrc $(SPHINX_BUILD) $(SPHINX_OPTS) -b html $(top_builddir)/docsrc $(top_builddir)/doc/html @touch $@ ## XXX doesn't detect if other rst sources are updated... doc/text/.sphinx-build.stamp: docsrc/.sphinx-build.stamp $(AM_V_GEN)DOCSRC=$(top_builddir)/docsrc $(SPHINX_BUILD) $(SPHINX_OPTS) -b text $(top_builddir)/docsrc $(top_builddir)/doc/text @touch $@ else clean-docsrc: man/.sphinx-build.stamp: @$(MKDIR_P) $(@D) $(AM_V_GEN)touch $@ @echo "warning: missing documentation dependencies. man pages will be empty" 1>&2 clean-man: doc/html/.sphinx-build.stamp: @echo "error: missing documentation dependencies. cannot build html documentation" 1>&2 @false doc/text/.sphinx-build.stamp: @echo "error: missing documentation dependencies. cannot build text documentation" 1>&2 @false endif .PHONY: man man-force clean-man man-force: ## trash the timestamps, so we always re-run sphinx-build. ## sphinx-build tracks source modifications itself, so forced ## rebuilds will still be a lot quicker once it has run once. @$(RM) docsrc/.sphinx-build.stamp man/.sphinx-build.stamp man: man-force $(dist_man1_MANS) $(dist_man3_MANS) $(dist_man5_MANS) $(dist_man8_MANS) # clean-man is conditionally defined above. We do not want to remove # man pages if the user is unable to rebuild them. .PHONY: doc-html doc-html-force clean-doc-html doc-html-force: @$(RM) docsrc/.sphinx-build.stamp doc/html/.sphinx-build.stamp doc-html: doc-html-force doc/html/.sphinx-build.stamp clean-doc-html: @$(RM) -r doc/html .PHONY: doc-text doc-text-force clean-doc-text doc-text-force: @$(RM) docsrc/.sphinx-build.stamp doc/text/.sphinx-build.stamp doc-text: doc-text-force doc/text/.sphinx-build.stamp clean-doc-text: @$(RM) -r doc/text GENERATED_EXAMPLES = doc/examples/imapd_conf/imapd.conf.sample .PHONY: doc-examples doc-examples-force clean-doc-examples doc-examples-force: doc-examples: doc-examples-force $(GENERATED_EXAMPLES) clean-doc-examples: @$(RM) $(GENERATED_EXAMPLES) .PHONY: doc clean-doc doc: man doc-html doc-text doc-examples clean-doc: clean-man clean-doc-html clean-doc-text clean-doc-examples .PHONY: clean-local clean-local: clean-doc clean-sphinx-cache clean-docsrc distgit: $(MKDIR_P) dist @echo "checking out the distribution from tag $(PACKAGE_NAME)-$(PACKAGE_VERSION)" git archive --format=tar --prefix=$(PACKAGE_NAME)-$(PACKAGE_VERSION)/ $(PACKAGE_NAME)-$(PACKAGE_VERSION) | tar -C dist -x -f - touch distgit dist-hook: find $(top_distdir) -type f -name .sphinx-build.stamp -delete find $(top_distdir) -type f -name .gitignore -delete rm -rf $(top_distdir)/$(SPHINX_CACHE) VERSION: tools/git-version.sh $(AM_V_GEN)$< > $@.NEW && mv $@.NEW $@ install-data-hook: if CMULOCAL $(INSTALL) -m 644 $(top_srcdir)/depot/depot.conf $(DESTDIR)/ endif install-exec-hook: if PERL for s in installsieve sieveshell; \ do \ $(MKDIR_P) $(top_builddir)/perl/sieve/scripts ; \ $(PERL_PREINSTALL) < $(top_srcdir)/perl/sieve/scripts/$$s.pl > $(top_builddir)/perl/sieve/scripts/$$s ;\ $(INSTALL) -m 755 $(top_builddir)/perl/sieve/scripts/$$s $(DESTDIR)$(bindir)/$$s ; \ done endif ## The @$(MKDIR_P) line is added due to a bug in Automake 1.10 and can be removed if using Automake 1.12. @$(MKDIR_P) $(DESTDIR)$(libexecdir) if SERVER cd $(DESTDIR)$(libexecdir) && \ $(LN_S) -f pop3d pop3proxyd && \ $(LN_S) -f imapd proxyd && \ $(LN_S) -f lmtpd lmtpproxyd endif ## The @$(MKDIR_P) line is added due to a bug in Automake 1.10 and can be removed if using Automake 1.12. @$(MKDIR_P) $(DESTDIR)$(bindir) cd $(DESTDIR)$(bindir) && \ $(LN_S) -f imtest httptest && \ $(LN_S) -f imtest lmtptest && \ $(LN_S) -f imtest mupdatetest && \ $(LN_S) -f imtest nntptest && \ $(LN_S) -f imtest pop3test && \ $(LN_S) -f imtest sivtest && \ $(LN_S) -f imtest smtptest && \ $(LN_S) -f imtest synctest uninstall-hook: cyrus-makemaker-uninstall-workaround if PERL for s in installsieve sieveshell; do \ rm -f $(DESTDIR)$(bindir)/$$s; \ done endif .PHONY: cyrus-makemaker-uninstall-workaround # Workaround for deprecated no-op MakeMaker uninstall. # Don't remove the packlists themselves, otherwise MakeMaker uninstall will # crash. Just remove the files MakeMaker uninstall won't remove itself. # # We have another layer of workaround in each module's Makefile that # removes the packlist after MakeMaker uninstall has finished ignoring it. cyrus-makemaker-uninstall-workaround: if PERL ( packlists=`find $(DESTDIR)$(libdir) -type f -name .packlist` && \ echo "uninstalling files from packlists: $$packlists" && \ rm -f `cat $$packlists` ) ( podfiles=`find $(DESTDIR)$(libdir) -type f -name perllocal.pod` && \ rm -f `echo $$podfiles` ) endif install-binsymlinks: ## Let's symlink everything else for prog in $(sbin_PROGRAMS); do $(LN_S) -f $(sbindir)/`basename $$prog` $(DESTDIR)$(bindir)/; done for prog in $(libexec_PROGRAMS); do $(LN_S) -f $(libexecdir)/`basename $$prog` $(DESTDIR)$(bindir)/; done SUFFIXES = .fig.png .fig.png: mkdir -p $(@D) fig2dev -L png $< $@ valgrind: $(MAKE) VG=yes check libtool: $(LIBTOOL_DEPS) $(SHELL) ./config.status libtool # Install for Cassandane cassandane: $(MAKE) DESTDIR=`cd ../inst ; /bin/pwd` install diff --git a/imap/icu_wrap.cpp b/imap/icu_wrap.cpp index 6ea805bcc..60b4c92a0 100644 --- a/imap/icu_wrap.cpp +++ b/imap/icu_wrap.cpp @@ -1,76 +1,76 @@ /* icu_wrap.c -- C++ hiding wrapper API for ICU * * Copyright (c) 1994-2019 Carnegie Mellon University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * 3. The name "Carnegie Mellon University" must not be used to * endorse or promote products derived from this software without * prior written permission. For permission or any legal * details, please contact * Carnegie Mellon University * Center for Technology Transfer and Enterprise Creation * 4615 Forbes Avenue * Suite 302 * Pittsburgh, PA 15213 * (412) 268-7393, fax: (412) 268-7395 * innovation@andrew.cmu.edu * * 4. Redistributions of any form whatsoever must retain the following * acknowledgment: * "This product includes software developed by Computing Services * at Carnegie Mellon University (http://www.cmu.edu/computing/)." * * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ extern "C" { #include "config.h" #include "string.h" #include "xmalloc.h" }; #include #include #include -extern "C" char *icu_getIDForWindowsID(const char *id) +extern "C" EXPORTED char *icu_getIDForWindowsID(const char *id) { UErrorCode status = U_ZERO_ERROR; UConverter *utf8cnv = ucnv_open("utf-8", &status); if (U_FAILURE(status)) return NULL; icu::UnicodeString uWinID {id, -1, utf8cnv, status}; ucnv_close(utf8cnv); icu::UnicodeString uID; icu::TimeZone::getIDForWindowsID(uWinID, NULL, uID, status); std::string str; uID.toUTF8String(str); if (!str.empty()) return xstrdup(str.c_str()); if (!strcasecmp(id, "Mid-Atlantic Standard Time")) { /* ICU doesn't map this ID */ return xstrdup("Atlantic/South_Georgia"); } return NULL; } diff --git a/imap/xapian_wrap.cpp b/imap/xapian_wrap.cpp index 367812bbd..c656bd8f5 100644 --- a/imap/xapian_wrap.cpp +++ b/imap/xapian_wrap.cpp @@ -1,2498 +1,2518 @@ #include #include #include #include #include #include #include #include #include extern "C" { #include #include "libconfig.h" #include "util.h" #include "search_engines.h" #include "search_part.h" #include "xmalloc.h" #include "xapian_wrap.h" #include "charset.h" #include "ptrarray.h" #include "parseaddr.h" /* generated headers are not necessarily in current directory */ #include "imap/imap_err.h" }; #include #include #include #ifdef HAVE_CLD2 #include #endif // from global.h extern int charset_flags; #define SLOT_CYRUSID 0 #define SLOT_DOCLANGS 1 #define SLOT_INDEXLEVEL 2 #define SLOT_INDEXVERSION 3 static const unsigned XAPIAN_MAX_TERM_LENGTH = 200; /* in UTF-8 bytes */ /* ====================================================================== */ static void make_cyrusid(struct buf *dst, const struct message_guid *guid, char doctype) { buf_reset(dst); buf_putc(dst, '*'); buf_putc(dst, doctype); buf_putc(dst, '*'); buf_appendcstr(dst, message_guid_encode(guid)); buf_cstring(dst); } /* ====================================================================== */ /* * A brief history of Xapian db versions: * Version 0: uses STEM_ALL for all terms, term prefixes don't start with 'X' * Version 1: term prefixes start with 'X' * Version 2: uses STEM_SOME for some terms * Version 3: removes all use of STEM_ALL * Version 4: indexes headers and bodies in separate documents * Version 5: indexes headers and bodies together and stems by language * Version 6: stores all detected languages of a document in slot SLOT_DOCLANGS (deprecated) * Version 7: indexes new DELIVEREDTO search part * Version 8: reintroduces language indexing for non-English text * Version 9: introduces index levels as keys to cyrusid metadata * Version 10: indexes new PRIORITY search part * Version 11: indexes LIST-ID as single value * Version 12: indexes email domains as single values. Supports subdomain search. * Version 13: indexes content-type and subtype separately * Version 14: adds SLOT_INDEXVERSION to documents * Version 15: receives indexed header fields and text in original format (rather than search form) * Version 16: indexes entire addr-spec as a single value. Prevents cross-matching localparts and domains */ #define XAPIAN_DB_CURRENT_VERSION 16 #define XAPIAN_DB_MIN_SUPPORTED_VERSION 5 static std::set read_db_versions(const Xapian::Database &database) { std::set versions; // db_version is a comma-separated list of version numbers std::string val = database.get_metadata("cyrus.db_version"); if (!val.empty()) { strarray_t *vstr = strarray_split(val.c_str(), ",", 0); for (int i = 0; i < strarray_size(vstr); i++) { int version = std::atoi(strarray_nth(vstr, i)); if (version) versions.insert(version); } strarray_free(vstr); } // Up to version 3 this was named stem version. val = database.get_metadata("cyrus.stem-version"); if (!val.empty()) { versions.insert(std::stoi(val)); } return versions; } static void write_db_versions(Xapian::WritableDatabase &database, std::set &versions) { std::ostringstream val; for (std::set::iterator it = versions.begin(); it != versions.end(); ++it) { if (it != versions.begin()) val << ","; val << *it; } database.set_metadata("cyrus.db_version", val.str()); database.set_metadata("cyrus.stem-version", ""); } /* ====================================================================== */ #define XAPIAN_LANG_COUNT_KEYPREFIX "lang.count" #define XAPIAN_LANG_DOC_KEYPREFIX "lang.doc" static std::string lang_prefix(const std::string& iso_lang, const char *prefix) { std::string ustr = std::string(prefix) + "XI" + iso_lang; std::transform(ustr.begin(), ustr.end(), ustr.begin(), ::toupper); return ustr; } static std::string lang_doc_key(const char *cyrusid) { std::string key(XAPIAN_LANG_DOC_KEYPREFIX "."); key += cyrusid; return key; } static std::string lang_count_key(const std::string& iso_lang) { std::string key(XAPIAN_LANG_COUNT_KEYPREFIX "."); key += iso_lang; return key; } static int calculate_language_counts(const Xapian::Database& db, std::map& lang_counts) { std::set db_versions = read_db_versions(db); if (db_versions.lower_bound(8) == db_versions.begin()) { // count all indexed body parts size_t nparts = 0; for (Xapian::TermIterator it = db.metadata_keys_begin("cyrusid.*P*"); it != db.metadata_keys_end("cyrusid.*P*"); ++it) { nparts++; } // count body parts with language metadata const std::string prefix{XAPIAN_LANG_DOC_KEYPREFIX ".*P*"}; size_t nlangparts = 0; for (Xapian::TermIterator it = db.metadata_keys_begin(prefix); it != db.metadata_keys_end(prefix); ++it) { lang_counts[db.get_metadata(*it)] += 1; nlangparts++; } // English or unknown language body parts have no metadata. lang_counts["en"] += nparts - nlangparts; // Sanity check data if (nparts < nlangparts) { return IMAP_IOERROR; } } return 0; } static void remove_legacy_metadata(Xapian::WritableDatabase& db) { const std::string prefix{XAPIAN_LANG_DOC_KEYPREFIX "."}; for (Xapian::TermIterator key = db.metadata_keys_begin(prefix); key != db.metadata_keys_end(prefix); ++key) { const std::string& val = db.get_metadata(*key); // Remove legacy keys and values. if ((*key).find('.') != std::string::npos || (!val.empty() && !isalpha(val[0]))) { db.set_metadata(*key, ""); } } for (Xapian::docid docid = 1; docid <= db.get_lastdocid(); ++docid) { try { Xapian::Document doc = db.get_document(docid); const std::string& val = doc.get_value(SLOT_DOCLANGS); // Remove legacy doclang slot values. if (!val.empty() && !isalpha(val[0])) { doc.remove_value(SLOT_DOCLANGS); } } catch (Xapian::DocNotFoundError e) { // ignore } } } static void write_language_counts(Xapian::WritableDatabase& db, const std::map& lang_counts) { for (Xapian::TermIterator it = db.metadata_keys_begin(XAPIAN_LANG_COUNT_KEYPREFIX); it != db.metadata_keys_end(XAPIAN_LANG_COUNT_KEYPREFIX); ++it) { db.set_metadata(*it, ""); } for (const std::pair& it : lang_counts) { db.set_metadata(lang_count_key(it.first), std::to_string(it.second)); } } static void read_language_counts(const Xapian::Database& db, std::map& lang_counts) { std::set db_versions = read_db_versions(db); if (db_versions.lower_bound(8) == db_versions.begin()) { const std::string prefix(XAPIAN_LANG_COUNT_KEYPREFIX "."); for (Xapian::TermIterator it = db.metadata_keys_begin(prefix); it != db.metadata_keys_end(prefix); ++it) { std::string iso_lang = (*it).substr(prefix.length()); unsigned count = std::stol(db.get_metadata(*it)); lang_counts[iso_lang] += count; } } } static void parse_doclangs(const std::string& val, std::set& doclangs) { if (val.empty() || !isalpha(val[0])) return; size_t base = 0, pos; while ((pos = val.find(',', base)) != std::string::npos) { doclangs.insert(val.substr(base, pos - base)); base = pos + 1; } doclangs.insert(val.substr(base)); } static std::string format_doclangs(const std::set& doclangs) { std::ostringstream val; for (std::set::iterator it = doclangs.begin(); it != doclangs.end(); ++it) { if (it != doclangs.begin()) val << ","; val << *it; } std::string s = val.str(); return s; } static std::string parse_langcode(const char *str) { std::string lstr(str); std::transform(lstr.begin(), lstr.end(), lstr.begin(), ::tolower); // accept syntax for two and three letter ISO 639 codes if (!(isalpha(lstr[0]) && isalpha(lstr[1]) && (lstr[2] == '\0' || (isalpha(lstr[2]) && lstr[3] == '\0')))) { return std::string(); } return lstr; } // Process-scoped, thread-unsafe cache of stoppers by ISO 639 code. static std::map> stoppers; static const Xapian::Stopper* get_stopper(const std::string& iso) { // Lookup cached entry. try { return stoppers.at(iso).get(); } catch (const std::out_of_range&) {}; // Lookup language name by ISO code. icu::Locale loc(iso.c_str()); if (loc.isBogus()) return NULL; // Read stopper file and add to cache. const char *swpath = config_getstring(IMAPOPT_SEARCH_STOPWORD_PATH); if (!swpath) return NULL; std::string lang_name; icu::UnicodeString ulang_name; loc.getDisplayLanguage(icu::Locale("en"), ulang_name); ulang_name.toLower(); ulang_name.toUTF8String(lang_name); // Open stopword file // XXX doesn't play nice with WIN32 paths std::string fname(std::string(swpath) + "/" + lang_name + ".txt"); errno = 0; std::ifstream inFile (fname); if (inFile.fail()) { syslog(LOG_DEBUG, "Xapian: could not open stopword file %s: %s", fname.c_str(), errno ? strerror(errno) : "unknown error"); return NULL; } // Create and store the Xapian stopper stoppers[iso].reset(new Xapian::SimpleStopper( std::istream_iterator(inFile), std::istream_iterator())); return stoppers[iso].get(); } class CyrusSearchStemmer : public Xapian::StemImplementation { charset_t utf8 {charset_lookupname("utf-8")}; std::map cache; Xapian::Stem stem {"en"}; public: virtual ~CyrusSearchStemmer() { charset_free(&utf8); } virtual std::string operator() (const std::string &word) override { // Is this word already in the cache? try { return cache.at(word); } catch (const std::out_of_range&) {} // Convert the word to search form std::unique_ptr q {charset_convert(word.c_str(), utf8, charset_flags), std::free}; std::string s = q ? stem(Xapian::Unicode::tolower(q.get())) : stem(word); if (s.size() > XAPIAN_MAX_TERM_LENGTH) return std::string{}; // Store the normalized word in the cache return cache[word] = s; } virtual std::string get_description () const override { return "Cyrus"; } }; class FrenchContractionStemmer : public Xapian::StemImplementation { Xapian::Stem stem {"fr"}; public: virtual std::string operator() (const std::string &word) override { size_t pos = 0; switch (word[0]) { case 'q': if (word.length() <= 3 || word[1] != 'u') { break; } pos++; // fall through case 'c': case 'd': case 'j': case 'l': case 'm': case 'n': case 's': case 't': // APOSTROPHE (U+0027) if (word.length() > pos + 2 && word[pos+1] == 0x27) { return stem(word.substr(pos + 2)); } // RIGHT SINGLE QUOTATION MARK (U+2019) // FULLWIDTH APOSTROPHE (U+FF07) else if (!word.compare(pos + 1, 3, "\xe2\x80\x99") || !word.compare(pos + 1, 3, "\xef\xbc\x87")) { return stem(word.substr(pos + 4)); } // fall through } // not a contraction return stem(word); } virtual std::string get_description () const override { return "fr-contraction"; } }; static Xapian::Stem get_stemmer(const std::string& iso_lang) { return iso_lang == "fr" ? Xapian::Stem{new FrenchContractionStemmer} : Xapian::Stem{iso_lang}; } #ifdef HAVE_CLD2 static std::string detect_language(const struct buf *part) { std::string iso_lang; bool reliable = false; CLD2::Language lang = CLD2::DetectLanguage(part->s, part->len, 1, &reliable); if (reliable && lang != CLD2::UNKNOWN_LANGUAGE) { std::string code(CLD2::LanguageCode(lang)); std::transform(code.begin(), code.end(), code.begin(), ::tolower); // Map CLD2 special codes to ISO 639. if (!code.compare("zh-Hant")) { code = "zh"; } else if (!code.compare("sr-ME" )) { code = "sr"; // not a political statement! } else if (!code.compare("xxx")) { code = ""; } iso_lang = parse_langcode(code.c_str()); } return iso_lang; } #endif /* HAVE_CLD2 */ /* ====================================================================== */ static uint8_t better_indexlevel(uint8_t levela, uint8_t levelb) { uint8_t a = levela & ~SEARCH_INDEXLEVEL_PARTIAL; uint8_t b = levelb & ~SEARCH_INDEXLEVEL_PARTIAL; if (a > b) return levela; if (a < b) return levelb; return (levela & SEARCH_INDEXLEVEL_PARTIAL) ? levelb : levela; } static uint8_t parse_indexlevel(const std::string& s) { uint8_t level = 0; if (hex_to_bin(s.c_str(), s.length(), &level) != 1) { return 0; } return level; } static std::string format_indexlevel(uint8_t level) { char hex[4]; bin_to_lchex(&level, 1, hex); return std::string(hex, 2); } /* ====================================================================== */ class CyrusMetadataCompactor : public Xapian::Compactor { public: CyrusMetadataCompactor() { } std::string resolve_duplicate_metadata(const std::string &key, size_t num_tags, const std::string tags[]) { if (key.rfind("cyrusid.", 0) == 0) { uint8_t indexlevel = parse_indexlevel(tags[0]); size_t bestpos = 0; for (size_t i = 1; i < num_tags; i++) { uint8_t level = parse_indexlevel(tags[i]); if (better_indexlevel(indexlevel, level) == level) { indexlevel = level; bestpos = i; } } return tags[bestpos]; } return tags[0]; } }; -int xapian_compact_dbs(const char *dest, const char **sources) +EXPORTED int xapian_compact_dbs(const char *dest, const char **sources) { int r = 0; Xapian::Database db; const char *thispath = "(unknown path)"; try { std::set db_versions; std::map lang_counts; std::vector subdbs; while (*sources) { thispath = *sources; Xapian::Database subdb(*sources++); db.add_database(subdb); subdbs.push_back(subdb); // Aggregate db versions. bool need_metadata = false; for (Xapian::docid docid = 1; docid <= subdb.get_lastdocid(); ++docid) { try { Xapian::Document doc = subdb.get_document(docid); const std::string& val = doc.get_value(SLOT_INDEXVERSION); if (!val.empty()) { int version = std::atoi(val.c_str()); if (version) db_versions.insert(version); } else need_metadata = true; } catch (Xapian::DocNotFoundError e) { // ignore } } if (need_metadata) { /* At least one document didn't have its index version set. * Read the legacy version from the metadata. */ std::set md_versions = read_db_versions(subdb); db_versions.insert(md_versions.begin(), md_versions.lower_bound(14)); } // Aggregate language counts. r = calculate_language_counts(subdb, lang_counts); if (r) { xsyslog(LOG_ERR, "IOERROR: corrupt language metadata", "path=<%s>", thispath); return r; } } thispath = "(unknown path)"; // Compact database. static CyrusMetadataCompactor comp; // FULLER because we never write to compression targets again. db.compact(dest, Xapian::Compactor::FULLER | Xapian::DBCOMPACT_MULTIPASS, 0, comp); Xapian::WritableDatabase newdb(dest); write_db_versions(newdb, db_versions); // Clean metadata. remove_legacy_metadata(newdb); // Reset language counts. write_language_counts(newdb, lang_counts); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); r = IMAP_IOERROR; } return r; } /* ====================================================================== */ static const char *get_term_prefix(int db_version, int partnum) { /* * We use term prefixes to store terms per search part. * In addition, each Xapian document contains a "XE" * prefix to indicate its document type, listed in * the XAPIAN_WRAP_DOCTYPE definitions. The "XE" prefix * MUST not be used for any search part. * */ static const char * const term_prefixes[SEARCH_NUM_PARTS] = { NULL, /* ANY */ "XF", /* FROM */ "XT", /* TO */ "XC", /* CC */ "XB", /* BCC */ "XS", /* SUBJECT */ "XL", /* LISTID */ "XY", /* TYPE */ "XH", /* HEADERS */ "", /* BODY */ "XO", /* LOCATION */ "XA", /* ATTACHMENTNAME */ "XAB", /* ATTACHMENTBODY */ "XDT", /* DELIVEREDTO */ "XI", /* LANGUAGE */ "XP" /* PRIORITY */ }; static const char * const term_prefixes_v0[SEARCH_NUM_PARTS] = { NULL, /* ANY */ "F", /* FROM */ "T", /* TO */ "C", /* CC */ "B", /* BCC */ "S", /* SUBJECT */ "L", /* LISTID */ "Y", /* TYPE */ "H", /* HEADERS */ "D", /* BODY */ "O", /* LOCATION */ "A", /* ATTACHMENTNAME */ "AB", /* ATTACHMENTBODY */ "E", /* DELIVEREDTO */ NULL, /* LANGUAGE */ NULL /* PRIORITY */ }; return db_version > 0 ? term_prefixes[partnum] : term_prefixes_v0[partnum]; } static Xapian::TermGenerator::stem_strategy get_stem_strategy(int db_version, int partnum) { static Xapian::TermGenerator::stem_strategy stem_strategy[SEARCH_NUM_PARTS] = { // Version 2 and higher Xapian::TermGenerator::STEM_NONE, /* ANY */ Xapian::TermGenerator::STEM_NONE, /* FROM */ Xapian::TermGenerator::STEM_NONE, /* TO */ Xapian::TermGenerator::STEM_NONE, /* CC */ Xapian::TermGenerator::STEM_NONE, /* BCC */ Xapian::TermGenerator::STEM_SOME, /* SUBJECT */ Xapian::TermGenerator::STEM_NONE, /* LISTID */ Xapian::TermGenerator::STEM_NONE, /* TYPE */ Xapian::TermGenerator::STEM_NONE, /* HEADERS */ Xapian::TermGenerator::STEM_SOME, /* BODY */ Xapian::TermGenerator::STEM_SOME, /* LOCATION */ Xapian::TermGenerator::STEM_NONE, /* ATTACHMENTNAME */ Xapian::TermGenerator::STEM_SOME, /* ATTACHMENTBODY */ Xapian::TermGenerator::STEM_NONE, /* DELIVEREDTO */ Xapian::TermGenerator::STEM_NONE, /* LANGUAGE */ Xapian::TermGenerator::STEM_NONE /* PRIORITY */ }; static Xapian::TermGenerator::stem_strategy stem_strategy_v1[SEARCH_NUM_PARTS] = { // Version 1: Stem bodies using STEM_SOME with stopwords Xapian::TermGenerator::STEM_NONE, /* ANY */ Xapian::TermGenerator::STEM_ALL, /* FROM */ Xapian::TermGenerator::STEM_ALL, /* TO */ Xapian::TermGenerator::STEM_ALL, /* CC */ Xapian::TermGenerator::STEM_ALL, /* BCC */ Xapian::TermGenerator::STEM_ALL, /* SUBJECT */ Xapian::TermGenerator::STEM_ALL, /* LISTID */ Xapian::TermGenerator::STEM_ALL, /* TYPE */ Xapian::TermGenerator::STEM_ALL, /* HEADERS */ Xapian::TermGenerator::STEM_SOME, /* BODY */ Xapian::TermGenerator::STEM_SOME, /* LOCATION */ Xapian::TermGenerator::STEM_NONE, /* ATTACHMENTNAME */ Xapian::TermGenerator::STEM_SOME, /* ATTACHMENTBODY */ Xapian::TermGenerator::STEM_ALL, /* DELIVEREDTO */ Xapian::TermGenerator::STEM_NONE, /* LANGUAGE */ Xapian::TermGenerator::STEM_NONE /* PRIORITY */ }; static Xapian::TermGenerator::stem_strategy stem_strategy_v0[SEARCH_NUM_PARTS] = { // Version 0: Initial version Xapian::TermGenerator::STEM_NONE, /* ANY */ Xapian::TermGenerator::STEM_ALL, /* FROM */ Xapian::TermGenerator::STEM_ALL, /* TO */ Xapian::TermGenerator::STEM_ALL, /* CC */ Xapian::TermGenerator::STEM_ALL, /* BCC */ Xapian::TermGenerator::STEM_ALL, /* SUBJECT */ Xapian::TermGenerator::STEM_ALL, /* LISTID */ Xapian::TermGenerator::STEM_ALL, /* TYPE */ Xapian::TermGenerator::STEM_ALL, /* HEADERS */ Xapian::TermGenerator::STEM_ALL, /* BODY */ Xapian::TermGenerator::STEM_ALL, /* LOCATION */ Xapian::TermGenerator::STEM_ALL, /* ATTACHMENTNAME */ Xapian::TermGenerator::STEM_ALL, /* ATTACHMENTBODY */ Xapian::TermGenerator::STEM_ALL, /* DELIVEREDTO */ Xapian::TermGenerator::STEM_NONE, /* LANGUAGE */ Xapian::TermGenerator::STEM_NONE /* PRIORITY */ }; switch (db_version) { case 0: return stem_strategy_v0[partnum]; case 1: return stem_strategy_v1[partnum]; default: return stem_strategy[partnum]; } } /* For all db paths in sources that are not using the latest database * version or not readable, report their paths in toreindex */ -void xapian_check_if_needs_reindex(const strarray_t *sources, strarray_t *toreindex, int always_upgrade) +EXPORTED void xapian_check_if_needs_reindex(const strarray_t *sources, + strarray_t *toreindex, + int always_upgrade) { // Check the version of all dbs in sources for (int i = 0; i < sources->count; i++) { const char *thispath = strarray_nth(sources, i); try { for (const int& it: read_db_versions(Xapian::Database{thispath})) { if (it < XAPIAN_DB_MIN_SUPPORTED_VERSION || (always_upgrade && (it != XAPIAN_DB_CURRENT_VERSION))) { strarray_add(toreindex, thispath); } } } catch (const Xapian::Error &err) { strarray_add(toreindex, thispath); } } } /* ====================================================================== */ static inline void add_boolean_nterm(Xapian::Document& doc, const std::string& term) { if (term.size() && term.size() < XAPIAN_MAX_TERM_LENGTH) { doc.add_boolean_term(term); } } struct xapian_dbw { // Database context. Xapian::WritableDatabase *database; ptrarray_t otherdbs; Xapian::TermGenerator *term_generator; Xapian::Stem *default_stemmer; const Xapian::Stopper* default_stopper; // Document context. Xapian::Document *document; char doctype; char *cyrusid; std::set *doclangs; std::vector *subjects; }; static int xapian_dbw_init(xapian_dbw_t *dbw) { dbw->default_stemmer = new Xapian::Stem(new CyrusSearchStemmer); dbw->default_stopper = get_stopper("en"); dbw->term_generator = new Xapian::TermGenerator; dbw->term_generator->set_max_word_length(XAPIAN_MAX_TERM_LENGTH); /* Always enable CJK word tokenization */ #ifdef USE_XAPIAN_CJK_WORDS dbw->term_generator->set_flags(Xapian::TermGenerator::FLAG_CJK_WORDS, ~Xapian::TermGenerator::FLAG_CJK_WORDS); #else dbw->term_generator->set_flags(Xapian::TermGenerator::FLAG_CJK_NGRAM, ~Xapian::TermGenerator::FLAG_CJK_NGRAM); #endif dbw->doclangs = new std::set; dbw->subjects = new std::vector; return 0; } -int xapian_dbw_open(const char **paths, xapian_dbw_t **dbwp, int mode, int nosync) +EXPORTED int xapian_dbw_open(const char **paths, xapian_dbw_t **dbwp, + int mode, int nosync) { xapian_dbw_t *dbw = (xapian_dbw_t *)xzmalloc(sizeof(xapian_dbw_t)); int r = 0; const char *thispath = *paths++; std::set db_versions; try { int flags = Xapian::DB_BACKEND_GLASS|Xapian::DB_RETRY_LOCK; if (nosync) flags |= Xapian::DB_DANGEROUS|Xapian::DB_NO_SYNC; try { dbw->database = new Xapian::WritableDatabase{thispath, flags|Xapian::DB_OPEN}; db_versions = read_db_versions(*dbw->database); } catch (Xapian::DatabaseOpeningError &e) { /* It's OK not to atomically create or open, since we can assume * the xapianactive file items to be locked. */ dbw->database = new Xapian::WritableDatabase{thispath, flags|Xapian::DB_CREATE}; } if (db_versions.find(XAPIAN_DB_CURRENT_VERSION) == db_versions.end()) { // Always index using latest database version. db_versions.insert(XAPIAN_DB_CURRENT_VERSION); write_db_versions(*dbw->database, db_versions); } r = xapian_dbw_init(dbw); } catch (const Xapian::DatabaseLockError &err) { /* somebody else is already indexing this user. They may be doing a different * mailbox, so we need to re-insert this mailbox into the queue! */ r = IMAP_MAILBOX_LOCKED; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); r = IMAP_IOERROR; } if (r) { xapian_dbw_close(dbw); return r; } /* open the read-only databases */ if (mode == XAPIAN_DBW_XAPINDEXED) { while (*paths) { try { thispath = *paths; ptrarray_append(&dbw->otherdbs, new Xapian::Database{*paths++}); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: reading database", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); } } } *dbwp = dbw; return 0; } -void xapian_dbw_close(xapian_dbw_t *dbw) +EXPORTED void xapian_dbw_close(xapian_dbw_t *dbw) { if (!dbw) return; try { delete dbw->database; delete dbw->term_generator; delete dbw->document; delete dbw->default_stemmer; delete dbw->doclangs; delete dbw->subjects; for (int i = 0; i < dbw->otherdbs.count; i++) { delete (Xapian::Database *)ptrarray_nth(&dbw->otherdbs, i); } ptrarray_fini(&dbw->otherdbs); free(dbw->cyrusid); free(dbw); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); } } -int xapian_dbw_begin_txn(xapian_dbw_t *dbw) +EXPORTED int xapian_dbw_begin_txn(xapian_dbw_t *dbw) { int r = 0; try { dbw->database->begin_transaction(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_dbw_commit_txn(xapian_dbw_t *dbw) +EXPORTED int xapian_dbw_commit_txn(xapian_dbw_t *dbw) { int r = 0; try { dbw->database->commit_transaction(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_dbw_cancel_txn(xapian_dbw_t *dbw) +EXPORTED int xapian_dbw_cancel_txn(xapian_dbw_t *dbw) { int r = 0; try { dbw->database->cancel_transaction(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_dbw_begin_doc(xapian_dbw_t *dbw, const struct message_guid *guid, char doctype) +EXPORTED int xapian_dbw_begin_doc(xapian_dbw_t *dbw, + const struct message_guid *guid, + char doctype) { int r = 0; try { delete dbw->document; dbw->document = new Xapian::Document; dbw->doctype = doctype; /* Set document id and type */ struct buf buf = BUF_INITIALIZER; make_cyrusid(&buf, guid, doctype); dbw->document->add_value(SLOT_CYRUSID, buf_cstring(&buf)); dbw->cyrusid = buf_release(&buf); add_boolean_nterm(*dbw->document, std::string("XE") + doctype); /* Initialize term generator */ dbw->term_generator->set_document(*dbw->document); dbw->term_generator->set_termpos(1); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } static int add_language_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); std::string val = parse_langcode(buf_cstring(part)); if (val.empty()) { syslog(LOG_INFO, "Xapian: not a valid ISO 639 code: %s", buf_cstring(part)); return 0; } add_boolean_nterm(*dbw->document, prefix + val); return 0; } static std::string parse_priority(const char *str) { const char *err; uint32_t u; if (parseuint32(str, &err, &u) == -1 || *err || u == 0) { return std::string(); } return std::to_string(u); } static int add_priority_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); if (buf_len(part)) { std::string val = parse_priority(buf_cstring(part)); if (val.empty()) { syslog(LOG_DEBUG, "Xapian: not a valid priority: %s", buf_cstring(part)); return 0; } add_boolean_nterm(*dbw->document, prefix + val); } return 0; } static std::string parse_listid(const char *str) { std::string val; /* Extract list-id */ const char *start = strrchr(str, '<'); if (start) { /* RFC2919 list-id header (with optional closing bracket) */ const char *end = strchr(++start, '>'); if (end) val = std::string(start, end - start); else val = std::string(start); } else { /* Groups-style header: 'list list-id[; contact list-contact]' * As seen at Google Group, Yahoo, et al. */ for (start = str; isspace(*start); start++) {} if (!strncasecmp("list", start, 4) && isspace(start[4])) { for (start = start + 4; isspace(*start); start++) {} if (*start) { const char *end = strchr(start, ';'); if (!end || end - start) { val = end ? std::string(start, end - start) : std::string{start}; } } } /* just raw value, that's OK too, like sentry creates. Parse up to first whitespace */ else { const char *end; for (end = start; *end && !isspace(*end); end++) {} val = std::string(start, end - start); } } /* Normalize list-id */ val.erase(std::remove_if(val.begin(), val.end(), isspace), val.end()); std::transform(val.begin(), val.end(), val.begin(), ::tolower); return val; } static int add_listid_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); /* Normalize list-id */ std::string val = parse_listid(buf_cstring(part)); val.erase(std::remove_if(val.begin(), val.end(), isspace), val.end()); std::transform(val.begin(), val.end(), val.begin(), ::tolower); if (val.empty()) { syslog(LOG_WARNING, "Xapian: not a valid list-id: %s", buf_cstring(part)); return 0; } add_boolean_nterm(*dbw->document, prefix + val); return 0; } static int add_email_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); std::string lpart = Xapian::Unicode::tolower(buf_cstring(part)); struct address_itr itr; address_itr_init(&itr, lpart.c_str(), 0); const struct address *addr; while ((addr = address_itr_next(&itr))) { if (addr->invalid) { continue; } if (addr->name) { dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); dbw->term_generator->index_text(Xapian::Utf8Iterator(addr->name), 1, prefix + 'N'); dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); dbw->term_generator->index_text(Xapian::Utf8Iterator(addr->name), 1, prefix); } if (addr->mailbox) { // index mailbox as single value std::string val(addr->mailbox); // ignore whitespace (as seen in quoted mailboxes) val.erase(std::remove_if(val.begin(), val.end(), isspace), val.end()); add_boolean_nterm(*dbw->document, prefix + 'L' + val); // index individual terms dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); dbw->term_generator->index_text(Xapian::Utf8Iterator(val), 1, prefix); } if (addr->domain && strcmp(addr->domain, "unspecified-domain")) { // index reversed domain std::string val; strarray_t *sa = strarray_split(addr->domain, ".", 0); val.reserve(buf_len(part)); for (int i = strarray_size(sa) - 1; i >= 0; i--) { val.append(strarray_nth(sa, i)); if (i > 0) { val.append(1, '.'); } } strarray_free(sa); add_boolean_nterm(*dbw->document, prefix + "D" + val); // index individual terms dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); dbw->term_generator->index_text(Xapian::Utf8Iterator(addr->domain, strlen(addr->domain)), 1, prefix); } // index entire addr-spec char *a = address_get_all(addr, /*canon_domain*/1); if (a) { add_boolean_nterm(*dbw->document, prefix + 'A' + std::string(a)); free(a); } } address_itr_fini(&itr); return 0; } static std::pair parse_content_type(const char *str) { std::pair ret; struct buf buf = BUF_INITIALIZER; const char *sep = strchr(str, '/'); if (sep) { /* type */ buf_setmap(&buf, str, sep - str); buf_lcase(&buf); buf_trim(&buf); ret.first = std::string(buf_cstring(&buf)); /* subtype */ buf_setcstr(&buf, sep + 1); buf_lcase(&buf); buf_trim(&buf); ret.second = std::string(buf_cstring(&buf)); } else { /* type or subtype */ buf_setcstr(&buf, str); buf_lcase(&buf); buf_trim(&buf); ret.first = std::string(buf_cstring(&buf)); } buf_free(&buf); return ret; } static int add_type_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { std::string prefix(get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)); std::pair ct = parse_content_type(buf_cstring(part)); if (!ct.first.empty()) { add_boolean_nterm(*dbw->document, prefix + "T" + ct.first); } if (!ct.second.empty()) { add_boolean_nterm(*dbw->document, prefix + "S" + ct.second); } if (!ct.first.empty() && !ct.second.empty()) { add_boolean_nterm(*dbw->document, prefix + ct.first + '/' + ct.second); } return 0; } -int add_text_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) +static int add_text_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) { const char *prefix = get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum); int r = 0; // Index text. Xapian::TermGenerator::stem_strategy stem_strategy = get_stem_strategy(XAPIAN_DB_CURRENT_VERSION, partnum); dbw->term_generator->set_stemming_strategy(stem_strategy); if (stem_strategy != Xapian::TermGenerator::STEM_NONE) { if (config_getswitch(IMAPOPT_SEARCH_INDEX_LANGUAGE)){ // Index by language. #ifndef HAVE_CLD2 // XXX is this really an "IOERROR"? xsyslog(LOG_ERR, "IOERROR: language indexing requires CLD2 library", NULL); return IMAP_IOERROR; #else if (search_part_is_body(partnum)) { const std::string iso_lang = detect_language(part); if (!iso_lang.empty()) { if (iso_lang.compare("en")) { // Stem and index by non-default language. try { dbw->term_generator->set_stemmer(get_stemmer(iso_lang)); dbw->term_generator->set_stopper(get_stopper(iso_lang)); dbw->term_generator->index_text(Xapian::Utf8Iterator(part->s, part->len), 1, lang_prefix(iso_lang, prefix)); } catch (const Xapian::InvalidArgumentError &err) { syslog(LOG_DEBUG, "Xapian: no stemmer for language %s", iso_lang.c_str()); } } if (dbw->doctype == 'P') { // Keep track of stemmer language. std::string key = lang_doc_key(dbw->cyrusid); dbw->database->set_metadata(key, iso_lang); dbw->document->add_value(SLOT_DOCLANGS, iso_lang); // Update language counts for body parts. key = lang_count_key(iso_lang); const std::string val = dbw->database->get_metadata(key); dbw->database->set_metadata(key, val.empty() ? "1" : std::to_string(std::stoi(val) + 1)); } // Store detected languages in document. dbw->doclangs->insert(iso_lang.c_str()); add_boolean_nterm(*dbw->document, std::string("XI") + iso_lang); } } else if (partnum == SEARCH_PART_SUBJECT) { // Keep subject text to index by language later. dbw->subjects->push_back(buf_cstring(part)); } #endif /* HAVE_CLD2 */ } // Index with default stemmer. dbw->term_generator->set_stemmer(*dbw->default_stemmer); dbw->term_generator->set_stopper(dbw->default_stopper); } else { // Index with no stemming. dbw->term_generator->set_stemmer(Xapian::Stem()); dbw->term_generator->set_stopper(NULL); } dbw->term_generator->index_text(Xapian::Utf8Iterator(part->s, part->len), 1, prefix); return r; } -int xapian_dbw_doc_part(xapian_dbw_t *dbw, const struct buf *part, int partnum) +EXPORTED int xapian_dbw_doc_part(xapian_dbw_t *dbw, + const struct buf *part, + int partnum) { int r = 0; if (!get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum)) { syslog(LOG_ERR, "xapian_wrapper: no prefix for partnum %d", partnum); return IMAP_INTERNAL; } try { // Handle search parts. switch (partnum) { case SEARCH_PART_PRIORITY: r = add_priority_part(dbw, part, partnum); break; case SEARCH_PART_LISTID: r = add_listid_part(dbw, part, partnum); break; case SEARCH_PART_LANGUAGE: r = add_language_part(dbw, part, partnum); break; case SEARCH_PART_FROM: case SEARCH_PART_TO: case SEARCH_PART_CC: case SEARCH_PART_BCC: case SEARCH_PART_DELIVEREDTO: r = add_email_part(dbw, part, partnum); break; case SEARCH_PART_TYPE: r = add_type_part(dbw, part, partnum); break; default: r = add_text_part(dbw, part, partnum); } // Finalize index. dbw->term_generator->increase_termpos(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_dbw_end_doc(xapian_dbw_t *dbw, uint8_t indexlevel) +EXPORTED int xapian_dbw_end_doc(xapian_dbw_t *dbw, uint8_t indexlevel) { int r = 0; assert(indexlevel > 0); try { if (config_getswitch(IMAPOPT_SEARCH_INDEX_LANGUAGE)){ // Keep track of languages used in this message. if (dbw->doctype == 'G') { std::string val = format_doclangs(*dbw->doclangs); dbw->database->set_metadata(lang_doc_key(dbw->cyrusid), val); dbw->document->add_value(SLOT_DOCLANGS, val); } // Index subjects by detected document languages. for (std::set::iterator it = dbw->doclangs->begin(); it != dbw->doclangs->end(); ++it) { std::string iso_lang = *it; if (iso_lang.compare("en")) { try { const char *tp = get_term_prefix(XAPIAN_DB_CURRENT_VERSION, SEARCH_PART_SUBJECT); std::string prefix = lang_prefix(iso_lang, tp); dbw->term_generator->set_stemmer(get_stemmer(iso_lang)); dbw->term_generator->set_stopper(get_stopper(iso_lang)); for (const std::string& subject : *dbw->subjects) dbw->term_generator->index_text(Xapian::Utf8Iterator(subject), 1, prefix); } catch (const Xapian::InvalidArgumentError &err) { // ignore unknown stemmer } } } } dbw->document->add_value(SLOT_INDEXLEVEL, format_indexlevel(indexlevel)); dbw->document->add_value(SLOT_INDEXVERSION, std::to_string(XAPIAN_DB_CURRENT_VERSION)); dbw->database->add_document(*dbw->document); dbw->database->set_metadata("cyrusid." + std::string(dbw->cyrusid), format_indexlevel(indexlevel)); delete dbw->document; dbw->document = 0; dbw->doctype = 0; free(dbw->cyrusid); dbw->cyrusid = NULL; dbw->doclangs->clear(); dbw->subjects->clear(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -unsigned long xapian_dbw_total_length(xapian_dbw_t *dbw) +EXPORTED unsigned long xapian_dbw_total_length(xapian_dbw_t *dbw) { unsigned long res = 0; try { res = dbw->database->get_total_length(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); } return res; } -uint8_t xapian_dbw_is_indexed(xapian_dbw_t *dbw, const struct message_guid *guid, char doctype) +EXPORTED uint8_t xapian_dbw_is_indexed(xapian_dbw_t *dbw, + const struct message_guid *guid, + char doctype) { struct buf buf = BUF_INITIALIZER; make_cyrusid(&buf, guid, doctype); std::string key = "cyrusid." + std::string(buf_cstring(&buf)); buf_free(&buf); /* indexed in the current DB? */ uint8_t indexlevel = parse_indexlevel(dbw->database->get_metadata(key)); if (indexlevel == SEARCH_INDEXLEVEL_BEST || (indexlevel && doctype == XAPIAN_WRAP_DOCTYPE_PART)) { return indexlevel; } /* indexed in other DBs? */ for (int i = 0; i < dbw->otherdbs.count; i++) { Xapian::Database *database = (Xapian::Database *)ptrarray_nth(&dbw->otherdbs, i); uint8_t level = parse_indexlevel(database->get_metadata(key)); if (level == SEARCH_INDEXLEVEL_BEST || (level && doctype == XAPIAN_WRAP_DOCTYPE_PART)) { return level; } else indexlevel = better_indexlevel(indexlevel, level); } return indexlevel; } /* ====================================================================== */ struct xapian_db { std::string *paths; Xapian::Database *database; // all but version 4 databases std::vector *subdbs; // all database subdbs Xapian::Stem *default_stemmer; const Xapian::Stopper* default_stopper; std::set *stem_languages; Xapian::QueryParser *parser; std::set *db_versions; xapian_dbw_t *dbw; }; static int xapian_db_init(xapian_db_t *db) { int r = 0; try { db->parser = new Xapian::QueryParser; db->parser->set_default_op(Xapian::Query::OP_AND); db->default_stemmer = new Xapian::Stem(new CyrusSearchStemmer); db->default_stopper = get_stopper("en"); // Determine stemmer languages (in addition to English). db->stem_languages = new std::set; std::map lang_counts; size_t total_doccount = 0; for (const Xapian::Database& subdb : *db->subdbs) { read_language_counts(subdb, lang_counts); total_doccount += subdb.get_doccount(); } total_doccount /= 2; // Crude estimate. for (std::pair& it : lang_counts) { if (it.first.compare("en") && ((double) it.second / total_doccount) >= 0.05) { db->stem_languages->insert(it.first); } } } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_db_open(const char **paths, xapian_db_t **dbp) +EXPORTED int xapian_db_open(const char **paths, xapian_db_t **dbp) { xapian_db_t *db = (xapian_db_t *)xzmalloc(sizeof(xapian_db_t)); const char *thispath = "(unknown)"; int r = 0; try { db->paths = new std::string; while (paths && *paths) { thispath = *paths++; Xapian::Database subdb {thispath}; std::set db_versions = read_db_versions(subdb); if (db_versions.empty()) { syslog(LOG_ERR, "xapian_wrapper: invalid db version in %s", thispath); r = IMAP_INTERNAL; goto done; } if (!db->db_versions) db->db_versions = new std::set; db->db_versions->insert(db_versions.begin(), db_versions.end()); // Check for experimental v4 indexes, they were bogus. if (db_versions.find(4) != db_versions.end()) { xsyslog(LOG_WARNING, "deprecated v4 index detected, " "search may return wrong results", "db=<%s>", thispath); } // Add database. if (!db->database) db->database = new Xapian::Database; db->database->add_database(subdb); // Xapian database has no API to access subdbs. if (!db->subdbs) db->subdbs = new std::vector; db->subdbs->push_back(subdb); db->paths->append(thispath).push_back(' '); } thispath = "(unknown)"; if (!db->database) { r = IMAP_NOTFOUND; goto done; } r = xapian_db_init(db); if (r) goto done; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); r = IMAP_IOERROR; } done: if (r) xapian_db_close(db); else *dbp = db; return r; } -int xapian_db_opendbw(struct xapian_dbw *dbw, xapian_db_t **dbp) +EXPORTED int xapian_db_opendbw(struct xapian_dbw *dbw, xapian_db_t **dbp) { xapian_db_t *db = (xapian_db_t *)xzmalloc(sizeof(xapian_db_t)); db->dbw = dbw; db->database = dbw->database; db->db_versions = new std::set(); std::set dbw_versions = read_db_versions(*dbw->database); db->db_versions->insert(dbw_versions.begin(), dbw_versions.end()); db->subdbs = new std::vector; db->subdbs->push_back(*dbw->database); int r = xapian_db_init(db); if (r) { xapian_db_close(db); db = NULL; } *dbp = db; return r; } -void xapian_db_close(xapian_db_t *db) +EXPORTED void xapian_db_close(xapian_db_t *db) { if (!db) return; try { if (!db->dbw) delete db->database; delete db->parser; delete db->paths; delete db->db_versions; delete db->default_stemmer; delete db->stem_languages; delete db->subdbs; free(db); } catch (const Xapian::Error &err) { /* XXX - memory leak? */ xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); } } -int xapian_db_langstats(xapian_db_t *db, ptrarray_t* lstats, size_t *nolang) +EXPORTED int xapian_db_langstats(xapian_db_t *db, ptrarray_t* lstats, + size_t *nolang) { std::map lang_counts; size_t total_part = 0; size_t total_lang = 0; for (const Xapian::Database& subdb : *db->subdbs) { // count body parts for (Xapian::TermIterator it = subdb.metadata_keys_begin("cyrusid.*P*"); it != subdb.metadata_keys_end("cyrusid.*P*"); ++it) { total_part++; } // cummulate language counts read_language_counts(subdb, lang_counts); } for (const std::pair& counts : lang_counts) { struct search_langstat *stat = (struct search_langstat*) xzmalloc(sizeof(struct search_langstat)); stat->iso_lang = xstrdup(counts.first.c_str()); stat->count = counts.second; ptrarray_append(lstats, stat); total_lang += counts.second; } *nolang = total_part > total_lang ? total_part - total_lang : 0; return 0; } -void xapian_query_add_stemmer(xapian_db_t *db, const char *iso_lang) +EXPORTED void xapian_query_add_stemmer(xapian_db_t *db, const char *iso_lang) { if (strcmp(iso_lang, "en")) db->stem_languages->insert(iso_lang); } static Xapian::Query* query_new_textmatch(const xapian_db_t *db, const char *match, const char *prefix, Xapian::TermGenerator::stem_strategy tg_stem_strategy) { unsigned flags = Xapian::QueryParser::FLAG_PHRASE | Xapian::QueryParser::FLAG_WILDCARD; std::string lmatch = Xapian::Unicode::tolower(match); if (tg_stem_strategy != Xapian::TermGenerator::STEM_NONE) { // Query without any stemmer. db->parser->set_stemmer(Xapian::Stem()); db->parser->set_stopper(NULL); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_NONE); Xapian::Query q = db->parser->parse_query(lmatch, flags, prefix); // Query with default stemmer. But don't stem stopwords. if (!db->default_stopper || !(*db->default_stopper)(lmatch)) { db->parser->set_stemmer(*db->default_stemmer); db->parser->set_stopper(db->default_stopper); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_SOME); q |= db->parser->parse_query(lmatch, flags, prefix); } // Stem query for each language detected in the index. for (const std::string& iso_lang : *db->stem_languages) { try { const Xapian::Stopper *stopper = get_stopper(iso_lang); db->parser->set_stemmer(get_stemmer(iso_lang)); db->parser->set_stopper(stopper); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_SOME); if (!stopper || !(*stopper)(lmatch)) { q |= db->parser->parse_query(lmatch, flags, lang_prefix(iso_lang, prefix)); } } catch (const Xapian::InvalidArgumentError &err) { syslog(LOG_INFO, "Xapian: no stemmer for language %s", iso_lang.c_str()); } } return new Xapian::Query(q); } else { db->parser->set_stemmer(Xapian::Stem()); db->parser->set_stopper(NULL); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_NONE); return new Xapian::Query {db->parser->parse_query(lmatch, flags, prefix)}; } } static Xapian::Query *query_new_language(const xapian_db_t *db __attribute__((unused)), const char *prefix, const char *str) { std::string val = parse_langcode(str); if (val.empty()) { syslog(LOG_DEBUG, "Xapian: invalid language in query: %s", str); return new Xapian::Query(Xapian::Query::MatchNothing); } return new Xapian::Query(std::string(prefix) + val); } static Xapian::Query *query_new_priority(const xapian_db_t *db __attribute__((unused)), const char *prefix, const char *str) { std::string val = parse_priority(str); if (val.empty()) { syslog(LOG_DEBUG, "Xapian: invalid priority in query: %s", str); return new Xapian::Query(Xapian::Query::MatchNothing); } return new Xapian::Query(std::string(prefix) + val); } static Xapian::Query *query_new_listid(const xapian_db_t *db, const char *prefix, const char *str) { Xapian::Query *q = NULL; std::string val = parse_listid(str); if (!val.empty()) { q = new Xapian::Query(std::string(prefix) + val); } else { syslog(LOG_DEBUG, "Xapian: invalid listid in query: %s", str); q = new Xapian::Query(Xapian::Query::MatchNothing); } if (db->db_versions->lower_bound(11) != db->db_versions->begin()) { // query in legacy format db->parser->set_stemmer(Xapian::Stem()); db->parser->set_stopper(NULL); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_NONE); q = new Xapian::Query(Xapian::Query::OP_OR, *q, db->parser->parse_query(str, 0, prefix)); } return q; } static Xapian::Query *query_new_email(const xapian_db_t *db, const char *_prefix, const char *str) { std::string prefix(_prefix); unsigned qpflags = Xapian::QueryParser::FLAG_PHRASE | Xapian::QueryParser::FLAG_WILDCARD; db->parser->set_stemmer(Xapian::Stem()); db->parser->set_stopper(NULL); db->parser->set_stemming_strategy(Xapian::QueryParser::STEM_NONE); std::string mystr = Xapian::Unicode::tolower(str); str = mystr.c_str(); const char *atsign = strchr(str, '@'); if (!atsign) { // query free text return new Xapian::Query{db->parser->parse_query(str, qpflags, prefix)}; } Xapian::Query q = Xapian::Query::MatchNothing; // query name and mailbox (unless just searching for '@domain') if (atsign > str) { struct address *addr = NULL; parseaddr_list(str, &addr); if (addr && addr->name) { Xapian::Query qq = db->parser->parse_query(addr->name, qpflags, prefix + 'N'); if (q.get_type() != q.LEAF_MATCH_NOTHING) { q &= qq; } else q = qq; } if (addr && addr->mailbox) { // strip the domain from the mailbox std::string mail(addr->mailbox); mail.erase(std::remove_if(mail.begin(), mail.end(), isspace), mail.end()); int wildcard = mail[mail.size()-1] == '*'; if (wildcard) { mail.resize(mail.size()-1); } if (!mail.empty()) { std::string term(prefix + 'L' + mail); Xapian::Query qq = wildcard ? Xapian::Query(Xapian::Query::OP_WILDCARD, term) : Xapian::Query(term); if (q.get_type() != q.LEAF_MATCH_NOTHING) { q &= qq; } else q = qq; } } // ignore @domain - it's being handled below if (addr) parseaddr_free(addr); } // query domain if (atsign[1]) { std::string domain; const char *dstart = atsign + 1; bool wildcard = *dstart == '*'; if (wildcard) dstart++; const char *dend; for (dend = dstart; *dend; dend++) { char c = *dend; if (Uisalnum(c) || c == '-' || c == '[' || c == ']' || c == ':') { continue; } else if (c == '.' && (dend-1 == dstart || dend[-2] != '.')) { continue; } else { break; } } if (dend > dstart) { strarray_t *sa = strarray_nsplit(dstart, dend - dstart, ".", 0); for (int i = strarray_size(sa) - 1; i >= 0; i--) { domain.append(strarray_nth(sa, i)); if (i > 0) { domain.append(1, '.'); } } strarray_free(sa); if (*dstart == '.') { domain.append(1, '.'); } } if (!domain.empty()) { std::string term(prefix + 'D' + domain); Xapian::Query qq = wildcard ? Xapian::Query(Xapian::Query::OP_WILDCARD, term) : Xapian::Query(term); { // FIXME - temporarily also search for '@' prefix std::string term2(prefix + '@' + domain); Xapian::Query qq2 = wildcard ? Xapian::Query(Xapian::Query::OP_WILDCARD, term2) : Xapian::Query(term2); qq |= qq2; } if (q.get_type() != q.LEAF_MATCH_NOTHING) { q &= qq; } else q = qq; } } if (q.get_type() == q.LEAF_MATCH_ALL) { q = Xapian::Query::MatchNothing; } // query in legacy format as well! if (db->db_versions->lower_bound(12) != db->db_versions->begin()) { q |= db->parser->parse_query(str, qpflags, prefix); } // query localpart@domain (ONLY if no wildcards) if ((atsign > str) && atsign[1] && !strchr(str, '*')) { struct address *addr = NULL; parseaddr_list(str, &addr); if (addr) { char *a = address_get_all(addr, /*canon_domain*/1); if (a) { // query 'A' term for index >= 16 std::string term(prefix + 'A' + std::string(a)); Xapian::Query qq = Xapian::Query(Xapian::Query::OP_AND, Xapian::Query(Xapian::Query::OP_VALUE_GE, Xapian::valueno(SLOT_INDEXVERSION), std::string("16")), Xapian::Query(term)); if (q.get_type() != q.LEAF_MATCH_NOTHING) { // otherwise, query 'L' + 'D' terms (as per above) Xapian::Query qq2 = Xapian::Query(Xapian::Query::OP_AND, Xapian::Query(Xapian::Query::OP_VALUE_LE, Xapian::valueno(SLOT_INDEXVERSION), std::string("15")), q); qq |= qq2; } q = qq; } parseaddr_free(addr); free(a); } } return new Xapian::Query(q); } static void append_alnum(struct buf *buf, const char *ss) { const unsigned char *s = (const unsigned char *)ss; for ( ; *s ; ++s) { if (Uisalnum(*s)) buf_putc(buf, *s); } } static Xapian::Query *query_new_type(const xapian_db_t *db __attribute__((unused)), const char *_prefix, const char *str) { std::pair ct = parse_content_type(str); std::string prefix(_prefix); Xapian::Query q = Xapian::Query::MatchNothing; bool query_legacy = db->db_versions->lower_bound(13) != db->db_versions->begin(); struct buf buf = BUF_INITIALIZER; unsigned qpflags = Xapian::QueryParser::FLAG_PHRASE | Xapian::QueryParser::FLAG_WILDCARD; if (!ct.first.empty() && ct.second.empty()) { /* Match either type or subtype */ if (ct.first != "*") { q = Xapian::Query(Xapian::Query::OP_OR, Xapian::Query(prefix + 'T' + ct.first), Xapian::Query(prefix + 'S' + ct.first)); if (query_legacy) { append_alnum(&buf, ct.first.c_str()); q |= db->parser->parse_query(buf_cstring(&buf), qpflags, prefix); } } } else if (ct.first == "*" || ct.second == "*") { /* Wildcard query */ if (!ct.first.empty() && ct.first != "*") { /* Match type */ q = Xapian::Query(prefix + 'T' + ct.first); if (query_legacy) { append_alnum(&buf, ct.first.c_str()); q |= db->parser->parse_query(buf_cstring(&buf), qpflags, prefix); } } if (!ct.second.empty() && ct.second != "*") { /* Match subtype */ q = Xapian::Query(prefix + 'S' + ct.second); if (query_legacy) { append_alnum(&buf, ct.second.c_str()); q |= db->parser->parse_query(buf_cstring(&buf), qpflags, prefix); } } } else if (!ct.first.empty() && !ct.second.empty()) { /* Verbatim search */ q = Xapian::Query(prefix + ct.first + '/' + ct.second); if (query_legacy) { append_alnum(&buf, ct.first.c_str()); buf_putc(&buf, '_'); append_alnum(&buf, ct.second.c_str()); q |= db->parser->parse_query(buf_cstring(&buf), qpflags, prefix); } } buf_free(&buf); return new Xapian::Query(q); } -Xapian::Query* +EXPORTED Xapian::Query * xapian_query_new_match_internal(const xapian_db_t *db, int partnum, const char *str) { const char *prefix = get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum); try { // Handle special value search parts. if (partnum == SEARCH_PART_LANGUAGE) { return query_new_language(db, prefix, str); } else if (partnum == SEARCH_PART_PRIORITY) { return query_new_priority(db, prefix, str); } else if (partnum == SEARCH_PART_LISTID) { return query_new_listid(db, prefix, str); } else if (partnum == SEARCH_PART_FROM || partnum == SEARCH_PART_TO || partnum == SEARCH_PART_CC || partnum == SEARCH_PART_BCC || partnum == SEARCH_PART_DELIVEREDTO) { return query_new_email(db, prefix, str); } else if (partnum == SEARCH_PART_TYPE) { return query_new_type(db, prefix, str); } // Don't stem queries for Thaana codepage (0780) or higher. for (const unsigned char *p = (const unsigned char *)str; *p; p++) { if (*p > 221) //has highbit return new Xapian::Query {db->parser->parse_query( str, #ifdef USE_XAPIAN_CJK_WORDS Xapian::QueryParser::FLAG_CJK_WORDS, #else Xapian::QueryParser::FLAG_CJK_NGRAM, #endif prefix)}; } // Stemable codepage. Xapian::TermGenerator::stem_strategy stem_strategy = get_stem_strategy(XAPIAN_DB_CURRENT_VERSION, partnum); Xapian::Query *qq = query_new_textmatch(db, str, prefix, stem_strategy); if (qq->get_type() == Xapian::Query::LEAF_MATCH_NOTHING) { delete qq; qq = NULL; } return qq; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); return 0; } } -xapian_query_t * +EXPORTED xapian_query_t * xapian_query_new_match(const xapian_db_t *db, int partnum, const char *str) { if (db->subdbs->empty()) { // no database to query return NULL; } const char *prefix = get_term_prefix(XAPIAN_DB_CURRENT_VERSION, partnum); if (!prefix) { return NULL; } int min_version = *db->db_versions->begin(); if (min_version < XAPIAN_DB_MIN_SUPPORTED_VERSION) { xsyslog(LOG_WARNING, "deprecated database version, reindex required", "version=<%d> min_supported_version=<%d> paths=<%s>", min_version, XAPIAN_DB_MIN_SUPPORTED_VERSION, db->paths->c_str()); } Xapian::Query *q = xapian_query_new_match_internal(db, partnum, str); if (min_version < 15) { /* Older versions indexed header fields in Cyrus search form */ charset_t utf8 = charset_lookupname("utf-8"); char *mystr = charset_convert(str, utf8, charset_flags); if (mystr) { Xapian::Query *qq = xapian_query_new_match_internal(db, partnum, mystr); if (qq && q) { *q |= *qq; } else if (!q) q = qq; } free(mystr); charset_free(&utf8); } return (xapian_query_t*) q; } -xapian_query_t *xapian_query_new_compound(const xapian_db_t *db __attribute__((unused)), - int is_or, xapian_query_t **children, int n) +EXPORTED xapian_query_t * +xapian_query_new_compound(const xapian_db_t *db __attribute__((unused)), + int is_or, xapian_query_t **children, int n) { try { // I want to use std::initializer_list here // but that requires "experimental" gcc C++0x support :( // 'compound' owns a refcount on each child. We need to // drop the one we got when we allocated the children Xapian::Query* compound = new Xapian::Query; if (is_or) for (int i = 0 ; i < n ; i++) { *compound |= *(Xapian::Query*)children[i]; delete (Xapian::Query*)children[i]; } else for (int i = 0 ; i < n ; i++) { if (compound->empty()) *compound = *(Xapian::Query*)children[i]; else *compound &= *(Xapian::Query*)children[i]; delete (Xapian::Query*)children[i]; } return (xapian_query_t *)compound; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); return 0; } } /* Xapian does not have an OP_NOT. WTF? We fake it with * OP_AND_NOT where the left child is MatchAll */ -xapian_query_t *xapian_query_new_not(const xapian_db_t *db __attribute__((unused)), - xapian_query_t *child) +EXPORTED xapian_query_t * +xapian_query_new_not(const xapian_db_t *db __attribute__((unused)), + xapian_query_t *child) { if (!child) return (xapian_query_t*) new Xapian::Query(Xapian::Query::MatchAll); try { Xapian::Query *qq = new Xapian::Query( Xapian::Query::OP_AND_NOT, Xapian::Query::MatchAll, *(Xapian::Query *)child); // 'compound' owns a refcount on each child. We need to // drop the one we got when we allocated the children delete (Xapian::Query *)child; return (xapian_query_t *)qq; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); return 0; } } -xapian_query_t *xapian_query_new_matchall(const xapian_db_t *db __attribute__((unused))) +EXPORTED xapian_query_t * +xapian_query_new_matchall(const xapian_db_t *db __attribute__((unused))) { return (xapian_query_t *) new Xapian::Query(Xapian::Query::MatchAll); } -xapian_query_t *xapian_query_new_has_doctype(const xapian_db_t *db __attribute__((unused)), - char doctype, xapian_query_t *child) +EXPORTED xapian_query_t * +xapian_query_new_has_doctype(const xapian_db_t *db __attribute__((unused)), + char doctype, xapian_query_t *child) { try { Xapian::Query *qq = new Xapian::Query( Xapian::Query::OP_FILTER, child ? *(Xapian::Query *)child : Xapian::Query::MatchAll, std::string("XE") + doctype); // 'compound' owns a refcount on each child. We need to // drop the one we got when we allocated the children delete (Xapian::Query *)child; return (xapian_query_t *)qq; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); return 0; } } -void xapian_query_free(xapian_query_t *qq) +EXPORTED void xapian_query_free(xapian_query_t *qq) { try { delete (Xapian::Query *)qq; } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); } } -int xapian_query_run(const xapian_db_t *db, const xapian_query_t *qq, - int (*cb)(void *data, size_t n, void *rock), void *rock) +EXPORTED int xapian_query_run(const xapian_db_t *db, const xapian_query_t *qq, + int (*cb)(void *data, size_t n, void *rock), void *rock) { const Xapian::Query *query = (const Xapian::Query *)qq; void *data = NULL; size_t n = 0; if (!db->database) return 0; try { Xapian::Database *database = db->database; Xapian::Enquire enquire(*database); enquire.set_query(*query); enquire.set_sort_by_value(0, false); // sort by cyrusid ascending Xapian::MSet matches = enquire.get_mset(0, database->get_doccount()); size_t size = matches.size(); if (size) data = xzmalloc(size * 41); for (Xapian::MSetIterator i = matches.begin() ; i != matches.end() ; ++i) { const Xapian::Document& d = i.get_document(); const std::string& cyrusid = d.get_value(SLOT_CYRUSID); /* ignore documents with no cyrusid. Shouldn't happen, but has been seen */ if (cyrusid.length() != 43) { xsyslog(LOG_ERR, "IOERROR: skipping document with zero-length cyrusid", "documentid=<%u> paths=<%s>", d.get_docid(), db->paths->c_str()); continue; } const char *cstr = cyrusid.c_str(); if (cstr[0] != '*' || !isalpha(cstr[1]) || cstr[2] != '*') { xsyslog(LOG_ERR, "IOERROR: skipping document with invalid cyrusid", "cyrusid=<%s> documentid=<%u> paths=<%s>", cstr, d.get_docid(), db->paths->c_str()); continue; } if (n >= size) throw Xapian::DatabaseError("Too many records in MSet"); char *entry = (char *) data + (41*n); memcpy(entry, cstr+3, 40); entry[40] = '\0'; ++n; } } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> query=<%s>", err.get_description().c_str(), query ? query->get_description().c_str() : ""); free(data); return IMAP_IOERROR; } if (!n) { free(data); return 0; } int r = cb(data, n, rock); free(data); return r; } /* ====================================================================== */ struct xapian_snipgen { Xapian::Stem *default_stemmer; xapian_db_t *db; Xapian::Database *memdb; std::vector *loose_terms; std::vector *queries; char *cyrusid; char doctype; struct buf *buf; const char *hi_start; const char *hi_end; const char *omit; size_t max_len; }; -xapian_snipgen_t *xapian_snipgen_new(xapian_db_t *db, - const char *hi_start, - const char *hi_end, - const char *omit) +EXPORTED xapian_snipgen_t * +xapian_snipgen_new(xapian_db_t *db, + const char *hi_start, + const char *hi_end, + const char *omit) { xapian_snipgen_t *snipgen = (xapian_snipgen_t *)xzmalloc(sizeof(xapian_snipgen_t)); snipgen->default_stemmer = new Xapian::Stem(new CyrusSearchStemmer); snipgen->db = db; snipgen->memdb = new Xapian::WritableDatabase(std::string(), Xapian::DB_BACKEND_INMEMORY); snipgen->buf = buf_new(); snipgen->hi_start = hi_start; snipgen->hi_end = hi_end; snipgen->omit = omit; snipgen->max_len = (size_t) config_getint(IMAPOPT_SEARCH_SNIPPET_LENGTH); return snipgen; } -void xapian_snipgen_free(xapian_snipgen_t *snipgen) +EXPORTED void xapian_snipgen_free(xapian_snipgen_t *snipgen) { if (!snipgen) return; delete snipgen->default_stemmer; delete snipgen->loose_terms; delete snipgen->queries; delete snipgen->memdb; free(snipgen->cyrusid); buf_destroy(snipgen->buf); free(snipgen); } static Xapian::Query xapian_snipgen_build_query(xapian_snipgen_t *snipgen, Xapian::Stem& stemmer) { Xapian::TermGenerator term_generator; Xapian::Query q; if (snipgen->loose_terms) { /* Add loose query terms */ term_generator.set_stemmer(stemmer); #ifdef USE_XAPIAN_CJK_WORDS term_generator.set_flags(Xapian::TermGenerator::FLAG_CJK_WORDS, ~Xapian::TermGenerator::FLAG_CJK_WORDS); #else term_generator.set_flags(Xapian::TermGenerator::FLAG_CJK_NGRAM, ~Xapian::TermGenerator::FLAG_CJK_NGRAM); #endif for(size_t i = 0; i < snipgen->loose_terms->size(); ++i) { term_generator.index_text(Xapian::Utf8Iterator((*snipgen->loose_terms)[i])); } const Xapian::Document& doc = term_generator.get_document(); q = Xapian::Query(Xapian::Query::OP_OR, doc.termlist_begin(), doc.termlist_end()); } if (snipgen->queries) { /* Add phrase queries */ unsigned flags = Xapian::QueryParser::FLAG_PHRASE| Xapian::QueryParser::FLAG_WILDCARD| #ifdef USE_XAPIAN_CJK_WORDS Xapian::QueryParser::FLAG_CJK_WORDS; #else Xapian::QueryParser::FLAG_CJK_NGRAM; #endif Xapian::QueryParser queryparser; queryparser.set_stemmer(stemmer); for(size_t i = 0; i < snipgen->queries->size(); ++i) { q |= queryparser.parse_query((*snipgen->queries)[i], flags);; } } return q; } -int xapian_snipgen_add_match(xapian_snipgen_t *snipgen, const char *match) +EXPORTED int xapian_snipgen_add_match(xapian_snipgen_t *snipgen, + const char *match) { size_t len = strlen(match); bool is_query = len > 1 && ((match[0] == '"' && match[len-1] == '"') || (strchr(match, '*') != NULL)); if (is_query) { if (!snipgen->queries) { snipgen->queries = new std::vector; } snipgen->queries->push_back(match); } else { if (!snipgen->loose_terms) { snipgen->loose_terms = new std::vector; } snipgen->loose_terms->push_back(match); } return 0; } -int xapian_snipgen_begin_doc(xapian_snipgen_t *snipgen, - const struct message_guid *guid, char doctype) +EXPORTED int xapian_snipgen_begin_doc(xapian_snipgen_t *snipgen, + const struct message_guid *guid, + char doctype) { struct buf buf = BUF_INITIALIZER; make_cyrusid(&buf, guid, doctype); snipgen->cyrusid = buf_release(&buf); snipgen->doctype = doctype; buf_reset(snipgen->buf); return 0; } -int xapian_snipgen_make_snippet(xapian_snipgen_t *snipgen, - const struct buf *part, - Xapian::Stem* stemmer) +EXPORTED int xapian_snipgen_make_snippet(xapian_snipgen_t *snipgen, + const struct buf *part, + Xapian::Stem* stemmer) { int r = 0; try { std::string text {buf_base(part), buf_len(part)}; Xapian::Enquire enquire(*snipgen->memdb); Xapian::Query qq = xapian_snipgen_build_query(snipgen, *stemmer); if (qq.empty()) return 0; enquire.set_query(qq); unsigned flags = Xapian::MSet::SNIPPET_EXHAUSTIVE | Xapian::MSet::SNIPPET_EMPTY_WITHOUT_MATCH; #ifdef USE_XAPIAN_CJK_WORDS flags |= Xapian::MSet::SNIPPET_CJK_WORDS; #endif const std::string snippet = enquire.get_mset(0, 0).snippet(text, snipgen->max_len - buf_len(snipgen->buf), *stemmer, flags, snipgen->hi_start, snipgen->hi_end, snipgen->omit); if (!snippet.empty()) { if (buf_len(snipgen->buf)) { buf_appendoverlap(snipgen->buf, snipgen->omit); } buf_appendcstr(snipgen->buf, snippet.c_str()); } } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s>", err.get_description().c_str()); r = IMAP_IOERROR; } return r; } -int xapian_snipgen_doc_part(xapian_snipgen_t *snipgen, const struct buf *part, - int partnum __attribute__((unused))) +EXPORTED int xapian_snipgen_doc_part(xapian_snipgen_t *snipgen, + const struct buf *part, + int partnum __attribute__((unused))) { // Ignore empty queries. if (!snipgen->loose_terms && !snipgen->queries) return 0; // Don't exceed allowed snippet length. if (buf_len(snipgen->buf) >= snipgen->max_len) return 0; if (config_getswitch(IMAPOPT_SEARCH_INDEX_LANGUAGE) && snipgen->db->database && snipgen->cyrusid) { std::set doclangs; // Lookup stemmer language for this document part, if any. std::string key = lang_doc_key(snipgen->cyrusid); for (const Xapian::Database& subdb : *snipgen->db->subdbs) { std::string val = subdb.get_metadata(key); if (!val.empty()) parse_doclangs(val, doclangs); break; } // Generate snippets for each detected message language. // The first non-empty snippet wins. size_t prev_size = buf_len(snipgen->buf); for (std::set::iterator it = doclangs.begin(); it != doclangs.end(); ++it) { const std::string& iso_lang = *it; if (iso_lang.compare("en")) { try { Xapian::Stem stemmer = get_stemmer(iso_lang); int r = xapian_snipgen_make_snippet(snipgen, part, &stemmer); if (!r && prev_size != buf_len(snipgen->buf)) { return 0; } } catch (const Xapian::InvalidArgumentError &err) { // ignore unknown stemmer } } } } /* Using a custom stemmer did not generate a snippet. * This could be because the query matched using the * default stemmer, so try generating a snippet with * that stemmer instead.*/ return xapian_snipgen_make_snippet(snipgen, part, snipgen->default_stemmer); } -int xapian_snipgen_end_doc(xapian_snipgen_t *snipgen, struct buf *buf) +EXPORTED int xapian_snipgen_end_doc(xapian_snipgen_t *snipgen, struct buf *buf) { buf_reset(buf); buf_copy(buf, snipgen->buf); buf_cstring(buf); buf_reset(snipgen->buf); delete snipgen->loose_terms; snipgen->loose_terms = NULL; delete snipgen->queries; snipgen->queries = NULL; free(snipgen->cyrusid); snipgen->cyrusid = NULL; snipgen->doctype = 0; return 0; } /* cb returns true if document should be copied, false if not */ -int xapian_filter(const char *dest, const char **sources, - int (*cb)(const char *cyrusid, void *rock), - void *rock) +EXPORTED int xapian_filter(const char *dest, const char **sources, + int (*cb)(const char *cyrusid, void *rock), + void *rock) { int r = 0; const char *thispath = "(unknown path)"; try { /* create a destination database */ Xapian::WritableDatabase destdb {dest, Xapian::DB_CREATE|Xapian::DB_BACKEND_GLASS}; /* With multiple databases as above, the docids are interleaved, so it * might be worth trying to open each source and copy its documents to * destdb in turn for better locality of reference, and so better cache * use. -- Olly on the mailing list */ std::vector srcdbs; // Open databases and aggregate database-level metadata. while (*sources) { thispath = *sources++; const Xapian::Database srcdb {thispath}; srcdbs.push_back(srcdb); } // Copy all matching documents. std::set db_versions; for (size_t i = 0; i < srcdbs.size(); ++i) { const Xapian::Database& srcdb = srcdbs.at(i); bool need_md_versions = false; std::set md_versions = read_db_versions(srcdb); /* copy all matching documents to the new DB */ for (Xapian::ValueIterator it = srcdb.valuestream_begin(SLOT_CYRUSID); it != srcdb.valuestream_end(SLOT_CYRUSID); ++it) { const std::string& cyrusid = *it; const std::string idkey {"cyrusid." + cyrusid}; // check if caller wants this cyrusid if (!cb(cyrusid.c_str(), rock)) { continue; } // is it already indexed? if (!destdb.get_metadata(idkey).empty()) { continue; } // is there a subsequent db with a better index level? (only for G docs) uint8_t indexlevel = parse_indexlevel(srcdb.get_metadata(idkey)); if (cyrusid[1] == XAPIAN_WRAP_DOCTYPE_MSG) { int found_better = 0; for (size_t j = i + 1; !found_better && j < srcdbs.size(); ++j) { uint8_t level = parse_indexlevel(srcdbs[j].get_metadata(idkey)); found_better = better_indexlevel(indexlevel, level) != indexlevel; } if (found_better) { continue; } } // add document Xapian::Document srcdoc = srcdb.get_document(it.get_docid()); Xapian::docid docid = destdb.add_document(srcdoc); destdb.set_metadata(idkey, format_indexlevel(indexlevel)); // copy document language metadata const std::string& langkey = lang_doc_key(cyrusid.c_str()); if (destdb.get_metadata(langkey).empty()) { std::string val = srcdb.get_metadata(langkey); if (!val.empty() && isalpha(val[0])) { destdb.set_metadata(langkey, val); } } const std::string& langval = srcdoc.get_value(SLOT_DOCLANGS); if (!langval.empty() && !isalpha(langval[0])) { destdb.get_document(docid).remove_value(SLOT_DOCLANGS); } // add document index version const std::string& verval = srcdoc.get_value(SLOT_INDEXVERSION); if (!verval.empty()) { int version = std::atoi(verval.c_str()); if (version) db_versions.insert(version); } else need_md_versions = true; } if (need_md_versions) { /* At least one added document didn't have its index * version slot set in this subdb. Read legacy versions. */ std::set md_versions = read_db_versions(srcdb); db_versions.insert(md_versions.begin(), md_versions.lower_bound(14)); } } thispath = "(unknown path)"; // set the versions write_db_versions(destdb, db_versions); // recalculate language counts std::map lang_counts; r = calculate_language_counts(destdb, lang_counts); if (r) { xsyslog(LOG_ERR, "IOERROR: corrupt metadata", "filter=<%s>", dest); return r; } write_language_counts(destdb, lang_counts); /* commit all changes explicitly */ destdb.commit(); } catch (const Xapian::Error &err) { xsyslog(LOG_ERR, "IOERROR: caught exception", "exception=<%s> path=<%s>", err.get_description().c_str(), thispath); r = IMAP_IOERROR; } return r; } -const char *xapian_version_string() +EXPORTED const char *xapian_version_string() { return Xapian::version_string(); } struct xapian_doc { Xapian::TermGenerator *termgen; Xapian::Document *doc; }; -xapian_doc_t *xapian_doc_new(void) +EXPORTED xapian_doc_t *xapian_doc_new(void) { xapian_doc_t *doc = (xapian_doc_t *) xzmalloc(sizeof(struct xapian_doc)); doc->doc = new Xapian::Document; doc->termgen = new Xapian::TermGenerator; doc->termgen->set_document(*doc->doc); return doc; } -void xapian_doc_index_text(xapian_doc_t *doc, const char *text, size_t len) +EXPORTED void xapian_doc_index_text(xapian_doc_t *doc, + const char *text, size_t len) { doc->termgen->index_text(Xapian::Utf8Iterator(text, len)); } -size_t xapian_doc_termcount(xapian_doc_t *doc) +EXPORTED size_t xapian_doc_termcount(xapian_doc_t *doc) { return doc->doc->termlist_count(); } -int xapian_doc_foreach_term(xapian_doc_t *doc, int(*cb)(const char*, void*), void *rock) +EXPORTED int xapian_doc_foreach_term(xapian_doc_t *doc, + int(*cb)(const char*, void*), + void *rock) { for (Xapian::TermIterator ti = doc->doc->termlist_begin(); ti != doc->doc->termlist_end(); ++ti) { int r = cb((*ti).c_str(), rock); if (r) return r; } return 0; } -void xapian_doc_reset(xapian_doc_t *doc) +EXPORTED void xapian_doc_reset(xapian_doc_t *doc) { doc->doc->clear_values(); } -extern void xapian_doc_close(xapian_doc_t *termgen); -void xapian_doc_close(xapian_doc_t *doc) +EXPORTED void xapian_doc_close(xapian_doc_t *doc) { delete doc->termgen; delete doc->doc; free(doc); }