diff --git a/.env b/.env deleted file mode 120000 index 0ec1a3a4..00000000 --- a/.env +++ /dev/null @@ -1 +0,0 @@ -src/.env \ No newline at end of file diff --git a/README.md b/README.md index b9cf9d90..3c7fc671 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,76 @@ -## Quickstart Instructions to try it out +# Quickstart Instructions to try it out -* Make sure you have docker and docker-compose available. +* Make sure you have podman available. * Change to the base directory of this repository. +* Run './kolabctl configure' to generate the .env file. * Run './kolabctl deploy' to start the deployment. * Add an /etc/hosts entry "127.0.0.1 kolab.local" -* navigate to https://kolab.local -* login as "admin@kolab.local" with password "simple123" (or whatever you have set), and create your users. +* Navigate to https://kolab.local +* Login as "admin@kolab.local" with the password set during the deploy step, and create your users. + +# Podman deployment + +The podman deployment can be managed using the kolabctl script, which is a wrapper around various podman commands. + +The kolabctl script can be used to: +* Build all images: './kolabctl build' +* Create a pod running all containers: './kolabctl deploy' +* Execute various maintenance tasks: +** Start/stop the pod +** Backup/restore the volumes +** Validate the deployment via './kolabctl selfcheck' + +## Host Requirements +* podman +* openssl + +## Update + +To update the containers without removing the data: + +* git pull +* Run "./kolabctl update" + +## Backup / Restore + +The "./kolabctl backup" script will stop all containers, snapshot the volumes to the backup/ directory, and restart the containers. + +"./kolabctl restore" will stop all containers, restore the volumes from tarballs in the backup/ directory, and restart the containers. + # Customization -To customize the installation, copy config.prod and adjust to your liking. You can then deploy the configuration using 'env CONFIG=config.custom ./kolabctl deploy'. +To customize the installation, copy config.prod and adjust to your liking. + +To generate a new .env based on that configuration 'env CONFIG=config.custom ./kolabctl configure --force'. +You can then deploy the configuration using 'env CONFIG=config.custom ./kolabctl deploy --reset'. -Please note that './kolabctl deploy' will remove any existing data. +Please note that './kolabctl deploy --reset' will remove any existing data. ## Alternative configurations Everything but config.prod is for development or demo purposes: * config.prod: A docker environment with just an admin account prepared. A starting point for a production environment. * config.demo: A docker environment with demo data included. * config.docker-dev: A development environment with everything running in docker. Includes a cyrus-murder. Don't use unless you know what you're doing. * config.host-dev: Run only dependencies in docker with ports exposed, and expect kolab4 to be run locally. Don't use unless you know what you're doing. * config.legacy: A docker environment that includes ldap and other legacy components. Don't use unless you know what you're doing. - # Use the ansible setup The ansible/ directory contains setup scripts to setup a fresh Fedora system with a kolab deployment. Modify the Makefile with the required variables and then execute `make setup`. This will configure the remote system and execute the above steps. -### Update - -To update the containers without removing the data: - -* git pull -* Run "./kolabctl update" +# Use the kubernetes deployment -### Backup / Restore +A helm chart and prebuilt images for a kubernetes deployment are available here: https://mirror.apheleia-it.ch/pub/kolab-kubernetes-latest.tar.gz -The "./kolabctl backup" script will stop all containers, snapshot the volumes to the backup/ directory, and restart the containers. - -"./kolabctl restore" will stop all containers, restore the volumes from tarballs in the backup/ directory, and restart the containers. +The tarball contains instructions as well as a kolabctl script to setup a k3s based deployment. +Please note: +* The prebuilt images are based on the same sources built by this repository. +* The images and helm chart come without support or guarantees for backwards comaptiblity. +* If you want to run this in production you need to either know what you're doing or get support. -### Requirements -* docker -* openssl diff --git a/ansible/grub.yml b/ansible/grub.yml deleted file mode 100644 index fcc59faf..00000000 --- a/ansible/grub.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - -- name: Check if cgroupv1 is configured - shell: grep -c 'systemd.unified_cgroup_hierarchy=0' /proc/cmdline - register: cgroup_status - ignore_errors: true - -- name: Install grubby - package: name=grubby state=installed - when: cgroup_status.stdout == "0" - -- name: Disable cgroupv2 - shell: grubby --update-kernel=ALL --args=\"systemd.unified_cgroup_hierarchy=0\" - when: cgroup_status.stdout == "0" - -- name: reboot - shell: sleep 2 && shutdown -r now "Ansible updates triggered" - async: 1 - poll: 0 - ignore_errors: true - when: cgroup_status.stdout == "0" - -- name: waiting for server to come back - local_action: wait_for host={{ansible_fqdn}} state=started timeout=600 delay=15 - when: cgroup_status.stdout == "0" - diff --git a/ansible/packages.yml b/ansible/packages.yml index 9737c558..9b9a431f 100644 --- a/ansible/packages.yml +++ b/ansible/packages.yml @@ -1,16 +1,11 @@ --- -- name: Add repository for docker-ce - ansible.builtin.command: dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo - args: - creates: /etc/yum.repos.d/docker-ce.repo - name: Install list of required packages ansible.builtin.dnf: allowerasing: true name: - git - tig - tmux - - docker - - docker-compose-plugin + - podman - certbot state: latest diff --git a/ansible/setup.yml b/ansible/setup.yml index 47d54bfa..3c63293c 100755 --- a/ansible/setup.yml +++ b/ansible/setup.yml @@ -1,137 +1,125 @@ #!/usr/bin/ansible-playbook - name: Setup kolab deployment on fedora server hosts: "{{ hostname }}" remote_user: root tasks: - - import_tasks: grub.yml - - name: Set hostname ansible.builtin.hostname: name: "{{ hostname }}" - import_tasks: packages.yml - - name: Put SELinux in permissive mode for docker - selinux: - policy: targeted - state: permissive - + # - name: Put SELinux in permissive mode for docker + # selinux: + # policy: targeted + # state: permissive + # - name: Setup user kolab ansible.builtin.user: name: kolab shell: /bin/bash - groups: wheel, audio, docker + groups: wheel, audio append: yes - name: sudo without password ansible.builtin.lineinfile: path: /etc/sudoers state: present regexp: '^%wheel\s' line: '%wheel ALL=(ALL) NOPASSWD: ALL' - - name: Start service docker, if not started - ansible.builtin.service: - name: docker - state: started - - import_tasks: certbot.yml - name: Delete kolab content & directory ansible.builtin.file: state: absent path: /home/kolab/kolab/ - name: get kolab git repo become: true become_user: kolab git: repo: https://git.kolab.org/source/kolab.git dest: /home/kolab/kolab version: "{{ git_branch }}" force: yes - name: Run bin/configure become: true become_user: kolab ansible.builtin.command: bin/configure.sh {{ config }} args: chdir: /home/kolab/kolab environment: HOST: "{{ hostname }}" OPENEXCHANGERATES_API_KEY: "{{ openexchangerates_api_key }}" FIREBASE_API_KEY: "{{ firebase_api_key }}" PUBLIC_IP: "{{ public_ip }}" ADMIN_PASSWORD: "{{ admin_password }}" KOLAB_GIT_REF: "{{ git_branch }}" - name: Permit receiving mail firewalld: port: 25/tcp permanent: yes state: enabled zone: FedoraServer - name: Permit http traffic firewalld: port: 80/tcp permanent: yes state: enabled zone: FedoraServer - name: Permit https traffic firewalld: port: 443/tcp permanent: yes state: enabled zone: FedoraServer - name: Permit TCP trafic for coturn firewalld: port: 3478/tcp permanent: yes state: enabled zone: FedoraServer - name: Permit TCP trafic for coturn firewalld: port: 5349/tcp permanent: yes state: enabled zone: FedoraServer - name: Permit UDP trafic for coturn firewalld: port: 3478/udp permanent: yes state: enabled zone: FedoraServer - name: Permit UDP trafic for coturn firewalld: port: 5349/udp permanent: yes state: enabled zone: FedoraServer - - name: Always restart docker before deploy (because of potential network issues otherwise) - ansible.builtin.service: - name: docker - state: restarted - - name: Deploy block: - - name: Run bin/deploy + - name: Run kolabctl deploy become: true become_user: kolab - ansible.builtin.command: bin/deploy.sh + ansible.builtin.command: kolabctl deploy args: chdir: /home/kolab/kolab environment: ADMIN_PASSWORD: "{{ admin_password }}" register: result always: - name: Print output from previous task with newlines ansible.builtin.debug: msg="{{result.stdout_lines}}" diff --git a/bin/backup.sh b/bin/backup.sh deleted file mode 100755 index 624547ab..00000000 --- a/bin/backup.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -mkdir -p backup - -backup_path="$(pwd)/backup/" - -function backup_volume { - volume_name=$1 - backup_destination=$2 - - echo "Backing up $volume_name to $backup_destination" - docker run --rm -v $volume_name:/data -v $backup_destination:/backup quay.io/centos/centos:stream8 tar -zcvf /backup/$volume_name.tar /data -} - -echo "Stopping containers" -docker compose stop - -echo "Backing up volumes" -volumes=($(docker volume ls -f name=kolab | awk '{if (NR > 1) print $2}')) -for v in "${volumes[@]}" -do - backup_volume $v $backup_path -done - -echo "Restarting containers" -docker compose start diff --git a/bin/configure.sh b/bin/configure.sh index a466bee6..8e3538b7 100755 --- a/bin/configure.sh +++ b/bin/configure.sh @@ -1,111 +1,116 @@ #!/bin/bash +# This script copies a deployment config over the root directory (installing it), +# and then generates necessary secrets if they are not yet existing. +# To avoid re-generating secrets store them in a config.secrets file, which will be appended to the .env file before checking +# for existing secrets. +# This script is no longer used if containers are used as the webapp container will overlay the config itself. # Uninstall the old config if [ -d config ]; then echo "Uninstalling the old config." find -L config/ -type f | while read file; do file=$(echo $file | sed -e 's|^config||g') file="./$file" rm -v $file done fi if [ "$1" == "" ]; then echo "Failed to find the configuration folder, please pass one as argument (e.g. config.demo)." exit 1 fi if [ ! -d $1 ]; then echo "Failed to find the configuration folder, please pass one as argument (e.g. config.demo)." exit 1 fi echo "Installing $1." # Link new config rm config ln -s $1 config # Install new config find -L config/ -type f | while read file; do dir=$(dirname $file | sed -e 's|^config||g') dir="./$dir" if [ ! -d $dir ]; then mkdir -p $dir fi cp -v $file $dir/ done if [ -f config.secrets ]; then # Add local secrets echo "" >> src/.env cat config.secrets >> src/.env fi # Generate random secrets if ! grep -q "COTURN_STATIC_SECRET" .env; then COTURN_STATIC_SECRET=$(openssl rand -hex 32); echo "COTURN_STATIC_SECRET=${COTURN_STATIC_SECRET}" >> src/.env fi if ! grep -q "MEET_WEBHOOK_TOKEN" .env; then MEET_WEBHOOK_TOKEN=$(openssl rand -hex 32); echo "MEET_WEBHOOK_TOKEN=${MEET_WEBHOOK_TOKEN}" >> src/.env fi if ! grep -q "MEET_SERVER_TOKEN" .env; then MEET_SERVER_TOKEN=$(openssl rand -hex 32); echo "MEET_SERVER_TOKEN=${MEET_SERVER_TOKEN}" >> src/.env fi if ! grep -q "APP_KEY=base64:" .env; then APP_KEY=$(openssl rand -base64 32); echo "APP_KEY=base64:${APP_KEY}" >> src/.env fi if ! grep -q "PASSPORT_PROXY_OAUTH_CLIENT_ID=" .env; then PASSPORT_PROXY_OAUTH_CLIENT_ID=$(uuidgen); echo "PASSPORT_PROXY_OAUTH_CLIENT_ID=${PASSPORT_PROXY_OAUTH_CLIENT_ID}" >> src/.env fi if ! grep -q "PASSPORT_PROXY_OAUTH_CLIENT_SECRET=" .env; then PASSPORT_PROXY_OAUTH_CLIENT_SECRET=$(openssl rand -base64 32); echo "PASSPORT_PROXY_OAUTH_CLIENT_SECRET=${PASSPORT_PROXY_OAUTH_CLIENT_SECRET}" >> src/.env fi if ! grep -q "PASSPORT_PUBLIC_KEY=|PASSPORT_PRIVATE_KEY=" .env; then PASSPORT_PRIVATE_KEY=$(openssl genrsa 4096); echo "PASSPORT_PRIVATE_KEY=\"${PASSPORT_PRIVATE_KEY}\"" >> src/.env PASSPORT_PUBLIC_KEY=$(echo "$PASSPORT_PRIVATE_KEY" | openssl rsa -pubout 2>/dev/null) echo "PASSPORT_PUBLIC_KEY=\"${PASSPORT_PUBLIC_KEY}\"" >> src/.env fi if ! grep -q "DES_KEY=" .env; then DES_KEY=$(openssl rand -base64 24); echo "DES_KEY=${DES_KEY}" >> src/.env fi bin/update-git-refs.sh # Customize configuration sed -i \ -e "s/{{ host }}/${HOST:-kolab.local}/g" \ -e "s/{{ openexchangerates_api_key }}/${OPENEXCHANGERATES_API_KEY}/g" \ -e "s/{{ firebase_api_key }}/${FIREBASE_API_KEY}/g" \ -e "s/{{ public_ip }}/${PUBLIC_IP:-172.18.0.1}/g" \ -e "s/{{ admin_password }}/${ADMIN_PASSWORD}/g" \ src/.env if [ -f /etc/letsencrypt/live/${HOST}/cert.pem ]; then echo "Using the available letsencrypt certificate for ${HOST}" cat >> .env << EOF KOLAB_SSL_CERTIFICATE=/etc/letsencrypt/live/${HOST}/cert.pem KOLAB_SSL_CERTIFICATE_FULLCHAIN=/etc/letsencrypt/live/${HOST}/fullchain.pem KOLAB_SSL_CERTIFICATE_KEY=/etc/letsencrypt/live/${HOST}/privkey.pem PROXY_SSL_CERTIFICATE=/etc/letsencrypt/live/${HOST}/fullchain.pem PROXY_SSL_CERTIFICATE_KEY=/etc/letsencrypt/live/${HOST}/privkey.pem EOF fi diff --git a/bin/deploy.sh b/bin/deploy.sh deleted file mode 100755 index 5dcf3b2e..00000000 --- a/bin/deploy.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -bin/quickstart.sh --nodev -if [[ -n $ADMIN_PASSWORD ]]; then - DOMAIN=$(grep APP_DOMAIN .env | tail -n1 | sed "s/APP_DOMAIN=//") - docker exec kolab-webapp ./artisan user:password "admin@$DOMAIN" "$ADMIN_PASSWORD" -fi diff --git a/bin/podman_shared b/bin/podman_shared new file mode 100644 index 00000000..9dd43171 --- /dev/null +++ b/bin/podman_shared @@ -0,0 +1,267 @@ +#!/bin/bash + +PODMAN=podman + + +podman__build() { + path=$1 + shift + name=$1 + shift + if [[ "$CACHE_REGISTRY" != "" ]]; then + CACHE_ARGS="--layers --cache-from=$CACHE_REGISTRY/$name --cache-to=$CACHE_REGISTRY/$name --cache-ttl=24h" + fi + podman build $@ $CACHE_ARGS $path -t $name +} + +podman__build_base() { + podman__build docker/base/ apheleia/almalinux9 -f almalinux9 + podman__build docker/swoole apheleia/swoole +} + +podman__build_webapp() { + podman__build docker/webapp kolab-webapp --ulimit nofile=65535:65535 \ + ${KOLAB_GIT_REMOTE:+"--build-arg=GIT_REMOTE=$KOLAB_GIT_REMOTE"} \ + ${KOLAB_GIT_REF:+"--build-arg=GIT_REF=$KOLAB_GIT_REF"} +} + +podman__build_meet() { + podman__build docker/meet kolab-meet --ulimit nofile=65535:65535 \ + ${KOLAB_GIT_REMOTE:+"--build-arg=GIT_REMOTE=$KOLAB_GIT_REMOTE"} \ + ${KOLAB_GIT_REF:+"--build-arg=GIT_REF=$KOLAB_GIT_REF"} +} + +podman__build_roundcube() { + podman__build docker/roundcube roundcube --ulimit nofile=65535:65535 \ + ${GIT_REMOTE_ROUNDCUBEMAIL:+"--build-arg=GIT_REMOTE_ROUNDCUBEMAIL=$GIT_REMOTE_ROUNDCUBEMAIL"} \ + ${GIT_REF_ROUNDCUBEMAIL:+"--build-arg=GIT_REF_ROUNDCUBEMAIL=$GIT_REF_ROUNDCUBEMAIL"} \ + ${GIT_REMOTE_ROUNDCUBEMAIL_PLUGINS:+"--build-arg=GIT_REMOTE_ROUNDCUBEMAIL_PLUGINS=$GIT_REMOTE_ROUNDCUBEMAIL_PLUGINS"} \ + ${GIT_REF_ROUNDCUBEMAIL_PLUGINS:+"--build-arg=GIT_REF_ROUNDCUBEMAIL_PLUGINS=$GIT_REF_ROUNDCUBEMAIL_PLUGINS"} \ + ${GIT_REMOTE_CHWALA:+"--build-arg=GIT_REMOTE_CHWALA=$GIT_REMOTE_CHWALA"} \ + ${GIT_REF_CHWALA:+"--build-arg=GIT_REF_CHWALA=$GIT_REF_CHWALA"} \ + ${GIT_REMOTE_SYNCROTON:+"--build-arg=GIT_REMOTE_SYNCROTON=$GIT_REMOTE_SYNCROTON"} \ + ${GIT_REF_SYNCROTON:+"--build-arg=GIT_REF_SYNCROTON=$GIT_REF_SYNCROTON"} \ + ${GIT_REMOTE_AUTOCONF:+"--build-arg=GIT_REMOTE_AUTOCONF=$GIT_REMOTE_AUTOCONF"} \ + ${GIT_REF_AUTOCONF:+"--build-arg=GIT_REF_AUTOCONF=$GIT_REF_AUTOCONF"} \ + ${GIT_REMOTE_IRONY:+"--build-arg=GIT_REMOTE_IRONY=$GIT_REMOTE_IRONY"} \ + ${GIT_REF_IRONY:+"--build-arg=GIT_REF_IRONY=$GIT_REF_IRONY"} \ + ${GIT_REMOTE_FREEBUSY:+"--build-arg=GIT_REMOTE_FREEBUSY=$GIT_REMOTE_FREEBUSY"} \ + ${GIT_REF_FREEBUSY:+"--build-arg=GIT_REF_FREEBUSY=$GIT_REF_FREEBUSY"} +} + +podman__build_postfix() { + podman__build docker/postfix kolab-postfix +} + +podman__build_all() { + podman__build_base + podman__build_webapp + podman__build_meet + podman build docker/postfix -t kolab-postfix + podman build docker/imap -t kolab-imap + ${IMAP_GIT_REMOTE:+"--build-arg=GIT_REMOTE=$IMAP_GIT_REMOTE"} \ + ${IMAP_GIT_REF:+"--build-arg=GIT_REF=$IMAP_GIT_REF"} + podman build docker/amavis -t kolab-amavis + podman build docker/collabora -t kolab-collabora --build-arg=REPOSITORY="https://www.collaboraoffice.com/repos/CollaboraOnline/23.05-CODE/CODE-rpm/" + podman build docker/mariadb -t mariadb + podman build docker/redis -t redis + podman build docker/proxy -t kolab-proxy + podman build docker/coturn -t kolab-coturn + podman build docker/utils -t kolab-utils + podman build docker/fluentbit -t fluentbit + podman__build_roundcube +} + +kolab__validate() { + POD=$1 + $PODMAN exec $POD-imap testsaslauthd -u cyrus-admin -p simple123 + $PODMAN exec $POD-imap testsaslauthd -u "john@kolab.org" -p simple123 + # Ensure the inbox is created + FOUND=false + for i in {1..60}; do + if $PODMAN exec $POD-imap bash -c 'echo "lm" | cyradm --auth PLAIN -u cyrus-admin -w simple123 --port 11143 localhost | grep "user/john@kolab.org"'; then + echo "Found mailbox"; + FOUND=true + break + else + echo "Waiting for mailbox"; + sleep 1; + fi + done + if ! $FOUND; then + echo "Failed to find the inbox for john@kolab.org" + exit 1 + fi +} + +podman__healthcheck() { + for CONTAINER in $@; do + echo "Waiting for ${CONTAINER} become healthy " + while [ $(podman healthcheck run ${CONTAINER}) ]; do + echo -n "."; sleep 5; + done + done +} + +podman__run_proxy() { + $PODMAN run -dt --pod $POD --name $POD-proxy --replace \ + -v $CERTS_PATH:/etc/certs:ro \ + -e APP_WEBSITE_DOMAIN \ + -e SSL_CERTIFICATE="/etc/certs/imap.hosted.com.cert" \ + -e SSL_CERTIFICATE_KEY="/etc/certs/imap.hosted.com.key" \ + -e WEBAPP_BACKEND="http://localhost:8000" \ + -e MEET_BACKEND="https://localhost:12443" \ + -e ROUNDCUBE_BACKEND="http://localhost:8080" \ + -e DAV_BACKEND="http://localhost:11080/dav" \ + -e COLLABORA_BACKEND="http://localhost:9980" \ + -e MATRIX_BACKEND="http://localhost:8008" \ + -e ELEMENT_BACKEND="http://localhost:8880" \ + kolab-proxy:latest +} + +podman__run_roundcube() { + $PODMAN run -dt --pod $POD --name $POD-roundcube --replace \ + -v ./ext:/src.orig:ro \ + -e APP_DOMAIN \ + -e DES_KEY \ + -e DB_HOST \ + -e DB_RC_DATABASE="roundcube" \ + -e DB_RC_USERNAME="roundcube" \ + -e DB_RC_PASSWORD="${DB_PASSWORD:?"missing env variable"}" \ + -e IMAP_HOST \ + -e IMAP_PORT \ + -e IMAP_ADMIN_LOGIN \ + -e IMAP_ADMIN_PASSWORD \ + -e MAIL_HOST \ + -e MAIL_PORT \ + -e IMAP_DEBUG \ + -e KOLAB_FILES_SERVER_URL=http://localhost:8080/chwala \ + -e FILEAPI_WOPI_OFFICE=http://localhost:9980 \ + -e FILEAPI_KOLABFILES_BASEURI=http://localhost:8000/api \ + -e FILE_API_SERVER_URL=http://localhost:8080/chwala/api/ \ + -e KOLAB_ADDRESSBOOK_CARDDAV_SERVER=http://localhost:11080/dav \ + -e CALENDAR_CALDAV_SERVER=http://localhost:11080/dav \ + -e TASKLIST_CALDAV_SERVER=http://localhost:11080/dav \ + roundcube:latest +} + +podman__run_mariadb() { + $PODMAN run -dt --pod $POD --name $POD-mariadb --replace \ + $MARIADB_STORAGE \ + -e MYSQL_ROOT_PASSWORD=${DB_ROOT_PASSWORD:?"missing env variable"} \ + -e TZ="+02:00" \ + -e DB_HKCCP_DATABASE="kolabdev" \ + -e DB_HKCCP_USERNAME="kolabdev" \ + -e DB_HKCCP_PASSWORD=${DB_PASSWORD:?"missing env variable"} \ + -e DB_KOLAB_DATABASE="kolab" \ + -e DB_KOLAB_USERNAME="kolab" \ + -e DB_KOLAB_PASSWORD=${DB_PASSWORD:?"missing env variable"} \ + -e DB_RC_DATABASE="roundcube" \ + -e DB_RC_USERNAME="roundcube" \ + -e DB_RC_PASSWORD=${DB_PASSWORD:?"missing env variable"} \ + --health-cmd "mysqladmin -u root ping && test -e /tmp/initialized" \ + mariadb:latest +} + +podman__run_redis() { + $PODMAN run -dt --pod $POD --name $POD-redis --replace \ + $REDIS_STORAGE \ + --health-cmd "redis-cli ping || exit 1" \ + redis:latest +} + +podman__run_minio() { + $PODMAN run -dt --pod $POD --name $POD-minio --replace \ + $MINIO_STORAGE \ + -e MINIO_ROOT_USER=${MINIO_USER:?"missing env variable"} \ + -e MINIO_ROOT_PASSWORD=${MINIO_PASSWORD:?"missing env variable"} \ + --health-cmd "mc ready local || exit 1" \ + --entrypoint sh \ + quay.io/minio/minio:latest -c 'mkdir -p /data/kolab && minio server /data --console-address ":9001"' +} + +podman__run_webapp() { + # We run with a fixed config.demo overlay and override the environment with ci/env + $PODMAN run -dt --pod $POD --name $POD-webapp --replace \ + --env-file=$1 \ + -v ./src:/src/kolabsrc.orig:ro \ + -v ./$2/src:/src/overlay:ro \ + -e NOENVFILE=true \ + -e APP_SERVICES_ALLOWED_DOMAINS="webapp,localhost,services.$HOST" \ + -e KOLAB_ROLE=combined \ + -e PASSPORT_PRIVATE_KEY="$PASSPORT_PRIVATE_KEY" \ + -e PASSPORT_PUBLIC_KEY="$PASSPORT_PUBLIC_KEY" \ + -e MINIO_ENDPOINT="http://localhost:9000" \ + --health-cmd "./artisan octane:status || exit 1" \ + kolab-webapp:latest +} + +podman__run_imap() { + $PODMAN run -dt --pod $POD --name $POD-imap --replace \ + $IMAP_SPOOL_STORAGE \ + $IMAP_LIB_STORAGE \ + -e APP_SERVICES_DOMAIN="localhost" \ + -e SERVICES_PORT=8000 \ + -e IMAP_ADMIN_LOGIN \ + -e IMAP_ADMIN_PASSWORD \ + --health-cmd "test -e /run/saslauthd/mux && kill -0 \$(cat /var/run/master.pid)" \ + kolab-imap:latest +} + +podman__run_postfix() { + $PODMAN run -dt --pod $POD --name $POD-postfix --replace \ + --privileged \ + $POSTFIX_SPOOL_STORAGE \ + $POSTFIX_LIB_STORAGE \ + -v $CERTS_PATH:/etc/certs:ro \ + -e SSL_CERTIFICATE="$KOLAB_SSL_CERTIFICATE" \ + -e SSL_CERTIFICATE_FULLCHAIN="$KOLAB_SSL_CERTIFICATE_FULLCHAIN" \ + -e SSL_CERTIFICATE_KEY="$KOLAB_SSL_CERTIFICATE_KEY" \ + -e APP_DOMAIN \ + -e APP_SERVICES_DOMAIN="localhost" \ + -e SERVICES_PORT=8000 \ + -e AMAVIS_HOST=127.0.0.1 \ + -e DB_HOST=127.0.0.1 \ + -e DB_USERNAME \ + -e DB_PASSWORD \ + -e DB_DATABASE \ + -e LMTP_DESTINATION="localhost:11024" \ + --health-cmd "test -e /run/saslauthd/mux && kill -0 \$(cat /var/spool/postfix/pid/master.pid)" \ + kolab-postfix:latest +} + +podman__run_amavis() { + $PODMAN run -dt --pod $POD --name $POD-amavis --replace \ + -e DB_HOST=localhost \ + -e DB_USERNAME \ + -e DB_PASSWORD \ + -e DB_DATABASE \ + kolab-amavis:latest +} + +podman__run_collabora() { + $PODMAN run -dt --pod $POD --name $POD-collabora --replace \ + --privileged \ + -e ALLOWED_HOSTS=${APP_DOMAIN} \ + kolab-collabora:latest +} + +podman__run_meet() { + $PODMAN run -dt --pod $POD --name $POD-meet --replace \ + -v ./meet/server:/src/meet:ro \ + -v $CERTS_PATH/meet.${APP_DOMAIN}.cert:/etc/pki/tls/certs/meet.${APP_DOMAIN}.cert \ + -v $CERTS_PATH/meet.${APP_DOMAIN}.key:/etc/pki/tls/private/meet.${APP_DOMAIN}.key \ + -e WEBRTC_LISTEN_IP=127.0.0.1 \ + -e WEBRTC_ANNOUNCED_ADDRESS=127.0.0.1 \ + -e PUBLIC_DOMAIN=$APP_DOMAIN \ + -e LISTENING_HOST=127.0.0.1 \ + -e LISTENING_PORT=12443 \ + -e DEBUG="*" \ + -e TURN_SERVER=none \ + -e AUTH_TOKEN=${MEET_SERVER_TOKEN} \ + -e WEBHOOK_TOKEN=${MEET_WEBHOOK_TOKEN} \ + -e WEBHOOK_URL=$APP_DOMAIN/api/webhooks/meet \ + -e SSL_CERT=/etc/pki/tls/certs/meet.$APP_DOMAIN.cert \ + -e SSL_KEY=/etc/pki/tls/private/meet.$APP_DOMAIN.key \ + kolab-meet:latest +} diff --git a/bin/quickstart.sh b/bin/quickstart.sh deleted file mode 100755 index c7a14a42..00000000 --- a/bin/quickstart.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -set -e -set -x - -base_dir=$(dirname $(dirname $0)) - -export DOCKER_BUILDKIT=1 -export BUILDKIT_PROGRESS=plain - -docker compose down -t 1 --remove-orphans -volumes=($(docker volume ls -f name=kolab | awk '{if (NR > 1) print $2}')) -for v in "${volumes[@]}" -do - docker volume rm $v || : -done - -# We can't use the following artisan commands because it will just block if redis is unavailable: -# src/artisan octane:stop >/dev/null 2>&1 || : -# src/artisan horizon:terminate >/dev/null 2>&1 || : -# we therefore just kill all artisan processes running. -pkill -9 -f artisan || : -pkill -9 -f swoole || : - -bin/regen-certs - -if [ "$1" == "--nodev" ]; then - echo "Starting everything in containers" - bin/build.sh - docker compose up -d --wait - exit 0 -fi -echo "Starting the development environment" - - -containers=(coturn mariadb meet pdns redis roundcube minio) - -if grep -q "ldap" docker-compose.override.yml; then - containers+=(ldap) -fi -# We grep for something that is unique to the container -if grep -q "kolab-init" docker-compose.override.yml; then - containers+=(kolab) -fi -if grep -q "imap" docker-compose.override.yml; then - containers+=(imap) -fi -if grep -q "postfix" docker-compose.override.yml; then - containers+=(postfix) -fi -if grep -q "imap-frontend" docker-compose.override.yml; then - containers+=(imap-frontend imap-backend imap-mupdate) -fi - - -docker compose build -docker compose up -d --wait ${containers[@]} - -pushd ${base_dir}/src/ - -rm -rf vendor/ composer.lock -php -dmemory_limit=-1 $(which composer) install -npm install -find bootstrap/cache/ -type f ! -name ".gitignore" -delete -./artisan key:generate -./artisan clear-compiled -./artisan cache:clear -./artisan horizon:install - -if rpm -qv chromium 2>/dev/null; then - chver=$(rpmquery --queryformat="%{VERSION}" chromium | awk -F'.' '{print $1}') - ./artisan dusk:chrome-driver ${chver} -fi - -if [ ! -f 'resources/countries.php' ]; then - ./artisan data:countries -fi - -npm run dev -popd - -pushd ${base_dir}/src/ -rm -rf database/database.sqlite -./artisan db:ping --wait -php -dmemory_limit=512M ./artisan migrate:refresh --seed -./artisan data:import || : -nohup ./artisan octane:start --host=$(grep OCTANE_HTTP_HOST .env | tail -n1 | sed "s/OCTANE_HTTP_HOST=//") > octane.out & -nohup ./artisan horizon > horizon.out & - -popd - -if grep -q "haproxy" docker-compose.override.yml; then - docker compose up --no-deps -d haproxy -fi -docker compose up --no-deps -d proxy diff --git a/bin/reconfigure.sh b/bin/reconfigure.sh deleted file mode 100755 index f5357b57..00000000 --- a/bin/reconfigure.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -cp src/.env .env.reconfigure.backup - -#FIXME: this will not remove files that have been removed in the updated overlay - -# Install new config -find -L config/ -type f | while read file; do - dir=$(dirname $file | sed -e 's|^config||g') - dir="./$dir" - - if [ ! -d $dir ]; then - mkdir -p $dir - fi - - cp -v $file $dir/ -done - -mv .env.reconfigure.backup src/.env diff --git a/bin/restore.sh b/bin/restore.sh deleted file mode 100755 index 38dc31b6..00000000 --- a/bin/restore.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -backup_path="$(pwd)/backup/" - -function restore_volume { - volume_name=$1 - backup_destination=$2 - - echo "Restoring $volume_name from $backup_destination" - docker run --rm -v $volume_name:/data -v $backup_destination:/backup quay.io/centos/centos:stream8 bash -c "rm -rf /data/* && tar xvf /backup/$volume_name.tar -C /data --strip 1" -} - -echo "Stopping containers" -docker compose stop - -# We currently expect the volumes to exist. -# We could alternatively create volumes form existing tar files -# for f in backup/*.tar; do -# echo "$(basename $f .tar)" ; -# done - -echo "Restoring volumes" -volumes=($(docker volume ls -f name=kolab | awk '{if (NR > 1) print $2}')) -for v in "${volumes[@]}" -do - restore_volume $v $backup_path -done -echo "Restarting containers" -docker compose start - diff --git a/bin/runci.sh b/bin/runci.sh new file mode 100755 index 00000000..5b99c037 --- /dev/null +++ b/bin/runci.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# This is how to run the ci container locally +# It needs privileges because it will run the podman containers inside the container (either via kolabctl or testctl) +# Mostly useful to make sure this works as expected. + +podman build docker/ci -t ci +mkdir /tmp/cicache +podman run --privileged --rm -ti -v /tmp/cicache:/var/lib/containers -e ROLE=test -e GIT_REF=dev/mollekopf ci:latest /init.sh diff --git a/bin/selfcheck.sh b/bin/selfcheck.sh deleted file mode 100755 index 8120d607..00000000 --- a/bin/selfcheck.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e -set -x - -APP_DOMAIN=$(grep APP_DOMAIN .env | tail -n1 | sed "s/APP_DOMAIN=//") -if [ -z "$ADMIN_PASSWORD" ]; then - ADMIN_PASSWORD="simple123" -fi -if [ -z "$ADMIN_USER" ]; then - ADMIN_USER="admin@$APP_DOMAIN" -fi - -docker compose exec postfix testsaslauthd -u "$ADMIN_USER" -p "$ADMIN_PASSWORD" -docker compose exec imap testsaslauthd -u "$ADMIN_USER" -p "$ADMIN_PASSWORD" - -docker compose -f docker-compose.yml -f docker-compose.build.yml run -ti --rm utils ./mailtransporttest.py --sender-username "$ADMIN_USER" --sender-password "$ADMIN_PASSWORD" --sender-host "$APP_DOMAIN" --recipient-username "$ADMIN_USER" --recipient-password "$ADMIN_PASSWORD" --recipient-host "$APP_DOMAIN" - -docker compose -f docker-compose.yml -f docker-compose.build.yml run -ti --rm utils ./kolabendpointtester.py --verbose --host "$APP_DOMAIN" --dav "https://$APP_DOMAIN/dav/" --imap "$APP_DOMAIN" --activesync "$APP_DOMAIN" --user "$ADMIN_USER" --password "$ADMIN_PASSWORD" - -echo "All tests have passed!" diff --git a/bin/start.sh b/bin/start.sh deleted file mode 100755 index 82252e6b..00000000 --- a/bin/start.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -docker compose up -d --wait diff --git a/bin/stop.sh b/bin/stop.sh deleted file mode 100755 index 3c4d8b24..00000000 --- a/bin/stop.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -docker compose down diff --git a/bin/update.sh b/bin/update.sh deleted file mode 100755 index 2f184a38..00000000 --- a/bin/update.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -docker compose down --remove-orphans - -docker pull quay.io/sclorg/mariadb-105-c9s -docker pull minio/minio:latest -docker pull almalinux:8 -docker pull almalinux:9 -docker pull fedora:35 -docker pull fedora:37 - -bin/reconfigure.sh -docker compose build -bin/regen-certs -docker compose up -d --wait diff --git a/ci/runchecks.sh b/ci/runchecks.sh deleted file mode 100755 index d41e2cb4..00000000 --- a/ci/runchecks.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -x -set -e - -# Setup -env HOST=kolab.local ADMIN_PASSWORD=simple123 bin/configure.sh config.demo -# docker compose pull --ignore-buildable -bin/quickstart.sh --nodev - -# Ensure the environment is functional -# env ADMIN_USER=john@kolab.org ADMIN_PASSWORD=simple123 bin/selfcheck.sh -ADMIN_USER=john@kolab.org -ADMIN_PASSWORD=simple123 -APP_DOMAIN=$(grep APP_DOMAIN .env | tail -n1 | sed "s/APP_DOMAIN=//") -docker compose exec postfix testsaslauthd -u "$ADMIN_USER" -p "$ADMIN_PASSWORD" -docker compose exec imap testsaslauthd -u "$ADMIN_USER" -p "$ADMIN_PASSWORD" -docker compose -f docker-compose.yml -f docker-compose.build.yml run -ti --rm utils ./kolabendpointtester.py --verbose --host "$APP_DOMAIN" --dav "https://$APP_DOMAIN/dav/" --imap "$APP_DOMAIN" --activesync "$APP_DOMAIN" --user "$ADMIN_USER" --password "$ADMIN_PASSWORD" diff --git a/ci/testctl b/ci/testctl index d493ff1a..8918d786 100755 --- a/ci/testctl +++ b/ci/testctl @@ -1,478 +1,408 @@ #!/bin/bash base_dir="$(dirname $(realpath "$0"))" pushd "${base_dir}" pushd .. set -e set -x PASSPORT_PRIVATE_KEY="-----BEGIN PRIVATE KEY----- MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCmYeRp7XXnPe8w X0iOJRpeskfUuOJ/Gqz5dsMIWFB6fPaI5/9tkMEyp+vCEF7eFXLBrXeQi6F/VNmV wn+dGEQhkhuDoEXr8Z4c333wLH8iOEF4WQbt/WF3ERdjmJt3vKry8B/OLNmmcK7j 4sz828h6L2ZT6GPcbGsNukxBMcIMOpflo0SLHy4VThdo6b1Q4nD2K/PX1ypyfFao nj3OfHBdSVLmTgd7BvB/azYFYWHP4INY8cylZWItDXuqPlBGSU2ff2xTKY/WRco/ djvrO9bM1WeI+8W36EeLHERru1QRpN22TgWCQ2dbLRsVrsMg8Ly6SMe8ceDXQt5C LKAN24jFt1UnBgr+qK1TrxkBtu5+V2WPYWhUvBLI/2qnFQh1GiWMKinWQO7rFCIC rRUcQBUu2AylmG0P/oPjPrjhAnxq3HguOn8cS1OeBpOH7+8tz0CeEdyVfT8maVs/ VWRZbEb0UjFLRNU+iVEGzz3jyQuKhOJ/2WuW0mJzF3pPQ64Dl+fLyXqF1KXNoPem evmmRjCZWfkWAEAWd3+yRfoOxGz55vaU1qGS81lnXnP1R5TZGXon24HHS9uRwHt6 JII+FEwgqr8K2TISDPxx7iQbXx8kcMUMBJG8aNoG73WVXmHs0uaEUsXMy9vtegeu //IPpNUTlbjsn8Ot+t68mTNLUZX74wIDAQABAoICAE5fZT8KVlPfJiikcWJXktTR aKmIj1Qs5ha6PQNUyk/wRhbWJUjge0jXtWNb37v/4WbexafGRgPbHYUAMal3kTw4 /RHi8JzD2uUh10pHQ3mEgz5jvTJkfMEfwWMuMulTazj1KB4vnTRb9t2saz+ebZA0 fKCAom1leoXkX+ADxrKI9Rz766EWxlfNyZQnKgCMMYabzIg6t6lm7VEO/PEjR7CB hfWrArYOXkG+6BrftLm9OVGv0GSGXZj4NWzLXnfFNrWvSYDg3nqhtDNxh6b2MGeb DGKHqipHVU/vOEGA44hOHwutM8YY5voZRJ1RjWOaUmPzPXaEM9NiEZydNaVhaEpq m7jNpu7S5xa2Eodt2iz2uQhnDHrYnGVCH5psal6TZAo9APWwwBOsFQ+nXwjxTeL9 +3JL6+jrP0eqzNVhl8c0cHJnBDpSVNG734RsK8XOxmJyq3Xt8Roi3Ud7gjy/FGpv XgzDpkFvd5uETn1VIuAfirm7MD8RbTIZAWCgqCrE7NuXOcnBGHuC955KF8OAx8np 8yCtlmBSXKifoIeeyu32L8s3g7md+xRuaU8yRtuClTLKG+6oRZYcaFNcVKKZzyu5 xnxUS6Haphd5/LhgnA3ujXkkNPdmHxPvJOWYABSNFeXzNF1npL/4wFLNvppMCPR1 v7M7AnbvyEvKm1Q2ePe9AoIBAQDigI4AJIaHeQiuqFSIWhm8NYkOZF0jfvWM7K8v 1IAE0WATP8KbeTINS2fUYZrNFs7S66Pl1WdPH7atVoi7QVcIoFhlYYRqILETpKJr z0dFLIiaajzQ9kTPzhLRDGBhO3TKb7RpFndYAuxzSw1C/3JHb4crD8kDIB8xVoba xvsXdVssqBQgScUrj1Ff4ZPtFhqLPsWnvdBpbM6LV/2t/CnTu4qU2szJZQNGP1Qf gEapbuZC6YFahXDTgYFTfn/vKzyKb/Fiskz3Rs9jgY08gRxIandeUqJIEoJi+CwZ q6twD8qKzGhB9nxSAOwhJzDg4SyhNnRQt5X8XQWVjpxs3HxnAoIBAQC8DPsIDN5r 7joZj5d4/k8Yg+q1ecySm9zYy9Lzf0WUFgRu9NW9UeUPRjGXhNo5VOxxB62rMZCJ E81ItxUVQwHH4S62ycBPbsYEapE/itS+KdEzWQP2u3HAkLD3N28snMlIhTJR8fXB GasWngs9Q7uB7Wk0niKa8T7fBDx9pOyjMlIPwo0lZCrUAnmjOgZ+RvvuGDgqpDdp h7JUxtFmsWPgBFNZtr5BTRcr5hWRoSXJgQODqpTQHjQddMWy7LCJg3qKLiKVIOd5 +iGzhUIZzo95FYiyt8Ojdt3Y0k5J99NOrOwAPNLvbC5TTshtA144E9uwEqBbTm+S RtLZeVBWZ1clAoIBAQC0j26jxnpH/MBjG2Vn3Quu8a50fqWQ6mCtGvD83BXBwXcp YSat8gtodbgrojNZUtlFYvug+GIGvW1O+TC+tfO/uLM+/mIkiDMhSZkBAJf8GOg8 0HvyyJ9KWSi+5XLfkBomVq4nJ/Wzf4Em16mWwzRCpjHGriq8BxtWpXeTaBQ6Ox+X ldWVd7lqZDGmkZju4zP91OiUM8i0gjyU8GwWCnL9iv+KcnHWCmR1134kLool/3Yn 2SV5F+89bHvAJ5OtAXadlWeEGkcoyJYC6P/CP9pgEB9gXddoRPkUFGpzfFqKVsxL oW9rRicM6BdUxn08h8SgL1zCC9fQ+ga9lpY0Yf/5AoIBAH7S5k5El5EE5mwsukRg hqmK9jUUAtLxiR0xQYD02dEIlE7cknYPEEOf3HxKnf5Cdv+35PlrAQZhs3YR+4cO XNoX1TBzml434BZEZNcM43Oosi1GIHU7b3kmXCMuYK0exGVDZ296lnp3vDoRtpTH 5GK44dYZvE7w2qz/p2g5XVqm6k80r4qDJps7XBuoW464gtnNvbuMas6iNLQWLk1q 32fKowgDRga2XiU+FFfV7a0bdGpNFfXSGOWwxlBobpsfb/pXKP2YZmSOPEJdYfoT pBFOY5Xcd3X8CZxcIW6jVABggP2cB8pvFEMdA/D5b4a0Zdo2ha1ulbJ6T2NZ/MN5 CH0CggEBAMLRnxLQRCgdyrYroqdSBU85fAk0uU//rn7i/1vQG6pUy4Dq6W/yBhFV /Fph6c9NXHUUbM3HlvyY2Ht4aUQl8d50wsyU6enxvpdwzti6N2WXyrEX4WtVqgNP OKHEu+mii3m6kOfvDD97AT4hAGzCZR4lkb06t49y7ua4NRZaKTrTiG3g2uTtBR81 /w1GtL+DNUEFzO1Iy2dscWxr76I+ZX6VlFHGneUlhyN9VJk8WHVI5xpVV9y7ay3I jXXFDgNqjqiSC6BU7iYpkVEKl/hvaGJU7CKLKFbxzBgseyY/7XsMHvWbwjK8a0Lm bakhie7hJBP7BoOup+dD5NQPlXBQ434= -----END PRIVATE KEY-----" PASSPORT_PUBLIC_KEY="-----BEGIN PUBLIC KEY----- MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEApmHkae115z3vMF9IjiUa XrJH1Ljifxqs+XbDCFhQenz2iOf/bZDBMqfrwhBe3hVywa13kIuhf1TZlcJ/nRhE IZIbg6BF6/GeHN998Cx/IjhBeFkG7f1hdxEXY5ibd7yq8vAfzizZpnCu4+LM/NvI ei9mU+hj3GxrDbpMQTHCDDqX5aNEix8uFU4XaOm9UOJw9ivz19cqcnxWqJ49znxw XUlS5k4Hewbwf2s2BWFhz+CDWPHMpWViLQ17qj5QRklNn39sUymP1kXKP3Y76zvW zNVniPvFt+hHixxEa7tUEaTdtk4FgkNnWy0bFa7DIPC8ukjHvHHg10LeQiygDduI xbdVJwYK/qitU68ZAbbufldlj2FoVLwSyP9qpxUIdRoljCop1kDu6xQiAq0VHEAV LtgMpZhtD/6D4z644QJ8atx4Ljp/HEtTngaTh+/vLc9AnhHclX0/JmlbP1VkWWxG 9FIxS0TVPolRBs8948kLioTif9lrltJicxd6T0OuA5fny8l6hdSlzaD3pnr5pkYw mVn5FgBAFnd/skX6DsRs+eb2lNahkvNZZ15z9UeU2Rl6J9uBx0vbkcB7eiSCPhRM IKq/CtkyEgz8ce4kG18fJHDFDASRvGjaBu91lV5h7NLmhFLFzMvb7XoHrv/yD6TV E5W47J/DrfrevJkzS1GV++MCAwEAAQ== -----END PUBLIC KEY-----" +export HOST=kolab.local + +export APP_WEBSITE_DOMAIN="$HOST" +export SSL_CERTIFICATE="/etc/certs/imap.hosted.com.cert" +export SSL_CERTIFICATE_KEY="/etc/certs/imap.hosted.com.key" +export APP_DOMAIN=$HOST +export DES_KEY=kBxUM/53N9p9abusAoT0ZEAxwI2pxFz/ +export DB_HOST=127.0.0.1 +export IMAP_HOST=localhost +export IMAP_PORT=11143 +export IMAP_ADMIN_LOGIN=cyrus-admin +export IMAP_ADMIN_PASSWORD=simple123 +export MAIL_HOST=localhost +export MAIL_PORT=10587 +export IMAP_DEBUG=true +export FILEAPI_WOPI_OFFICE=https://$HOST +export CALENDAR_CALDAV_SERVER=http://localhost:11080/dav +export KOLAB_ADDRESSBOOK_CARDDAV_SERVER=http://localhost:11080/dav +export DB_ROOT_PASSWORD=simple123 +export DB_HKCCP_PASSWORD=simple123 +export DB_KOLAB_PASSWORD=simple123 +export DB_RC_PASSWORD=simple123 +export DB_PASSWORD=simple123 +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=simple123 +export MINIO_USER=minio +export MINIO_PASSWORD=simple123 +export MEET_SERVER_TOKEN=simple123 +export MEET_WEBHOOK_TOKEN=simple123 + +export CERTS_PATH=./ci/certs +export IMAP_SPOOL_STORAGE=--mount=type=tmpfs,tmpfs-size=128M,tmpfs-mode=777,destination=/var/spool/imap,U=true,notmpcopyup +export IMAP_LIB_STORAGE=--mount=type=tmpfs,tmpfs-size=128M,tmpfs-mode=777,destination=/var/lib/imap,U=true,notmpcopyup +export MARIADB_STORAGE=--mount=type=tmpfs,tmpfs-size=512M,destination=/var/lib/mysql,U=true +export REDIS_STORAGE=--mount=type=tmpfs,tmpfs-size=128M,destination=/var/lib/redis,U=true +export MINIO_STORAGE=--mount=type=tmpfs,tmpfs-size=128M,destination=/data,U=true + + +export PODMAN_IGNORE_CGROUPSV1_WARNING=true PODMAN="podman" +source bin/podman_shared + # Teardown the currently running environment kolab__teardown() { $PODMAN pod rm --force tests $PODMAN pod rm --force dev } # Build all containers required for testing kolab__build() { - source bin/build.sh - podman__build_base - podman__build_webapp - podman__build_meet - - podman__build docker/imap kolab-imap - podman__build docker/mariadb mariadb - podman__build docker/redis redis - podman__build docker/proxy kolab-proxy - podman__build docker/tests kolab-tests --ulimit nofile=65535:65535 - podman__build_roundcube - - env CERT_DIR=ci/certs APP_DOMAIN=kolab.local bin/regen-certs + if [[ $1 != "" ]]; then + if declare -f "podman__build_$1" >/dev/null 2>&1; then + podman__build_$1 + else + podman__build docker/$1 $1 + fi + else + podman__build_base + podman__build_webapp + podman__build_meet + + podman__build docker/imap kolab-imap + podman__build docker/mariadb mariadb + podman__build docker/redis redis + podman__build docker/proxy kolab-proxy + podman__build_roundcube + + podman__build docker/tests kolab-tests --ulimit nofile=65535:65535 + env CERT_DIR=ci/certs APP_DOMAIN=$HOST bin/regen-certs + fi } # Setup the test environment kolab__setup() { echo "Build" kolab__build echo "Setup" - POD=tests + export POD=tests # Create the pod first $PODMAN pod create --replace --name $POD podman__run_mariadb podman__run_redis podman__healthcheck $POD-mariadb $POD-redis - podman__run_webapp config.demo + podman__run_imap + + podman__run_webapp ci/env config.demo podman__healthcheck $POD-webapp - podman__run_imap podman__healthcheck $POD-imap # Ensure all commands are processed echo "Flushing work queue" $PODMAN exec -ti $POD-webapp ./artisan queue:work --stop-when-empty podman__run_minio podman__healthcheck $POD-minio # Validate the test environment kolab__validate $POD } # "testsuite" # "quicktest" # "tests/Feature/Jobs/WalletCheckTest.php" kolab__test() { - POD=tests + export POD=tests $PODMAN run -ti --pod tests --name $POD-kolab-tests --replace \ --env-file=ci/env \ -v ./src:/src/kolabsrc.orig:ro \ -e APP_SERVICES_DOMAINS="localhost" \ -e PASSPORT_PRIVATE_KEY="$PASSPORT_PRIVATE_KEY" \ -e PASSPORT_PUBLIC_KEY="$PASSPORT_PUBLIC_KEY" \ -e APP_URL="http://kolab.local" \ -e APP_PUBLIC_URL="http://kolab.local" \ -e APP_HEADER_CSP="" \ -e APP_HEADER_XFO="" \ -e ASSET_URL="http://kolab.local" \ -e MEET_SERVER_URLS="http://kolab.local/meetmedia/api/" \ kolab-tests:latest /init.sh $@ } kolab__proxytest() { $PODMAN run -ti --pod tests --name $POD-proxy-tests --replace \ -v ./ci/certs/:/etc/certs/:ro \ --env-file=ci/env \ -e SSL_CERTIFICATE=/etc/certs/imap.hosted.com.cert \ -e SSL_CERTIFICATE_KEY=/etc/certs/imap.hosted.com.key \ kolab-proxy:latest /init.sh validate } kolab__lint() { - kolab__test lint + $PODMAN run --rm -ti \ + -v ./src:/src/kolabsrc.orig:ro \ + kolab-tests:latest /init.sh lint } # Setup the test environment and run a complete testsuite kolab__testrun() { echo "Setup" kolab__setup echo "Test" kolab__test testsuite } # Setup the test environment and run all testsuites kolab__testrun_complete() { echo "Setup" kolab__setup echo "Test" kolab__test lint kolab__test testsuite kolab__rctest syncroton lint kolab__rctest syncroton testsuite kolab__rctest irony lint # kolab__rctest irony testsuite kolab__rctest roundcubemail-plugins-kolab lint # kolab__rctest roundcubemail-plugins-kolab testsuite } # Get a shell inside the test container to run/debug tests kolab__shell() { kolab__test shell } # Run the roundcube testsuite kolab__rctest() { $PODMAN run -t --pod tests --name $POD-roundcube --replace \ -v ./ext:/src.orig:ro \ -e APP_DOMAIN=kolab.local \ -e DES_KEY=kBxUM/53N9p9abusAoT0ZEAxwI2pxFz/ \ -e DB_HOST=127.0.0.1 \ -e DB_RC_DATABASE=roundcube \ -e DB_RC_USERNAME=roundcube \ -e DB_RC_PASSWORD=simple123 \ -e IMAP_HOST=localhost \ -e IMAP_PORT=11143 \ -e IMAP_ADMIN_LOGIN=cyrus-admin \ -e IMAP_ADMIN_PASSWORD=simple123 \ -e IMAP_DEBUG=false \ -e SQL_DEBUG=false \ -e ACTIVESYNC_DEBUG=false \ -e RUN_MIGRATIONS=true \ -e MAIL_HOST=localhost \ -e MAIL_PORT=10587 \ -e FILEAPI_WOPI_OFFICE=https://kolab.local \ -e CALENDAR_CALDAV_SERVER=http://imap:11080/dav \ -e KOLAB_ADDRESSBOOK_CARDDAV_SERVER=http://imap:11080/dav \ roundcube:latest ./init.sh $@ } # Get a shell inside the roundcube test container to run/debug tests kolab__rcshell() { $PODMAN run -ti --pod tests --name $POD-roundcube --replace \ -v ./ext:/src.orig:ro \ -e APP_DOMAIN=kolab.local \ -e DES_KEY=kBxUM/53N9p9abusAoT0ZEAxwI2pxFz/ \ -e DB_HOST=127.0.0.1 \ -e DB_RC_DATABASE=roundcube \ -e DB_RC_USERNAME=roundcube \ -e DB_RC_PASSWORD=simple123 \ -e IMAP_HOST=localhost \ -e IMAP_PORT=11143 \ -e IMAP_ADMIN_LOGIN=cyrus-admin \ -e IMAP_ADMIN_PASSWORD=simple123 \ -e MAIL_HOST=localhost \ -e MAIL_PORT=10587 \ -e FILEAPI_WOPI_OFFICE=https://kolab.local \ -e CALENDAR_CALDAV_SERVER=http://localhost:11080/dav \ -e KOLAB_ADDRESSBOOK_CARDDAV_SERVER=http://localhost:11080/dav \ roundcube:latest ./init.sh shell } kolab__validate() { POD=$1 $PODMAN exec $POD-imap testsaslauthd -u cyrus-admin -p simple123 $PODMAN exec $POD-imap testsaslauthd -u "john@kolab.org" -p simple123 # Ensure the inbox is created FOUND=false for i in {1..60}; do if $PODMAN exec $POD-imap bash -c 'echo "lm" | cyradm --auth PLAIN -u cyrus-admin -w simple123 --port 11143 localhost | grep "user/john@kolab.org"'; then echo "Found mailbox"; FOUND=true break else echo "Waiting for mailbox"; sleep 1; fi done if ! $FOUND; then echo "Failed to find the inbox for john@kolab.org" exit 1 fi } -podman__healthcheck() { - for CONTAINER in $@; do - echo "Waiting for ${CONTAINER} become healthy " - while [ $(podman healthcheck run ${CONTAINER}) ]; do - echo -n "."; sleep 5; - done - done -} - -podman__run_proxy() { - $PODMAN run -dt --pod $POD --name $POD-proxy --replace \ - -v ./ci/certs:/etc/certs:ro \ - -e APP_WEBSITE_DOMAIN="kolab.local" \ - -e SSL_CERTIFICATE="/etc/certs/imap.hosted.com.cert" \ - -e SSL_CERTIFICATE_KEY="/etc/certs/imap.hosted.com.key" \ - -e WEBAPP_BACKEND="http://localhost:8000" \ - -e MEET_BACKEND="https://localhost:12443" \ - -e ROUNDCUBE_BACKEND="http://localhost:8080" \ - -e DAV_BACKEND="http://localhost:11080/dav" \ - -e COLLABORA_BACKEND="http://localhost:9980" \ - kolab-proxy:latest -} - -podman__run_roundcube() { - $PODMAN run -dt --pod $POD --name $POD-roundcube --replace \ - -v ./ext:/src.orig:ro \ - -e APP_DOMAIN=kolab.local \ - -e DES_KEY=kBxUM/53N9p9abusAoT0ZEAxwI2pxFz/ \ - -e DB_HOST=127.0.0.1 \ - -e DB_RC_DATABASE=roundcube \ - -e DB_RC_USERNAME=roundcube \ - -e DB_RC_PASSWORD=simple123 \ - -e IMAP_HOST=localhost \ - -e IMAP_PORT=11143 \ - -e IMAP_ADMIN_LOGIN=cyrus-admin \ - -e IMAP_ADMIN_PASSWORD=simple123 \ - -e MAIL_HOST=localhost \ - -e MAIL_PORT=10587 \ - -e IMAP_DEBUG=true \ - -e FILEAPI_WOPI_OFFICE=https://kolab.local \ - -e CALENDAR_CALDAV_SERVER=http://localhost:11080/dav \ - -e KOLAB_ADDRESSBOOK_CARDDAV_SERVER=http://localhost:11080/dav \ - roundcube:latest -} - -podman__run_mariadb() { - $PODMAN run -dt --pod $POD --name $POD-mariadb --replace \ - --mount=type=tmpfs,tmpfs-size=512M,destination=/var/lib/mysql,U=true \ - -e MYSQL_ROOT_PASSWORD=simple123 \ - -e TZ="+02:00" \ - -e DB_HKCCP_DATABASE=kolabdev \ - -e DB_HKCCP_USERNAME=kolabdev \ - -e DB_HKCCP_PASSWORD=simple123 \ - -e DB_KOLAB_DATABASE=kolab \ - -e DB_KOLAB_USERNAME=kolab \ - -e DB_KOLAB_PASSWORD=simple123 \ - -e DB_RC_DATABASE=roundcube \ - -e DB_RC_USERNAME=roundcube \ - -e DB_RC_PASSWORD=simple123 \ - --health-cmd "mysqladmin -u root ping && test -e /tmp/initialized" \ - mariadb:latest -} - -podman__run_redis() { - $PODMAN run -dt --pod $POD --name $POD-redis --replace \ - --mount=type=tmpfs,tmpfs-size=128M,destination=/var/lib/redis,U=true \ - --health-cmd "redis-cli ping || exit 1" \ - redis:latest -} - -podman__run_minio() { - $PODMAN run -dt --pod $POD --name $POD-minio --replace \ - --mount=type=tmpfs,tmpfs-size=128M,destination=/data,U=true \ - -e MINIO_ROOT_USER=minio \ - -e MINIO_ROOT_PASSWORD=simple123 \ - --health-cmd "mc ready local || exit 1" \ - --entrypoint sh \ - quay.io/minio/minio:latest -c 'mkdir -p /data/kolab && minio server /data --console-address ":9001"' -} - -podman__run_webapp() { - # We run with a fixed config.demo overlay and override the environment with ci/env - $PODMAN run -dt --pod $POD --name $POD-webapp --replace \ - --env-file=ci/env \ - -v ./src:/src/kolabsrc.orig:ro \ - -v ./$1/src:/src/overlay:ro \ - -e NOENVFILE=true \ - -e APP_SERVICES_ALLOWED_DOMAINS="webapp,localhost,services.kolab.local" \ - -e KOLAB_ROLE=combined \ - -e PASSPORT_PRIVATE_KEY="$PASSPORT_PRIVATE_KEY" \ - -e PASSPORT_PUBLIC_KEY="$PASSPORT_PUBLIC_KEY" \ - --health-cmd "./artisan octane:status || exit 1" \ - kolab-webapp:latest -} - -podman__run_imap() { - $PODMAN run -dt --pod $POD --name $POD-imap --replace \ - --mount=type=tmpfs,tmpfs-size=128M,tmpfs-mode=777,destination=/var/spool/imap,U=true,notmpcopyup \ - --mount=type=tmpfs,tmpfs-size=128M,tmpfs-mode=777,destination=/var/lib/imap,U=true,notmpcopyup \ - -e APP_SERVICES_DOMAIN="localhost" \ - -e SERVICES_PORT=8000 \ - -e IMAP_ADMIN_LOGIN=cyrus-admin \ - -e IMAP_ADMIN_PASSWORD=simple123 \ - --health-cmd "test -e /run/saslauthd/mux && kill -0 \$(cat /var/run/master.pid)" \ - kolab-imap:latest -} - -podman__run_meet() { - $PODMAN run -dt --pod $POD --name $POD-meet --replace \ - -v ./meet/server:/src/meet:ro \ - -v ./ci/certs/meet.kolab.local.cert:/etc/pki/tls/certs/meet.kolab.local.cert \ - -v ./ci/certs/meet.kolab.local.key:/etc/pki/tls/private/meet.kolab.local.key \ - -e WEBRTC_LISTEN_IP=127.0.0.1 \ - -e WEBRTC_ANNOUNCED_ADDRESS=127.0.0.1 \ - -e PUBLIC_DOMAIN=kolab.local \ - -e LISTENING_HOST=127.0.0.1 \ - -e LISTENING_PORT=12443 \ - -e DEBUG="*" \ - -e TURN_SERVER=none \ - -e AUTH_TOKEN=simple123 \ - -e WEBHOOK_TOKEN=simple123 \ - -e WEBHOOK_URL=kolab.local/api/webhooks/meet \ - -e SSL_CERT=/etc/pki/tls/certs/meet.kolab.local.cert \ - -e SSL_KEY=/etc/pki/tls/private/meet.kolab.local.key \ - kolab-meet:latest +kolab__run() { + export POD=dev + podman__run_$1 } kolab__deploy() { - POD=dev + export POD=dev # Create the pod first $PODMAN pod create \ --replace \ --add-host=kolab.local:127.0.0.1 \ --publish "443:6443" \ --publish "465:6465" \ --publish "587:6587" \ --publish "143:6143" \ --publish "993:6993" \ --publish "44444:44444/udp" \ --publish "44444:44444/tcp" \ --name $POD podman__run_mariadb podman__run_redis podman__healthcheck $POD-mariadb $POD-redis - podman__run_webapp config.prod + podman__run_imap + + podman__run_webapp ci/env config.prod podman__healthcheck $POD-webapp - podman__run_imap podman__healthcheck $POD-imap # Ensure all commands are processed echo "Flushing work queue" $PODMAN exec -ti $POD-webapp ./artisan queue:work --stop-when-empty $PODMAN exec $POD-webapp ./artisan user:password "admin@kolab.local" "simple123" podman__run_minio podman__healthcheck $POD-minio podman__run_meet podman__run_roundcube podman__run_proxy } kolab__generate_mail() { $PODMAN run --pod=dev -t --rm kolab-utils:latest ./generatemail.py --maxAttachmentSize=3 --type=mail --count 100 --username admin@kolab.local --password simple123 --host localhost --port 11143 INBOX } kolab__syncroton_sync() { $PODMAN run -t --network=host --add-host=kolab.local:127.0.0.1 --rm kolab-utils:latest ./activesynccli.py --host kolab.local --user admin@kolab.local --password simple123 sync 38b950ebd62cd9a66929c89615d0fc04 } kolab__db() { POD=dev $PODMAN exec -ti $POD-mariadb /bin/bash -c "mysql -h 127.0.0.1 -u kolabdev --password=simple123 kolabdev" } kolab__help() { cat </dev/null 2>&1; then "kolab__$cmdname" "${@:1}" else echo "Function $cmdname not recognized" >&2 kolab__help exit 1 fi diff --git a/config.prod/src/.env b/config.prod/src/.env index bd150af6..28a4daad 100644 --- a/config.prod/src/.env +++ b/config.prod/src/.env @@ -1,149 +1,147 @@ APP_NAME=Kolab APP_ENV=local APP_KEY= APP_DEBUG=true APP_URL=https://{{ host }} APP_PUBLIC_URL=https://{{ host }} APP_DOMAIN={{ host }} APP_WEBSITE_DOMAIN={{ host }} APP_THEME=default APP_TENANT_ID=5 APP_LOCALE=en APP_LOCALES= APP_WITH_ADMIN=1 APP_WITH_RESELLER=1 APP_WITH_SERVICES=1 APP_WITH_FILES=1 APP_WITH_WALLET=0 APP_WITH_SIGNUP=0 APP_LDAP=0 APP_IMAP=1 APP_HEADER_CSP="connect-src 'self'; child-src 'self'; font-src 'self'; form-action 'self' data:; frame-ancestors 'self'; img-src blob: data: 'self' *; media-src 'self'; object-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-eval' 'unsafe-inline'; default-src 'self';" APP_HEADER_XFO=sameorigin ASSET_URL=https://{{ host }} WEBMAIL_URL=/roundcubemail/ SUPPORT_URL=/support LOG_CHANNEL=stdout LOG_SLOW_REQUESTS=5 LOG_DEPRECATIONS_CHANNEL=null LOG_LEVEL=debug DB_CONNECTION=mysql DB_DATABASE=kolabdev -DB_HOST=mariadb +DB_HOST=127.0.0.1 DB_PASSWORD={{ admin_password }} DB_ROOT_PASSWORD={{ admin_password }} DB_PORT=3306 DB_USERNAME=kolabdev BROADCAST_DRIVER=redis CACHE_DRIVER=redis QUEUE_CONNECTION=redis SESSION_DRIVER=file SESSION_LIFETIME=120 -OPENEXCHANGERATES_API_KEY="from openexchangerates.org" - -MFA_DSN=mysql://roundcube:{{ admin_password }}@mariadb/roundcube +MFA_DSN=mysql://roundcube:{{ admin_password }}@127.0.0.1/roundcube MFA_TOTP_DIGITS=6 MFA_TOTP_INTERVAL=30 MFA_TOTP_DIGEST=sha1 -IMAP_URI=imap:11143 -IMAP_HOST=172.18.0.12 +IMAP_URI=127.0.0.1:11143 +IMAP_HOST=127.0.0.1 IMAP_PORT=11143 IMAP_GUAM_PORT=11143 IMAP_ADMIN_LOGIN=cyrus-admin IMAP_ADMIN_PASSWORD={{ admin_password }} IMAP_VERIFY_HOST=false IMAP_VERIFY_PEER=false IMAP_WITH_GROUPWARE_DEFAULT_FOLDERS=false -SMTP_HOST=172.18.0.13 +SMTP_HOST=127.0.0.1 SMTP_PORT=10587 COTURN_PUBLIC_IP='{{ public_ip }}' -MEET_SERVER_URLS=https://{{ host }}/meetmedia/api/ +MEET_SERVER_URLS=https://127.0.0.1:12443/meetmedia/api/ MEET_SERVER_VERIFY_TLS=false -MEET_WEBRTC_LISTEN_IP='172.18.0.1' +MEET_WEBRTC_LISTEN_IP='127.0.0.1' MEET_PUBLIC_DOMAIN={{ host }} -MEET_TURN_SERVER='turn:172.18.0.1:3478' -MEET_LISTENING_HOST=172.18.0.1 +MEET_TURN_SERVER='turn:127.0.0.1:3478' +MEET_LISTENING_HOST=127.0.0.1 PGP_ENABLE=true PGP_BINARY=/usr/bin/gpg PGP_AGENT=/usr/bin/gpg-agent PGP_GPGCONF=/usr/bin/gpgconf PGP_LENGTH= -REDIS_HOST=redis +REDIS_HOST=127.0.0.1 REDIS_PASSWORD=null REDIS_PORT=6379 OCTANE_HTTP_HOST={{ host }} SWOOLE_PACKAGE_MAX_LENGTH=10485760 MAIL_MAILER=smtp -MAIL_HOST=172.18.0.7 -MAIL_PORT=587 +MAIL_HOST=127.0.0.1 +MAIL_PORT=10587 MAIL_USERNAME="noreply@{{ host }}" MAIL_PASSWORD="{{ admin_password }}" MAIL_ENCRYPTION=starttls MAIL_FROM_ADDRESS="noreply@{{ host }}" MAIL_FROM_NAME="{{ host }}" MAIL_REPLYTO_ADDRESS="noreply@{{ host }}" MAIL_REPLYTO_NAME=null MAIL_VERIFY_PEER='false' RATELIMIT_WHITELIST="noreply@{{ host }}" DNS_TTL=3600 DNS_SPF="v=spf1 mx -all" DNS_STATIC="%s. MX 10 ext-mx01.mykolab.com." DNS_COPY_FROM=null MIX_ASSET_PATH='/' PASSWORD_POLICY= COMPANY_NAME=kolab.org COMPANY_ADDRESS= COMPANY_DETAILS= COMPANY_EMAIL= COMPANY_LOGO= COMPANY_FOOTER= VAT_COUNTRIES=CH,LI VAT_RATE=7.7 KB_ACCOUNT_DELETE= KB_ACCOUNT_SUSPENDED= KB_PAYMENT_SYSTEM= KOLAB_SSL_CERTIFICATE=/etc/certs/kolab.hosted.com.cert KOLAB_SSL_CERTIFICATE_FULLCHAIN=/etc/certs/kolab.hosted.com.chain.pem KOLAB_SSL_CERTIFICATE_KEY=/etc/certs/kolab.hosted.com.key PROXY_SSL_CERTIFICATE=/etc/certs/imap.hosted.com.cert PROXY_SSL_CERTIFICATE_KEY=/etc/certs/imap.hosted.com.key OPENEXCHANGERATES_API_KEY={{ openexchangerates_api_key }} FIREBASE_API_KEY={{ firebase_api_key }} MINIO_USER=minio MINIO_PASSWORD={{ admin_password }} MINIO_BUCKET=kolab FILESYSTEM_DISK=minio -TRUSTED_PROXIES="::ffff:172.18.0.7/8" +TRUSTED_PROXIES="::ffff:127.0.0.1/8" diff --git a/docker-compose.build.yml b/docker-compose.build.yml deleted file mode 100644 index 7b1a2e75..00000000 --- a/docker-compose.build.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: '3' -services: - swoole: - build: - context: ./docker/swoole/ - image: apheleia/swoole - almalinux8: - build: - context: ./docker/base/ - dockerfile: almalinux8 - image: apheleia/almalinux8 - almalinux9: - build: - context: ./docker/base/ - dockerfile: almalinux9 - image: apheleia/almalinux9 - tests: - build: - context: ./docker/tests/ - container_name: kolab-tests - image: kolab-tests - fluentbit: - build: - context: ./docker/fluentbit/ - container_name: fluentbit - image: fluentbit - utils: - build: - context: ./docker/utils/ - container_name: kolab-utils - image: kolab-utils - dns: 172.18.0.11 - extra_hosts: - - "${APP_DOMAIN}:172.18.0.7" - networks: - kolab: - ipv4_address: 172.18.0.27 diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100644 index 7ee6c374..00000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3' -services: - roundcube: - networks: - kolab: - ipv4_address: 172.18.0.31 diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index ec2dbfac..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,249 +0,0 @@ -version: '3' -services: - coturn: - build: - context: ./docker/coturn/ - container_name: kolab-coturn - healthcheck: - interval: 10s - test: "kill -0 $$(cat /tmp/turnserver.pid)" - timeout: 5s - retries: 30 - environment: - - TURN_PUBLIC_IP=${COTURN_PUBLIC_IP} - - TURN_LISTEN_PORT=3478 - - TURN_STATIC_SECRET=${COTURN_STATIC_SECRET} - hostname: sturn.mgmt.com - image: kolab-coturn - network_mode: host - restart: on-failure - roundcube: - build: - context: ./docker/roundcube/ - args: - GIT_REF_ROUNDCUBEMAIL: ${GIT_REF_ROUNDCUBEMAIL} - GIT_REMOTE_ROUNDCUBEMAIL: ${GIT_REMOTE_ROUNDCUBEMAIL} - GIT_REF_ROUNDCUBEMAIL_PLUGINS: ${GIT_REF_ROUNDCUBEMAIL_PLUGINS} - GIT_REMOTE_ROUNDCUBEMAIL_PLUGINS: ${GIT_REMOTE_ROUNDCUBEMAIL_PLUGINS} - GIT_REF_CHWALA: ${GIT_REF_CHWALA} - GIT_REMOTE_CHWALA: ${GIT_REMOTE_CHWALA} - GIT_REF_SYNCROTON: ${GIT_REF_SYNCROTON} - GIT_REMOTE_SYNCROTON: ${GIT_REMOTE_SYNCROTON} - GIT_REF_AUTOCONF: ${GIT_REF_AUTOCONF} - GIT_REMOTE_AUTOCONF: ${GIT_REMOTE_AUTOCONF} - GIT_REF_IRONY: ${GIT_REF_IRONY} - GIT_REMOTE_IRONY: ${GIT_REMOTE_IRONY} - GIT_REF_FREEBUSY: ${GIT_REF_FREEBUSY} - GIT_REMOTE_FREEBUSY: ${GIT_REMOTE_FREEBUSY} - container_name: kolab-roundcube - hostname: roundcube.hosted.com - restart: on-failure - depends_on: - mariadb: - condition: service_healthy - pdns: - condition: service_healthy - environment: - - APP_DOMAIN=${APP_DOMAIN} - - DES_KEY=${DES_KEY} - - DB_HOST=mariadb - - DB_ROOT_PASSWORD=${DB_ROOT_PASSWORD} - - DB_RC_DATABASE=roundcube - - DB_RC_USERNAME=roundcube - - DB_RC_PASSWORD=${DB_PASSWORD:?"DB_PASSWORD is missing"} - - IMAP_HOST=imap - - IMAP_PORT=11143 - - IMAP_ADMIN_LOGIN=${IMAP_ADMIN_LOGIN} - - IMAP_ADMIN_PASSWORD=${IMAP_ADMIN_PASSWORD} - - MAIL_HOST=postfix - - MAIL_PORT=10587 - healthcheck: - interval: 10s - test: "kill -0 $$(cat /run/httpd/httpd.pid)" - timeout: 5s - retries: 30 - # This makes docker's dns, resolve via pdns for this container. - # Please note it does not affect /etc/resolv.conf - dns: 172.18.0.11 - image: roundcube - extra_hosts: - - "${APP_DOMAIN}:172.18.0.7" - networks: - kolab: - ipv4_address: 172.18.0.9 - ports: - - "8080:8080" - tmpfs: - - /tmp - - /var/tmp - volumes: - - ./ext/:/src.orig/:ro - - roundcube:/data - mariadb: - build: - context: ./docker/mariadb/ - container_name: kolab-mariadb - restart: on-failure - environment: - - MYSQL_ROOT_PASSWORD=${DB_ROOT_PASSWORD} - - TZ="+02:00" - - DB_HKCCP_DATABASE=${DB_DATABASE} - - DB_HKCCP_USERNAME=${DB_USERNAME} - - DB_HKCCP_PASSWORD=${DB_PASSWORD} - - DB_KOLAB_DATABASE=kolab - - DB_KOLAB_USERNAME=kolab - - DB_KOLAB_PASSWORD=${DB_PASSWORD:?"DB_PASSWORD is missing"} - - DB_RC_DATABASE=roundcube - - DB_RC_USERNAME=roundcube - - DB_RC_PASSWORD=${DB_PASSWORD:?"DB_PASSWORD is missing"} - healthcheck: - interval: 10s - test: "mysqladmin -u root ping && test -e /tmp/initialized" - timeout: 5s - retries: 30 - image: mariadb - networks: - kolab: - ipv4_address: 172.18.0.3 - volumes: - - mariadb:/var/lib/mysql - pdns: - build: - context: ./docker/pdns/ - container_name: kolab-pdns - restart: on-failure - tty: true - hostname: pdns - depends_on: - mariadb: - condition: service_healthy - healthcheck: - interval: 10s - test: "pdns_control rping || exit 1" - timeout: 5s - retries: 30 - image: kolab-pdns - environment: - - ROLE=both - - DB_HOST=mariadb - - DB_DATABASE=${DB_DATABASE:?DB_DATABASE} - - DB_USERNAME=${DB_USERNAME:?DB_USERNAME} - - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD} - networks: - kolab: - ipv4_address: 172.18.0.11 - tmpfs: - - /run - - /tmp - - /var/run - - /var/tmp - volumes: - - /sys/fs/cgroup:/sys/fs/cgroup:ro - redis: - build: - context: ./docker/redis/ - healthcheck: - interval: 10s - test: "redis-cli ping || exit 1" - timeout: 5s - retries: 30 - container_name: kolab-redis - restart: on-failure - hostname: redis - image: redis - networks: - - kolab - volumes: - - ./docker/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro - webapp: - build: - context: ./docker/webapp/ - args: - GIT_REF: ${KOLAB_GIT_REF} - GIT_REMOTE: ${KOLAB_GIT_REMOTE} - container_name: kolab-webapp - restart: on-failure - image: kolab-webapp - healthcheck: - interval: 10s - test: "./artisan octane:status || exit 1" - timeout: 5s - retries: 30 - start_period: 5m - depends_on: - redis: - condition: service_healthy - networks: - kolab: - ipv4_address: 172.18.0.4 - extra_hosts: - - "${APP_DOMAIN}:172.18.0.7" - volumes: - - ./src:/src/kolabsrc.orig:ro - ports: - - "8000:8000" - meet: - build: - context: ./docker/meet/ - args: - GIT_REF: ${KOLAB_GIT_REF} - GIT_REMOTE: ${KOLAB_GIT_REMOTE} - container_name: kolab-meet - restart: on-failure - healthcheck: - interval: 10s - test: "curl --insecure -H 'X-AUTH-TOKEN: ${MEET_SERVER_TOKEN}' --fail https://${MEET_LISTENING_HOST}:12443/meetmedia/api/health || exit 1" - timeout: 5s - retries: 30 - start_period: 5m - environment: - - WEBRTC_LISTEN_IP=${MEET_WEBRTC_LISTEN_IP:?err} - - PUBLIC_DOMAIN=${MEET_PUBLIC_DOMAIN:?err} - - LISTENING_HOST=${MEET_LISTENING_HOST:?err} - - LISTENING_PORT=12443 - - TURN_SERVER=${MEET_TURN_SERVER} - - TURN_STATIC_SECRET=${COTURN_STATIC_SECRET} - - AUTH_TOKEN=${MEET_SERVER_TOKEN:?err} - - WEBHOOK_TOKEN=${MEET_WEBHOOK_TOKEN:?err} - - WEBHOOK_URL=${APP_PUBLIC_URL:?err}/api/webhooks/meet - - SSL_CERT=/etc/pki/tls/certs/meet.${APP_WEBSITE_DOMAIN:?err}.cert - - SSL_KEY=/etc/pki/tls/private/meet.${APP_WEBSITE_DOMAIN:?err}.key - network_mode: host - image: kolab-meet - volumes: - - ./meet/server:/src/meet/:ro - - ./docker/certs/meet.${APP_WEBSITE_DOMAIN}.cert:/etc/pki/tls/certs/meet.${APP_WEBSITE_DOMAIN}.cert - - ./docker/certs/meet.${APP_WEBSITE_DOMAIN}.key:/etc/pki/tls/private/meet.${APP_WEBSITE_DOMAIN}.key - minio: - container_name: kolab-minio - restart: on-failure - healthcheck: - interval: 10s - test: "mc ready local || exit 1" - timeout: 5s - retries: 30 - start_period: 5m - environment: - - MINIO_ROOT_USER=${MINIO_USER} - - MINIO_ROOT_PASSWORD=${MINIO_PASSWORD} - image: minio/minio - networks: - kolab: - ipv4_address: 172.18.0.14 - ports: - - "9000:9000" - - "9001:9001" - entrypoint: sh - command: -c 'mkdir -p /data/${MINIO_BUCKET} && minio server /data --console-address ":9001"' - volumes: - - minio:/data -networks: - kolab: - driver: bridge - ipam: - config: - - subnet: "172.18.0.0/24" -volumes: - mariadb: - minio: - roundcube: diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index ac84c795..624ed79c 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -1,10 +1,11 @@ FROM quay.io/podman/stable RUN dnf -y install git openssl COPY init.sh /init.sh +ENV ROLE=test ENV GIT_REF=master ENV GIT_REMOTE=https://git.kolab.org/source/kolab.git CMD [ "/init.sh" ] diff --git a/docker/ci/init.sh b/docker/ci/init.sh index c032d924..fb0ddf32 100755 --- a/docker/ci/init.sh +++ b/docker/ci/init.sh @@ -1,95 +1,105 @@ #!/bin/bash function check_success() { if [[ "$1" == "0" ]]; then echo "1"; else echo "0"; fi; } function checkout() { if [ ! -d "$1" ]; then git clone "$2" "$1" || exit pushd "$1" || exit git checkout "$3" || exit popd || exit fi } function pin_commit() { git ls-remote --exit-code -h "$1" "refs/heads/$2" | awk '{print $1}' } if [[ "$CACHE_REGISTRY" != "" ]]; then cat <> /etc/containers/registries.conf [[registry]] prefix = "$CACHE_REGISTRY" insecure = true location = "$CACHE_REGISTRY" EOF fi # This is the code that we are going to test checkout kolab "$GIT_REMOTE" "$GIT_REF" pushd kolab || exit # This are the pinned commits that are going to be used for the base images export KOLAB_GIT_REMOTE=https://git.kolab.org/source/kolab export KOLAB_GIT_REF=$(pin_commit "$KOLAB_GIT_REMOTE" "dev/mollekopf") export GIT_REMOTE_ROUNDCUBEMAIL=https://git.kolab.org/source/roundcubemail.git export GIT_REF_ROUNDCUBEMAIL=$(pin_commit "$GIT_REMOTE_ROUNDCUBEMAIL" "dev/kolab-1.5") export GIT_REMOTE_ROUNDCUBEMAIL_PLUGINS=https://git.kolab.org/diffusion/RPK/roundcubemail-plugins-kolab.git export GIT_REF_ROUNDCUBEMAIL_PLUGINS=$(pin_commit "$GIT_REMOTE_ROUNDCUBEMAIL_PLUGINS" "master") export GIT_REMOTE_CHWALA=https://git.kolab.org/diffusion/C/chwala.git export GIT_REF_CHWALA=$(pin_commit "$GIT_REMOTE_CHWALA" "dev/mollekopf") export GIT_REMOTE_SYNCROTON=https://git.kolab.org/diffusion/S/syncroton.git export GIT_REF_SYNCROTON=$(pin_commit "$GIT_REMOTE_SYNCROTON" "master") export GIT_REMOTE_AUTOCONF=https://git.kolab.org/diffusion/AC/autoconf.git export GIT_REF_AUTOCONF=$(pin_commit "$GIT_REMOTE_AUTOCONF" "master") export GIT_REMOTE_IRONY=https://git.kolab.org/source/iRony.git export GIT_REF_IRONY=$(pin_commit "$GIT_REMOTE_IRONY" "master") export GIT_REMOTE_FREEBUSY=https://git.kolab.org/diffusion/F/freebusy.git export GIT_REF_FREEBUSY=$(pin_commit "$GIT_REMOTE_FREEBUSY" "master") export IMAP_GIT_REMOTE=https://git.kolab.org/source/cyrus-imapd export IMAP_GIT_REF=$(pin_commit "$GIT_REMOTE_FREEBUSY" "dev/kolab-3.6") # Execute -ci/testctl build -BUILD_RESULT=$(check_success $?) -ci/testctl lint -LINT_RESULT=$(check_success $?) -ci/testctl testrun -TESTRUN_RESULT=$(check_success $?) +if [[ $ROLE == "test" ]]; then + ci/testctl build + BUILD_RESULT=$(check_success $?) + ci/testctl lint + LINT_RESULT=$(check_success $?) + ci/testctl testrun + TESTRUN_RESULT=$(check_success $?) +elif [[ $ROLE == "deploy" ]]; then + env ADMIN_PASSWORD=simple123 PUBLIC_IP=127.0.0.1 ./kolabctl configure + env ADMIN_PASSWORD=simple123 ./kolabctl deploy + DEPLOY_RESULT=$(check_success $?) + env ADMIN_PASSWORD=simple123 ./kolabctl selfcheck + SELFCHECK_RESULT=$(check_success $?) +fi + +HOST=${HOST:-$HOSTNAME} # Publish test results if [[ "$PROMETHEUS_PUSHGATEWAY" != "" ]]; then EPOCH=$(date +"%s") METRICS=$( cat < /etc/pki/tls/private/postfix.pem chown postfix:mail /etc/pki/tls/private/postfix.pem chmod 655 /etc/pki/tls/private/postfix.pem +fi chown -R postfix:mail /var/lib/postfix chown -R postfix:mail /var/spool/postfix /usr/sbin/postfix set-permissions sed -i -r \ -e "s|APP_SERVICES_DOMAIN|$APP_SERVICES_DOMAIN|g" \ -e "s|SERVICES_PORT|$SERVICES_PORT|g" \ /etc/saslauthd.conf /usr/sbin/saslauthd -m /run/saslauthd -a httpform -d & # If host mounting /var/spool/postfix, we need to delete old pid file before # starting services rm -f /var/spool/postfix/pid/master.pid /usr/libexec/postfix/aliasesdb /usr/libexec/postfix/chroot-update sed -i -r \ -e "s|LMTP_DESTINATION|$LMTP_DESTINATION|g" \ -e "s|APP_DOMAIN|$APP_DOMAIN|g" \ -e "s|MYNETWORKS|$MYNETWORKS|g" \ -e "s|AMAVIS_HOST|$AMAVIS_HOST|g" \ /etc/postfix/main.cf sed -i -r \ -e "s|MYNETWORKS|$MYNETWORKS|g" \ -e "s|AMAVIS_HOST|$AMAVIS_HOST|g" \ /etc/postfix/master.cf sed -i -r \ -e "s|SERVICES_HOST|http://$APP_SERVICES_DOMAIN:$SERVICES_PORT|g" \ /usr/libexec/postfix/kolab_policy* sed -i -r \ -e "s|DB_HOST|$DB_HOST|g" \ -e "s|DB_USERNAME|$DB_USERNAME|g" \ -e "s|DB_PASSWORD|$DB_PASSWORD|g" \ -e "s|DB_DATABASE|$DB_DATABASE|g" \ /etc/postfix/sql/* # echo "/$APP_DOMAIN/ lmtp:$LMTP_DESTINATION" >> /etc/postfix/transport # postmap /etc/postfix/transport /usr/sbin/postfix check exec /usr/sbin/postfix -c /etc/postfix start-fg diff --git a/docker/roundcube/Dockerfile b/docker/roundcube/Dockerfile index d8b20148..6878e83c 100644 --- a/docker/roundcube/Dockerfile +++ b/docker/roundcube/Dockerfile @@ -1,126 +1,126 @@ FROM apheleia/almalinux9 ENV HOME=/opt/app-root/src # Add kolab RUN rpm --import https://mirror.apheleia-it.ch/repos/Kolab:/16/key.asc && \ rpm -Uvh https://mirror.apheleia-it.ch/repos/Kolab:/16/kolab-16-for-el9.rpm # Install php modules RUN sed -i -e '/^ssl/d' /etc/yum.repos.d/kolab*.repo && \ dnf config-manager --enable kolab-16 &&\ dnf -y --setopt=install_weak_deps=False --setopt tsflags= install php-kolab php-kolabformat \ composer \ diffutils \ file \ git \ make \ unzip \ curl-minimal \ mariadb \ which \ rsync \ openssl-devel \ httpd \ patch \ php-cli \ php-common \ php-devel \ php-ldap \ php-opcache \ php-pecl-apcu \ php-mysqlnd \ php-gd \ php-fpm \ php-pear \ ImageMagick \ re2c \ npm \ wget && \ dnf clean all RUN npm install -g less less-plugin-clean-css WORKDIR ${HOME} COPY rootfs/opt/app-root/src/build.sh /opt/app-root/src/ COPY rootfs/opt/app-root/src/update.sh /opt/app-root/src/ COPY rootfs/opt/app-root/src/composer.json /opt/app-root/src/ COPY rootfs/opt/app-root/src/roundcubemail-config-templates /opt/app-root/src/roundcubemail-config-templates ARG GIT_REF_ROUNDCUBEMAIL=dev/kolab-1.5 ARG GIT_REMOTE_ROUNDCUBEMAIL=https://git.kolab.org/source/roundcubemail.git ARG GIT_REF_ROUNDCUBEMAIL_PLUGINS=master ARG GIT_REMOTE_ROUNDCUBEMAIL_PLUGINS=https://git.kolab.org/diffusion/RPK/roundcubemail-plugins-kolab.git ARG GIT_REF_CHWALA=master ARG GIT_REMOTE_CHWALA=https://git.kolab.org/diffusion/C/chwala.git ARG GIT_REF_SYNCROTON=master ARG GIT_REMOTE_SYNCROTON=https://git.kolab.org/diffusion/S/syncroton.git ARG GIT_REF_AUTOCONF=master ARG GIT_REMOTE_AUTOCONF=https://git.kolab.org/diffusion/AC/autoconf.git ARG GIT_REF_IRONY=master ARG GIT_REMOTE_IRONY=https://git.kolab.org/source/iRony.git ARG GIT_REF_FREEBUSY=master ARG GIT_REMOTE_FREEBUSY=https://git.kolab.org/diffusion/F/freebusy.git RUN /opt/app-root/src/build.sh && \ chgrp -R 0 /opt/app-root/src && \ chmod -R g=u /opt/app-root/src && \ mkdir -p /run/php-fpm && \ chmod 777 /run/php-fpm && \ mkdir -p /run/httpd && \ chmod 777 /run/httpd && \ mkdir -p /data && \ chmod 777 /data && \ chmod -R 777 /etc/php.ini /etc/httpd /var/log/httpd /var/lib/httpd /data && \ chown -R 1001:0 /opt/app-root/src /data COPY /rootfs / -RUN chmod -R 777 /etc/php.ini /etc/httpd +RUN chmod -R 777 /etc/php.ini /etc/httpd /opt/app-root/src/*.sh VOLUME /data ENV RUN_MIGRATIONS=true ENV KOLABOBJECTS_COMPAT_MODE=false # ENV FILEAPI_KOLABFILES_BASEURI= # ENV FILEAPI_WOPI_OFFICE= # ENV FILE_API_URL= # ENV FILE_API_SERVER_URL= # ENV CALENDAR_CALDAV_SERVER= # ENV TASKLIST_CALDAV_SERVER= # ENV KOLAB_ADDRESSBOOK_CARDDAV_SERVER= # ENV KOLAB_FILES_URL= # ENV KOLAB_FILES_SERVER_URL= # ENV IMAP_HOST= # ENV IMAP_PORT= # ENV IMAP_TLS= # ENV IMAP_PROXY_PROTOCOL= # ENV IMAP_ADMIN_LOGIN= # ENV IMAP_ADMIN_PASSWORD= # ENV DB_RC_USERNAME= # ENV DB_RC_PASSWORD= # ENV DB_RC_DATABASE= # ENV DB_HOST= # ENV DES_KEY= # ENV APP_DOMAIN= # ENV PROXY_WHITELIST= # ENV SUBMISSION_HOST= # ENV SUBMISSION_PORT= # ENV SUBMISSION_ENCRYPTION= # ENV DISABLED_PLUGINS= # ENV EXTRA_PLUGINS= # ENV SQL_DEBUG= # ENV MEMCACHE_DEBUG= # ENV IMAP_DEBUG= # ENV SMTP_DEBUG= # ENV DAV_DEBUG= # ENV ACTIVESYNC_DEBUG= USER 1001 EXPOSE 8080 # https://httpd.apache.org/docs/2.4/stopping.html#gracefulstop STOPSIGNAL SIGWINCH CMD [ "/opt/app-root/src/init.sh" ] diff --git a/docker/roundcube/rootfs/opt/app-root/src/checkconnections.sh b/docker/roundcube/rootfs/opt/app-root/src/checkconnections.sh index 1650e762..1277b657 100755 --- a/docker/roundcube/rootfs/opt/app-root/src/checkconnections.sh +++ b/docker/roundcube/rootfs/opt/app-root/src/checkconnections.sh @@ -1,104 +1,105 @@ #!/bin/bash set -e #Check all connections that roundcube requires. Should probably be a php script. #* imap #* chwala # # External access # file_api_url # kolab_files_url # kolab_addressbook_carddav_url # calendar_caldav_url -IMAP_HOST=$(./getconfig.php default_host) -IMAP_PORT=$(./getconfig.php default_port) -echo "IMAP : $IMAP_HOST:$IMAP_PORT" -echo "a01 LOGOUT" | telnet $IMAP_HOST $IMAP_PORT | grep "Connected to imap" -echo "IMAP is OK" - -MAIL_HOST=$(./getconfig.php smtp_server) -MAIL_PORT=$(./getconfig.php smtp_port) -echo "SMTP : $MAIL_HOST:$MAIL_PORT" -echo "quit" | telnet $MAIL_HOST $MAIL_PORT | grep "Connected to postfix" -echo "SMTP is OK" +# FIXME the telnet checks no longer seem to work (fails with remote connection closed on LOGOUT...) +#IMAP_HOST=$(./getconfig.php default_host) +#IMAP_PORT=$(./getconfig.php default_port) +#echo "IMAP : $IMAP_HOST:$IMAP_PORT" +#echo "a01 LOGOUT" | telnet $IMAP_HOST $IMAP_PORT | grep "Connected to imap" +#echo "IMAP is OK" + +#MAIL_HOST=$(./getconfig.php smtp_server) +#MAIL_PORT=$(./getconfig.php smtp_port) +#echo "SMTP : $MAIL_HOST:$MAIL_PORT" +#echo "quit" | telnet $MAIL_HOST $MAIL_PORT | grep "Connected to postfix" +#echo "SMTP is OK" #FIXME in newer mariadb-shell variants there is --dsn, but in older mysql client version there doesn't seem to be something like it # MYSQL_DSN=$(./getconfig.php db_dsnw) # echo "Mysql : $IMAP_HOST:$IMAP_PORT" # mysql --batch 'describe table foobar?' # echo "IMAP is OK" URL=$(./getconfig.php fileapi_wopi_office) echo "WOPI office: $URL" curl -sD /dev/stderr "$URL/hosting/discovery" -k | grep "" echo "WOPI office is OK" URL=$(./getconfig.php kolab_files_server_url) if [[ $URL == "" ]]; then URL=$(./getconfig.php kolab_files_url) fi echo "Chwala url: $URL" curl -sD /dev/stderr "$URL/api/" -k | grep "Invalid session" echo "Chwala is OK" if [[ "$(./getconfig.php fileapi_backend)" == "kolabfiles" ]]; then URL=$(./getconfig.php fileapi_kolabfiles_baseuri) echo "Kolabfiles $URL" # We expect a 401 if the api call exists in this location (even unauthenticated). curl -s -o /dev/null -w "%{http_code}" "${URL}/v4/fs" | grep "401" echo "Kolabfiles API is OK" fi if [[ "$(./getconfig.php kolab_addressbook_driver)" == "carddav" ]]; then # $config['kolab_addressbook_carddav_server'] = "https://" . ($_SERVER["HTTP_HOST"] ?? null) . "/dav"; URL=$(./getconfig.php kolab_addressbook_carddav_server) echo "Carddav $URL" curl -sD /dev/stderr -H "Content-Type: application/xml" -X PROPFIND -H "Depth: infinity" --data '' $URL -k | grep "405 Method Not Allowed" echo "Carddav is OK" #FIXME this is for external access, so we can't test this here #FIXME username/host/addressbook substitution # $config['kolab_addressbook_carddav_url'] = 'http://%h/dav/addressbooks/%u/%i'; # URL=$(./getconfig.php kolab_addressbook_carddav_url) # echo "Carddav $URL" # curl -sD /dev/stderr -H "Content-Type: application/xml" -X PROPFIND -H "Depth: infinity" --data '' $URL -k # echo "Carddav is OK" fi if [[ "$(./getconfig.php calendar_driver)" == "caldav" ]]; then #$config['calendar_caldav_server'] = "https://" . ($_SERVER["HTTP_HOST"] ?? null) . "/dav"; URL=$(./getconfig.php calendar_caldav_server) echo "Caldav $URL" curl -sD /dev/stderr -H "Content-Type: application/xml" -X PROPFIND -H "Depth: infinity" --data '' $URL -k | grep "405 Method Not Allowed" echo "Caldav is OK" #FIXME this is for external access, so we can't test this here #$config['calendar_caldav_url'] = 'http://%h/dav/calendars/%u/%i'; fi if [[ "$(./getconfig.php tasklist_driver)" == "caldav" ]]; then #$config['calendar_caldav_server'] = "https://" . ($_SERVER["HTTP_HOST"] ?? null) . "/dav"; URL=$(./getconfig.php tasklist_caldav_server) echo "Tasklist caldav $URL" curl -sD /dev/stderr -H "Content-Type: application/xml" -X PROPFIND -H "Depth: infinity" --data '' $URL -k | grep "405 Method Not Allowed" echo "Tasklist caldav is OK" fi if [[ "$(./getconfig.php calendar_driver)" == "caldav" ]]; then #$config['calendar_caldav_server'] = "https://" . ($_SERVER["HTTP_HOST"] ?? null) . "/dav"; URL=$(./getconfig.php calendar_caldav_server) echo "Caldav $URL" curl -sD /dev/stderr -H "Content-Type: application/xml" -X PROPFIND -H "Depth: infinity" --data '' $URL -k | grep "405 Method Not Allowed" echo "Caldav is OK" #FIXME this is for external access, so we can't test this here #$config['calendar_caldav_url'] = 'http://%h/dav/calendars/%u/%i'; fi echo "All checks complete" diff --git a/kolabctl b/kolabctl index d087b55f..d0df346f 100755 --- a/kolabctl +++ b/kolabctl @@ -1,101 +1,402 @@ #!/bin/bash set -e CONFIG=${CONFIG:-"config.prod"} -HOST=${HOST:-"kolab.local"} -BRANCH=${BRANCH:-"master"} +export HOST=${HOST:-"kolab.local"} +BRANCH=${BRANCH:-"dev/mollekopf"} +OPENEXCHANGERATES_API_KEY=${OPENEXCHANGERATES_API_KEY} +FIREBASE_API_KEY=${FIREBASE_API_KEY} +PUBLIC_IP=${PUBLIC_IP} + +export CERTS_PATH=./docker/certs + +export POD=kolab-prod +export IMAP_SPOOL_STORAGE="--mount=type=volume,src=$POD-imap-spool,destination=/var/spool/imap,U=true" +export IMAP_LIB_STORAGE="--mount=type=volume,src=$POD-imap-lib,destination=/var/lib/imap,U=true" +export POSTFIX_SPOOL_STORAGE="--mount=type=volume,src=$POD-postfix-spool,destination=/var/spool/imap,U=true" +export POSTFIX_LIB_STORAGE="--mount=type=volume,src=$POD-postfix-lib,destination=/var/lib/imap,U=true" +export MARIADB_STORAGE="--mount=type=volume,src=$POD-mariadb-data,destination=/var/lib/mysql,U=true" +export REDIS_STORAGE="--mount=type=volume,src=$POD-redis-data,destination=/var/lib/redis,U=true" +export MINIO_STORAGE="--mount=type=volume,src=$POD-minio-data,destination=/data,U=true" + +export PODMAN_IGNORE_CGROUPSV1_WARNING=true + +source bin/podman_shared + + +__export_env() { + source src/.env + export APP_WEBSITE_DOMAIN + export APP_DOMAIN + export DB_HOST + export IMAP_HOST + export IMAP_PORT + export IMAP_ADMIN_LOGIN + export IMAP_ADMIN_PASSWORD + export MAIL_HOST + export MAIL_PORT + export IMAP_DEBUG + export FILEAPI_WOPI_OFFICE + export CALENDAR_CALDAV_SERVER + export KOLAB_ADDRESSBOOK_CARDDAV_SERVER + export DB_ROOT_PASSWORD + export DB_USERNAME + export DB_PASSWORD + export DB_DATABASE + export MINIO_USER + export MINIO_PASSWORD + export PASSPORT_PRIVATE_KEY + export PASSPORT_PUBLIC_KEY + export DES_KEY + export MEET_SERVER_TOKEN + export MEET_WEBHOOK_TOKEN + export KOLAB_SSL_CERTIFICATE + export KOLAB_SSL_CERTIFICATE_FULLCHAIN + export KOLAB_SSL_CERTIFICATE_KEY +} + +kolab__configure() { + if [[ "$1" == "--force" ]]; then + rm src/.env + fi + + # Generate the .env once with all the necessary secrets + if [[ -f src/.env ]]; then + echo "src/.env already exists, not regenerating" + return + fi + + cp "$CONFIG/src/.env" src/.env + + if [[ -z $ADMIN_PASSWORD ]]; then + echo "Please enter your new admin password for the admin@$HOST user:" + read -r ADMIN_PASSWORD + fi + + if [[ -z $PUBLIC_IP ]]; then + PUBLIC_IP=$(ip -o route get to 8.8.8.8 | sed -n 's/.*src \([0-9.]\+\).*/\1/p') + fi + + # Generate random secrets + if ! grep -q "COTURN_STATIC_SECRET" src/.env; then + COTURN_STATIC_SECRET=$(openssl rand -hex 32); + echo "COTURN_STATIC_SECRET=${COTURN_STATIC_SECRET}" >> src/.env + fi + + if ! grep -q "MEET_WEBHOOK_TOKEN" src/.env; then + MEET_WEBHOOK_TOKEN=$(openssl rand -hex 32); + echo "MEET_WEBHOOK_TOKEN=${MEET_WEBHOOK_TOKEN}" >> src/.env + fi + + if ! grep -q "MEET_SERVER_TOKEN" src/.env; then + MEET_SERVER_TOKEN=$(openssl rand -hex 32); + echo "MEET_SERVER_TOKEN=${MEET_SERVER_TOKEN}" >> src/.env + fi + + if ! grep -q "APP_KEY=base64:" src/.env; then + APP_KEY=$(openssl rand -base64 32); + echo "APP_KEY=base64:${APP_KEY}" >> src/.env + fi + + if ! grep -q "PASSPORT_PROXY_OAUTH_CLIENT_ID=" src/.env; then + PASSPORT_PROXY_OAUTH_CLIENT_ID=$(uuidgen); + echo "PASSPORT_PROXY_OAUTH_CLIENT_ID=${PASSPORT_PROXY_OAUTH_CLIENT_ID}" >> src/.env + fi + + if ! grep -q "PASSPORT_PROXY_OAUTH_CLIENT_SECRET=" src/.env; then + PASSPORT_PROXY_OAUTH_CLIENT_SECRET=$(openssl rand -base64 32); + echo "PASSPORT_PROXY_OAUTH_CLIENT_SECRET=${PASSPORT_PROXY_OAUTH_CLIENT_SECRET}" >> src/.env + fi + + if ! grep -q "PASSPORT_PUBLIC_KEY=|PASSPORT_PRIVATE_KEY=" src/.env; then + PASSPORT_PRIVATE_KEY=$(openssl genrsa 4096); + echo "PASSPORT_PRIVATE_KEY=\"${PASSPORT_PRIVATE_KEY}\"" >> src/.env + + PASSPORT_PUBLIC_KEY=$(echo "$PASSPORT_PRIVATE_KEY" | openssl rsa -pubout 2>/dev/null) + echo "PASSPORT_PUBLIC_KEY=\"${PASSPORT_PUBLIC_KEY}\"" >> src/.env + fi + + if ! grep -q "DES_KEY=" src/.env; then + DES_KEY=$(openssl rand -base64 24); + echo "DES_KEY=${DES_KEY}" >> src/.env + fi + + # Customize configuration + sed -i \ + -e "s/{{ host }}/${HOST}/g" \ + -e "s/{{ openexchangerates_api_key }}/${OPENEXCHANGERATES_API_KEY}/g" \ + -e "s/{{ firebase_api_key }}/${FIREBASE_API_KEY}/g" \ + -e "s/{{ public_ip }}/${PUBLIC_IP}/g" \ + -e "s/{{ admin_password }}/${ADMIN_PASSWORD}/g" \ + src/.env + + if [ -f /etc/letsencrypt/live/${HOST}/cert.pem ]; then + echo "Using the available letsencrypt certificate for ${HOST}" + cat >> src/.env << EOF +KOLAB_SSL_CERTIFICATE=/etc/letsencrypt/live/${HOST}/cert.pem +KOLAB_SSL_CERTIFICATE_FULLCHAIN=/etc/letsencrypt/live/${HOST}/fullchain.pem +KOLAB_SSL_CERTIFICATE_KEY=/etc/letsencrypt/live/${HOST}/privkey.pem +PROXY_SSL_CERTIFICATE=/etc/letsencrypt/live/${HOST}/fullchain.pem +PROXY_SSL_CERTIFICATE_KEY=/etc/letsencrypt/live/${HOST}/privkey.pem +EOF + fi +} kolab__deploy() { - echo "Please enter your new admin password for the admin@$HOST user:" - read -r ADMIN_PASSWORD + if [[ -z $ADMIN_PASSWORD ]]; then + echo "Please enter your new admin password for the admin@$HOST user:" + read -r ADMIN_PASSWORD + fi echo "Deploying $CONFIG from branch $BRANCH on $HOST" - command env KOLAB_GIT_REF=$BRANCH HOST=$HOST ADMIN_PASSWORD="$ADMIN_PASSWORD" bin/configure.sh "$CONFIG" - command env ADMIN_PASSWORD="$ADMIN_PASSWORD" bin/deploy.sh + + if [[ ! -f src/.env ]]; then + echo "Missing src/.env file, run 'kolabctl configure' to generate" + exit 1 + fi + + if [[ "$1" == "--reset" ]]; then + kolab__reset --force + fi + + __export_env + + podman volume create $POD-imap-spool --ignore -l=kolab + podman volume create $POD-imap-lib --ignore -l=kolab + podman volume create $POD-postfix-spool --ignore -l=kolab + podman volume create $POD-postfix-lib --ignore -l=kolab + podman volume create $POD-mariadb-data --ignore -l=kolab + podman volume create $POD-redis-data --ignore -l=kolab + podman volume create $POD-minio-data --ignore -l=kolab + + kolab__build + + # Create the pod first + $PODMAN pod create \ + --replace \ + --add-host=$HOST:127.0.0.1 \ + --publish "443:6443" \ + --publish "465:6465" \ + --publish "587:6587" \ + --publish "143:6143" \ + --publish "993:6993" \ + --publish "44444:44444/udp" \ + --publish "44444:44444/tcp" \ + --name $POD + + podman__run_mariadb + podman__run_redis + + podman__healthcheck $POD-mariadb $POD-redis + + # Make imap available to the webapp seeder, but don't expect it to be healthy until it can authenticate against the webapp + podman__run_imap + + podman__run_webapp src/.env $CONFIG + podman__healthcheck $POD-webapp + + podman__healthcheck $POD-imap + + # Ensure all commands are processed + echo "Flushing work queue" + $PODMAN exec -ti $POD-webapp ./artisan queue:work --stop-when-empty + + if [[ -n $ADMIN_PASSWORD ]]; then + podman exec $POD-webapp ./artisan user:password "admin@$APP_DOMAIN" "$ADMIN_PASSWORD" + fi + + podman__run_minio + podman__healthcheck $POD-minio + + podman__run_meet + + podman__run_roundcube + podman__run_postfix + podman__run_amavis + podman__run_collabora + podman__run_proxy +} + +kolab__reset() { + if [[ "$1" == "--force" ]]; then + REPLY="y" + else + read -p "Are you sure? This will delete the pod including all data. Type y to confirm." -n 1 -r + echo + fi + if [[ "$REPLY" =~ ^[Yy]$ ]]; + then + podman pod rm --force $POD + volumes=($(podman volume ls -f name=$POD | awk '{if (NR > 1) print $2}')) + for v in "${volumes[@]}" + do + podman volume rm --force $v + done + fi } kolab__start() { - command bin/start.sh + podman pod start $POD } kolab__stop() { - command bin/stop.sh + podman pod stop $POD } kolab__update() { - command bin/update.sh + kolab__stop + + podman pull quay.io/sclorg/mariadb-105-c9s + podman pull minio/minio:latest + podman pull almalinux:9 + + kolab__build + + kolab__start } kolab__backup() { - command bin/backup.sh + backup_path="$(pwd)/backup/" + mkdir -p "$backup_path" + + echo "Stopping containers" + kolab__stop + + echo "Backing up volumes" + volumes=($(podman volume ls -f name=$POD | awk '{if (NR > 1) print $2}')) + for v in "${volumes[@]}" + do + podman export -o="$backup_path/$v.tar" + done + + echo "Restarting containers" + kolab__start } kolab__restore() { - command bin/restore.sh + backup_path="$(pwd)/backup/" + + echo "Stopping containers" + kolab__stop + + # We currently expect the volumes to exist. + # We could alternatively create volumes form existing tar files + # for f in backup/*.tar; do + # echo "$(basename $f .tar)" ; + # done + + echo "Restoring volumes" + volumes=($(podman volume ls -f name=$POD | awk '{if (NR > 1) print $2}')) + for v in "${volumes[@]}" + do + podman import $v "$backup_path/$v.tar" + done + echo "Restarting containers" + kolab__start } kolab__selfcheck() { - command bin/selfcheck.sh + set -e + set -x + + APP_DOMAIN=$(grep APP_DOMAIN src/.env | tail -n1 | sed "s/APP_DOMAIN=//") + if [ -z "$ADMIN_PASSWORD" ]; then + ADMIN_PASSWORD="simple123" + fi + if [ -z "$ADMIN_USER" ]; then + ADMIN_USER="admin@$APP_DOMAIN" + fi + + # We skip mollie and openexchange + podman exec $POD-webapp ./artisan status:health --check DB --check Redis --check IMAP --check Roundcube --check Meet --check DAV + podman exec $POD-postfix testsaslauthd -u "$ADMIN_USER" -p "$ADMIN_PASSWORD" + podman exec $POD-imap testsaslauthd -u "$ADMIN_USER" -p "$ADMIN_PASSWORD" + + # podman run -ti --rm utils ./mailtransporttest.py --sender-username "$ADMIN_USER" --sender-password "$ADMIN_PASSWORD" --sender-host "127.0.0.1" --recipient-username "$ADMIN_USER" --recipient-password "$ADMIN_PASSWORD" --recipient-host "127.0.0.1" --recipient-port "11143" + + # podman run -ti --rm utils ./kolabendpointtester.py --verbose --host "$APP_DOMAIN" --dav "https://$APP_DOMAIN/dav/" --imap "$APP_DOMAIN" --activesync "$APP_DOMAIN" --user "$ADMIN_USER" --password "$ADMIN_PASSWORD" + + echo "All tests have passed!" } kolab__ps() { - command docker compose ps + command podman pod $POD ps } kolab__exec() { - command docker compose exec -ti $@ + container=$1 + shift + command podman exec -ti $POD-$container $@ } kolab__run() { - command docker compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.build.yml run --rm -ti $@ + __export_env + podman__run_$1 } kolab__build() { - command docker compose -f docker-compose.yml -f docker-compose.override.yml -f docker-compose.build.yml build $@ + if [[ $1 != "" ]]; then + podman__build_$1 + else + podman__build_base + podman__build_webapp + podman__build_meet + + podman__build docker/imap kolab-imap + podman__build docker/mariadb mariadb + podman__build docker/redis redis + podman__build docker/proxy kolab-proxy + podman__build_roundcube + + podman__build docker/utils utils + podman__build docker/postfix kolab-postfix + podman__build docker/amavis kolab-amavis + podman__build docker/collabora kolab-collabora --build-arg=REPOSITORY="https://www.collaboraoffice.com/repos/CollaboraOnline/23.05-CODE/CODE-rpm/" + env CERT_DIR=docker/certs APP_DOMAIN=$HOST bin/regen-certs + fi } kolab__cyradm() { - # command docker compose exec -ti imap cyradm --auth PLAIN -u admin@kolab.local -w simple123 --port 11143 localhost + # command podman exec -ti $POD-imap cyradm --auth PLAIN -u admin@kolab.local -w simple123 --port 11143 localhost if [[ "$@" ]]; then - command docker compose exec -ti imap echo "$@" | cyradm --auth PLAIN -u $(grep IMAP_ADMIN_LOGIN .env | cut -d '=' -f 2 ) -w $(grep IMAP_ADMIN_PASSWORD .env | cut -d '=' -f 2 ) --port 11143 localhost + command podman exec -ti $POD-imap echo "$@" | cyradm --auth PLAIN -u $(grep IMAP_ADMIN_LOGIN src/.env | cut -d '=' -f 2 ) -w $(grep IMAP_ADMIN_PASSWORD src/.env | cut -d '=' -f 2 ) --port 11143 localhost else - command docker compose exec -ti imap cyradm --auth PLAIN -u $(grep IMAP_ADMIN_LOGIN .env | cut -d '=' -f 2 ) -w $(grep IMAP_ADMIN_PASSWORD .env | cut -d '=' -f 2 ) --port 11143 localhost + command podman exec -ti $POD-imap cyradm --auth PLAIN -u $(grep IMAP_ADMIN_LOGIN src/.env | cut -d '=' -f 2 ) -w $(grep IMAP_ADMIN_PASSWORD src/.env | cut -d '=' -f 2 ) --port 11143 localhost fi } kolab__shell() { - command docker compose exec -ti $1 /bin/bash + kolab__exec $1 /bin/bash } kolab__logs() { - command docker compose logs -f $1 + command podman logs -f $POD-$1 } - kolab__help() { cat </dev/null 2>&1; then "kolab__$cmdname" "${@:1}" else echo "Function $cmdname not recognized" >&2 kolab__help exit 1 fi