Docker/cron/docassemble-cron-daily.sh

Rama master · réplica de lectura del repositorio original.

#! /bin/bash

export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}"
export DA_DEFAULT_LOCAL="local3.14"

export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate"
source "${DA_ACTIVATE}"
export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}"
export CONTAINERROLE=":${CONTAINERROLE:-all}:"
source /dev/stdin < <(su -c "source \"${DA_ACTIVATE}\" && python -m docassemble.base.read_config \"${DA_CONFIG_FILE}\"" www-data)
export LOGDIRECTORY="${LOGDIRECTORY:-${DA_ROOT}/log}"

set -- $LOCALE
export LANG=$1

# Renew Let's Encrypt certificate; back up the certificates; copy certificates to exim
if [[ $CONTAINERROLE =~ .*:(all):.* ]] && [ "${USEHTTPS:-false}" == "true" ] && [ "${USELETSENCRYPT:-false}" == "true" ] && [ -f /etc/letsencrypt/da_using_lets_encrypt ]; then
    export USE_PYTHON_3=1
    if [ "${DAWEBSERVER:-nginx}" = "apache" ]; then
	certbot renew --apache --cert-name "${DAHOSTNAME}" --pre-hook "supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 stop apache2" --post-hook "/etc/init.d/apache2 stop; supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 start apache2"
    fi
    if [ "${DAWEBSERVER:-nginx}" = "nginx" ]; then
	certbot renew --nginx --cert-name "${DAHOSTNAME}" --pre-hook "supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 stop nginx" --post-hook "nginx -s stop &> /dev/null; supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 start nginx"
    fi
    cd /
    rm -f /tmp/letsencrypt.tar.gz
    tar -zcf /tmp/letsencrypt.tar.gz etc/letsencrypt
    if [ "${S3ENABLE:-false}" == "true" ]; then
	s4cmd -f put /tmp/letsencrypt.tar.gz "s3://${S3BUCKET}/letsencrypt.tar.gz"
	if [ "${DAWEBSERVER:-nginx}" = "apache" ]; then
	    s4cmd dsync "/etc/apache2/sites-available" "s3://${S3BUCKET}/apache"
	fi
    elif [ "${AZUREENABLE:-false}" == "true" ]; then
	az storage blob upload --no-progress --overwrite true --only-show-errors --output none --container-name "${AZURECONTAINER}" -f /tmp/letsencrypt.tar.gz -n "letsencrypt.tar.gz"
	if [ "${DAWEBSERVER:-nginx}" = "apache" ]; then
	    for the_file in $( find /etc/apache2/sites-available/ -type f ); do
		target_file=`basename ${the_file}`
		az storage blob upload --no-progress --overwrite true --only-show-errors --output none --container-name "${AZURECONTAINER}" -f "${the_file}" -n "apache/${target_file}"
	    done
	fi
    else
	cp /tmp/letsencrypt.tar.gz "${DA_ROOT}/backup/letsencrypt.tar.gz"
    fi
    rm -f /tmp/letsencrypt.tar.gz
    if [ "$ENABLEEMAILSERVER" == "true" ]; then
	if [ ! -f /etc/ssl/docassemble/exim.crt ] && [ ! -f /etc/ssl/docassemble/exim.key ]; then
	    cp "/etc/letsencrypt/live/${DAHOSTNAME}/fullchain.pem" /etc/exim4/exim.crt
	    cp "/etc/letsencrypt/live/${DAHOSTNAME}/privkey.pem" /etc/exim4/exim.key
	    chown root:Debian-exim /etc/exim4/exim.crt
	    chown root:Debian-exim /etc/exim4/exim.key
	    chmod 640 /etc/exim4/exim.crt
	    chmod 640 /etc/exim4/exim.key
	    supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 stop exim4
	    supervisorctl ${DASUPERVISOROPTS}--serverurl http://localhost:9001 start exim4
	fi
    fi
fi

# Run cron jobs in docassemble interviews
if [[ $CONTAINERROLE =~ .*:(all|cron):.* ]]; then
    "${DA_ROOT}/webapp/run-cron.sh" cron_daily
fi

# Backup log files. If not using cloud storage, also backup configuration and uploaded files.
if [ "${S3ENABLE:-false}" == "true" ]; then
    if [[ $CONTAINERROLE =~ .*:(all|log):.* ]]; then
        s4cmd dsync "${DA_ROOT}/log" "s3://${S3BUCKET}/log"
    fi
elif [ "${AZUREENABLE:-false}" == "true" ]; then
    if [[ $CONTAINERROLE =~ .*:(all|log):.* ]]; then
	let LOGDIRECTORYLENGTH=${#LOGDIRECTORY}+2
	for the_file in $(find "${LOGDIRECTORY}" -type f | cut -c ${LOGDIRECTORYLENGTH}-); do
	    az storage blob upload --no-progress --overwrite true --only-show-errors --output none --container-name "${AZURECONTAINER}" -f "${LOGDIRECTORY}/${the_file}" -n "log/${the_file}"
	done
    fi
elif [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then
    if [[ $CONTAINERROLE =~ .*:(all|web|celery|log|cron):.* ]]; then
	rsync -auq --delete "${LOGDIRECTORY}" "${DA_ROOT}/backup/"
    fi
    if [[ $CONTAINERROLE =~ .*:(all|cron):.* ]]; then
	rm -f "${DA_ROOT}/backup/config.yml"
	cp "${DA_CONFIG_FILE}" "${DA_ROOT}/backup/config.yml"
	rsync -auq --delete "${DA_ROOT}/files" "${DA_ROOT}/backup/"
    fi
fi

# if using a single server, back up web server logs so that they can be restored.
if [[ $CONTAINERROLE =~ .*:(all):.* ]]; then
    if [ "${S3ENABLE:-false}" == "true" ]; then
	if [ "${DAWEBSERVER:-nginx}" = "apache" ]; then
	    s4cmd dsync "/var/log/apache2" "s3://${S3BUCKET}/apachelogs"
	fi
	if [ "${DAWEBSERVER:-nginx}" = "nginx" ]; then
	    s4cmd dsync "/var/log/nginx" "s3://${S3BUCKET}/nginxlogs"
	fi
    elif [ "${AZUREENABLE:-false}" == "true" ]; then
	if [ "${DAWEBSERVER:-nginx}" = "apache" ]; then
	    for the_file in $(find /var/log/apache2 -type f | cut -c 18-); do
		az storage blob upload --no-progress --overwrite true --only-show-errors --output none --container-name "${AZURECONTAINER}" -f "/var/log/apache2/${the_file}" -n "apachelogs/${the_file}"
	    done
	fi
	if [ "${DAWEBSERVER:-nginx}" = "nginx" ]; then
	    for the_file in $(find /var/log/nginx -type f | cut -c 16-); do
		az storage blob upload --no-progress --overwrite true --only-show-errors --output none --container-name "${AZURECONTAINER}" -f "/var/log/nginx/${the_file}" -n "nginxlogs/${the_file}"
	    done
	fi
    elif [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then
	if [ "${DAWEBSERVER:-nginx}" = "apache" ]; then
	    mkdir -p "${DA_ROOT}/backup/apachelogs"
	    rsync -auq --delete /var/log/apache2/ "${DA_ROOT}/backup/apachelogs/"
	fi
	if [ "${DAWEBSERVER:-nginx}" = "nginx" ]; then
	    mkdir -p "${DA_ROOT}/backup/nginxlogs"
	    rsync -auq /var/log/nginx/ "${DA_ROOT}/backup/nginxlogs/"
	fi
    fi
fi

# If this container is running a SQL server, back up PostgreSQL
if [[ $CONTAINERROLE =~ .*:(all|sql):.* ]]; then
    PGBACKUPDIR=`mktemp -d`
    chown postgres:postgres "${PGBACKUPDIR}"
    su postgres -c 'psql -Atc "SELECT datname FROM pg_database" postgres' | grep -v -e template -e postgres | awk -v backupdir="$PGBACKUPDIR" '{print "cd /tmp; su postgres -c \"pg_dump -F c -f " backupdir "/" $1 " " $1 "\""}' | bash
    if [ "${S3ENABLE:-false}" == "true" ]; then
	s4cmd dsync "$PGBACKUPDIR" "s3://${S3BUCKET}/postgres"
    elif [ "${AZUREENABLE:-false}" == "true" ]; then
	for the_file in $( find "$PGBACKUPDIR/" -type f ); do
	    target_file=`basename ${the_file}`
	    az storage blob upload --no-progress --overwrite true --only-show-errors --output none --container-name "${AZURECONTAINER}" -f "${the_file}" -n "postgres/${target_file}"
	done
    elif [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then
	rsync -auq "$PGBACKUPDIR/" "${DA_ROOT}/backup/postgres"
    fi
fi

# If this container is running a Redis server, back up Redis
if [[ $CONTAINERROLE =~ .*:(all|redis):.* ]] && [ -f /var/lib/redis/dump.rdb ]; then
    if [ "${S3ENABLE:-false}" == "true" ]; then
	s4cmd -f put "/var/lib/redis/dump.rdb" "s3://${S3BUCKET}/redis.rdb"
    elif [ "${AZUREENABLE:-false}" == "true" ]; then
	az storage blob upload --no-progress --overwrite true --only-show-errors --output none --container-name "${AZURECONTAINER}" -f "/var/lib/redis/dump.rdb" -n "redis.rdb"
    elif [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then
	cp /var/lib/redis/dump.rdb "${DA_ROOT}/backup/redis.rdb"
    fi
fi

# Create a rolling backup directory for today
if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ] && [ "${DABACKUPDAYS}" != "0" ]; then
    MONTHDAY=$(date +%m-%d)
    BACKUPDIR="${DA_ROOT}/backup/$MONTHDAY"
    rm -rf $BACKUPDIR
    mkdir -p $BACKUPDIR

    # Copy Let's Encrypt certificates to rolling backup
    if [[ $CONTAINERROLE =~ .*:(all):.* ]] && [ "${USEHTTPS:-false}" == "true" ] && [ "${USELETSENCRYPT:-false}" == "true" ] && [ -f /etc/letsencrypt/da_using_lets_encrypt ]; then
	cd /
	rm -f /tmp/letsencrypt.tar.gz
	tar -zcf /tmp/letsencrypt.tar.gz etc/letsencrypt
	cp /tmp/letsencrypt.tar.gz "${BACKUPDIR}/"
	rm -f /tmp/letsencrypt.tar.gz
	cd /tmp
    fi

    # Copy uploaded files, configuration, and logs to the rolling backup
    if [[ $CONTAINERROLE =~ .*:(all|web|celery|log|cron):.* ]]; then
	if [ "${BACKUPFILESTORAGE:-true}" == "true" ] && [ "${S3ENABLE:-false}" == "false" ] && [ "${AZUREENABLE:-false}" == "false" ]; then
	    rsync -auq "${DA_ROOT}/files" "${BACKUPDIR}/"
	fi
	rsync -auq "${DA_ROOT}/config" "${BACKUPDIR}/"
	rsync -auq "${LOGDIRECTORY}" "${BACKUPDIR}/"
    fi

    # If this container is a redis server, copy the Redis database to the rolling backup
    if [[ $CONTAINERROLE =~ .*:(all|redis):.* ]]; then
	if [ -f /var/lib/redis/dump.rdb ]; then
	    cp /var/lib/redis/dump.rdb "${BACKUPDIR}/redis.rdb"
	fi
    elif [[ $CONTAINERROLE =~ .*:cron:.* ]] && [ "${REDIS:-redis://localhost}" != "redis://localhost" ]; then
	$REDISCLI --rdb "${BACKUPDIR}/redis.rdb" &> /dev/null
    fi

    # If using PostgreSQL, copy the SQL database to the rolling backup
    if [ "$DBTYPE" == "postgresql" ]; then
	if [[ $CONTAINERROLE =~ .*:(all|sql):.* ]]; then
	    rsync -auq "$PGBACKUPDIR/" "${BACKUPDIR}/postgres"
	elif [[ $CONTAINERROLE =~ .*:cron:.* ]] && [ "${DBHOST:-localhost}" != "localhost" ] && [ "${DBBACKUP:-true}" == "true" ]; then
	    PGBACKUPDIR=`mktemp -d`
	    export PGPASSWORD="${DBPASSWORD:-abc123}"
	    if [ "$DBPORT" != "" ]; then
		export PGPORT="${DBPORT}"
	    fi
	    if [ "$DBSSLMODE" != "" ]; then
		export PGSSLMODE="${DBSSLMODE}"
	    fi
	    if [ "$DBSSLCERT" != "" ]; then
		export PGSSLCERT="/etc/ssl/docassemble/${DBSSLCERT}"
	    fi
	    if [ "$DBSSLKEY" != "" ]; then
		export PGSSLKEY="/etc/ssl/docassemble/${DBSSLKEY}"
	    fi
	    if [ "$DBSSLROOTCERT" != "" ]; then
		export PGSSLROOTCERT="/etc/ssl/docassemble/${DBSSLROOTCERT}"
	    fi

	    pg_dump -F c -f "${PGBACKUPDIR}/${DBNAME}" -h "${DBHOST}" -U "${DBUSER:-docassemble}" -w -p "${DBPORT:-5432}" "${DBNAME}"
	    unset PGPASSWORD
	    unset PGSSLMODE
	    unset PGSSLCERT
	    unset PGSSLKEY
	    unset PGSSLROOTCERT
	    rsync -auq "$PGBACKUPDIR/" "${BACKUPDIR}/postgres"
	    rm -rf "${PGBACKUPDIR}"
	fi
    fi

    # Delete old rolling backup directories. If syncing to Azure, we do this below.
    if [ "${AZUREENABLE:-false}" == "false" ]; then
	rm -rf `find "${DA_ROOT}/backup" -maxdepth 1 -path '*[0-9][0-9]-[0-9][0-9]' -a -type 'd' -a -mtime +${DABACKUPDAYS:-14} -print`
    fi

    # If using cloud storage, copy the rolling backup to the cloud
    if [ "${S3ENABLE:-false}" == "true" ]; then
	if [[ $CONTAINERROLE =~ .*:(all):.* ]]; then
	    BACKUPTARGET="s3://${S3BUCKET}/backup"
	else
	    if [ "${EC2:-false}" == "true" ]; then
		export LOCAL_HOSTNAME=`curl http://169.254.169.254/latest/meta-data/local-hostname`
	    else
		export LOCAL_HOSTNAME=`hostname --fqdn`
	    fi
	    BACKUPTARGET="s3://${S3BUCKET}/backup/${LOCAL_HOSTNAME}"
	fi
	s4cmd --delete-removed dsync "${DA_ROOT}/backup" "${BACKUPTARGET}"
    elif [ "${AZUREENABLE:-false}" == "true" ]; then
	if [[ $CONTAINERROLE =~ .*:(all):.* ]]; then
	    BACKUPTARGET="backup"
	else
	    if [ "${EC2:-false}" == "true" ]; then
		export LOCAL_HOSTNAME=`curl http://169.254.169.254/latest/meta-data/local-hostname`
	    else
		export LOCAL_HOSTNAME=`hostname --fqdn`
	    fi
	    BACKUPTARGET="backup/${LOCAL_HOSTNAME}"
	fi
	CURRENTBACKUP="${DA_ROOT}/backup"
	let CUTPOINT=${#CURRENTBACKUP}+2
	for the_file in $( find "${CURRENTBACKUP}/" -type f | cut -c ${CUTPOINT}- ); do
	    az storage blob upload --no-progress --overwrite true --only-show-errors --output none --container-name "${AZURECONTAINER}" -f "${CURRENTBACKUP}/${the_file}" -n "${BACKUPTARGET}/${the_file}"
	done
	for the_dir in $( find "${CURRENTBACKUP}" -maxdepth 1 -path '*[0-9][0-9]-[0-9][0-9]' -a -type 'd' -a -mtime +${DABACKUPDAYS:-14} -print | cut -c ${CUTPOINT}- ); do
	    for the_file in $( find "${CURRENTBACKUP}/${the_dir}" -type f | cut -c ${CUTPOINT}- ); do
		az storage blob delete --only-show-errors --output none --container-name "${AZURECONTAINER}" -n "${BACKUPTARGET}/${the_file}"
	    done
	    rm -rf "${CURRENTBACKUP}/$the_dir"
	done
    fi
fi

if [[ $CONTAINERROLE =~ .*:(all|sql):.* ]]; then
    rm -rf "${PGBACKUPDIR}"
fi