Add BorgBackup disaster recovery system

- Full backup of PostgreSQL, Docker volumes, nginx config, SSL certs, crontabs, OpenDKIM
- Daily backups at 03:00 UTC with 7d/4w/3m retention
- Local storage at /opt/borg-backups/docfast
- Restore testing verified
- Documentation for disaster recovery procedures

scripts/borg-restore.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+# DocFast BorgBackup Restore Script
+# Restores from Borg backup for disaster recovery
+
+set -euo pipefail
+
+# Configuration
+BORG_REPO="/opt/borg-backups/docfast"
+RESTORE_DIR="/tmp/docfast-restore-$$"
+LOG_FILE="/var/log/docfast-restore.log"
+
+# Usage function
+usage() {
+    echo "Usage: $0 [list|restore] [archive-name]"
+    echo "  list                    - List available archives"
+    echo "  restore <archive-name>  - Restore specific archive"
+    echo "  restore latest          - Restore latest archive"
+    echo ""
+    echo "Examples:"
+    echo "  $0 list"
+    echo "  $0 restore docfast-2026-02-15_0300"
+    echo "  $0 restore latest"
+    exit 1
+}
+
+# Logging function
+log() {
+    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
+}
+
+# Error handler
+error_exit() {
+    log "ERROR: $1"
+    cleanup
+    exit 1
+}
+
+# Cleanup function
+cleanup() {
+    if [[ -d "$RESTORE_DIR" ]]; then
+        log "Cleaning up temporary directory: $RESTORE_DIR"
+        rm -rf "$RESTORE_DIR"
+    fi
+}
+
+# Trap cleanup on exit
+trap cleanup EXIT
+
+# Check if repository exists
+if [[ ! -d "$BORG_REPO" ]]; then
+    error_exit "Borg repository not found: $BORG_REPO"
+fi
+
+# Set up environment
+export BORG_PASSPHRASE="docfast-backup-$(date +%Y)"
+export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes
+mkdir -p "$(dirname "$LOG_FILE")"
+
+# Parse command line
+case "${1:-}" in
+    "list")
+        log "Listing available archives..."
+        borg list "$BORG_REPO"
+        exit 0
+        ;;
+    "restore")
+        ARCHIVE_NAME="${2:-}"
+        if [[ -z "$ARCHIVE_NAME" ]]; then
+            usage
+        fi
+        
+        if [[ "$ARCHIVE_NAME" == "latest" ]]; then
+            log "Finding latest archive..."
+            ARCHIVE_NAME=$(borg list --short "$BORG_REPO" | grep "^docfast-" | tail -1)
+            if [[ -z "$ARCHIVE_NAME" ]]; then
+                error_exit "No archives found in repository"
+            fi
+            log "Latest archive found: $ARCHIVE_NAME"
+        fi
+        ;;
+    *)
+        usage
+        ;;
+esac
+
+log "Starting restore of archive: $ARCHIVE_NAME"
+
+# Verify archive exists
+if ! borg list "$BORG_REPO::$ARCHIVE_NAME" >/dev/null 2>&1; then
+    error_exit "Archive not found: $ARCHIVE_NAME"
+fi
+
+# Create restore directory
+mkdir -p "$RESTORE_DIR"
+log "Restoring to temporary directory: $RESTORE_DIR"
+
+# Extract archive
+log "Extracting archive..."
+cd "$RESTORE_DIR"
+borg extract --verbose --list "$BORG_REPO::$ARCHIVE_NAME"
+
+log "Archive extracted successfully. Restore data available at: $RESTORE_DIR"
+echo ""
+echo "RESTORE LOCATIONS:"
+echo "=================="
+echo "PostgreSQL dump:     $RESTORE_DIR/tmp/docfast-backup-*/docfast-db.dump"
+echo "Docker volumes:      $RESTORE_DIR/tmp/docfast-backup-*/docker-volumes/"
+echo "Nginx config:        $RESTORE_DIR/tmp/docfast-backup-*/nginx/"
+echo "SSL certificates:    $RESTORE_DIR/tmp/docfast-backup-*/letsencrypt/"
+echo "Crontabs:           $RESTORE_DIR/tmp/docfast-backup-*/crontabs/"
+echo "OpenDKIM keys:      $RESTORE_DIR/tmp/docfast-backup-*/opendkim/"
+echo "DocFast app files:  $RESTORE_DIR/tmp/docfast-backup-*/docfast-app/"
+echo "System info:        $RESTORE_DIR/tmp/docfast-backup-*/system/"
+echo ""
+echo "MANUAL RESTORE STEPS:"
+echo "====================="
+echo "1. Stop DocFast service:"
+echo "   systemctl stop docker"
+echo ""
+echo "2. Restore PostgreSQL database:"
+echo "   sudo -u postgres dropdb docfast"
+echo "   sudo -u postgres createdb -O docfast docfast"
+echo "   sudo -u postgres pg_restore -d docfast $RESTORE_DIR/tmp/docfast-backup-*/docfast-db.dump"
+echo ""
+echo "3. Restore Docker volumes:"
+echo "   cp -r $RESTORE_DIR/tmp/docfast-backup-*/docker-volumes/* /var/lib/docker/volumes/"
+echo ""
+echo "4. Restore configuration files:"
+echo "   cp -r $RESTORE_DIR/tmp/docfast-backup-*/nginx/* /etc/nginx/"
+echo "   cp -r $RESTORE_DIR/tmp/docfast-backup-*/letsencrypt/* /etc/letsencrypt/"
+echo "   cp -r $RESTORE_DIR/tmp/docfast-backup-*/opendkim/* /etc/opendkim/"
+echo "   cp -r $RESTORE_DIR/tmp/docfast-backup-*/docfast-app/* /opt/docfast/"
+echo ""
+echo "5. Restore crontabs:"
+echo "   cp $RESTORE_DIR/tmp/docfast-backup-*/crontabs/root /var/spool/cron/crontabs/root"
+echo "   chmod 600 /var/spool/cron/crontabs/root"
+echo ""
+echo "6. Set correct permissions:"
+echo "   chown -R opendkim:opendkim /etc/opendkim/keys"
+echo "   chown -R postgres:postgres /var/lib/postgresql"
+echo ""
+echo "7. Start services:"
+echo "   systemctl start postgresql"
+echo "   systemctl start docker"
+echo "   cd /opt/docfast && docker-compose up -d"
+echo ""
+echo "WARNING: This script does NOT automatically restore files to prevent"
+echo "accidental overwrites. Follow the manual steps above carefully."
+
+log "Restore extraction completed. Follow manual steps to complete restoration."