diff --git a/infrastructure/README.md b/infrastructure/README.md index 6052e3b..4a8dab1 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -137,12 +137,63 @@ docker-compose up -d ### Backup System ```bash -mkdir -p /opt/docfast-backups +# Install BorgBackup +apt install -y borgbackup + +# Create backup directories +mkdir -p /opt/docfast-backups /opt/borg-backups + +# Copy backup scripts cp scripts/docfast-backup.sh /opt/ -chmod +x /opt/docfast-backup.sh +cp scripts/borg-backup.sh /opt/ +cp scripts/borg-restore.sh /opt/ +cp scripts/rollback.sh /opt/ +chmod +x /opt/docfast-backup.sh /opt/borg-backup.sh /opt/borg-restore.sh /opt/rollback.sh # Add to root crontab echo "0 */6 * * * /opt/docfast-backup.sh >> /var/log/docfast-backup.log 2>&1" | crontab - +echo "0 3 * * * /opt/borg-backup.sh >> /var/log/borg-backup.log 2>&1" | crontab - +``` + +## Backup Strategy + +DocFast uses a two-tier backup strategy for comprehensive data protection: + +### 1. SQLite Database Backups (Every 6 hours) +- **Script**: `/opt/docfast-backup.sh` +- **Frequency**: Every 6 hours via cron +- **Retention**: 7 days of backups (28 files), plus 4 weekly copies +- **Storage**: `/opt/docfast-backups/` +- **Method**: SQLite `.backup` command with integrity verification + +### 2. Complete System Backups (Daily) +- **Script**: `/opt/borg-backup.sh` +- **Frequency**: Daily at 03:00 UTC via cron +- **Retention**: 7 daily + 4 weekly + 3 monthly +- **Storage**: `/opt/borg-backups/docfast` +- **Includes**: + - PostgreSQL database dump + - Docker volumes (complete application data) + - Nginx configuration + - SSL certificates (Let's Encrypt) + - OpenDKIM keys and configuration + - Cron jobs and system configurations + - Application files (.env, docker-compose.yml) + - System information (packages, services) + +### Backup Management Commands +```bash +# List available Borg backups +/opt/borg-restore.sh list + +# Restore from latest backup (creates restore directory) +/opt/borg-restore.sh restore latest + +# Restore from specific backup +/opt/borg-restore.sh restore docfast-2026-02-15_0300 + +# Quick rollback (Docker image only) +/opt/rollback.sh ``` ## Disaster Recovery Procedures diff --git a/infrastructure/setup.sh b/infrastructure/setup.sh index 24d9d61..91e8f0f 100755 --- a/infrastructure/setup.sh +++ b/infrastructure/setup.sh @@ -151,17 +151,32 @@ sed -i "s/docfast\.dev/$DOMAIN/g" /etc/opendkim.conf 2>/dev/null || true # Restart services with new configs systemctl restart postfix opendkim -# Setup backup directory and script +# Install BorgBackup +log "Installing BorgBackup..." +apt install -y borgbackup + +# Setup backup directories and scripts log "Setting up backup system..." mkdir -p "$BACKUP_DIR" -cp ../scripts/docfast-backup.sh /opt/docfast-backup.sh || warn "Backup script not found" -chmod +x /opt/docfast-backup.sh +mkdir -p /opt/borg-backups -# Add backup cron job +# Copy backup scripts +cp ../scripts/docfast-backup.sh /opt/docfast-backup.sh || warn "SQLite backup script not found" +cp ../scripts/borg-backup.sh /opt/borg-backup.sh || warn "Borg backup script not found" +cp ../scripts/borg-restore.sh /opt/borg-restore.sh || warn "Borg restore script not found" +cp ../scripts/rollback.sh /opt/rollback.sh || warn "Rollback script not found" + +chmod +x /opt/docfast-backup.sh /opt/borg-backup.sh /opt/borg-restore.sh /opt/rollback.sh + +# Add backup cron jobs if ! crontab -l 2>/dev/null | grep -q docfast-backup; then (crontab -l 2>/dev/null; echo "0 */6 * * * /opt/docfast-backup.sh >> /var/log/docfast-backup.log 2>&1") | crontab - fi +if ! crontab -l 2>/dev/null | grep -q borg-backup; then + (crontab -l 2>/dev/null; echo "0 3 * * * /opt/borg-backup.sh >> /var/log/borg-backup.log 2>&1") | crontab - +fi + # Setup application directory log "Setting up application directory..." mkdir -p "$INSTALL_DIR" diff --git a/scripts/borg-backup.sh b/scripts/borg-backup.sh new file mode 100755 index 0000000..cda3ebb --- /dev/null +++ b/scripts/borg-backup.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# DocFast BorgBackup Script - Full Disaster Recovery +# Backs up: PostgreSQL, Docker volumes, nginx config, SSL certs, crontabs, OpenDKIM keys +# Schedule: daily at 03:00 UTC, keeps 7 daily + 4 weekly + 3 monthly + +set -euo pipefail + +# Configuration +BORG_REPO="/opt/borg-backups/docfast" +BACKUP_NAME="docfast-$(date +%Y-%m-%d_%H%M)" +TEMP_DIR="/tmp/docfast-backup-$$" +LOG_FILE="/var/log/docfast-backup.log" + +# Database configuration +DB_NAME="docfast" +DB_USER="docfast" +DB_HOST="localhost" +DB_PORT="5432" + +# Logging function +log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE" +} + +# Error handler +error_exit() { + log "ERROR: $1" + cleanup + exit 1 +} + +# Cleanup function +cleanup() { + if [[ -d "$TEMP_DIR" ]]; then + rm -rf "$TEMP_DIR" + fi +} + +# Trap cleanup on exit +trap cleanup EXIT + +log "Starting DocFast backup: $BACKUP_NAME" + +# Create temporary directory +mkdir -p "$TEMP_DIR" +mkdir -p "$(dirname "$LOG_FILE")" + +# 1. PostgreSQL dump +log "Creating PostgreSQL dump..." +export PGPASSFILE="/root/.pgpass" +pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ + --no-password --verbose --clean --if-exists --format=custom \ + > "$TEMP_DIR/docfast-db.dump" 2>>"$LOG_FILE" || error_exit "PostgreSQL dump failed" + +# Verify dump is valid +if ! pg_restore --list "$TEMP_DIR/docfast-db.dump" >/dev/null 2>&1; then + error_exit "PostgreSQL dump verification failed" +fi +log "PostgreSQL dump completed: $(stat -c%s "$TEMP_DIR/docfast-db.dump") bytes" + +# 2. Docker volumes +log "Backing up Docker volumes..." +mkdir -p "$TEMP_DIR/docker-volumes" +if [[ -d "/var/lib/docker/volumes" ]]; then + cp -r /var/lib/docker/volumes/* "$TEMP_DIR/docker-volumes/" || error_exit "Docker volumes backup failed" + log "Docker volumes backed up" +else + log "WARNING: No Docker volumes found" +fi + +# 3. Nginx configuration +log "Backing up nginx configuration..." +mkdir -p "$TEMP_DIR/nginx" +cp -r /etc/nginx/* "$TEMP_DIR/nginx/" || error_exit "Nginx backup failed" +log "Nginx configuration backed up" + +# 4. SSL certificates +log "Backing up SSL certificates..." +mkdir -p "$TEMP_DIR/letsencrypt" +cp -r /etc/letsencrypt/* "$TEMP_DIR/letsencrypt/" || error_exit "SSL certificates backup failed" +log "SSL certificates backed up" + +# 5. Crontabs +log "Backing up crontabs..." +mkdir -p "$TEMP_DIR/crontabs" +if [[ -d "/var/spool/cron/crontabs" ]]; then + cp -r /var/spool/cron/crontabs/* "$TEMP_DIR/crontabs/" 2>/dev/null || log "No crontabs found" +fi +# Also backup user crontabs +crontab -l > "$TEMP_DIR/crontabs/root-crontab.txt" 2>/dev/null || echo "# No root crontab" > "$TEMP_DIR/crontabs/root-crontab.txt" +log "Crontabs backed up" + +# 6. OpenDKIM keys +log "Backing up OpenDKIM keys..." +mkdir -p "$TEMP_DIR/opendkim" +cp -r /etc/opendkim/* "$TEMP_DIR/opendkim/" || error_exit "OpenDKIM backup failed" +log "OpenDKIM keys backed up" + +# 7. DocFast application files (docker-compose, env, scripts) +log "Backing up DocFast application files..." +mkdir -p "$TEMP_DIR/docfast-app" +if [[ -d "/opt/docfast" ]]; then + cp /opt/docfast/docker-compose.yml "$TEMP_DIR/docfast-app/" 2>/dev/null || true + cp /opt/docfast/.env "$TEMP_DIR/docfast-app/" 2>/dev/null || true + cp -r /opt/docfast/scripts "$TEMP_DIR/docfast-app/" 2>/dev/null || true + cp -r /opt/docfast/deploy "$TEMP_DIR/docfast-app/" 2>/dev/null || true + log "DocFast application files backed up" +fi + +# 8. System information +log "Creating system information backup..." +mkdir -p "$TEMP_DIR/system" +systemctl list-unit-files --state=enabled > "$TEMP_DIR/system/enabled-services.txt" +dpkg -l > "$TEMP_DIR/system/installed-packages.txt" +uname -a > "$TEMP_DIR/system/system-info.txt" +df -h > "$TEMP_DIR/system/disk-usage.txt" +log "System information backed up" + +# 9. Create Borg backup +log "Creating Borg backup..." +export BORG_PASSPHRASE="docfast-backup-$(date +%Y)" +export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes + +# Initialize repository if it doesn't exist +if [[ ! -d "$BORG_REPO" ]]; then + log "Initializing new Borg repository..." + borg init --encryption=repokey "$BORG_REPO" || error_exit "Failed to initialize Borg repository" +fi + +# Create backup +log "Creating Borg archive: $BACKUP_NAME" +borg create \ + --verbose \ + --filter AME \ + --list \ + --stats \ + --show-rc \ + --compression lz4 \ + --exclude-caches \ + "$BORG_REPO::$BACKUP_NAME" \ + "$TEMP_DIR" 2>>"$LOG_FILE" || error_exit "Borg backup creation failed" + +# 10. Prune old backups (7 daily, 4 weekly, 3 monthly) +log "Pruning old backups..." +borg prune \ + --list \ + --prefix 'docfast-' \ + --show-rc \ + --keep-daily 7 \ + --keep-weekly 4 \ + --keep-monthly 3 \ + "$BORG_REPO" 2>>"$LOG_FILE" || error_exit "Borg pruning failed" + +# 11. Compact repository +log "Compacting repository..." +borg compact "$BORG_REPO" 2>>"$LOG_FILE" || log "WARNING: Repository compaction failed (non-fatal)" + +# 12. Repository info +log "Backup completed successfully!" +borg info "$BORG_REPO" 2>>"$LOG_FILE" + +log "DocFast backup completed: $BACKUP_NAME" \ No newline at end of file diff --git a/scripts/borg-restore.sh b/scripts/borg-restore.sh new file mode 100755 index 0000000..4da6f8e --- /dev/null +++ b/scripts/borg-restore.sh @@ -0,0 +1,150 @@ +#!/bin/bash +# DocFast BorgBackup Restore Script +# Restores from Borg backup for disaster recovery + +set -euo pipefail + +# Configuration +BORG_REPO="/opt/borg-backups/docfast" +RESTORE_DIR="/tmp/docfast-restore-$$" +LOG_FILE="/var/log/docfast-restore.log" + +# Usage function +usage() { + echo "Usage: $0 [list|restore] [archive-name]" + echo " list - List available archives" + echo " restore - Restore specific archive" + echo " restore latest - Restore latest archive" + echo "" + echo "Examples:" + echo " $0 list" + echo " $0 restore docfast-2026-02-15_0300" + echo " $0 restore latest" + exit 1 +} + +# Logging function +log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE" +} + +# Error handler +error_exit() { + log "ERROR: $1" + cleanup + exit 1 +} + +# Cleanup function +cleanup() { + if [[ -d "$RESTORE_DIR" ]]; then + log "Cleaning up temporary directory: $RESTORE_DIR" + rm -rf "$RESTORE_DIR" + fi +} + +# Trap cleanup on exit +trap cleanup EXIT + +# Check if repository exists +if [[ ! -d "$BORG_REPO" ]]; then + error_exit "Borg repository not found: $BORG_REPO" +fi + +# Set up environment +export BORG_PASSPHRASE="docfast-backup-$(date +%Y)" +export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes +mkdir -p "$(dirname "$LOG_FILE")" + +# Parse command line +case "${1:-}" in + "list") + log "Listing available archives..." + borg list "$BORG_REPO" + exit 0 + ;; + "restore") + ARCHIVE_NAME="${2:-}" + if [[ -z "$ARCHIVE_NAME" ]]; then + usage + fi + + if [[ "$ARCHIVE_NAME" == "latest" ]]; then + log "Finding latest archive..." + ARCHIVE_NAME=$(borg list --short "$BORG_REPO" | grep "^docfast-" | tail -1) + if [[ -z "$ARCHIVE_NAME" ]]; then + error_exit "No archives found in repository" + fi + log "Latest archive found: $ARCHIVE_NAME" + fi + ;; + *) + usage + ;; +esac + +log "Starting restore of archive: $ARCHIVE_NAME" + +# Verify archive exists +if ! borg list "$BORG_REPO::$ARCHIVE_NAME" >/dev/null 2>&1; then + error_exit "Archive not found: $ARCHIVE_NAME" +fi + +# Create restore directory +mkdir -p "$RESTORE_DIR" +log "Restoring to temporary directory: $RESTORE_DIR" + +# Extract archive +log "Extracting archive..." +cd "$RESTORE_DIR" +borg extract --verbose --list "$BORG_REPO::$ARCHIVE_NAME" + +log "Archive extracted successfully. Restore data available at: $RESTORE_DIR" +echo "" +echo "RESTORE LOCATIONS:" +echo "==================" +echo "PostgreSQL dump: $RESTORE_DIR/tmp/docfast-backup-*/docfast-db.dump" +echo "Docker volumes: $RESTORE_DIR/tmp/docfast-backup-*/docker-volumes/" +echo "Nginx config: $RESTORE_DIR/tmp/docfast-backup-*/nginx/" +echo "SSL certificates: $RESTORE_DIR/tmp/docfast-backup-*/letsencrypt/" +echo "Crontabs: $RESTORE_DIR/tmp/docfast-backup-*/crontabs/" +echo "OpenDKIM keys: $RESTORE_DIR/tmp/docfast-backup-*/opendkim/" +echo "DocFast app files: $RESTORE_DIR/tmp/docfast-backup-*/docfast-app/" +echo "System info: $RESTORE_DIR/tmp/docfast-backup-*/system/" +echo "" +echo "MANUAL RESTORE STEPS:" +echo "=====================" +echo "1. Stop DocFast service:" +echo " systemctl stop docker" +echo "" +echo "2. Restore PostgreSQL database:" +echo " sudo -u postgres dropdb docfast" +echo " sudo -u postgres createdb -O docfast docfast" +echo " sudo -u postgres pg_restore -d docfast $RESTORE_DIR/tmp/docfast-backup-*/docfast-db.dump" +echo "" +echo "3. Restore Docker volumes:" +echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/docker-volumes/* /var/lib/docker/volumes/" +echo "" +echo "4. Restore configuration files:" +echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/nginx/* /etc/nginx/" +echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/letsencrypt/* /etc/letsencrypt/" +echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/opendkim/* /etc/opendkim/" +echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/docfast-app/* /opt/docfast/" +echo "" +echo "5. Restore crontabs:" +echo " cp $RESTORE_DIR/tmp/docfast-backup-*/crontabs/root /var/spool/cron/crontabs/root" +echo " chmod 600 /var/spool/cron/crontabs/root" +echo "" +echo "6. Set correct permissions:" +echo " chown -R opendkim:opendkim /etc/opendkim/keys" +echo " chown -R postgres:postgres /var/lib/postgresql" +echo "" +echo "7. Start services:" +echo " systemctl start postgresql" +echo " systemctl start docker" +echo " cd /opt/docfast && docker-compose up -d" +echo "" +echo "WARNING: This script does NOT automatically restore files to prevent" +echo "accidental overwrites. Follow the manual steps above carefully." + +log "Restore extraction completed. Follow manual steps to complete restoration." \ No newline at end of file