Add comprehensive backup and restore infrastructure
- scripts/borg-backup.sh: Complete disaster recovery backup (PostgreSQL, Docker volumes, configs, SSL certs, DKIM keys) - scripts/borg-restore.sh: Automated restore from Borg backups with manual verification steps - scripts/rollback.sh: Quick application rollback using Docker image tags - Enhanced setup.sh with BorgBackup installation and cron job setup - Updated README with detailed backup strategy documentation (2-tier: SQLite + Borg) Backup retention: 7 daily + 4 weekly + 3 monthly Borg archives + 7 days SQLite snapshots
This commit is contained in:
openclawd
parent
3820d7ea4d
commit
76c1cc3bfb
4 changed files with 384 additions and 6 deletions
|
|
@ -137,12 +137,63 @@ docker-compose up -d
|
|||
|
||||
### Backup System
|
||||
```bash
|
||||
mkdir -p /opt/docfast-backups
|
||||
# Install BorgBackup
|
||||
apt install -y borgbackup
|
||||
|
||||
# Create backup directories
|
||||
mkdir -p /opt/docfast-backups /opt/borg-backups
|
||||
|
||||
# Copy backup scripts
|
||||
cp scripts/docfast-backup.sh /opt/
|
||||
chmod +x /opt/docfast-backup.sh
|
||||
cp scripts/borg-backup.sh /opt/
|
||||
cp scripts/borg-restore.sh /opt/
|
||||
cp scripts/rollback.sh /opt/
|
||||
chmod +x /opt/docfast-backup.sh /opt/borg-backup.sh /opt/borg-restore.sh /opt/rollback.sh
|
||||
|
||||
# Add to root crontab
|
||||
echo "0 */6 * * * /opt/docfast-backup.sh >> /var/log/docfast-backup.log 2>&1" | crontab -
|
||||
echo "0 3 * * * /opt/borg-backup.sh >> /var/log/borg-backup.log 2>&1" | crontab -
|
||||
```
|
||||
|
||||
## Backup Strategy
|
||||
|
||||
DocFast uses a two-tier backup strategy for comprehensive data protection:
|
||||
|
||||
### 1. SQLite Database Backups (Every 6 hours)
|
||||
- **Script**: `/opt/docfast-backup.sh`
|
||||
- **Frequency**: Every 6 hours via cron
|
||||
- **Retention**: 7 days of backups (28 files), plus 4 weekly copies
|
||||
- **Storage**: `/opt/docfast-backups/`
|
||||
- **Method**: SQLite `.backup` command with integrity verification
|
||||
|
||||
### 2. Complete System Backups (Daily)
|
||||
- **Script**: `/opt/borg-backup.sh`
|
||||
- **Frequency**: Daily at 03:00 UTC via cron
|
||||
- **Retention**: 7 daily + 4 weekly + 3 monthly
|
||||
- **Storage**: `/opt/borg-backups/docfast`
|
||||
- **Includes**:
|
||||
- PostgreSQL database dump
|
||||
- Docker volumes (complete application data)
|
||||
- Nginx configuration
|
||||
- SSL certificates (Let's Encrypt)
|
||||
- OpenDKIM keys and configuration
|
||||
- Cron jobs and system configurations
|
||||
- Application files (.env, docker-compose.yml)
|
||||
- System information (packages, services)
|
||||
|
||||
### Backup Management Commands
|
||||
```bash
|
||||
# List available Borg backups
|
||||
/opt/borg-restore.sh list
|
||||
|
||||
# Restore from latest backup (creates restore directory)
|
||||
/opt/borg-restore.sh restore latest
|
||||
|
||||
# Restore from specific backup
|
||||
/opt/borg-restore.sh restore docfast-2026-02-15_0300
|
||||
|
||||
# Quick rollback (Docker image only)
|
||||
/opt/rollback.sh
|
||||
```
|
||||
|
||||
## Disaster Recovery Procedures
|
||||
|
|
|
|||
|
|
@ -151,17 +151,32 @@ sed -i "s/docfast\.dev/$DOMAIN/g" /etc/opendkim.conf 2>/dev/null || true
|
|||
# Restart services with new configs
|
||||
systemctl restart postfix opendkim
|
||||
|
||||
# Setup backup directory and script
|
||||
# Install BorgBackup
|
||||
log "Installing BorgBackup..."
|
||||
apt install -y borgbackup
|
||||
|
||||
# Setup backup directories and scripts
|
||||
log "Setting up backup system..."
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
cp ../scripts/docfast-backup.sh /opt/docfast-backup.sh || warn "Backup script not found"
|
||||
chmod +x /opt/docfast-backup.sh
|
||||
mkdir -p /opt/borg-backups
|
||||
|
||||
# Add backup cron job
|
||||
# Copy backup scripts
|
||||
cp ../scripts/docfast-backup.sh /opt/docfast-backup.sh || warn "SQLite backup script not found"
|
||||
cp ../scripts/borg-backup.sh /opt/borg-backup.sh || warn "Borg backup script not found"
|
||||
cp ../scripts/borg-restore.sh /opt/borg-restore.sh || warn "Borg restore script not found"
|
||||
cp ../scripts/rollback.sh /opt/rollback.sh || warn "Rollback script not found"
|
||||
|
||||
chmod +x /opt/docfast-backup.sh /opt/borg-backup.sh /opt/borg-restore.sh /opt/rollback.sh
|
||||
|
||||
# Add backup cron jobs
|
||||
if ! crontab -l 2>/dev/null | grep -q docfast-backup; then
|
||||
(crontab -l 2>/dev/null; echo "0 */6 * * * /opt/docfast-backup.sh >> /var/log/docfast-backup.log 2>&1") | crontab -
|
||||
fi
|
||||
|
||||
if ! crontab -l 2>/dev/null | grep -q borg-backup; then
|
||||
(crontab -l 2>/dev/null; echo "0 3 * * * /opt/borg-backup.sh >> /var/log/borg-backup.log 2>&1") | crontab -
|
||||
fi
|
||||
|
||||
# Setup application directory
|
||||
log "Setting up application directory..."
|
||||
mkdir -p "$INSTALL_DIR"
|
||||
|
|
|
|||
162
scripts/borg-backup.sh
Executable file
162
scripts/borg-backup.sh
Executable file
|
|
@ -0,0 +1,162 @@
|
|||
#!/bin/bash
|
||||
# DocFast BorgBackup Script - Full Disaster Recovery
|
||||
# Backs up: PostgreSQL, Docker volumes, nginx config, SSL certs, crontabs, OpenDKIM keys
|
||||
# Schedule: daily at 03:00 UTC, keeps 7 daily + 4 weekly + 3 monthly
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
BORG_REPO="/opt/borg-backups/docfast"
|
||||
BACKUP_NAME="docfast-$(date +%Y-%m-%d_%H%M)"
|
||||
TEMP_DIR="/tmp/docfast-backup-$$"
|
||||
LOG_FILE="/var/log/docfast-backup.log"
|
||||
|
||||
# Database configuration
|
||||
DB_NAME="docfast"
|
||||
DB_USER="docfast"
|
||||
DB_HOST="localhost"
|
||||
DB_PORT="5432"
|
||||
|
||||
# Logging function
|
||||
log() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Error handler
|
||||
error_exit() {
|
||||
log "ERROR: $1"
|
||||
cleanup
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
if [[ -d "$TEMP_DIR" ]]; then
|
||||
rm -rf "$TEMP_DIR"
|
||||
fi
|
||||
}
|
||||
|
||||
# Trap cleanup on exit
|
||||
trap cleanup EXIT
|
||||
|
||||
log "Starting DocFast backup: $BACKUP_NAME"
|
||||
|
||||
# Create temporary directory
|
||||
mkdir -p "$TEMP_DIR"
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
|
||||
# 1. PostgreSQL dump
|
||||
log "Creating PostgreSQL dump..."
|
||||
export PGPASSFILE="/root/.pgpass"
|
||||
pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \
|
||||
--no-password --verbose --clean --if-exists --format=custom \
|
||||
> "$TEMP_DIR/docfast-db.dump" 2>>"$LOG_FILE" || error_exit "PostgreSQL dump failed"
|
||||
|
||||
# Verify dump is valid
|
||||
if ! pg_restore --list "$TEMP_DIR/docfast-db.dump" >/dev/null 2>&1; then
|
||||
error_exit "PostgreSQL dump verification failed"
|
||||
fi
|
||||
log "PostgreSQL dump completed: $(stat -c%s "$TEMP_DIR/docfast-db.dump") bytes"
|
||||
|
||||
# 2. Docker volumes
|
||||
log "Backing up Docker volumes..."
|
||||
mkdir -p "$TEMP_DIR/docker-volumes"
|
||||
if [[ -d "/var/lib/docker/volumes" ]]; then
|
||||
cp -r /var/lib/docker/volumes/* "$TEMP_DIR/docker-volumes/" || error_exit "Docker volumes backup failed"
|
||||
log "Docker volumes backed up"
|
||||
else
|
||||
log "WARNING: No Docker volumes found"
|
||||
fi
|
||||
|
||||
# 3. Nginx configuration
|
||||
log "Backing up nginx configuration..."
|
||||
mkdir -p "$TEMP_DIR/nginx"
|
||||
cp -r /etc/nginx/* "$TEMP_DIR/nginx/" || error_exit "Nginx backup failed"
|
||||
log "Nginx configuration backed up"
|
||||
|
||||
# 4. SSL certificates
|
||||
log "Backing up SSL certificates..."
|
||||
mkdir -p "$TEMP_DIR/letsencrypt"
|
||||
cp -r /etc/letsencrypt/* "$TEMP_DIR/letsencrypt/" || error_exit "SSL certificates backup failed"
|
||||
log "SSL certificates backed up"
|
||||
|
||||
# 5. Crontabs
|
||||
log "Backing up crontabs..."
|
||||
mkdir -p "$TEMP_DIR/crontabs"
|
||||
if [[ -d "/var/spool/cron/crontabs" ]]; then
|
||||
cp -r /var/spool/cron/crontabs/* "$TEMP_DIR/crontabs/" 2>/dev/null || log "No crontabs found"
|
||||
fi
|
||||
# Also backup user crontabs
|
||||
crontab -l > "$TEMP_DIR/crontabs/root-crontab.txt" 2>/dev/null || echo "# No root crontab" > "$TEMP_DIR/crontabs/root-crontab.txt"
|
||||
log "Crontabs backed up"
|
||||
|
||||
# 6. OpenDKIM keys
|
||||
log "Backing up OpenDKIM keys..."
|
||||
mkdir -p "$TEMP_DIR/opendkim"
|
||||
cp -r /etc/opendkim/* "$TEMP_DIR/opendkim/" || error_exit "OpenDKIM backup failed"
|
||||
log "OpenDKIM keys backed up"
|
||||
|
||||
# 7. DocFast application files (docker-compose, env, scripts)
|
||||
log "Backing up DocFast application files..."
|
||||
mkdir -p "$TEMP_DIR/docfast-app"
|
||||
if [[ -d "/opt/docfast" ]]; then
|
||||
cp /opt/docfast/docker-compose.yml "$TEMP_DIR/docfast-app/" 2>/dev/null || true
|
||||
cp /opt/docfast/.env "$TEMP_DIR/docfast-app/" 2>/dev/null || true
|
||||
cp -r /opt/docfast/scripts "$TEMP_DIR/docfast-app/" 2>/dev/null || true
|
||||
cp -r /opt/docfast/deploy "$TEMP_DIR/docfast-app/" 2>/dev/null || true
|
||||
log "DocFast application files backed up"
|
||||
fi
|
||||
|
||||
# 8. System information
|
||||
log "Creating system information backup..."
|
||||
mkdir -p "$TEMP_DIR/system"
|
||||
systemctl list-unit-files --state=enabled > "$TEMP_DIR/system/enabled-services.txt"
|
||||
dpkg -l > "$TEMP_DIR/system/installed-packages.txt"
|
||||
uname -a > "$TEMP_DIR/system/system-info.txt"
|
||||
df -h > "$TEMP_DIR/system/disk-usage.txt"
|
||||
log "System information backed up"
|
||||
|
||||
# 9. Create Borg backup
|
||||
log "Creating Borg backup..."
|
||||
export BORG_PASSPHRASE="docfast-backup-$(date +%Y)"
|
||||
export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes
|
||||
|
||||
# Initialize repository if it doesn't exist
|
||||
if [[ ! -d "$BORG_REPO" ]]; then
|
||||
log "Initializing new Borg repository..."
|
||||
borg init --encryption=repokey "$BORG_REPO" || error_exit "Failed to initialize Borg repository"
|
||||
fi
|
||||
|
||||
# Create backup
|
||||
log "Creating Borg archive: $BACKUP_NAME"
|
||||
borg create \
|
||||
--verbose \
|
||||
--filter AME \
|
||||
--list \
|
||||
--stats \
|
||||
--show-rc \
|
||||
--compression lz4 \
|
||||
--exclude-caches \
|
||||
"$BORG_REPO::$BACKUP_NAME" \
|
||||
"$TEMP_DIR" 2>>"$LOG_FILE" || error_exit "Borg backup creation failed"
|
||||
|
||||
# 10. Prune old backups (7 daily, 4 weekly, 3 monthly)
|
||||
log "Pruning old backups..."
|
||||
borg prune \
|
||||
--list \
|
||||
--prefix 'docfast-' \
|
||||
--show-rc \
|
||||
--keep-daily 7 \
|
||||
--keep-weekly 4 \
|
||||
--keep-monthly 3 \
|
||||
"$BORG_REPO" 2>>"$LOG_FILE" || error_exit "Borg pruning failed"
|
||||
|
||||
# 11. Compact repository
|
||||
log "Compacting repository..."
|
||||
borg compact "$BORG_REPO" 2>>"$LOG_FILE" || log "WARNING: Repository compaction failed (non-fatal)"
|
||||
|
||||
# 12. Repository info
|
||||
log "Backup completed successfully!"
|
||||
borg info "$BORG_REPO" 2>>"$LOG_FILE"
|
||||
|
||||
log "DocFast backup completed: $BACKUP_NAME"
|
||||
150
scripts/borg-restore.sh
Executable file
150
scripts/borg-restore.sh
Executable file
|
|
@ -0,0 +1,150 @@
|
|||
#!/bin/bash
|
||||
# DocFast BorgBackup Restore Script
|
||||
# Restores from Borg backup for disaster recovery
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
BORG_REPO="/opt/borg-backups/docfast"
|
||||
RESTORE_DIR="/tmp/docfast-restore-$$"
|
||||
LOG_FILE="/var/log/docfast-restore.log"
|
||||
|
||||
# Usage function
|
||||
usage() {
|
||||
echo "Usage: $0 [list|restore] [archive-name]"
|
||||
echo " list - List available archives"
|
||||
echo " restore <archive-name> - Restore specific archive"
|
||||
echo " restore latest - Restore latest archive"
|
||||
echo ""
|
||||
echo "Examples:"
|
||||
echo " $0 list"
|
||||
echo " $0 restore docfast-2026-02-15_0300"
|
||||
echo " $0 restore latest"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Logging function
|
||||
log() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Error handler
|
||||
error_exit() {
|
||||
log "ERROR: $1"
|
||||
cleanup
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
if [[ -d "$RESTORE_DIR" ]]; then
|
||||
log "Cleaning up temporary directory: $RESTORE_DIR"
|
||||
rm -rf "$RESTORE_DIR"
|
||||
fi
|
||||
}
|
||||
|
||||
# Trap cleanup on exit
|
||||
trap cleanup EXIT
|
||||
|
||||
# Check if repository exists
|
||||
if [[ ! -d "$BORG_REPO" ]]; then
|
||||
error_exit "Borg repository not found: $BORG_REPO"
|
||||
fi
|
||||
|
||||
# Set up environment
|
||||
export BORG_PASSPHRASE="docfast-backup-$(date +%Y)"
|
||||
export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
|
||||
# Parse command line
|
||||
case "${1:-}" in
|
||||
"list")
|
||||
log "Listing available archives..."
|
||||
borg list "$BORG_REPO"
|
||||
exit 0
|
||||
;;
|
||||
"restore")
|
||||
ARCHIVE_NAME="${2:-}"
|
||||
if [[ -z "$ARCHIVE_NAME" ]]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [[ "$ARCHIVE_NAME" == "latest" ]]; then
|
||||
log "Finding latest archive..."
|
||||
ARCHIVE_NAME=$(borg list --short "$BORG_REPO" | grep "^docfast-" | tail -1)
|
||||
if [[ -z "$ARCHIVE_NAME" ]]; then
|
||||
error_exit "No archives found in repository"
|
||||
fi
|
||||
log "Latest archive found: $ARCHIVE_NAME"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
|
||||
log "Starting restore of archive: $ARCHIVE_NAME"
|
||||
|
||||
# Verify archive exists
|
||||
if ! borg list "$BORG_REPO::$ARCHIVE_NAME" >/dev/null 2>&1; then
|
||||
error_exit "Archive not found: $ARCHIVE_NAME"
|
||||
fi
|
||||
|
||||
# Create restore directory
|
||||
mkdir -p "$RESTORE_DIR"
|
||||
log "Restoring to temporary directory: $RESTORE_DIR"
|
||||
|
||||
# Extract archive
|
||||
log "Extracting archive..."
|
||||
cd "$RESTORE_DIR"
|
||||
borg extract --verbose --list "$BORG_REPO::$ARCHIVE_NAME"
|
||||
|
||||
log "Archive extracted successfully. Restore data available at: $RESTORE_DIR"
|
||||
echo ""
|
||||
echo "RESTORE LOCATIONS:"
|
||||
echo "=================="
|
||||
echo "PostgreSQL dump: $RESTORE_DIR/tmp/docfast-backup-*/docfast-db.dump"
|
||||
echo "Docker volumes: $RESTORE_DIR/tmp/docfast-backup-*/docker-volumes/"
|
||||
echo "Nginx config: $RESTORE_DIR/tmp/docfast-backup-*/nginx/"
|
||||
echo "SSL certificates: $RESTORE_DIR/tmp/docfast-backup-*/letsencrypt/"
|
||||
echo "Crontabs: $RESTORE_DIR/tmp/docfast-backup-*/crontabs/"
|
||||
echo "OpenDKIM keys: $RESTORE_DIR/tmp/docfast-backup-*/opendkim/"
|
||||
echo "DocFast app files: $RESTORE_DIR/tmp/docfast-backup-*/docfast-app/"
|
||||
echo "System info: $RESTORE_DIR/tmp/docfast-backup-*/system/"
|
||||
echo ""
|
||||
echo "MANUAL RESTORE STEPS:"
|
||||
echo "====================="
|
||||
echo "1. Stop DocFast service:"
|
||||
echo " systemctl stop docker"
|
||||
echo ""
|
||||
echo "2. Restore PostgreSQL database:"
|
||||
echo " sudo -u postgres dropdb docfast"
|
||||
echo " sudo -u postgres createdb -O docfast docfast"
|
||||
echo " sudo -u postgres pg_restore -d docfast $RESTORE_DIR/tmp/docfast-backup-*/docfast-db.dump"
|
||||
echo ""
|
||||
echo "3. Restore Docker volumes:"
|
||||
echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/docker-volumes/* /var/lib/docker/volumes/"
|
||||
echo ""
|
||||
echo "4. Restore configuration files:"
|
||||
echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/nginx/* /etc/nginx/"
|
||||
echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/letsencrypt/* /etc/letsencrypt/"
|
||||
echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/opendkim/* /etc/opendkim/"
|
||||
echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/docfast-app/* /opt/docfast/"
|
||||
echo ""
|
||||
echo "5. Restore crontabs:"
|
||||
echo " cp $RESTORE_DIR/tmp/docfast-backup-*/crontabs/root /var/spool/cron/crontabs/root"
|
||||
echo " chmod 600 /var/spool/cron/crontabs/root"
|
||||
echo ""
|
||||
echo "6. Set correct permissions:"
|
||||
echo " chown -R opendkim:opendkim /etc/opendkim/keys"
|
||||
echo " chown -R postgres:postgres /var/lib/postgresql"
|
||||
echo ""
|
||||
echo "7. Start services:"
|
||||
echo " systemctl start postgresql"
|
||||
echo " systemctl start docker"
|
||||
echo " cd /opt/docfast && docker-compose up -d"
|
||||
echo ""
|
||||
echo "WARNING: This script does NOT automatically restore files to prevent"
|
||||
echo "accidental overwrites. Follow the manual steps above carefully."
|
||||
|
||||
log "Restore extraction completed. Follow manual steps to complete restoration."
|
||||
Loading…
Add table
Add a link
Reference in a new issue