diff --git a/BACKUP_PROCEDURES.md b/BACKUP_PROCEDURES.md deleted file mode 100644 index 52106ca..0000000 --- a/BACKUP_PROCEDURES.md +++ /dev/null @@ -1,184 +0,0 @@ -# DocFast Backup & Disaster Recovery Procedures - -## Overview -DocFast now uses BorgBackup for full disaster recovery backups. The system backs up all critical components needed to restore the service on a new server. - -## What is Backed Up -- **PostgreSQL database** - Full database dump with schema and data -- **Docker volumes** - Application data and files -- **Nginx configuration** - Web server configuration -- **SSL certificates** - Let's Encrypt certificates and keys -- **Crontabs** - Scheduled tasks -- **OpenDKIM keys** - Email authentication keys -- **DocFast application files** - docker-compose.yml, .env, scripts -- **System information** - Installed packages, enabled services, disk usage - -## Backup Location & Schedule - -### Current Setup (Local) -- **Location**: `/opt/borg-backups/docfast` -- **Schedule**: Daily at 03:00 UTC -- **Retention**: 7 daily + 4 weekly + 3 monthly backups -- **Compression**: LZ4 (fast compression/decompression) -- **Encryption**: repokey mode (encrypted with passphrase) - -### Security -- **Passphrase**: `docfast-backup-YYYY` (where YYYY is current year) -- **Key backup**: Stored in `/opt/borg-backups/docfast-key-backup.txt` -- **⚠️ IMPORTANT**: Both passphrase AND key are required for restore! - -## Scripts - -### Backup Script: `/opt/docfast-borg-backup.sh` -- Automated backup creation -- Runs via cron daily at 03:00 UTC -- Logs to `/var/log/docfast-backup.log` -- Auto-prunes old backups - -### Restore Script: `/opt/docfast-borg-restore.sh` -- List available backups: `./docfast-borg-restore.sh list` -- Restore specific backup: `./docfast-borg-restore.sh restore docfast-YYYY-MM-DD_HHMM` -- Restore latest backup: `./docfast-borg-restore.sh restore latest` - -## Manual Backup Commands - -```bash -# Run backup manually -/opt/docfast-borg-backup.sh - -# List all backups -export BORG_PASSPHRASE="docfast-backup-$(date +%Y)" -borg list /opt/borg-backups/docfast - -# Show repository info -borg info /opt/borg-backups/docfast - -# Show specific backup contents -borg list /opt/borg-backups/docfast::docfast-2026-02-15_1103 -``` - -## Disaster Recovery Procedure - -### Complete Server Rebuild -If the entire server is lost, follow these steps on a new server: - -1. **Install dependencies**: - ```bash - apt update && apt install -y docker.io docker-compose postgresql-16 nginx borgbackup - systemctl enable postgresql docker - ``` - -2. **Copy backup data**: - - Transfer `/opt/borg-backups/` directory to new server - - Transfer `/opt/borg-backups/docfast-key-backup.txt` - -3. **Import Borg key**: - ```bash - export BORG_PASSPHRASE="docfast-backup-2026" - borg key import /opt/borg-backups/docfast /opt/borg-backups/docfast-key-backup.txt - ``` - -4. **Restore latest backup**: - ```bash - /opt/docfast-borg-restore.sh restore latest - ``` - -5. **Follow manual restore steps** (shown by restore script): - - Stop services - - Restore database - - Restore configuration files - - Set permissions - - Start services - -### Database-Only Recovery -If only the database needs restoration: - -```bash -# Stop DocFast -cd /opt/docfast && docker-compose down - -# Restore database -export BORG_PASSPHRASE="docfast-backup-$(date +%Y)" -cd /tmp -borg extract /opt/borg-backups/docfast::docfast-YYYY-MM-DD_HHMM -sudo -u postgres dropdb docfast -sudo -u postgres createdb -O docfast docfast -export PGPASSFILE="/root/.pgpass" -pg_restore -d docfast /tmp/tmp/docfast-backup-*/docfast-db.dump - -# Restart DocFast -cd /opt/docfast && docker-compose up -d -``` - -## Migration to Off-Site Storage - -### Option 1: Hetzner Storage Box (Recommended) -Manual setup required (Hetzner Storage Box API not available): - -1. **Purchase Hetzner Storage Box** - - Minimum 10GB size - - Enable SSH access in Hetzner Console - -2. **Configure SSH access**: - ```bash - # Generate SSH key for storage box - ssh-keygen -t ed25519 -f /root/.ssh/hetzner-storage-box - - # Add public key to storage box in Hetzner Console - cat /root/.ssh/hetzner-storage-box.pub - ``` - -3. **Update backup script**: - Change `BORG_REPO` in `/opt/docfast-borg-backup.sh`: - ```bash - BORG_REPO="ssh://uXXXXXX@uXXXXXX.your-storagebox.de:23/./docfast-backups" - ``` - -4. **Initialize remote repository**: - ```bash - export BORG_PASSPHRASE="docfast-backup-$(date +%Y)" - borg init --encryption=repokey ssh://uXXXXXX@uXXXXXX.your-storagebox.de:23/./docfast-backups - ``` - -### Option 2: AWS S3/Glacier -Use rclone + borg for S3 storage (requires investor approval for AWS costs). - -## Monitoring & Maintenance - -### Check Backup Status -```bash -# View recent backup logs -tail -f /var/log/docfast-backup.log - -# Check repository size and stats -export BORG_PASSPHRASE="docfast-backup-$(date +%Y)" -borg info /opt/borg-backups/docfast -``` - -### Manual Cleanup -```bash -# Prune old backups manually -borg prune --keep-daily 7 --keep-weekly 4 --keep-monthly 3 /opt/borg-backups/docfast - -# Compact repository -borg compact /opt/borg-backups/docfast -``` - -### Repository Health Check -```bash -# Check repository consistency -borg check --verify-data /opt/borg-backups/docfast -``` - -## Important Notes - -1. **Test restores regularly** - Run restore test monthly -2. **Monitor backup logs** - Check for failures in `/var/log/docfast-backup.log` -3. **Keep key safe** - Store `/opt/borg-backups/docfast-key-backup.txt` securely off-site -4. **Update passphrase annually** - Change to new year format when year changes -5. **Local storage limit** - Current server has ~19GB available, monitor usage - -## Migration Timeline -- **Immediate**: Local BorgBackup operational (✅ Complete) -- **Phase 2**: Off-site storage setup (requires Storage Box purchase or AWS approval) -- **Phase 3**: Automated off-site testing and monitoring \ No newline at end of file diff --git a/scripts/borg-backup.sh b/scripts/borg-backup.sh deleted file mode 100755 index cda3ebb..0000000 --- a/scripts/borg-backup.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/bin/bash -# DocFast BorgBackup Script - Full Disaster Recovery -# Backs up: PostgreSQL, Docker volumes, nginx config, SSL certs, crontabs, OpenDKIM keys -# Schedule: daily at 03:00 UTC, keeps 7 daily + 4 weekly + 3 monthly - -set -euo pipefail - -# Configuration -BORG_REPO="/opt/borg-backups/docfast" -BACKUP_NAME="docfast-$(date +%Y-%m-%d_%H%M)" -TEMP_DIR="/tmp/docfast-backup-$$" -LOG_FILE="/var/log/docfast-backup.log" - -# Database configuration -DB_NAME="docfast" -DB_USER="docfast" -DB_HOST="localhost" -DB_PORT="5432" - -# Logging function -log() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE" -} - -# Error handler -error_exit() { - log "ERROR: $1" - cleanup - exit 1 -} - -# Cleanup function -cleanup() { - if [[ -d "$TEMP_DIR" ]]; then - rm -rf "$TEMP_DIR" - fi -} - -# Trap cleanup on exit -trap cleanup EXIT - -log "Starting DocFast backup: $BACKUP_NAME" - -# Create temporary directory -mkdir -p "$TEMP_DIR" -mkdir -p "$(dirname "$LOG_FILE")" - -# 1. PostgreSQL dump -log "Creating PostgreSQL dump..." -export PGPASSFILE="/root/.pgpass" -pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" \ - --no-password --verbose --clean --if-exists --format=custom \ - > "$TEMP_DIR/docfast-db.dump" 2>>"$LOG_FILE" || error_exit "PostgreSQL dump failed" - -# Verify dump is valid -if ! pg_restore --list "$TEMP_DIR/docfast-db.dump" >/dev/null 2>&1; then - error_exit "PostgreSQL dump verification failed" -fi -log "PostgreSQL dump completed: $(stat -c%s "$TEMP_DIR/docfast-db.dump") bytes" - -# 2. Docker volumes -log "Backing up Docker volumes..." -mkdir -p "$TEMP_DIR/docker-volumes" -if [[ -d "/var/lib/docker/volumes" ]]; then - cp -r /var/lib/docker/volumes/* "$TEMP_DIR/docker-volumes/" || error_exit "Docker volumes backup failed" - log "Docker volumes backed up" -else - log "WARNING: No Docker volumes found" -fi - -# 3. Nginx configuration -log "Backing up nginx configuration..." -mkdir -p "$TEMP_DIR/nginx" -cp -r /etc/nginx/* "$TEMP_DIR/nginx/" || error_exit "Nginx backup failed" -log "Nginx configuration backed up" - -# 4. SSL certificates -log "Backing up SSL certificates..." -mkdir -p "$TEMP_DIR/letsencrypt" -cp -r /etc/letsencrypt/* "$TEMP_DIR/letsencrypt/" || error_exit "SSL certificates backup failed" -log "SSL certificates backed up" - -# 5. Crontabs -log "Backing up crontabs..." -mkdir -p "$TEMP_DIR/crontabs" -if [[ -d "/var/spool/cron/crontabs" ]]; then - cp -r /var/spool/cron/crontabs/* "$TEMP_DIR/crontabs/" 2>/dev/null || log "No crontabs found" -fi -# Also backup user crontabs -crontab -l > "$TEMP_DIR/crontabs/root-crontab.txt" 2>/dev/null || echo "# No root crontab" > "$TEMP_DIR/crontabs/root-crontab.txt" -log "Crontabs backed up" - -# 6. OpenDKIM keys -log "Backing up OpenDKIM keys..." -mkdir -p "$TEMP_DIR/opendkim" -cp -r /etc/opendkim/* "$TEMP_DIR/opendkim/" || error_exit "OpenDKIM backup failed" -log "OpenDKIM keys backed up" - -# 7. DocFast application files (docker-compose, env, scripts) -log "Backing up DocFast application files..." -mkdir -p "$TEMP_DIR/docfast-app" -if [[ -d "/opt/docfast" ]]; then - cp /opt/docfast/docker-compose.yml "$TEMP_DIR/docfast-app/" 2>/dev/null || true - cp /opt/docfast/.env "$TEMP_DIR/docfast-app/" 2>/dev/null || true - cp -r /opt/docfast/scripts "$TEMP_DIR/docfast-app/" 2>/dev/null || true - cp -r /opt/docfast/deploy "$TEMP_DIR/docfast-app/" 2>/dev/null || true - log "DocFast application files backed up" -fi - -# 8. System information -log "Creating system information backup..." -mkdir -p "$TEMP_DIR/system" -systemctl list-unit-files --state=enabled > "$TEMP_DIR/system/enabled-services.txt" -dpkg -l > "$TEMP_DIR/system/installed-packages.txt" -uname -a > "$TEMP_DIR/system/system-info.txt" -df -h > "$TEMP_DIR/system/disk-usage.txt" -log "System information backed up" - -# 9. Create Borg backup -log "Creating Borg backup..." -export BORG_PASSPHRASE="docfast-backup-$(date +%Y)" -export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes - -# Initialize repository if it doesn't exist -if [[ ! -d "$BORG_REPO" ]]; then - log "Initializing new Borg repository..." - borg init --encryption=repokey "$BORG_REPO" || error_exit "Failed to initialize Borg repository" -fi - -# Create backup -log "Creating Borg archive: $BACKUP_NAME" -borg create \ - --verbose \ - --filter AME \ - --list \ - --stats \ - --show-rc \ - --compression lz4 \ - --exclude-caches \ - "$BORG_REPO::$BACKUP_NAME" \ - "$TEMP_DIR" 2>>"$LOG_FILE" || error_exit "Borg backup creation failed" - -# 10. Prune old backups (7 daily, 4 weekly, 3 monthly) -log "Pruning old backups..." -borg prune \ - --list \ - --prefix 'docfast-' \ - --show-rc \ - --keep-daily 7 \ - --keep-weekly 4 \ - --keep-monthly 3 \ - "$BORG_REPO" 2>>"$LOG_FILE" || error_exit "Borg pruning failed" - -# 11. Compact repository -log "Compacting repository..." -borg compact "$BORG_REPO" 2>>"$LOG_FILE" || log "WARNING: Repository compaction failed (non-fatal)" - -# 12. Repository info -log "Backup completed successfully!" -borg info "$BORG_REPO" 2>>"$LOG_FILE" - -log "DocFast backup completed: $BACKUP_NAME" \ No newline at end of file diff --git a/scripts/borg-restore.sh b/scripts/borg-restore.sh deleted file mode 100755 index 4da6f8e..0000000 --- a/scripts/borg-restore.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash -# DocFast BorgBackup Restore Script -# Restores from Borg backup for disaster recovery - -set -euo pipefail - -# Configuration -BORG_REPO="/opt/borg-backups/docfast" -RESTORE_DIR="/tmp/docfast-restore-$$" -LOG_FILE="/var/log/docfast-restore.log" - -# Usage function -usage() { - echo "Usage: $0 [list|restore] [archive-name]" - echo " list - List available archives" - echo " restore - Restore specific archive" - echo " restore latest - Restore latest archive" - echo "" - echo "Examples:" - echo " $0 list" - echo " $0 restore docfast-2026-02-15_0300" - echo " $0 restore latest" - exit 1 -} - -# Logging function -log() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE" -} - -# Error handler -error_exit() { - log "ERROR: $1" - cleanup - exit 1 -} - -# Cleanup function -cleanup() { - if [[ -d "$RESTORE_DIR" ]]; then - log "Cleaning up temporary directory: $RESTORE_DIR" - rm -rf "$RESTORE_DIR" - fi -} - -# Trap cleanup on exit -trap cleanup EXIT - -# Check if repository exists -if [[ ! -d "$BORG_REPO" ]]; then - error_exit "Borg repository not found: $BORG_REPO" -fi - -# Set up environment -export BORG_PASSPHRASE="docfast-backup-$(date +%Y)" -export BORG_RELOCATED_REPO_ACCESS_IS_OK=yes -mkdir -p "$(dirname "$LOG_FILE")" - -# Parse command line -case "${1:-}" in - "list") - log "Listing available archives..." - borg list "$BORG_REPO" - exit 0 - ;; - "restore") - ARCHIVE_NAME="${2:-}" - if [[ -z "$ARCHIVE_NAME" ]]; then - usage - fi - - if [[ "$ARCHIVE_NAME" == "latest" ]]; then - log "Finding latest archive..." - ARCHIVE_NAME=$(borg list --short "$BORG_REPO" | grep "^docfast-" | tail -1) - if [[ -z "$ARCHIVE_NAME" ]]; then - error_exit "No archives found in repository" - fi - log "Latest archive found: $ARCHIVE_NAME" - fi - ;; - *) - usage - ;; -esac - -log "Starting restore of archive: $ARCHIVE_NAME" - -# Verify archive exists -if ! borg list "$BORG_REPO::$ARCHIVE_NAME" >/dev/null 2>&1; then - error_exit "Archive not found: $ARCHIVE_NAME" -fi - -# Create restore directory -mkdir -p "$RESTORE_DIR" -log "Restoring to temporary directory: $RESTORE_DIR" - -# Extract archive -log "Extracting archive..." -cd "$RESTORE_DIR" -borg extract --verbose --list "$BORG_REPO::$ARCHIVE_NAME" - -log "Archive extracted successfully. Restore data available at: $RESTORE_DIR" -echo "" -echo "RESTORE LOCATIONS:" -echo "==================" -echo "PostgreSQL dump: $RESTORE_DIR/tmp/docfast-backup-*/docfast-db.dump" -echo "Docker volumes: $RESTORE_DIR/tmp/docfast-backup-*/docker-volumes/" -echo "Nginx config: $RESTORE_DIR/tmp/docfast-backup-*/nginx/" -echo "SSL certificates: $RESTORE_DIR/tmp/docfast-backup-*/letsencrypt/" -echo "Crontabs: $RESTORE_DIR/tmp/docfast-backup-*/crontabs/" -echo "OpenDKIM keys: $RESTORE_DIR/tmp/docfast-backup-*/opendkim/" -echo "DocFast app files: $RESTORE_DIR/tmp/docfast-backup-*/docfast-app/" -echo "System info: $RESTORE_DIR/tmp/docfast-backup-*/system/" -echo "" -echo "MANUAL RESTORE STEPS:" -echo "=====================" -echo "1. Stop DocFast service:" -echo " systemctl stop docker" -echo "" -echo "2. Restore PostgreSQL database:" -echo " sudo -u postgres dropdb docfast" -echo " sudo -u postgres createdb -O docfast docfast" -echo " sudo -u postgres pg_restore -d docfast $RESTORE_DIR/tmp/docfast-backup-*/docfast-db.dump" -echo "" -echo "3. Restore Docker volumes:" -echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/docker-volumes/* /var/lib/docker/volumes/" -echo "" -echo "4. Restore configuration files:" -echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/nginx/* /etc/nginx/" -echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/letsencrypt/* /etc/letsencrypt/" -echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/opendkim/* /etc/opendkim/" -echo " cp -r $RESTORE_DIR/tmp/docfast-backup-*/docfast-app/* /opt/docfast/" -echo "" -echo "5. Restore crontabs:" -echo " cp $RESTORE_DIR/tmp/docfast-backup-*/crontabs/root /var/spool/cron/crontabs/root" -echo " chmod 600 /var/spool/cron/crontabs/root" -echo "" -echo "6. Set correct permissions:" -echo " chown -R opendkim:opendkim /etc/opendkim/keys" -echo " chown -R postgres:postgres /var/lib/postgresql" -echo "" -echo "7. Start services:" -echo " systemctl start postgresql" -echo " systemctl start docker" -echo " cd /opt/docfast && docker-compose up -d" -echo "" -echo "WARNING: This script does NOT automatically restore files to prevent" -echo "accidental overwrites. Follow the manual steps above carefully." - -log "Restore extraction completed. Follow manual steps to complete restoration." \ No newline at end of file diff --git a/src/routes/health.ts b/src/routes/health.ts index 5d27dbb..bbee73a 100644 --- a/src/routes/health.ts +++ b/src/routes/health.ts @@ -1,55 +1,21 @@ import { Router } from "express"; import { getPoolStats } from "../services/browser.js"; -import { pool } from "../services/db.js"; export const healthRouter = Router(); -healthRouter.get("/", async (_req, res) => { - const poolStats = getPoolStats(); - let databaseStatus: any; - let overallStatus = "ok"; - let httpStatus = 200; - - // Check database connectivity - try { - const client = await pool.connect(); - try { - const result = await client.query('SELECT version()'); - const version = result.rows[0]?.version || 'Unknown'; - // Extract just the PostgreSQL version number (e.g., "PostgreSQL 15.4") - const versionMatch = version.match(/PostgreSQL ([\d.]+)/); - const shortVersion = versionMatch ? `PostgreSQL ${versionMatch[1]}` : 'PostgreSQL'; - - databaseStatus = { - status: "ok", - version: shortVersion - }; - } finally { - client.release(); - } - } catch (error: any) { - databaseStatus = { - status: "error", - message: error.message || "Database connection failed" - }; - overallStatus = "degraded"; - httpStatus = 503; - } - - const response = { - status: overallStatus, +healthRouter.get("/", (_req, res) => { + const pool = getPoolStats(); + res.json({ + status: "ok", version: "0.2.1", - database: databaseStatus, pool: { - size: poolStats.poolSize, - active: poolStats.totalPages - poolStats.availablePages, - available: poolStats.availablePages, - queueDepth: poolStats.queueDepth, - pdfCount: poolStats.pdfCount, - restarting: poolStats.restarting, - uptimeSeconds: Math.round(poolStats.uptimeMs / 1000), + size: pool.poolSize, + active: pool.totalPages - pool.availablePages, + available: pool.availablePages, + queueDepth: pool.queueDepth, + pdfCount: pool.pdfCount, + restarting: pool.restarting, + uptimeSeconds: Math.round(pool.uptimeMs / 1000), }, - }; - - res.status(httpStatus).json(response); -}); \ No newline at end of file + }); +}); diff --git a/src/routes/health.ts.backup b/src/routes/health.ts.backup deleted file mode 100644 index bbee73a..0000000 --- a/src/routes/health.ts.backup +++ /dev/null @@ -1,21 +0,0 @@ -import { Router } from "express"; -import { getPoolStats } from "../services/browser.js"; - -export const healthRouter = Router(); - -healthRouter.get("/", (_req, res) => { - const pool = getPoolStats(); - res.json({ - status: "ok", - version: "0.2.1", - pool: { - size: pool.poolSize, - active: pool.totalPages - pool.availablePages, - available: pool.availablePages, - queueDepth: pool.queueDepth, - pdfCount: pool.pdfCount, - restarting: pool.restarting, - uptimeSeconds: Math.round(pool.uptimeMs / 1000), - }, - }); -});