fix: compile TypeScript in Docker build — dist/ was never built in CI, connection resilience code was missing from images

Dockerfile

@@ -13,11 +13,15 @@ RUN groupadd --gid 1001 docfast \
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 
+# Build stage - compile TypeScript
 WORKDIR /app
-COPY package*.json ./
-RUN npm install --omit=dev
+COPY package*.json tsconfig.json ./
+RUN npm install
+COPY src/ src/
+RUN npx tsc
 
-COPY dist/ dist/
+# Remove dev dependencies
+RUN npm prune --omit=dev
 COPY scripts/ scripts/
 COPY public/ public/
 RUN node scripts/build-html.cjs

dist/middleware/usage.js

@@ -1,6 +1,6 @@
 import { isProKey } from "../services/keys.js";
 import logger from "../services/logger.js";
-import pool from "../services/db.js";
+import { queryWithRetry, connectWithRetry } from "../services/db.js";
 const FREE_TIER_LIMIT = 100;
 const PRO_TIER_LIMIT = 5000;
 // In-memory cache, periodically synced to PostgreSQL
@@ -17,7 +17,7 @@ function getMonthKey() {
 }
 export async function loadUsageData() {
     try {
-        const result = await pool.query("SELECT key, count, month_key FROM usage");
+        const result = await queryWithRetry("SELECT key, count, month_key FROM usage");
         usage = new Map();
         for (const row of result.rows) {
             usage.set(row.key, { count: row.count, monthKey: row.month_key });
@@ -34,7 +34,7 @@ async function flushDirtyEntries() {
     if (dirtyKeys.size === 0)
         return;
     const keysToFlush = [...dirtyKeys];
-    const client = await pool.connect();
+    const client = await connectWithRetry();
     try {
         await client.query("BEGIN");
         for (const key of keysToFlush) {

dist/routes/health.js

@@ -5,28 +5,37 @@ import { pool } from "../services/db.js";
 const require = createRequire(import.meta.url);
 const { version: APP_VERSION } = require("../../package.json");
 export const healthRouter = Router();
+const HEALTH_CHECK_TIMEOUT_MS = 3000;
 healthRouter.get("/", async (_req, res) => {
     const poolStats = getPoolStats();
     let databaseStatus;
     let overallStatus = "ok";
     let httpStatus = 200;
-    // Check database connectivity
+    // Check database connectivity with a real query and timeout
     try {
-        const client = await pool.connect();
-        try {
-            const result = await client.query('SELECT version()');
-            const version = result.rows[0]?.version || 'Unknown';
-            // Extract just the PostgreSQL version number (e.g., "PostgreSQL 15.4")
-            const versionMatch = version.match(/PostgreSQL ([\d.]+)/);
-            const shortVersion = versionMatch ? `PostgreSQL ${versionMatch[1]}` : 'PostgreSQL';
-            databaseStatus = {
-                status: "ok",
-                version: shortVersion
-            };
-        }
-        finally {
-            client.release();
-        }
+        const dbCheck = async () => {
+            const client = await pool.connect();
+            try {
+                // Use SELECT 1 as a lightweight liveness probe
+                await client.query('SELECT 1');
+                const result = await client.query('SELECT version()');
+                const version = result.rows[0]?.version || 'Unknown';
+                const versionMatch = version.match(/PostgreSQL ([\d.]+)/);
+                const shortVersion = versionMatch ? `PostgreSQL ${versionMatch[1]}` : 'PostgreSQL';
+                client.release();
+                return { status: "ok", version: shortVersion };
+            }
+            catch (queryErr) {
+                // Destroy the bad connection so it doesn't go back to the pool
+                try {
+                    client.release(true);
+                }
+                catch (_) { }
+                throw queryErr;
+            }
+        };
+        const timeout = new Promise((_resolve, reject) => setTimeout(() => reject(new Error("Database health check timed out")), HEALTH_CHECK_TIMEOUT_MS));
+        databaseStatus = await Promise.race([dbCheck(), timeout]);
     }
     catch (error) {
         databaseStatus = {

dist/services/db.js

@@ -1,6 +1,20 @@
 import pg from "pg";
 import logger from "./logger.js";
 const { Pool } = pg;
+// Transient error codes from PgBouncer / PostgreSQL that warrant retry
+const TRANSIENT_ERRORS = new Set([
+    "ECONNRESET",
+    "ECONNREFUSED",
+    "EPIPE",
+    "ETIMEDOUT",
+    "CONNECTION_LOST",
+    "57P01", // admin_shutdown
+    "57P02", // crash_shutdown
+    "57P03", // cannot_connect_now
+    "08006", // connection_failure
+    "08003", // connection_does_not_exist
+    "08001", // sqlclient_unable_to_establish_sqlconnection
+]);
 const pool = new Pool({
     host: process.env.DATABASE_HOST || "172.17.0.1",
     port: parseInt(process.env.DATABASE_PORT || "5432", 10),
@@ -8,13 +22,119 @@ const pool = new Pool({
     user: process.env.DATABASE_USER || "docfast",
     password: process.env.DATABASE_PASSWORD || "docfast",
     max: 10,
-    idleTimeoutMillis: 30000,
+    idleTimeoutMillis: 10000, // Evict idle connections after 10s (was 30s) — faster cleanup of stale sockets
+    connectionTimeoutMillis: 5000, // Don't wait forever for a connection
+    allowExitOnIdle: false,
+    keepAlive: true, // TCP keepalive to detect dead connections
+    keepAliveInitialDelayMillis: 10000, // Start keepalive probes after 10s idle
 });
-pool.on("error", (err) => {
-    logger.error({ err }, "Unexpected PostgreSQL pool error");
+// Handle errors on idle clients — pg.Pool automatically removes the client
+// after emitting this event, so we just log it.
+pool.on("error", (err, client) => {
+    logger.error({ err }, "Unexpected error on idle PostgreSQL client — evicted from pool");
 });
+/**
+ * Determine if an error is transient (PgBouncer failover, network blip)
+ */
+export function isTransientError(err) {
+    if (!err)
+        return false;
+    const code = err.code || "";
+    const msg = (err.message || "").toLowerCase();
+    if (TRANSIENT_ERRORS.has(code))
+        return true;
+    if (msg.includes("no available server"))
+        return true; // PgBouncer specific
+    if (msg.includes("connection terminated"))
+        return true;
+    if (msg.includes("connection refused"))
+        return true;
+    if (msg.includes("server closed the connection"))
+        return true;
+    if (msg.includes("timeout expired"))
+        return true;
+    return false;
+}
+/**
+ * Execute a query with automatic retry on transient errors.
+ *
+ * KEY FIX: On transient error, we destroy the bad connection (client.release(true))
+ * so the pool creates a fresh TCP connection on the next attempt, instead of
+ * reusing a dead socket from the pool.
+ */
+export async function queryWithRetry(queryText, params, maxRetries = 3) {
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        let client;
+        try {
+            client = await pool.connect();
+            const result = await client.query(queryText, params);
+            client.release(); // Return healthy connection to pool
+            return result;
+        }
+        catch (err) {
+            // Destroy the bad connection so pool doesn't reuse it
+            if (client) {
+                try {
+                    client.release(true);
+                }
+                catch (_) { /* already destroyed */ }
+            }
+            lastError = err;
+            if (!isTransientError(err) || attempt === maxRetries) {
+                throw err;
+            }
+            const delayMs = Math.min(1000 * Math.pow(2, attempt), 5000); // 1s, 2s, 4s (capped at 5s)
+            logger.warn({ err: err.message, code: err.code, attempt: attempt + 1, maxRetries, delayMs }, "Transient DB error, destroying bad connection and retrying...");
+            await new Promise(resolve => setTimeout(resolve, delayMs));
+        }
+    }
+    throw lastError;
+}
+/**
+ * Connect with retry — for operations that need a client (transactions).
+ * On transient connect errors, waits and retries so the pool can establish
+ * fresh connections to the new PgBouncer pod.
+ */
+export async function connectWithRetry(maxRetries = 3) {
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            const client = await pool.connect();
+            // Validate the connection is actually alive
+            try {
+                await client.query("SELECT 1");
+            }
+            catch (validationErr) {
+                // Connection is dead — destroy it and retry
+                try {
+                    client.release(true);
+                }
+                catch (_) { }
+                if (!isTransientError(validationErr) || attempt === maxRetries) {
+                    throw validationErr;
+                }
+                const delayMs = Math.min(1000 * Math.pow(2, attempt), 5000);
+                logger.warn({ err: validationErr.message, code: validationErr.code, attempt: attempt + 1 }, "Connection validation failed, destroying and retrying...");
+                await new Promise(resolve => setTimeout(resolve, delayMs));
+                continue;
+            }
+            return client;
+        }
+        catch (err) {
+            lastError = err;
+            if (!isTransientError(err) || attempt === maxRetries) {
+                throw err;
+            }
+            const delayMs = Math.min(1000 * Math.pow(2, attempt), 5000);
+            logger.warn({ err: err.message, code: err.code, attempt: attempt + 1, maxRetries, delayMs }, "Transient DB connect error, retrying...");
+            await new Promise(resolve => setTimeout(resolve, delayMs));
+        }
+    }
+    throw lastError;
+}
 export async function initDatabase() {
-    const client = await pool.connect();
+    const client = await connectWithRetry();
     try {
         await client.query(`
       CREATE TABLE IF NOT EXISTS api_keys (

dist/services/keys.js

@@ -1,11 +1,11 @@
 import { randomBytes } from "crypto";
 import logger from "./logger.js";
-import pool from "./db.js";
+import { queryWithRetry } from "./db.js";
 // In-memory cache for fast lookups, synced with PostgreSQL
 let keysCache = [];
 export async function loadKeys() {
     try {
-        const result = await pool.query("SELECT key, tier, email, created_at, stripe_customer_id FROM api_keys");
+        const result = await queryWithRetry("SELECT key, tier, email, created_at, stripe_customer_id FROM api_keys");
         keysCache = result.rows.map((r) => ({
             key: r.key,
             tier: r.tier,
@@ -25,7 +25,7 @@ export async function loadKeys() {
             const entry = { key: k, tier: "pro", email: "seed@docfast.dev", createdAt: new Date().toISOString() };
             keysCache.push(entry);
             // Upsert into DB
-            await pool.query(`INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4)
+            await queryWithRetry(`INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4)
          ON CONFLICT (key) DO NOTHING`, [k, "pro", "seed@docfast.dev", new Date().toISOString()]).catch(() => { });
         }
     }
@@ -55,7 +55,7 @@ export async function createFreeKey(email) {
         email: email || "",
         createdAt: new Date().toISOString(),
     };
-    await pool.query("INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4)", [entry.key, entry.tier, entry.email, entry.createdAt]);
+    await queryWithRetry("INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4)", [entry.key, entry.tier, entry.email, entry.createdAt]);
     keysCache.push(entry);
     return entry;
 }
@@ -63,7 +63,7 @@ export async function createProKey(email, stripeCustomerId) {
     const existing = keysCache.find((k) => k.stripeCustomerId === stripeCustomerId);
     if (existing) {
         existing.tier = "pro";
-        await pool.query("UPDATE api_keys SET tier = 'pro' WHERE key = $1", [existing.key]);
+        await queryWithRetry("UPDATE api_keys SET tier = 'pro' WHERE key = $1", [existing.key]);
         return existing;
     }
     const entry = {
@@ -73,7 +73,7 @@ export async function createProKey(email, stripeCustomerId) {
         createdAt: new Date().toISOString(),
         stripeCustomerId,
     };
-    await pool.query("INSERT INTO api_keys (key, tier, email, created_at, stripe_customer_id) VALUES ($1, $2, $3, $4, $5)", [entry.key, entry.tier, entry.email, entry.createdAt, entry.stripeCustomerId]);
+    await queryWithRetry("INSERT INTO api_keys (key, tier, email, created_at, stripe_customer_id) VALUES ($1, $2, $3, $4, $5)", [entry.key, entry.tier, entry.email, entry.createdAt, entry.stripeCustomerId]);
     keysCache.push(entry);
     return entry;
 }
@@ -81,7 +81,7 @@ export async function downgradeByCustomer(stripeCustomerId) {
     const entry = keysCache.find((k) => k.stripeCustomerId === stripeCustomerId);
     if (entry) {
         entry.tier = "free";
-        await pool.query("UPDATE api_keys SET tier = 'free' WHERE stripe_customer_id = $1", [stripeCustomerId]);
+        await queryWithRetry("UPDATE api_keys SET tier = 'free' WHERE stripe_customer_id = $1", [stripeCustomerId]);
         return true;
     }
     return false;
@@ -94,7 +94,7 @@ export async function updateKeyEmail(apiKey, newEmail) {
     if (!entry)
         return false;
     entry.email = newEmail;
-    await pool.query("UPDATE api_keys SET email = $1 WHERE key = $2", [newEmail, apiKey]);
+    await queryWithRetry("UPDATE api_keys SET email = $1 WHERE key = $2", [newEmail, apiKey]);
     return true;
 }
 export async function updateEmailByCustomer(stripeCustomerId, newEmail) {
@@ -102,6 +102,6 @@ export async function updateEmailByCustomer(stripeCustomerId, newEmail) {
     if (!entry)
         return false;
     entry.email = newEmail;
-    await pool.query("UPDATE api_keys SET email = $1 WHERE stripe_customer_id = $2", [newEmail, stripeCustomerId]);
+    await queryWithRetry("UPDATE api_keys SET email = $1 WHERE stripe_customer_id = $2", [newEmail, stripeCustomerId]);
     return true;
 }

dist/services/verification.js

@@ -1,21 +1,21 @@
 import { randomBytes, randomInt, timingSafeEqual } from "crypto";
 import logger from "./logger.js";
-import pool from "./db.js";
+import { queryWithRetry } from "./db.js";
 const TOKEN_EXPIRY_MS = 24 * 60 * 60 * 1000;
 const CODE_EXPIRY_MS = 15 * 60 * 1000;
 const MAX_ATTEMPTS = 3;
 export async function createVerification(email, apiKey) {
     // Check for existing unexpired, unverified
-    const existing = await pool.query("SELECT * FROM verifications WHERE email = $1 AND verified_at IS NULL AND created_at > NOW() - INTERVAL '24 hours' LIMIT 1", [email]);
+    const existing = await queryWithRetry("SELECT * FROM verifications WHERE email = $1 AND verified_at IS NULL AND created_at > NOW() - INTERVAL '24 hours' LIMIT 1", [email]);
     if (existing.rows.length > 0) {
         const r = existing.rows[0];
         return { email: r.email, token: r.token, apiKey: r.api_key, createdAt: r.created_at.toISOString(), verifiedAt: null };
     }
     // Remove old unverified
-    await pool.query("DELETE FROM verifications WHERE email = $1 AND verified_at IS NULL", [email]);
+    await queryWithRetry("DELETE FROM verifications WHERE email = $1 AND verified_at IS NULL", [email]);
     const token = randomBytes(32).toString("hex");
     const now = new Date().toISOString();
-    await pool.query("INSERT INTO verifications (email, token, api_key, created_at) VALUES ($1, $2, $3, $4)", [email, token, apiKey, now]);
+    await queryWithRetry("INSERT INTO verifications (email, token, api_key, created_at) VALUES ($1, $2, $3, $4)", [email, token, apiKey, now]);
     return { email, token, apiKey, createdAt: now, verifiedAt: null };
 }
 export function verifyToken(token) {
@@ -27,7 +27,7 @@ export function verifyToken(token) {
 // In-memory cache for verifications (loaded on startup, updated on changes)
 let verificationsCache = [];
 export async function loadVerifications() {
-    const result = await pool.query("SELECT * FROM verifications");
+    const result = await queryWithRetry("SELECT * FROM verifications");
     verificationsCache = result.rows.map((r) => ({
         email: r.email,
         token: r.token,
@@ -56,11 +56,11 @@ function verifyTokenSync(token) {
         return { status: "expired" };
     v.verifiedAt = new Date().toISOString();
     // Update DB async
-    pool.query("UPDATE verifications SET verified_at = $1 WHERE token = $2", [v.verifiedAt, token]).catch((err) => logger.error({ err }, "Failed to update verification"));
+    queryWithRetry("UPDATE verifications SET verified_at = $1 WHERE token = $2", [v.verifiedAt, token]).catch((err) => logger.error({ err }, "Failed to update verification"));
     return { status: "ok", verification: v };
 }
 export async function createPendingVerification(email) {
-    await pool.query("DELETE FROM pending_verifications WHERE email = $1", [email]);
+    await queryWithRetry("DELETE FROM pending_verifications WHERE email = $1", [email]);
     const now = new Date();
     const pending = {
         email,
@@ -69,38 +69,38 @@ export async function createPendingVerification(email) {
         expiresAt: new Date(now.getTime() + CODE_EXPIRY_MS).toISOString(),
         attempts: 0,
     };
-    await pool.query("INSERT INTO pending_verifications (email, code, created_at, expires_at, attempts) VALUES ($1, $2, $3, $4, $5)", [pending.email, pending.code, pending.createdAt, pending.expiresAt, pending.attempts]);
+    await queryWithRetry("INSERT INTO pending_verifications (email, code, created_at, expires_at, attempts) VALUES ($1, $2, $3, $4, $5)", [pending.email, pending.code, pending.createdAt, pending.expiresAt, pending.attempts]);
     return pending;
 }
 export async function verifyCode(email, code) {
     const cleanEmail = email.trim().toLowerCase();
-    const result = await pool.query("SELECT * FROM pending_verifications WHERE email = $1", [cleanEmail]);
+    const result = await queryWithRetry("SELECT * FROM pending_verifications WHERE email = $1", [cleanEmail]);
     const pending = result.rows[0];
     if (!pending)
         return { status: "invalid" };
     if (new Date() > new Date(pending.expires_at)) {
-        await pool.query("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]);
+        await queryWithRetry("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]);
         return { status: "expired" };
     }
     if (pending.attempts >= MAX_ATTEMPTS) {
-        await pool.query("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]);
+        await queryWithRetry("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]);
         return { status: "max_attempts" };
     }
-    await pool.query("UPDATE pending_verifications SET attempts = attempts + 1 WHERE email = $1", [cleanEmail]);
+    await queryWithRetry("UPDATE pending_verifications SET attempts = attempts + 1 WHERE email = $1", [cleanEmail]);
     const a = Buffer.from(pending.code, "utf8");
     const b = Buffer.from(code, "utf8");
     const codeMatch = a.length === b.length && timingSafeEqual(a, b);
     if (!codeMatch) {
         return { status: "invalid" };
     }
-    await pool.query("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]);
+    await queryWithRetry("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]);
     return { status: "ok" };
 }
 export async function isEmailVerified(email) {
-    const result = await pool.query("SELECT 1 FROM verifications WHERE email = $1 AND verified_at IS NOT NULL LIMIT 1", [email]);
+    const result = await queryWithRetry("SELECT 1 FROM verifications WHERE email = $1 AND verified_at IS NOT NULL LIMIT 1", [email]);
     return result.rows.length > 0;
 }
 export async function getVerifiedApiKey(email) {
-    const result = await pool.query("SELECT api_key FROM verifications WHERE email = $1 AND verified_at IS NOT NULL LIMIT 1", [email]);
+    const result = await queryWithRetry("SELECT api_key FROM verifications WHERE email = $1 AND verified_at IS NOT NULL LIMIT 1", [email]);
     return result.rows[0]?.api_key ?? null;
 }