From e611609580329c2654014c2180f9dd7035ee557e Mon Sep 17 00:00:00 2001 From: OpenClaw Deployer Date: Wed, 18 Feb 2026 16:19:59 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20compile=20TypeScript=20in=20Docker=20bui?= =?UTF-8?q?ld=20=E2=80=94=20dist/=20was=20never=20built=20in=20CI,=20conne?= =?UTF-8?q?ction=20resilience=20code=20was=20missing=20from=20images?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 10 ++- dist/middleware/usage.js | 6 +- dist/routes/health.js | 41 ++++++----- dist/services/db.js | 128 ++++++++++++++++++++++++++++++++-- dist/services/keys.js | 18 ++--- dist/services/verification.js | 30 ++++---- 6 files changed, 183 insertions(+), 50 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1143405..16c8ef5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,15 @@ RUN groupadd --gid 1001 docfast \ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium +# Build stage - compile TypeScript WORKDIR /app -COPY package*.json ./ -RUN npm install --omit=dev +COPY package*.json tsconfig.json ./ +RUN npm install +COPY src/ src/ +RUN npx tsc -COPY dist/ dist/ +# Remove dev dependencies +RUN npm prune --omit=dev COPY scripts/ scripts/ COPY public/ public/ RUN node scripts/build-html.cjs diff --git a/dist/middleware/usage.js b/dist/middleware/usage.js index c3251af..6dd2f5e 100644 --- a/dist/middleware/usage.js +++ b/dist/middleware/usage.js @@ -1,6 +1,6 @@ import { isProKey } from "../services/keys.js"; import logger from "../services/logger.js"; -import pool from "../services/db.js"; +import { queryWithRetry, connectWithRetry } from "../services/db.js"; const FREE_TIER_LIMIT = 100; const PRO_TIER_LIMIT = 5000; // In-memory cache, periodically synced to PostgreSQL @@ -17,7 +17,7 @@ function getMonthKey() { } export async function loadUsageData() { try { - const result = await pool.query("SELECT key, count, month_key FROM usage"); + const result = await queryWithRetry("SELECT key, count, month_key FROM usage"); usage = new Map(); for (const row of result.rows) { usage.set(row.key, { count: row.count, monthKey: row.month_key }); @@ -34,7 +34,7 @@ async function flushDirtyEntries() { if (dirtyKeys.size === 0) return; const keysToFlush = [...dirtyKeys]; - const client = await pool.connect(); + const client = await connectWithRetry(); try { await client.query("BEGIN"); for (const key of keysToFlush) { diff --git a/dist/routes/health.js b/dist/routes/health.js index 700dd4b..cf3a146 100644 --- a/dist/routes/health.js +++ b/dist/routes/health.js @@ -5,28 +5,37 @@ import { pool } from "../services/db.js"; const require = createRequire(import.meta.url); const { version: APP_VERSION } = require("../../package.json"); export const healthRouter = Router(); +const HEALTH_CHECK_TIMEOUT_MS = 3000; healthRouter.get("/", async (_req, res) => { const poolStats = getPoolStats(); let databaseStatus; let overallStatus = "ok"; let httpStatus = 200; - // Check database connectivity + // Check database connectivity with a real query and timeout try { - const client = await pool.connect(); - try { - const result = await client.query('SELECT version()'); - const version = result.rows[0]?.version || 'Unknown'; - // Extract just the PostgreSQL version number (e.g., "PostgreSQL 15.4") - const versionMatch = version.match(/PostgreSQL ([\d.]+)/); - const shortVersion = versionMatch ? `PostgreSQL ${versionMatch[1]}` : 'PostgreSQL'; - databaseStatus = { - status: "ok", - version: shortVersion - }; - } - finally { - client.release(); - } + const dbCheck = async () => { + const client = await pool.connect(); + try { + // Use SELECT 1 as a lightweight liveness probe + await client.query('SELECT 1'); + const result = await client.query('SELECT version()'); + const version = result.rows[0]?.version || 'Unknown'; + const versionMatch = version.match(/PostgreSQL ([\d.]+)/); + const shortVersion = versionMatch ? `PostgreSQL ${versionMatch[1]}` : 'PostgreSQL'; + client.release(); + return { status: "ok", version: shortVersion }; + } + catch (queryErr) { + // Destroy the bad connection so it doesn't go back to the pool + try { + client.release(true); + } + catch (_) { } + throw queryErr; + } + }; + const timeout = new Promise((_resolve, reject) => setTimeout(() => reject(new Error("Database health check timed out")), HEALTH_CHECK_TIMEOUT_MS)); + databaseStatus = await Promise.race([dbCheck(), timeout]); } catch (error) { databaseStatus = { diff --git a/dist/services/db.js b/dist/services/db.js index 4d4b85a..1814736 100644 --- a/dist/services/db.js +++ b/dist/services/db.js @@ -1,6 +1,20 @@ import pg from "pg"; import logger from "./logger.js"; const { Pool } = pg; +// Transient error codes from PgBouncer / PostgreSQL that warrant retry +const TRANSIENT_ERRORS = new Set([ + "ECONNRESET", + "ECONNREFUSED", + "EPIPE", + "ETIMEDOUT", + "CONNECTION_LOST", + "57P01", // admin_shutdown + "57P02", // crash_shutdown + "57P03", // cannot_connect_now + "08006", // connection_failure + "08003", // connection_does_not_exist + "08001", // sqlclient_unable_to_establish_sqlconnection +]); const pool = new Pool({ host: process.env.DATABASE_HOST || "172.17.0.1", port: parseInt(process.env.DATABASE_PORT || "5432", 10), @@ -8,13 +22,119 @@ const pool = new Pool({ user: process.env.DATABASE_USER || "docfast", password: process.env.DATABASE_PASSWORD || "docfast", max: 10, - idleTimeoutMillis: 30000, + idleTimeoutMillis: 10000, // Evict idle connections after 10s (was 30s) — faster cleanup of stale sockets + connectionTimeoutMillis: 5000, // Don't wait forever for a connection + allowExitOnIdle: false, + keepAlive: true, // TCP keepalive to detect dead connections + keepAliveInitialDelayMillis: 10000, // Start keepalive probes after 10s idle }); -pool.on("error", (err) => { - logger.error({ err }, "Unexpected PostgreSQL pool error"); +// Handle errors on idle clients — pg.Pool automatically removes the client +// after emitting this event, so we just log it. +pool.on("error", (err, client) => { + logger.error({ err }, "Unexpected error on idle PostgreSQL client — evicted from pool"); }); +/** + * Determine if an error is transient (PgBouncer failover, network blip) + */ +export function isTransientError(err) { + if (!err) + return false; + const code = err.code || ""; + const msg = (err.message || "").toLowerCase(); + if (TRANSIENT_ERRORS.has(code)) + return true; + if (msg.includes("no available server")) + return true; // PgBouncer specific + if (msg.includes("connection terminated")) + return true; + if (msg.includes("connection refused")) + return true; + if (msg.includes("server closed the connection")) + return true; + if (msg.includes("timeout expired")) + return true; + return false; +} +/** + * Execute a query with automatic retry on transient errors. + * + * KEY FIX: On transient error, we destroy the bad connection (client.release(true)) + * so the pool creates a fresh TCP connection on the next attempt, instead of + * reusing a dead socket from the pool. + */ +export async function queryWithRetry(queryText, params, maxRetries = 3) { + let lastError; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + let client; + try { + client = await pool.connect(); + const result = await client.query(queryText, params); + client.release(); // Return healthy connection to pool + return result; + } + catch (err) { + // Destroy the bad connection so pool doesn't reuse it + if (client) { + try { + client.release(true); + } + catch (_) { /* already destroyed */ } + } + lastError = err; + if (!isTransientError(err) || attempt === maxRetries) { + throw err; + } + const delayMs = Math.min(1000 * Math.pow(2, attempt), 5000); // 1s, 2s, 4s (capped at 5s) + logger.warn({ err: err.message, code: err.code, attempt: attempt + 1, maxRetries, delayMs }, "Transient DB error, destroying bad connection and retrying..."); + await new Promise(resolve => setTimeout(resolve, delayMs)); + } + } + throw lastError; +} +/** + * Connect with retry — for operations that need a client (transactions). + * On transient connect errors, waits and retries so the pool can establish + * fresh connections to the new PgBouncer pod. + */ +export async function connectWithRetry(maxRetries = 3) { + let lastError; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + const client = await pool.connect(); + // Validate the connection is actually alive + try { + await client.query("SELECT 1"); + } + catch (validationErr) { + // Connection is dead — destroy it and retry + try { + client.release(true); + } + catch (_) { } + if (!isTransientError(validationErr) || attempt === maxRetries) { + throw validationErr; + } + const delayMs = Math.min(1000 * Math.pow(2, attempt), 5000); + logger.warn({ err: validationErr.message, code: validationErr.code, attempt: attempt + 1 }, "Connection validation failed, destroying and retrying..."); + await new Promise(resolve => setTimeout(resolve, delayMs)); + continue; + } + return client; + } + catch (err) { + lastError = err; + if (!isTransientError(err) || attempt === maxRetries) { + throw err; + } + const delayMs = Math.min(1000 * Math.pow(2, attempt), 5000); + logger.warn({ err: err.message, code: err.code, attempt: attempt + 1, maxRetries, delayMs }, "Transient DB connect error, retrying..."); + await new Promise(resolve => setTimeout(resolve, delayMs)); + } + } + throw lastError; +} export async function initDatabase() { - const client = await pool.connect(); + const client = await connectWithRetry(); try { await client.query(` CREATE TABLE IF NOT EXISTS api_keys ( diff --git a/dist/services/keys.js b/dist/services/keys.js index 6b751fc..88ad4a4 100644 --- a/dist/services/keys.js +++ b/dist/services/keys.js @@ -1,11 +1,11 @@ import { randomBytes } from "crypto"; import logger from "./logger.js"; -import pool from "./db.js"; +import { queryWithRetry } from "./db.js"; // In-memory cache for fast lookups, synced with PostgreSQL let keysCache = []; export async function loadKeys() { try { - const result = await pool.query("SELECT key, tier, email, created_at, stripe_customer_id FROM api_keys"); + const result = await queryWithRetry("SELECT key, tier, email, created_at, stripe_customer_id FROM api_keys"); keysCache = result.rows.map((r) => ({ key: r.key, tier: r.tier, @@ -25,7 +25,7 @@ export async function loadKeys() { const entry = { key: k, tier: "pro", email: "seed@docfast.dev", createdAt: new Date().toISOString() }; keysCache.push(entry); // Upsert into DB - await pool.query(`INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4) + await queryWithRetry(`INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4) ON CONFLICT (key) DO NOTHING`, [k, "pro", "seed@docfast.dev", new Date().toISOString()]).catch(() => { }); } } @@ -55,7 +55,7 @@ export async function createFreeKey(email) { email: email || "", createdAt: new Date().toISOString(), }; - await pool.query("INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4)", [entry.key, entry.tier, entry.email, entry.createdAt]); + await queryWithRetry("INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4)", [entry.key, entry.tier, entry.email, entry.createdAt]); keysCache.push(entry); return entry; } @@ -63,7 +63,7 @@ export async function createProKey(email, stripeCustomerId) { const existing = keysCache.find((k) => k.stripeCustomerId === stripeCustomerId); if (existing) { existing.tier = "pro"; - await pool.query("UPDATE api_keys SET tier = 'pro' WHERE key = $1", [existing.key]); + await queryWithRetry("UPDATE api_keys SET tier = 'pro' WHERE key = $1", [existing.key]); return existing; } const entry = { @@ -73,7 +73,7 @@ export async function createProKey(email, stripeCustomerId) { createdAt: new Date().toISOString(), stripeCustomerId, }; - await pool.query("INSERT INTO api_keys (key, tier, email, created_at, stripe_customer_id) VALUES ($1, $2, $3, $4, $5)", [entry.key, entry.tier, entry.email, entry.createdAt, entry.stripeCustomerId]); + await queryWithRetry("INSERT INTO api_keys (key, tier, email, created_at, stripe_customer_id) VALUES ($1, $2, $3, $4, $5)", [entry.key, entry.tier, entry.email, entry.createdAt, entry.stripeCustomerId]); keysCache.push(entry); return entry; } @@ -81,7 +81,7 @@ export async function downgradeByCustomer(stripeCustomerId) { const entry = keysCache.find((k) => k.stripeCustomerId === stripeCustomerId); if (entry) { entry.tier = "free"; - await pool.query("UPDATE api_keys SET tier = 'free' WHERE stripe_customer_id = $1", [stripeCustomerId]); + await queryWithRetry("UPDATE api_keys SET tier = 'free' WHERE stripe_customer_id = $1", [stripeCustomerId]); return true; } return false; @@ -94,7 +94,7 @@ export async function updateKeyEmail(apiKey, newEmail) { if (!entry) return false; entry.email = newEmail; - await pool.query("UPDATE api_keys SET email = $1 WHERE key = $2", [newEmail, apiKey]); + await queryWithRetry("UPDATE api_keys SET email = $1 WHERE key = $2", [newEmail, apiKey]); return true; } export async function updateEmailByCustomer(stripeCustomerId, newEmail) { @@ -102,6 +102,6 @@ export async function updateEmailByCustomer(stripeCustomerId, newEmail) { if (!entry) return false; entry.email = newEmail; - await pool.query("UPDATE api_keys SET email = $1 WHERE stripe_customer_id = $2", [newEmail, stripeCustomerId]); + await queryWithRetry("UPDATE api_keys SET email = $1 WHERE stripe_customer_id = $2", [newEmail, stripeCustomerId]); return true; } diff --git a/dist/services/verification.js b/dist/services/verification.js index 92e36d8..c61e4b6 100644 --- a/dist/services/verification.js +++ b/dist/services/verification.js @@ -1,21 +1,21 @@ import { randomBytes, randomInt, timingSafeEqual } from "crypto"; import logger from "./logger.js"; -import pool from "./db.js"; +import { queryWithRetry } from "./db.js"; const TOKEN_EXPIRY_MS = 24 * 60 * 60 * 1000; const CODE_EXPIRY_MS = 15 * 60 * 1000; const MAX_ATTEMPTS = 3; export async function createVerification(email, apiKey) { // Check for existing unexpired, unverified - const existing = await pool.query("SELECT * FROM verifications WHERE email = $1 AND verified_at IS NULL AND created_at > NOW() - INTERVAL '24 hours' LIMIT 1", [email]); + const existing = await queryWithRetry("SELECT * FROM verifications WHERE email = $1 AND verified_at IS NULL AND created_at > NOW() - INTERVAL '24 hours' LIMIT 1", [email]); if (existing.rows.length > 0) { const r = existing.rows[0]; return { email: r.email, token: r.token, apiKey: r.api_key, createdAt: r.created_at.toISOString(), verifiedAt: null }; } // Remove old unverified - await pool.query("DELETE FROM verifications WHERE email = $1 AND verified_at IS NULL", [email]); + await queryWithRetry("DELETE FROM verifications WHERE email = $1 AND verified_at IS NULL", [email]); const token = randomBytes(32).toString("hex"); const now = new Date().toISOString(); - await pool.query("INSERT INTO verifications (email, token, api_key, created_at) VALUES ($1, $2, $3, $4)", [email, token, apiKey, now]); + await queryWithRetry("INSERT INTO verifications (email, token, api_key, created_at) VALUES ($1, $2, $3, $4)", [email, token, apiKey, now]); return { email, token, apiKey, createdAt: now, verifiedAt: null }; } export function verifyToken(token) { @@ -27,7 +27,7 @@ export function verifyToken(token) { // In-memory cache for verifications (loaded on startup, updated on changes) let verificationsCache = []; export async function loadVerifications() { - const result = await pool.query("SELECT * FROM verifications"); + const result = await queryWithRetry("SELECT * FROM verifications"); verificationsCache = result.rows.map((r) => ({ email: r.email, token: r.token, @@ -56,11 +56,11 @@ function verifyTokenSync(token) { return { status: "expired" }; v.verifiedAt = new Date().toISOString(); // Update DB async - pool.query("UPDATE verifications SET verified_at = $1 WHERE token = $2", [v.verifiedAt, token]).catch((err) => logger.error({ err }, "Failed to update verification")); + queryWithRetry("UPDATE verifications SET verified_at = $1 WHERE token = $2", [v.verifiedAt, token]).catch((err) => logger.error({ err }, "Failed to update verification")); return { status: "ok", verification: v }; } export async function createPendingVerification(email) { - await pool.query("DELETE FROM pending_verifications WHERE email = $1", [email]); + await queryWithRetry("DELETE FROM pending_verifications WHERE email = $1", [email]); const now = new Date(); const pending = { email, @@ -69,38 +69,38 @@ export async function createPendingVerification(email) { expiresAt: new Date(now.getTime() + CODE_EXPIRY_MS).toISOString(), attempts: 0, }; - await pool.query("INSERT INTO pending_verifications (email, code, created_at, expires_at, attempts) VALUES ($1, $2, $3, $4, $5)", [pending.email, pending.code, pending.createdAt, pending.expiresAt, pending.attempts]); + await queryWithRetry("INSERT INTO pending_verifications (email, code, created_at, expires_at, attempts) VALUES ($1, $2, $3, $4, $5)", [pending.email, pending.code, pending.createdAt, pending.expiresAt, pending.attempts]); return pending; } export async function verifyCode(email, code) { const cleanEmail = email.trim().toLowerCase(); - const result = await pool.query("SELECT * FROM pending_verifications WHERE email = $1", [cleanEmail]); + const result = await queryWithRetry("SELECT * FROM pending_verifications WHERE email = $1", [cleanEmail]); const pending = result.rows[0]; if (!pending) return { status: "invalid" }; if (new Date() > new Date(pending.expires_at)) { - await pool.query("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]); + await queryWithRetry("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]); return { status: "expired" }; } if (pending.attempts >= MAX_ATTEMPTS) { - await pool.query("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]); + await queryWithRetry("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]); return { status: "max_attempts" }; } - await pool.query("UPDATE pending_verifications SET attempts = attempts + 1 WHERE email = $1", [cleanEmail]); + await queryWithRetry("UPDATE pending_verifications SET attempts = attempts + 1 WHERE email = $1", [cleanEmail]); const a = Buffer.from(pending.code, "utf8"); const b = Buffer.from(code, "utf8"); const codeMatch = a.length === b.length && timingSafeEqual(a, b); if (!codeMatch) { return { status: "invalid" }; } - await pool.query("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]); + await queryWithRetry("DELETE FROM pending_verifications WHERE email = $1", [cleanEmail]); return { status: "ok" }; } export async function isEmailVerified(email) { - const result = await pool.query("SELECT 1 FROM verifications WHERE email = $1 AND verified_at IS NOT NULL LIMIT 1", [email]); + const result = await queryWithRetry("SELECT 1 FROM verifications WHERE email = $1 AND verified_at IS NOT NULL LIMIT 1", [email]); return result.rows.length > 0; } export async function getVerifiedApiKey(email) { - const result = await pool.query("SELECT api_key FROM verifications WHERE email = $1 AND verified_at IS NOT NULL LIMIT 1", [email]); + const result = await queryWithRetry("SELECT api_key FROM verifications WHERE email = $1 AND verified_at IS NOT NULL LIMIT 1", [email]); return result.rows[0]?.api_key ?? null; }