fix: database connection resilience — retry on transient errors, TCP keepalive, health check timeout

- Enable TCP keepalive on pg.Pool to detect dead connections
- Add connectionTimeoutMillis (5s) to prevent hanging on stale connections
- Add queryWithRetry() with exponential backoff for transient DB errors
- Add connectWithRetry() for transaction-based operations
- Detect PgBouncer "no available server" and other transient errors
- Health check has 3s timeout and returns 503 on DB failure
- All DB operations in keys, verification, usage use retry logic

Fixes BUG-075: PgBouncer failover causes permanent pod failures

src/services/keys.ts

@@ -1,6 +1,7 @@
 import { randomBytes } from "crypto";
 import logger from "./logger.js";
 import pool from "./db.js";
+import { queryWithRetry, connectWithRetry } from "./db.js";
 
 export interface ApiKey {
   key: string;
@@ -15,7 +16,7 @@ let keysCache: ApiKey[] = [];
 
 export async function loadKeys(): Promise<void> {
   try {
-    const result = await pool.query(
+    const result = await queryWithRetry(
       "SELECT key, tier, email, created_at, stripe_customer_id FROM api_keys"
     );
     keysCache = result.rows.map((r) => ({
@@ -37,7 +38,7 @@ export async function loadKeys(): Promise<void> {
       const entry: ApiKey = { key: k, tier: "pro", email: "seed@docfast.dev", createdAt: new Date().toISOString() };
       keysCache.push(entry);
       // Upsert into DB
-      await pool.query(
+      await queryWithRetry(
         `INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4)
          ON CONFLICT (key) DO NOTHING`,
         [k, "pro", "seed@docfast.dev", new Date().toISOString()]
@@ -76,7 +77,7 @@ export async function createFreeKey(email?: string): Promise<ApiKey> {
     createdAt: new Date().toISOString(),
   };
 
-  await pool.query(
+  await queryWithRetry(
     "INSERT INTO api_keys (key, tier, email, created_at) VALUES ($1, $2, $3, $4)",
     [entry.key, entry.tier, entry.email, entry.createdAt]
   );
@@ -88,7 +89,7 @@ export async function createProKey(email: string, stripeCustomerId: string): Pro
   const existing = keysCache.find((k) => k.stripeCustomerId === stripeCustomerId);
   if (existing) {
     existing.tier = "pro";
-    await pool.query("UPDATE api_keys SET tier = 'pro' WHERE key = $1", [existing.key]);
+    await queryWithRetry("UPDATE api_keys SET tier = 'pro' WHERE key = $1", [existing.key]);
     return existing;
   }
 
@@ -100,7 +101,7 @@ export async function createProKey(email: string, stripeCustomerId: string): Pro
     stripeCustomerId,
   };
 
-  await pool.query(
+  await queryWithRetry(
     "INSERT INTO api_keys (key, tier, email, created_at, stripe_customer_id) VALUES ($1, $2, $3, $4, $5)",
     [entry.key, entry.tier, entry.email, entry.createdAt, entry.stripeCustomerId]
   );
@@ -112,7 +113,7 @@ export async function downgradeByCustomer(stripeCustomerId: string): Promise<boo
   const entry = keysCache.find((k) => k.stripeCustomerId === stripeCustomerId);
   if (entry) {
     entry.tier = "free";
-    await pool.query("UPDATE api_keys SET tier = 'free' WHERE stripe_customer_id = $1", [stripeCustomerId]);
+    await queryWithRetry("UPDATE api_keys SET tier = 'free' WHERE stripe_customer_id = $1", [stripeCustomerId]);
     return true;
   }
   return false;
@@ -126,7 +127,7 @@ export async function updateKeyEmail(apiKey: string, newEmail: string): Promise<
   const entry = keysCache.find((k) => k.key === apiKey);
   if (!entry) return false;
   entry.email = newEmail;
-  await pool.query("UPDATE api_keys SET email = $1 WHERE key = $2", [newEmail, apiKey]);
+  await queryWithRetry("UPDATE api_keys SET email = $1 WHERE key = $2", [newEmail, apiKey]);
   return true;
 }
 
@@ -134,6 +135,6 @@ export async function updateEmailByCustomer(stripeCustomerId: string, newEmail:
   const entry = keysCache.find(k => k.stripeCustomerId === stripeCustomerId);
   if (!entry) return false;
   entry.email = newEmail;
-  await pool.query("UPDATE api_keys SET email = $1 WHERE stripe_customer_id = $2", [newEmail, stripeCustomerId]);
+  await queryWithRetry("UPDATE api_keys SET email = $1 WHERE stripe_customer_id = $2", [newEmail, stripeCustomerId]);
   return true;
 }