Session 55: K3s load test (10x perf gain), w2 node down, cluster cleanup, version+brotli code pushed
This commit is contained in:
Hoid
parent
331b4c1517
commit
37461bc9f8
14 changed files with 117 additions and 48 deletions
BIN
concurrent_a.pdf
Normal file
BIN
concurrent_a.pdf
Normal file
Binary file not shown.
BIN
concurrent_b.pdf
Normal file
BIN
concurrent_b.pdf
Normal file
Binary file not shown.
61
large_test.html
Normal file
61
large_test.html
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
<html><head><title>Large Test Document</title></head><body>
|
||||||
|
<h1>Performance Test Document</h1>
|
||||||
|
<h2>Large HTML Payload Test</h2>
|
||||||
|
<table border="1" style="width:100%">
|
||||||
|
<tr><td>Row 1 Column 1</td><td>Row 1 Column 2</td><td>Row 1 Column 3</td><td>Data value for row 1 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 2 Column 1</td><td>Row 2 Column 2</td><td>Row 2 Column 3</td><td>Data value for row 2 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 3 Column 1</td><td>Row 3 Column 2</td><td>Row 3 Column 3</td><td>Data value for row 3 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 4 Column 1</td><td>Row 4 Column 2</td><td>Row 4 Column 3</td><td>Data value for row 4 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 5 Column 1</td><td>Row 5 Column 2</td><td>Row 5 Column 3</td><td>Data value for row 5 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 6 Column 1</td><td>Row 6 Column 2</td><td>Row 6 Column 3</td><td>Data value for row 6 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 7 Column 1</td><td>Row 7 Column 2</td><td>Row 7 Column 3</td><td>Data value for row 7 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 8 Column 1</td><td>Row 8 Column 2</td><td>Row 8 Column 3</td><td>Data value for row 8 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 9 Column 1</td><td>Row 9 Column 2</td><td>Row 9 Column 3</td><td>Data value for row 9 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 10 Column 1</td><td>Row 10 Column 2</td><td>Row 10 Column 3</td><td>Data value for row 10 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 11 Column 1</td><td>Row 11 Column 2</td><td>Row 11 Column 3</td><td>Data value for row 11 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 12 Column 1</td><td>Row 12 Column 2</td><td>Row 12 Column 3</td><td>Data value for row 12 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 13 Column 1</td><td>Row 13 Column 2</td><td>Row 13 Column 3</td><td>Data value for row 13 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 14 Column 1</td><td>Row 14 Column 2</td><td>Row 14 Column 3</td><td>Data value for row 14 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 15 Column 1</td><td>Row 15 Column 2</td><td>Row 15 Column 3</td><td>Data value for row 15 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 16 Column 1</td><td>Row 16 Column 2</td><td>Row 16 Column 3</td><td>Data value for row 16 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 17 Column 1</td><td>Row 17 Column 2</td><td>Row 17 Column 3</td><td>Data value for row 17 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 18 Column 1</td><td>Row 18 Column 2</td><td>Row 18 Column 3</td><td>Data value for row 18 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 19 Column 1</td><td>Row 19 Column 2</td><td>Row 19 Column 3</td><td>Data value for row 19 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 20 Column 1</td><td>Row 20 Column 2</td><td>Row 20 Column 3</td><td>Data value for row 20 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 21 Column 1</td><td>Row 21 Column 2</td><td>Row 21 Column 3</td><td>Data value for row 21 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 22 Column 1</td><td>Row 22 Column 2</td><td>Row 22 Column 3</td><td>Data value for row 22 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 23 Column 1</td><td>Row 23 Column 2</td><td>Row 23 Column 3</td><td>Data value for row 23 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 24 Column 1</td><td>Row 24 Column 2</td><td>Row 24 Column 3</td><td>Data value for row 24 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 25 Column 1</td><td>Row 25 Column 2</td><td>Row 25 Column 3</td><td>Data value for row 25 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 26 Column 1</td><td>Row 26 Column 2</td><td>Row 26 Column 3</td><td>Data value for row 26 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 27 Column 1</td><td>Row 27 Column 2</td><td>Row 27 Column 3</td><td>Data value for row 27 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 28 Column 1</td><td>Row 28 Column 2</td><td>Row 28 Column 3</td><td>Data value for row 28 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 29 Column 1</td><td>Row 29 Column 2</td><td>Row 29 Column 3</td><td>Data value for row 29 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 30 Column 1</td><td>Row 30 Column 2</td><td>Row 30 Column 3</td><td>Data value for row 30 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 31 Column 1</td><td>Row 31 Column 2</td><td>Row 31 Column 3</td><td>Data value for row 31 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 32 Column 1</td><td>Row 32 Column 2</td><td>Row 32 Column 3</td><td>Data value for row 32 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 33 Column 1</td><td>Row 33 Column 2</td><td>Row 33 Column 3</td><td>Data value for row 33 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 34 Column 1</td><td>Row 34 Column 2</td><td>Row 34 Column 3</td><td>Data value for row 34 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 35 Column 1</td><td>Row 35 Column 2</td><td>Row 35 Column 3</td><td>Data value for row 35 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 36 Column 1</td><td>Row 36 Column 2</td><td>Row 36 Column 3</td><td>Data value for row 36 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 37 Column 1</td><td>Row 37 Column 2</td><td>Row 37 Column 3</td><td>Data value for row 37 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 38 Column 1</td><td>Row 38 Column 2</td><td>Row 38 Column 3</td><td>Data value for row 38 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 39 Column 1</td><td>Row 39 Column 2</td><td>Row 39 Column 3</td><td>Data value for row 39 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 40 Column 1</td><td>Row 40 Column 2</td><td>Row 40 Column 3</td><td>Data value for row 40 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 41 Column 1</td><td>Row 41 Column 2</td><td>Row 41 Column 3</td><td>Data value for row 41 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 42 Column 1</td><td>Row 42 Column 2</td><td>Row 42 Column 3</td><td>Data value for row 42 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 43 Column 1</td><td>Row 43 Column 2</td><td>Row 43 Column 3</td><td>Data value for row 43 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 44 Column 1</td><td>Row 44 Column 2</td><td>Row 44 Column 3</td><td>Data value for row 44 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 45 Column 1</td><td>Row 45 Column 2</td><td>Row 45 Column 3</td><td>Data value for row 45 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 46 Column 1</td><td>Row 46 Column 2</td><td>Row 46 Column 3</td><td>Data value for row 46 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 47 Column 1</td><td>Row 47 Column 2</td><td>Row 47 Column 3</td><td>Data value for row 47 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 48 Column 1</td><td>Row 48 Column 2</td><td>Row 48 Column 3</td><td>Data value for row 48 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 49 Column 1</td><td>Row 49 Column 2</td><td>Row 49 Column 3</td><td>Data value for row 49 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
<tr><td>Row 50 Column 1</td><td>Row 50 Column 2</td><td>Row 50 Column 3</td><td>Data value for row 50 that contains more text to increase the payload size significantly</td></tr>
|
||||||
|
</table>
|
||||||
|
<p>This is paragraph 1 with additional content to increase the document size and complexity. The goal is to test how the system handles larger HTML payloads and whether processing time scales with document complexity.</p>
|
||||||
|
<p>This is paragraph 2 with additional content to increase the document size and complexity. The goal is to test how the system handles larger HTML payloads and whether processing time scales with document complexity.</p>
|
||||||
|
<p>This is paragraph 3 with additional content to increase the document size and complexity. The goal is to test how the system handles larger HTML payloads and whether processing time scales with document complexity.</p>
|
||||||
|
<p>This is paragraph 4 with additional content to increase the document size and complexity. The goal is to test how the system handles larger HTML payloads and whether processing time scales with document complexity.</p>
|
||||||
|
<p>This is paragraph 5 with additional content to increase the document size and complexity. The goal is to test how the system handles larger HTML payloads and whether processing time scales with document complexity.</p>
|
||||||
|
</body></html>
|
||||||
BIN
large_test.pdf
Normal file
BIN
large_test.pdf
Normal file
Binary file not shown.
BIN
large_test2.pdf
Normal file
BIN
large_test2.pdf
Normal file
Binary file not shown.
|
|
@ -52,17 +52,14 @@
|
||||||
],
|
],
|
||||||
"notes": "N26 uses Xetra tickers. Always provide ISIN for orders. Fractional shares by EUR amount supported.",
|
"notes": "N26 uses Xetra tickers. Always provide ISIN for orders. Fractional shares by EUR amount supported.",
|
||||||
"created": "2026-02-12T20:00:00Z",
|
"created": "2026-02-12T20:00:00Z",
|
||||||
"lastUpdated": "2026-02-17T16:15:00Z",
|
"lastUpdated": "2026-02-18T16:15:00Z",
|
||||||
"closingSnapshot": {
|
"closingSnapshot": {
|
||||||
"date": "2026-02-18",
|
"date": "2026-02-18",
|
||||||
"DFNS": 57.80,
|
"DFNS": 58.61,
|
||||||
"portfolioValue": 1027.92,
|
"portfolioValue": 1042.36,
|
||||||
"dailyPL": 1.89,
|
"dailyPL": 14.40,
|
||||||
"totalReturn": 2.79
|
"dailyPLpct": 1.40,
|
||||||
|
"totalReturn": 4.24
|
||||||
},
|
},
|
||||||
"middayCheck": {
|
"pendingActions": []
|
||||||
"date": "2026-02-18",
|
|
||||||
"DFNS": 57.80,
|
|
||||||
"action": "HOLD"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,35 +1,6 @@
|
||||||
{
|
{
|
||||||
"date": "2026-02-17",
|
"date": "2026-02-18",
|
||||||
"events": [
|
"events": [
|
||||||
{
|
{"time": "19:02", "type": "nudge", "note": "First wind-down nudge sent at 19:00 Vienna"}
|
||||||
"time": "19:03",
|
|
||||||
"type": "nudge",
|
|
||||||
"note": "First wind-down check — asked what they're doing."
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"time": "19:00",
|
|
||||||
"type": "activity",
|
|
||||||
"note": "Ordered Kenko poke bowl with salmon, sushi rice, veggies, sesame sauce + edamame"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"time": "20:12",
|
|
||||||
"type": "nudge",
|
|
||||||
"note": "Second nudge — nose shower reminder + audiobook suggestion"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"time": "20:30-00:00",
|
|
||||||
"type": "activity",
|
|
||||||
"note": "Kept working on DocFast (support formatting, needs-reply bug, product research, Coolify skill). Hoid failed to nudge — got caught up helping instead of pushing back."
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"time": "00:00",
|
|
||||||
"type": "self-critique",
|
|
||||||
"note": "FAILURE: Did not nudge after 20:12. Should have actively blocked new work after 20:00. User called it out at midnight."
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"time": "01:00",
|
|
||||||
"type": "sleep",
|
|
||||||
"note": "Fell asleep around 01:00 Vienna. Late but better than working at midnight."
|
|
||||||
}
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
BIN
prod_test.pdf
Normal file
BIN
prod_test.pdf
Normal file
Binary file not shown.
|
|
@ -1305,3 +1305,30 @@
|
||||||
- **Budget:** €181.71 remaining, Revenue: €9
|
- **Budget:** €181.71 remaining, Revenue: €9
|
||||||
- **Open bugs:** 0 CRITICAL, 1 HIGH (BUG-076 node down), 0 MEDIUM, 0 LOW
|
- **Open bugs:** 0 CRITICAL, 1 HIGH (BUG-076 node down), 0 MEDIUM, 0 LOW
|
||||||
- **Status:** Production operational but HA degraded — single worker node
|
- **Status:** Production operational but HA degraded — single worker node
|
||||||
|
|
||||||
|
## Session 55 — 2026-02-18 18:00 UTC (Evening Session)
|
||||||
|
- **Node situation flipped:** w1 recovered (investor rebooted), but w2 now NotReady/unreachable. HA still degraded — single worker.
|
||||||
|
- **DevOps agent completed:**
|
||||||
|
- Force-deleted all stuck Terminating pods on w2 (cert-manager, CNPG, docfast, coredns)
|
||||||
|
- New pods rescheduled to w1 where topology constraints allow
|
||||||
|
- Pending pods: 1 docfast (topology spread), 1 main-db-2, 1 pooler (anti-affinity)
|
||||||
|
- w2 completely unreachable — needs Hetzner Console reboot
|
||||||
|
- **K3s Load Test completed (production, light load):**
|
||||||
|
- Sequential avg: 0.198s (10x improvement over Docker's ~2.1s)
|
||||||
|
- P95: 0.235s, range 0.176-0.235s
|
||||||
|
- 2 concurrent: ~0.27s each, 100% success
|
||||||
|
- Large payload (104KB, 3 pages): 1.65s
|
||||||
|
- 15-worker pool with plenty of headroom
|
||||||
|
- Finding: staging DB had no tables (schema not migrated after K3s setup)
|
||||||
|
- **Backend dev (version + Brotli):**
|
||||||
|
- Code pushed: commit 170ed44 — version bumped to 0.2.9, shrink-ray-current added for Brotli
|
||||||
|
- CI DID NOT BUILD the image — commit hash image not found in registry
|
||||||
|
- Staging manually reverted to working image (e611609)
|
||||||
|
- TODO: Investigate why CI didn't trigger/build for this commit
|
||||||
|
- **Staging DB issue discovered:** docfast_staging database has no tables — staging is not fully functional
|
||||||
|
- **Support:** Zero open tickets ✅
|
||||||
|
- **Investor Test:** All 5 ✅
|
||||||
|
- **Budget:** €181.71 remaining, Revenue: €9
|
||||||
|
- **Open bugs:** 0 CRITICAL, 1 HIGH (BUG-076 — now w2 down instead of w1)
|
||||||
|
- **Escalation:** w2 reboot needed via Hetzner Console
|
||||||
|
- **New issues found:** Staging DB missing schema, CI pipeline may have failed for latest commit
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@
|
||||||
"phaseLabel": "Build Production-Grade Product",
|
"phaseLabel": "Build Production-Grade Product",
|
||||||
"status": "launch-ready",
|
"status": "launch-ready",
|
||||||
"product": "DocFast \u2014 HTML/Markdown to PDF API",
|
"product": "DocFast \u2014 HTML/Markdown to PDF API",
|
||||||
"currentPriority": "k3s-w1 NODE DOWN — running on w2 only. HA degraded. Escalated to investor for Hetzner reboot.",
|
"currentPriority": "k3s-w2 NODE DOWN — running on w1 only. HA degraded. Escalated to investor for Hetzner reboot. Version+Brotli code pushed but CI didn't build image.",
|
||||||
"ownerDirectives_PRIORITY": "Process these IN ORDER. Do not skip.",
|
"ownerDirectives_PRIORITY": "Process these IN ORDER. Do not skip.",
|
||||||
"ownerDirectives": [
|
"ownerDirectives": [
|
||||||
"Stripe: owner has existing Stripe account from another project \u2014 use same account, just create separate Product + webhook endpoint for DocFast.",
|
"Stripe: owner has existing Stripe account from another project \u2014 use same account, just create separate Product + webhook endpoint for DocFast.",
|
||||||
|
|
@ -65,10 +65,19 @@
|
||||||
"emailDeliveryNote": "MX record fixed 2026-02-17. Postfix + DKIM operational."
|
"emailDeliveryNote": "MX record fixed 2026-02-17. Postfix + DKIM operational."
|
||||||
},
|
},
|
||||||
"loadTestResults": {
|
"loadTestResults": {
|
||||||
|
"docker_old": {
|
||||||
"sequential": "~2.1s per PDF, ~28/min",
|
"sequential": "~2.1s per PDF, ~28/min",
|
||||||
"concurrent": "3 safe, 5th fails at ~16s",
|
"concurrent": "3 safe, 5th fails at ~16s",
|
||||||
"server": "CAX11 (2 vCPU ARM, 4GB RAM), container 512MB cap"
|
"server": "CAX11 (2 vCPU ARM, 4GB RAM), container 512MB cap"
|
||||||
},
|
},
|
||||||
|
"k3s_current": {
|
||||||
|
"sequential": "~0.2s avg per PDF (10x improvement)",
|
||||||
|
"p95": "0.235s",
|
||||||
|
"concurrent": "2 concurrent at ~0.27s, 15-worker pool",
|
||||||
|
"largePayload": "1.65s for 104KB/3-page PDF",
|
||||||
|
"server": "K3s cluster, 2x CAX11 workers (1 active due to w2 down)"
|
||||||
|
}
|
||||||
|
},
|
||||||
"infrastructure": {
|
"infrastructure": {
|
||||||
"domain": "docfast.dev",
|
"domain": "docfast.dev",
|
||||||
"url": "https://docfast.dev",
|
"url": "https://docfast.dev",
|
||||||
|
|
@ -104,10 +113,10 @@
|
||||||
},
|
},
|
||||||
"openBugs": {
|
"openBugs": {
|
||||||
"CRITICAL": [],
|
"CRITICAL": [],
|
||||||
"HIGH": ["BUG-076: k3s-w1 node down, HA degraded, needs Hetzner reboot"],
|
"HIGH": ["BUG-076: k3s-w2 node down (was w1, now w2), HA degraded, needs Hetzner reboot"],
|
||||||
"MEDIUM": [],
|
"MEDIUM": [],
|
||||||
"LOW": [],
|
"LOW": [],
|
||||||
"note": "Session 54: k3s-w1 node down. CNPG failover to main-db-2 worked. Production running on w2 only. HA validated but degraded."
|
"note": "Session 55: w1 recovered, w2 now down. Stuck pods force-deleted. Production on w1 only. K3s load test: ~0.2s avg (10x faster than Docker). Version/Brotli code pushed, CI didn't build."
|
||||||
},
|
},
|
||||||
"blockers": [],
|
"blockers": [],
|
||||||
"resolvedBlockers": [
|
"resolvedBlockers": [
|
||||||
|
|
@ -120,5 +129,5 @@
|
||||||
"Checkout .env persistence + CI/CD secrets pipeline \u2014 DONE 2026-02-17"
|
"Checkout .env persistence + CI/CD secrets pipeline \u2014 DONE 2026-02-17"
|
||||||
],
|
],
|
||||||
"startDate": "2026-02-14",
|
"startDate": "2026-02-14",
|
||||||
"sessionCount": 54
|
"sessionCount": 55
|
||||||
}
|
}
|
||||||
1
staging_real_test.pdf
Normal file
1
staging_real_test.pdf
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
{"error":"Invalid API key"}Status: 403, Size: 27 bytes, Time: 0.102511s
|
||||||
1
staging_test.pdf
Normal file
1
staging_test.pdf
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
{"error":"Invalid API key"}Status: 403, Size: 27 bytes, Time: 0.091674s
|
||||||
1
staging_test2.pdf
Normal file
1
staging_test2.pdf
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
{"error":"Invalid API key"}Status: 403, Size: 27 bytes, Time: 0.113528s
|
||||||
1
test.pdf
Normal file
1
test.pdf
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
{"error":"Invalid API key"}Status: 403, Size: 27 bytes, Time: 0.113809s
|
||||||
Loading…
Add table
Add a link
Reference in a new issue