runward 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE.md +13 -0
- package/README.md +128 -0
- package/dist/cli.js +76 -0
- package/dist/commands/check.js +50 -0
- package/dist/commands/doctor.js +75 -0
- package/dist/commands/init.js +90 -0
- package/dist/commands/status.js +49 -0
- package/dist/commands/update.js +48 -0
- package/dist/lib/constants.js +2 -0
- package/dist/lib/mission.js +96 -0
- package/dist/lib/paths.js +28 -0
- package/dist/lib/styles.js +60 -0
- package/dist/lib/tools.js +59 -0
- package/dist/lib/write.js +30 -0
- package/package.json +29 -0
- package/templates/mission/adr/ADR-0000-template.md +38 -0
- package/templates/mission/architecture.md +60 -0
- package/templates/mission/decision-matrix.md +37 -0
- package/templates/mission/evaluation-rubric.md +60 -0
- package/templates/mission/floor.md +47 -0
- package/templates/mission/framing.md +58 -0
- package/templates/mission/mission-contract.md +58 -0
- package/templates/mission/observability-schema.md +53 -0
- package/templates/mission/port-contract.md +63 -0
- package/templates/mission/reference-stack.md +70 -0
- package/templates/mission/runbook.md +70 -0
- package/templates/mission/shared-bricks.md +74 -0
- package/templates/mission/threat-model.md +53 -0
- package/templates/rules/async-job-guardrails.md +95 -0
- package/templates/rules/async-post-turn-pipeline.md +96 -0
- package/templates/rules/async-scheduled-maintenance.md +95 -0
- package/templates/rules/cache-three-tier-architecture.md +44 -0
- package/templates/rules/checklist-day-zero-project.md +97 -0
- package/templates/rules/checklist-pre-production-observability.md +99 -0
- package/templates/rules/checklist-pre-production-performance.md +109 -0
- package/templates/rules/checklist-pre-production-resilience.md +99 -0
- package/templates/rules/checklist-pre-production-security.md +86 -0
- package/templates/rules/config-secrets-boundary.md +57 -0
- package/templates/rules/config-typing-zod.md +78 -0
- package/templates/rules/contracts-governance.md +64 -0
- package/templates/rules/data-memory-consolidation.md +81 -0
- package/templates/rules/data-memory-invalidation.md +90 -0
- package/templates/rules/data-memory-scoring.md +154 -0
- package/templates/rules/data-migrations-forward-only.md +55 -0
- package/templates/rules/data-orphan-cleanup.md +142 -0
- package/templates/rules/data-ttl-types.md +100 -0
- package/templates/rules/eval-loop.md +52 -0
- package/templates/rules/frontier-deterministic-boundary.md +91 -0
- package/templates/rules/hexa-adapter-pattern.md +85 -0
- package/templates/rules/hexa-architecture.md +84 -0
- package/templates/rules/hexa-move-deterministic-out.md +93 -0
- package/templates/rules/hexa-recommended-stack.md +33 -0
- package/templates/rules/hexa-typescript-native.md +80 -0
- package/templates/rules/observability-alert-configuration.md +160 -0
- package/templates/rules/observability-llm-metrics.md +150 -0
- package/templates/rules/observability-startup-provider-log.md +128 -0
- package/templates/rules/observability-structured-json-logs.md +138 -0
- package/templates/rules/patterns-memory-router-tiered.md +139 -0
- package/templates/rules/patterns-prompt-compiler.md +126 -0
- package/templates/rules/patterns-request-id-propagation.md +137 -0
- package/templates/rules/process-adr-and-journal.md +50 -0
- package/templates/rules/provider-llm-auto-detection.md +54 -0
- package/templates/rules/provider-no-crash-missing-env.md +67 -0
- package/templates/rules/quality-codebase-metrics.md +115 -0
- package/templates/rules/quality-zod-input-validation.md +131 -0
- package/templates/rules/resilience-fail-open.md +65 -0
- package/templates/rules/resilience-multi-provider-fallback.md +100 -0
- package/templates/rules/resilience-retry-backoff.md +115 -0
- package/templates/rules/resilience-retryable-errors.md +107 -0
- package/templates/rules/routing-confidence-upgrade.md +89 -0
- package/templates/rules/routing-model-cost-ratios.md +68 -0
- package/templates/rules/routing-smart-complexity.md +131 -0
- package/templates/rules/scaling-db-connection-pooling.md +121 -0
- package/templates/rules/scaling-distributed-rate-limiting.md +141 -0
- package/templates/rules/scaling-state-externalization.md +84 -0
- package/templates/rules/security-prompt-injection.md +63 -0
- package/templates/rules/state-event-sourcing.md +66 -0
- package/templates/rules/tools-registry-pattern.md +153 -0
- package/templates/rules/tools-scope-atomicity.md +103 -0
- package/templates/targets/AGENTS.md +32 -0
- package/templates/workflows/architect.md +50 -0
- package/templates/workflows/brownfield.md +52 -0
- package/templates/workflows/decision-loop.md +52 -0
- package/templates/workflows/floor.md +60 -0
- package/templates/workflows/frame.md +69 -0
- package/templates/workflows/govern.md +55 -0
- package/templates/workflows/handover.md +55 -0
- package/templates/workflows/iterate.md +52 -0
- package/templates/workflows/method.md +53 -0
- package/templates/workflows/review.md +59 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Post-Turn Pipeline (Keep the Interactive Turn Thin)
|
|
3
|
+
impact: HIGH
|
|
4
|
+
impactDescription: Keeps response latency bound to what the answer needs by moving extraction, consolidation and summarization into an isolated, event-driven pipeline that runs after the turn
|
|
5
|
+
tags: [async, pipeline, events, latency, memory, provenance]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Post-Turn Pipeline
|
|
9
|
+
|
|
10
|
+
> **The interactive turn does what the answer needs. Everything else leaves the turn.**
|
|
11
|
+
|
|
12
|
+
A turn that extracts facts, consolidates memory and writes episode summaries before replying makes the user pay for work the response never needed. The turn stays thin: retrieve context, decide, answer, persist the interaction event. Everything that serves the *next* turn rather than this one — fact extraction, memory consolidation, episode summarization, pre-warming the likely next context — moves to a post-turn pipeline, triggered by an event emitted when the turn completes.
|
|
13
|
+
|
|
14
|
+
Three properties are non-negotiable:
|
|
15
|
+
|
|
16
|
+
1. **The provenance fingerprint is deposited at emission.** The event carries a fingerprint of what the model actually saw. By the time a pipeline step runs — possibly minutes later, possibly on retry — working state may have changed; the step works from the recorded truth, not from whatever memory looks like now.
|
|
17
|
+
2. **Each step is isolated with its own bounded retry.** A flaky summarizer retries on its own schedule without touching its siblings.
|
|
18
|
+
3. **One step failing never fails the others.** No shared fate: fact extraction completes even when summarization is down.
|
|
19
|
+
|
|
20
|
+
**Incorrect:**
|
|
21
|
+
|
|
22
|
+
```typescript
|
|
23
|
+
// Everything inline: the user waits for work the answer does not need.
|
|
24
|
+
async function handleTurn(input: UserInput): Promise<Reply> {
|
|
25
|
+
const context = await memory.retrieve(input);
|
|
26
|
+
const reply = await orchestrator.answer(input, context);
|
|
27
|
+
|
|
28
|
+
await extractFacts(reply); // BAD: not needed to answer
|
|
29
|
+
await consolidateMemory(); // BAD: batch work on the hot path
|
|
30
|
+
await summarizeEpisode(reply); // BAD: if this throws, the turn fails
|
|
31
|
+
await prewarmNextContext(input); // BAD: p95 now includes four jobs
|
|
32
|
+
|
|
33
|
+
return reply;
|
|
34
|
+
}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
**Correct:**
|
|
38
|
+
|
|
39
|
+
```typescript
|
|
40
|
+
// Thin turn: answer, persist, emit. The pipeline runs after the reply is out.
|
|
41
|
+
async function handleTurn(input: UserInput): Promise<Reply> {
|
|
42
|
+
const context = await memory.retrieve(input);
|
|
43
|
+
const reply = await orchestrator.answer(input, context);
|
|
44
|
+
|
|
45
|
+
await events.emit({
|
|
46
|
+
type: 'turn.completed',
|
|
47
|
+
turnId: reply.turnId,
|
|
48
|
+
provenance: fingerprint(context), // what the model saw, sealed now
|
|
49
|
+
occurredAt: new Date(),
|
|
50
|
+
});
|
|
51
|
+
return reply; // latency = retrieve + answer + one event write
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Each step: its own handler, its own bounded retry, no shared fate.
|
|
55
|
+
const steps: PipelineStep[] = [
|
|
56
|
+
{ name: 'extract-facts', run: extractFacts },
|
|
57
|
+
{ name: 'consolidate', run: consolidateMemory },
|
|
58
|
+
{ name: 'summarize', run: summarizeEpisode },
|
|
59
|
+
{ name: 'prewarm-context', run: prewarmNextContext },
|
|
60
|
+
];
|
|
61
|
+
|
|
62
|
+
events.on('turn.completed', async (event) => {
|
|
63
|
+
const results = await Promise.allSettled(
|
|
64
|
+
steps.map(step =>
|
|
65
|
+
withRetry(() => step.run(event), { maxRetries: 3, name: step.name })
|
|
66
|
+
)
|
|
67
|
+
);
|
|
68
|
+
results.forEach((r, i) => {
|
|
69
|
+
if (r.status === 'rejected') {
|
|
70
|
+
log.error('pipeline.step.failed', {
|
|
71
|
+
step: steps[i].name,
|
|
72
|
+
turnId: event.turnId,
|
|
73
|
+
error: String(r.reason),
|
|
74
|
+
});
|
|
75
|
+
// Siblings have already succeeded or failed on their own.
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
**Why this matters:**
|
|
82
|
+
|
|
83
|
+
- **Latency**: the user waits for the answer, not for the housekeeping. Four background jobs on the hot path can double p95 for zero visible value.
|
|
84
|
+
- **Resilience**: a degraded summarizer no longer takes fact extraction — or the turn itself — down with it. Each step degrades alone.
|
|
85
|
+
- **Audit**: because the fingerprint is sealed at emission, a step that runs late or on retry still reads exactly what the turn saw. Provenance survives asynchrony.
|
|
86
|
+
- **Cost**: pipeline steps are off the hot path, so they can run on cheaper model tiers and absorb backpressure without touching the user experience.
|
|
87
|
+
|
|
88
|
+
Post-turn steps are background jobs like any other: they carry the four guardrails — bounded retry, idempotency, bounded concurrency, job observability (see *Background Job Guardrails*).
|
|
89
|
+
|
|
90
|
+
**Checklist:**
|
|
91
|
+
|
|
92
|
+
- [ ] The interactive turn performs only what the response requires; everything else is triggered by a turn-completed event.
|
|
93
|
+
- [ ] The event carries the provenance fingerprint, deposited at emission — steps never read mutable state to learn what the model saw.
|
|
94
|
+
- [ ] Each pipeline step has its own bounded retry, independent of its siblings.
|
|
95
|
+
- [ ] A step's failure is logged with the turn id and isolated; the other steps complete.
|
|
96
|
+
- [ ] Pipeline steps follow the background-job guardrails (idempotency keys, concurrency caps, queue metrics).
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Scheduled Maintenance (The Cron Updates and Cools, It Never Destroys Truth)
|
|
3
|
+
impact: MEDIUM
|
|
4
|
+
impactDescription: Keeps memory, derived views and the journal healthy over months of operation through periodic decay, pruning and cold archiving that never touch the source of truth
|
|
5
|
+
tags: [async, cron, maintenance, memory, retention, archive]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Scheduled Maintenance
|
|
9
|
+
|
|
10
|
+
> **The cron updates and cools. It never destroys the truth.**
|
|
11
|
+
|
|
12
|
+
Some upkeep has no triggering event — its trigger is the passage of time. Memory scores decay as facts age; derived links drift past their retention; the journal accumulates segments nobody reads hot anymore; trends only appear across weeks of aggregates; reminders come due; legally imposed erasure deadlines arrive. This work belongs to scheduled maintenance jobs, not to the interactive turn and not to the post-turn pipeline.
|
|
13
|
+
|
|
14
|
+
One dividing line governs everything the cron touches:
|
|
15
|
+
|
|
16
|
+
- **Derived data** — scores, derived links, indexes, caches, summaries — may be decayed, pruned or rebuilt freely: it is recomputable from the journal.
|
|
17
|
+
- **The immutable journal** — the source of truth — is never deleted by maintenance. Old segments are moved to cold storage, checksum verified, and remain retrievable for audit and replay.
|
|
18
|
+
- **Enforced erasure** — right-to-be-forgotten, legal retention limits — is the only true deletion, and it is an explicit, audited command with its own path. It is never a side effect of a cleanup batch.
|
|
19
|
+
|
|
20
|
+
**Incorrect:**
|
|
21
|
+
|
|
22
|
+
```typescript
|
|
23
|
+
// A "cleanup" cron that quietly destroys the source of truth.
|
|
24
|
+
cron.schedule('0 3 * * *', async () => {
|
|
25
|
+
// BAD: the journal is the truth — audit and replay just died
|
|
26
|
+
await db.query(`DELETE FROM journal WHERE created_at < now() - interval '90 days'`);
|
|
27
|
+
// BAD: silent disappearance — nothing records why the fact vanished
|
|
28
|
+
await db.query(`DELETE FROM memories WHERE score < 0.2`);
|
|
29
|
+
});
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
**Correct:**
|
|
33
|
+
|
|
34
|
+
```typescript
|
|
35
|
+
// Each job declares what it touches; none of them deletes truth.
|
|
36
|
+
const maintenance: MaintenanceJob[] = [
|
|
37
|
+
{
|
|
38
|
+
name: 'decay-scores', schedule: '@daily',
|
|
39
|
+
run: () => memoryStore.applyDecay({ halfLifeDays: 30 }),
|
|
40
|
+
// Adjusts ranking; low scores demote, they do not delete.
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
name: 'prune-derived-links', schedule: '@daily',
|
|
44
|
+
run: () => linkStore.pruneDerived({ olderThan: RETENTION }),
|
|
45
|
+
// Derived only: every pruned link is recomputable from the journal.
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
name: 'archive-journal-segments', schedule: '@weekly',
|
|
49
|
+
run: async () => {
|
|
50
|
+
const segments = await journal.segmentsOlderThan(HOT_WINDOW);
|
|
51
|
+
for (const s of segments) {
|
|
52
|
+
const archived = await coldStorage.put(s);
|
|
53
|
+
if (archived.checksum !== s.checksum) throw new Error('archive mismatch');
|
|
54
|
+
await journal.releaseHotCopy(s.id, { archivedAt: archived.location });
|
|
55
|
+
}
|
|
56
|
+
// Moved and verified — never deleted. Audit can still unfold it.
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
name: 'detect-trends', schedule: '@weekly',
|
|
61
|
+
run: () => trends.computeOverAggregates({ window: '90d' }),
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
name: 'due-reminders', schedule: '@hourly',
|
|
65
|
+
run: () => reminders.emitDue(),
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
name: 'enforced-erasure', schedule: '@daily',
|
|
69
|
+
run: async () => {
|
|
70
|
+
// The only true deletion: explicit requests, own path, audited.
|
|
71
|
+
for (const req of await erasureRequests.pending()) {
|
|
72
|
+
await eraseAcrossStores(req.subjectId, req.scope);
|
|
73
|
+
await auditLog.append({ type: 'erasure.executed', request: req.id });
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
];
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
**Why this matters:**
|
|
81
|
+
|
|
82
|
+
- **Retrieval quality**: without decay and pruning, ranking drowns in stale facts; the cron is what keeps recall honest as the system ages.
|
|
83
|
+
- **Bounded cost without lost history**: cold archiving caps hot-storage growth while audit and replay keep working — the truth cools, it does not vanish.
|
|
84
|
+
- **Compliance both ways**: enforced erasure is provable (explicit request, audit record), and nothing *else* can silently delete regulated data, because no other job is allowed to.
|
|
85
|
+
- **Recovery**: after a bad consolidation or a corrupted index, every derived structure can be rebuilt — precisely because maintenance never touched the journal.
|
|
86
|
+
|
|
87
|
+
Maintenance jobs are background jobs like any other: bounded retry, idempotency, bounded concurrency, queue metrics (see *Background Job Guardrails*). A weekly job that fails silently for a month is a slow outage.
|
|
88
|
+
|
|
89
|
+
**Checklist:**
|
|
90
|
+
|
|
91
|
+
- [ ] Every maintenance job is classified: derived (may decay/prune/rebuild), journal (archive only, verify then release), or erasure (explicit and audited).
|
|
92
|
+
- [ ] No scheduled job issues deletes against the journal; archiving verifies checksums before releasing hot copies.
|
|
93
|
+
- [ ] Score decay demotes and invalidation supersedes — nothing silently disappears from memory.
|
|
94
|
+
- [ ] Enforced erasure runs on explicit requests through its own path, and writes an audit record per execution.
|
|
95
|
+
- [ ] Maintenance jobs follow the background-job guardrails and alert when a scheduled run is missed or failing.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Prompt Cache Stability (a Cost Lever, Provider-Specific)
|
|
3
|
+
impact: MEDIUM
|
|
4
|
+
impactDescription: Cuts input-token cost when a provider bills cached prefixes cheaper, by keeping the stable part of the prompt byte-identical across requests
|
|
5
|
+
tags: [cache, llm, cost, observability]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Prompt Cache Stability
|
|
9
|
+
|
|
10
|
+
This is a **cost lever, not an architecture principle.** Cost is driven first by the model boundary (keep deterministic work out of the model) and by routing; prompt caching is a secondary optimisation, and only where the provider bills a cached prefix cheaper than a fresh one. Reach for it once cost is measured and the prompt has a genuinely stable prefix, not before.
|
|
11
|
+
|
|
12
|
+
> **Cache stability beats token reduction.** A layer that varies between requests gets a 0% cache hit and costs more than the same layer sent stable. A 15K-token stable prefix at 90% hit is cheaper than a 10K-token prefix at 0%.
|
|
13
|
+
|
|
14
|
+
**Order the prompt from most stable to most dynamic, and mark the stable cut points:**
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
Layer 1 - CORE identity, methodology, format, safety rules changes: never (mid-session)
|
|
18
|
+
Layer 2 - TOOLS generic tool schemas and guidelines changes: on feature updates
|
|
19
|
+
Layer 3 - DYNAMIC retrieved context, history, the user message changes: every request
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
**Correct:**
|
|
23
|
+
|
|
24
|
+
```typescript
|
|
25
|
+
const response = await model.complete({
|
|
26
|
+
system: [
|
|
27
|
+
{ text: corePrompt, cacheable: true }, // stable prefix, cached
|
|
28
|
+
{ text: toolsPrompt, cacheable: true }, // semi-stable, cached
|
|
29
|
+
{ text: dynamicContext }, // never cached
|
|
30
|
+
],
|
|
31
|
+
messages,
|
|
32
|
+
});
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
**What breaks a cache (anti-patterns):** injecting user/tenant data into Layer 1-2, timestamps in stable layers, conditionally including sections, reordering content within a cached layer. Keep variable content in Layer 3, always.
|
|
36
|
+
|
|
37
|
+
**Measure it or skip it:** track cache-read vs cache-write tokens in your LLM metrics. If the hit rate on the stable prefix is not materially above zero, the prefix is not actually stable and the lever is doing nothing.
|
|
38
|
+
|
|
39
|
+
**Checklist:**
|
|
40
|
+
|
|
41
|
+
- [ ] Cost is measured first; caching is applied as a deliberate lever, not by default.
|
|
42
|
+
- [ ] The stable prefix is byte-identical across requests (no per-request variation).
|
|
43
|
+
- [ ] Variable content lives only in the dynamic layer.
|
|
44
|
+
- [ ] Cache hit rate on the stable prefix is monitored, not assumed.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Day-Zero Project Setup Checklist
|
|
3
|
+
impact: CRITICAL
|
|
4
|
+
impactDescription: Starts a project on the foundation that prevents costly refactors, with the core engineering disciplines wired from the first commit
|
|
5
|
+
tags: [checklist, architecture, setup, project]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Day-Zero Project Setup Checklist
|
|
9
|
+
|
|
10
|
+
The foundation to lay before the first feature. Sober by default: only what runs everywhere, with the disciplines that are expensive to retrofit wired from commit one.
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
ARCHITECTURE (30 min)
|
|
14
|
+
├── [ ] Hexagonal structure (core/, adapters/, infrastructure/)
|
|
15
|
+
├── [ ] Ports defined (even if empty); the model is one of them
|
|
16
|
+
├── [ ] Config typed and validated once with Zod (no scattered process.env)
|
|
17
|
+
├── [ ] Unified error model
|
|
18
|
+
└── [ ] Structured JSON logger with request-id propagation
|
|
19
|
+
|
|
20
|
+
BOUNDARIES (15 min) [the core disciplines]
|
|
21
|
+
├── [ ] Model behind a port, resolved at the boundary (no vendor SDK in core)
|
|
22
|
+
├── [ ] Deterministic boundary planned: facts from a store, model writes prose
|
|
23
|
+
├── [ ] Optional services degrade gracefully (absent != crash); logged at startup
|
|
24
|
+
└── [ ] Secrets read only in typed config; never in code, never in the model
|
|
25
|
+
|
|
26
|
+
QUALITY (15 min)
|
|
27
|
+
├── [ ] Lint + format configured
|
|
28
|
+
├── [ ] Pre-commit hooks
|
|
29
|
+
├── [ ] Domain test template (the floor's success criterion as a test)
|
|
30
|
+
├── [ ] ADR template + first ADRs for structural choices (with revisit triggers)
|
|
31
|
+
├── [ ] JOURNAL.md for per-session entries
|
|
32
|
+
└── [ ] Minimal README
|
|
33
|
+
|
|
34
|
+
INFRA (15 min)
|
|
35
|
+
├── [ ] .env.example with the required vars only, optional ones commented
|
|
36
|
+
├── [ ] Forward-only migration runner (numbered, contiguous, branch-first)
|
|
37
|
+
├── [ ] CI pipeline (lint + test + schema-drift)
|
|
38
|
+
├── [ ] Healthcheck + readiness endpoint
|
|
39
|
+
└── [ ] Conventional commits
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
**Minimum directory structure:**
|
|
43
|
+
|
|
44
|
+
This is the layout of the reference scaffold; follow it so every project speaks the same structure.
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
src/
|
|
48
|
+
├── core/ # Pure: no framework, no vendor SDK reaches here
|
|
49
|
+
│ ├── domain/ # entities, value objects, zod types, guards
|
|
50
|
+
│ ├── application/ # use-cases (orchestration)
|
|
51
|
+
│ └── ports/
|
|
52
|
+
│ ├── in/ # primary ports (how the app is driven)
|
|
53
|
+
│ └── out/ # secondary ports (model, store, clock, tools)
|
|
54
|
+
├── adapters/
|
|
55
|
+
│ ├── primary/ # http, cli
|
|
56
|
+
│ └── secondary/ # model, persistence, tools
|
|
57
|
+
├── infrastructure/
|
|
58
|
+
│ ├── config/ # zod-typed, validated once; the only reader of env
|
|
59
|
+
│ ├── container.ts # wires adapters to ports (DI)
|
|
60
|
+
│ ├── middleware.ts # the single cross-cutting surface (logs, request-id, cost)
|
|
61
|
+
│ ├── registry.ts # tool registry (an index, not a bus)
|
|
62
|
+
│ └── observability/
|
|
63
|
+
├── eval/ # behavioural eval harness
|
|
64
|
+
db/ (when persisted)
|
|
65
|
+
└── migrations/ # 0001_init.sql, forward-only
|
|
66
|
+
test/ # domain · contract (schema-drift) · integration · cost-cap
|
|
67
|
+
CLAUDE.md · JOURNAL.md · README.md · docs/adr/
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
**Minimum `.env.example` (required only; optional commented):**
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
# Required
|
|
74
|
+
DATABASE_URL=postgresql://user:pass@host/db?sslmode=verify-full&channel_binding=require
|
|
75
|
+
MODEL_GATEWAY_URL= # or MODEL_PROVIDER + MODEL
|
|
76
|
+
MODEL=
|
|
77
|
+
|
|
78
|
+
# Optional (feature lights up only if present)
|
|
79
|
+
# SEARCH_API_KEY=
|
|
80
|
+
# MEMORY_URL=
|
|
81
|
+
# TELEMETRY_TOKEN=
|
|
82
|
+
LOG_LEVEL=info
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
**Startup log (resolved config, never secrets):**
|
|
86
|
+
|
|
87
|
+
```typescript
|
|
88
|
+
log.info("[STARTUP] config", {
|
|
89
|
+
env: config.env,
|
|
90
|
+
model: config.model.provider,
|
|
91
|
+
database: !!config.database.url,
|
|
92
|
+
search: config.optional.search,
|
|
93
|
+
memory: config.optional.memory,
|
|
94
|
+
});
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
The split is deliberate: the *Boundaries* block is what separates an agentic system that holds up from one that does not. It is cheap on day zero and expensive on day ninety.
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Pre-Production Observability Checklist
|
|
3
|
+
impact: HIGH
|
|
4
|
+
impactDescription: Ensures you can monitor and debug production issues
|
|
5
|
+
tags: [checklist, observability, production, deployment]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Pre-Production Observability Checklist
|
|
9
|
+
|
|
10
|
+
Observability items to validate before production deployment.
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
OBSERVABILITY CHECKLIST
|
|
14
|
+
├── [ ] Structured logs (JSON format)
|
|
15
|
+
├── [ ] Request ID propagated through all services
|
|
16
|
+
├── [ ] LLM metrics tracked (tokens, cost, latency, model)
|
|
17
|
+
├── [ ] Error rates tracked and alerted
|
|
18
|
+
├── [ ] Alerts configured for critical metrics
|
|
19
|
+
├── [ ] Dashboard for operational monitoring
|
|
20
|
+
├── [ ] Log retention policy defined
|
|
21
|
+
└── [ ] Sensitive data redacted from logs
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
**Structured Logging:**
|
|
25
|
+
|
|
26
|
+
```typescript
|
|
27
|
+
// BAD: Incorrect: unstructured logs
|
|
28
|
+
console.log(`User ${userId} created project ${projectId}`);
|
|
29
|
+
|
|
30
|
+
// GOOD: Correct: structured JSON
|
|
31
|
+
logger.info('Project created', {
|
|
32
|
+
userId,
|
|
33
|
+
projectId,
|
|
34
|
+
tenantId,
|
|
35
|
+
requestId: ctx.requestId,
|
|
36
|
+
duration: Date.now() - start,
|
|
37
|
+
});
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
**Request ID Propagation:**
|
|
41
|
+
|
|
42
|
+
```typescript
|
|
43
|
+
// Middleware to create/propagate request ID
|
|
44
|
+
function requestIdMiddleware(req: Request, res: Response, next: NextFunction) {
|
|
45
|
+
const requestId = req.headers['x-request-id'] as string || crypto.randomUUID();
|
|
46
|
+
req.requestId = requestId;
|
|
47
|
+
res.setHeader('x-request-id', requestId);
|
|
48
|
+
|
|
49
|
+
// Add to async local storage for logging
|
|
50
|
+
asyncLocalStorage.run({ requestId }, next);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// All logs include requestId
|
|
54
|
+
const logger = {
|
|
55
|
+
info(message: string, data: object) {
|
|
56
|
+
const ctx = asyncLocalStorage.getStore();
|
|
57
|
+
console.log(JSON.stringify({
|
|
58
|
+
level: 'info',
|
|
59
|
+
message,
|
|
60
|
+
requestId: ctx?.requestId,
|
|
61
|
+
timestamp: new Date().toISOString(),
|
|
62
|
+
...data,
|
|
63
|
+
}));
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
**LLM Metrics:**
|
|
69
|
+
|
|
70
|
+
```typescript
|
|
71
|
+
// Track every LLM call
|
|
72
|
+
interface LLMMetrics {
|
|
73
|
+
model: string;
|
|
74
|
+
inputTokens: number;
|
|
75
|
+
outputTokens: number;
|
|
76
|
+
latencyMs: number;
|
|
77
|
+
cached: boolean;
|
|
78
|
+
estimatedCost: number;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function logLLMMetrics(metrics: LLMMetrics) {
|
|
82
|
+
logger.info('LLM request completed', metrics);
|
|
83
|
+
|
|
84
|
+
// Send to metrics service
|
|
85
|
+
telemetry.record('llm_request', {
|
|
86
|
+
...metrics,
|
|
87
|
+
timestamp: Date.now(),
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
**Alert Configuration:**
|
|
93
|
+
|
|
94
|
+
| Metric | Threshold | Action |
|
|
95
|
+
|--------|-----------|--------|
|
|
96
|
+
| Error rate | > 5% over 5 min | Page on-call |
|
|
97
|
+
| LLM latency P95 | > 5s | Investigate |
|
|
98
|
+
| Cache hit rate | < 50% | Investigate |
|
|
99
|
+
| Cost per request | > $0.10 | Review routing |
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Pre-Production Performance Checklist
|
|
3
|
+
impact: HIGH
|
|
4
|
+
impactDescription: Ensures application performs well under production load
|
|
5
|
+
tags: [checklist, performance, production, deployment]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Pre-Production Performance Checklist
|
|
9
|
+
|
|
10
|
+
Performance items to validate before production deployment.
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
PERFORMANCE CHECKLIST
|
|
14
|
+
├── [ ] LLM cache 3-tier implemented
|
|
15
|
+
├── [ ] Smart routing active (Fast/Balanced/Deep)
|
|
16
|
+
├── [ ] DB connection pooling configured
|
|
17
|
+
├── [ ] DB queries indexed (check slow query log)
|
|
18
|
+
├── [ ] Frontend assets optimized (images, bundle) if a UI is served
|
|
19
|
+
├── [ ] Code splitting enabled
|
|
20
|
+
├── [ ] Bundle size analyzed (<500KB initial)
|
|
21
|
+
├── [ ] API response times measured (<200ms P95 for non-LLM)
|
|
22
|
+
└── [ ] Load testing completed
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
**Cache Verification:**
|
|
26
|
+
|
|
27
|
+
```typescript
|
|
28
|
+
// Log cache hit rates at startup and periodically
|
|
29
|
+
setInterval(() => {
|
|
30
|
+
const stats = cacheStats.get();
|
|
31
|
+
logger.info('Cache statistics', {
|
|
32
|
+
layer1HitRate: stats.layer1.hits / stats.layer1.total,
|
|
33
|
+
layer2HitRate: stats.layer2.hits / stats.layer2.total,
|
|
34
|
+
totalSavings: stats.estimatedSavings,
|
|
35
|
+
});
|
|
36
|
+
}, 60000);
|
|
37
|
+
|
|
38
|
+
// Alert if cache performance degrades
|
|
39
|
+
if (stats.layer1HitRate < 0.8) {
|
|
40
|
+
logger.warn('Layer 1 cache hit rate below threshold', { rate: stats.layer1HitRate });
|
|
41
|
+
}
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
**Database Performance:**
|
|
45
|
+
|
|
46
|
+
```typescript
|
|
47
|
+
// Enable slow query logging
|
|
48
|
+
const pool = new Pool({
|
|
49
|
+
connectionString: process.env.DATABASE_URL,
|
|
50
|
+
max: 20,
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
// Log slow queries
|
|
54
|
+
pool.on('query', (event) => {
|
|
55
|
+
if (event.duration > 100) { // > 100ms
|
|
56
|
+
logger.warn('Slow query detected', {
|
|
57
|
+
query: event.query.substring(0, 200),
|
|
58
|
+
duration: event.duration,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
**Bundle Analysis:**
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
# Analyze bundle size
|
|
68
|
+
npm run build -- --analyze
|
|
69
|
+
|
|
70
|
+
# Check for large dependencies
|
|
71
|
+
npx @next/bundle-analyzer
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**Load Testing:**
|
|
75
|
+
|
|
76
|
+
```typescript
|
|
77
|
+
// k6 load test script
|
|
78
|
+
import http from 'k6/http';
|
|
79
|
+
import { check, sleep } from 'k6';
|
|
80
|
+
|
|
81
|
+
export const options = {
|
|
82
|
+
stages: [
|
|
83
|
+
{ duration: '2m', target: 50 }, // Ramp up
|
|
84
|
+
{ duration: '5m', target: 50 }, // Sustain
|
|
85
|
+
{ duration: '2m', target: 100 }, // Spike
|
|
86
|
+
{ duration: '2m', target: 0 }, // Ramp down
|
|
87
|
+
],
|
|
88
|
+
thresholds: {
|
|
89
|
+
http_req_duration: ['p(95)<500'], // 95% of requests < 500ms
|
|
90
|
+
http_req_failed: ['rate<0.01'], // Error rate < 1%
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
export default function () {
|
|
95
|
+
const res = http.get('https://api.example.com/health');
|
|
96
|
+
check(res, { 'status is 200': (r) => r.status === 200 });
|
|
97
|
+
sleep(1);
|
|
98
|
+
}
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
**Performance Targets:**
|
|
102
|
+
|
|
103
|
+
| Metric | Target | Critical |
|
|
104
|
+
|--------|--------|----------|
|
|
105
|
+
| API response (non-LLM) | <200ms P95 | >500ms |
|
|
106
|
+
| LLM response (Fast) | <500ms P95 | >2s |
|
|
107
|
+
| LLM response (Balanced) | <2s P95 | >5s |
|
|
108
|
+
| Initial bundle size | <500KB | >1MB |
|
|
109
|
+
| Cache hit rate (L1) | >80% | <60% |
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Pre-Production Resilience Checklist
|
|
3
|
+
impact: CRITICAL
|
|
4
|
+
impactDescription: Ensures application handles failures gracefully in production
|
|
5
|
+
tags: [checklist, resilience, production, deployment]
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Pre-Production Resilience Checklist
|
|
9
|
+
|
|
10
|
+
Resilience items that MUST be validated before any production deployment.
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
RESILIENCE CHECKLIST
|
|
14
|
+
├── [ ] LLM retry with exponential backoff
|
|
15
|
+
├── [ ] Timeout on ALL external requests (LLM, APIs, DB)
|
|
16
|
+
├── [ ] Healthcheck endpoint (/health or /api/health)
|
|
17
|
+
├── [ ] Readiness probe (for Kubernetes)
|
|
18
|
+
├── [ ] Graceful shutdown (handle SIGTERM)
|
|
19
|
+
├── [ ] Fallback providers configured (a model gateway or manual)
|
|
20
|
+
├── [ ] Circuit breaker for flaky services
|
|
21
|
+
├── [ ] Fail-open for non-critical services
|
|
22
|
+
└── [ ] Error boundaries in UI (React)
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
**Incorrect:**
|
|
26
|
+
|
|
27
|
+
```typescript
|
|
28
|
+
// No timeout, no retry
|
|
29
|
+
const response = await fetch(externalApi); // BAD: Can hang forever
|
|
30
|
+
|
|
31
|
+
// No graceful shutdown
|
|
32
|
+
process.on('SIGTERM', () => process.exit(0)); // BAD: Drops in-flight requests
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
**Correct:**
|
|
36
|
+
|
|
37
|
+
```typescript
|
|
38
|
+
// Timeout on external calls
|
|
39
|
+
const controller = new AbortController();
|
|
40
|
+
const timeout = setTimeout(() => controller.abort(), 30000);
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
const response = await fetch(externalApi, { signal: controller.signal });
|
|
44
|
+
} finally {
|
|
45
|
+
clearTimeout(timeout);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Graceful shutdown
|
|
49
|
+
let isShuttingDown = false;
|
|
50
|
+
|
|
51
|
+
process.on('SIGTERM', async () => {
|
|
52
|
+
console.info('[SHUTDOWN] Received SIGTERM, starting graceful shutdown');
|
|
53
|
+
isShuttingDown = true;
|
|
54
|
+
|
|
55
|
+
// Stop accepting new requests
|
|
56
|
+
server.close();
|
|
57
|
+
|
|
58
|
+
// Wait for in-flight requests (max 30s)
|
|
59
|
+
await Promise.race([
|
|
60
|
+
waitForInflightRequests(),
|
|
61
|
+
sleep(30000),
|
|
62
|
+
]);
|
|
63
|
+
|
|
64
|
+
// Close database connections
|
|
65
|
+
await db.destroy();
|
|
66
|
+
|
|
67
|
+
console.info('[SHUTDOWN] Graceful shutdown complete');
|
|
68
|
+
process.exit(0);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// Healthcheck endpoint
|
|
72
|
+
app.get('/health', (req, res) => {
|
|
73
|
+
if (isShuttingDown) {
|
|
74
|
+
return res.status(503).json({ status: 'shutting_down' });
|
|
75
|
+
}
|
|
76
|
+
res.json({ status: 'healthy', timestamp: new Date().toISOString() });
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
// Readiness probe (checks dependencies)
|
|
80
|
+
app.get('/ready', async (req, res) => {
|
|
81
|
+
try {
|
|
82
|
+
await db.query('SELECT 1');
|
|
83
|
+
res.json({ status: 'ready' });
|
|
84
|
+
} catch (error) {
|
|
85
|
+
res.status(503).json({ status: 'not_ready', error: error.message });
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
**LLM Retry Configuration:**
|
|
91
|
+
|
|
92
|
+
```typescript
|
|
93
|
+
const LLM_RETRY_CONFIG = {
|
|
94
|
+
maxRetries: 3,
|
|
95
|
+
initialDelayMs: 1000,
|
|
96
|
+
maxDelayMs: 10000,
|
|
97
|
+
retryableErrors: ['rate_limit_error', 'overloaded_error', 'api_error'],
|
|
98
|
+
};
|
|
99
|
+
```
|