multi-forge 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- forge/__init__.py +3 -0
- forge/_extensions/agents/.gitkeep +0 -0
- forge/_extensions/commands/.gitkeep +0 -0
- forge/_extensions/skills/analyze/SKILL.md +87 -0
- forge/_extensions/skills/challenge/SKILL.md +91 -0
- forge/_extensions/skills/consensus/SKILL.md +120 -0
- forge/_extensions/skills/consensus/resources/code_consensus_evaluation.md +94 -0
- forge/_extensions/skills/consensus/resources/consensus_evaluation.md +70 -0
- forge/_extensions/skills/consensus/resources/synthesis.md +101 -0
- forge/_extensions/skills/debate/SKILL.md +116 -0
- forge/_extensions/skills/debate/resources/code_debate_evaluation.md +101 -0
- forge/_extensions/skills/debate/resources/debate_evaluation.md +90 -0
- forge/_extensions/skills/panel/SKILL.md +141 -0
- forge/_extensions/skills/panel/resources/synthesis.md +103 -0
- forge/_extensions/skills/qa/SKILL.md +704 -0
- forge/_extensions/skills/qa/resources/checklist/0-enable.md +78 -0
- forge/_extensions/skills/qa/resources/checklist/1-preflight.md +24 -0
- forge/_extensions/skills/qa/resources/checklist/10-resume.md +143 -0
- forge/_extensions/skills/qa/resources/checklist/11-config.md +150 -0
- forge/_extensions/skills/qa/resources/checklist/12-search.md +58 -0
- forge/_extensions/skills/qa/resources/checklist/13-guard.md +237 -0
- forge/_extensions/skills/qa/resources/checklist/14-workflow.md +305 -0
- forge/_extensions/skills/qa/resources/checklist/15-skills.md +155 -0
- forge/_extensions/skills/qa/resources/checklist/16-handoff.md +224 -0
- forge/_extensions/skills/qa/resources/checklist/17-info.md +50 -0
- forge/_extensions/skills/qa/resources/checklist/18-disable.md +84 -0
- forge/_extensions/skills/qa/resources/checklist/19-uninstall.md +146 -0
- forge/_extensions/skills/qa/resources/checklist/2-extensions.md +188 -0
- forge/_extensions/skills/qa/resources/checklist/20-cleanup.md +36 -0
- forge/_extensions/skills/qa/resources/checklist/3-auth.md +234 -0
- forge/_extensions/skills/qa/resources/checklist/4-proxy.md +481 -0
- forge/_extensions/skills/qa/resources/checklist/5-session.md +541 -0
- forge/_extensions/skills/qa/resources/checklist/6-hooks.md +275 -0
- forge/_extensions/skills/qa/resources/checklist/7-costs.md +309 -0
- forge/_extensions/skills/qa/resources/checklist/8-status-line.md +174 -0
- forge/_extensions/skills/qa/resources/checklist/9-direct-commands.md +146 -0
- forge/_extensions/skills/qa/resources/checklist.md +103 -0
- forge/_extensions/skills/qa/resources/report-template.md +62 -0
- forge/_extensions/skills/qa/scripts/start-container.sh +529 -0
- forge/_extensions/skills/qa/scripts/walkthrough-state.py +1137 -0
- forge/_extensions/skills/review/SKILL.md +125 -0
- forge/_extensions/skills/review/references/claude-4.6.md +474 -0
- forge/_extensions/skills/review/references/claude-4.7.md +710 -0
- forge/_extensions/skills/review/references/gemini-3.1.md +546 -0
- forge/_extensions/skills/review/references/gpt-5.5.md +490 -0
- forge/_extensions/skills/review/references/skills-writing-guide.md +1588 -0
- forge/_extensions/skills/review/resources/code-anthropic.md +160 -0
- forge/_extensions/skills/review/resources/code-gemini.md +184 -0
- forge/_extensions/skills/review/resources/code-openai.md +203 -0
- forge/_extensions/skills/review/resources/code.md +160 -0
- forge/_extensions/skills/review-docs/SKILL.md +121 -0
- forge/_extensions/skills/review-docs/resources/docs-anthropic.md +170 -0
- forge/_extensions/skills/review-docs/resources/docs-gemini.md +204 -0
- forge/_extensions/skills/review-docs/resources/docs-openai.md +231 -0
- forge/_extensions/skills/review-docs/resources/docs.md +170 -0
- forge/_extensions/skills/smoke-test/SKILL.md +27 -0
- forge/_extensions/skills/smoke-test/scripts/smoke-test.sh +118 -0
- forge/_extensions/skills/understand/SKILL.md +148 -0
- forge/_extensions/skills/understand/resources/code-anthropic.md +163 -0
- forge/_extensions/skills/understand/resources/code-gemini.md +194 -0
- forge/_extensions/skills/understand/resources/code-openai.md +181 -0
- forge/_extensions/skills/understand/resources/code.md +163 -0
- forge/_extensions/skills/understand/resources/docs-anthropic.md +177 -0
- forge/_extensions/skills/understand/resources/docs-gemini.md +202 -0
- forge/_extensions/skills/understand/resources/docs-openai.md +191 -0
- forge/_extensions/skills/understand/resources/docs.md +177 -0
- forge/_extensions/skills/walkthrough/SKILL.md +599 -0
- forge/_extensions/skills/walkthrough/resources/checklist.md +765 -0
- forge/_extensions/skills/walkthrough/scripts/run-in-repo.sh +118 -0
- forge/_extensions/skills/walkthrough/scripts/setup-test-repo.sh +198 -0
- forge/_extensions/skills/walkthrough/scripts/walkthrough-state.py +1137 -0
- forge/backend/__init__.py +174 -0
- forge/backend/adapters/__init__.py +38 -0
- forge/backend/adapters/litellm.py +158 -0
- forge/backend/creation.py +89 -0
- forge/backend/registry.py +178 -0
- forge/cli/__init__.py +16 -0
- forge/cli/auth.py +483 -0
- forge/cli/backend.py +298 -0
- forge/cli/claude.py +411 -0
- forge/cli/config_cmd.py +303 -0
- forge/cli/extensions.py +1001 -0
- forge/cli/gc.py +165 -0
- forge/cli/guard.py +1018 -0
- forge/cli/guards.py +106 -0
- forge/cli/handoff.py +110 -0
- forge/cli/hooks/__init__.py +36 -0
- forge/cli/hooks/_group.py +20 -0
- forge/cli/hooks/_helpers.py +149 -0
- forge/cli/hooks/commands.py +1677 -0
- forge/cli/hooks/direct_commands.py +1304 -0
- forge/cli/hooks/install.py +232 -0
- forge/cli/hooks/policy.py +151 -0
- forge/cli/hooks/read_hygiene.py +74 -0
- forge/cli/hooks/verification.py +370 -0
- forge/cli/logs.py +406 -0
- forge/cli/main.py +292 -0
- forge/cli/proxy.py +1821 -0
- forge/cli/proxy_costs.py +313 -0
- forge/cli/search.py +416 -0
- forge/cli/session.py +892 -0
- forge/cli/session_addendum.py +81 -0
- forge/cli/session_fork.py +750 -0
- forge/cli/session_handoff.py +141 -0
- forge/cli/session_lifecycle.py +2053 -0
- forge/cli/session_manage.py +1336 -0
- forge/cli/session_memory.py +201 -0
- forge/cli/status_line.py +1398 -0
- forge/cli/workflow.py +1964 -0
- forge/config/__init__.py +110 -0
- forge/config/dataclass_utils.py +88 -0
- forge/config/defaults/__init__.py +0 -0
- forge/config/defaults/backends/__init__.py +0 -0
- forge/config/defaults/backends/litellm.yaml +196 -0
- forge/config/defaults/templates/__init__.py +0 -0
- forge/config/defaults/templates/litellm-anthropic-local.yaml +33 -0
- forge/config/defaults/templates/litellm-anthropic.yaml +24 -0
- forge/config/defaults/templates/litellm-gemini-flash-local.yaml +37 -0
- forge/config/defaults/templates/litellm-gemini-local.yaml +32 -0
- forge/config/defaults/templates/litellm-gemini-test.yaml +34 -0
- forge/config/defaults/templates/litellm-gemini.yaml +21 -0
- forge/config/defaults/templates/litellm-openai-codex-local.yaml +36 -0
- forge/config/defaults/templates/litellm-openai-local.yaml +38 -0
- forge/config/defaults/templates/litellm-openai.yaml +28 -0
- forge/config/defaults/templates/openrouter-anthropic.yaml +23 -0
- forge/config/defaults/templates/openrouter-deepseek.yaml +26 -0
- forge/config/defaults/templates/openrouter-gemini-flash.yaml +26 -0
- forge/config/defaults/templates/openrouter-gemini.yaml +23 -0
- forge/config/defaults/templates/openrouter-glm.yaml +23 -0
- forge/config/defaults/templates/openrouter-kimi.yaml +30 -0
- forge/config/defaults/templates/openrouter-minimax.yaml +26 -0
- forge/config/defaults/templates/openrouter-openai-codex.yaml +23 -0
- forge/config/defaults/templates/openrouter-openai.yaml +28 -0
- forge/config/defaults/templates/openrouter-qwen.yaml +25 -0
- forge/config/loader.py +675 -0
- forge/config/schema.py +448 -0
- forge/core/__init__.py +5 -0
- forge/core/auth/__init__.py +67 -0
- forge/core/auth/capabilities.py +219 -0
- forge/core/auth/credentials_file.py +244 -0
- forge/core/auth/protocols.py +18 -0
- forge/core/auth/secrets.py +243 -0
- forge/core/auth/template_secrets.py +112 -0
- forge/core/data/__init__.py +5 -0
- forge/core/data/model_catalog.yaml +1522 -0
- forge/core/data/pricing.yaml +140 -0
- forge/core/data/system_prompt_addendums/__init__.py +0 -0
- forge/core/data/system_prompt_addendums/gemini.md +330 -0
- forge/core/data/system_prompt_addendums/openai.md +328 -0
- forge/core/llm/__init__.py +231 -0
- forge/core/llm/clients/__init__.py +14 -0
- forge/core/llm/clients/base.py +115 -0
- forge/core/llm/clients/litellm.py +619 -0
- forge/core/llm/clients/openai_compat.py +244 -0
- forge/core/llm/clients/openrouter.py +234 -0
- forge/core/llm/credentials.py +439 -0
- forge/core/llm/detection.py +86 -0
- forge/core/llm/errors.py +44 -0
- forge/core/llm/protocols.py +80 -0
- forge/core/llm/types.py +176 -0
- forge/core/logging.py +146 -0
- forge/core/models/__init__.py +91 -0
- forge/core/models/catalog.py +467 -0
- forge/core/models/pricing.py +165 -0
- forge/core/models/types.py +167 -0
- forge/core/naming.py +212 -0
- forge/core/ops/__init__.py +73 -0
- forge/core/ops/context.py +141 -0
- forge/core/ops/gc.py +802 -0
- forge/core/ops/proxy.py +146 -0
- forge/core/ops/resolution.py +135 -0
- forge/core/ops/session.py +344 -0
- forge/core/ops/session_context.py +548 -0
- forge/core/paths.py +38 -0
- forge/core/process.py +54 -0
- forge/core/reactive/__init__.py +38 -0
- forge/core/reactive/cost_tracking.py +300 -0
- forge/core/reactive/env.py +180 -0
- forge/core/reactive/proxy.py +78 -0
- forge/core/reactive/routing.py +622 -0
- forge/core/reactive/session_runner.py +185 -0
- forge/core/reactive/structured_output.py +62 -0
- forge/core/reactive/tagger.py +94 -0
- forge/core/reactive/throttle.py +132 -0
- forge/core/state/__init__.py +59 -0
- forge/core/state/exceptions.py +59 -0
- forge/core/state/io.py +140 -0
- forge/core/state/lock.py +99 -0
- forge/core/state/timestamps.py +60 -0
- forge/core/transcript.py +78 -0
- forge/core/typing_helpers.py +24 -0
- forge/core/workqueue/__init__.py +67 -0
- forge/core/workqueue/queue.py +552 -0
- forge/core/workqueue/types.py +63 -0
- forge/guard/__init__.py +26 -0
- forge/guard/deterministic/__init__.py +26 -0
- forge/guard/deterministic/base.py +158 -0
- forge/guard/deterministic/coding_standards.py +256 -0
- forge/guard/deterministic/registry.py +148 -0
- forge/guard/deterministic/tdd.py +171 -0
- forge/guard/engine.py +216 -0
- forge/guard/protocols.py +91 -0
- forge/guard/queries.py +96 -0
- forge/guard/semantic/__init__.py +34 -0
- forge/guard/semantic/promotion.py +18 -0
- forge/guard/semantic/supervisor.py +813 -0
- forge/guard/semantic/verdict.py +183 -0
- forge/guard/store.py +124 -0
- forge/guard/team/__init__.py +6 -0
- forge/guard/team/config.py +24 -0
- forge/guard/team/handlers.py +209 -0
- forge/guard/team/prompts.py +41 -0
- forge/guard/types.py +125 -0
- forge/guard/workflow/__init__.py +17 -0
- forge/guard/workflow/branches.py +67 -0
- forge/guard/workflow/config.py +63 -0
- forge/guard/workflow/divergence.py +113 -0
- forge/guard/workflow/policy.py +87 -0
- forge/guard/workflow/stages.py +205 -0
- forge/install/__init__.py +55 -0
- forge/install/cli.py +281 -0
- forge/install/exceptions.py +163 -0
- forge/install/hooks.py +109 -0
- forge/install/installer.py +1037 -0
- forge/install/models.py +321 -0
- forge/install/preset.py +272 -0
- forge/install/settings_merge.py +831 -0
- forge/install/tracking.py +238 -0
- forge/install/version.py +141 -0
- forge/proxy/__init__.py +0 -0
- forge/proxy/base_client.py +181 -0
- forge/proxy/client_adapter.py +476 -0
- forge/proxy/client_factory.py +531 -0
- forge/proxy/converters.py +1206 -0
- forge/proxy/cost_logger.py +132 -0
- forge/proxy/cost_tracker.py +242 -0
- forge/proxy/data_models.py +338 -0
- forge/proxy/error_hints.py +92 -0
- forge/proxy/metrics.py +222 -0
- forge/proxy/model_spec.py +158 -0
- forge/proxy/proxies.py +333 -0
- forge/proxy/proxy_identity.py +134 -0
- forge/proxy/proxy_orchestrator.py +1018 -0
- forge/proxy/proxy_startup.py +54 -0
- forge/proxy/server.py +1561 -0
- forge/proxy/utils.py +537 -0
- forge/review/__init__.py +6 -0
- forge/review/adversarial.py +111 -0
- forge/review/consensus.py +236 -0
- forge/review/engine.py +356 -0
- forge/review/models.py +437 -0
- forge/review/resources/__init__.py +5 -0
- forge/review/resources/codereview-performance.md +85 -0
- forge/review/resources/codereview-quick.md +75 -0
- forge/review/resources/codereview-security.md +92 -0
- forge/review/resources/codereview.md +85 -0
- forge/review/resources/docreview-quick.md +75 -0
- forge/review/resources/docreview.md +86 -0
- forge/review/resources/thinkdeep.md +89 -0
- forge/review/routing.py +368 -0
- forge/review/synthesis.py +73 -0
- forge/runtime_config.py +438 -0
- forge/search/__init__.py +55 -0
- forge/search/bm25_store.py +264 -0
- forge/search/content_store.py +197 -0
- forge/search/engine.py +352 -0
- forge/search/exceptions.py +51 -0
- forge/search/extractor.py +234 -0
- forge/search/index_state.py +295 -0
- forge/search/store.py +215 -0
- forge/search/tokenizer.py +24 -0
- forge/session/__init__.py +130 -0
- forge/session/active.py +339 -0
- forge/session/artifacts.py +202 -0
- forge/session/claude/__init__.py +50 -0
- forge/session/claude/cleanup.py +105 -0
- forge/session/claude/invoke.py +236 -0
- forge/session/claude/paths.py +200 -0
- forge/session/cleanup.py +216 -0
- forge/session/config.py +34 -0
- forge/session/direct_model.py +107 -0
- forge/session/effective.py +169 -0
- forge/session/exceptions.py +255 -0
- forge/session/handoff.py +881 -0
- forge/session/handoff_agent.py +544 -0
- forge/session/hooks/__init__.py +35 -0
- forge/session/hooks/models.py +73 -0
- forge/session/hooks/session_start.py +507 -0
- forge/session/identity.py +84 -0
- forge/session/index.py +553 -0
- forge/session/manager.py +1506 -0
- forge/session/models.py +572 -0
- forge/session/overrides.py +344 -0
- forge/session/plan_resolution.py +286 -0
- forge/session/prev_sessions.py +128 -0
- forge/session/store.py +431 -0
- forge/session/validation.py +47 -0
- forge/session/worktree/__init__.py +65 -0
- forge/session/worktree/cleanup.py +262 -0
- forge/session/worktree/config_copy.py +203 -0
- forge/session/worktree/create.py +332 -0
- forge/sidecar/__init__.py +29 -0
- forge/sidecar/container.py +161 -0
- forge/sidecar/docker.py +86 -0
- forge/sidecar/secrets.py +19 -0
- multi_forge-0.2.0.dist-info/METADATA +242 -0
- multi_forge-0.2.0.dist-info/RECORD +311 -0
- multi_forge-0.2.0.dist-info/WHEEL +4 -0
- multi_forge-0.2.0.dist-info/entry_points.txt +2 -0
- multi_forge-0.2.0.dist-info/licenses/LICENSE +203 -0
- multi_forge-0.2.0.dist-info/licenses/NOTICE +14 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
<!-- prereq: 0.3, 2.1, 5.1 -->
|
|
2
|
+
|
|
3
|
+
## 6. Hooks Testing
|
|
4
|
+
|
|
5
|
+
Note: Install hooks via `forge extension enable` (full) or `forge hook enable` (hooks-only; writes to
|
|
6
|
+
`settings.local.json`).
|
|
7
|
+
|
|
8
|
+
### 6.1 Verify Hook Configuration
|
|
9
|
+
|
|
10
|
+
<!-- auto -->
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
# Check hooks in whichever settings file was used during enable.
|
|
14
|
+
# `forge extension enable` writes to the scope's settings file;
|
|
15
|
+
# `forge hook enable --local` always writes to settings.local.json.
|
|
16
|
+
cat $CLAUDE_HOME/settings.local.json | jq '.hooks' 2>/dev/null || \
|
|
17
|
+
cat $CLAUDE_HOME/settings.json | jq '.hooks'
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
- [ ] `PreToolUse` hooks configured (policy-check)
|
|
21
|
+
- [ ] `PostToolUse` hooks configured (plan-write)
|
|
22
|
+
- [ ] `Stop` hook configured
|
|
23
|
+
- [ ] `UserPromptSubmit` hook configured
|
|
24
|
+
- [ ] `SessionStart` hook configured
|
|
25
|
+
|
|
26
|
+
### 6.2 Install Hooks Only (Optional)
|
|
27
|
+
|
|
28
|
+
<!-- auto -->
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
# Install hooks only (no commands/skills)
|
|
32
|
+
forge hook enable --user
|
|
33
|
+
forge hook enable --local
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
- [ ] Hooks-only install works (writes to settings.local.json)
|
|
37
|
+
|
|
38
|
+
### 6.3 Test Hook Manually
|
|
39
|
+
|
|
40
|
+
<!-- auto -->
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
cd $FORGE_TEST_REPO
|
|
44
|
+
|
|
45
|
+
# Test the status-line command with the real stdin contract.
|
|
46
|
+
BASE_URL=$(jq -r '.intent.proxy.base_url // empty' .forge/sessions/test-session-1/forge.session.json)
|
|
47
|
+
mkdir -p .forge/walkthrough
|
|
48
|
+
cat > .forge/walkthrough/status-line-transcript.jsonl <<EOF
|
|
49
|
+
{"requestId":"req-001","message":{"role":"user","content":[{"type":"text","text":"Read the config file."}]}}
|
|
50
|
+
{"requestId":"req-001","message":{"role":"assistant","content":[{"type":"text","text":"I'll inspect it."},{"type":"tool_use","id":"tool-001","name":"Read","input":{"file_path":"${FORGE_TEST_REPO}/config.yaml"}}]}}
|
|
51
|
+
{"requestId":"req-001","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool-001","content":"timeout: 10"}]}}
|
|
52
|
+
{"requestId":"req-002","message":{"role":"user","content":[{"type":"text","text":"Update the timeout and run tests."}]}}
|
|
53
|
+
{"requestId":"req-002","message":{"role":"assistant","content":[{"type":"tool_use","id":"tool-002","name":"Edit","input":{"file_path":"${FORGE_TEST_REPO}/config.yaml"}},{"type":"tool_use","id":"tool-003","name":"Bash","input":{"command":"uv run pytest"}}]}}
|
|
54
|
+
EOF
|
|
55
|
+
STATUS_INPUT=$(jq -nc \
|
|
56
|
+
--arg cwd "$FORGE_TEST_REPO" \
|
|
57
|
+
--arg transcript "$FORGE_TEST_REPO/.forge/walkthrough/status-line-transcript.jsonl" \
|
|
58
|
+
'{
|
|
59
|
+
workspace: {current_dir: $cwd},
|
|
60
|
+
model: {display_name: "Opus 4.6"},
|
|
61
|
+
transcript_path: $transcript
|
|
62
|
+
}')
|
|
63
|
+
|
|
64
|
+
echo "$STATUS_INPUT" | FORGE_SESSION=test-session-1 ANTHROPIC_BASE_URL="$BASE_URL" forge status-line
|
|
65
|
+
|
|
66
|
+
# Test user-prompt-submit with a %help command
|
|
67
|
+
echo '{"prompt": "%help"}' | FORGE_SESSION=test-session-1 forge hook user-prompt-submit
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
- [ ] Status line outputs session/model info (and proxy info if available)
|
|
71
|
+
- [ ] `%help` returns help text (or decision payload)
|
|
72
|
+
|
|
73
|
+
### 6.4 Smoke Test SessionStart Hook
|
|
74
|
+
|
|
75
|
+
<!-- auto -->
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
cd $FORGE_TEST_REPO
|
|
79
|
+
|
|
80
|
+
# Use the candidate UUID already stored in the session manifest
|
|
81
|
+
SESSION_ID=$(cat .forge/sessions/test-session-1/forge.session.json | jq -r '.confirmed.claude_session_id')
|
|
82
|
+
|
|
83
|
+
echo "{\"session_id\":\"$SESSION_ID\",\"transcript_path\":\".forge/walkthrough/mock-transcript.jsonl\",\"source\":\"startup\"}" | FORGE_SESSION=test-session-1 forge hook session-start
|
|
84
|
+
|
|
85
|
+
# Verify manifest updated
|
|
86
|
+
cat .forge/sessions/test-session-1/forge.session.json | jq '.confirmed.transcript_path'
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
- [ ] Hook returns JSON success
|
|
90
|
+
- [ ] Manifest has `confirmed.transcript_path` set to the provided value
|
|
91
|
+
|
|
92
|
+
### 6.5 Smoke Test plan-write Hook (Plan Path Recorded)
|
|
93
|
+
|
|
94
|
+
<!-- auto -->
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
cd $FORGE_TEST_REPO
|
|
98
|
+
|
|
99
|
+
SESSION_ID=$(cat .forge/sessions/test-session-1/forge.session.json | jq -r '.confirmed.claude_session_id')
|
|
100
|
+
|
|
101
|
+
mkdir -p .claude/plans
|
|
102
|
+
echo "# Test Plan" > .claude/plans/test-plan.md
|
|
103
|
+
|
|
104
|
+
echo "{\"hook_event_name\":\"PostToolUse\",\"tool_input\":{\"file_path\":\".claude/plans/test-plan.md\"},\"session_id\":\"$SESSION_ID\"}" | FORGE_SESSION=test-session-1 forge hook plan-write
|
|
105
|
+
|
|
106
|
+
# Verify manifest recorded latest plan path
|
|
107
|
+
cat .forge/sessions/test-session-1/forge.session.json | jq '.confirmed.latest_plan_path'
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
- [ ] Hook returns `action: recorded`
|
|
111
|
+
- [ ] Manifest has `confirmed.latest_plan_path` pointing to `.claude/plans/test-plan.md`
|
|
112
|
+
|
|
113
|
+
### 6.6 Smoke Test exit-plan-mode Hook (Approved Snapshot)
|
|
114
|
+
|
|
115
|
+
<!-- auto -->
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
cd $FORGE_TEST_REPO
|
|
119
|
+
|
|
120
|
+
SESSION_ID=$(cat .forge/sessions/test-session-1/forge.session.json | jq -r '.confirmed.claude_session_id')
|
|
121
|
+
|
|
122
|
+
echo "{\"hook_event_name\":\"PreToolUse\",\"session_id\":\"$SESSION_ID\"}" | FORGE_SESSION=test-session-1 forge hook exit-plan-mode
|
|
123
|
+
|
|
124
|
+
# Verify snapshot exists
|
|
125
|
+
ls -la .forge/artifacts/test-session-1/plans/ | head -50
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
- [ ] Hook returns `action: snapshotted`
|
|
129
|
+
- [ ] Snapshot file created under `.forge/artifacts/test-session-1/plans/`
|
|
130
|
+
|
|
131
|
+
### 6.7 Smoke Test Stop Hook (Transcript Copy + Queue Markers)
|
|
132
|
+
|
|
133
|
+
<!-- auto -->
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
cd $FORGE_TEST_REPO
|
|
137
|
+
|
|
138
|
+
SESSION_ID=$(cat .forge/sessions/test-session-1/forge.session.json | jq -r '.confirmed.claude_session_id')
|
|
139
|
+
|
|
140
|
+
cat > .forge/walkthrough/mock-stop-transcript.jsonl << 'EOF'
|
|
141
|
+
{"type":"assistant","message":{"content":"(mock transcript)"}}
|
|
142
|
+
EOF
|
|
143
|
+
|
|
144
|
+
echo "{\"hook_event_name\":\"Stop\",\"session_id\":\"$SESSION_ID\",\"transcript_path\":\".forge/walkthrough/mock-stop-transcript.jsonl\"}" | FORGE_SESSION=test-session-1 forge hook stop
|
|
145
|
+
|
|
146
|
+
# Verify transcript snapshot copied into artifacts
|
|
147
|
+
ls -la .forge/artifacts/test-session-1/transcripts/ | head -50
|
|
148
|
+
test -f ".forge/artifacts/test-session-1/transcripts/${SESSION_ID}.jsonl"
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
- [ ] Hook returns JSON success
|
|
152
|
+
- [ ] Transcript copied to `.forge/artifacts/test-session-1/transcripts/<session_id>.jsonl`
|
|
153
|
+
|
|
154
|
+
### 6.8 Smoke Test pre-compact Hook (Transcript Capture)
|
|
155
|
+
|
|
156
|
+
<!-- auto -->
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
# pre-compact captures transcript before compaction (always exit 0)
|
|
160
|
+
jq -nc --arg cwd "$FORGE_TEST_REPO" \
|
|
161
|
+
'{session_id: "test-uuid", transcript_path: "/tmp/test.jsonl", cwd: $cwd}' \
|
|
162
|
+
| FORGE_SESSION=test-session-1 forge hook pre-compact
|
|
163
|
+
echo "exit=$?"
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
- [ ] Exit code is 0
|
|
167
|
+
|
|
168
|
+
### 6.9 Smoke Test policy-check Hook (Fail-Open)
|
|
169
|
+
|
|
170
|
+
<!-- auto -->
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
# Default session has policy disabled, so this should allow (exit 0)
|
|
174
|
+
echo '{"hook_event_name":"PreToolUse","tool_name":"Write","tool_input":{"file_path":"src/example.py","content":"x"}}' | FORGE_SESSION=test-session-1 forge hook policy-check
|
|
175
|
+
echo "exit=$?"
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
- [ ] Exit code is 0 (allowed)
|
|
179
|
+
|
|
180
|
+
### 6.10 End-to-End Stop Hook (Real Session Exit)
|
|
181
|
+
|
|
182
|
+
<!-- prereq: 4.2 -->
|
|
183
|
+
|
|
184
|
+
<!-- requires: api_key -->
|
|
185
|
+
|
|
186
|
+
<!-- human:guided -->
|
|
187
|
+
|
|
188
|
+
Start a real Claude session via Forge, perform a small action, then exit. Verify that hooks actually fired and wrote to
|
|
189
|
+
the session manifest and artifacts — this catches "hooks wired but not firing" regressions that manual `forge hook ...`
|
|
190
|
+
invocations (steps 6.3–6.9) cannot detect.
|
|
191
|
+
|
|
192
|
+
In the **container shell**, clean up and start a session:
|
|
193
|
+
|
|
194
|
+
```
|
|
195
|
+
forge session delete hook-e2e-test --force 2>/dev/null || true
|
|
196
|
+
forge session start hook-e2e-test --proxy "$FORGE_QA_OPENAI_PROXY"
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
Inside the launched Claude session, do a small action (e.g., "write hello to /tmp/test.txt" or "read the repository
|
|
200
|
+
README and tell me the title"), then exit Claude (Ctrl+C or `/exit`).
|
|
201
|
+
|
|
202
|
+
After Claude exits, run these exact checks in the **container shell**:
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
MANIFEST=".forge/sessions/hook-e2e-test/forge.session.json"
|
|
206
|
+
|
|
207
|
+
# Confirmed fields written by Stop hook
|
|
208
|
+
jq '.confirmed | {claude_session_id, transcript_path, confirmed_by, confirmed_at}' "$MANIFEST"
|
|
209
|
+
|
|
210
|
+
# Programmatic manifest assertions
|
|
211
|
+
jq -e '.confirmed.transcript_path | strings | length > 0' "$MANIFEST"
|
|
212
|
+
jq -e '.confirmed.claude_session_id | strings | length > 0' "$MANIFEST"
|
|
213
|
+
jq -e '.confirmed.confirmed_by == "hook:stop"' "$MANIFEST"
|
|
214
|
+
|
|
215
|
+
# Transcript artifact copied
|
|
216
|
+
test -d .forge/artifacts/hook-e2e-test/transcripts
|
|
217
|
+
find .forge/artifacts/hook-e2e-test/transcripts -type f -name '*.jsonl' -print -quit | grep -q .
|
|
218
|
+
|
|
219
|
+
# Stop hook log exists
|
|
220
|
+
ls ~/.forge/logs/hooks/stop.*.log | tail -1
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
- [ ] Claude session starts and runs with hooks active
|
|
224
|
+
- [ ] After exit, `confirmed.transcript_path` is set in session manifest
|
|
225
|
+
- [ ] After exit, `confirmed.claude_session_id` is set (reconciled from actual session)
|
|
226
|
+
- [ ] Transcript artifact copied to `.forge/artifacts/hook-e2e-test/transcripts/`
|
|
227
|
+
- [ ] Stop hook ran automatically (check `confirmed_by` = "hook:stop")
|
|
228
|
+
|
|
229
|
+
### 6.11 WorktreeCreate Hook (Claude-Native Worktree)
|
|
230
|
+
|
|
231
|
+
<!-- prereq: 4.2 -->
|
|
232
|
+
|
|
233
|
+
<!-- requires: api_key -->
|
|
234
|
+
|
|
235
|
+
<!-- human:guided -->
|
|
236
|
+
|
|
237
|
+
Verify that Claude Code's native worktree creation (via `--worktree` or the Agent tool with `isolation: "worktree"`)
|
|
238
|
+
triggers Forge's WorktreeCreate hook, which creates the worktree and auto-installs extensions.
|
|
239
|
+
|
|
240
|
+
In the **container shell**, clean up and start a worktree session:
|
|
241
|
+
|
|
242
|
+
```
|
|
243
|
+
forge session delete wt-hook-test --yes --force 2>/dev/null || true
|
|
244
|
+
WORKTREE_PATH="${FORGE_TEST_REPO}-wt-hook-test"
|
|
245
|
+
git worktree remove "$WORKTREE_PATH" --force 2>/dev/null || true
|
|
246
|
+
git branch -D wt-hook-test 2>/dev/null || true
|
|
247
|
+
forge session start wt-hook-test --worktree --proxy "$FORGE_QA_OPENAI_PROXY"
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Inside the launched Claude session, verify the status line is visible and type `%help` (should list Forge direct
|
|
251
|
+
commands), then exit Claude (`/exit`).
|
|
252
|
+
|
|
253
|
+
After Claude exits, verify:
|
|
254
|
+
|
|
255
|
+
```bash
|
|
256
|
+
WORKTREE_PATH="${FORGE_TEST_REPO}-wt-hook-test"
|
|
257
|
+
|
|
258
|
+
# Worktree was created
|
|
259
|
+
ls -d "$WORKTREE_PATH" 2>/dev/null || echo "worktree not found"
|
|
260
|
+
git worktree list | grep wt-hook-test
|
|
261
|
+
|
|
262
|
+
# Forge extensions installed in the worktree
|
|
263
|
+
cat "$WORKTREE_PATH/.claude/settings.local.json" 2>/dev/null | jq '.hooks | keys'
|
|
264
|
+
|
|
265
|
+
# Cleanup
|
|
266
|
+
forge session delete wt-hook-test --yes --force
|
|
267
|
+
git worktree list | grep wt-hook-test && echo "FAIL: worktree not removed" || echo "OK: worktree cleaned up"
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
- [ ] Worktree created by Forge's WorktreeCreate hook (not Claude Code's default)
|
|
271
|
+
- [ ] Forge extensions installed in the worktree (hooks in settings.local.json)
|
|
272
|
+
- [ ] Status line visible in the worktree session
|
|
273
|
+
- [ ] Worktree cleaned up after session delete
|
|
274
|
+
|
|
275
|
+
---
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
<!-- prereq: 0.3 -->
|
|
2
|
+
|
|
3
|
+
## 7. Cost Tracking & Spend Caps
|
|
4
|
+
|
|
5
|
+
### 7.1 Cost CLI (Empty State)
|
|
6
|
+
|
|
7
|
+
<!-- auto -->
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
# Use a guaranteed-empty proxy_id for empty-state tests.
|
|
11
|
+
# Other sections (e.g., section 4 guided sessions) may have created real cost logs,
|
|
12
|
+
# so we cannot assume global cost logs are empty.
|
|
13
|
+
forge proxy costs qa-no-such-proxy 2>&1
|
|
14
|
+
echo "---"
|
|
15
|
+
forge proxy costs qa-no-such-proxy --period all 2>&1
|
|
16
|
+
echo "---"
|
|
17
|
+
forge proxy costs qa-no-such-proxy --json
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
- [ ] `forge proxy costs qa-no-such-proxy` shows `No cost data for today (qa-no-such-proxy).`
|
|
21
|
+
- [ ] `--period all` shows `No cost data for all (qa-no-such-proxy).`
|
|
22
|
+
- [ ] `--json` returns valid JSON with `total_cost_micros: 0` and `total_requests: 0`
|
|
23
|
+
|
|
24
|
+
### 7.2 Cost CLI (JSON Structure)
|
|
25
|
+
|
|
26
|
+
<!-- auto -->
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
# Verify JSON output schema using the empty-proxy filter (guaranteed empty)
|
|
30
|
+
forge proxy costs qa-no-such-proxy --json | python3 -c "
|
|
31
|
+
import json, sys
|
|
32
|
+
d = json.load(sys.stdin)
|
|
33
|
+
fields = {'period','proxy_id','total_cost_micros','total_cost_usd','total_requests','interactive_cost_micros','by_verb','by_model','estimated'}
|
|
34
|
+
missing = fields - set(d.keys())
|
|
35
|
+
print(f'MISSING={missing}' if missing else 'ALL_FIELDS_PRESENT')
|
|
36
|
+
print(f'period={d[\"period\"]}')
|
|
37
|
+
print(f'estimated={d[\"estimated\"]}')
|
|
38
|
+
"
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
- [ ] JSON contains all required fields: `period`, `proxy_id`, `total_cost_micros`, `total_cost_usd`, `total_requests`,
|
|
42
|
+
`interactive_cost_micros`, `by_verb`, `by_model`, `estimated`
|
|
43
|
+
- [ ] `period` is `today`
|
|
44
|
+
- [ ] `estimated` is `true`
|
|
45
|
+
|
|
46
|
+
### 7.3 Seed Fixture Request Logs
|
|
47
|
+
|
|
48
|
+
<!-- auto -->
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
# Seed QA-prefixed fixture request logs matching cost_logger.py record schema.
|
|
52
|
+
# Uses qa-fixture prefix and PID 99999 to avoid collision with real proxy logs.
|
|
53
|
+
mkdir -p ~/.forge/costs/requests
|
|
54
|
+
cat > ~/.forge/costs/requests/qa-fixture_99999.jsonl <<'EOF'
|
|
55
|
+
{"ts":"2026-05-01T00:00:00Z","proxy_id":"qa-fixture","model":"test/gemini-2.5-flash","tier":"haiku","input_tokens":200,"output_tokens":80,"cached_tokens":0,"cost_micros":300,"estimated":true,"pricing_source":"catalog","latency_ms":120.0,"failed":false,"request_id":"req-qa-001"}
|
|
56
|
+
{"ts":"2026-05-01T00:01:00Z","proxy_id":"qa-fixture","model":"test/gemini-3.1-pro-preview","tier":"sonnet","input_tokens":500,"output_tokens":150,"cached_tokens":50,"cost_micros":1200,"estimated":true,"pricing_source":"catalog","latency_ms":350.0,"failed":false,"request_id":"req-qa-002"}
|
|
57
|
+
{"ts":"2026-05-01T00:02:00Z","proxy_id":"qa-fixture","model":"test/gemini-3.1-pro-preview","tier":"opus","input_tokens":1000,"output_tokens":400,"cached_tokens":100,"cost_micros":3500,"estimated":true,"pricing_source":"catalog","latency_ms":800.0,"failed":false,"request_id":"req-qa-003"}
|
|
58
|
+
EOF
|
|
59
|
+
|
|
60
|
+
# Verify fixture is readable -- filter by qa-fixture to isolate from real proxy logs
|
|
61
|
+
forge proxy costs qa-fixture --period all --json
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
- [ ] Fixture file created at `~/.forge/costs/requests/qa-fixture_99999.jsonl`
|
|
65
|
+
- [ ] `forge proxy costs qa-fixture --period all --json` shows `total_cost_micros` of 5000 (300 + 1200 + 3500)
|
|
66
|
+
- [ ] `total_requests` is 3
|
|
67
|
+
- [ ] `by_model` contains both `test/gemini-2.5-flash` and `test/gemini-3.1-pro-preview`
|
|
68
|
+
|
|
69
|
+
### 7.4 Seed Fixture Verb Logs
|
|
70
|
+
|
|
71
|
+
<!-- auto -->
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Seed QA-prefixed fixture verb logs matching cost_tracking.py verb record schema.
|
|
75
|
+
mkdir -p ~/.forge/costs/verbs
|
|
76
|
+
cat > ~/.forge/costs/verbs/qa-fixture_99999.jsonl <<'EOF'
|
|
77
|
+
{"ts":"2026-05-01T00:05:00Z","verb":"qa-fixture-panel","total_cost_micros":1500,"estimated":true,"input_tokens":700,"output_tokens":230,"cached_tokens":50,"request_count":2,"duration_ms":1200.0,"per_proxy":[{"base_url":"http://localhost:8084","cost_micros":1500,"input_tokens":700,"output_tokens":230,"cached_tokens":50,"request_count":2}]}
|
|
78
|
+
EOF
|
|
79
|
+
|
|
80
|
+
# Verify verb attribution appears. Do not proxy-filter this check: verb logs are scoped
|
|
81
|
+
# by resolved proxy base_url, while qa-fixture is only a request-log proxy_id fixture.
|
|
82
|
+
forge proxy costs --period all 2>&1
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
- [ ] Fixture file created at `~/.forge/costs/verbs/qa-fixture_99999.jsonl`
|
|
86
|
+
- [ ] `forge proxy costs --period all` shows `qa-fixture-panel` verb in output
|
|
87
|
+
- [ ] Verb cost attributed to `qa-fixture-panel` (1500 micros)
|
|
88
|
+
|
|
89
|
+
### 7.5 Cost CLI Breakdowns
|
|
90
|
+
|
|
91
|
+
<!-- auto -->
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
# By-model breakdown -- filter to qa-fixture to isolate from real proxy logs
|
|
95
|
+
forge proxy costs qa-fixture --by-model --period all 2>&1
|
|
96
|
+
|
|
97
|
+
echo "---"
|
|
98
|
+
|
|
99
|
+
# JSON with proxy_id filter
|
|
100
|
+
forge proxy costs qa-fixture --period all --json
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
- [ ] `--by-model` table shows model names with cost and token columns
|
|
104
|
+
- [ ] JSON output has `proxy_id: "qa-fixture"`
|
|
105
|
+
- [ ] Filtered `total_requests` is 3 (only qa-fixture records)
|
|
106
|
+
- [ ] Rich table output captured via `2>&1` (console uses stderr)
|
|
107
|
+
|
|
108
|
+
### 7.6 Malformed Log Resilience
|
|
109
|
+
|
|
110
|
+
<!-- auto -->
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
# Append non-JSON garbage lines to the fixture request log
|
|
114
|
+
echo 'THIS_IS_NOT_JSON' >> ~/.forge/costs/requests/qa-fixture_99999.jsonl
|
|
115
|
+
echo '<<<CORRUPT>>>' >> ~/.forge/costs/requests/qa-fixture_99999.jsonl
|
|
116
|
+
|
|
117
|
+
# Cost CLI should skip malformed lines -- filter to qa-fixture for deterministic count
|
|
118
|
+
forge proxy costs qa-fixture --period all --json 2>&1
|
|
119
|
+
echo "EXIT=$?"
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
- [ ] Command succeeds (exit 0) despite malformed lines
|
|
123
|
+
- [ ] Valid records still returned (`total_requests` is 3, not 5)
|
|
124
|
+
- [ ] No traceback or error on stderr
|
|
125
|
+
|
|
126
|
+
### 7.7 Spend Cap Configuration via CLI
|
|
127
|
+
|
|
128
|
+
<!-- prereq: 4.2 -->
|
|
129
|
+
|
|
130
|
+
<!-- auto -->
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
# Set spend caps on the test proxy from section 4
|
|
134
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.caps.per_day=20.00
|
|
135
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.caps.per_month=100.00
|
|
136
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.cap_mode=post
|
|
137
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.on_cap_hit=reject
|
|
138
|
+
|
|
139
|
+
# Validate config is healthy after cap changes
|
|
140
|
+
forge proxy validate "$FORGE_QA_GEMINI_PROXY"
|
|
141
|
+
|
|
142
|
+
# Show raw YAML to verify caps appear
|
|
143
|
+
forge proxy show "$FORGE_QA_GEMINI_PROXY" --raw
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
- [ ] `costs.caps.per_day` appears in raw YAML as `20.0` (float, not string `"20.00"`)
|
|
147
|
+
- [ ] `costs.caps.per_month` appears as `100.0`
|
|
148
|
+
- [ ] `cap_mode` is `post`
|
|
149
|
+
- [ ] `on_cap_hit` is `reject`
|
|
150
|
+
- [ ] Config validates successfully after setting caps
|
|
151
|
+
- [ ] Raw YAML shows complete `costs:` section with `caps`, `cap_mode`, `on_cap_hit`
|
|
152
|
+
|
|
153
|
+
### 7.8 Spend Cap Config Validation (Invalid Values)
|
|
154
|
+
|
|
155
|
+
<!-- prereq: 4.2 -->
|
|
156
|
+
|
|
157
|
+
<!-- auto -->
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
# Invalid cap_mode -- should be rejected
|
|
161
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.cap_mode=invalid 2>&1; echo "EXIT=$?"
|
|
162
|
+
|
|
163
|
+
# Invalid on_cap_hit -- should be rejected
|
|
164
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.on_cap_hit=invalid 2>&1; echo "EXIT=$?"
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
- [ ] Invalid `cap_mode` rejected with validation error (exit non-zero)
|
|
168
|
+
- [ ] Invalid `on_cap_hit` rejected with validation error (exit non-zero)
|
|
169
|
+
- [ ] Error messages reference valid values (`post`/`strict` and `reject`/`warn`)
|
|
170
|
+
|
|
171
|
+
### 7.9 Spend Cap Enforcement (Reject Mode)
|
|
172
|
+
|
|
173
|
+
<!-- prereq: 4.2 -->
|
|
174
|
+
|
|
175
|
+
<!-- requires: api_key -->
|
|
176
|
+
|
|
177
|
+
<!-- human:guided -->
|
|
178
|
+
|
|
179
|
+
Seed a current-timestamp cost log so the proxy's cost tracker bootstraps above the cap, then make a request to verify
|
|
180
|
+
rejection. This avoids depending on a real request landing above a tiny cap (which is non-deterministic for cheap
|
|
181
|
+
models).
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
# Set a low daily cap on the working QA OpenAI proxy in the container
|
|
185
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.caps.per_day=0.01
|
|
186
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.on_cap_hit=reject
|
|
187
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.cap_mode=post
|
|
188
|
+
|
|
189
|
+
# Seed a cost log with a current timestamp so the tracker bootstraps above the cap.
|
|
190
|
+
# The tracker reads YYYY-MM_*.jsonl files on startup (bootstrap_from_logs).
|
|
191
|
+
mkdir -p ~/.forge/costs/requests
|
|
192
|
+
MONTH=$(date -u +%Y-%m)
|
|
193
|
+
TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
194
|
+
echo "{\"ts\":\"$TS\",\"proxy_id\":\"$FORGE_QA_OPENAI_PROXY\",\"model\":\"seed\",\"tier\":\"sonnet\",\"input_tokens\":0,\"output_tokens\":0,\"cached_tokens\":0,\"cost_micros\":50000,\"estimated\":true,\"pricing_source\":\"catalog\",\"latency_ms\":0,\"failed\":false,\"request_id\":\"req-qa-cap-seed\"}" \
|
|
195
|
+
> ~/.forge/costs/requests/${MONTH}_qa-cap-seed.jsonl
|
|
196
|
+
|
|
197
|
+
# Restart proxy so it bootstraps from the seeded log (--force bypasses shared-port check)
|
|
198
|
+
forge proxy stop "$FORGE_QA_OPENAI_PROXY" --force 2>/dev/null || true
|
|
199
|
+
forge proxy start "$FORGE_QA_OPENAI_PROXY"
|
|
200
|
+
|
|
201
|
+
# Make a request -- should be rejected immediately
|
|
202
|
+
forge claude start --proxy "$FORGE_QA_OPENAI_PROXY"
|
|
203
|
+
# Say "hello" -- expect rejection or error about spend cap, then exit (/exit)
|
|
204
|
+
|
|
205
|
+
# Clean up seeded log
|
|
206
|
+
rm -f ~/.forge/costs/requests/${MONTH}_qa-cap-seed.jsonl
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
- [ ] After proxy restart, the seeded cost triggers the daily cap
|
|
210
|
+
- [ ] Proxy returns HTTP 429 or Claude reports a `spend_cap_exceeded` error
|
|
211
|
+
- [ ] Error message includes current spend and limit amounts
|
|
212
|
+
- [ ] Error message suggests `forge proxy set <id> costs.caps.per_day=<amount>` to adjust
|
|
213
|
+
|
|
214
|
+
### 7.10 Spend Cap Enforcement (Warn Mode)
|
|
215
|
+
|
|
216
|
+
<!-- prereq: 4.2 -->
|
|
217
|
+
|
|
218
|
+
<!-- requires: api_key -->
|
|
219
|
+
|
|
220
|
+
<!-- human:guided -->
|
|
221
|
+
|
|
222
|
+
Switch to warn mode and verify requests succeed with a warning header instead of being blocked. Uses the same seeded
|
|
223
|
+
cost log approach for deterministic cap triggering.
|
|
224
|
+
|
|
225
|
+
```
|
|
226
|
+
# Use the same deterministic cap settings as 7.9, then switch to warn mode.
|
|
227
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.caps.per_day=0.01
|
|
228
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.cap_mode=post
|
|
229
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.on_cap_hit=warn
|
|
230
|
+
|
|
231
|
+
# Re-seed the cost log (cleanup from 7.9 removed it)
|
|
232
|
+
mkdir -p ~/.forge/costs/requests
|
|
233
|
+
MONTH=$(date -u +%Y-%m)
|
|
234
|
+
TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
235
|
+
echo "{\"ts\":\"$TS\",\"proxy_id\":\"$FORGE_QA_OPENAI_PROXY\",\"model\":\"seed\",\"tier\":\"sonnet\",\"input_tokens\":0,\"output_tokens\":0,\"cached_tokens\":0,\"cost_micros\":50000,\"estimated\":true,\"pricing_source\":\"catalog\",\"latency_ms\":0,\"failed\":false,\"request_id\":\"req-qa-cap-warn\"}" \
|
|
236
|
+
> ~/.forge/costs/requests/${MONTH}_qa-cap-seed.jsonl
|
|
237
|
+
|
|
238
|
+
# Restart proxy so it bootstraps with the seeded cost (--force bypasses shared-port check)
|
|
239
|
+
forge proxy stop "$FORGE_QA_OPENAI_PROXY" --force 2>/dev/null || true
|
|
240
|
+
forge proxy start "$FORGE_QA_OPENAI_PROXY"
|
|
241
|
+
|
|
242
|
+
# Make a direct request and capture response headers.
|
|
243
|
+
BASE_URL=$(jq -r --arg id "$FORGE_QA_OPENAI_PROXY" '.proxies[$id].base_url' ~/.forge/proxies/index.json)
|
|
244
|
+
curl -sS -D /tmp/qa-spend-warn.headers -o /tmp/qa-spend-warn.body \
|
|
245
|
+
-w 'HTTP=%{http_code}\n' \
|
|
246
|
+
-H 'content-type: application/json' \
|
|
247
|
+
-H 'x-api-key: test' \
|
|
248
|
+
-H 'user-agent: claude-code/qa-spend-warn' \
|
|
249
|
+
"$BASE_URL/v1/messages" \
|
|
250
|
+
-d '{"model":"claude-3-5-haiku-20241022","max_tokens":16,"temperature":0,"messages":[{"role":"user","content":"Reply with exactly one word: ok"}]}'
|
|
251
|
+
|
|
252
|
+
# Verify the response was allowed and included the warn-mode header.
|
|
253
|
+
grep -i '^x-spend-warning:' /tmp/qa-spend-warn.headers
|
|
254
|
+
cat /tmp/qa-spend-warn.body | jq -r '._request_id // empty'
|
|
255
|
+
# If curl did not report HTTP=200, inspect the proxy error details:
|
|
256
|
+
# cat /tmp/qa-spend-warn.body | jq .
|
|
257
|
+
# forge logs --tail proxy
|
|
258
|
+
|
|
259
|
+
# Optional Claude smoke: run with debug output, say "hello", then exit (/exit).
|
|
260
|
+
# The deterministic header check above is the source of truth for this step.
|
|
261
|
+
forge claude start --proxy "$FORGE_QA_OPENAI_PROXY" -- --debug
|
|
262
|
+
|
|
263
|
+
# Clean up seeded log
|
|
264
|
+
rm -f ~/.forge/costs/requests/${MONTH}_qa-cap-seed.jsonl /tmp/qa-spend-warn.headers /tmp/qa-spend-warn.body
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
- [ ] Request succeeds (not blocked) in warn mode
|
|
268
|
+
- [ ] `curl` reports `HTTP=200`
|
|
269
|
+
- [ ] `grep -i '^x-spend-warning:' /tmp/qa-spend-warn.headers` prints the spend-cap warning header
|
|
270
|
+
- [ ] Optional Claude debug run also succeeds (no `spend_cap_exceeded` block)
|
|
271
|
+
|
|
272
|
+
### 7.11 Cleanup Fixture Cost Logs
|
|
273
|
+
|
|
274
|
+
<!-- auto -->
|
|
275
|
+
|
|
276
|
+
```bash
|
|
277
|
+
# Remove only QA fixture files -- do not touch real proxy cost logs
|
|
278
|
+
rm -f ~/.forge/costs/requests/qa-fixture_*.jsonl
|
|
279
|
+
rm -f ~/.forge/costs/verbs/qa-fixture_*.jsonl
|
|
280
|
+
|
|
281
|
+
# Remove cap-seed logs from 7.9/7.10 (in case cleanup within those steps failed)
|
|
282
|
+
rm -f ~/.forge/costs/requests/*_qa-cap-seed.jsonl
|
|
283
|
+
|
|
284
|
+
# Verify cleanup: no QA-owned cost fixture files remain
|
|
285
|
+
ls ~/.forge/costs/requests/qa-fixture_*.jsonl 2>&1 || echo "QA_REQUEST_LOGS_CLEAN"
|
|
286
|
+
ls ~/.forge/costs/verbs/qa-fixture_*.jsonl 2>&1 || echo "QA_VERB_LOGS_CLEAN"
|
|
287
|
+
ls ~/.forge/costs/requests/*_qa-cap-seed.jsonl 2>&1 || echo "QA_CAP_SEED_LOGS_CLEAN"
|
|
288
|
+
|
|
289
|
+
# Reset spend caps on test proxies
|
|
290
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.caps.per_day=none 2>/dev/null || true
|
|
291
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.caps.per_month=none 2>/dev/null || true
|
|
292
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.on_cap_hit=reject 2>/dev/null || true
|
|
293
|
+
forge proxy set "$FORGE_QA_GEMINI_PROXY" costs.cap_mode=post 2>/dev/null || true
|
|
294
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.caps.per_day=none 2>/dev/null || true
|
|
295
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.caps.per_month=none 2>/dev/null || true
|
|
296
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.on_cap_hit=reject 2>/dev/null || true
|
|
297
|
+
forge proxy set "$FORGE_QA_OPENAI_PROXY" costs.cap_mode=post 2>/dev/null || true
|
|
298
|
+
|
|
299
|
+
# Restart the QA OpenAI proxy so the running proxy drops seeded spend/cap state from 7.9/7.10
|
|
300
|
+
forge proxy stop "$FORGE_QA_OPENAI_PROXY" --force 2>/dev/null || true
|
|
301
|
+
forge proxy start "$FORGE_QA_OPENAI_PROXY"
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
- [ ] QA fixture request logs removed (no `qa-fixture_*.jsonl` in `requests/`)
|
|
305
|
+
- [ ] QA fixture verb logs removed (no `qa-fixture_*.jsonl` in `verbs/`)
|
|
306
|
+
- [ ] QA cap seed logs removed (no `*_qa-cap-seed.jsonl` in `requests/`)
|
|
307
|
+
- [ ] Spend caps reset on QA OpenAI and Gemini test proxies
|
|
308
|
+
|
|
309
|
+
---
|