broharness 0.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. broharness-0.0.0/.github/workflows/publish.yml +39 -0
  2. broharness-0.0.0/.gitignore +13 -0
  3. broharness-0.0.0/.python-version +1 -0
  4. broharness-0.0.0/DIAGRAM.md +22 -0
  5. broharness-0.0.0/PKG-INFO +268 -0
  6. broharness-0.0.0/README.md +257 -0
  7. broharness-0.0.0/notebooks/bedrock.ipynb +417 -0
  8. broharness-0.0.0/notebooks/dev.ipynb +556 -0
  9. broharness-0.0.0/notebooks/flow_idea.ipynb +1226 -0
  10. broharness-0.0.0/notebooks/from_scratch.ipynb +533 -0
  11. broharness-0.0.0/notebooks/recipe.ipynb +571 -0
  12. broharness-0.0.0/notebooks/skill_tool_flow_idea.ipynb +457 -0
  13. broharness-0.0.0/pyproject.toml +29 -0
  14. broharness-0.0.0/skills/ds-mentor/SKILL.md +54 -0
  15. broharness-0.0.0/skills/ds-mentor/references/common-pitfalls.md +53 -0
  16. broharness-0.0.0/skills/ds-mentor/references/metrics.md +48 -0
  17. broharness-0.0.0/skills/ds-mentor/references/model-basics.md +53 -0
  18. broharness-0.0.0/skills/ds-mentor/references/pandas-gotchas.md +60 -0
  19. broharness-0.0.0/skills/ds-mentor/references/stats-basics.md +54 -0
  20. broharness-0.0.0/skills/file-ops/SKILL.md +153 -0
  21. broharness-0.0.0/skills/file-ops/scripts/create_file.py +67 -0
  22. broharness-0.0.0/skills/file-ops/scripts/delete_file.py +60 -0
  23. broharness-0.0.0/skills/file-ops/scripts/list_directory.py +74 -0
  24. broharness-0.0.0/skills/file-ops/scripts/read_file.py +78 -0
  25. broharness-0.0.0/skills/file-ops/scripts/update_file.py +63 -0
  26. broharness-0.0.0/skills/skill-call/SKILL.md +82 -0
  27. broharness-0.0.0/skills/tell-joke/SKILL.md +46 -0
  28. broharness-0.0.0/skills/tell-joke/references/dad-joke.md +17 -0
  29. broharness-0.0.0/skills/tell-joke/references/knock-knock.md +17 -0
  30. broharness-0.0.0/skills/tell-joke/references/one-liner.md +19 -0
  31. broharness-0.0.0/skills/tell-joke/references/pun-joke.md +17 -0
  32. broharness-0.0.0/skills/tell-joke/references/riddle.md +18 -0
  33. broharness-0.0.0/skills/tool-call/SKILL.md +92 -0
  34. broharness-0.0.0/src/broharness/__init__.py +5 -0
  35. broharness-0.0.0/src/broharness/codeblock.py +105 -0
  36. broharness-0.0.0/src/broharness/data_model.py +174 -0
  37. broharness-0.0.0/src/broharness/debug.py +42 -0
  38. broharness-0.0.0/src/broharness/flows/__init__.py +0 -0
  39. broharness-0.0.0/src/broharness/flows/answer.py +107 -0
  40. broharness-0.0.0/src/broharness/flows/ask_user_question.py +28 -0
  41. broharness-0.0.0/src/broharness/flows/fail_recovery.py +30 -0
  42. broharness-0.0.0/src/broharness/flows/skill_call.py +72 -0
  43. broharness-0.0.0/src/broharness/flows/tool_call.py +85 -0
  44. broharness-0.0.0/src/broharness/flows/tool_use.py +151 -0
  45. broharness-0.0.0/src/broharness/harness.py +51 -0
  46. broharness-0.0.0/src/broharness/llms/__init__.py +0 -0
  47. broharness-0.0.0/src/broharness/llms/bedrock.py +30 -0
  48. broharness-0.0.0/src/broharness/toolblock.py +70 -0
  49. broharness-0.0.0/uv.lock +910 -0
@@ -0,0 +1,39 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*.*.*"
7
+
8
+ jobs:
9
+ build:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v4
13
+
14
+ - name: Install uv
15
+ uses: astral-sh/setup-uv@v3
16
+
17
+ - name: Build sdist and wheel
18
+ run: uv build
19
+
20
+ - name: Upload build artifacts
21
+ uses: actions/upload-artifact@v4
22
+ with:
23
+ name: dist
24
+ path: dist/
25
+
26
+ publish:
27
+ needs: build
28
+ runs-on: ubuntu-latest
29
+ permissions:
30
+ id-token: write # required for PyPI trusted publishing
31
+ steps:
32
+ - name: Download build artifacts
33
+ uses: actions/download-artifact@v4
34
+ with:
35
+ name: dist
36
+ path: dist/
37
+
38
+ - name: Publish to PyPI
39
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,13 @@
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+
12
+ # Local SQLite lead store (generated by notebooks/sales_agent.ipynb)
13
+ data/
@@ -0,0 +1 @@
1
+ 3.12
@@ -0,0 +1,22 @@
1
+ ```mermaid
2
+ flowchart TD
3
+ A[User request] --> B["skill-call (tools: load_skill, ask_followup_question)"]
4
+ B -->|picks load_skill x N| C["load_skill(skill_name) - code, free"]
5
+ B -->|picks ask_followup_question| Q1[Ask user, wait for reply] --> A
6
+ B -->|no tools picked| E1[Answer - no skill needed]
7
+
8
+ C --> D["Build tools for this skill: load_skill_extension, ask_followup_question"]
9
+ D --> T["tool-call (current tool set)"]
10
+
11
+ T -->|picks load_skill_extension| L["load_skill_extension(skill_name, path) - code, free"]
12
+ L --> D2[Rebuild tools with newly loaded script or reference]
13
+ D2 --> T
14
+
15
+ T -->|picks ask_followup_question| Q2[Ask user, wait for reply] --> T
16
+
17
+ T -->|picks a real tool, e.g. read_file| X[Run tool - code]
18
+ X --> T
19
+
20
+ T -->|no tools picked| F[Answer to user]
21
+
22
+ ```
@@ -0,0 +1,268 @@
1
+ Metadata-Version: 2.5
2
+ Name: broharness
3
+ Version: 0.0.0
4
+ Summary: Add your description here
5
+ Requires-Python: >=3.12
6
+ Requires-Dist: broflow>=0.2.0
7
+ Requires-Dist: brollm>=0.2.0
8
+ Requires-Dist: broskill>=0.1.1
9
+ Requires-Dist: brospec>=0.1.0
10
+ Description-Content-Type: text/markdown
11
+
12
+ # broharness
13
+
14
+ A small, prompt-based tool-calling harness for models without native
15
+ tool-calling — built from scratch to study the mechanism, modeled loosely
16
+ on how Claude Code discovers and runs its own skills. Users write their
17
+ own skills as plain folders (a `SKILL.md` plus optional bundled scripts);
18
+ `broharness` is just the fixed engine that discovers them, routes a
19
+ request to one, and runs a tool loop against it.
20
+
21
+ **This is a test version (`v0.0.0`)** — the mechanism works and is
22
+ exercised end to end against a real model, but it's a study project, not a
23
+ finished product. See [Known limitations](#known-limitations-honest-about-the-test-version)
24
+ before relying on it for anything beyond experimentation.
25
+
26
+ ## What this is, in one shape
27
+
28
+ ```python
29
+ from pathlib import Path
30
+ from broskill import SkillControl, ToolControl
31
+ from broharness import Harness
32
+ from broharness.data_model import State
33
+ from broharness.llms.bedrock import UserMessage
34
+
35
+ SKILL_DIR = Path("skills")
36
+ sc = SkillControl(SKILL_DIR)
37
+ tc = ToolControl(sc)
38
+
39
+ h = Harness() # the fixed orchestration -- owns no config of its own
40
+ state = State(
41
+ root=Path("."),
42
+ skill_dir=SKILL_DIR,
43
+ messages=[UserMessage("what's in skills/file-ops/SKILL.md?")],
44
+ session_messages=[...],
45
+ skill_control=sc,
46
+ tool_control=tc,
47
+ tools={"load_skill": sc.load_skill, "load_skill_extension": sc.load_skill_extension, "load_tool": tc.load_tool},
48
+ session_tools={...},
49
+ debug=[...],
50
+ )
51
+ state = h.run(state) # one turn: State in, State out
52
+ print(state.messages[-1]["content"][0]["text"])
53
+ ```
54
+
55
+ `Harness` is deliberately lean: it builds the fixed task flow once and
56
+ does nothing else. Everything about a run — which skills directory, which
57
+ model per role, the system prompt, whether to print step traces — lives on
58
+ `State`, built by the caller. This keeps the two testable and controllable
59
+ separately, and keeps `Harness` reusable across many independently-built
60
+ `State`s. See `notebooks/from_scratch.ipynb` for the fully hand-assembled
61
+ version (useful for understanding every moving part) and
62
+ `notebooks/recipe.ipynb` for the packaged, day-to-day usage shown above.
63
+
64
+ ## Repo layout
65
+
66
+ ```
67
+ src/broharness/
68
+ data_model.py Process enum, LLMUse (per-role model ids), State,
69
+ shared helpers (usage tracking, anti-hallucination
70
+ guards, debug tracing)
71
+ harness.py Harness -- builds the fixed TaskRegistry/Flow, run(state)
72
+ codeblock.py parses a model's response into a strict JSON contract
73
+ toolblock.py the harness's own meta-tools (load_skill,
74
+ load_skill_extension, load_tool, ask_user_question)
75
+ flows/ the six tasks that make up the fixed orchestration
76
+ llms/bedrock.py the one supported LLM call today (AWS Bedrock)
77
+
78
+ skills/ one folder per skill -- see "What a skill is" below
79
+ notebooks/
80
+ from_scratch.ipynb hand-assembles everything, cell by cell -- the
81
+ teaching version
82
+ recipe.ipynb the packaged Harness API, mirrored from the above
83
+ (bedrock.ipynb, dev.ipynb, flow_idea.ipynb, skill_tool_flow_idea.ipynb
84
+ are earlier exploration drafts, superseded by the two above)
85
+ ```
86
+
87
+ ## What a skill is
88
+
89
+ A skill is a folder under `skills/` with a `SKILL.md`: YAML frontmatter
90
+ (`name`, `description`, ...) plus a free-form body of instructions, and
91
+ optionally a `scripts/` folder of executable Python scripts and/or a
92
+ `references/` folder of supporting documents.
93
+
94
+ ```
95
+ ---
96
+ name: file-ops
97
+ description: Create, read, update, delete, or list files in the project.
98
+ Use when the user wants to see what's in a file, find files matching a
99
+ pattern, write a new file, change a file's content, or remove a file.
100
+ version: v0.1.0
101
+ tags: [filesystem]
102
+ status: experiment
103
+ ---
104
+
105
+ # File Operations
106
+
107
+ ## Instructions
108
+ ...
109
+ ## Errors
110
+ ...
111
+ ## Tools
112
+ - `scripts/read_file.py` -- ...
113
+ ```
114
+
115
+ The `description` is what a routing model reasons over when deciding
116
+ whether to trigger a skill — it's read for every skill, every turn; the
117
+ full body is only loaded for the one skill chosen. A vague description
118
+ means a skill that never gets picked, or gets picked for the wrong
119
+ request.
120
+
121
+ **Scripts are auto-registered.** The moment a skill loads, every
122
+ `scripts/*.py` file in it becomes directly callable by name — no separate
123
+ registration step needed (this used to require the model to call
124
+ `load_tool` first; it proved unreliable even after explicit corrective
125
+ errors, so it's now automatic and free). A script just needs a
126
+ `get_args()` returning an `argparse.ArgumentParser`; its help text becomes
127
+ the tool's description shown to the model.
128
+
129
+ **References are opt-in.** A `references/*.md` file (e.g. a style guide, a
130
+ detail doc) is *not* auto-loaded — a skill's instructions point to it, and
131
+ the model calls `load_skill_extension` to pull it in only when actually
132
+ needed. This is the progressive-disclosure half of the design: a skill's
133
+ `SKILL.md` should stay short, and reference files carry the detail that
134
+ isn't needed on every single call.
135
+
136
+ ## How the fixed flow works
137
+
138
+ Every request runs through the same six tasks (`src/broharness/flows/`):
139
+
140
+ ```
141
+ skill_call → tool_call → tool_use → answer
142
+ ↑ ↑ ↓ ↑ ↓
143
+ └──── fail_recovery ────┘ └── ask_user_question
144
+ ```
145
+
146
+ - **`skill_call`** — given every skill's `(name, description)`, picks
147
+ zero or more skills to load (or none, if the request needs no skill).
148
+ - **`tool_call`** — given the loaded skill's instructions and its
149
+ auto-registered tools, picks which tool(s) to call, if any.
150
+ - **`tool_use`** — actually runs the chosen tool: a skill's own script (as
151
+ a real subprocess, isolated from the harness process), one of the
152
+ harness's own meta-tools (`load_skill`, `load_skill_extension`,
153
+ `load_tool`), or `ask_user_question` (blocks on real `input()`).
154
+ - **`answer`** — writes the final natural-language reply, grounded in
155
+ whatever was actually fetched. It has no tool-calling ability of its
156
+ own on purpose — this is where a small model is most tempted to
157
+ hallucinate a plausible-sounding answer instead of admitting something
158
+ wasn't found, so its prompt is the most heavily guarded part of the
159
+ harness (see below).
160
+ - **`fail_recovery`** — a bounded retry (`state.max_retries`, default 3)
161
+ for any task that errors, falling through to `answer` once exhausted
162
+ rather than looping forever.
163
+
164
+ Both the model's response format (a single JSON codeblock, nothing else)
165
+ and this task graph are a fixed contract — a skill only ever supplies
166
+ *content* (instructions, scripts, references), never orchestration logic.
167
+
168
+ ## Guardrails this harness actually enforces
169
+
170
+ Built in response to specific, reproduced failures against a real (small,
171
+ 12B) model — not speculative hardening:
172
+
173
+ - **Anti-hallucination in `answer`** — its prompt explicitly forbids
174
+ stating a fact/file content not actually present in what was fetched,
175
+ and a separate check (`tool_results_are_empty`) calls out a "nothing
176
+ found" result explicitly, since a quiet empty-result message read enough
177
+ like content that the model would sometimes invent a plausible answer
178
+ around it anyway.
179
+ - **Real-question detection, not "ends in `?`"** — a naive
180
+ `text.endswith('?')` check misfires on a persona whose sentences
181
+ habitually end in a rhetorical tag ("...you know?", "...right?"),
182
+ derailing a complete answer into an unwanted clarification loop.
183
+ `looks_like_a_question()` requires the final sentence to actually start
184
+ like a question (a WH-word or auxiliary verb).
185
+ - **Per-conversation skill-loaded checks** — `load_skill_extension` is
186
+ checked against *this conversation's* `state.registered_skills`, not
187
+ delegated straight to the skill-loading library's own internal
188
+ tracking, which turned out to persist across unrelated runs sharing the
189
+ same `SkillControl` instance and could let a wrong `skill_name` silently
190
+ succeed instead of failing loudly.
191
+ - **Per-turn state flushing** — `State.flush_turn()` clears everything
192
+ only valid for the turn that just finished (`tool_results`,
193
+ `candidated_tools`, `executed_calls`, ...) before a new turn starts,
194
+ since a previous turn's fetched content sitting in `State` was answering
195
+ the *next*, unrelated question.
196
+
197
+ ## Skills included
198
+
199
+ - **`skill-call` / `tool-call`** — the meta-skills that carry the
200
+ tool-calling contract prompt itself (`default: true`, always loaded).
201
+ Not something you'd normally touch when adding a new skill.
202
+ - **`file-ops`** — create/read/update/delete/list files in the project.
203
+ Destructive actions (`update_file`, `delete_file`) require confirming
204
+ with the user first; `delete_file` never accepts a glob pattern.
205
+ - **`tell-joke`** — dad jokes, puns, knock-knock jokes, one-liners,
206
+ riddles — reference-driven style guides, not canned joke lists.
207
+ - **`ds-mentor`** — explains data science/ML/statistics concepts to a
208
+ junior data scientist (metrics, stats basics, model fundamentals, common
209
+ pitfalls, pandas gotchas), grounded in curated reference material.
210
+
211
+ ## Running it
212
+
213
+ Needs AWS credentials resolvable by `boto3` (env vars,
214
+ `~/.aws/credentials`, SSO profile, ...) with Bedrock model access granted
215
+ for `google.gemma-3-12b-it` in `us-east-1` — the only LLM backend wired up
216
+ today (`src/broharness/llms/bedrock.py`).
217
+
218
+ ```
219
+ uv sync
220
+ ```
221
+
222
+ `uv sync` installs this repo's own `src/broharness` package in editable
223
+ mode. Then open `notebooks/recipe.ipynb` (packaged `Harness` usage) or
224
+ `notebooks/from_scratch.ipynb` (every piece assembled by hand) and run it
225
+ top to bottom in your own kernel.
226
+
227
+ ## Building your own skill
228
+
229
+ 1. `skills/<your-skill>/SKILL.md` with frontmatter (`name`, `description`,
230
+ `version`, `tags`, `status`) and an `## Instructions` section written
231
+ for a model that can only act through the tool-call contract — assume
232
+ nothing carries over between calls except what's in `State`.
233
+ 2. Optional `skills/<your-skill>/scripts/*.py` — any script with a
234
+ `get_args()` returning an `argparse.ArgumentParser` is auto-registered
235
+ as a callable tool the moment the skill loads. Runs as a real
236
+ subprocess; only stdout/stderr comes back, never its source.
237
+ 3. Optional `skills/<your-skill>/references/*.md` — loaded on demand via
238
+ `load_skill_extension`, not automatically. Good for detail that
239
+ shouldn't bloat the main `SKILL.md` (error taxonomies, style guides,
240
+ longer reference docs).
241
+ 4. Add an `## Errors` section if your scripts can fail in more than one
242
+ way — spell out each distinct error message and what the model should
243
+ do in response, rather than one generic "something went wrong."
244
+ `file-ops/SKILL.md` is the fullest example of this pattern.
245
+ 5. Test it directly (`uv run skills/<your-skill>/scripts/foo.py --arg ...`)
246
+ before testing it through the harness — isolates a script bug from a
247
+ prompting/routing issue.
248
+
249
+ ## Known limitations (honest, about the test version)
250
+
251
+ - **Meta-skills aren't cleanly separated from user skills.** `skill-call`
252
+ and `tool-call` currently live in the same `skills/` folder a user would
253
+ put their own skills in, and are referenced by hardcoded name in
254
+ `flows/skill_call.py`/`flows/tool_call.py` rather than via the `default`
255
+ flag the underlying skill library already exposes. A `skills/` folder
256
+ missing those two will break.
257
+ - **Single LLM backend.** Only AWS Bedrock (`llms/bedrock.py`) is wired
258
+ up; `Harness(llm=...)` accepts any compatible callable, but nothing else
259
+ has been tested against it yet.
260
+ - **Small-model retry stubbornness.** The harness bounds failures with
261
+ retries and a graceful fallback (see Guardrails above), but a small
262
+ model sometimes repeats the exact same wrong tool call verbatim across
263
+ every retry rather than self-correcting — this is a capability limit of
264
+ the model tested against, not something further prompt tuning reliably
265
+ fixes.
266
+ - **No cross-skill tool sharing.** A skill's auto-registered tools are
267
+ only visible while that skill is loaded; a second skill can't reuse the
268
+ first one's scripts without duplicating them.
@@ -0,0 +1,257 @@
1
+ # broharness
2
+
3
+ A small, prompt-based tool-calling harness for models without native
4
+ tool-calling — built from scratch to study the mechanism, modeled loosely
5
+ on how Claude Code discovers and runs its own skills. Users write their
6
+ own skills as plain folders (a `SKILL.md` plus optional bundled scripts);
7
+ `broharness` is just the fixed engine that discovers them, routes a
8
+ request to one, and runs a tool loop against it.
9
+
10
+ **This is a test version (`v0.0.0`)** — the mechanism works and is
11
+ exercised end to end against a real model, but it's a study project, not a
12
+ finished product. See [Known limitations](#known-limitations-honest-about-the-test-version)
13
+ before relying on it for anything beyond experimentation.
14
+
15
+ ## What this is, in one shape
16
+
17
+ ```python
18
+ from pathlib import Path
19
+ from broskill import SkillControl, ToolControl
20
+ from broharness import Harness
21
+ from broharness.data_model import State
22
+ from broharness.llms.bedrock import UserMessage
23
+
24
+ SKILL_DIR = Path("skills")
25
+ sc = SkillControl(SKILL_DIR)
26
+ tc = ToolControl(sc)
27
+
28
+ h = Harness() # the fixed orchestration -- owns no config of its own
29
+ state = State(
30
+ root=Path("."),
31
+ skill_dir=SKILL_DIR,
32
+ messages=[UserMessage("what's in skills/file-ops/SKILL.md?")],
33
+ session_messages=[...],
34
+ skill_control=sc,
35
+ tool_control=tc,
36
+ tools={"load_skill": sc.load_skill, "load_skill_extension": sc.load_skill_extension, "load_tool": tc.load_tool},
37
+ session_tools={...},
38
+ debug=[...],
39
+ )
40
+ state = h.run(state) # one turn: State in, State out
41
+ print(state.messages[-1]["content"][0]["text"])
42
+ ```
43
+
44
+ `Harness` is deliberately lean: it builds the fixed task flow once and
45
+ does nothing else. Everything about a run — which skills directory, which
46
+ model per role, the system prompt, whether to print step traces — lives on
47
+ `State`, built by the caller. This keeps the two testable and controllable
48
+ separately, and keeps `Harness` reusable across many independently-built
49
+ `State`s. See `notebooks/from_scratch.ipynb` for the fully hand-assembled
50
+ version (useful for understanding every moving part) and
51
+ `notebooks/recipe.ipynb` for the packaged, day-to-day usage shown above.
52
+
53
+ ## Repo layout
54
+
55
+ ```
56
+ src/broharness/
57
+ data_model.py Process enum, LLMUse (per-role model ids), State,
58
+ shared helpers (usage tracking, anti-hallucination
59
+ guards, debug tracing)
60
+ harness.py Harness -- builds the fixed TaskRegistry/Flow, run(state)
61
+ codeblock.py parses a model's response into a strict JSON contract
62
+ toolblock.py the harness's own meta-tools (load_skill,
63
+ load_skill_extension, load_tool, ask_user_question)
64
+ flows/ the six tasks that make up the fixed orchestration
65
+ llms/bedrock.py the one supported LLM call today (AWS Bedrock)
66
+
67
+ skills/ one folder per skill -- see "What a skill is" below
68
+ notebooks/
69
+ from_scratch.ipynb hand-assembles everything, cell by cell -- the
70
+ teaching version
71
+ recipe.ipynb the packaged Harness API, mirrored from the above
72
+ (bedrock.ipynb, dev.ipynb, flow_idea.ipynb, skill_tool_flow_idea.ipynb
73
+ are earlier exploration drafts, superseded by the two above)
74
+ ```
75
+
76
+ ## What a skill is
77
+
78
+ A skill is a folder under `skills/` with a `SKILL.md`: YAML frontmatter
79
+ (`name`, `description`, ...) plus a free-form body of instructions, and
80
+ optionally a `scripts/` folder of executable Python scripts and/or a
81
+ `references/` folder of supporting documents.
82
+
83
+ ```
84
+ ---
85
+ name: file-ops
86
+ description: Create, read, update, delete, or list files in the project.
87
+ Use when the user wants to see what's in a file, find files matching a
88
+ pattern, write a new file, change a file's content, or remove a file.
89
+ version: v0.1.0
90
+ tags: [filesystem]
91
+ status: experiment
92
+ ---
93
+
94
+ # File Operations
95
+
96
+ ## Instructions
97
+ ...
98
+ ## Errors
99
+ ...
100
+ ## Tools
101
+ - `scripts/read_file.py` -- ...
102
+ ```
103
+
104
+ The `description` is what a routing model reasons over when deciding
105
+ whether to trigger a skill — it's read for every skill, every turn; the
106
+ full body is only loaded for the one skill chosen. A vague description
107
+ means a skill that never gets picked, or gets picked for the wrong
108
+ request.
109
+
110
+ **Scripts are auto-registered.** The moment a skill loads, every
111
+ `scripts/*.py` file in it becomes directly callable by name — no separate
112
+ registration step needed (this used to require the model to call
113
+ `load_tool` first; it proved unreliable even after explicit corrective
114
+ errors, so it's now automatic and free). A script just needs a
115
+ `get_args()` returning an `argparse.ArgumentParser`; its help text becomes
116
+ the tool's description shown to the model.
117
+
118
+ **References are opt-in.** A `references/*.md` file (e.g. a style guide, a
119
+ detail doc) is *not* auto-loaded — a skill's instructions point to it, and
120
+ the model calls `load_skill_extension` to pull it in only when actually
121
+ needed. This is the progressive-disclosure half of the design: a skill's
122
+ `SKILL.md` should stay short, and reference files carry the detail that
123
+ isn't needed on every single call.
124
+
125
+ ## How the fixed flow works
126
+
127
+ Every request runs through the same six tasks (`src/broharness/flows/`):
128
+
129
+ ```
130
+ skill_call → tool_call → tool_use → answer
131
+ ↑ ↑ ↓ ↑ ↓
132
+ └──── fail_recovery ────┘ └── ask_user_question
133
+ ```
134
+
135
+ - **`skill_call`** — given every skill's `(name, description)`, picks
136
+ zero or more skills to load (or none, if the request needs no skill).
137
+ - **`tool_call`** — given the loaded skill's instructions and its
138
+ auto-registered tools, picks which tool(s) to call, if any.
139
+ - **`tool_use`** — actually runs the chosen tool: a skill's own script (as
140
+ a real subprocess, isolated from the harness process), one of the
141
+ harness's own meta-tools (`load_skill`, `load_skill_extension`,
142
+ `load_tool`), or `ask_user_question` (blocks on real `input()`).
143
+ - **`answer`** — writes the final natural-language reply, grounded in
144
+ whatever was actually fetched. It has no tool-calling ability of its
145
+ own on purpose — this is where a small model is most tempted to
146
+ hallucinate a plausible-sounding answer instead of admitting something
147
+ wasn't found, so its prompt is the most heavily guarded part of the
148
+ harness (see below).
149
+ - **`fail_recovery`** — a bounded retry (`state.max_retries`, default 3)
150
+ for any task that errors, falling through to `answer` once exhausted
151
+ rather than looping forever.
152
+
153
+ Both the model's response format (a single JSON codeblock, nothing else)
154
+ and this task graph are a fixed contract — a skill only ever supplies
155
+ *content* (instructions, scripts, references), never orchestration logic.
156
+
157
+ ## Guardrails this harness actually enforces
158
+
159
+ Built in response to specific, reproduced failures against a real (small,
160
+ 12B) model — not speculative hardening:
161
+
162
+ - **Anti-hallucination in `answer`** — its prompt explicitly forbids
163
+ stating a fact/file content not actually present in what was fetched,
164
+ and a separate check (`tool_results_are_empty`) calls out a "nothing
165
+ found" result explicitly, since a quiet empty-result message read enough
166
+ like content that the model would sometimes invent a plausible answer
167
+ around it anyway.
168
+ - **Real-question detection, not "ends in `?`"** — a naive
169
+ `text.endswith('?')` check misfires on a persona whose sentences
170
+ habitually end in a rhetorical tag ("...you know?", "...right?"),
171
+ derailing a complete answer into an unwanted clarification loop.
172
+ `looks_like_a_question()` requires the final sentence to actually start
173
+ like a question (a WH-word or auxiliary verb).
174
+ - **Per-conversation skill-loaded checks** — `load_skill_extension` is
175
+ checked against *this conversation's* `state.registered_skills`, not
176
+ delegated straight to the skill-loading library's own internal
177
+ tracking, which turned out to persist across unrelated runs sharing the
178
+ same `SkillControl` instance and could let a wrong `skill_name` silently
179
+ succeed instead of failing loudly.
180
+ - **Per-turn state flushing** — `State.flush_turn()` clears everything
181
+ only valid for the turn that just finished (`tool_results`,
182
+ `candidated_tools`, `executed_calls`, ...) before a new turn starts,
183
+ since a previous turn's fetched content sitting in `State` was answering
184
+ the *next*, unrelated question.
185
+
186
+ ## Skills included
187
+
188
+ - **`skill-call` / `tool-call`** — the meta-skills that carry the
189
+ tool-calling contract prompt itself (`default: true`, always loaded).
190
+ Not something you'd normally touch when adding a new skill.
191
+ - **`file-ops`** — create/read/update/delete/list files in the project.
192
+ Destructive actions (`update_file`, `delete_file`) require confirming
193
+ with the user first; `delete_file` never accepts a glob pattern.
194
+ - **`tell-joke`** — dad jokes, puns, knock-knock jokes, one-liners,
195
+ riddles — reference-driven style guides, not canned joke lists.
196
+ - **`ds-mentor`** — explains data science/ML/statistics concepts to a
197
+ junior data scientist (metrics, stats basics, model fundamentals, common
198
+ pitfalls, pandas gotchas), grounded in curated reference material.
199
+
200
+ ## Running it
201
+
202
+ Needs AWS credentials resolvable by `boto3` (env vars,
203
+ `~/.aws/credentials`, SSO profile, ...) with Bedrock model access granted
204
+ for `google.gemma-3-12b-it` in `us-east-1` — the only LLM backend wired up
205
+ today (`src/broharness/llms/bedrock.py`).
206
+
207
+ ```
208
+ uv sync
209
+ ```
210
+
211
+ `uv sync` installs this repo's own `src/broharness` package in editable
212
+ mode. Then open `notebooks/recipe.ipynb` (packaged `Harness` usage) or
213
+ `notebooks/from_scratch.ipynb` (every piece assembled by hand) and run it
214
+ top to bottom in your own kernel.
215
+
216
+ ## Building your own skill
217
+
218
+ 1. `skills/<your-skill>/SKILL.md` with frontmatter (`name`, `description`,
219
+ `version`, `tags`, `status`) and an `## Instructions` section written
220
+ for a model that can only act through the tool-call contract — assume
221
+ nothing carries over between calls except what's in `State`.
222
+ 2. Optional `skills/<your-skill>/scripts/*.py` — any script with a
223
+ `get_args()` returning an `argparse.ArgumentParser` is auto-registered
224
+ as a callable tool the moment the skill loads. Runs as a real
225
+ subprocess; only stdout/stderr comes back, never its source.
226
+ 3. Optional `skills/<your-skill>/references/*.md` — loaded on demand via
227
+ `load_skill_extension`, not automatically. Good for detail that
228
+ shouldn't bloat the main `SKILL.md` (error taxonomies, style guides,
229
+ longer reference docs).
230
+ 4. Add an `## Errors` section if your scripts can fail in more than one
231
+ way — spell out each distinct error message and what the model should
232
+ do in response, rather than one generic "something went wrong."
233
+ `file-ops/SKILL.md` is the fullest example of this pattern.
234
+ 5. Test it directly (`uv run skills/<your-skill>/scripts/foo.py --arg ...`)
235
+ before testing it through the harness — isolates a script bug from a
236
+ prompting/routing issue.
237
+
238
+ ## Known limitations (honest, about the test version)
239
+
240
+ - **Meta-skills aren't cleanly separated from user skills.** `skill-call`
241
+ and `tool-call` currently live in the same `skills/` folder a user would
242
+ put their own skills in, and are referenced by hardcoded name in
243
+ `flows/skill_call.py`/`flows/tool_call.py` rather than via the `default`
244
+ flag the underlying skill library already exposes. A `skills/` folder
245
+ missing those two will break.
246
+ - **Single LLM backend.** Only AWS Bedrock (`llms/bedrock.py`) is wired
247
+ up; `Harness(llm=...)` accepts any compatible callable, but nothing else
248
+ has been tested against it yet.
249
+ - **Small-model retry stubbornness.** The harness bounds failures with
250
+ retries and a graceful fallback (see Guardrails above), but a small
251
+ model sometimes repeats the exact same wrong tool call verbatim across
252
+ every retry rather than self-correcting — this is a capability limit of
253
+ the model tested against, not something further prompt tuning reliably
254
+ fixes.
255
+ - **No cross-skill tool sharing.** A skill's auto-registered tools are
256
+ only visible while that skill is loaded; a second skill can't reuse the
257
+ first one's scripts without duplicating them.