simt-dev 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +170 -0
- package/README.md +300 -82
- package/docs/decisions/ADR-058-one-tool-three-channels-and-the-order-is-forced.md +13 -0
- package/docs/decisions/ADR-059-the-version-a-repository-runs-is-the-one-it-pinned.md +1 -1
- package/docs/decisions/ADR-097-the-channels-publish-by-hand-and-the-version-rides-in-as-an-input.md +71 -0
- package/docs/decisions/ADR-098-the-fourth-channel-is-one-file-and-it-needs-no-toolchain.md +149 -0
- package/package.json +8 -4
- package/src/advice/advice.mjs +229 -0
- package/src/agenthooks/render.mjs +87 -21
- package/src/argv.mjs +25 -0
- package/src/bench/ablate.mjs +1 -3
- package/src/bench/grade.mjs +1 -1
- package/src/bench/mine.mjs +3 -3
- package/src/bench/probes.mjs +1 -3
- package/src/bench/report.mjs +1 -3
- package/src/bench/run.mjs +4 -18
- package/src/bench/tasks.mjs +36 -0
- package/src/bench/variants.mjs +1 -3
- package/src/brief/assemble.mjs +24 -4
- package/src/brief/packet.mjs +4 -1
- package/src/ci/dependabot.mjs +1 -1
- package/src/ci/workflow.mjs +40 -12
- package/src/commands/agenthook.mjs +6 -2
- package/src/commands/bench.mjs +35 -3
- package/src/commands/ci.mjs +1 -1
- package/src/commands/compare.mjs +1 -1
- package/src/commands/distill.mjs +36 -0
- package/src/commands/gate.mjs +2 -2
- package/src/commands/graph.mjs +2 -2
- package/src/commands/harness.mjs +87 -14
- package/src/commands/help.mjs +54 -4
- package/src/commands/hooks.mjs +1 -1
- package/src/commands/index.mjs +29 -3
- package/src/commands/init.mjs +2 -2
- package/src/commands/local.mjs +11 -0
- package/src/commands/mcp.mjs +1 -1
- package/src/commands/pack.mjs +246 -0
- package/src/commands/plugin.mjs +61 -0
- package/src/commands/review.mjs +65 -0
- package/src/commands/rules.mjs +1 -1
- package/src/commands/skills.mjs +3 -34
- package/src/commands/skillscompile.mjs +1 -3
- package/src/commands/skillsinstall.mjs +1 -1
- package/src/commands/sync.mjs +8 -8
- package/src/commands/tokens.mjs +1 -1
- package/src/commands/upgrade.mjs +1 -1
- package/src/commands/why.mjs +6 -3
- package/src/compare/discover.mjs +1 -1
- package/src/compare/hooksstate.mjs +19 -7
- package/src/compare/learn.mjs +2 -2
- package/src/compare/row.mjs +3 -3
- package/src/decisions/index.mjs +1 -1
- package/src/distill/distill.mjs +246 -0
- package/src/distill/engine.mjs +187 -0
- package/src/distribute/release.mjs +142 -0
- package/src/graph/load.mjs +9 -8
- package/src/hooks/dispatcher.mjs +3 -2
- package/src/hooks/shim.mjs +11 -6
- package/src/index/govern.mjs +1 -1
- package/src/index/map.mjs +1 -1
- package/src/index/persist.mjs +2 -2
- package/src/index/store.mjs +50 -20
- package/src/ledger/index.mjs +2 -2
- package/src/local/client.mjs +22 -8
- package/src/local/config.mjs +52 -0
- package/src/locate.mjs +22 -1
- package/src/mcp/prompts.mjs +118 -0
- package/src/mcp/resources.mjs +96 -22
- package/src/mcp/server.mjs +33 -6
- package/src/mcp/tools.mjs +12 -12
- package/src/measure/session.mjs +1 -1
- package/src/memory/decide.mjs +1 -1
- package/src/memory/lessons.mjs +14 -14
- package/src/name.mjs +11 -1
- package/src/package-version.mjs +12 -10
- package/src/packs/manage.mjs +563 -0
- package/src/packs.mjs +15 -5
- package/src/plugins.mjs +164 -0
- package/src/projection/loaders.mjs +84 -10
- package/src/projection/manifest.mjs +4 -4
- package/src/projection/project.mjs +3 -3
- package/src/projection/rules.mjs +24 -4
- package/src/projection/surfaces.mjs +20 -2
- package/src/resolve/body.mjs +46 -0
- package/src/review/critic.mjs +509 -0
- package/src/toolwords.mjs +8 -5
- package/src/tsconfig.mjs +3 -3
- package/src/upgrade/plan.mjs +1 -1
- /package/bin/{simt-dev.mjs → simt.mjs} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# ADR-059 — The version a repository runs is the one it pinned, and the gate refuses otherwise
|
|
2
2
|
|
|
3
|
-
Status: **accepted** · 2026-10-12 · affects all repos
|
|
3
|
+
Status: **accepted** · 2026-10-12 · affects all repos · ruling 3's no-npm install and remedy superseded by [ADR-098](ADR-098-the-fourth-channel-is-one-file-and-it-needs-no-toolchain.md)
|
|
4
4
|
|
|
5
5
|
## Context
|
|
6
6
|
|
package/docs/decisions/ADR-097-the-channels-publish-by-hand-and-the-version-rides-in-as-an-input.md
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# ADR-097 — the channels publish by hand, and the version rides in as an input
|
|
2
|
+
|
|
3
|
+
Status: **accepted** · 2026-10-25 · ruling 3's Homebrew precondition is superseded by [ADR-098](ADR-098-the-fourth-channel-is-one-file-and-it-needs-no-toolchain.md), which also restates what each channel now waits on
|
|
4
|
+
|
|
5
|
+
**Affects:** `.github/workflows/publish.yml`,
|
|
6
|
+
`.github/workflows/publish-channels.yml`, `scripts/publish-local.mjs`,
|
|
7
|
+
`docs/decisions/ADR-058-one-tool-three-channels-and-the-order-is-forced.md`
|
|
8
|
+
|
|
9
|
+
## Context
|
|
10
|
+
|
|
11
|
+
ADR-058 ruled the three channels and the order between them — npm first, PyPI and
|
|
12
|
+
Homebrew only once the npm version is live — and its ruling 7 also pinned the
|
|
13
|
+
mechanics: the two channel jobs lived inside `publish.yml`, next to the staging job,
|
|
14
|
+
and a maintainer re-ran them through "Re-run failed jobs" once `npm stage approve` had
|
|
15
|
+
landed. The same ruling recorded why no `workflow_dispatch` could replace that:
|
|
16
|
+
on a dispatch, `github.ref_name` is the branch, and the version guard would compare a
|
|
17
|
+
tag against a branch.
|
|
18
|
+
|
|
19
|
+
Running it that way taught something the ruling did not predict. The channel jobs fail
|
|
20
|
+
on every release **by design** — the stage is not approvable until a human decides, so
|
|
21
|
+
a healthy release hours in is a release whose run is red. A run that is red when
|
|
22
|
+
nothing is wrong is the argument ADR-058 itself used against a gate that is red on the
|
|
23
|
+
day it lands: it teaches the reader to ignore red. And the only supported recovery is
|
|
24
|
+
re-running part of an old run, an action buried in the run page's retry menu rather
|
|
25
|
+
than in anything a maintainer can name.
|
|
26
|
+
|
|
27
|
+
The branch problem stays, but it points at a different fix. A dispatch cannot carry a
|
|
28
|
+
tag in `ref_name`, but it doesn't have to: the version is one string, it is already
|
|
29
|
+
written in exactly one place (`package.json`), and the maintainer who just approved
|
|
30
|
+
the stage knows it — they typed or copied it to get here.
|
|
31
|
+
|
|
32
|
+
## Ruling
|
|
33
|
+
|
|
34
|
+
1. **`publish.yml` owns npm staging and stops there.** The tag workflow keeps the
|
|
35
|
+
version guard and the staging job, and no longer carries a job designed to sit red
|
|
36
|
+
for hours. A green publish run means "staged", and nothing in it promises more.
|
|
37
|
+
|
|
38
|
+
2. **The channels run from `.github/workflows/publish-channels.yml`, a
|
|
39
|
+
`workflow_dispatch` whose `version` input is the release (e.g. `0.5.0`).** The
|
|
40
|
+
version is the *input*, not the ref, which is what dissolves the branch-vs-tag
|
|
41
|
+
failure ADR-058 recorded: the input travels into the guard, and the guard refuses
|
|
42
|
+
to run unless tag `v<version>` exists on the default branch's history and its
|
|
43
|
+
checkout's `package.json` agrees with it. A dispatch cannot guess; the input makes
|
|
44
|
+
guessing impossible.
|
|
45
|
+
|
|
46
|
+
3. **The ordering ruling is unchanged.** Both channel jobs still begin with the check
|
|
47
|
+
that `npm view` serves the version (PyPI) and that the tarball is downloadable
|
|
48
|
+
(Homebrew), and both still name the remedy — `npm stage approve <stage-id>` —
|
|
49
|
+
when it is not. What moved is who runs the steps and how the version travels;
|
|
50
|
+
what did not move is that npm is live before either channel publishes.
|
|
51
|
+
|
|
52
|
+
4. **The channel jobs keep their identity unchanged.** PyPI still publishes with
|
|
53
|
+
trusted publishing against the `pypi` environment; Homebrew still audits, renders
|
|
54
|
+
and pushes the formula to the tap. Nothing about the artefacts changed, so ADR-058
|
|
55
|
+
rulings 1–6 read as written.
|
|
56
|
+
|
|
57
|
+
## Consequences
|
|
58
|
+
|
|
59
|
+
A maintainer's release tour is two named actions instead of one retry:
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
npm stage approve <stage-id>
|
|
63
|
+
gh workflow run publish-channels.yml -f version=<VER>
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Red is honest again: a red channel run means a channel defect, and the release page
|
|
67
|
+
carries no run whose expected end state is red. `scripts/publish-local.mjs` keeps its
|
|
68
|
+
role as the not-CI twin — it mirrors `publish.yml` for npm and
|
|
69
|
+
`publish-channels.yml` for the two channels, and its header now says exactly that,
|
|
70
|
+
because a twin of a file whose name no longer holds what it did is a comment that
|
|
71
|
+
misdescribes the artefact (the estate's own rule against decoration that lies).
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
---
|
|
2
|
+
simt-affects: ["scripts/build-sea.mjs", "src/distribute/release.mjs", "src/locate.mjs", "src/package-version.mjs", "src/hooks/shim.mjs", "src/ci/workflow.mjs", ".github/workflows/publish.yml", ".github/workflows/publish-channels.yml", "Formula/simt-dev.rb", "scripts/release-formula.mjs", "scripts/publish-local.mjs", ".gitignore", "test/binary.test.mjs", "test/ci.test.mjs", "test/packaging.test.mjs"]
|
|
3
|
+
simt-supersedes: [{ adr: "ADR-058", ruling: 5 }, { adr: "ADR-058", ruling: 7, kind: "narrowed" }, { adr: "ADR-059", ruling: 3, kind: "superseded" }, { adr: "ADR-097", ruling: 3, kind: "narrowed" }]
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# ADR-098 — The fourth channel is one file, and it needs no toolchain
|
|
7
|
+
|
|
8
|
+
Status: **accepted** · 2026-10-26 · supersedes ADR-058 ruling 5 · narrows ADR-058 ruling 7 and ADR-097 ruling 3 · supersedes ADR-059 ruling 3
|
|
9
|
+
|
|
10
|
+
Affects: `scripts/build-sea.mjs`, `src/distribute/release.mjs`, `src/locate.mjs`,
|
|
11
|
+
`src/hooks/shim.mjs`, `src/ci/workflow.mjs`, `.github/workflows/publish.yml`,
|
|
12
|
+
`.github/workflows/publish-channels.yml`, `Formula/simt-dev.rb`,
|
|
13
|
+
`scripts/release-formula.mjs`, `scripts/publish-local.mjs`, `test/binary.test.mjs`
|
|
14
|
+
|
|
15
|
+
## Context
|
|
16
|
+
|
|
17
|
+
Review 0002's S35 ruled that no estate repository and no CI runner may be unable to
|
|
18
|
+
obtain this tool, and ADR-059 shipped its first half: the remedy and the consumer CI
|
|
19
|
+
step branch by repository kind. The half that did not ship is the one this record
|
|
20
|
+
settles. For a repository without npm — a Gradle build, a Connect IQ project — every
|
|
21
|
+
channel that existed needed a toolchain the repository does not have: the npm tarball
|
|
22
|
+
needs npm, the wheel needs a Python, and the formula, until this record, declared
|
|
23
|
+
`depends_on "node"` and so installed a command that could not start without one. The CI
|
|
24
|
+
step that stood in for the missing channel ran `npm i -g simt-dev@<v>`, which needs the
|
|
25
|
+
npm the repository kind is defined by not having. The one repository where the gate had
|
|
26
|
+
no toolchain already was also the one where obtaining the tool required installing one.
|
|
27
|
+
|
|
28
|
+
Two properties of the package made the alternative cheap. It has zero runtime
|
|
29
|
+
dependencies and no build step (ADR-012), so "packaging" is copying ~1.3 MB of text
|
|
30
|
+
around a pinned runtime; and the text reads its own shipped files — `rules/`,
|
|
31
|
+
`templates/`, `docs/decisions/` — relative to `import.meta.url`, so it needs real files
|
|
32
|
+
at real paths, not a module graph. Node's single-executable-application support embeds
|
|
33
|
+
a script in an official node binary, and postject injects that blob into any target's
|
|
34
|
+
copy of one; the missing half was a payload that materializes as a tree.
|
|
35
|
+
|
|
36
|
+
## Ruling
|
|
37
|
+
|
|
38
|
+
1. **A fourth artefact is released: one file per target, carrying the same payload the
|
|
39
|
+
npm tarball ships.** `scripts/build-sea.mjs` collects exactly `package.json`'s
|
|
40
|
+
`files` set plus `package.json` itself, and wraps it in a pinned official node base
|
|
41
|
+
at the version `engines` names. The targets are `darwin-arm64`, `darwin-x64` and
|
|
42
|
+
`linux-x64` — the set the one-line remedy and the consumer CI step can name — and a
|
|
43
|
+
platform outside it gets a loud 404, never a silent install of nothing. Each asset is
|
|
44
|
+
published to the GitHub release beside its `.sha256`, with a build-provenance
|
|
45
|
+
attestation binding the digest to the tag workflow's OIDC identity.
|
|
46
|
+
|
|
47
|
+
2. **The payload materializes as real files before the tool runs.** The SEA entry is a
|
|
48
|
+
small CommonJS bootstrap (Node 22's SEA host runs the entry as CJS, whatever the
|
|
49
|
+
entry's own shape) that writes the embedded file set into
|
|
50
|
+
`${XDG_CACHE_HOME:-$HOME/.cache}/simt-dev/<node version>/<digest>/` and then imports
|
|
51
|
+
`bin/simt.mjs` from there. The digest is a hash of the canonical payload, so one
|
|
52
|
+
directory per payload, written to a staging directory and renamed into place, with a
|
|
53
|
+
marker file making a second run a no-op. Real files at real paths are what keep every
|
|
54
|
+
`import.meta.url`-relative read in `src/` working unmodified; a bootstrap that inlined
|
|
55
|
+
the tool instead would have to re-implement each one.
|
|
56
|
+
|
|
57
|
+
3. **Determinism is the release contract, and the release checks it.** The same tree
|
|
58
|
+
builds byte-identical assets on any host: the node base is pinned, the SEA config uses
|
|
59
|
+
relative paths so no checkout path lands in the blob, and darwin binaries are ad-hoc
|
|
60
|
+
codesigned with a fixed identifier (without one, codesign derives the identifier from
|
|
61
|
+
the file path, which is what made two checkouts differ). Because of that, the tag
|
|
62
|
+
workflow re-builds each asset and fails the release when its checksum is not the one
|
|
63
|
+
`Formula/simt-dev.rb` in the tagged tree promises — the equality is checked, not
|
|
64
|
+
assumed.
|
|
65
|
+
|
|
66
|
+
4. **Homebrew's artefact becomes the binary (supersedes ADR-058 ruling 5).** The formula
|
|
67
|
+
carries one `url`/`sha256` pair per target, selected by OS and CPU at install time,
|
|
68
|
+
and no longer declares `depends_on "node"`: the binary is the runtime. The "not
|
|
69
|
+
`std_npm_args`" half of ADR-058 ruling 5 stands unchanged — the formula still does no
|
|
70
|
+
build-time fetch of a package manager. `scripts/release-formula.mjs` patches each
|
|
71
|
+
pair by content, one url and the sha256 line that follows it, and refuses a formula
|
|
72
|
+
whose layout has drifted rather than patching blind.
|
|
73
|
+
|
|
74
|
+
5. **The order each channel waits on is what that channel downloads (narrows ADR-058
|
|
75
|
+
ruling 7 and ADR-097 ruling 3).** PyPI still begins by checking that `npm view` serves
|
|
76
|
+
the version, because the npm channel is the one a maintainer has to approve by hand.
|
|
77
|
+
Homebrew now begins by checking that the release assets exist, because the formula
|
|
78
|
+
downloads those — a formula naming a tag whose binaries were never built installs a
|
|
79
|
+
404 just as surely as one naming an unapproved npm version. The ordering rule
|
|
80
|
+
survives; only its second premise moved.
|
|
81
|
+
|
|
82
|
+
6. **The repository's own copy is checked before `PATH`, and the version pin still
|
|
83
|
+
decides.** The locator resolves `$SIMT_DEV_BIN`, then `node_modules`, then
|
|
84
|
+
`.simt/bin/simt`, then `PATH` — so the bytes a repository downloaded are the bytes its
|
|
85
|
+
gate runs, and a machine-wide install cannot shadow them. The dispatcher's version
|
|
86
|
+
comparison (ADR-059 rulings 1 and 2) is unchanged and still refuses a stale binary,
|
|
87
|
+
whatever path it was found by.
|
|
88
|
+
|
|
89
|
+
7. **A repository without npm installs the binary with one command, and its CI needs no
|
|
90
|
+
Node (supersedes ADR-059 ruling 3's no-npm remedy).** The remedy the shim prints for
|
|
91
|
+
such a repository is one line — `curl` the asset for this machine into `.simt/bin` and
|
|
92
|
+
`chmod` it — with the platform derived at run time from `uname`, because the remedy is
|
|
93
|
+
committed text that a colleague on another machine will read. The generated consumer
|
|
94
|
+
workflow downloads the same asset, verifies it against the published `.sha256`, and
|
|
95
|
+
installs it into `.simt/bin` with no `setup-node` step at all; the install fails loudly
|
|
96
|
+
rather than letting a gate pass that never started. The remedy stays one line a reader
|
|
97
|
+
copies (it carries no checksum — the dispatcher re-verifies the version, and the
|
|
98
|
+
automated CI install is where the published checksum is checked).
|
|
99
|
+
|
|
100
|
+
## Consequences
|
|
101
|
+
|
|
102
|
+
- **The acceptance criterion is executable.** A Gradle fixture with every directory
|
|
103
|
+
containing `node` removed from `PATH` runs the release binary's `check` and its
|
|
104
|
+
`agent-hook` path, and the rendered hook shim hands over to `.simt/bin/simt` and
|
|
105
|
+
refuses when it is absent (`test/binary.test.mjs`). The test skips rather than fails on
|
|
106
|
+
a host that is not a release target or cannot reach `nodejs.org` for the base, because
|
|
107
|
+
a contributor's suite must still go green (ADR-031).
|
|
108
|
+
- **One `files` list feeds three artefacts.** The npm tarball, the wheel and the binary
|
|
109
|
+
payload all derive from `package.json`'s `files`, so the parity ADR-058 ruled survives
|
|
110
|
+
the new artefact instead of being restated in a third place.
|
|
111
|
+
- **The assets are large — one node runtime each.** That is the price of the only channel
|
|
112
|
+
a repository with no toolchain can use, and it is paid per release, not per install:
|
|
113
|
+
the payload is materialized once per node version and digest and reused after that.
|
|
114
|
+
- **The formula's committed checksums are the deterministic build of this tree.** Editing
|
|
115
|
+
anything in the payload changes them, and a release whose build disagrees with the
|
|
116
|
+
formula fails at tag time — which is the moment the maintainer still has the tree in
|
|
117
|
+
front of them. The committed values are never a placeholder: they are what the next
|
|
118
|
+
tag of this tree must hash to.
|
|
119
|
+
- **A repository that never downloads the binary is unaffected.** The locator's new
|
|
120
|
+
branch is one `[ -x … ]` test that is false when the path is absent, and the npm and
|
|
121
|
+
wheel channels do not consult it.
|
|
122
|
+
|
|
123
|
+
## Rejected
|
|
124
|
+
|
|
125
|
+
- **Bun and Deno `compile`.** Evaluated on their documented behaviour, not measured. The
|
|
126
|
+
package's `engines` floor is Node 22.20.0, its index is `node:sqlite` (ADR-089), and
|
|
127
|
+
every latency figure this plan records was taken on that runtime. A binary compiled by
|
|
128
|
+
another runtime would be the one channel not running the runtime the other three run,
|
|
129
|
+
so the records would stop describing it, and the estate would have to re-measure per
|
|
130
|
+
channel to say anything true about it. Node's SEA keeps every channel on the declared
|
|
131
|
+
runtime, and its base is an official node build — the thing that can be pinned and
|
|
132
|
+
attested byte-for-byte. If the floor ever moves to another runtime, the bootstrap is
|
|
133
|
+
the only part that needs rewriting, because the payload is plain files.
|
|
134
|
+
- **A bundler that inlines the tool into one script.** The tool reads `rules/`,
|
|
135
|
+
`templates/` and its own decision records at run time; a bundle would have to
|
|
136
|
+
re-implement every `import.meta.url` root computation, and ADR-012's zero-dependency
|
|
137
|
+
rule leaves no bundler to depend on. The payload here is runtime-agnostic text; only
|
|
138
|
+
the wrapper is node-specific.
|
|
139
|
+
- **An archive plus a runner script.** Two files is an install step, and the channel
|
|
140
|
+
exists so that a repository with no toolchain gets one file that runs.
|
|
141
|
+
- **A checksum inside the one-line remedy.** The remedy is text a reader copies out of an
|
|
142
|
+
error message; adding a second verification command makes it no longer one line. The
|
|
143
|
+
download is TLS from the release, the dispatcher re-verifies the version the repository
|
|
144
|
+
pinned, and the automated install — the CI step, which no human is watching — is where
|
|
145
|
+
the published `.sha256` is checked before anything runs.
|
|
146
|
+
- **`npm i -g simt-dev@<v>` for a repository without npm** (what ADR-059 ruling 3 ruled).
|
|
147
|
+
The command needs npm, which is the one thing the repository kind is defined by not
|
|
148
|
+
having; that this was the remedy for the kind it could not serve is the defect S35
|
|
149
|
+
found.
|
package/package.json
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "simt-dev",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.6.0",
|
|
4
|
+
"description": "An intelligence amplifier for AI coding agents: rules and dual gates adopted by one command, plus local retrieval, task briefs, benches and repo memory — deterministic by default, with local-model adapters that propose and never select.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Said Elimam",
|
|
7
7
|
"private": false,
|
|
8
8
|
"type": "module",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "https://github.com/QuazarMedia/simt-dev"
|
|
12
|
+
},
|
|
9
13
|
"engines": {
|
|
10
14
|
"node": ">=22.20.0"
|
|
11
15
|
},
|
|
@@ -14,7 +18,7 @@
|
|
|
14
18
|
"linux"
|
|
15
19
|
],
|
|
16
20
|
"bin": {
|
|
17
|
-
"simt": "bin/simt
|
|
21
|
+
"simt": "bin/simt.mjs"
|
|
18
22
|
},
|
|
19
23
|
"files": [
|
|
20
24
|
"bin",
|
|
@@ -38,6 +42,6 @@
|
|
|
38
42
|
],
|
|
39
43
|
"scripts": {
|
|
40
44
|
"test": "node --test --test-concurrency=4 test/*.test.mjs",
|
|
41
|
-
"lint": "node bin/simt
|
|
45
|
+
"lint": "node bin/simt.mjs doctor"
|
|
42
46
|
}
|
|
43
47
|
}
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// Model and effort advice, read from the repository's own bench history (plan 0003 R40).
|
|
2
|
+
//
|
|
3
|
+
// The mapping the design row asks for — task features to the cheapest tier that resolved
|
|
4
|
+
// similar tasks — is data, not opinion: the records `bench run` stored on disk already say
|
|
5
|
+
// which model classes resolved which mined tasks and what those runs cost. So the tier
|
|
6
|
+
// ladder is the estate's own evidence sorted by price, and no vendor name is written into
|
|
7
|
+
// the code: a repository whose history has one class of run gets advice about that class,
|
|
8
|
+
// and one whose history has three gets a ladder with three steps.
|
|
9
|
+
//
|
|
10
|
+
// The properties this exists to keep:
|
|
11
|
+
//
|
|
12
|
+
// * Advice, never a choice. The output is a sentence in the brief header along with the
|
|
13
|
+
// bench numbers behind it; the agent (or the human) names the model, not simt-dev
|
|
14
|
+
// (P2). Nothing here writes a model id into a config, a lock or a command.
|
|
15
|
+
// * Deterministic. No model, no network, no clock (P1): the same runs directory yields
|
|
16
|
+
// the same advice, so a reviewable claim about cost is possible.
|
|
17
|
+
// * Quiet in thin evidence. A neighborhood with no resolved run yields a caution, not a
|
|
18
|
+
// made-up tier; a history with no runs at all yields no advice line.
|
|
19
|
+
//
|
|
20
|
+
// The measure of "cheapest" follows the bench's own report: cost in dollars when the run
|
|
21
|
+
// records or a prices table price the tokens, total tokens otherwise. Ordering by a number
|
|
22
|
+
// that does not exist for any run would be nondeterminism, not economics.
|
|
23
|
+
import { classifyTask } from '../brief/kind.mjs';
|
|
24
|
+
import { computeRunCost, isRunResolved, loadPrices, loadRunRecords } from '../bench/report.mjs';
|
|
25
|
+
import { loadTasks } from '../bench/tasks.mjs';
|
|
26
|
+
import { CONFIG_DIR } from '../name.mjs';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Every advice report carries its row's accept bar (ADR-076: a metric is a measurement or
|
|
30
|
+
* it is not claimed).
|
|
31
|
+
*/
|
|
32
|
+
export const METRIC_NOTE =
|
|
33
|
+
'advice is advisory until a held-out bench shows >= 25% cost cut at delta R >= -2 (plan 0003 R40)';
|
|
34
|
+
|
|
35
|
+
const TIERS = ['small', 'default', 'frontier'];
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The ladder can specialize beyond kind only while there is still evidence to stand on:
|
|
39
|
+
* below this many same-kind tasks the kind is the whole neighborhood, and a narrower
|
|
40
|
+
* window makes evidence worse, not "more similar".
|
|
41
|
+
*/
|
|
42
|
+
const NARROW_MIN_POOL = 6;
|
|
43
|
+
|
|
44
|
+
/** The narrowed window needs at least these resolved runs or it is not evidence either. */
|
|
45
|
+
const NARROW_MIN_RESOLVED = 3;
|
|
46
|
+
|
|
47
|
+
/** A file window is a step from the estimate in either direction — a factor of two in size. */
|
|
48
|
+
const WINDOW_HALF = 2;
|
|
49
|
+
|
|
50
|
+
function median(values) {
|
|
51
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
52
|
+
if (sorted.length === 0) return 0;
|
|
53
|
+
const mid = Math.floor(sorted.length / 2);
|
|
54
|
+
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Tier for a rank in the cheapest-first ladder: thirds, not absolute cost — an estate's
|
|
59
|
+
* "frontier" relative to its own history means more than a number the code guessed.
|
|
60
|
+
* The cheapest of two observed classes is 'small' and the dearer one 'default'; a class
|
|
61
|
+
* is 'frontier' only once a ladder has three steps, because a two-step ladder cannot
|
|
62
|
+
* name a top third honestly.
|
|
63
|
+
*/
|
|
64
|
+
export function tierOfRank(rank, count) {
|
|
65
|
+
if (count < 3) return rank === 0 ? TIERS[0] : TIERS[1];
|
|
66
|
+
if (rank * 3 < count) return TIERS[0];
|
|
67
|
+
if (rank * 3 >= count * 2) return TIERS[2];
|
|
68
|
+
return TIERS[1];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Total tokens a usage report carries — the measure that exists even with no prices file.
|
|
73
|
+
*/
|
|
74
|
+
function tokensOf(usage = {}) {
|
|
75
|
+
const num = (v) => Number(v ?? 0);
|
|
76
|
+
return num(usage.input) + num(usage.cacheWrite ?? usage.cache_write) + num(usage.cacheRead ?? usage.cache_read) + num(usage.output);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Load bench history as advice-ready run rows.
|
|
81
|
+
*
|
|
82
|
+
* A descriptor missing for a run leaves the row with `kind: null`; such a run can join no
|
|
83
|
+
* neighborhood, which is the honest answer — a task whose prompt was never mined is a
|
|
84
|
+
* task whose kind would be a guess.
|
|
85
|
+
*/
|
|
86
|
+
export async function loadHistory(root) {
|
|
87
|
+
const [records, prices, tasks] = await Promise.all([
|
|
88
|
+
loadRunRecords(root),
|
|
89
|
+
// An unreadable prices file would throw out of a brief, whose callers only asked for
|
|
90
|
+
// a packet; the bench has its own report for surfacing a broken prices file loudly.
|
|
91
|
+
loadPrices(root).catch(() => null),
|
|
92
|
+
loadTasks(root),
|
|
93
|
+
]);
|
|
94
|
+
const byId = new Map(tasks.map((task) => [task.id, task]));
|
|
95
|
+
const runs = records.map((record) => {
|
|
96
|
+
const descriptor = byId.get(record.task) ?? null;
|
|
97
|
+
return {
|
|
98
|
+
task: record.task,
|
|
99
|
+
model: record.model,
|
|
100
|
+
resolved: isRunResolved(record),
|
|
101
|
+
cost: computeRunCost(record, prices),
|
|
102
|
+
tokens: tokensOf(record.usage),
|
|
103
|
+
toolCalls: Number(record?.usage?.toolCalls ?? record?.usage?.tool_calls ?? 0),
|
|
104
|
+
files: Number(descriptor?.gold?.files ?? record?.diff?.files ?? 0),
|
|
105
|
+
kind: descriptor ? classifyTask(descriptor.prompt ?? '', { paths: [] }).kind : null,
|
|
106
|
+
};
|
|
107
|
+
});
|
|
108
|
+
return { runs };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Advise from bench history for one task's features.
|
|
113
|
+
*
|
|
114
|
+
* @param {{kind: string, files: number}} features the task kind, and the estimate of how
|
|
115
|
+
* many files it will touch (the brief's scope size; a history run carries the mined
|
|
116
|
+
* task's gold stat in the same role)
|
|
117
|
+
* @param {{runs: object[]}} history rows from loadHistory
|
|
118
|
+
* @returns null when the history holds no runs; otherwise `{ picked, caution }` where
|
|
119
|
+
* `picked` is `{ line, basis, metric }` when a tier was named (and `caution` null), and
|
|
120
|
+
* `caution` explains the missing neighborhood when no tier can honestly be named
|
|
121
|
+
*/
|
|
122
|
+
export function advise(features, history) {
|
|
123
|
+
const runs = history?.runs ?? [];
|
|
124
|
+
if (runs.length === 0) return null;
|
|
125
|
+
const kind = features.kind;
|
|
126
|
+
const sameKind = runs.filter((run) => run.kind === kind);
|
|
127
|
+
const taskCount = new Set(sameKind.map((run) => run.task)).size;
|
|
128
|
+
if (sameKind.length === 0) {
|
|
129
|
+
return {
|
|
130
|
+
picked: null,
|
|
131
|
+
caution: `bench history holds no ${kind} task, so no cheap tier was advised`,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// The measure is chosen once, across all resolved history: a ladder built on dollars
|
|
136
|
+
// while the pick ran on tokens would compare a number with a different number's name.
|
|
137
|
+
const measure = runs.some((run) => run.resolved && run.cost > 0) ? 'cost' : 'tokens';
|
|
138
|
+
|
|
139
|
+
// Kind decides the neighborhood. The file window only specializes a neighborhood that
|
|
140
|
+
// has already got evidence to spare (NARROW_MIN_*): a factor-of-two window around the
|
|
141
|
+
// estimate refines "bugfix" to "small bugfix" when the kind has runs to lend, and says
|
|
142
|
+
// nothing when it has not.
|
|
143
|
+
let pool = sameKind;
|
|
144
|
+
if (features.files > 0 && sameKind.length >= NARROW_MIN_POOL) {
|
|
145
|
+
const near = sameKind.filter(
|
|
146
|
+
(run) =>
|
|
147
|
+
run.files > 0 &&
|
|
148
|
+
run.files >= Math.ceil(features.files / WINDOW_HALF) &&
|
|
149
|
+
run.files <= features.files * WINDOW_HALF,
|
|
150
|
+
);
|
|
151
|
+
if (near.filter((run) => run.resolved).length >= NARROW_MIN_RESOLVED) pool = near;
|
|
152
|
+
}
|
|
153
|
+
const poolTaskCount = new Set(pool.map((run) => run.task)).size;
|
|
154
|
+
|
|
155
|
+
// The ladder orders every model class the estate ever ran, cheapest first, so the
|
|
156
|
+
// advice can say where its pick sits in the estate's whole history. Uncost evidence is
|
|
157
|
+
// still cost position: a class that never resolved one of these tasks occupies a rung
|
|
158
|
+
// in the price ladder, and the resolved floor below — not the ladder — decides whether
|
|
159
|
+
// the pick is allowed to stand on it.
|
|
160
|
+
const ladderValues = new Map();
|
|
161
|
+
for (const run of runs) {
|
|
162
|
+
const values = ladderValues.get(run.model) ?? [];
|
|
163
|
+
values.push(measure === 'cost' ? run.cost : run.tokens);
|
|
164
|
+
ladderValues.set(run.model, values);
|
|
165
|
+
}
|
|
166
|
+
const ladderNames = [...ladderValues.keys()].sort(
|
|
167
|
+
(a, b) => median(ladderValues.get(a)) - median(ladderValues.get(b)) || a.localeCompare(b),
|
|
168
|
+
);
|
|
169
|
+
|
|
170
|
+
const resolvedByModel = new Map();
|
|
171
|
+
for (const run of pool) {
|
|
172
|
+
if (!run.resolved) continue;
|
|
173
|
+
const values = resolvedByModel.get(run.model) ?? [];
|
|
174
|
+
values.push(measure === 'cost' ? run.cost : run.tokens);
|
|
175
|
+
resolvedByModel.set(run.model, values);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// "Resolved similar tasks" is a floor, not a style: a class that never resolved one of
|
|
179
|
+
// these tasks is cheaper because it failed for less money.
|
|
180
|
+
const qualifying = ladderNames.filter((model) => resolvedByModel.has(model));
|
|
181
|
+
if (qualifying.length === 0) {
|
|
182
|
+
return {
|
|
183
|
+
picked: null,
|
|
184
|
+
caution: `no model resolved any of the ${taskCount} ${kind} task${taskCount === 1 ? '' : 's'} in bench history, so no cheap tier was advised`,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Ties break on the model name, so the same history always yields the same pick (P4).
|
|
189
|
+
const cheapest = [...qualifying].sort(
|
|
190
|
+
(a, b) => median(resolvedByModel.get(a)) - median(resolvedByModel.get(b)) || a.localeCompare(b),
|
|
191
|
+
)[0];
|
|
192
|
+
const ladderRank = ladderNames.indexOf(cheapest);
|
|
193
|
+
const tier = tierOfRank(ladderRank, ladderNames.length);
|
|
194
|
+
|
|
195
|
+
// Effort advice is relative to the estate too: heavier turns of tool use than the
|
|
196
|
+
// estate's own resolved median is what "plan first" means here.
|
|
197
|
+
const resolvedRuns = runs.filter((run) => run.resolved);
|
|
198
|
+
const estateToolCalls = median(resolvedRuns.map((run) => run.toolCalls));
|
|
199
|
+
const resolvedRunsOfCheapest = pool.filter((run) => run.resolved && run.model === cheapest);
|
|
200
|
+
const pickToolCalls = median(resolvedRunsOfCheapest.map((run) => run.toolCalls));
|
|
201
|
+
const effort = estateToolCalls > 0 && pickToolCalls > estateToolCalls ? 'plan first' : 'no plan mode';
|
|
202
|
+
|
|
203
|
+
const pickMedian = median(resolvedByModel.get(cheapest));
|
|
204
|
+
const pickUnit = measure === 'cost' ? `cost $${pickMedian.toFixed(4)}` : `tokens ${Math.round(pickMedian)}`;
|
|
205
|
+
|
|
206
|
+
return {
|
|
207
|
+
picked: {
|
|
208
|
+
line: `${tier} model, ${effort}`,
|
|
209
|
+
basis: `from ${poolTaskCount} ${kind} task${poolTaskCount === 1 ? '' : 's'} in bench history; cheapest that resolved: ${cheapest} (median ${pickUnit}); a human picks the model, this only ranks classes`,
|
|
210
|
+
metric: METRIC_NOTE,
|
|
211
|
+
},
|
|
212
|
+
caution: null,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* The brief's door into advice (plan 0003 R40): history on disk in, advice out, or null
|
|
218
|
+
* when the history has no runs at all — a brief in a repository without bench history
|
|
219
|
+
* carries every other section and no made-up opinion about models.
|
|
220
|
+
*
|
|
221
|
+
* `kind` comes in already classified when the caller is the brief itself, so the packet
|
|
222
|
+
* never carries two opinions about one task.
|
|
223
|
+
*/
|
|
224
|
+
export async function briefAdvice(root, { task = '', scope = [], kind = null } = {}) {
|
|
225
|
+
const history = await loadHistory(root);
|
|
226
|
+
if (history.runs.length === 0) return null;
|
|
227
|
+
const known = kind ?? classifyTask(task, { paths: scope }).kind;
|
|
228
|
+
return advise({ kind: known, files: scope.length }, history);
|
|
229
|
+
}
|