kibsu 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kibsu-0.1.0/LICENSE +21 -0
- kibsu-0.1.0/PKG-INFO +250 -0
- kibsu-0.1.0/README.md +217 -0
- kibsu-0.1.0/kibsu/__init__.py +6 -0
- kibsu-0.1.0/kibsu/__main__.py +139 -0
- kibsu-0.1.0/kibsu/audit.py +468 -0
- kibsu-0.1.0/kibsu/check.py +534 -0
- kibsu-0.1.0/kibsu/config.py +63 -0
- kibsu-0.1.0/kibsu/discover.py +323 -0
- kibsu-0.1.0/kibsu/gate.py +720 -0
- kibsu-0.1.0/kibsu/guide.py +281 -0
- kibsu-0.1.0/kibsu/index.py +235 -0
- kibsu-0.1.0/kibsu/install.py +317 -0
- kibsu-0.1.0/kibsu/learn.py +313 -0
- kibsu-0.1.0/kibsu/report.py +325 -0
- kibsu-0.1.0/kibsu/survey.py +173 -0
- kibsu-0.1.0/kibsu/tokens.py +344 -0
- kibsu-0.1.0/kibsu.egg-info/PKG-INFO +250 -0
- kibsu-0.1.0/kibsu.egg-info/SOURCES.txt +32 -0
- kibsu-0.1.0/kibsu.egg-info/dependency_links.txt +1 -0
- kibsu-0.1.0/kibsu.egg-info/top_level.txt +1 -0
- kibsu-0.1.0/pyproject.toml +22 -0
- kibsu-0.1.0/setup.cfg +4 -0
- kibsu-0.1.0/tests/test_audit.py +117 -0
- kibsu-0.1.0/tests/test_check.py +144 -0
- kibsu-0.1.0/tests/test_discover.py +75 -0
- kibsu-0.1.0/tests/test_gate.py +262 -0
- kibsu-0.1.0/tests/test_guide.py +110 -0
- kibsu-0.1.0/tests/test_index.py +108 -0
- kibsu-0.1.0/tests/test_install.py +239 -0
- kibsu-0.1.0/tests/test_learn.py +85 -0
- kibsu-0.1.0/tests/test_report.py +84 -0
- kibsu-0.1.0/tests/test_survey.py +343 -0
- kibsu-0.1.0/tests/test_tokens.py +83 -0
kibsu-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mohammed Bajalan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
kibsu-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kibsu
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reads your coding-agent instructions and reports which of them can actually be verified.
|
|
5
|
+
Author: Mohammed Bajalan
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Mohammed Bajalan
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/M-Bajalan/kibsu
|
|
29
|
+
Requires-Python: >=3.8
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# Kibsu
|
|
35
|
+
|
|
36
|
+
**What can your agents not do in this repository yet?**
|
|
37
|
+
|
|
38
|
+
You have written instructions for coding agents — `AGENTS.md`, `CLAUDE.md`, a `skills/` directory,
|
|
39
|
+
`.cursorrules`. Kibsu reads them alongside your git history and reports which of those instructions
|
|
40
|
+
anyone could actually verify were followed, and which ones only work when a human remembers.
|
|
41
|
+
|
|
42
|
+
It writes nothing. Run it, then run `git status`.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## Run it
|
|
47
|
+
|
|
48
|
+
No dependencies. Python 3.8+.
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
python -m kibsu /path/to/any/repo
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Real output, against a public repository at commit `3dcbd5c`:
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
WHAT YOUR AGENTS CANNOT DO HERE YET
|
|
58
|
+
/tmp/superpowers
|
|
59
|
+
--------------------------------------------------------------------------
|
|
60
|
+
x Find your docs 89 markdown files, no index. An agent must grep the tree.
|
|
61
|
+
+ Know your conventions consistent frontmatter (description, name) - enforceable.
|
|
62
|
+
+ Prove they followed 19.8% of your procedural instructions are verifiable
|
|
63
|
+
(above the 11.1% public median).
|
|
64
|
+
x Resume after a break your instructions promise no artifacts at all - nothing
|
|
65
|
+
survives the session, and nothing can be checked.
|
|
66
|
+
! Follow your own rules no index to check history against.
|
|
67
|
+
--------------------------------------------------------------------------
|
|
68
|
+
2 of 5 ready.
|
|
69
|
+
Nothing was written to this repo - run `git status` to confirm.
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Reproduce it:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
git clone https://github.com/obra/superpowers /tmp/superpowers && cd /tmp/superpowers && git checkout 3dcbd5c
|
|
76
|
+
python -m kibsu /tmp/superpowers && git -C /tmp/superpowers status --porcelain
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The second command prints nothing. That is the point.
|
|
80
|
+
|
|
81
|
+
`+` ready · `x` not ready · `!` not applicable here · `?` **could not be checked**
|
|
82
|
+
|
|
83
|
+
A check that could not run gets its own line and its own mark, and the summary says so out loud.
|
|
84
|
+
Printing one line fewer would make "2 of 4" and "2 of 5" indistinguishable, and the check that
|
|
85
|
+
cannot run is reliably the uncomfortable one.
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## The five questions
|
|
90
|
+
|
|
91
|
+
| | question | what it actually tests |
|
|
92
|
+
|---|---|---|
|
|
93
|
+
| 1 | **Find your docs** | is there an index, or must an agent grep the whole tree on every task? |
|
|
94
|
+
| 2 | **Know your conventions** | is there a consistent document shape to enforce, or does every file differ? |
|
|
95
|
+
| 3 | **Prove they followed** | what fraction of your instructions could a reviewer verify afterwards? |
|
|
96
|
+
| 4 | **Resume after a break** | do the files your instructions promise actually get produced? |
|
|
97
|
+
| 5 | **Follow your own rules** | how often have your own commits broken your own written rule? |
|
|
98
|
+
|
|
99
|
+
Question 5 is a **replay**, not an opinion. It walks your last N commits and asks, for each one,
|
|
100
|
+
whether the index you claim to maintain would have gone stale. The answer is a count of your own
|
|
101
|
+
commits, which is considerably harder to argue with than a score.
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## What "verifiable" means
|
|
106
|
+
|
|
107
|
+
An instruction is **CHECKABLE** if a reviewer could tell, from the repository alone, whether it
|
|
108
|
+
happened:
|
|
109
|
+
|
|
110
|
+
- it is a tick-box, **or**
|
|
111
|
+
- it contains a runnable command, **or**
|
|
112
|
+
- it names a concrete file artifact, **or**
|
|
113
|
+
- it refers to an exit code, a diff, or an assertion
|
|
114
|
+
|
|
115
|
+
Everything else is **CLAIMABLE**: the only evidence it happened is the agent saying that it did.
|
|
116
|
+
|
|
117
|
+
**The metric is deliberately biased toward CHECKABLE.** Every ambiguous instruction is counted as
|
|
118
|
+
checkable — a bare path mention counts, a bare command word counts. Reported figures are therefore
|
|
119
|
+
**ceilings**. The true numbers are lower. `--definitions` prints the whole ruleset so you can argue
|
|
120
|
+
with it.
|
|
121
|
+
|
|
122
|
+
### Genres, and why they matter
|
|
123
|
+
|
|
124
|
+
Scoring every document on checkability is the obvious way to get this wrong, so units are split
|
|
125
|
+
first and procedure-only figures are reported separately:
|
|
126
|
+
|
|
127
|
+
| genre | describes | is checkability a fair test? |
|
|
128
|
+
|---|---|---|
|
|
129
|
+
| **procedure** | what to do, in order | yes — this is the headline |
|
|
130
|
+
| **doctrine** | how to *think* ("name the assumption before building") | **no.** Produces judgement, not files. 0% here is the genre working, not a defect |
|
|
131
|
+
| **persona** | who the agent is ("You are a senior Rust engineer…") | no — it promises nothing |
|
|
132
|
+
| **reference** | lookup material: tables, options, definitions | partially |
|
|
133
|
+
|
|
134
|
+
**Genre is declared in frontmatter** (`genre: doctrine`), not guessed. Auto-detection still runs as
|
|
135
|
+
a fallback, and **any disagreement between declaration and detection is printed** — a declaration
|
|
136
|
+
cannot quietly buy a better score.
|
|
137
|
+
|
|
138
|
+
That design exists because auto-detection failed. Ten numbered *principles* are structurally
|
|
139
|
+
identical to ten numbered *steps*, and every heuristic tried turned out to be a prior belief
|
|
140
|
+
expressed as a regular expression.
|
|
141
|
+
|
|
142
|
+
### Phantom artifacts
|
|
143
|
+
|
|
144
|
+
Some instructions mandate an output file. Kibsu extracts those filenames and searches the working
|
|
145
|
+
tree **and the full git history** for them. An artifact that has never existed, in any commit, is a
|
|
146
|
+
**phantom** — an instruction no model has ever been caught skipping, because nothing was ever
|
|
147
|
+
looking.
|
|
148
|
+
|
|
149
|
+
Scoped to artifacts a document claims are produced *inside its own repository*. Scaffolding
|
|
150
|
+
instructions that generate files in the **user's** project are excluded, with the reason printed.
|
|
151
|
+
On a shallow clone the result is `UNKNOWN`, never zero.
|
|
152
|
+
|
|
153
|
+
---
|
|
154
|
+
|
|
155
|
+
## The survey
|
|
156
|
+
|
|
157
|
+
Eight public instruction collections, measured at pinned commits. Raw per-repo JSON, each carrying
|
|
158
|
+
the SHA it was measured at, is in [`evidence/`](evidence/).
|
|
159
|
+
|
|
160
|
+
Regenerate the whole table yourself:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
python -m kibsu survey
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
> **This section previously linked to an `evidence/` directory that had never existed in any
|
|
167
|
+
> commit** — a phantom artifact, in the README of a tool built to detect phantom artifacts. It was
|
|
168
|
+
> found by an outside reviewer, not by me, and not by four of my own review agents that had passed
|
|
169
|
+
> the same file. The directory now exists because the survey was actually re-run; see below for the
|
|
170
|
+
> one number that changed when it was.
|
|
171
|
+
|
|
172
|
+
| repo | sha | units | instr | all% | **proc%** | phantom |
|
|
173
|
+
|---|---|---:|---:|---:|---:|---:|
|
|
174
|
+
| davila7/claude-code-templates | `91d14a7` | 891 | 12,578 | 21.3% | **22.9%** | 34/74 (46%) |
|
|
175
|
+
| obra/superpowers | `3dcbd5c` | 14 | 267 | 18.0% | **19.8%** | — |
|
|
176
|
+
| wshobson/agents | `c4b82b0` | 180 | 807 | 10.8% | **12.8%** | 4/7 (57%) |
|
|
177
|
+
| contains-studio/agents | `a5a480c` | 37 | 564 | 12.4% | **12.7%** | — |
|
|
178
|
+
| anthropics/skills | `b29e7cf` | 18 | 310 | 15.8% | **9.4%** | 6/22 (27%) |
|
|
179
|
+
| sanjeed5/awesome-cursor-rules-mdc | `8fbf269` | 5 | 124 | 8.9% | **8.9%** | — |
|
|
180
|
+
| vijaythecoder/awesome-claude-agents | `2050f3c` | 33 | 186 | 7.5% | **6.2%** | — |
|
|
181
|
+
| VoltAgent/awesome-claude-code-subagents | `947b44c` | 154 | 2,775 | 2.1% | **2.2%** | — |
|
|
182
|
+
|
|
183
|
+
**median procedure-only: 11.1%** · min 2.2% · max 22.9%
|
|
184
|
+
**in-scope mandated artifacts: 103 distinct, 44 never existed in any commit (43%)**
|
|
185
|
+
|
|
186
|
+
### One number moved, and that is the point of the SHA column
|
|
187
|
+
|
|
188
|
+
An earlier revision of this table published **41 phantoms of 99 (41%)** and carried **no commit
|
|
189
|
+
SHAs**. Re-measuring produced **44 of 103 (43%)**.
|
|
190
|
+
|
|
191
|
+
Every checkable figure — units, instructions, all%, proc%, the median, min and max — matched
|
|
192
|
+
exactly. The entire difference is the phantom-artifact count for
|
|
193
|
+
`davila7/claude-code-templates`, the most actively maintained collection of the eight: `31/70`
|
|
194
|
+
became `34/74` as the repository grew.
|
|
195
|
+
|
|
196
|
+
Nothing was wrong with the original measurement. It simply **could not be reproduced**, because it
|
|
197
|
+
recorded no commit to reproduce it against — the same defect this tool reports as `? COULD NOT
|
|
198
|
+
CHECK` in other people's repositories. Both figures were probably right about different commits,
|
|
199
|
+
and there was no way to tell.
|
|
200
|
+
|
|
201
|
+
Every row now carries the SHA it was measured at, and the raw per-repo JSON is in
|
|
202
|
+
[`evidence/`](evidence/).
|
|
203
|
+
|
|
204
|
+
Two further repositories fell below the sample floor (≥5 procedure units **and** ≥50 procedure
|
|
205
|
+
instructions). They are reported but **not ranked**: a percentage from fifteen instructions is noise
|
|
206
|
+
wearing a number, and one of the two would otherwise have topped the table at 66.7% off a single
|
|
207
|
+
unit.
|
|
208
|
+
|
|
209
|
+
### The part that was not expected
|
|
210
|
+
|
|
211
|
+
**Five of the eight mandate zero artifacts.** Not "promised and missing" — *never promised anything
|
|
212
|
+
at all*. The field splits into two distinct failure modes: collections that make no verifiable claim
|
|
213
|
+
in the first place, and collections whose claims fail roughly 40% of the time.
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## Honest limits
|
|
218
|
+
|
|
219
|
+
Read these before quoting anything above.
|
|
220
|
+
|
|
221
|
+
- **`obra/superpowers` ships a separate eval harness** that drives real agent CLIs and grades
|
|
222
|
+
whether a skill was followed. Kibsu reads only *documents*, so it cannot see that. The real
|
|
223
|
+
enforceability of that repo is higher than 19.8%. This is the strongest counter-example to the
|
|
224
|
+
entire approach, and it belongs at the top of the limits rather than buried at the bottom.
|
|
225
|
+
- **The metric is a heuristic.** Biased generous by design. Ceilings, not measurements.
|
|
226
|
+
- **Persona collections are not badly written**, they are a different genre. Scoring them on
|
|
227
|
+
checkability was the first mistake this tool made.
|
|
228
|
+
- **Low checkability has not been shown to produce worse outcomes.** Nobody has demonstrated that.
|
|
229
|
+
The demonstrated finding is that *nobody can tell either way* — the weaker claim, the more
|
|
230
|
+
defensible one, and the only one worth acting on.
|
|
231
|
+
- **N is small** for several repositories. Sample floors are enforced, not assumed.
|
|
232
|
+
- **Genre auto-detection is weak.** Declare it.
|
|
233
|
+
|
|
234
|
+
---
|
|
235
|
+
|
|
236
|
+
## The name
|
|
237
|
+
|
|
238
|
+
*kibsu* (𒆠𒍑, KI.UŠ) is Akkadian, derived from *kabāsu*, "to tread":
|
|
239
|
+
|
|
240
|
+
> **1.** "tread, imprint" of a foot — *kibsu redû*, "to follow a track"; *ša kibsi*, "tracker"
|
|
241
|
+
> **2.** "track, route"; transferred, "mode of life, behaviour, course of life"
|
|
242
|
+
> **3.** mathematical: "way of calculation"
|
|
243
|
+
|
|
244
|
+
— *A Concise Dictionary of Akkadian*, Black, George & Postgate, Harrassowitz ²2000
|
|
245
|
+
|
|
246
|
+
The trace left behind, the course of conduct, and the method of reckoning. The tool is all three.
|
|
247
|
+
|
|
248
|
+
## License
|
|
249
|
+
|
|
250
|
+
MIT. See [LICENSE](LICENSE) and [PROVENANCE.md](PROVENANCE.md).
|
kibsu-0.1.0/README.md
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
# Kibsu
|
|
2
|
+
|
|
3
|
+
**What can your agents not do in this repository yet?**
|
|
4
|
+
|
|
5
|
+
You have written instructions for coding agents — `AGENTS.md`, `CLAUDE.md`, a `skills/` directory,
|
|
6
|
+
`.cursorrules`. Kibsu reads them alongside your git history and reports which of those instructions
|
|
7
|
+
anyone could actually verify were followed, and which ones only work when a human remembers.
|
|
8
|
+
|
|
9
|
+
It writes nothing. Run it, then run `git status`.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Run it
|
|
14
|
+
|
|
15
|
+
No dependencies. Python 3.8+.
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
python -m kibsu /path/to/any/repo
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Real output, against a public repository at commit `3dcbd5c`:
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
WHAT YOUR AGENTS CANNOT DO HERE YET
|
|
25
|
+
/tmp/superpowers
|
|
26
|
+
--------------------------------------------------------------------------
|
|
27
|
+
x Find your docs 89 markdown files, no index. An agent must grep the tree.
|
|
28
|
+
+ Know your conventions consistent frontmatter (description, name) - enforceable.
|
|
29
|
+
+ Prove they followed 19.8% of your procedural instructions are verifiable
|
|
30
|
+
(above the 11.1% public median).
|
|
31
|
+
x Resume after a break your instructions promise no artifacts at all - nothing
|
|
32
|
+
survives the session, and nothing can be checked.
|
|
33
|
+
! Follow your own rules no index to check history against.
|
|
34
|
+
--------------------------------------------------------------------------
|
|
35
|
+
2 of 5 ready.
|
|
36
|
+
Nothing was written to this repo - run `git status` to confirm.
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Reproduce it:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
git clone https://github.com/obra/superpowers /tmp/superpowers && cd /tmp/superpowers && git checkout 3dcbd5c
|
|
43
|
+
python -m kibsu /tmp/superpowers && git -C /tmp/superpowers status --porcelain
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The second command prints nothing. That is the point.
|
|
47
|
+
|
|
48
|
+
`+` ready · `x` not ready · `!` not applicable here · `?` **could not be checked**
|
|
49
|
+
|
|
50
|
+
A check that could not run gets its own line and its own mark, and the summary says so out loud.
|
|
51
|
+
Printing one line fewer would make "2 of 4" and "2 of 5" indistinguishable, and the check that
|
|
52
|
+
cannot run is reliably the uncomfortable one.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## The five questions
|
|
57
|
+
|
|
58
|
+
| | question | what it actually tests |
|
|
59
|
+
|---|---|---|
|
|
60
|
+
| 1 | **Find your docs** | is there an index, or must an agent grep the whole tree on every task? |
|
|
61
|
+
| 2 | **Know your conventions** | is there a consistent document shape to enforce, or does every file differ? |
|
|
62
|
+
| 3 | **Prove they followed** | what fraction of your instructions could a reviewer verify afterwards? |
|
|
63
|
+
| 4 | **Resume after a break** | do the files your instructions promise actually get produced? |
|
|
64
|
+
| 5 | **Follow your own rules** | how often have your own commits broken your own written rule? |
|
|
65
|
+
|
|
66
|
+
Question 5 is a **replay**, not an opinion. It walks your last N commits and asks, for each one,
|
|
67
|
+
whether the index you claim to maintain would have gone stale. The answer is a count of your own
|
|
68
|
+
commits, which is considerably harder to argue with than a score.
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## What "verifiable" means
|
|
73
|
+
|
|
74
|
+
An instruction is **CHECKABLE** if a reviewer could tell, from the repository alone, whether it
|
|
75
|
+
happened:
|
|
76
|
+
|
|
77
|
+
- it is a tick-box, **or**
|
|
78
|
+
- it contains a runnable command, **or**
|
|
79
|
+
- it names a concrete file artifact, **or**
|
|
80
|
+
- it refers to an exit code, a diff, or an assertion
|
|
81
|
+
|
|
82
|
+
Everything else is **CLAIMABLE**: the only evidence it happened is the agent saying that it did.
|
|
83
|
+
|
|
84
|
+
**The metric is deliberately biased toward CHECKABLE.** Every ambiguous instruction is counted as
|
|
85
|
+
checkable — a bare path mention counts, a bare command word counts. Reported figures are therefore
|
|
86
|
+
**ceilings**. The true numbers are lower. `--definitions` prints the whole ruleset so you can argue
|
|
87
|
+
with it.
|
|
88
|
+
|
|
89
|
+
### Genres, and why they matter
|
|
90
|
+
|
|
91
|
+
Scoring every document on checkability is the obvious way to get this wrong, so units are split
|
|
92
|
+
first and procedure-only figures are reported separately:
|
|
93
|
+
|
|
94
|
+
| genre | describes | is checkability a fair test? |
|
|
95
|
+
|---|---|---|
|
|
96
|
+
| **procedure** | what to do, in order | yes — this is the headline |
|
|
97
|
+
| **doctrine** | how to *think* ("name the assumption before building") | **no.** Produces judgement, not files. 0% here is the genre working, not a defect |
|
|
98
|
+
| **persona** | who the agent is ("You are a senior Rust engineer…") | no — it promises nothing |
|
|
99
|
+
| **reference** | lookup material: tables, options, definitions | partially |
|
|
100
|
+
|
|
101
|
+
**Genre is declared in frontmatter** (`genre: doctrine`), not guessed. Auto-detection still runs as
|
|
102
|
+
a fallback, and **any disagreement between declaration and detection is printed** — a declaration
|
|
103
|
+
cannot quietly buy a better score.
|
|
104
|
+
|
|
105
|
+
That design exists because auto-detection failed. Ten numbered *principles* are structurally
|
|
106
|
+
identical to ten numbered *steps*, and every heuristic tried turned out to be a prior belief
|
|
107
|
+
expressed as a regular expression.
|
|
108
|
+
|
|
109
|
+
### Phantom artifacts
|
|
110
|
+
|
|
111
|
+
Some instructions mandate an output file. Kibsu extracts those filenames and searches the working
|
|
112
|
+
tree **and the full git history** for them. An artifact that has never existed, in any commit, is a
|
|
113
|
+
**phantom** — an instruction no model has ever been caught skipping, because nothing was ever
|
|
114
|
+
looking.
|
|
115
|
+
|
|
116
|
+
Scoped to artifacts a document claims are produced *inside its own repository*. Scaffolding
|
|
117
|
+
instructions that generate files in the **user's** project are excluded, with the reason printed.
|
|
118
|
+
On a shallow clone the result is `UNKNOWN`, never zero.
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## The survey
|
|
123
|
+
|
|
124
|
+
Eight public instruction collections, measured at pinned commits. Raw per-repo JSON, each carrying
|
|
125
|
+
the SHA it was measured at, is in [`evidence/`](evidence/).
|
|
126
|
+
|
|
127
|
+
Regenerate the whole table yourself:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
python -m kibsu survey
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
> **This section previously linked to an `evidence/` directory that had never existed in any
|
|
134
|
+
> commit** — a phantom artifact, in the README of a tool built to detect phantom artifacts. It was
|
|
135
|
+
> found by an outside reviewer, not by me, and not by four of my own review agents that had passed
|
|
136
|
+
> the same file. The directory now exists because the survey was actually re-run; see below for the
|
|
137
|
+
> one number that changed when it was.
|
|
138
|
+
|
|
139
|
+
| repo | sha | units | instr | all% | **proc%** | phantom |
|
|
140
|
+
|---|---|---:|---:|---:|---:|---:|
|
|
141
|
+
| davila7/claude-code-templates | `91d14a7` | 891 | 12,578 | 21.3% | **22.9%** | 34/74 (46%) |
|
|
142
|
+
| obra/superpowers | `3dcbd5c` | 14 | 267 | 18.0% | **19.8%** | — |
|
|
143
|
+
| wshobson/agents | `c4b82b0` | 180 | 807 | 10.8% | **12.8%** | 4/7 (57%) |
|
|
144
|
+
| contains-studio/agents | `a5a480c` | 37 | 564 | 12.4% | **12.7%** | — |
|
|
145
|
+
| anthropics/skills | `b29e7cf` | 18 | 310 | 15.8% | **9.4%** | 6/22 (27%) |
|
|
146
|
+
| sanjeed5/awesome-cursor-rules-mdc | `8fbf269` | 5 | 124 | 8.9% | **8.9%** | — |
|
|
147
|
+
| vijaythecoder/awesome-claude-agents | `2050f3c` | 33 | 186 | 7.5% | **6.2%** | — |
|
|
148
|
+
| VoltAgent/awesome-claude-code-subagents | `947b44c` | 154 | 2,775 | 2.1% | **2.2%** | — |
|
|
149
|
+
|
|
150
|
+
**median procedure-only: 11.1%** · min 2.2% · max 22.9%
|
|
151
|
+
**in-scope mandated artifacts: 103 distinct, 44 never existed in any commit (43%)**
|
|
152
|
+
|
|
153
|
+
### One number moved, and that is the point of the SHA column
|
|
154
|
+
|
|
155
|
+
An earlier revision of this table published **41 phantoms of 99 (41%)** and carried **no commit
|
|
156
|
+
SHAs**. Re-measuring produced **44 of 103 (43%)**.
|
|
157
|
+
|
|
158
|
+
Every checkable figure — units, instructions, all%, proc%, the median, min and max — matched
|
|
159
|
+
exactly. The entire difference is the phantom-artifact count for
|
|
160
|
+
`davila7/claude-code-templates`, the most actively maintained collection of the eight: `31/70`
|
|
161
|
+
became `34/74` as the repository grew.
|
|
162
|
+
|
|
163
|
+
Nothing was wrong with the original measurement. It simply **could not be reproduced**, because it
|
|
164
|
+
recorded no commit to reproduce it against — the same defect this tool reports as `? COULD NOT
|
|
165
|
+
CHECK` in other people's repositories. Both figures were probably right about different commits,
|
|
166
|
+
and there was no way to tell.
|
|
167
|
+
|
|
168
|
+
Every row now carries the SHA it was measured at, and the raw per-repo JSON is in
|
|
169
|
+
[`evidence/`](evidence/).
|
|
170
|
+
|
|
171
|
+
Two further repositories fell below the sample floor (≥5 procedure units **and** ≥50 procedure
|
|
172
|
+
instructions). They are reported but **not ranked**: a percentage from fifteen instructions is noise
|
|
173
|
+
wearing a number, and one of the two would otherwise have topped the table at 66.7% off a single
|
|
174
|
+
unit.
|
|
175
|
+
|
|
176
|
+
### The part that was not expected
|
|
177
|
+
|
|
178
|
+
**Five of the eight mandate zero artifacts.** Not "promised and missing" — *never promised anything
|
|
179
|
+
at all*. The field splits into two distinct failure modes: collections that make no verifiable claim
|
|
180
|
+
in the first place, and collections whose claims fail roughly 40% of the time.
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
184
|
+
## Honest limits
|
|
185
|
+
|
|
186
|
+
Read these before quoting anything above.
|
|
187
|
+
|
|
188
|
+
- **`obra/superpowers` ships a separate eval harness** that drives real agent CLIs and grades
|
|
189
|
+
whether a skill was followed. Kibsu reads only *documents*, so it cannot see that. The real
|
|
190
|
+
enforceability of that repo is higher than 19.8%. This is the strongest counter-example to the
|
|
191
|
+
entire approach, and it belongs at the top of the limits rather than buried at the bottom.
|
|
192
|
+
- **The metric is a heuristic.** Biased generous by design. Ceilings, not measurements.
|
|
193
|
+
- **Persona collections are not badly written**, they are a different genre. Scoring them on
|
|
194
|
+
checkability was the first mistake this tool made.
|
|
195
|
+
- **Low checkability has not been shown to produce worse outcomes.** Nobody has demonstrated that.
|
|
196
|
+
The demonstrated finding is that *nobody can tell either way* — the weaker claim, the more
|
|
197
|
+
defensible one, and the only one worth acting on.
|
|
198
|
+
- **N is small** for several repositories. Sample floors are enforced, not assumed.
|
|
199
|
+
- **Genre auto-detection is weak.** Declare it.
|
|
200
|
+
|
|
201
|
+
---
|
|
202
|
+
|
|
203
|
+
## The name
|
|
204
|
+
|
|
205
|
+
*kibsu* (𒆠𒍑, KI.UŠ) is Akkadian, derived from *kabāsu*, "to tread":
|
|
206
|
+
|
|
207
|
+
> **1.** "tread, imprint" of a foot — *kibsu redû*, "to follow a track"; *ša kibsi*, "tracker"
|
|
208
|
+
> **2.** "track, route"; transferred, "mode of life, behaviour, course of life"
|
|
209
|
+
> **3.** mathematical: "way of calculation"
|
|
210
|
+
|
|
211
|
+
— *A Concise Dictionary of Akkadian*, Black, George & Postgate, Harrassowitz ²2000
|
|
212
|
+
|
|
213
|
+
The trace left behind, the course of conduct, and the method of reckoning. The tool is all three.
|
|
214
|
+
|
|
215
|
+
## License
|
|
216
|
+
|
|
217
|
+
MIT. See [LICENSE](LICENSE) and [PROVENANCE.md](PROVENANCE.md).
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Command-line entry point for kibsu.
|
|
2
|
+
|
|
3
|
+
Run as `python -m kibsu <command> ...`. Tier A tools (discover, index, install, tokens,
|
|
4
|
+
survey) and Tier B tools (check, report, guide, audit) are ported and registered below;
|
|
5
|
+
Tier C (learn, gate) is now fully ported too.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import importlib
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _cmd_version(args):
|
|
16
|
+
print("kibsu %s" % __version__)
|
|
17
|
+
return 0
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# Tools that carry their own argparse.ArgumentParser and handle their own -h/--help. Their
|
|
21
|
+
# argv is forwarded untouched (see _forward below) rather than re-parsed by argparse
|
|
22
|
+
# subparsers here: argparse.REMAINDER does not reliably swallow a bare "--help" as the very
|
|
23
|
+
# first token of a subparser with no other options defined, so dispatch for these is
|
|
24
|
+
# done by slicing sys.argv directly, before kibsu's own top-level parser ever runs.
|
|
25
|
+
_FORWARDED = ("discover", "index", "install", "tokens", "check", "report", "guide", "audit", "learn", "gate")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _forward(subcommand, extra_args):
|
|
29
|
+
"""Hand off to a ported tool's own main(), unmodified.
|
|
30
|
+
|
|
31
|
+
discover.py / index.py / install.py / tokens.py each carry their own
|
|
32
|
+
argparse.ArgumentParser and call parse_args() with no explicit argv, so it reads
|
|
33
|
+
sys.argv[1:]. To reuse that logic exactly as ported, sys.argv is swapped for the
|
|
34
|
+
duration of the call to `[subcommand] + extra_args`; each tool's own ArgumentParser has
|
|
35
|
+
prog= set to "python -m kibsu <subcommand>", so its own --help / usage / error output is
|
|
36
|
+
correct regardless of what sys.argv[0] happens to be under `python -m kibsu`.
|
|
37
|
+
"""
|
|
38
|
+
module = importlib.import_module("." + subcommand, __package__)
|
|
39
|
+
old_argv = sys.argv
|
|
40
|
+
try:
|
|
41
|
+
sys.argv = [subcommand] + list(extra_args)
|
|
42
|
+
return module.main()
|
|
43
|
+
finally:
|
|
44
|
+
sys.argv = old_argv
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _cmd_survey(args):
|
|
48
|
+
# survey.py has no argparse of its own (see that file) - it reads sys.argv[1] directly
|
|
49
|
+
# as an optional local-repo path, and unconditionally clones + audits ten public repos
|
|
50
|
+
# on every real run regardless of that argument. So --help is handled entirely by THIS
|
|
51
|
+
# subparser (add_help defaults to True below) and never reaches survey.main() at all;
|
|
52
|
+
# that is deliberate, not an oversight, and keeps `--help` free of network side effects.
|
|
53
|
+
from . import survey
|
|
54
|
+
old_argv = sys.argv
|
|
55
|
+
try:
|
|
56
|
+
sys.argv = ["survey", args.local] if args.local else ["survey"]
|
|
57
|
+
survey.main()
|
|
58
|
+
finally:
|
|
59
|
+
sys.argv = old_argv
|
|
60
|
+
return 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# Subcommand registry: name -> (help text, handler(args) -> exit code). Only used for
|
|
64
|
+
# commands NOT in _FORWARDED - those are dispatched earlier, in main(), by slicing argv.
|
|
65
|
+
_SUBCOMMANDS = {
|
|
66
|
+
"version": ("print the installed kibsu version", _cmd_version),
|
|
67
|
+
"discover": ("what is configured in a repo, and what actually runs", None),
|
|
68
|
+
"index": ("build a deterministic markdown index with a derived taxonomy", None),
|
|
69
|
+
"install": ("wire the check gate to git commit, reversibly (needs the check tool - "
|
|
70
|
+
"see its own --help)", None),
|
|
71
|
+
"tokens": ("model-tier subagent guard, cost ledger, and spend report", None),
|
|
72
|
+
"survey": ("clone public agent-instruction repos, audit each, print the distribution",
|
|
73
|
+
_cmd_survey),
|
|
74
|
+
"check": ("check the repo against its own index - the pre-commit gate (needs an index - "
|
|
75
|
+
"see the index tool)", None),
|
|
76
|
+
"report": ("read-only readiness report: what an agent cannot do here yet", None),
|
|
77
|
+
"guide": ("what an agent actually has to remember, vs. what a mechanism enforces", None),
|
|
78
|
+
"audit": ("measure the checkable:claimable ratio of an agent skill set", None),
|
|
79
|
+
"learn": ("does the shared knowledge base still tell the truth - dangling links, rotted "
|
|
80
|
+
"citations, orphans", None),
|
|
81
|
+
"gate": ("the commit gate: runs the commands configured under \"gates\" in .kibsu.json and "
|
|
82
|
+
"blocks only on a NEW finding (needs a baseline - see its own --help)", None),
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def build_parser():
|
|
87
|
+
parser = argparse.ArgumentParser(
|
|
88
|
+
prog="kibsu",
|
|
89
|
+
description=(
|
|
90
|
+
"Kibsu reads your coding-agent instructions (AGENTS.md, CLAUDE.md, "
|
|
91
|
+
"skills, memory) and reports which of them can actually be "
|
|
92
|
+
"verified from the repository."
|
|
93
|
+
),
|
|
94
|
+
)
|
|
95
|
+
parser.add_argument(
|
|
96
|
+
"--version",
|
|
97
|
+
action="version",
|
|
98
|
+
version="kibsu %s" % __version__,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
subparsers = parser.add_subparsers(dest="command", metavar="<command>")
|
|
102
|
+
|
|
103
|
+
for name, (help_text, _handler) in _SUBCOMMANDS.items():
|
|
104
|
+
if name == "survey":
|
|
105
|
+
sp = subparsers.add_parser(name, help=help_text)
|
|
106
|
+
sp.add_argument(
|
|
107
|
+
"local",
|
|
108
|
+
nargs="?",
|
|
109
|
+
default=None,
|
|
110
|
+
help="optional local repo to audit for comparison (this still clones and "
|
|
111
|
+
"audits public repos over the network regardless)",
|
|
112
|
+
)
|
|
113
|
+
else:
|
|
114
|
+
# For _FORWARDED names this subparser only exists so `python -m kibsu --help`
|
|
115
|
+
# lists them; real invocations are intercepted in main() before parse_args runs.
|
|
116
|
+
subparsers.add_parser(name, help=help_text)
|
|
117
|
+
|
|
118
|
+
return parser
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def main(argv=None):
|
|
122
|
+
raw = sys.argv[1:] if argv is None else list(argv)
|
|
123
|
+
|
|
124
|
+
if raw and raw[0] in _FORWARDED:
|
|
125
|
+
return _forward(raw[0], raw[1:])
|
|
126
|
+
|
|
127
|
+
parser = build_parser()
|
|
128
|
+
args = parser.parse_args(raw)
|
|
129
|
+
|
|
130
|
+
if not args.command:
|
|
131
|
+
parser.print_help()
|
|
132
|
+
return 0
|
|
133
|
+
|
|
134
|
+
_help_text, handler = _SUBCOMMANDS[args.command]
|
|
135
|
+
return handler(args)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
if __name__ == "__main__":
|
|
139
|
+
sys.exit(main())
|