repo-context-optimizer 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. repo_context_optimizer-0.1.0/LICENSE +21 -0
  2. repo_context_optimizer-0.1.0/PKG-INFO +443 -0
  3. repo_context_optimizer-0.1.0/README.md +428 -0
  4. repo_context_optimizer-0.1.0/pyproject.toml +35 -0
  5. repo_context_optimizer-0.1.0/setup.cfg +4 -0
  6. repo_context_optimizer-0.1.0/src/rco/__init__.py +3 -0
  7. repo_context_optimizer-0.1.0/src/rco/__main__.py +6 -0
  8. repo_context_optimizer-0.1.0/src/rco/build.py +88 -0
  9. repo_context_optimizer-0.1.0/src/rco/canon.py +127 -0
  10. repo_context_optimizer-0.1.0/src/rco/check.py +229 -0
  11. repo_context_optimizer-0.1.0/src/rco/cli.py +202 -0
  12. repo_context_optimizer-0.1.0/src/rco/diff.py +266 -0
  13. repo_context_optimizer-0.1.0/src/rco/extract/__init__.py +38 -0
  14. repo_context_optimizer-0.1.0/src/rco/extract/cli.py +896 -0
  15. repo_context_optimizer-0.1.0/src/rco/extract/python.py +160 -0
  16. repo_context_optimizer-0.1.0/src/rco/extract/routes.py +297 -0
  17. repo_context_optimizer-0.1.0/src/rco/extract/sql.py +755 -0
  18. repo_context_optimizer-0.1.0/src/rco/fixture.py +87 -0
  19. repo_context_optimizer-0.1.0/src/rco/forest.py +208 -0
  20. repo_context_optimizer-0.1.0/src/rco/forest_model.py +238 -0
  21. repo_context_optimizer-0.1.0/src/rco/gitread.py +97 -0
  22. repo_context_optimizer-0.1.0/src/rco/imports.py +103 -0
  23. repo_context_optimizer-0.1.0/src/rco/model.py +115 -0
  24. repo_context_optimizer-0.1.0/src/rco/modules.py +19 -0
  25. repo_context_optimizer-0.1.0/src/rco/output.py +103 -0
  26. repo_context_optimizer-0.1.0/src/rco/pack.py +266 -0
  27. repo_context_optimizer-0.1.0/src/rco/pin.py +171 -0
  28. repo_context_optimizer-0.1.0/src/rco/prose/__init__.py +60 -0
  29. repo_context_optimizer-0.1.0/src/rco/prose/check.py +54 -0
  30. repo_context_optimizer-0.1.0/src/rco/prose/llm.py +36 -0
  31. repo_context_optimizer-0.1.0/src/rco/prose/mine.py +75 -0
  32. repo_context_optimizer-0.1.0/src/rco/redact.py +198 -0
  33. repo_context_optimizer-0.1.0/src/rco/render.py +48 -0
  34. repo_context_optimizer-0.1.0/src/rco/store/__init__.py +151 -0
  35. repo_context_optimizer-0.1.0/src/rco/store/sqlite.py +201 -0
  36. repo_context_optimizer-0.1.0/src/rco/triage.py +158 -0
  37. repo_context_optimizer-0.1.0/src/rco/update.py +164 -0
  38. repo_context_optimizer-0.1.0/src/repo_context_optimizer.egg-info/PKG-INFO +443 -0
  39. repo_context_optimizer-0.1.0/src/repo_context_optimizer.egg-info/SOURCES.txt +73 -0
  40. repo_context_optimizer-0.1.0/src/repo_context_optimizer.egg-info/dependency_links.txt +1 -0
  41. repo_context_optimizer-0.1.0/src/repo_context_optimizer.egg-info/entry_points.txt +2 -0
  42. repo_context_optimizer-0.1.0/src/repo_context_optimizer.egg-info/top_level.txt +1 -0
  43. repo_context_optimizer-0.1.0/tests/test_acceptance.py +134 -0
  44. repo_context_optimizer-0.1.0/tests/test_build.py +118 -0
  45. repo_context_optimizer-0.1.0/tests/test_check.py +110 -0
  46. repo_context_optimizer-0.1.0/tests/test_check_hidden_code.py +447 -0
  47. repo_context_optimizer-0.1.0/tests/test_check_hygiene.py +378 -0
  48. repo_context_optimizer-0.1.0/tests/test_check_role_trailer.py +154 -0
  49. repo_context_optimizer-0.1.0/tests/test_diff.py +179 -0
  50. repo_context_optimizer-0.1.0/tests/test_extract_cli.py +225 -0
  51. repo_context_optimizer-0.1.0/tests/test_extract_python.py +101 -0
  52. repo_context_optimizer-0.1.0/tests/test_extract_routes.py +144 -0
  53. repo_context_optimizer-0.1.0/tests/test_extract_sql.py +224 -0
  54. repo_context_optimizer-0.1.0/tests/test_forest.py +98 -0
  55. repo_context_optimizer-0.1.0/tests/test_golden.py +29 -0
  56. repo_context_optimizer-0.1.0/tests/test_imports.py +47 -0
  57. repo_context_optimizer-0.1.0/tests/test_output.py +158 -0
  58. repo_context_optimizer-0.1.0/tests/test_pack.py +214 -0
  59. repo_context_optimizer-0.1.0/tests/test_phase0_baseline.py +92 -0
  60. repo_context_optimizer-0.1.0/tests/test_phase2_contract.py +138 -0
  61. repo_context_optimizer-0.1.0/tests/test_phase2_flow.py +41 -0
  62. repo_context_optimizer-0.1.0/tests/test_pin.py +159 -0
  63. repo_context_optimizer-0.1.0/tests/test_prose_check.py +30 -0
  64. repo_context_optimizer-0.1.0/tests/test_prose_mine.py +44 -0
  65. repo_context_optimizer-0.1.0/tests/test_prose_queue.py +77 -0
  66. repo_context_optimizer-0.1.0/tests/test_redact.py +225 -0
  67. repo_context_optimizer-0.1.0/tests/test_replay.py +72 -0
  68. repo_context_optimizer-0.1.0/tests/test_skeleton.py +38 -0
  69. repo_context_optimizer-0.1.0/tests/test_spin_run.py +167 -0
  70. repo_context_optimizer-0.1.0/tests/test_store_sqlite.py +121 -0
  71. repo_context_optimizer-0.1.0/tests/test_test_select.py +136 -0
  72. repo_context_optimizer-0.1.0/tests/test_triage.py +90 -0
  73. repo_context_optimizer-0.1.0/tests/test_update.py +105 -0
  74. repo_context_optimizer-0.1.0/tests/test_update_props.py +242 -0
  75. repo_context_optimizer-0.1.0/tests/test_wheel_smoke.py +53 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 si4ai
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,443 @@
1
+ Metadata-Version: 2.4
2
+ Name: repo-context-optimizer
3
+ Version: 0.1.0
4
+ Summary: Deterministic repo context for coding agents: contract Trees, contract diffs, and a map checker.
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/si4ai/repo-context-optimizer
7
+ Classifier: Programming Language :: Python :: 3
8
+ Classifier: Programming Language :: Python :: 3.12
9
+ Classifier: Operating System :: OS Independent
10
+ Classifier: Development Status :: 2 - Pre-Alpha
11
+ Requires-Python: >=3.12
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Dynamic: license-file
15
+
16
+ # repo-context-optimizer
17
+
18
+ Deterministic repo context for coding agents: contract Trees, an architecture Forest, contract diffs, and rebase triage. The goal is to replace LLM tokens with code wherever a contract can be computed.
19
+
20
+ **Status:** 0.1.0 is the first installable release (phase 2). `pip install repo-context-optimizer` installs the `rco` command. Trees, the map checker, `rco diff`, `rco update`, `rco pack`, and `rco forest` work. Mined prose is on; the LLM path is off. The release is rough on purpose: Python import edges only, package-grain modules, no override file, no mill backend, no triage, and no A/B claim. Later phases remain design.
21
+
22
+ **Team:** Builder and grok-tui split the development work, Auditor checks the design and models, and Validator passes and merges each PR. A docs desk joins once there is something to document.
23
+
24
+ ### If you are an agent
25
+
26
+ Run `rco pack --for <files you will touch>`.
27
+ Read the pack.
28
+ Open raw files only on demand.
29
+
30
+ ## Why
31
+
32
+ Cold-start agents waste tokens rediscovering a repository: they read too many files, guess at module boundaries, and re-learn contracts that code already knows. Parallel agents on a busy repo pay again in rebase churn, often rereading full diffs when only a contract changed, or waking an agent when no contract they use has moved.
33
+
34
+ RCO's rule: **if code can compute it, code computes it.** CPU, memory, and disk are cheaper than tokens. LLM use is minimized and optional. `rco` is designed to work with zero tokens.
35
+
36
+ ## Just works
37
+
38
+ The default in 0.1.0:
39
+
40
+ ```
41
+ pip install repo-context-optimizer # installs the `rco` command
42
+ cd any-git-repo
43
+ rco pack # SHA=HEAD; no --for: whole-repo Forest only; --budget 8000
44
+ rco pack --for src/cli.py # Forest + Trees for those modules and one import hop
45
+ rco diff # BASE=merge-base(default branch, HEAD); HEAD=HEAD
46
+ ```
47
+
48
+ `--budget` is in estimated tokens. The estimator is approximate (characters / 4). The design default is 8000. `--for` is optional. These defaults match the [Planned commands](#planned-commands) section.
49
+
50
+ Zero config is the goal: the SQLite backend is the default, with no `rco.toml` required. Cache lives in SQLite under `.rco/`, which rco adds to `.git/info/exclude`, never to `.gitignore`. Modules are detected automatically. Parsers for common languages come bundled. An unknown language falls back to a file outline instead of an error. No LLM, keys, or network are required.
51
+
52
+ `rco.toml`, other backends, adapters, and MCP are all opt-in. If a configured backend fails to load, rco falls back to SQLite and prints the exact command to fix it.
53
+
54
+ ### Acceptance tests (zero-config goal)
55
+
56
+ | Repo | Expected behavior |
57
+ |---|---|
58
+ | Small Python repo | Modules detected from the package layout; Trees extracted; `rco pack` and `rco diff` run with the defaults above. |
59
+ | Polyglot repo | Bundled parsers used where available; unknown languages fall back to a file outline, not an error. |
60
+ | Large monorepo | Pack respects `--budget`; `rco update` re-extracts only modules whose tree ids changed. Cost and hit rate are Phase 0 measurements. |
61
+ | Repo with submodules | Submodules are skipped, with a note in the output. |
62
+ | LFS repo | LFS pointer files are treated as opaque: not fetched, not parsed as the real blob. |
63
+ | Empty repo | Empty pack, exit 0. |
64
+
65
+ ## Concepts
66
+
67
+ ```mermaid
68
+ flowchart LR
69
+ R[repo at a SHA] --> M[Modules]
70
+ M -->|parse, no LLM| T[Trees<br/>micro-contracts]
71
+ T -->|import graph| F[Forest<br/>macro architecture]
72
+ F -.->|optional, lazy| P[Forest prose]
73
+ T --> K[rco pack]
74
+ F --> K
75
+ P -.-> K
76
+ ```
77
+
78
+ ### Module
79
+
80
+ A module is a logical unit of the repo, not necessarily a directory.
81
+
82
+ Detection order:
83
+
84
+ 1. Language manifests first (`pyproject.toml`, `package.json`, `Cargo.toml`, `go.mod`, `pom.xml`, and similar).
85
+ 2. Import-graph clustering (for example Louvain community detection) when packages are not already explicit.
86
+ 3. Directory fallback for whatever remains, with size caps (for example a maximum file count or line count per module) so summaries stay small enough to be useful.
87
+
88
+ A checked-in override file can pin boundaries; the detector only falls back to heuristics where that file is silent. A check reports any file that belongs to no module. The override file format is still an open question.
89
+
90
+ ### Tree (micro-contracts)
91
+
92
+ A Tree is a deterministic contract extracted from a module. It is never written by an LLM.
93
+
94
+ - **Code contracts:** language-native skeletons with function and method bodies stripped, via Python's `ast` module and tree-sitter for other languages. Public classes and functions, signatures, type hints, the first docstring line, constants, and `__all__` (or the language's equivalent).
95
+ - **Non-code contracts,** which often matter more across a repo than signatures alone: CLI commands and flags, SQL schema tables and columns, HTTP routes, and message or event shapes.
96
+
97
+ Output is canonical (sorted JSON, no line numbers), so identical input always yields byte-identical output. The Tree hash is the sha256 of that JSON.
98
+
99
+ ### Forest (macro architecture)
100
+
101
+ A Forest describes how modules fit together. Most of it is computed, not generated: layer; depends-on and used-by, from the import graph; entry points; and the commands, tables, and routes a module owns.
102
+
103
+ A small, optional LLM prose part may add purpose, invariants, and gotchas. rco works with the prose turned off; generation rules live under [Phase 2](#roadmap). A repo-level Forest can render the module graph (for example as a mermaid diagram) plus a short paragraph per layer.
104
+
105
+ ## Fingerprints
106
+
107
+ Hashes are keyed by content, not by commit.
108
+
109
+ - **ModuleHash** = the module's own content (git's tree object id for the module's paths, or sha256 of its file hashes) plus the extractor version. **The commit SHA is not part of this key:** including it would invalidate every module on every merge.
110
+ - **TreeHash** = sha256 of the canonical Tree JSON.
111
+ - **ForestHash** = TreeHash plus prompt version plus model id, so a prompt or model change invalidates only the prose.
112
+ - A separate index maps `commit SHA -> module hashes`. A new commit mostly points at existing cache entries. A lookup by tree hash on a miss still resolves identical module states across rebased histories.
113
+
114
+ Generated summaries are not committed to the default branch. Generated files in the repo would themselves cause rebase conflicts, which is the problem this tool is meant to avoid.
115
+
116
+ ## Incremental update after each merge
117
+
118
+ The cost of each merge is meant to scale with what the merge changed, not with the size of the repo. Git already says which files moved. How many modules a typical merge touches, and how long a re-extract takes, are Phase 0 measurements.
119
+
120
+ ```mermaid
121
+ flowchart TD
122
+ A[merge lands: OLD to NEW] --> B["git diff --name-only OLD NEW<br/>map files to modules"]
123
+ B --> C{module git tree id<br/>changed?}
124
+ C -->|no| S[skip module]
125
+ C -->|yes| D[re-extract Tree<br/>parse, no LLM]
126
+ D --> E{TreeHash changed?}
127
+ E -->|no, internal change| G[copy Forest forward to NEW]
128
+ E -->|yes, contract moved| H[recompute import edges<br/>Forest now reads as stale]
129
+ S --> I
130
+ G --> I
131
+ H --> I{every module of NEW<br/>in the store?}
132
+ I -->|no| D
133
+ I -->|yes| J["publish pointer<br/>one atomic compare-and-swap"]
134
+ J --> K[queue debounced prose rewrites]
135
+ J --> L[contract diff, rco triage hints,<br/>contract-change notices]
136
+ ```
137
+
138
+ The same update in order, showing where the four SPIN rules sit:
139
+
140
+ ```mermaid
141
+ sequenceDiagram
142
+ participant M as merge
143
+ participant U as rco update
144
+ participant S as store
145
+ participant P as pointer
146
+ participant Q as prose queue
147
+ M->>U: OLD to NEW
148
+ U->>S: write Trees for changed modules
149
+ U->>S: check every module of NEW is present
150
+ Note over U,S: rule 2: complete SHAs only
151
+ U->>P: compare-and-swap OLD to NEW
152
+ Note over U,P: rule 1: one atomic pointer write
153
+ alt pointer still reads OLD
154
+ P-->>U: swapped to NEW
155
+ U->>Q: queue prose rewrites
156
+ Note over U,Q: rule 4: prose after the pointer
157
+ else pointer already moved past OLD (late finisher)
158
+ P-->>U: swap refused
159
+ Note over U,P: write nothing, the newer update already covers NEW or later
160
+ end
161
+ Note over P,Q: rule 3: packs compute staleness from TreeHashes
162
+ ```
163
+
164
+ Staleness is never stored: a pack sees a stale Forest because its TreeHash no longer matches the current Tree's. Several merges in a short window share one prose rewrite per module.
165
+
166
+ Properties, checked by the SPIN model in `verify/spin/` (see [Verification](#verification)):
167
+
168
+ - **The pointer write is one atomic compare-and-swap:** move `main` from OLD to NEW only if it still reads OLD and NEW descends from it (for example `git update-ref` with the old value, or a conditional database update). A separate "check, then write" is not enough: two updaters can both pass the check, and a late update then moves the pointer backwards.
169
+ - **Publish only a complete SHA:** before the pointer write, check every module of the new SHA against the store, not only the modules in the diff. Otherwise a slow or crashed earlier update leaves a module unbuilt and the pointer moves past it for good.
170
+ - **Staleness is computed, not stored:** a pack compares each Forest's TreeHash with the current Tree's. A stored stale flag can be left looking fresh by a race.
171
+ - **Queue the prose rewrite after the pointer write,** so a rewrite never describes a Tree that is already out of date and leaves its Forest stale for good.
172
+ - Idempotent and content-addressed: running the same update twice equals running it once. There is no hold: mill holds do not auto-expire, so a lock taken for regeneration could be left stuck.
173
+ - The new SHA's pointer moves as soon as its Trees are complete. Forest prose catches up asynchronously. A stale Forest is shown as stale, not as if it were current.
174
+
175
+ Triggers: a git `post-merge` hook, a CI job on push to the default branch, or a merge webhook.
176
+
177
+ ## Planned commands
178
+
179
+ `rco tree`, `rco check`, and `rco diff` have landed; see [Just works](#just-works). The [output contract](#output-contract) applies to every shipped command. `rco triage` and `rco notice` (phase 5, hints only) have landed. `rco pack` and `rco update` are not implemented yet. Phase 4 adds `rco pin` and `rco drift` (pinned contracts, a report and never a block; see `docs/design/pin-contract.md`). Defaults match the [quick start](#just-works).
180
+
181
+ ### `rco pack [SHA] [--for FILES] [--budget N]`
182
+
183
+ - `SHA` defaults to `HEAD`.
184
+ - `--for` is optional. If omitted, the pack is the whole-repo Forest only. If present, the pack is the repo Forest, full Trees for the named modules, and Trees one import hop out; anything beyond that is trimmed to its name and one line.
185
+ - `--budget N` is in estimated tokens (characters / 4, approximate). Design default: 8000.
186
+
187
+ The pack is the map; raw files are fetched on demand.
188
+
189
+ Which command reads and writes what:
190
+
191
+ ```mermaid
192
+ flowchart LR
193
+ G[git history] --> UP[rco update]
194
+ UP -->|Trees, Forest, pointer| ST[(store)]
195
+ ST --> PK[rco pack]
196
+ ST --> DF[rco diff]
197
+ G --> DF
198
+ DF -->|contract changes| TR[rco triage]
199
+ B[open branches] --> TR
200
+ PK --> A[agent]
201
+ TR -->|clean or structural brief| A
202
+ ```
203
+
204
+ #### Pack output (example)
205
+
206
+ This sample is about 120 estimated tokens (characters / 4; the estimator is approximate).
207
+
208
+ ```markdown
209
+ # rco pack (SHA=abc1234, budget=8000 estimated tokens)
210
+
211
+ ## Forest
212
+ - tw: desk CLI and process runner (depends on: proc, client)
213
+ - server: HTTP API and schema (used by: tw, wall)
214
+ - wall: browser UI
215
+
216
+ ## Trees (full, for --for tw/cli.py)
217
+ ### tw
218
+ def run(argv: list[str]) -> int
219
+ def stop_run(run_id: str) -> None
220
+ CLI: tw run, tw stop
221
+
222
+ ## One hop (name and one line)
223
+ - proc: process supervision for tw runs
224
+ - client: shared client library
225
+
226
+ # budget used: ~120 / 8000 estimated tokens
227
+ ```
228
+
229
+ ### `rco diff [BASE] [HEAD]`
230
+
231
+ `BASE` defaults to the merge-base of the repository's default branch and `HEAD`. `HEAD` defaults to `HEAD`.
232
+
233
+ Lists added, removed, and changed signatures, flags, columns, and routes. For each change it lists callers as `file:line` from a reverse index.
234
+
235
+ ### `rco triage`
236
+
237
+ Triage never rebases, merges, or pushes anyone's branch. For each open branch it reports either `clean: no changed file, no changed contract used` or a structural brief, as a hint. The branch owner or a merge-main step acts on that hint.
238
+
239
+ Usage: `rco triage BRANCH... [--onto TARGET]` (TARGET defaults to the default branch). The cross-PR contract notice is `rco notice BRANCH... [--base BASE] [--head HEAD]`: it lists which other open branches change files that use a contract this PR changes. It names branches, not desks, and always exits 0, so a CI job running it is a visible hint, never a block. Both see Python callers only; CLI, SQL, and route changes show up as files changed on both sides.
240
+
241
+ An unchanged contract can still hide a behavior change, so tests still run. The hypothesis, to be measured in Phase 5, is that many branches are contract-clean and need no structural update.
242
+
243
+ ### `rco update`
244
+
245
+ The incremental post-merge updater described above.
246
+
247
+ ### Structural brief (example)
248
+
249
+ The brief is generated from the contract diff and the caller index. It is not written by a model.
250
+
251
+ ```markdown
252
+ ### STRUCTURAL UPDATE REQUIRED
253
+ Module `auth_service` in the target branch updated its interface contracts.
254
+
255
+ #### Contract changes (upstream):
256
+ - REMOVED: `validateToken(token: string): boolean`
257
+ + ADDED: `validateToken(token: string, options?: AuthOptions): Promise<AuthResult>`
258
+
259
+ #### Impacted files in your branch:
260
+ - `src/controllers/user_controller.ts` (calls `validateToken`)
261
+
262
+ #### Task:
263
+ Update callers in `src/controllers/user_controller.ts` to match the new `AuthResult` signature.
264
+ Do not re-read or modify unaffected modules.
265
+ ```
266
+
267
+ ## Output contract
268
+
269
+ Output goes to stdout by default, or to `--out FILE`. `--format markdown` (default) or `--format json`.
270
+
271
+ Exit codes, so agents can script the commands:
272
+
273
+ | Code | Meaning |
274
+ |---|---|
275
+ | 0 | Ok (clean pack, no contract findings, triage all clean). |
276
+ | 1 | Findings (contract changed, Forest stale, or triage emitted a structural brief). `rco notice` never exits 1. |
277
+ | 2 | Usage error. |
278
+ | 3 | Internal error. |
279
+
280
+ ## Secrets
281
+
282
+ Before any pack output or storage, rco runs a redaction and hygiene pass over extracted constants, docstrings, and other literals. Redacted values are replaced with a placeholder and are never written through. What is redacted:
283
+
284
+ - Private keys and PEM blocks
285
+ - Bearer tokens, API keys, passwords, and other high-entropy secret-shaped strings
286
+ - Connection strings and URLs that embed credentials
287
+ - Credential-like assignments (for example `password = "..."`, `api_key = "..."`)
288
+
289
+ Signatures, type names, route paths, flag names, and table or column names are kept. Public identifiers are not treated as secrets.
290
+
291
+ ## Storage backends
292
+
293
+ Storage sits behind one small interface, chosen in `rco.toml`:
294
+
295
+ ```toml
296
+ backend = "sqlite" | "mill" | "mcp" | "<entry point>"
297
+ ```
298
+
299
+ The five calls are: put a node, get a node, list nodes by SHA, set a pointer (like a label), and post a notice (a stale Forest or an affected PR). Notices are optional; a backend that can only store still works, and notices go to stdout.
300
+
301
+ ```mermaid
302
+ flowchart LR
303
+ rco[rco] --> I["storage interface<br/>put node, get node, list by SHA,<br/>set pointer, post notice"]
304
+ I --> Q[(sqlite: default<br/>one database under .rco/)]
305
+ I --> ML[mill: optional extra<br/>hashes, pointers, scores]
306
+ I --> MC[mcp: five calls as tools<br/>on a named server]
307
+ I --> AD[adapter: Python entry point<br/>pip install rco-whatever]
308
+ ```
309
+
310
+ An adapter is a small Python class that implements the five calls and registers through an entry point, so installing it makes it show up with no change to rco. The `mcp` backend maps the same five calls to tools on a server the user names, so any shared-memory tool that exposes MCP works with no Python adapter.
311
+
312
+ A conformance test suite is the check that an adapter is honest. Every backend runs it against itself.
313
+
314
+ The SQLite backend (`sqlite`) is the default: one database under `.rco/`. Tree and Forest JSON live in this content store (or another backend's equivalent). They are not copied into mill cards.
315
+
316
+ The mill backend is an optional extra (`pip install repo-context-optimizer[mill]`) that talks to the mill (si4ai/the-mill). It calls only existing mill commands; it adds no new mill verbs.
317
+
318
+ - **Content:** Tree and Forest JSON stay in the SQLite content store. The mill holds only hashes and pointers. There is no mill tree node per module per SHA. Mill trees outline mail and todo. Card text has a 4096-character budget.
319
+ - **label** holds pointers such as `rco/main` or `rco/0.8.11`. A new label needs `mill create label` (Admin) on limited trust.
320
+ - **Staleness** is a score, `stale Forests outstanding`, and optionally one rollup card per repo. Not a card per module. Cards start only when a human asks.
321
+ - **mail** may carry a contract-diff notice. Mapping a PR to a desk requires a per-desk GitHub identity, which is not available while desks share one account. Letters are batched, one per affected PR, and addressed; every wake costs tokens.
322
+ - **tw** is the mill's recorded command runner. From a desk's sitting `mill connect DESK`, `tw rco update` records the run: start, finished, and failed.
323
+ - **score** also tracks LLM tokens rco itself spends (when prose is on) and stale Forests outstanding. Tokens and reads before the first useful action are not produced by rco; see Metrics.
324
+
325
+ ## Metrics
326
+
327
+ Claim no savings until they are measured against a baseline.
328
+
329
+ - Tokens and reads before the first useful action, and wrong guesses on cold runs, come from cold-agent logs, not from rco itself. Phase 0 uses the mill's cold-test kit and an existing baseline.
330
+ - Branches triage reported clean (hypothesis, measured in Phase 5). Tests still run, because an unchanged contract can hide a behavior change.
331
+ - LLM tokens spent per module and per merge, when prose is on (from rco's own logs).
332
+ - Stale Forests outstanding (score).
333
+
334
+ ## Roadmap
335
+
336
+ Development order: a reliable map first, then pins and drift checks, then rebase and merge tools built on the map.
337
+
338
+ ```mermaid
339
+ flowchart LR
340
+ P0[Phase 0<br/>baseline] --> P1[Phase 1<br/>Trees, diff,<br/>map checker]
341
+ P1 --> P2[Phase 2<br/>Forest,<br/>optional prose]
342
+ P2 --> P3[Phase 3<br/>rco pack<br/>A/B on cold agents]
343
+ P3 --> P4[Phase 4<br/>pinned contracts]
344
+ P4 --> P5[Phase 5<br/>triage hints]
345
+ P5 --> P6[mill backend,<br/>MCP adapter]
346
+ ```
347
+
348
+ ### First slices
349
+
350
+ Each slice lands before the next starts, and each has a done-when that a check can confirm.
351
+
352
+ ```mermaid
353
+ flowchart TD
354
+ S1["1. Empty repo<br/>README, role-only hygiene and role-trailer checks, CI, test selection (ci/test-select.toml), verify/spin/"] --> S2
355
+ S2["2. Map checker<br/>planted-drift self-test, recorded time limit"] --> S3
356
+ S3["3. Pointer writer<br/>atomic CAS, complete SHAs, computed staleness"] --> S4
357
+ S4["4. Forest prose<br/>queued after the pointer write"] --> S5
358
+ S5["5. rco update and rco pack<br/>end to end on this repo and the mill"]
359
+ ```
360
+
361
+ | Slice | Done when |
362
+ |---|---|
363
+ | 1. Empty repo | CI is green on `main` and selects jobs from [`ci/test-select.toml`](ci/test-select.toml); the hygiene check fails a planted personal name in a test PR; the role-trailer check fails a commit with no known role, so the self-merge check works from the first PR; a desk can take a card from worktree to PR. |
364
+ | 2. Map checker | It lists every mismatch (missing, extra, changed) with its path and exits nonzero on any; output is byte-identical for the same inputs; the planted-drift self-test fails if any drift is missed; a time limit is recorded with the RCO SHA. |
365
+ | 3. Pointer writer | The SPIN model is committed beside the code and passes all seven properties; a crash between store write and pointer write never publishes a partial SHA. |
366
+ | 4. Forest prose | A test shows no prose is ever written for a SHA the pointer has not published. |
367
+ | 5. Update and pack | `tw rco update` runs after each merge on this repo and the mill and publishes the merge SHA; `rco pack --for FILES` returns a pack within `--budget`; this is where dogfooding starts. |
368
+
369
+ Until slice 5 lands, the pipeline ends at merge; from slice 2 on, the map checker reports drift read-only on this repo and the mill.
370
+
371
+ - **Phase 0.** Baseline, using the mill's cold-test kit and an existing baseline: tokens and reads before the first useful action, plus wrong guesses, from cold-agent logs. Also measure extract cost and how many modules a typical merge touches.
372
+ - **Phase 1.** Trees, hashes, and `rco diff` for Python, the CLI, SQL, and routes. Zero LLM. Deterministic tests: same SHA in, same bytes out. Redaction pass and the output contract (stdout/`--out`, format flags, exit codes). The map checker (see [Verification](#verification)) ships with the first Tree builder, so the first map anyone reads comes with a measured accuracy.
373
+ - **Phase 2.** The Forest, computed first, with optional prose, plus staleness and name-in-Tree checks. When prose is on, it is generated lazily from the Tree (not raw source), the first time an agent asks for it in a pack; mined first from existing docstrings, module headers, and READMEs, with the model called only when nothing useful exists; written to a fixed template with a hard length cap, using a cheap model; rewritten only when that module's contract hash changes, never on internal edits, and debounced across merges; and checked by code: any name the prose cites must exist in the Tree, or the entry is flagged.
374
+ - **Phase 3.** `rco pack`, A/B tested on cold agents.
375
+ - **Phase 4.** Pinned contracts: a developer reviews a Tree and commits it as the intended contract; every PR re-extracts and reports drift from the pin as a visible check, not a block. The map then works both ways: it describes the code, and the code is checked against it.
376
+ - **Phase 5.** `rco triage` as a hint (never rebases or pushes), plus a cross-PR contract notice in CI (a visible hint, not a block), subject to the identity constraint above. Rebase and merge tools come only after the map is reliable.
377
+ - Then the mill backend and the MCP adapter.
378
+
379
+ Language: Python first, so `pip install repo-context-optimizer` just works. Port only a hot path if measurement shows one. The LLM Forest comes after the deterministic diff and triage, because those are meant to save tokens with zero LLM spend.
380
+
381
+ Dynamic dispatch (command tables, `getattr`, and similar registries) will need custom extractors, or the CLI Tree will be incomplete. The Forest prose will drift unless the staleness signal is actually read.
382
+
383
+ ## Verification
384
+
385
+ Three layers, the same ones the mill uses for its own protocol.
386
+
387
+ ```mermaid
388
+ flowchart LR
389
+ V1["1. Is the design right?<br/>SPIN model, verify/spin/"] --> V2["2. Does the code follow it?<br/>map checker, property tests,<br/>history replay"]
390
+ V2 --> V3["3. Does it run?<br/>smoke tests, exit codes,<br/>timing, cold-agent A/B"]
391
+ ```
392
+
393
+ 1. **Is the design right?** A SPIN (or TLA+) model of `rco update` with racing merges, a late update, a crash midway, and a rebased history. Properties: the pointer only moves forward (a new SHA must descend from the current one, checked and written in one atomic compare-and-swap); a published pointer always has its Trees; a stale Forest is never shown as current; an update run twice equals one run; no changed module is skipped; every Forest eventually becomes fresh. Status: modelled in `verify/spin/rco_update.pml` (linear history, two modules, two updaters, one crash, one duplicate). The design as first written failed all seven properties; with the four rules above it passes all seven. Rebased history and a lossy store are not modelled yet.
394
+ 2. **Does the code follow the design, and is the map true?** The map checker takes a repo, a SHA, and a map, and reports each claim as true, false, or missing, with no LLM:
395
+ - Contracts: every function, flag, table, column, and route in a Tree exists at that SHA with that shape, checked by a fresh parse and by real outputs (`--help`, the live schema, the route table). It also reports what the Tree missed.
396
+ - Edges: every depends-on and used-by link matches the import graph at that SHA.
397
+ - Freshness: the map's SHA and module hashes match the code.
398
+ - Prose names: every name the Forest prose cites exists in the Tree.
399
+ - The checker checks itself: maps with planted errors (a removed flag, a renamed column, a missing edge) must all be flagged.
400
+ Alongside it: property tests (Hypothesis) that mirror the model's properties over random edit and merge sequences; a history replay where incremental updates must equal a full rebuild at every SHA of the default branch; and golden fixture repos with known Trees and Forests.
401
+ 3. **Does it run?** The zero-config acceptance table as smoke tests, the exit-code contract, timing and budget on the replay, and the cold-agent A/B, which is the only evidence that the map helps.
402
+
403
+ When `--budget` trims a pack, the pack names what it trimmed.
404
+
405
+ ## Open questions
406
+
407
+ - **PyPI package name.** Publish as `repo-context-optimizer` (installs the `rco` command). It was unclaimed on PyPI on 2026-11-09; the first release claims it.
408
+ - **Which prose pieces truly need an LLM.** Find out empirically: ship Phase 1 with no LLM, then see what cold agents still ask for or get wrong.
409
+ - **Module override file format.**
410
+ - **Default `--budget`.** 8000 estimated tokens is the design default; confirm against Phase 0 pack sizes.
411
+
412
+ ## Proposed: merge planning (mill tool, scope TBD)
413
+
414
+ **Status:** a proposal. The human and the desks will discuss scope before any work starts.
415
+
416
+ **Where it lives:** in the mill, as a host report plus a desk CLI command. The optimizer later feeds it a file- and function-level map in place of the plain diffs.
417
+
418
+ **What it does:**
419
+
420
+ - For open PRs, or a card's planned file list, it reports which ones collide on the same files or lines.
421
+ - It groups the rest into lanes that can land in any order.
422
+ - It suggests a merge order and names which PR needs a rebase after each merge.
423
+ - It previews collisions before code is written.
424
+
425
+ **Design:** deterministic, with no LLM. Data gathering is kept separate from the conflict graph so the optimizer's map can plug in later. It is a visible check or hint, never a hard block.
426
+
427
+ **Quick win:** generate the CHANGELOG at release time, so PRs stop conflicting on it.
428
+
429
+ **Possible split:** Builder takes the server report and the graph; grok-tui takes the CLI. The two agree on the JSON shape first.
430
+
431
+ **Open questions for the scope discussion:** line-level or function-level granularity; whether it is advisory only or wired into Validator; the CHANGELOG format.
432
+
433
+ ## Prior art
434
+
435
+ AST skeletonizers, ctags, tree-sitter, Aider's repo map, and various repo packers already extract structure or pack context. RCO's difference is content-addressed incremental Trees and Forest, a contract diff, rebase triage that reports a hint rather than touching branches, and pluggable storage.
436
+
437
+ ## Process and GitHub setup
438
+
439
+ This repository dogfoods a portable process: checks over rules, report-only CI reports that show facts, and an independent Validator that passes and merges. A hidden-code report (`tools/check_hidden_code.py`) runs on every PR and push. It exits 0 on findings; a crash fails the job. The recipe for pointing the same process at a user repository is in [`docs/github-setup.md`](docs/github-setup.md).
440
+
441
+ ## License
442
+
443
+ MIT. See [LICENSE](LICENSE).