quirework 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- quirework/__init__.py +3 -0
- quirework/__main__.py +3 -0
- quirework/cli.py +431 -0
- quirework/core/__init__.py +1 -0
- quirework/core/artifact_paths.py +86 -0
- quirework/core/checks.py +235 -0
- quirework/core/frontmatter.py +83 -0
- quirework/core/globs.py +45 -0
- quirework/core/pinned.py +169 -0
- quirework/core/usage.py +604 -0
- quirework/delivery/__init__.py +1 -0
- quirework/delivery/artifacts.py +210 -0
- quirework/delivery/text.py +166 -0
- quirework/manuscript/__init__.py +1 -0
- quirework/manuscript/backfill.py +1972 -0
- quirework/manuscript/editions.py +111 -0
- quirework/manuscript/footprint.py +95 -0
- quirework/manuscript/formats.py +208 -0
- quirework/manuscript/lifecycle.py +191 -0
- quirework/manuscript/notes.py +2301 -0
- quirework/manuscript/pages.py +284 -0
- quirework/manuscript/quarto.py +416 -0
- quirework/manuscript/quotes.py +512 -0
- quirework/manuscript/rename.py +309 -0
- quirework/manuscript/sources.py +286 -0
- quirework/manuscript/trail.py +1875 -0
- quirework/manuscript/units.py +416 -0
- quirework/project.py +645 -0
- quirework/provenance/__init__.py +1 -0
- quirework/provenance/agyfilter.py +436 -0
- quirework/provenance/backup/__init__.py +469 -0
- quirework/provenance/backup/base.py +290 -0
- quirework/provenance/backup/command.py +107 -0
- quirework/provenance/backup/directory.py +142 -0
- quirework/provenance/backup/git.py +150 -0
- quirework/provenance/backup/rclone.py +153 -0
- quirework/provenance/capture/__init__.py +1 -0
- quirework/provenance/capture/commands.py +28 -0
- quirework/provenance/capture/extract_artifacts.py +110 -0
- quirework/provenance/capture/extract_files.py +118 -0
- quirework/provenance/capture/fetch_conversation.py +127 -0
- quirework/provenance/capture/fetch_gemini.py +149 -0
- quirework/provenance/capture/render_transcript.py +126 -0
- quirework/provenance/capture/transform.py +22 -0
- quirework/provenance/catalog.py +733 -0
- quirework/provenance/ccfilter.py +1126 -0
- quirework/provenance/codexfilter.py +883 -0
- quirework/provenance/coldread.py +1731 -0
- quirework/provenance/curate.py +67 -0
- quirework/provenance/humaninput.py +811 -0
- quirework/provenance/ingest.py +31 -0
- quirework/provenance/ingestion/__init__.py +1 -0
- quirework/provenance/ingestion/agy.py +192 -0
- quirework/provenance/ingestion/claude.py +493 -0
- quirework/provenance/ingestion/codex.py +296 -0
- quirework/provenance/lineage.py +148 -0
- quirework/provenance/metrics.py +1247 -0
- quirework/provenance/rulings.py +478 -0
- quirework/provenance/scope.py +73 -0
- quirework/provenance/search.py +455 -0
- quirework/provenance/semantic.py +475 -0
- quirework/provenance/show.py +1213 -0
- quirework/provenance/store.py +1271 -0
- quirework/provenance/trees.py +133 -0
- quirework/provenance/worktrees.py +199 -0
- quirework/providers/__init__.py +1 -0
- quirework/providers/blind.py +2192 -0
- quirework/providers/canary.py +685 -0
- quirework/providers/catalog.py +44 -0
- quirework/providers/claude_login.py +92 -0
- quirework/providers/exits.py +196 -0
- quirework/providers/harness.py +830 -0
- quirework/providers/images.py +92 -0
- quirework/providers/models.py +1284 -0
- quirework/providers/quota.py +588 -0
- quirework/providers/reads.py +474 -0
- quirework/providers/replies.py +175 -0
- quirework/publish/__init__.py +1 -0
- quirework/publish/backends.py +112 -0
- quirework/publish/bookstage.py +1086 -0
- quirework/publish/build.py +60 -0
- quirework/publish/editions.py +48 -0
- quirework/publish/epubcheck.py +287 -0
- quirework/publish/experiment.py +1097 -0
- quirework/publish/figfonts.py +234 -0
- quirework/publish/fignumbers.py +293 -0
- quirework/publish/figures.py +729 -0
- quirework/publish/folios.py +62 -0
- quirework/publish/fonts.py +307 -0
- quirework/publish/graybuild.py +73 -0
- quirework/publish/hugo.py +117 -0
- quirework/publish/kindle.js +411 -0
- quirework/publish/kindle.py +1393 -0
- quirework/publish/panelkit.py +918 -0
- quirework/publish/plans.py +651 -0
- quirework/publish/proof.py +792 -0
- quirework/publish/redpen.py +1654 -0
- quirework/publish/render.py +942 -0
- quirework/publish/targets.py +216 -0
- quirework/publish/variant.py +918 -0
- quirework/runs/__init__.py +1 -0
- quirework/runs/__main__.py +15 -0
- quirework/runs/commands.py +278 -0
- quirework/runs/coverage.py +86 -0
- quirework/runs/engine.py +2143 -0
- quirework/runs/feed.py +853 -0
- quirework/runs/observe.py +379 -0
- quirework/runs/storage.py +155 -0
- quirework/settings.py +492 -0
- quirework/sops/__init__.py +1 -0
- quirework/sops/catalog.py +386 -0
- quirework/sops/commands.py +567 -0
- quirework/sops/composition.py +382 -0
- quirework/sops/instruments.py +184 -0
- quirework/sops/personas.py +296 -0
- quirework/sops/review.py +2568 -0
- quirework/sops/room.py +2791 -0
- quirework/sops/roundtable.py +2578 -0
- quirework/sops/sections.py +169 -0
- quirework/sops/workshop.py +1629 -0
- quirework/templates/backup.toml +39 -0
- quirework/templates/book.toml +88 -0
- quirework/templates/run.toml +32 -0
- quirework/templates/site.toml +90 -0
- quirework/templates/tells.toml +64 -0
- quirework/templates/terminology.toml +20 -0
- quirework/templates/tmpfiles.conf +42 -0
- quirework/testing.py +345 -0
- quirework/text/__init__.py +1 -0
- quirework/text/jargon.py +545 -0
- quirework/text/mdwrap.py +1001 -0
- quirework/text/tells.py +547 -0
- quirework-0.1.0.dist-info/METADATA +444 -0
- quirework-0.1.0.dist-info/RECORD +138 -0
- quirework-0.1.0.dist-info/WHEEL +5 -0
- quirework-0.1.0.dist-info/entry_points.txt +2 -0
- quirework-0.1.0.dist-info/licenses/LICENSE +21 -0
- quirework-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1731 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Content-addressed cache for the authorship cold-read analysis.
|
|
3
|
+
|
|
4
|
+
The expensive step in the cold-read is a *blind* sub-agent reading one
|
|
5
|
+
transcript and writing a report. The corpus is almost entirely append-only
|
|
6
|
+
(the web conversations are frozen; a finished cc-session never changes), so a
|
|
7
|
+
refresh should only (re)analyze sources whose analysis is missing or stale.
|
|
8
|
+
|
|
9
|
+
This indexes each source by a cache key and tracks which have a current
|
|
10
|
+
report::
|
|
11
|
+
|
|
12
|
+
key = sha256(source bytes) + filter_version_for(source) + PROMPT_VERSIONS[scope]
|
|
13
|
+
(one recipe version per *reader*, not one for the corpus, so
|
|
14
|
+
bumping the Codex parser leaves Claude Code reports alone;
|
|
15
|
+
a bump revalidates against the recorded render hash first --
|
|
16
|
+
a recipe change that did not alter what the analyst read
|
|
17
|
+
does not re-buy the report. The source hash still gates first,
|
|
18
|
+
with one exception: a Claude Code capture that gained its
|
|
19
|
+
session's sub-agents, whose render is unchanged by construction)
|
|
20
|
+
|
|
21
|
+
Anything that changes what an analysis *means* -- the source content, the
|
|
22
|
+
filter recipe, or the analyst prompt -- changes the key and misses the cache
|
|
23
|
+
deliberately. The analyst **model** is recorded per source as provenance but
|
|
24
|
+
is not part of the key: model choice is an economic per-run decision (Sonnet
|
|
25
|
+
for most, Opus where depth is worth the tokens), and a mixed-model corpus
|
|
26
|
+
must be stable, not converge back to the default by silent re-analysis.
|
|
27
|
+
|
|
28
|
+
What is cached and committed:
|
|
29
|
+
.quirework/provenance/coldread/manifest.json index (points at raw logs)
|
|
30
|
+
.quirework/provenance/coldread/reports/<key>.md one analyst report per source
|
|
31
|
+
|
|
32
|
+
What is NOT committed (regenerated on demand into ~/tmp):
|
|
33
|
+
the filtered transcript renders the analysts read -- they contain verbatim
|
|
34
|
+
prompts (firehose-adjacent) and are deterministic from raw + the filter.
|
|
35
|
+
|
|
36
|
+
Subcommands:
|
|
37
|
+
status dry-run: hit / MISS per source + tokens
|
|
38
|
+
triage free, mechanical relevance prioritizer over pending
|
|
39
|
+
captures (by files edited): 'keep' if it edited the
|
|
40
|
+
document, 'review' (needs an intelligent read) if not
|
|
41
|
+
(--review-list prints just the review-set paths)
|
|
42
|
+
render write filtered renders for every miss into the staging
|
|
43
|
+
dir; a long tail of small sealed sessions is packed into
|
|
44
|
+
batch files (--batch-under N, --batch-max M)
|
|
45
|
+
verify-blind scan staged renders for CLAUDE.md fingerprints
|
|
46
|
+
analyze run each staged render through a sandboxed, blind
|
|
47
|
+
`claude -p` (neutral CWD, tools off, transcript inlined),
|
|
48
|
+
gate on its blindness-check, and auto-record clean reports
|
|
49
|
+
(--only SUBSTR, --model M, --limit N, --oldest,
|
|
50
|
+
--dry-run, --timeout S)
|
|
51
|
+
record SRC R register report R for a single source SRC
|
|
52
|
+
record-batch R SRC... register one shared report R across several sources
|
|
53
|
+
discards per session: commits made vs commits that reached main
|
|
54
|
+
(a discrepancy detector for authority the transcript
|
|
55
|
+
cannot record; --min N to set the flag threshold)
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
import contextlib
|
|
59
|
+
import datetime
|
|
60
|
+
import fcntl
|
|
61
|
+
import hashlib
|
|
62
|
+
import json
|
|
63
|
+
import os
|
|
64
|
+
import re
|
|
65
|
+
import shutil
|
|
66
|
+
import subprocess
|
|
67
|
+
import sys
|
|
68
|
+
from pathlib import Path
|
|
69
|
+
|
|
70
|
+
from quirework.project import current_project
|
|
71
|
+
from quirework.provenance import store
|
|
72
|
+
from quirework.provenance.store import reference as source_reference
|
|
73
|
+
from quirework.providers.blind import BLIND_SETTINGS
|
|
74
|
+
from quirework.providers.blind import NO_SETTING_SOURCES
|
|
75
|
+
from quirework.providers.blind import MIN_TOOLS
|
|
76
|
+
from quirework.providers.blind import NO_MCP
|
|
77
|
+
from quirework.providers.blind import confine
|
|
78
|
+
from quirework.providers.blind import seat_dir
|
|
79
|
+
from quirework.providers import canary
|
|
80
|
+
from quirework.provenance import lineage as LINEAGE
|
|
81
|
+
from quirework.provenance import rulings as RULINGS
|
|
82
|
+
from quirework.provenance import scope as SCOPE
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _analyst_argv(model):
|
|
86
|
+
"""The blind analyst's command; the prompt arrives on stdin."""
|
|
87
|
+
return [
|
|
88
|
+
"claude",
|
|
89
|
+
"-p",
|
|
90
|
+
"--model",
|
|
91
|
+
model,
|
|
92
|
+
*BLIND_SETTINGS,
|
|
93
|
+
*NO_SETTING_SOURCES,
|
|
94
|
+
*MIN_TOOLS,
|
|
95
|
+
*NO_MCP,
|
|
96
|
+
]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _analyst_twin(prompt, model, workdir, session):
|
|
100
|
+
return canary.Twin(_analyst_argv(model), prompt.encode())
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _render_base():
|
|
104
|
+
return current_project().path(
|
|
105
|
+
"provenance.rendered", str(current_project().cache_root / "coldread/rendered")
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# Parts of the cache key that live outside the source bytes: one prompt version
|
|
110
|
+
# per scope, tracking that scope's brief. Bump a scope's number when its brief
|
|
111
|
+
# changes in a way that should invalidate its cached reports.
|
|
112
|
+
#
|
|
113
|
+
# Per-scope, NOT one shared number, and the distinction is not cosmetic: the two
|
|
114
|
+
# briefs are edited independently and read disjoint halves of the corpus, so a
|
|
115
|
+
# single counter would have made every edit to the making brief silently
|
|
116
|
+
# re-buy ~180 book reports that the edit never touched. That is the same
|
|
117
|
+
# over-invalidation the render-hash revalidation exists to undo (see
|
|
118
|
+
# `render_sha256`), reintroduced one layer up.
|
|
119
|
+
PROMPT_VERSIONS = {
|
|
120
|
+
SCOPE.WORK: 2,
|
|
121
|
+
# 3: states the standing-instructions baseline -- the assistant works under
|
|
122
|
+
# rules it did not author in-session, so conformance to a convention reads
|
|
123
|
+
# as prior human direction and only the edges are diagnostic (2026-08-13).
|
|
124
|
+
SCOPE.MAKING: 3,
|
|
125
|
+
}
|
|
126
|
+
# The analysis model, selected by the author: Sonnet reads-and-classifies well
|
|
127
|
+
# at a fraction of Opus cost, and the id here is what the CLI alias below
|
|
128
|
+
# resolves to (recorded per the take-it-from-the-human doctrine).
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# Renders shorter than this (after filtering) are stub sessions -- skipped.
|
|
132
|
+
MIN_RENDER_CHARS = 500
|
|
133
|
+
|
|
134
|
+
# Render-time batching: sealed sources whose render is below this many tokens
|
|
135
|
+
# are concatenated (within an era) into batch files, capped at batch-max, so a
|
|
136
|
+
# long tail of tiny sessions does not become a long tail of agents. Only sealed
|
|
137
|
+
# sources batch -- the live session always renders standalone -- so a batch of
|
|
138
|
+
# unchanging sessions never goes stale.
|
|
139
|
+
DEFAULT_BATCH_UNDER = 8000
|
|
140
|
+
DEFAULT_BATCH_MAX = 120000
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# Blind analysis runs as a headless `claude -p` from a neutral CWD (no project
|
|
144
|
+
# CLAUDE.md or path-keyed memory loads there) with all file/exec tools disabled
|
|
145
|
+
# and the transcript inlined, so the analyst is structurally unable to read the
|
|
146
|
+
# project's own framing. (Agent-tool sub-agents inherit CLAUDE.md/MEMORY.md and
|
|
147
|
+
# are NOT blind -- confirmed by test; do not use them here.)
|
|
148
|
+
def _reports_tmp():
|
|
149
|
+
return current_project().path(
|
|
150
|
+
"provenance.reports", str(current_project().cache_root / "coldread/reports")
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# One brief per scope: the book run asks who originated the book's ideas and
|
|
155
|
+
# prose, the making run asks the same of the instruments and the process rules
|
|
156
|
+
# (see `scope.py`). Each carries its own PROMPT_VERSIONS entry, and the scope
|
|
157
|
+
# is folded into the cache key so a session that changes scope is re-analyzed
|
|
158
|
+
# under the brief it will actually be read with, rather than cache-hitting on a
|
|
159
|
+
# report written to answer the other question.
|
|
160
|
+
|
|
161
|
+
ANALYZE_TIMEOUT_S = 600
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
from quirework.provenance import ccfilter as FLT
|
|
165
|
+
|
|
166
|
+
FILTER_VERSION = FLT.FILTER_VERSION
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _options():
|
|
170
|
+
"""Read policy from the selected project for this operation."""
|
|
171
|
+
from types import SimpleNamespace
|
|
172
|
+
|
|
173
|
+
MODEL = current_project().data("provenance.coldread_policy", {}).get("model", None)
|
|
174
|
+
CLI_MODEL = (
|
|
175
|
+
current_project().data("provenance.coldread_policy", {}).get("cli_model", None)
|
|
176
|
+
)
|
|
177
|
+
policy = current_project().data("provenance.coldread_policy", {})
|
|
178
|
+
_DOC_PREFIXES = tuple(policy.get("doc_prefixes", ()))
|
|
179
|
+
_DOC_DIRS = tuple(policy.get("doc_dirs", ()))
|
|
180
|
+
# Undeclared, the document is the work's footprint, working roots
|
|
181
|
+
# included: drafting in a scratch directory is still authorship. A policy
|
|
182
|
+
# that names its directories replaces it rather than adding to it.
|
|
183
|
+
FOOTPRINT = None
|
|
184
|
+
if "doc_dirs" not in policy:
|
|
185
|
+
from quirework.manuscript.footprint import footprint
|
|
186
|
+
|
|
187
|
+
FOOTPRINT = footprint(current_project())
|
|
188
|
+
return SimpleNamespace(
|
|
189
|
+
CLI_MODEL=CLI_MODEL,
|
|
190
|
+
MODEL=MODEL,
|
|
191
|
+
FOOTPRINT=FOOTPRINT,
|
|
192
|
+
_DOC_DIRS=_DOC_DIRS,
|
|
193
|
+
_DOC_PREFIXES=_DOC_PREFIXES,
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _settings():
|
|
198
|
+
"""Resolve paths from the current operation, without import-time state."""
|
|
199
|
+
from types import SimpleNamespace
|
|
200
|
+
|
|
201
|
+
REPORTS = (
|
|
202
|
+
current_project().path("provenance.coldread", ".quirework/provenance/coldread")
|
|
203
|
+
/ "reports"
|
|
204
|
+
)
|
|
205
|
+
MANIFEST = (
|
|
206
|
+
current_project().path("provenance.coldread", ".quirework/provenance/coldread")
|
|
207
|
+
/ "manifest.json"
|
|
208
|
+
)
|
|
209
|
+
ANALYST_PROMPTS = {
|
|
210
|
+
SCOPE.WORK: current_project().path(
|
|
211
|
+
"provenance.coldread", ".quirework/provenance/coldread"
|
|
212
|
+
)
|
|
213
|
+
/ "analyst-prompt.md",
|
|
214
|
+
SCOPE.MAKING: current_project().path(
|
|
215
|
+
"provenance.coldread", ".quirework/provenance/coldread"
|
|
216
|
+
)
|
|
217
|
+
/ "analyst-prompt-making.md",
|
|
218
|
+
}
|
|
219
|
+
ANALYST_PROMPT = ANALYST_PROMPTS[SCOPE.WORK]
|
|
220
|
+
LOCK = (
|
|
221
|
+
current_project().path("provenance.coldread", ".quirework/provenance/coldread")
|
|
222
|
+
/ ".lock"
|
|
223
|
+
)
|
|
224
|
+
SWEEP_LOCK = (
|
|
225
|
+
current_project().path("provenance.coldread", ".quirework/provenance/coldread")
|
|
226
|
+
/ ".sweep.lock"
|
|
227
|
+
)
|
|
228
|
+
return SimpleNamespace(
|
|
229
|
+
ANALYST_PROMPT=ANALYST_PROMPT,
|
|
230
|
+
ANALYST_PROMPTS=ANALYST_PROMPTS,
|
|
231
|
+
LOCK=LOCK,
|
|
232
|
+
MANIFEST=MANIFEST,
|
|
233
|
+
REPORTS=REPORTS,
|
|
234
|
+
SWEEP_LOCK=SWEEP_LOCK,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def filter_version_for(path):
|
|
239
|
+
"""The recipe version that produced this source's render.
|
|
240
|
+
|
|
241
|
+
One number per *reader*, not one for the corpus. Each session format has
|
|
242
|
+
its own parser, and they move independently: bumping the Codex reader must
|
|
243
|
+
not invalidate three hundred Claude Code reports, and a `ccfilter` change
|
|
244
|
+
must not silently leave Codex reports validated against a recipe that no
|
|
245
|
+
longer exists -- which it did, because the version folded into the key was
|
|
246
|
+
always `ccfilter`'s. A web `.md` is read straight off disk, so no parser
|
|
247
|
+
version can alter it and `ccfilter`'s number is the harmless default.
|
|
248
|
+
"""
|
|
249
|
+
from quirework.provenance import codexfilter as CODEX_FLT
|
|
250
|
+
|
|
251
|
+
return FLT.policy_version(
|
|
252
|
+
CODEX_FLT.FILTER_VERSION
|
|
253
|
+
if store.family_of_reference(path) == store.CODEX
|
|
254
|
+
else FLT.FILTER_VERSION
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
# --------------------------------------------------------------------------- #
|
|
259
|
+
# sources
|
|
260
|
+
# --------------------------------------------------------------------------- #
|
|
261
|
+
def _pending(family):
|
|
262
|
+
"""Captures in the store not yet accepted into this ledger."""
|
|
263
|
+
return set(store.pending(family, "*.jsonl"))
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def sources(scope=SCOPE.WORK):
|
|
267
|
+
"""``(sources, resumed)``: what a run of ``scope`` reads, and what it skips.
|
|
268
|
+
|
|
269
|
+
``sources`` lists every accepted source transcript as ``(path, era)``.
|
|
270
|
+
Only accepted sessions are evidence: a pending capture has not been read
|
|
271
|
+
and kept, and an accepted version never changes, so a report on it is
|
|
272
|
+
never about a log that has since grown.
|
|
273
|
+
|
|
274
|
+
Acceptance does not say evidence *for what*, and ``scope`` does: ``'work'``
|
|
275
|
+
(the default, and every era-A web dialogue) or ``'making'`` for the
|
|
276
|
+
apparatus half, listed in ``making-sessions.txt``. Pass ``None`` for the
|
|
277
|
+
whole corpus regardless of scope.
|
|
278
|
+
|
|
279
|
+
``resumed`` lists, as ``(path, holders)``, each Claude Code capture left
|
|
280
|
+
out because a source in the same run holds its whole dialogue: a session
|
|
281
|
+
resumed under a new id, whose copy renders the same conversation and what
|
|
282
|
+
followed. Reading both would weigh one conversation twice. Ownership is
|
|
283
|
+
decided among this run's own sources (`lineage`), so a session stays in a
|
|
284
|
+
run whose question its copy is not asked, and no render changes, so every
|
|
285
|
+
cached report stays a hit.
|
|
286
|
+
"""
|
|
287
|
+
making = SCOPE.making_ids()
|
|
288
|
+
out = [
|
|
289
|
+
(p, "a-web")
|
|
290
|
+
for p in store.files(store.WEB, "*.md")
|
|
291
|
+
if SCOPE.wants(p, scope, making)
|
|
292
|
+
]
|
|
293
|
+
out += [
|
|
294
|
+
(p, "b-agy")
|
|
295
|
+
for p in store.files(store.AGY, "*.jsonl")
|
|
296
|
+
if SCOPE.wants(p, scope, making)
|
|
297
|
+
]
|
|
298
|
+
cc = [p for p in store.files(store.CC, "*.jsonl") if SCOPE.wants(p, scope, making)]
|
|
299
|
+
own = LINEAGE.owners(cc)
|
|
300
|
+
resumed = []
|
|
301
|
+
for p in cc:
|
|
302
|
+
holders = own.held_by(p)
|
|
303
|
+
if holders:
|
|
304
|
+
resumed.append((p, sorted(Path(h) for h in holders)))
|
|
305
|
+
else:
|
|
306
|
+
out.append((p, "c-refinement"))
|
|
307
|
+
out += [
|
|
308
|
+
(p, "d-codex")
|
|
309
|
+
for p in store.files(store.CODEX, "*.jsonl")
|
|
310
|
+
if SCOPE.wants(p, scope, making)
|
|
311
|
+
]
|
|
312
|
+
# A review sheet's ruling is the author deciding, item by item, what the
|
|
313
|
+
# assistant put to him, with his reasons in his own words: evidence of who
|
|
314
|
+
# directed the work as much as any conversation. It is read only where the
|
|
315
|
+
# project declares its archive, and it is never a pending capture.
|
|
316
|
+
out += [(p, "e-rulings") for p in RULINGS.rounds() if SCOPE.wants(p, scope, making)]
|
|
317
|
+
return out, resumed
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def iter_sources(scope=SCOPE.WORK):
|
|
321
|
+
"""Yield ``(path, era)`` for every source a run of ``scope`` reads (`sources`)."""
|
|
322
|
+
yield from sources(scope)[0]
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def render_source(path):
|
|
326
|
+
"""Return ``(text, stats)``: filtered for jsonl, as-is for web ``.md``.
|
|
327
|
+
|
|
328
|
+
Web dialogues deliberately stay unredacted: they predate and are the source
|
|
329
|
+
of the project framing, so a marker there is usually the human coining it
|
|
330
|
+
live (evidence) — verify-blind flags them REVIEW, never FAIL."""
|
|
331
|
+
family = store.family_of(path)
|
|
332
|
+
if RULINGS.holds(path):
|
|
333
|
+
# The archive is written for the operator and names the project as
|
|
334
|
+
# freely as a Claude Code session does, so it takes the same
|
|
335
|
+
# redaction before an analyst reads it.
|
|
336
|
+
text = FLT.redact(path.read_text(encoding="utf-8", errors="replace"))
|
|
337
|
+
return text, {"chars": len(text)}
|
|
338
|
+
if path.suffix == ".jsonl":
|
|
339
|
+
if family == store.CODEX:
|
|
340
|
+
from quirework.provenance import codexfilter as CODEX_FLT
|
|
341
|
+
|
|
342
|
+
return CODEX_FLT.filter_session(path)
|
|
343
|
+
if family == store.AGY:
|
|
344
|
+
from quirework.provenance import catalog as CAT
|
|
345
|
+
|
|
346
|
+
text, stats = CAT._agy_render(path)
|
|
347
|
+
# The agy render has no filter pass of its own, and GEMINI.md
|
|
348
|
+
# injects the same project framing CLAUDE.md does — so the same
|
|
349
|
+
# wrapper-strip + redaction ccfilter applies to cc sessions runs
|
|
350
|
+
# here (verify-blind caught the leak as a FAIL).
|
|
351
|
+
text = FLT.redact(FLT.strip_wrappers(text))
|
|
352
|
+
return text, {**stats, "chars": len(text)}
|
|
353
|
+
return FLT.filter_session(str(path))
|
|
354
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
355
|
+
return text, {"chars": len(text)}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# --------------------------------------------------------------------------- #
|
|
359
|
+
# manifest + keys
|
|
360
|
+
# --------------------------------------------------------------------------- #
|
|
361
|
+
def load_manifest():
|
|
362
|
+
if _settings().MANIFEST.exists():
|
|
363
|
+
return json.loads(_settings().MANIFEST.read_text())
|
|
364
|
+
return {
|
|
365
|
+
"schema": 1,
|
|
366
|
+
"filter_version": FILTER_VERSION,
|
|
367
|
+
"prompt_version": PROMPT_VERSIONS[SCOPE.WORK],
|
|
368
|
+
"default_model": _options().MODEL,
|
|
369
|
+
"updated": None,
|
|
370
|
+
"sources": {},
|
|
371
|
+
"integrated_into": ".quirework/provenance/authorship-coldread.md",
|
|
372
|
+
"integrated_keys": [],
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
# The manifest is one JSON blob rewritten whole, and `analyze` is a long
|
|
377
|
+
# sequence of read-modify-write cycles against it. Nothing shards the work --
|
|
378
|
+
# two concurrent sweeps walk the same `sorted(renders)` in the same order --
|
|
379
|
+
# so without a lock they interleave: A loads, B loads, A records, B records,
|
|
380
|
+
# and B's whole-file write silently drops A's entry. The report file survives
|
|
381
|
+
# on disk (its name is content-addressed) but is orphaned from the manifest,
|
|
382
|
+
# so it reads as a miss and gets paid for again. Nothing is corrupted and a
|
|
383
|
+
# later serial run self-heals; the cost is money and bookkeeping.
|
|
384
|
+
# Two locks, deliberately distinct files. A single long-held lock would
|
|
385
|
+
# self-deadlock: `analyze` would hold it for the whole sweep and then call
|
|
386
|
+
# `cmd_record`, whose own `open()` creates a second file description, and
|
|
387
|
+
# flock would block the process against itself.
|
|
388
|
+
# .sweep.lock held for a whole `analyze` run, non-blocking -> a second
|
|
389
|
+
# sweep fails fast instead of racing.
|
|
390
|
+
# .lock held for the few ms of one manifest read-modify-write, so a
|
|
391
|
+
# hand-run `record` during a sweep cannot clobber an entry.
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
@contextlib.contextmanager
|
|
395
|
+
def _flock(path, blocking=True):
|
|
396
|
+
"""Hold an exclusive flock on ``path`` (its own file, never the manifest).
|
|
397
|
+
|
|
398
|
+
The lock files are separate from the manifest because `save_manifest`
|
|
399
|
+
replaces it via `os.replace`, which swaps the inode -- a lock held on the
|
|
400
|
+
old one would silently stop conflicting with anything.
|
|
401
|
+
"""
|
|
402
|
+
current_project().path(
|
|
403
|
+
"provenance.coldread", ".quirework/provenance/coldread"
|
|
404
|
+
).mkdir(parents=True, exist_ok=True)
|
|
405
|
+
flags = fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)
|
|
406
|
+
with open(path, "a+") as fh:
|
|
407
|
+
try:
|
|
408
|
+
fcntl.flock(fh.fileno(), flags)
|
|
409
|
+
except BlockingIOError:
|
|
410
|
+
raise AlreadyRunning(_lock_holder(fh)) from None
|
|
411
|
+
fh.seek(0)
|
|
412
|
+
fh.truncate()
|
|
413
|
+
fh.write(
|
|
414
|
+
f"{os.getpid()} {datetime.datetime.now().isoformat(timespec='seconds')}\n"
|
|
415
|
+
)
|
|
416
|
+
fh.flush()
|
|
417
|
+
try:
|
|
418
|
+
yield
|
|
419
|
+
finally:
|
|
420
|
+
fcntl.flock(fh.fileno(), fcntl.LOCK_UN)
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def manifest_lock():
|
|
424
|
+
"""Serialize one manifest read-modify-write across processes."""
|
|
425
|
+
return _flock(_settings().LOCK, blocking=True)
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def persist_revalidations(m):
|
|
429
|
+
"""Write back entries `is_hit` carried across a filter bump.
|
|
430
|
+
|
|
431
|
+
Without this the revalidation is recomputed on every run -- and it renders
|
|
432
|
+
each candidate to do so, which is the expensive part. Re-reads the manifest
|
|
433
|
+
under the lock and copies over only the revalidation fields, so a report
|
|
434
|
+
recorded by another process in the meantime is not clobbered.
|
|
435
|
+
"""
|
|
436
|
+
if not m.pop("_dirty", None):
|
|
437
|
+
return
|
|
438
|
+
today = datetime.date.today().isoformat()
|
|
439
|
+
with manifest_lock():
|
|
440
|
+
cur = load_manifest()
|
|
441
|
+
groups = [(m.get("sources", {}), cur["sources"])]
|
|
442
|
+
groups += [
|
|
443
|
+
(entries, cur.get("persona_sources", {}).get(key, {}))
|
|
444
|
+
for key, entries in m.get("persona_sources", {}).items()
|
|
445
|
+
]
|
|
446
|
+
for observed, current_entries in groups:
|
|
447
|
+
for rel, e in observed.items():
|
|
448
|
+
current = filter_version_for(rel)
|
|
449
|
+
if e.get("filter_version") != current or "render_sha256" not in e:
|
|
450
|
+
continue
|
|
451
|
+
ce = current_entries.get(rel)
|
|
452
|
+
# Another process may have recorded a different source while
|
|
453
|
+
# this process was rendering. Merge only the same evidence.
|
|
454
|
+
if (
|
|
455
|
+
ce is not None
|
|
456
|
+
and ce.get("filter_version") != current
|
|
457
|
+
and all(
|
|
458
|
+
ce.get(k) == e.get(k)
|
|
459
|
+
for k in ("sha256", "prompt_version", "report")
|
|
460
|
+
)
|
|
461
|
+
and SCOPE.normalize(ce.get("scope"))
|
|
462
|
+
== SCOPE.normalize(e.get("scope"))
|
|
463
|
+
):
|
|
464
|
+
ce["filter_version"] = current
|
|
465
|
+
ce["render_sha256"] = e["render_sha256"]
|
|
466
|
+
ce.setdefault("revalidated", today)
|
|
467
|
+
save_manifest(cur)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def sweep_lock():
|
|
471
|
+
"""Refuse to start a second concurrent `analyze` (nothing shards the work)."""
|
|
472
|
+
return _flock(_settings().SWEEP_LOCK, blocking=False)
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
class AlreadyRunning(RuntimeError):
|
|
476
|
+
"""Another process holds the cold-read lock."""
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _lock_holder(fh):
|
|
480
|
+
try:
|
|
481
|
+
fh.seek(0)
|
|
482
|
+
return fh.read().strip() or "unknown"
|
|
483
|
+
except OSError:
|
|
484
|
+
return "unknown"
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def save_manifest(m):
|
|
488
|
+
m.pop("_dirty", None)
|
|
489
|
+
m["filter_version"] = FILTER_VERSION
|
|
490
|
+
m["prompt_version"] = PROMPT_VERSIONS[SCOPE.WORK]
|
|
491
|
+
# The corpus is mixed-model by design; only the per-source entries say
|
|
492
|
+
# which model read which source. The top level records the default.
|
|
493
|
+
m.pop("model", None)
|
|
494
|
+
m["default_model"] = _options().MODEL
|
|
495
|
+
m["updated"] = datetime.date.today().isoformat()
|
|
496
|
+
# Atomic: a kill mid-write must not leave a truncated manifest, and a
|
|
497
|
+
# concurrent reader must never see a half-written one (observed as a
|
|
498
|
+
# JSONDecodeError while a sweep was running).
|
|
499
|
+
tmp = _settings().MANIFEST.with_suffix(".json.tmp")
|
|
500
|
+
tmp.write_text(json.dumps(m, indent=2, ensure_ascii=False) + "\n")
|
|
501
|
+
os.replace(tmp, _settings().MANIFEST)
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def sha256_file(p):
|
|
505
|
+
return FLT.sha256_file(str(p))
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
_DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})")
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def source_date(path):
|
|
512
|
+
"""Earliest date in a source transcript, as ``YYYY-MM-DD`` (or ``""``).
|
|
513
|
+
|
|
514
|
+
Read from the data itself rather than the catalog or the file mtime: a
|
|
515
|
+
mirrored session's mtime is when it was copied, not when it was held.
|
|
516
|
+
"""
|
|
517
|
+
try:
|
|
518
|
+
if path.suffix == ".jsonl":
|
|
519
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
520
|
+
for line in fh:
|
|
521
|
+
m = _DATE_RE.search(line)
|
|
522
|
+
if m:
|
|
523
|
+
return m.group(1)
|
|
524
|
+
return ""
|
|
525
|
+
head = path.read_text(encoding="utf-8", errors="replace")[:4000]
|
|
526
|
+
m = _DATE_RE.search(head)
|
|
527
|
+
return m.group(1) if m else ""
|
|
528
|
+
except OSError:
|
|
529
|
+
return ""
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def render_date(render_path):
|
|
533
|
+
"""Earliest source date across a render (a batch holds several)."""
|
|
534
|
+
dates = []
|
|
535
|
+
try:
|
|
536
|
+
text = render_path.read_text(encoding="utf-8", errors="replace")
|
|
537
|
+
except OSError:
|
|
538
|
+
return ""
|
|
539
|
+
for rel in re.findall(r"<!-- coldread-source: (\S+)", text):
|
|
540
|
+
d = source_date(current_project().root / rel)
|
|
541
|
+
if d:
|
|
542
|
+
dates.append(d)
|
|
543
|
+
return min(dates) if dates else ""
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def render_sha256(path):
|
|
547
|
+
"""Hash of the text an analyst actually reads for ``path``.
|
|
548
|
+
|
|
549
|
+
The cache used to key on the *source* bytes plus a global
|
|
550
|
+
``FILTER_VERSION``, which meant any filter change invalidated every
|
|
551
|
+
report — including the ones whose render the change never touched. The
|
|
552
|
+
v2->v3 bump made that concrete: 218 sources, 64 renders actually changed,
|
|
553
|
+
and 154 reports would have been re-bought identical. Era A is the clearest
|
|
554
|
+
case, since a web ``.md`` is read straight off disk and no filter version
|
|
555
|
+
can alter it.
|
|
556
|
+
|
|
557
|
+
So the render hash is the honest key: it is exactly what the analyst saw.
|
|
558
|
+
Recorded on every report, and used to revalidate across a filter bump."""
|
|
559
|
+
text, _ = render_source(path)
|
|
560
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def report_name(sha, scope=SCOPE.WORK, filter_version=None, persona=None):
|
|
564
|
+
"""Cache filename for one source's report under one scope."""
|
|
565
|
+
tag = f"-{scope}"
|
|
566
|
+
version = FILTER_VERSION if filter_version is None else filter_version
|
|
567
|
+
variant = "-persona-" + persona["sha256"] if persona else ""
|
|
568
|
+
return f"{sha[:12]}-f{version}p{PROMPT_VERSIONS[scope]}{tag}{variant}.md"
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _persona_entries(manifest, persona=None):
|
|
572
|
+
if persona is None:
|
|
573
|
+
return manifest["sources"]
|
|
574
|
+
return manifest.setdefault("persona_sources", {}).setdefault(persona["sha256"], {})
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def is_hit(manifest, relpath, sha, path=None, persona=None):
|
|
578
|
+
"""A report is current iff source bytes, filter, and prompt all match.
|
|
579
|
+
|
|
580
|
+
The model is deliberately NOT a validity criterion (author ruling): model
|
|
581
|
+
choice is an economic decision made per run — some sources read by Sonnet,
|
|
582
|
+
others by Opus — so it is recorded per source as *provenance* and never
|
|
583
|
+
invalidates. When it was part of the key, a mixed corpus was unstable:
|
|
584
|
+
any non-default report read as stale, and the next default sweep silently
|
|
585
|
+
re-analyzed and overwrote it. Redoing a source under a different model is
|
|
586
|
+
an explicit act (delete its manifest entry, or `analyze --only` after
|
|
587
|
+
doing so), never a side effect."""
|
|
588
|
+
e = _persona_entries(manifest, persona).get(relpath)
|
|
589
|
+
if not e:
|
|
590
|
+
return False
|
|
591
|
+
# A report answers the question its scope's brief asked. If the source has
|
|
592
|
+
# since changed scope, the cached report is about the other question and is
|
|
593
|
+
# not a hit -- entries written before scopes existed are the work's by
|
|
594
|
+
# default, which is what they were, and so are entries recorded under the
|
|
595
|
+
# label the work's scope once had. The prompt version is then checked
|
|
596
|
+
# against *that scope's* brief, so an edit to one brief never invalidates
|
|
597
|
+
# the other's.
|
|
598
|
+
scope = SCOPE.scope_of_path(relpath)
|
|
599
|
+
if SCOPE.normalize(e.get("scope", SCOPE.WORK)) != scope:
|
|
600
|
+
return False
|
|
601
|
+
if e.get("prompt_version") != PROMPT_VERSIONS[scope]:
|
|
602
|
+
return False
|
|
603
|
+
if not (
|
|
604
|
+
current_project().path("provenance.coldread", ".quirework/provenance/coldread")
|
|
605
|
+
/ e.get("report", "")
|
|
606
|
+
).exists():
|
|
607
|
+
return False
|
|
608
|
+
current = filter_version_for(relpath)
|
|
609
|
+
moved = e.get("sha256") != sha
|
|
610
|
+
if not moved and e.get("filter_version") == current:
|
|
611
|
+
return True
|
|
612
|
+
# The source hash gates first: the render hash is a second check, never a
|
|
613
|
+
# bypass -- with one exception. A Claude Code capture that names sub-agents
|
|
614
|
+
# can differ from the bytes an analyst read only in side-thread records,
|
|
615
|
+
# which the render drops, so its render settles whether anything the
|
|
616
|
+
# analyst read changed. A session whose dialogue grew renders differently
|
|
617
|
+
# and still misses.
|
|
618
|
+
if moved and not (path is not None and _names_subagents(path)):
|
|
619
|
+
return False
|
|
620
|
+
# The filter recipe moved, or the bytes did as above. That only invalidates
|
|
621
|
+
# this report if it changed what the analyst read -- so ask, rather than
|
|
622
|
+
# assuming (see `render_sha256`). Rendering is the expensive part, so it
|
|
623
|
+
# happens only on this path.
|
|
624
|
+
recorded = e.get("render_sha256")
|
|
625
|
+
if recorded and path is not None and recorded == render_sha256(path):
|
|
626
|
+
if e.get("sha256") != sha:
|
|
627
|
+
# Keep the bytes the analyst was actually given on the record.
|
|
628
|
+
e.setdefault("analyzed_sha256", e.get("sha256"))
|
|
629
|
+
e["sha256"] = sha
|
|
630
|
+
e["filter_version"] = current # revalidated, not re-bought
|
|
631
|
+
manifest["_dirty"] = True
|
|
632
|
+
return True
|
|
633
|
+
return False
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def _names_subagents(path):
|
|
637
|
+
"""Whether ``path`` is a Claude Code capture whose header lists sub-agents."""
|
|
638
|
+
if store.family_of(path) != store.CC:
|
|
639
|
+
return False
|
|
640
|
+
header = FLT.capture_header(path)
|
|
641
|
+
return bool(header and header.get("subagents"))
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def classify(path, manifest, persona=None):
|
|
645
|
+
"""Return ``'hit'`` or ``'MISS'`` for an accepted source, with its key."""
|
|
646
|
+
rel = str(source_reference(path, current_project()))
|
|
647
|
+
sha = sha256_file(path)
|
|
648
|
+
return (
|
|
649
|
+
("hit" if is_hit(manifest, rel, sha, path, persona=persona) else "MISS"),
|
|
650
|
+
rel,
|
|
651
|
+
sha,
|
|
652
|
+
)
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def _era_for_rel(rel):
|
|
656
|
+
if RULINGS.holds(rel):
|
|
657
|
+
return "e-rulings"
|
|
658
|
+
family = store.family_of_reference(rel)
|
|
659
|
+
if family == store.CODEX:
|
|
660
|
+
return "d-codex"
|
|
661
|
+
if family == store.AGY:
|
|
662
|
+
return "b-agy"
|
|
663
|
+
if family == store.WEB:
|
|
664
|
+
return "a-web"
|
|
665
|
+
return "c-refinement"
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
# --------------------------------------------------------------------------- #
|
|
669
|
+
# subcommands
|
|
670
|
+
# --------------------------------------------------------------------------- #
|
|
671
|
+
def cmd_status(_args):
|
|
672
|
+
from quirework.sops import personas
|
|
673
|
+
|
|
674
|
+
scope, _args = _parse_scope(_args)
|
|
675
|
+
if _args:
|
|
676
|
+
if len(_args) != 2 or _args[0] != "--persona":
|
|
677
|
+
raise ValueError("status accepts --scope and --persona ID")
|
|
678
|
+
persona = personas.resolve(_args[1])
|
|
679
|
+
else:
|
|
680
|
+
persona = None
|
|
681
|
+
manifest = load_manifest()
|
|
682
|
+
n = {"hit": 0, "MISS": 0}
|
|
683
|
+
miss_tokens = 0
|
|
684
|
+
rows = []
|
|
685
|
+
read, resumed = sources(scope)
|
|
686
|
+
for path, era in read:
|
|
687
|
+
state, rel, sha = classify(path, manifest, persona=persona)
|
|
688
|
+
n[state] += 1
|
|
689
|
+
est = 0
|
|
690
|
+
if state != "hit":
|
|
691
|
+
body, _ = render_source(path)
|
|
692
|
+
est = len(body) // 4
|
|
693
|
+
miss_tokens += est
|
|
694
|
+
# Which model read this source (provenance; the corpus is mixed-model
|
|
695
|
+
# by design, so the column is how the mix stays visible).
|
|
696
|
+
model = (_persona_entries(manifest, persona).get(rel) or {}).get("model", "")
|
|
697
|
+
model = model.replace("claude-", "") if state == "hit" else "-"
|
|
698
|
+
rows.append((state, era, rel, sha[:10], est, model))
|
|
699
|
+
|
|
700
|
+
persist_revalidations(manifest)
|
|
701
|
+
|
|
702
|
+
width = max((len(r[2]) for r in rows), default=10)
|
|
703
|
+
mwidth = max((len(r[5]) for r in rows), default=5)
|
|
704
|
+
print(
|
|
705
|
+
f"{'state':9} {'era':13} {'source':{width}} {'sha':10} {'~tok':>8} {'model':{mwidth}}"
|
|
706
|
+
)
|
|
707
|
+
print("-" * (9 + 13 + width + 10 + 9 + mwidth + 5))
|
|
708
|
+
for state, era, rel, sha, est, model in rows:
|
|
709
|
+
tok = f"{est:,}" if est else "-"
|
|
710
|
+
print(f"{state:9} {era:13} {rel:{width}} {sha:10} {tok:>8} {model}")
|
|
711
|
+
|
|
712
|
+
print()
|
|
713
|
+
waiting = sum(len(_pending(f)) for f in (store.CC, store.AGY, store.CODEX))
|
|
714
|
+
if waiting:
|
|
715
|
+
print(
|
|
716
|
+
f"pending: {waiting} captured session(s) not accepted yet -- triage, "
|
|
717
|
+
"then accept or reject them (`quirework provenance store status`)"
|
|
718
|
+
)
|
|
719
|
+
print(
|
|
720
|
+
f"scope={scope or 'all'} "
|
|
721
|
+
f"hits={n['hit']} misses={n['MISS']} "
|
|
722
|
+
f"to-analyze tokens≈{miss_tokens:,} "
|
|
723
|
+
f"(≈{max(1, -(-miss_tokens // 150000))} blind sub-agent(s) @150k ctx)"
|
|
724
|
+
)
|
|
725
|
+
if resumed:
|
|
726
|
+
project = current_project()
|
|
727
|
+
print(
|
|
728
|
+
f"resumed: {len(resumed)} session(s) are read once, in the capture "
|
|
729
|
+
"that continued them under a new id:"
|
|
730
|
+
)
|
|
731
|
+
for path, holders in resumed:
|
|
732
|
+
names = ", ".join(str(source_reference(h, project)) for h in holders)
|
|
733
|
+
print(f" {source_reference(path, project)} -> {names}")
|
|
734
|
+
# The other scope is a separate artifact, not a subset -- say so, or a run
|
|
735
|
+
# that reports "0 misses" reads as a finished corpus when half of it was
|
|
736
|
+
# never in this run's frame.
|
|
737
|
+
if scope is not None:
|
|
738
|
+
other = SCOPE.MAKING if scope == SCOPE.WORK else SCOPE.WORK
|
|
739
|
+
n_other = sum(1 for _ in iter_sources(other))
|
|
740
|
+
print(
|
|
741
|
+
f"note: {n_other} source(s) are scope={other} and are not in this "
|
|
742
|
+
f"run; see `coldread status --scope {other}`."
|
|
743
|
+
)
|
|
744
|
+
if n["MISS"]:
|
|
745
|
+
flag = "" if scope in (None, SCOPE.WORK) else f" --scope {scope}"
|
|
746
|
+
print(
|
|
747
|
+
f"next: quirework provenance coldread render{flag} then spawn blind "
|
|
748
|
+
f"analysts from",
|
|
749
|
+
_render_out(scope or SCOPE.WORK),
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def _render_out(scope=SCOPE.WORK):
|
|
754
|
+
"""Staging directory for one analysis scope."""
|
|
755
|
+
return _render_base().with_name(_render_base().name + f"-{scope}")
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
def _parse_scope(args):
|
|
759
|
+
"""Pull `--scope work|making|all` out of an arg list; returns (scope, rest)."""
|
|
760
|
+
scope, rest, i = SCOPE.WORK, [], 0
|
|
761
|
+
while i < len(args):
|
|
762
|
+
if args[i] == "--scope":
|
|
763
|
+
want = args[i + 1]
|
|
764
|
+
if want == "all":
|
|
765
|
+
scope = None
|
|
766
|
+
elif want in SCOPE.SCOPES:
|
|
767
|
+
scope = want
|
|
768
|
+
else:
|
|
769
|
+
raise SystemExit(
|
|
770
|
+
f"coldread: --scope must be one of "
|
|
771
|
+
f"{', '.join(SCOPE.SCOPES)}, all (got {want!r})"
|
|
772
|
+
)
|
|
773
|
+
i += 2
|
|
774
|
+
else:
|
|
775
|
+
rest.append(args[i])
|
|
776
|
+
i += 1
|
|
777
|
+
return scope, rest
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def _parse_render_flags(args):
|
|
781
|
+
bu, bm = DEFAULT_BATCH_UNDER, DEFAULT_BATCH_MAX
|
|
782
|
+
i = 0
|
|
783
|
+
while i < len(args):
|
|
784
|
+
if args[i] == "--batch-under":
|
|
785
|
+
bu = int(args[i + 1])
|
|
786
|
+
i += 2
|
|
787
|
+
elif args[i] == "--batch-max":
|
|
788
|
+
bm = int(args[i + 1])
|
|
789
|
+
i += 2
|
|
790
|
+
else:
|
|
791
|
+
print(f"render: ignoring unknown arg {args[i]}", file=sys.stderr)
|
|
792
|
+
i += 1
|
|
793
|
+
return bu, bm
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _write_batch(outname, era, members, out=None):
|
|
797
|
+
if out is None:
|
|
798
|
+
out = _render_base()
|
|
799
|
+
parts = [
|
|
800
|
+
f"<!-- batch:{era} sessions:{len(members)} -->",
|
|
801
|
+
"",
|
|
802
|
+
f"# Batched short sessions ({era})",
|
|
803
|
+
"",
|
|
804
|
+
"Several short, independent sessions are concatenated below.",
|
|
805
|
+
"Analyze EACH session on its own and give one short report per session,",
|
|
806
|
+
"in the order they appear. Do not blend them into a single narrative.",
|
|
807
|
+
"",
|
|
808
|
+
]
|
|
809
|
+
for i, (stem, _rel, sha, body, _est) in enumerate(members, 1):
|
|
810
|
+
parts.append("=" * 70)
|
|
811
|
+
parts.append(
|
|
812
|
+
f"## SESSION {i}/{len(members)} — {era} / {stem} (sha {sha[:12]})"
|
|
813
|
+
)
|
|
814
|
+
parts.append("")
|
|
815
|
+
parts.append(body)
|
|
816
|
+
parts.append("")
|
|
817
|
+
members_hdr = " ".join(f"{rel}={sha}" for _stem, rel, sha, _b, _e in members)
|
|
818
|
+
header = f"<!-- coldread-batch: {members_hdr} -->\n\n"
|
|
819
|
+
(out / outname).write_text(header + "\n".join(parts), encoding="utf-8")
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def _write_corpus_index(singles, batches, out=None):
|
|
823
|
+
if out is None:
|
|
824
|
+
out = _render_base()
|
|
825
|
+
lines = [
|
|
826
|
+
"# Blind cold-read staging corpus",
|
|
827
|
+
"",
|
|
828
|
+
"Each analyst reads ONE file below and nothing else in this repo.",
|
|
829
|
+
"Batch files hold several short sessions; analyze each independently.",
|
|
830
|
+
"Register results with `quirework provenance coldread record` (singles) or",
|
|
831
|
+
"`record-batch` (batches).",
|
|
832
|
+
"",
|
|
833
|
+
"## Single-session renders",
|
|
834
|
+
"",
|
|
835
|
+
"| render | ~tokens | source | sha |",
|
|
836
|
+
"| --- | --- | --- | --- |",
|
|
837
|
+
]
|
|
838
|
+
for outname, _state, rel, sha, est in singles:
|
|
839
|
+
lines.append(f"| {outname} | {est:,} | {rel} | {sha[:12]} |")
|
|
840
|
+
lines += ["", "## Batches", ""]
|
|
841
|
+
if not batches:
|
|
842
|
+
lines.append("_none_")
|
|
843
|
+
for outname, _era, toks, members in batches:
|
|
844
|
+
rels = " ".join(m[1] for m in members)
|
|
845
|
+
lines.append(f"### {outname} (~{toks:,} tok, {len(members)} sessions)")
|
|
846
|
+
lines.append("")
|
|
847
|
+
for stem, rel, sha, _body, est in members:
|
|
848
|
+
lines.append(f"- {rel} (~{est:,} tok, sha {sha[:12]})")
|
|
849
|
+
lines.append("")
|
|
850
|
+
lines.append("After analysis, register the shared report:")
|
|
851
|
+
lines.append("")
|
|
852
|
+
lines.append("```")
|
|
853
|
+
lines.append(f"quirework provenance coldread record-batch <REPORT.md> {rels}")
|
|
854
|
+
lines.append("```")
|
|
855
|
+
lines.append("")
|
|
856
|
+
(out / "corpus-index.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
def cmd_render(args):
|
|
860
|
+
scope, args = _parse_scope(args)
|
|
861
|
+
batch_under, batch_max = _parse_render_flags(args)
|
|
862
|
+
out = _render_out(scope or SCOPE.WORK)
|
|
863
|
+
manifest = load_manifest()
|
|
864
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
865
|
+
|
|
866
|
+
singles = [] # (outname, state, rel, sha, est)
|
|
867
|
+
smalls = {} # era -> [(stem, rel, sha, body, est)]
|
|
868
|
+
for path, era in iter_sources(scope):
|
|
869
|
+
state, rel, sha = classify(path, manifest)
|
|
870
|
+
if state == "hit":
|
|
871
|
+
continue
|
|
872
|
+
body, _ = render_source(path)
|
|
873
|
+
if len(body.strip()) < MIN_RENDER_CHARS:
|
|
874
|
+
continue # stub session
|
|
875
|
+
est = len(body) // 4
|
|
876
|
+
if batch_under and est < batch_under:
|
|
877
|
+
smalls.setdefault(era, []).append((path.stem, rel, sha, body, est))
|
|
878
|
+
else:
|
|
879
|
+
outname = f"{era}__{path.stem}.md"
|
|
880
|
+
header = f"<!-- coldread-source: {rel} {sha} -->\n\n"
|
|
881
|
+
(out / outname).write_text(header + body, encoding="utf-8")
|
|
882
|
+
singles.append((outname, state, rel, sha, est))
|
|
883
|
+
|
|
884
|
+
batches = [] # (outname, era, tokens, members)
|
|
885
|
+
for era, items in smalls.items():
|
|
886
|
+
items.sort(key=lambda x: -x[4]) # largest-first bin packing
|
|
887
|
+
bins = [] # [tokens, [members]]
|
|
888
|
+
for it in items:
|
|
889
|
+
for b in bins:
|
|
890
|
+
if b[0] + it[4] <= batch_max:
|
|
891
|
+
b[0] += it[4]
|
|
892
|
+
b[1].append(it)
|
|
893
|
+
break
|
|
894
|
+
else:
|
|
895
|
+
bins.append([it[4], [it]])
|
|
896
|
+
for n, (toks, members) in enumerate(bins, 1):
|
|
897
|
+
outname = f"{era}__batch-{n:02d}.md"
|
|
898
|
+
_write_batch(outname, era, members, out)
|
|
899
|
+
batches.append((outname, era, toks, members))
|
|
900
|
+
|
|
901
|
+
_write_corpus_index(singles, batches, out)
|
|
902
|
+
batched = sum(len(m) for *_, m in batches)
|
|
903
|
+
print(
|
|
904
|
+
f"staged {len(singles)} single render(s) + {len(batches)} batch(es) "
|
|
905
|
+
f"covering {batched} short session(s) -> {out}"
|
|
906
|
+
)
|
|
907
|
+
print(f"index: {out / 'corpus-index.md'}")
|
|
908
|
+
print(
|
|
909
|
+
f"verify blindness before spawning: quirework provenance coldread verify-blind {out}"
|
|
910
|
+
)
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def cmd_verify_blind(args):
|
|
914
|
+
"""Scan renders for project-framing fingerprints.
|
|
915
|
+
|
|
916
|
+
Local-session hits are a hard FAIL: project guidance is injected there, so
|
|
917
|
+
a fingerprint means filtering missed an injection. Web (era-a) hits are a
|
|
918
|
+
REVIEW warning, not a failure: those transcripts predate and are the source
|
|
919
|
+
of the guidance, so the phrases are usually the human coining the framing
|
|
920
|
+
live (evidence), not contamination.
|
|
921
|
+
"""
|
|
922
|
+
target = Path(args[0]).expanduser() if args else _render_base()
|
|
923
|
+
files = [target] if target.is_file() else sorted(target.glob("*.md"))
|
|
924
|
+
if not files:
|
|
925
|
+
print(f"no renders found in {target}", file=sys.stderr)
|
|
926
|
+
return 1
|
|
927
|
+
fails = reviews = 0
|
|
928
|
+
for f in files:
|
|
929
|
+
if f.name == "corpus-index.md":
|
|
930
|
+
continue
|
|
931
|
+
low = f.read_text(encoding="utf-8", errors="replace").lower()
|
|
932
|
+
hits = sorted({m for m in FLT.CONTAMINATION_MARKERS if m.lower() in low})
|
|
933
|
+
if not hits:
|
|
934
|
+
continue
|
|
935
|
+
era = f.name.split("__", 1)[0] # render names are <era>__<stem>.md
|
|
936
|
+
if era == "a-web":
|
|
937
|
+
reviews += 1
|
|
938
|
+
print(
|
|
939
|
+
f"REVIEW {f.name}: web dialogue contains {hits} (likely evidence; eyeball)"
|
|
940
|
+
)
|
|
941
|
+
else:
|
|
942
|
+
fails += 1
|
|
943
|
+
print(f"FAIL {f.name}: injected framing leaked: {hits}")
|
|
944
|
+
if fails:
|
|
945
|
+
print(f"FAIL: {fails} local-session render(s) carry leaked project framing")
|
|
946
|
+
return 1
|
|
947
|
+
msg = f"OK: no injection leaks in {len(files)} render(s)"
|
|
948
|
+
if reviews:
|
|
949
|
+
msg += f"; {reviews} web render(s) flagged for manual review"
|
|
950
|
+
print(msg)
|
|
951
|
+
return 0
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
def _model_id(cli_model=None):
|
|
955
|
+
"""The provenance id recorded for a run selected as `cli_model`.
|
|
956
|
+
|
|
957
|
+
The default alias resolves to the configured MODEL; any other selection is
|
|
958
|
+
recorded verbatim — the id records what the operator *selected*, the
|
|
959
|
+
strongest attribution available here ("Model metadata is not
|
|
960
|
+
trustworthy"). Provenance only: the model never affects cache validity
|
|
961
|
+
(see `is_hit`)."""
|
|
962
|
+
return (
|
|
963
|
+
_options().MODEL
|
|
964
|
+
if cli_model in (None, _options().CLI_MODEL, _options().MODEL)
|
|
965
|
+
else cli_model
|
|
966
|
+
)
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
def cmd_record(args, model=None, persona=None):
|
|
970
|
+
if len(args) != 2:
|
|
971
|
+
print(
|
|
972
|
+
"usage: quirework provenance coldread record SOURCE_PATH REPORT_FILE",
|
|
973
|
+
file=sys.stderr,
|
|
974
|
+
)
|
|
975
|
+
return 2
|
|
976
|
+
src, report = args
|
|
977
|
+
path = Path(src)
|
|
978
|
+
if not path.is_absolute():
|
|
979
|
+
path = current_project().root / src
|
|
980
|
+
rel = str(source_reference(path, current_project()))
|
|
981
|
+
sha = sha256_file(path)
|
|
982
|
+
scope = SCOPE.scope_of_path(path)
|
|
983
|
+
_settings().REPORTS.mkdir(parents=True, exist_ok=True)
|
|
984
|
+
dest = _settings().REPORTS / report_name(
|
|
985
|
+
sha, scope, filter_version_for(path), persona
|
|
986
|
+
)
|
|
987
|
+
shutil.copyfile(report, dest)
|
|
988
|
+
with manifest_lock():
|
|
989
|
+
manifest = load_manifest()
|
|
990
|
+
era = _era_for_rel(rel)
|
|
991
|
+
_persona_entries(manifest, persona)[rel] = {
|
|
992
|
+
"sha256": sha,
|
|
993
|
+
"render_sha256": render_sha256(path),
|
|
994
|
+
"era": era,
|
|
995
|
+
"scope": scope,
|
|
996
|
+
"report": f"reports/{dest.name}",
|
|
997
|
+
"filter_version": filter_version_for(path),
|
|
998
|
+
"prompt_version": PROMPT_VERSIONS[scope],
|
|
999
|
+
"model": _model_id(model),
|
|
1000
|
+
"analyzed": datetime.date.today().isoformat(),
|
|
1001
|
+
**({"persona": persona} if persona else {}),
|
|
1002
|
+
}
|
|
1003
|
+
save_manifest(manifest)
|
|
1004
|
+
print(
|
|
1005
|
+
f"recorded {rel}\n -> {dest.relative_to(current_project().path('provenance.coldread', '.quirework/provenance/coldread'))}"
|
|
1006
|
+
)
|
|
1007
|
+
return 0
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def cmd_record_batch(args, model=None, persona=None):
|
|
1011
|
+
if len(args) < 2:
|
|
1012
|
+
print(
|
|
1013
|
+
"usage: quirework provenance coldread record-batch REPORT_FILE SOURCE...",
|
|
1014
|
+
file=sys.stderr,
|
|
1015
|
+
)
|
|
1016
|
+
return 2
|
|
1017
|
+
report, srcs = args[0], args[1:]
|
|
1018
|
+
paths = [
|
|
1019
|
+
(Path(s) if Path(s).is_absolute() else current_project().root / s) for s in srcs
|
|
1020
|
+
]
|
|
1021
|
+
shas = {str(source_reference(p, current_project())): sha256_file(p) for p in paths}
|
|
1022
|
+
batch_id = hashlib.sha256("".join(sorted(shas.values())).encode()).hexdigest()[:12]
|
|
1023
|
+
# A batch is rendered within one era *and* one scope (see `cmd_render`), so
|
|
1024
|
+
# the members agree; take the scope from the first and tag the file with it.
|
|
1025
|
+
scope = SCOPE.scope_of_path(paths[0])
|
|
1026
|
+
tag = "" if scope == SCOPE.WORK else f"-{scope}"
|
|
1027
|
+
# A batch is one era, so one reader and one recipe version (see above).
|
|
1028
|
+
version = filter_version_for(paths[0])
|
|
1029
|
+
_settings().REPORTS.mkdir(parents=True, exist_ok=True)
|
|
1030
|
+
variant = "-persona-" + persona["sha256"] if persona else ""
|
|
1031
|
+
dest = (
|
|
1032
|
+
_settings().REPORTS
|
|
1033
|
+
/ f"batch-{batch_id}-f{version}p{PROMPT_VERSIONS[scope]}{tag}{variant}.md"
|
|
1034
|
+
)
|
|
1035
|
+
shutil.copyfile(report, dest)
|
|
1036
|
+
with manifest_lock():
|
|
1037
|
+
manifest = load_manifest()
|
|
1038
|
+
today = datetime.date.today().isoformat()
|
|
1039
|
+
for rel, sha in shas.items():
|
|
1040
|
+
_persona_entries(manifest, persona)[rel] = {
|
|
1041
|
+
"sha256": sha,
|
|
1042
|
+
"render_sha256": render_sha256(current_project().root / rel),
|
|
1043
|
+
"era": _era_for_rel(rel),
|
|
1044
|
+
"scope": scope,
|
|
1045
|
+
"report": f"reports/{dest.name}",
|
|
1046
|
+
"batch_id": batch_id,
|
|
1047
|
+
"filter_version": version,
|
|
1048
|
+
"prompt_version": PROMPT_VERSIONS[scope],
|
|
1049
|
+
"model": _model_id(model),
|
|
1050
|
+
"analyzed": today,
|
|
1051
|
+
**({"persona": persona} if persona else {}),
|
|
1052
|
+
}
|
|
1053
|
+
save_manifest(manifest)
|
|
1054
|
+
print(
|
|
1055
|
+
f"recorded batch {batch_id} ({len(shas)} sources)\n -> {dest.relative_to(current_project().path('provenance.coldread', '.quirework/provenance/coldread'))}"
|
|
1056
|
+
)
|
|
1057
|
+
return 0
|
|
1058
|
+
|
|
1059
|
+
|
|
1060
|
+
def _load_brief(scope=SCOPE.WORK):
|
|
1061
|
+
text = _settings().ANALYST_PROMPTS[scope].read_text(encoding="utf-8")
|
|
1062
|
+
# Everything before the first '---' rule is maintainer-facing preamble (it
|
|
1063
|
+
# names the meta-purpose of the analysis) and must NOT reach the analyst.
|
|
1064
|
+
# Send only the analyst-facing brief after it.
|
|
1065
|
+
parts = re.split(r"(?m)^---\s*$", text, maxsplit=1)
|
|
1066
|
+
return (parts[1] if len(parts) > 1 else text).strip()
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def _parse_render_header(path):
|
|
1070
|
+
"""Return {kind, members:[(rel,sha)], body} from a staged render's header."""
|
|
1071
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
1072
|
+
if not lines:
|
|
1073
|
+
return None
|
|
1074
|
+
head, body = lines[0], "\n".join(lines[1:]).lstrip("\n")
|
|
1075
|
+
m = re.match(r"<!-- coldread-source:\s+(\S+)\s+(\S+)\s+-->", head)
|
|
1076
|
+
if m:
|
|
1077
|
+
return {"kind": "single", "members": [(m.group(1), m.group(2))], "body": body}
|
|
1078
|
+
m = re.match(r"<!-- coldread-batch:\s+(.*?)\s*-->", head)
|
|
1079
|
+
if m:
|
|
1080
|
+
members = []
|
|
1081
|
+
for tok in m.group(1).split():
|
|
1082
|
+
rel, _, sha = tok.partition("=")
|
|
1083
|
+
members.append((rel, sha))
|
|
1084
|
+
return {"kind": "batch", "members": members, "body": body}
|
|
1085
|
+
return None
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
def _blindness_ok(report):
|
|
1089
|
+
"""Gate: a report is usable only if its analyst declared no prior context.
|
|
1090
|
+
|
|
1091
|
+
Reads the answer on the BLINDNESS CHECK line itself (not a keyword scan of
|
|
1092
|
+
the whole head, which mistook a "yes" anywhere for a contamination claim).
|
|
1093
|
+
"""
|
|
1094
|
+
m = re.search(r"blindness check\b\s*:?\s*\n?\s*(.+)", report, flags=re.IGNORECASE)
|
|
1095
|
+
if not m:
|
|
1096
|
+
return False, "no blindness-check line"
|
|
1097
|
+
answer = m.group(1).strip()[:120].lower()
|
|
1098
|
+
if re.match(r"\(?\s*(yes|partial)\b", answer):
|
|
1099
|
+
return False, "analyst declared prior context"
|
|
1100
|
+
if re.search(
|
|
1101
|
+
r"\b(no|none|no prior|no context|no knowledge|no background|uncontaminated)\b",
|
|
1102
|
+
answer,
|
|
1103
|
+
):
|
|
1104
|
+
return True, "clean"
|
|
1105
|
+
return False, "blindness declaration unclear"
|
|
1106
|
+
|
|
1107
|
+
|
|
1108
|
+
def cmd_analyze(args):
|
|
1109
|
+
import argparse
|
|
1110
|
+
from quirework.sops import personas
|
|
1111
|
+
|
|
1112
|
+
scope, args = _parse_scope(args)
|
|
1113
|
+
stage = _render_out(scope or SCOPE.WORK)
|
|
1114
|
+
parser = argparse.ArgumentParser(prog="quirework provenance coldread analyze")
|
|
1115
|
+
parser.add_argument("--only")
|
|
1116
|
+
parser.add_argument("--model", default=_options().CLI_MODEL)
|
|
1117
|
+
parser.add_argument("--persona")
|
|
1118
|
+
parser.add_argument("--dry-run", action="store_true")
|
|
1119
|
+
parser.add_argument("--oldest", action="store_true")
|
|
1120
|
+
parser.add_argument("--limit", type=int)
|
|
1121
|
+
parser.add_argument("--timeout", type=int, default=ANALYZE_TIMEOUT_S)
|
|
1122
|
+
selected = parser.parse_args(args)
|
|
1123
|
+
only, model, dry, oldest = (
|
|
1124
|
+
selected.only,
|
|
1125
|
+
selected.model,
|
|
1126
|
+
selected.dry_run,
|
|
1127
|
+
selected.oldest,
|
|
1128
|
+
)
|
|
1129
|
+
limit, tmo = selected.limit, selected.timeout
|
|
1130
|
+
if (limit is not None and limit < 0) or tmo <= 0:
|
|
1131
|
+
raise ValueError("limit must be nonnegative and timeout must be positive")
|
|
1132
|
+
persona = (
|
|
1133
|
+
personas.resolve(selected.persona) if selected.persona is not None else None
|
|
1134
|
+
)
|
|
1135
|
+
if persona and persona["profile"]["id"] == personas._options().DEFAULT:
|
|
1136
|
+
raise ValueError(
|
|
1137
|
+
"an independent authorship analyst cannot use the co-author persona"
|
|
1138
|
+
)
|
|
1139
|
+
|
|
1140
|
+
renders = sorted(p for p in stage.glob("*.md") if p.name != "corpus-index.md")
|
|
1141
|
+
if oldest:
|
|
1142
|
+
# Walk the project's history in order, so an incremental sweep covers a
|
|
1143
|
+
# contiguous stretch of the arc rather than an alphabetical scatter of
|
|
1144
|
+
# it. Undated renders sort last, keeping the era prefix as the fallback.
|
|
1145
|
+
renders.sort(key=lambda p: (render_date(p) or "9999", p.name))
|
|
1146
|
+
if only:
|
|
1147
|
+
renders = [p for p in renders if only in p.name]
|
|
1148
|
+
if not renders:
|
|
1149
|
+
print(f"no renders to analyze in {stage}; run `render` first", file=sys.stderr)
|
|
1150
|
+
return 1
|
|
1151
|
+
|
|
1152
|
+
# One brief per scope, loaded once. The brief a render is analyzed under is
|
|
1153
|
+
# taken from the render's own members below, not from the run's flag: a
|
|
1154
|
+
# render staged under one scope and analyzed after the session changed scope
|
|
1155
|
+
# would otherwise be asked the wrong question, silently.
|
|
1156
|
+
briefs = {s: _load_brief(s) for s in SCOPE.SCOPES}
|
|
1157
|
+
_reports_tmp().mkdir(parents=True, exist_ok=True)
|
|
1158
|
+
done = 0
|
|
1159
|
+
# Circuit breaker: a run of consecutive `claude` failures is not this
|
|
1160
|
+
# render's problem, it is the account's (the 5-hour usage cap, observed
|
|
1161
|
+
# 2026-07-28: one capped run churned through 106 renders, every one exit 1).
|
|
1162
|
+
# Abort the sweep instead; the cache resumes exactly where it stopped.
|
|
1163
|
+
consec_fail = 0
|
|
1164
|
+
MAX_CONSEC_FAIL = 5
|
|
1165
|
+
proven = None # proven at the first analysis that launches
|
|
1166
|
+
for rp in renders:
|
|
1167
|
+
if consec_fail >= MAX_CONSEC_FAIL:
|
|
1168
|
+
print(
|
|
1169
|
+
f"\nSTOP: {consec_fail} consecutive claude failures — likely "
|
|
1170
|
+
f"the usage cap, not the renders. Re-run "
|
|
1171
|
+
f"`quirework provenance coldread analyze` when the window resets; "
|
|
1172
|
+
f"recorded reports are cache hits and are skipped."
|
|
1173
|
+
)
|
|
1174
|
+
return 1
|
|
1175
|
+
info = _parse_render_header(rp)
|
|
1176
|
+
if not info:
|
|
1177
|
+
print(f"SKIP {rp.name}: no coldread header (re-run render)")
|
|
1178
|
+
continue
|
|
1179
|
+
manifest = load_manifest()
|
|
1180
|
+
present = all(
|
|
1181
|
+
(current_project().root / rel).exists() for rel, _ in info["members"]
|
|
1182
|
+
)
|
|
1183
|
+
cached = present and all(
|
|
1184
|
+
is_hit(
|
|
1185
|
+
manifest,
|
|
1186
|
+
rel,
|
|
1187
|
+
sha256_file(current_project().root / rel),
|
|
1188
|
+
current_project().root / rel,
|
|
1189
|
+
persona=persona,
|
|
1190
|
+
)
|
|
1191
|
+
for rel, _ in info["members"]
|
|
1192
|
+
)
|
|
1193
|
+
persist_revalidations(manifest)
|
|
1194
|
+
if cached:
|
|
1195
|
+
print(f"cached {rp.name} (skip)", flush=True)
|
|
1196
|
+
continue
|
|
1197
|
+
if limit is not None and done >= limit:
|
|
1198
|
+
print(f"limit {limit} reached; stopping")
|
|
1199
|
+
break
|
|
1200
|
+
if dry:
|
|
1201
|
+
print(
|
|
1202
|
+
f"would analyze {rp.name} [{info['kind']}, {len(info['members'])} source(s)]"
|
|
1203
|
+
)
|
|
1204
|
+
done += 1
|
|
1205
|
+
continue
|
|
1206
|
+
|
|
1207
|
+
rscope = SCOPE.scope_of_path(info["members"][0][0])
|
|
1208
|
+
brief = briefs[rscope]
|
|
1209
|
+
prompt = (
|
|
1210
|
+
brief.replace("{TRANSCRIPT}", info["body"])
|
|
1211
|
+
if "{TRANSCRIPT}" in brief
|
|
1212
|
+
else brief + "\n\nTRANSCRIPT(S) TO ANALYZE:\n\n" + info["body"]
|
|
1213
|
+
)
|
|
1214
|
+
if persona:
|
|
1215
|
+
prompt = personas.compose(prompt, persona)
|
|
1216
|
+
print(
|
|
1217
|
+
f"analyzing {rp.name} [{info['kind']}, {len(info['members'])} src] "
|
|
1218
|
+
f"via claude -p --model {model} ...",
|
|
1219
|
+
flush=True,
|
|
1220
|
+
)
|
|
1221
|
+
if proven is None:
|
|
1222
|
+
try:
|
|
1223
|
+
proven = canary.require("cold-read analyst", "claude", _analyst_twin)
|
|
1224
|
+
except canary.CanaryError as exc:
|
|
1225
|
+
print(f"\nSTOP: {exc}", flush=True)
|
|
1226
|
+
return 1
|
|
1227
|
+
try:
|
|
1228
|
+
sd = seat_dir()
|
|
1229
|
+
res = subprocess.run(
|
|
1230
|
+
confine(_analyst_argv(model), sd, runners=["claude"], canary=proven),
|
|
1231
|
+
input=prompt,
|
|
1232
|
+
text=True,
|
|
1233
|
+
capture_output=True,
|
|
1234
|
+
cwd=str(sd),
|
|
1235
|
+
timeout=tmo,
|
|
1236
|
+
)
|
|
1237
|
+
except subprocess.TimeoutExpired:
|
|
1238
|
+
print(f" TIMEOUT after {tmo}s -- skipped", flush=True)
|
|
1239
|
+
continue
|
|
1240
|
+
if res.returncode != 0:
|
|
1241
|
+
# A capped/errored run may say nothing on stderr; show stdout's
|
|
1242
|
+
# tail too so the reason is not a blank.
|
|
1243
|
+
why = (res.stderr.strip() or res.stdout.strip())[:200]
|
|
1244
|
+
print(f" claude exited {res.returncode}: {why} -- skipped", flush=True)
|
|
1245
|
+
consec_fail += 1
|
|
1246
|
+
continue
|
|
1247
|
+
report = res.stdout.strip()
|
|
1248
|
+
if not report:
|
|
1249
|
+
print(" empty report -- skipped", flush=True)
|
|
1250
|
+
consec_fail += 1
|
|
1251
|
+
continue
|
|
1252
|
+
consec_fail = 0
|
|
1253
|
+
|
|
1254
|
+
report_path = _reports_tmp() / (rp.stem + ".report.md")
|
|
1255
|
+
report_path.write_text(report + "\n", encoding="utf-8")
|
|
1256
|
+
ok, reason = _blindness_ok(report)
|
|
1257
|
+
if not ok:
|
|
1258
|
+
bad = _reports_tmp() / (rp.stem + ".CONTAMINATED.md")
|
|
1259
|
+
report_path.replace(bad)
|
|
1260
|
+
print(
|
|
1261
|
+
f" REFUSED ({reason}) -- quarantined {bad.name}, NOT recorded",
|
|
1262
|
+
flush=True,
|
|
1263
|
+
)
|
|
1264
|
+
continue
|
|
1265
|
+
|
|
1266
|
+
rels = [rel for rel, _ in info["members"]]
|
|
1267
|
+
if info["kind"] == "single":
|
|
1268
|
+
cmd_record([rels[0], str(report_path)], model=model, persona=persona)
|
|
1269
|
+
else:
|
|
1270
|
+
cmd_record_batch([str(report_path), *rels], model=model, persona=persona)
|
|
1271
|
+
done += 1
|
|
1272
|
+
|
|
1273
|
+
print(
|
|
1274
|
+
f"\n{'would analyze' if dry else 'analyzed and recorded'} {done} render(s) this run"
|
|
1275
|
+
)
|
|
1276
|
+
return 0
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
# Paths that ARE the document (edits to these mean the session did authorship
|
|
1280
|
+
# work); everything else is tooling/config/provenance = "meta".
|
|
1281
|
+
# NB: the illustrations directory is deliberately excluded -- figures are
|
|
1282
|
+
# mechanical execution, not authorship. Counting them as "doc" auto-keeps
|
|
1283
|
+
# plumbing sessions and hides them from the intelligent read.
|
|
1284
|
+
|
|
1285
|
+
|
|
1286
|
+
def _repo_relative(fp):
|
|
1287
|
+
"""A recorded edit path, made relative to the project tree it names.
|
|
1288
|
+
|
|
1289
|
+
A session records absolute paths, and the checkout it ran in is not
|
|
1290
|
+
necessarily the one this process was started from, nor one that still
|
|
1291
|
+
exists. The trees come from the corpus's recorded list, never from the
|
|
1292
|
+
worktrees present right now, so the same corpus classifies the same way on
|
|
1293
|
+
every machine and after any `git worktree remove` (see `trees`).
|
|
1294
|
+
"""
|
|
1295
|
+
from quirework.provenance import trees
|
|
1296
|
+
|
|
1297
|
+
return trees.relative(trees.recorded(current_project()), fp)
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
def _classify_edit(fp):
|
|
1301
|
+
rel = _repo_relative(fp)
|
|
1302
|
+
options = _options()
|
|
1303
|
+
s = rel.lower()
|
|
1304
|
+
if s.startswith(options._DOC_PREFIXES) or s.startswith(options._DOC_DIRS):
|
|
1305
|
+
return "doc"
|
|
1306
|
+
if options.FOOTPRINT is not None and options.FOOTPRINT.covers(rel, working=True):
|
|
1307
|
+
return "doc"
|
|
1308
|
+
return "meta"
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
def _edit_targets(jsonl_path):
|
|
1312
|
+
"""Count doc vs meta file edits in a session, and tally edited basenames."""
|
|
1313
|
+
doc = meta = 0
|
|
1314
|
+
files = {}
|
|
1315
|
+
with open(jsonl_path, encoding="utf-8") as fh:
|
|
1316
|
+
for line in fh:
|
|
1317
|
+
line = line.strip()
|
|
1318
|
+
if not line:
|
|
1319
|
+
continue
|
|
1320
|
+
try:
|
|
1321
|
+
obj = json.loads(line)
|
|
1322
|
+
except json.JSONDecodeError:
|
|
1323
|
+
continue
|
|
1324
|
+
if obj.get("type") != "assistant":
|
|
1325
|
+
continue
|
|
1326
|
+
content = (obj.get("message") or {}).get("content")
|
|
1327
|
+
if not isinstance(content, list):
|
|
1328
|
+
continue
|
|
1329
|
+
for b in content:
|
|
1330
|
+
if not (isinstance(b, dict) and b.get("type") == "tool_use"):
|
|
1331
|
+
continue
|
|
1332
|
+
if b.get("name") not in ("Edit", "Write", "NotebookEdit"):
|
|
1333
|
+
continue
|
|
1334
|
+
inp = b.get("input") or {}
|
|
1335
|
+
fp = inp.get("file_path") or inp.get("notebook_path")
|
|
1336
|
+
if not fp:
|
|
1337
|
+
continue
|
|
1338
|
+
if _classify_edit(fp) == "doc":
|
|
1339
|
+
doc += 1
|
|
1340
|
+
else:
|
|
1341
|
+
meta += 1
|
|
1342
|
+
bn = os.path.basename(fp)
|
|
1343
|
+
files[bn] = files.get(bn, 0) + 1
|
|
1344
|
+
return doc, meta, files
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
def cmd_triage(args):
|
|
1348
|
+
"""Mechanical relevance prioritizer over local sessions (free, no model calls).
|
|
1349
|
+
|
|
1350
|
+
A session that edits document files (the manuscript, the spec, retained
|
|
1351
|
+
analysis, scratch notes, the companion collections) is 'keep'.
|
|
1352
|
+
A session that does not is 'review':
|
|
1353
|
+
it might still be a substantive discussion that only touched tooling, so it
|
|
1354
|
+
needs an intelligent read before any removal -- the mechanics alone never
|
|
1355
|
+
drop a file.
|
|
1356
|
+
|
|
1357
|
+
'keep' is a PRIORITIZATION signal, not a final verdict: confirm the
|
|
1358
|
+
top-edited column shows canonical content before trusting it, since a single
|
|
1359
|
+
incidental edit near the essay can auto-keep an otherwise plumbing session.
|
|
1360
|
+
"""
|
|
1361
|
+
from quirework.provenance import catalog as CAT
|
|
1362
|
+
from quirework.provenance import codexfilter as CODEX_FLT
|
|
1363
|
+
|
|
1364
|
+
review_only = "--review-list" in args
|
|
1365
|
+
show_all = "--all" in args
|
|
1366
|
+
# Keyed like the rows below, which hold each session's reference.
|
|
1367
|
+
waiting = {
|
|
1368
|
+
str(source_reference(p, current_project()))
|
|
1369
|
+
for family in (store.CC, store.AGY, store.CODEX)
|
|
1370
|
+
for p in _pending(family)
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
def captured(family):
|
|
1374
|
+
"""Pending captures, and with --all the accepted ones they have not
|
|
1375
|
+
replaced, in name order."""
|
|
1376
|
+
chosen = {p.name: p for p in store.files(family, "*.jsonl")} if show_all else {}
|
|
1377
|
+
chosen.update({p.name: p for p in store.pending(family, "*.jsonl")})
|
|
1378
|
+
return [chosen[name] for name in sorted(chosen)]
|
|
1379
|
+
|
|
1380
|
+
rows = []
|
|
1381
|
+
started = {} # reference -> (path, kind) for the kinds that record a cwd
|
|
1382
|
+
for p in captured(store.CC):
|
|
1383
|
+
rel = str(source_reference(p, current_project()))
|
|
1384
|
+
started[rel] = (p, "cc")
|
|
1385
|
+
body, _ = FLT.filter_session(str(p))
|
|
1386
|
+
if len(body.strip()) < MIN_RENDER_CHARS:
|
|
1387
|
+
continue
|
|
1388
|
+
doc, meta, files = _edit_targets(p)
|
|
1389
|
+
verdict = "keep" if doc > 0 else "review"
|
|
1390
|
+
rows.append((rel, len(body) // 4, doc, meta, verdict, files))
|
|
1391
|
+
for p in captured(store.AGY):
|
|
1392
|
+
rel = str(source_reference(p, current_project()))
|
|
1393
|
+
body, _ = CAT._agy_render(p)
|
|
1394
|
+
if len(body.strip()) < MIN_RENDER_CHARS:
|
|
1395
|
+
continue
|
|
1396
|
+
doc, meta, files = CAT._agy_edit_targets(p)
|
|
1397
|
+
verdict = "keep" if doc > 0 else "review"
|
|
1398
|
+
rows.append((rel, len(body) // 4, doc, meta, verdict, files))
|
|
1399
|
+
for p in captured(store.CODEX):
|
|
1400
|
+
rel = str(source_reference(p, current_project()))
|
|
1401
|
+
started[rel] = (p, "codex")
|
|
1402
|
+
body, _ = CODEX_FLT.filter_session(p)
|
|
1403
|
+
if len(body.strip()) < MIN_RENDER_CHARS:
|
|
1404
|
+
continue
|
|
1405
|
+
doc, meta, files = CODEX_FLT.edit_targets(p, _classify_edit)
|
|
1406
|
+
verdict = "keep" if doc > 0 else "review"
|
|
1407
|
+
rows.append((rel, len(body) // 4, doc, meta, verdict, files))
|
|
1408
|
+
short_pending = []
|
|
1409
|
+
if not show_all:
|
|
1410
|
+
short_pending = sorted(waiting - {r[0] for r in rows})
|
|
1411
|
+
rows.sort(key=lambda r: -r[1])
|
|
1412
|
+
listed = [started[r[0]] for r in rows if r[0] in started]
|
|
1413
|
+
found = CAT.unplaced(
|
|
1414
|
+
[p for p, kind in listed if kind == "cc"],
|
|
1415
|
+
[p for p, kind in listed if kind == "codex"],
|
|
1416
|
+
)
|
|
1417
|
+
if found:
|
|
1418
|
+
print("triage: the verdicts below undercount document edits.", file=sys.stderr)
|
|
1419
|
+
CAT.report_unplaced(found)
|
|
1420
|
+
|
|
1421
|
+
if review_only:
|
|
1422
|
+
for rel, _t, _d, _m, verdict, _f in rows:
|
|
1423
|
+
if verdict == "review":
|
|
1424
|
+
print(rel)
|
|
1425
|
+
return 0
|
|
1426
|
+
|
|
1427
|
+
if not rows:
|
|
1428
|
+
if not show_all:
|
|
1429
|
+
if short_pending:
|
|
1430
|
+
print(
|
|
1431
|
+
f"nothing above the {MIN_RENDER_CHARS}-char render floor, but "
|
|
1432
|
+
f"{len(short_pending)} pending session(s) fell below "
|
|
1433
|
+
"it -- these can still be a real stub, a crashed run, or "
|
|
1434
|
+
"evaluation-seat exhaust; read each before curating:"
|
|
1435
|
+
)
|
|
1436
|
+
for rel in short_pending:
|
|
1437
|
+
print(f" {rel}")
|
|
1438
|
+
else:
|
|
1439
|
+
print(
|
|
1440
|
+
"nothing pending triage: every captured session is "
|
|
1441
|
+
"accepted (--all lists them anyway)."
|
|
1442
|
+
)
|
|
1443
|
+
else:
|
|
1444
|
+
print("no sessions to triage.")
|
|
1445
|
+
return 0
|
|
1446
|
+
|
|
1447
|
+
w = max(len(r[0]) for r in rows)
|
|
1448
|
+
print("=== session relevance triage (mechanical prioritizer: by files edited) ===")
|
|
1449
|
+
if not show_all:
|
|
1450
|
+
print("(pending captures only; --all includes accepted sessions)")
|
|
1451
|
+
print(
|
|
1452
|
+
f"{'verdict':7} {'~tok':>7} {'doc':>4} {'meta':>5} {'session':{w}} top edited"
|
|
1453
|
+
)
|
|
1454
|
+
for rel, toks, doc, meta, verdict, files in rows:
|
|
1455
|
+
top = ", ".join(
|
|
1456
|
+
f"{n}x{c}" for n, c in sorted(files.items(), key=lambda x: -x[1])[:3]
|
|
1457
|
+
)
|
|
1458
|
+
print(
|
|
1459
|
+
f"{verdict:7} {toks:>7,} {doc:>4} {meta:>5} {rel:{w}} {top or '(no edits)'}"
|
|
1460
|
+
)
|
|
1461
|
+
nkeep = sum(r[4] == "keep" for r in rows)
|
|
1462
|
+
nrev = sum(r[4] == "review" for r in rows)
|
|
1463
|
+
print(
|
|
1464
|
+
f"\nkeep={nkeep} (edited the document) review={nrev} (no doc edits -> need an "
|
|
1465
|
+
f"intelligent read before removal)"
|
|
1466
|
+
)
|
|
1467
|
+
print(
|
|
1468
|
+
"'keep' is a prioritization signal, not a final verdict: confirm the top-edited"
|
|
1469
|
+
)
|
|
1470
|
+
print("Confirm that the top-edited column shows configured work material,")
|
|
1471
|
+
print("not only project guides or tooling files.")
|
|
1472
|
+
print("web/era-a sessions are not shown; judge those manually.")
|
|
1473
|
+
if short_pending:
|
|
1474
|
+
print(
|
|
1475
|
+
f"\n{len(short_pending)} more pending session(s) fell below the "
|
|
1476
|
+
f"{MIN_RENDER_CHARS}-char render floor and are NOT in the table above "
|
|
1477
|
+
"-- still pending, read each before curating:"
|
|
1478
|
+
)
|
|
1479
|
+
for rel in short_pending:
|
|
1480
|
+
print(f" {rel}")
|
|
1481
|
+
return 0
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
USAGE = """usage: quirework provenance coldread [SUBCOMMAND] [OPTIONS]
|
|
1485
|
+
|
|
1486
|
+
Blind authorship cold-read: a content-addressed cache over per-source analyst
|
|
1487
|
+
reports. The cache key folds in the source bytes, the filter recipe, the prompt
|
|
1488
|
+
and the model, so changing any of them misses deliberately -- a report is never
|
|
1489
|
+
silently reused under a different recipe.
|
|
1490
|
+
|
|
1491
|
+
subcommands (default: status):
|
|
1492
|
+
status per-source cache state (hit / MISS)
|
|
1493
|
+
triage mechanical keep/review prioritizer over captured
|
|
1494
|
+
cc, agy, and codex sessions (free, no model calls)
|
|
1495
|
+
--review-list print only the 'review' sessions
|
|
1496
|
+
--all include accepted sessions (default: only the
|
|
1497
|
+
pending captures awaiting curation)
|
|
1498
|
+
render write the filtered transcripts the analysts read
|
|
1499
|
+
--batch-under N batch sources under N tokens
|
|
1500
|
+
--batch-max N cap a batch at N tokens
|
|
1501
|
+
verify-blind scan renders for project-framing fingerprints
|
|
1502
|
+
(a cc-session hit is a hard failure)
|
|
1503
|
+
analyze fan out one blind `claude -p` per render
|
|
1504
|
+
--only SUBSTR restrict to renders matching SUBSTR
|
|
1505
|
+
--model M analyst model (default: the configured one)
|
|
1506
|
+
--persona ID reusable analyst profile; separate cache namespace
|
|
1507
|
+
status --persona ID inspect that profile's cache
|
|
1508
|
+
--limit N analyze at most N renders
|
|
1509
|
+
--oldest walk the corpus oldest-first (incremental sweeps
|
|
1510
|
+
then cover a contiguous stretch of the arc)
|
|
1511
|
+
--timeout S per-analyst timeout in seconds
|
|
1512
|
+
--dry-run print what would run, run nothing
|
|
1513
|
+
record SOURCE REPORT cache one report against its source
|
|
1514
|
+
record-batch REPORT SRC.. cache one batched report against its sources
|
|
1515
|
+
|
|
1516
|
+
options:
|
|
1517
|
+
-h, --help show this help message and exit
|
|
1518
|
+
"""
|
|
1519
|
+
|
|
1520
|
+
|
|
1521
|
+
# --------------------------------------------------------------------------- #
|
|
1522
|
+
# discards -- the authority a transcript cannot record
|
|
1523
|
+
# --------------------------------------------------------------------------- #
|
|
1524
|
+
# A blind analyst reads a session and judges who drove it. But work is also
|
|
1525
|
+
# rejected *after* the session, in git, and that leaves no trace in any
|
|
1526
|
+
# transcript: the author let an agent run with little review, several commits
|
|
1527
|
+
# landed, and he discarded them with a hard reset. No filter version can fix
|
|
1528
|
+
# this; the transcript records the talk, not what became of the artifact.
|
|
1529
|
+
#
|
|
1530
|
+
# WHAT THIS ACTUALLY FOUND, recorded because it is a negative result and the
|
|
1531
|
+
# temptation is to quietly keep the tool and drop the finding. Over this corpus
|
|
1532
|
+
# it flags 28 of 495 commits (6%) across 16 sessions -- and those sessions'
|
|
1533
|
+
# reports already read as human-led ("The human drove the substance
|
|
1534
|
+
# throughout", "human-orchestrated throughout"). It does NOT surface sessions
|
|
1535
|
+
# where an analyst over-credited the AI, which is what it was built to catch.
|
|
1536
|
+
#
|
|
1537
|
+
# Worse for the premise: of the 15 canon commits discarded on 2026-07-12, only
|
|
1538
|
+
# 3 are attributable to a session still in the corpus. The rest were made by
|
|
1539
|
+
# sessions since curated out, or outside a session entirely. So the detector
|
|
1540
|
+
# cannot see the very event that motivated it. Keep it as a cheap check, and
|
|
1541
|
+
# do not mistake a clean run for evidence that nothing was thrown away.
|
|
1542
|
+
#
|
|
1543
|
+
# (An earlier attribution pass matched any SHA appearing in a transcript and
|
|
1544
|
+
# scored 7e133544 with 13 discarded commits. It created none -- it was a
|
|
1545
|
+
# session *reviewing* another's commits and quoting their SHAs. Mentioning is
|
|
1546
|
+
# not making, which is why only `git commit`'s own echo is counted below.)
|
|
1547
|
+
#
|
|
1548
|
+
# So this reports the outcome beside the report, and deliberately does no more.
|
|
1549
|
+
# It is a DISCREPANCY DETECTOR, not a weight: it flags sessions worth a second
|
|
1550
|
+
# look and never adjusts a verdict. Two reasons for that restraint. Handing an
|
|
1551
|
+
# analyst "12 of 15 commits were discarded" leads the witness -- the fact
|
|
1552
|
+
# carries its interpretation with it. And letting the *summarizer* reweight on
|
|
1553
|
+
# it hands the corpus's best-documented failure mode (an AI narrating the
|
|
1554
|
+
# collaboration as more human-directed than the transcript supports) a lever
|
|
1555
|
+
# pointing straight at the flattering conclusion.
|
|
1556
|
+
#
|
|
1557
|
+
# Read the numbers as a question, never as an answer, and mind the caveats:
|
|
1558
|
+
# * survival != endorsement. A commit in main may mean it was approved, or
|
|
1559
|
+
# that nobody looked. A discard may be rejection, supersession, or an
|
|
1560
|
+
# abandoned branch.
|
|
1561
|
+
# * volume != weight. A one-word "no" that stops a chapter outweighs fifteen
|
|
1562
|
+
# discarded commits and shows up here as nothing.
|
|
1563
|
+
# * it only sees authority exercised AFTER a commit. Most correction happens
|
|
1564
|
+
# in the dialogue, before anything is committed.
|
|
1565
|
+
|
|
1566
|
+
# `git commit` echoes "[main 1a2b3c4] subject", which is proof the commit was
|
|
1567
|
+
# MADE here -- unlike a bare SHA, which a session that merely *reviews* commits
|
|
1568
|
+
# quotes constantly.
|
|
1569
|
+
_COMMIT_OUT = re.compile(
|
|
1570
|
+
r"\[(?:detached HEAD|[\w./-]+)\s+([0-9a-f]{7,40})\]\s+([^\\\"\n]+)"
|
|
1571
|
+
)
|
|
1572
|
+
|
|
1573
|
+
|
|
1574
|
+
def _main_index():
|
|
1575
|
+
"""``(short-sha prefixes, subjects)`` for everything reachable from main."""
|
|
1576
|
+
|
|
1577
|
+
def run(fmt):
|
|
1578
|
+
try:
|
|
1579
|
+
return subprocess.run(
|
|
1580
|
+
[
|
|
1581
|
+
"git",
|
|
1582
|
+
"-C",
|
|
1583
|
+
str(current_project().root),
|
|
1584
|
+
"log",
|
|
1585
|
+
"--format=" + fmt,
|
|
1586
|
+
"main",
|
|
1587
|
+
],
|
|
1588
|
+
capture_output=True,
|
|
1589
|
+
text=True,
|
|
1590
|
+
check=True,
|
|
1591
|
+
).stdout.splitlines()
|
|
1592
|
+
except Exception:
|
|
1593
|
+
return []
|
|
1594
|
+
|
|
1595
|
+
shas = {h[:7] for h in run("%H")}
|
|
1596
|
+
return shas, {s.strip() for s in run("%s")}
|
|
1597
|
+
|
|
1598
|
+
|
|
1599
|
+
def session_commits(path):
|
|
1600
|
+
"""``[(sha, subject)]`` for commits created in this session's transcript."""
|
|
1601
|
+
out, seen = [], set()
|
|
1602
|
+
try:
|
|
1603
|
+
text = open(path, encoding="utf-8", errors="replace").read()
|
|
1604
|
+
except OSError:
|
|
1605
|
+
return out
|
|
1606
|
+
for sha, subj in _COMMIT_OUT.findall(text):
|
|
1607
|
+
subj = subj.strip().rstrip("\\n").strip()
|
|
1608
|
+
if sha[:7] in seen or not subj:
|
|
1609
|
+
continue
|
|
1610
|
+
seen.add(sha[:7])
|
|
1611
|
+
out.append((sha[:7], subj))
|
|
1612
|
+
return out
|
|
1613
|
+
|
|
1614
|
+
|
|
1615
|
+
def cmd_discards(args):
|
|
1616
|
+
"""Per session: commits made, how many survive in main, how many were cut."""
|
|
1617
|
+
threshold = 1
|
|
1618
|
+
for i, a in enumerate(args):
|
|
1619
|
+
if a == "--min" and i + 1 < len(args):
|
|
1620
|
+
threshold = int(args[i + 1])
|
|
1621
|
+
main_shas, main_subjects = _main_index()
|
|
1622
|
+
if not main_shas:
|
|
1623
|
+
print("discards: cannot read `main`; is this a git repo?", file=sys.stderr)
|
|
1624
|
+
return 2
|
|
1625
|
+
manifest = load_manifest()
|
|
1626
|
+
rows = []
|
|
1627
|
+
# Whole corpus, both scopes: this counts commits a session made and how many
|
|
1628
|
+
# survive, which is as true of a session that built an instrument as of one
|
|
1629
|
+
# that wrote a chapter. Scope decides which cold-read *reads* a session, not
|
|
1630
|
+
# which tools may count it. A resumed session's commits are counted once,
|
|
1631
|
+
# in the capture that continued it (see `sources`).
|
|
1632
|
+
for path, era in iter_sources(None):
|
|
1633
|
+
if path.suffix != ".jsonl":
|
|
1634
|
+
continue
|
|
1635
|
+
made = session_commits(path)
|
|
1636
|
+
if not made:
|
|
1637
|
+
continue
|
|
1638
|
+
# The project folds follow-up fixes into the previous commit (soft reset
|
|
1639
|
+
# + re-commit), so a KEPT commit routinely changes SHA. Matching the
|
|
1640
|
+
# subject as well is what separates a rewrite from a rejection: over
|
|
1641
|
+
# this corpus, SHA-only scored 150 discards where 47 were real.
|
|
1642
|
+
gone = [
|
|
1643
|
+
(s, subj)
|
|
1644
|
+
for s, subj in made
|
|
1645
|
+
if s not in main_shas and subj not in main_subjects
|
|
1646
|
+
]
|
|
1647
|
+
rel = str(source_reference(path, current_project()))
|
|
1648
|
+
rows.append(
|
|
1649
|
+
(
|
|
1650
|
+
len(gone),
|
|
1651
|
+
len(made),
|
|
1652
|
+
path.stem[:8],
|
|
1653
|
+
source_date(path),
|
|
1654
|
+
(manifest["sources"].get(rel) or {}).get("report", "-"),
|
|
1655
|
+
gone,
|
|
1656
|
+
)
|
|
1657
|
+
)
|
|
1658
|
+
rows.sort(key=lambda r: (-r[0], r[3]))
|
|
1659
|
+
|
|
1660
|
+
flagged = [r for r in rows if r[0] >= threshold]
|
|
1661
|
+
print(f"{'session':10} {'date':11} {'made':>5} {'cut':>4} report")
|
|
1662
|
+
print("-" * 74)
|
|
1663
|
+
for gone_n, made_n, sess, date, report, _ in flagged:
|
|
1664
|
+
print(f"{sess:10} {date:11} {made_n:>5} {gone_n:>4} {report.split('/')[-1]}")
|
|
1665
|
+
tot_made = sum(r[1] for r in rows)
|
|
1666
|
+
tot_gone = sum(r[0] for r in rows)
|
|
1667
|
+
print(
|
|
1668
|
+
f"\n{len(rows)} session(s) committed; {tot_made} commit(s) made, "
|
|
1669
|
+
f"{tot_gone} never reached main ({tot_gone / tot_made:.0%})."
|
|
1670
|
+
)
|
|
1671
|
+
print(f"{len(flagged)} session(s) at or above the --min {threshold} threshold.")
|
|
1672
|
+
if flagged:
|
|
1673
|
+
print(
|
|
1674
|
+
"\nThis is a QUESTION, not an adjustment. Where a session's report credits\n"
|
|
1675
|
+
"the AI with substance that was later cut, the report is not wrong about\n"
|
|
1676
|
+
"the transcript -- the transcript could not see the reset. Read the two\n"
|
|
1677
|
+
"together; do not silently reweight the verdict."
|
|
1678
|
+
)
|
|
1679
|
+
return 0
|
|
1680
|
+
|
|
1681
|
+
|
|
1682
|
+
def _analyze_locked(args):
|
|
1683
|
+
"""`analyze` under the sweep lock: one at a time, and say so if not."""
|
|
1684
|
+
try:
|
|
1685
|
+
with sweep_lock():
|
|
1686
|
+
return cmd_analyze(args)
|
|
1687
|
+
except AlreadyRunning as e:
|
|
1688
|
+
print(
|
|
1689
|
+
f"analyze: another cold-read sweep is already running ({e}).\n"
|
|
1690
|
+
f" Nothing shards the work -- two sweeps walk the same renders in\n"
|
|
1691
|
+
f" the same order, pay for the same analysts, and clobber each\n"
|
|
1692
|
+
f" other's manifest entries. Wait for it, or stop it first.\n"
|
|
1693
|
+
f" Lock: {_settings().SWEEP_LOCK}",
|
|
1694
|
+
file=sys.stderr,
|
|
1695
|
+
)
|
|
1696
|
+
return 1
|
|
1697
|
+
|
|
1698
|
+
|
|
1699
|
+
def main(argv):
|
|
1700
|
+
cmds = {
|
|
1701
|
+
"status": cmd_status,
|
|
1702
|
+
"triage": cmd_triage,
|
|
1703
|
+
"render": cmd_render,
|
|
1704
|
+
"verify-blind": cmd_verify_blind,
|
|
1705
|
+
"analyze": _analyze_locked,
|
|
1706
|
+
"record": cmd_record,
|
|
1707
|
+
"record-batch": cmd_record_batch,
|
|
1708
|
+
"discards": cmd_discards,
|
|
1709
|
+
}
|
|
1710
|
+
# Checked across the whole argv, not just argv[0]: every subcommand parses
|
|
1711
|
+
# flags with an "ignore unknown arg" loop, so a `coldread analyze --help`
|
|
1712
|
+
# that reached the subcommand would shrug off the flag and *run the
|
|
1713
|
+
# analysis*. There is no positional here that can legitimately be `-h`.
|
|
1714
|
+
if any(a in ("-h", "--help") for a in argv):
|
|
1715
|
+
print(USAGE, end="")
|
|
1716
|
+
return 0
|
|
1717
|
+
cmd = argv[0] if argv else "status"
|
|
1718
|
+
if cmd not in cmds:
|
|
1719
|
+
print(
|
|
1720
|
+
f"unknown command: {cmd}\nsubcommands: {', '.join(cmds)}", file=sys.stderr
|
|
1721
|
+
)
|
|
1722
|
+
return 2
|
|
1723
|
+
try:
|
|
1724
|
+
return cmds[cmd](argv[1:]) or 0
|
|
1725
|
+
except (ValueError, OSError, subprocess.SubprocessError) as exc:
|
|
1726
|
+
print(f"coldread: {exc}", file=sys.stderr)
|
|
1727
|
+
return 2
|
|
1728
|
+
|
|
1729
|
+
|
|
1730
|
+
if __name__ == "__main__":
|
|
1731
|
+
sys.exit(main(sys.argv[1:]))
|