seqforge 2026.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqforge/__init__.py +16 -0
- seqforge/cli/__init__.py +38 -0
- seqforge/cli/__main__.py +8 -0
- seqforge/cli/_common.py +105 -0
- seqforge/cli/compose.py +119 -0
- seqforge/cli/eval.py +103 -0
- seqforge/cli/harvest.py +417 -0
- seqforge/cli/hook.py +247 -0
- seqforge/cli/io.py +502 -0
- seqforge/cli/kb.py +348 -0
- seqforge/cli/manifest.py +536 -0
- seqforge/cli/probe.py +43 -0
- seqforge/cli/processing.py +192 -0
- seqforge/cli/project.py +52 -0
- seqforge/cli/resolve.py +55 -0
- seqforge/cli/root.py +66 -0
- seqforge/cli/run.py +463 -0
- seqforge/cli/schema.py +41 -0
- seqforge/compose/__init__.py +28 -0
- seqforge/compose/core.py +515 -0
- seqforge/compose/gates.py +113 -0
- seqforge/compose/params.py +447 -0
- seqforge/e2e.py +1926 -0
- seqforge/evals/__init__.py +78 -0
- seqforge/evals/case.py +382 -0
- seqforge/evals/grade.py +300 -0
- seqforge/evals/run.py +420 -0
- seqforge/harvest/__init__.py +121 -0
- seqforge/harvest/extract.py +319 -0
- seqforge/harvest/fields.py +212 -0
- seqforge/harvest/normalize.py +537 -0
- seqforge/harvest/prep.py +41 -0
- seqforge/harvest/providers.py +321 -0
- seqforge/harvest/verify.py +251 -0
- seqforge/hooks/__init__.py +33 -0
- seqforge/hooks/guards.py +214 -0
- seqforge/io/__init__.py +61 -0
- seqforge/io/archive.py +450 -0
- seqforge/io/attributes.py +190 -0
- seqforge/io/biosample/attributes.json +6341 -0
- seqforge/io/efo/labels.json +55 -0
- seqforge/io/efo.py +138 -0
- seqforge/io/onlist.py +661 -0
- seqforge/io/onlists/3M-february-2018.codes.gz +0 -0
- seqforge/io/onlists/737K-arc-v1.codes.gz +0 -0
- seqforge/io/onlists/737K-august-2016.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3.codes.gz +0 -0
- seqforge/io/onlists/index.json +74 -0
- seqforge/io/remote.py +659 -0
- seqforge/io/taxonomy.py +194 -0
- seqforge/kb/__init__.py +62 -0
- seqforge/kb/anchor.py +169 -0
- seqforge/kb/generate.py +147 -0
- seqforge/kb/loader.py +152 -0
- seqforge/kb/roundtrip.py +112 -0
- seqforge/kb/schema.py +422 -0
- seqforge/kb/specs/10x-3p-gex/spec.yaml +62 -0
- seqforge/kb/specs/10x-3p-gex-v2/README.md +41 -0
- seqforge/kb/specs/10x-3p-gex-v2/spec.yaml +83 -0
- seqforge/kb/specs/10x-3p-gex-v3/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3/spec.yaml +118 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/spec.yaml +124 -0
- seqforge/kb/specs/bd-rhapsody-wta/README.md +103 -0
- seqforge/kb/specs/bd-rhapsody-wta/spec.yaml +130 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced/spec.yaml +99 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v1/spec.yaml +93 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v2/spec.yaml +81 -0
- seqforge/kb/specs/bulk-rnaseq-pe/README.md +35 -0
- seqforge/kb/specs/bulk-rnaseq-pe/spec.yaml +97 -0
- seqforge/kb/specs/splitseq/README.md +51 -0
- seqforge/kb/specs/splitseq/spec.yaml +157 -0
- seqforge/manifest/__init__.py +61 -0
- seqforge/manifest/fill.py +531 -0
- seqforge/manifest/hash.py +77 -0
- seqforge/manifest/instruct.py +114 -0
- seqforge/manifest/policy.py +409 -0
- seqforge/manifest/validate.py +274 -0
- seqforge/models/__init__.py +268 -0
- seqforge/models/assertion.py +68 -0
- seqforge/models/base.py +100 -0
- seqforge/models/blocker.py +71 -0
- seqforge/models/conflict.py +47 -0
- seqforge/models/dataset.py +320 -0
- seqforge/models/evidenced.py +54 -0
- seqforge/models/observation.py +157 -0
- seqforge/models/processing.py +231 -0
- seqforge/models/records.py +145 -0
- seqforge/models/resolve.py +216 -0
- seqforge/probe/__init__.py +46 -0
- seqforge/probe/core.py +232 -0
- seqforge/probe/signals.py +250 -0
- seqforge/probe/streaming.py +118 -0
- seqforge/project.py +177 -0
- seqforge/py.typed +0 -0
- seqforge/resolve/__init__.py +98 -0
- seqforge/resolve/assign.py +204 -0
- seqforge/resolve/cache.py +119 -0
- seqforge/resolve/confuse.py +215 -0
- seqforge/resolve/engine.py +646 -0
- seqforge/resolve/escalate.py +668 -0
- seqforge/resolve/evaluators.py +306 -0
- seqforge/resolve/geometry.py +89 -0
- seqforge/resolve/group.py +85 -0
- seqforge/resolve/records.py +550 -0
- seqforge/resolve/scoring.py +373 -0
- seqforge/resolve/window.py +206 -0
- seqforge/workflows/__init__.py +234 -0
- seqforge/workflows/cram.py +117 -0
- seqforge/workflows/h5ad.py +368 -0
- seqforge/workflows/map/star.smk +101 -0
- seqforge/workflows/map/starsolo.smk +360 -0
- seqforge/workflows/qc.py +157 -0
- seqforge/workspace.py +125 -0
- seqforge-2026.7.1.dist-info/METADATA +125 -0
- seqforge-2026.7.1.dist-info/RECORD +124 -0
- seqforge-2026.7.1.dist-info/WHEEL +4 -0
- seqforge-2026.7.1.dist-info/entry_points.txt +2 -0
- seqforge-2026.7.1.dist-info/licenses/LICENSE +21 -0
seqforge/cli/harvest.py
ADDED
|
@@ -0,0 +1,417 @@
|
|
|
1
|
+
"""`seqforge harvest` -- prose/metadata -> span-verified Assertions (the one LLM touchpoint).
|
|
2
|
+
|
|
3
|
+
`_harvest_extract_pipeline` is the stage body, returned as a value so `seqforge run` can chain it.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from enum import StrEnum
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
12
|
+
|
|
13
|
+
import typer
|
|
14
|
+
from pydantic import ValidationError
|
|
15
|
+
|
|
16
|
+
from ..manifest import instructions_from_assertions
|
|
17
|
+
from ..workspace import documents_dir, logs_dir, readable
|
|
18
|
+
from ._common import _emit, _StageOut
|
|
19
|
+
from .root import harvest_app
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from ..harvest.normalize import PdfBackend
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class PdfBackendChoice(StrEnum):
|
|
26
|
+
"""Which engine opens a PDF, exposed as ``--pdf-backend``. ``pymupdf`` (AGPL-3.0) is the default
|
|
27
|
+
because it read every real manuscript in the eval, including ones ``pypdf`` (BSD) cannot parse;
|
|
28
|
+
``pypdf`` stays as the permissive fallback. Neither reorders geometrically — tables come from
|
|
29
|
+
pdfplumber either way, so the choice is really which reader survives more files."""
|
|
30
|
+
|
|
31
|
+
pypdf = "pypdf"
|
|
32
|
+
pymupdf = "pymupdf"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@harvest_app.command("normalize")
|
|
36
|
+
def harvest_normalize(
|
|
37
|
+
docs: list[Path] = typer.Argument(
|
|
38
|
+
None, help="Reference documents to cite (.txt/.md/.pdf/.xlsx)."
|
|
39
|
+
),
|
|
40
|
+
instruction: list[Path] = typer.Option(
|
|
41
|
+
[],
|
|
42
|
+
"--instruction",
|
|
43
|
+
help="Document(s) authored FOR seqforge (e.g. alignment_instruction.md).",
|
|
44
|
+
),
|
|
45
|
+
pdf_backend: PdfBackendChoice = typer.Option(
|
|
46
|
+
PdfBackendChoice.pymupdf,
|
|
47
|
+
"--pdf-backend",
|
|
48
|
+
help="PDF text extractor: pymupdf (default, AGPL, reads more PDFs) | pypdf (BSD fallback).",
|
|
49
|
+
),
|
|
50
|
+
workspace: Path = typer.Option(
|
|
51
|
+
Path("."), "-C", "--workspace", help="Root for seqforge/ state."
|
|
52
|
+
),
|
|
53
|
+
) -> None:
|
|
54
|
+
"""Extract each document ONCE into the canonical text that spans are computed against.
|
|
55
|
+
|
|
56
|
+
A document's ROLE is the flag it arrived under, never its filename: only an --instruction document
|
|
57
|
+
may set processing.*. `alignment_instruction.md` is a convention you pass here, load-bearing
|
|
58
|
+
nowhere — a filename trigger would be spoofable by renaming a downloaded PDF.
|
|
59
|
+
"""
|
|
60
|
+
from ..harvest import normalize_document
|
|
61
|
+
|
|
62
|
+
backend = cast("PdfBackend", pdf_backend.value)
|
|
63
|
+
outdir = documents_dir(workspace)
|
|
64
|
+
outdir.mkdir(parents=True, exist_ok=True)
|
|
65
|
+
rows = []
|
|
66
|
+
for doc, role in _roled(docs, instruction):
|
|
67
|
+
try:
|
|
68
|
+
nd = normalize_document(doc, role=role, pdf_backend=backend)
|
|
69
|
+
except (OSError, RuntimeError) as exc:
|
|
70
|
+
typer.echo(f"{doc}: {exc}", err=True)
|
|
71
|
+
raise typer.Exit(1) from exc
|
|
72
|
+
target = outdir / _document_filename(nd)
|
|
73
|
+
target.write_text(nd.text)
|
|
74
|
+
rows.append(
|
|
75
|
+
{
|
|
76
|
+
"source": nd.source_basename,
|
|
77
|
+
"role": nd.role,
|
|
78
|
+
"scope": nd.scope,
|
|
79
|
+
"subject": nd.subject,
|
|
80
|
+
"doc_sha256": nd.doc_sha256,
|
|
81
|
+
"normalized_sha256": nd.normalized_sha256,
|
|
82
|
+
"normalizer_version": nd.normalizer_version,
|
|
83
|
+
"n_chars": nd.n_chars,
|
|
84
|
+
"path": str(target.relative_to(Path(workspace))),
|
|
85
|
+
}
|
|
86
|
+
)
|
|
87
|
+
typer.echo(json.dumps({"normalized": rows}, indent=2))
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _document_filename(doc: Any) -> str:
|
|
91
|
+
"""``paper.pdf`` -> ``paper-3f8a1c2d9b04.txt``; a record -> ``sample-SAMN40935621-....txt``.
|
|
92
|
+
|
|
93
|
+
The hash stays, because two documents can share a name and the identity is the hash. But a
|
|
94
|
+
directory of bare 64-hex filenames is a directory you cannot read, and an early build's document
|
|
95
|
+
cache was exactly that: nothing in it said which file was the paper. The source
|
|
96
|
+
name is already known -- we opened the file -- so printing it costs nothing and no model is
|
|
97
|
+
involved in producing it.
|
|
98
|
+
"""
|
|
99
|
+
return readable(Path(doc.source_basename).stem, doc.doc_sha256) + ".txt"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _roled(docs: list[Path] | None, instruction: list[Path] | None) -> list[tuple[Path, str]]:
|
|
103
|
+
"""Pair each document with the ROLE its flag assigned. Code owns role; a filename never does."""
|
|
104
|
+
pairs: list[tuple[Path, str]] = [(d, "reference") for d in (docs or [])]
|
|
105
|
+
pairs += [(d, "instruction") for d in (instruction or [])]
|
|
106
|
+
if not pairs:
|
|
107
|
+
typer.echo("give at least one document, or --instruction FILE", err=True)
|
|
108
|
+
raise typer.Exit(2)
|
|
109
|
+
return pairs
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@harvest_app.command("extract")
|
|
113
|
+
def harvest_extract(
|
|
114
|
+
docs: list[Path] = typer.Argument(
|
|
115
|
+
None, help="Reference documents to cite (.txt/.md/.pdf/.xlsx)."
|
|
116
|
+
),
|
|
117
|
+
instruction: list[Path] = typer.Option(
|
|
118
|
+
[],
|
|
119
|
+
"--instruction",
|
|
120
|
+
help="Document(s) authored FOR seqforge; only these may set processing.*.",
|
|
121
|
+
),
|
|
122
|
+
records_path: Path | None = typer.Option(
|
|
123
|
+
None,
|
|
124
|
+
"--records",
|
|
125
|
+
help="A record set from `seqforge io records`. Each record's free text becomes its OWN "
|
|
126
|
+
"document, which is how a claim gets to name a sample.",
|
|
127
|
+
),
|
|
128
|
+
provider: str | None = typer.Option(
|
|
129
|
+
None, "--provider", help="anthropic | deepseek | openai-compatible (default: auto-detect)."
|
|
130
|
+
),
|
|
131
|
+
model: str | None = typer.Option(
|
|
132
|
+
None, "--model", help="Override the model (default: the provider's own default)."
|
|
133
|
+
),
|
|
134
|
+
verify: bool = typer.Option(
|
|
135
|
+
True, "--verify/--no-verify", help="Span-verify the drafts immediately."
|
|
136
|
+
),
|
|
137
|
+
pdf_backend: PdfBackendChoice = typer.Option(
|
|
138
|
+
PdfBackendChoice.pymupdf,
|
|
139
|
+
"--pdf-backend",
|
|
140
|
+
help="PDF text extractor: pymupdf (default, AGPL, reads more PDFs) | pypdf (BSD fallback).",
|
|
141
|
+
),
|
|
142
|
+
workspace: Path = typer.Option(
|
|
143
|
+
Path("."), "-C", "--workspace", help="Root for seqforge/ state."
|
|
144
|
+
),
|
|
145
|
+
) -> None:
|
|
146
|
+
"""The ONE LLM touchpoint: prose -> AssertionDraft[] -> (verified) Assertion[].
|
|
147
|
+
|
|
148
|
+
The model only proposes `{field, value, quote}`; code computes the offsets and decides what
|
|
149
|
+
survives — which is what makes the provider swappable. Auto-detects DEEPSEEK_API_KEY /
|
|
150
|
+
ANTHROPIC_API_KEY. Exit 1 if the LLM surface is unavailable, 4 if any claim fails verification.
|
|
151
|
+
|
|
152
|
+
**`--records` is how a claim names a sample.** Each archive record is rendered as its own
|
|
153
|
+
document and asked only what a record at that level can answer: a BioSample's document is asked
|
|
154
|
+
for sample attributes and never for a chemistry; an experiment's protocol paragraph is asked for
|
|
155
|
+
the chemistry and nothing else. Since a sample's document contains one sample's prose, "which
|
|
156
|
+
sample" is answered by which file we handed the model — the model never names one, and cannot.
|
|
157
|
+
"""
|
|
158
|
+
_emit(
|
|
159
|
+
_harvest_extract_pipeline(
|
|
160
|
+
docs=docs,
|
|
161
|
+
instruction=instruction,
|
|
162
|
+
records_path=records_path,
|
|
163
|
+
provider=provider,
|
|
164
|
+
model=model,
|
|
165
|
+
verify=verify,
|
|
166
|
+
workspace=workspace,
|
|
167
|
+
pdf_backend=cast("PdfBackend", pdf_backend.value),
|
|
168
|
+
)
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _harvest_extract_pipeline(
|
|
173
|
+
*,
|
|
174
|
+
docs: list[Path] | None,
|
|
175
|
+
instruction: list[Path] | None,
|
|
176
|
+
records_path: Path | None,
|
|
177
|
+
provider: str | None,
|
|
178
|
+
model: str | None,
|
|
179
|
+
verify: bool,
|
|
180
|
+
workspace: Path,
|
|
181
|
+
pdf_backend: PdfBackend = "pymupdf",
|
|
182
|
+
) -> _StageOut:
|
|
183
|
+
"""The body of ``harvest extract``, returned as a value so ``seqforge run`` can chain it.
|
|
184
|
+
|
|
185
|
+
The one LLM stage, and the one place ``run`` cannot be fully deterministic — hence ``--no-llm``,
|
|
186
|
+
which is the caller choosing not to enter here at all. Every exit is a ``_StageOut``: exit 1 if no
|
|
187
|
+
provider or the endpoint fails, exit 4 if a claim fails the span tripwire (a rejected claim needs
|
|
188
|
+
a human, not a silent drop). On success it still writes ``assertions.json`` and the rendered
|
|
189
|
+
documents to disk, because a span citation is only checkable while the exact text survives.
|
|
190
|
+
"""
|
|
191
|
+
from ..harvest import (
|
|
192
|
+
ExtractionOutcome,
|
|
193
|
+
ExtractUnavailable,
|
|
194
|
+
NormalizedDoc,
|
|
195
|
+
ProviderUnavailable,
|
|
196
|
+
UnreadableDocument,
|
|
197
|
+
extract_drafts,
|
|
198
|
+
has_prose,
|
|
199
|
+
normalize_document,
|
|
200
|
+
normalize_record,
|
|
201
|
+
resolve_provider,
|
|
202
|
+
verify_drafts,
|
|
203
|
+
)
|
|
204
|
+
from ..kb import load_all_specs
|
|
205
|
+
from ..models.records import ArchiveRecordSet
|
|
206
|
+
|
|
207
|
+
specs = load_all_specs()
|
|
208
|
+
logs = logs_dir(workspace)
|
|
209
|
+
logs.mkdir(parents=True, exist_ok=True)
|
|
210
|
+
try:
|
|
211
|
+
llm = resolve_provider(provider)
|
|
212
|
+
except ProviderUnavailable as exc:
|
|
213
|
+
return _StageOut({"error": "no_provider", "detail": str(exc)}, 1, err=True)
|
|
214
|
+
chosen = model or llm.default_model()
|
|
215
|
+
|
|
216
|
+
all_drafts = []
|
|
217
|
+
normalized = []
|
|
218
|
+
usage_total: dict[str, int] = {}
|
|
219
|
+
extractor = None
|
|
220
|
+
sources: list[tuple[object, str]] = [(d, r) for d, r in _roled(docs, instruction)]
|
|
221
|
+
for doc, role in sources:
|
|
222
|
+
try:
|
|
223
|
+
nd = normalize_document(doc, role=role, pdf_backend=pdf_backend) # type: ignore[arg-type]
|
|
224
|
+
except UnreadableDocument as exc:
|
|
225
|
+
# A document that yields no quotable text is a refusal, not a silent empty extraction:
|
|
226
|
+
# surface it with a nonzero exit exactly like a missing provider, so `run` halts here
|
|
227
|
+
# rather than emitting a manifest that is silent about a paper it could not read.
|
|
228
|
+
return _StageOut(
|
|
229
|
+
{"error": "unreadable_document", "detail": str(exc), "document": str(doc)},
|
|
230
|
+
1,
|
|
231
|
+
err=True,
|
|
232
|
+
)
|
|
233
|
+
normalized.append(nd)
|
|
234
|
+
|
|
235
|
+
if records_path is not None:
|
|
236
|
+
records = ArchiveRecordSet.model_validate_json(records_path.read_text())
|
|
237
|
+
# Only records that HAVE prose, and only levels we ask anything of. A record with an empty
|
|
238
|
+
# ask costs an API call to be told nothing; `fields_for` already knows which those are.
|
|
239
|
+
from ..harvest.fields import fields_for
|
|
240
|
+
|
|
241
|
+
for record in records.records:
|
|
242
|
+
if has_prose(record) and fields_for(record.level, "reference"):
|
|
243
|
+
normalized.append(normalize_record(record))
|
|
244
|
+
|
|
245
|
+
# The one place `run` cannot be deterministic, and now the one it need not be slow. Each document
|
|
246
|
+
# is an independent, network-bound LLM call, so they go out concurrently on a THREAD pool (I/O, so
|
|
247
|
+
# threads release the GIL — processes would only add IPC). Results are reassembled in `normalized`
|
|
248
|
+
# order below, so assertions.json is byte-identical no matter which call returned first.
|
|
249
|
+
def _extract(nd: NormalizedDoc) -> ExtractionOutcome:
|
|
250
|
+
return extract_drafts(nd, specs, provider=llm, model=chosen)
|
|
251
|
+
|
|
252
|
+
outcomes: dict[str, ExtractionOutcome] = {}
|
|
253
|
+
try:
|
|
254
|
+
if len(normalized) > 1:
|
|
255
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
256
|
+
|
|
257
|
+
with ThreadPoolExecutor(max_workers=min(8, len(normalized))) as pool:
|
|
258
|
+
futures = {pool.submit(_extract, nd): nd for nd in normalized}
|
|
259
|
+
for fut in futures:
|
|
260
|
+
outcomes[futures[fut].doc_sha256] = fut.result()
|
|
261
|
+
else:
|
|
262
|
+
for nd in normalized:
|
|
263
|
+
outcomes[nd.doc_sha256] = _extract(nd)
|
|
264
|
+
except ExtractUnavailable as exc:
|
|
265
|
+
return _StageOut({"error": "llm_unavailable", "detail": str(exc)}, 1, err=True)
|
|
266
|
+
|
|
267
|
+
usage_records: list[dict[str, object]] = []
|
|
268
|
+
extract_rejected: list[dict[str, object]] = []
|
|
269
|
+
for nd in normalized:
|
|
270
|
+
outcome = outcomes[nd.doc_sha256]
|
|
271
|
+
all_drafts.extend(outcome.drafts)
|
|
272
|
+
extract_rejected.extend(outcome.rejected)
|
|
273
|
+
extractor = outcome.extractor
|
|
274
|
+
for k, v in outcome.usage.items():
|
|
275
|
+
usage_total[k] = usage_total.get(k, 0) + v
|
|
276
|
+
usage_records.append(
|
|
277
|
+
{
|
|
278
|
+
"document": {"scope": nd.scope, "subject": nd.subject, "doc_sha256": nd.doc_sha256},
|
|
279
|
+
"provider": outcome.provider,
|
|
280
|
+
"model": outcome.model,
|
|
281
|
+
"mode": outcome.mode,
|
|
282
|
+
"usage": outcome.usage,
|
|
283
|
+
}
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
# The cost ledger (disk is state). Written whether or not we go on to verify, because the call
|
|
287
|
+
# happened and cost tokens regardless. `n_calls` is per-document; `cache_read_tokens > 0` means the
|
|
288
|
+
# stable KB prefix was served from cache, so a second run over the same documents is much cheaper.
|
|
289
|
+
(logs / "usage.json").write_text(
|
|
290
|
+
json.dumps(
|
|
291
|
+
{
|
|
292
|
+
"provider": llm.name,
|
|
293
|
+
"model": chosen,
|
|
294
|
+
"prompt_version": extractor.prompt_version if extractor else None,
|
|
295
|
+
"totals": {**usage_total, "n_calls": len(normalized)},
|
|
296
|
+
"calls": usage_records,
|
|
297
|
+
},
|
|
298
|
+
indent=2,
|
|
299
|
+
)
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
payload: dict[str, object] = {
|
|
303
|
+
"provider": llm.name,
|
|
304
|
+
"model": chosen,
|
|
305
|
+
"n_drafts": len(all_drafts),
|
|
306
|
+
# Drafts the model returned malformed (e.g. `value: null`) — dropped, not fatal (#5). Surfaced
|
|
307
|
+
# so a run is not silent about a batch that was partly lost, but never an exit code: a flaky
|
|
308
|
+
# token is a provider hiccup we tolerate, not a claim a human must weigh in on.
|
|
309
|
+
"n_extract_rejected": len(extract_rejected),
|
|
310
|
+
"extract_rejected": extract_rejected,
|
|
311
|
+
"usage": {**usage_total, "n_calls": len(normalized)},
|
|
312
|
+
"usage_by_document": usage_records,
|
|
313
|
+
"usage_path": str(logs / "usage.json"),
|
|
314
|
+
"drafts": [d.model_dump(mode="json") for d in all_drafts],
|
|
315
|
+
}
|
|
316
|
+
if not verify:
|
|
317
|
+
return _StageOut(payload, 0)
|
|
318
|
+
|
|
319
|
+
assert extractor is not None
|
|
320
|
+
report = verify_drafts(all_drafts, normalized, extractor=extractor)
|
|
321
|
+
instruction_docs = frozenset(d.doc_sha256 for d in normalized if d.role == "instruction")
|
|
322
|
+
# An OBJECT, not a bare list, and the `instruction_docs` key is the reason. Which documents were
|
|
323
|
+
# authored FOR seqforge is what decides whether an assertion may touch `processing.*` --
|
|
324
|
+
# and it lived only in this process's memory, so the artifact could not reconstruct the
|
|
325
|
+
# instructable surface and `processing new` had no way to consume it. The join existed in
|
|
326
|
+
# `fill_processing` the whole time and nothing could reach it.
|
|
327
|
+
# `document_subjects` is the same idea one level up: which RECORD each document was rendered
|
|
328
|
+
# from. It is what lets `manifest fill` tell a sample's own alias (a declaration about that
|
|
329
|
+
# sample) from a paper about six samples (an inference about each), and it too lived only in this
|
|
330
|
+
# process's memory. Code owns both mappings because code chose both documents.
|
|
331
|
+
(logs / "assertions.json").write_text(
|
|
332
|
+
json.dumps(
|
|
333
|
+
{
|
|
334
|
+
"instruction_docs": sorted(instruction_docs),
|
|
335
|
+
"document_subjects": [
|
|
336
|
+
{"doc_sha256": d.doc_sha256, "scope": d.scope, "subject": d.subject}
|
|
337
|
+
for d in sorted(normalized, key=lambda d: d.doc_sha256)
|
|
338
|
+
],
|
|
339
|
+
"assertions": [a.model_dump(mode="json") for a in report.assertions],
|
|
340
|
+
},
|
|
341
|
+
indent=2,
|
|
342
|
+
)
|
|
343
|
+
)
|
|
344
|
+
# The rendered documents, on disk, under readable names. A span citation is only checkable if the
|
|
345
|
+
# exact text it was greppedded against still exists -- and for a record-derived document these
|
|
346
|
+
# bytes exist nowhere else, because we made them.
|
|
347
|
+
docdir = documents_dir(workspace)
|
|
348
|
+
docdir.mkdir(parents=True, exist_ok=True)
|
|
349
|
+
for nd in normalized:
|
|
350
|
+
(docdir / _document_filename(nd)).write_text(nd.text)
|
|
351
|
+
payload["n_accepted"] = report.n_accepted
|
|
352
|
+
payload["n_rejected"] = len(report.rejected)
|
|
353
|
+
# what the user may act on: verified directives, projected onto the instructable surface
|
|
354
|
+
instructions, conflicts = instructions_from_assertions(
|
|
355
|
+
report.assertions, instruction_docs=instruction_docs
|
|
356
|
+
)
|
|
357
|
+
payload["instructions"] = [
|
|
358
|
+
{"field": i.field, "value": i.value, "basis": i.basis, "evidence": i.evidence}
|
|
359
|
+
for i in instructions
|
|
360
|
+
]
|
|
361
|
+
payload["conflicts"] = [c.model_dump(mode="json") for c in conflicts]
|
|
362
|
+
payload["rejected"] = report.rejected
|
|
363
|
+
payload["assertions"] = [a.model_dump(mode="json") for a in report.assertions]
|
|
364
|
+
# Exit 4 when the author must weigh in: two instructions disagreeing has no tiebreak, and a claim
|
|
365
|
+
# that failed the span tripwire needs a human rather than a silent drop.
|
|
366
|
+
code = 4 if (conflicts or report.rejected) else 0
|
|
367
|
+
return _StageOut(payload, code)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
@harvest_app.command("verify")
|
|
371
|
+
def harvest_verify(
|
|
372
|
+
drafts_json: Path = typer.Argument(..., help="AssertionDraft[] JSON (from `harvest extract`)."),
|
|
373
|
+
docs: list[Path] = typer.Option(..., "--doc", help="Source document(s) the drafts cite."),
|
|
374
|
+
model_id: str = typer.Option("unknown", help="Model that produced the drafts (provenance)."),
|
|
375
|
+
prompt_version: str = typer.Option("unknown", help="Prompt version (provenance)."),
|
|
376
|
+
pdf_backend: PdfBackendChoice = typer.Option(
|
|
377
|
+
PdfBackendChoice.pymupdf,
|
|
378
|
+
"--pdf-backend",
|
|
379
|
+
help="PDF extractor — must match the one `extract` used, or the canonical text differs.",
|
|
380
|
+
),
|
|
381
|
+
) -> None:
|
|
382
|
+
"""Grep each quote back into the canonical text + check it entails the value. Exit 4 if any fail.
|
|
383
|
+
|
|
384
|
+
Both flags are code-owned, so a hallucinated or mis-attributed claim fails closed.
|
|
385
|
+
"""
|
|
386
|
+
from ..harvest import normalize_document, verify_drafts
|
|
387
|
+
from ..models.assertion import AssertionDraft, ExtractorProvenance
|
|
388
|
+
|
|
389
|
+
try:
|
|
390
|
+
raw = json.loads(drafts_json.read_text())
|
|
391
|
+
drafts = [AssertionDraft.model_validate(d) for d in raw]
|
|
392
|
+
except (OSError, ValidationError, ValueError) as exc:
|
|
393
|
+
typer.echo(f"cannot read drafts {drafts_json}: {exc}", err=True)
|
|
394
|
+
raise typer.Exit(2) from exc
|
|
395
|
+
|
|
396
|
+
normalized = [
|
|
397
|
+
normalize_document(d, pdf_backend=cast("PdfBackend", pdf_backend.value)) for d in docs
|
|
398
|
+
]
|
|
399
|
+
report = verify_drafts(
|
|
400
|
+
drafts,
|
|
401
|
+
normalized,
|
|
402
|
+
extractor=ExtractorProvenance(model_id=model_id, prompt_version=prompt_version),
|
|
403
|
+
)
|
|
404
|
+
typer.echo(
|
|
405
|
+
json.dumps(
|
|
406
|
+
{
|
|
407
|
+
"n_drafts": len(drafts),
|
|
408
|
+
"n_accepted": report.n_accepted,
|
|
409
|
+
"n_rejected": len(report.rejected),
|
|
410
|
+
"assertions": [a.model_dump(mode="json") for a in report.assertions],
|
|
411
|
+
"rejected": report.rejected,
|
|
412
|
+
},
|
|
413
|
+
indent=2,
|
|
414
|
+
)
|
|
415
|
+
)
|
|
416
|
+
if report.rejected:
|
|
417
|
+
raise typer.Exit(4) # a rejected claim needs a human, not a silent drop
|
seqforge/cli/hook.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""`seqforge hook` -- the agent hooks (pre/post-tool-use, stop) as mechanism, plus install/check."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
from pydantic import ValidationError
|
|
10
|
+
|
|
11
|
+
from ..manifest import FillError, exit_code_for_report, validate_manifest
|
|
12
|
+
from ._common import _load_manifest
|
|
13
|
+
from .root import hook_app
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@hook_app.command("pre-tool-use")
|
|
17
|
+
def hook_pre_tool_use() -> None:
|
|
18
|
+
"""Deny an unbounded FASTQ stream or an absolute path in a manifest.
|
|
19
|
+
|
|
20
|
+
Reads the hook payload on stdin, emits a permissionDecision on stdout. Exit 0 always: the decision
|
|
21
|
+
travels in the JSON, and a crashing guard must never wedge the agent.
|
|
22
|
+
"""
|
|
23
|
+
from ..hooks import pre_tool_use
|
|
24
|
+
|
|
25
|
+
payload = _hook_payload()
|
|
26
|
+
denial = pre_tool_use(payload)
|
|
27
|
+
if denial is None:
|
|
28
|
+
raise typer.Exit(0)
|
|
29
|
+
typer.echo(
|
|
30
|
+
json.dumps(
|
|
31
|
+
{
|
|
32
|
+
"hookSpecificOutput": {
|
|
33
|
+
"hookEventName": "PreToolUse",
|
|
34
|
+
"permissionDecision": "deny",
|
|
35
|
+
"permissionDecisionReason": denial.message(),
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
)
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@hook_app.command("post-tool-use")
|
|
43
|
+
def hook_post_tool_use() -> None:
|
|
44
|
+
"""After any manifest edit, re-run `manifest validate`. The model does not grade its own work."""
|
|
45
|
+
from ..hooks import post_tool_use_targets
|
|
46
|
+
|
|
47
|
+
payload = _hook_payload()
|
|
48
|
+
target = post_tool_use_targets(payload)
|
|
49
|
+
if target is None or not Path(target).is_file():
|
|
50
|
+
raise typer.Exit(0)
|
|
51
|
+
try:
|
|
52
|
+
manifest = _load_manifest(Path(target))
|
|
53
|
+
report = validate_manifest(manifest)
|
|
54
|
+
except (FillError, ValidationError, ValueError, OSError) as exc:
|
|
55
|
+
typer.echo(
|
|
56
|
+
json.dumps(
|
|
57
|
+
{
|
|
58
|
+
"decision": "block",
|
|
59
|
+
"reason": f"{target} did not parse as a Manifest: {exc}",
|
|
60
|
+
}
|
|
61
|
+
)
|
|
62
|
+
)
|
|
63
|
+
raise typer.Exit(0) from None
|
|
64
|
+
if report.ok:
|
|
65
|
+
typer.echo(
|
|
66
|
+
json.dumps(
|
|
67
|
+
{
|
|
68
|
+
"hookSpecificOutput": {
|
|
69
|
+
"hookEventName": "PostToolUse",
|
|
70
|
+
"additionalContext": f"manifest validate: OK ({Path(target).name}).",
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
raise typer.Exit(0)
|
|
76
|
+
codes = [str(getattr(b.code, "value", b.code)) for b in report.blockers]
|
|
77
|
+
typer.echo(
|
|
78
|
+
json.dumps(
|
|
79
|
+
{
|
|
80
|
+
"decision": "block",
|
|
81
|
+
"reason": (
|
|
82
|
+
f"manifest validate FAILED on {Path(target).name} (exit "
|
|
83
|
+
f"{exit_code_for_report(report)}): {codes}. Refusal is the contract — fix "
|
|
84
|
+
"the manifest; do not proceed as though it validated."
|
|
85
|
+
),
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@hook_app.command("stop")
|
|
92
|
+
def hook_stop(
|
|
93
|
+
workspace: Path = typer.Option(Path("."), "-C", "--workspace", help="Root holding seqforge/."),
|
|
94
|
+
) -> None:
|
|
95
|
+
"""Refuse to end the turn while questions.md is non-empty — ambiguity routes to a human."""
|
|
96
|
+
from ..hooks import stop_decision
|
|
97
|
+
|
|
98
|
+
payload = _hook_payload()
|
|
99
|
+
reason = stop_decision(payload, workspace=workspace)
|
|
100
|
+
if reason is None:
|
|
101
|
+
raise typer.Exit(0)
|
|
102
|
+
typer.echo(json.dumps({"decision": "block", "reason": reason}))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@hook_app.command("install")
|
|
106
|
+
def hook_install(
|
|
107
|
+
workspace: Path = typer.Option(Path("."), "-C", "--workspace", help="Project root."),
|
|
108
|
+
force: bool = typer.Option(False, "--force", help="Overwrite an existing hooks block."),
|
|
109
|
+
) -> None:
|
|
110
|
+
"""Write the three hooks into .claude/settings.json, merging with whatever is already there."""
|
|
111
|
+
from ..hooks import HOOKS_VERSION
|
|
112
|
+
|
|
113
|
+
settings_path = Path(workspace) / ".claude" / "settings.json"
|
|
114
|
+
settings_path.parent.mkdir(parents=True, exist_ok=True)
|
|
115
|
+
settings: dict[str, object] = {}
|
|
116
|
+
if settings_path.is_file():
|
|
117
|
+
try:
|
|
118
|
+
settings = json.loads(settings_path.read_text() or "{}")
|
|
119
|
+
except json.JSONDecodeError as exc:
|
|
120
|
+
typer.echo(f"{settings_path} is not valid JSON: {exc}", err=True)
|
|
121
|
+
raise typer.Exit(1) from exc
|
|
122
|
+
existing = settings.get("hooks")
|
|
123
|
+
if existing and not force:
|
|
124
|
+
typer.echo(
|
|
125
|
+
json.dumps(
|
|
126
|
+
{
|
|
127
|
+
"ok": False,
|
|
128
|
+
"reason": f"{settings_path} already defines hooks; re-run with --force to replace",
|
|
129
|
+
},
|
|
130
|
+
indent=2,
|
|
131
|
+
),
|
|
132
|
+
err=True,
|
|
133
|
+
)
|
|
134
|
+
raise typer.Exit(2)
|
|
135
|
+
|
|
136
|
+
run = "${CLAUDE_PROJECT_DIR}/.claude/hooks/seqforge-hook.sh"
|
|
137
|
+
settings["hooks"] = {
|
|
138
|
+
"PreToolUse": [
|
|
139
|
+
{
|
|
140
|
+
"matcher": "Bash|Write|Edit|NotebookEdit|Read|Grep|Glob",
|
|
141
|
+
"hooks": [{"type": "command", "command": f"{run} pre-tool-use", "args": []}],
|
|
142
|
+
}
|
|
143
|
+
],
|
|
144
|
+
"PostToolUse": [
|
|
145
|
+
{
|
|
146
|
+
"matcher": "Write|Edit|NotebookEdit",
|
|
147
|
+
"hooks": [{"type": "command", "command": f"{run} post-tool-use", "args": []}],
|
|
148
|
+
}
|
|
149
|
+
],
|
|
150
|
+
"Stop": [{"hooks": [{"type": "command", "command": f"{run} stop", "args": []}]}],
|
|
151
|
+
}
|
|
152
|
+
settings_path.write_text(json.dumps(settings, indent=2) + "\n")
|
|
153
|
+
|
|
154
|
+
shim = Path(workspace) / ".claude" / "hooks" / "seqforge-hook.sh"
|
|
155
|
+
shim.parent.mkdir(parents=True, exist_ok=True)
|
|
156
|
+
shim.write_text(
|
|
157
|
+
"#!/usr/bin/env bash\n"
|
|
158
|
+
"# Generated by `seqforge hook install`. The guard logic lives in seqforge.hooks (typed +\n"
|
|
159
|
+
"# tested); this only routes the event. Fails OPEN by design: a broken hook must not wedge\n"
|
|
160
|
+
"# the agent, and a guard that can hang is worse than the risk it manages.\n"
|
|
161
|
+
"set -uo pipefail\n"
|
|
162
|
+
'cd "${CLAUDE_PROJECT_DIR:-.}" || exit 0\n'
|
|
163
|
+
'exec pixi run -q -- python -m seqforge.cli hook "$@" 2>/dev/null || exit 0\n'
|
|
164
|
+
)
|
|
165
|
+
shim.chmod(0o755)
|
|
166
|
+
typer.echo(
|
|
167
|
+
json.dumps(
|
|
168
|
+
{
|
|
169
|
+
"ok": True,
|
|
170
|
+
"hooks_version": HOOKS_VERSION,
|
|
171
|
+
"settings": str(settings_path),
|
|
172
|
+
"shim": str(shim),
|
|
173
|
+
"events": ["PreToolUse", "PostToolUse", "Stop"],
|
|
174
|
+
},
|
|
175
|
+
indent=2,
|
|
176
|
+
)
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@hook_app.command("check")
|
|
181
|
+
def hook_check(
|
|
182
|
+
workspace: Path = typer.Option(Path("."), "-C", "--workspace", help="Root holding seqforge/."),
|
|
183
|
+
) -> None:
|
|
184
|
+
"""Self-test: prove each guard fires. A guard nobody has watched deny is not a guard.
|
|
185
|
+
|
|
186
|
+
A hook that silently never fires is indistinguishable from one that always allows — so this
|
|
187
|
+
exercises every rule against a known-bad payload and reports what it caught.
|
|
188
|
+
"""
|
|
189
|
+
from ..hooks import HOOKS_VERSION, pre_tool_use, questions_outstanding
|
|
190
|
+
|
|
191
|
+
cases = [
|
|
192
|
+
(
|
|
193
|
+
"unbounded FASTQ",
|
|
194
|
+
{"tool_name": "Bash", "tool_input": {"command": "zcat big.fastq.gz | wc -l"}},
|
|
195
|
+
),
|
|
196
|
+
(
|
|
197
|
+
"allows a bounded stream",
|
|
198
|
+
{"tool_name": "Bash", "tool_input": {"command": "zcat big.fastq.gz | head -n 400"}},
|
|
199
|
+
),
|
|
200
|
+
(
|
|
201
|
+
"allows the seqforge verb",
|
|
202
|
+
{"tool_name": "Bash", "tool_input": {"command": "seqforge probe big.fastq.gz"}},
|
|
203
|
+
),
|
|
204
|
+
(
|
|
205
|
+
"absolute path in manifest",
|
|
206
|
+
{
|
|
207
|
+
"tool_name": "Write",
|
|
208
|
+
"tool_input": {
|
|
209
|
+
"file_path": "manifest.yaml",
|
|
210
|
+
"file_text": "genome: /data/ref/hg38.fa\n",
|
|
211
|
+
},
|
|
212
|
+
},
|
|
213
|
+
),
|
|
214
|
+
]
|
|
215
|
+
results = []
|
|
216
|
+
for name, payload in cases:
|
|
217
|
+
denial = pre_tool_use(payload)
|
|
218
|
+
results.append(
|
|
219
|
+
{"case": name, "denied": denial is not None, "rule": denial.rule if denial else None}
|
|
220
|
+
)
|
|
221
|
+
typer.echo(
|
|
222
|
+
json.dumps(
|
|
223
|
+
{
|
|
224
|
+
"hooks_version": HOOKS_VERSION,
|
|
225
|
+
"open_questions": [str(p) for p in questions_outstanding(workspace)],
|
|
226
|
+
"checks": results,
|
|
227
|
+
},
|
|
228
|
+
indent=2,
|
|
229
|
+
)
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _hook_payload() -> dict[str, object]:
|
|
234
|
+
"""Read the hook event from stdin. A malformed payload means NO OPINION, never a crash."""
|
|
235
|
+
import sys
|
|
236
|
+
|
|
237
|
+
try:
|
|
238
|
+
raw = sys.stdin.read()
|
|
239
|
+
except (OSError, ValueError):
|
|
240
|
+
return {}
|
|
241
|
+
if not raw.strip():
|
|
242
|
+
return {}
|
|
243
|
+
try:
|
|
244
|
+
parsed = json.loads(raw)
|
|
245
|
+
except json.JSONDecodeError:
|
|
246
|
+
return {}
|
|
247
|
+
return parsed if isinstance(parsed, dict) else {}
|