seqforge 2026.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqforge/__init__.py +16 -0
- seqforge/cli/__init__.py +38 -0
- seqforge/cli/__main__.py +8 -0
- seqforge/cli/_common.py +105 -0
- seqforge/cli/compose.py +119 -0
- seqforge/cli/eval.py +103 -0
- seqforge/cli/harvest.py +417 -0
- seqforge/cli/hook.py +247 -0
- seqforge/cli/io.py +502 -0
- seqforge/cli/kb.py +348 -0
- seqforge/cli/manifest.py +536 -0
- seqforge/cli/probe.py +43 -0
- seqforge/cli/processing.py +192 -0
- seqforge/cli/project.py +52 -0
- seqforge/cli/resolve.py +55 -0
- seqforge/cli/root.py +66 -0
- seqforge/cli/run.py +463 -0
- seqforge/cli/schema.py +41 -0
- seqforge/compose/__init__.py +28 -0
- seqforge/compose/core.py +515 -0
- seqforge/compose/gates.py +113 -0
- seqforge/compose/params.py +447 -0
- seqforge/e2e.py +1926 -0
- seqforge/evals/__init__.py +78 -0
- seqforge/evals/case.py +382 -0
- seqforge/evals/grade.py +300 -0
- seqforge/evals/run.py +420 -0
- seqforge/harvest/__init__.py +121 -0
- seqforge/harvest/extract.py +319 -0
- seqforge/harvest/fields.py +212 -0
- seqforge/harvest/normalize.py +537 -0
- seqforge/harvest/prep.py +41 -0
- seqforge/harvest/providers.py +321 -0
- seqforge/harvest/verify.py +251 -0
- seqforge/hooks/__init__.py +33 -0
- seqforge/hooks/guards.py +214 -0
- seqforge/io/__init__.py +61 -0
- seqforge/io/archive.py +450 -0
- seqforge/io/attributes.py +190 -0
- seqforge/io/biosample/attributes.json +6341 -0
- seqforge/io/efo/labels.json +55 -0
- seqforge/io/efo.py +138 -0
- seqforge/io/onlist.py +661 -0
- seqforge/io/onlists/3M-february-2018.codes.gz +0 -0
- seqforge/io/onlists/737K-arc-v1.codes.gz +0 -0
- seqforge/io/onlists/737K-august-2016.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3.codes.gz +0 -0
- seqforge/io/onlists/index.json +74 -0
- seqforge/io/remote.py +659 -0
- seqforge/io/taxonomy.py +194 -0
- seqforge/kb/__init__.py +62 -0
- seqforge/kb/anchor.py +169 -0
- seqforge/kb/generate.py +147 -0
- seqforge/kb/loader.py +152 -0
- seqforge/kb/roundtrip.py +112 -0
- seqforge/kb/schema.py +422 -0
- seqforge/kb/specs/10x-3p-gex/spec.yaml +62 -0
- seqforge/kb/specs/10x-3p-gex-v2/README.md +41 -0
- seqforge/kb/specs/10x-3p-gex-v2/spec.yaml +83 -0
- seqforge/kb/specs/10x-3p-gex-v3/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3/spec.yaml +118 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/spec.yaml +124 -0
- seqforge/kb/specs/bd-rhapsody-wta/README.md +103 -0
- seqforge/kb/specs/bd-rhapsody-wta/spec.yaml +130 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced/spec.yaml +99 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v1/spec.yaml +93 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v2/spec.yaml +81 -0
- seqforge/kb/specs/bulk-rnaseq-pe/README.md +35 -0
- seqforge/kb/specs/bulk-rnaseq-pe/spec.yaml +97 -0
- seqforge/kb/specs/splitseq/README.md +51 -0
- seqforge/kb/specs/splitseq/spec.yaml +157 -0
- seqforge/manifest/__init__.py +61 -0
- seqforge/manifest/fill.py +531 -0
- seqforge/manifest/hash.py +77 -0
- seqforge/manifest/instruct.py +114 -0
- seqforge/manifest/policy.py +409 -0
- seqforge/manifest/validate.py +274 -0
- seqforge/models/__init__.py +268 -0
- seqforge/models/assertion.py +68 -0
- seqforge/models/base.py +100 -0
- seqforge/models/blocker.py +71 -0
- seqforge/models/conflict.py +47 -0
- seqforge/models/dataset.py +320 -0
- seqforge/models/evidenced.py +54 -0
- seqforge/models/observation.py +157 -0
- seqforge/models/processing.py +231 -0
- seqforge/models/records.py +145 -0
- seqforge/models/resolve.py +216 -0
- seqforge/probe/__init__.py +46 -0
- seqforge/probe/core.py +232 -0
- seqforge/probe/signals.py +250 -0
- seqforge/probe/streaming.py +118 -0
- seqforge/project.py +177 -0
- seqforge/py.typed +0 -0
- seqforge/resolve/__init__.py +98 -0
- seqforge/resolve/assign.py +204 -0
- seqforge/resolve/cache.py +119 -0
- seqforge/resolve/confuse.py +215 -0
- seqforge/resolve/engine.py +646 -0
- seqforge/resolve/escalate.py +668 -0
- seqforge/resolve/evaluators.py +306 -0
- seqforge/resolve/geometry.py +89 -0
- seqforge/resolve/group.py +85 -0
- seqforge/resolve/records.py +550 -0
- seqforge/resolve/scoring.py +373 -0
- seqforge/resolve/window.py +206 -0
- seqforge/workflows/__init__.py +234 -0
- seqforge/workflows/cram.py +117 -0
- seqforge/workflows/h5ad.py +368 -0
- seqforge/workflows/map/star.smk +101 -0
- seqforge/workflows/map/starsolo.smk +360 -0
- seqforge/workflows/qc.py +157 -0
- seqforge/workspace.py +125 -0
- seqforge-2026.7.1.dist-info/METADATA +125 -0
- seqforge-2026.7.1.dist-info/RECORD +124 -0
- seqforge-2026.7.1.dist-info/WHEEL +4 -0
- seqforge-2026.7.1.dist-info/entry_points.txt +2 -0
- seqforge-2026.7.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"""Provider layer — the LLM is a swappable component, not a foundation.
|
|
2
|
+
|
|
3
|
+
seqforge is a compiler whose only LLM touchpoint proposes claims that code then re-verifies from
|
|
4
|
+
first principles. That makes the provider genuinely pluggable: nothing downstream trusts the model,
|
|
5
|
+
so the choice is about cost and extraction quality, never about correctness guarantees.
|
|
6
|
+
|
|
7
|
+
Two providers ship:
|
|
8
|
+
|
|
9
|
+
- ``anthropic`` — strict ``json_schema`` structured output; the returned shape is guaranteed.
|
|
10
|
+
- ``openai-compatible`` — any OpenAI-shaped endpoint via ``base_url``. **DeepSeek** is a preset; so
|
|
11
|
+
are vLLM, Ollama, Together, and friends. These offer ``response_format={"type": "json_object"}``
|
|
12
|
+
only: valid JSON is guaranteed, the *shape* is not.
|
|
13
|
+
|
|
14
|
+
**That capability gap is contained, not papered over.** For json-object providers we put the schema
|
|
15
|
+
and a worked example in the prompt, and then — as always — ``ExtractionResult.model_validate_json``
|
|
16
|
+
is the gate. A provider that returns the wrong shape fails validation and the batch is refused; it
|
|
17
|
+
cannot produce a half-parsed assertion. This is exactly the division of labor working: agents propose, code decides.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from typing import Any, Protocol
|
|
26
|
+
|
|
27
|
+
#: Anthropic. Adaptive thinking + strict schema.
|
|
28
|
+
ANTHROPIC_DEFAULT_MODEL = "claude-opus-4-8"
|
|
29
|
+
|
|
30
|
+
#: DeepSeek V4. `-pro` is the default for the accuracy-critical extraction stage; `-flash` is ~3x
|
|
31
|
+
#: cheaper and also V4 (1M ctx) if throughput matters more than recall — pass --model to switch.
|
|
32
|
+
#: NB `deepseek-chat` / `deepseek-reasoner` are deprecated (2026-07-24) aliases onto V4-Flash; we
|
|
33
|
+
#: name a V4 model explicitly so nothing breaks when they are withdrawn.
|
|
34
|
+
DEEPSEEK_DEFAULT_MODEL = "deepseek-v4-pro"
|
|
35
|
+
DEEPSEEK_BASE_URL = "https://api.deepseek.com"
|
|
36
|
+
|
|
37
|
+
#: How many extra times to re-issue a json_object request that came back with EMPTY content. DeepSeek
|
|
38
|
+
#: documents that json_object mode intermittently returns an empty body, and v4-pro does it often
|
|
39
|
+
#: enough to abort a whole harvest (#4). An empty body is a provider hiccup, not "the document says
|
|
40
|
+
#: nothing" (that is a well-formed `{"drafts": []}`), so a bounded retry recovers it instead of failing
|
|
41
|
+
#: the dataset. A real API error is not retried here — it raises on the first attempt.
|
|
42
|
+
_EMPTY_CONTENT_RETRIES = 3
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ProviderUnavailable(RuntimeError):
|
|
46
|
+
"""No usable provider: SDK missing, credential absent, or the endpoint failed."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class LLMResponse:
|
|
51
|
+
"""Raw model output plus normalized usage. The text is UNVALIDATED — the caller decides."""
|
|
52
|
+
|
|
53
|
+
text: str
|
|
54
|
+
usage: dict[str, int]
|
|
55
|
+
#: How the call was made, recorded for the cost/provenance ledger: reasoning ``thinking`` mode,
|
|
56
|
+
#: the ``max_tokens`` ceiling, and which structured-output ``response_format`` was in force. The
|
|
57
|
+
#: same prompt at a different effort is a different run, and this is what lets a reader see it.
|
|
58
|
+
mode: dict[str, Any] = field(default_factory=dict)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class LLMProvider(Protocol):
|
|
62
|
+
"""What extraction needs from a model: JSON text back, and a name to record in provenance."""
|
|
63
|
+
|
|
64
|
+
name: str
|
|
65
|
+
|
|
66
|
+
def default_model(self) -> str: ...
|
|
67
|
+
|
|
68
|
+
def complete_json(
|
|
69
|
+
self, *, system: str, user: str, schema: dict[str, Any], model: str, max_tokens: int
|
|
70
|
+
) -> LLMResponse: ...
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class AnthropicProvider:
|
|
74
|
+
"""Claude via the official SDK: strict schema, explicit prefix caching, adaptive thinking."""
|
|
75
|
+
|
|
76
|
+
name = "anthropic"
|
|
77
|
+
|
|
78
|
+
def __init__(self, *, client: Any | None = None, api_key: str | None = None) -> None:
|
|
79
|
+
self._client = client
|
|
80
|
+
self._api_key = api_key
|
|
81
|
+
|
|
82
|
+
def default_model(self) -> str:
|
|
83
|
+
return ANTHROPIC_DEFAULT_MODEL
|
|
84
|
+
|
|
85
|
+
def _resolve(self) -> Any:
|
|
86
|
+
if self._client is not None:
|
|
87
|
+
return self._client
|
|
88
|
+
try:
|
|
89
|
+
import anthropic
|
|
90
|
+
except ImportError as exc: # pragma: no cover - host dependent
|
|
91
|
+
raise ProviderUnavailable("the `anthropic` SDK is not installed") from exc
|
|
92
|
+
return (
|
|
93
|
+
anthropic.Anthropic(api_key=self._api_key) if self._api_key else anthropic.Anthropic()
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
def complete_json(
|
|
97
|
+
self, *, system: str, user: str, schema: dict[str, Any], model: str, max_tokens: int
|
|
98
|
+
) -> LLMResponse:
|
|
99
|
+
client = self._resolve()
|
|
100
|
+
try:
|
|
101
|
+
response = client.messages.create(
|
|
102
|
+
model=model,
|
|
103
|
+
max_tokens=max_tokens,
|
|
104
|
+
# cache breakpoint on the last system block: render order is tools -> system ->
|
|
105
|
+
# messages, so the stable prefix caches and the volatile document does not.
|
|
106
|
+
system=[{"type": "text", "text": system, "cache_control": {"type": "ephemeral"}}],
|
|
107
|
+
messages=[{"role": "user", "content": user}],
|
|
108
|
+
output_config={"format": {"type": "json_schema", "schema": schema}},
|
|
109
|
+
thinking={"type": "adaptive"},
|
|
110
|
+
)
|
|
111
|
+
except Exception as exc:
|
|
112
|
+
raise ProviderUnavailable(f"anthropic call failed: {exc}") from exc
|
|
113
|
+
text = next((b.text for b in response.content if getattr(b, "type", None) == "text"), "")
|
|
114
|
+
return LLMResponse(
|
|
115
|
+
text=text,
|
|
116
|
+
usage=_anthropic_usage(response),
|
|
117
|
+
mode={
|
|
118
|
+
"thinking": "adaptive",
|
|
119
|
+
"max_tokens": max_tokens,
|
|
120
|
+
"response_format": "json_schema",
|
|
121
|
+
},
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class OpenAICompatibleProvider:
|
|
126
|
+
"""Any OpenAI-shaped endpoint (DeepSeek, vLLM, Ollama, Together, ...) selected by ``base_url``.
|
|
127
|
+
|
|
128
|
+
These expose ``json_object`` mode only, so the schema travels in the prompt and Pydantic — not
|
|
129
|
+
the provider — enforces the shape. Prefix caching is automatic server-side (DeepSeek reports it
|
|
130
|
+
as cache hit/miss tokens), so there is no ``cache_control`` to place; keeping the prefix stable
|
|
131
|
+
is the whole job.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
name = "openai-compatible"
|
|
135
|
+
|
|
136
|
+
def __init__(
|
|
137
|
+
self,
|
|
138
|
+
*,
|
|
139
|
+
base_url: str,
|
|
140
|
+
api_key: str | None = None,
|
|
141
|
+
default_model: str = DEEPSEEK_DEFAULT_MODEL,
|
|
142
|
+
name: str | None = None,
|
|
143
|
+
client: Any | None = None,
|
|
144
|
+
) -> None:
|
|
145
|
+
self.base_url = base_url
|
|
146
|
+
self._api_key = api_key
|
|
147
|
+
self._default_model = default_model
|
|
148
|
+
self._client = client
|
|
149
|
+
if name:
|
|
150
|
+
self.name = name
|
|
151
|
+
|
|
152
|
+
def default_model(self) -> str:
|
|
153
|
+
return self._default_model
|
|
154
|
+
|
|
155
|
+
def _resolve(self) -> Any:
|
|
156
|
+
if self._client is not None:
|
|
157
|
+
return self._client
|
|
158
|
+
try:
|
|
159
|
+
from openai import OpenAI
|
|
160
|
+
except ImportError as exc: # pragma: no cover - host dependent
|
|
161
|
+
raise ProviderUnavailable(
|
|
162
|
+
"the `openai` SDK is not installed (it is the client for OpenAI-compatible "
|
|
163
|
+
"endpoints such as DeepSeek)"
|
|
164
|
+
) from exc
|
|
165
|
+
if not self._api_key:
|
|
166
|
+
raise ProviderUnavailable(f"no API key for {self.name} ({self.base_url})")
|
|
167
|
+
return OpenAI(api_key=self._api_key, base_url=self.base_url)
|
|
168
|
+
|
|
169
|
+
def complete_json(
|
|
170
|
+
self, *, system: str, user: str, schema: dict[str, Any], model: str, max_tokens: int
|
|
171
|
+
) -> LLMResponse:
|
|
172
|
+
client = self._resolve()
|
|
173
|
+
# Re-issue on EMPTY content, bounded (#4). DeepSeek's json_object mode intermittently returns
|
|
174
|
+
# an empty body; that is a provider hiccup, not the document saying nothing (which is a
|
|
175
|
+
# well-formed `{"drafts": []}`), so a few retries recover it instead of aborting the harvest.
|
|
176
|
+
# Usage is ACCUMULATED across attempts: an empty response still cost tokens, and the harvest
|
|
177
|
+
# ledger is meant to reflect what the calls actually cost, not just the final one.
|
|
178
|
+
usage_total: dict[str, int] = {}
|
|
179
|
+
for _ in range(_EMPTY_CONTENT_RETRIES + 1):
|
|
180
|
+
try:
|
|
181
|
+
response = client.chat.completions.create(
|
|
182
|
+
model=model,
|
|
183
|
+
max_tokens=max_tokens,
|
|
184
|
+
messages=[
|
|
185
|
+
{"role": "system", "content": system},
|
|
186
|
+
{"role": "user", "content": user},
|
|
187
|
+
],
|
|
188
|
+
# json_object guarantees valid JSON, NOT the right shape — Pydantic checks shape.
|
|
189
|
+
response_format={"type": "json_object"},
|
|
190
|
+
)
|
|
191
|
+
except Exception as exc:
|
|
192
|
+
# A real API error is not a content hiccup — fail on the first attempt, do not retry.
|
|
193
|
+
raise ProviderUnavailable(f"{self.name} call failed: {exc}") from exc
|
|
194
|
+
for key, val in _openai_usage(response).items():
|
|
195
|
+
usage_total[key] = usage_total.get(key, 0) + val
|
|
196
|
+
choice = response.choices[0] if response.choices else None
|
|
197
|
+
text = (getattr(choice.message, "content", None) or "") if choice else ""
|
|
198
|
+
if text.strip():
|
|
199
|
+
# `thinking` is the MODEL's own (v4-pro/-reasoner reason inherently); the API takes no
|
|
200
|
+
# toggle, so it is reported as the model name's business, not a flag we set.
|
|
201
|
+
# `response_format` is the weaker json_object contract, which is why Pydantic — not the
|
|
202
|
+
# provider — enforces the shape.
|
|
203
|
+
return LLMResponse(
|
|
204
|
+
text=text,
|
|
205
|
+
usage=usage_total, # summed over every attempt, including the empty ones
|
|
206
|
+
mode={
|
|
207
|
+
"thinking": "model-default",
|
|
208
|
+
"max_tokens": max_tokens,
|
|
209
|
+
"response_format": "json_object",
|
|
210
|
+
},
|
|
211
|
+
)
|
|
212
|
+
# Every attempt came back empty: refuse loudly rather than let it read as "says nothing". The
|
|
213
|
+
# hint is provider-agnostic (this class also serves vLLM/Ollama/Together) and names no
|
|
214
|
+
# soon-deprecated model.
|
|
215
|
+
raise ProviderUnavailable(
|
|
216
|
+
f"{self.name} returned empty content in JSON mode on {_EMPTY_CONTENT_RETRIES + 1} "
|
|
217
|
+
f"attempts (a known json_object-mode failure; try a different model or provider)"
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def deepseek_provider(api_key: str | None = None, **kwargs: Any) -> OpenAICompatibleProvider:
|
|
222
|
+
"""DeepSeek preset of the OpenAI-compatible provider."""
|
|
223
|
+
return OpenAICompatibleProvider(
|
|
224
|
+
base_url=DEEPSEEK_BASE_URL,
|
|
225
|
+
api_key=api_key or os.environ.get("DEEPSEEK_API_KEY"),
|
|
226
|
+
default_model=DEEPSEEK_DEFAULT_MODEL,
|
|
227
|
+
name="deepseek",
|
|
228
|
+
**kwargs,
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def resolve_provider(name: str | None = None) -> LLMProvider:
|
|
233
|
+
"""Pick a provider explicitly, or auto-detect from the environment.
|
|
234
|
+
|
|
235
|
+
Explicit beats implicit: ``--provider`` / ``SEQFORGE_LLM_PROVIDER`` wins. Otherwise we take
|
|
236
|
+
whichever credential is present, and refuse (listing the options) rather than guess when neither
|
|
237
|
+
is — an extraction that silently picks a different model than you expected is a provenance bug.
|
|
238
|
+
"""
|
|
239
|
+
choice = (name or os.environ.get("SEQFORGE_LLM_PROVIDER") or "").strip().lower()
|
|
240
|
+
if choice in ("deepseek", "deepseek-v4"):
|
|
241
|
+
return deepseek_provider()
|
|
242
|
+
if choice == "anthropic":
|
|
243
|
+
return AnthropicProvider()
|
|
244
|
+
if choice in ("openai-compatible", "custom"):
|
|
245
|
+
base = os.environ.get("SEQFORGE_LLM_BASE_URL")
|
|
246
|
+
if not base:
|
|
247
|
+
raise ProviderUnavailable("provider 'openai-compatible' needs SEQFORGE_LLM_BASE_URL")
|
|
248
|
+
return OpenAICompatibleProvider(
|
|
249
|
+
base_url=base,
|
|
250
|
+
api_key=os.environ.get("SEQFORGE_LLM_API_KEY"),
|
|
251
|
+
default_model=os.environ.get("SEQFORGE_LLM_MODEL", DEEPSEEK_DEFAULT_MODEL),
|
|
252
|
+
name="openai-compatible",
|
|
253
|
+
)
|
|
254
|
+
if choice:
|
|
255
|
+
raise ProviderUnavailable(
|
|
256
|
+
f"unknown provider {choice!r}; known: anthropic, deepseek, openai-compatible"
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
if os.environ.get("DEEPSEEK_API_KEY"):
|
|
260
|
+
return deepseek_provider()
|
|
261
|
+
if os.environ.get("ANTHROPIC_API_KEY"):
|
|
262
|
+
return AnthropicProvider()
|
|
263
|
+
raise ProviderUnavailable(
|
|
264
|
+
"no LLM credential found. Set DEEPSEEK_API_KEY or ANTHROPIC_API_KEY, or pass "
|
|
265
|
+
"--provider with SEQFORGE_LLM_BASE_URL/-API_KEY for any OpenAI-compatible endpoint."
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _anthropic_usage(response: Any) -> dict[str, int]:
|
|
270
|
+
usage = getattr(response, "usage", None)
|
|
271
|
+
if usage is None:
|
|
272
|
+
return {}
|
|
273
|
+
return {
|
|
274
|
+
"input_tokens": int(getattr(usage, "input_tokens", 0) or 0),
|
|
275
|
+
"output_tokens": int(getattr(usage, "output_tokens", 0) or 0),
|
|
276
|
+
"cache_read_tokens": int(getattr(usage, "cache_read_input_tokens", 0) or 0),
|
|
277
|
+
"cache_write_tokens": int(getattr(usage, "cache_creation_input_tokens", 0) or 0),
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _openai_usage(response: Any) -> dict[str, int]:
|
|
282
|
+
usage = getattr(response, "usage", None)
|
|
283
|
+
if usage is None:
|
|
284
|
+
return {}
|
|
285
|
+
out = {
|
|
286
|
+
"input_tokens": int(getattr(usage, "prompt_tokens", 0) or 0),
|
|
287
|
+
"output_tokens": int(getattr(usage, "completion_tokens", 0) or 0),
|
|
288
|
+
}
|
|
289
|
+
# DeepSeek reports automatic prefix caching this way; normalize onto the common key.
|
|
290
|
+
hit = getattr(usage, "prompt_cache_hit_tokens", None)
|
|
291
|
+
if hit is not None:
|
|
292
|
+
out["cache_read_tokens"] = int(hit or 0)
|
|
293
|
+
return out
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def schema_prompt(schema: dict[str, Any]) -> str:
|
|
297
|
+
"""The json-mode contract: say 'json', show the schema, show an example (DeepSeek requires both).
|
|
298
|
+
|
|
299
|
+
Harmless on providers that enforce a strict schema, so extraction keeps ONE prompt across
|
|
300
|
+
providers — one ``prompt_version``, one thing for evals to compare.
|
|
301
|
+
"""
|
|
302
|
+
example = {
|
|
303
|
+
"drafts": [
|
|
304
|
+
{
|
|
305
|
+
"field": "library.chemistry",
|
|
306
|
+
"value": "10x-3p-gex-v3",
|
|
307
|
+
"span": {
|
|
308
|
+
"doc_sha256": "<echo the sha given below>",
|
|
309
|
+
"quote": "Chromium Single Cell 3' v3",
|
|
310
|
+
"context": None,
|
|
311
|
+
},
|
|
312
|
+
"llm_confidence": 0.95,
|
|
313
|
+
}
|
|
314
|
+
]
|
|
315
|
+
}
|
|
316
|
+
return (
|
|
317
|
+
"Return a single json object matching this JSON Schema exactly:\n"
|
|
318
|
+
f"{json.dumps(schema, separators=(',', ':'))}\n\n"
|
|
319
|
+
"Example of a well-formed json response (an empty `drafts` list is valid and common):\n"
|
|
320
|
+
f"{json.dumps(example, indent=2)}\n"
|
|
321
|
+
)
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""``harvest verify`` — the hallucination tripwire, owned by code and failing closed.
|
|
2
|
+
|
|
3
|
+
The LLM emits only ``{field, value, quote}``; it never emits offsets, because models cannot count
|
|
4
|
+
characters and a wrong offset would reject a truthful claim. Code searches the normalized text,
|
|
5
|
+
computes the offsets, and sets **both** verification flags. An Assertion only reaches ``manifest fill``
|
|
6
|
+
if all three hold:
|
|
7
|
+
|
|
8
|
+
- ``field`` is in the allowlist — the claim names a manifest path the model may set at all. This is
|
|
9
|
+
the *only* check that is not about the document, and it is the one the other two cannot stand in
|
|
10
|
+
for: see :mod:`seqforge.harvest.fields` for the real quote that entails a real value on a field
|
|
11
|
+
nobody ever authorized.
|
|
12
|
+
- ``span_verified`` — the quote really occurs in the cited document. Catches **fabricated provenance**.
|
|
13
|
+
- ``entailment_ok`` — the quote actually *supports the value*. Catches the more common and more
|
|
14
|
+
dangerous failure: a **real quote mis-attached to a wrong value** (a verbatim "single-cell RNA-seq"
|
|
15
|
+
span pinned to "10x 3' v3.1"). Without this, span-verification alone is theatre — a model can quote
|
|
16
|
+
the paper faithfully and still invent the conclusion.
|
|
17
|
+
|
|
18
|
+
All three are deterministic. Anything unverifiable is rejected, never waved through.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import re
|
|
24
|
+
from collections.abc import Callable
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
|
|
27
|
+
from ..kb import load_all_specs
|
|
28
|
+
from ..models.assertion import Assertion, AssertionDraft, ExtractorProvenance, SourceSpan
|
|
29
|
+
from .fields import PERMITTED_FIELDS, permitted_for
|
|
30
|
+
from .normalize import NormalizedDoc, page_for_offset
|
|
31
|
+
from .prep import normalize_prep_type
|
|
32
|
+
|
|
33
|
+
_WS = re.compile(r"\s+")
|
|
34
|
+
_TOKEN = re.compile(r"[a-z0-9']+")
|
|
35
|
+
#: tokens too generic to carry entailment weight on their own
|
|
36
|
+
_STOPWORDS = frozenset(
|
|
37
|
+
{"the", "a", "an", "of", "for", "with", "and", "or", "was", "were", "used", "using", "kit"}
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class VerifyReport:
|
|
43
|
+
"""Outcome of verifying a batch of drafts: what survived, and precisely why the rest did not."""
|
|
44
|
+
|
|
45
|
+
assertions: list[Assertion]
|
|
46
|
+
rejected: list[dict[str, object]]
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def n_accepted(self) -> int:
|
|
50
|
+
return len(self.assertions)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _squash(text: str) -> str:
|
|
54
|
+
return _WS.sub(" ", text).strip()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _tokens(text: str) -> list[str]:
|
|
58
|
+
return _TOKEN.findall(text.lower().replace("-", " "))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def find_span(text: str, quote: str) -> tuple[int, int] | None:
|
|
62
|
+
"""Locate ``quote`` in ``text``, tolerating whitespace differences. Returns ``(start, end)``.
|
|
63
|
+
|
|
64
|
+
Exact first; then a whitespace-flexible regex, because a quote copied across a line wrap differs
|
|
65
|
+
from the canonical text only in runs of space — rejecting that would punish an honest quote.
|
|
66
|
+
"""
|
|
67
|
+
if not quote.strip():
|
|
68
|
+
return None
|
|
69
|
+
idx = text.find(quote)
|
|
70
|
+
if idx >= 0:
|
|
71
|
+
return idx, idx + len(quote)
|
|
72
|
+
squashed = _squash(quote)
|
|
73
|
+
idx = text.find(squashed)
|
|
74
|
+
if idx >= 0:
|
|
75
|
+
return idx, idx + len(squashed)
|
|
76
|
+
pattern = r"\s+".join(re.escape(part) for part in squashed.split(" ") if part)
|
|
77
|
+
if not pattern:
|
|
78
|
+
return None
|
|
79
|
+
match = re.search(pattern, text)
|
|
80
|
+
return (match.start(), match.end()) if match else None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _kb_chemistry_aliases(value: str) -> list[str]:
|
|
84
|
+
"""A paper says "Chromium Single Cell 3' v3", not "10x-3p-gex-v3". The KB curates those aliases."""
|
|
85
|
+
forms: list[str] = []
|
|
86
|
+
for tech_id, spec in load_all_specs().items():
|
|
87
|
+
candidates = {tech_id, spec.identity.id, spec.identity.name, *spec.identity.aliases}
|
|
88
|
+
if any(_squash(c).lower() == _squash(value).lower() for c in candidates):
|
|
89
|
+
forms.extend(candidates)
|
|
90
|
+
return forms
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
#: field -> extra acceptable surface forms. EXACT-match dispatch, replacing a substring test
|
|
94
|
+
#: (`if "chemistry" in field or "assay" in field`) that would have misfired the moment a field was
|
|
95
|
+
#: named e.g. `processing.assay_override`.
|
|
96
|
+
#:
|
|
97
|
+
#: **`processing.quantification` is deliberately absent, and that absence is the design.** Its values
|
|
98
|
+
#: are STARsolo's own spellings, so a quote naming the feature already matches the bare value —
|
|
99
|
+
#: `entails` lowercases and substring-matches, so "pass --soloFeatures GeneFull" needs no alias. An
|
|
100
|
+
#: alias table here could therefore only ever LOOSEN the check, and loosening is exactly the hazard:
|
|
101
|
+
#: teach it `"nuclei" -> GeneFull` and the sentence "we prepared single nuclei" entails a processing
|
|
102
|
+
#: decision, at which point span verification is theatre — the model would be INFERRING the decision from a biological
|
|
103
|
+
#: fact, and that inference is code's to own. The instruction document's documented contract is **name
|
|
104
|
+
#: the STARsolo feature**; "count introns too" is correctly rejected as not-entailed.
|
|
105
|
+
#:
|
|
106
|
+
#: That rigor is affordable only because of the all-five default: the case that most tempts you to add the alias — a
|
|
107
|
+
#: nuclear prep — is the case the all-five default already covers. If the default ever narrows,
|
|
108
|
+
#: revisit this comment at the same time.
|
|
109
|
+
_ALIAS_SOURCES: dict[str, Callable[[str], list[str]]] = {
|
|
110
|
+
"library.chemistry": _kb_chemistry_aliases,
|
|
111
|
+
"library.assay": _kb_chemistry_aliases,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def surface_forms(field: str, value: str) -> list[str]:
|
|
116
|
+
"""Acceptable surface forms for a value — the value itself plus any curated aliases for its field."""
|
|
117
|
+
extra = _ALIAS_SOURCES.get(field)
|
|
118
|
+
return list(dict.fromkeys([value, *(extra(value) if extra else [])]))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def entails(quote: str, field: str, value: str) -> bool:
|
|
122
|
+
"""Does ``quote`` support ``value``? True iff some surface form is carried by the quote.
|
|
123
|
+
|
|
124
|
+
A form matches when it is a substring, or when all of its significant tokens appear in the quote
|
|
125
|
+
(order-independent — "Chromium Single Cell 3' v3" carries the alias "Chromium 3' v3"). Purely
|
|
126
|
+
generic tokens cannot entail on their own, so a quote saying only "single-cell RNA-seq" can never
|
|
127
|
+
entail a specific chemistry version.
|
|
128
|
+
|
|
129
|
+
**Know what this check cannot do.** Its power comes entirely from ``value`` being drawn from a
|
|
130
|
+
controlled vocabulary: for ``library.chemistry`` the value is a KB id, so the quote must contain a
|
|
131
|
+
real alias, and a quote about "droplet-based single-cell" cannot smuggle in a v3 chemistry. For a
|
|
132
|
+
free-text field the model supplies a value copied *out of* the quote, so ``form in quote`` is
|
|
133
|
+
trivially true and this returns True for anything. **Entailment is vacuous when value ⊆ quote.**
|
|
134
|
+
|
|
135
|
+
So span verification is a tripwire for fabricated and mis-attributed claims, NOT for field-assignment errors. A
|
|
136
|
+
real quote filed under the wrong field passes here by construction — `eval run --llm` caught
|
|
137
|
+
exactly that (worm husbandry filed as an experimental `condition`). The defense for free-text
|
|
138
|
+
fields is the prompt's operational definition of the field plus the evals corpus that measures it,
|
|
139
|
+
not this function. Tightening the matcher would not help; there is nothing here left to check.
|
|
140
|
+
|
|
141
|
+
**``library.prep_type`` is the one field checked by NORMALIZED match, not surface forms.** It is a
|
|
142
|
+
controlled biological concept (cells vs nuclei), so a terse quote ("snRNA-seq") must support a
|
|
143
|
+
verbose value ("single-nucleus RNA sequencing (snRNA-seq)") and a cell quote must never support a
|
|
144
|
+
nucleus value. Both sides are normalized to one of two preps and compared. This lets a nuclei claim
|
|
145
|
+
through — after which :func:`policy.resolve_features`, not the model, promotes ``GeneFull`` — while
|
|
146
|
+
never letting prose entail a *counting* decision: ``prep_type`` names no STARsolo feature, which is
|
|
147
|
+
exactly why ``processing.quantification`` is deliberately NOT handled this way (see ``_ALIAS_SOURCES``).
|
|
148
|
+
"""
|
|
149
|
+
if field == "library.prep_type":
|
|
150
|
+
qp = normalize_prep_type(quote)
|
|
151
|
+
return qp is not None and qp == normalize_prep_type(value)
|
|
152
|
+
q = _squash(quote).lower()
|
|
153
|
+
q_tokens = set(_tokens(q))
|
|
154
|
+
for form in surface_forms(field, value):
|
|
155
|
+
f = _squash(form).lower()
|
|
156
|
+
if f and f in q:
|
|
157
|
+
return True
|
|
158
|
+
f_tokens = [t for t in _tokens(f) if t not in _STOPWORDS]
|
|
159
|
+
if f_tokens and set(f_tokens) <= q_tokens:
|
|
160
|
+
return True
|
|
161
|
+
return False
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def verify_drafts(
|
|
165
|
+
drafts: list[AssertionDraft],
|
|
166
|
+
docs: list[NormalizedDoc],
|
|
167
|
+
*,
|
|
168
|
+
extractor: ExtractorProvenance,
|
|
169
|
+
) -> VerifyReport:
|
|
170
|
+
"""Compose code-owned :class:`Assertion`s from LLM drafts. Only fully-verified claims survive."""
|
|
171
|
+
by_sha = {d.doc_sha256: d for d in docs}
|
|
172
|
+
assertions: list[Assertion] = []
|
|
173
|
+
rejected: list[dict[str, object]] = []
|
|
174
|
+
|
|
175
|
+
for i, draft in enumerate(drafts):
|
|
176
|
+
# FIRST, and before anything about the document's CONTENT: may the model set this field at
|
|
177
|
+
# all? A quote can be real and entailing on a field that was never on offer, so no amount of
|
|
178
|
+
# document-checking below can substitute for this one (see .fields).
|
|
179
|
+
if draft.field not in PERMITTED_FIELDS:
|
|
180
|
+
rejected.append(
|
|
181
|
+
_reject(draft, "field_not_permitted", f"{draft.field!r} is not an assertable field")
|
|
182
|
+
)
|
|
183
|
+
continue
|
|
184
|
+
doc = by_sha.get(draft.span.doc_sha256)
|
|
185
|
+
if doc is not None and not permitted_for(draft.field, doc.scope, doc.role):
|
|
186
|
+
# ...and may it set this field from THIS document? A downloaded methods PDF may never
|
|
187
|
+
# steer the pipeline, and a BioSample record has no opinion about the chemistry. Both
|
|
188
|
+
# scope and role are code-owned — the flag the document arrived under, and the record it
|
|
189
|
+
# was rendered from — so this is a deterministic refusal, not a judgement about the
|
|
190
|
+
# sentence.
|
|
191
|
+
rejected.append(
|
|
192
|
+
_reject(
|
|
193
|
+
draft,
|
|
194
|
+
"field_not_permitted_for_doc",
|
|
195
|
+
f"{draft.field!r} may not be set by {doc.source_basename!r}, which is a "
|
|
196
|
+
f"{doc.scope}-scoped {doc.role} document",
|
|
197
|
+
)
|
|
198
|
+
)
|
|
199
|
+
continue
|
|
200
|
+
if doc is None:
|
|
201
|
+
rejected.append(
|
|
202
|
+
_reject(draft, "unknown_doc", f"no normalized doc {draft.span.doc_sha256}")
|
|
203
|
+
)
|
|
204
|
+
continue
|
|
205
|
+
found = find_span(doc.text, draft.span.quote)
|
|
206
|
+
if found is None:
|
|
207
|
+
# fabricated provenance: the quote is not in the document it cites
|
|
208
|
+
rejected.append(
|
|
209
|
+
_reject(draft, "span_not_found", "quote does not occur in the document")
|
|
210
|
+
)
|
|
211
|
+
continue
|
|
212
|
+
if not entails(draft.span.quote, draft.field, draft.value):
|
|
213
|
+
# real quote, wrong value — the failure span-verification alone would miss
|
|
214
|
+
rejected.append(
|
|
215
|
+
_reject(draft, "not_entailed", f"quote does not support value {draft.value!r}")
|
|
216
|
+
)
|
|
217
|
+
continue
|
|
218
|
+
start, end = found
|
|
219
|
+
assertions.append(
|
|
220
|
+
Assertion(
|
|
221
|
+
id=f"assert-{draft.span.doc_sha256[:8]}-{i}",
|
|
222
|
+
field=draft.field,
|
|
223
|
+
value=draft.value,
|
|
224
|
+
span=SourceSpan(
|
|
225
|
+
doc_sha256=draft.span.doc_sha256,
|
|
226
|
+
quote=draft.span.quote,
|
|
227
|
+
context=draft.span.context,
|
|
228
|
+
char_start=start, # computed by code, never by the model
|
|
229
|
+
char_end=end,
|
|
230
|
+
# ...and the page the offset falls on, same discipline: code reads it off the
|
|
231
|
+
# document's page index, the model's value (if any) is discarded. None unless it
|
|
232
|
+
# is a PDF, where "p.4" is a real, checkable location.
|
|
233
|
+
page=page_for_offset(doc.pages, start),
|
|
234
|
+
),
|
|
235
|
+
span_verified=True,
|
|
236
|
+
entailment_ok=True,
|
|
237
|
+
llm_confidence=draft.llm_confidence,
|
|
238
|
+
extractor=extractor,
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
return VerifyReport(assertions=assertions, rejected=rejected)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _reject(draft: AssertionDraft, reason: str, detail: str) -> dict[str, object]:
|
|
245
|
+
return {
|
|
246
|
+
"field": draft.field,
|
|
247
|
+
"value": draft.value,
|
|
248
|
+
"quote": draft.span.quote[:120],
|
|
249
|
+
"reason": reason,
|
|
250
|
+
"detail": detail,
|
|
251
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""``hooks`` — policy becomes mechanism (design §4.2).
|
|
2
|
+
|
|
3
|
+
`CLAUDE.md` can *say* "never read a whole FASTQ". Only a hook can stop one. These are the checked
|
|
4
|
+
edges of the rules: bounded reads, no absolute path in a manifest, and code — not the model — decides
|
|
5
|
+
whether an edit validated.
|
|
6
|
+
|
|
7
|
+
Wire them with ``seqforge hook install`` — see :mod:`.guards` for why the logic is typed and tested
|
|
8
|
+
rather than living in a shell script.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from .guards import (
|
|
14
|
+
HOOKS_VERSION,
|
|
15
|
+
Denial,
|
|
16
|
+
check_absolute_path_write,
|
|
17
|
+
check_unbounded_fastq,
|
|
18
|
+
post_tool_use_targets,
|
|
19
|
+
pre_tool_use,
|
|
20
|
+
questions_outstanding,
|
|
21
|
+
stop_decision,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"HOOKS_VERSION",
|
|
26
|
+
"Denial",
|
|
27
|
+
"pre_tool_use",
|
|
28
|
+
"post_tool_use_targets",
|
|
29
|
+
"stop_decision",
|
|
30
|
+
"check_unbounded_fastq",
|
|
31
|
+
"check_absolute_path_write",
|
|
32
|
+
"questions_outstanding",
|
|
33
|
+
]
|