seqforge 2026.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqforge/__init__.py +16 -0
- seqforge/cli/__init__.py +38 -0
- seqforge/cli/__main__.py +8 -0
- seqforge/cli/_common.py +105 -0
- seqforge/cli/compose.py +119 -0
- seqforge/cli/eval.py +103 -0
- seqforge/cli/harvest.py +417 -0
- seqforge/cli/hook.py +247 -0
- seqforge/cli/io.py +502 -0
- seqforge/cli/kb.py +348 -0
- seqforge/cli/manifest.py +536 -0
- seqforge/cli/probe.py +43 -0
- seqforge/cli/processing.py +192 -0
- seqforge/cli/project.py +52 -0
- seqforge/cli/resolve.py +55 -0
- seqforge/cli/root.py +66 -0
- seqforge/cli/run.py +463 -0
- seqforge/cli/schema.py +41 -0
- seqforge/compose/__init__.py +28 -0
- seqforge/compose/core.py +515 -0
- seqforge/compose/gates.py +113 -0
- seqforge/compose/params.py +447 -0
- seqforge/e2e.py +1926 -0
- seqforge/evals/__init__.py +78 -0
- seqforge/evals/case.py +382 -0
- seqforge/evals/grade.py +300 -0
- seqforge/evals/run.py +420 -0
- seqforge/harvest/__init__.py +121 -0
- seqforge/harvest/extract.py +319 -0
- seqforge/harvest/fields.py +212 -0
- seqforge/harvest/normalize.py +537 -0
- seqforge/harvest/prep.py +41 -0
- seqforge/harvest/providers.py +321 -0
- seqforge/harvest/verify.py +251 -0
- seqforge/hooks/__init__.py +33 -0
- seqforge/hooks/guards.py +214 -0
- seqforge/io/__init__.py +61 -0
- seqforge/io/archive.py +450 -0
- seqforge/io/attributes.py +190 -0
- seqforge/io/biosample/attributes.json +6341 -0
- seqforge/io/efo/labels.json +55 -0
- seqforge/io/efo.py +138 -0
- seqforge/io/onlist.py +661 -0
- seqforge/io/onlists/3M-february-2018.codes.gz +0 -0
- seqforge/io/onlists/737K-arc-v1.codes.gz +0 -0
- seqforge/io/onlists/737K-august-2016.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3.codes.gz +0 -0
- seqforge/io/onlists/index.json +74 -0
- seqforge/io/remote.py +659 -0
- seqforge/io/taxonomy.py +194 -0
- seqforge/kb/__init__.py +62 -0
- seqforge/kb/anchor.py +169 -0
- seqforge/kb/generate.py +147 -0
- seqforge/kb/loader.py +152 -0
- seqforge/kb/roundtrip.py +112 -0
- seqforge/kb/schema.py +422 -0
- seqforge/kb/specs/10x-3p-gex/spec.yaml +62 -0
- seqforge/kb/specs/10x-3p-gex-v2/README.md +41 -0
- seqforge/kb/specs/10x-3p-gex-v2/spec.yaml +83 -0
- seqforge/kb/specs/10x-3p-gex-v3/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3/spec.yaml +118 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/spec.yaml +124 -0
- seqforge/kb/specs/bd-rhapsody-wta/README.md +103 -0
- seqforge/kb/specs/bd-rhapsody-wta/spec.yaml +130 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced/spec.yaml +99 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v1/spec.yaml +93 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v2/spec.yaml +81 -0
- seqforge/kb/specs/bulk-rnaseq-pe/README.md +35 -0
- seqforge/kb/specs/bulk-rnaseq-pe/spec.yaml +97 -0
- seqforge/kb/specs/splitseq/README.md +51 -0
- seqforge/kb/specs/splitseq/spec.yaml +157 -0
- seqforge/manifest/__init__.py +61 -0
- seqforge/manifest/fill.py +531 -0
- seqforge/manifest/hash.py +77 -0
- seqforge/manifest/instruct.py +114 -0
- seqforge/manifest/policy.py +409 -0
- seqforge/manifest/validate.py +274 -0
- seqforge/models/__init__.py +268 -0
- seqforge/models/assertion.py +68 -0
- seqforge/models/base.py +100 -0
- seqforge/models/blocker.py +71 -0
- seqforge/models/conflict.py +47 -0
- seqforge/models/dataset.py +320 -0
- seqforge/models/evidenced.py +54 -0
- seqforge/models/observation.py +157 -0
- seqforge/models/processing.py +231 -0
- seqforge/models/records.py +145 -0
- seqforge/models/resolve.py +216 -0
- seqforge/probe/__init__.py +46 -0
- seqforge/probe/core.py +232 -0
- seqforge/probe/signals.py +250 -0
- seqforge/probe/streaming.py +118 -0
- seqforge/project.py +177 -0
- seqforge/py.typed +0 -0
- seqforge/resolve/__init__.py +98 -0
- seqforge/resolve/assign.py +204 -0
- seqforge/resolve/cache.py +119 -0
- seqforge/resolve/confuse.py +215 -0
- seqforge/resolve/engine.py +646 -0
- seqforge/resolve/escalate.py +668 -0
- seqforge/resolve/evaluators.py +306 -0
- seqforge/resolve/geometry.py +89 -0
- seqforge/resolve/group.py +85 -0
- seqforge/resolve/records.py +550 -0
- seqforge/resolve/scoring.py +373 -0
- seqforge/resolve/window.py +206 -0
- seqforge/workflows/__init__.py +234 -0
- seqforge/workflows/cram.py +117 -0
- seqforge/workflows/h5ad.py +368 -0
- seqforge/workflows/map/star.smk +101 -0
- seqforge/workflows/map/starsolo.smk +360 -0
- seqforge/workflows/qc.py +157 -0
- seqforge/workspace.py +125 -0
- seqforge-2026.7.1.dist-info/METADATA +125 -0
- seqforge-2026.7.1.dist-info/RECORD +124 -0
- seqforge-2026.7.1.dist-info/WHEEL +4 -0
- seqforge-2026.7.1.dist-info/entry_points.txt +2 -0
- seqforge-2026.7.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
"""Per-technology scoring: evidence matrix -> injective assignment -> cardinality-normalized score.
|
|
2
|
+
|
|
3
|
+
For technology ``t`` with roles ``R_t`` (its reads) and files ``F``, cell ``M[r][f]`` is ``FORBIDDEN``
|
|
4
|
+
if any ``requires(r)`` gate FAILs or any ``excludes(r)`` gate PASSes, else the normalized weighted
|
|
5
|
+
``supports(r)`` sum in ``[0, 1]``. ``FORBIDDEN`` is an internal ``Cell(forbidden=True)`` flag, never a
|
|
6
|
+
``±inf`` — serialized it is ``{"status": "forbidden"}`` so no infinity ever crosses the JSON boundary.
|
|
7
|
+
|
|
8
|
+
``score(t) = raw / |R_t| - (λ / |R_t|)·|F \\ A*|`` is cardinality-normalized so a 2-role 10x and a
|
|
9
|
+
6-role SPLiT-seq are comparable. The filename prior enters as a sub-threshold ``β``-scaled nudge that
|
|
10
|
+
can only break an exact byte-tie.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections import defaultdict
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
|
|
18
|
+
from ..io import OnlistNotAvailable, OnlistRegistry
|
|
19
|
+
from ..kb.schema import OnlistHitRate, Read, SegmentLength, Spec
|
|
20
|
+
from ..models.resolve import TechScore
|
|
21
|
+
from .assign import AssignmentResult, best_assignment
|
|
22
|
+
from .evaluators import Outcome, evaluate, onlist_admits_over_length, read_length_compatible
|
|
23
|
+
from .window import WindowProbe
|
|
24
|
+
|
|
25
|
+
_LAMBDA = 0.25 # penalty per unassigned (leftover) file, cardinality-normalized
|
|
26
|
+
_BETA = 0.01 # filename-prior coefficient (<< min support weight -> tie-break only)
|
|
27
|
+
_GLOBAL_COEF = 0.001 # tech-global supports (header_index) contribute a sub-threshold tie-break
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class Cell:
|
|
32
|
+
"""One evidence-matrix cell: forbidden, or a finite support value in ``[0, 1]``."""
|
|
33
|
+
|
|
34
|
+
forbidden: bool
|
|
35
|
+
value: float
|
|
36
|
+
reason: str = ""
|
|
37
|
+
|
|
38
|
+
def to_json(self) -> dict[str, object]:
|
|
39
|
+
if self.forbidden:
|
|
40
|
+
return {"status": "forbidden", "reason": self.reason}
|
|
41
|
+
return {"status": "scored", "value": round(self.value, 4)}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class TechEvaluation:
|
|
46
|
+
"""The full scored verdict for one technology against the dataset's files."""
|
|
47
|
+
|
|
48
|
+
tech: str
|
|
49
|
+
roles: list[str]
|
|
50
|
+
file_shas: list[str]
|
|
51
|
+
matrix: dict[str, list[Cell]]
|
|
52
|
+
assignment: AssignmentResult
|
|
53
|
+
score: TechScore
|
|
54
|
+
rung: int
|
|
55
|
+
used_onlist: bool
|
|
56
|
+
equivalence_members: list[str]
|
|
57
|
+
barcode_role_ids: list[str]
|
|
58
|
+
unfillable_role_ids: list[str]
|
|
59
|
+
cdna_role_fillable: bool
|
|
60
|
+
#: A barcode role's onlist POSITIVELY hit — the whitelist actually identified barcodes in the data
|
|
61
|
+
#: (best hit clears the floor-anchored admission bar), not merely that an onlist was consulted. This
|
|
62
|
+
#: is the honest "this data IS barcoded" signal escalate uses so the barcodeless bulk fallback never
|
|
63
|
+
#: shadows a real single-cell library, WITHOUT hijacking genuine bulk (whose barcode window sits at
|
|
64
|
+
#: the whitelist floor, so this stays False even when a barcode read passes on over-length geometry).
|
|
65
|
+
barcode_onlist_hit: bool = False
|
|
66
|
+
#: A barcode role's onlist whitelist was REGISTERED and materializable — we had a list to check
|
|
67
|
+
#: against. Lets F1b (escalate) tell "checked, missed" (refuse) from "never checked" (abstain).
|
|
68
|
+
barcode_onlist_available: bool = False
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def valid(self) -> bool:
|
|
72
|
+
return self.score.status == "scored"
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def value(self) -> float:
|
|
76
|
+
return self.score.value if self.score.value is not None else float("-inf")
|
|
77
|
+
|
|
78
|
+
def role_assignment_shas(self) -> dict[str, str]:
|
|
79
|
+
"""role_id -> file sha256 for the winning assignment (empty if forbidden)."""
|
|
80
|
+
return {self.roles[r]: self.file_shas[f] for r, f in self.assignment.mapping.items()}
|
|
81
|
+
|
|
82
|
+
def matrix_json(self) -> dict[str, dict[str, dict[str, object]]]:
|
|
83
|
+
"""JSON-safe evidence matrix: ``{role: {file_sha: {status, ...}}}`` — no ``±inf``."""
|
|
84
|
+
out: dict[str, dict[str, dict[str, object]]] = {}
|
|
85
|
+
for role, cells in self.matrix.items():
|
|
86
|
+
out[role] = {self.file_shas[f]: cells[f].to_json() for f in range(len(cells))}
|
|
87
|
+
return out
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def filename_prior(read: Read, basename: str) -> float:
|
|
91
|
+
"""1.0 if the file's name carries the read's conventional token (``_R1_`` -> ``R1``), else 0.0."""
|
|
92
|
+
if not read.file_hint:
|
|
93
|
+
return 0.0
|
|
94
|
+
token = read.file_hint.strip("_")
|
|
95
|
+
return 1.0 if token and token in basename else 0.0
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _score_cell(
|
|
99
|
+
read: Read,
|
|
100
|
+
wp: WindowProbe,
|
|
101
|
+
spec: Spec,
|
|
102
|
+
registry: OnlistRegistry,
|
|
103
|
+
requires: list[object],
|
|
104
|
+
excludes: list[object],
|
|
105
|
+
supports: list[tuple[object, float]],
|
|
106
|
+
) -> tuple[Cell, bool]:
|
|
107
|
+
used_onlist = False
|
|
108
|
+
if read_length_compatible(read, wp) == Outcome.FAIL:
|
|
109
|
+
reason = (
|
|
110
|
+
f"read-length incompatible (mode {wp.mode_length} vs {read.min_len}..{read.max_len})"
|
|
111
|
+
)
|
|
112
|
+
return Cell(forbidden=True, value=0.0, reason=reason), used_onlist
|
|
113
|
+
for t in requires:
|
|
114
|
+
ev = evaluate(t, read, wp, spec, registry)
|
|
115
|
+
used_onlist = used_onlist or ev.used_onlist
|
|
116
|
+
if ev.outcome == Outcome.FAIL:
|
|
117
|
+
# A `segment_length` FAIL in the over-length DEAD ZONE (canonical < mode < over_length_min)
|
|
118
|
+
# is not necessarily a wrong read: an R1 over-sequenced to e.g. 75 bp is a real barcode
|
|
119
|
+
# read whose CB/UMI still sit at the fixed offsets — over_length_min is deliberately high
|
|
120
|
+
# (100) so a 60-94 bp cDNA is not admitted on length alone. The WHITELIST is the
|
|
121
|
+
# disambiguator: a genuine cDNA of the same length misses it, a real barcode hits it. So if
|
|
122
|
+
# the barcode onlist hits, admit as over-length (rung 3); else keep the FAIL. This is the
|
|
123
|
+
# one place a rung-3 result overrides a rung-0-2 length gate, and it only ever ADMITS
|
|
124
|
+
# (#7 — GSE126954's over-sequenced SRX5411291, which the v2 length gate otherwise forbids).
|
|
125
|
+
if _over_length_admitted_by_onlist(t, read, wp, spec, registry, supports):
|
|
126
|
+
used_onlist = True
|
|
127
|
+
continue
|
|
128
|
+
return Cell(
|
|
129
|
+
forbidden=True, value=0.0, reason=f"requires FAIL: {ev.detail}"
|
|
130
|
+
), used_onlist
|
|
131
|
+
for t in excludes:
|
|
132
|
+
ev = evaluate(t, read, wp, spec, registry)
|
|
133
|
+
used_onlist = used_onlist or ev.used_onlist
|
|
134
|
+
if ev.outcome == Outcome.PASS:
|
|
135
|
+
return Cell(
|
|
136
|
+
forbidden=True, value=0.0, reason=f"excludes matched: {ev.detail}"
|
|
137
|
+
), used_onlist
|
|
138
|
+
total_w = sum(w for _, w in supports)
|
|
139
|
+
value = 0.0
|
|
140
|
+
if total_w > 0:
|
|
141
|
+
acc = 0.0
|
|
142
|
+
for when, weight in supports:
|
|
143
|
+
ev = evaluate(when, read, wp, spec, registry)
|
|
144
|
+
used_onlist = used_onlist or ev.used_onlist
|
|
145
|
+
acc += weight * ev.score
|
|
146
|
+
value = acc / total_w
|
|
147
|
+
return Cell(forbidden=False, value=value, reason="scored"), used_onlist
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _over_length_admitted_by_onlist(
|
|
151
|
+
test: object,
|
|
152
|
+
read: Read,
|
|
153
|
+
wp: WindowProbe,
|
|
154
|
+
spec: Spec,
|
|
155
|
+
registry: OnlistRegistry,
|
|
156
|
+
supports: list[tuple[object, float]],
|
|
157
|
+
) -> bool:
|
|
158
|
+
"""Admit a barcode read over-sequenced into the length dead zone IFF its barcode prefix hits the
|
|
159
|
+
whitelist. Deliberately narrow and additive: it fires ONLY on a ``segment_length`` FAIL whose mode
|
|
160
|
+
is strictly between the canonical ``length`` and ``over_length_min`` (a read at/below the canonical
|
|
161
|
+
length, or already ``>= over_length_min``, does not reach here), and ONLY when an ``onlist_hit_rate``
|
|
162
|
+
support clears the FLOOR-ANCHORED admission bar (:func:`onlist_admits_over_length`) — a LOWER bar
|
|
163
|
+
than the support's own ``min`` PASS threshold, because admission asks "barcode or cDNA?" not
|
|
164
|
+
"confident barcode?", so it can admit a read whose exact hit rate sits below ``min`` yet far above
|
|
165
|
+
chance. A cDNA read of the same length hits the whitelist at its floor and stays forbidden, so
|
|
166
|
+
rung-0-2 separability between single-cell and cDNA-only chemistries is preserved.
|
|
167
|
+
"""
|
|
168
|
+
if not isinstance(test, SegmentLength) or test.over_length_min is None:
|
|
169
|
+
return False
|
|
170
|
+
if not (test.length < wp.mode_length < test.over_length_min):
|
|
171
|
+
return False # not the dead zone: canonical is exact-checked, >= over_length_min already PASSes
|
|
172
|
+
for when, _weight in supports:
|
|
173
|
+
# A FLOOR-ANCHORED bar, not the support `min`: admission asks "barcode or cDNA?", not
|
|
174
|
+
# "confident barcode?". Seqforge matches exactly (no 1MM correction), so a real over-sequenced
|
|
175
|
+
# barcode read with ordinary error hits below the 0.6 support gate yet far above chance -- the
|
|
176
|
+
# gate rejected SRX5411291 and it fell to bulk. See `onlist_admits_over_length`.
|
|
177
|
+
if isinstance(when, OnlistHitRate) and onlist_admits_over_length(
|
|
178
|
+
when, read, wp, spec, registry
|
|
179
|
+
):
|
|
180
|
+
return True
|
|
181
|
+
return False
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _clears_onlist_bar(
|
|
185
|
+
read: Read,
|
|
186
|
+
wp: WindowProbe,
|
|
187
|
+
supports: list[tuple[object, float]],
|
|
188
|
+
spec: Spec,
|
|
189
|
+
registry: OnlistRegistry,
|
|
190
|
+
) -> bool:
|
|
191
|
+
"""True iff some onlist support for this barcode ``read`` clears the floor-anchored admission bar on
|
|
192
|
+
``wp`` — the file's barcode prefix hits the whitelist far above the ~1e-4..1e-3 random floor, i.e.
|
|
193
|
+
this read LOOKS barcoded. The predicate both the F1a seating constraint and ``barcode_onlist_hit``
|
|
194
|
+
are built from."""
|
|
195
|
+
return any(
|
|
196
|
+
isinstance(when, OnlistHitRate)
|
|
197
|
+
and onlist_admits_over_length(when, read, wp, spec, registry)
|
|
198
|
+
for when, _w in supports
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _barcode_onlist_available(
|
|
203
|
+
spec: Spec,
|
|
204
|
+
registry: OnlistRegistry,
|
|
205
|
+
barcode_role_ids: list[str],
|
|
206
|
+
sup_by: dict[str, list[tuple[object, float]]],
|
|
207
|
+
) -> bool:
|
|
208
|
+
"""True iff at least one barcode role's onlist whitelist is REGISTERED and materializable — we had a
|
|
209
|
+
list to check against. When False the whitelist was never consulted, so a ``barcode_onlist_hit`` of
|
|
210
|
+
False means 'could not check', not 'barcode absent': F1b must abstain, not refuse."""
|
|
211
|
+
for rid in barcode_role_ids:
|
|
212
|
+
for when, _w in sup_by[rid]:
|
|
213
|
+
if not isinstance(when, OnlistHitRate):
|
|
214
|
+
continue
|
|
215
|
+
ref = spec.onlists.get(when.onlist)
|
|
216
|
+
if ref is None or not registry.has(ref.registry):
|
|
217
|
+
continue
|
|
218
|
+
try:
|
|
219
|
+
registry.packed(ref.registry)
|
|
220
|
+
except OnlistNotAvailable:
|
|
221
|
+
continue
|
|
222
|
+
return True
|
|
223
|
+
return False
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _global_support(
|
|
227
|
+
global_supports: list[tuple[object, float]],
|
|
228
|
+
reads: list[Read],
|
|
229
|
+
wps: list[WindowProbe],
|
|
230
|
+
spec: Spec,
|
|
231
|
+
registry: OnlistRegistry,
|
|
232
|
+
) -> float:
|
|
233
|
+
"""Normalized score of read-less supports (e.g. ``header_index``), max over files."""
|
|
234
|
+
if not global_supports or not wps:
|
|
235
|
+
return 0.0
|
|
236
|
+
total_w = sum(w for _, w in global_supports)
|
|
237
|
+
if total_w <= 0:
|
|
238
|
+
return 0.0
|
|
239
|
+
acc = 0.0
|
|
240
|
+
for when, weight in global_supports:
|
|
241
|
+
best = max(evaluate(when, reads[0], wp, spec, registry).score for wp in wps)
|
|
242
|
+
acc += weight * best
|
|
243
|
+
return acc / total_w
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def build_tech_evaluation(
|
|
247
|
+
spec: Spec, wps: list[WindowProbe], registry: OnlistRegistry
|
|
248
|
+
) -> TechEvaluation:
|
|
249
|
+
"""Score one technology against the dataset's files (the evidence matrix + joint assignment)."""
|
|
250
|
+
reads_by_id = {r.id: r for r in spec.reads}
|
|
251
|
+
roles = [r.id for r in spec.reads]
|
|
252
|
+
n_files = len(wps)
|
|
253
|
+
file_shas = [wp.observation.file.sha256 for wp in wps]
|
|
254
|
+
barcode_role_ids = [r.id for r in spec.reads if any(el.type == "barcode" for el in r.elements)]
|
|
255
|
+
|
|
256
|
+
req_by: dict[str, list[object]] = defaultdict(list)
|
|
257
|
+
exc_by: dict[str, list[object]] = defaultdict(list)
|
|
258
|
+
sup_by: dict[str, list[tuple[object, float]]] = defaultdict(list)
|
|
259
|
+
global_sup: list[tuple[object, float]] = []
|
|
260
|
+
for t in spec.signature.requires:
|
|
261
|
+
rid = getattr(t, "read", None)
|
|
262
|
+
if rid is not None:
|
|
263
|
+
req_by[rid].append(t)
|
|
264
|
+
for t in spec.signature.excludes:
|
|
265
|
+
rid = getattr(t, "read", None)
|
|
266
|
+
if rid is not None:
|
|
267
|
+
exc_by[rid].append(t)
|
|
268
|
+
for s in spec.signature.supports:
|
|
269
|
+
rid = getattr(s.when, "read", None)
|
|
270
|
+
if rid is not None:
|
|
271
|
+
sup_by[rid].append((s.when, s.weight))
|
|
272
|
+
else:
|
|
273
|
+
global_sup.append((s.when, s.weight))
|
|
274
|
+
|
|
275
|
+
matrix: dict[str, list[Cell]] = {}
|
|
276
|
+
score_m: list[list[float]] = []
|
|
277
|
+
forbidden_m: list[list[bool]] = []
|
|
278
|
+
prior_m: list[list[float]] = []
|
|
279
|
+
used_onlist = False
|
|
280
|
+
for rid in roles:
|
|
281
|
+
read = reads_by_id[rid]
|
|
282
|
+
cells: list[Cell] = []
|
|
283
|
+
row_score: list[float] = []
|
|
284
|
+
row_forbid: list[bool] = []
|
|
285
|
+
row_prior: list[float] = []
|
|
286
|
+
for wp in wps:
|
|
287
|
+
cell, uo = _score_cell(read, wp, spec, registry, req_by[rid], exc_by[rid], sup_by[rid])
|
|
288
|
+
used_onlist = used_onlist or uo
|
|
289
|
+
cells.append(cell)
|
|
290
|
+
row_score.append(cell.value)
|
|
291
|
+
row_forbid.append(cell.forbidden)
|
|
292
|
+
row_prior.append(_BETA * filename_prior(read, wp.observation.file.basename))
|
|
293
|
+
matrix[rid] = cells
|
|
294
|
+
score_m.append(row_score)
|
|
295
|
+
forbidden_m.append(row_forbid)
|
|
296
|
+
prior_m.append(row_prior)
|
|
297
|
+
|
|
298
|
+
# F1a — a barcode role must be seated on a read that LOOKS barcoded. Per barcode role, which files
|
|
299
|
+
# clear the floor-anchored onlist bar (prefix hits the whitelist far above chance)? Computed once and
|
|
300
|
+
# reused for the seating constraint here and for the `barcode_onlist_hit` signal below.
|
|
301
|
+
barcode_clears: dict[str, list[bool]] = {
|
|
302
|
+
rid: [_clears_onlist_bar(reads_by_id[rid], wp, sup_by[rid], spec, registry) for wp in wps]
|
|
303
|
+
for rid in barcode_role_ids
|
|
304
|
+
}
|
|
305
|
+
# When some file clears, forbid seating the role on any file that does NOT — so sum-maximization
|
|
306
|
+
# cannot park the barcode role on a cDNA-length mate that merely out-scored the real barcode read on
|
|
307
|
+
# other supports. The swap it prevents: a real barcode read carrying ordinary sequencing error can
|
|
308
|
+
# score higher as cDNA than as its own barcode, and the true cDNA read of a low-diversity library
|
|
309
|
+
# scores low on both roles, so `(barcode->cDNA)+(cDNA->barcode)` beats the honest seat (PRJNA658829
|
|
310
|
+
# SRR12575567). A file that clears the bar is always a barcode — a real cDNA hits the whitelist at the
|
|
311
|
+
# ~1e-4..1e-3 random floor, far below the bar — so forcing a clearing read into barcode is never
|
|
312
|
+
# wrong. No file clearing => a no-op here; escalate's F1b then refuses rather than compose a barcode
|
|
313
|
+
# read that matches no whitelist.
|
|
314
|
+
for rid, clears in barcode_clears.items():
|
|
315
|
+
if not any(clears):
|
|
316
|
+
continue
|
|
317
|
+
ri = roles.index(rid)
|
|
318
|
+
for f in range(n_files):
|
|
319
|
+
if not clears[f] and not forbidden_m[ri][f]:
|
|
320
|
+
forbidden_m[ri][f] = True
|
|
321
|
+
matrix[rid][f] = Cell(
|
|
322
|
+
forbidden=True,
|
|
323
|
+
value=0.0,
|
|
324
|
+
reason="barcode role: another read hits the onlist and this one does not",
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
assignment = best_assignment(len(roles), n_files, score_m, forbidden_m, prior_m)
|
|
328
|
+
global_bonus = _global_support(global_sup, list(reads_by_id.values()), wps, spec, registry)
|
|
329
|
+
|
|
330
|
+
# Did a barcode role's onlist actually hit (not just get consulted)? True iff some file cleared the
|
|
331
|
+
# floor-anchored admission bar above (reusing F1a's `barcode_clears`) — the "this data is barcoded"
|
|
332
|
+
# signal escalate uses to keep the barcodeless fallback from shadowing a real single-cell library,
|
|
333
|
+
# and to refuse (F1b) when a barcoded winner has no whitelist-hitting read. Genuine bulk sits at the
|
|
334
|
+
# whitelist floor and stays False even when its read passes over-length geometry.
|
|
335
|
+
barcode_onlist_hit = any(any(clears) for clears in barcode_clears.values())
|
|
336
|
+
barcode_onlist_available = _barcode_onlist_available(spec, registry, barcode_role_ids, sup_by)
|
|
337
|
+
unfillable_role_ids = [roles[i] for i in assignment.unfillable_roles]
|
|
338
|
+
cdna_role_fillable = any(
|
|
339
|
+
any(el.type in ("cdna", "gdna") for el in reads_by_id[rid].elements)
|
|
340
|
+
and any(not c.forbidden for c in matrix[rid])
|
|
341
|
+
for rid in roles
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
if assignment.valid:
|
|
345
|
+
raw_norm = assignment.raw / len(roles)
|
|
346
|
+
penalty = (_LAMBDA / len(roles)) * len(assignment.unassigned_files)
|
|
347
|
+
value = raw_norm - penalty + _GLOBAL_COEF * global_bonus
|
|
348
|
+
score = TechScore(technology=spec.identity.id, status="scored", value=round(value, 6))
|
|
349
|
+
else:
|
|
350
|
+
reason = (
|
|
351
|
+
f"unfillable role(s): {unfillable_role_ids}"
|
|
352
|
+
if unfillable_role_ids
|
|
353
|
+
else "no valid injective role assignment"
|
|
354
|
+
)
|
|
355
|
+
score = TechScore(technology=spec.identity.id, status="forbidden", reason=reason)
|
|
356
|
+
|
|
357
|
+
equivalence = [c.id for c in spec.confusable_with if c.relationship == "processing_equivalent"]
|
|
358
|
+
return TechEvaluation(
|
|
359
|
+
tech=spec.identity.id,
|
|
360
|
+
roles=roles,
|
|
361
|
+
file_shas=file_shas,
|
|
362
|
+
matrix=matrix,
|
|
363
|
+
assignment=assignment,
|
|
364
|
+
score=score,
|
|
365
|
+
rung=3 if used_onlist else 2,
|
|
366
|
+
used_onlist=used_onlist,
|
|
367
|
+
equivalence_members=equivalence,
|
|
368
|
+
barcode_role_ids=barcode_role_ids,
|
|
369
|
+
unfillable_role_ids=unfillable_role_ids,
|
|
370
|
+
cdna_role_fillable=cdna_role_fillable,
|
|
371
|
+
barcode_onlist_hit=barcode_onlist_hit,
|
|
372
|
+
barcode_onlist_available=barcode_onlist_available,
|
|
373
|
+
)
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""``WindowProbe`` — the bounded, role-conditioned view the resolver scores against (§3.1).
|
|
2
|
+
|
|
3
|
+
The structural :class:`Observation` is deliberately role-free and carries no raw sequences, but the
|
|
4
|
+
resolver needs role-conditioned answers — distinct-ratio and onlist hit-rate over *arbitrary*
|
|
5
|
+
``[start, end)`` windows a candidate technology proposes. ``WindowProbe`` pairs the Observation with
|
|
6
|
+
the same bounded, in-memory sample that produced it (from :func:`probe.probe_sample`) and answers
|
|
7
|
+
those window queries. It never re-reads the file: the sample is already within the budget.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
|
|
14
|
+
from ..io import HitResult, Orientation, PackedOnlist, onlist_hit_rate
|
|
15
|
+
from ..io.onlist import pack_barcode, revcomp
|
|
16
|
+
from ..kb.schema import Read
|
|
17
|
+
from ..models.observation import CycleComposition, Observation
|
|
18
|
+
|
|
19
|
+
_IUPAC = {
|
|
20
|
+
"A": "A", "C": "C", "G": "G", "T": "T",
|
|
21
|
+
"R": "AG", "Y": "CT", "S": "GC", "W": "AT", "K": "GT", "M": "AC",
|
|
22
|
+
"B": "CGT", "D": "AGT", "H": "ACT", "V": "ACG", "N": "ACGT",
|
|
23
|
+
} # fmt: skip
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class WindowProbe:
|
|
28
|
+
"""An Observation plus its bounded sampled sequences, queryable over arbitrary windows.
|
|
29
|
+
|
|
30
|
+
Most windows are a fixed ``[start, end)`` column. **Anchored** (floating) elements are not — their
|
|
31
|
+
per-read position is recovered by :func:`seqforge.kb.anchor.resolve_windows`, and the
|
|
32
|
+
``anchored_*`` methods answer distinct-ratio / onlist-hit over those per-read frames instead. The
|
|
33
|
+
full per-read frame is resolved once and memoized (``_frame_cache``, keyed by the ``Read`` object)
|
|
34
|
+
so scoring three cell-label blocks on one read does not realign it three times.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
observation: Observation
|
|
38
|
+
seqs: list[str]
|
|
39
|
+
#: read object identity -> per-sampled-read resolved element windows (``None`` where the frame was
|
|
40
|
+
#: not found). Mutated in place; the frozen dataclass forbids rebinding the attribute, not filling
|
|
41
|
+
#: the dict. ``compare=False`` so two probes with equal seqs stay equal regardless of what was cached.
|
|
42
|
+
_frame_cache: dict[int, list[dict[str, tuple[int, int]] | None]] = field(
|
|
43
|
+
default_factory=dict, compare=False, repr=False
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def n_sampled(self) -> int:
|
|
48
|
+
return len(self.seqs)
|
|
49
|
+
|
|
50
|
+
def _frames(self, read: Read) -> list[dict[str, tuple[int, int]] | None]:
|
|
51
|
+
"""Per-read resolved element windows for an anchored layout, memoized per ``Read``."""
|
|
52
|
+
key = id(read)
|
|
53
|
+
cached = self._frame_cache.get(key)
|
|
54
|
+
if cached is None:
|
|
55
|
+
from ..kb.anchor import resolve_windows
|
|
56
|
+
|
|
57
|
+
cached = [resolve_windows(s, read) for s in self.seqs]
|
|
58
|
+
self._frame_cache[key] = cached
|
|
59
|
+
return cached
|
|
60
|
+
|
|
61
|
+
def anchored_windows(self, read: Read, element_name: str) -> list[tuple[int, int] | None]:
|
|
62
|
+
"""The per-read ``[start, end)`` of one floating element (``None`` where the frame was lost)."""
|
|
63
|
+
return [f.get(element_name) if f is not None else None for f in self._frames(read)]
|
|
64
|
+
|
|
65
|
+
def anchored_distinct_ratio(self, read: Read, element_name: str) -> float | None:
|
|
66
|
+
"""``distinct/total`` of a floating element's per-read slices; ``None`` if no frame resolved."""
|
|
67
|
+
windows = self.anchored_windows(read, element_name)
|
|
68
|
+
slices = [
|
|
69
|
+
self.seqs[i][s:e] for i, w in enumerate(windows) if w is not None for s, e in (w,)
|
|
70
|
+
]
|
|
71
|
+
slices = [s for s in slices if s]
|
|
72
|
+
if not slices:
|
|
73
|
+
return None
|
|
74
|
+
return len(set(slices)) / len(slices)
|
|
75
|
+
|
|
76
|
+
def anchored_onlist_hit(
|
|
77
|
+
self,
|
|
78
|
+
read: Read,
|
|
79
|
+
element_name: str,
|
|
80
|
+
onlist: PackedOnlist,
|
|
81
|
+
orientation: Orientation = "either",
|
|
82
|
+
) -> HitResult:
|
|
83
|
+
"""Whitelist hit-rate of a floating element, sliced per read at its resolved frame.
|
|
84
|
+
|
|
85
|
+
The anchored twin of :func:`~seqforge.io.onlist.onlist_hit_rate`: no offset scan (the frame IS
|
|
86
|
+
the offset), forward and/or reverse-complement per ``orientation``. ``n_tested`` counts reads
|
|
87
|
+
whose frame resolved to a window of the onlist's width; a lost frame simply does not contribute.
|
|
88
|
+
"""
|
|
89
|
+
windows = self.anchored_windows(read, element_name)
|
|
90
|
+
strands = (
|
|
91
|
+
["forward"]
|
|
92
|
+
if orientation == "forward"
|
|
93
|
+
else ["revcomp"]
|
|
94
|
+
if orientation == "revcomp"
|
|
95
|
+
else ["forward", "revcomp"]
|
|
96
|
+
)
|
|
97
|
+
best = HitResult(
|
|
98
|
+
hit_rate=0.0, orientation="forward", offset=0, n_tested=0, floor=onlist.floor
|
|
99
|
+
)
|
|
100
|
+
for strand in strands:
|
|
101
|
+
tested = 0
|
|
102
|
+
hits = 0
|
|
103
|
+
for i, w in enumerate(windows):
|
|
104
|
+
if w is None:
|
|
105
|
+
continue
|
|
106
|
+
sub = self.seqs[i][w[0] : w[1]]
|
|
107
|
+
if len(sub) != onlist.width:
|
|
108
|
+
continue
|
|
109
|
+
tested += 1
|
|
110
|
+
code = pack_barcode(revcomp(sub) if strand == "revcomp" else sub)
|
|
111
|
+
if code is not None and onlist.contains(code):
|
|
112
|
+
hits += 1
|
|
113
|
+
if tested and hits / tested > best.hit_rate:
|
|
114
|
+
best = HitResult(
|
|
115
|
+
hit_rate=hits / tested,
|
|
116
|
+
orientation=strand, # type: ignore[arg-type]
|
|
117
|
+
offset=0,
|
|
118
|
+
n_tested=tested,
|
|
119
|
+
floor=onlist.floor,
|
|
120
|
+
)
|
|
121
|
+
return best
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def mode_length(self) -> int:
|
|
125
|
+
return self.observation.read_length.mode
|
|
126
|
+
|
|
127
|
+
def distinct_ratio(self, start: int, end: int) -> float | None:
|
|
128
|
+
"""``distinct/total`` over ``[start, end)`` (role-conditioned; a supports signal, never a gate)."""
|
|
129
|
+
window = [s[start:end] for s in self.seqs if len(s) >= end]
|
|
130
|
+
if not window:
|
|
131
|
+
return None
|
|
132
|
+
return len(set(window)) / len(window)
|
|
133
|
+
|
|
134
|
+
def composition_window(self, start: int, end: int | None) -> list[CycleComposition]:
|
|
135
|
+
"""Per-cycle composition over cycles ``[start, end)`` (``end=None`` => to the longest read)."""
|
|
136
|
+
comps = self.observation.per_cycle_composition
|
|
137
|
+
stop = len(comps) if end is None else min(end, len(comps))
|
|
138
|
+
return [c for c in comps[start:stop]]
|
|
139
|
+
|
|
140
|
+
def onlist_hit(
|
|
141
|
+
self, start: int, onlist: PackedOnlist, orientation: Orientation = "either"
|
|
142
|
+
) -> HitResult:
|
|
143
|
+
"""Best whitelist hit anchored at ``start`` (width from the onlist), fwd + revcomp + offset scan."""
|
|
144
|
+
return onlist_hit_rate(self.seqs, start, onlist, orientation=orientation)
|
|
145
|
+
|
|
146
|
+
def motif_rate(
|
|
147
|
+
self,
|
|
148
|
+
motif: str,
|
|
149
|
+
*,
|
|
150
|
+
where: str = "anywhere",
|
|
151
|
+
search_start: int | None = None,
|
|
152
|
+
search_end: int | None = None,
|
|
153
|
+
max_mismatch: int = 1,
|
|
154
|
+
) -> float | None:
|
|
155
|
+
"""Fraction of reads matching an IUPAC ``motif`` (<= ``max_mismatch``) in the search window."""
|
|
156
|
+
m = len(motif)
|
|
157
|
+
if m == 0:
|
|
158
|
+
return None
|
|
159
|
+
tested = 0
|
|
160
|
+
matched = 0
|
|
161
|
+
for seq in self.seqs:
|
|
162
|
+
if len(seq) < m:
|
|
163
|
+
continue
|
|
164
|
+
tested += 1
|
|
165
|
+
if self._read_has_motif(seq, motif, where, search_start, search_end, max_mismatch):
|
|
166
|
+
matched += 1
|
|
167
|
+
if tested == 0:
|
|
168
|
+
return None
|
|
169
|
+
return matched / tested
|
|
170
|
+
|
|
171
|
+
@staticmethod
|
|
172
|
+
def _read_has_motif(
|
|
173
|
+
seq: str,
|
|
174
|
+
motif: str,
|
|
175
|
+
where: str,
|
|
176
|
+
search_start: int | None,
|
|
177
|
+
search_end: int | None,
|
|
178
|
+
max_mismatch: int,
|
|
179
|
+
) -> bool:
|
|
180
|
+
m = len(motif)
|
|
181
|
+
if where == "read_start":
|
|
182
|
+
starts = [0]
|
|
183
|
+
elif where == "read_end":
|
|
184
|
+
starts = [len(seq) - m]
|
|
185
|
+
elif where == "window":
|
|
186
|
+
lo = search_start or 0
|
|
187
|
+
hi = search_end if search_end is not None else len(seq) - m
|
|
188
|
+
starts = list(range(lo, hi + 1))
|
|
189
|
+
else: # anywhere
|
|
190
|
+
starts = list(range(0, len(seq) - m + 1))
|
|
191
|
+
for pos in starts:
|
|
192
|
+
if pos < 0 or pos + m > len(seq):
|
|
193
|
+
continue
|
|
194
|
+
if _motif_matches(seq[pos : pos + m], motif, max_mismatch):
|
|
195
|
+
return True
|
|
196
|
+
return False
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _motif_matches(window: str, motif: str, max_mismatch: int) -> bool:
|
|
200
|
+
mism = 0
|
|
201
|
+
for base, code in zip(window, motif, strict=True):
|
|
202
|
+
if base not in _IUPAC.get(code, code):
|
|
203
|
+
mism += 1
|
|
204
|
+
if mism > max_mismatch:
|
|
205
|
+
return False
|
|
206
|
+
return True
|