seqforge 2026.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqforge/__init__.py +16 -0
- seqforge/cli/__init__.py +38 -0
- seqforge/cli/__main__.py +8 -0
- seqforge/cli/_common.py +105 -0
- seqforge/cli/compose.py +119 -0
- seqforge/cli/eval.py +103 -0
- seqforge/cli/harvest.py +417 -0
- seqforge/cli/hook.py +247 -0
- seqforge/cli/io.py +502 -0
- seqforge/cli/kb.py +348 -0
- seqforge/cli/manifest.py +536 -0
- seqforge/cli/probe.py +43 -0
- seqforge/cli/processing.py +192 -0
- seqforge/cli/project.py +52 -0
- seqforge/cli/resolve.py +55 -0
- seqforge/cli/root.py +66 -0
- seqforge/cli/run.py +463 -0
- seqforge/cli/schema.py +41 -0
- seqforge/compose/__init__.py +28 -0
- seqforge/compose/core.py +515 -0
- seqforge/compose/gates.py +113 -0
- seqforge/compose/params.py +447 -0
- seqforge/e2e.py +1926 -0
- seqforge/evals/__init__.py +78 -0
- seqforge/evals/case.py +382 -0
- seqforge/evals/grade.py +300 -0
- seqforge/evals/run.py +420 -0
- seqforge/harvest/__init__.py +121 -0
- seqforge/harvest/extract.py +319 -0
- seqforge/harvest/fields.py +212 -0
- seqforge/harvest/normalize.py +537 -0
- seqforge/harvest/prep.py +41 -0
- seqforge/harvest/providers.py +321 -0
- seqforge/harvest/verify.py +251 -0
- seqforge/hooks/__init__.py +33 -0
- seqforge/hooks/guards.py +214 -0
- seqforge/io/__init__.py +61 -0
- seqforge/io/archive.py +450 -0
- seqforge/io/attributes.py +190 -0
- seqforge/io/biosample/attributes.json +6341 -0
- seqforge/io/efo/labels.json +55 -0
- seqforge/io/efo.py +138 -0
- seqforge/io/onlist.py +661 -0
- seqforge/io/onlists/3M-february-2018.codes.gz +0 -0
- seqforge/io/onlists/737K-arc-v1.codes.gz +0 -0
- seqforge/io/onlists/737K-august-2016.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3.codes.gz +0 -0
- seqforge/io/onlists/index.json +74 -0
- seqforge/io/remote.py +659 -0
- seqforge/io/taxonomy.py +194 -0
- seqforge/kb/__init__.py +62 -0
- seqforge/kb/anchor.py +169 -0
- seqforge/kb/generate.py +147 -0
- seqforge/kb/loader.py +152 -0
- seqforge/kb/roundtrip.py +112 -0
- seqforge/kb/schema.py +422 -0
- seqforge/kb/specs/10x-3p-gex/spec.yaml +62 -0
- seqforge/kb/specs/10x-3p-gex-v2/README.md +41 -0
- seqforge/kb/specs/10x-3p-gex-v2/spec.yaml +83 -0
- seqforge/kb/specs/10x-3p-gex-v3/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3/spec.yaml +118 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/spec.yaml +124 -0
- seqforge/kb/specs/bd-rhapsody-wta/README.md +103 -0
- seqforge/kb/specs/bd-rhapsody-wta/spec.yaml +130 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced/spec.yaml +99 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v1/spec.yaml +93 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v2/spec.yaml +81 -0
- seqforge/kb/specs/bulk-rnaseq-pe/README.md +35 -0
- seqforge/kb/specs/bulk-rnaseq-pe/spec.yaml +97 -0
- seqforge/kb/specs/splitseq/README.md +51 -0
- seqforge/kb/specs/splitseq/spec.yaml +157 -0
- seqforge/manifest/__init__.py +61 -0
- seqforge/manifest/fill.py +531 -0
- seqforge/manifest/hash.py +77 -0
- seqforge/manifest/instruct.py +114 -0
- seqforge/manifest/policy.py +409 -0
- seqforge/manifest/validate.py +274 -0
- seqforge/models/__init__.py +268 -0
- seqforge/models/assertion.py +68 -0
- seqforge/models/base.py +100 -0
- seqforge/models/blocker.py +71 -0
- seqforge/models/conflict.py +47 -0
- seqforge/models/dataset.py +320 -0
- seqforge/models/evidenced.py +54 -0
- seqforge/models/observation.py +157 -0
- seqforge/models/processing.py +231 -0
- seqforge/models/records.py +145 -0
- seqforge/models/resolve.py +216 -0
- seqforge/probe/__init__.py +46 -0
- seqforge/probe/core.py +232 -0
- seqforge/probe/signals.py +250 -0
- seqforge/probe/streaming.py +118 -0
- seqforge/project.py +177 -0
- seqforge/py.typed +0 -0
- seqforge/resolve/__init__.py +98 -0
- seqforge/resolve/assign.py +204 -0
- seqforge/resolve/cache.py +119 -0
- seqforge/resolve/confuse.py +215 -0
- seqforge/resolve/engine.py +646 -0
- seqforge/resolve/escalate.py +668 -0
- seqforge/resolve/evaluators.py +306 -0
- seqforge/resolve/geometry.py +89 -0
- seqforge/resolve/group.py +85 -0
- seqforge/resolve/records.py +550 -0
- seqforge/resolve/scoring.py +373 -0
- seqforge/resolve/window.py +206 -0
- seqforge/workflows/__init__.py +234 -0
- seqforge/workflows/cram.py +117 -0
- seqforge/workflows/h5ad.py +368 -0
- seqforge/workflows/map/star.smk +101 -0
- seqforge/workflows/map/starsolo.smk +360 -0
- seqforge/workflows/qc.py +157 -0
- seqforge/workspace.py +125 -0
- seqforge-2026.7.1.dist-info/METADATA +125 -0
- seqforge-2026.7.1.dist-info/RECORD +124 -0
- seqforge-2026.7.1.dist-info/WHEEL +4 -0
- seqforge-2026.7.1.dist-info/entry_points.txt +2 -0
- seqforge-2026.7.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""Tier A structural signals — computed from a bounded sample, with no KB and no role assignment.
|
|
2
|
+
|
|
3
|
+
Every function here reports *structure* (composition, segmentation, recurrence, header grammar,
|
|
4
|
+
integrity). None of it assigns a role — ``constant`` is "a constant span", never "the TSO". That
|
|
5
|
+
interpretation is the resolver's job.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
from typing import Literal
|
|
13
|
+
|
|
14
|
+
from ..models.observation import (
|
|
15
|
+
ConstantSegment,
|
|
16
|
+
CycleComposition,
|
|
17
|
+
HomopolymerSegment,
|
|
18
|
+
RandomSegment,
|
|
19
|
+
ReadLengthProfile,
|
|
20
|
+
ReadNameGrammar,
|
|
21
|
+
Segment,
|
|
22
|
+
WindowDistinctRatio,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
_BASE_IDX = {"A": 0, "C": 1, "G": 2, "T": 3}
|
|
26
|
+
_PURE_THRESHOLD = 0.9 # a cycle whose dominant base fraction >= this is "constant sequence"
|
|
27
|
+
_HOMOPOLYMER_MIN = (
|
|
28
|
+
4 # a run of >= this many identical dominant bases is a homopolymer, not a linker
|
|
29
|
+
)
|
|
30
|
+
_SRA_HEADER = re.compile(r"^[SED]RR\d+\.\d+")
|
|
31
|
+
_ILLUMINA_INDEX = re.compile(r"[ /]([ACGTN]{6,})(?:\+([ACGTN]{6,}))?\s*$")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def per_cycle_composition(seqs: list[str]) -> list[CycleComposition]:
|
|
35
|
+
"""Fraction of A/C/G/T/N at each 0-based cycle, over reads long enough to reach that cycle."""
|
|
36
|
+
if not seqs:
|
|
37
|
+
return []
|
|
38
|
+
max_len = max(len(s) for s in seqs)
|
|
39
|
+
counts = [[0, 0, 0, 0, 0] for _ in range(max_len)]
|
|
40
|
+
denom = [0] * max_len
|
|
41
|
+
for s in seqs:
|
|
42
|
+
for i, ch in enumerate(s):
|
|
43
|
+
counts[i][_BASE_IDX.get(ch, 4)] += 1 # non-ACGT (incl. N) -> the N bucket
|
|
44
|
+
denom[i] += 1
|
|
45
|
+
out: list[CycleComposition] = []
|
|
46
|
+
for i in range(max_len):
|
|
47
|
+
d = denom[i] or 1
|
|
48
|
+
c = counts[i]
|
|
49
|
+
out.append(
|
|
50
|
+
CycleComposition(cycle=i, a=c[0] / d, c=c[1] / d, g=c[2] / d, t=c[3] / d, n=c[4] / d)
|
|
51
|
+
)
|
|
52
|
+
return out
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _dominant(comp: CycleComposition) -> tuple[str, float]:
|
|
56
|
+
"""Return the dominant ACGT base and its fraction for one cycle."""
|
|
57
|
+
pairs = (("A", comp.a), ("C", comp.c), ("G", comp.g), ("T", comp.t))
|
|
58
|
+
base, frac = max(pairs, key=lambda p: p[1])
|
|
59
|
+
return base, frac
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _entropy_bits(comp: CycleComposition) -> float:
|
|
63
|
+
"""Shannon entropy (bits) of the ACGT distribution at one cycle; ~2.0 for uniform."""
|
|
64
|
+
total = comp.a + comp.c + comp.g + comp.t
|
|
65
|
+
if total <= 0:
|
|
66
|
+
return 0.0
|
|
67
|
+
bits = 0.0
|
|
68
|
+
for p in (comp.a, comp.c, comp.g, comp.t):
|
|
69
|
+
q = p / total
|
|
70
|
+
if q > 0:
|
|
71
|
+
bits -= q * math.log2(q)
|
|
72
|
+
return bits
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def segment(comps: list[CycleComposition]) -> list[Segment]:
|
|
76
|
+
"""Merge cycles into constant / homopolymer / random segments (structural, role-free).
|
|
77
|
+
|
|
78
|
+
A cycle whose dominant base fraction >= ``_PURE_THRESHOLD`` is "constant sequence". Within a run
|
|
79
|
+
of pure cycles, a run of the same dominant base (>= ``_HOMOPOLYMER_MIN``) is a homopolymer (polyT
|
|
80
|
+
capture / polyA tail); a stretch of varying pure bases is a linker/adapter/TSO (constant).
|
|
81
|
+
Everything else is random (CB/UMI/cDNA candidate). Index == cycle by construction.
|
|
82
|
+
"""
|
|
83
|
+
if not comps:
|
|
84
|
+
return []
|
|
85
|
+
labels: list[tuple[str, str, float]] = [] # (kind, dominant_base, purity) per cycle
|
|
86
|
+
for comp in comps:
|
|
87
|
+
base, frac = _dominant(comp)
|
|
88
|
+
labels.append(("pure", base, frac) if frac >= _PURE_THRESHOLD else ("random", base, frac))
|
|
89
|
+
|
|
90
|
+
segments: list[Segment] = []
|
|
91
|
+
i = 0
|
|
92
|
+
n = len(labels)
|
|
93
|
+
while i < n:
|
|
94
|
+
kind = labels[i][0]
|
|
95
|
+
j = i + 1
|
|
96
|
+
while j < n and labels[j][0] == kind:
|
|
97
|
+
j += 1
|
|
98
|
+
if kind == "random":
|
|
99
|
+
mean_bits = sum(_entropy_bits(comps[k]) for k in range(i, j)) / (j - i)
|
|
100
|
+
segments.append(RandomSegment(start=i, end=j, mean_entropy_bits=mean_bits))
|
|
101
|
+
else:
|
|
102
|
+
segments.extend(_split_pure_run(labels, i, j))
|
|
103
|
+
i = j
|
|
104
|
+
return segments
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _split_pure_run(labels: list[tuple[str, str, float]], lo: int, hi: int) -> list[Segment]:
|
|
108
|
+
"""Split a run of pure cycles ``[lo, hi)`` into homopolymer + constant (linker) segments."""
|
|
109
|
+
out: list[Segment] = []
|
|
110
|
+
const_start: int | None = None
|
|
111
|
+
|
|
112
|
+
def flush_constant(end: int) -> None:
|
|
113
|
+
nonlocal const_start
|
|
114
|
+
if const_start is None:
|
|
115
|
+
return
|
|
116
|
+
bases = [labels[k][1] for k in range(const_start, end)]
|
|
117
|
+
purity = sum(labels[k][2] for k in range(const_start, end)) / (end - const_start)
|
|
118
|
+
out.append(
|
|
119
|
+
ConstantSegment(start=const_start, end=end, consensus="".join(bases), purity=purity)
|
|
120
|
+
)
|
|
121
|
+
const_start = None
|
|
122
|
+
|
|
123
|
+
k = lo
|
|
124
|
+
while k < hi:
|
|
125
|
+
base = labels[k][1]
|
|
126
|
+
r = k + 1
|
|
127
|
+
while r < hi and labels[r][1] == base:
|
|
128
|
+
r += 1
|
|
129
|
+
if r - k >= _HOMOPOLYMER_MIN:
|
|
130
|
+
flush_constant(k)
|
|
131
|
+
out.append(
|
|
132
|
+
HomopolymerSegment(
|
|
133
|
+
base=base, # type: ignore[arg-type] # single ACGT char
|
|
134
|
+
start=k,
|
|
135
|
+
end=r,
|
|
136
|
+
mean_run=float(r - k),
|
|
137
|
+
)
|
|
138
|
+
)
|
|
139
|
+
elif const_start is None:
|
|
140
|
+
const_start = k
|
|
141
|
+
k = r
|
|
142
|
+
flush_constant(hi)
|
|
143
|
+
return out
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def read_length_profile(seqs: list[str]) -> ReadLengthProfile:
|
|
147
|
+
"""Mode, distinct-count, min/max, and (only when variable) percentiles of read length."""
|
|
148
|
+
lengths = sorted(len(s) for s in seqs)
|
|
149
|
+
if not lengths:
|
|
150
|
+
return ReadLengthProfile(mode=0, n_distinct=1, min_len=0, max_len=0)
|
|
151
|
+
freq: dict[int, int] = {}
|
|
152
|
+
for length in lengths:
|
|
153
|
+
freq[length] = freq.get(length, 0) + 1
|
|
154
|
+
mode = max(freq, key=lambda k: freq[k])
|
|
155
|
+
n_distinct = len(freq)
|
|
156
|
+
percentiles = None
|
|
157
|
+
if n_distinct > 1:
|
|
158
|
+
percentiles = {
|
|
159
|
+
"p1": lengths[max(0, (len(lengths) * 1) // 100 - 1)],
|
|
160
|
+
"p50": lengths[len(lengths) // 2],
|
|
161
|
+
"p99": lengths[min(len(lengths) - 1, (len(lengths) * 99) // 100)],
|
|
162
|
+
}
|
|
163
|
+
return ReadLengthProfile(
|
|
164
|
+
mode=mode,
|
|
165
|
+
n_distinct=n_distinct,
|
|
166
|
+
min_len=lengths[0],
|
|
167
|
+
max_len=lengths[-1],
|
|
168
|
+
percentiles=percentiles,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def window_distinct_ratio(seqs: list[str], start: int, end: int) -> float | None:
|
|
173
|
+
"""distinct/total over an explicit ``[start, end)`` window (role-conditioned; used by resolve)."""
|
|
174
|
+
window = [s[start:end] for s in seqs if len(s) >= end]
|
|
175
|
+
if not window:
|
|
176
|
+
return None
|
|
177
|
+
return len(set(window)) / len(window)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def distinct_ratios(seqs: list[str], segments: list[Segment]) -> list[WindowDistinctRatio]:
|
|
181
|
+
"""distinct/total over each random segment (candidate CB/UMI/cDNA window). Supports-only signal."""
|
|
182
|
+
out: list[WindowDistinctRatio] = []
|
|
183
|
+
for seg in segments:
|
|
184
|
+
if not isinstance(seg, RandomSegment):
|
|
185
|
+
continue
|
|
186
|
+
window = [s[seg.start : seg.end] for s in seqs if len(s) >= seg.end]
|
|
187
|
+
if not window:
|
|
188
|
+
continue
|
|
189
|
+
ratio = len(set(window)) / len(window)
|
|
190
|
+
out.append(
|
|
191
|
+
WindowDistinctRatio(
|
|
192
|
+
start=seg.start, end=seg.end, distinct_ratio=ratio, n_sampled=len(window)
|
|
193
|
+
)
|
|
194
|
+
)
|
|
195
|
+
return out
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def parse_read_name(name: str | None) -> ReadNameGrammar:
|
|
199
|
+
"""Parse an Illumina header; detect SRA-normalized headers (the index has been stripped)."""
|
|
200
|
+
if not name:
|
|
201
|
+
return ReadNameGrammar(parsed=False)
|
|
202
|
+
if _SRA_HEADER.match(name):
|
|
203
|
+
return ReadNameGrammar(parsed=False, sra_normalized=True)
|
|
204
|
+
fields = name.split(":")
|
|
205
|
+
if len(fields) >= 7:
|
|
206
|
+
index_match = _ILLUMINA_INDEX.search(name)
|
|
207
|
+
index = index_match.group(1) if index_match else None
|
|
208
|
+
lane: int | None
|
|
209
|
+
tile: int | None
|
|
210
|
+
try:
|
|
211
|
+
lane = int(fields[3])
|
|
212
|
+
tile = int(fields[4])
|
|
213
|
+
except ValueError:
|
|
214
|
+
lane = None
|
|
215
|
+
tile = None
|
|
216
|
+
return ReadNameGrammar(
|
|
217
|
+
parsed=True,
|
|
218
|
+
instrument=fields[0],
|
|
219
|
+
run=fields[1],
|
|
220
|
+
flowcell=fields[2],
|
|
221
|
+
lane=lane,
|
|
222
|
+
tile=tile,
|
|
223
|
+
index=index,
|
|
224
|
+
)
|
|
225
|
+
return ReadNameGrammar(parsed=False)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def quality_encoding(
|
|
229
|
+
min_ord: int | None, max_ord: int | None
|
|
230
|
+
) -> Literal["phred33", "phred64", "unknown"]:
|
|
231
|
+
"""Infer the Phred offset from the observed quality-char ordinal range."""
|
|
232
|
+
if min_ord is None or max_ord is None:
|
|
233
|
+
return "unknown"
|
|
234
|
+
if min_ord < 64:
|
|
235
|
+
return "phred33"
|
|
236
|
+
if max_ord > 74:
|
|
237
|
+
return "phred64"
|
|
238
|
+
return "unknown"
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def n_rate(seqs: list[str]) -> float:
|
|
242
|
+
"""Fraction of non-ACGT (N) bases across the sample."""
|
|
243
|
+
total = 0
|
|
244
|
+
n_count = 0
|
|
245
|
+
for s in seqs:
|
|
246
|
+
total += len(s)
|
|
247
|
+
for ch in s:
|
|
248
|
+
if ch not in _BASE_IDX:
|
|
249
|
+
n_count += 1
|
|
250
|
+
return (n_count / total) if total else 0.0
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Bounded FASTQ streaming — the bounded-read invariant made mechanical.
|
|
2
|
+
|
|
3
|
+
Decompress a gzip FASTQ incrementally and stop at whichever budget trips first: ``max_reads`` records
|
|
4
|
+
or ``max_bytes`` *decompressed* bytes. There is no random-access seek plan and no whole-file
|
|
5
|
+
decompression; a code path that can touch a whole multi-GB FASTQ is a bug.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import gzip
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import IO
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class StreamSample:
|
|
18
|
+
"""A bounded sample of a FASTQ, plus the byte/record accounting that proves it stayed in budget."""
|
|
19
|
+
|
|
20
|
+
seqs: list[str] = field(default_factory=list)
|
|
21
|
+
first_name: str | None = None
|
|
22
|
+
qual_min_ord: int | None = None
|
|
23
|
+
qual_max_ord: int | None = None
|
|
24
|
+
n_reads: int = 0
|
|
25
|
+
decompressed_bytes: int = 0
|
|
26
|
+
compressed_bytes: int = 0
|
|
27
|
+
truncated: bool = False
|
|
28
|
+
ok: bool = True
|
|
29
|
+
budget_exhausted: bool = False
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def sample_fastq_stream(fileobj: IO[bytes], max_reads: int, max_bytes: int) -> StreamSample:
|
|
33
|
+
"""Read at most ``max_reads`` records / ``max_bytes`` decompressed bytes from a gzip FASTQ stream.
|
|
34
|
+
|
|
35
|
+
Source-agnostic: ``fileobj`` is any binary reader positioned at the gzip magic — a local file
|
|
36
|
+
(:func:`sample_fastq_gz`) or an in-memory range-read *prefix* (``io.remote.probe_remote``). A prefix
|
|
37
|
+
that ends mid-member is the normal remote case, and it is handled by the same path as a truncated
|
|
38
|
+
upload: the cut is caught and the trailing partial record dropped. ``compressed_bytes`` is the
|
|
39
|
+
reader's final position; the caller owns opening and closing ``fileobj``.
|
|
40
|
+
|
|
41
|
+
Parameters
|
|
42
|
+
----------
|
|
43
|
+
fileobj
|
|
44
|
+
A binary stream of gzip-compressed FASTQ bytes (a whole file, or a bounded head prefix).
|
|
45
|
+
max_reads
|
|
46
|
+
Hard cap on records read.
|
|
47
|
+
max_bytes
|
|
48
|
+
Hard cap on *decompressed* bytes read. Whichever cap trips first stops the stream.
|
|
49
|
+
|
|
50
|
+
Returns
|
|
51
|
+
-------
|
|
52
|
+
StreamSample
|
|
53
|
+
The sampled sequences and the byte/record accounting. ``truncated`` is set if the gzip
|
|
54
|
+
stream ends mid-member before either budget or a clean EOF; ``ok`` is False on a
|
|
55
|
+
gzip/format error.
|
|
56
|
+
"""
|
|
57
|
+
sample = StreamSample()
|
|
58
|
+
try:
|
|
59
|
+
with gzip.GzipFile(fileobj=fileobj) as gz:
|
|
60
|
+
line_iter = iter(gz)
|
|
61
|
+
while sample.n_reads < max_reads and sample.decompressed_bytes < max_bytes:
|
|
62
|
+
try:
|
|
63
|
+
header = next(line_iter, None)
|
|
64
|
+
if header is None: # clean EOF, fewer reads than the budget
|
|
65
|
+
break
|
|
66
|
+
seq = next(line_iter, None)
|
|
67
|
+
plus = next(line_iter, None)
|
|
68
|
+
qual = next(line_iter, None)
|
|
69
|
+
except (EOFError, gzip.BadGzipFile, OSError):
|
|
70
|
+
# gzip stream cut mid-member (truncated upload, or a bounded range-read prefix).
|
|
71
|
+
sample.truncated = True
|
|
72
|
+
break
|
|
73
|
+
if seq is None or plus is None or qual is None:
|
|
74
|
+
# a partial final record => the stream was cut mid-record.
|
|
75
|
+
sample.truncated = True
|
|
76
|
+
break
|
|
77
|
+
|
|
78
|
+
name = header.decode("ascii", "replace").rstrip("\n")
|
|
79
|
+
seq_s = seq.decode("ascii", "replace").rstrip("\n")
|
|
80
|
+
qual_s = qual.decode("ascii", "replace").rstrip("\n")
|
|
81
|
+
|
|
82
|
+
if sample.first_name is None:
|
|
83
|
+
sample.first_name = name.lstrip("@")
|
|
84
|
+
sample.seqs.append(seq_s)
|
|
85
|
+
_update_qual_ords(sample, qual_s)
|
|
86
|
+
sample.n_reads += 1
|
|
87
|
+
sample.decompressed_bytes += len(header) + len(seq) + len(plus) + len(qual)
|
|
88
|
+
except (gzip.BadGzipFile, OSError):
|
|
89
|
+
sample.ok = False
|
|
90
|
+
finally:
|
|
91
|
+
sample.compressed_bytes = fileobj.tell()
|
|
92
|
+
|
|
93
|
+
sample.budget_exhausted = sample.n_reads >= max_reads or sample.decompressed_bytes >= max_bytes
|
|
94
|
+
return sample
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def sample_fastq_gz(path: str | Path, max_reads: int, max_bytes: int) -> StreamSample:
|
|
98
|
+
"""Read a bounded head of a LOCAL gzip FASTQ. Thin wrapper over :func:`sample_fastq_stream`.
|
|
99
|
+
|
|
100
|
+
Opens the file, hands the reader to the source-agnostic sampler, and closes it. ``gzip.GzipFile``
|
|
101
|
+
does not close a ``fileobj`` it was handed, so the ``tell()`` inside the sampler runs before this
|
|
102
|
+
``close()``.
|
|
103
|
+
"""
|
|
104
|
+
raw = open(path, "rb") # noqa: SIM115 - closed explicitly in finally
|
|
105
|
+
try:
|
|
106
|
+
return sample_fastq_stream(raw, max_reads, max_bytes)
|
|
107
|
+
finally:
|
|
108
|
+
raw.close()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _update_qual_ords(sample: StreamSample, qual: str) -> None:
|
|
112
|
+
"""Track the min/max quality-char ordinal (used to infer the Phred offset)."""
|
|
113
|
+
if not qual:
|
|
114
|
+
return
|
|
115
|
+
ords = [ord(c) for c in qual]
|
|
116
|
+
lo, hi = min(ords), max(ords)
|
|
117
|
+
sample.qual_min_ord = lo if sample.qual_min_ord is None else min(sample.qual_min_ord, lo)
|
|
118
|
+
sample.qual_max_ord = hi if sample.qual_max_ord is None else max(sample.qual_max_ord, hi)
|
seqforge/project.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Project-level views over a multi-assay compile: the flat ``sample_metadata.tsv`` and ``project.yaml``.
|
|
2
|
+
|
|
3
|
+
A large project splits into **assays**, one :class:`~seqforge.models.dataset.DatasetManifest` each.
|
|
4
|
+
Those manifests are the source of truth; this module unions them back into the "one study" views a
|
|
5
|
+
human reads — a flat table with one row per sample across every assay, and a small index mapping each
|
|
6
|
+
assay to its subdir / chemistry / sample count. Both are **derived** from the manifests (so they cannot
|
|
7
|
+
drift from them) and **deterministic** (sorted rows and columns), so regenerating is a no-op.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
import yaml
|
|
16
|
+
|
|
17
|
+
from .models.dataset import DatasetManifest
|
|
18
|
+
from .workspace import state_dir
|
|
19
|
+
|
|
20
|
+
#: Common BioSample attributes, ordered for readability. Any *other* resolved attribute follows,
|
|
21
|
+
#: sorted — the column set is the union of what actually resolved, never a hand-fixed schema.
|
|
22
|
+
_PREFERRED_ATTRS: tuple[str, ...] = (
|
|
23
|
+
"strain",
|
|
24
|
+
"genotype",
|
|
25
|
+
"age",
|
|
26
|
+
"dev_stage",
|
|
27
|
+
"sex",
|
|
28
|
+
"tissue",
|
|
29
|
+
"cell_type",
|
|
30
|
+
"cell_line",
|
|
31
|
+
"treatment",
|
|
32
|
+
"source_name",
|
|
33
|
+
"disease",
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
_LEADING: tuple[str, ...] = ("sample_id", "accession", "assay", "organism")
|
|
37
|
+
_TRAILING: tuple[str, ...] = ("n_files", "files")
|
|
38
|
+
|
|
39
|
+
#: The project-level file names, written at the top of ``seqforge/`` (never inside an assay subdir).
|
|
40
|
+
SAMPLE_METADATA_TSV = "sample_metadata.tsv"
|
|
41
|
+
PROJECT_YAML = "project.yaml"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def sample_metadata_table(
|
|
45
|
+
manifests: list[DatasetManifest],
|
|
46
|
+
) -> tuple[list[str], list[dict[str, str]]]:
|
|
47
|
+
"""One row per sample across every manifest. Returns ``(columns, rows)``, both deterministic.
|
|
48
|
+
|
|
49
|
+
Columns: ``sample_id, accession, assay, organism``, then every resolved BioSample attribute
|
|
50
|
+
(common ones first, the rest sorted), then ``n_files, files``. An empty cell means the attribute
|
|
51
|
+
did not resolve for that sample — absence, honestly, rather than a guessed value.
|
|
52
|
+
"""
|
|
53
|
+
rows: list[dict[str, str]] = []
|
|
54
|
+
seen_attrs: set[str] = set()
|
|
55
|
+
for manifest in manifests:
|
|
56
|
+
chemistry = manifest.library.chemistry.value[0]
|
|
57
|
+
organism = str(manifest.experiment.organism.value)
|
|
58
|
+
for sample in manifest.experiment.samples:
|
|
59
|
+
attrs = {key: ev.value for key, ev in sample.attributes.items()}
|
|
60
|
+
seen_attrs.update(attrs)
|
|
61
|
+
rows.append(
|
|
62
|
+
{
|
|
63
|
+
"sample_id": sample.sample_id,
|
|
64
|
+
"accession": sample.accession or "",
|
|
65
|
+
"assay": chemistry,
|
|
66
|
+
"organism": organism,
|
|
67
|
+
"n_files": str(len(sample.file_uris)),
|
|
68
|
+
"files": ";".join(sorted(Path(uri).name for uri in sample.file_uris)),
|
|
69
|
+
**attrs,
|
|
70
|
+
}
|
|
71
|
+
)
|
|
72
|
+
attr_cols = [a for a in _PREFERRED_ATTRS if a in seen_attrs]
|
|
73
|
+
attr_cols += sorted(seen_attrs - set(_PREFERRED_ATTRS))
|
|
74
|
+
columns = [*_LEADING, *attr_cols, *_TRAILING]
|
|
75
|
+
rows.sort(key=lambda r: (r["assay"], r["sample_id"]))
|
|
76
|
+
return columns, rows
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def format_tsv(columns: list[str], rows: list[dict[str, str]]) -> str:
|
|
80
|
+
"""Render ``(columns, rows)`` as a TSV string (trailing newline). Tabs/newlines in a value are
|
|
81
|
+
squashed to spaces so a stray attribute value can never break the table's shape."""
|
|
82
|
+
|
|
83
|
+
def cell(value: str) -> str:
|
|
84
|
+
return " ".join(str(value).split())
|
|
85
|
+
|
|
86
|
+
lines = ["\t".join(columns)]
|
|
87
|
+
lines += ["\t".join(cell(r.get(c, "")) for c in columns) for r in rows]
|
|
88
|
+
return "\n".join(lines) + "\n"
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def project_index(assays: list[dict[str, Any]]) -> dict[str, Any]:
|
|
92
|
+
"""The ``project.yaml`` payload: the assays (sorted by chemistry) plus project-wide counts.
|
|
93
|
+
|
|
94
|
+
Each assay entry names its chemistry, its subdir (``None`` for a single-assay flat project), its
|
|
95
|
+
sample count, and its manifest / pipeline paths — enough to discover every per-assay output.
|
|
96
|
+
"""
|
|
97
|
+
entries = sorted(assays, key=lambda a: str(a.get("chemistry", "")))
|
|
98
|
+
return {
|
|
99
|
+
"n_assays": len(entries),
|
|
100
|
+
"n_samples": sum(int(a.get("n_samples", 0)) for a in entries),
|
|
101
|
+
"assays": entries,
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def discover_assays(workspace: str | Path) -> list[tuple[str | None, Path]]:
|
|
106
|
+
"""``(subdir, manifest_path)`` for each assay under ``seqforge/``.
|
|
107
|
+
|
|
108
|
+
A top-level ``manifest.yaml`` is a single, flat assay (subdir ``None``); otherwise every
|
|
109
|
+
``seqforge/<assay>/manifest.yaml`` is one assay. Sorted, so the result is deterministic.
|
|
110
|
+
"""
|
|
111
|
+
root = state_dir(workspace)
|
|
112
|
+
top = root / "manifest.yaml"
|
|
113
|
+
if top.is_file():
|
|
114
|
+
return [(None, top)]
|
|
115
|
+
if not root.is_dir():
|
|
116
|
+
return []
|
|
117
|
+
found = [
|
|
118
|
+
(child.name, child / "manifest.yaml")
|
|
119
|
+
for child in root.iterdir()
|
|
120
|
+
if child.is_dir() and (child / "manifest.yaml").is_file()
|
|
121
|
+
]
|
|
122
|
+
return sorted(found, key=lambda pair: pair[0] or "")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _relative_to(path: Any, workspace: str | Path) -> Any:
|
|
126
|
+
"""A path made relative to ``workspace`` when it is under it, else left untouched (None passes).
|
|
127
|
+
|
|
128
|
+
Keeps ``project.yaml`` machine-independent: a workspace's absolute location must not leak into an
|
|
129
|
+
index that is content-addressed reproducible and re-generatable anywhere the outputs are copied.
|
|
130
|
+
"""
|
|
131
|
+
if not path:
|
|
132
|
+
return path
|
|
133
|
+
try:
|
|
134
|
+
return str(Path(path).resolve().relative_to(Path(workspace).resolve()))
|
|
135
|
+
except ValueError:
|
|
136
|
+
return path
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def write_project_views(workspace: str | Path, assays: list[dict[str, Any]]) -> tuple[Path, Path]:
|
|
140
|
+
"""Write ``sample_metadata.tsv`` + ``project.yaml`` at the project top and return their paths.
|
|
141
|
+
|
|
142
|
+
``assays`` is one dict per assay carrying at least ``chemistry``, ``subdir``, ``n_samples`` and a
|
|
143
|
+
``manifest`` path (optionally ``pipeline``/``snakefile``); the manifests are loaded to build the
|
|
144
|
+
flat table. Written even for a single-assay project — the "one study" view is always useful. Paths
|
|
145
|
+
in ``project.yaml`` are stored relative to ``workspace`` so the index is portable.
|
|
146
|
+
"""
|
|
147
|
+
root = state_dir(workspace)
|
|
148
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
149
|
+
manifests = [
|
|
150
|
+
DatasetManifest.model_validate(yaml.safe_load(Path(a["manifest"]).read_text()))
|
|
151
|
+
for a in assays
|
|
152
|
+
]
|
|
153
|
+
columns, rows = sample_metadata_table(manifests)
|
|
154
|
+
tsv_path = root / SAMPLE_METADATA_TSV
|
|
155
|
+
tsv_path.write_text(format_tsv(columns, rows))
|
|
156
|
+
|
|
157
|
+
portable = [
|
|
158
|
+
{
|
|
159
|
+
k: (_relative_to(v, workspace) if k in ("manifest", "snakefile", "pipeline") else v)
|
|
160
|
+
for k, v in a.items()
|
|
161
|
+
}
|
|
162
|
+
for a in assays
|
|
163
|
+
]
|
|
164
|
+
yaml_path = root / PROJECT_YAML
|
|
165
|
+
yaml_path.write_text(yaml.safe_dump(project_index(portable), sort_keys=True))
|
|
166
|
+
return tsv_path, yaml_path
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
__all__ = [
|
|
170
|
+
"SAMPLE_METADATA_TSV",
|
|
171
|
+
"PROJECT_YAML",
|
|
172
|
+
"sample_metadata_table",
|
|
173
|
+
"format_tsv",
|
|
174
|
+
"project_index",
|
|
175
|
+
"discover_assays",
|
|
176
|
+
"write_project_views",
|
|
177
|
+
]
|
seqforge/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""``resolve`` — the scoring engine: bytes + KB (+ optional hypothesis) -> a ranked, escalated verdict.
|
|
2
|
+
|
|
3
|
+
Deterministic and LLM-free. Signature-test evaluators score a JSON-safe evidence matrix
|
|
4
|
+
``M[role][file]``; a cardinality-normalized joint role-assignment picks the best injective
|
|
5
|
+
files->roles map per technology; escalation turns the ranked candidates into exactly one of
|
|
6
|
+
``Decision`` / ``Conflict`` / ``Question`` / ``Blocker`` with rung provenance. Every artifact is
|
|
7
|
+
content-addressed under ``.seqforge/``. The only interpretive input is a span-verified
|
|
8
|
+
``hypothesis`` that steers control flow — it never enters the matrix (§3.4).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
#: CalVer YYYY.M.PATCH; bumped when scoring/assignment/escalation semantics change. Folded into the
|
|
14
|
+
#: dataset cache key so a resolver change invalidates stale candidates.
|
|
15
|
+
#: 2026.7.1 — `resolve_runs`: files are grouped into runs and each run is assigned on its own
|
|
16
|
+
#: bytes. A dataset resolved as one library dropped every file but one pair per role.
|
|
17
|
+
#: 2026.7.2 — over-length onlist admission: a barcode read over-sequenced into the length dead zone
|
|
18
|
+
#: (canonical < mode < over_length_min) is admitted when its barcode prefix hits the whitelist, so a
|
|
19
|
+
#: previously-forbidden over-sequenced read now resolves to its chemistry (#7).
|
|
20
|
+
#: 2026.7.3 — over-length admission uses a FLOOR-ANCHORED bar, not the support `min`: admission asks
|
|
21
|
+
#: "barcode vs cDNA" (chance ≈ whitelist floor), not "confident barcode" (0.6). A real over-sequenced
|
|
22
|
+
#: barcode read with ordinary sequencing error hit below 0.6 on exact match and fell to bulk (#7,
|
|
23
|
+
#: GSE126954 SRX5411291); the floor-anchored bar admits it while still rejecting a same-length cDNA.
|
|
24
|
+
#: 2026.7.4 — multi-lane surplus absorption: a run sequenced across N lanes holds N files per role, but
|
|
25
|
+
#: the injective assignment fills each role once; the surplus same-length lane files are now absorbed
|
|
26
|
+
#: into their role (was NO_VALID_ROLE_ASSIGNMENT), so a multi-lane 10x dataset resolves (GSE208154).
|
|
27
|
+
#: 2026.7.5 — surplus absorption matches by READ DESIGNATION (R1/R2/…) + length, not de-laned filename.
|
|
28
|
+
#: One accession sequenced across several flowcells carries a different flowcell id per file, so the
|
|
29
|
+
#: lanes of one read de-laned to different names and the cross-flowcell surplus stayed unassigned;
|
|
30
|
+
#: matching on the designation the sequencer wrote fuses them (GSE208154 is 2 flowcells x 8 lanes x
|
|
31
|
+
#: {R1,R2,I1} per run, which 2026.7.4's de-lane equality could not absorb across the flowcell boundary).
|
|
32
|
+
#: 2026.7.6 — role assignment optimizes (coverage, score) lexicographically, not score alone: a file
|
|
33
|
+
#: eligible for exactly one role claims it before a multi-role file can. GSE208154's real cDNA reads
|
|
34
|
+
#: have low-diversity 5′ ends, so a 28 bp barcode read out-scored the 91 bp cDNA read for the cDNA role;
|
|
35
|
+
#: score-max then took a barcode file for cDNA and orphaned every cDNA-length file (absorption could not
|
|
36
|
+
#: recover — the cDNA rep was itself a barcode read). The 91 bp reads are forbidden for the barcode role
|
|
37
|
+
#: (dead zone), so cDNA is their only home; coverage now seats them there. No-op for one-file-per-role
|
|
38
|
+
#: runs (injectivity already forces the map), so the other 12 worm datasets are unaffected.
|
|
39
|
+
#: 2026.7.7 — hierarchical descent: resolve_dataset scores a length-FEASIBLE pool (drawn from runnable
|
|
40
|
+
#: specs via the scorer's own read-length gate) instead of a flat loop over the whole KB; escalate still
|
|
41
|
+
#: receives the full KB. Provably winner-invariant — a length-infeasible spec would have scored
|
|
42
|
+
#: forbidden — so the winner equals a flat full scan; this only narrows which specs are scored as the KB
|
|
43
|
+
#: grows, and reads sibling confusability off the tree instead of hand-declared cliques.
|
|
44
|
+
#: 2026.7.8 — family-level chemistry authority: a WITHIN-family asserted-vs-observed geometry difference
|
|
45
|
+
#: (asserted v2, observed v3 — both 10x-3p-gex leaves) is no longer a blocking conflict. A paper names
|
|
46
|
+
#: the assay family reliably and the leaf vaguely; the bytes decide the leaf, so the disagreement is
|
|
47
|
+
#: recorded as a `resolved` conflict (auditable, non-blocking) instead of exit 4. A CROSS-family
|
|
48
|
+
#: difference still blocks. Auto-resolves GSE229022 ("10x 3' v2/v3" in prose, byte-provably v3).
|
|
49
|
+
#: 2026.7.9 — cross-family honesty made symmetric: a BULK chemistry asserted but a barcoded single-cell
|
|
50
|
+
#: library observed now surfaces a conflict (exit 4), the mirror of single-cell-asserted-but-bulk-observed
|
|
51
|
+
#: (which already did). Both directions of a wrong data-vs-paper pairing are now caught, not just one.
|
|
52
|
+
#: 2026.7.10 — BD Rhapsody Enhanced beads: anchored barcode elements are located by their adjacent
|
|
53
|
+
#: anchor sequence before the onlist window is read, so an Enhanced (diversity-spacer) library scores its
|
|
54
|
+
#: CLS whitelists at the right offset instead of missing them at a fixed one (#53).
|
|
55
|
+
#: 2026.7.11 — barcode-role seating + barcode-absent refusal (F1): the barcode role may only seat on a
|
|
56
|
+
#: read that clears the onlist bar when some read does (no equal-length swap onto a cDNA mate), and a
|
|
57
|
+
#: barcoded winner with NO whitelist-hitting read now refuses (BARCODE_READ_ABSENT) instead of composing
|
|
58
|
+
#: a pipeline STARsolo would run to an empty matrix.
|
|
59
|
+
#: 2026.7.12 — barcode-absent refusal keys on ALL valid candidates, not just the top: an over-length
|
|
60
|
+
#: v2/v3 tie where v2 edges v3 on score (top=v2, 737K misses) while v3's 3M list hits must NOT refuse —
|
|
61
|
+
#: the data is barcoded and resolves to v3. Only a dataset where no barcoded leaf hits is barcode-absent
|
|
62
|
+
#: (PRJNA658829 SRR12575567 was false-blocked before this).
|
|
63
|
+
RESOLVE_VERSION = "2026.7.12"
|
|
64
|
+
|
|
65
|
+
from .cache import Cache, dataset_id # noqa: E402
|
|
66
|
+
from .engine import ( # noqa: E402
|
|
67
|
+
Hypothesis,
|
|
68
|
+
MultiRunOutput,
|
|
69
|
+
ResolveOutput,
|
|
70
|
+
RunResolution,
|
|
71
|
+
exit_code_for,
|
|
72
|
+
resolve_dataset,
|
|
73
|
+
resolve_runs,
|
|
74
|
+
role_of_sha_for,
|
|
75
|
+
)
|
|
76
|
+
from .group import group_runs, run_key # noqa: E402
|
|
77
|
+
from .scoring import Cell, TechEvaluation, build_tech_evaluation # noqa: E402
|
|
78
|
+
from .window import WindowProbe # noqa: E402
|
|
79
|
+
|
|
80
|
+
__all__ = [
|
|
81
|
+
"RESOLVE_VERSION",
|
|
82
|
+
"resolve_dataset",
|
|
83
|
+
"resolve_runs",
|
|
84
|
+
"ResolveOutput",
|
|
85
|
+
"MultiRunOutput",
|
|
86
|
+
"RunResolution",
|
|
87
|
+
"role_of_sha_for",
|
|
88
|
+
"group_runs",
|
|
89
|
+
"run_key",
|
|
90
|
+
"Hypothesis",
|
|
91
|
+
"exit_code_for",
|
|
92
|
+
"build_tech_evaluation",
|
|
93
|
+
"TechEvaluation",
|
|
94
|
+
"Cell",
|
|
95
|
+
"WindowProbe",
|
|
96
|
+
"Cache",
|
|
97
|
+
"dataset_id",
|
|
98
|
+
]
|