seqforge 2026.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqforge/__init__.py +16 -0
- seqforge/cli/__init__.py +38 -0
- seqforge/cli/__main__.py +8 -0
- seqforge/cli/_common.py +105 -0
- seqforge/cli/compose.py +119 -0
- seqforge/cli/eval.py +103 -0
- seqforge/cli/harvest.py +417 -0
- seqforge/cli/hook.py +247 -0
- seqforge/cli/io.py +502 -0
- seqforge/cli/kb.py +348 -0
- seqforge/cli/manifest.py +536 -0
- seqforge/cli/probe.py +43 -0
- seqforge/cli/processing.py +192 -0
- seqforge/cli/project.py +52 -0
- seqforge/cli/resolve.py +55 -0
- seqforge/cli/root.py +66 -0
- seqforge/cli/run.py +463 -0
- seqforge/cli/schema.py +41 -0
- seqforge/compose/__init__.py +28 -0
- seqforge/compose/core.py +515 -0
- seqforge/compose/gates.py +113 -0
- seqforge/compose/params.py +447 -0
- seqforge/e2e.py +1926 -0
- seqforge/evals/__init__.py +78 -0
- seqforge/evals/case.py +382 -0
- seqforge/evals/grade.py +300 -0
- seqforge/evals/run.py +420 -0
- seqforge/harvest/__init__.py +121 -0
- seqforge/harvest/extract.py +319 -0
- seqforge/harvest/fields.py +212 -0
- seqforge/harvest/normalize.py +537 -0
- seqforge/harvest/prep.py +41 -0
- seqforge/harvest/providers.py +321 -0
- seqforge/harvest/verify.py +251 -0
- seqforge/hooks/__init__.py +33 -0
- seqforge/hooks/guards.py +214 -0
- seqforge/io/__init__.py +61 -0
- seqforge/io/archive.py +450 -0
- seqforge/io/attributes.py +190 -0
- seqforge/io/biosample/attributes.json +6341 -0
- seqforge/io/efo/labels.json +55 -0
- seqforge/io/efo.py +138 -0
- seqforge/io/onlist.py +661 -0
- seqforge/io/onlists/3M-february-2018.codes.gz +0 -0
- seqforge/io/onlists/737K-arc-v1.codes.gz +0 -0
- seqforge/io/onlists/737K-august-2016.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3.codes.gz +0 -0
- seqforge/io/onlists/index.json +74 -0
- seqforge/io/remote.py +659 -0
- seqforge/io/taxonomy.py +194 -0
- seqforge/kb/__init__.py +62 -0
- seqforge/kb/anchor.py +169 -0
- seqforge/kb/generate.py +147 -0
- seqforge/kb/loader.py +152 -0
- seqforge/kb/roundtrip.py +112 -0
- seqforge/kb/schema.py +422 -0
- seqforge/kb/specs/10x-3p-gex/spec.yaml +62 -0
- seqforge/kb/specs/10x-3p-gex-v2/README.md +41 -0
- seqforge/kb/specs/10x-3p-gex-v2/spec.yaml +83 -0
- seqforge/kb/specs/10x-3p-gex-v3/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3/spec.yaml +118 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/spec.yaml +124 -0
- seqforge/kb/specs/bd-rhapsody-wta/README.md +103 -0
- seqforge/kb/specs/bd-rhapsody-wta/spec.yaml +130 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced/spec.yaml +99 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v1/spec.yaml +93 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v2/spec.yaml +81 -0
- seqforge/kb/specs/bulk-rnaseq-pe/README.md +35 -0
- seqforge/kb/specs/bulk-rnaseq-pe/spec.yaml +97 -0
- seqforge/kb/specs/splitseq/README.md +51 -0
- seqforge/kb/specs/splitseq/spec.yaml +157 -0
- seqforge/manifest/__init__.py +61 -0
- seqforge/manifest/fill.py +531 -0
- seqforge/manifest/hash.py +77 -0
- seqforge/manifest/instruct.py +114 -0
- seqforge/manifest/policy.py +409 -0
- seqforge/manifest/validate.py +274 -0
- seqforge/models/__init__.py +268 -0
- seqforge/models/assertion.py +68 -0
- seqforge/models/base.py +100 -0
- seqforge/models/blocker.py +71 -0
- seqforge/models/conflict.py +47 -0
- seqforge/models/dataset.py +320 -0
- seqforge/models/evidenced.py +54 -0
- seqforge/models/observation.py +157 -0
- seqforge/models/processing.py +231 -0
- seqforge/models/records.py +145 -0
- seqforge/models/resolve.py +216 -0
- seqforge/probe/__init__.py +46 -0
- seqforge/probe/core.py +232 -0
- seqforge/probe/signals.py +250 -0
- seqforge/probe/streaming.py +118 -0
- seqforge/project.py +177 -0
- seqforge/py.typed +0 -0
- seqforge/resolve/__init__.py +98 -0
- seqforge/resolve/assign.py +204 -0
- seqforge/resolve/cache.py +119 -0
- seqforge/resolve/confuse.py +215 -0
- seqforge/resolve/engine.py +646 -0
- seqforge/resolve/escalate.py +668 -0
- seqforge/resolve/evaluators.py +306 -0
- seqforge/resolve/geometry.py +89 -0
- seqforge/resolve/group.py +85 -0
- seqforge/resolve/records.py +550 -0
- seqforge/resolve/scoring.py +373 -0
- seqforge/resolve/window.py +206 -0
- seqforge/workflows/__init__.py +234 -0
- seqforge/workflows/cram.py +117 -0
- seqforge/workflows/h5ad.py +368 -0
- seqforge/workflows/map/star.smk +101 -0
- seqforge/workflows/map/starsolo.smk +360 -0
- seqforge/workflows/qc.py +157 -0
- seqforge/workspace.py +125 -0
- seqforge-2026.7.1.dist-info/METADATA +125 -0
- seqforge-2026.7.1.dist-info/RECORD +124 -0
- seqforge-2026.7.1.dist-info/WHEEL +4 -0
- seqforge-2026.7.1.dist-info/entry_points.txt +2 -0
- seqforge-2026.7.1.dist-info/licenses/LICENSE +21 -0
seqforge/kb/loader.py
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""Discover and load KB ``spec.yaml`` files, validating each against the :class:`Spec` schema."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import yaml
|
|
9
|
+
|
|
10
|
+
from .schema import Spec
|
|
11
|
+
|
|
12
|
+
SPECS_DIR = Path(__file__).parent / "specs"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def list_spec_ids() -> list[str]:
|
|
16
|
+
"""Return the ids of every technology directory that ships a ``spec.yaml``."""
|
|
17
|
+
if not SPECS_DIR.is_dir():
|
|
18
|
+
return []
|
|
19
|
+
return sorted(p.name for p in SPECS_DIR.iterdir() if (p / "spec.yaml").is_file())
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def load_spec(tech_id: str) -> Spec:
|
|
23
|
+
"""Load and validate one technology's ``spec.yaml``."""
|
|
24
|
+
path = SPECS_DIR / tech_id / "spec.yaml"
|
|
25
|
+
if not path.is_file():
|
|
26
|
+
raise FileNotFoundError(f"no KB spec for {tech_id!r} at {path}")
|
|
27
|
+
data = yaml.safe_load(path.read_text())
|
|
28
|
+
return Spec.model_validate(data)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def load_all_specs() -> dict[str, Spec]:
|
|
32
|
+
"""Load and validate every KB spec (the deterministic core of ``kb lint``)."""
|
|
33
|
+
return {tech_id: load_spec(tech_id) for tech_id in list_spec_ids()}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def runnable_spec_ids() -> list[str]:
|
|
37
|
+
"""Ids of specs that compile to a recipe — leaves and runnable families, not abstract nodes.
|
|
38
|
+
|
|
39
|
+
An abstract family node declares no ``backend``: it classifies during descent but is never scored,
|
|
40
|
+
composed, or params-gated. Tests and tools over "every chemistry" collect from here (derived from
|
|
41
|
+
the KB, not hand-maintained) so a family node never masquerades as a runnable one.
|
|
42
|
+
"""
|
|
43
|
+
return [tech_id for tech_id in list_spec_ids() if load_spec(tech_id).backend is not None]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class KbTree:
|
|
48
|
+
"""The KB as a parent/child forest.
|
|
49
|
+
|
|
50
|
+
A **family** node narrows to its children; a **leaf** is a concrete, runnable chemistry. Siblings
|
|
51
|
+
(same parent) are confusable-by-construction, decided by the parent's ``children_decided_by`` — so
|
|
52
|
+
the resolver reads sibling relationships off the tree instead of a hand-declared ``confusable_with``
|
|
53
|
+
clique. ``children`` is ``node id -> sorted child ids``.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
specs: dict[str, Spec]
|
|
57
|
+
children: dict[str, list[str]]
|
|
58
|
+
|
|
59
|
+
def children_of(self, tech_id: str) -> list[str]:
|
|
60
|
+
return list(self.children.get(tech_id, []))
|
|
61
|
+
|
|
62
|
+
def is_family(self, tech_id: str) -> bool:
|
|
63
|
+
return bool(self.children.get(tech_id))
|
|
64
|
+
|
|
65
|
+
def parent_of(self, tech_id: str) -> str | None:
|
|
66
|
+
return self.specs[tech_id].parent
|
|
67
|
+
|
|
68
|
+
def siblings_of(self, tech_id: str) -> list[str]:
|
|
69
|
+
"""Other children of this node's parent (empty for a root)."""
|
|
70
|
+
parent = self.specs[tech_id].parent
|
|
71
|
+
if parent is None:
|
|
72
|
+
return []
|
|
73
|
+
return [c for c in self.children.get(parent, []) if c != tech_id]
|
|
74
|
+
|
|
75
|
+
def ancestors_of(self, tech_id: str) -> list[str]:
|
|
76
|
+
"""The parent chain from this node up to (and including) its root, nearest first."""
|
|
77
|
+
out: list[str] = []
|
|
78
|
+
seen: set[str] = set()
|
|
79
|
+
cur = self.specs[tech_id].parent
|
|
80
|
+
while cur is not None and cur not in seen:
|
|
81
|
+
out.append(cur)
|
|
82
|
+
seen.add(cur)
|
|
83
|
+
cur = self.specs[cur].parent if cur in self.specs else None
|
|
84
|
+
return out
|
|
85
|
+
|
|
86
|
+
def runnable_descendants_of(self, tech_id: str) -> list[str]:
|
|
87
|
+
"""Every leaf / runnable family at or below this node (what descent may terminate on)."""
|
|
88
|
+
out: list[str] = []
|
|
89
|
+
stack = [tech_id]
|
|
90
|
+
seen: set[str] = set()
|
|
91
|
+
while stack:
|
|
92
|
+
node = stack.pop()
|
|
93
|
+
if node in seen:
|
|
94
|
+
continue
|
|
95
|
+
seen.add(node)
|
|
96
|
+
if self.specs[node].backend is not None:
|
|
97
|
+
out.append(node)
|
|
98
|
+
stack.extend(self.children.get(node, []))
|
|
99
|
+
return sorted(out)
|
|
100
|
+
|
|
101
|
+
def leaves(self) -> list[str]:
|
|
102
|
+
"""Ids with no children — the concrete, compilable chemistries."""
|
|
103
|
+
return sorted(i for i in self.specs if not self.children.get(i))
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def build_tree(specs: dict[str, Spec]) -> KbTree:
|
|
107
|
+
"""Build and VALIDATE the KB forest; raise ``ValueError`` on a malformed tree.
|
|
108
|
+
|
|
109
|
+
Checks: every ``parent`` resolves; no cycles; ``node_kind`` agrees with having children; every
|
|
110
|
+
abstract family (a family with no backend) has at least one runnable descendant, so descent from it
|
|
111
|
+
can always terminate at something that compiles.
|
|
112
|
+
"""
|
|
113
|
+
children: dict[str, list[str]] = {}
|
|
114
|
+
for tech_id, spec in specs.items():
|
|
115
|
+
parent = spec.parent
|
|
116
|
+
if parent is None:
|
|
117
|
+
continue
|
|
118
|
+
if parent == tech_id:
|
|
119
|
+
raise ValueError(f"{tech_id!r}: a node cannot be its own parent")
|
|
120
|
+
if parent not in specs:
|
|
121
|
+
raise ValueError(f"{tech_id!r}: parent {parent!r} is not a known spec")
|
|
122
|
+
children.setdefault(parent, []).append(tech_id)
|
|
123
|
+
for parent in children:
|
|
124
|
+
children[parent] = sorted(children[parent])
|
|
125
|
+
|
|
126
|
+
for tech_id in specs: # no parent cycles: every chain must terminate at a root
|
|
127
|
+
seen: set[str] = set()
|
|
128
|
+
cur: str | None = tech_id
|
|
129
|
+
while cur is not None:
|
|
130
|
+
if cur in seen:
|
|
131
|
+
raise ValueError(f"parent cycle through {tech_id!r}")
|
|
132
|
+
seen.add(cur)
|
|
133
|
+
cur = specs[cur].parent
|
|
134
|
+
|
|
135
|
+
tree = KbTree(specs=specs, children=children)
|
|
136
|
+
for tech_id, spec in specs.items():
|
|
137
|
+
has_children = bool(children.get(tech_id))
|
|
138
|
+
if has_children and spec.node_kind != "family":
|
|
139
|
+
raise ValueError(f"{tech_id!r} has children but is not declared node_kind: family")
|
|
140
|
+
if spec.node_kind == "family" and not has_children:
|
|
141
|
+
raise ValueError(f"{tech_id!r} is declared a family but has no children")
|
|
142
|
+
if spec.node_kind == "family" and spec.backend is None:
|
|
143
|
+
if not tree.runnable_descendants_of(tech_id):
|
|
144
|
+
raise ValueError(
|
|
145
|
+
f"abstract family {tech_id!r} has no runnable descendant to compile"
|
|
146
|
+
)
|
|
147
|
+
return tree
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def load_tree() -> KbTree:
|
|
151
|
+
"""Load every spec and build the validated KB tree."""
|
|
152
|
+
return build_tree(load_all_specs())
|
seqforge/kb/roundtrip.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""``kb roundtrip`` — the self-test: spec -> synth FASTQ -> probe -> recover; assert == declared.
|
|
2
|
+
|
|
3
|
+
Generic over any spec: for every declared read it checks that the probe recovers the declared fixed
|
|
4
|
+
length, that barcode windows recur (low distinct-ratio), and that UMI windows are ~unique (high
|
|
5
|
+
distinct-ratio). Uses a temp directory; touches no real data.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import tempfile
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from ..probe import probe_file
|
|
15
|
+
from ..probe.signals import window_distinct_ratio
|
|
16
|
+
from .anchor import resolve_windows
|
|
17
|
+
from .generate import generate_reads, write_fastq_gz
|
|
18
|
+
from .loader import load_spec
|
|
19
|
+
from .schema import Read
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _write_fastq_gz(path: Path, seqs: list[str]) -> None:
|
|
23
|
+
write_fastq_gz(path, seqs)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _anchored_distinct_ratio(seqs: list[str], read: Read, element_name: str) -> float | None:
|
|
27
|
+
"""``distinct/total`` of a FLOATING element, sliced at each read's phase-detected frame.
|
|
28
|
+
|
|
29
|
+
A fixed element has a constant window :func:`window_distinct_ratio` can slice; an anchored one does
|
|
30
|
+
not (the diversity insert staggers it per read), so recover its frame first. This is what makes the
|
|
31
|
+
round-trip actually verify anchoring: if the resolver could not recover the staggered cell-label
|
|
32
|
+
blocks, their slices would not recur and ``barcode_recurs`` would go red.
|
|
33
|
+
"""
|
|
34
|
+
slices = [
|
|
35
|
+
s[a:b]
|
|
36
|
+
for s in seqs
|
|
37
|
+
if (w := resolve_windows(s, read)) is not None and element_name in w
|
|
38
|
+
for a, b in (w[element_name],)
|
|
39
|
+
if s[a:b]
|
|
40
|
+
]
|
|
41
|
+
if not slices:
|
|
42
|
+
return None
|
|
43
|
+
return len(set(slices)) / len(slices)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def run_roundtrip(tech_id: str, *, n: int = 2000, seed: int = 0) -> dict[str, Any]:
|
|
47
|
+
"""Round-trip one technology and return ``{tech, passed, checks:[...]}``."""
|
|
48
|
+
spec = load_spec(tech_id)
|
|
49
|
+
reads = generate_reads(spec, n=n, seed=seed)
|
|
50
|
+
checks: list[dict[str, Any]] = []
|
|
51
|
+
|
|
52
|
+
with tempfile.TemporaryDirectory() as td:
|
|
53
|
+
for read in spec.reads:
|
|
54
|
+
seqs = reads[read.id]
|
|
55
|
+
path = Path(td) / f"{read.id}.fastq.gz"
|
|
56
|
+
_write_fastq_gz(path, seqs)
|
|
57
|
+
obs = probe_file(path)
|
|
58
|
+
|
|
59
|
+
if read.min_len is not None and read.min_len == read.max_len:
|
|
60
|
+
checks.append(
|
|
61
|
+
{
|
|
62
|
+
"read": read.id,
|
|
63
|
+
"check": "length",
|
|
64
|
+
"ok": obs.read_length.mode == read.min_len,
|
|
65
|
+
"declared": read.min_len,
|
|
66
|
+
"recovered": obs.read_length.mode,
|
|
67
|
+
}
|
|
68
|
+
)
|
|
69
|
+
# an open-ended cDNA/gDNA read must probe back as variable-length (non-vacuous for the
|
|
70
|
+
# no-barcode bulk branch, whose only structural claim is "two variable cDNA reads").
|
|
71
|
+
has_open_cdna = any(
|
|
72
|
+
el.type in ("cdna", "gdna") and el.end is None for el in read.elements
|
|
73
|
+
)
|
|
74
|
+
if has_open_cdna and read.min_len != read.max_len:
|
|
75
|
+
checks.append(
|
|
76
|
+
{
|
|
77
|
+
"read": read.id,
|
|
78
|
+
"check": "cdna_variable",
|
|
79
|
+
"ok": obs.read_length.n_distinct > 1,
|
|
80
|
+
"n_distinct": obs.read_length.n_distinct,
|
|
81
|
+
}
|
|
82
|
+
)
|
|
83
|
+
for el in read.elements:
|
|
84
|
+
if el.start is not None and el.end is not None:
|
|
85
|
+
ratio = window_distinct_ratio(seqs, el.start, el.end)
|
|
86
|
+
elif el.anchor is not None:
|
|
87
|
+
# a floating element: recover its per-read frame rather than skipping it (pre-#43
|
|
88
|
+
# every anchored element was skipped, so the round-trip proved nothing about them).
|
|
89
|
+
ratio = _anchored_distinct_ratio(seqs, read, el.name)
|
|
90
|
+
else:
|
|
91
|
+
continue # a variable-length insert with no anchor (the VB itself): no window to
|
|
92
|
+
# check — its recovery is proven by the downstream anchored elements resolving.
|
|
93
|
+
if el.type == "barcode" and el.onlist:
|
|
94
|
+
checks.append(
|
|
95
|
+
{
|
|
96
|
+
"read": read.id,
|
|
97
|
+
"check": f"barcode_recurs:{el.name}",
|
|
98
|
+
"ok": ratio is not None and ratio < 0.5,
|
|
99
|
+
"ratio": ratio,
|
|
100
|
+
}
|
|
101
|
+
)
|
|
102
|
+
elif el.type == "umi":
|
|
103
|
+
checks.append(
|
|
104
|
+
{
|
|
105
|
+
"read": read.id,
|
|
106
|
+
"check": f"umi_unique:{el.name}",
|
|
107
|
+
"ok": ratio is not None and ratio > 0.7,
|
|
108
|
+
"ratio": ratio,
|
|
109
|
+
}
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
return {"tech": tech_id, "passed": all(c["ok"] for c in checks), "checks": checks}
|
seqforge/kb/schema.py
ADDED
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
"""The KB ``spec.yaml`` schema — machine-checkable, closed-vocabulary, self-validating.
|
|
2
|
+
|
|
3
|
+
One directory per technology: ``kb/specs/<tech>/{spec.yaml, README.md}``. ``spec.yaml`` declares the
|
|
4
|
+
read layout (element coordinates), onlist references, a detection ``signature`` (requires / supports /
|
|
5
|
+
excludes), a ``backend`` param template, and a ``confusable_with`` list. Every model forbids extra
|
|
6
|
+
keys, so a typo fails validation exactly where the DSL is executed. The signature test vocabulary is
|
|
7
|
+
*exactly* the scorer's evaluator set.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
from typing import Annotated, Literal
|
|
14
|
+
|
|
15
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
16
|
+
|
|
17
|
+
ElementType = Literal[
|
|
18
|
+
"barcode", "umi", "cdna", "gdna", "linker", "poly_a", "poly_t", "fixed", "index", "diversity"
|
|
19
|
+
]
|
|
20
|
+
# ``diversity`` = a variable-length insert of random bases at a read's 5' end (BD Rhapsody Enhanced's
|
|
21
|
+
# 0-3 bp diversity/phasing insert). It carries no barcode/UMI value — its ONLY effect is to STAGGER
|
|
22
|
+
# every downstream element by its per-read length, which is why it is modelled as a first-class element
|
|
23
|
+
# (``min_len``/``max_len``): the generator draws a per-read length for it and the anchored resolver
|
|
24
|
+
# recovers the frame the stagger created (see ``kb/anchor.py``). It is not a linker/fixed (no literal
|
|
25
|
+
# sequence) and not a counted feature, so ``compose`` skips it and it maps to a ``linker`` manifest role.
|
|
26
|
+
Mechanism = Literal["none", "onlist", "metadata", "alignment", "user"]
|
|
27
|
+
Decidable = Literal["reads", "onlist", "metadata", "alignment", "user"]
|
|
28
|
+
Orientation = Literal["forward", "revcomp", "either"]
|
|
29
|
+
SeqspecRegion = Literal[
|
|
30
|
+
"barcode",
|
|
31
|
+
"umi",
|
|
32
|
+
"cdna",
|
|
33
|
+
"gdna",
|
|
34
|
+
"index5",
|
|
35
|
+
"index7",
|
|
36
|
+
"linker",
|
|
37
|
+
"poly_A",
|
|
38
|
+
"poly_t",
|
|
39
|
+
"custom_primer",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
_ONLIST_TOKEN = re.compile(r"^\{onlist:([A-Za-z0-9._-]+)\}$")
|
|
43
|
+
_ANY_BRACE = re.compile(r"\{[^}]*\}")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class _Forbid(BaseModel):
|
|
47
|
+
"""Base that forbids unknown keys, so the closed vocabulary is enforced where it is executed."""
|
|
48
|
+
|
|
49
|
+
model_config = ConfigDict(extra="forbid")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Anchor(_Forbid):
|
|
53
|
+
"""Locate a variable-length / floating element (e.g. inDrop's post-W1 barcode)."""
|
|
54
|
+
|
|
55
|
+
relative_to: Literal["read_start", "read_end", "element"] = "read_start"
|
|
56
|
+
ref_element: str | None = None
|
|
57
|
+
ref_side: Literal["start", "end"] = "end"
|
|
58
|
+
offset: int = 0
|
|
59
|
+
motif: str | None = None
|
|
60
|
+
max_mismatch: int = 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Element(_Forbid):
|
|
64
|
+
"""One element of a read. 0-based half-open ``[start, end)``; ``end=None`` => open-ended (cDNA)."""
|
|
65
|
+
|
|
66
|
+
type: ElementType
|
|
67
|
+
name: str
|
|
68
|
+
start: int | None = None
|
|
69
|
+
end: int | None = None
|
|
70
|
+
min_len: int | None = None
|
|
71
|
+
max_len: int | None = None
|
|
72
|
+
anchor: Anchor | None = None
|
|
73
|
+
sequence: str | None = None
|
|
74
|
+
onlist: str | None = None
|
|
75
|
+
seqspec_region_type: SeqspecRegion
|
|
76
|
+
|
|
77
|
+
@model_validator(mode="after")
|
|
78
|
+
def _addressable(self) -> Element:
|
|
79
|
+
fixed = self.start is not None and self.end is not None
|
|
80
|
+
opened = self.start is not None and self.end is None
|
|
81
|
+
varlen = self.min_len is not None or self.max_len is not None
|
|
82
|
+
anchored = self.anchor is not None
|
|
83
|
+
if self.type in ("linker", "fixed") and self.sequence is None:
|
|
84
|
+
raise ValueError(f"element {self.name!r}: linker/fixed needs a literal `sequence`")
|
|
85
|
+
if self.type in ("cdna", "gdna"):
|
|
86
|
+
return self # open-ended is legal
|
|
87
|
+
if not (fixed or opened or anchored or varlen):
|
|
88
|
+
raise ValueError(f"element {self.name!r}: give [start,end), an anchor, or min/max_len")
|
|
89
|
+
return self
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class Read(_Forbid):
|
|
93
|
+
"""A read (== one FASTQ). ``id`` is a ROLE label (R1/R2/bc/cdna), never a filename claim."""
|
|
94
|
+
|
|
95
|
+
id: str
|
|
96
|
+
seqspec_read_id: str
|
|
97
|
+
file_hint: str | None = None
|
|
98
|
+
strand: Literal["pos", "neg"] = "pos"
|
|
99
|
+
min_len: int | None = None
|
|
100
|
+
max_len: int | None = None
|
|
101
|
+
elements: list[Element]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class OnlistRef(_Forbid):
|
|
105
|
+
"""Alias -> pooch-registry name. URL/sha256/length/orientation live in the registry, never here."""
|
|
106
|
+
|
|
107
|
+
registry: str
|
|
108
|
+
role: Literal["cell_barcode", "sample_index", "feature", "atac_barcode"]
|
|
109
|
+
expected_orientation: Orientation = "forward"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# ---- signature tests: a CLOSED set == the scorer's evaluators ----
|
|
113
|
+
class _Seg(_Forbid):
|
|
114
|
+
"""A test addressed to a segment by element name XOR (start, end)."""
|
|
115
|
+
|
|
116
|
+
read: str
|
|
117
|
+
element: str | None = None
|
|
118
|
+
start: int | None = None
|
|
119
|
+
end: int | None = None
|
|
120
|
+
|
|
121
|
+
@model_validator(mode="after")
|
|
122
|
+
def _one_address(self) -> _Seg:
|
|
123
|
+
by_name = self.element is not None
|
|
124
|
+
by_coord = self.start is not None and self.end is not None
|
|
125
|
+
if by_name == by_coord:
|
|
126
|
+
raise ValueError("address a segment by element name XOR (start, end)")
|
|
127
|
+
return self
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class ReadCount(_Forbid):
|
|
131
|
+
test: Literal["read_count"]
|
|
132
|
+
roles: int # biological + barcode ROLE count, never raw file count
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class SegmentLength(_Forbid):
|
|
136
|
+
test: Literal["segment_length"]
|
|
137
|
+
read: str
|
|
138
|
+
length: int
|
|
139
|
+
tolerance: int = 0
|
|
140
|
+
#: Over-length escape. A read whose mode is ``>= over_length_min`` PASSES regardless of ``length``
|
|
141
|
+
#: — an insert-bearing / over-sequenced barcode read (e.g. a 10x R1 sequenced to 150 bp: CB+UMI in
|
|
142
|
+
#: bp0-28, the rest junk). Keep it strictly above ``length`` so a canonical read is never
|
|
143
|
+
#: "over-length" and the exact-length gate still separates near-neighbours (v2's 26 vs v3's 28).
|
|
144
|
+
over_length_min: int | None = None
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class HasSegment(_Seg):
|
|
148
|
+
test: Literal["has_segment"]
|
|
149
|
+
kind: Literal["constant", "random", "polyT", "polyA"]
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class DistinctRatio(_Seg):
|
|
153
|
+
test: Literal["distinct_ratio"]
|
|
154
|
+
expect: Literal["low", "high"] # SUPPORTS-only; depth-dependent, never a gate
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class OnlistHitRate(_Seg):
|
|
158
|
+
test: Literal["onlist_hit_rate"]
|
|
159
|
+
onlist: str
|
|
160
|
+
orientation: Orientation = "either"
|
|
161
|
+
min: float
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class MotifPresent(_Forbid):
|
|
165
|
+
test: Literal["motif_present"]
|
|
166
|
+
read: str
|
|
167
|
+
motif: str
|
|
168
|
+
where: Literal["read_start", "read_end", "anywhere", "window"] = "anywhere"
|
|
169
|
+
search_start: int | None = None
|
|
170
|
+
search_end: int | None = None
|
|
171
|
+
max_mismatch: int = 1
|
|
172
|
+
min_rate: float = 0.5
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class BaseComposition(_Seg):
|
|
176
|
+
test: Literal["base_composition"]
|
|
177
|
+
base: Literal["A", "C", "G", "T", "N"]
|
|
178
|
+
min_fraction: float
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class HeaderIndex(_Forbid):
|
|
182
|
+
test: Literal["header_index"]
|
|
183
|
+
present: bool
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
Test = Annotated[
|
|
187
|
+
ReadCount
|
|
188
|
+
| SegmentLength
|
|
189
|
+
| HasSegment
|
|
190
|
+
| DistinctRatio
|
|
191
|
+
| OnlistHitRate
|
|
192
|
+
| MotifPresent
|
|
193
|
+
| BaseComposition
|
|
194
|
+
| HeaderIndex,
|
|
195
|
+
Field(discriminator="test"),
|
|
196
|
+
]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class Support(_Forbid):
|
|
200
|
+
when: Test
|
|
201
|
+
weight: float = 1.0
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
class Signature(_Forbid):
|
|
205
|
+
requires: list[Test] # hard AND-gates (no distinct_ratio here — it's depth-dependent)
|
|
206
|
+
supports: list[Support] # additive evidence (onlist + distinct_ratio live here)
|
|
207
|
+
excludes: list[Test] # anti-gates: any pass => disqualify
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
KB_PARSE_KEYS: frozenset[str] = frozenset(
|
|
211
|
+
{
|
|
212
|
+
"soloType",
|
|
213
|
+
"soloCBstart",
|
|
214
|
+
"soloCBlen",
|
|
215
|
+
"soloUMIstart",
|
|
216
|
+
"soloUMIlen",
|
|
217
|
+
"soloCBwhitelist",
|
|
218
|
+
"soloCBposition",
|
|
219
|
+
"soloUMIposition",
|
|
220
|
+
"soloStrand",
|
|
221
|
+
"soloAdapterSequence",
|
|
222
|
+
"soloBarcodeReadLength",
|
|
223
|
+
}
|
|
224
|
+
)
|
|
225
|
+
"""Every key the KB may declare. Each says how to **parse** reads, and each is decided by bytes.
|
|
226
|
+
|
|
227
|
+
The line is parse vs. count. What to COUNT — ``soloFeatures``, ``quantMode`` — is *intent*: it belongs
|
|
228
|
+
to the processing manifest, where a user may instruct it and a gate may check it. ``soloFeatures``
|
|
229
|
+
sat here because this is where the aligner's flags live, and it cost a measured **40.7 % of a nuclear
|
|
230
|
+
library**: 10x 3' v3.1 chemistry is byte-identical for cells and nuclei, so counting was never a
|
|
231
|
+
chemistry property at all.
|
|
232
|
+
|
|
233
|
+
The disjointness of this set from the instructable surface is what makes "a user instruction
|
|
234
|
+
contradicts the observed bytes" **structurally inexpressible** rather than merely deprioritized — the
|
|
235
|
+
user has no vocabulary in which to say it. That is the strongest form of that guarantee available, and it is why
|
|
236
|
+
moving a key across this line has to be an explicit, gated act.
|
|
237
|
+
"""
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
class Backend(_Forbid):
|
|
241
|
+
"""A data template mapping to a workflow module. Only ``{onlist:<alias>}`` interpolation is legal."""
|
|
242
|
+
|
|
243
|
+
module: str
|
|
244
|
+
params: dict[str, str | int | float | list[str]]
|
|
245
|
+
|
|
246
|
+
@model_validator(mode="after")
|
|
247
|
+
def _only_parse_keys(self) -> Backend:
|
|
248
|
+
"""A count/reference/runtime knob may not be declared here.
|
|
249
|
+
|
|
250
|
+
``params`` is a ``dict``, so ``extra="forbid"`` cannot reach inside it — hence an explicit
|
|
251
|
+
validator. It fires in ``load_spec``, in ``kb lint``, and in every test that loads a spec,
|
|
252
|
+
which is what makes the parse/count line a property of the DSL rather than a convention.
|
|
253
|
+
"""
|
|
254
|
+
stray = sorted(set(self.params) - KB_PARSE_KEYS)
|
|
255
|
+
if stray:
|
|
256
|
+
raise ValueError(
|
|
257
|
+
f"backend.params declares non-parse key(s) {stray}: backend.params says how to "
|
|
258
|
+
f"PARSE reads (byte-decided); what to COUNT belongs in the processing manifest, "
|
|
259
|
+
f"where a user may instruct it. Known parse keys: {sorted(KB_PARSE_KEYS)}"
|
|
260
|
+
)
|
|
261
|
+
return self
|
|
262
|
+
|
|
263
|
+
def check_tokens(self, onlist_aliases: set[str]) -> None:
|
|
264
|
+
"""Reject any interpolation token that is not a declared ``{onlist:<alias>}``."""
|
|
265
|
+
for value in self._strings():
|
|
266
|
+
for match in _ANY_BRACE.finditer(value):
|
|
267
|
+
token = _ONLIST_TOKEN.match(match.group(0))
|
|
268
|
+
if token is None:
|
|
269
|
+
raise ValueError(
|
|
270
|
+
f"illegal template expression {match.group(0)!r} "
|
|
271
|
+
"(only {onlist:<alias>} is allowed)"
|
|
272
|
+
)
|
|
273
|
+
if token.group(1) not in onlist_aliases:
|
|
274
|
+
raise ValueError(f"unknown onlist alias {token.group(1)!r}")
|
|
275
|
+
|
|
276
|
+
def _strings(self) -> list[str]:
|
|
277
|
+
out: list[str] = []
|
|
278
|
+
for value in self.params.values():
|
|
279
|
+
if isinstance(value, str):
|
|
280
|
+
out.append(value)
|
|
281
|
+
elif isinstance(value, list):
|
|
282
|
+
out.extend(v for v in value if isinstance(v, str))
|
|
283
|
+
return out
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class Confusable(_Forbid):
|
|
287
|
+
id: str
|
|
288
|
+
relationship: Literal["processing_equivalent", "processing_divergent"]
|
|
289
|
+
distinguishable_by: list[Mechanism]
|
|
290
|
+
note: str = ""
|
|
291
|
+
|
|
292
|
+
@model_validator(mode="after")
|
|
293
|
+
def _shape(self) -> Confusable:
|
|
294
|
+
if self.relationship == "processing_divergent" and self.distinguishable_by == ["none"]:
|
|
295
|
+
raise ValueError("a processing_divergent pair cannot be distinguishable_by [none]")
|
|
296
|
+
return self
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
class Identity(_Forbid):
|
|
300
|
+
id: str
|
|
301
|
+
version: str
|
|
302
|
+
name: str
|
|
303
|
+
aliases: list[str] = Field(default_factory=list)
|
|
304
|
+
assay_ontology: list[str] = Field(default_factory=list)
|
|
305
|
+
modality: Literal["rna", "atac", "multi"] = "rna"
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
class Spec(_Forbid):
|
|
309
|
+
"""A complete, self-validating technology specification (one node in the KB tree)."""
|
|
310
|
+
|
|
311
|
+
schema_version: int
|
|
312
|
+
identity: Identity
|
|
313
|
+
reads: list[Read]
|
|
314
|
+
onlists: dict[str, OnlistRef]
|
|
315
|
+
signature: Signature
|
|
316
|
+
#: The runnable STARsolo/STAR template. ``None`` on an ABSTRACT family node — a classifier that
|
|
317
|
+
#: narrows to its children but has no single recipe (``10x-3p-gex`` parses v2 and v3 differently).
|
|
318
|
+
#: A leaf, and a runnable family like ``bulk``, always declare one.
|
|
319
|
+
backend: Backend | None = None
|
|
320
|
+
#: The family id this node descends from; ``None`` at a root. Siblings (same parent) are
|
|
321
|
+
#: confusable-by-construction, decided by the parent's ``children_decided_by`` — which is why a
|
|
322
|
+
#: sibling clique of ``processing_divergent`` ``confusable_with`` edges collapses to one ``parent``.
|
|
323
|
+
parent: str | None = None
|
|
324
|
+
node_kind: Literal["family", "leaf"] = "leaf"
|
|
325
|
+
#: How a FAMILY node's children are told apart (``onlist`` / ``metadata`` / ``alignment``). Empty on
|
|
326
|
+
#: a leaf. Replaces the per-sibling ``distinguishable_by`` for the divergent-tie question.
|
|
327
|
+
children_decided_by: list[Mechanism] = Field(default_factory=list)
|
|
328
|
+
confusable_with: list[Confusable] = Field(default_factory=list)
|
|
329
|
+
|
|
330
|
+
def require_backend(self) -> Backend:
|
|
331
|
+
"""The runnable backend, or a clear error if this is an abstract family node.
|
|
332
|
+
|
|
333
|
+
Only leaves and runnable families compile; descent always resolves to one of those, so every
|
|
334
|
+
compose/params/policy site is entitled to a backend. This turns the ``Backend | None`` into a
|
|
335
|
+
checked ``Backend`` at exactly those call sites.
|
|
336
|
+
"""
|
|
337
|
+
if self.backend is None:
|
|
338
|
+
raise ValueError(
|
|
339
|
+
f"{self.identity.id!r} is an abstract family node with no runnable backend; "
|
|
340
|
+
"only leaves and runnable families compile"
|
|
341
|
+
)
|
|
342
|
+
return self.backend
|
|
343
|
+
|
|
344
|
+
@property
|
|
345
|
+
def decidable_by(self) -> list[Decidable]:
|
|
346
|
+
"""How this technology can be separated from the ones it is confusable with. **Derived.**
|
|
347
|
+
|
|
348
|
+
This was a hand-typed field on every spec, and two of them carried the comment "CI-computed
|
|
349
|
+
union over the divergent confusables". No CI computed it. Nothing read it either — `escalate`
|
|
350
|
+
builds a Question's ``decidable_by`` from ``confusable_with[].distinguishable_by``, inline,
|
|
351
|
+
which is precisely the union the comment described. So the field was a claim about behaviour
|
|
352
|
+
that caused no behaviour: a comment with a list's syntax, free to drift from the thing it
|
|
353
|
+
claimed to summarize, with nothing to notice.
|
|
354
|
+
|
|
355
|
+
That is the exact shape of `RegistryEntry.fetchable` before it was derived, and of
|
|
356
|
+
`required_config` before that. Deriving it is the only fix that stays fixed.
|
|
357
|
+
|
|
358
|
+
``processing_equivalent`` twins are excluded on purpose: §12 says two entries with identical
|
|
359
|
+
``backend.params`` are declared equivalent and recorded together, so there is nothing to
|
|
360
|
+
decide between them and no mechanism that could.
|
|
361
|
+
"""
|
|
362
|
+
out: set[Decidable] = set()
|
|
363
|
+
for c in self.confusable_with:
|
|
364
|
+
if c.relationship == "processing_divergent":
|
|
365
|
+
out.update(m for m in c.distinguishable_by if m != "none")
|
|
366
|
+
return sorted(out) # type: ignore[arg-type]
|
|
367
|
+
|
|
368
|
+
@model_validator(mode="after")
|
|
369
|
+
def _node_shape(self) -> Spec:
|
|
370
|
+
"""A leaf must be runnable; ``children_decided_by`` is a family-only knob."""
|
|
371
|
+
if self.node_kind == "leaf":
|
|
372
|
+
if self.backend is None:
|
|
373
|
+
raise ValueError(
|
|
374
|
+
f"{self.identity.id!r}: a leaf node must declare a runnable backend"
|
|
375
|
+
)
|
|
376
|
+
if self.children_decided_by:
|
|
377
|
+
raise ValueError(
|
|
378
|
+
f"{self.identity.id!r}: children_decided_by is only meaningful on a family node"
|
|
379
|
+
)
|
|
380
|
+
return self
|
|
381
|
+
|
|
382
|
+
@model_validator(mode="after")
|
|
383
|
+
def _cross_refs(self) -> Spec:
|
|
384
|
+
aliases = set(self.onlists)
|
|
385
|
+
read_ids = {r.id for r in self.reads}
|
|
386
|
+
elements_by_read = {r.id: {e.name for e in r.elements} for r in self.reads}
|
|
387
|
+
|
|
388
|
+
# every onlist alias referenced by an element must be declared
|
|
389
|
+
for read in self.reads:
|
|
390
|
+
for el in read.elements:
|
|
391
|
+
if el.onlist and el.onlist not in aliases:
|
|
392
|
+
raise ValueError(f"element {el.name!r}: unknown onlist {el.onlist!r}")
|
|
393
|
+
if el.anchor and el.anchor.ref_element:
|
|
394
|
+
if el.anchor.ref_element not in elements_by_read[read.id]:
|
|
395
|
+
raise ValueError(
|
|
396
|
+
f"element {el.name!r}: anchor ref_element "
|
|
397
|
+
f"{el.anchor.ref_element!r} not in read {read.id!r}"
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
# every signature test must reference a declared read (and element/onlist)
|
|
401
|
+
tests: list[Test] = [
|
|
402
|
+
*self.signature.requires,
|
|
403
|
+
*self.signature.excludes,
|
|
404
|
+
*(s.when for s in self.signature.supports),
|
|
405
|
+
]
|
|
406
|
+
for t in tests:
|
|
407
|
+
read = getattr(t, "read", None)
|
|
408
|
+
if read is not None and read not in read_ids:
|
|
409
|
+
raise ValueError(f"signature test references unknown read {read!r}")
|
|
410
|
+
element = getattr(t, "element", None)
|
|
411
|
+
if element is not None and read is not None:
|
|
412
|
+
if element not in elements_by_read.get(read, set()):
|
|
413
|
+
raise ValueError(
|
|
414
|
+
f"signature test references unknown element {element!r} in read {read!r}"
|
|
415
|
+
)
|
|
416
|
+
onlist = getattr(t, "onlist", None)
|
|
417
|
+
if onlist is not None and onlist not in aliases:
|
|
418
|
+
raise ValueError(f"signature test references unknown onlist {onlist!r}")
|
|
419
|
+
|
|
420
|
+
if self.backend is not None:
|
|
421
|
+
self.backend.check_tokens(aliases)
|
|
422
|
+
return self
|