seqforge 2026.7.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. seqforge/__init__.py +16 -0
  2. seqforge/cli/__init__.py +38 -0
  3. seqforge/cli/__main__.py +8 -0
  4. seqforge/cli/_common.py +105 -0
  5. seqforge/cli/compose.py +119 -0
  6. seqforge/cli/eval.py +103 -0
  7. seqforge/cli/harvest.py +417 -0
  8. seqforge/cli/hook.py +247 -0
  9. seqforge/cli/io.py +502 -0
  10. seqforge/cli/kb.py +348 -0
  11. seqforge/cli/manifest.py +536 -0
  12. seqforge/cli/probe.py +43 -0
  13. seqforge/cli/processing.py +192 -0
  14. seqforge/cli/project.py +52 -0
  15. seqforge/cli/resolve.py +55 -0
  16. seqforge/cli/root.py +66 -0
  17. seqforge/cli/run.py +463 -0
  18. seqforge/cli/schema.py +41 -0
  19. seqforge/compose/__init__.py +28 -0
  20. seqforge/compose/core.py +515 -0
  21. seqforge/compose/gates.py +113 -0
  22. seqforge/compose/params.py +447 -0
  23. seqforge/e2e.py +1926 -0
  24. seqforge/evals/__init__.py +78 -0
  25. seqforge/evals/case.py +382 -0
  26. seqforge/evals/grade.py +300 -0
  27. seqforge/evals/run.py +420 -0
  28. seqforge/harvest/__init__.py +121 -0
  29. seqforge/harvest/extract.py +319 -0
  30. seqforge/harvest/fields.py +212 -0
  31. seqforge/harvest/normalize.py +537 -0
  32. seqforge/harvest/prep.py +41 -0
  33. seqforge/harvest/providers.py +321 -0
  34. seqforge/harvest/verify.py +251 -0
  35. seqforge/hooks/__init__.py +33 -0
  36. seqforge/hooks/guards.py +214 -0
  37. seqforge/io/__init__.py +61 -0
  38. seqforge/io/archive.py +450 -0
  39. seqforge/io/attributes.py +190 -0
  40. seqforge/io/biosample/attributes.json +6341 -0
  41. seqforge/io/efo/labels.json +55 -0
  42. seqforge/io/efo.py +138 -0
  43. seqforge/io/onlist.py +661 -0
  44. seqforge/io/onlists/3M-february-2018.codes.gz +0 -0
  45. seqforge/io/onlists/737K-arc-v1.codes.gz +0 -0
  46. seqforge/io/onlists/737K-august-2016.codes.gz +0 -0
  47. seqforge/io/onlists/bd-rhapsody-cls1-384.codes.gz +0 -0
  48. seqforge/io/onlists/bd-rhapsody-cls1.codes.gz +0 -0
  49. seqforge/io/onlists/bd-rhapsody-cls2-384.codes.gz +0 -0
  50. seqforge/io/onlists/bd-rhapsody-cls2.codes.gz +0 -0
  51. seqforge/io/onlists/bd-rhapsody-cls3-384.codes.gz +0 -0
  52. seqforge/io/onlists/bd-rhapsody-cls3.codes.gz +0 -0
  53. seqforge/io/onlists/index.json +74 -0
  54. seqforge/io/remote.py +659 -0
  55. seqforge/io/taxonomy.py +194 -0
  56. seqforge/kb/__init__.py +62 -0
  57. seqforge/kb/anchor.py +169 -0
  58. seqforge/kb/generate.py +147 -0
  59. seqforge/kb/loader.py +152 -0
  60. seqforge/kb/roundtrip.py +112 -0
  61. seqforge/kb/schema.py +422 -0
  62. seqforge/kb/specs/10x-3p-gex/spec.yaml +62 -0
  63. seqforge/kb/specs/10x-3p-gex-v2/README.md +41 -0
  64. seqforge/kb/specs/10x-3p-gex-v2/spec.yaml +83 -0
  65. seqforge/kb/specs/10x-3p-gex-v3/README.md +56 -0
  66. seqforge/kb/specs/10x-3p-gex-v3/spec.yaml +118 -0
  67. seqforge/kb/specs/10x-3p-gex-v3.1/README.md +56 -0
  68. seqforge/kb/specs/10x-3p-gex-v3.1/spec.yaml +124 -0
  69. seqforge/kb/specs/bd-rhapsody-wta/README.md +103 -0
  70. seqforge/kb/specs/bd-rhapsody-wta/spec.yaml +130 -0
  71. seqforge/kb/specs/bd-rhapsody-wta-enhanced/spec.yaml +99 -0
  72. seqforge/kb/specs/bd-rhapsody-wta-enhanced-v1/spec.yaml +93 -0
  73. seqforge/kb/specs/bd-rhapsody-wta-enhanced-v2/spec.yaml +81 -0
  74. seqforge/kb/specs/bulk-rnaseq-pe/README.md +35 -0
  75. seqforge/kb/specs/bulk-rnaseq-pe/spec.yaml +97 -0
  76. seqforge/kb/specs/splitseq/README.md +51 -0
  77. seqforge/kb/specs/splitseq/spec.yaml +157 -0
  78. seqforge/manifest/__init__.py +61 -0
  79. seqforge/manifest/fill.py +531 -0
  80. seqforge/manifest/hash.py +77 -0
  81. seqforge/manifest/instruct.py +114 -0
  82. seqforge/manifest/policy.py +409 -0
  83. seqforge/manifest/validate.py +274 -0
  84. seqforge/models/__init__.py +268 -0
  85. seqforge/models/assertion.py +68 -0
  86. seqforge/models/base.py +100 -0
  87. seqforge/models/blocker.py +71 -0
  88. seqforge/models/conflict.py +47 -0
  89. seqforge/models/dataset.py +320 -0
  90. seqforge/models/evidenced.py +54 -0
  91. seqforge/models/observation.py +157 -0
  92. seqforge/models/processing.py +231 -0
  93. seqforge/models/records.py +145 -0
  94. seqforge/models/resolve.py +216 -0
  95. seqforge/probe/__init__.py +46 -0
  96. seqforge/probe/core.py +232 -0
  97. seqforge/probe/signals.py +250 -0
  98. seqforge/probe/streaming.py +118 -0
  99. seqforge/project.py +177 -0
  100. seqforge/py.typed +0 -0
  101. seqforge/resolve/__init__.py +98 -0
  102. seqforge/resolve/assign.py +204 -0
  103. seqforge/resolve/cache.py +119 -0
  104. seqforge/resolve/confuse.py +215 -0
  105. seqforge/resolve/engine.py +646 -0
  106. seqforge/resolve/escalate.py +668 -0
  107. seqforge/resolve/evaluators.py +306 -0
  108. seqforge/resolve/geometry.py +89 -0
  109. seqforge/resolve/group.py +85 -0
  110. seqforge/resolve/records.py +550 -0
  111. seqforge/resolve/scoring.py +373 -0
  112. seqforge/resolve/window.py +206 -0
  113. seqforge/workflows/__init__.py +234 -0
  114. seqforge/workflows/cram.py +117 -0
  115. seqforge/workflows/h5ad.py +368 -0
  116. seqforge/workflows/map/star.smk +101 -0
  117. seqforge/workflows/map/starsolo.smk +360 -0
  118. seqforge/workflows/qc.py +157 -0
  119. seqforge/workspace.py +125 -0
  120. seqforge-2026.7.1.dist-info/METADATA +125 -0
  121. seqforge-2026.7.1.dist-info/RECORD +124 -0
  122. seqforge-2026.7.1.dist-info/WHEEL +4 -0
  123. seqforge-2026.7.1.dist-info/entry_points.txt +2 -0
  124. seqforge-2026.7.1.dist-info/licenses/LICENSE +21 -0
seqforge/kb/loader.py ADDED
@@ -0,0 +1,152 @@
1
+ """Discover and load KB ``spec.yaml`` files, validating each against the :class:`Spec` schema."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+
8
+ import yaml
9
+
10
+ from .schema import Spec
11
+
12
+ SPECS_DIR = Path(__file__).parent / "specs"
13
+
14
+
15
+ def list_spec_ids() -> list[str]:
16
+ """Return the ids of every technology directory that ships a ``spec.yaml``."""
17
+ if not SPECS_DIR.is_dir():
18
+ return []
19
+ return sorted(p.name for p in SPECS_DIR.iterdir() if (p / "spec.yaml").is_file())
20
+
21
+
22
+ def load_spec(tech_id: str) -> Spec:
23
+ """Load and validate one technology's ``spec.yaml``."""
24
+ path = SPECS_DIR / tech_id / "spec.yaml"
25
+ if not path.is_file():
26
+ raise FileNotFoundError(f"no KB spec for {tech_id!r} at {path}")
27
+ data = yaml.safe_load(path.read_text())
28
+ return Spec.model_validate(data)
29
+
30
+
31
+ def load_all_specs() -> dict[str, Spec]:
32
+ """Load and validate every KB spec (the deterministic core of ``kb lint``)."""
33
+ return {tech_id: load_spec(tech_id) for tech_id in list_spec_ids()}
34
+
35
+
36
+ def runnable_spec_ids() -> list[str]:
37
+ """Ids of specs that compile to a recipe — leaves and runnable families, not abstract nodes.
38
+
39
+ An abstract family node declares no ``backend``: it classifies during descent but is never scored,
40
+ composed, or params-gated. Tests and tools over "every chemistry" collect from here (derived from
41
+ the KB, not hand-maintained) so a family node never masquerades as a runnable one.
42
+ """
43
+ return [tech_id for tech_id in list_spec_ids() if load_spec(tech_id).backend is not None]
44
+
45
+
46
+ @dataclass(frozen=True)
47
+ class KbTree:
48
+ """The KB as a parent/child forest.
49
+
50
+ A **family** node narrows to its children; a **leaf** is a concrete, runnable chemistry. Siblings
51
+ (same parent) are confusable-by-construction, decided by the parent's ``children_decided_by`` — so
52
+ the resolver reads sibling relationships off the tree instead of a hand-declared ``confusable_with``
53
+ clique. ``children`` is ``node id -> sorted child ids``.
54
+ """
55
+
56
+ specs: dict[str, Spec]
57
+ children: dict[str, list[str]]
58
+
59
+ def children_of(self, tech_id: str) -> list[str]:
60
+ return list(self.children.get(tech_id, []))
61
+
62
+ def is_family(self, tech_id: str) -> bool:
63
+ return bool(self.children.get(tech_id))
64
+
65
+ def parent_of(self, tech_id: str) -> str | None:
66
+ return self.specs[tech_id].parent
67
+
68
+ def siblings_of(self, tech_id: str) -> list[str]:
69
+ """Other children of this node's parent (empty for a root)."""
70
+ parent = self.specs[tech_id].parent
71
+ if parent is None:
72
+ return []
73
+ return [c for c in self.children.get(parent, []) if c != tech_id]
74
+
75
+ def ancestors_of(self, tech_id: str) -> list[str]:
76
+ """The parent chain from this node up to (and including) its root, nearest first."""
77
+ out: list[str] = []
78
+ seen: set[str] = set()
79
+ cur = self.specs[tech_id].parent
80
+ while cur is not None and cur not in seen:
81
+ out.append(cur)
82
+ seen.add(cur)
83
+ cur = self.specs[cur].parent if cur in self.specs else None
84
+ return out
85
+
86
+ def runnable_descendants_of(self, tech_id: str) -> list[str]:
87
+ """Every leaf / runnable family at or below this node (what descent may terminate on)."""
88
+ out: list[str] = []
89
+ stack = [tech_id]
90
+ seen: set[str] = set()
91
+ while stack:
92
+ node = stack.pop()
93
+ if node in seen:
94
+ continue
95
+ seen.add(node)
96
+ if self.specs[node].backend is not None:
97
+ out.append(node)
98
+ stack.extend(self.children.get(node, []))
99
+ return sorted(out)
100
+
101
+ def leaves(self) -> list[str]:
102
+ """Ids with no children — the concrete, compilable chemistries."""
103
+ return sorted(i for i in self.specs if not self.children.get(i))
104
+
105
+
106
+ def build_tree(specs: dict[str, Spec]) -> KbTree:
107
+ """Build and VALIDATE the KB forest; raise ``ValueError`` on a malformed tree.
108
+
109
+ Checks: every ``parent`` resolves; no cycles; ``node_kind`` agrees with having children; every
110
+ abstract family (a family with no backend) has at least one runnable descendant, so descent from it
111
+ can always terminate at something that compiles.
112
+ """
113
+ children: dict[str, list[str]] = {}
114
+ for tech_id, spec in specs.items():
115
+ parent = spec.parent
116
+ if parent is None:
117
+ continue
118
+ if parent == tech_id:
119
+ raise ValueError(f"{tech_id!r}: a node cannot be its own parent")
120
+ if parent not in specs:
121
+ raise ValueError(f"{tech_id!r}: parent {parent!r} is not a known spec")
122
+ children.setdefault(parent, []).append(tech_id)
123
+ for parent in children:
124
+ children[parent] = sorted(children[parent])
125
+
126
+ for tech_id in specs: # no parent cycles: every chain must terminate at a root
127
+ seen: set[str] = set()
128
+ cur: str | None = tech_id
129
+ while cur is not None:
130
+ if cur in seen:
131
+ raise ValueError(f"parent cycle through {tech_id!r}")
132
+ seen.add(cur)
133
+ cur = specs[cur].parent
134
+
135
+ tree = KbTree(specs=specs, children=children)
136
+ for tech_id, spec in specs.items():
137
+ has_children = bool(children.get(tech_id))
138
+ if has_children and spec.node_kind != "family":
139
+ raise ValueError(f"{tech_id!r} has children but is not declared node_kind: family")
140
+ if spec.node_kind == "family" and not has_children:
141
+ raise ValueError(f"{tech_id!r} is declared a family but has no children")
142
+ if spec.node_kind == "family" and spec.backend is None:
143
+ if not tree.runnable_descendants_of(tech_id):
144
+ raise ValueError(
145
+ f"abstract family {tech_id!r} has no runnable descendant to compile"
146
+ )
147
+ return tree
148
+
149
+
150
+ def load_tree() -> KbTree:
151
+ """Load every spec and build the validated KB tree."""
152
+ return build_tree(load_all_specs())
@@ -0,0 +1,112 @@
1
+ """``kb roundtrip`` — the self-test: spec -> synth FASTQ -> probe -> recover; assert == declared.
2
+
3
+ Generic over any spec: for every declared read it checks that the probe recovers the declared fixed
4
+ length, that barcode windows recur (low distinct-ratio), and that UMI windows are ~unique (high
5
+ distinct-ratio). Uses a temp directory; touches no real data.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import tempfile
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ from ..probe import probe_file
15
+ from ..probe.signals import window_distinct_ratio
16
+ from .anchor import resolve_windows
17
+ from .generate import generate_reads, write_fastq_gz
18
+ from .loader import load_spec
19
+ from .schema import Read
20
+
21
+
22
+ def _write_fastq_gz(path: Path, seqs: list[str]) -> None:
23
+ write_fastq_gz(path, seqs)
24
+
25
+
26
+ def _anchored_distinct_ratio(seqs: list[str], read: Read, element_name: str) -> float | None:
27
+ """``distinct/total`` of a FLOATING element, sliced at each read's phase-detected frame.
28
+
29
+ A fixed element has a constant window :func:`window_distinct_ratio` can slice; an anchored one does
30
+ not (the diversity insert staggers it per read), so recover its frame first. This is what makes the
31
+ round-trip actually verify anchoring: if the resolver could not recover the staggered cell-label
32
+ blocks, their slices would not recur and ``barcode_recurs`` would go red.
33
+ """
34
+ slices = [
35
+ s[a:b]
36
+ for s in seqs
37
+ if (w := resolve_windows(s, read)) is not None and element_name in w
38
+ for a, b in (w[element_name],)
39
+ if s[a:b]
40
+ ]
41
+ if not slices:
42
+ return None
43
+ return len(set(slices)) / len(slices)
44
+
45
+
46
+ def run_roundtrip(tech_id: str, *, n: int = 2000, seed: int = 0) -> dict[str, Any]:
47
+ """Round-trip one technology and return ``{tech, passed, checks:[...]}``."""
48
+ spec = load_spec(tech_id)
49
+ reads = generate_reads(spec, n=n, seed=seed)
50
+ checks: list[dict[str, Any]] = []
51
+
52
+ with tempfile.TemporaryDirectory() as td:
53
+ for read in spec.reads:
54
+ seqs = reads[read.id]
55
+ path = Path(td) / f"{read.id}.fastq.gz"
56
+ _write_fastq_gz(path, seqs)
57
+ obs = probe_file(path)
58
+
59
+ if read.min_len is not None and read.min_len == read.max_len:
60
+ checks.append(
61
+ {
62
+ "read": read.id,
63
+ "check": "length",
64
+ "ok": obs.read_length.mode == read.min_len,
65
+ "declared": read.min_len,
66
+ "recovered": obs.read_length.mode,
67
+ }
68
+ )
69
+ # an open-ended cDNA/gDNA read must probe back as variable-length (non-vacuous for the
70
+ # no-barcode bulk branch, whose only structural claim is "two variable cDNA reads").
71
+ has_open_cdna = any(
72
+ el.type in ("cdna", "gdna") and el.end is None for el in read.elements
73
+ )
74
+ if has_open_cdna and read.min_len != read.max_len:
75
+ checks.append(
76
+ {
77
+ "read": read.id,
78
+ "check": "cdna_variable",
79
+ "ok": obs.read_length.n_distinct > 1,
80
+ "n_distinct": obs.read_length.n_distinct,
81
+ }
82
+ )
83
+ for el in read.elements:
84
+ if el.start is not None and el.end is not None:
85
+ ratio = window_distinct_ratio(seqs, el.start, el.end)
86
+ elif el.anchor is not None:
87
+ # a floating element: recover its per-read frame rather than skipping it (pre-#43
88
+ # every anchored element was skipped, so the round-trip proved nothing about them).
89
+ ratio = _anchored_distinct_ratio(seqs, read, el.name)
90
+ else:
91
+ continue # a variable-length insert with no anchor (the VB itself): no window to
92
+ # check — its recovery is proven by the downstream anchored elements resolving.
93
+ if el.type == "barcode" and el.onlist:
94
+ checks.append(
95
+ {
96
+ "read": read.id,
97
+ "check": f"barcode_recurs:{el.name}",
98
+ "ok": ratio is not None and ratio < 0.5,
99
+ "ratio": ratio,
100
+ }
101
+ )
102
+ elif el.type == "umi":
103
+ checks.append(
104
+ {
105
+ "read": read.id,
106
+ "check": f"umi_unique:{el.name}",
107
+ "ok": ratio is not None and ratio > 0.7,
108
+ "ratio": ratio,
109
+ }
110
+ )
111
+
112
+ return {"tech": tech_id, "passed": all(c["ok"] for c in checks), "checks": checks}
seqforge/kb/schema.py ADDED
@@ -0,0 +1,422 @@
1
+ """The KB ``spec.yaml`` schema — machine-checkable, closed-vocabulary, self-validating.
2
+
3
+ One directory per technology: ``kb/specs/<tech>/{spec.yaml, README.md}``. ``spec.yaml`` declares the
4
+ read layout (element coordinates), onlist references, a detection ``signature`` (requires / supports /
5
+ excludes), a ``backend`` param template, and a ``confusable_with`` list. Every model forbids extra
6
+ keys, so a typo fails validation exactly where the DSL is executed. The signature test vocabulary is
7
+ *exactly* the scorer's evaluator set.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import re
13
+ from typing import Annotated, Literal
14
+
15
+ from pydantic import BaseModel, ConfigDict, Field, model_validator
16
+
17
+ ElementType = Literal[
18
+ "barcode", "umi", "cdna", "gdna", "linker", "poly_a", "poly_t", "fixed", "index", "diversity"
19
+ ]
20
+ # ``diversity`` = a variable-length insert of random bases at a read's 5' end (BD Rhapsody Enhanced's
21
+ # 0-3 bp diversity/phasing insert). It carries no barcode/UMI value — its ONLY effect is to STAGGER
22
+ # every downstream element by its per-read length, which is why it is modelled as a first-class element
23
+ # (``min_len``/``max_len``): the generator draws a per-read length for it and the anchored resolver
24
+ # recovers the frame the stagger created (see ``kb/anchor.py``). It is not a linker/fixed (no literal
25
+ # sequence) and not a counted feature, so ``compose`` skips it and it maps to a ``linker`` manifest role.
26
+ Mechanism = Literal["none", "onlist", "metadata", "alignment", "user"]
27
+ Decidable = Literal["reads", "onlist", "metadata", "alignment", "user"]
28
+ Orientation = Literal["forward", "revcomp", "either"]
29
+ SeqspecRegion = Literal[
30
+ "barcode",
31
+ "umi",
32
+ "cdna",
33
+ "gdna",
34
+ "index5",
35
+ "index7",
36
+ "linker",
37
+ "poly_A",
38
+ "poly_t",
39
+ "custom_primer",
40
+ ]
41
+
42
+ _ONLIST_TOKEN = re.compile(r"^\{onlist:([A-Za-z0-9._-]+)\}$")
43
+ _ANY_BRACE = re.compile(r"\{[^}]*\}")
44
+
45
+
46
+ class _Forbid(BaseModel):
47
+ """Base that forbids unknown keys, so the closed vocabulary is enforced where it is executed."""
48
+
49
+ model_config = ConfigDict(extra="forbid")
50
+
51
+
52
+ class Anchor(_Forbid):
53
+ """Locate a variable-length / floating element (e.g. inDrop's post-W1 barcode)."""
54
+
55
+ relative_to: Literal["read_start", "read_end", "element"] = "read_start"
56
+ ref_element: str | None = None
57
+ ref_side: Literal["start", "end"] = "end"
58
+ offset: int = 0
59
+ motif: str | None = None
60
+ max_mismatch: int = 0
61
+
62
+
63
+ class Element(_Forbid):
64
+ """One element of a read. 0-based half-open ``[start, end)``; ``end=None`` => open-ended (cDNA)."""
65
+
66
+ type: ElementType
67
+ name: str
68
+ start: int | None = None
69
+ end: int | None = None
70
+ min_len: int | None = None
71
+ max_len: int | None = None
72
+ anchor: Anchor | None = None
73
+ sequence: str | None = None
74
+ onlist: str | None = None
75
+ seqspec_region_type: SeqspecRegion
76
+
77
+ @model_validator(mode="after")
78
+ def _addressable(self) -> Element:
79
+ fixed = self.start is not None and self.end is not None
80
+ opened = self.start is not None and self.end is None
81
+ varlen = self.min_len is not None or self.max_len is not None
82
+ anchored = self.anchor is not None
83
+ if self.type in ("linker", "fixed") and self.sequence is None:
84
+ raise ValueError(f"element {self.name!r}: linker/fixed needs a literal `sequence`")
85
+ if self.type in ("cdna", "gdna"):
86
+ return self # open-ended is legal
87
+ if not (fixed or opened or anchored or varlen):
88
+ raise ValueError(f"element {self.name!r}: give [start,end), an anchor, or min/max_len")
89
+ return self
90
+
91
+
92
+ class Read(_Forbid):
93
+ """A read (== one FASTQ). ``id`` is a ROLE label (R1/R2/bc/cdna), never a filename claim."""
94
+
95
+ id: str
96
+ seqspec_read_id: str
97
+ file_hint: str | None = None
98
+ strand: Literal["pos", "neg"] = "pos"
99
+ min_len: int | None = None
100
+ max_len: int | None = None
101
+ elements: list[Element]
102
+
103
+
104
+ class OnlistRef(_Forbid):
105
+ """Alias -> pooch-registry name. URL/sha256/length/orientation live in the registry, never here."""
106
+
107
+ registry: str
108
+ role: Literal["cell_barcode", "sample_index", "feature", "atac_barcode"]
109
+ expected_orientation: Orientation = "forward"
110
+
111
+
112
+ # ---- signature tests: a CLOSED set == the scorer's evaluators ----
113
+ class _Seg(_Forbid):
114
+ """A test addressed to a segment by element name XOR (start, end)."""
115
+
116
+ read: str
117
+ element: str | None = None
118
+ start: int | None = None
119
+ end: int | None = None
120
+
121
+ @model_validator(mode="after")
122
+ def _one_address(self) -> _Seg:
123
+ by_name = self.element is not None
124
+ by_coord = self.start is not None and self.end is not None
125
+ if by_name == by_coord:
126
+ raise ValueError("address a segment by element name XOR (start, end)")
127
+ return self
128
+
129
+
130
+ class ReadCount(_Forbid):
131
+ test: Literal["read_count"]
132
+ roles: int # biological + barcode ROLE count, never raw file count
133
+
134
+
135
+ class SegmentLength(_Forbid):
136
+ test: Literal["segment_length"]
137
+ read: str
138
+ length: int
139
+ tolerance: int = 0
140
+ #: Over-length escape. A read whose mode is ``>= over_length_min`` PASSES regardless of ``length``
141
+ #: — an insert-bearing / over-sequenced barcode read (e.g. a 10x R1 sequenced to 150 bp: CB+UMI in
142
+ #: bp0-28, the rest junk). Keep it strictly above ``length`` so a canonical read is never
143
+ #: "over-length" and the exact-length gate still separates near-neighbours (v2's 26 vs v3's 28).
144
+ over_length_min: int | None = None
145
+
146
+
147
+ class HasSegment(_Seg):
148
+ test: Literal["has_segment"]
149
+ kind: Literal["constant", "random", "polyT", "polyA"]
150
+
151
+
152
+ class DistinctRatio(_Seg):
153
+ test: Literal["distinct_ratio"]
154
+ expect: Literal["low", "high"] # SUPPORTS-only; depth-dependent, never a gate
155
+
156
+
157
+ class OnlistHitRate(_Seg):
158
+ test: Literal["onlist_hit_rate"]
159
+ onlist: str
160
+ orientation: Orientation = "either"
161
+ min: float
162
+
163
+
164
+ class MotifPresent(_Forbid):
165
+ test: Literal["motif_present"]
166
+ read: str
167
+ motif: str
168
+ where: Literal["read_start", "read_end", "anywhere", "window"] = "anywhere"
169
+ search_start: int | None = None
170
+ search_end: int | None = None
171
+ max_mismatch: int = 1
172
+ min_rate: float = 0.5
173
+
174
+
175
+ class BaseComposition(_Seg):
176
+ test: Literal["base_composition"]
177
+ base: Literal["A", "C", "G", "T", "N"]
178
+ min_fraction: float
179
+
180
+
181
+ class HeaderIndex(_Forbid):
182
+ test: Literal["header_index"]
183
+ present: bool
184
+
185
+
186
+ Test = Annotated[
187
+ ReadCount
188
+ | SegmentLength
189
+ | HasSegment
190
+ | DistinctRatio
191
+ | OnlistHitRate
192
+ | MotifPresent
193
+ | BaseComposition
194
+ | HeaderIndex,
195
+ Field(discriminator="test"),
196
+ ]
197
+
198
+
199
+ class Support(_Forbid):
200
+ when: Test
201
+ weight: float = 1.0
202
+
203
+
204
+ class Signature(_Forbid):
205
+ requires: list[Test] # hard AND-gates (no distinct_ratio here — it's depth-dependent)
206
+ supports: list[Support] # additive evidence (onlist + distinct_ratio live here)
207
+ excludes: list[Test] # anti-gates: any pass => disqualify
208
+
209
+
210
+ KB_PARSE_KEYS: frozenset[str] = frozenset(
211
+ {
212
+ "soloType",
213
+ "soloCBstart",
214
+ "soloCBlen",
215
+ "soloUMIstart",
216
+ "soloUMIlen",
217
+ "soloCBwhitelist",
218
+ "soloCBposition",
219
+ "soloUMIposition",
220
+ "soloStrand",
221
+ "soloAdapterSequence",
222
+ "soloBarcodeReadLength",
223
+ }
224
+ )
225
+ """Every key the KB may declare. Each says how to **parse** reads, and each is decided by bytes.
226
+
227
+ The line is parse vs. count. What to COUNT — ``soloFeatures``, ``quantMode`` — is *intent*: it belongs
228
+ to the processing manifest, where a user may instruct it and a gate may check it. ``soloFeatures``
229
+ sat here because this is where the aligner's flags live, and it cost a measured **40.7 % of a nuclear
230
+ library**: 10x 3' v3.1 chemistry is byte-identical for cells and nuclei, so counting was never a
231
+ chemistry property at all.
232
+
233
+ The disjointness of this set from the instructable surface is what makes "a user instruction
234
+ contradicts the observed bytes" **structurally inexpressible** rather than merely deprioritized — the
235
+ user has no vocabulary in which to say it. That is the strongest form of that guarantee available, and it is why
236
+ moving a key across this line has to be an explicit, gated act.
237
+ """
238
+
239
+
240
+ class Backend(_Forbid):
241
+ """A data template mapping to a workflow module. Only ``{onlist:<alias>}`` interpolation is legal."""
242
+
243
+ module: str
244
+ params: dict[str, str | int | float | list[str]]
245
+
246
+ @model_validator(mode="after")
247
+ def _only_parse_keys(self) -> Backend:
248
+ """A count/reference/runtime knob may not be declared here.
249
+
250
+ ``params`` is a ``dict``, so ``extra="forbid"`` cannot reach inside it — hence an explicit
251
+ validator. It fires in ``load_spec``, in ``kb lint``, and in every test that loads a spec,
252
+ which is what makes the parse/count line a property of the DSL rather than a convention.
253
+ """
254
+ stray = sorted(set(self.params) - KB_PARSE_KEYS)
255
+ if stray:
256
+ raise ValueError(
257
+ f"backend.params declares non-parse key(s) {stray}: backend.params says how to "
258
+ f"PARSE reads (byte-decided); what to COUNT belongs in the processing manifest, "
259
+ f"where a user may instruct it. Known parse keys: {sorted(KB_PARSE_KEYS)}"
260
+ )
261
+ return self
262
+
263
+ def check_tokens(self, onlist_aliases: set[str]) -> None:
264
+ """Reject any interpolation token that is not a declared ``{onlist:<alias>}``."""
265
+ for value in self._strings():
266
+ for match in _ANY_BRACE.finditer(value):
267
+ token = _ONLIST_TOKEN.match(match.group(0))
268
+ if token is None:
269
+ raise ValueError(
270
+ f"illegal template expression {match.group(0)!r} "
271
+ "(only {onlist:<alias>} is allowed)"
272
+ )
273
+ if token.group(1) not in onlist_aliases:
274
+ raise ValueError(f"unknown onlist alias {token.group(1)!r}")
275
+
276
+ def _strings(self) -> list[str]:
277
+ out: list[str] = []
278
+ for value in self.params.values():
279
+ if isinstance(value, str):
280
+ out.append(value)
281
+ elif isinstance(value, list):
282
+ out.extend(v for v in value if isinstance(v, str))
283
+ return out
284
+
285
+
286
+ class Confusable(_Forbid):
287
+ id: str
288
+ relationship: Literal["processing_equivalent", "processing_divergent"]
289
+ distinguishable_by: list[Mechanism]
290
+ note: str = ""
291
+
292
+ @model_validator(mode="after")
293
+ def _shape(self) -> Confusable:
294
+ if self.relationship == "processing_divergent" and self.distinguishable_by == ["none"]:
295
+ raise ValueError("a processing_divergent pair cannot be distinguishable_by [none]")
296
+ return self
297
+
298
+
299
+ class Identity(_Forbid):
300
+ id: str
301
+ version: str
302
+ name: str
303
+ aliases: list[str] = Field(default_factory=list)
304
+ assay_ontology: list[str] = Field(default_factory=list)
305
+ modality: Literal["rna", "atac", "multi"] = "rna"
306
+
307
+
308
+ class Spec(_Forbid):
309
+ """A complete, self-validating technology specification (one node in the KB tree)."""
310
+
311
+ schema_version: int
312
+ identity: Identity
313
+ reads: list[Read]
314
+ onlists: dict[str, OnlistRef]
315
+ signature: Signature
316
+ #: The runnable STARsolo/STAR template. ``None`` on an ABSTRACT family node — a classifier that
317
+ #: narrows to its children but has no single recipe (``10x-3p-gex`` parses v2 and v3 differently).
318
+ #: A leaf, and a runnable family like ``bulk``, always declare one.
319
+ backend: Backend | None = None
320
+ #: The family id this node descends from; ``None`` at a root. Siblings (same parent) are
321
+ #: confusable-by-construction, decided by the parent's ``children_decided_by`` — which is why a
322
+ #: sibling clique of ``processing_divergent`` ``confusable_with`` edges collapses to one ``parent``.
323
+ parent: str | None = None
324
+ node_kind: Literal["family", "leaf"] = "leaf"
325
+ #: How a FAMILY node's children are told apart (``onlist`` / ``metadata`` / ``alignment``). Empty on
326
+ #: a leaf. Replaces the per-sibling ``distinguishable_by`` for the divergent-tie question.
327
+ children_decided_by: list[Mechanism] = Field(default_factory=list)
328
+ confusable_with: list[Confusable] = Field(default_factory=list)
329
+
330
+ def require_backend(self) -> Backend:
331
+ """The runnable backend, or a clear error if this is an abstract family node.
332
+
333
+ Only leaves and runnable families compile; descent always resolves to one of those, so every
334
+ compose/params/policy site is entitled to a backend. This turns the ``Backend | None`` into a
335
+ checked ``Backend`` at exactly those call sites.
336
+ """
337
+ if self.backend is None:
338
+ raise ValueError(
339
+ f"{self.identity.id!r} is an abstract family node with no runnable backend; "
340
+ "only leaves and runnable families compile"
341
+ )
342
+ return self.backend
343
+
344
+ @property
345
+ def decidable_by(self) -> list[Decidable]:
346
+ """How this technology can be separated from the ones it is confusable with. **Derived.**
347
+
348
+ This was a hand-typed field on every spec, and two of them carried the comment "CI-computed
349
+ union over the divergent confusables". No CI computed it. Nothing read it either — `escalate`
350
+ builds a Question's ``decidable_by`` from ``confusable_with[].distinguishable_by``, inline,
351
+ which is precisely the union the comment described. So the field was a claim about behaviour
352
+ that caused no behaviour: a comment with a list's syntax, free to drift from the thing it
353
+ claimed to summarize, with nothing to notice.
354
+
355
+ That is the exact shape of `RegistryEntry.fetchable` before it was derived, and of
356
+ `required_config` before that. Deriving it is the only fix that stays fixed.
357
+
358
+ ``processing_equivalent`` twins are excluded on purpose: §12 says two entries with identical
359
+ ``backend.params`` are declared equivalent and recorded together, so there is nothing to
360
+ decide between them and no mechanism that could.
361
+ """
362
+ out: set[Decidable] = set()
363
+ for c in self.confusable_with:
364
+ if c.relationship == "processing_divergent":
365
+ out.update(m for m in c.distinguishable_by if m != "none")
366
+ return sorted(out) # type: ignore[arg-type]
367
+
368
+ @model_validator(mode="after")
369
+ def _node_shape(self) -> Spec:
370
+ """A leaf must be runnable; ``children_decided_by`` is a family-only knob."""
371
+ if self.node_kind == "leaf":
372
+ if self.backend is None:
373
+ raise ValueError(
374
+ f"{self.identity.id!r}: a leaf node must declare a runnable backend"
375
+ )
376
+ if self.children_decided_by:
377
+ raise ValueError(
378
+ f"{self.identity.id!r}: children_decided_by is only meaningful on a family node"
379
+ )
380
+ return self
381
+
382
+ @model_validator(mode="after")
383
+ def _cross_refs(self) -> Spec:
384
+ aliases = set(self.onlists)
385
+ read_ids = {r.id for r in self.reads}
386
+ elements_by_read = {r.id: {e.name for e in r.elements} for r in self.reads}
387
+
388
+ # every onlist alias referenced by an element must be declared
389
+ for read in self.reads:
390
+ for el in read.elements:
391
+ if el.onlist and el.onlist not in aliases:
392
+ raise ValueError(f"element {el.name!r}: unknown onlist {el.onlist!r}")
393
+ if el.anchor and el.anchor.ref_element:
394
+ if el.anchor.ref_element not in elements_by_read[read.id]:
395
+ raise ValueError(
396
+ f"element {el.name!r}: anchor ref_element "
397
+ f"{el.anchor.ref_element!r} not in read {read.id!r}"
398
+ )
399
+
400
+ # every signature test must reference a declared read (and element/onlist)
401
+ tests: list[Test] = [
402
+ *self.signature.requires,
403
+ *self.signature.excludes,
404
+ *(s.when for s in self.signature.supports),
405
+ ]
406
+ for t in tests:
407
+ read = getattr(t, "read", None)
408
+ if read is not None and read not in read_ids:
409
+ raise ValueError(f"signature test references unknown read {read!r}")
410
+ element = getattr(t, "element", None)
411
+ if element is not None and read is not None:
412
+ if element not in elements_by_read.get(read, set()):
413
+ raise ValueError(
414
+ f"signature test references unknown element {element!r} in read {read!r}"
415
+ )
416
+ onlist = getattr(t, "onlist", None)
417
+ if onlist is not None and onlist not in aliases:
418
+ raise ValueError(f"signature test references unknown onlist {onlist!r}")
419
+
420
+ if self.backend is not None:
421
+ self.backend.check_tokens(aliases)
422
+ return self