seqforge 2026.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqforge/__init__.py +16 -0
- seqforge/cli/__init__.py +38 -0
- seqforge/cli/__main__.py +8 -0
- seqforge/cli/_common.py +105 -0
- seqforge/cli/compose.py +119 -0
- seqforge/cli/eval.py +103 -0
- seqforge/cli/harvest.py +417 -0
- seqforge/cli/hook.py +247 -0
- seqforge/cli/io.py +502 -0
- seqforge/cli/kb.py +348 -0
- seqforge/cli/manifest.py +536 -0
- seqforge/cli/probe.py +43 -0
- seqforge/cli/processing.py +192 -0
- seqforge/cli/project.py +52 -0
- seqforge/cli/resolve.py +55 -0
- seqforge/cli/root.py +66 -0
- seqforge/cli/run.py +463 -0
- seqforge/cli/schema.py +41 -0
- seqforge/compose/__init__.py +28 -0
- seqforge/compose/core.py +515 -0
- seqforge/compose/gates.py +113 -0
- seqforge/compose/params.py +447 -0
- seqforge/e2e.py +1926 -0
- seqforge/evals/__init__.py +78 -0
- seqforge/evals/case.py +382 -0
- seqforge/evals/grade.py +300 -0
- seqforge/evals/run.py +420 -0
- seqforge/harvest/__init__.py +121 -0
- seqforge/harvest/extract.py +319 -0
- seqforge/harvest/fields.py +212 -0
- seqforge/harvest/normalize.py +537 -0
- seqforge/harvest/prep.py +41 -0
- seqforge/harvest/providers.py +321 -0
- seqforge/harvest/verify.py +251 -0
- seqforge/hooks/__init__.py +33 -0
- seqforge/hooks/guards.py +214 -0
- seqforge/io/__init__.py +61 -0
- seqforge/io/archive.py +450 -0
- seqforge/io/attributes.py +190 -0
- seqforge/io/biosample/attributes.json +6341 -0
- seqforge/io/efo/labels.json +55 -0
- seqforge/io/efo.py +138 -0
- seqforge/io/onlist.py +661 -0
- seqforge/io/onlists/3M-february-2018.codes.gz +0 -0
- seqforge/io/onlists/737K-arc-v1.codes.gz +0 -0
- seqforge/io/onlists/737K-august-2016.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3.codes.gz +0 -0
- seqforge/io/onlists/index.json +74 -0
- seqforge/io/remote.py +659 -0
- seqforge/io/taxonomy.py +194 -0
- seqforge/kb/__init__.py +62 -0
- seqforge/kb/anchor.py +169 -0
- seqforge/kb/generate.py +147 -0
- seqforge/kb/loader.py +152 -0
- seqforge/kb/roundtrip.py +112 -0
- seqforge/kb/schema.py +422 -0
- seqforge/kb/specs/10x-3p-gex/spec.yaml +62 -0
- seqforge/kb/specs/10x-3p-gex-v2/README.md +41 -0
- seqforge/kb/specs/10x-3p-gex-v2/spec.yaml +83 -0
- seqforge/kb/specs/10x-3p-gex-v3/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3/spec.yaml +118 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/spec.yaml +124 -0
- seqforge/kb/specs/bd-rhapsody-wta/README.md +103 -0
- seqforge/kb/specs/bd-rhapsody-wta/spec.yaml +130 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced/spec.yaml +99 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v1/spec.yaml +93 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v2/spec.yaml +81 -0
- seqforge/kb/specs/bulk-rnaseq-pe/README.md +35 -0
- seqforge/kb/specs/bulk-rnaseq-pe/spec.yaml +97 -0
- seqforge/kb/specs/splitseq/README.md +51 -0
- seqforge/kb/specs/splitseq/spec.yaml +157 -0
- seqforge/manifest/__init__.py +61 -0
- seqforge/manifest/fill.py +531 -0
- seqforge/manifest/hash.py +77 -0
- seqforge/manifest/instruct.py +114 -0
- seqforge/manifest/policy.py +409 -0
- seqforge/manifest/validate.py +274 -0
- seqforge/models/__init__.py +268 -0
- seqforge/models/assertion.py +68 -0
- seqforge/models/base.py +100 -0
- seqforge/models/blocker.py +71 -0
- seqforge/models/conflict.py +47 -0
- seqforge/models/dataset.py +320 -0
- seqforge/models/evidenced.py +54 -0
- seqforge/models/observation.py +157 -0
- seqforge/models/processing.py +231 -0
- seqforge/models/records.py +145 -0
- seqforge/models/resolve.py +216 -0
- seqforge/probe/__init__.py +46 -0
- seqforge/probe/core.py +232 -0
- seqforge/probe/signals.py +250 -0
- seqforge/probe/streaming.py +118 -0
- seqforge/project.py +177 -0
- seqforge/py.typed +0 -0
- seqforge/resolve/__init__.py +98 -0
- seqforge/resolve/assign.py +204 -0
- seqforge/resolve/cache.py +119 -0
- seqforge/resolve/confuse.py +215 -0
- seqforge/resolve/engine.py +646 -0
- seqforge/resolve/escalate.py +668 -0
- seqforge/resolve/evaluators.py +306 -0
- seqforge/resolve/geometry.py +89 -0
- seqforge/resolve/group.py +85 -0
- seqforge/resolve/records.py +550 -0
- seqforge/resolve/scoring.py +373 -0
- seqforge/resolve/window.py +206 -0
- seqforge/workflows/__init__.py +234 -0
- seqforge/workflows/cram.py +117 -0
- seqforge/workflows/h5ad.py +368 -0
- seqforge/workflows/map/star.smk +101 -0
- seqforge/workflows/map/starsolo.smk +360 -0
- seqforge/workflows/qc.py +157 -0
- seqforge/workspace.py +125 -0
- seqforge-2026.7.1.dist-info/METADATA +125 -0
- seqforge-2026.7.1.dist-info/RECORD +124 -0
- seqforge-2026.7.1.dist-info/WHEEL +4 -0
- seqforge-2026.7.1.dist-info/entry_points.txt +2 -0
- seqforge-2026.7.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Project verified :class:`Assertion`s onto the instructable surface. Deterministic; no LLM.
|
|
2
|
+
|
|
3
|
+
A user may tell seqforge what to do, in prose, in a document they hand us:
|
|
4
|
+
|
|
5
|
+
seqforge harvest extract PAPER.pdf --instruction notes.md
|
|
6
|
+
# ^ reference ^ instruction
|
|
7
|
+
|
|
8
|
+
**The document's ROLE decides the basis, not the model's reading of the sentence.** An instruction is
|
|
9
|
+
imperative ("align this in GeneFull mode") and a claim is declarative ("we used GeneFull"), and it is
|
|
10
|
+
tempting to ask the LLM to tell them apart. Don't: that classification has no quote to check it
|
|
11
|
+
against, so it lands in exactly the class :func:`~seqforge.harvest.verify.entails` is provably blind
|
|
12
|
+
to — and this model's one known failure mode is *field misassignment*, a real quote correctly copied
|
|
13
|
+
onto the wrong field. Role is decided by **which flag the document arrived under**, which code owns
|
|
14
|
+
and a shell history records.
|
|
15
|
+
|
|
16
|
+
Role also subsumes mood by fiat, and fiat is the right tool here precisely because it is checkable: if
|
|
17
|
+
you wrote a descriptive sentence inside a file you handed us *for this purpose*, we honor it. You
|
|
18
|
+
authored the file for us; every sentence in it is addressed to us.
|
|
19
|
+
|
|
20
|
+
Note what cannot reach this module: a ``processing.*`` draft from a **reference** document.
|
|
21
|
+
:mod:`seqforge.harvest.fields` refuses it at verify time, so a downloaded methods PDF can never steer
|
|
22
|
+
the pipeline. That is a deliberate narrowing of "instructions may live among the unstructured
|
|
23
|
+
metadata", and it costs nothing: with the default counting everything, a paper saying "we used
|
|
24
|
+
GeneFull" describes a subset of what we already compute.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from collections.abc import Sequence
|
|
30
|
+
from dataclasses import dataclass
|
|
31
|
+
|
|
32
|
+
from ..models.assertion import Assertion
|
|
33
|
+
from ..models.base import Basis
|
|
34
|
+
from ..models.conflict import Conflict, ConflictPosition
|
|
35
|
+
|
|
36
|
+
#: The fields an *instruction document* may set. A strict subset of the instructable surface: a CLI
|
|
37
|
+
#: flag may set more (threads, annotation_name), because a flag is typed by a human at run time while
|
|
38
|
+
#: this is a model's reading of prose.
|
|
39
|
+
#:
|
|
40
|
+
#: `annotation_name` is deliberately absent — it is a liulab-genome registry name (`WS298`), a
|
|
41
|
+
#: vocabulary no paper writes in, so asking for it would only invite a guess.
|
|
42
|
+
INSTRUCTABLE_FIELDS: frozenset[str] = frozenset(
|
|
43
|
+
{
|
|
44
|
+
"processing.quantification",
|
|
45
|
+
"processing.genome.assembly",
|
|
46
|
+
}
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class Instruction:
|
|
52
|
+
"""One processing directive: span-verified, carrying the basis its document's ROLE assigned."""
|
|
53
|
+
|
|
54
|
+
field: str
|
|
55
|
+
value: str
|
|
56
|
+
basis: Basis
|
|
57
|
+
evidence: list[str]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def instructions_from_assertions(
|
|
61
|
+
assertions: Sequence[Assertion], *, instruction_docs: frozenset[str] = frozenset()
|
|
62
|
+
) -> tuple[list[Instruction], list[Conflict]]:
|
|
63
|
+
"""Project verified assertions onto the instructable surface.
|
|
64
|
+
|
|
65
|
+
``instruction_docs`` is the set of ``doc_sha256`` handed to us under ``--instruction``; code knows
|
|
66
|
+
it because code chose it. Anything sourced elsewhere is dropped rather than downgraded — a
|
|
67
|
+
reference document has no business setting intent at all.
|
|
68
|
+
|
|
69
|
+
Two instructions disagreeing **at the same precedence** on one field is a :class:`Conflict`
|
|
70
|
+
(exit 4): there is no tiebreak, and a disagreement is surfaced for intent exactly as it is for
|
|
71
|
+
truth. Note this
|
|
72
|
+
is a *same-basis* disagreement; an instruction differing from a policy default is not a conflict at
|
|
73
|
+
all — that is what an instruction IS.
|
|
74
|
+
"""
|
|
75
|
+
out: list[Instruction] = []
|
|
76
|
+
for a in assertions:
|
|
77
|
+
if a.field not in INSTRUCTABLE_FIELDS:
|
|
78
|
+
continue # experiment.*/library.* travel a different path; off-surface fields are dropped
|
|
79
|
+
if not (a.span_verified and a.entailment_ok):
|
|
80
|
+
continue # belt and braces: verify already refuses to emit these
|
|
81
|
+
if a.span.doc_sha256 not in instruction_docs:
|
|
82
|
+
continue # a reference doc may not steer the pipeline (see the module docstring)
|
|
83
|
+
out.append(
|
|
84
|
+
Instruction(field=a.field, value=a.value, basis="user_confirmed", evidence=[a.id])
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
conflicts: list[Conflict] = []
|
|
88
|
+
by_field: dict[str, list[Instruction]] = {}
|
|
89
|
+
for ins in out:
|
|
90
|
+
by_field.setdefault(ins.field, []).append(ins)
|
|
91
|
+
for field, group in sorted(by_field.items()):
|
|
92
|
+
values = {i.value for i in group}
|
|
93
|
+
if len(values) > 1:
|
|
94
|
+
conflicts.append(
|
|
95
|
+
Conflict(
|
|
96
|
+
id=f"conflict-instruction-{field.replace('.', '-')}",
|
|
97
|
+
field=field,
|
|
98
|
+
positions=[
|
|
99
|
+
ConflictPosition(
|
|
100
|
+
value=i.value, basis=i.basis, evidence=list(i.evidence), confidence=1.0
|
|
101
|
+
)
|
|
102
|
+
for i in group
|
|
103
|
+
],
|
|
104
|
+
kind="asserted_vs_asserted",
|
|
105
|
+
# the first real consumer of Decidable's long-unused "user" member: only the
|
|
106
|
+
# person who wrote both sentences can say which one they meant.
|
|
107
|
+
decidable_by=["user"],
|
|
108
|
+
status="open",
|
|
109
|
+
)
|
|
110
|
+
)
|
|
111
|
+
return out, conflicts
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
__all__ = ["Instruction", "INSTRUCTABLE_FIELDS", "instructions_from_assertions"]
|
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
"""Processing policy — the default a user gets when they instruct nothing.
|
|
2
|
+
|
|
3
|
+
seqforge picks the best default pipeline option; a user instruction overrides it. This module is the
|
|
4
|
+
"picks" half: small, explicit, and derived. The aligner and runtime env follow from the KB's backend
|
|
5
|
+
module, never from a guess, and the runtime env is a **literal** ``liulab-runtime`` name —
|
|
6
|
+
there is no profile-indirection layer to invent here.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Sequence
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
|
|
14
|
+
from ..harvest.prep import normalize_prep_type as _normalize_prep_type
|
|
15
|
+
from ..kb.schema import Spec
|
|
16
|
+
from ..models.assertion import Assertion
|
|
17
|
+
from ..models.base import Basis
|
|
18
|
+
from ..models.blocker import BlockerSubject, ValidationWarning
|
|
19
|
+
from ..models.dataset import DatasetManifest
|
|
20
|
+
from ..models.evidenced import EvidencedBool, EvidencedStr
|
|
21
|
+
from ..models.processing import (
|
|
22
|
+
BulkQuant,
|
|
23
|
+
EvidencedGenome,
|
|
24
|
+
EvidencedQuantification,
|
|
25
|
+
EvidencedRuntimeEnv,
|
|
26
|
+
GenomeRef,
|
|
27
|
+
ProcessingSection,
|
|
28
|
+
Quantification,
|
|
29
|
+
ResourceHints,
|
|
30
|
+
RuntimeEnv,
|
|
31
|
+
SoloFeature,
|
|
32
|
+
SoloQuant,
|
|
33
|
+
)
|
|
34
|
+
from ..workflows import get_module
|
|
35
|
+
from .instruct import Instruction
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class PolicyError(RuntimeError):
|
|
39
|
+
"""Intent cannot be resolved — a required choice has no safe default (refuse, don't guess)."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
#: KB backend module -> the aligner name recorded in `processing.aligner`.
|
|
43
|
+
_ALIGNER_FOR_MODULE = {"map/starsolo": "starsolo", "map/star": "star"}
|
|
44
|
+
|
|
45
|
+
DEFAULT_SOLO_FEATURES: tuple[SoloFeature, ...] = (
|
|
46
|
+
"Gene",
|
|
47
|
+
"GeneFull",
|
|
48
|
+
"GeneFull_ExonOverIntron",
|
|
49
|
+
"GeneFull_Ex50pAS",
|
|
50
|
+
"Velocyto",
|
|
51
|
+
)
|
|
52
|
+
"""Count everything; do not ask which.
|
|
53
|
+
|
|
54
|
+
One alignment, five counting rules, one pass. Download and alignment dominate the cost by orders of
|
|
55
|
+
magnitude, and count matrices are small — so we emit every answer and let the consumer choose. That
|
|
56
|
+
**dissolves** the cells-vs-nuclei question rather than answering it, which is the sibling of the §12
|
|
57
|
+
benign rule: §12 says never escalate an ambiguity that cannot change the output; this says never
|
|
58
|
+
escalate one whose every answer you can afford to emit.
|
|
59
|
+
|
|
60
|
+
We measured the alternative. ``--soloFeatures Gene`` silently discards **40.7 %** of a nuclear library
|
|
61
|
+
(`kb e2e-introns` on ce11: Gene=1186 exonic-only vs GeneFull=1940). STARsolo exits 0 and the matrix
|
|
62
|
+
merely looks thin — the same failure shape as a strand inversion.
|
|
63
|
+
|
|
64
|
+
**Exactly scRecounter's five, in scRecounter's order, and deliberately no SJ.** Reasons, ranked:
|
|
65
|
+
|
|
66
|
+
1. ``Gene`` first, so the primary matrix matches the common whole-cell expectation, while
|
|
67
|
+
``GeneFull`` sits right there for the nuclear case. Order only names the primary; nothing is
|
|
68
|
+
dropped.
|
|
69
|
+
2. It satisfies STARsolo's "Velocyto requires Gene" by construction.
|
|
70
|
+
3. **SJ is out, and the reason belongs in code rather than inherited silently.** A splice-junction
|
|
71
|
+
matrix has a *different feature axis* — it is not a drop-in alternative count of the same thing,
|
|
72
|
+
nothing downstream consumes it, and it costs disk for no training signal today. One entry away if
|
|
73
|
+
that changes. scRecounter's five is a *precedent*, not a derivation; adopting it wholesale without
|
|
74
|
+
saying this would import someone else's unstated scope decision.
|
|
75
|
+
4. Following a known-good precedent that runs at scale on real public data beats our own reasoning
|
|
76
|
+
here, and it is citable: ArcInstitute/scRecounter, workflows/star_full.nf.
|
|
77
|
+
|
|
78
|
+
**The cost is measured (2026-07-15), and it has a KNEE.** `kb e2e-cost` on hg38 + GENCODE v50 + 10x
|
|
79
|
+
3' v3, all five features, 16 threads, 5 000 cells, reads simulated from real hg38 sequence with
|
|
80
|
+
barcodes drawn from the real 6 794 880-entry whitelist:
|
|
81
|
+
|
|
82
|
+
=========== ========== ==================
|
|
83
|
+
reads peak RSS delta
|
|
84
|
+
=========== ========== ==================
|
|
85
|
+
10 000 000 34.570 GB —
|
|
86
|
+
40 000 000 34.600 GB +30 MB
|
|
87
|
+
100 000 000 34.659 GB +59 MB
|
|
88
|
+
250 000 000 44.055 GB **+9.4 GB**
|
|
89
|
+
=========== ========== ==================
|
|
90
|
+
|
|
91
|
+
Read that bottom row before quoting any of the others. Up to ~100 M reads the number is the *genome
|
|
92
|
+
index* (~30 GB resident before a read is parsed) and depth is irrelevant — 10x the reads cost 89 MB.
|
|
93
|
+
Then it stops being flat. Peak RSS is really ``max(alignment_peak, solo_peak(reads))``: the alignment
|
|
94
|
+
phase is index-bound and flat, the **Solo counting phase grows with depth**, and it overtakes the
|
|
95
|
+
index somewhere between 100 M and 250 M. Watching the 250 M run live shows it directly — RSS sits at
|
|
96
|
+
~17 GB early in Solo, climbs past the 34.6 GB alignment peak, and tops out at 44 GB while writing
|
|
97
|
+
five matrices.
|
|
98
|
+
|
|
99
|
+
**Provisioning, honestly:**
|
|
100
|
+
|
|
101
|
+
- ≤ 100 M reads: ~35 GB. Solid — three points.
|
|
102
|
+
- 250 M reads: ~44 GB. One point.
|
|
103
|
+
- **> 250 M: UNMEASURED.** If ``solo_peak`` is ~linear the crossover implies ~180-190 bytes/read,
|
|
104
|
+
putting 500 M near ~88 GB — but that is arithmetic on a single point, not a measurement, and this
|
|
105
|
+
docstring has now been wrong once for exactly that reason. Give a deep human library **128 GB**
|
|
106
|
+
until somebody measures 500 M.
|
|
107
|
+
|
|
108
|
+
**How this docstring was wrong for three hours, because the lesson outlives the number.** The first
|
|
109
|
+
three points were fitted and reported ``max_residual_gb: 0.0`` — a perfect line — projecting 34.8 GB
|
|
110
|
+
at 250 M. Reality: 44.055 GB, a **9.3 GB / 27 % under-estimate from a fit that reported zero error**.
|
|
111
|
+
Earlier the same day ``_fit_line`` was fixed to refuse *two*-point fits, on the grounds that a line
|
|
112
|
+
through two points fits exactly and so its residual cannot falsify anything. That was right and it was
|
|
113
|
+
not enough: three *collinear* points inside one regime cannot falsify either. They were genuinely
|
|
114
|
+
linear; the residual was genuinely 0.0; the model was genuinely wrong. **A residual can only falsify
|
|
115
|
+
within the range sampled — it can never report that the range itself was too narrow.** The four-point
|
|
116
|
+
fit does say so (``max_residual_gb: 2.312``), one point too late to have helped.
|
|
117
|
+
|
|
118
|
+
The ``--outSAMtype`` gap was measured on the same principle: the sweep ran ``None``, the shipped
|
|
119
|
+
module runs ``BAM Unsorted``, and one variable changed gives **34.600 -> 35.345 GB (+745 MB) and
|
|
120
|
+
+19 % wall-clock** at 40 M. Measured at one depth, and after the knee, "one depth" is a warning rather
|
|
121
|
+
than a footnote.
|
|
122
|
+
|
|
123
|
+
Read every number with its configuration or not at all: peak RSS includes STAR's per-thread buffers,
|
|
124
|
+
so these are peaks **at 16 threads**. That is why ``kb e2e-fit`` refuses to merge runs differing in
|
|
125
|
+
threads, cells, assembly or outSAMtype.
|
|
126
|
+
|
|
127
|
+
Reproducibility is not assumed: the 40 M point was re-measured on a different node, through a
|
|
128
|
+
different code path (32 sharded FASTQs vs one file), on *different reads*, and landed on **34.600 GB**
|
|
129
|
+
again — identical to three decimals.
|
|
130
|
+
|
|
131
|
+
**The instrument that produced these had a floor, found 2026-07-15 and fixed.** ``wait4``'s
|
|
132
|
+
``ru_maxrss`` reports ``max(parent_rss_at_fork, child_peak)`` on Linux — a child's address space
|
|
133
|
+
begins as a copy of its parent's, and ``exec`` never lowers the high-water mark. Measured: beside an
|
|
134
|
+
879 MB parent, a child allocating 1 MB reported ``879260 KiB``, the parent's RSS to the byte. **Every
|
|
135
|
+
number in this table stands**, because the floor was the harness (~1 GB at the very worst) and these
|
|
136
|
+
readings are 34–44 GB — but that is an argument, not a check, so ``kb e2e-cost`` now records
|
|
137
|
+
``harness_peak_rss_kib`` beside every reading and takes the peak from the child's own ``/proc``
|
|
138
|
+
``VmHWM``, which ``exec`` resets. The lesson is the same one this docstring already carries twice: an
|
|
139
|
+
instrument that cannot be wrong in a way you would notice has not been checked.
|
|
140
|
+
|
|
141
|
+
**None of this changes the default.** Velocyto stays: the knee is a property of counting 250 M reads
|
|
142
|
+
at all, the marginal cost of the fifth rule over the first is not what moves this number, and the
|
|
143
|
+
pre-registered kill rule (">2x wall-clock or over the mem_gb hint => drop to four") is about the
|
|
144
|
+
*feature set*, not the depth. What changed is the memory request, which is what the measurement was
|
|
145
|
+
for. If it ever does prove pathological the default drops to four and ``--quantify`` restores it: an
|
|
146
|
+
expensive default is not a trap precisely *because* the processing manifest exists to override it.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass(frozen=True)
|
|
151
|
+
class ProcessingOverrides:
|
|
152
|
+
"""CLI-typed overrides — the top of the precedence ladder.
|
|
153
|
+
|
|
154
|
+
A flag outranks an instruction document because it is more specific and it is later in time: both
|
|
155
|
+
are the user talking to seqforge, one just talks now. It also outranks it in *trust*, which is why
|
|
156
|
+
it may set fields an instruction document may not (``threads``, ``annotation_name``): a flag is
|
|
157
|
+
typed by a human, a document is read by a model.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
assembly: str | None = None
|
|
161
|
+
annotation_name: str | None = None
|
|
162
|
+
features: tuple[SoloFeature, ...] | None = None # --quantify: EXACT replacement, not a union
|
|
163
|
+
threads: int | None = None
|
|
164
|
+
environment: RuntimeEnv | None = None
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@dataclass(frozen=True)
|
|
168
|
+
class ProcessingDefaults:
|
|
169
|
+
"""Policy-derived processing intent for one chemistry."""
|
|
170
|
+
|
|
171
|
+
module: str
|
|
172
|
+
aligner: str
|
|
173
|
+
quantification: Quantification
|
|
174
|
+
environment: RuntimeEnv
|
|
175
|
+
variant_calling: bool
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def processing_defaults(spec: Spec) -> ProcessingDefaults:
|
|
179
|
+
"""Derive the processing section's policy defaults from the identified chemistry's backend."""
|
|
180
|
+
module = spec.require_backend().module
|
|
181
|
+
aligner = _ALIGNER_FOR_MODULE.get(module, module.rsplit("/", 1)[-1])
|
|
182
|
+
# Asked of the MODULE, which is the only thing that knows what software it needs. This was a
|
|
183
|
+
# hardcoded `"align-rna"` sitting beside a module that also declared `align-rna` — two owners of
|
|
184
|
+
# one fact, harmless only because no rule read the env. The moment `starsolo_count` grew a
|
|
185
|
+
# `container:`, that pair could disagree into a container with no STAR in it. An ATAC module
|
|
186
|
+
# declaring `align-dna` is now simply right, instead of being overridden here by a comment
|
|
187
|
+
# promising someone would remember.
|
|
188
|
+
environment: RuntimeEnv = get_module(module).env
|
|
189
|
+
# Counting is MODULE-scoped: soloFeatures is meaningless to plain STAR, and quantMode is
|
|
190
|
+
# meaningless to STARsolo. A processing manifest that carried one shape unconditionally would be
|
|
191
|
+
# a type error the moment it met the other module.
|
|
192
|
+
#
|
|
193
|
+
# Keyed on the block the module READS, not on its name. `if module == "map/starsolo"` was the
|
|
194
|
+
# same silent fall-through as `param_block_key`'s and `_read_files_in`'s before it: any third
|
|
195
|
+
# module quietly gets `quantMode=GeneCounts`, which is a real and wrong instruction to an aligner
|
|
196
|
+
# that may not take it. `param_block` refuses a module whose contract is neither solo nor bulk,
|
|
197
|
+
# so the `else` here can only be bulk — by construction rather than by hope.
|
|
198
|
+
quantification: Quantification = (
|
|
199
|
+
SoloQuant(features=list(DEFAULT_SOLO_FEATURES))
|
|
200
|
+
if get_module(module).param_block == "solo"
|
|
201
|
+
else BulkQuant(mode="GeneCounts")
|
|
202
|
+
)
|
|
203
|
+
return ProcessingDefaults(
|
|
204
|
+
module=module,
|
|
205
|
+
aligner=aligner,
|
|
206
|
+
quantification=quantification,
|
|
207
|
+
environment=environment,
|
|
208
|
+
variant_calling=False,
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# `_normalize_prep_type` and its cells-vs-nuclei vocabulary moved to `harvest.prep` (harvest reads the
|
|
213
|
+
# prose; this module consumes the normalized result), imported at the top under the same private name.
|
|
214
|
+
# `harvest.verify` now shares that exact vocabulary to entail a `library.prep_type` quote, instead of
|
|
215
|
+
# mirroring it — one place to teach a new synonym, not two that can silently drift apart.
|
|
216
|
+
def prep_type_from_assertions(assertions: Sequence[Assertion]) -> str | None:
|
|
217
|
+
"""The cells-vs-nuclei prep, normalized from a span-verified ``library.prep_type`` claim.
|
|
218
|
+
|
|
219
|
+
``None`` if the paper does not say, or if two verified claims disagree — never a guess between
|
|
220
|
+
them. The model FOUND the biology ("single nuclei") with a quote that greps back; this reads that
|
|
221
|
+
record and normalizes its wording. It names no feature: the biology -> feature mapping is
|
|
222
|
+
:func:`resolve_features`'s and code's alone, which is why sourcing this field from prose is safe.
|
|
223
|
+
"""
|
|
224
|
+
values: set[str] = set()
|
|
225
|
+
for a in assertions:
|
|
226
|
+
if a.field != "library.prep_type" or not (a.span_verified and a.entailment_ok):
|
|
227
|
+
continue
|
|
228
|
+
norm = _normalize_prep_type(a.value)
|
|
229
|
+
if norm is not None:
|
|
230
|
+
values.add(norm)
|
|
231
|
+
return next(iter(values)) if len(values) == 1 else None
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def resolve_features(
|
|
235
|
+
*,
|
|
236
|
+
instructions: Sequence[Instruction] = (),
|
|
237
|
+
override: tuple[SoloFeature, ...] | None = None,
|
|
238
|
+
prep_type: str | None = None,
|
|
239
|
+
) -> tuple[list[SoloFeature], Basis, list[str], list[ValidationWarning]]:
|
|
240
|
+
"""Fold policy + instructions + a flag into ONE ordered feature list, with its provenance.
|
|
241
|
+
|
|
242
|
+
**Prose promotes; it never narrows.** "This dataset should be aligned in GeneFull mode" is
|
|
243
|
+
ambiguous: *instead of* Gene, or *make sure* GeneFull is computed? We take the second — it is the
|
|
244
|
+
charitable reading, it is the cheap one, and it is consistent with counting everything by default.
|
|
245
|
+
So an instructed feature is UNIONed with the default and promoted to the front, where it
|
|
246
|
+
becomes primary. Nothing is dropped.
|
|
247
|
+
|
|
248
|
+
That is also the safety argument for letting a model source this field at all: because the default
|
|
249
|
+
computes everything, a hallucinated instruction can only mislabel which matrix is primary — it
|
|
250
|
+
cannot destroy signal. The blast radius of the one failure span-verification provably cannot catch
|
|
251
|
+
is a wrong label on a matrix we still computed.
|
|
252
|
+
|
|
253
|
+
**A flag replaces exactly.** The user typed the whole list; they mean it. Narrowing is the only
|
|
254
|
+
irreversible act available here, so it warns rather than passing silently.
|
|
255
|
+
|
|
256
|
+
**A single-nucleus prep only REORDERS.** With no flag and no instruction, a span-verified
|
|
257
|
+
``prep_type`` of ``single-nucleus`` promotes ``GeneFull`` to primary — a nuclear library is
|
|
258
|
+
~1/3 intronic, so a Gene-first primary silently under-counts it (ce11: Gene=1186 vs GeneFull=1940,
|
|
259
|
+
a 40.7% loss). Still all five features, one alignment, one pass; only ``adata.X`` changes. This is
|
|
260
|
+
the model finding biology and code deciding processing — the split the whole compiler is built on.
|
|
261
|
+
"""
|
|
262
|
+
warnings: list[ValidationWarning] = []
|
|
263
|
+
default = list(DEFAULT_SOLO_FEATURES)
|
|
264
|
+
|
|
265
|
+
if override is not None:
|
|
266
|
+
features = list(dict.fromkeys(override))
|
|
267
|
+
dropped = [f for f in default if f not in features]
|
|
268
|
+
if dropped:
|
|
269
|
+
warnings.append(
|
|
270
|
+
ValidationWarning(
|
|
271
|
+
code="FEATURES_NARROWED",
|
|
272
|
+
message=(
|
|
273
|
+
f"--quantify drops {dropped} from the default. Counting is cheap next to the "
|
|
274
|
+
f"alignment you are already paying for, and dropping is the only "
|
|
275
|
+
f"irreversible act here: --soloFeatures Gene alone was measured to discard "
|
|
276
|
+
f"40.7% of a nuclear library."
|
|
277
|
+
),
|
|
278
|
+
subject=BlockerSubject(kind="field", ref="processing.quantification"),
|
|
279
|
+
)
|
|
280
|
+
)
|
|
281
|
+
return features, "user_confirmed", ["cli:--quantify"], warnings
|
|
282
|
+
|
|
283
|
+
named = [i for i in instructions if i.field == "processing.quantification"]
|
|
284
|
+
if named:
|
|
285
|
+
# promote, do not substitute: instructed features move to the front, the rest of the default
|
|
286
|
+
# follows in its own order. `dict.fromkeys` keeps first-seen order and de-duplicates.
|
|
287
|
+
promoted = [i.value for i in named]
|
|
288
|
+
features = list(dict.fromkeys([*promoted, *default])) # type: ignore[list-item]
|
|
289
|
+
evidence = [e for i in named for e in i.evidence]
|
|
290
|
+
return features, "user_confirmed", evidence, warnings
|
|
291
|
+
|
|
292
|
+
if prep_type == "single-nucleus":
|
|
293
|
+
# No flag, no instruction — but a verified nuclei prep. Promote GeneFull to primary the same
|
|
294
|
+
# way an instruction would, using the same union idiom so nothing is dropped and Gene still
|
|
295
|
+
# follows (Velocyto's "requires Gene" holds by construction).
|
|
296
|
+
features = list(dict.fromkeys(["GeneFull", *default]))
|
|
297
|
+
return features, "inferred", ["policy:genefull-primary-for-single-nucleus"], warnings
|
|
298
|
+
|
|
299
|
+
return default, "inferred", ["policy:default-solo-features"], warnings
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def resolve_processing(
|
|
303
|
+
*,
|
|
304
|
+
spec: Spec,
|
|
305
|
+
dataset: DatasetManifest,
|
|
306
|
+
instructions: Sequence[Instruction] = (),
|
|
307
|
+
overrides: ProcessingOverrides | None = None,
|
|
308
|
+
prep_type: str | None = None,
|
|
309
|
+
) -> tuple[ProcessingSection, list[ValidationWarning]]:
|
|
310
|
+
"""THE single place precedence lives: policy default -> instruction -> CLI flag.
|
|
311
|
+
|
|
312
|
+
Pure: no bytes, no disk, no LLM, no network. A pure function of three inputs is exactly what you
|
|
313
|
+
want owning the rule that decides what gets counted.
|
|
314
|
+
|
|
315
|
+
Precedence is **silent** by design. A flag overriding an instruction, or an instruction overriding
|
|
316
|
+
a policy default, is not an ambiguity — it is what an instruction IS, and stopping to ask would
|
|
317
|
+
make the pipeline unusable by the people telling it what to do. What IS surfaced is a
|
|
318
|
+
*same-precedence* disagreement, which has no tiebreak; :func:`instructions_from_assertions` raises
|
|
319
|
+
those as ``Conflict``s (exit 4) before this function ever runs.
|
|
320
|
+
"""
|
|
321
|
+
ov = overrides or ProcessingOverrides()
|
|
322
|
+
defaults = processing_defaults(spec)
|
|
323
|
+
rung = dataset.library.chemistry.rung
|
|
324
|
+
|
|
325
|
+
quant: Quantification
|
|
326
|
+
warnings: list[ValidationWarning] = []
|
|
327
|
+
if isinstance(defaults.quantification, SoloQuant):
|
|
328
|
+
features, basis, evidence, warnings = resolve_features(
|
|
329
|
+
instructions=instructions, override=ov.features, prep_type=prep_type
|
|
330
|
+
)
|
|
331
|
+
quant = SoloQuant(features=features)
|
|
332
|
+
else:
|
|
333
|
+
# bulk: counting is module-scoped, and there is nothing here a user needs to instruct —
|
|
334
|
+
# --quantMode GeneCounts already emits all three strand columns.
|
|
335
|
+
quant = defaults.quantification
|
|
336
|
+
basis, evidence = "inferred", ["policy:default-bulk-quant-mode"]
|
|
337
|
+
|
|
338
|
+
instructed = _instructed_entry(instructions, "processing.genome.assembly")
|
|
339
|
+
assembly = ov.assembly or (instructed.value if instructed else None)
|
|
340
|
+
if assembly is None:
|
|
341
|
+
raise PolicyError(
|
|
342
|
+
f"no genome: this dataset's organism is taxid {dataset.experiment.organism.value}. "
|
|
343
|
+
"Pass --assembly/--annotation, or name an assembly in an --instruction document. "
|
|
344
|
+
"seqforge will not guess: taxid -> preferred assembly (hg38 vs hg19 vs T2T) is a policy "
|
|
345
|
+
"call, and that map belongs to liulab-genome."
|
|
346
|
+
)
|
|
347
|
+
if ov.annotation_name is None:
|
|
348
|
+
raise PolicyError(
|
|
349
|
+
"no annotation: --annotation names a GTF REGISTERED with liulab-genome (e.g. WS298). "
|
|
350
|
+
"It is a registry name, not something a paper writes, so there is nothing to infer."
|
|
351
|
+
)
|
|
352
|
+
# §7's ladder: a CLI flag and an --instruction document are BOTH `user_confirmed` and differ only
|
|
353
|
+
# in PRECEDENCE — they are the same user, one talking later. The channel lives in `evidence`.
|
|
354
|
+
#
|
|
355
|
+
# This read `"user_confirmed" if ov.assembly else "asserted"`, and `asserted` is what a database
|
|
356
|
+
# or a paper says — the opposite of the user talking. The branch could only fire when the assembly
|
|
357
|
+
# came from an instruction, and no production caller ever passed one, so it was dead code AND
|
|
358
|
+
# wrong. Making the path reachable is what surfaced it.
|
|
359
|
+
genome_basis: Basis
|
|
360
|
+
genome_evidence: list[str]
|
|
361
|
+
if ov.assembly:
|
|
362
|
+
genome_basis, genome_evidence = "user_confirmed", ["cli:--assembly"]
|
|
363
|
+
elif instructed is not None:
|
|
364
|
+
genome_basis, genome_evidence = instructed.basis, list(instructed.evidence)
|
|
365
|
+
else: # pragma: no cover - `assembly is None` already raised above
|
|
366
|
+
genome_basis, genome_evidence = "inferred", []
|
|
367
|
+
|
|
368
|
+
section = ProcessingSection(
|
|
369
|
+
genome=EvidencedGenome(
|
|
370
|
+
value=GenomeRef(
|
|
371
|
+
assembly=assembly,
|
|
372
|
+
annotation_name=ov.annotation_name,
|
|
373
|
+
ncbi_taxid=dataset.experiment.organism.value,
|
|
374
|
+
),
|
|
375
|
+
basis=genome_basis,
|
|
376
|
+
evidence=genome_evidence,
|
|
377
|
+
confidence=0.9,
|
|
378
|
+
rung=0,
|
|
379
|
+
),
|
|
380
|
+
aligner=EvidencedStr(value=defaults.aligner, basis="inferred", confidence=0.95, rung=rung),
|
|
381
|
+
quantification=EvidencedQuantification(
|
|
382
|
+
value=quant, basis=basis, evidence=evidence, confidence=0.9, rung=rung
|
|
383
|
+
),
|
|
384
|
+
variant_calling=EvidencedBool(
|
|
385
|
+
value=defaults.variant_calling, basis="inferred", confidence=0.9, rung=0
|
|
386
|
+
),
|
|
387
|
+
environment=EvidencedRuntimeEnv(
|
|
388
|
+
value=ov.environment or defaults.environment,
|
|
389
|
+
basis="user_confirmed" if ov.environment else "inferred",
|
|
390
|
+
confidence=0.95,
|
|
391
|
+
rung=0,
|
|
392
|
+
),
|
|
393
|
+
resources=ResourceHints(threads=ov.threads) if ov.threads else ResourceHints(),
|
|
394
|
+
)
|
|
395
|
+
return section, warnings
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _instructed_entry(instructions: Sequence[Instruction], field: str) -> Instruction | None:
|
|
399
|
+
"""The instruction for a single-valued field, if any. Same-field conflicts are already out.
|
|
400
|
+
|
|
401
|
+
Returns the Instruction rather than its value, because the basis and evidence are the point: an
|
|
402
|
+
instruction is the USER talking (`user_confirmed`), and its evidence names the assertion whose
|
|
403
|
+
quote greps back into the document. Returning a bare string is what let the caller stamp
|
|
404
|
+
`asserted` — a database's basis — on something a human wrote for us.
|
|
405
|
+
"""
|
|
406
|
+
for i in instructions:
|
|
407
|
+
if i.field == field:
|
|
408
|
+
return i
|
|
409
|
+
return None
|