seqforge 2026.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqforge/__init__.py +16 -0
- seqforge/cli/__init__.py +38 -0
- seqforge/cli/__main__.py +8 -0
- seqforge/cli/_common.py +105 -0
- seqforge/cli/compose.py +119 -0
- seqforge/cli/eval.py +103 -0
- seqforge/cli/harvest.py +417 -0
- seqforge/cli/hook.py +247 -0
- seqforge/cli/io.py +502 -0
- seqforge/cli/kb.py +348 -0
- seqforge/cli/manifest.py +536 -0
- seqforge/cli/probe.py +43 -0
- seqforge/cli/processing.py +192 -0
- seqforge/cli/project.py +52 -0
- seqforge/cli/resolve.py +55 -0
- seqforge/cli/root.py +66 -0
- seqforge/cli/run.py +463 -0
- seqforge/cli/schema.py +41 -0
- seqforge/compose/__init__.py +28 -0
- seqforge/compose/core.py +515 -0
- seqforge/compose/gates.py +113 -0
- seqforge/compose/params.py +447 -0
- seqforge/e2e.py +1926 -0
- seqforge/evals/__init__.py +78 -0
- seqforge/evals/case.py +382 -0
- seqforge/evals/grade.py +300 -0
- seqforge/evals/run.py +420 -0
- seqforge/harvest/__init__.py +121 -0
- seqforge/harvest/extract.py +319 -0
- seqforge/harvest/fields.py +212 -0
- seqforge/harvest/normalize.py +537 -0
- seqforge/harvest/prep.py +41 -0
- seqforge/harvest/providers.py +321 -0
- seqforge/harvest/verify.py +251 -0
- seqforge/hooks/__init__.py +33 -0
- seqforge/hooks/guards.py +214 -0
- seqforge/io/__init__.py +61 -0
- seqforge/io/archive.py +450 -0
- seqforge/io/attributes.py +190 -0
- seqforge/io/biosample/attributes.json +6341 -0
- seqforge/io/efo/labels.json +55 -0
- seqforge/io/efo.py +138 -0
- seqforge/io/onlist.py +661 -0
- seqforge/io/onlists/3M-february-2018.codes.gz +0 -0
- seqforge/io/onlists/737K-arc-v1.codes.gz +0 -0
- seqforge/io/onlists/737K-august-2016.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls1.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls2.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3-384.codes.gz +0 -0
- seqforge/io/onlists/bd-rhapsody-cls3.codes.gz +0 -0
- seqforge/io/onlists/index.json +74 -0
- seqforge/io/remote.py +659 -0
- seqforge/io/taxonomy.py +194 -0
- seqforge/kb/__init__.py +62 -0
- seqforge/kb/anchor.py +169 -0
- seqforge/kb/generate.py +147 -0
- seqforge/kb/loader.py +152 -0
- seqforge/kb/roundtrip.py +112 -0
- seqforge/kb/schema.py +422 -0
- seqforge/kb/specs/10x-3p-gex/spec.yaml +62 -0
- seqforge/kb/specs/10x-3p-gex-v2/README.md +41 -0
- seqforge/kb/specs/10x-3p-gex-v2/spec.yaml +83 -0
- seqforge/kb/specs/10x-3p-gex-v3/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3/spec.yaml +118 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/README.md +56 -0
- seqforge/kb/specs/10x-3p-gex-v3.1/spec.yaml +124 -0
- seqforge/kb/specs/bd-rhapsody-wta/README.md +103 -0
- seqforge/kb/specs/bd-rhapsody-wta/spec.yaml +130 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced/spec.yaml +99 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v1/spec.yaml +93 -0
- seqforge/kb/specs/bd-rhapsody-wta-enhanced-v2/spec.yaml +81 -0
- seqforge/kb/specs/bulk-rnaseq-pe/README.md +35 -0
- seqforge/kb/specs/bulk-rnaseq-pe/spec.yaml +97 -0
- seqforge/kb/specs/splitseq/README.md +51 -0
- seqforge/kb/specs/splitseq/spec.yaml +157 -0
- seqforge/manifest/__init__.py +61 -0
- seqforge/manifest/fill.py +531 -0
- seqforge/manifest/hash.py +77 -0
- seqforge/manifest/instruct.py +114 -0
- seqforge/manifest/policy.py +409 -0
- seqforge/manifest/validate.py +274 -0
- seqforge/models/__init__.py +268 -0
- seqforge/models/assertion.py +68 -0
- seqforge/models/base.py +100 -0
- seqforge/models/blocker.py +71 -0
- seqforge/models/conflict.py +47 -0
- seqforge/models/dataset.py +320 -0
- seqforge/models/evidenced.py +54 -0
- seqforge/models/observation.py +157 -0
- seqforge/models/processing.py +231 -0
- seqforge/models/records.py +145 -0
- seqforge/models/resolve.py +216 -0
- seqforge/probe/__init__.py +46 -0
- seqforge/probe/core.py +232 -0
- seqforge/probe/signals.py +250 -0
- seqforge/probe/streaming.py +118 -0
- seqforge/project.py +177 -0
- seqforge/py.typed +0 -0
- seqforge/resolve/__init__.py +98 -0
- seqforge/resolve/assign.py +204 -0
- seqforge/resolve/cache.py +119 -0
- seqforge/resolve/confuse.py +215 -0
- seqforge/resolve/engine.py +646 -0
- seqforge/resolve/escalate.py +668 -0
- seqforge/resolve/evaluators.py +306 -0
- seqforge/resolve/geometry.py +89 -0
- seqforge/resolve/group.py +85 -0
- seqforge/resolve/records.py +550 -0
- seqforge/resolve/scoring.py +373 -0
- seqforge/resolve/window.py +206 -0
- seqforge/workflows/__init__.py +234 -0
- seqforge/workflows/cram.py +117 -0
- seqforge/workflows/h5ad.py +368 -0
- seqforge/workflows/map/star.smk +101 -0
- seqforge/workflows/map/starsolo.smk +360 -0
- seqforge/workflows/qc.py +157 -0
- seqforge/workspace.py +125 -0
- seqforge-2026.7.1.dist-info/METADATA +125 -0
- seqforge-2026.7.1.dist-info/RECORD +124 -0
- seqforge-2026.7.1.dist-info/WHEEL +4 -0
- seqforge-2026.7.1.dist-info/entry_points.txt +2 -0
- seqforge-2026.7.1.dist-info/licenses/LICENSE +21 -0
seqforge/models/base.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Shared scalars, controlled vocabularies, and the ``Evidenced[T]`` three-truths carrier.
|
|
2
|
+
|
|
3
|
+
These are the atoms every other model is built from. ``Evidenced[T]`` wraps every interpretive
|
|
4
|
+
manifest field so a value never travels without its provenance.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Annotated, Generic, Literal, TypeVar
|
|
10
|
+
|
|
11
|
+
from pydantic import (
|
|
12
|
+
AfterValidator,
|
|
13
|
+
BaseModel,
|
|
14
|
+
ConfigDict,
|
|
15
|
+
Field,
|
|
16
|
+
PositiveInt,
|
|
17
|
+
StringConstraints,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _reject_absolute_or_local(value: str) -> str:
|
|
22
|
+
"""Reject absolute or local filesystem paths in a manifest URI.
|
|
23
|
+
|
|
24
|
+
A manifest URI is a relative path, a non-file scheme (``s3://``, ``gs://``, ``https://``,
|
|
25
|
+
``sra:``), or a bare accession — never ``/abs``, ``~/x``, ``C:\\...``, a UNC ``\\\\host``, or a
|
|
26
|
+
``file://`` absolute path.
|
|
27
|
+
"""
|
|
28
|
+
bad = (
|
|
29
|
+
value.startswith(("/", "~"))
|
|
30
|
+
or value.startswith("file:///")
|
|
31
|
+
or (len(value) > 1 and value[1] == ":") # Windows drive, e.g. C:\
|
|
32
|
+
or value.startswith("\\\\") # UNC \\host\share
|
|
33
|
+
)
|
|
34
|
+
if bad:
|
|
35
|
+
raise ValueError(f"absolute/local path forbidden in a manifest URI: {value!r}")
|
|
36
|
+
return value
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ---- scalars & controlled vocabulary ----
|
|
40
|
+
Sha256 = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{64}$")]
|
|
41
|
+
"""Lowercase 64-hex sha256 digest (of file bytes, canonical text, or a bounded content key)."""
|
|
42
|
+
|
|
43
|
+
Uri = Annotated[str, StringConstraints(min_length=1), AfterValidator(_reject_absolute_or_local)]
|
|
44
|
+
"""A manifest URI: relative path / non-file scheme / bare accession. Never an absolute local path."""
|
|
45
|
+
|
|
46
|
+
LocalPath = Annotated[str, StringConstraints(min_length=1)]
|
|
47
|
+
"""An internal-only local filesystem path (e.g. where probe read bytes). Never enters a Manifest."""
|
|
48
|
+
|
|
49
|
+
AssayTerm = Annotated[str, StringConstraints(pattern=r"^(EFO|OBI):\d{4,}$")]
|
|
50
|
+
"""EFO/OBI assay CURIE, e.g. ``EFO:0009922``."""
|
|
51
|
+
|
|
52
|
+
NcbiTaxid = PositiveInt
|
|
53
|
+
"""NCBI taxonomy id, e.g. 9606, 10090, 559292, 6239."""
|
|
54
|
+
|
|
55
|
+
Accession = Annotated[
|
|
56
|
+
str,
|
|
57
|
+
StringConstraints(pattern=r"^([SED]R[RXPS]\d+|GS[EM]\d+|PRJ[A-Z]{2}\d+|SAM[NED][A-Z]?\d+)$"),
|
|
58
|
+
]
|
|
59
|
+
"""NCBI/ENA/DDBJ run/experiment/study/sample, GEO, BioProject, or BioSample accession."""
|
|
60
|
+
|
|
61
|
+
ChemistryId = str
|
|
62
|
+
"""KB primary key, e.g. ``10x-3p-gex-v3``. Open vocabulary, validated against the KB in code."""
|
|
63
|
+
|
|
64
|
+
Confidence = Annotated[float, Field(ge=0.0, le=1.0)]
|
|
65
|
+
Rung = Annotated[int, Field(ge=0, le=7)]
|
|
66
|
+
Basis = Literal["observed", "asserted", "inferred", "user_confirmed"]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
T = TypeVar("T")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class Evidenced(BaseModel, Generic[T]):
|
|
73
|
+
"""A single manifest field value tagged with its epistemic provenance.
|
|
74
|
+
|
|
75
|
+
Wraps every interpretive manifest field. ``basis`` records how we know it
|
|
76
|
+
(``observed`` from bytes, ``asserted`` from humans/DBs, ``inferred`` derived,
|
|
77
|
+
``user_confirmed``); disagreement across bases becomes a first-class ``Conflict``, never a
|
|
78
|
+
silent merge. ``rung`` is the cheapest escalation-ladder step that settled the field.
|
|
79
|
+
|
|
80
|
+
``confidence`` is **optional, and ``None`` is the informative value**: it means no judgement was
|
|
81
|
+
made, so there is no confidence to report. Copying a ``strain = CQ758`` out of a BioSample record
|
|
82
|
+
involves no interpretation — the record says it, we transcribed it, and ``basis="asserted"`` plus
|
|
83
|
+
the record accession in ``evidence`` already says everything true about how we know it. Writing
|
|
84
|
+
``1.0`` there would invite exactly the question it cannot answer ("you are certain the strain is
|
|
85
|
+
CQ758?"); we are certain the *record declares* it, which is a different claim and the one we make.
|
|
86
|
+
|
|
87
|
+
So: a number here means somebody or something weighed evidence and could have been wrong. That is
|
|
88
|
+
the winning candidate's score, or a language model's advisory self-report on reading prose. It is
|
|
89
|
+
never a decoration, and it is never copied from a neighbouring field — a value repeated across
|
|
90
|
+
four fields is one judgement wearing four hats, which is the ``processing``-masquerading-as-a-truth
|
|
91
|
+
mistake in miniature.
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
model_config = ConfigDict(frozen=True)
|
|
95
|
+
|
|
96
|
+
value: T
|
|
97
|
+
basis: Basis
|
|
98
|
+
evidence: list[str] = Field(default_factory=list)
|
|
99
|
+
confidence: Confidence | None = None
|
|
100
|
+
rung: Rung
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""``Blocker`` — refusal as an exit code.
|
|
2
|
+
|
|
3
|
+
A ``Blocker`` is ALWAYS fatal (its presence => nonzero exit). Advisory diagnostics are a separate
|
|
4
|
+
:class:`ValidationWarning` (renamed from the design's ``Warning`` to avoid shadowing the builtin), so
|
|
5
|
+
branching code never inspects a severity to decide whether something blocks.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from enum import StrEnum
|
|
11
|
+
from typing import Literal
|
|
12
|
+
|
|
13
|
+
from pydantic import BaseModel, Field
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BlockerCode(StrEnum):
|
|
17
|
+
"""The stable contract the CLI/skill branch on."""
|
|
18
|
+
|
|
19
|
+
MISSING_TECHNICAL_READ = "MISSING_TECHNICAL_READ"
|
|
20
|
+
#: The winning barcoded chemistry's barcode role IS filled, but no seated read carries
|
|
21
|
+
#: whitelist-matchable barcodes though the chemistry declares a whitelist — STARsolo would read
|
|
22
|
+
#: barcodes from a read that matches nothing and report ~0 valid barcodes at exit 0. Distinct from
|
|
23
|
+
#: MISSING_TECHNICAL_READ, where the role is structurally UNFILLABLE (no read of the right shape).
|
|
24
|
+
BARCODE_READ_ABSENT = "BARCODE_READ_ABSENT"
|
|
25
|
+
TRUNCATED_GZIP = "TRUNCATED_GZIP"
|
|
26
|
+
CORRUPT_FASTQ = "CORRUPT_FASTQ"
|
|
27
|
+
UNSUPPORTED_TECHNOLOGY = "UNSUPPORTED_TECHNOLOGY"
|
|
28
|
+
PRETRIMMED_VARIABLE_LENGTH = "PRETRIMMED_VARIABLE_LENGTH"
|
|
29
|
+
NO_VALID_ROLE_ASSIGNMENT = "NO_VALID_ROLE_ASSIGNMENT"
|
|
30
|
+
ONLIST_VERIFICATION_FAILED = "ONLIST_VERIFICATION_FAILED"
|
|
31
|
+
UNRESOLVED_CONFLICT = "UNRESOLVED_CONFLICT"
|
|
32
|
+
MISSING_CONTROLLED_VOCAB = "MISSING_CONTROLLED_VOCAB"
|
|
33
|
+
ABSOLUTE_PATH = "ABSOLUTE_PATH"
|
|
34
|
+
#: A processing manifest's genome does not belong to the dataset's organism. A wrong-but-VALID
|
|
35
|
+
#: assembly is the most catastrophic silent failure available here: it aligns, exits 0, and emits
|
|
36
|
+
#: a plausible matrix in the wrong coordinate space. Nothing downstream would ever notice.
|
|
37
|
+
GENOME_ORGANISM_MISMATCH = "GENOME_ORGANISM_MISMATCH"
|
|
38
|
+
#: A processing manifest bound to a different dataset than the one being compiled.
|
|
39
|
+
DATASET_PIN_MISMATCH = "DATASET_PIN_MISMATCH"
|
|
40
|
+
#: An archive record was supplied and does not account for the files on disk. Only ever raised
|
|
41
|
+
#: when a record EXISTS: a dataset with no accession has nothing to join and is not a refusal.
|
|
42
|
+
#: Half-joining is the failure this exists to prevent — the files it could not place would get no
|
|
43
|
+
#: sample facts, and a manifest that is confidently right about four samples and silent about two
|
|
44
|
+
#: reads as a manifest about six.
|
|
45
|
+
RECORD_JOIN_INCOMPLETE = "RECORD_JOIN_INCOMPLETE"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class BlockerSubject(BaseModel):
|
|
49
|
+
"""What the refusal is about. ``ref`` is a basename / dotted path / dataset id — never a path."""
|
|
50
|
+
|
|
51
|
+
kind: Literal["file", "field", "dataset"]
|
|
52
|
+
ref: str
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class Blocker(BaseModel):
|
|
56
|
+
"""A structured refusal emitted alongside a nonzero exit. ``remedy`` MUST be actionable."""
|
|
57
|
+
|
|
58
|
+
id: str
|
|
59
|
+
code: BlockerCode
|
|
60
|
+
message: str
|
|
61
|
+
remedy: str
|
|
62
|
+
subject: BlockerSubject
|
|
63
|
+
evidence: list[str] = Field(default_factory=list)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class ValidationWarning(BaseModel):
|
|
67
|
+
"""A non-blocking advisory note (exits 0). Kept distinct from :class:`Blocker`, which is fatal."""
|
|
68
|
+
|
|
69
|
+
code: str
|
|
70
|
+
message: str
|
|
71
|
+
subject: BlockerSubject
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""``Conflict`` — a surfaced disagreement between truths, never auto-picked.
|
|
2
|
+
|
|
3
|
+
``positions`` generalizes the common observed/asserted pair; ``status="benign"`` is the §12 escape
|
|
4
|
+
hatch for two confusable KB entries that emit identical ``backend.params``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel, Field
|
|
12
|
+
|
|
13
|
+
from .base import Basis, Confidence, Rung
|
|
14
|
+
|
|
15
|
+
Decidable = Literal["reads", "onlist", "metadata", "alignment", "user"]
|
|
16
|
+
"""How a divergent pair could be separated. Includes ``onlist`` — the rung-3 mechanism §12 uses."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ConflictPosition(BaseModel):
|
|
20
|
+
"""One side of a disagreement. ``value`` is the canonical string form (fields are heterogeneous)."""
|
|
21
|
+
|
|
22
|
+
value: str
|
|
23
|
+
basis: Basis
|
|
24
|
+
evidence: list[str] = Field(default_factory=list)
|
|
25
|
+
confidence: Confidence
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Resolution(BaseModel):
|
|
29
|
+
"""How an open conflict was ultimately settled."""
|
|
30
|
+
|
|
31
|
+
chosen_value: str
|
|
32
|
+
basis: Basis
|
|
33
|
+
rung: Rung
|
|
34
|
+
decided_by: Literal["code", "user", "benign_equivalence"]
|
|
35
|
+
note: str | None = None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Conflict(BaseModel):
|
|
39
|
+
"""A first-class, surfaced disagreement between truths."""
|
|
40
|
+
|
|
41
|
+
id: str
|
|
42
|
+
field: str
|
|
43
|
+
positions: list[ConflictPosition] = Field(min_length=2)
|
|
44
|
+
kind: Literal["observed_vs_asserted", "asserted_vs_asserted", "other"]
|
|
45
|
+
decidable_by: list[Decidable]
|
|
46
|
+
status: Literal["open", "resolved", "benign"] = "open"
|
|
47
|
+
resolution: Resolution | None = None
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
""":class:`DatasetManifest` — a finished assay. Two truths, one lifetime, IMMUTABLE.
|
|
2
|
+
|
|
3
|
+
``library`` is physical truth about molecules and sequencer output (authority = **evidence**);
|
|
4
|
+
``experiment`` is biological/metadata truth (authority = **metadata and humans**). Both are claims
|
|
5
|
+
about *what the data is*. A finished assay does not change, so neither does this file: it is
|
|
6
|
+
content-addressed, and nothing downstream may write to it. When a new fact arrives, the manifest is
|
|
7
|
+
rebuilt from evidence and gets a new hash. It is never patched.
|
|
8
|
+
|
|
9
|
+
**What is deliberately absent is the point.** There is no ``processing`` section here. What we choose
|
|
10
|
+
to *do* with an assay is intent, not truth, and it lives in
|
|
11
|
+
:mod:`~seqforge.models.processing` — of which there are many per dataset. The old three-section
|
|
12
|
+
manifest made "three truths" (the three *bases*) and "three sections" line up by coincidence, and the
|
|
13
|
+
pun cost us: ``processing`` inherited the grammar of a truth — ``Evidenced`` fields, an "authority", a
|
|
14
|
+
uniform ``basis="inferred"`` stamped on by construction — and then ``compose`` read almost none of
|
|
15
|
+
them. A field that is never read can never produce the ``Conflict`` the three-truths rule promises. ``base.py`` listing
|
|
16
|
+
**four** bases against three sections was the tell.
|
|
17
|
+
|
|
18
|
+
This module imports nothing from ``processing``, and must not. That independence is the two-artifact
|
|
19
|
+
split expressed as an import graph: a dataset cannot know how it will be processed, because it will be
|
|
20
|
+
processed many ways.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import Literal
|
|
26
|
+
|
|
27
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
|
28
|
+
|
|
29
|
+
from .base import Accession, AssayTerm, ChemistryId, Evidenced, Sha256, Uri
|
|
30
|
+
from .evidenced import (
|
|
31
|
+
EvidencedAccessionList,
|
|
32
|
+
EvidencedChemistrySet,
|
|
33
|
+
EvidencedStr,
|
|
34
|
+
EvidencedTaxid,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ReadAnchor(BaseModel):
|
|
39
|
+
"""A floating element's addressing, carried into the IR so the manifest records what STAGGERS.
|
|
40
|
+
|
|
41
|
+
A fixed element has a ``start``; an anchored one does not — its position floats per read (BD
|
|
42
|
+
Rhapsody Enhanced's 0-3 bp diversity insert shifts every cell-label block downstream). The manifest
|
|
43
|
+
is *what the data is*, so it must say an element floats rather than pin it to a wrong constant. This
|
|
44
|
+
mirrors the KB DSL's ``kb.schema.Anchor`` field-for-field (``test_read_anchor_mirrors_the_kb_anchor``
|
|
45
|
+
keeps the two from drifting); it is duplicated rather than imported because the IR is the
|
|
46
|
+
schema-export source of truth and must not pull the ``kb``->``probe``->``models`` import chain back
|
|
47
|
+
into ``models``. It carries no interpretation, so it is a plain ``BaseModel``, not ``Evidenced``.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
relative_to: Literal["read_start", "read_end", "element"] = "read_start"
|
|
51
|
+
ref_element: str | None = None
|
|
52
|
+
ref_side: Literal["start", "end"] = "end"
|
|
53
|
+
offset: int = 0
|
|
54
|
+
motif: str | None = None
|
|
55
|
+
max_mismatch: int = 0
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ReadElement(BaseModel):
|
|
59
|
+
"""One ordered sub-region of a read; a superset/adapter over a seqspec Region.
|
|
60
|
+
|
|
61
|
+
Adds a derived ``start`` and an interpretive ``role`` that seqspec Regions do not carry. seqspec
|
|
62
|
+
``sequence_type`` maps: fixed -> ``sequence``; random -> random ACGT; onlist -> ``onlist_ref``. An
|
|
63
|
+
``anchor`` (with no fixed ``start``) marks an element whose position floats per read.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
role: Literal["CB", "UMI", "cDNA", "gDNA", "index", "linker", "polyT", "polyA"]
|
|
67
|
+
region_type: Literal[
|
|
68
|
+
"barcode",
|
|
69
|
+
"umi",
|
|
70
|
+
"cdna",
|
|
71
|
+
"gdna",
|
|
72
|
+
"index5",
|
|
73
|
+
"index7",
|
|
74
|
+
"linker",
|
|
75
|
+
"poly_A",
|
|
76
|
+
"poly_t",
|
|
77
|
+
"custom_primer",
|
|
78
|
+
]
|
|
79
|
+
start: int | None = None
|
|
80
|
+
length: int | None = None
|
|
81
|
+
min_len: int | None = None
|
|
82
|
+
max_len: int | None = None
|
|
83
|
+
sequence: str | None = None
|
|
84
|
+
onlist_ref: str | None = None
|
|
85
|
+
#: Present iff the element floats (no fixed ``start``); records the KB's declared anchor verbatim.
|
|
86
|
+
anchor: ReadAnchor | None = None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class ReadDef(BaseModel):
|
|
90
|
+
"""A sequencing read (== one FASTQ), aligned to a seqspec Read. ``read_id`` is a role, not a file."""
|
|
91
|
+
|
|
92
|
+
read_id: str
|
|
93
|
+
strand: Literal["pos", "neg"]
|
|
94
|
+
min_len: int = Field(ge=0)
|
|
95
|
+
max_len: int = Field(ge=0)
|
|
96
|
+
elements: list[ReadElement]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class ReadLayout(BaseModel):
|
|
100
|
+
"""Full physical library structure = ordered reads x ordered elements."""
|
|
101
|
+
|
|
102
|
+
modality: Literal["rna", "atac", "protein", "crispr", "dna"]
|
|
103
|
+
reads: list[ReadDef]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class EvidencedReadLayout(Evidenced[ReadLayout]):
|
|
107
|
+
"""An ``Evidenced`` read layout."""
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class Onlist(BaseModel):
|
|
111
|
+
"""Barcode-whitelist registry entry; fetched via pooch + hash-verified, never vendored.
|
|
112
|
+
|
|
113
|
+
``orientation_hint`` is a non-authoritative default; the authoritative orientation lives per-read
|
|
114
|
+
in the KB (one list can be forward for GEX and reverse-complemented for ATAC).
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
name: str
|
|
118
|
+
uri: Uri
|
|
119
|
+
sha256: Sha256
|
|
120
|
+
length: int = Field(ge=1)
|
|
121
|
+
orientation_hint: Literal["forward", "reverse_complement"] | None = None
|
|
122
|
+
n_entries: int | None = None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class FileInventoryItem(BaseModel):
|
|
126
|
+
"""One physical file + checksum + assigned role. No absolute path.
|
|
127
|
+
|
|
128
|
+
``read_id`` is a plain string, not ``Evidenced``, and that is not a demotion. The role assignment
|
|
129
|
+
and the chemistry come out of **one** joint optimization — ``Candidate`` is literally
|
|
130
|
+
``(technology, score, role_assignment)``, and the score scores the pair — so the confidence in
|
|
131
|
+
"this file is R1" and the confidence in "this library is v3" are the same number, and
|
|
132
|
+
``library.chemistry`` is where it lives. Twelve files each carrying a copy of it was the pilot's
|
|
133
|
+
manifest saying one thing thirteen times.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
uri: Uri
|
|
137
|
+
basename: str
|
|
138
|
+
sha256: Sha256
|
|
139
|
+
size_bytes: int = Field(gt=0)
|
|
140
|
+
#: The read role the joint optimization assigned, e.g. ``R1``, or :data:`INDEX_ROLE` for a
|
|
141
|
+
#: technical sample-index read set aside from the pipeline. ``None`` = unassigned, which
|
|
142
|
+
#: ``validate`` surfaces: an unassigned file is one ``compose`` will silently skip.
|
|
143
|
+
read_id: str | None = None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
#: The ``read_id`` marking a technical sample-index read (10x I1/I2). STARsolo consumes only the
|
|
147
|
+
#: CB+UMI and cDNA reads, so an 8/10 bp index file is a leftover of an *already-decided* run — set
|
|
148
|
+
#: aside under this role rather than left unassigned (which blocks) or forced into a layout (which
|
|
149
|
+
#: would demand an index file of every clean sample). It is not a layout read and never becomes a
|
|
150
|
+
#: unit; the length gate that assigns it lives in ``resolve/engine.py`` (:func:`index_tagged_roles`).
|
|
151
|
+
INDEX_ROLE = "index"
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class AssayLabel(BaseModel):
|
|
155
|
+
"""One chemistry, spelled three ways: our key, EFO's id, and EFO's words.
|
|
156
|
+
|
|
157
|
+
This exists because ``assay: EFO:0009922`` beside ``chemistry: [10x-3p-gex-v3,
|
|
158
|
+
10x-3p-gex-v3.1]`` was two puzzles at once. *What is the difference between them?* — none, they
|
|
159
|
+
are the same fact in two vocabularies. *Why is one a list and the other not?* — because the
|
|
160
|
+
§12 equivalence class has two members and the assay field could only hold one, so it silently
|
|
161
|
+
dropped v3.1's CURIE.
|
|
162
|
+
|
|
163
|
+
Making it a label per chemistry answers both and removes a third problem nobody had noticed: an
|
|
164
|
+
assay CURIE that disagrees with the chemistry it names is now **inexpressible** rather than
|
|
165
|
+
merely absent. There is nowhere to write it.
|
|
166
|
+
|
|
167
|
+
``name`` is EFO's own label, generated into ``io/efo/labels.json`` from EBI's OLS4. Never typed
|
|
168
|
+
here: a label we maintain by hand is a label that drifts from the ontology it claims to quote.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
model_config = ConfigDict(frozen=True)
|
|
172
|
+
|
|
173
|
+
#: The KB's primary key, e.g. ``10x-3p-gex-v3``.
|
|
174
|
+
chemistry: ChemistryId
|
|
175
|
+
#: The controlled-vocabulary id, e.g. ``EFO:0009922``.
|
|
176
|
+
curie: AssayTerm
|
|
177
|
+
#: What EFO calls it, e.g. ``10x 3' v3``. The thing a human reads.
|
|
178
|
+
name: str
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class LibrarySection(BaseModel):
|
|
182
|
+
"""Physical truth. Authority = EVIDENCE.
|
|
183
|
+
|
|
184
|
+
**One decision, one confidence.** ``chemistry`` is the decision — the joint optimization over
|
|
185
|
+
(which technology, which file is which read) that ``resolve score`` performs — and it is the only
|
|
186
|
+
``Evidenced`` field here. Everything else follows from it: ``assay`` is the same answer in EFO's
|
|
187
|
+
vocabulary, ``read_layout`` is the KB's declared structure for that chemistry filled in with
|
|
188
|
+
measured lengths, ``files[].read_id`` is the assignment half of the very same optimization, and
|
|
189
|
+
``onlists`` are the whitelists that chemistry uses.
|
|
190
|
+
|
|
191
|
+
They used to each carry their own ``Evidenced`` envelope, and the pilot's manifest showed what
|
|
192
|
+
that bought: ``confidence: 0.750672`` printed four times, identical, because it was always one
|
|
193
|
+
number about one decision wearing four hats. Four envelopes cannot disagree — they were filled
|
|
194
|
+
from the same variable — so they were never four truths, and we never asked for four. We ask
|
|
195
|
+
that a value not travel without its provenance, which is exactly what one honest envelope does.
|
|
196
|
+
A field repeated is not provenance; it is decoration that looks like provenance, which is worse
|
|
197
|
+
than none.
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
chemistry: EvidencedChemistrySet
|
|
201
|
+
#: One per chemistry in the equivalence class, same order. Derived, so it carries no confidence.
|
|
202
|
+
assay: list[AssayLabel] = Field(default_factory=list)
|
|
203
|
+
read_layout: ReadLayout
|
|
204
|
+
onlists: list[Onlist] = Field(default_factory=list)
|
|
205
|
+
files: list[FileInventoryItem]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class SampleGroup(BaseModel):
|
|
209
|
+
"""One biological sample, the files that carry it, and what is declared about it.
|
|
210
|
+
|
|
211
|
+
``attributes`` is keyed by an **NCBI harmonized BioSample attribute name** — ``strain``,
|
|
212
|
+
``tissue``, ``dev_stage``, ``genotype``, all 960 of them — and the validator refuses anything
|
|
213
|
+
else. Two named fields (``tissue``, ``condition``) used to sit here instead, and both were wrong
|
|
214
|
+
in the same way:
|
|
215
|
+
|
|
216
|
+
- ``condition`` is not a vocabulary anyone else uses. We invented it, and a field named
|
|
217
|
+
"condition" accepts anything you can call a condition; on the pilot a language model filed worm
|
|
218
|
+
husbandry into it. NCBI's ``treatment`` / ``genotype`` / ``disease`` are the real keys, each
|
|
219
|
+
with a definition somebody else maintains.
|
|
220
|
+
- Two typed fields cannot hold ``strain``, and ``strain`` is the only structured field that
|
|
221
|
+
separates the pilot's wild-type samples from its daf-2 mutants.
|
|
222
|
+
|
|
223
|
+
An open dict rather than 960 pydantic fields, because a typed list mirroring somebody else's
|
|
224
|
+
vocabulary is the shape this repo keeps getting bitten by: it rots the moment they add an
|
|
225
|
+
attribute, and nothing goes red. The vocabulary is the file NCBI's list was generated into, and
|
|
226
|
+
the validator reads *it*.
|
|
227
|
+
"""
|
|
228
|
+
|
|
229
|
+
sample_id: str
|
|
230
|
+
#: The archive's id for this sample, when there is one. ``None`` for a dataset that never went
|
|
231
|
+
#: near an archive — which is most of them, and not a defect.
|
|
232
|
+
accession: Accession | None = None
|
|
233
|
+
attributes: dict[str, EvidencedStr] = Field(default_factory=dict)
|
|
234
|
+
file_uris: list[Uri] = Field(default_factory=list)
|
|
235
|
+
|
|
236
|
+
@field_validator("attributes")
|
|
237
|
+
@classmethod
|
|
238
|
+
def _keys_are_ncbi_attributes(cls, value: dict[str, EvidencedStr]) -> dict[str, EvidencedStr]:
|
|
239
|
+
"""Fail closed on a key NCBI does not define: no field enters a manifest unvalidated."""
|
|
240
|
+
from ..io.attributes import is_attribute
|
|
241
|
+
|
|
242
|
+
unknown = sorted(k for k in value if not is_attribute(k))
|
|
243
|
+
if unknown:
|
|
244
|
+
raise ValueError(
|
|
245
|
+
f"not NCBI harmonized BioSample attribute name(s): {unknown}. A sample attribute's "
|
|
246
|
+
f"key space is NCBI's 960 curated names (`seqforge io attributes list`), not ours."
|
|
247
|
+
)
|
|
248
|
+
return value
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
class Study(BaseModel):
|
|
252
|
+
"""The study these files came from, as the archive declares it.
|
|
253
|
+
|
|
254
|
+
Deliberately **not** ``Evidenced``: none of it is an interpretation. The record says the title is
|
|
255
|
+
X and we copied X, exactly as we copy a file's ``sha256``. The abstract is deliberately absent —
|
|
256
|
+
it is prose, it belongs in a document a quote can grep back into, and pasting a paragraph of
|
|
257
|
+
English into a content-addressed manifest would make the dataset's identity depend on it.
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
accession: Accession | None = None
|
|
261
|
+
title: str | None = None
|
|
262
|
+
center: str | None = None
|
|
263
|
+
data_type: str | None = None
|
|
264
|
+
released: str | None = None
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
class ExperimentSection(BaseModel):
|
|
268
|
+
"""Biological/metadata truth. Authority = METADATA + humans."""
|
|
269
|
+
|
|
270
|
+
organism: EvidencedTaxid
|
|
271
|
+
accessions: EvidencedAccessionList
|
|
272
|
+
samples: list[SampleGroup]
|
|
273
|
+
study: Study | None = None
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
class DatasetProvenance(BaseModel):
|
|
277
|
+
"""Binds a dataset manifest to the bytes and the KB that read them.
|
|
278
|
+
|
|
279
|
+
``workflow_version`` is deliberately ABSENT. Nothing in ``library``/``experiment`` depends on
|
|
280
|
+
which Snakemake module will one day run: the assay happened before we had an opinion about it.
|
|
281
|
+
It belongs to the processing manifest, and folding it in here would make a dataset's identity
|
|
282
|
+
churn every time a rule file changed — which is exactly the coupling the two-artifact split removes.
|
|
283
|
+
"""
|
|
284
|
+
|
|
285
|
+
dataset_hash: str
|
|
286
|
+
kb_version: str
|
|
287
|
+
seqforge_version: str
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
class DatasetManifest(BaseModel):
|
|
291
|
+
"""A finished assay: what the bench did. Two truths, two authorities, one lifetime.
|
|
292
|
+
|
|
293
|
+
Structural only: semantic cross-checks (referential integrity — every experiment ``file_uri`` in
|
|
294
|
+
the library inventory — and the no-absolute-path sweep) are enforced by ``manifest validate``
|
|
295
|
+
as ``Blocker``s, not as construction-time ``ValidationError``s.
|
|
296
|
+
"""
|
|
297
|
+
|
|
298
|
+
model_config = ConfigDict(frozen=True)
|
|
299
|
+
|
|
300
|
+
library: LibrarySection
|
|
301
|
+
experiment: ExperimentSection
|
|
302
|
+
provenance: DatasetProvenance
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
__all__ = [
|
|
306
|
+
"ReadAnchor",
|
|
307
|
+
"ReadElement",
|
|
308
|
+
"ReadDef",
|
|
309
|
+
"ReadLayout",
|
|
310
|
+
"EvidencedReadLayout",
|
|
311
|
+
"Onlist",
|
|
312
|
+
"FileInventoryItem",
|
|
313
|
+
"AssayLabel",
|
|
314
|
+
"LibrarySection",
|
|
315
|
+
"SampleGroup",
|
|
316
|
+
"Study",
|
|
317
|
+
"ExperimentSection",
|
|
318
|
+
"DatasetProvenance",
|
|
319
|
+
"DatasetManifest",
|
|
320
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Named ``Evidenced[T]`` specializations over the base scalars — stable ``$defs`` for schema export.
|
|
2
|
+
|
|
3
|
+
Pydantic generics are structural: ``Evidenced[str]`` has no stable class name, so exporting it inlines
|
|
4
|
+
an anonymous schema and the ``$defs`` churn every time an unrelated field moves. Naming each one keeps
|
|
5
|
+
the exported JSON Schema (design §1.8, the single source of truth) diffable.
|
|
6
|
+
|
|
7
|
+
These are the wrappers whose payload is a **base scalar**, so they can be shared by both halves of the
|
|
8
|
+
manifest without either half importing the other. A wrapper over a *domain* type lives next to that
|
|
9
|
+
type — :class:`~seqforge.models.dataset.EvidencedReadLayout` in ``dataset``,
|
|
10
|
+
:class:`~seqforge.models.processing.EvidencedGenome` in ``processing``. That is not a stylistic
|
|
11
|
+
preference: it is what keeps ``dataset`` and ``processing`` from importing each other, and that
|
|
12
|
+
independence is the two-artifact split expressed as an import graph rather than as a comment.
|
|
13
|
+
|
|
14
|
+
base -> evidenced -> dataset
|
|
15
|
+
-> processing (dataset and processing never meet)
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from .base import Accession, AssayTerm, ChemistryId, Evidenced, NcbiTaxid
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class EvidencedStr(Evidenced[str]):
|
|
24
|
+
"""An ``Evidenced`` string field."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class EvidencedBool(Evidenced[bool]):
|
|
28
|
+
"""An ``Evidenced`` boolean field."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class EvidencedTaxid(Evidenced[NcbiTaxid]):
|
|
32
|
+
"""An ``Evidenced`` NCBI taxid."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class EvidencedAssay(Evidenced[AssayTerm]):
|
|
36
|
+
"""An ``Evidenced`` EFO/OBI assay CURIE."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class EvidencedChemistrySet(Evidenced[list[ChemistryId]]):
|
|
40
|
+
"""An ``Evidenced`` chemistry equivalence class (benign twins recorded together, §12)."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class EvidencedAccessionList(Evidenced[list[Accession]]):
|
|
44
|
+
"""An ``Evidenced`` list of accessions."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"EvidencedStr",
|
|
49
|
+
"EvidencedBool",
|
|
50
|
+
"EvidencedTaxid",
|
|
51
|
+
"EvidencedAssay",
|
|
52
|
+
"EvidencedChemistrySet",
|
|
53
|
+
"EvidencedAccessionList",
|
|
54
|
+
]
|