seqforge 2026.7.1__py3-none-any.whl

seqforge/models/observation.py

@@ -0,0 +1,157 @@
+"""``Observation`` — deterministic, LLM-free, network-free probe output for ONE file.
+
+Cached by file sha256. Reports structural signals ONLY; it MUST NOT assign roles — mapping
+``constant -> linker/TSO``, ``random -> CB|UMI|cDNA``, ``homopolymer-T -> polyT`` is the resolver's
+job, scored and second-guessable.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .base import Confidence, LocalPath, Sha256
+
+
+class CycleComposition(BaseModel):
+    """Base fractions at one 0-based cycle; ``a + c + g + t + n ~= 1.0``."""
+
+    cycle: int = Field(ge=0)
+    a: float
+    c: float
+    g: float
+    t: float
+    n: float
+
+
+class ConstantSegment(BaseModel):
+    """A cycle span where one base dominates (>~90%): a linker/adapter/TSO candidate.
+
+    Structural only — the role is NOT assigned here.
+    """
+
+    kind: Literal["constant"] = "constant"
+    start: int = Field(ge=0)
+    end: int = Field(ge=0)
+    consensus: str
+    purity: Confidence
+
+
+class RandomSegment(BaseModel):
+    """A near-uniform ACGT span: a CB/UMI/cDNA candidate (role NOT assigned)."""
+
+    kind: Literal["random"] = "random"
+    start: int = Field(ge=0)
+    end: int = Field(ge=0)
+    mean_entropy_bits: float
+
+
+class HomopolymerSegment(BaseModel):
+    """A run of one base (polyT capture / polyA tail): structural only."""
+
+    kind: Literal["homopolymer"] = "homopolymer"
+    base: Literal["A", "C", "G", "T"]
+    start: int = Field(ge=0)
+    end: int = Field(ge=0)
+    mean_run: float
+
+
+Segment = Annotated[
+    ConstantSegment | RandomSegment | HomopolymerSegment,
+    Field(discriminator="kind"),
+]
+
+
+class FileIdentity(BaseModel):
+    """Content identity of one FASTQ. Observation is internal, so a LOCAL path is allowed here only.
+
+    ``sha256`` is a **content-address**, not a whole-file hash. Locally it is a bounded key over the
+    basename + head sample + size + gzip ISIZE (``probe.core._content_key``, issue #37). For hosted
+    bytes it is derived from the provider md5 (``probe.core.content_key_from_md5``, issue #39) — a
+    64-hex name that is a pure function of the md5, so two hosted files with the same md5 dedup. Either
+    way fingerprinting never reads a whole FASTQ; a remote probe range-reads only a bounded head.
+    """
+
+    sha256: Sha256
+    size_bytes: int = Field(gt=0)
+    basename: str
+    local_uri: LocalPath | None = None
+
+
+class ProbeProvenance(BaseModel):
+    """What the bounded probe did under its read/byte budget.
+
+    ``bytes_read`` is decompressed; ``compressed_bytes_read`` drives ``estimated_total_reads``
+    (avoids the compression-ratio undercount).
+    """
+
+    n_reads_sampled: int = Field(ge=0)
+    bytes_read: int = Field(ge=0)
+    compressed_bytes_read: int = Field(ge=0)
+    tool_version: str
+    params_hash: str
+
+
+class ReadLengthProfile(BaseModel):
+    """Read-length summary. ``n_distinct > 1`` on a fixed-geometry read -> PRETRIMMED_VARIABLE_LENGTH."""
+
+    mode: int = Field(ge=0)
+    n_distinct: int = Field(ge=1)
+    min_len: int = Field(ge=0)
+    max_len: int = Field(ge=0)
+    percentiles: dict[str, int] | None = None
+
+
+class WindowDistinctRatio(BaseModel):
+    """``distinct/total`` over a candidate cycle window.
+
+    DEPTH-DEPENDENT: a supports signal only, never a gate. Normalize with ``4^len`` and sampled-N
+    before interpreting (see the scorer).
+    """
+
+    start: int = Field(ge=0)
+    end: int = Field(ge=0)
+    distinct_ratio: Confidence
+    n_sampled: int = Field(ge=1)
+
+
+class ReadNameGrammar(BaseModel):
+    """Parsed Illumina header; all optional. ``sra_normalized`` flags an ``@SRR....N`` rewrite."""
+
+    parsed: bool = False
+    instrument: str | None = None
+    run: str | None = None
+    flowcell: str | None = None
+    lane: int | None = None
+    tile: int | None = None
+    index: str | None = None
+    sra_normalized: bool = False
+
+
+class GzipIntegrity(BaseModel):
+    """Gzip stream integrity. ``truncated`` -> TRUNCATED_GZIP Blocker downstream."""
+
+    ok: bool
+    truncated: bool
+    bgzf: bool | None = None
+    member_count: int | None = None
+
+
+class Observation(BaseModel):
+    """Structural, role-free probe output for one file, cached by ``file.sha256``."""
+
+    model_config = ConfigDict(frozen=True)
+
+    file: FileIdentity
+    probe: ProbeProvenance
+    per_cycle_composition: list[CycleComposition]
+    segments: list[Segment]
+    read_length: ReadLengthProfile
+    distinct_value_windows: list[WindowDistinctRatio]
+    read_name: ReadNameGrammar
+    quality_encoding: Literal["phred33", "phred64", "unknown"]
+    n_rate: Confidence
+    estimated_total_reads: int = Field(ge=0)
+    est_method: Literal["isize", "compressed_ratio"]
+    gzip: GzipIntegrity

seqforge/models/processing.py

@@ -0,0 +1,231 @@
+""":class:`ProcessingManifest` — one way to process a dataset. Many per dataset.
+
+A finished assay is immutable; what you *do* with it is a choice, and there are several defensible
+ones. So the manifest is two artifacts with two lifetimes:
+
+- :class:`~seqforge.models.dataset.DatasetManifest` — the **IR**. What the data *is*. One per dataset.
+- :class:`ProcessingManifest` — the **flags**. What to *do* with it. Many per dataset.
+
+That is the compiler metaphor doing work rather than decorating it. Same IR + different flags =
+different binaries; same dataset + different processing manifests = different pipelines, with the
+dataset hash unchanged. ``-O2`` does not get to edit the IR, and neither does anything in this file.
+
+**This section is intent, not truth, and it has no authority.** Every field is still ``Evidenced``,
+but for a different reason than in ``dataset``: there, ``basis`` records HOW WE KNOW; here it records
+WHO DECIDED. A corpus row reading "GeneFull because the user's instruction file said so" is
+categorically different from "GeneFull because policy defaults to all five", and that difference has
+to survive into the training corpus. ``user_confirmed`` — which has sat in the ``Basis`` literal since
+the beginning without a single writer — is the basis this section exists to carry.
+
+This module imports nothing from ``dataset``, and must not (see that module's header).
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from .base import Accession, Evidenced, NcbiTaxid, Sha256
+from .evidenced import EvidencedBool, EvidencedStr
+
+
+class GenomeRef(BaseModel):
+    """liulab-genome selection: UCSC assembly id + a REGISTERED GTF name. Never a path."""
+
+    assembly: str
+    annotation_name: str
+    ncbi_taxid: NcbiTaxid | None = None
+
+
+class EvidencedGenome(Evidenced[GenomeRef]):
+    """An ``Evidenced`` genome reference."""
+
+
+RuntimeEnv = Literal["align-rna", "align-dna", "ml", "ml-gpu"]
+"""A literal liulab-runtime environment name — the env name IS the identifier (no profile layer)."""
+
+
+class EvidencedRuntimeEnv(Evidenced[RuntimeEnv]):
+    """An ``Evidenced`` liulab-runtime environment name."""
+
+
+SoloFeature = Literal[
+    "Gene",
+    "SJ",
+    "GeneFull",
+    "GeneFull_ExonOverIntron",
+    "GeneFull_Ex50pAS",
+    "Velocyto",
+]
+"""STARsolo's complete ``--soloFeatures`` vocabulary.
+
+Closed on purpose, and it is the closure that does the safety work: ``verify.entails`` is **vacuous
+when value ⊆ quote**, so span-verification only bites for a controlled vocabulary. Because this one is closed,
+"aligned in GeneFull mode" entails ``GeneFull`` and "count introns too" does not — and the second
+rejection is the *right* answer, not a gap to paper over with aliases.
+"""
+
+
+class SoloQuant(BaseModel):
+    """STARsolo counting. An ORDERED list; element 0 is the primary matrix.
+
+    Order is a seqforge-side annotation with no aligner-side referent: STARsolo writes one
+    ``Solo.out/<Feature>/`` per value and does not care about order, so ``[Gene, GeneFull]`` and
+    ``[GeneFull, Gene]`` produce byte-identical output. What the order buys is a deterministic answer
+    to "which matrix is THE matrix" without a second field — and because it has no effect on the
+    aligner, ``compose`` projects it out to an explicit ``primary_feature`` key rather than leaving a
+    positional convention load-bearing for every downstream reader.
+
+    A known cells-vs-nuclei prep fact may only REORDER this list, never shorten it. Compute is spent
+    once, count matrices are small, and dropping a feature is the only irreversible act available.
+    """
+
+    kind: Literal["solo"] = "solo"
+    features: list[SoloFeature] = Field(min_length=1)
+
+    @model_validator(mode="after")
+    def _starsolo_rules(self) -> SoloQuant:
+        if len(set(self.features)) != len(self.features):
+            raise ValueError(f"duplicate soloFeatures: {self.features}")
+        # STARsolo's docs: "Velocyto quantification requires Gene features". A real aligner
+        # constraint, so it is a validator and not a comment — STAR would error out, but only AFTER
+        # the download and the alignment we were amortizing. Refuse first, with a remedy. This
+        # is also the clearest proof that a closed vocabulary is not by itself armor: no enum can
+        # express "this member requires that one".
+        if "Velocyto" in self.features and "Gene" not in self.features:
+            raise ValueError(
+                "STARsolo requires Gene features for Velocyto quantification; add 'Gene' to features"
+            )
+        return self
+
+
+class BulkQuant(BaseModel):
+    """Plain-STAR counting.
+
+    There is no strandedness knob here and there never needs to be: ``--quantMode GeneCounts``
+    already emits all three strand columns in ``ReadsPerGene.out.tab``. ``bulk-rnaseq-pe``'s spec
+    long carried a comment promising that "library strandedness is resolved at compose from
+    processing policy", and no code ever implemented it — because there was never a decision to make.
+    Same law as ``soloFeatures``: when the aligner computes every alternative in one pass and the
+    outputs are small, compute them all and let the consumer choose.
+    """
+
+    kind: Literal["bulk"] = "bulk"
+    mode: Literal["GeneCounts", "TranscriptomeSAM", "None"] = "GeneCounts"
+
+
+Quantification = Annotated[SoloQuant | BulkQuant, Field(discriminator="kind")]
+"""What to COUNT, discriminated by aligner family (the house style: cf. ``Segment``, ``Test``)."""
+
+
+class EvidencedQuantification(Evidenced[Quantification]):
+    """An ``Evidenced`` counting decision — the field that used to be decorative.
+
+    Policy set it to the string ``"gene"``, wrote it to the manifest, and ``compose`` then ignored it
+    and read ``soloFeatures`` from the KB instead: two sources of truth for one decision, unable to
+    disagree only because one was never consulted.
+    """
+
+
+class ResourceHints(BaseModel):
+    """Advisory resource requests for the workflow scheduler."""
+
+    threads: int = Field(ge=1, default=8)
+    mem_gb: int = Field(ge=1, default=32)
+    disk_gb: int | None = None
+    gpus: int = Field(ge=0, default=0)
+
+
+class ProcessingSection(BaseModel):
+    """INTENT — what we choose to do to a finished assay. Not a truth; no authority.
+
+    ``basis`` here records WHO DECIDED, on this ladder (highest first):
+
+    ==================  =================  ==========================
+    source              basis              evidence
+    ==================  =================  ==========================
+    a CLI flag          ``user_confirmed`` ``["cli:--quantify"]``
+    an instruction doc  ``user_confirmed`` ``["assert-..."]``
+    reference prose     ``asserted``       ``["assert-..."]``
+    a policy default    ``inferred``       ``["policy:<rule>"]``
+    ==================  =================  ==========================
+
+    The first two share a basis and differ only in precedence — both are the user talking to seqforge,
+    one just talks later — so the *channel* lives in ``evidence``. That is what design §1.0's open
+    note already prescribed, and it is why no ``policy_default`` basis is needed: once a section can
+    carry a **varying** basis, ``inferred`` plus an evidence ref naming the rule is distinguishable by
+    inspection.
+    """
+
+    # `extra="forbid"` is that enforcement at the model, not just at the gate. The instructable
+    # surface is *enumerated*; an unknown key must be a validation error, never a silent drop. It was
+    # a silent drop until 2026-07-15 — `ProcessingSection(soloStrand="Reverse")` constructed happily
+    # and discarded the field — which is pydantic's default, and the wrong default here: this is the
+    # artifact a user hands us, so an unrecognised key is either a typo or an attempt to reach a
+    # parse decision, and both deserve to fail loudly rather than be dropped on the floor.
+    model_config = ConfigDict(extra="forbid")
+
+    genome: EvidencedGenome
+    aligner: EvidencedStr
+    quantification: EvidencedQuantification
+    variant_calling: EvidencedBool
+    environment: EvidencedRuntimeEnv
+    resources: ResourceHints = Field(default_factory=ResourceHints)
+
+
+class DatasetPin(BaseModel):
+    """Which dataset a processing manifest is bound to."""
+
+    dataset_hash: Sha256
+    accessions: list[Accession] = Field(default_factory=list)  # human-readable, advisory only
+
+
+class ProcessingProvenance(BaseModel):
+    """Binds a processing manifest to the module source that will execute it."""
+
+    processing_hash: str
+    workflow_version: str
+    seqforge_version: str
+
+
+class ProcessingManifest(BaseModel):
+    """One way to process a dataset. Many per dataset — that plurality IS the design.
+
+    ``dataset is None`` => a **template**: it applies to any dataset, which is what lets one file drive
+    10^4 of them (this is scRecounter's uniform reprocessing, and it is the half of the design that a
+    mandatory pin would destroy — you would have 10^4 near-identical files, none of which anyone
+    reads, and the file would stop carrying signal).
+
+    ``dataset is not None`` => **bound**: ``compose`` refuses any dataset whose hash differs, with a
+    ``Blocker`` (exit 3), and never auto-repins.
+
+    ``compose`` always writes the bound form it actually used next to the config it produced, so the
+    default path leaves recoverable state on disk without demanding an input file. Disk is
+    *state*, not *input*.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    processing_id: str  # a human slug, e.g. "default-2026.7" / "genefull-primary"
+    dataset: DatasetPin | None = None
+    processing: ProcessingSection
+    provenance: ProcessingProvenance
+
+
+__all__ = [
+    "GenomeRef",
+    "EvidencedGenome",
+    "RuntimeEnv",
+    "EvidencedRuntimeEnv",
+    "SoloFeature",
+    "SoloQuant",
+    "BulkQuant",
+    "Quantification",
+    "EvidencedQuantification",
+    "ResourceHints",
+    "ProcessingSection",
+    "DatasetPin",
+    "ProcessingProvenance",
+    "ProcessingManifest",
+]

seqforge/models/records.py

@@ -0,0 +1,145 @@
+"""``ArchiveRecord`` — what a public archive *declares* about a dataset, before anyone interprets it.
+
+A record is not a truth and not a manifest. It is a transcript: this is what the submitter typed and
+the archive stored, split into the two halves that need different machinery.
+
+- ``attributes`` is the **structured** half (``strain = CQ758``). Code parses it. No model is
+  involved, and none is needed: it is already a key and a value.
+- ``free_text`` is the **prose** half (``"Rep3 daf2 reads"``, a study abstract, a protocol
+  paragraph). Code cannot parse it; that is job (a), and it is what harvest is for.
+
+**Every record is optional, and that is a requirement rather than an accident.** seqforge compiles
+FASTQ that arrives with an accession, FASTQ that arrives with a README, and FASTQ that arrives with
+nothing. There is no archive for a freshly-sequenced plate on a lab filesystem, so no code path may
+assume one exists, and "no record" must produce a quieter manifest rather than a refusal. What a
+record adds when it *is* there is per-sample subject identity — the thing a dataset-level document
+can never supply.
+
+**The hierarchy is the archive's, and the join is ours.** ``parent`` points one level up
+(run -> experiment -> sample -> project) and is copied out of the record, never inferred: the archive
+already knows which run came from which sample, and re-deriving that from filenames would be a guess
+where a fact was available. What code still has to decide is the last hop — record run to *file on
+disk* — because the archive does not know what you downloaded or what you named it.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+#: The four levels of the archive's own hierarchy. ``project`` is the study; ``sample`` is the
+#: biological material; ``experiment`` is the library prep; ``run`` is one sequencing run == the
+#: files. Other archives use other words for the same four things.
+RecordLevel = Literal["project", "sample", "experiment", "run"]
+
+
+class FreeText(BaseModel):
+    """One piece of prose from a record, and what the archive called it.
+
+    ``label`` is the archive's own field name (``sample_alias``, ``design_description``), kept so a
+    quote can be traced to the field it came out of rather than to an anonymous blob of text.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    label: str
+    text: str
+
+
+class RecordAttribute(BaseModel):
+    """One structured key/value a record declares.
+
+    ``harmonized`` records whether ``name`` is one of NCBI's 960 curated attribute names or the
+    submitter's own invention. Both are kept: an unharmonized attribute is a real thing the submitter
+    said, and dropping it would lose information, while promoting it into the controlled key space
+    would be a guess. Only a harmonized attribute may become a manifest sample fact.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    value: str
+    harmonized: bool = False
+    #: The submitter's raw tag, when it differs from ``name``. Provenance for the harmonization.
+    raw_name: str | None = None
+
+
+class ArchiveRecord(BaseModel):
+    """One level of one archive record, as fetched. A transcript, not an interpretation."""
+
+    model_config = ConfigDict(frozen=True)
+
+    level: RecordLevel
+    #: The record's id in whatever namespace produced it. Not typed as ``Accession``: this is the
+    #: archive's word for the record, and seqforge is not only ever handed NCBI accessions.
+    accession: str
+    #: The record one level up, by its own id. Copied from the record; never inferred.
+    parent: str | None = None
+    attributes: list[RecordAttribute] = Field(default_factory=list)
+    free_text: list[FreeText] = Field(default_factory=list)
+    #: Filenames the archive says this record's data was submitted as. The only thing that can join a
+    #: record to a file whose name no longer contains the accession.
+    filenames: list[str] = Field(default_factory=list)
+
+    def attribute(self, name: str) -> str | None:
+        """The value of a harmonized attribute, or ``None``. Never raises — absence is normal."""
+        for attr in self.attributes:
+            if attr.name == name and attr.harmonized:
+                return attr.value
+        return None
+
+    def text(self, label: str) -> str | None:
+        for ft in self.free_text:
+            if ft.label == label:
+                return ft.text
+        return None
+
+
+class ArchiveRecordSet(BaseModel):
+    """Every record fetched for one query, across all four levels.
+
+    Content-addressed and cached under the workspace: a record is a fact about the archive at a
+    moment, so re-fetching it should be a choice rather than a side effect of re-running.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    #: Which archive, and how. e.g. ``ncbi-sra+biosample``.
+    source: str
+    #: What was asked for. The accession a human typed.
+    query: str
+    records: list[ArchiveRecord] = Field(default_factory=list)
+
+    def at(self, level: RecordLevel) -> list[ArchiveRecord]:
+        return [r for r in self.records if r.level == level]
+
+    def by_accession(self, accession: str) -> ArchiveRecord | None:
+        for r in self.records:
+            if r.accession == accession:
+                return r
+        return None
+
+    def ancestor(self, record: ArchiveRecord, level: RecordLevel) -> ArchiveRecord | None:
+        """Walk ``parent`` up to ``level``. The join, and it is pure record-following.
+
+        Bounded by the number of records so a record set with a parent cycle (an archive bug, or a
+        hand-written one) terminates rather than hanging.
+        """
+        seen: set[str] = set()
+        current: ArchiveRecord | None = record
+        while current is not None and current.accession not in seen:
+            if current.level == level:
+                return current
+            seen.add(current.accession)
+            current = self.by_accession(current.parent) if current.parent else None
+        return None
+
+
+__all__ = [
+    "RecordLevel",
+    "FreeText",
+    "RecordAttribute",
+    "ArchiveRecord",
+    "ArchiveRecordSet",
+]