seqforge 2026.7.1__py3-none-any.whl

seqforge/models/resolve.py

@@ -0,0 +1,216 @@
+"""Score / compile output models — the wire formats between ``score`` and ``compile``.
+
+Every stage output is a first-class model so ``schema export`` references only types that exist and
+every stdout object round-trips through JSON Schema. ``TechScore`` is JSON-safe: no ``+/-inf`` ever
+crosses the JSON boundary.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from .base import Accession, Basis, ChemistryId, Rung, Sha256, Uri
+from .blocker import Blocker, ValidationWarning
+from .conflict import Conflict, ConflictPosition, Decidable
+from .evidenced import EvidencedStr, EvidencedTaxid
+from .processing import RuntimeEnv
+
+
+class TechScore(BaseModel):
+    """JSON-safe technology score. ``forbidden`` == a requires/excludes gate failed."""
+
+    technology: ChemistryId
+    status: Literal["forbidden", "scored"]
+    value: float | None = None
+    reason: str | None = None
+
+
+class RoleAssignment(BaseModel):
+    """The bipartite files->roles solution. ``assignment`` maps role_id -> file sha256."""
+
+    assignment: dict[str, str]
+    unassigned: list[str] = Field(default_factory=list)
+
+
+class Candidate(BaseModel):
+    """One ranked technology candidate with its role assignment and per-field deciding rungs."""
+
+    technology: ChemistryId
+    score: TechScore
+    role_assignment: RoleAssignment
+    rung_resolved: dict[str, int]
+    equivalence_members: list[ChemistryId] = Field(default_factory=list)
+    evidence: list[str] = Field(default_factory=list)
+
+
+class Question(BaseModel):
+    """A human-facing question. The code decides the option set; a human/agent picks among it."""
+
+    id: str
+    field: str
+    prompt: str
+    options: list[str]
+    decidable_by: list[Decidable]
+    rung: Rung
+
+
+class Decision(BaseModel):
+    """A persisted answer to an already-posed question (agents propose, code decides)."""
+
+    question_id: str
+    chosen: str
+    basis: Basis
+    actor: Literal["user", "agent", "code"]
+    evidence: list[str] = Field(default_factory=list)
+
+
+class ResolveResult(BaseModel):
+    """The output of ``resolve score``: ranked candidates, surfaced conflicts, and open questions."""
+
+    dataset_id: str
+    kb_version: str
+    rung_reached: Rung
+    candidates: list[Candidate]
+    conflicts: list[Conflict]
+    questions: list[Question]
+    blockers: list[Blocker] = Field(default_factory=list)
+
+
+class ResolvedSample(BaseModel):
+    """One biological sample, the files that carry it, and what we know about it.
+
+    ``sample_id`` always exists and is always code's: it is the archive's sample accession when a
+    record was joined, and the run grouping otherwise. There is no path on which a language model
+    names a sample — that is the whole reason a per-record document works.
+
+    ``attributes`` is keyed by an NCBI harmonized attribute name (``strain``, ``tissue``,
+    ``dev_stage``). Open-keyed rather than a fixed set of typed fields, because the key space is
+    NCBI's 960 and mirroring 960 names into pydantic fields is the hand-maintained contract this repo
+    keeps getting bitten by. Enforcement lives in the validator, against the shipped vocabulary.
+    """
+
+    sample_id: str
+    accession: Accession | None = None
+    attributes: dict[str, EvidencedStr] = Field(default_factory=dict)
+    #: The files carrying this sample, by content hash. ``fill`` turns these into manifest URIs; the
+    #: resolver does not know what a URI is and should not.
+    file_shas: list[Sha256] = Field(default_factory=list)
+
+
+class ProjectFacts(BaseModel):
+    """The study, as the archive declares it. Structured facts only (design decision, 2026-07-16).
+
+    Not ``Evidenced``: none of this is an interpretation. The record says the title is X and we copied
+    X, exactly as we copy a file's ``sha256`` — a basis and a confidence would be theatre. The study
+    *abstract* is deliberately absent: it is prose, it belongs in a document a quote can grep back
+    into, and pasting it into a content-addressed manifest would make the dataset's identity depend on
+    a paragraph of English.
+    """
+
+    accession: Accession | None = None
+    title: str | None = None
+    center: str | None = None
+    data_type: str | None = None
+    released: str | None = None
+
+
+class MetadataResolution(BaseModel):
+    """The output of the metadata resolver — the sibling of ``resolve score``, over records not bytes.
+
+    Same discipline and the same shape of answer, because it has the same ways of being wrong: it
+    emits evidenced values, and can refuse with a ``Blocker`` (a record whose runs do not match the
+    files on disk). A sample-attribute *disagreement* is different from a refusal: the resolver decides
+    it — a stronger authority wins, equal authorities leave the field null — so it is a non-blocking
+    ``warning``, not something that stops the dataset compiling. A dataset with no archive record and
+    no prose resolves to samples-with-no-facts, which is a real answer and the honest one — most
+    sequencing data has never had an accession.
+    """
+
+    samples: list[ResolvedSample] = Field(default_factory=list)
+    project: ProjectFacts | None = None
+    #: The organism, when a record or a document declares it. ``None`` means nobody said, and the
+    #: caller must supply it — never a default, because a wrong taxid aligns cleanly against the wrong
+    #: genome and nothing downstream ever asks again.
+    organism: EvidencedTaxid | None = None
+    #: Non-blocking notes on sample attributes the resolver decided under disagreement (precedence, or
+    #: null when it could not pick). Surfaced so the resolution is never silent; they never block.
+    warnings: list[ValidationWarning] = Field(default_factory=list)
+    blockers: list[Blocker] = Field(default_factory=list)
+
+
+class ArbitrationRequest(BaseModel):
+    """LLM job (b) INPUT schema (opt-in ``resolve adjudicate``)."""
+
+    conflict_id: str
+    positions: list[ConflictPosition]
+
+
+class ArbitrationResponse(BaseModel):
+    """LLM job (b) OUTPUT schema — references a position by index, re-derives no values."""
+
+    conflict_id: str
+    chosen_index: int
+    rationale: str
+
+
+class ValidationReport(BaseModel):
+    """The output of ``manifest validate``."""
+
+    ok: bool
+    blockers: list[Blocker]
+    conflicts: list[Conflict]
+    warnings: list[ValidationWarning] = Field(default_factory=list)
+
+
+class ModuleSelection(BaseModel):
+    """One selected, versioned workflow module and the runtime env it runs in."""
+
+    name: str
+    version: str
+    env: RuntimeEnv
+
+
+class ComposeResult(BaseModel):
+    """The output of ``compose``: selected modules, emitted config paths, and the gate verdicts.
+
+    ``gate`` maps a gate name (``params`` / ``wiring`` / ``e2e``) to its verdict. ``skip`` is
+    first-class and distinct from ``pass``: the wiring and e2e gates depend on a toolchain seqforge
+    does not own (snakemake; STAR + liulab-genome + network), and a gate that reports ``pass``
+    because it never ran would let green CI be mistaken for coverage.
+    """
+
+    modules: list[ModuleSelection]
+    #: The run wrapper — **the thing a user submits**, and the reason `compose` exists. It is named
+    #: here beside the config rather than left to be discovered on disk: it was previously written as
+    #: a side effect of a gate that could not run, so `compose` reported success while emitting no
+    #: runnable artifact at all. A deliverable absent from the result object is a deliverable nobody
+    #: notices is missing.
+    snakefile_path: Uri
+    config_path: Uri
+    units_path: Uri
+    gate: dict[str, Literal["pass", "fail", "skip"]]
+    params_preview: dict[str, object]
+
+
+class RunResult(BaseModel):
+    """The output of the headless ``run`` entrypoint."""
+
+    dataset_id: str
+    stages: dict[str, str]
+    exit: int
+    artifacts: dict[str, Uri]
+    provenance_id: str
+
+
+class EvalReport(BaseModel):
+    """The output of ``eval run``: the metrics tracked on every prompt/KB/resolve change."""
+
+    n_cases: int
+    field_accuracy: float
+    false_accept_rate: float
+    false_refuse_rate: float
+    questions_asked: dict[str, float]
+    cost: dict[str, float]
+    per_case: list[dict[str, object]]

seqforge/probe/__init__.py

@@ -0,0 +1,46 @@
+"""``probe`` — deterministic, bounded FASTQ fingerprinting (no LLM, no network).
+
+Turns bytes into an :class:`~seqforge.models.Observation` from a bounded, head-limited decompressed
+stream. Every touch is bounded by a read budget (``--max-reads``, default 200 000) and a byte cap
+(``--max-bytes``, default 256 MB decompressed); wall-clock is never a budget, and a code path that
+*can* stream a whole multi-GB FASTQ is a bug.
+
+Tier A (this module) computes structural signals with no KB: per-cycle base composition, segmentation
+(constant/random/homopolymer), read-length profile, distinct-value ratios, read-name grammar,
+N-rate, quality encoding, gzip integrity, and an extrapolated read-count estimate. It assigns **no
+roles** — that interpretation belongs to ``resolve``.
+"""
+
+from __future__ import annotations
+
+#: CalVer YYYY.M.PATCH, bumped whenever probe output semantics change; folded into the Observation
+#: cache key. Component/tool-stamp versions use CalVer just like the package version.
+#: 2026.7.1 — content-address from a bounded local key (head + size + gzip ISIZE), not a whole-file
+#: sha256 (issue #37); the file identity string changes, so cached observations recompute once.
+PROBE_VERSION = "2026.7.1"
+
+#: Default bounded-read budget. Overridable per call by the CLI.
+DEFAULT_MAX_READS = 200_000
+
+#: Default decompressed-byte cap. Whichever budget trips first stops the stream.
+DEFAULT_MAX_BYTES = 256 * 1024 * 1024
+
+# Imported last: core depends on the budget constants above (keeps the package import acyclic).
+from .core import (  # noqa: E402
+    build_observation,
+    content_key_from_md5,
+    probe_file,
+    probe_sample,
+    remote_content_key,
+)
+
+__all__ = [
+    "PROBE_VERSION",
+    "DEFAULT_MAX_READS",
+    "DEFAULT_MAX_BYTES",
+    "build_observation",
+    "content_key_from_md5",
+    "remote_content_key",
+    "probe_file",
+    "probe_sample",
+]

seqforge/probe/core.py

@@ -0,0 +1,232 @@
+"""``probe_file`` — orchestrate bounded streaming + Tier A signals into an :class:`Observation`."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from pathlib import Path
+from typing import Literal
+
+from ..models.observation import (
+    FileIdentity,
+    GzipIntegrity,
+    Observation,
+    ProbeProvenance,
+)
+from . import DEFAULT_MAX_BYTES, DEFAULT_MAX_READS, PROBE_VERSION
+from . import signals as sig
+from .streaming import StreamSample, sample_fastq_gz
+
+
+def _content_key(basename: str, size_bytes: int, isize: int | None, seqs: list[str]) -> str:
+    """Content-address a FASTQ from bounded, already-sampled data — never a whole-file read.
+
+    A file's identity here is a *name*: stable for the same file, distinct across files. It combines
+    the basename, the compressed size, the gzip ISIZE trailer (uncompressed size mod 2^32), and a hash
+    of the bounded head sample — all in hand after :func:`~seqforge.probe.streaming.sample_fastq_gz`,
+    so no extra bytes are read. The basename is part of the identity because a dataset's files are
+    distinguished by name (``_1``/``_2``, lane, flowcell): two files with identical reads but different
+    names are different files, and downstream maps (``dataset_uris``, role assignment) require one
+    sha per file. The whole-file sha256 this replaces captured the name incidentally (the gzip filename
+    header) and forced the entire file to be read — which was never the point (issue #37). At
+    10^4-dataset scale the durable identity is the provider md5, injected via
+    ``probe_sample(..., sha256=...)``.
+    """
+    h = hashlib.sha256()
+    h.update(
+        f"seqforge-content-key\x00{basename}\x00{size_bytes}\x00{isize}\x00{len(seqs)}\n".encode()
+    )
+    for s in seqs:
+        h.update(s.encode("ascii", "replace"))
+        h.update(b"\n")
+    return h.hexdigest()
+
+
+def content_key_from_md5(md5: str) -> str:
+    """Derive the 64-hex content-address of a file whose PROVIDER md5 is known (issue #39).
+
+    ENA/SRA publish a per-file md5 over the *hosted* bytes. It is 32 hex, but a
+    :class:`~seqforge.models.observation.FileIdentity` ``sha256`` is a 64-hex content-address — a
+    *name*, not a recomputed file hash (see :func:`_content_key`). This maps the provider md5 into that
+    space injectively: identical md5 -> identical address, so two hosted files with the same md5 dedup
+    correctly, and **no byte of the file is read**. Unlike the local key it carries no basename — for
+    hosted bytes an identical md5 legitimately means identical content. This is the durable, machine-
+    independent identity a remote probe (``io.remote.probe_remote``) stamps via ``sha256=``.
+    """
+    m = md5.strip().lower()
+    if not re.fullmatch(r"[0-9a-f]{32}", m):
+        raise ValueError(f"not a 32-hex md5: {md5!r}")
+    return hashlib.sha256(f"seqforge-provider-md5\x00{m}".encode()).hexdigest()
+
+
+def remote_content_key(basename: str, size_bytes: int, seqs: list[str]) -> str:
+    """A bounded content key for a REMOTE file with no provider md5 — the local key minus the ISIZE.
+
+    The gzip ISIZE trailer is unreachable without the file's tail, so a remote fingerprint passes
+    ``isize=None``; otherwise this is exactly :func:`_content_key` (basename + size + head sample).
+    Prefer :func:`content_key_from_md5` whenever the provider md5 is known — that is the durable
+    identity that matches the hosted bytes.
+    """
+    return _content_key(basename, size_bytes, None, seqs)
+
+
+def _params_hash(max_reads: int, max_bytes: int) -> str:
+    return hashlib.sha256(f"{max_reads}:{max_bytes}".encode()).hexdigest()[:16]
+
+
+def _gzip_isize(path: Path) -> int | None:
+    """The gzip ISIZE trailer: uncompressed size mod 2^32 (O(1); unreliable for >4GB / multi-member)."""
+    try:
+        with open(path, "rb") as fh:
+            fh.seek(-4, 2)
+            return int.from_bytes(fh.read(4), "little")
+    except OSError:
+        return None
+
+
+def _estimate_reads(
+    file_size: int,
+    n_reads: int,
+    decompressed_bytes: int,
+    compressed_bytes: int,
+    budget_exhausted: bool,
+    isize: int | None,
+) -> tuple[int, Literal["isize", "compressed_ratio"]]:
+    """Extrapolate total reads without reading the whole file.
+
+    Prefer the gzip ISIZE trailer (uncompressed size / average record size); fall back to the
+    compressed-size ratio. If the whole (small) file was read, the sampled count is exact. ``isize``
+    is read once by the caller (an O(1) seek) and shared with the content key.
+    """
+    if n_reads == 0:
+        return 0, "compressed_ratio"
+    if not budget_exhausted:
+        return n_reads, "isize"  # read to EOF: the count is exact
+    avg_record = decompressed_bytes / n_reads
+    if isize is not None and avg_record > 0 and isize > decompressed_bytes:
+        return int(isize / avg_record), "isize"
+    if compressed_bytes > 0:
+        return int(file_size * n_reads / compressed_bytes), "compressed_ratio"
+    return n_reads, "compressed_ratio"
+
+
+def build_observation(
+    sample: StreamSample,
+    *,
+    size_bytes: int,
+    sha256: str,
+    basename: str,
+    local_uri: str | None,
+    isize: int | None,
+    max_reads: int = DEFAULT_MAX_READS,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+) -> tuple[Observation, list[str]]:
+    """Assemble the Tier-A :class:`Observation` from an already-sampled ``StreamSample``.
+
+    The pure, source-agnostic core of a probe: it runs the signal pipeline over ``sample.seqs`` and
+    stamps identity/provenance, reading no bytes itself. Both a local probe (:func:`probe_sample`,
+    ``sample`` from a file) and a remote probe (``io.remote.probe_remote``, ``sample`` from a bounded
+    range-read prefix) call it, so a URL resolves to a library exactly as a local file does. ``sha256``
+    is the fully-formed 64-hex content-address the caller chose (a provider md5 via
+    :func:`content_key_from_md5`, or a bounded local/remote key via :func:`_content_key`); ``isize`` is
+    the gzip ISIZE trailer when reachable (local) and ``None`` when it is not (a remote prefix has no
+    tail), which simply falls the read estimate back to the compressed-size ratio.
+    """
+    comps = sig.per_cycle_composition(sample.seqs)
+    segments = sig.segment(comps)
+    read_length = sig.read_length_profile(sample.seqs)
+    windows = sig.distinct_ratios(sample.seqs, segments)
+    read_name = sig.parse_read_name(sample.first_name)
+    quality = sig.quality_encoding(sample.qual_min_ord, sample.qual_max_ord)
+    nrate = sig.n_rate(sample.seqs)
+
+    estimated_total, est_method = _estimate_reads(
+        size_bytes,
+        sample.n_reads,
+        sample.decompressed_bytes,
+        sample.compressed_bytes,
+        sample.budget_exhausted,
+        isize,
+    )
+
+    observation = Observation(
+        file=FileIdentity(
+            sha256=sha256,
+            size_bytes=size_bytes,
+            basename=basename,
+            local_uri=local_uri,
+        ),
+        probe=ProbeProvenance(
+            n_reads_sampled=sample.n_reads,
+            bytes_read=sample.decompressed_bytes,
+            compressed_bytes_read=sample.compressed_bytes,
+            tool_version=PROBE_VERSION,
+            params_hash=_params_hash(max_reads, max_bytes),
+        ),
+        per_cycle_composition=comps,
+        segments=segments,
+        read_length=read_length,
+        distinct_value_windows=windows,
+        read_name=read_name,
+        quality_encoding=quality,
+        n_rate=nrate,
+        estimated_total_reads=estimated_total,
+        est_method=est_method,
+        gzip=GzipIntegrity(ok=sample.ok, truncated=sample.truncated),
+    )
+    return observation, sample.seqs
+
+
+def probe_sample(
+    path: str | Path,
+    *,
+    max_reads: int = DEFAULT_MAX_READS,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    sha256: str | None = None,
+) -> tuple[Observation, list[str]]:
+    """Fingerprint one LOCAL FASTQ gzip and ALSO return its bounded sampled sequences.
+
+    :class:`Observation` is structural + role-free and cached to disk; the raw sampled ``seqs`` are
+    the same bounded, in-memory sample used to build it. ``resolve`` needs those seqs to answer
+    role-conditioned distinct-ratio / onlist-hit-rate over arbitrary windows (a ``WindowProbe``),
+    which the structural Observation deliberately does not carry. The sample stays within the
+    budget — this returns it, it does not re-read the file.
+    """
+    p = Path(path)
+    sample = sample_fastq_gz(p, max_reads=max_reads, max_bytes=max_bytes)
+    file_size = p.stat().st_size
+    isize = _gzip_isize(p)
+    return build_observation(
+        sample,
+        size_bytes=file_size,
+        sha256=sha256 or _content_key(p.name, file_size, isize, sample.seqs),
+        basename=p.name,
+        local_uri=str(p),
+        isize=isize,
+        max_reads=max_reads,
+        max_bytes=max_bytes,
+    )
+
+
+def probe_file(
+    path: str | Path,
+    *,
+    max_reads: int = DEFAULT_MAX_READS,
+    max_bytes: int = DEFAULT_MAX_BYTES,
+    sha256: str | None = None,
+) -> Observation:
+    """Fingerprint one FASTQ gzip into a role-free :class:`Observation` under a bounded budget.
+
+    Parameters
+    ----------
+    path
+        Local path to a gzip-compressed FASTQ.
+    max_reads, max_bytes
+        The read budget and decompressed-byte cap.
+    sha256
+        Precomputed content identity (e.g. a provider md5); if omitted, a bounded local content key is
+        derived from the head sample + size + gzip ISIZE (see :func:`_content_key`) — never a
+        whole-file read.
+    """
+    observation, _seqs = probe_sample(path, max_reads=max_reads, max_bytes=max_bytes, sha256=sha256)
+    return observation