seqforge 2026.7.1__py3-none-any.whl

seqforge/resolve/evaluators.py

@@ -0,0 +1,306 @@
+"""Signature-test evaluators — the CLOSED set that mirrors ``kb.schema`` exactly (§3.1).
+
+``evaluate(test, read, wp, spec, registry)`` returns an :class:`Evaluation` carrying both a gate
+``outcome`` (``PASS`` / ``FAIL`` / ``ABSTAIN``) and a supports ``score`` in ``[0, 1]``. The caller
+uses ``outcome`` for ``requires`` / ``excludes`` gates and ``score`` for ``supports`` weighting.
+
+Two invariants hold regardless of where a test is placed:
+- **``ABSTAIN`` never gates.** "The probe cannot see this signal" is not "the signal is absent" — an
+  abstaining requires/excludes test is a pass-through, not a rejection (an SRA-normalized
+  ``header_index`` must not reject every SRA dataset).
+- **``distinct_ratio`` never gates.** It is depth-dependent: its gate outcome is forced to ``ABSTAIN``
+  so a misplaced ``requires`` cannot use it; its supports ``score`` remains meaningful.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import StrEnum
+
+from ..io import OnlistNotAvailable, OnlistRegistry
+from ..kb.schema import (
+    BaseComposition,
+    DistinctRatio,
+    Element,
+    HasSegment,
+    HeaderIndex,
+    MotifPresent,
+    OnlistHitRate,
+    Read,
+    SegmentLength,
+    Spec,
+)
+from ..models.observation import CycleComposition
+from .window import WindowProbe
+
+_CONSTANT_PURITY = 0.9  # mean max-base fraction over a window to call it "constant sequence"
+_RANDOM_MAXFRAC = 0.55  # mean max-base fraction below this is "near-uniform random"
+
+
+class Outcome(StrEnum):
+    PASS = "pass"
+    FAIL = "fail"
+    ABSTAIN = "abstain"
+
+
+@dataclass(frozen=True)
+class Evaluation:
+    """A test's gate ``outcome`` and its supports ``score`` in ``[0, 1]``, with a short reason."""
+
+    outcome: Outcome
+    score: float
+    detail: str = ""
+    #: True iff a real (materialized) onlist was consulted — lifts the deciding rung to 3.
+    used_onlist: bool = False
+
+
+def _clip(x: float) -> float:
+    return max(0.0, min(1.0, x))
+
+
+def _window_for(test: object, read: Read) -> tuple[int, int | None]:
+    """Resolve a test's FIXED target window from ``element`` name XOR explicit ``(start, end)``.
+
+    Only valid for fixed-offset elements. A **floating** element has no constant window — its per-read
+    frame is resolved by :meth:`WindowProbe.anchored_windows`, and the callers below route to it via
+    :func:`_anchored_element` before ever reaching here.
+    """
+    element = getattr(test, "element", None)
+    if element is not None:
+        for el in read.elements:
+            if el.name == element:
+                return (el.start or 0), el.end
+        return 0, None
+    start = getattr(test, "start", None)
+    end = getattr(test, "end", None)
+    return (start or 0), end
+
+
+def _anchored_element(test: object, read: Read) -> Element | None:
+    """The floating element a test targets by name, or ``None`` if it targets a fixed one / coordinates.
+
+    This is where ``el.anchor`` — dropped by every consuming layer before #43 — is finally read on the
+    scoring path: an ``onlist_hit_rate`` / ``distinct_ratio`` addressed to an anchored element must be
+    answered over the per-read frame, not a constant column.
+    """
+    name = getattr(test, "element", None)
+    if name is None:
+        return None
+    for el in read.elements:
+        if el.name == name and el.anchor is not None:
+            return el
+    return None
+
+
+def _mean_max_fraction(wp: WindowProbe, start: int, end: int | None) -> float | None:
+    comps = wp.composition_window(start, end)
+    if not comps:
+        return None
+    total = 0.0
+    for c in comps:
+        total += max(c.a, c.c, c.g, c.t)
+    return total / len(comps)
+
+
+def _base_fraction(c: CycleComposition, base: str) -> float:
+    return {"A": c.a, "C": c.c, "G": c.g, "T": c.t, "N": c.n}[base]
+
+
+def _mean_base_fraction(wp: WindowProbe, start: int, end: int | None, base: str) -> float | None:
+    comps = wp.composition_window(start, end)
+    if not comps:
+        return None
+    return sum(_base_fraction(c, base) for c in comps) / len(comps)
+
+
+def evaluate(
+    test: object, read: Read, wp: WindowProbe, spec: Spec, registry: OnlistRegistry
+) -> Evaluation:
+    """Evaluate one signature test against a file's :class:`WindowProbe`."""
+    if isinstance(test, SegmentLength):
+        return _eval_segment_length(test, wp)
+    if isinstance(test, HasSegment):
+        return _eval_has_segment(test, read, wp)
+    if isinstance(test, DistinctRatio):
+        return _eval_distinct_ratio(test, read, wp)
+    if isinstance(test, OnlistHitRate):
+        return _eval_onlist(test, read, wp, spec, registry)
+    if isinstance(test, MotifPresent):
+        return _eval_motif(test, wp)
+    if isinstance(test, BaseComposition):
+        return _eval_base_composition(test, read, wp)
+    if isinstance(test, HeaderIndex):
+        return _eval_header_index(test, wp)
+    # read_count is a dataset-level global, handled by the assignment feasibility check.
+    return Evaluation(Outcome.ABSTAIN, 0.0, "not a per-cell test")
+
+
+def _eval_segment_length(test: SegmentLength, wp: WindowProbe) -> Evaluation:
+    mode = wp.mode_length
+    if mode == 0:
+        return Evaluation(Outcome.FAIL, 0.0, "no reads")
+    if test.over_length_min is not None and mode >= test.over_length_min:
+        # An over-sequenced / insert-bearing barcode read: CB+UMI live at the declared offsets and the
+        # trailing bases are junk STARsolo ignores. Canonical exactness is preserved because
+        # over_length_min sits strictly above `length` (a 28 bp read is never "over-length").
+        return Evaluation(
+            Outcome.PASS, 1.0, f"mode={mode} >= over_length_min={test.over_length_min}"
+        )
+    diff = abs(mode - test.length)
+    outcome = Outcome.PASS if diff <= test.tolerance else Outcome.FAIL
+    score = _clip(1.0 - diff / max(1.0, test.length * 0.1))
+    return Evaluation(outcome, score, f"mode={mode} vs {test.length}±{test.tolerance}")
+
+
+def _eval_has_segment(test: HasSegment, read: Read, wp: WindowProbe) -> Evaluation:
+    start, end = _window_for(test, read)
+    if test.kind in ("constant", "random"):
+        mmf = _mean_max_fraction(wp, start, end)
+        if mmf is None:
+            return Evaluation(Outcome.ABSTAIN, 0.0, "window unreadable")
+        if test.kind == "constant":
+            outcome = Outcome.PASS if mmf >= _CONSTANT_PURITY else Outcome.FAIL
+            return Evaluation(outcome, _clip((mmf - 0.5) / 0.4), f"mean_maxfrac={mmf:.2f}")
+        outcome = Outcome.PASS if mmf <= _RANDOM_MAXFRAC else Outcome.FAIL
+        return Evaluation(outcome, _clip((_RANDOM_MAXFRAC - mmf) / 0.3), f"mean_maxfrac={mmf:.2f}")
+    base = "T" if test.kind == "polyT" else "A"
+    frac = _mean_base_fraction(wp, start, end, base)
+    if frac is None:
+        return Evaluation(Outcome.ABSTAIN, 0.0, "window unreadable")
+    outcome = Outcome.PASS if frac >= 0.8 else Outcome.FAIL
+    return Evaluation(outcome, _clip(frac), f"{base}-fraction={frac:.2f}")
+
+
+def _eval_distinct_ratio(test: DistinctRatio, read: Read, wp: WindowProbe) -> Evaluation:
+    """SUPPORTS-only: the gate outcome is forced to ABSTAIN so it can never gate (depth-dependent)."""
+    anchored = _anchored_element(test, read)
+    if anchored is not None:
+        ratio = wp.anchored_distinct_ratio(read, anchored.name)
+        detail = "anchored "
+    else:
+        start, end = _window_for(test, read)
+        if end is None:
+            return Evaluation(Outcome.ABSTAIN, 0.0, "open-ended window")
+        ratio = wp.distinct_ratio(start, end)
+        detail = ""
+    if ratio is None:
+        return Evaluation(Outcome.ABSTAIN, 0.0, "window unreadable")
+    score = _clip(1.0 - ratio) if test.expect == "low" else _clip(ratio)
+    return Evaluation(
+        Outcome.ABSTAIN, score, f"{detail}distinct_ratio={ratio:.3f} expect={test.expect}"
+    )
+
+
+def _eval_onlist(
+    test: OnlistHitRate, read: Read, wp: WindowProbe, spec: Spec, registry: OnlistRegistry
+) -> Evaluation:
+    ref = spec.onlists.get(test.onlist)
+    if ref is None:
+        return Evaluation(Outcome.ABSTAIN, 0.0, f"unknown onlist alias {test.onlist!r}")
+    if not registry.has(ref.registry):
+        return Evaluation(Outcome.ABSTAIN, 0.0, f"onlist {ref.registry!r} not registered")
+    try:
+        packed = registry.packed(ref.registry)
+    except OnlistNotAvailable:
+        return Evaluation(Outcome.ABSTAIN, 0.0, f"onlist {ref.registry!r} not materialized")
+    anchored = _anchored_element(test, read)
+    if anchored is not None:
+        hit = wp.anchored_onlist_hit(read, anchored.name, packed, orientation=test.orientation)
+    else:
+        start, _ = _window_for(test, read)
+        hit = wp.onlist_hit(start, packed, orientation=test.orientation)
+    outcome = Outcome.PASS if hit.hit_rate >= test.min else Outcome.FAIL
+    detail = f"hit={hit.hit_rate:.2f} min={test.min} {hit.orientation}@Δ{hit.offset} floor={hit.floor:.1e}"
+    return Evaluation(outcome, hit.score(test.min), detail, used_onlist=True)
+
+
+#: The over-length admission (scoring._over_length_admitted_by_onlist) asks a NARROWER question than
+#: the onlist support gate. The support ``min`` (e.g. v2's 0.6) means "does this read carry confident,
+#: 1MM-correctable barcodes?" — the bar STARsolo's own CB correction is measured against. Admission
+#: asks only "is this over-sequenced dead-zone read barcode-bearing rather than cDNA?" A cDNA (or any
+#: non-barcode) read hits a barcode whitelist at its CHANCE FLOOR — ``n_entries / 4**width`` ≈ 1e-4 to
+#: ~2e-3 for the 10x whitelists — with negligible variance over a 200k-read sample, so a rate a few
+#: hundredfold above the floor is decisive. Seqforge matches barcodes EXACTLY (no 1MM correction), so a
+#: real over-sequenced barcode read with ordinary sequencing error sits well under 0.6 yet vastly above
+#: the floor (GSE126954's SRX5411291 — the perfect-whitelist fixtures hit ~1.0 and never exposed this).
+#: The floor-anchored bar admits it without ever admitting a same-length cDNA read.
+_OVERLENGTH_ADMISSION_MIN = 0.05  # absolute floor: a meaningful barcode signal, not chance
+_OVERLENGTH_ADMISSION_FLOOR_MULT = 50.0  # and always well clear of THIS whitelist's chance floor
+
+
+def onlist_admits_over_length(
+    test: OnlistHitRate, read: Read, wp: WindowProbe, spec: Spec, registry: OnlistRegistry
+) -> bool:
+    """True iff an onlist test's window hits the whitelist far enough above chance to call the read
+    barcode-bearing (not cDNA) — the floor-anchored admission bar, NOT the support ``min``.
+
+    Mirrors :func:`_eval_onlist`'s window/whitelist resolution exactly, but decides on a lower,
+    floor-derived threshold. Any unresolved / unmaterialized onlist yields ``False`` (no admission
+    without a whitelist to check against), matching the gate's ABSTAIN.
+    """
+    ref = spec.onlists.get(test.onlist)
+    if ref is None or not registry.has(ref.registry):
+        return False
+    try:
+        packed = registry.packed(ref.registry)
+    except OnlistNotAvailable:
+        return False
+    anchored = _anchored_element(test, read)
+    if anchored is not None:
+        hit = wp.anchored_onlist_hit(read, anchored.name, packed, orientation=test.orientation)
+    else:
+        start, _ = _window_for(test, read)
+        hit = wp.onlist_hit(start, packed, orientation=test.orientation)
+    bar = max(_OVERLENGTH_ADMISSION_MIN, hit.floor * _OVERLENGTH_ADMISSION_FLOOR_MULT)
+    return hit.hit_rate >= bar
+
+
+def _eval_motif(test: MotifPresent, wp: WindowProbe) -> Evaluation:
+    rate = wp.motif_rate(
+        test.motif,
+        where=test.where,
+        search_start=test.search_start,
+        search_end=test.search_end,
+        max_mismatch=test.max_mismatch,
+    )
+    if rate is None:
+        return Evaluation(Outcome.ABSTAIN, 0.0, "no reads long enough")
+    outcome = Outcome.PASS if rate >= test.min_rate else Outcome.FAIL
+    return Evaluation(outcome, _clip(rate / max(test.min_rate, 1e-9)), f"motif_rate={rate:.2f}")
+
+
+def _eval_base_composition(test: BaseComposition, read: Read, wp: WindowProbe) -> Evaluation:
+    start, end = _window_for(test, read)
+    frac = _mean_base_fraction(wp, start, end, test.base)
+    if frac is None:
+        return Evaluation(Outcome.ABSTAIN, 0.0, "window unreadable")
+    outcome = Outcome.PASS if frac >= test.min_fraction else Outcome.FAIL
+    return Evaluation(
+        outcome, _clip(frac / max(test.min_fraction, 1e-9)), f"{test.base}={frac:.2f}"
+    )
+
+
+def _eval_header_index(test: HeaderIndex, wp: WindowProbe) -> Evaluation:
+    grammar = wp.observation.read_name
+    if grammar.sra_normalized:
+        return Evaluation(Outcome.ABSTAIN, 0.0, "SRA-normalized header (index stripped)")
+    has_index = grammar.index is not None
+    outcome = Outcome.PASS if has_index == test.present else Outcome.FAIL
+    return Evaluation(outcome, 1.0 if has_index == test.present else 0.0, f"has_index={has_index}")
+
+
+def read_length_compatible(read: Read, wp: WindowProbe) -> Outcome:
+    """Implicit per-role gate from the Read's declared ``min_len`` / ``max_len`` (a real requires).
+
+    A file can fill a role only if its mode length is compatible with the read's declared geometry:
+    a fixed read demands an exact mode; a variable read demands ``mode >= min_len`` (and ``<= max_len``
+    when declared). An empty file fails (it cannot fill any role).
+    """
+    mode = wp.mode_length
+    if mode == 0:
+        return Outcome.FAIL
+    if read.min_len is not None and mode < read.min_len:
+        return Outcome.FAIL
+    if read.max_len is not None and mode > read.max_len:
+        return Outcome.FAIL
+    return Outcome.PASS

seqforge/resolve/geometry.py

@@ -0,0 +1,89 @@
+"""Byte-derived geometry feasibility — the winner-invariance foundation for descent scoring.
+
+``length_feasible(spec, wps)`` asks the necessary question the full scorer asks first: can this spec's
+reads be assigned, one-to-one, to length-compatible files? It reuses the scorer's own
+``read_length_compatible`` gate (:mod:`.evaluators`) and its bipartite matcher (:mod:`.assign`), so a
+spec it rejects is one ``build_tech_evaluation`` would also reject — ``_score_cell`` forbids a cell
+*unconditionally* on ``read_length_compatible == FAIL`` before any onlist logic runs (scoring.py). That
+makes it a **proven necessary condition** for a valid score: narrowing the candidate set to
+length-feasible specs can never drop a spec that full scoring would have made a winner. Descent scoring
+and the confusability CI guard both stand on this.
+
+``geometry_fingerprint`` is a coarser, dataset-independent key for grouping and diagnostics only;
+``length_feasible`` is the correctness predicate.
+"""
+
+from __future__ import annotations
+
+import json
+
+from ..kb.schema import SegmentLength, Spec
+from .assign import best_assignment
+from .evaluators import Outcome, read_length_compatible
+from .window import WindowProbe
+
+
+def length_feasible(spec: Spec, wps: list[WindowProbe]) -> bool:
+    """True iff every read can be matched one-to-one to a length-compatible file.
+
+    Uses only ``read_length_compatible`` — the scorer's first, unconditional gate — so a ``False`` here
+    provably implies ``build_tech_evaluation(spec, wps, ...).valid is False``. The converse does not
+    hold (a length-feasible spec may still be forbidden by a finer requires/excludes test); that is
+    correct for a necessary condition and simply means the spec is scored and rejected normally.
+    """
+    n_roles = len(spec.reads)
+    n_files = len(wps)
+    if n_files < n_roles:
+        return False
+    forbidden = [
+        [read_length_compatible(read, wp) == Outcome.FAIL for wp in wps] for read in spec.reads
+    ]
+    score = [[0.0] * n_files for _ in range(n_roles)]
+    prior = [[0.0] * n_files for _ in range(n_roles)]
+    return best_assignment(n_roles, n_files, score, forbidden, prior).valid
+
+
+def geometry_could_accept(spec: Spec, probes: list[object]) -> bool:
+    """Pairwise necessary condition for ``accepts_at_rungs_0_2(spec, probes)`` (confuse.py).
+
+    Validity requires length-feasibility, so this is a sound skip for the confusability CI guard: a
+    ``False`` here means the pair cannot be confusable at rungs 0-2, and the full scorer need not run.
+    Mirrors ``accepts_at_rungs_0_2``'s ``WindowProbe`` filtering so it is a drop-in guard.
+    """
+    wps = [p for p in probes if isinstance(p, WindowProbe)]
+    return length_feasible(spec, wps)
+
+
+def geometry_fingerprint(spec: Spec) -> str:
+    """A deterministic, dataset-independent key grouping specs by read geometry (diagnostics only).
+
+    File order is irrelevant (reads are sorted by a canonical descriptor), so two specs that differ
+    only in which mate is R1 vs R2 collide. Not a correctness predicate — see :func:`length_feasible`.
+    """
+    seg_by_read: dict[str, SegmentLength] = {
+        t.read: t for t in spec.signature.requires if isinstance(t, SegmentLength)
+    }
+    reads_desc: list[dict[str, object]] = []
+    for read in spec.reads:
+        el_types = {el.type for el in read.elements}
+        if el_types & {"cdna", "gdna"}:
+            kind = f"bio:{read.strand}"
+        elif "barcode" in el_types:
+            kind = "barcode"
+        else:
+            kind = "other"
+        seg = seg_by_read.get(read.id)
+        reads_desc.append(
+            {
+                "kind": kind,
+                "min_len": read.min_len,
+                "max_len": read.max_len,
+                "seg_length": seg.length if seg else None,
+                "over_length_min": seg.over_length_min if seg else None,
+                "open_ended": any(
+                    el.type in ("cdna", "gdna") and el.end is None for el in read.elements
+                ),
+            }
+        )
+    reads_desc.sort(key=lambda d: json.dumps(d, sort_keys=True))
+    return json.dumps({"n_reads": len(spec.reads), "reads": reads_desc}, sort_keys=True)

seqforge/resolve/group.py

@@ -0,0 +1,85 @@
+"""Split a pile of FASTQ files into the **runs** they came from, by filename.
+
+**Filenames lie about roles. They do not lie about identity — and the difference is the whole design
+of this module.**
+
+`fasterq-dump`'s `_1` / `_2` suffixes say nothing about which read is the barcode and which is the
+cDNA: they are an artifact of dump order, and inferring roles from them is exactly the guess this
+project exists to refuse. Roles are decided by bytes, in `resolve`, and nothing here touches them.
+
+But `SRR28716558` is not an interpretation. It is an accession the archive assigned, printed on the
+file by the tool that wrote it. Grouping by it is a rung-1 signal used for the one thing rung 1 is
+allowed to do: a weak, checkable prior about which files belong together. So filenames *group*; bytes
+*assign*. If the grouping is wrong the chemistry check downstream disagrees loudly, because two runs
+of the same library resolve to the same chemistry and a mis-grouped pair does not.
+
+**Why this exists.** `resolve_dataset` scores one set of files as ONE library, which is correct and
+always was. The bug was that nobody split first: hand it a 6-run dataset's 12 files and it does a
+single global role assignment, picks the best (R1, R2) pair out of all 12, and leaves the other **ten**
+with no role at all. `_units` then skips them, `validate` passes clean, and you get a content-addressed
+manifest that has quietly dropped 5/6 of the data. Exit 0. That is the failure class §5 of the brief
+exists to prevent, and the pilot dataset is exactly 6 runs.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from pathlib import Path
+
+#: A mate/read token at the END of a stem: `_1`, `_2`, `_R1`, `.R2`, and Illumina's `_R1_001`.
+#:
+#: `[1-4]` rather than `[12]` on purpose: `fasterq-dump --include-technical` emits `_1.._4` for a
+#: 10x run (I1/I2/R1/R2), and a `_3` that failed to match here would become its own bogus "run"
+#: instead of joining its siblings. Extra files inside a group are fine — a leftover is what
+#: `resolve`'s assignment penalty is for; a leftover in a group of its own is not.
+_MATE = re.compile(r"^(?P<stem>.+?)[._](?:R|read[-_]?)?(?P<mate>[1-4])(?:[._]\d{3})?$", re.I)
+
+#: Extensions to strip before looking for a mate token. Longest first — `.fastq.gz` before `.gz`.
+_EXTS = (".fastq.gz", ".fq.gz", ".fastq.bz2", ".fastq.xz", ".fastq", ".fq", ".gz")
+
+#: A LEADING SRA/ENA/DDBJ run accession the dump tool printed on the file. Unlike the mate token, an
+#: accession is a real identity the archive assigned (see the module docstring), so it OUTRANKS the
+#: mate heuristic below. It has to: an original-format download can carry the submitter's own lane
+#: naming *after* the accession — `SRR36109512_11314-RM-1_S1_L005_R1_001` — so the two mate files
+#: differ only in `_R1_`/`_R2_` buried mid-name, where the end-anchored mate strip cannot see it. The
+#: strip then keys each file to the whole `..._S1_L005` stem, every file becomes its own singleton
+#: "run", and the record join (`records.py`, `by_accession.get(run_key(...))`) misses every file
+#: (#6, GSE310667). Keying on the accession rejoins the mates and lands the join.
+_SRA_RUN = re.compile(r"^([SED]RR\d+)(?=[._])")
+
+
+def _strip_ext(name: str) -> str:
+    lowered = name.lower()
+    for ext in _EXTS:
+        if lowered.endswith(ext):
+            return name[: -len(ext)]
+    return name
+
+
+def run_key(path: str | Path) -> str:
+    """The run a file belongs to, derived from its name. Never a claim about the file's ROLE.
+
+    `SRR28716558_1.fastq.gz` -> `SRR28716558`; `SRR36109512_11314-RM-1_S1_L005_R1_001.fastq.gz` ->
+    `SRR36109512` (a leading accession wins over any submitter suffix); `x_S1_L001_R1_001.fastq.gz` ->
+    `x_S1_L001`; a name with no accession and no mate token is its own run, which is the right answer
+    for a single-end library.
+    """
+    stem = _strip_ext(Path(path).name)
+    sra = _SRA_RUN.match(stem)
+    if sra is not None:
+        return sra.group(1)
+    match = _MATE.match(stem)
+    return match.group("stem") if match else stem
+
+
+def group_runs(paths: Sequence[str | Path]) -> dict[str, list[Path]]:
+    """Group `paths` into runs, preserving input order within each run and sorting the runs by key.
+
+    Every input path lands in exactly one group; nothing is dropped, deduplicated, or reordered across
+    groups. That is worth stating because the bug this module fixes was files silently disappearing.
+    """
+    groups: dict[str, list[Path]] = {}
+    for path in paths:
+        groups.setdefault(run_key(path), []).append(Path(path))
+    return {key: groups[key] for key in sorted(groups)}