seqforge 2026.7.1__py3-none-any.whl

seqforge/resolve/assign.py

@@ -0,0 +1,204 @@
+"""Joint role->file assignment: the injective, cardinality-normalized optimization (§3.3).
+
+An assignment ``A: R_t -> F`` is **injective** (each role a distinct file). ``valid(A)`` selects no
+forbidden cell and fills every role. Over valid ``A`` we optimize ``(coverage, Σ(cell + β·prior))``
+lexicographically — coverage first (see below), then score. The common single-cell case (``|F| <= 4``)
+is solved by brute force over all injective maps; larger ``|F|`` uses an O(n^3) Hungarian on
+``-(bonus + cell + β·prior)`` (forbidden as a large finite cost; the coverage bonus is dropped from the
+reported score) with a post-check that no selected edge is a forbidden edge (an all-forbidden role =>
+unfillable => not a padded win).
+
+**Coverage precedes score.** A file eligible for exactly one role (forbidden for every other) can be
+placed nowhere else, so an assignment that leaves it orphaned while a multi-role file takes its role
+is coverage-wrong even when it scores higher. We therefore optimize ``(coverage, Σ(cell + β·prior))``
+lexicographically: first the count of single-role-eligible files that claim their sole role, then the
+score. This is a no-op wherever injectivity already forces the map — the common one-file-per-role case
+— and only bites a multi-file-per-role run (many lanes/flowcells under one accession), where a short
+barcode read may out-score the real long cDNA read for the cDNA role yet the cDNA-length reads are the
+only files that role can take. The coverage term orders the search; it is excluded from the reported
+score, which stays the honest ``Σ(cell + β·prior)`` of the chosen map.
+
+The filename prior is a *sub-threshold* nudge (``β << min(weight)``): it can only break an exact
+byte-tie, never override a gate or flip validity.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from itertools import permutations
+from math import factorial, inf
+
+#: Brute-force whenever the number of injective maps P(F, R) is at most this; else Hungarian.
+_BRUTE_CAP = 40_320  # 8!
+_BIG = 1e9  # a finite forbidden-edge cost for the Hungarian path (never selected if avoidable)
+#: Per-cell selection bonus for a single-role-eligible file claiming its sole role. Large enough that
+#: one more such placement outranks any score/prior configuration, small enough to sit under ``_BIG``;
+#: excluded from the reported score, so it orders coverage above score and nothing more.
+_COVERAGE_BONUS = 1e3
+
+
+@dataclass(frozen=True)
+class AssignmentResult:
+    """The best valid injective assignment for one technology (or an invalid verdict + reason)."""
+
+    valid: bool
+    #: role index -> file index (only for a valid assignment)
+    mapping: dict[int, int] = field(default_factory=dict)
+    #: file indices left over (index reads / ignored), penalized at rate lambda
+    unassigned_files: list[int] = field(default_factory=list)
+    raw: float = -inf
+    #: roles forbidden on EVERY file (structurally unfillable) — drives MISSING_TECHNICAL_READ
+    unfillable_roles: list[int] = field(default_factory=list)
+
+
+def _n_injective(n_files: int, n_roles: int) -> int:
+    if n_files < n_roles:
+        return 0
+    return factorial(n_files) // factorial(n_files - n_roles)
+
+
+def best_assignment(
+    n_roles: int,
+    n_files: int,
+    score: list[list[float]],
+    forbidden: list[list[bool]],
+    prior: list[list[float]],
+) -> AssignmentResult:
+    """Return the best valid injective role->file assignment — coverage first, then score (or invalid).
+
+    ``score``/``forbidden``/``prior`` are ``n_roles x n_files``. ``score`` is the finite support
+    value in ``[0, 1]``; ``prior`` is the sub-threshold filename nudge already scaled by ``β``. Among
+    valid maps, one with more single-role-eligible files in their sole role wins; ties break on score.
+    """
+    unfillable = [
+        r for r in range(n_roles) if n_files == 0 or all(forbidden[r][f] for f in range(n_files))
+    ]
+    if n_files < n_roles or unfillable:
+        return AssignmentResult(
+            valid=False,
+            unassigned_files=list(range(n_files)),
+            unfillable_roles=unfillable,
+        )
+
+    # ``exclusive[r][f]``: file f is eligible for role r and for no other role — it can be placed
+    # nowhere else, so coverage demands r be represented by one of its exclusive files when it has any.
+    n_eligible = [sum(not forbidden[r][f] for r in range(n_roles)) for f in range(n_files)]
+    exclusive = [
+        [(not forbidden[r][f]) and n_eligible[f] == 1 for f in range(n_files)]
+        for r in range(n_roles)
+    ]
+
+    if _n_injective(n_files, n_roles) <= _BRUTE_CAP:
+        chosen = _brute(n_roles, n_files, score, forbidden, prior, exclusive)
+    else:
+        chosen = _hungarian_assign(n_roles, n_files, score, forbidden, prior, exclusive)
+
+    if chosen is None:  # a forbidden pattern blocks every full injective map
+        return AssignmentResult(valid=False, unassigned_files=list(range(n_files)))
+
+    mapping, raw = chosen
+    assigned = set(mapping.values())
+    unassigned = [f for f in range(n_files) if f not in assigned]
+    return AssignmentResult(valid=True, mapping=mapping, unassigned_files=unassigned, raw=raw)
+
+
+def _brute(
+    n_roles: int,
+    n_files: int,
+    score: list[list[float]],
+    forbidden: list[list[bool]],
+    prior: list[list[float]],
+    exclusive: list[list[bool]],
+) -> tuple[dict[int, int], float] | None:
+    best_key: tuple[int, float] | None = None  # (coverage, raw), lexicographic
+    best: tuple[int, ...] | None = None
+    for perm in permutations(range(n_files), n_roles):
+        if any(forbidden[r][perm[r]] for r in range(n_roles)):
+            continue
+        coverage = sum(exclusive[r][perm[r]] for r in range(n_roles))
+        raw = sum(score[r][perm[r]] + prior[r][perm[r]] for r in range(n_roles))
+        key = (coverage, raw)
+        if best_key is None or key > best_key:
+            best_key = key
+            best = perm
+    if best is None or best_key is None:
+        return None
+    return {r: best[r] for r in range(n_roles)}, best_key[1]
+
+
+def _hungarian_assign(
+    n_roles: int,
+    n_files: int,
+    score: list[list[float]],
+    forbidden: list[list[bool]],
+    prior: list[list[float]],
+    exclusive: list[list[bool]],
+) -> tuple[dict[int, int], float] | None:
+    n = max(n_roles, n_files)
+    # square cost: minimize -(coverage_bonus + score + prior); forbidden -> _BIG; dummy cols -> 0. The
+    # coverage bonus (>> any score sum) makes a single-role-eligible file claim its sole role first; it
+    # is dropped from the reported ``raw``, which stays the honest Σ(score + prior) of the chosen map.
+    cost = [[0.0] * n for _ in range(n)]
+    for r in range(n_roles):
+        for f in range(n_files):
+            if forbidden[r][f]:
+                cost[r][f] = _BIG
+            else:
+                bonus = _COVERAGE_BONUS if exclusive[r][f] else 0.0
+                cost[r][f] = -(bonus + score[r][f] + prior[r][f])
+    col_for_row = _hungarian(cost)
+    mapping: dict[int, int] = {}
+    raw = 0.0
+    for r in range(n_roles):
+        f = col_for_row[r]
+        if f >= n_files or forbidden[r][f]:  # forced onto a dummy or forbidden edge -> no valid A
+            return None
+        mapping[r] = f
+        raw += score[r][f] + prior[r][f]
+    return mapping, raw
+
+
+def _hungarian(cost: list[list[float]]) -> list[int]:
+    """O(n^3) min-cost perfect assignment on a square matrix; returns ``col_for_row[i]``."""
+    n = len(cost)
+    u = [0.0] * (n + 1)
+    v = [0.0] * (n + 1)
+    p = [0] * (n + 1)  # p[j] = row matched to column j (1-indexed; 0 = unmatched)
+    way = [0] * (n + 1)
+    for i in range(1, n + 1):
+        p[0] = i
+        j0 = 0
+        minv = [inf] * (n + 1)
+        used = [False] * (n + 1)
+        while True:
+            used[j0] = True
+            i0 = p[j0]
+            delta = inf
+            j1 = -1
+            for j in range(1, n + 1):
+                if not used[j]:
+                    cur = cost[i0 - 1][j - 1] - u[i0] - v[j]
+                    if cur < minv[j]:
+                        minv[j] = cur
+                        way[j] = j0
+                    if minv[j] < delta:
+                        delta = minv[j]
+                        j1 = j
+            for j in range(n + 1):
+                if used[j]:
+                    u[p[j]] += delta
+                    v[j] -= delta
+                else:
+                    minv[j] -= delta
+            j0 = j1
+            if p[j0] == 0:
+                break
+        while j0:
+            j1 = way[j0]
+            p[j0] = p[j1]
+            j0 = j1
+    col_for_row = [0] * n
+    for j in range(1, n + 1):
+        if p[j]:
+            col_for_row[p[j] - 1] = j - 1
+    return col_for_row

seqforge/resolve/cache.py

@@ -0,0 +1,119 @@
+"""Content-addressed, resumable ``seqforge/`` artifacts: disk is state, context is cache.
+
+Per-file :class:`Observation` keyed by its content-address (a bounded local key or a provider md5 —
+never a whole-file scan; see :func:`~seqforge.probe.core._content_key`); the dataset
+:class:`ResolveResult` keyed by ``dataset_id = sha256(sorted(file_shas) ⊕ kb_version)`` with
+``probe_version`` / ``resolve_version`` folded in — so a probe or scorer change invalidates the cache
+without a manual bump. :func:`resume_key` adds a stat-only pointer (``realpath+size+mtime`` →
+``dataset_id``) so an unchanged re-run rebuilds the answer without reading a FASTQ byte. Every write is
+atomic-ish (write-then-rename); a corrupt/absent artifact reads back as ``None`` (recompute), never a
+crash.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections.abc import Iterable
+from pathlib import Path
+
+from ..models.observation import Observation
+from ..models.resolve import ResolveResult
+from ..workspace import cache_dir
+
+
+def dataset_id(
+    file_shas: list[str], kb_version: str, probe_version: str, resolve_version: str
+) -> str:
+    """The content-addressed dataset id — stable under file order, sensitive to tool versions."""
+    key = "\n".join(sorted(file_shas))
+    key += f"|kb={kb_version}|probe={probe_version}|resolve={resolve_version}"
+    return hashlib.sha256(key.encode()).hexdigest()
+
+
+def resume_key(
+    paths: Iterable[str | Path], kb_version: str, probe_version: str, resolve_version: str
+) -> str | None:
+    """A stat-only key over the input files: same (realpath, size, mtime) set -> same key.
+
+    This is the LOCAL "did anything change since the last run?" check that lets a re-run skip probing
+    and read ZERO FASTQ bytes. It is deliberately NOT the content-addressed :func:`dataset_id` (which
+    needs the bytes): mtime is the standard cheap heuristic, and ``--no-cache`` is the escape hatch.
+    Returns ``None`` if any file is missing — nothing to resume against, so probe afresh.
+    """
+    parts: list[str] = []
+    for p in paths:
+        try:
+            st = os.stat(p)
+        except OSError:
+            return None
+        parts.append(f"{os.path.realpath(p)}|{st.st_size}|{st.st_mtime_ns}")
+    key = "\n".join(sorted(parts))
+    key += f"|kb={kb_version}|probe={probe_version}|resolve={resolve_version}"
+    return hashlib.sha256(key.encode()).hexdigest()
+
+
+class Cache:
+    """Reader/writer for the ``seqforge/cache/`` artifact tree rooted at a workspace.
+
+    These are content-addressed, resumable, and safe to delete — so they live under ``cache/``, not
+    beside the manifest a human reads.
+    """
+
+    def __init__(self, workspace: str | Path) -> None:
+        self.root = cache_dir(workspace)
+
+    def _obs_path(self, sha: str) -> Path:
+        return self.root / "observations" / f"{sha}.json"
+
+    def _resolve_path(self, ds_id: str) -> Path:
+        return self.root / "candidates" / f"{ds_id}.json"
+
+    def read_observation(self, sha: str) -> Observation | None:
+        path = self._obs_path(sha)
+        if not path.is_file():
+            return None
+        try:
+            return Observation.model_validate_json(path.read_text())
+        except (ValueError, OSError):
+            return None
+
+    def write_observation(self, obs: Observation) -> None:
+        self._write(self._obs_path(obs.file.sha256), obs.model_dump_json(indent=2))
+
+    def read_resolve(self, ds_id: str) -> ResolveResult | None:
+        path = self._resolve_path(ds_id)
+        if not path.is_file():
+            return None
+        try:
+            return ResolveResult.model_validate_json(path.read_text())
+        except (ValueError, OSError):
+            return None
+
+    def write_resolve(self, ds_id: str, result: ResolveResult) -> None:
+        self._write(self._resolve_path(ds_id), result.model_dump_json(indent=2))
+
+    def _resume_path(self, key: str) -> Path:
+        return self.root / "resume" / f"{key}.json"
+
+    def read_resume(self, key: str) -> dict[str, object] | None:
+        """Read a stat-keyed resume pointer (see :func:`resume_key`); ``None`` if absent/corrupt."""
+        path = self._resume_path(key)
+        if not path.is_file():
+            return None
+        try:
+            data = json.loads(path.read_text())
+        except (ValueError, OSError):
+            return None
+        return data if isinstance(data, dict) else None
+
+    def write_resume(self, key: str, payload: dict[str, object]) -> None:
+        self._write(self._resume_path(key), json.dumps(payload, indent=2))
+
+    @staticmethod
+    def _write(path: Path, text: str) -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(path.suffix + ".tmp")
+        tmp.write_text(text)
+        tmp.replace(path)

seqforge/resolve/confuse.py

@@ -0,0 +1,215 @@
+"""Confusability helpers — the ``§12`` benign rule and its ``backend_identical`` biconditional.
+
+Two technologies are **processing-equivalent** iff, after resolving every ``{onlist:alias}`` to its
+registry name and normalizing key order **and** the read->role placement, their ``backend.params``
+canonical forms are byte-equal. Including role placement matters: two techs that differ only in
+*which* read is biological must not be called benign. The CI biconditional is
+``backend_identical(A, B) <=> declared processing_equivalent`` (§2.4); this module provides the
+``backend_identical`` primitive and the declared-relationship lookups the resolver consults at
+runtime to decide a benign record-both vs a divergent tie.
+
+Since counting moved out of ``backend.params``, this predicate means exactly *"these two
+chemistries parse reads identically"* — which is what ``processing_equivalent`` should have meant all
+along, and it makes the rule **stronger**, not weaker: two specs differing only in what they count are
+no longer distinguishable here, because that difference is no longer a chemistry fact at all. It is
+the processing manifest's to make, per dataset.
+
+**List order is significant** and is never normalized; see :func:`_resolve_value`.
+"""
+
+from __future__ import annotations
+
+import json
+
+from ..kb.schema import Spec
+
+
+def _role_placement(spec: Spec) -> list[str]:
+    """Canonical biological/technical read ordering (``readFilesIn`` order is cDNA read first)."""
+    kinds: list[str] = []
+    for read in spec.reads:
+        el_types = {el.type for el in read.elements}
+        if el_types & {"cdna", "gdna"}:
+            kinds.append(f"bio:{read.strand}")
+        elif "barcode" in el_types:
+            kinds.append("barcode")
+        else:
+            kinds.append("other")
+    return sorted(kinds)
+
+
+def canonical_backend(spec: Spec) -> str:
+    """A canonical, onlist-resolved, role-aware serialization of a spec's ``backend``.
+
+    An ABSTRACT family node has no backend, so it canonicalizes to a per-id sentinel: no two nodes
+    share it, so a classifier is never ``backend_identical`` to — and thus never a false
+    processing-equivalent twin of — a leaf.
+    """
+    if spec.backend is None:
+        return json.dumps({"abstract_node": spec.identity.id}, sort_keys=True)
+    resolved: dict[str, object] = {}
+    for key, value in spec.backend.params.items():
+        resolved[key] = _resolve_value(value, spec)
+    # Fold in the DERIVED geometry (soloCB/UMIposition, soloAdapterSequence). Those are byte-decided
+    # parse facts read off the element coordinates, not declared here — and they are exactly where two
+    # chemistries with identical DECLARED params can still parse reads differently. The original BD bead
+    # and the Enhanced-96 bead share soloType, whitelists and strand; only their geometry differs (fixed
+    # offsets vs an adapter-anchored, diversity-insert-staggered frame). Comparing declared params alone
+    # called them byte-identical -> §12-benign -> one config for both. Local import: `compose` reads the
+    # KB, so importing it at module load would knot resolve<->compose. (#43)
+    from ..compose.params import derived_params
+
+    resolved.update(derived_params(spec))
+    payload = {
+        "module": spec.backend.module,
+        "params": resolved,
+        "placement": _role_placement(spec),
+    }
+    return json.dumps(payload, sort_keys=True)
+
+
+def _resolve_value(value: object, spec: Spec) -> object:
+    if isinstance(value, str):
+        return _resolve_token(value, spec)
+    if isinstance(value, list):
+        # ORDER IS PRESERVED, and it must be. This used to sort, justified by exactly one comment:
+        # "normalize list order so soloFeatures=[Gene,GeneFull] == [GeneFull,Gene]". soloFeatures has
+        # since moved to the processing manifest (it says what to COUNT, not how to parse), and
+        # with it the only reason the sort existed.
+        #
+        # What it would sort NOW is the only list-valued parse param left: splitseq's
+        # `soloCBwhitelist: [round1, round2, round3]` — which is POSITIONAL. The rounds map to CB
+        # positions in order. Sorting it made `backend_identical` return True for a spec against
+        # itself-with-rounds-permuted: two chemistries that parse reads DIFFERENTLY, declared byte-
+        # equal, hence §12-benign, hence one config emitted for both. It never fired only by the
+        # alphabetical accident that round1 < round2 < round3.
+        return [_resolve_token(v, spec) if isinstance(v, str) else v for v in value]
+    return value
+
+
+def _resolve_token(value: str, spec: Spec) -> str:
+    if value.startswith("{onlist:") and value.endswith("}"):
+        alias = value[len("{onlist:") : -1]
+        ref = spec.onlists.get(alias)
+        return f"registry:{ref.registry}" if ref else value
+    return value
+
+
+def backend_identical(a: Spec, b: Spec) -> bool:
+    """True iff two specs compile to byte-equal, onlist-resolved, role-aware backends."""
+    return canonical_backend(a) == canonical_backend(b)
+
+
+def accepts_at_rungs_0_2(spec: Spec, probes: list[object]) -> bool:
+    """Would ``spec`` claim this data using only the CHEAP probes — no onlist, no network?
+
+    The onlist is withheld by handing the evaluator an **empty registry**, so every
+    ``onlist_hit_rate`` test abstains and the verdict rests on geometry, segmentation, distinct-value
+    ratios and header grammar alone. That is precisely rungs 0-2 (§5), expressed by removing the
+    rung-3 evidence rather than by reimplementing the scorer without it.
+
+    This is the primitive behind :func:`rung02_separable`, and it is why "ask the human" can be a
+    computed property instead of a prompt hope.
+    """
+    from ..io import OnlistRegistry
+    from .scoring import build_tech_evaluation
+    from .window import WindowProbe
+
+    wps = [p for p in probes if isinstance(p, WindowProbe)]
+    return build_tech_evaluation(spec, wps, OnlistRegistry(offline=True)).valid
+
+
+def rung02_separable(a: Spec, a_probes: list[object], b: Spec, b_probes: list[object]) -> bool:
+    """Do the cheap probes tell these two chemistries apart at all? (design §2.4, fact 1)
+
+    Separable iff **neither** spec accepts the other's data on geometry alone. If A would happily
+    claim B's reads, no amount of scoring rigour separates them below rung 3 — the honest thing is
+    for the KB to *say so* via ``confusable_with``, so the resolver knows to reach for the onlist or
+    a human rather than picking the alphabetically-luckier entry.
+
+    Some distinctions are provably undecidable from reads (10x 3' and 5' share CB/UMI geometry;
+    inDrop v2 and v3 share oligos). The system must KNOW that rather than guess, which is the whole
+    point of computing this instead of hand-maintaining a truth table.
+    """
+    return not (accepts_at_rungs_0_2(a, b_probes) or accepts_at_rungs_0_2(b, a_probes))
+
+
+def declared_equivalents(spec: Spec) -> set[str]:
+    """Ids the spec declares as ``processing_equivalent`` twins (benign, ``§12`` record-both)."""
+    return {c.id for c in spec.confusable_with if c.relationship == "processing_equivalent"}
+
+
+def declared_divergent(spec: Spec) -> set[str]:
+    """Ids the spec declares as ``processing_divergent`` (a real disagreement to decide)."""
+    return {c.id for c in spec.confusable_with if c.relationship == "processing_divergent"}
+
+
+def is_processing_equivalent(a: Spec, b_id: str) -> bool:
+    """Does ``a`` declare ``b_id`` as a processing-equivalent twin?"""
+    return b_id in declared_equivalents(a)
+
+
+# ---- tree-sourced confusability: siblings replace hand-declared divergent cliques ----
+def share_parent(specs: dict[str, Spec], a: str, b: str) -> bool:
+    """True iff ``a`` and ``b`` are siblings — the same non-null parent in the KB tree."""
+    pa = specs[a].parent if a in specs else None
+    pb = specs[b].parent if b in specs else None
+    return pa is not None and pa == pb
+
+
+def is_tree_kin(specs: dict[str, Spec], a: str, b: str) -> bool:
+    """True iff ``a`` and ``b`` are parent-child or siblings — a confusability the tree DECLARES.
+
+    A divergent sibling clique (v2/v3/v3.1) collapses to one ``parent`` link, so the under-declaration
+    guard treats tree kin the way it treats an explicit ``confusable_with`` edge: already declared.
+    """
+    if a not in specs or b not in specs:
+        return False
+    if specs[a].parent == b or specs[b].parent == a:
+        return True
+    return share_parent(specs, a, b)
+
+
+def sibling_decided_by(specs: dict[str, Spec], a: str, b: str) -> list[str]:
+    """If ``a`` and ``b`` are siblings, the mechanisms their parent declares separate its children.
+
+    This is where the divergent-tie question now reads ``decidable_by`` from — the parent's
+    ``children_decided_by`` — instead of the per-sibling ``distinguishable_by`` edge that was deleted.
+    """
+    if not share_parent(specs, a, b):
+        return []
+    parent = specs[a].parent
+    if parent is None or parent not in specs:
+        return []
+    return [m for m in specs[parent].children_decided_by if m != "none"]
+
+
+def _root_of(specs: dict[str, Spec], tech: str) -> str:
+    """The family-root ancestor of ``tech`` — walk ``parent`` links to the top of its tree."""
+    cur = tech
+    seen: set[str] = set()
+    while cur in specs and cur not in seen:
+        parent = specs[cur].parent
+        if parent is None:
+            break
+        seen.add(cur)
+        cur = parent
+    return cur
+
+
+def same_family(specs: dict[str, Spec], a: str, b: str) -> bool:
+    """True iff ``a`` and ``b`` share a family root — the assay family a paper reliably names.
+
+    The policy this encodes: harvest is trusted at the FAMILY level (10x 3' gene-expression), the bytes
+    at the LEAF level (v2 vs v3). A within-family disagreement (asserted v2, observed v3) is therefore
+    not a blocking conflict — the bytes decide the leaf. It is broader than ``is_tree_kin`` (which is
+    parent-child OR siblings only): two cousins under a deeper tree are the same family yet not kin.
+
+    Every assay family is its own root today (``10x-3p-gex``, ``bulk-rnaseq-pe``, ``splitseq``,
+    ``bd-rhapsody-wta`` are distinct roots), so a shared root IS the same family. A future super-root
+    over two genuinely different families would need a KB-lint guard against over-suppression; none
+    exists now, and none is needed while roots and families coincide.
+    """
+    if a not in specs or b not in specs:
+        return False
+    return _root_of(specs, a) == _root_of(specs, b)