seqforge 2026.7.1__py3-none-any.whl

seqforge/compose/core.py

@@ -0,0 +1,515 @@
+"""``compose`` — (dataset, processing) -> ``Snakefile`` + ``config.yaml`` + ``units.tsv``.
+
+**Emit data, never code.** The composer selects a hand-written, versioned module and emits its
+*configuration*; it never writes rule source. It is a pure function of **both** its inputs plus two
+versioned ones recorded in provenance (the KB and the workflow modules) — it needs no FASTQ on disk,
+local or remote. Two inputs, still no I/O: a processing manifest is data, not a side channel.
+
+Purity across *both* is what makes "same dataset + different processing = different pipeline, same
+dataset hash" a fact rather than a hope. Output is keyed by ``run_id = H(dataset ⊕ processing ⊕ kb ⊕
+workflow)``, so composing one dataset two ways yields two directories instead of the second silently
+overwriting the first — which is what a fixed ``.seqforge/pipeline/`` path used to do, in exactly the
+case the split exists to enable.
+
+The machine-independent/machine-specific boundary lives here: the **manifest** carries URIs and a
+``liulab-genome`` assembly id; the **config** it compiles to is the resolved, machine-specific
+instantiation, so paths appear here and only here.
+
+The gate has three parts (design §4.1) — this module always runs the deterministic **params** gate;
+``wiring`` (``snakemake -n -p``) now really runs, because `snakemake-minimal` is a declared
+dependency; ``e2e`` (the real count-matrix run) is reported as ``skip``, never silently as ``pass``.
+
+**The Snakefile is the deliverable, and it is emitted here, unconditionally.** It used to be written
+inside ``wiring_gate`` — so the product of the compiler was a side effect of a validation step, and
+that step returned early whenever ``snakemake`` was absent from PATH, which it always was. `compose`
+exited 0 and emitted nothing a user could run. A gate may read the deliverable; it may not be the
+reason the deliverable exists.
+"""
+
+from __future__ import annotations
+
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from ..io import DEFAULT_REGISTRY, OnlistNotAvailable, OnlistRegistry
+from ..kb import load_spec
+from ..kb.schema import Spec
+from ..manifest.hash import run_id
+from ..models.dataset import INDEX_ROLE, DatasetManifest
+from ..models.processing import (
+    DatasetPin,
+    ProcessingManifest,
+    Quantification,
+    RuntimeEnv,
+    SoloQuant,
+)
+from ..models.resolve import ComposeResult, ModuleSelection
+from ..resolve.group import run_key
+from ..workflows import WorkflowModule, container_uri, get_module
+from ..workspace import readable, state_dir
+from .params import (
+    derived_params,
+    find_read_with_role,
+    param_block_key,
+    params_gate,
+    processing_params,
+    render_param,
+)
+
+
+class ComposeError(RuntimeError):
+    """The manifest cannot be compiled (unknown chemistry/module, or an unresolvable whitelist)."""
+
+
+#: Names of the artifacts a run directory carries, beside the wrapper that reads them.
+_CONFIG_NAME = "config.yaml"
+_UNITS_TSV_NAME = "units.tsv"
+_SNAKEFILE_NAME = "Snakefile"
+
+#: The run wrapper: **the deliverable**. A user submits this file.
+#:
+#: It is data, not code: it names a hand-written module and points it at a config. There is no
+#: rule body, no shell block, and no logic here — `test_the_generated_wrapper_contains_no_rule_source`
+#: holds the line.
+#:
+#: **It uses `module` / `use rule *`, not `include:`, and that is load-bearing rather than stylistic.**
+#: Snakemake's default target is the first rule defined in the *main* Snakefile, and an `include:`d
+#: rule is not one — so the old `configfile:` + `include:` wrapper parsed clean, listed all three
+#: rules, and then did **nothing at all** when run: "Nothing to be done", exit 0. Measured 2026-07-15;
+#: the same module content inlined builds 3 jobs, and via `include:` builds 0. `use rule * from m as *`
+#: re-declares the rules in this workflow, which makes `all` a real default target and `snakemake` with
+#: no arguments do the right thing. This is also what design.md §4.1 specified all along.
+_WRAPPER = """# GENERATED by `seqforge compose` -- this file is DATA: it names a hand-written module
+# and points it at a config. Rule source is never generated; the rules live in the module below,
+# `{snakefile}`, COPIED into this directory beside this Snakefile at the exact version this run is
+# pinned to. The copy is deliberate: a run directory carries its own recipe, so it stays readable and
+# reproducible even after it is moved off the machine that composed it -- no path back into an
+# installed package. (Running it still needs `seqforge` and `genome` importable; the copy makes the
+# pipeline legible, not standalone.)
+#
+# Run it:  snakemake --profile <your-cluster-profile> --software-deployment-method apptainer
+#
+# The default target is `all`, which produces one <sample>.h5ad per sample in units.tsv (plus a
+# <sample>.velocyto.h5ad when Velocyto is counted).
+#
+# `--software-deployment-method apptainer` is what makes the aligner rule run inside the
+# liulab-runtime image named in config.yaml. WITHOUT IT SNAKEMAKE IGNORES THE `container:` DIRECTIVE
+# ENTIRELY and STAR comes from your PATH -- the run still works, and is no longer pinned. Never
+# needed for the packaging step, which runs in whatever environment has seqforge.
+configfile: "config.yaml"
+
+
+module {alias}:
+    snakefile:
+        "{snakefile}"
+    config:
+        config
+
+
+use rule * from {alias} as *
+"""
+
+
+def _module_alias(name: str) -> str:
+    """`map/starsolo` -> `starsolo`: a Snakemake module needs a Python-identifier-shaped name."""
+    return name.rsplit("/", 1)[-1].replace("-", "_")
+
+
+def render_wrapper(module_name: str, snakefile: str) -> str:
+    """The run wrapper's text, for the module `module_name` at the LOCAL name `snakefile`.
+
+    ``snakefile`` is a bare filename (``starsolo.smk``), not a package path: the composer copies the
+    module into the run directory and the wrapper imports its neighbour, so the `snakefile:` line
+    resolves relative to this Snakefile. Snakemake resolves a module's snakefile against the including
+    file's directory, which is exactly where the copy lands.
+    """
+    return _WRAPPER.format(alias=_module_alias(module_name), snakefile=snakefile)
+
+
+@dataclass(frozen=True)
+class ComposePlan:
+    """The composed artifacts, before they are written (compose is a pure function of the manifest)."""
+
+    config: dict[str, object]
+    units: list[dict[str, str]]
+    module: ModuleSelection
+    spec: Spec
+    #: relative path -> the registry name `rule onlist` will build it from. NOT the barcodes:
+    #: see `_resolve_token` and workflows/map/starsolo.smk for why compose does not write 111 MB.
+    onlist_files: dict[str, list[str]]
+
+
+def plan(
+    manifest: DatasetManifest,
+    processing: ProcessingManifest,
+    *,
+    registry: OnlistRegistry | None = None,
+    outdir: str = "results",
+    fastq_dir: str | Path | None = None,
+    sif_dir: str | Path | None = None,
+) -> ComposePlan:
+    """Build the config + units + module selection for one (dataset, processing) pair.
+
+    Both inputs are positional and required. Composing one dataset two ways is literally two calls
+    with one argument changed, producing two ``run_id``s over one unchanged ``dataset_hash``.
+
+    ``threads`` is deliberately NOT an argument: it lives in ``processing.resources.threads``. Having
+    it in both places was the same two-sources-of-truth disease this whole change is curing.
+    ``outdir`` and ``fastq_dir`` stay arguments, because they are paths — machine facts, which a
+    machine-independent manifest forbids carrying. ``threads`` is intent; a path is an invocation.
+
+    ``fastq_dir`` is where this machine keeps the bytes. A manifest URI is a *relative* name, so
+    without it ``units.tsv`` carries bare basenames and the pipeline resolves them against the run
+    directory — where the FASTQs are not. That is not a stricter kind of purity, it is a pipeline that
+    cannot find its input. Given a directory, ``units.tsv`` carries resolved absolute paths: the
+    manifest stays machine-independent and the *config* becomes the machine-specific instantiation,
+    which is the boundary this module's docstring has always drawn. Still no I/O — the path is joined,
+    never read, so ``plan`` stays pure and needs no FASTQ on disk.
+    """
+    registry = registry if registry is not None else DEFAULT_REGISTRY
+    _check_pin(manifest, processing)
+    chemistry = manifest.library.chemistry.value
+    if not chemistry:
+        raise ComposeError("manifest.library.chemistry is empty; nothing to compile")
+    # A processing-equivalent class emits identical params by construction (§12), so the first
+    # member compiles for all of them — that is exactly what makes the ambiguity benign.
+    spec = load_spec(chemistry[0])
+    try:
+        module = get_module(spec.require_backend().module)
+    except KeyError as exc:
+        raise ComposeError(str(exc)) from exc
+
+    onlist_files: dict[str, list[str]] = {}
+    intent = processing.processing
+    _check_env(intent.environment.value, module)
+    # Two owners, one block. The KB says how to PARSE; the processing manifest says what to
+    # COUNT. params_gate proves the key sets stay disjoint and that both halves arrive verbatim.
+    params = _resolve_params(manifest, spec, registry, onlist_files)
+    # Derived before processing, both after the KB: three owners, one key each, and `param_owners`
+    # is the definition all three agree on. A complex chemistry locates its barcodes by quadruple,
+    # computed from the element model rather than hand-entered (see `derived_params`).
+    params.update(derived_params(spec))
+    params.update(
+        {k: render_param(v) for k, v in processing_params(intent.quantification.value).items()}
+    )
+
+    config: dict[str, object] = {
+        "chemistry": list(chemistry),
+        "genome": {
+            "assembly": intent.genome.value.assembly,
+            "annotation": intent.genome.value.annotation_name,
+        },
+        # The env name RESOLVED to an image this machine can run. `config["env"]` used to sit here:
+        # emitted, and read by no rule — the env was recorded and then ignored, so a pipeline ran
+        # against whatever STAR happened to be on PATH. One key, one owner, and now something reads
+        # it. The manifest still carries the env *name*; this is its machine-specific rendering,
+        # exactly like a fastq path.
+        "container": container_uri(intent.environment.value, sif_dir),
+        "threads": intent.resources.threads,
+        # The advisory memory hint, in MB, so the finalize CRAM sort can be given a real `-m` budget
+        # rather than single-threading samtools' default. An intent knob (like threads), never a
+        # machine fact — the scheduler honours it when the user submits.
+        "mem_mb": intent.resources.mem_gb * 1024,
+        "outdir": outdir,
+        "read_files_in": _read_files_in(manifest, module),
+        "samples": [s.sample_id for s in manifest.experiment.samples],
+        # Emitted by the composer like every other key. It used to be injected by the run wrapper at
+        # parse time, and `required_config` documented that arrangement — but the wrapper was written
+        # inside a gate that could not run, so the injector named by the contract did not exist and
+        # every compiled pipeline was one `KeyError` from the start. One owner, no special case.
+        "units_tsv": _UNITS_TSV_NAME,
+    }
+    config[param_block_key(spec)] = params
+    primary = _primary_feature(intent.quantification.value)
+    if primary is not None:
+        # Top level, NOT inside config["solo"] — that block must stay "every key is a STAR CLI flag"
+        # for the gate's coverage check, and STARsolo has no --primaryFeature. Order in the manifest
+        # is a seqforge-side annotation with no aligner-side referent (STARsolo writes one
+        # Solo.out/<Feature>/ per value and does not care about order), so it gets projected out to
+        # an explicit value rather than leaving a positional convention load-bearing for every
+        # downstream reader.
+        config["primary_feature"] = primary
+
+    return ComposePlan(
+        config=config,
+        units=_units(manifest, fastq_dir),
+        module=ModuleSelection(name=module.name, version=module.version, env=module.env),
+        spec=spec,
+        onlist_files=onlist_files,
+    )
+
+
+def _primary_feature(quant: Quantification) -> str | None:
+    """Which ``Solo.out/<Feature>/`` is THE matrix for this run. ``None`` for bulk (no such split)."""
+    return quant.features[0] if isinstance(quant, SoloQuant) else None
+
+
+def _check_env(env: RuntimeEnv, module: WorkflowModule) -> None:
+    """The recipe's runtime env must be the one the module needs. Refuse; never quietly correct.
+
+    The env is read off the processing manifest, not off the module, because that is the artifact
+    that *records the decision* — and `ProcessingSection` forbids unknown keys precisely so that a
+    user's stated intent is never silently dropped. But a module knows what software it needs, and
+    ``map/starsolo`` in an env with no STAR is not a preference, it is a run that dies on a node.
+
+    So both are read and they must agree. Until `container:` landed, `processing.environment` was
+    emitted into the config and read by nothing, while policy hardcoded ``"align-rna"`` beside a
+    module that also declared it — two owners who could not disagree only because nobody asked
+    either. Policy now asks the module, which makes this check redundant *for the default path* and
+    load-bearing for the one where a user hand-writes a processing.yaml.
+    """
+    if env != module.env:
+        raise ComposeError(
+            f"processing.environment is {env!r} but module {module.name!r} needs {module.env!r}. "
+            f"The env is what supplies the aligner, so this run would start in a container without "
+            f"one. Drop the field to take the module's own env, or pick a module that wants {env!r}."
+        )
+
+
+def _check_pin(manifest: DatasetManifest, processing: ProcessingManifest) -> None:
+    """A BOUND processing manifest must name this dataset. Never auto-repin."""
+    pin = processing.dataset
+    if pin is None:
+        return  # a template: portable by design — that is what drives a corpus
+    if pin.dataset_hash != manifest.provenance.dataset_hash:
+        raise ComposeError(
+            f"processing manifest {processing.processing_id!r} is pinned to dataset "
+            f"{pin.dataset_hash[:12]}… but this dataset is "
+            f"{manifest.provenance.dataset_hash[:12]}…. Re-run `seqforge processing new` against "
+            f"this dataset, or drop the pin to make it a template."
+        )
+
+
+def compose(
+    manifest: DatasetManifest,
+    processing: ProcessingManifest,
+    *,
+    registry: OnlistRegistry | None = None,
+    workspace: str | Path = ".",
+    outdir: str = "results",
+    fastq_dir: str | Path | None = None,
+    sif_dir: str | Path | None = None,
+    run_wiring_gate: bool = True,
+    subdir: str | None = None,
+) -> ComposeResult:
+    """Compile a (dataset, processing) pair into a runnable configuration + run the compose gates.
+
+    ``subdir`` roots the output one level deeper — ``seqforge/<subdir>/pipeline/…`` instead of
+    ``seqforge/pipeline/…`` — so a heterogeneous project can compile one assay per subdirectory. It
+    defaults to ``None`` (the flat, single-assay layout), which is byte-identical to before.
+    """
+    p = plan(
+        manifest, processing, registry=registry, outdir=outdir, fastq_dir=fastq_dir, sif_dir=sif_dir
+    )
+
+    # Keyed by the RUN, not by the workspace. A fixed `.seqforge/pipeline/` path meant recipe B
+    # silently overwrote recipe A's config, units, and materialized onlists — and "one dataset, many
+    # recipes" is precisely the case this whole change exists to enable.
+    rid = run_id(
+        dataset_hash=manifest.provenance.dataset_hash,
+        processing_hash=processing.provenance.processing_hash,
+        kb_version=manifest.provenance.kb_version,
+        workflow_version=processing.provenance.workflow_version,
+    )
+    # `pipeline/default-a3f8c19d2b04/`, not `pipeline/a3f8c19d…696/`. The run_id stays the identity
+    # -- it is what keeps two recipes over one dataset apart -- but the recipe already has a name and
+    # printing it costs nothing. The pilot's workspace was a directory of 64-hex names in which
+    # nothing said which pipeline was which.
+    root_parts = (subdir,) if subdir else ()
+    pipeline_dir = state_dir(
+        workspace, *root_parts, "pipeline", readable(processing.processing_id, rid)
+    )
+    pipeline_dir.mkdir(parents=True, exist_ok=True)
+    config_path = pipeline_dir / _CONFIG_NAME
+    units_path = pipeline_dir / _UNITS_TSV_NAME
+
+    # The whitelists are NOT written here. `rule onlist` builds each one when a job needs it and
+    # `temp()` deletes it after — see workflows/map/starsolo.smk. Compose used to expand 6 794 880
+    # packed barcodes into 111 MB of text per run directory, at compile time, for a file STAR opens
+    # once; compiling one dataset three ways cost a third of a gigabyte of identical bytes that
+    # nothing ever cleaned up. What compose owes the pipeline is the whitelist's NAME, which is a
+    # decision, and `_resolve_token` still verifies the registry can produce it before emitting one.
+    config_path.write_text(yaml.safe_dump(p.config, sort_keys=True))
+    units_path.write_text(_units_tsv(p.units))
+    # The Snakefile is THE DELIVERABLE, so the composer writes it — always, unconditionally, like the
+    # config beside it. It used to be written inside `wiring_gate`, which returned early when
+    # `snakemake` was absent from PATH, and `snakemake` was in no dependency table: the product was a
+    # side effect of a validation step that could not fire, so `compose` emitted no Snakefile at all.
+    # A gate may now read this file; it may not be the reason it exists.
+    # Copy the hand-written module in beside the wrapper, and point the wrapper at the local copy.
+    # This is still "emit data, never code": the module is checked-in source COPIED verbatim, not
+    # generated -- `test_shipped_modules_are_hand_written_not_generated` guards the source, and the
+    # wrapper itself still carries no rule body. What the copy buys is a run directory that reads and
+    # reproduces on its own, instead of a `snakefile:` line pointing at an absolute path inside
+    # whatever seqforge checkout happened to compose it.
+    module = get_module(p.module.name)
+    local_module_name = module.snakefile.name
+    shutil.copy2(module.snakefile, pipeline_dir / local_module_name)
+    snakefile_path = pipeline_dir / _SNAKEFILE_NAME
+    snakefile_path.write_text(render_wrapper(module.name, local_module_name))
+    # The resolved, dataset-BOUND processing manifest that produced this config, beside it. Disk is
+    # state, not input: the default path takes no --processing and must still
+    # leave behind exactly what decided the run.
+    bound = (
+        processing
+        if processing.dataset is not None
+        else processing.model_copy(
+            update={
+                "dataset": DatasetPin(
+                    dataset_hash=manifest.provenance.dataset_hash,
+                    accessions=list(manifest.experiment.accessions.value),
+                )
+            }
+        )
+    )
+    (pipeline_dir / "processing.lock.yaml").write_text(
+        yaml.safe_dump(bound.model_dump(mode="json"), sort_keys=True)
+    )
+
+    from .gates import e2e_gate, wiring_gate
+
+    status, problems = params_gate(manifest, processing, p.spec, p.config)
+    gate: dict[str, str] = {
+        "params": status,
+        "wiring": wiring_gate(pipeline_dir, p) if run_wiring_gate else "skip",
+        "e2e": e2e_gate(),
+    }
+
+    preview: dict[str, object] = dict(p.config)
+    # a failing gate must say WHY; the verdict alone is not actionable
+    preview["params_problems"] = problems
+
+    return ComposeResult(
+        modules=[p.module],
+        snakefile_path=str(snakefile_path.relative_to(Path(workspace))),
+        config_path=str(config_path.relative_to(Path(workspace))),
+        units_path=str(units_path.relative_to(Path(workspace))),
+        gate=gate,  # type: ignore[arg-type]
+        params_preview=preview,
+    )
+
+
+def _resolve_params(
+    manifest: DatasetManifest,
+    spec: Spec,
+    registry: OnlistRegistry,
+    onlist_files: dict[str, list[str]],
+) -> dict[str, object]:
+    """Render the KB backend params for a CLI, resolving every ``{onlist:alias}`` to a real path."""
+    out: dict[str, object] = {}
+    for key, value in spec.require_backend().params.items():
+        if isinstance(value, list):
+            rendered = [_resolve_token(v, spec, registry, onlist_files) for v in value]
+            out[key] = " ".join(str(r) for r in rendered)
+        else:
+            out[key] = render_param(_resolve_token(value, spec, registry, onlist_files))
+    return out
+
+
+def _resolve_token(
+    value: object, spec: Spec, registry: OnlistRegistry, onlist_files: dict[str, list[str]]
+) -> object:
+    if not (isinstance(value, str) and value.startswith("{onlist:") and value.endswith("}")):
+        return value
+    alias = value[len("{onlist:") : -1]
+    ref = spec.onlists.get(alias)
+    if ref is None:
+        raise ComposeError(f"backend references undeclared onlist alias {alias!r}")
+    name = ref.registry
+    rel = f"onlists/{name}.txt"
+    try:
+        registry.packed(name)
+    except OnlistNotAvailable as exc:
+        raise ComposeError(
+            f"onlist {name!r} is not materialized, so --soloCBwhitelist cannot be emitted: {exc}. "
+            "Register it (URL + sha256) or run with a registry that can fetch it."
+        ) from exc
+    # Verified, not materialized. `registry.packed(name)` above is what proves the whitelist EXISTS
+    # and is the declared barcode set -- refusing here, at compile time, is the rung-3 promise
+    # `compose` makes. What it must not do is write the 111 MB: that is `rule onlist`'s job, and
+    # `temp()`'s. The name is recorded so the gate and the tests can still see which list was chosen.
+    onlist_files[rel] = [name]
+    return rel
+
+
+def _read_files_in(manifest: DatasetManifest, module: WorkflowModule) -> dict[str, str]:
+    """Map the manifest's reads to the keys the module hands its aligner.
+
+    Dispatch is on the module's declared ``read_layout_kind``, **not** on its name. This was
+    `if spec.backend.module == "map/starsolo": ... else: <bulk>`, which is a bug rather than an
+    abstraction gap: it makes "not starsolo" mean "bulk paired-end", so any third module would have
+    silently emitted `mate1`/`mate2` for a chemistry that has neither and produced a wrong command
+    line with no error anywhere.
+    """
+    if module.read_layout_kind == "barcoded":
+        cdna = find_read_with_role(manifest, "cDNA") or find_read_with_role(manifest, "gDNA")
+        barcode = find_read_with_role(manifest, "CB")
+        if cdna is None or barcode is None:
+            raise ComposeError("a barcoded chemistry needs both a cDNA read and a CB-bearing read")
+        return {"cdna": cdna.read_id, "barcode": barcode.read_id}
+    reads = manifest.library.read_layout.reads
+    if len(reads) < 2:
+        raise ComposeError(f"bulk paired-end needs 2 reads, found {len(reads)}")
+    return {"mate1": reads[0].read_id, "mate2": reads[1].read_id}
+
+
+def _resolve_uri(uri: str, fastq_dir: str | Path | None) -> str:
+    """A manifest URI -> the path this machine will actually open.
+
+    A non-file scheme (``s3://``, ``https://``, a bare accession) is passed through untouched: it is
+    not ours to resolve, and Snakemake storage plugins or a future `fetch` module own that. Only a
+    relative local name gets joined to ``fastq_dir``.
+    """
+    if "://" in uri or fastq_dir is None:
+        return uri
+    return str((Path(fastq_dir) / uri).resolve())
+
+
+def _units(manifest: DatasetManifest, fastq_dir: str | Path | None = None) -> list[dict[str, str]]:
+    """One row per (sample, run, read role, file). Falls back to a single implicit sample.
+
+    ``run`` is which sequencing run a file came from, from the SAME `resolve.group.run_key` that
+    grouped the dataset into runs during resolution. Recording it here is what lets the mapping module
+    pair a pooled sample's mates by run: STAR reads `--readFilesIn` mate-by-mate and desyncs if cDNA of
+    run K is joined with barcodes of run J, and `sample.file_uris` gives no order guarantee. Deriving
+    the run once, in the one function that owns the notion, keeps the workflow from re-parsing names.
+    """
+    by_uri = {f.uri: f for f in manifest.library.files}
+    rows: list[dict[str, str]] = []
+    samples = manifest.experiment.samples
+    if not samples:
+        for f in manifest.library.files:
+            if f.read_id is not None and f.read_id != INDEX_ROLE:
+                rows.append(
+                    {
+                        "sample_id": "sample1",
+                        "run": run_key(f.uri),
+                        "read_id": f.read_id,
+                        "path": _resolve_uri(f.uri, fastq_dir),
+                    }
+                )
+        return rows
+    for sample in samples:
+        for uri in sample.file_uris:
+            item = by_uri.get(uri)
+            if item is None or item.read_id is None or item.read_id == INDEX_ROLE:
+                continue  # unassigned files, and set-aside technical index reads, never become units
+            rows.append(
+                {
+                    "sample_id": sample.sample_id,
+                    "run": run_key(item.uri),
+                    "read_id": item.read_id,
+                    "path": _resolve_uri(item.uri, fastq_dir),
+                }
+            )
+    return rows
+
+
+def _units_tsv(rows: list[dict[str, str]]) -> str:
+    header = ["sample_id", "run", "read_id", "path"]
+    lines = ["\t".join(header)]
+    lines += ["\t".join(r[h] for h in header) for r in rows]
+    return "\n".join(lines) + "\n"

seqforge/compose/gates.py

@@ -0,0 +1,113 @@
+"""The compose gate's other two parts: **wiring** and **e2e** (design §4.1, parts 1 and 3).
+
+Both depend on a toolchain seqforge does not own, so both report **``skip``** — never ``pass`` — when
+that toolchain is absent. A gate that silently reports ``pass`` because it did not run is worse than
+no gate at all: green CI would then be mistaken for coverage.
+
+- **wiring** (`snakemake -n -p`): needs the `snakemake` binary, which is now a declared dependency
+  (`[tool.pixi.feature.wf]`), so this gate actually runs. It works in a **throwaway copy** of the run
+  directory and never writes into the run directory itself — see the incident note on `_replica`.
+- **e2e** (the real count-matrix run): needs STAR + liulab-genome + network. It is a Linux/cluster
+  operation, deliberately NOT run inside `compose`; `seqforge kb e2e` owns it.
+
+**Why `-p`, and why no `--lint`.** Both were measured on 2026-07-15 rather than reasoned about:
+
+- `-p` forces Snakemake to *format* every `shell:` block while planning. Without it a dry run never
+  renders the command, so a `KeyError` on a missing param — `starsolo.smk` dereferencing
+  `soloCBstart` for a `CB_UMI_Complex` chemistry that has no such key — plans clean and dies on a
+  compute node. `-p` is the difference between this gate catching that and rubber-stamping it.
+- `--lint` was in this gate and is now gone. It fails on *every* rule we ship, for a missing `log:`
+  directive and "mixed rules and functions in same snakefile" — style opinions, not wiring facts. A
+  gate that is red for a correct config teaches people to ignore it, and then it guards nothing.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .core import ComposePlan
+
+
+def have(binary: str) -> bool:
+    return shutil.which(binary) is not None
+
+
+def _replica(pipeline_dir: Path, plan: ComposePlan) -> Path:
+    """Copy the compiled artifacts to a scratch dir and stand in zero-byte FASTQs **there**.
+
+    This gate needs its inputs to exist — `snakemake -n` raises `MissingInputException` otherwise —
+    and `compose` is a pure function that runs with no FASTQ on disk. So the dry run is given
+    zero-byte stand-ins: it validates *wiring*, not data.
+
+    **The stand-ins go in a throwaway copy, and that is the whole point of this function.** They used
+    to be touched straight into the run directory, at `pipeline_dir / row["path"]`, and never removed.
+    That was invisible only because `snakemake` was in no dependency table, so this gate never ran. The
+    moment it did, the run directory would contain zero-byte files named exactly like the FASTQs, STAR
+    would read them, and the pipeline would emit an empty matrix and **exit 0** — a silent, plausible,
+    wrong answer, which is the one failure class this project exists to prevent.
+
+    An **absolute** unit path is skipped rather than stood in: it names a real FASTQ the caller pointed
+    us at with `--fastq-dir`, and `scratch / "/abs/path"` is `/abs/path`, so touching it would create a
+    zero-byte file *at the real location*. If such a file is genuinely missing the dry run fails, and
+    failing is right — the caller said it was there.
+    """
+    scratch = Path(tempfile.mkdtemp(prefix="seqforge-wiring-"))
+    for item in pipeline_dir.iterdir():
+        if item.name == ".snakemake":
+            continue
+        dst = scratch / item.name
+        if item.is_dir():
+            shutil.copytree(item, dst)
+        else:
+            shutil.copy2(item, dst)
+    for row in plan.units:
+        path = Path(row["path"])
+        if path.is_absolute():
+            continue
+        target = scratch / path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.touch(exist_ok=True)
+    return scratch
+
+
+def wiring_gate(pipeline_dir: Path, plan: ComposePlan) -> str:
+    """`snakemake -n -p` over a throwaway replica. ``skip`` only if snakemake is absent."""
+    if not have("snakemake"):
+        return "skip"
+    scratch = _replica(pipeline_dir, plan)
+    try:
+        proc = subprocess.run(
+            ["snakemake", "-d", str(scratch), "-s", str(scratch / "Snakefile"), "-n", "-p"],
+            capture_output=True,
+            text=True,
+            timeout=300,
+        )
+        if proc.returncode != 0:
+            return "fail"
+        # A dry run that plans NOTHING exits 0 and says "Nothing to be done", which is exactly what a
+        # workflow with no reachable target does — and for most of this repo's life that is what the
+        # generated wrapper produced, because an `include:`d `rule all` is not a default target. A
+        # gate cannot tell "correct" from "planned nothing" by exit code, so it must look.
+        if "Nothing to be done" in (proc.stdout or ""):
+            return "fail"
+        return "pass"
+    finally:
+        shutil.rmtree(scratch, ignore_errors=True)
+
+
+def e2e_gate() -> str:
+    """The real count-matrix run is owned by ``seqforge kb e2e`` — never implicitly run by compose."""
+    if not (have("STAR") and _have_genome()):
+        return "skip"
+    return "skip"  # available but deliberately not run here; invoke `seqforge kb e2e`
+
+
+def _have_genome() -> bool:
+    import importlib.util
+
+    return importlib.util.find_spec("genome") is not None