genokeeper 0.1.0__py3-none-any.whl

genokeeper/__init__.py

@@ -0,0 +1,5 @@
+"""genokeeper: findings from the user's own whole-genome data, kept on this machine."""
+
+from importlib.metadata import version
+
+__version__ = version("genokeeper")

genokeeper/align.py

@@ -0,0 +1,164 @@
+"""Stage 2a alignment: the platform gate on the delivered FASTQ, and bwa mem to a
+coordinate-sorted, duplicate-marked CRAM on the GRCh38 no-alt FASTA.
+
+The platform gate stops reads the T7 checkpoint was not trained on: a DNBSEQ-G400 run
+(flowcell `V...`) or a DNBSEQ-T7+ run (flowcell `ML...`, base qualities binned to four
+levels). Its anchor comes from the FASTQ itself, the flowcell ID the sequencer writes into
+every read name and the quality characters of every base, which a mislabelled delivery or a
+file mix-up cannot move because no one downstream of the instrument rewrites them. A
+sequence archive (SRA, ENA, DDBJ) puts its run accession first and keeps the instrument's
+name as the second field; the gate reads that one.
+"""
+
+import gzip
+import os
+import re
+import secrets
+import shlex
+import shutil
+import subprocess
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from genokeeper.tools import ToolFailed, gated, require
+
+# The original DNBSEQ-T7 names its flowcells E...: the platform the T7 checkpoint wants.
+T7_FLOWCELL = "E"
+# T7+ bins base qualities to four levels; an unbinned T7 run writes dozens.
+MIN_QUALITY_LEVELS = 5
+_FLOWCELL = re.compile(r"^@([A-Z]+)\d")
+_ARCHIVED = re.compile(r"^@[SED]RR\d+\.\d+\s+(?=[A-Z]+\d)")
+_KNOWN = {"E": "DNBSEQ-T7", "V": "DNBSEQ-G400", "ML": "DNBSEQ-T7+"}
+
+
+@dataclass(frozen=True)
+class Platform:
+    """What the FASTQ's first reads show: the flowcell's letter prefix and how many distinct
+    base quality values occur."""
+
+    flowcell_prefix: str
+    quality_levels: int
+
+
+class PlatformRefused(RuntimeError):
+    """The FASTQ is not from the platform the T7 checkpoint was trained on."""
+
+
+def _records(fastq: Path) -> Iterator[tuple[str, str]]:
+    """(name line, quality line) of each record, gzipped or plain."""
+    with fastq.open("rb") as raw:
+        gzipped = raw.read(2) == b"\x1f\x8b"
+    opener = gzip.open if gzipped else open
+    with opener(fastq, "rt") as f:
+        while name := f.readline():
+            f.readline()
+            f.readline()
+            yield name.rstrip("\n"), f.readline().rstrip("\n")
+
+
+def check_platform(
+    fastq1: Path, *, reads: int = 100_000, names: bool = True
+) -> Platform:
+    """The platform of the first `reads` records of `fastq1`; raises `PlatformRefused` unless
+    every read name carries an E... flowcell and the qualities take at least
+    `MIN_QUALITY_LEVELS` values. Only aggregates leave this function.
+
+    `names=False` skips the flowcell tests and reports the first alignment's `E`: a FASTQ
+    written back from a calibration CRAM carries ENA's accession alone (bwa kept the
+    name's first word), and its reads passed the full gate when that CRAM was aligned. The
+    quality test runs as before."""
+    prefixes: set[str] = set()
+    qualities: set[str] = set()
+    seen = 0
+    for name, quality in _records(fastq1):
+        match = _FLOWCELL.match(_ARCHIVED.sub("@", name, count=1))
+        prefixes.add(match.group(1) if match else "?")
+        qualities.update(quality)
+        seen += 1
+        if seen == reads:
+            break
+    if not seen:
+        raise PlatformRefused(f"{fastq1}: no reads")
+    if not names:
+        prefixes = {T7_FLOWCELL}
+    if len(prefixes) > 1:
+        raise PlatformRefused(
+            f"{fastq1}: reads from flowcells {' and '.join(sorted(prefixes))}; one run's"
+            " FASTQ comes from one kind of flowcell"
+        )
+    prefix = prefixes.pop()
+    if prefix != T7_FLOWCELL:
+        kind = _KNOWN.get(prefix, "an unknown platform")
+        raise PlatformRefused(
+            f"{fastq1}: flowcell prefix {prefix} ({kind}); the T7 checkpoint is for the"
+            f" original DNBSEQ-T7 ({T7_FLOWCELL}...)"
+        )
+    if len(qualities) < MIN_QUALITY_LEVELS:
+        raise PlatformRefused(
+            f"{fastq1}: {len(qualities)} base quality levels (binned, as DNBSEQ-T7+ writes"
+            f" them); the T7 checkpoint was trained on unbinned qualities"
+        )
+    return Platform(flowcell_prefix=prefix, quality_levels=len(qualities))
+
+
+def align(
+    fastq1: Path, fastq2: Path, fasta: Path, out: Path, *, sample: str, threads: int
+) -> Path:
+    """`bwa mem | samtools fixmate -m | samtools sort | samtools markdup -O cram` into `out`,
+    indexed, as one unit behind the core gate (`dna:align`). bwa finds its index beside
+    `fasta` (`<fasta>.bwt` and the rest, as the bootstrap unpacks it).
+
+    Every stage's exit status is checked: the first that failed raises `ToolFailed` with its
+    stderr. The CRAM and its index are written under temp names and renamed only when every
+    stage succeeded, so a failed or killed run leaves no `out`."""
+    require("bwa", "samtools")
+    work = out.parent / f".align-{secrets.token_hex(4)}"
+    work.mkdir()
+    tmp = out.with_name(f".{out.name}.{secrets.token_hex(4)}.cram")
+    t = str(threads)
+    group = f"@RG\\tID:{sample}\\tSM:{sample}\\tPL:DNBSEQ"
+    stages = [
+        # -K: a fixed batch size makes the output independent of the thread count.
+        ["bwa", "mem", "-t", t, "-K", "100000000", "-Y", "-R", group, str(fasta)]
+        + [str(fastq1), str(fastq2)],
+        ["samtools", "fixmate", "-m", "-u", "-@", t, "-", "-"],
+        ["samtools", "sort", "-u", "-@", t, "-T", str(work / "sort"), "-"],
+        ["samtools", "markdup", "-@", t, "-T", str(work / "markdup")]
+        + ["-O", "cram", "--reference", str(fasta), "-", str(tmp)],
+        ["samtools", "index", "-@", t, str(tmp)],
+    ]
+    status = work / "status"
+    pipe = " | ".join(
+        f"{shlex.join(stage)} 2> {shlex.quote(str(work / f'err{i}'))}"
+        for i, stage in enumerate(stages[:-1])
+    )
+    index = f"{shlex.join(stages[-1])} 2> {shlex.quote(str(work / 'err4'))}"
+    script = (
+        f'{pipe}\ns=("${{PIPESTATUS[@]}}")\n'
+        f'if [ "${{s[*]}}" = "0 0 0 0" ]; then {index}; s+=($?); fi\n'
+        f'echo "${{s[*]}}" > {shlex.quote(str(status))}\n'
+    )
+    try:
+        ran = subprocess.run(
+            gated("dna:align", ["bash", "-c", script]),
+            check=False,  # each stage's status is read from the status file below
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            text=True,
+            errors="replace",
+        )
+        codes = [int(c) for c in status.read_text().split()] if status.exists() else []
+        if not codes:
+            raise ToolFailed(ran.returncode, ["bash", "-c", script], None, ran.stderr)
+        for i, code in enumerate(codes):
+            if code:
+                stderr = (work / f"err{i}").read_text(errors="replace")
+                raise ToolFailed(code, stages[i], None, stderr)
+        os.replace(f"{tmp}.crai", f"{out}.crai")
+        os.replace(tmp, out)
+    finally:
+        tmp.unlink(missing_ok=True)
+        Path(f"{tmp}.crai").unlink(missing_ok=True)
+        shutil.rmtree(work, ignore_errors=True)
+    return out

genokeeper/ancestry/__init__.py

@@ -0,0 +1,4 @@
+"""Ancestry beyond pgsc_calc's similarity (slice 3f, "Admixture"): supervised admixture
+proportions against the panel's superpopulations (`admixture`) and the archaic share by
+hmmix (`archaic`). Both read the user's genotypes in their containers, offline, behind the
+core gate as `dna:ancestry`."""

genokeeper/ancestry/admixture.py

@@ -0,0 +1,95 @@
+"""Supervised admixture proportions by fastmixture 1.3.0 (slice 3f, "Admixture").
+
+The panel's reference (`typing_files.build_panel_reference`) is its unrelated samples'
+LD-pruned SNPs (`pruned.bim`), fastmixture's supervised P file with the superpopulations as
+labels 1..K (AMR unlabelled) and `groups.json` naming each label. The user's genotypes at
+the pruned SNPs (`pruned_alleles`) are written by `pgsc.write_target_vcf` (reference rows
+explicit, an unknown call `./.`, one row per `pruned.bim` row, where a position can hold two
+or three: the projection needs every SNP of the P file), turned into a bed by one plink2 call in pgsc_calc's image (`to_bed`; variant IDs `chrom:pos:ref:alt`, as
+the panel's), and projected on the P file (`run_projection`). Both containers run offline,
+gated `dna:ancestry`.
+"""
+
+import json
+from collections.abc import Sequence
+from pathlib import Path
+
+from genokeeper.containers import PLATFORMS, run_container
+from genokeeper.genotype.gvcf import Allele
+from genokeeper.typing_files import LABEL, SEED
+
+PROJECTION = "projection"
+
+
+def pruned_alleles(bim: Path) -> list[Allele]:
+    """The pruned SNPs of the panel's `pruned.bim` in its order, as `(chr<N>, pos, REF,
+    ALT)`: plink2 writes ALT as A1 and REF as A2."""
+    alleles: list[Allele] = []
+    for line in bim.read_text().splitlines():
+        chrom, _id, _cm, pos, a1, a2 = line.split("\t")
+        alleles.append((f"chr{chrom.removeprefix('chr')}", int(pos), a2, a1))
+    return alleles
+
+
+def read_groups(path: Path) -> tuple[str, ...]:
+    """`groups.json`'s labelled groups in label order (1..K)."""
+    groups: dict[str, int] = json.loads(path.read_text())["groups"]
+    return tuple(sorted(groups, key=groups.__getitem__))
+
+
+def read_q(path: Path, groups: Sequence[str]) -> dict[str, float]:
+    """fastmixture's one-sample Q file, column k the k-th of `groups`; `ValueError` for
+    other than one row of `len(groups)` columns."""
+    rows = [line.split() for line in path.read_text().splitlines() if line.strip()]
+    if len(rows) != 1 or len(rows[0]) != len(groups):
+        width = len(rows[0]) if rows else 0
+        raise ValueError(
+            f"{path}: {len(rows)} rows of {width} columns, not one of {len(groups)}"
+        )
+    return dict(zip(groups, (float(v) for v in rows[0]), strict=True))
+
+
+def to_bed(vcf: Path, out: Path) -> Path:
+    """`vcf` (the user's calls at `pruned_alleles`) as the PLINK bed `out` with its bim
+    and fam, by plink2 in pgsc_calc's image; `out`."""
+    out.parent.mkdir(parents=True, exist_ok=True)
+    run_container(
+        "pgsc_plink2",
+        [
+            "plink2", "--vcf", f"/in/{vcf.name}", "--set-all-var-ids", "@:#:$r:$a",
+            "--make-bed", "--out", f"/out/{out.stem}",
+        ],
+        mounts=[(vcf.parent, "/in", True), (out.parent, "/out", False)],
+        platform=PLATFORMS["pgsc_plink2"],
+        label=LABEL,
+    )  # fmt: skip
+    if not out.is_file():
+        raise RuntimeError(f"plink2 wrote no {out.name}")
+    return out
+
+
+def run_projection(bed: Path, p_file: Path, k: int, out: Path) -> dict[str, float]:
+    """The user's `bed` projected on the supervised `p_file` (K `k`, seed `SEED`) by
+    fastmixture, its Q file written into `out`; the proportions labelled by the
+    `groups.json` beside `p_file`. `ValueError` when `k` is not that file's K."""
+    groups = read_groups(p_file.parent / "groups.json")
+    if len(groups) != k:
+        raise ValueError(
+            f"K {k}, but {p_file.parent / 'groups.json'} has {len(groups)}"
+        )
+    out.mkdir(parents=True, exist_ok=True)
+    run_container(
+        "fastmixture",
+        [
+            "fastmixture", "--bfile", f"/in/{bed.stem}", "--K", str(k),
+            "--seed", str(SEED), "--threads", "1",
+            "--projection", f"/ref/{p_file.name}", "--out", f"/out/{PROJECTION}",
+        ],
+        mounts=[(bed.parent, "/in", True), (p_file.parent, "/ref", True), (out, "/out", False)],
+        platform=PLATFORMS["fastmixture"],
+        label=LABEL,
+    )  # fmt: skip
+    q = out / f"{PROJECTION}.K{k}.s{SEED}.Q"
+    if not q.is_file():
+        raise RuntimeError(f"fastmixture wrote no {q.name}")
+    return read_q(q, groups)

genokeeper/ancestry/archaic.py

@@ -0,0 +1,147 @@
+"""The archaic share by hmmix 0.9.2 (slice 3f, "Admixture"; ruling R5).
+
+`run_hmmix` runs, in one hmmix container (offline, gated `dna:ancestry`), the steps Task 1
+recorded: the user's PASS calls split one VCF per autosome (hmmix pairs each with its
+ancestral FASTA by the varying part of the names), `create_ingroup` against the 1000G+HGDP
+outgroup on the strict callability mask (refused when the observations come back nearly
+empty, as they do silently without vcftools), `train`, and `decode -admixpop` with the four
+high-coverage archaic genomes, which writes `decoded.diploid.txt`.
+
+A decoded `Archaic` segment is Neanderthal when the most derived SNPs it shares with any of
+the three Neanderthals (Altai, Vindija, Chagyrskaya) outnumber those it shares with
+Denisova, Denisovan when Denisova's are more, and unclassified on a tie (`Segment.origin`;
+hmmix's README reads its example segment this way, and the published anchor kept fragments
+that share more with a Neanderthal than with Denisova). `archaic_share` takes the
+Neanderthal segments' raw span over `ARCHAIC_GENOME_BP` (R15: the anchor and the statement
+are the Neanderthal share; the Denisovan span is kept beside it), the published
+anchor's fixed denominator (Coll Macià et al. 2026, Mol Biol Evol 43:msag134: 2 ×
+2,550,757,832 bp, autosomes, both haplotypes), not the strict mask's sum: the unphased
+segments' span stands for both haplotypes', as archaic segments rarely overlap between them.
+"""
+
+import csv
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from genokeeper.containers import PLATFORMS, run_container
+from genokeeper.typing_files import LABEL
+
+ARCHAIC_GENOME_BP = 2 * 2_550_757_832
+DECODED = "decoded.diploid.txt"
+NEANDERTHALS = ("AltaiNeandertal", "Vindija33.19", "Chagyrskaya-Phalanx")
+DENISOVA = "Denisova"
+# Fewer observation rows than this means create_ingroup failed silently (Task 5's HG002
+# truth gave 133,532).
+MIN_OBS_ROWS = 1000
+
+_SCRIPT = """\
+set -euo pipefail
+vcf="$1"; work="$2"; min_obs="$3"
+cd "$work"
+mkdir -p anc vcf
+chroms=$(seq 1 22)
+tar -xzf /h/hg38_ancestral.tar.gz -C anc \
+  $(for c in $chroms; do printf 'hg38_ancestral/homo_sapiens_ancestor_%s.fa ' "$c"; done)
+ind=$(bcftools query -l "$vcf" | head -n 1)
+for c in $chroms; do
+  bcftools view -f PASS -r "chr$c" "$vcf" -Oz -o "vcf/owner.chr$c.vcf.gz"
+  bcftools index -f "vcf/owner.chr$c.vcf.gz"
+done
+join() { local IFS=,; echo "$*"; }
+vcfs=$(join $(for c in $chroms; do echo "vcf/owner.chr$c.vcf.gz"; done))
+ancs=$(join $(for c in $chroms; do echo "anc/hg38_ancestral/homo_sapiens_ancestor_$c.fa"; done))
+arch=$(join $(for c in $chroms; do echo "/h/archaic/individuals_highcov.$c.bcf"; done))
+hmmix create_ingroup -ind="$ind" -vcf="$vcfs" -weights=/h/hg38_strick_callability_mask.bed \
+  -outgroup=/h/hg38_Outgroup_1000g_HGDP.txt -ancestral="$ancs" -out=obs
+obs="obs.$ind.txt"
+[ "$(wc -l < "$obs")" -gt "$min_obs" ] || { echo "hmmix: $obs nearly empty" >&2; exit 3; }
+hmmix train -obs="$obs" -weights=/h/hg38_strick_callability_mask.bed -mutrates=/h/hg38_mutationrate.bed \
+  -out=trained.json
+hmmix decode -obs="$obs" -weights=/h/hg38_strick_callability_mask.bed -mutrates=/h/hg38_mutationrate.bed \
+  -param=trained.json -admixpop="$arch" -out=decoded
+rm -rf anc vcf
+"""
+
+
+@dataclass(frozen=True)
+class Segment:
+    """One decoded segment: its span (0-based, end exclusive), state (`Human` or
+    `Archaic`), and the most derived SNPs it shares with one Neanderthal and with
+    Denisova."""
+
+    chrom: str
+    start: int
+    end: int
+    state: str
+    neanderthal: int
+    denisova: int
+
+    @property
+    def origin(self) -> Literal["neanderthal", "denisovan"] | None:
+        """Which archaic group an `Archaic` segment shares more derived SNPs with; None for
+        a tie or a `Human` segment."""
+        if self.state != "Archaic" or self.neanderthal == self.denisova:
+            return None
+        return "neanderthal" if self.neanderthal > self.denisova else "denisovan"
+
+
+@dataclass(frozen=True)
+class ArchaicShare:
+    """The Neanderthal and Denisovan segments' summed spans, and the Neanderthal span's
+    share of the denominator."""
+
+    neanderthal_bp: int
+    denisovan_bp: int
+    share: float
+
+
+def run_hmmix(vcf: Path, files: Path, out: Path) -> Path:
+    """hmmix on the user's indexed `vcf` with its helper files `files`
+    (`typing_files.TypingFiles.hmmix`), its work and `DECODED` in `out`; the decoded file.
+    `RuntimeError` when it exits 0 without it, and what `run_container` raises."""
+    out.mkdir(parents=True, exist_ok=True)
+    run_container(
+        "hmmix",
+        ["bash", "-c", _SCRIPT, "hmmix", f"/in/{vcf.name}", "/w", str(MIN_OBS_ROWS)],
+        mounts=[(vcf.parent, "/in", True), (files, "/h", True), (out, "/w", False)],
+        platform=PLATFORMS["hmmix"],
+        label=LABEL,
+    )
+    decoded = out / DECODED
+    if not decoded.is_file():
+        raise RuntimeError(f"hmmix exited 0 without {DECODED}")
+    return decoded
+
+
+def read_segments(path: Path) -> list[Segment]:
+    """Each segment of hmmix's `decode -admixpop` output."""
+    with path.open(newline="", encoding="utf-8") as f:
+        return [
+            Segment(
+                chrom=row["chrom"],
+                start=int(row["start"]),
+                end=int(row["end"]),
+                state=row["state"],
+                neanderthal=max(int(row[name]) for name in NEANDERTHALS),
+                denisova=int(row[DENISOVA]),
+            )
+            for row in csv.DictReader(f, delimiter="\t")
+        ]
+
+
+def archaic_share(
+    segments: Sequence[Segment], genome_bp: int = ARCHAIC_GENOME_BP
+) -> ArchaicShare:
+    """The Neanderthal and Denisovan segments' raw spans, and the Neanderthal span over
+    `genome_bp`; unclassified segments count for neither."""
+    spans = {"neanderthal": 0, "denisovan": 0}
+    for s in segments:
+        if s.origin is not None:
+            spans[s.origin] += s.end - s.start
+    return ArchaicShare(
+        neanderthal_bp=spans["neanderthal"],
+        denisovan_bp=spans["denisovan"],
+        share=spans["neanderthal"] / genome_bp,
+    )

genokeeper/annotate/__init__.py

@@ -0,0 +1,2 @@
+"""Variant annotation for slice 1e: Ensembl VEP with its plugins, and SpliceAI
+(`genokeeper.annotate.vep`)."""