allelix 1.8.1__py3-none-any.whl

allelix/annotators/gnomad.py

@@ -0,0 +1,212 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 dial481
+"""gnomAD population frequency enrichment.
+
+gnomAD is not a clinical annotator — it does not produce Annotation
+objects. It enriches existing annotations with population allele
+frequency context. The pipeline calls ``bulk_lookup()`` after all
+annotators have run, and stamps each annotation's ``allele_frequency``
+field.
+
+License: ODbL v1.0 (Open Database License). We extract only rsID +
+allele frequencies (no SpliceAI or other restrictively licensed fields).
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from typing import TYPE_CHECKING, ClassVar
+
+from allelix.annotators.base import Annotator, LicenseDescriptor
+from allelix.databases._versions import GNOMAD_SCHEMA_VERSION
+from allelix.databases.gnomad_loader import (
+    GNOMAD_CACHE_URL,
+    GNOMAD_DB_FILENAME,
+    GNOMAD_EXPECTED_SHA256,
+    install_prebuilt_cache,
+)
+from allelix.databases.manager import (
+    download,
+    get_database_info,
+    verify_file_hash,
+)
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from allelix.models import Annotation, Variant
+
+logger = logging.getLogger(__name__)
+
+_BULK_BATCH_SIZE = 900
+
+
+class GnomadAnnotator(Annotator):
+    """Population frequency enrichment from gnomAD.
+
+    Subclasses Annotator for ``db update`` / ``db status`` / ``is_ready()``
+    integration. ``annotate()`` always returns ``[]`` — gnomAD does not
+    participate in the per-variant annotation loop.
+    """
+
+    name: ClassVar[str] = "gnomad"
+    display_name: ClassVar[str] = "gnomAD"
+    attribution: ClassVar[str] = "gnomAD"
+    requires_download: ClassVar[bool] = True
+    server_driven_freshness: ClassVar[bool] = False
+    license: ClassVar[LicenseDescriptor] = LicenseDescriptor(
+        spdx="ODbL-1.0",
+        license_url="https://opendatacommons.org/licenses/odbl/1-0/",
+        attribution_text=("Population frequencies sourced from gnomAD, used under ODbL v1.0."),
+        source_url="https://gnomad.broadinstitute.org",
+        commercial_ok=True,
+    )
+
+    def __init__(self, data_dir: Path) -> None:
+        """Bind to the data directory."""
+        super().__init__(data_dir)
+        self._db_path = data_dir / GNOMAD_DB_FILENAME
+        self._conn: sqlite3.Connection | None = None
+
+    def _connection(self) -> sqlite3.Connection:
+        if self._conn is None:
+            if not self._db_path.exists():
+                raise FileNotFoundError(
+                    f"gnomAD cache not found at {self._db_path}. Run `allelix db update` first."
+                )
+            self._conn = sqlite3.connect(self._db_path)
+        return self._conn
+
+    def setup(self) -> None:
+        """Download the pre-built gnomAD exome frequency cache from HuggingFace."""
+        gz_path = self.data_dir / "gnomad.sqlite.gz"
+        download(GNOMAD_CACHE_URL, gz_path)
+        verify_file_hash(gz_path, "sha256", GNOMAD_EXPECTED_SHA256)
+        install_prebuilt_cache(
+            gz_path,
+            self._db_path,
+            source_url=GNOMAD_CACHE_URL,
+        )
+        try:
+            gz_path.unlink()
+        except OSError:
+            logger.warning("Could not remove staged file at %s", gz_path)
+
+    def is_ready(self) -> bool:
+        """True when the gnomAD SQLite cache exists with current schema version."""
+        info = get_database_info(self._db_path, "gnomad")
+        if info is None:
+            return False
+        tag = info.get("local_version_tag") or ""
+        return tag == f"sv:{GNOMAD_SCHEMA_VERSION}" or not tag
+
+    def version(self) -> str | None:
+        """Return the cached database version, or None."""
+        info = get_database_info(self._db_path, "gnomad")
+        return info["version"] if info else None
+
+    def record_count(self) -> int | None:
+        """Return the number of rsIDs in the cache, or None."""
+        info = get_database_info(self._db_path, "gnomad")
+        return info["record_count"] if info else None
+
+    def close(self) -> None:
+        """Close the SQLite connection if open."""
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+    def fetch_remote_signal(self) -> str | None:
+        """Code-driven source — no runtime freshness probe (ADR-0030)."""
+        return None
+
+    def cached_remote_signal(self) -> str | None:
+        """Code-driven source — no cached signal to compare (ADR-0030)."""
+        return None
+
+    def annotate(self, variant: Variant) -> list[Annotation]:
+        """Not used — gnomAD enriches, does not annotate. Always returns []."""
+        return []
+
+    def lookup(self, rsid: str) -> float | None:
+        """Return global allele frequency for a single rsID, or None."""
+        conn = self._connection()
+        row = conn.execute(
+            "SELECT MAX(af) FROM gnomad_frequencies WHERE rsid = ?", (rsid,)
+        ).fetchone()
+        return row[0] if row else None
+
+    def bulk_lookup(self, rsids: set[str]) -> dict[str, float]:
+        """Return ``{rsid: af}`` for all rsIDs found in the cache.
+
+        Fallback for annotations without a known alt allele. Uses MAX to
+        resolve multi-allelic sites. Prefer ``bulk_lookup_by_alt`` when alt
+        is available.
+
+        Batches into chunks of 900 to stay within SQLite's variable limit.
+        """
+        if not rsids:
+            return {}
+        conn = self._connection()
+        result: dict[str, float] = {}
+        rsid_list = list(rsids)
+        for i in range(0, len(rsid_list), _BULK_BATCH_SIZE):
+            batch = rsid_list[i : i + _BULK_BATCH_SIZE]
+            placeholders = ",".join("?" * len(batch))
+            rows = conn.execute(
+                f"SELECT rsid, MAX(af) FROM gnomad_frequencies"
+                f" WHERE rsid IN ({placeholders}) GROUP BY rsid",
+                batch,
+            ).fetchall()
+            for rsid, af in rows:
+                if af is not None:
+                    result[rsid] = af
+        return result
+
+    def bulk_resolve_coordinates(
+        self, rsids: set[str]
+    ) -> dict[str, list[tuple[str, int, str, str]]]:
+        """Return ``{rsid: [(chrom, pos, ref, alt), ...]}`` from the gnomAD cache.
+
+        Maps rsIDs to genomic coordinates for coordinate-based lookups
+        (CADD, future VCF-keyed sources). Multi-allelic sites return
+        multiple tuples per rsid.
+        """
+        if not rsids:
+            return {}
+        conn = self._connection()
+        result: dict[str, list[tuple[str, int, str, str]]] = {}
+        rsid_list = list(rsids)
+        for i in range(0, len(rsid_list), _BULK_BATCH_SIZE):
+            batch = rsid_list[i : i + _BULK_BATCH_SIZE]
+            placeholders = ",".join("?" * len(batch))
+            rows = conn.execute(
+                f"SELECT rsid, chrom, pos, ref, alt FROM gnomad_frequencies"
+                f" WHERE rsid IN ({placeholders})",
+                batch,
+            ).fetchall()
+            for rsid, chrom, pos, ref, alt in rows:
+                result.setdefault(rsid, []).append((chrom, pos, ref, alt))
+        return result
+
+    def bulk_lookup_by_alt(self, keys: set[tuple[str, str]]) -> dict[tuple[str, str], float]:
+        """Return ``{(rsid, alt): af}`` for exact allele matches."""
+        if not keys:
+            return {}
+        conn = self._connection()
+        result: dict[tuple[str, str], float] = {}
+        key_list = list(keys)
+        batch_size = _BULK_BATCH_SIZE // 2
+        for i in range(0, len(key_list), batch_size):
+            batch = key_list[i : i + batch_size]
+            clauses = " OR ".join(["(rsid = ? AND alt = ?)"] * len(batch))
+            params = [v for rsid, alt in batch for v in (rsid, alt)]
+            rows = conn.execute(
+                f"SELECT rsid, alt, af FROM gnomad_frequencies WHERE {clauses}",
+                params,
+            ).fetchall()
+            for rsid, alt, af in rows:
+                if af is not None:
+                    result[(rsid, alt)] = af
+        return result

allelix/annotators/gwas.py

@@ -0,0 +1,354 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 dial481
+"""GWAS Catalog annotator. Source-attributed trait associations."""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import sqlite3
+import zipfile
+from typing import TYPE_CHECKING, ClassVar
+
+from allelix.annotators.base import Annotator, LicenseDescriptor
+from allelix.databases.gwas_loader import (
+    _CATEGORIZER_VERSION,
+    _REQUIRED_GWAS_COLUMNS,
+    GWAS_CATALOG_URL,
+    GWAS_DB_FILENAME,
+    load_gwas_tsv,
+    schema_is_current,
+)
+from allelix.databases.manager import (
+    _ensure_local_version_tag_column,
+    download,
+    get_database_info,
+    head_request_headers,
+)
+from allelix.models import Annotation
+
+_EXCLUDED_TRAIT_CATEGORIES = frozenset(
+    {
+        "body_measurement",
+        "lipid_measurement",
+        "hematological_measurement",
+        "other_measurement",
+        "behavioral",
+    }
+)
+
+_MUST_INCLUDE_RSIDS = frozenset(
+    {
+        "rs10737680",  # CFH — age-related macular degeneration
+        "rs11209026",  # IL23R — inflammatory bowel disease
+        "rs9271366",  # HLA-DRB1 — multiple sclerosis
+    }
+)
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from allelix.models import Variant
+
+logger = logging.getLogger(__name__)
+
+
+def _magnitude(p_value: float | None, or_beta: float | None) -> float:
+    """Derive magnitude from p-value and optional effect size."""
+    if p_value is None:
+        base = 2.0
+    elif p_value < 5e-100:
+        base = 8.0
+    elif p_value < 5e-20:
+        base = 7.0
+    elif p_value < 5e-8:
+        base = 6.0
+    elif p_value < 5e-6:
+        base = 4.0
+    elif p_value < 5e-4:
+        base = 3.0
+    else:
+        base = 2.0
+
+    if or_beta is not None and or_beta > 0:
+        if or_beta >= 3.0 or or_beta <= 0.33:
+            base = min(base + 1.0, 9.0)
+        elif or_beta >= 2.0 or or_beta <= 0.5:
+            base = min(base + 0.5, 9.0)
+
+    return base
+
+
+_UNKNOWN_RISK_ALLELE_MAG_CAP = 3.0
+
+
+class GWASCatalogAnnotator(Annotator):
+    """Annotates variants with GWAS Catalog trait associations."""
+
+    name: ClassVar[str] = "gwas"
+    display_name: ClassVar[str] = "GWAS Catalog"
+    attribution: ClassVar[str] = "GWAS Catalog"
+    requires_download: ClassVar[bool] = True
+    license: ClassVar[LicenseDescriptor] = LicenseDescriptor(
+        spdx="custom-embl-ebi",
+        license_url="https://www.ebi.ac.uk/gwas/docs/about",
+        attribution_text=(
+            "GWAS Catalog data sourced from NHGRI-EBI GWAS Catalog,"
+            " available under EMBL-EBI Terms of Use."
+        ),
+        source_url="https://www.ebi.ac.uk/gwas/",
+        commercial_ok=True,
+    )
+
+    def __init__(self, data_dir: Path, *, filter_traits: bool = True) -> None:
+        """Initialize with path to the data directory."""
+        super().__init__(data_dir)
+        self._db_path = data_dir / GWAS_DB_FILENAME
+        self._conn: sqlite3.Connection | None = None
+        self._filter_traits = filter_traits
+
+    def _connection(self) -> sqlite3.Connection:
+        if self._conn is None:
+            self._conn = sqlite3.connect(self._db_path)
+        return self._conn
+
+    def setup(self) -> None:
+        """Download GWAS Catalog associations ZIP, extract TSV, and ingest."""
+        url = GWAS_CATALOG_URL
+        signal = self.fetch_remote_signal()
+        if signal is None:
+            msg = (
+                "gwas: cannot verify remote freshness signal. "
+                "Refresh aborted to avoid persisting an incomplete cache stamp. "
+                "Retry, or pass --force if you accept that next `db update` "
+                "will re-download to re-establish the signal."
+            )
+            raise RuntimeError(msg)
+        zip_path = self.data_dir / "gwas_catalog_associations.zip"
+        tsv_path = self.data_dir / "gwas_catalog_associations.tsv"
+        # No content-hash verification: EBI publishes no checksum for this
+        # file and the content is mutable, so there is nothing to pin or
+        # fetch. TLS + Content-Length truncation guard only. See ADR-0029.
+        download(url, zip_path)
+        try:
+            with zipfile.ZipFile(zip_path) as zf:
+                tsv_names = [n for n in zf.namelist() if n.endswith(".tsv")]
+                if not tsv_names:
+                    msg = f"No .tsv file found in {zip_path}"
+                    raise RuntimeError(msg)
+                zf.extract(tsv_names[0], self.data_dir)
+                extracted = self.data_dir / tsv_names[0]
+                if extracted != tsv_path:
+                    extracted.rename(tsv_path)
+            load_gwas_tsv(tsv_path, self._db_path, source_url=url, remote_signal=signal)
+        finally:
+            try:
+                zip_path.unlink()
+            except FileNotFoundError:
+                pass
+            except OSError:
+                logger.warning("Could not remove staged file at %s", zip_path)
+
+    def is_ready(self) -> bool:
+        """Return True when the local GWAS cache exists and has current schema.
+
+        Handles three states:
+
+        1. Current tag in ``local_version_tag`` — ready.
+        2. No tag (legacy cache or ``|cv:`` still in ``remote_signal``) —
+           one-shot migration via ``_stamp_existing_gwas_cache``.
+        3. Stale tag (categorizer bumped) — auto-reingest from cached TSV
+           if still on disk.
+        """
+        info = get_database_info(self._db_path, "gwas")
+        if info is None:
+            return False
+        tag = info.get("local_version_tag") or ""
+        if tag == f"cv:{_CATEGORIZER_VERSION}":
+            return _has_current_gwas_columns(self._db_path)
+        if not tag and _stamp_existing_gwas_cache(self._db_path):
+            return _has_current_gwas_columns(self._db_path)
+        tsv_path = self.data_dir / "gwas_catalog_associations.tsv"
+        if tsv_path.exists():
+            print(
+                "GWAS categorizer changed — re-ingesting from cached TSV...",
+                flush=True,
+            )
+            try:
+                load_gwas_tsv(
+                    tsv_path,
+                    self._db_path,
+                    source_url=GWAS_CATALOG_URL,
+                    remote_signal=self.cached_remote_signal(),
+                )
+            except Exception:
+                logger.warning("Auto-reingest from cached TSV failed", exc_info=True)
+                return False
+            return schema_is_current(self._db_path)
+        return False
+
+    def version(self) -> str | None:
+        """Return the cached database version string, or None."""
+        info = get_database_info(self._db_path, "gwas")
+        return info["version"] if info else None
+
+    def record_count(self) -> int | None:
+        """Return the number of cached GWAS association records, or None."""
+        info = get_database_info(self._db_path, "gwas")
+        return info["record_count"] if info else None
+
+    def close(self) -> None:
+        """Close the SQLite connection if open."""
+        if self._conn is not None:
+            self._conn.close()
+            self._conn = None
+
+    def fetch_remote_signal(self) -> str | None:
+        """Probe the GWAS Catalog URL for ETag or Last-Modified."""
+        headers = head_request_headers(GWAS_CATALOG_URL)
+        if headers is None:
+            return None
+        etag = headers.get("ETag") or headers.get("Etag")
+        last_modified = headers.get("Last-Modified") or headers.get("Last-modified")
+        if etag:
+            return f"etag:{etag.strip()}"
+        if last_modified:
+            return f"lm:{last_modified.strip()}"
+        return None
+
+    def cached_remote_signal(self) -> str | None:
+        """Return the remote signal stored during the last successful ingest."""
+        info = get_database_info(self._db_path, "gwas")
+        if not info or not info["remote_signal"]:
+            return None
+        return info["remote_signal"] or None
+
+    def annotate(self, variant: Variant) -> list[Annotation]:
+        """Return GWAS Catalog annotations for variants the user carries.
+
+        Carrier matching uses the risk allele when specified. When the
+        GWAS entry doesn't specify a risk allele, the annotation fires
+        on rsid match alone with a magnitude penalty.
+        """
+        if variant.is_no_call:
+            return []
+
+        sql = (
+            "SELECT risk_allele, trait, p_value, or_beta, gene, "
+            "study_accession, pubmed_id, trait_category "
+            "FROM gwas_associations WHERE rsid = ?"
+        )
+        rows = self._connection().execute(sql, (variant.rsid,)).fetchall()
+        annotations: list[Annotation] = []
+        user_diploid = _user_diploid(variant)
+
+        for row in rows:
+            (
+                risk_allele,
+                trait,
+                p_value,
+                or_beta,
+                gene,
+                study_accession,
+                pubmed_id,
+                trait_category,
+            ) = row
+
+            if self._filter_traits and trait_category in _EXCLUDED_TRAIT_CATEGORIES:
+                continue
+
+            if risk_allele is not None:
+                if variant.allele1 != risk_allele and variant.allele2 != risk_allele:
+                    continue
+                mag = _magnitude(p_value, or_beta)
+                risk_note = ""
+            else:
+                # ADR-0024: unknown risk allele fires on rsID match alone
+                # but capped at 3.0 so it doesn't pass typical --min-magnitude
+                # thresholds. Without knowing which allele is the risk allele,
+                # we can't apply the carrier rule (ADR-0007).
+                mag = min(_magnitude(p_value, or_beta), _UNKNOWN_RISK_ALLELE_MAG_CAP)
+                risk_note = " (risk allele not specified in study)"
+
+            p_str = f"p={p_value:.1e}" if p_value is not None else "p=N/A"
+            gene_str = gene or "—"
+            description = f"GWAS Catalog: {trait} ({p_str}, gene: {gene_str}){risk_note}"
+
+            references: list[str] = []
+            if pubmed_id:
+                references.append(f"pubmed:{pubmed_id}")
+            if study_accession:
+                references.append(f"gwas:{study_accession}")
+
+            annotations.append(
+                Annotation(
+                    source=self.name,
+                    rsid=variant.rsid,
+                    significance="gwas_association",
+                    category="trait",
+                    magnitude=mag,
+                    description=description,
+                    attribution=self.attribution,
+                    genotype_match=user_diploid,
+                    references=references,
+                    condition=trait,
+                    gene=gene or "",
+                    alt="",
+                    is_must_include=variant.rsid in _MUST_INCLUDE_RSIDS,
+                )
+            )
+        return annotations
+
+
+def _user_diploid(variant: Variant) -> str:
+    """Sorted two-letter diploid for SNVs; indel passthrough verbatim."""
+    a1, a2 = variant.allele1, variant.allele2
+    if len(a1) == 1 and len(a2) == 1:
+        return "".join(sorted((a1, a2)))
+    return f"{a1}/{a2}"
+
+
+def _has_current_gwas_columns(db_path: Path) -> bool:
+    """True iff the gwas_associations table has the required columns."""
+    try:
+        with contextlib.closing(sqlite3.connect(db_path)) as conn:
+            cols = {row[1] for row in conn.execute("PRAGMA table_info(gwas_associations)")}
+            return _REQUIRED_GWAS_COLUMNS.issubset(cols)
+    except sqlite3.DatabaseError:
+        return False
+
+
+def _stamp_existing_gwas_cache(db_path: Path) -> bool:
+    """One-shot migration: stamp ``local_version_tag`` on a GWAS cache.
+
+    Handles legacy caches with ``|cv:N`` baked into ``remote_signal``
+    by moving the tag and cleaning the signal. Returns True if the
+    current categorizer version is now stamped.
+    """
+    if not db_path.exists():
+        return False
+    tag = f"cv:{_CATEGORIZER_VERSION}"
+    try:
+        with contextlib.closing(sqlite3.connect(db_path)) as conn:
+            _ensure_local_version_tag_column(conn)
+            row = conn.execute(
+                "SELECT remote_signal, local_version_tag FROM database_versions WHERE name='gwas'"
+            ).fetchone()
+            if not row:
+                return False
+            sig, existing_tag = row
+            if existing_tag == tag:
+                return True
+            if existing_tag is not None:
+                return False
+            clean_signal = (sig or "").split("|cv:")[0]
+            conn.execute(
+                "UPDATE database_versions "
+                "SET remote_signal = ?, local_version_tag = ? "
+                "WHERE name = 'gwas'",
+                (clean_signal, tag),
+            )
+            conn.commit()
+        return True
+    except (sqlite3.OperationalError, sqlite3.DatabaseError):
+        return False