allelix 1.8.1__py3-none-any.whl

allelix/databases/snpedia_parser.py

@@ -0,0 +1,342 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 dial481
+"""Parse raw SNPedia wiki markup into structured genotype rows.
+
+Called automatically by the SNPedia annotator when raw pages exist but
+the structured ``snpedia_genotypes`` table does not. Can also be invoked
+standalone via ``scripts/parse_snpedia.py``.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import sqlite3
+from datetime import UTC, datetime
+
+import mwparserfromhell
+
+logger = logging.getLogger(__name__)
+
+_PARSER_VERSION = 6
+
+
+def _parse_title_prefix(title: str) -> tuple[str, str] | None:
+    """Extract (prefix, number) from a title like 'Rs12345' or 'I4000178'.
+
+    Returns None if the title doesn't start with Rs or I followed by digits.
+    """
+    if title.startswith("Rs") or title.startswith("I"):
+        prefix = "Rs" if title.startswith("Rs") else "I"
+        rest = title[len(prefix) :]
+        digits = []
+        for ch in rest:
+            if ch.isdigit():
+                digits.append(ch)
+            else:
+                break
+        if digits:
+            return prefix, "".join(digits)
+    return None
+
+
+def _parse_title_alleles(title: str) -> tuple[str, str] | None:
+    """Extract alleles from a title like 'Rs12345(A;G)' or 'I4000178(C;T)'.
+
+    Returns None if the title doesn't contain a valid (allele;allele) suffix.
+    """
+    paren_start = title.find("(")
+    if paren_start == -1 or not title.endswith(")"):
+        return None
+    inner = title[paren_start + 1 : -1]
+    semi = inner.find(";")
+    if semi == -1:
+        return None
+    a1 = inner[:semi].strip()
+    a2 = inner[semi + 1 :].strip()
+    if a1 and a2:
+        return a1, a2
+    return None
+
+
+def _tmpl_param(tmpl: object, name: str) -> str:
+    """Extract a named parameter from a mwparserfromhell template."""
+    if tmpl.has(name):
+        return str(tmpl.get(name).value).strip()
+    return ""
+
+
+_STRUCTURED_SCHEMA = """
+CREATE TABLE IF NOT EXISTS snpedia_genotypes (
+    rsid TEXT NOT NULL,
+    allele1 TEXT NOT NULL,
+    allele2 TEXT NOT NULL,
+    magnitude REAL,
+    repute TEXT,
+    summary TEXT,
+    gene TEXT,
+    scraped_at TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_snpedia_rsid_alleles
+    ON snpedia_genotypes(rsid, allele1, allele2);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_snpedia_genotype_dedup
+    ON snpedia_genotypes(rsid, allele1, allele2, COALESCE(summary, ''));
+
+CREATE TABLE IF NOT EXISTS database_versions (
+    name TEXT PRIMARY KEY,
+    source_url TEXT NOT NULL,
+    version TEXT,
+    downloaded_at TEXT NOT NULL,
+    record_count INTEGER NOT NULL,
+    remote_signal TEXT,
+    local_version_tag TEXT
+);
+"""
+
+
+def parser_is_current(conn: sqlite3.Connection) -> bool:
+    """Return True if the cache was built by the current parser version.
+
+    Checks ``local_version_tag`` first.  If absent, falls back to the
+    legacy ``|pv:N`` suffix in ``remote_signal`` and migrates the tag
+    in-place — avoiding a full re-parse just to populate the column.
+    """
+    tag = f"pv:{_PARSER_VERSION}"
+    try:
+        row = conn.execute(
+            "SELECT local_version_tag FROM database_versions WHERE name='snpedia'"
+        ).fetchone()
+        if row and row[0] == tag:
+            return True
+    except sqlite3.OperationalError:
+        pass
+    try:
+        row = conn.execute(
+            "SELECT remote_signal FROM database_versions WHERE name='snpedia'"
+        ).fetchone()
+        if row and row[0] and f"|pv:{_PARSER_VERSION}" in row[0]:
+            from allelix.databases.manager import _ensure_local_version_tag_column
+
+            _ensure_local_version_tag_column(conn)
+            clean_signal = row[0].split("|pv:")[0]
+            conn.execute(
+                "UPDATE database_versions "
+                "SET remote_signal = ?, local_version_tag = ? WHERE name = 'snpedia'",
+                (clean_signal, tag),
+            )
+            conn.commit()
+            return True
+    except sqlite3.OperationalError:
+        pass
+    return False
+
+
+def _dedupe_existing(conn: sqlite3.Connection) -> int:
+    """Collapse pre-existing duplicate rows in old caches. Returns rows removed."""
+    before = conn.execute("SELECT COUNT(*) FROM snpedia_genotypes").fetchone()[0]
+    conn.execute("""
+        DELETE FROM snpedia_genotypes
+        WHERE rowid NOT IN (
+            SELECT MIN(rowid) FROM snpedia_genotypes
+            GROUP BY rsid, UPPER(allele1), UPPER(allele2), COALESCE(summary, '')
+        )
+    """)
+    after = conn.execute("SELECT COUNT(*) FROM snpedia_genotypes").fetchone()[0]
+    return before - after
+
+
+def detect_raw_table(conn: sqlite3.Connection) -> str | None:
+    """Return the name of the raw pages table, or None if absent."""
+    tables = {
+        row[0]
+        for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
+    }
+    if "_raw_pages" in tables:
+        return "_raw_pages"
+    if "pages" in tables:
+        return "pages"
+    return None
+
+
+def has_structured_table(conn: sqlite3.Connection) -> bool:
+    """Return True if snpedia_genotypes exists and has rows."""
+    try:
+        count = conn.execute("SELECT COUNT(*) FROM snpedia_genotypes").fetchone()[0]
+        return count > 0
+    except sqlite3.OperationalError:
+        return False
+
+
+def parse_raw_pages(db_path: str, *, verbose: bool = False) -> int:
+    """Parse raw wiki markup into structured genotype rows.
+
+    Returns the number of structured rows created.
+    """
+    with contextlib.closing(sqlite3.connect(db_path)) as conn:
+        return _parse_raw_pages_inner(conn, verbose=verbose)
+
+
+def _parse_raw_pages_inner(conn: sqlite3.Connection, *, verbose: bool = False) -> int:
+    """Inner parser logic. Caller owns the connection lifecycle."""
+    raw_table = detect_raw_table(conn)
+    if raw_table is None:
+        return 0
+
+    if verbose:
+        logger.info("Parsing SNPedia raw pages from '%s' table", raw_table)
+
+    conn.execute("DROP INDEX IF EXISTS idx_snpedia_genotype_dedup")
+
+    has_table = conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='snpedia_genotypes'"
+    ).fetchone()
+    if has_table:
+        deduped = _dedupe_existing(conn)
+        if deduped:
+            logger.info("Backfill dedupe: removed %d duplicate row(s)", deduped)
+
+    conn.executescript(_STRUCTURED_SCHEMA)
+    conn.execute("DELETE FROM snpedia_genotypes")
+
+    # Build gene map from SNP pages
+    gene_map: dict[str, str] = {}
+    snp_rows = conn.execute(
+        f"SELECT title, content FROM {raw_table} WHERE category = 'snp'"
+    ).fetchall()
+    print(f"  Building gene map from {len(snp_rows)} SNP pages...", flush=True)
+    for title, content in snp_rows:
+        parsed_prefix = _parse_title_prefix(title)
+        if not parsed_prefix or not content:
+            continue
+        prefix, num = parsed_prefix
+        snp_key = f"{prefix.lower()}{num}"
+        try:
+            wikicode = mwparserfromhell.parse(content)
+            for template in wikicode.filter_templates():
+                tname = template.name.strip().lower()
+                if tname in ("rsnum", "snp"):
+                    gene = _tmpl_param(template, "Gene")
+                    if gene:
+                        gene_map[snp_key] = gene
+                    break
+                if tname == "23andme snp":
+                    gene = _tmpl_param(template, "Gene_s")
+                    if gene:
+                        gene_map[snp_key] = gene
+                    break
+        except Exception:
+            logger.debug("Failed to parse SNP page %s", title, exc_info=True)
+            continue
+
+    print(f"  Gene map: {len(gene_map)} mappings built.", flush=True)
+
+    # Parse genotype pages
+    genotype_rows = conn.execute(
+        f"SELECT title, content, scraped_at FROM {raw_table} WHERE category = 'genotype'"
+    ).fetchall()
+    print(f"  Parsing {len(genotype_rows)} genotype pages...", flush=True)
+
+    batch: list[tuple[str, str, str, float | None, str | None, str | None, str | None, str]] = []
+
+    for title, content, scraped_at in genotype_rows:
+        parsed_prefix = _parse_title_prefix(title)
+        if not parsed_prefix or not content:
+            continue
+
+        prefix, num = parsed_prefix
+        snp_id = f"{prefix.lower()}{num}"
+
+        try:
+            wikicode = mwparserfromhell.parse(content)
+        except Exception:
+            logger.debug("Failed to parse genotype page %s", title, exc_info=True)
+            continue
+
+        templates = [
+            t for t in wikicode.filter_templates() if t.name.strip().lower() == "genotype"
+        ]
+        if not templates:
+            continue
+
+        tmpl = templates[0]
+
+        allele1 = _tmpl_param(tmpl, "allele1").upper()
+        allele2 = _tmpl_param(tmpl, "allele2").upper()
+        if not allele1 or not allele2:
+            title_alleles = _parse_title_alleles(title)
+            if not title_alleles:
+                continue
+            allele1, allele2 = title_alleles[0].upper(), title_alleles[1].upper()
+            if not allele1 or not allele2:
+                continue
+
+        if allele1 > allele2:
+            allele1, allele2 = allele2, allele1
+
+        mag_str = _tmpl_param(tmpl, "magnitude")
+        magnitude: float | None = None
+        if mag_str:
+            try:
+                magnitude = float(mag_str)
+            except ValueError:
+                magnitude = None
+
+        repute = _tmpl_param(tmpl, "repute") or None
+        summary = _tmpl_param(tmpl, "summary") or None
+        gene = gene_map.get(snp_id) or None
+
+        batch.append((snp_id, allele1, allele2, magnitude, repute, summary, gene, scraped_at))
+
+        if len(batch) >= 1000:
+            conn.executemany(
+                "INSERT OR IGNORE INTO snpedia_genotypes "
+                "(rsid, allele1, allele2, magnitude, repute, summary, gene, scraped_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                batch,
+            )
+            batch.clear()
+
+    if batch:
+        conn.executemany(
+            "INSERT OR IGNORE INTO snpedia_genotypes "
+            "(rsid, allele1, allele2, magnitude, repute, summary, gene, scraped_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            batch,
+        )
+
+    row_count = conn.execute("SELECT COUNT(*) FROM snpedia_genotypes").fetchone()[0]
+
+    date_row = conn.execute(f"SELECT MIN(scraped_at) FROM {raw_table}").fetchone()
+    scrape_date = date_row[0][:10] if date_row and date_row[0] else "unknown"
+
+    existing_signal = ""
+    try:
+        sig_row = conn.execute(
+            "SELECT remote_signal FROM database_versions WHERE name = 'snpedia'"
+        ).fetchone()
+        if sig_row and sig_row[0]:
+            existing_signal = sig_row[0].split("|pv:")[0]
+    except sqlite3.OperationalError:
+        pass
+
+    conn.execute("DELETE FROM database_versions WHERE name = 'snpedia'")
+    conn.execute(
+        "INSERT INTO database_versions "
+        "(name, source_url, version, downloaded_at, record_count, "
+        "remote_signal, local_version_tag) "
+        "VALUES (?, ?, ?, ?, ?, ?, ?)",
+        (
+            "snpedia",
+            "https://bots.snpedia.com/api.php",
+            f"scraped {scrape_date} ({row_count} genotypes)",
+            datetime.now(UTC).isoformat(),
+            row_count,
+            existing_signal,
+            f"pv:{_PARSER_VERSION}",
+        ),
+    )
+
+    conn.commit()
+    return row_count

allelix/exporters/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 dial481
+"""Format exporters for parsed genotype data."""

allelix/exporters/plink.py

@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 dial481
+"""PLINK1 binary format (.bed/.bim/.fam) exporter."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from allelix.utils.allele import complement, is_strand_ambiguous
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+    from pathlib import Path
+
+    from allelix.models import Variant
+
+_BED_MAGIC = bytes([0x6C, 0x1B, 0x01])
+
+_CHROM_CODES = {
+    "X": "23",
+    "Y": "24",
+    "MT": "26",
+}
+
+
+def _orient_genotype(
+    allele1: str,
+    allele2: str,
+    ref: str,
+    alt: str,
+) -> tuple[str, str] | None:
+    """Map user alleles to {ref, alt} in a consistent orientation.
+
+    Returns None for palindromic sites, indels, or alleles that don't fit.
+    Both alleles are tested in the same orientation — no mixed-strand.
+    """
+    if len(allele1) != 1 or len(allele2) != 1:
+        return None
+    if is_strand_ambiguous(ref, alt):
+        return None
+
+    pair = {allele1, allele2}
+    if pair <= {ref, alt}:
+        return (allele1, allele2)
+
+    c1, c2 = complement(allele1), complement(allele2)
+    if {c1, c2} <= {ref, alt}:
+        return (c1, c2)
+
+    return None
+
+
+def export_plink(
+    variants: Iterator[Variant],
+    prefix: Path,
+    build: str,
+    ref_alt_map: dict[str, tuple[str, str]] | None = None,
+) -> tuple[int, int, int, int]:
+    """Write .bed/.bim/.fam from parsed variants.
+
+    Args:
+        variants: Parsed variant iterator (consumed once).
+        prefix: Base path for output files.
+        build: Genome build label (informational, not used for liftover).
+        ref_alt_map: ``{rsid: (ref, alt)}`` from gnomAD coordinate resolution.
+            When provided, uses ref/alt to assign A1/A2 for proper allele coding.
+            When None or rsid missing, falls back to ``A2="0"`` for homozygotes.
+
+    Returns:
+        ``(variants_written, no_calls_skipped, indels_skipped, monomorphic_count)``
+
+    Note:
+        No-call variants and indels (multi-character alleles) are dropped.
+        PLINK1 BIM is SNV-only (single-character A1/A2). Indels would
+        produce non-standard BIM rows that downstream tools may reject.
+    """
+    fam_path = prefix.with_suffix(".fam")
+    bim_path = prefix.with_suffix(".bim")
+    bed_path = prefix.with_suffix(".bed")
+
+    fam_path.write_text("0\tSAMPLE\t0\t0\t0\t-9\n")
+
+    written = 0
+    skipped = 0
+    indels = 0
+    monomorphic = 0
+
+    with bim_path.open("w") as bim_f, bed_path.open("wb") as bed_f:
+        bed_f.write(_BED_MAGIC)
+
+        for v in variants:
+            if v.is_no_call:
+                skipped += 1
+                continue
+
+            if len(v.allele1) != 1 or len(v.allele2) != 1:
+                indels += 1
+                continue
+
+            chrom_code = _CHROM_CODES.get(v.chromosome, v.chromosome)
+            a1: str
+            a2: str
+            bed_code: int
+
+            if ref_alt_map and v.rsid in ref_alt_map:
+                ref, alt = ref_alt_map[v.rsid]
+                resolved = _orient_genotype(v.allele1, v.allele2, ref, alt)
+                if resolved is not None:
+                    r1, r2 = resolved
+                    a1 = ref
+                    a2 = alt
+                    a2_count = sum(1 for a in (r1, r2) if a == alt)
+                    if a2_count == 0:
+                        bed_code = 0b00
+                    elif a2_count == 1:
+                        bed_code = 0b10
+                    else:
+                        bed_code = 0b11
+                else:
+                    a1, a2, bed_code, is_mono = _fallback_coding(v)
+                    if is_mono:
+                        monomorphic += 1
+            else:
+                a1, a2, bed_code, is_mono = _fallback_coding(v)
+                if is_mono:
+                    monomorphic += 1
+
+            bim_f.write(f"{chrom_code}\t{v.rsid}\t0\t{v.position}\t{a1}\t{a2}\n")
+            bed_f.write(bytes([bed_code]))
+            written += 1
+
+    return written, skipped, indels, monomorphic
+
+
+def _fallback_coding(v: Variant) -> tuple[str, str, int, bool]:
+    """Fallback allele coding when ref/alt is unknown.
+
+    Returns ``(a1, a2, bed_code, is_monomorphic)``.
+    """
+    if v.is_heterozygous:
+        alleles = sorted([v.allele1, v.allele2])
+        return alleles[0], alleles[1], 0b10, False
+
+    return v.allele1, "0", 0b00, True

allelix/models.py

@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 dial481
+"""Core data models for genotype variants and reference annotations.
+
+Trust boundary: parsers are responsible for validating raw input. Model
+constructors do not enforce chromosome names, position bounds, or allele
+encodings — they trust their caller. If a Variant or Annotation is
+constructed by code outside the `allelix.parsers` package, the caller owns
+the validation.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+NO_CALL_MARKER = "-"
+DEFAULT_BUILD = "GRCh37"
+
+
+@dataclass
+class Variant:
+    """A single genotype call: which alleles a sample carries at one position.
+
+    All parsers normalize to this representation. Downstream code (annotators,
+    reports) only sees Variants, never raw file formats.
+
+    Attributes:
+        rsid: dbSNP reference identifier (e.g., "rs1801133").
+        chromosome: Chromosome name. "1"-"22", "X", "Y", or "MT".
+        position: 1-based genomic coordinate in the given build.
+        allele1: First observed allele. A/T/G/C, multi-base for indels, or "-" for no-call.
+        allele2: Second observed allele. Same encoding as allele1.
+        build: Reference genome build. "GRCh37" (hg19) or "GRCh38" (hg38).
+    """
+
+    rsid: str
+    chromosome: str
+    position: int
+    allele1: str
+    allele2: str
+    build: str = DEFAULT_BUILD
+
+    @property
+    def is_heterozygous(self) -> bool:
+        """True if the two alleles differ (and neither is a no-call)."""
+        if self.is_no_call:
+            return False
+        return self.allele1 != self.allele2
+
+    @property
+    def is_no_call(self) -> bool:
+        """True if either allele is the no-call marker.
+
+        Typically indicates assay failure at this position, but the precise
+        meaning is format-dependent (some VCFs use `-` for indel deletions).
+        """
+        return self.allele1 == NO_CALL_MARKER or self.allele2 == NO_CALL_MARKER
+
+    @property
+    def genotype(self) -> str:
+        """Human-readable genotype string (e.g., "C/T")."""
+        return f"{self.allele1}/{self.allele2}"
+
+
+@dataclass
+class Annotation:
+    """A claim about a variant sourced from a specific reference database.
+
+    Allelix never asserts variant significance directly — every Annotation is
+    attributed to its source database. See README § Regulatory Posture.
+
+    Attributes:
+        source: Lowercase database identifier (e.g., "clinvar", "pharmgkb").
+        rsid: The variant this annotation applies to.
+        significance: Source-prefixed classification (e.g., "clinvar_pathogenic").
+        category: Coarse filter bucket. Use non-diagnostic labels: "clinical",
+            "pharma", "carrier", "trait", "methylation". Never bare medical terms
+            like "pathogenic" — those would read as Allelix's own classification.
+        magnitude: 0-10 importance score (SNPedia-style).
+        description: Human-readable explanation.
+        attribution: Display name of the source ("ClinVar", "PharmGKB", ...).
+        genotype_match: Which genotype triggers this annotation (e.g., "T/T").
+        references: PubMed IDs or URLs supporting the claim.
+        condition: Disease or condition name, if applicable.
+        gene: Gene symbol, if known.
+        review_status: ClinVar review status (CLNREVSTAT), empty for non-ClinVar.
+        is_must_include: Internal flag for GWAS rollup; excluded from public output.
+    """
+
+    source: str
+    rsid: str
+    significance: str
+    category: str
+    magnitude: float
+    description: str
+    attribution: str
+    genotype_match: str
+    references: list[str] = field(default_factory=list)
+    condition: str = ""
+    gene: str = ""
+    review_status: str = ""
+    alt: str = ""
+    is_must_include: bool = False
+    allele_frequency: float | None = None
+    am_pathogenicity: float | None = None
+    am_class: str = ""
+    cadd_phred: float | None = None
+
+    @property
+    def zygosity(self) -> str:
+        """Classify the genotype call as Heterozygous, Homozygous, or No Call."""
+        if NO_CALL_MARKER in self.genotype_match:
+            return "No Call"
+        parts = self.genotype_match.split("/")
+        if len(parts) != 2:
+            return "Homozygous" if len(set(self.genotype_match)) == 1 else "Heterozygous"
+        return "Heterozygous" if parts[0] != parts[1] else "Homozygous"

allelix/parsers/__init__.py

@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2026 dial481
+"""Parser registry. Auto-detection tries each registered parser; first match wins."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from allelix.parsers.ancestrydna import AncestryDNAParser
+from allelix.parsers.base import GenotypeParser
+from allelix.parsers.ftdna import FTDNAParser
+from allelix.parsers.livingdna import LivingDNAParser
+from allelix.parsers.myhappygenes import MyHappyGenesParser
+from allelix.parsers.myheritage import MyHeritageParser
+from allelix.parsers.twentythreeandme import TwentyThreeAndMeParser
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+PARSERS: list[GenotypeParser] = [
+    MyHappyGenesParser(),
+    TwentyThreeAndMeParser(),
+    AncestryDNAParser(),
+    LivingDNAParser(),
+    MyHeritageParser(),
+    FTDNAParser(),
+]
+
+
+class ParserNotFoundError(ValueError):
+    """Raised when no parser can handle a file or a named parser does not exist."""
+
+
+def get_parser_by_name(name: str) -> GenotypeParser:
+    """Look up a parser by its `name` attribute.
+
+    Args:
+        name: Lowercase parser identifier (e.g., "myhappygenes").
+
+    Raises:
+        ParserNotFoundError: If no registered parser has that name.
+    """
+    for parser in PARSERS:
+        if parser.name == name:
+            return parser
+    available = ", ".join(p.name for p in PARSERS)
+    raise ParserNotFoundError(f"Unknown parser {name!r}. Available: {available}")
+
+
+def detect_parser(file_path: Path) -> GenotypeParser:
+    """Auto-detect the parser for a file. First match wins.
+
+    Args:
+        file_path: Path to the genotype file.
+
+    Raises:
+        ParserNotFoundError: If no parser recognizes the format.
+    """
+    for parser in PARSERS:
+        if parser.can_parse(file_path):
+            return parser
+    raise ParserNotFoundError(
+        f"No parser recognized {file_path.name!r}. Try forcing a format with --format <name>."
+    )
+
+
+__all__ = [
+    "PARSERS",
+    "GenotypeParser",
+    "ParserNotFoundError",
+    "detect_parser",
+    "get_parser_by_name",
+]