gwas-sumstats-parser 0.5.0__py3-none-any.whl

gwas_sumstats_parser/__init__.py

@@ -0,0 +1,33 @@
+"""Shared vocabulary, parsing and standardisation for GWAS summary-statistics
+files.
+
+This package is the common ingest layer consumed by both ``merit`` and
+``gwas-norm``. It owns the column vocabulary, the value parsers and the
+route table that says which quantity is derived from which columns; the
+consumers own everything downstream of a standardised row.
+"""
+
+from __future__ import annotations
+
+# The "as" form marks this as a deliberate re-export, so callers can read
+# gwas_sumstats_parser.__version__ while the literal lives only in _version.py
+from ._version import __version__ as __version__
+from .assembly import AssemblyResult, check_assembly, load_probe_table
+from .reader import DroppingReader, Reader
+from .errors import ParseError
+from .vocabulary import COLUMNS, Column
+
+# Import examples into the main package
+from .example_data import examples
+
+__all__ = [
+    "AssemblyResult",
+    "COLUMNS",
+    "Column",
+    "DroppingReader",
+    "ParseError",
+    "Reader",
+    "check_assembly",
+    "examples",
+    "load_probe_table",
+]

gwas_sumstats_parser/_version.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

gwas_sumstats_parser/assembly.py

@@ -0,0 +1,463 @@
+"""Check which of two genome builds an indexed file's coordinates follow.
+
+The check is standalone: parsing never invokes it. It looks up a table of
+probe variants, each with a position in build A and in build B, through the
+reader's indexed region query, and reports how many probes it found at each
+build's position together with a decision that may be undecided.
+"""
+
+from __future__ import annotations
+
+import csv
+from dataclasses import dataclass
+from importlib import resources
+from os import PathLike
+from pathlib import Path
+from typing import Any
+
+from .errors import ParseError
+from .opening import detect_index, open_source
+from .parsers import (
+    compile_allele_pattern,
+    parse_allele,
+    parse_chromosome,
+    parse_number,
+    parse_position,
+)
+from .reader import DroppingReader, Region
+
+SPECIES_KEY = "#species"
+BUILDS_KEY = "#builds"
+PROBE_COLUMNS = (
+    "probe_id",
+    "chr_a",
+    "pos_a",
+    "chr_b",
+    "pos_b",
+    "ref",
+    "alt",
+    "coverage_score",
+)
+BUNDLED_PROBES = ("probe_data", "assembly_probes.tsv")
+UNKNOWN = "unknown"
+AMBIGUOUS = "ambiguous"
+# Reader settings the check decides itself (specification 001, A-008): a
+# probe match needs both alleles, and the widest alphabet loses no row.
+FIXED_READER_SETTINGS: dict[str, object] = {
+    "allele_level": "iupac_indel",
+    "require_other_allele": True,
+}
+_PROBE_ALLELE_PATTERN = compile_allele_pattern("iupac_indel")
+
+
+@dataclass(frozen=True)
+class Probe:
+    """One probe variant with its position in each of the two builds.
+
+    Parameters
+    ----------
+    probe_id : str
+        Identifier of the probe.
+    chr_a, chr_b : str
+        Chromosome in build A and in build B, as written in the table.
+    pos_a, pos_b : int
+        One-based position in build A and in build B.
+    ref, alt : str
+        The two alleles; matching ignores which is which.
+    coverage_score : float
+        How commonly the probe appears in files; carried, not used in scoring.
+    """
+
+    probe_id: str
+    chr_a: str
+    pos_a: int
+    chr_b: str
+    pos_b: int
+    ref: str
+    alt: str
+    coverage_score: float
+
+
+@dataclass(frozen=True)
+class ProbeTable:
+    """A species, its two builds in A then B order, and the probes.
+
+    Parameters
+    ----------
+    species : str
+        Species label from the table's species comment.
+    builds : tuple[str, str]
+        Build names from the table's builds comment, A first.
+    probes : tuple[Probe, ...]
+        Probe records in file order.
+    """
+
+    species: str
+    builds: tuple[str, str]
+    probes: tuple[Probe, ...]
+
+
+@dataclass(frozen=True)
+class AssemblyResult:
+    """Evidence and decision of one assembly check.
+
+    Parameters
+    ----------
+    species : str
+        Species label of the probe table used.
+    builds : tuple[str, str]
+        The two build names examined, A first.
+    decision : str
+        One of the two build names, ``"ambiguous"`` when both builds score
+        well and too close together, or ``"unknown"`` when too few probes
+        were found or neither build scores well enough.
+    hits : dict[str, int]
+        Per build, the probes found at that build's position with the
+        probe's two alleles.
+    scores : dict[str, float]
+        Per build, its hits divided by the examined count; zero when nothing
+        was examined.
+    examined : int
+        Probes found at their position in either build, whatever the alleles.
+    failed_rows : int
+        Fetched rows that failed to parse and were skipped as no evidence.
+    """
+
+    species: str
+    builds: tuple[str, str]
+    decision: str
+    hits: dict[str, int]
+    scores: dict[str, float]
+    examined: int
+    failed_rows: int
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def load_probe_table(path: str | PathLike[str] | None = None) -> ProbeTable:
+    """Read and validate a probe table; without a path, the bundled human one.
+
+    Parameters
+    ----------
+    path : str or PathLike[str] or None
+        Tab-separated probe table. ``None`` reads the human GRCh37/GRCh38
+        table shipped inside the package.
+
+    Returns
+    -------
+    ProbeTable
+        Species, builds and probes.
+
+    Raises
+    ------
+    ParseError
+        If the species or builds comment is absent or malformed, a required
+        column is absent, there are no probes, or a cell is invalid. A
+        supplied table never falls back to the human labels.
+
+    Notes
+    -----
+    The first line is ``#species``, a tab, and the species. The second is
+    ``#builds``, a tab, and exactly two different build names separated by a
+    tab, A then B. The third is a header holding at least ``probe_id``,
+    ``chr_a``, ``pos_a``, ``chr_b``, ``pos_b``, ``ref``, ``alt`` and
+    ``coverage_score``, in any order. Alleles use the widest allele alphabet,
+    the one the check reads file rows with.
+    """
+    if path is None:
+        text = (
+            resources.files(__package__).joinpath(*BUNDLED_PROBES).read_text()
+        )
+    else:
+        text = Path(path).read_text()
+    lines = text.splitlines()
+    species = _comment_values(lines, 0, SPECIES_KEY)
+    if len(species) != 1:
+        raise ParseError(
+            "probe table must begin with a #species comment naming one species",
+            error_func=load_probe_table,
+            error_value=lines[0] if lines else "",
+        )
+    builds = _comment_values(lines, 1, BUILDS_KEY)
+    if len(builds) != 2 or builds[0] == builds[1]:
+        raise ParseError(
+            "probe table needs a #builds comment naming exactly two builds",
+            error_func=load_probe_table,
+            error_value=lines[1] if len(lines) > 1 else "",
+        )
+    records = csv.DictReader(lines[2:], delimiter="\t")
+    header = records.fieldnames or []
+    for name in PROBE_COLUMNS:
+        if name not in header:
+            raise ParseError(
+                "probe table lacks a required column",
+                error_func=load_probe_table,
+                error_value=name,
+            )
+    probes = tuple(_probe(record) for record in records)
+    if not probes:
+        raise ParseError(
+            "probe table holds no probes",
+            error_func=load_probe_table,
+            error_value=path,
+        )
+    return ProbeTable(species[0], (builds[0], builds[1]), probes)
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def _comment_values(lines: list[str], index: int, key: str) -> list[str]:
+    """Return the non-empty values of the comment expected on one line.
+
+    An absent line, or a line that does not start with the key, gives an
+    empty list, which no caller accepts.
+    """
+    if index >= len(lines):
+        return []
+    cells = lines[index].split("\t")
+    if cells[0] != key:
+        return []
+    return [cell.strip() for cell in cells[1:] if cell.strip()]
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def _probe(record: dict[str, str | None]) -> Probe:
+    """Convert one table record, reporting the first invalid cell."""
+    for name in PROBE_COLUMNS:
+        if record[name] is None:
+            raise ParseError(
+                "probe record has too few cells",
+                error_func=load_probe_table,
+                error_value=name,
+            )
+    cells: dict[str, Any] = record
+    return Probe(
+        probe_id=cells["probe_id"].strip(),
+        chr_a=cells["chr_a"].strip(),
+        pos_a=parse_position(cells["pos_a"]),
+        chr_b=cells["chr_b"].strip(),
+        pos_b=parse_position(cells["pos_b"]),
+        ref=parse_allele(cells["ref"], _PROBE_ALLELE_PATTERN),
+        alt=parse_allele(cells["alt"], _PROBE_ALLELE_PATTERN),
+        coverage_score=float(parse_number(cells["coverage_score"])),
+    )
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def score_assembly(
+    species: str,
+    builds: tuple[str, str],
+    hits: tuple[int, int],
+    examined: int,
+    *,
+    failed_rows: int = 0,
+    match_threshold: float = 0.8,
+    confidence_delta_threshold: float = 0.1,
+    min_examined: int = 5,
+) -> AssemblyResult:
+    """Turn hit counts into per-build scores and a decision.
+
+    Parameters
+    ----------
+    species : str
+        Species label to report.
+    builds : tuple[str, str]
+        Build names, A first.
+    hits : tuple[int, int]
+        Probes matched at the build A position and at the build B position.
+    examined : int
+        Probes found at their position in either build.
+    failed_rows : int
+        Fetched rows that failed to parse, reported unchanged.
+    match_threshold : float
+        Minimum score of the better build.
+    confidence_delta_threshold : float
+        Minimum difference between the two scores.
+    min_examined : int
+        Minimum examined probes.
+
+    Returns
+    -------
+    AssemblyResult
+        Evidence and decision.
+
+    Notes
+    -----
+    The decision is ``"unknown"`` when ``examined`` is below its minimum or
+    the better score is below its minimum; otherwise ``"ambiguous"`` when the
+    score difference is below its minimum; otherwise the better build. A
+    value equal to its minimum passes. The difference is the difference of
+    the hit counts divided by ``examined``, which is exact where subtracting
+    the two rounded scores is not. When the scores are level and the minimum
+    difference is zero, build B is returned.
+    """
+    hits_a, hits_b = hits
+    if examined > 0:
+        score_a = hits_a / examined
+        score_b = hits_b / examined
+        difference = abs(hits_a - hits_b) / examined
+    else:
+        score_a = score_b = difference = 0.0
+    if examined < min_examined or max(score_a, score_b) < match_threshold:
+        decision = UNKNOWN
+    elif difference < confidence_delta_threshold:
+        decision = AMBIGUOUS
+    elif hits_a > hits_b:
+        decision = builds[0]
+    else:
+        decision = builds[1]
+    return AssemblyResult(
+        species=species,
+        builds=builds,
+        decision=decision,
+        hits={builds[0]: hits_a, builds[1]: hits_b},
+        scores={builds[0]: score_a, builds[1]: score_b},
+        examined=examined,
+        failed_rows=failed_rows,
+    )
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def check_assembly(
+    path: str | PathLike[str],
+    mapping: dict[str, str],
+    *,
+    effect_type: str,
+    probes: str | PathLike[str] | None = None,
+    match_threshold: float = 0.8,
+    confidence_delta_threshold: float = 0.1,
+    min_examined: int = 5,
+    **reader_settings: Any,
+) -> AssemblyResult:
+    """Check an indexed file against the two builds of a probe table.
+
+    Parameters
+    ----------
+    path : str or PathLike[str]
+        Block-compressed file with a neighbouring tabix index.
+    mapping : dict[str, str]
+        Vocabulary names mapped to source headers, as for the reader. The
+        reader's required quantities apply, and the other allele is always
+        required.
+    effect_type : str
+        Interpretation of the effect column, as for the reader.
+    probes : str or PathLike[str] or None
+        Probe table for any species; ``None`` uses the bundled human
+        GRCh37/GRCh38 table. See :func:`load_probe_table` for the format.
+    match_threshold : float
+        Minimum score of the better build.
+    confidence_delta_threshold : float
+        Minimum difference between the two scores.
+    min_examined : int
+        Minimum probes found at their position in either build.
+    **reader_settings : object
+        Further reader settings describing the file, such as
+        ``logged_pvalue``, ``chrpos_spec`` or ``chromosome_synonyms``, handed
+        unchanged to the reader. ``allele_level`` and
+        ``require_other_allele`` are fixed by the check and refused.
+
+    Returns
+    -------
+    AssemblyResult
+        Species, builds, decision, per-build hits and scores, the examined
+        count and the number of rows skipped because they failed to parse.
+
+    Raises
+    ------
+    ParseError
+        If a fixed reader setting is supplied, the probe table is invalid,
+        the file has no index, or the reader refuses the configuration.
+
+    Notes
+    -----
+    Rows are read with the widest allele alphabet and unusable rows are
+    dropped, so one malformed row never prevents a decision; it counts as no
+    evidence and is reported in ``failed_rows``. Probe and file chromosomes
+    are normalised alike, so ``chr1`` and ``1`` agree. A probe whose
+    chromosome the supplied synonyms reject is not looked up. See
+    :func:`score_assembly` for the decision rule.
+    """
+    for setting in FIXED_READER_SETTINGS:
+        if setting in reader_settings:
+            raise ParseError(
+                "this reader setting is fixed by the assembly check",
+                error_func=check_assembly,
+                error_value=setting,
+            )
+    if detect_index(path) is None:
+        raise ParseError(
+            "an index is required for the assembly check",
+            error_func=check_assembly,
+            error_value=path,
+        )
+    table = load_probe_table(probes)
+    with open_source(path) as source:
+        reader = DroppingReader(
+            source,
+            mapping,
+            effect_type=effect_type,
+            **FIXED_READER_SETTINGS,
+            **reader_settings,
+        )
+        hits, examined = _count(reader, table.probes)
+        failed_rows = len(reader.failures)
+    return score_assembly(
+        table.species,
+        table.builds,
+        hits,
+        examined,
+        failed_rows=failed_rows,
+        match_threshold=match_threshold,
+        confidence_delta_threshold=confidence_delta_threshold,
+        min_examined=min_examined,
+    )
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def _count(
+    reader: DroppingReader, probes: tuple[Probe, ...]
+) -> tuple[tuple[int, int], int]:
+    """Fetch every probe position once and count hits and examined probes.
+
+    Returns the build A and build B hit counts, then the examined count.
+    """
+    # The reader's own normalised synonym table, so a probe chromosome is
+    # spelled exactly as the reader spells a row's chromosome.
+    synonyms = reader.chromosome_synonyms
+    places: list[tuple[Probe, tuple[str, int] | None, tuple[str, int] | None]]
+    places = []
+    regions: list[Region] = []
+    for probe in probes:
+        keys: list[tuple[str, int] | None] = []
+        for chromosome, position in (
+            (probe.chr_a, probe.pos_a),
+            (probe.chr_b, probe.pos_b),
+        ):
+            try:
+                name = parse_chromosome(chromosome, synonyms)
+            except ParseError:
+                keys.append(None)
+                continue
+            keys.append((name, position))
+            regions.append((chromosome, position, position))
+        places.append((probe, keys[0], keys[1]))
+    found: set[tuple[str, int]] = set()
+    matched: set[tuple[str, int, frozenset[str]]] = set()
+    if regions:  # A query without regions would scan the whole file.
+        chromosome_column = reader.column_indices["chr_name"]
+        position_column = reader.column_indices["start_pos"]
+        effect_column = reader.column_indices["effect_allele"]
+        other_column = reader.column_indices["other_allele"]
+        for row in reader.fetch(*regions):
+            place = (row[chromosome_column], row[position_column])
+            found.add(place)  # type: ignore[arg-type]
+            alleles = frozenset((row[effect_column], row[other_column]))
+            matched.add((*place, alleles))  # type: ignore[arg-type]
+    hits_a = hits_b = examined = 0
+    for probe, key_a, key_b in places:
+        alleles = frozenset((probe.ref, probe.alt))
+        if key_a in found or key_b in found:
+            examined += 1
+        if key_a is not None and (*key_a, alleles) in matched:
+            hits_a += 1
+        if key_b is not None and (*key_b, alleles) in matched:
+            hits_b += 1
+    return (hits_a, hits_b), examined

gwas_sumstats_parser/burgess.py

@@ -0,0 +1,139 @@
+"""Reference-calibrated scalar effects and standalone calibration estimation."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from math import sqrt
+from typing import Any
+
+import numpy as np
+
+from .errors import ParseError
+from .resolver import Route
+from .stats import _number, set_effect_size_pvalue
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def average_variation(
+    standard_errors: Sequence[float], frequencies: Sequence[float]
+) -> float:
+    """Estimate calibration from supplied reference-study numeric sequences.
+
+    Parameters
+    ----------
+    standard_errors, frequencies : Sequence[float]
+        Paired reference errors and allele frequencies from the same study
+        and effect scale as the target variants.
+
+    Returns
+    -------
+    float
+        Mean finite SE*sqrt(f*(1-f)) term, or NaN when none survive.
+        This calibration incorporates study precision; it is not the
+        phenotype standard deviation. Comparable reference and target
+        precision, including sufficiently similar sample sizes, is assumed.
+    """
+    errors = np.asarray(standard_errors, dtype=float)
+    frequency = np.asarray(frequencies, dtype=float)
+    with np.errstate(invalid="ignore", over="ignore"):
+        terms = errors * np.sqrt(frequency * (1 - frequency))
+    finite = terms[np.isfinite(terms)]
+    return float(np.mean(finite)) if finite.size else float("nan")
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def set_standard_error_burgess(frequency: Any, average_variation: Any) -> float:
+    """Estimate error from reference calibration on the target effect scale.
+
+    Parameters
+    ----------
+    frequency, average_variation : object
+        Frequency strictly between zero and one, and positive finite
+        calibration from the same study and effect scale. Calibration
+        incorporates study precision, assuming comparable reference and
+        target precision including sufficiently similar sample sizes.
+
+    Returns
+    -------
+    float
+        Calibration divided by sqrt(f*(1-f)); no sample count is used.
+    """
+    method = set_standard_error_burgess
+    f = _number(frequency, method)
+    calibration = _number(average_variation, method)
+    if not 0 < f < 1:
+        raise ParseError(
+            "Burgess frequency must be strictly between zero and one",
+            error_func=method,
+            error_value=frequency,
+        )
+    if calibration <= 0:
+        raise ParseError(
+            "average_variation must be positive",
+            error_func=method,
+            error_value=average_variation,
+        )
+    return _number(calibration / sqrt(f * (1 - f)), method)
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def set_effect_size_burgess(
+    logarithm: Any, frequency: Any, average_variation: Any, direction: Any
+) -> float:
+    """Estimate a signed linear or log-odds effect using reference precision.
+
+    Parameters
+    ----------
+    logarithm, frequency, average_variation, direction : object
+        Negative base-ten probability log, allele frequency, same-study
+        calibration on the target effect scale, and explicit sign (+1/-1).
+        Calibration already incorporates study precision; comparable
+        reference and target precision is assumed.
+
+    Returns
+    -------
+    float
+        Signed normal score times the calibrated error. The reference scale
+        determines linear units or log odds; no sample count is used.
+    """
+    try:
+        error = set_standard_error_burgess(frequency, average_variation)
+        return set_effect_size_pvalue(error, logarithm, direction)
+    except ParseError as failure:
+        failure.error_func = set_effect_size_burgess
+        raise
+
+
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def burgess_routes(effect_type: str) -> tuple[Route, ...]:
+    """Supply fallback routes for explicitly declared effect directions.
+
+    Parameters
+    ----------
+    effect_type : str
+        Source interpretation, restricted to linear or log-odds direction.
+
+    Returns
+    -------
+    tuple[Route, ...]
+        Reference-calibrated routes to append after ordinary alternatives.
+    """
+    if effect_type not in {"direction_beta", "direction_log_or"}:
+        return ()
+    return (
+        Route(
+            "standard_error",
+            ("effect_allele_freq", "average_variation"),
+            set_standard_error_burgess,
+        ),
+        Route(
+            "effect_size",
+            (
+                "mlog10_pvalue",
+                "effect_allele_freq",
+                "average_variation",
+                "effect_direction",
+            ),
+            set_effect_size_burgess,
+        ),
+    )