statgen 0.3.0__tar.gz → 0.3.2__tar.gz

{statgen-0.3.0 → statgen-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statgen
-Version: 0.3.0
+Version: 0.3.2
 Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{statgen-0.3.0 → statgen-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "statgen"
-version = "0.3.0"
+version = "0.3.2"
 description = "Statistical genetics data objects and loaders for Python and MATLAB/Octave."
 readme = "README.md"
 requires-python = ">=3.10"

{statgen-0.3.0 → statgen-0.3.2}/statgen/__init__.py

@@ -1,5 +1,5 @@
 from ._verbosity import get_verbosity, set_verbosity
 
-__version__ = "0.3.0"
+__version__ = "0.3.2"
 
 __all__ = ["__version__", "get_verbosity", "set_verbosity"]

{statgen-0.3.0 → statgen-0.3.2}/statgen/_bfile_utils.py

@@ -135,6 +135,10 @@ def _parse_bim(path: Path) -> pd.DataFrame:
         raise ValueError(
             f"{path}:{idx + 1}: a2 must be uppercase DNA bases (A/C/G/T): {df['a2'].iat[idx]!r}"
         )
+    same_allele = df["a1"].eq(df["a2"])
+    if same_allele.any():
+        idx = int(same_allele.idxmax())
+        raise ValueError(f"{path}:{idx + 1}: a1 and a2 must differ")
 
     cm_num = pd.to_numeric(df["cm"], errors="coerce")
     bad_cm = cm_num.isna()

{statgen-0.3.0 → statgen-0.3.2}/statgen/_utils.py

@@ -8,6 +8,14 @@ _ALLELE_HASH_P = 2_147_483_647
 _ALLELE_HASH_BASE1 = 257
 _ALLELE_HASH_BASE2 = 263
 _ALLELE_HASH_MAX_CHARS = 150
+_SINGLE_BASE_BYTES = np.frombuffer(b"ACGT", dtype=np.uint8).astype(np.int64) + 1
+_SINGLE_BASE_HASH64 = (
+    (
+        ((_ALLELE_HASH_BASE1 + _SINGLE_BASE_BYTES) % _ALLELE_HASH_P).astype(np.uint64)
+        << np.uint64(32)
+    )
+    + ((_ALLELE_HASH_BASE2 + _SINGLE_BASE_BYTES) % _ALLELE_HASH_P).astype(np.uint64)
+)
 
 
 def validate_requested_shards(shards, available_labels, where: str) -> list[str]:
@@ -55,7 +63,38 @@ def allele_hash64(alleles) -> np.ndarray:
         strings = [s[:_ALLELE_HASH_MAX_CHARS] for s in strings]
         lengths = np.minimum(lengths, _ALLELE_HASH_MAX_CHARS)
 
-    encoded = [s.encode("utf-8") for s in strings]
+    out = np.empty(n, dtype=np.uint64)
+    single_base = lengths == 1
+    if np.any(single_base):
+        single_idx = np.flatnonzero(single_base)
+        single_strings = np.array([strings[i] for i in single_idx], dtype=object)
+        assigned = np.zeros(single_idx.size, dtype=bool)
+        for base, hash_value in zip(("A", "C", "G", "T"), _SINGLE_BASE_HASH64):
+            matches = single_strings == base
+            if np.any(matches):
+                out[single_idx[matches]] = hash_value
+                assigned[matches] = True
+
+        if np.any(~assigned):
+            other_idx = single_idx[~assigned]
+            encoded = [strings[i].encode("utf-8") for i in other_idx]
+            h1, h2 = _allele_hash_lanes_from_utf8(encoded)
+            out[other_idx] = (
+                h1.astype(np.uint64) << np.uint64(32)
+            ) + h2.astype(np.uint64)
+
+    if np.all(single_base):
+        return out
+
+    multi_idx = np.flatnonzero(~single_base)
+    encoded = [strings[i].encode("utf-8") for i in multi_idx]
+    h1, h2 = _allele_hash_lanes_from_utf8(encoded)
+    out[multi_idx] = (h1.astype(np.uint64) << np.uint64(32)) + h2.astype(np.uint64)
+    return out
+
+
+def _allele_hash_lanes_from_utf8(encoded) -> tuple[np.ndarray, np.ndarray]:
+    n = len(encoded)
     byte_lengths = np.fromiter((len(b) for b in encoded), dtype=np.int64, count=n)
     max_len = int(byte_lengths.max(initial=0))
     mat = np.zeros((n, max_len), dtype=np.uint8)
@@ -74,4 +113,4 @@ def allele_hash64(alleles) -> np.ndarray:
         h1[active] = (h1[active] * _ALLELE_HASH_BASE1 + x) % _ALLELE_HASH_P
         h2[active] = (h2[active] * _ALLELE_HASH_BASE2 + x) % _ALLELE_HASH_P
 
-    return (h1.astype(np.uint64) << np.uint64(32)) + h2.astype(np.uint64)
+    return h1, h2

{statgen-0.3.0 → statgen-0.3.2}/statgen/reference.py

@@ -39,17 +39,6 @@ def _is_strand_ambiguous(a1_hash64, a2_hash64) -> np.ndarray:
     )
 
 
-def _validate_distinct_alleles(label: str, a1_arr: np.ndarray, a2_arr: np.ndarray) -> None:
-    same = np.asarray(a1_arr, dtype=object).reshape(-1) == np.asarray(
-        a2_arr, dtype=object
-    ).reshape(-1)
-    if np.any(same):
-        idx0 = int(np.flatnonzero(same)[0])
-        raise ValueError(
-            f"Reference shard {label} variant {idx0 + 1}: a1 and a2 must differ"
-        )
-
-
 def _checksum_from_arrays(
     chr_arr: np.ndarray, bp_arr: np.ndarray, a1_arr: np.ndarray, a2_arr: np.ndarray
 ) -> str:
@@ -92,7 +81,6 @@ class ReferenceShard:
             raise ValueError("ReferenceShard vector lengths must match")
         if self._chr.size and not np.all(self._chr == self._label):
             raise ValueError(f"Reference shard {self._label} contains multiple chr labels")
-        _validate_distinct_alleles(self._label, self._a1, self._a2)
         self._a1_hash64 = allele_hash64(self._a1)
         self._a2_hash64 = allele_hash64(self._a2)
         self._checksum = _checksum_from_arrays(self._chr, self._bp, self._a1, self._a2)
@@ -175,7 +163,6 @@ class ReferenceShard:
             raise ValueError("Invalid reference cache: panel-wide vector lengths mismatch")
         if obj._chr.size and not np.all(obj._chr == obj._label):
             raise ValueError(f"Invalid reference cache: shard {obj._label} contains multiple chr labels")
-        _validate_distinct_alleles(obj._label, obj._a1, obj._a2)
         obj._a1_hash64 = (
             allele_hash64(obj._a1)
             if a1_hash64 is None

{statgen-0.3.0 → statgen-0.3.2}/statgen/sumstats.py

@@ -1,15 +1,19 @@
 import json
+import re
 import warnings
+import gzip
 from pathlib import Path
 
 import numpy as np
 import pandas as pd
-from pandas.errors import ParserError
+from pandas.errors import EmptyDataError, ParserError
 
-from ._utils import allele_hash64, validate_requested_shards
+from ._utils import CANONICAL_CHR_ORDER, allele_hash64, validate_requested_shards
 from ._variant_match import match_shard_numeric, numeric_variant_keys
 
 _CACHE_SCHEMA = "sumstats_cache/0.1"
+_IGNORED_CHR = {"Y", "MT"}
+_DNA_ALLELE_RE = re.compile(r"^[ACGT]+$")
 _REQUIRED_COLS = ("chr", "bp", "a1", "a2", "p")
 _OPTIONAL_COLS = ("z", "n", "beta", "se", "eaf", "info")
 _OPTIONAL_VECTOR_FIELDS = ("z", "n", "beta", "se", "eaf", "info")
@@ -18,64 +22,128 @@ _SUMSTATS_COL_MAP = {
     "effectallele": "a1",
     "otherallele": "a2",
 }
+_NUMERIC_NA_VALUES = ["", "NaN", "nan", "NA", "N/A", "na", "NULL", "null"]
 
 
-def _canonicalize_columns(df: pd.DataFrame, path: Path) -> pd.DataFrame:
-    columns = [_SUMSTATS_COL_MAP.get(str(c).lower(), str(c).lower()) for c in df.columns]
+def _canonicalize_column_names(columns, path: Path) -> list[str]:
+    columns = [_SUMSTATS_COL_MAP.get(str(c).lower(), str(c).lower()) for c in columns]
     duplicates = sorted({c for c in columns if columns.count(c) > 1})
     if duplicates:
         raise ValueError(f"{path}: duplicate columns after column normalization: {', '.join(duplicates)}")
+    return columns
+
+
+def _canonicalize_columns(df: pd.DataFrame, path: Path) -> pd.DataFrame:
     out = df.copy()
-    out.columns = columns
+    out.columns = _canonicalize_column_names(df.columns, path)
     return out
 
 
+def _read_sumstats_header(path: Path) -> list[str]:
+    opener = gzip.open if str(path).lower().endswith(".gz") else open
+    try:
+        with opener(path, "rt", newline="") as f:
+            header = f.readline()
+    except OSError as exc:
+        raise ValueError(f"{path}: malformed TSV") from exc
+    if header == "":
+        raise ValueError(f"{path}: malformed TSV")
+    return header.rstrip("\r\n").split("\t")
+
+
 def _parse_sumstats(path: Path) -> pd.DataFrame:
+    raw_columns = _read_sumstats_header(path)
+    columns = _canonicalize_column_names(raw_columns, path)
+    missing = [c for c in _REQUIRED_COLS if c not in columns]
+    if missing:
+        raise ValueError(f"{path}: missing required columns: {', '.join(missing)}")
+
+    selected = [c in (*_REQUIRED_COLS, *_OPTIONAL_COLS) for c in columns]
+    selected_raw = [c for c, keep in zip(raw_columns, selected) if keep]
+    selected_names = [c for c, keep in zip(columns, selected) if keep]
+    char_cols = {
+        raw: str
+        for raw, canonical in zip(selected_raw, selected_names)
+        if canonical in {"chr", "a1", "a2"}
+    }
+    numeric_cols = {
+        raw: float
+        for raw, canonical in zip(selected_raw, selected_names)
+        if canonical not in {"chr", "a1", "a2"}
+    }
+    numeric_na_values = {raw: _NUMERIC_NA_VALUES for raw in numeric_cols}
+
     try:
         df = pd.read_csv(
             path,
             sep="\t",
             compression="infer",
-            dtype=str,
+            usecols=selected_raw,
+            dtype={**char_cols, **numeric_cols},
             keep_default_na=False,
-            na_filter=False,
+            na_values=numeric_na_values,
         )
-    except ParserError as exc:
+    except (EmptyDataError, ParserError, ValueError) as exc:
         raise ValueError(f"{path}: malformed TSV") from exc
 
-    df = _canonicalize_columns(df, path)
+    df.columns = selected_names
 
     missing = [c for c in _REQUIRED_COLS if c not in df.columns]
     if missing:
         raise ValueError(f"{path}: missing required columns: {', '.join(missing)}")
 
-    out = df.copy()
-    bp_num = pd.to_numeric(out["bp"], errors="coerce")
-    bad_bp = bp_num.isna() | (np.floor(bp_num) != bp_num)
-    if bad_bp.any():
-        idx = int(bad_bp.idxmax())
-        raise ValueError(f"{path}: row {idx + 2}: bp is not an integer: {out['bp'].iat[idx]!r}")
-    out["bp"] = bp_num.astype(np.int64)
-
+    out = df
     for col in ("chr", "a1", "a2"):
         bad_empty = out[col].eq("")
         if bad_empty.any():
             idx = int(bad_empty.idxmax())
             raise ValueError(f"{path}: row {idx + 2}: {col} must be non-empty")
 
-    p_vals = pd.to_numeric(out["p"], errors="coerce")
-    p_arr = p_vals.to_numpy(dtype=float)
-    bad_p = p_vals.isna() | ~np.isfinite(p_arr) | (p_arr < 0.0) | (p_arr > 1.0)
+    chr_col = out["chr"]
+    chr_style = chr_col.str.lower().str.startswith("chr")
+    if chr_style.any():
+        idx = int(chr_style.idxmax())
+        raise ValueError(f"{path}: row {idx + 2}: chr-style labels (e.g., chr1/chrX) are not allowed")
+    known_chr = chr_col.isin(CANONICAL_CHR_ORDER) | chr_col.isin(_IGNORED_CHR)
+    if not known_chr.all():
+        idx = int((~known_chr).idxmax())
+        raise ValueError(
+            f"{path}: row {idx + 2}: unsupported chr label {chr_col.iat[idx]!r}; expected 1-22, X (Y/MT are ignored)"
+        )
+    out = out.loc[chr_col.isin(CANONICAL_CHR_ORDER)].copy()
+
+    bad_a1_syntax = ~out["a1"].str.fullmatch(_DNA_ALLELE_RE.pattern)
+    if bad_a1_syntax.any():
+        idx = int(bad_a1_syntax.idxmax())
+        raise ValueError(
+            f"{path}: row {idx + 2}: a1 must be uppercase DNA bases (A/C/G/T): {out.loc[idx, 'a1']!r}"
+        )
+    bad_a2_syntax = ~out["a2"].str.fullmatch(_DNA_ALLELE_RE.pattern)
+    if bad_a2_syntax.any():
+        idx = int(bad_a2_syntax.idxmax())
+        raise ValueError(
+            f"{path}: row {idx + 2}: a2 must be uppercase DNA bases (A/C/G/T): {out.loc[idx, 'a2']!r}"
+        )
+    same_allele = out["a1"].eq(out["a2"])
+    if same_allele.any():
+        idx = int(same_allele.idxmax())
+        raise ValueError(f"{path}: row {idx + 2}: a1 and a2 must differ")
+
+    bad_bp = out["bp"].isna() | (np.floor(out["bp"]) != out["bp"])
+    if bad_bp.any():
+        idx = int(bad_bp.idxmax())
+        raise ValueError(f"{path}: row {idx + 2}: bp is not an integer: {out.loc[idx, 'bp']!r}")
+    out["bp"] = out["bp"].astype(np.int64)
+
+    p_arr = out["p"].to_numpy(dtype=float)
+    bad_p = out["p"].isna() | ~np.isfinite(p_arr) | (p_arr < 0.0) | (p_arr > 1.0)
     if bad_p.any():
         idx = int(bad_p.idxmax())
-        raise ValueError(f"{path}: row {idx + 2}: p must be finite numeric in [0, 1]: {out['p'].iat[idx]!r}")
-    out["p"] = p_vals.astype(float)
+        raise ValueError(f"{path}: row {idx + 2}: p must be finite numeric in [0, 1]: {out.loc[idx, 'p']!r}")
 
     for opt in _OPTIONAL_COLS:
         if opt in out.columns:
-            vals = pd.to_numeric(out[opt], errors="coerce").astype(float)
-            vals = vals.mask(~np.isfinite(vals.to_numpy(dtype=float)))
-            out[opt] = vals
+            out[opt] = out[opt].mask(~np.isfinite(out[opt].to_numpy(dtype=float)))
 
     return out
 

{statgen-0.3.0 → statgen-0.3.2}/statgen.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statgen
-Version: 0.3.0
+Version: 0.3.2
 Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown