statgen 0.2.0__tar.gz → 0.2.2__tar.gz

{statgen-0.2.0 → statgen-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statgen
-Version: 0.2.0
+Version: 0.2.2
 Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{statgen-0.2.0 → statgen-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "statgen"
-version = "0.2.0"
+version = "0.2.2"
 description = "Statistical genetics data objects and loaders for Python and MATLAB/Octave."
 readme = "README.md"
 requires-python = ">=3.10"

statgen-0.2.2/statgen/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.2"

{statgen-0.2.0 → statgen-0.2.2}/statgen/_ld_npz.py

@@ -19,30 +19,51 @@ from ._ld_schema import (
 
 def validate_npz_distribution(path, check_payload_structure=False) -> dict:
     path = Path(path)
+    if not path.is_dir():
+        raise ValueError("validate_npz_distribution: path must identify a panel root directory")
+
+    manifest = read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
+    runtime = manifest["runtime_format"]
+    seen_reference_bim = set()
+    for entry in manifest["shards"]:
+        shard_path = path / entry["file"]
+        require_file(shard_path)
+        file_md5 = md5_file(shard_path)
+        if file_md5 != entry.get("file_md5"):
+            raise ValueError(f"{shard_path}: file_md5 does not match manifest")
+        _ld_r, _a1freq, meta = read_npz_shard(
+            shard_path,
+            check_payload_structure=check_payload_structure,
+        )
+        validate_manifest_entry_agreement(entry, meta, shard_path)
+        reference_key = (
+            entry["reference_bim"],
+            entry["chr"],
+            int(entry["num_snp"]),
+            entry["reference_checksum"],
+        )
+        if reference_key not in seen_reference_bim:
+            _validate_bundled_reference(root=path, entry=entry)
+            seen_reference_bim.add(reference_key)
+
+    return {"ok": True}
 
-    if path.is_dir():
-        manifest = read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
-        runtime = manifest["runtime_format"]
-        for entry in manifest["shards"]:
-            shard_path = path / entry["file"]
-            require_file(shard_path)
-            file_md5 = md5_file(shard_path)
-            if file_md5 != entry.get("file_md5"):
-                raise ValueError(f"{shard_path}: file_md5 does not match manifest")
-            _ld_r, _a1freq, meta = read_npz_shard(
-                shard_path,
-                check_payload_structure=check_payload_structure,
-            )
-            validate_manifest_entry_agreement(entry, meta, shard_path)
-    else:
-        runtime = runtime_from_suffix(path)
-        if runtime != PY_RUNTIME_FORMAT:
-            raise ValueError(f"{path}: unsupported LD runtime {runtime!r}")
-        read_npz_shard(path, check_payload_structure=check_payload_structure)
 
+def validate_npz_shard_file(path, check_payload_structure=False) -> dict:
+    path = Path(path)
+    runtime = runtime_from_suffix(path)
+    if runtime != PY_RUNTIME_FORMAT:
+        raise ValueError(f"{path}: unsupported LD runtime {runtime!r}")
+    read_npz_shard(path, check_payload_structure=check_payload_structure)
     return {"ok": True}
 
 
+def _validate_bundled_reference(*, root: Path, entry: dict) -> None:
+    from ._ld_reference import validate_bundled_reference_bim
+
+    validate_bundled_reference_bim(root / entry["reference_bim"], entry, target="manifest")
+
+
 def read_npz_shard(path: Path, check_payload_structure: bool):
     require_file(path)
     with np.load(path, allow_pickle=False) as data:

statgen-0.2.2/statgen/_ld_reference.py

@@ -0,0 +1,57 @@
+from pathlib import Path
+
+from ._ld_schema import require_file
+from ._utils import validate_requested_shards
+from .reference import ReferencePanel, load_reference
+
+
+def validate_bundled_reference_bim(path, entry: dict, *, target: str):
+    path = Path(path)
+    require_file(path)
+    panel = load_reference(path)
+    ref_shards = panel.shards
+    if len(ref_shards) != 1:
+        raise ValueError(f"{path}: bundled reference_bim must contain exactly one shard")
+    ref = ref_shards[0]
+    validate_reference_shard_for_ld_entry(ref, entry, path, target=target)
+    return ref
+
+
+def validate_reference_shard_for_ld_entry(ref_shard, entry: dict, path, *, target: str) -> None:
+    path = Path(path)
+    if ref_shard.label != entry["chr"]:
+        raise ValueError(f"{path}: bundled reference_bim chr does not match {target}")
+    if ref_shard.num_snp != int(entry["num_snp"]):
+        raise ValueError(f"{path}: bundled reference_bim num_snp does not match {target}")
+    if ref_shard.checksum != entry["reference_checksum"]:
+        raise ValueError(f"{path}: bundled reference_bim reference_checksum does not match {target}")
+
+
+def load_bundled_reference_panel(root, manifest: dict, shards, *, where: str) -> ReferencePanel:
+    root = Path(root)
+    available = _manifest_chr_labels(manifest)
+    selected = validate_requested_shards(shards, available, where)
+    entries_by_chr = {}
+    for entry in manifest["shards"]:
+        entries_by_chr.setdefault(entry["chr"], []).append(entry)
+
+    ref_shards = []
+    for label in selected:
+        entries = entries_by_chr[label]
+        reference_path = root / entries[0]["reference_bim"]
+        ref_shard = validate_bundled_reference_bim(reference_path, entries[0], target="manifest")
+        for entry in entries[1:]:
+            validate_reference_shard_for_ld_entry(ref_shard, entry, reference_path, target="manifest")
+        ref_shards.append(ref_shard)
+    return ReferencePanel(ref_shards)
+
+
+def _manifest_chr_labels(manifest: dict) -> list[str]:
+    labels = []
+    seen = set()
+    for entry in manifest["shards"]:
+        label = entry["chr"]
+        if label not in seen:
+            labels.append(label)
+            seen.add(label)
+    return labels

{statgen-0.2.0 → statgen-0.2.2}/statgen/_ld_schema.py

@@ -33,17 +33,30 @@ def read_manifest(path: Path, expected_runtime: str | None) -> dict:
         raise ValueError(f"{path}: shards must be a non-empty list")
 
     seen = set()
+    reference_bim_by_chr = {}
     for i, entry in enumerate(shards):
         validate_manifest_entry(entry, f"{path}:shards[{i}]")
         key = (entry["chr"], entry["sex"])
         if key in seen:
             raise ValueError(f"{path}: duplicate manifest entry for chr {key[0]} sex {key[1]!r}")
         seen.add(key)
+        prior_reference_bim = reference_bim_by_chr.setdefault(entry["chr"], entry["reference_bim"])
+        if entry["reference_bim"] != prior_reference_bim:
+            raise ValueError(f"{path}: manifest entries for chr {entry['chr']} must share reference_bim")
     return manifest
 
 
 def validate_manifest_entry(entry: dict, where: str) -> None:
-    required = {"chr", "sex", "file", "file_md5", "num_snp", "nnz", "reference_checksum"}
+    required = {
+        "chr",
+        "sex",
+        "file",
+        "file_md5",
+        "num_snp",
+        "nnz",
+        "reference_checksum",
+        "reference_bim",
+    }
     missing = sorted(required.difference(entry))
     if missing:
         raise ValueError(f"{where}: missing required fields: {', '.join(missing)}")
@@ -58,6 +71,7 @@ def validate_manifest_entry(entry: dict, where: str) -> None:
         raise ValueError(f"{where}: reference_checksum must be a non-empty string")
     if not isinstance(entry["file_md5"], str) or len(entry["file_md5"]) != 32:
         raise ValueError(f"{where}: file_md5 must be a lowercase MD5 hex string")
+    validate_reference_bim_filename(entry["reference_bim"], f"{where}: reference_bim")
 
 
 def validate_shard_metadata(meta: dict, path: Path, expected_format: str) -> None:
@@ -73,6 +87,8 @@ def validate_shard_metadata(meta: dict, path: Path, expected_format: str) -> Non
         "diagonal",
         "value",
         "reference_checksum",
+        "reference_bim",
+        "num_monomorphic_snps",
     }
     missing = sorted(required.difference(meta))
     if missing:
@@ -86,6 +102,12 @@ def validate_shard_metadata(meta: dict, path: Path, expected_format: str) -> Non
     validate_chr_sex(meta["chr"], meta["sex"], f"{path}: metadata")
     validate_positive_int(meta["num_snp"], f"{path}: metadata num_snp")
     validate_nonnegative_int(meta["nnz"], f"{path}: metadata nnz")
+    validate_nonnegative_int(
+        meta["num_monomorphic_snps"],
+        f"{path}: metadata num_monomorphic_snps",
+    )
+    if int(meta["num_monomorphic_snps"]) > int(meta["num_snp"]):
+        raise ValueError(f"{path}: metadata num_monomorphic_snps must not exceed num_snp")
     if int(meta["nnz"]) >= 2**31:
         raise ValueError(f"{path}: CSC32 metadata nnz must be < 2^31")
     if meta["matrix"] != "symmetric":
@@ -96,6 +118,7 @@ def validate_shard_metadata(meta: dict, path: Path, expected_format: str) -> Non
         raise ValueError(f"{path}: metadata value must be 'r'")
     if not isinstance(meta["reference_checksum"], str) or meta["reference_checksum"] == "":
         raise ValueError(f"{path}: metadata reference_checksum must be a non-empty string")
+    validate_reference_bim_filename(meta["reference_bim"], f"{path}: metadata reference_bim")
     if expected_format == NPZ_FORMAT:
         if meta.get("sparse_layout") != "csc":
             raise ValueError(f"{path}: metadata sparse_layout must be 'csc'")
@@ -119,7 +142,7 @@ def validate_chrx_sex(sex, where: str) -> str:
 
 
 def validate_manifest_entry_agreement(entry: dict, meta: dict, path: Path) -> None:
-    for key in ("chr", "sex", "num_snp", "nnz", "reference_checksum"):
+    for key in ("chr", "sex", "num_snp", "nnz", "reference_checksum", "reference_bim"):
         if entry.get(key) != meta.get(key):
             raise ValueError(f"{path}: manifest/per-file metadata mismatch for {key}")
     suffix_runtime = runtime_from_suffix(path)
@@ -127,6 +150,19 @@ def validate_manifest_entry_agreement(entry: dict, meta: dict, path: Path) -> No
         raise ValueError(f"{path}: manifest file extension and metadata format disagree")
 
 
+def default_reference_bim_filename(chr_label: str) -> str:
+    return f"reference_chr{chr_label}.bim"
+
+
+def validate_reference_bim_filename(value, where: str) -> str:
+    if not isinstance(value, str) or value == "":
+        raise ValueError(f"{where} must be a non-empty filename")
+    path = Path(value)
+    if path.is_absolute() or len(path.parts) != 1 or value in {".", ".."} or ".." in value:
+        raise ValueError(f"{where} must be a plain relative filename")
+    return value
+
+
 def runtime_from_suffix(path: Path) -> str:
     if path.suffix == ".npz":
         return PY_RUNTIME_FORMAT

{statgen-0.2.0 → statgen-0.2.2}/statgen/_ld_writer.py

@@ -14,11 +14,14 @@ from ._ld_schema import (
     NPZ_REQUIRED,
     PY_RUNTIME_FORMAT,
     SHARD_SCHEMA,
+    default_reference_bim_filename,
     md5_file,
     validate_chr_sex,
     validate_positive_int,
+    validate_reference_bim_filename,
     validate_shard_metadata,
 )
+from ._ld_reference import validate_bundled_reference_bim
 from ._utils import CHR_RANK
 
 
@@ -70,8 +73,14 @@ def write_ld_npz_shards(root, shard_specs) -> list[dict]:
             raise ValueError(f"duplicate LD shard output file: {file_key}")
         seen_files.add(file_key)
 
-        shard_payload = {k: v for k, v in normalized.items() if k != "file"}
+        shard_payload = {
+            k: v
+            for k, v in normalized.items()
+            if k not in {"file", "reference_shard"}
+        }
         meta = _write_ld_npz_shard(path, **shard_payload)
+        reference_bim = meta["reference_bim"]
+        _ensure_reference_bim(root / reference_bim, normalized["reference_shard"], meta)
         records.append(
             {
                 "chr": meta["chr"],
@@ -81,6 +90,7 @@ def write_ld_npz_shards(root, shard_specs) -> list[dict]:
                 "num_snp": meta["num_snp"],
                 "nnz": meta["nnz"],
                 "reference_checksum": meta["reference_checksum"],
+                "reference_bim": reference_bim,
             }
         )
     return records
@@ -131,6 +141,7 @@ def _ld_npz_manifest_record(path: Path, root: Path) -> dict:
         "num_snp": meta["num_snp"],
         "nnz": meta["nnz"],
         "reference_checksum": meta["reference_checksum"],
+        "reference_bim": _require_reference_bim(root, meta),
     }
 
 
@@ -155,6 +166,7 @@ def _normalize_ld_shard_write_spec(spec: dict) -> dict:
         "build_metadata",
         "extra_metadata",
         "file",
+        "reference_bim",
     }
     unknown = sorted(set(spec).difference(allowed))
     if unknown:
@@ -178,12 +190,26 @@ def _normalize_ld_shard_write_spec(spec: dict) -> dict:
     reference_checksum = spec.get("reference_checksum", getattr(ref, "checksum", None))
     if not isinstance(reference_checksum, str) or reference_checksum == "":
         raise ValueError("LD shard write spec reference_checksum must be a non-empty string")
+    reference_bim = validate_reference_bim_filename(
+        spec.get("reference_bim", default_reference_bim_filename(chr_label)),
+        "LD shard write spec reference_bim",
+    )
+    if ref is None:
+        raise ValueError("LD shard write spec must provide reference_shard so reference_bim can be bundled")
+    if getattr(ref, "label", None) != chr_label:
+        raise ValueError("LD shard write spec reference_shard label must match chr")
+    if getattr(ref, "num_snp", None) != num_snp:
+        raise ValueError("LD shard write spec reference_shard num_snp must match num_snp")
+    if getattr(ref, "checksum", None) != reference_checksum:
+        raise ValueError("LD shard write spec reference_shard checksum must match reference_checksum")
 
     return {
         "chr_label": chr_label,
         "sex": sex,
         "num_snp": num_snp,
         "reference_checksum": reference_checksum,
+        "reference_shard": ref,
+        "reference_bim": reference_bim,
         "a1freq": _coerce_a1freq(spec.get("a1freq"), num_snp),
         "ld_pairs": spec.get("ld_pairs"),
         "build_metadata": dict(spec.get("build_metadata", {})),
@@ -199,6 +225,7 @@ def _write_ld_npz_shard(
     sex,
     num_snp,
     reference_checksum,
+    reference_bim,
     a1freq,
     ld_pairs,
     build_metadata=None,
@@ -217,7 +244,9 @@ def _write_ld_npz_shard(
         sex=sex,
         num_snp=num_snp,
         nnz=nnz,
+        num_monomorphic_snps=int(np.count_nonzero((a1freq == 0.0) | (a1freq == 1.0))),
         reference_checksum=reference_checksum,
+        reference_bim=reference_bim,
         build_metadata=build_metadata,
         extra_metadata=extra_metadata,
     )
@@ -337,7 +366,9 @@ def _ld_npz_metadata(
     sex,
     num_snp,
     nnz,
+    num_monomorphic_snps,
     reference_checksum,
+    reference_bim,
     build_metadata=None,
     extra_metadata=None,
 ) -> dict:
@@ -354,7 +385,9 @@ def _ld_npz_metadata(
         "matrix": "symmetric",
         "diagonal": "explicit_unit",
         "value": "r",
+        "num_monomorphic_snps": int(num_monomorphic_snps),
         "reference_checksum": reference_checksum,
+        "reference_bim": reference_bim,
     }
     build = dict(LD_BUILD_METADATA_DEFAULTS)
     if build_metadata:
@@ -362,6 +395,17 @@ def _ld_npz_metadata(
     meta.update(build)
     if extra_metadata:
         meta.update(extra_metadata)
+    meta.update(
+        {
+            "chr": chr_label,
+            "sex": sex,
+            "num_snp": int(num_snp),
+            "nnz": int(nnz),
+            "num_monomorphic_snps": int(num_monomorphic_snps),
+            "reference_checksum": reference_checksum,
+            "reference_bim": reference_bim,
+        }
+    )
     validate_shard_metadata(meta, Path("<generated>"), expected_format=NPZ_FORMAT)
     return meta
 
@@ -372,6 +416,37 @@ def _default_ld_shard_filename(chr_label: str, sex, suffix: str) -> str:
     return f"ld_chr{chr_label}{suffix}"
 
 
+def _ensure_reference_bim(path: Path, reference_shard, meta: dict) -> None:
+    if path.exists():
+        _validate_existing_reference_bim(path, meta)
+        return
+    _write_reference_bim_from_shard(path, reference_shard)
+    _validate_existing_reference_bim(path, meta)
+
+
+def _write_reference_bim_from_shard(path: Path, reference_shard) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8", newline="") as f:
+        for chr_label, snp, bp, a1, a2 in zip(
+            reference_shard.chr,
+            reference_shard.snp,
+            reference_shard.bp,
+            reference_shard.a1,
+            reference_shard.a2,
+        ):
+            f.write(f"{chr_label}\t{snp}\t0\t{int(bp)}\t{a1}\t{a2}\n")
+
+
+def _validate_existing_reference_bim(path: Path, meta: dict) -> None:
+    validate_bundled_reference_bim(path, meta, target="LD shard")
+
+
+def _require_reference_bim(root: Path, meta: dict) -> str:
+    reference_bim = meta["reference_bim"]
+    _validate_existing_reference_bim(root / reference_bim, meta)
+    return reference_bim
+
+
 def _write_deterministic_npz(path: Path, arrays: dict[str, np.ndarray]) -> None:
     with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
         for name, array in arrays.items():

statgen-0.2.2/statgen/_utils.py

@@ -0,0 +1,77 @@
+import warnings
+
+import numpy as np
+
+CANONICAL_CHR_ORDER = [str(i) for i in range(1, 23)] + ["X"]
+CHR_RANK = {c: i for i, c in enumerate(CANONICAL_CHR_ORDER)}
+_ALLELE_HASH_P = 2_147_483_647
+_ALLELE_HASH_BASE1 = 257
+_ALLELE_HASH_BASE2 = 263
+_ALLELE_HASH_MAX_CHARS = 150
+
+
+def validate_requested_shards(shards, available_labels, where: str) -> list[str]:
+    labels = list(available_labels)
+    if shards is None:
+        return labels
+    if isinstance(shards, str):
+        raise ValueError(f"{where}: shards must be a non-empty list of unique canonical contig labels")
+
+    requested = list(shards)
+    if not requested:
+        raise ValueError(f"{where}: shards must be a non-empty list of unique canonical contig labels")
+
+    seen = set()
+    prev_rank = None
+    for label in requested:
+        if label not in CHR_RANK:
+            raise ValueError(f"{where}: unsupported shard label {label!r}; expected canonical labels 1-22 or X")
+        if label in seen:
+            raise ValueError(f"{where}: duplicate shard label {label!r} in shards")
+        rank = CHR_RANK[label]
+        if prev_rank is not None and rank <= prev_rank:
+            raise ValueError(f"{where}: shards must be in canonical subsequence order")
+        seen.add(label)
+        prev_rank = rank
+        if label not in labels:
+            raise ValueError(f"{where}: requested shard {label!r} is not present")
+    return requested
+
+
+def allele_hash64(alleles) -> np.ndarray:
+    arr = np.asarray(alleles, dtype=object).reshape(-1)
+    n = arr.size
+    if n == 0:
+        return np.array([], dtype=np.uint64)
+
+    strings = [str(allele) for allele in arr]
+    lengths = np.fromiter((len(s) for s in strings), dtype=np.int64, count=n)
+    if np.any(lengths > _ALLELE_HASH_MAX_CHARS):
+        warnings.warn(
+            "Allele length exceeds 150 characters; hashing uses first 150 characters",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+        strings = [s[:_ALLELE_HASH_MAX_CHARS] for s in strings]
+        lengths = np.minimum(lengths, _ALLELE_HASH_MAX_CHARS)
+
+    encoded = [s.encode("utf-8") for s in strings]
+    byte_lengths = np.fromiter((len(b) for b in encoded), dtype=np.int64, count=n)
+    max_len = int(byte_lengths.max(initial=0))
+    mat = np.zeros((n, max_len), dtype=np.uint8)
+    if max_len:
+        flat = np.frombuffer(b"".join(encoded), dtype=np.uint8)
+        row_idx = np.repeat(np.arange(n, dtype=np.int64), byte_lengths)
+        starts = np.cumsum(byte_lengths, dtype=np.int64) - byte_lengths
+        col_idx = np.arange(flat.size, dtype=np.int64) - np.repeat(starts, byte_lengths)
+        mat[row_idx, col_idx] = flat
+
+    h1 = np.ones(n, dtype=np.int64)
+    h2 = np.ones(n, dtype=np.int64)
+    for j in range(max_len):
+        active = j < byte_lengths
+        x = mat[active, j].astype(np.int64) + 1
+        h1[active] = (h1[active] * _ALLELE_HASH_BASE1 + x) % _ALLELE_HASH_P
+        h2[active] = (h2[active] * _ALLELE_HASH_BASE2 + x) % _ALLELE_HASH_P
+
+    return (h1.astype(np.uint64) << np.uint64(32)) + h2.astype(np.uint64)

{statgen-0.2.0 → statgen-0.2.2}/statgen/annotations.py

@@ -1,5 +1,4 @@
 import json
-import io
 from pathlib import Path
 
 import numpy as np
@@ -12,27 +11,6 @@ from ._utils import validate_requested_shards
 _CACHE_SCHEMA = "annotations_cache/0.1"
 
 
-def _is_ignored_bed_line(line: str) -> bool:
-    return (
-        line == ""
-        or line.startswith("#")
-        or line.startswith("track ")
-        or line.startswith("browser ")
-    )
-
-
-def _read_bed_payload(path: Path) -> str:
-    kept = []
-    with open(path, "r", encoding="utf-8") as f:
-        for raw in f:
-            line = raw.rstrip("\r\n")
-            if _is_ignored_bed_line(line):
-                continue
-            kept.append(line)
-    if not kept:
-        raise ValueError(f"{path}: BED file is empty")
-    return "\n".join(kept) + "\n"
-
 
 def _as_sparse_binary(mat) -> sparse.csr_matrix:
     if sparse.issparse(mat):
@@ -94,14 +72,16 @@ def _annotation_name_from_path(path: Path) -> str:
 
 
 def _parse_bed(path: Path) -> dict[str, np.ndarray]:
-    payload = _read_bed_payload(path)
+    # Lines beginning with '#' and blank lines are skipped natively.
+    # 'track'/'browser' metadata lines are not supported; prefix with '#' if present.
     try:
         df = pd.read_csv(
-            io.StringIO(payload),
-            sep=r"\s+",
+            path,
+            sep="\t",
             header=None,
             dtype=str,
-            engine="python",
+            comment="#",
+            skip_blank_lines=True,
             keep_default_na=False,
             na_filter=False,
         )
@@ -113,7 +93,7 @@ def _parse_bed(path: Path) -> dict[str, np.ndarray]:
     if df.shape[0] == 0:
         raise ValueError(f"{path}: BED file is empty")
     if df.shape[1] < 3:
-        raise ValueError(f"{path}: BED must have at least 3 whitespace-separated columns")
+        raise ValueError(f"{path}: BED must have at least 3 tab-separated columns")
 
     chr_col = df.iloc[:, 0]
     start_raw = df.iloc[:, 1]

{statgen-0.2.0 → statgen-0.2.2}/statgen/ld.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+import warnings
 
 import numpy as np
 from scipy import sparse
@@ -15,7 +16,16 @@ from ._utils import validate_requested_shards
 
 
 class LDShard:
-    def __init__(self, chr_label, sex, num_snp, ld_r, a1freq, reference_checksum):
+    def __init__(
+        self,
+        chr_label,
+        sex,
+        num_snp,
+        ld_r,
+        a1freq,
+        reference_checksum,
+        num_monomorphic_snps=0,
+    ):
         self._chr = str(chr_label)
         self._sex = sex
         self._num_snp = int(num_snp)
@@ -23,6 +33,7 @@ class LDShard:
         self._ld_r2 = _build_ld_r2(self._ld_r)
         self._a1freq = np.asarray(a1freq, dtype=np.float32).reshape(-1)
         self._reference_checksum = str(reference_checksum)
+        self._num_monomorphic_snps = int(num_monomorphic_snps)
 
     @property
     def chr(self) -> str:
@@ -56,11 +67,16 @@ class LDShard:
     def checksum(self) -> str:
         return self._reference_checksum
 
+    @property
+    def num_monomorphic_snps(self) -> int:
+        return self._num_monomorphic_snps
+
 
 class LDPanel:
-    def __init__(self, shard_groups, default_chrX_sex="female"):
+    def __init__(self, shard_groups, default_chrX_sex="female", reference=None):
         self._shard_groups = [list(g) for g in shard_groups]
         self._default_chrX_sex = validate_chrx_sex(default_chrX_sex, "default_chrX_sex")
+        self._reference = reference
         self._shard_offsets = []
         pos = 0
         for group in self._shard_groups:
@@ -77,6 +93,8 @@ class LDPanel:
             )
             pos += first.num_snp
         self._num_snp = pos
+        if self._reference is not None:
+            self._validate_reference_shape()
         self._validate_default_chrX_sex()
 
     @property
@@ -108,6 +126,10 @@ class LDPanel:
     def default_chrX_sex(self) -> str:
         return self._default_chrX_sex
 
+    @property
+    def reference(self):
+        return self._reference
+
     def a1freq(self, chrX_sex=None) -> np.ndarray:
         shards = [shard for shard, _start, _stop in self.iter_shards_with_offsets(chrX_sex)]
         if not shards:
@@ -118,7 +140,8 @@ class LDPanel:
         available = [group[0].label for group in self._shard_groups]
         selected = validate_requested_shards(shards, available, "LDPanel.select_shards")
         by_label = {group[0].label: group for group in self._shard_groups}
-        return LDPanel([by_label[label] for label in selected], self._default_chrX_sex)
+        reference = None if self._reference is None else self._reference.select_shards(selected)
+        return LDPanel([by_label[label] for label in selected], self._default_chrX_sex, reference)
 
     def multiply_r2(self, M, chrX_sex=None):
         if sparse.issparse(M):
@@ -156,6 +179,17 @@ class LDPanel:
                         f"got {self._default_chrX_sex!r}, present {sorted(present)!r}"
                     )
 
+    def _validate_reference_shape(self) -> None:
+        ref_shards = list(self._reference.shards)
+        if len(ref_shards) != len(self._shard_groups):
+            raise ValueError("LDPanel reference shard count must match LD shard groups")
+        for ref_shard, group in zip(ref_shards, self._shard_groups):
+            first = group[0]
+            if ref_shard.label != first.label:
+                raise ValueError("LDPanel reference shard labels must match LD shard groups")
+            if ref_shard.num_snp != first.num_snp:
+                raise ValueError("LDPanel reference shard sizes must match LD shard groups")
+
     def _multiply_r2_dense(self, M, chrX_sex=None):
         arr = np.asarray(M)
         if arr.ndim == 1:
@@ -213,26 +247,32 @@ class LDPanel:
         return sparse.vstack(parts, format=M.getformat()).astype(out_dtype)
 
 
-def load_ld(path, reference, default_chrX_sex=None) -> LDPanel:
+def load_ld(path, reference=None, shards=None, default_chrX_sex=None) -> LDPanel:
     default_chrX_sex = validate_chrx_sex(
         "female" if default_chrX_sex is None else default_chrX_sex,
         "default_chrX_sex",
     )
     path = Path(path)
+    if not path.is_dir():
+        raise ValueError("load_ld: path must identify a panel root directory")
+
+    manifest = read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
+    if reference is None:
+        reference = _load_bundled_reference(path, manifest, shards)
+    elif shards is not None:
+        reference = reference.select_shards(shards)
+
     ref_shards = list(reference.shards)
     if not ref_shards:
         raise ValueError("load_ld: reference must contain at least one shard")
 
-    if path.is_dir():
-        manifest = read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
-        groups = _load_panel_root(path, manifest, ref_shards)
-    else:
-        groups = _load_single_shard(path, ref_shards)
-
-    return LDPanel(groups, default_chrX_sex=default_chrX_sex)
+    groups = _load_panel_root(path, manifest, ref_shards)
+    return LDPanel(groups, default_chrX_sex=default_chrX_sex, reference=reference)
 
 
 def validate_ld_distribution(path, check_payload_structure=False) -> dict:
+    if not Path(path).is_dir():
+        raise ValueError("validate_ld_distribution: path must identify a panel root directory")
     return validate_npz_distribution(path, check_payload_structure=check_payload_structure)
 
 
@@ -311,6 +351,12 @@ def _write_ld_npz_distribution(*args, **kwargs):
     return write_ld_npz_distribution(*args, **kwargs)
 
 
+def _load_bundled_reference(root: Path, manifest: dict, shards):
+    from ._ld_reference import load_bundled_reference_panel
+
+    return load_bundled_reference_panel(root, manifest, shards, where="load_ld")
+
+
 def _load_panel_root(root: Path, manifest: dict, ref_shards: list) -> list[list[LDShard]]:
     entries = list(manifest["shards"])
     groups = []
@@ -346,22 +392,6 @@ def _load_panel_root(root: Path, manifest: dict, ref_shards: list) -> list[list[
     return groups
 
 
-def _load_single_shard(path: Path, ref_shards: list) -> list[list[LDShard]]:
-    shard, _meta = _load_npz_shard(path)
-    if len(ref_shards) != 1:
-        raise ValueError(
-            "single-shard LD loads require a single-shard reference; "
-            f"reference has {len(ref_shards)} shards"
-        )
-    if ref_shards[0].label != shard.chr:
-        raise ValueError(
-            "single-shard LD chromosome does not match single-shard reference: "
-            f"LD chr {shard.chr!r}, reference chr {ref_shards[0].label!r}"
-        )
-    _validate_reference_compatibility(shard, ref_shards[0], path)
-    return [[shard]]
-
-
 def _load_npz_shard(path: Path) -> tuple[LDShard, dict]:
     require_file(path)
     ld_r, a1freq, meta = read_npz_shard(path, check_payload_structure=False)
@@ -372,7 +402,15 @@ def _load_npz_shard(path: Path) -> tuple[LDShard, dict]:
         ld_r,
         a1freq,
         meta["reference_checksum"],
+        int(meta["num_monomorphic_snps"]),
     )
+    if shard.num_monomorphic_snps:
+        warnings.warn(
+            f"{path}: LD shard contains {shard.num_monomorphic_snps} monomorphic SNPs; "
+            "LD involving those SNPs is undefined and represented by omitted off-diagonal entries",
+            RuntimeWarning,
+            stacklevel=2,
+        )
     return shard, meta
 
 

{statgen-0.2.0 → statgen-0.2.2}/statgen/reference.py

@@ -8,7 +8,7 @@ import numpy as np
 import pandas as pd
 from pandas.errors import ParserError
 
-from ._utils import CANONICAL_CHR_ORDER, CHR_RANK, validate_requested_shards
+from ._utils import CANONICAL_CHR_ORDER, CHR_RANK, allele_hash64, validate_requested_shards
 
 logger = logging.getLogger(__name__)
 
@@ -16,8 +16,6 @@ _CACHE_SCHEMA = "reference_cache/0.1"
 _BIM_NCOLS = 6
 _IGNORED_CHR = {"Y", "MT"}
 _DNA_ALLELE_RE = re.compile(r"^[ACGT]+$")
-
-
 def _validate_reference_sort_order(
     chr_col: pd.Series,
     bp_col: np.ndarray,
@@ -34,23 +32,21 @@ def _validate_reference_sort_order(
             f"{path}:{idx + 1}: chr must use canonical labels 1-22 or X"
         )
 
-    order_df = pd.DataFrame(
-        {
-            "chr_rank": chr_rank.to_numpy(dtype=np.int64),
-            "bp": np.asarray(bp_col, dtype=np.int64),
-            "a1": a1_col.to_numpy(dtype=object),
-            "a2": a2_col.to_numpy(dtype=object),
-            "line": np.asarray(line_numbers, dtype=np.int64),
-        }
-    )
+    order_df = pd.DataFrame({
+        "chr_rank": chr_rank.to_numpy(dtype=np.int64),
+        "bp": np.asarray(bp_col, dtype=np.int64),
+        "a1_hash64": allele_hash64(a1_col.to_numpy(dtype=object)),
+        "a2_hash64": allele_hash64(a2_col.to_numpy(dtype=object)),
+        "line": np.asarray(line_numbers, dtype=np.int64),
+    })
 
     dup_mask = order_df.duplicated(
-        subset=["chr_rank", "bp", "a1", "a2"], keep="first"
+        subset=["chr_rank", "bp", "a1_hash64", "a2_hash64"], keep="first"
     )
     if dup_mask.any():
         line = int(order_df.loc[dup_mask.idxmax(), "line"])
         raise ValueError(
-            f"{path}:{line}: duplicate (chr, bp, a1, a2) tuple is not allowed"
+            f"{path}:{line}: duplicate (chr, bp, a1_hash64, a2_hash64) matching key is not allowed"
         )
 
     prev = order_df.shift(1)
@@ -58,26 +54,14 @@ def _validate_reference_sort_order(
         (order_df["chr_rank"] < prev["chr_rank"])
         | (
             (order_df["chr_rank"] == prev["chr_rank"])
-            & (
-                (order_df["bp"] < prev["bp"])
-                | (
-                    (order_df["bp"] == prev["bp"])
-                    & (
-                        (order_df["a1"] < prev["a1"])
-                        | (
-                            (order_df["a1"] == prev["a1"])
-                            & (order_df["a2"] < prev["a2"])
-                        )
-                    )
-                )
-            )
+            & (order_df["bp"] < prev["bp"])
         )
     )
     bad_order = bad_order.fillna(False)
     if bad_order.any():
         line = int(order_df.loc[bad_order.idxmax(), "line"])
         raise ValueError(
-            f"{path}:{line}: rows must be sorted by (chr_rank, bp, a1, a2) in canonical contig order"
+            f"{path}:{line}: rows must be sorted by (chr_rank, bp) in canonical contig order"
         )
 
 
@@ -221,6 +205,18 @@ class ReferenceShard:
         self._bp = np.asarray(bp_arr, dtype=np.int64)
         self._a1 = np.asarray(a1_arr, dtype=object)
         self._a2 = np.asarray(a2_arr, dtype=object)
+        if not (
+            self._chr.size
+            == self._snp.size
+            == self._bp.size
+            == self._a1.size
+            == self._a2.size
+        ):
+            raise ValueError("ReferenceShard vector lengths must match")
+        if self._chr.size and not np.all(self._chr == self._label):
+            raise ValueError(f"Reference shard {self._label} contains multiple chr labels")
+        self._a1_hash64 = allele_hash64(self._a1)
+        self._a2_hash64 = allele_hash64(self._a2)
         self._checksum = _checksum_from_arrays(self._chr, self._bp, self._a1, self._a2)
 
     @property
@@ -251,12 +247,31 @@ class ReferenceShard:
     def a2(self) -> np.ndarray:
         return self._a2
 
+    @property
+    def a1_hash64(self) -> np.ndarray:
+        return self._a1_hash64
+
+    @property
+    def a2_hash64(self) -> np.ndarray:
+        return self._a2_hash64
+
     @property
     def checksum(self) -> str:
         return self._checksum
 
     @classmethod
-    def _from_arrays(cls, label, chr_arr, snp_arr, bp_arr, a1_arr, a2_arr, checksum):
+    def _from_arrays(
+        cls,
+        label,
+        chr_arr,
+        snp_arr,
+        bp_arr,
+        a1_arr,
+        a2_arr,
+        checksum,
+        a1_hash64=None,
+        a2_hash64=None,
+    ):
         obj = cls.__new__(cls)
         obj._label = label
         obj._chr = np.asarray(chr_arr, dtype=object)
@@ -264,6 +279,28 @@ class ReferenceShard:
         obj._bp = np.asarray(bp_arr, dtype=np.int64)
         obj._a1 = np.asarray(a1_arr, dtype=object)
         obj._a2 = np.asarray(a2_arr, dtype=object)
+        if not (
+            obj._chr.size
+            == obj._snp.size
+            == obj._bp.size
+            == obj._a1.size
+            == obj._a2.size
+        ):
+            raise ValueError("Invalid reference cache: panel-wide vector lengths mismatch")
+        if obj._chr.size and not np.all(obj._chr == obj._label):
+            raise ValueError(f"Invalid reference cache: shard {obj._label} contains multiple chr labels")
+        obj._a1_hash64 = (
+            allele_hash64(obj._a1)
+            if a1_hash64 is None
+            else np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
+        )
+        obj._a2_hash64 = (
+            allele_hash64(obj._a2)
+            if a2_hash64 is None
+            else np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
+        )
+        if obj._a1_hash64.size != obj._bp.size or obj._a2_hash64.size != obj._bp.size:
+            raise ValueError("Invalid reference cache: allele hash vector lengths mismatch")
         obj._checksum = checksum
         return obj
 
@@ -313,6 +350,18 @@ class ReferencePanel:
             return np.array([], dtype=object)
         return np.concatenate([s.a2 for s in self._shards])
 
+    @property
+    def a1_hash64(self) -> np.ndarray:
+        if not self._shards:
+            return np.array([], dtype=np.uint64)
+        return np.concatenate([s.a1_hash64 for s in self._shards])
+
+    @property
+    def a2_hash64(self) -> np.ndarray:
+        if not self._shards:
+            return np.array([], dtype=np.uint64)
+        return np.concatenate([s.a2_hash64 for s in self._shards])
+
     @property
     def shard_offsets(self) -> list:
         return list(self._shard_offsets)
@@ -327,6 +376,13 @@ class ReferencePanel:
         by_label = {s.label: s for s in self._shards}
         return ReferencePanel([by_label[label] for label in selected])
 
+    def validate_checksums(self) -> bool:
+        for shard in self._shards:
+            computed = _checksum_from_arrays(shard.chr, shard.bp, shard.a1, shard.a2)
+            if computed != shard.checksum:
+                raise ValueError(f"Reference checksum mismatch for shard {shard.label}")
+        return True
+
     def is_object_compatible(self, obj) -> bool:
         log_fn = logger.warning
 
@@ -441,12 +497,18 @@ def load_reference(path, shards=None) -> ReferencePanel:
     return ReferencePanel(out_shards)
 
 
-def save_reference_cache(panel: ReferencePanel, path) -> None:
+def save_reference_cache(panel: ReferencePanel, path, mode: str = "full") -> None:
+    mode = str(mode).lower()
+    if mode not in {"full", "thin"}:
+        raise ValueError("save_reference_cache mode must be 'full' or 'thin'")
+
     # Metadata (schema, labels, checksums) as a compact JSON blob stored in the npz.
     # SNP-axis numeric vectors (bp) and string arrays are stored as native
     # binary numpy arrays — not JSON — per the performance contract.
+    # Python keeps reference caches full even when mode='thin' is requested.
     meta = {
         "schema": _CACHE_SCHEMA,
+        "mode": "full",
         "shard_labels": [s.label for s in panel.shards],
         "shard_checksums": [s.checksum for s in panel.shards],
     }
@@ -460,6 +522,8 @@ def save_reference_cache(panel: ReferencePanel, path) -> None:
         arrays[p + "bp"]  = s.bp
         arrays[p + "a1"]  = np.asarray(s.a1, dtype=str)
         arrays[p + "a2"]  = np.asarray(s.a2, dtype=str)
+        arrays[p + "a1_hash64"] = s.a1_hash64
+        arrays[p + "a2_hash64"] = s.a2_hash64
     np.savez_compressed(path, **arrays)
 
 
@@ -469,6 +533,11 @@ def load_reference_cache(path, shards=None) -> ReferencePanel:
         schema = meta.get("schema")
         if schema != _CACHE_SCHEMA:
             raise ValueError(f"Unsupported reference cache schema: {schema!r}")
+        mode = meta.get("mode")
+        if mode is None:
+            raise ValueError("reference cache missing mode; delete and rebuild old cache")
+        if mode != "full":
+            raise ValueError(f"Unsupported Python reference cache mode: {mode!r}")
 
         labels = list(meta.get("shard_labels", []))
         checksums = list(meta.get("shard_checksums", []))
@@ -481,6 +550,8 @@ def load_reference_cache(path, shards=None) -> ReferencePanel:
         for label in selected:
             i = label_to_index[label]
             p = f"s{i}_"
+            if p + "a1_hash64" not in data or p + "a2_hash64" not in data:
+                raise ValueError("reference cache missing a1_hash64/a2_hash64; rebuild cache")
             shard_objs.append(
                 ReferenceShard._from_arrays(
                     label,
@@ -490,6 +561,8 @@ def load_reference_cache(path, shards=None) -> ReferencePanel:
                     data[p + "a1"],
                     data[p + "a2"],
                     checksums[i],
+                    data[p + "a1_hash64"],
+                    data[p + "a2_hash64"],
                 )
             )
     return ReferencePanel(shard_objs)

{statgen-0.2.0 → statgen-0.2.2}/statgen/sumstats.py

@@ -5,20 +5,77 @@ import numpy as np
 import pandas as pd
 from pandas.errors import ParserError
 
-from ._utils import validate_requested_shards
+from ._utils import allele_hash64, validate_requested_shards
 
 _CACHE_SCHEMA = "sumstats_cache/0.1"
 _REQUIRED_COLS = ("chr", "bp", "a1", "a2", "z", "n")
 _OPTIONAL_COLS = ("p", "beta", "se", "eaf", "info")
+_SUMSTATS_COL_MAP = {
+    "pos": "bp",
+    "effectallele": "a1",
+    "otherallele": "a2",
+}
+
+
+_KEY_DTYPE = np.dtype([("bp", "<i8"), ("a1_hash64", "<u8"), ("a2_hash64", "<u8")])
+
+
+def _numeric_keys(bp_arr, a1_hash64, a2_hash64) -> np.ndarray:
+    bp = np.asarray(bp_arr, dtype=np.int64).reshape(-1)
+    a1h = np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
+    a2h = np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
+    if not (bp.size == a1h.size == a2h.size):
+        raise ValueError("numeric key vector lengths mismatch")
+    keys = np.empty(bp.size, dtype=_KEY_DTYPE)
+    keys["bp"] = bp
+    keys["a1_hash64"] = a1h
+    keys["a2_hash64"] = a2h
+    return keys
+
+
+def _check_unique_sorted_keys(sorted_keys: np.ndarray, where: str) -> None:
+    if sorted_keys.size < 2:
+        return
+    dup = sorted_keys[1:] == sorted_keys[:-1]
+    if np.any(dup):
+        pos = int(np.flatnonzero(dup)[0] + 1)
+        key = sorted_keys[pos]
+        raise ValueError(
+            f"Ambiguous duplicate sumstats/reference matching key in {where}: "
+            f"bp={int(key['bp'])}, a1_hash64={int(key['a1_hash64'])}, "
+            f"a2_hash64={int(key['a2_hash64'])}"
+        )
 
 
-def _key_from_arrays(chr_arr, bp_arr, a1_arr, a2_arr) -> pd.Series:
-    return (
-        pd.Series(chr_arr, dtype="string")
-        .str.cat(pd.Series(bp_arr, dtype=np.int64).astype("string"), sep=":")
-        .str.cat(pd.Series(a1_arr, dtype="string"), sep=":")
-        .str.cat(pd.Series(a2_arr, dtype="string"), sep=":")
-    )
+def _match_shard_numeric(ref_keys: np.ndarray, src_keys: np.ndarray, shard_label: str) -> np.ndarray:
+    if src_keys.size == 0:
+        return np.full(ref_keys.size, -1, dtype=np.int64)
+
+    ref_order = np.argsort(ref_keys, kind="mergesort", order=("bp", "a1_hash64", "a2_hash64"))
+    src_order = np.argsort(src_keys, kind="mergesort", order=("bp", "a1_hash64", "a2_hash64"))
+    ref_sorted = ref_keys[ref_order]
+    src_sorted = src_keys[src_order]
+    _check_unique_sorted_keys(ref_sorted, f"reference shard {shard_label}")
+    _check_unique_sorted_keys(src_sorted, f"sumstats shard {shard_label}")
+
+    pos = np.searchsorted(src_sorted, ref_keys)
+    in_range = pos < src_sorted.size
+    matched = np.zeros(ref_keys.size, dtype=bool)
+    matched[in_range] = src_sorted[pos[in_range]] == ref_keys[in_range]
+
+    out = np.full(ref_keys.size, -1, dtype=np.int64)
+    out[matched] = src_order[pos[matched]]
+    return out
+
+
+def _canonicalize_columns(df: pd.DataFrame, path: Path) -> pd.DataFrame:
+    columns = [_SUMSTATS_COL_MAP.get(str(c).lower(), str(c).lower()) for c in df.columns]
+    duplicates = sorted({c for c in columns if columns.count(c) > 1})
+    if duplicates:
+        raise ValueError(f"{path}: duplicate columns after column normalization: {', '.join(duplicates)}")
+    out = df.copy()
+    out.columns = columns
+    return out
 
 
 def _parse_sumstats(path: Path) -> pd.DataFrame:
@@ -34,6 +91,8 @@ def _parse_sumstats(path: Path) -> pd.DataFrame:
     except ParserError as exc:
         raise ValueError(f"{path}: malformed TSV") from exc
 
+    df = _canonicalize_columns(df, path)
+
     missing = [c for c in _REQUIRED_COLS if c not in df.columns]
     if missing:
         raise ValueError(f"{path}: missing required columns: {', '.join(missing)}")
@@ -289,40 +348,45 @@ def _build_sumstats_from_aligned(
 
 
 def _build_sumstats_panel(df: pd.DataFrame, reference) -> Sumstats:
-    ref_chr = np.asarray(reference.chr, dtype=object)
-    ref_bp = np.asarray(reference.bp, dtype=np.int64)
-    ref_a1 = np.asarray(reference.a1, dtype=object)
-    ref_a2 = np.asarray(reference.a2, dtype=object)
-    ref_keys = _key_from_arrays(ref_chr, ref_bp, ref_a1, ref_a2)
-
-    src_keys = _key_from_arrays(
-        df["chr"].to_numpy(dtype=object),
-        df["bp"].to_numpy(dtype=np.int64),
-        df["a1"].to_numpy(dtype=object),
-        df["a2"].to_numpy(dtype=object),
-    )
-    src = df.copy()
-    src["_key"] = src_keys
-
-    key_to = {}
-    for col in ("z", "n") + _OPTIONAL_COLS:
-        if col in src.columns:
-            key_to[col] = src.set_index("_key")[col]
-
-    aligned_z = key_to["z"].reindex(ref_keys).to_numpy(dtype=float)
-    aligned_n = key_to["n"].reindex(ref_keys).to_numpy(dtype=float)
-    if "p" in key_to:
-        aligned_p = key_to["p"].reindex(ref_keys).to_numpy(dtype=float)
-        aligned_logp = _derive_logp(aligned_p)
+    n = int(reference.num_snp)
+    aligned_z = np.full(n, np.nan, dtype=float)
+    aligned_n = np.full(n, np.nan, dtype=float)
+    aligned_p = np.full(n, np.nan, dtype=float) if "p" in df.columns else None
+
+    aligned_optional = {
+        col: (np.full(n, np.nan, dtype=float) if col in df.columns else None)
+        for col in ("beta", "se", "eaf", "info")
+    }
+
+    src_chr = df["chr"].to_numpy(dtype=object)
+    src_bp = df["bp"].to_numpy(dtype=np.int64)
+    src_a1_hash64 = allele_hash64(df["a1"].to_numpy(dtype=object))
+    src_a2_hash64 = allele_hash64(df["a2"].to_numpy(dtype=object))
+
+    for ref_shard, off in zip(reference.shards, reference.shard_offsets):
+        start = int(off["start0"])
+        stop = int(off["stop0"])
+        src_mask = src_chr == ref_shard.label
+        src_idx = np.flatnonzero(src_mask)
+        ref_keys = _numeric_keys(ref_shard.bp, ref_shard.a1_hash64, ref_shard.a2_hash64)
+        src_keys = _numeric_keys(src_bp[src_idx], src_a1_hash64[src_idx], src_a2_hash64[src_idx])
+        local_match = _match_shard_numeric(ref_keys, src_keys, ref_shard.label)
+        has_match = local_match >= 0
+        matched_src = src_idx[local_match[has_match]]
+        aligned_ix = np.arange(start, stop, dtype=np.int64)[has_match]
+
+        aligned_z[aligned_ix] = df["z"].to_numpy(dtype=float)[matched_src]
+        aligned_n[aligned_ix] = df["n"].to_numpy(dtype=float)[matched_src]
+        if aligned_p is not None:
+            aligned_p[aligned_ix] = df["p"].to_numpy(dtype=float)[matched_src]
+        for col, out in aligned_optional.items():
+            if out is not None:
+                out[aligned_ix] = df[col].to_numpy(dtype=float)[matched_src]
+
+    if aligned_p is None:
+        aligned_logp = np.full(n, np.nan, dtype=float)
     else:
-        aligned_logp = np.full(ref_keys.size, np.nan, dtype=float)
-
-    aligned_optional = {}
-    for col in ("beta", "se", "eaf", "info"):
-        if col in key_to:
-            aligned_optional[col] = key_to[col].reindex(ref_keys).to_numpy(dtype=float)
-        else:
-            aligned_optional[col] = None
+        aligned_logp = _derive_logp(aligned_p)
 
     return _build_sumstats_from_aligned(
         reference,

{statgen-0.2.0 → statgen-0.2.2}/statgen.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statgen
-Version: 0.2.0
+Version: 0.2.2
 Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{statgen-0.2.0 → statgen-0.2.2}/statgen.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ README.md
 pyproject.toml
 statgen/__init__.py
 statgen/_ld_npz.py
+statgen/_ld_reference.py
 statgen/_ld_schema.py
 statgen/_ld_writer.py
 statgen/_utils.py

statgen-0.2.0/statgen/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.0"

statgen-0.2.0/statgen/_utils.py

@@ -1,30 +0,0 @@
-CANONICAL_CHR_ORDER = [str(i) for i in range(1, 23)] + ["X"]
-CHR_RANK = {c: i for i, c in enumerate(CANONICAL_CHR_ORDER)}
-
-
-def validate_requested_shards(shards, available_labels, where: str) -> list[str]:
-    labels = list(available_labels)
-    if shards is None:
-        return labels
-    if isinstance(shards, str):
-        raise ValueError(f"{where}: shards must be a non-empty list of unique canonical contig labels")
-
-    requested = list(shards)
-    if not requested:
-        raise ValueError(f"{where}: shards must be a non-empty list of unique canonical contig labels")
-
-    seen = set()
-    prev_rank = None
-    for label in requested:
-        if label not in CHR_RANK:
-            raise ValueError(f"{where}: unsupported shard label {label!r}; expected canonical labels 1-22 or X")
-        if label in seen:
-            raise ValueError(f"{where}: duplicate shard label {label!r} in shards")
-        rank = CHR_RANK[label]
-        if prev_rank is not None and rank <= prev_rank:
-            raise ValueError(f"{where}: shards must be in canonical subsequence order")
-        seen.add(label)
-        prev_rank = rank
-        if label not in labels:
-            raise ValueError(f"{where}: requested shard {label!r} is not present")
-    return requested