statgen 0.2.4__tar.gz → 0.2.6__tar.gz

{statgen-0.2.4 → statgen-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statgen
-Version: 0.2.4
+Version: 0.2.6
 Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{statgen-0.2.4 → statgen-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "statgen"
-version = "0.2.4"
+version = "0.2.6"
 description = "Statistical genetics data objects and loaders for Python and MATLAB/Octave."
 readme = "README.md"
 requires-python = ">=3.10"

statgen-0.2.6/statgen/__init__.py

@@ -0,0 +1,5 @@
+from ._verbosity import get_verbosity, set_verbosity
+
+__version__ = "0.2.6"
+
+__all__ = ["__version__", "get_verbosity", "set_verbosity"]

{statgen-0.2.4 → statgen-0.2.6}/statgen/_ld_npz.py

@@ -23,7 +23,14 @@ def validate_npz_distribution(path, check_payload_structure=False) -> dict:
         raise ValueError("validate_npz_distribution: path must identify a panel root directory")
 
     manifest = read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
-    runtime = manifest["runtime_format"]
+    reference_cache_path = path / manifest["reference_cache"]
+    require_file(reference_cache_path)
+    if md5_file(reference_cache_path) != manifest["reference_cache_md5"]:
+        raise ValueError(f"{reference_cache_path}: reference_cache_md5 does not match manifest")
+    from .reference import load_reference_cache
+
+    reference = load_reference_cache(reference_cache_path)
+    ref_by_label = {s.label: s for s in reference.shards}
     seen_reference_bim = set()
     for entry in manifest["shards"]:
         shard_path = path / entry["file"]
@@ -36,6 +43,10 @@ def validate_npz_distribution(path, check_payload_structure=False) -> dict:
             check_payload_structure=check_payload_structure,
         )
         validate_manifest_entry_agreement(entry, meta, shard_path)
+        ref_shard = ref_by_label.get(entry["chr"])
+        if ref_shard is None:
+            raise ValueError(f"{reference_cache_path}: reference cache missing shard {entry['chr']}")
+        _validate_reference_cache_entry(ref_shard, entry, reference_cache_path)
         reference_key = (
             entry["reference_bim"],
             entry["chr"],
@@ -64,6 +75,13 @@ def _validate_bundled_reference(*, root: Path, entry: dict) -> None:
     validate_bundled_reference_bim(root / entry["reference_bim"], entry, target="manifest")
 
 
+def _validate_reference_cache_entry(ref_shard, entry: dict, path: Path) -> None:
+    if ref_shard.num_snp != int(entry["num_snp"]):
+        raise ValueError(f"{path}: reference cache num_snp does not match manifest")
+    if ref_shard.checksum != entry["reference_checksum"]:
+        raise ValueError(f"{path}: reference cache reference_checksum does not match manifest")
+
+
 def read_npz_shard(path: Path, check_payload_structure: bool):
     require_file(path)
     with np.load(path, allow_pickle=False) as data:

statgen-0.2.6/statgen/_ld_reference.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+from ._ld_schema import require_file
+from .reference import load_reference
+
+
+def validate_bundled_reference_bim(path, entry: dict, *, target: str):
+    path = Path(path)
+    require_file(path)
+    panel = load_reference(path)
+    ref_shards = panel.shards
+    if len(ref_shards) != 1:
+        raise ValueError(f"{path}: bundled reference_bim must contain exactly one shard")
+    ref = ref_shards[0]
+    validate_reference_shard_for_ld_entry(ref, entry, path, target=target)
+    return ref
+
+
+def validate_reference_shard_for_ld_entry(ref_shard, entry: dict, path, *, target: str) -> None:
+    path = Path(path)
+    if ref_shard.label != entry["chr"]:
+        raise ValueError(f"{path}: bundled reference_bim chr does not match {target}")
+    if ref_shard.num_snp != int(entry["num_snp"]):
+        raise ValueError(f"{path}: bundled reference_bim num_snp does not match {target}")
+    if ref_shard.checksum != entry["reference_checksum"]:
+        raise ValueError(f"{path}: bundled reference_bim reference_checksum does not match {target}")

{statgen-0.2.4 → statgen-0.2.6}/statgen/_ld_schema.py

@@ -28,6 +28,13 @@ def read_manifest(path: Path, expected_runtime: str | None) -> dict:
         raise ValueError(f"{path}: unsupported LD runtime_format {runtime!r}")
     if expected_runtime is not None and runtime != expected_runtime:
         raise ValueError(f"{path}: expected runtime_format {expected_runtime!r}, got {runtime!r}")
+    validate_plain_relative_filename(
+        manifest.get("reference_cache"),
+        f"{path}: reference_cache",
+    )
+    reference_cache_md5 = manifest.get("reference_cache_md5")
+    if not isinstance(reference_cache_md5, str) or len(reference_cache_md5) != 32:
+        raise ValueError(f"{path}: reference_cache_md5 must be a lowercase MD5 hex string")
     shards = manifest.get("shards")
     if not isinstance(shards, list) or not shards:
         raise ValueError(f"{path}: shards must be a non-empty list")
@@ -71,7 +78,7 @@ def validate_manifest_entry(entry: dict, where: str) -> None:
         raise ValueError(f"{where}: reference_checksum must be a non-empty string")
     if not isinstance(entry["file_md5"], str) or len(entry["file_md5"]) != 32:
         raise ValueError(f"{where}: file_md5 must be a lowercase MD5 hex string")
-    validate_reference_bim_filename(entry["reference_bim"], f"{where}: reference_bim")
+    validate_plain_relative_filename(entry["reference_bim"], f"{where}: reference_bim")
 
 
 def validate_shard_metadata(meta: dict, path: Path, expected_format: str) -> None:
@@ -118,7 +125,7 @@ def validate_shard_metadata(meta: dict, path: Path, expected_format: str) -> Non
         raise ValueError(f"{path}: metadata value must be 'r'")
     if not isinstance(meta["reference_checksum"], str) or meta["reference_checksum"] == "":
         raise ValueError(f"{path}: metadata reference_checksum must be a non-empty string")
-    validate_reference_bim_filename(meta["reference_bim"], f"{path}: metadata reference_bim")
+    validate_plain_relative_filename(meta["reference_bim"], f"{path}: metadata reference_bim")
     if expected_format == NPZ_FORMAT:
         if meta.get("sparse_layout") != "csc":
             raise ValueError(f"{path}: metadata sparse_layout must be 'csc'")
@@ -154,7 +161,7 @@ def default_reference_bim_filename(chr_label: str) -> str:
     return f"reference_chr{chr_label}.bim"
 
 
-def validate_reference_bim_filename(value, where: str) -> str:
+def validate_plain_relative_filename(value, where: str) -> str:
     if not isinstance(value, str) or value == "":
         raise ValueError(f"{where} must be a non-empty filename")
     path = Path(value)
@@ -163,6 +170,9 @@ def validate_reference_bim_filename(value, where: str) -> str:
     return value
 
 
+validate_reference_bim_filename = validate_plain_relative_filename
+
+
 def runtime_from_suffix(path: Path) -> str:
     if path.suffix == ".npz":
         return PY_RUNTIME_FORMAT

{statgen-0.2.4 → statgen-0.2.6}/statgen/_ld_writer.py

@@ -22,9 +22,12 @@ from ._ld_schema import (
     validate_shard_metadata,
 )
 from ._ld_reference import validate_bundled_reference_bim
+from .reference import ReferencePanel, load_reference, save_reference_cache
 from ._utils import CHR_RANK
 
 
+REFERENCE_CACHE_FILENAME = "reference_cache.npz"
+
 LD_BUILD_METADATA_DEFAULTS = {
     "build_tool": "statgen",
     "build_command": "synthetic/internal LD artifact writer",
@@ -101,10 +104,13 @@ def write_ld_npz_manifest(root, records, *, validate=True) -> dict:
     records = list(records)
     if not records:
         raise ValueError("LD manifest must contain at least one shard")
+    reference_cache = _write_reference_cache(root, records, REFERENCE_CACHE_FILENAME)
     manifest = {
         "object_type": "ld_panel_manifest",
         "schema_version": MANIFEST_SCHEMA,
         "runtime_format": PY_RUNTIME_FORMAT,
+        "reference_cache": reference_cache.name,
+        "reference_cache_md5": md5_file(reference_cache),
         "shards": records,
     }
     _write_json(root / "ld_manifest.json", manifest)
@@ -117,7 +123,11 @@ def create_ld_npz_manifest(root, *, validate=True) -> dict:
     root = Path(root)
     if not root.is_dir():
         raise FileNotFoundError(f"LD distribution root not found: {root}")
-    records = [_ld_npz_manifest_record(path, root) for path in root.glob("*.npz")]
+    records = [
+        _ld_npz_manifest_record(path, root)
+        for path in root.glob("*.npz")
+        if path.name != REFERENCE_CACHE_FILENAME
+    ]
     records.sort(key=_ld_manifest_sort_key)
     return write_ld_npz_manifest(root, records, validate=validate)
 
@@ -447,6 +457,25 @@ def _require_reference_bim(root: Path, meta: dict) -> str:
     return reference_bim
 
 
+def _write_reference_cache(root: Path, records: list[dict], file_name: str) -> Path:
+    ref_shards = []
+    seen = set()
+    for record in records:
+        label = record["chr"]
+        if label in seen:
+            continue
+        seen.add(label)
+        panel = load_reference(root / record["reference_bim"])
+        if len(panel.shards) != 1:
+            raise ValueError(f"{root / record['reference_bim']}: bundled reference_bim must contain exactly one shard")
+        ref = panel.shards[0]
+        validate_bundled_reference_bim(root / record["reference_bim"], record, target="manifest")
+        ref_shards.append(ref)
+    cache_path = root / file_name
+    save_reference_cache(ReferencePanel(ref_shards), cache_path)
+    return cache_path
+
+
 def _write_deterministic_npz(path: Path, arrays: dict[str, np.ndarray]) -> None:
     with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
         for name, array in arrays.items():

statgen-0.2.6/statgen/_verbosity.py

@@ -0,0 +1,20 @@
+import sys
+
+_LEVELS = ("quiet", "info")
+_VERBOSITY = "info"
+
+
+def set_verbosity(level: str) -> None:
+    global _VERBOSITY
+    if level not in _LEVELS:
+        raise ValueError("verbosity must be one of: quiet, info")
+    _VERBOSITY = level
+
+
+def get_verbosity() -> str:
+    return _VERBOSITY
+
+
+def info(message: str) -> None:
+    if _VERBOSITY == "info":
+        print(message, file=sys.stderr)

{statgen-0.2.4 → statgen-0.2.6}/statgen/ld.py

@@ -13,6 +13,7 @@ from ._ld_schema import (
     validate_manifest_entry_agreement,
 )
 from ._utils import validate_requested_shards
+from ._verbosity import info as _log_info
 
 
 class LDShard:
@@ -245,20 +246,13 @@ class LDPanel:
         return sparse.vstack(parts, format=M.getformat()).astype(out_dtype)
 
 
-def load_ld(path, reference=None, shards=None, default_chrX_sex=None) -> LDPanel:
+def load_ld(path, shards=None, default_chrX_sex=None) -> LDPanel:
     default_chrX_sex = validate_chrx_sex(
         "female" if default_chrX_sex is None else default_chrX_sex,
         "default_chrX_sex",
     )
-    path = Path(path)
-    if not path.is_dir():
-        raise ValueError("load_ld: path must identify a panel root directory")
-
-    manifest = read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
-    if reference is None:
-        reference = _load_bundled_reference(path, manifest, shards)
-    elif shards is not None:
-        reference = reference.select_shards(shards)
+    path, manifest = _read_panel_manifest(path, "load_ld")
+    reference = _load_manifest_reference_cache(path, manifest, shards)
 
     ref_shards = list(reference.shards)
     if not ref_shards:
@@ -268,6 +262,11 @@ def load_ld(path, reference=None, shards=None, default_chrX_sex=None) -> LDPanel
     return LDPanel(groups, default_chrX_sex=default_chrX_sex, reference=reference)
 
 
+def load_ld_reference(path, shards=None):
+    path, manifest = _read_panel_manifest(path, "load_ld_reference")
+    return _load_manifest_reference_cache(path, manifest, shards)
+
+
 def validate_ld_distribution(path, check_payload_structure=False) -> dict:
     if not Path(path).is_dir():
         raise ValueError("validate_ld_distribution: path must identify a panel root directory")
@@ -349,10 +348,19 @@ def _write_ld_npz_distribution(*args, **kwargs):
     return write_ld_npz_distribution(*args, **kwargs)
 
 
-def _load_bundled_reference(root: Path, manifest: dict, shards):
-    from ._ld_reference import load_bundled_reference_panel
+def _read_panel_manifest(path, where: str) -> tuple[Path, dict]:
+    path = Path(path)
+    if not path.is_dir():
+        raise ValueError(f"{where}: path must identify a panel root directory")
+    return path, read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
+
+
+def _load_manifest_reference_cache(root: Path, manifest: dict, shards):
+    from .reference import load_reference_cache
 
-    return load_bundled_reference_panel(root, manifest, shards, where="load_ld")
+    reference_cache_path = root / manifest["reference_cache"]
+    require_file(reference_cache_path)
+    return load_reference_cache(reference_cache_path, shards=shards)
 
 
 def _load_panel_root(root: Path, manifest: dict, ref_shards: list) -> list[list[LDShard]]:
@@ -378,6 +386,10 @@ def _load_panel_root(root: Path, manifest: dict, ref_shards: list) -> list[list[
         seen_sex = set()
         for entry in selected:
             shard_path = root / entry["file"]
+            label = str(entry.get("chr"))
+            sex = entry.get("sex")
+            suffix = "" if sex is None else f" ({sex})"
+            _log_info(f"statgen.load_ld: loading shard {label}{suffix} from {shard_path}")
             shard, meta = _load_npz_shard(shard_path)
             validate_manifest_entry_agreement(entry, meta, shard_path)
             _validate_reference_compatibility(shard, ref_shard, shard_path)

{statgen-0.2.4 → statgen-0.2.6}/statgen/reference.py

@@ -1,7 +1,9 @@
 import hashlib
+import io
 import json
 import logging
 from pathlib import Path
+import zipfile
 
 import numpy as np
 import pandas as pd
@@ -339,8 +341,8 @@ class ReferencePanel:
                 ok = False
         return ok
 
-    def save_cache(self, path, mode: str = "full", format=None) -> None:
-        save_reference_cache(self, path, mode=mode, format=format)
+    def save_cache(self, path, format=None) -> None:
+        save_reference_cache(self, path, format=format)
 
 
 def load_reference(path, shards=None) -> ReferencePanel:
@@ -401,18 +403,12 @@ def load_reference(path, shards=None) -> ReferencePanel:
     return ReferencePanel(out_shards)
 
 
-def save_reference_cache(panel: ReferencePanel, path, mode: str = "full", format=None) -> None:
-    mode = str(mode).lower()
-    if mode not in {"full", "thin"}:
-        raise ValueError("save_reference_cache mode must be 'full' or 'thin'")
-
+def save_reference_cache(panel: ReferencePanel, path, format=None) -> None:
     # Metadata (schema, labels, checksums) as a compact JSON blob stored in the npz.
     # SNP-axis numeric vectors (bp) and string arrays are stored as native
     # binary numpy arrays — not JSON — per the performance contract.
-    # Python keeps reference caches full even when mode='thin' is requested.
     meta = {
         "schema": _CACHE_SCHEMA,
-        "mode": "full",
         "shard_labels": [s.label for s in panel.shards],
         "shard_checksums": [s.checksum for s in panel.shards],
     }
@@ -428,7 +424,7 @@ def save_reference_cache(panel: ReferencePanel, path, mode: str = "full", format
         arrays[p + "a2"]  = np.asarray(s.a2, dtype=str)
         arrays[p + "a1_hash64"] = s.a1_hash64
         arrays[p + "a2_hash64"] = s.a2_hash64
-    np.savez_compressed(path, **arrays)
+    _write_deterministic_npz(Path(path), arrays)
 
 
 def load_reference_cache(path, shards=None) -> ReferencePanel:
@@ -437,11 +433,6 @@ def load_reference_cache(path, shards=None) -> ReferencePanel:
         schema = meta.get("schema")
         if schema != _CACHE_SCHEMA:
             raise ValueError(f"Unsupported reference cache schema: {schema!r}")
-        mode = meta.get("mode")
-        if mode is None:
-            raise ValueError("reference cache missing mode; delete and rebuild old cache")
-        if mode != "full":
-            raise ValueError(f"Unsupported Python reference cache mode: {mode!r}")
 
         labels = list(meta.get("shard_labels", []))
         checksums = list(meta.get("shard_checksums", []))
@@ -470,3 +461,15 @@ def load_reference_cache(path, shards=None) -> ReferencePanel:
                 )
             )
     return ReferencePanel(shard_objs)
+
+
+def _write_deterministic_npz(path: Path, arrays: dict[str, np.ndarray]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        for name, array in arrays.items():
+            buf = io.BytesIO()
+            np.save(buf, array, allow_pickle=False)
+            info = zipfile.ZipInfo(f"{name}.npy", date_time=(1980, 1, 1, 0, 0, 0))
+            info.compress_type = zipfile.ZIP_DEFLATED
+            info.external_attr = 0o644 << 16
+            zf.writestr(info, buf.getvalue())

{statgen-0.2.4 → statgen-0.2.6}/statgen.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: statgen
-Version: 0.2.4
+Version: 0.2.6
 Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{statgen-0.2.4 → statgen-0.2.6}/statgen.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ statgen/_ld_schema.py
 statgen/_ld_writer.py
 statgen/_utils.py
 statgen/_variant_match.py
+statgen/_verbosity.py
 statgen/annotations.py
 statgen/genotype.py
 statgen/ld.py

statgen-0.2.4/statgen/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.4"

statgen-0.2.4/statgen/_ld_reference.py

@@ -1,57 +0,0 @@
-from pathlib import Path
-
-from ._ld_schema import require_file
-from ._utils import validate_requested_shards
-from .reference import ReferencePanel, load_reference
-
-
-def validate_bundled_reference_bim(path, entry: dict, *, target: str):
-    path = Path(path)
-    require_file(path)
-    panel = load_reference(path)
-    ref_shards = panel.shards
-    if len(ref_shards) != 1:
-        raise ValueError(f"{path}: bundled reference_bim must contain exactly one shard")
-    ref = ref_shards[0]
-    validate_reference_shard_for_ld_entry(ref, entry, path, target=target)
-    return ref
-
-
-def validate_reference_shard_for_ld_entry(ref_shard, entry: dict, path, *, target: str) -> None:
-    path = Path(path)
-    if ref_shard.label != entry["chr"]:
-        raise ValueError(f"{path}: bundled reference_bim chr does not match {target}")
-    if ref_shard.num_snp != int(entry["num_snp"]):
-        raise ValueError(f"{path}: bundled reference_bim num_snp does not match {target}")
-    if ref_shard.checksum != entry["reference_checksum"]:
-        raise ValueError(f"{path}: bundled reference_bim reference_checksum does not match {target}")
-
-
-def load_bundled_reference_panel(root, manifest: dict, shards, *, where: str) -> ReferencePanel:
-    root = Path(root)
-    available = _manifest_chr_labels(manifest)
-    selected = validate_requested_shards(shards, available, where)
-    entries_by_chr = {}
-    for entry in manifest["shards"]:
-        entries_by_chr.setdefault(entry["chr"], []).append(entry)
-
-    ref_shards = []
-    for label in selected:
-        entries = entries_by_chr[label]
-        reference_path = root / entries[0]["reference_bim"]
-        ref_shard = validate_bundled_reference_bim(reference_path, entries[0], target="manifest")
-        for entry in entries[1:]:
-            validate_reference_shard_for_ld_entry(ref_shard, entry, reference_path, target="manifest")
-        ref_shards.append(ref_shard)
-    return ReferencePanel(ref_shards)
-
-
-def _manifest_chr_labels(manifest: dict) -> list[str]:
-    labels = []
-    seen = set()
-    for entry in manifest["shards"]:
-        label = entry["chr"]
-        if label not in seen:
-            labels.append(label)
-            seen.add(label)
-    return labels