statgen 0.2.5__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: statgen
3
- Version: 0.2.5
3
+ Version: 0.3.0
4
4
  Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "statgen"
7
- version = "0.2.5"
7
+ version = "0.3.0"
8
8
  description = "Statistical genetics data objects and loaders for Python and MATLAB/Octave."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1,5 @@
1
+ from ._verbosity import get_verbosity, set_verbosity
2
+
3
+ __version__ = "0.3.0"
4
+
5
+ __all__ = ["__version__", "get_verbosity", "set_verbosity"]
@@ -17,6 +17,7 @@ _FAM_NAMES = ["fid", "iid", "father_id", "mother_id", "sex", "pheno"]
17
17
  _PLOIDY_NAMES = ["ploidy_male", "ploidy_female"]
18
18
  _DNA_ALLELE_RE = re.compile(r"^[ACGT]+$")
19
19
  _BED_LOOKUP_INT8 = None
20
+ _BED_MAX_READ_BYTES = 64 * 1024 * 1024
20
21
 
21
22
 
22
23
  def _read_csv_strict(path: Path, *, sep, names, description: str | None = None) -> pd.DataFrame:
@@ -256,20 +257,33 @@ def _bed_lookup_int8() -> np.ndarray:
256
257
  def _read_bed_rows_int8(path: Path, source_rows: np.ndarray, source_num_sample: int) -> np.ndarray:
257
258
  path = Path(path)
258
259
  source_rows = np.asarray(source_rows, dtype=np.int64).reshape(-1)
259
- bytes_per_snp = (int(source_num_sample) + 3) // 4
260
- decoded = np.empty((int(source_num_sample), source_rows.size), dtype=np.int8)
260
+ n_samples = int(source_num_sample)
261
+ bytes_per_snp = (n_samples + 3) // 4
262
+ if source_rows.size == 0:
263
+ return np.empty((n_samples, 0), dtype=np.int8)
264
+ unique_rows, inverse = np.unique(source_rows, return_inverse=True)
265
+ unique_decoded = np.empty((n_samples, unique_rows.size), dtype=np.int8)
266
+ rows_per_read = unique_rows.size if bytes_per_snp == 0 else max(1, _BED_MAX_READ_BYTES // bytes_per_snp)
261
267
  lookup = _bed_lookup_int8()
262
268
  with open(path, "rb") as f:
263
- # PERF: loop over requested SNPs retained; each SNP is a separate BED row
264
- # requiring an individual seek. Inner decode is vectorized across subjects.
265
- for j, source_row0 in enumerate(source_rows):
266
- f.seek(3 + int(source_row0) * bytes_per_snp)
267
- packed = f.read(bytes_per_snp)
268
- if len(packed) != bytes_per_snp:
269
- raise ValueError(
270
- f"{path}: short BED read for source SNP row {int(source_row0)}: "
271
- f"expected {bytes_per_snp} bytes, got {len(packed)}"
272
- )
273
- byte_values = np.frombuffer(packed, dtype=np.uint8)
274
- decoded[:, j] = lookup[byte_values].reshape(-1)[:source_num_sample]
275
- return decoded
269
+ group_starts = np.r_[0, np.flatnonzero(np.diff(unique_rows) != 1) + 1]
270
+ group_stops = np.r_[group_starts[1:], unique_rows.size]
271
+ # PERF: one seek/read per contiguous BED row block; requested order and
272
+ # repeats are restored after block decoding. Blocks are capped to
273
+ # bound transient packed/decode allocations for large requests.
274
+ for start, stop in zip(group_starts, group_stops, strict=True):
275
+ for chunk_start in range(start, stop, rows_per_read):
276
+ chunk_stop = min(chunk_start + rows_per_read, stop)
277
+ block_rows = unique_rows[chunk_start:chunk_stop]
278
+ expected_bytes = int(bytes_per_snp * block_rows.size)
279
+ f.seek(3 + int(block_rows[0]) * bytes_per_snp)
280
+ packed = f.read(expected_bytes)
281
+ if len(packed) != expected_bytes:
282
+ raise ValueError(
283
+ f"{path}: short BED read for source SNP rows {int(block_rows[0])}-{int(block_rows[-1])}: "
284
+ f"expected {expected_bytes} bytes, got {len(packed)}"
285
+ )
286
+ byte_values = np.frombuffer(packed, dtype=np.uint8)
287
+ block_decoded = lookup[byte_values].reshape(block_rows.size, bytes_per_snp * 4)
288
+ unique_decoded[:, chunk_start:chunk_stop] = block_decoded[:, :n_samples].T
289
+ return unique_decoded[:, inverse]
@@ -23,7 +23,14 @@ def validate_npz_distribution(path, check_payload_structure=False) -> dict:
23
23
  raise ValueError("validate_npz_distribution: path must identify a panel root directory")
24
24
 
25
25
  manifest = read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
26
- runtime = manifest["runtime_format"]
26
+ reference_cache_path = path / manifest["reference_cache"]
27
+ require_file(reference_cache_path)
28
+ if md5_file(reference_cache_path) != manifest["reference_cache_md5"]:
29
+ raise ValueError(f"{reference_cache_path}: reference_cache_md5 does not match manifest")
30
+ from .reference import load_reference_cache
31
+
32
+ reference = load_reference_cache(reference_cache_path)
33
+ ref_by_label = {s.label: s for s in reference.shards}
27
34
  seen_reference_bim = set()
28
35
  for entry in manifest["shards"]:
29
36
  shard_path = path / entry["file"]
@@ -36,6 +43,10 @@ def validate_npz_distribution(path, check_payload_structure=False) -> dict:
36
43
  check_payload_structure=check_payload_structure,
37
44
  )
38
45
  validate_manifest_entry_agreement(entry, meta, shard_path)
46
+ ref_shard = ref_by_label.get(entry["chr"])
47
+ if ref_shard is None:
48
+ raise ValueError(f"{reference_cache_path}: reference cache missing shard {entry['chr']}")
49
+ _validate_reference_cache_entry(ref_shard, entry, reference_cache_path)
39
50
  reference_key = (
40
51
  entry["reference_bim"],
41
52
  entry["chr"],
@@ -64,6 +75,13 @@ def _validate_bundled_reference(*, root: Path, entry: dict) -> None:
64
75
  validate_bundled_reference_bim(root / entry["reference_bim"], entry, target="manifest")
65
76
 
66
77
 
78
+ def _validate_reference_cache_entry(ref_shard, entry: dict, path: Path) -> None:
79
+ if ref_shard.num_snp != int(entry["num_snp"]):
80
+ raise ValueError(f"{path}: reference cache num_snp does not match manifest")
81
+ if ref_shard.checksum != entry["reference_checksum"]:
82
+ raise ValueError(f"{path}: reference cache reference_checksum does not match manifest")
83
+
84
+
67
85
  def read_npz_shard(path: Path, check_payload_structure: bool):
68
86
  require_file(path)
69
87
  with np.load(path, allow_pickle=False) as data:
@@ -0,0 +1,26 @@
1
+ from pathlib import Path
2
+
3
+ from ._ld_schema import require_file
4
+ from .reference import load_reference
5
+
6
+
7
+ def validate_bundled_reference_bim(path, entry: dict, *, target: str):
8
+ path = Path(path)
9
+ require_file(path)
10
+ panel = load_reference(path)
11
+ ref_shards = panel.shards
12
+ if len(ref_shards) != 1:
13
+ raise ValueError(f"{path}: bundled reference_bim must contain exactly one shard")
14
+ ref = ref_shards[0]
15
+ validate_reference_shard_for_ld_entry(ref, entry, path, target=target)
16
+ return ref
17
+
18
+
19
+ def validate_reference_shard_for_ld_entry(ref_shard, entry: dict, path, *, target: str) -> None:
20
+ path = Path(path)
21
+ if ref_shard.label != entry["chr"]:
22
+ raise ValueError(f"{path}: bundled reference_bim chr does not match {target}")
23
+ if ref_shard.num_snp != int(entry["num_snp"]):
24
+ raise ValueError(f"{path}: bundled reference_bim num_snp does not match {target}")
25
+ if ref_shard.checksum != entry["reference_checksum"]:
26
+ raise ValueError(f"{path}: bundled reference_bim reference_checksum does not match {target}")
@@ -28,6 +28,13 @@ def read_manifest(path: Path, expected_runtime: str | None) -> dict:
28
28
  raise ValueError(f"{path}: unsupported LD runtime_format {runtime!r}")
29
29
  if expected_runtime is not None and runtime != expected_runtime:
30
30
  raise ValueError(f"{path}: expected runtime_format {expected_runtime!r}, got {runtime!r}")
31
+ validate_plain_relative_filename(
32
+ manifest.get("reference_cache"),
33
+ f"{path}: reference_cache",
34
+ )
35
+ reference_cache_md5 = manifest.get("reference_cache_md5")
36
+ if not isinstance(reference_cache_md5, str) or len(reference_cache_md5) != 32:
37
+ raise ValueError(f"{path}: reference_cache_md5 must be a lowercase MD5 hex string")
31
38
  shards = manifest.get("shards")
32
39
  if not isinstance(shards, list) or not shards:
33
40
  raise ValueError(f"{path}: shards must be a non-empty list")
@@ -71,7 +78,7 @@ def validate_manifest_entry(entry: dict, where: str) -> None:
71
78
  raise ValueError(f"{where}: reference_checksum must be a non-empty string")
72
79
  if not isinstance(entry["file_md5"], str) or len(entry["file_md5"]) != 32:
73
80
  raise ValueError(f"{where}: file_md5 must be a lowercase MD5 hex string")
74
- validate_reference_bim_filename(entry["reference_bim"], f"{where}: reference_bim")
81
+ validate_plain_relative_filename(entry["reference_bim"], f"{where}: reference_bim")
75
82
 
76
83
 
77
84
  def validate_shard_metadata(meta: dict, path: Path, expected_format: str) -> None:
@@ -118,7 +125,7 @@ def validate_shard_metadata(meta: dict, path: Path, expected_format: str) -> Non
118
125
  raise ValueError(f"{path}: metadata value must be 'r'")
119
126
  if not isinstance(meta["reference_checksum"], str) or meta["reference_checksum"] == "":
120
127
  raise ValueError(f"{path}: metadata reference_checksum must be a non-empty string")
121
- validate_reference_bim_filename(meta["reference_bim"], f"{path}: metadata reference_bim")
128
+ validate_plain_relative_filename(meta["reference_bim"], f"{path}: metadata reference_bim")
122
129
  if expected_format == NPZ_FORMAT:
123
130
  if meta.get("sparse_layout") != "csc":
124
131
  raise ValueError(f"{path}: metadata sparse_layout must be 'csc'")
@@ -154,7 +161,7 @@ def default_reference_bim_filename(chr_label: str) -> str:
154
161
  return f"reference_chr{chr_label}.bim"
155
162
 
156
163
 
157
- def validate_reference_bim_filename(value, where: str) -> str:
164
+ def validate_plain_relative_filename(value, where: str) -> str:
158
165
  if not isinstance(value, str) or value == "":
159
166
  raise ValueError(f"{where} must be a non-empty filename")
160
167
  path = Path(value)
@@ -163,6 +170,9 @@ def validate_reference_bim_filename(value, where: str) -> str:
163
170
  return value
164
171
 
165
172
 
173
+ validate_reference_bim_filename = validate_plain_relative_filename
174
+
175
+
166
176
  def runtime_from_suffix(path: Path) -> str:
167
177
  if path.suffix == ".npz":
168
178
  return PY_RUNTIME_FORMAT
@@ -22,9 +22,12 @@ from ._ld_schema import (
22
22
  validate_shard_metadata,
23
23
  )
24
24
  from ._ld_reference import validate_bundled_reference_bim
25
+ from .reference import ReferencePanel, load_reference, save_reference_cache
25
26
  from ._utils import CHR_RANK
26
27
 
27
28
 
29
+ REFERENCE_CACHE_FILENAME = "reference_cache.npz"
30
+
28
31
  LD_BUILD_METADATA_DEFAULTS = {
29
32
  "build_tool": "statgen",
30
33
  "build_command": "synthetic/internal LD artifact writer",
@@ -101,10 +104,13 @@ def write_ld_npz_manifest(root, records, *, validate=True) -> dict:
101
104
  records = list(records)
102
105
  if not records:
103
106
  raise ValueError("LD manifest must contain at least one shard")
107
+ reference_cache = _write_reference_cache(root, records, REFERENCE_CACHE_FILENAME)
104
108
  manifest = {
105
109
  "object_type": "ld_panel_manifest",
106
110
  "schema_version": MANIFEST_SCHEMA,
107
111
  "runtime_format": PY_RUNTIME_FORMAT,
112
+ "reference_cache": reference_cache.name,
113
+ "reference_cache_md5": md5_file(reference_cache),
108
114
  "shards": records,
109
115
  }
110
116
  _write_json(root / "ld_manifest.json", manifest)
@@ -117,7 +123,11 @@ def create_ld_npz_manifest(root, *, validate=True) -> dict:
117
123
  root = Path(root)
118
124
  if not root.is_dir():
119
125
  raise FileNotFoundError(f"LD distribution root not found: {root}")
120
- records = [_ld_npz_manifest_record(path, root) for path in root.glob("*.npz")]
126
+ records = [
127
+ _ld_npz_manifest_record(path, root)
128
+ for path in root.glob("*.npz")
129
+ if path.name != REFERENCE_CACHE_FILENAME
130
+ ]
121
131
  records.sort(key=_ld_manifest_sort_key)
122
132
  return write_ld_npz_manifest(root, records, validate=validate)
123
133
 
@@ -447,6 +457,25 @@ def _require_reference_bim(root: Path, meta: dict) -> str:
447
457
  return reference_bim
448
458
 
449
459
 
460
+ def _write_reference_cache(root: Path, records: list[dict], file_name: str) -> Path:
461
+ ref_shards = []
462
+ seen = set()
463
+ for record in records:
464
+ label = record["chr"]
465
+ if label in seen:
466
+ continue
467
+ seen.add(label)
468
+ panel = load_reference(root / record["reference_bim"])
469
+ if len(panel.shards) != 1:
470
+ raise ValueError(f"{root / record['reference_bim']}: bundled reference_bim must contain exactly one shard")
471
+ ref = panel.shards[0]
472
+ validate_bundled_reference_bim(root / record["reference_bim"], record, target="manifest")
473
+ ref_shards.append(ref)
474
+ cache_path = root / file_name
475
+ save_reference_cache(ReferencePanel(ref_shards), cache_path)
476
+ return cache_path
477
+
478
+
450
479
  def _write_deterministic_npz(path: Path, arrays: dict[str, np.ndarray]) -> None:
451
480
  with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
452
481
  for name, array in arrays.items():
@@ -0,0 +1,20 @@
1
+ import sys
2
+
3
+ _LEVELS = ("quiet", "info")
4
+ _VERBOSITY = "info"
5
+
6
+
7
+ def set_verbosity(level: str) -> None:
8
+ global _VERBOSITY
9
+ if level not in _LEVELS:
10
+ raise ValueError("verbosity must be one of: quiet, info")
11
+ _VERBOSITY = level
12
+
13
+
14
+ def get_verbosity() -> str:
15
+ return _VERBOSITY
16
+
17
+
18
+ def info(message: str) -> None:
19
+ if _VERBOSITY == "info":
20
+ print(message, file=sys.stderr)
@@ -12,13 +12,13 @@ _CACHE_SCHEMA = "annotations_cache/0.1"
12
12
 
13
13
 
14
14
 
15
- def _as_sparse_binary(mat) -> sparse.csr_matrix:
15
+ def _as_sparse_binary(mat, name: str = "annomat") -> sparse.csr_matrix:
16
16
  if sparse.issparse(mat):
17
17
  csr = mat.tocsr(copy=True)
18
18
  else:
19
19
  arr = np.asarray(mat)
20
20
  if arr.ndim != 2:
21
- raise ValueError("annomat must be a 2D matrix")
21
+ raise ValueError(f"{name} must be a 2D matrix")
22
22
  csr = sparse.csr_matrix(arr)
23
23
 
24
24
  # Guard against CSR matrices carrying explicit stored zeros in .data.
@@ -30,7 +30,7 @@ def _as_sparse_binary(mat) -> sparse.csr_matrix:
30
30
  bad = data != 1
31
31
  if bad.any():
32
32
  idx = int(np.flatnonzero(bad)[0])
33
- raise ValueError(f"annomat contains non-binary value: {data[idx]!r}")
33
+ raise ValueError(f"{name} contains non-binary value: {data[idx]!r}")
34
34
 
35
35
  csr = csr.astype(np.uint8)
36
36
  csr.data[:] = 1
@@ -336,17 +336,17 @@ class AnnotationPanel:
336
336
 
337
337
  return AnnotationPanel(out_shards, lhs_names + rhs_names)
338
338
 
339
- def save_cache(self, path, format=None) -> None:
340
- save_annotations_cache(self, path, format=format)
339
+ def save_cache(self, path) -> None:
340
+ save_annotations_cache(self, path)
341
341
 
342
342
 
343
- def create_annotations(reference, annomat, annonames) -> AnnotationPanel:
344
- names = _coerce_annonames(annonames)
343
+ def create_annotations(reference, annotation_matrix, annotation_names) -> AnnotationPanel:
344
+ names = _coerce_annonames(annotation_names)
345
345
  n = int(reference.num_snp)
346
- mat = _as_sparse_binary(annomat)
346
+ mat = _as_sparse_binary(annotation_matrix, "annotation_matrix")
347
347
  if mat.shape != (n, len(names)):
348
348
  raise ValueError(
349
- f"annomat shape mismatch: expected ({n}, {len(names)}), got {mat.shape}"
349
+ f"annotation_matrix shape mismatch: expected ({n}, {len(names)}), got {mat.shape}"
350
350
  )
351
351
 
352
352
  out_shards = []
@@ -369,7 +369,7 @@ def create_annotation(reference, annovec, annoname) -> AnnotationPanel:
369
369
  raise ValueError("annoname must be non-empty")
370
370
  vec = _coerce_annovec(annovec, int(reference.num_snp))
371
371
  mat = sparse.csr_matrix(vec.reshape(-1, 1))
372
- return create_annotations(reference, mat, [name])
372
+ return create_annotations(reference, annotation_matrix=mat, annotation_names=[name])
373
373
 
374
374
 
375
375
  def load_annotations(bed_paths, reference) -> AnnotationPanel:
@@ -384,10 +384,10 @@ def load_annotations(bed_paths, reference) -> AnnotationPanel:
384
384
  raise FileNotFoundError(f"BED file not found: {p}")
385
385
 
386
386
  annomat, annonames = _paint_annotations(paths, reference)
387
- return create_annotations(reference, annomat, annonames)
387
+ return create_annotations(reference, annotation_matrix=annomat, annotation_names=annonames)
388
388
 
389
389
 
390
- def save_annotations_cache(panel: AnnotationPanel, path, format=None) -> None:
390
+ def save_annotations_cache(panel: AnnotationPanel, path) -> None:
391
391
  meta = {
392
392
  "schema": _CACHE_SCHEMA,
393
393
  "shard_labels": [s.label for s in panel.shards],
@@ -519,8 +519,8 @@ class GenotypePanel:
519
519
  same_ploidy = ploidy_male.reshape(1, -1)
520
520
  geno[np.ix_(is_unknown, cols)] *= same_ploidy / 2.0
521
521
 
522
- def save_cache(self, path, format=None) -> None:
523
- save_genotype_cache(self, path, format=format)
522
+ def save_cache(self, path) -> None:
523
+ save_genotype_cache(self, path)
524
524
 
525
525
 
526
526
  def _fam_equal(lhs: pd.DataFrame, rhs: pd.DataFrame) -> bool:
@@ -642,7 +642,11 @@ def load_genotype(bfile_prefix, reference) -> GenotypePanel:
642
642
  else:
643
643
  source_layout = "non_sharded"
644
644
  shared = _load_source_record(prefix, None)
645
- if "X" in ref_labels and Path(prefix + ".ploidy").is_file() is False:
645
+ if (
646
+ "X" in ref_labels
647
+ and (shared.bim["chr"] == "X").any()
648
+ and Path(prefix + ".ploidy").is_file() is False
649
+ ):
646
650
  warnings.warn(
647
651
  f"{prefix}: chrX genotype source has no .ploidy sidecar; "
648
652
  "defaulting chrX rows to male/female ploidy (1, 2)",
@@ -692,7 +696,7 @@ def load_genotype(bfile_prefix, reference) -> GenotypePanel:
692
696
  )
693
697
 
694
698
 
695
- def save_genotype_cache(panel: GenotypePanel, path, format=None) -> None:
699
+ def save_genotype_cache(panel: GenotypePanel, path) -> None:
696
700
  meta = {
697
701
  "schema": _CACHE_SCHEMA,
698
702
  "n_shards": len(panel.shards),
@@ -13,6 +13,7 @@ from ._ld_schema import (
13
13
  validate_manifest_entry_agreement,
14
14
  )
15
15
  from ._utils import validate_requested_shards
16
+ from ._verbosity import info as _log_info
16
17
 
17
18
 
18
19
  class LDShard:
@@ -245,20 +246,13 @@ class LDPanel:
245
246
  return sparse.vstack(parts, format=M.getformat()).astype(out_dtype)
246
247
 
247
248
 
248
- def load_ld(path, reference=None, shards=None, default_chrX_sex=None) -> LDPanel:
249
+ def load_ld(path, shards=None, default_chrX_sex=None) -> LDPanel:
249
250
  default_chrX_sex = validate_chrx_sex(
250
251
  "female" if default_chrX_sex is None else default_chrX_sex,
251
252
  "default_chrX_sex",
252
253
  )
253
- path = Path(path)
254
- if not path.is_dir():
255
- raise ValueError("load_ld: path must identify a panel root directory")
256
-
257
- manifest = read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
258
- if reference is None:
259
- reference = _load_bundled_reference(path, manifest, shards)
260
- elif shards is not None:
261
- reference = reference.select_shards(shards)
254
+ path, manifest = _read_panel_manifest(path, "load_ld")
255
+ reference = _load_manifest_reference_cache(path, manifest, shards)
262
256
 
263
257
  ref_shards = list(reference.shards)
264
258
  if not ref_shards:
@@ -268,6 +262,11 @@ def load_ld(path, reference=None, shards=None, default_chrX_sex=None) -> LDPanel
268
262
  return LDPanel(groups, default_chrX_sex=default_chrX_sex, reference=reference)
269
263
 
270
264
 
265
+ def load_ld_reference(path, shards=None):
266
+ path, manifest = _read_panel_manifest(path, "load_ld_reference")
267
+ return _load_manifest_reference_cache(path, manifest, shards)
268
+
269
+
271
270
  def validate_ld_distribution(path, check_payload_structure=False) -> dict:
272
271
  if not Path(path).is_dir():
273
272
  raise ValueError("validate_ld_distribution: path must identify a panel root directory")
@@ -349,10 +348,19 @@ def _write_ld_npz_distribution(*args, **kwargs):
349
348
  return write_ld_npz_distribution(*args, **kwargs)
350
349
 
351
350
 
352
- def _load_bundled_reference(root: Path, manifest: dict, shards):
353
- from ._ld_reference import load_bundled_reference_panel
351
+ def _read_panel_manifest(path, where: str) -> tuple[Path, dict]:
352
+ path = Path(path)
353
+ if not path.is_dir():
354
+ raise ValueError(f"{where}: path must identify a panel root directory")
355
+ return path, read_manifest(path / "ld_manifest.json", expected_runtime=PY_RUNTIME_FORMAT)
356
+
357
+
358
+ def _load_manifest_reference_cache(root: Path, manifest: dict, shards):
359
+ from .reference import load_reference_cache
354
360
 
355
- return load_bundled_reference_panel(root, manifest, shards, where="load_ld")
361
+ reference_cache_path = root / manifest["reference_cache"]
362
+ require_file(reference_cache_path)
363
+ return load_reference_cache(reference_cache_path, shards=shards)
356
364
 
357
365
 
358
366
  def _load_panel_root(root: Path, manifest: dict, ref_shards: list) -> list[list[LDShard]]:
@@ -378,6 +386,10 @@ def _load_panel_root(root: Path, manifest: dict, ref_shards: list) -> list[list[
378
386
  seen_sex = set()
379
387
  for entry in selected:
380
388
  shard_path = root / entry["file"]
389
+ label = str(entry.get("chr"))
390
+ sex = entry.get("sex")
391
+ suffix = "" if sex is None else f" ({sex})"
392
+ _log_info(f"statgen.load_ld: loading shard {label}{suffix} from {shard_path}")
381
393
  shard, meta = _load_npz_shard(shard_path)
382
394
  validate_manifest_entry_agreement(entry, meta, shard_path)
383
395
  _validate_reference_compatibility(shard, ref_shard, shard_path)
@@ -1,7 +1,9 @@
1
1
  import hashlib
2
+ import io
2
3
  import json
3
4
  import logging
4
5
  from pathlib import Path
6
+ import zipfile
5
7
 
6
8
  import numpy as np
7
9
  import pandas as pd
@@ -339,8 +341,8 @@ class ReferencePanel:
339
341
  ok = False
340
342
  return ok
341
343
 
342
- def save_cache(self, path, mode: str = "full", format=None) -> None:
343
- save_reference_cache(self, path, mode=mode, format=format)
344
+ def save_cache(self, path) -> None:
345
+ save_reference_cache(self, path)
344
346
 
345
347
 
346
348
  def load_reference(path, shards=None) -> ReferencePanel:
@@ -401,18 +403,12 @@ def load_reference(path, shards=None) -> ReferencePanel:
401
403
  return ReferencePanel(out_shards)
402
404
 
403
405
 
404
- def save_reference_cache(panel: ReferencePanel, path, mode: str = "full", format=None) -> None:
405
- mode = str(mode).lower()
406
- if mode not in {"full", "thin"}:
407
- raise ValueError("save_reference_cache mode must be 'full' or 'thin'")
408
-
406
+ def save_reference_cache(panel: ReferencePanel, path) -> None:
409
407
  # Metadata (schema, labels, checksums) as a compact JSON blob stored in the npz.
410
408
  # SNP-axis numeric vectors (bp) and string arrays are stored as native
411
409
  # binary numpy arrays — not JSON — per the performance contract.
412
- # Python keeps reference caches full even when mode='thin' is requested.
413
410
  meta = {
414
411
  "schema": _CACHE_SCHEMA,
415
- "mode": "full",
416
412
  "shard_labels": [s.label for s in panel.shards],
417
413
  "shard_checksums": [s.checksum for s in panel.shards],
418
414
  }
@@ -428,7 +424,7 @@ def save_reference_cache(panel: ReferencePanel, path, mode: str = "full", format
428
424
  arrays[p + "a2"] = np.asarray(s.a2, dtype=str)
429
425
  arrays[p + "a1_hash64"] = s.a1_hash64
430
426
  arrays[p + "a2_hash64"] = s.a2_hash64
431
- np.savez_compressed(path, **arrays)
427
+ _write_deterministic_npz(Path(path), arrays)
432
428
 
433
429
 
434
430
  def load_reference_cache(path, shards=None) -> ReferencePanel:
@@ -437,11 +433,6 @@ def load_reference_cache(path, shards=None) -> ReferencePanel:
437
433
  schema = meta.get("schema")
438
434
  if schema != _CACHE_SCHEMA:
439
435
  raise ValueError(f"Unsupported reference cache schema: {schema!r}")
440
- mode = meta.get("mode")
441
- if mode is None:
442
- raise ValueError("reference cache missing mode; delete and rebuild old cache")
443
- if mode != "full":
444
- raise ValueError(f"Unsupported Python reference cache mode: {mode!r}")
445
436
 
446
437
  labels = list(meta.get("shard_labels", []))
447
438
  checksums = list(meta.get("shard_checksums", []))
@@ -470,3 +461,15 @@ def load_reference_cache(path, shards=None) -> ReferencePanel:
470
461
  )
471
462
  )
472
463
  return ReferencePanel(shard_objs)
464
+
465
+
466
+ def _write_deterministic_npz(path: Path, arrays: dict[str, np.ndarray]) -> None:
467
+ path.parent.mkdir(parents=True, exist_ok=True)
468
+ with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
469
+ for name, array in arrays.items():
470
+ buf = io.BytesIO()
471
+ np.save(buf, array, allow_pickle=False)
472
+ info = zipfile.ZipInfo(f"{name}.npy", date_time=(1980, 1, 1, 0, 0, 0))
473
+ info.compress_type = zipfile.ZIP_DEFLATED
474
+ info.external_attr = 0o644 << 16
475
+ zf.writestr(info, buf.getvalue())
@@ -252,8 +252,8 @@ class Sumstats:
252
252
  by_label = {s.label: s for s in self._shards}
253
253
  return Sumstats([by_label[label] for label in selected])
254
254
 
255
- def save_cache(self, path, format=None) -> None:
256
- save_sumstats_cache(self, path, format=format)
255
+ def save_cache(self, path) -> None:
256
+ save_sumstats_cache(self, path)
257
257
 
258
258
 
259
259
  def _derive_logp(p_vals: np.ndarray) -> np.ndarray:
@@ -393,26 +393,26 @@ def load_sumstats(path, reference) -> Sumstats:
393
393
 
394
394
  def create_sumstats(
395
395
  reference,
396
- pvec,
397
- zvec=None,
398
- nvec=None,
399
- beta_vec=None,
400
- se_vec=None,
401
- eaf_vec=None,
402
- info_vec=None,
396
+ p,
397
+ z=None,
398
+ n=None,
399
+ beta=None,
400
+ se=None,
401
+ eaf=None,
402
+ info=None,
403
403
  ) -> Sumstats:
404
- n = int(reference.num_snp)
405
- aligned_p = _coerce_aligned_vec("pvec", pvec, n=n, allow_inf=True)
406
- _validate_pvec(aligned_p, "pvec")
404
+ num_snp_value = int(reference.num_snp)
405
+ aligned_p = _coerce_aligned_vec("p", p, n=num_snp_value, allow_inf=True)
406
+ _validate_pvec(aligned_p, "p")
407
407
  aligned_logp = _derive_logp(aligned_p)
408
408
 
409
409
  aligned_optional = {
410
- "z": None if zvec is None else _coerce_aligned_vec("zvec", zvec, n=n, allow_inf=False),
411
- "n": None if nvec is None else _coerce_aligned_vec("nvec", nvec, n=n, allow_inf=False),
412
- "beta": None if beta_vec is None else _coerce_aligned_vec("beta_vec", beta_vec, n=n, allow_inf=False),
413
- "se": None if se_vec is None else _coerce_aligned_vec("se_vec", se_vec, n=n, allow_inf=False),
414
- "eaf": None if eaf_vec is None else _coerce_aligned_vec("eaf_vec", eaf_vec, n=n, allow_inf=False),
415
- "info": None if info_vec is None else _coerce_aligned_vec("info_vec", info_vec, n=n, allow_inf=False),
410
+ "z": None if z is None else _coerce_aligned_vec("z", z, n=num_snp_value, allow_inf=False),
411
+ "n": None if n is None else _coerce_aligned_vec("n", n, n=num_snp_value, allow_inf=False),
412
+ "beta": None if beta is None else _coerce_aligned_vec("beta", beta, n=num_snp_value, allow_inf=False),
413
+ "se": None if se is None else _coerce_aligned_vec("se", se, n=num_snp_value, allow_inf=False),
414
+ "eaf": None if eaf is None else _coerce_aligned_vec("eaf", eaf, n=num_snp_value, allow_inf=False),
415
+ "info": None if info is None else _coerce_aligned_vec("info", info, n=num_snp_value, allow_inf=False),
416
416
  }
417
417
 
418
418
  return _build_sumstats_from_aligned(
@@ -422,7 +422,7 @@ def create_sumstats(
422
422
  )
423
423
 
424
424
 
425
- def save_sumstats_cache(sumstats: Sumstats, path, format=None) -> None:
425
+ def save_sumstats_cache(sumstats: Sumstats, path) -> None:
426
426
  meta = {
427
427
  "schema": _CACHE_SCHEMA,
428
428
  "shard_labels": [s.label for s in sumstats.shards],
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: statgen
3
- Version: 0.2.5
3
+ Version: 0.3.0
4
4
  Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -8,6 +8,7 @@ statgen/_ld_schema.py
8
8
  statgen/_ld_writer.py
9
9
  statgen/_utils.py
10
10
  statgen/_variant_match.py
11
+ statgen/_verbosity.py
11
12
  statgen/annotations.py
12
13
  statgen/genotype.py
13
14
  statgen/ld.py
@@ -1 +0,0 @@
1
- __version__ = "0.2.5"
@@ -1,57 +0,0 @@
1
- from pathlib import Path
2
-
3
- from ._ld_schema import require_file
4
- from ._utils import validate_requested_shards
5
- from .reference import ReferencePanel, load_reference
6
-
7
-
8
- def validate_bundled_reference_bim(path, entry: dict, *, target: str):
9
- path = Path(path)
10
- require_file(path)
11
- panel = load_reference(path)
12
- ref_shards = panel.shards
13
- if len(ref_shards) != 1:
14
- raise ValueError(f"{path}: bundled reference_bim must contain exactly one shard")
15
- ref = ref_shards[0]
16
- validate_reference_shard_for_ld_entry(ref, entry, path, target=target)
17
- return ref
18
-
19
-
20
- def validate_reference_shard_for_ld_entry(ref_shard, entry: dict, path, *, target: str) -> None:
21
- path = Path(path)
22
- if ref_shard.label != entry["chr"]:
23
- raise ValueError(f"{path}: bundled reference_bim chr does not match {target}")
24
- if ref_shard.num_snp != int(entry["num_snp"]):
25
- raise ValueError(f"{path}: bundled reference_bim num_snp does not match {target}")
26
- if ref_shard.checksum != entry["reference_checksum"]:
27
- raise ValueError(f"{path}: bundled reference_bim reference_checksum does not match {target}")
28
-
29
-
30
- def load_bundled_reference_panel(root, manifest: dict, shards, *, where: str) -> ReferencePanel:
31
- root = Path(root)
32
- available = _manifest_chr_labels(manifest)
33
- selected = validate_requested_shards(shards, available, where)
34
- entries_by_chr = {}
35
- for entry in manifest["shards"]:
36
- entries_by_chr.setdefault(entry["chr"], []).append(entry)
37
-
38
- ref_shards = []
39
- for label in selected:
40
- entries = entries_by_chr[label]
41
- reference_path = root / entries[0]["reference_bim"]
42
- ref_shard = validate_bundled_reference_bim(reference_path, entries[0], target="manifest")
43
- for entry in entries[1:]:
44
- validate_reference_shard_for_ld_entry(ref_shard, entry, reference_path, target="manifest")
45
- ref_shards.append(ref_shard)
46
- return ReferencePanel(ref_shards)
47
-
48
-
49
- def _manifest_chr_labels(manifest: dict) -> list[str]:
50
- labels = []
51
- seen = set()
52
- for entry in manifest["shards"]:
53
- label = entry["chr"]
54
- if label not in seen:
55
- labels.append(label)
56
- seen.add(label)
57
- return labels
File without changes
File without changes
File without changes