statgen 0.2.6__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {statgen-0.2.6 → statgen-0.3.1}/PKG-INFO +1 -1
- {statgen-0.2.6 → statgen-0.3.1}/pyproject.toml +1 -1
- {statgen-0.2.6 → statgen-0.3.1}/statgen/__init__.py +1 -1
- {statgen-0.2.6 → statgen-0.3.1}/statgen/_bfile_utils.py +33 -15
- {statgen-0.2.6 → statgen-0.3.1}/statgen/_utils.py +41 -2
- {statgen-0.2.6 → statgen-0.3.1}/statgen/annotations.py +12 -12
- {statgen-0.2.6 → statgen-0.3.1}/statgen/genotype.py +8 -4
- {statgen-0.2.6 → statgen-0.3.1}/statgen/reference.py +3 -16
- {statgen-0.2.6 → statgen-0.3.1}/statgen/sumstats.py +112 -44
- {statgen-0.2.6 → statgen-0.3.1}/statgen.egg-info/PKG-INFO +1 -1
- {statgen-0.2.6 → statgen-0.3.1}/README.md +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/setup.cfg +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen/_ld_npz.py +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen/_ld_reference.py +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen/_ld_schema.py +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen/_ld_writer.py +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen/_variant_match.py +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen/_verbosity.py +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen/ld.py +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen.egg-info/SOURCES.txt +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen.egg-info/dependency_links.txt +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen.egg-info/requires.txt +0 -0
- {statgen-0.2.6 → statgen-0.3.1}/statgen.egg-info/top_level.txt +0 -0
|
@@ -17,6 +17,7 @@ _FAM_NAMES = ["fid", "iid", "father_id", "mother_id", "sex", "pheno"]
|
|
|
17
17
|
_PLOIDY_NAMES = ["ploidy_male", "ploidy_female"]
|
|
18
18
|
_DNA_ALLELE_RE = re.compile(r"^[ACGT]+$")
|
|
19
19
|
_BED_LOOKUP_INT8 = None
|
|
20
|
+
_BED_MAX_READ_BYTES = 64 * 1024 * 1024
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
def _read_csv_strict(path: Path, *, sep, names, description: str | None = None) -> pd.DataFrame:
|
|
@@ -134,6 +135,10 @@ def _parse_bim(path: Path) -> pd.DataFrame:
|
|
|
134
135
|
raise ValueError(
|
|
135
136
|
f"{path}:{idx + 1}: a2 must be uppercase DNA bases (A/C/G/T): {df['a2'].iat[idx]!r}"
|
|
136
137
|
)
|
|
138
|
+
same_allele = df["a1"].eq(df["a2"])
|
|
139
|
+
if same_allele.any():
|
|
140
|
+
idx = int(same_allele.idxmax())
|
|
141
|
+
raise ValueError(f"{path}:{idx + 1}: a1 and a2 must differ")
|
|
137
142
|
|
|
138
143
|
cm_num = pd.to_numeric(df["cm"], errors="coerce")
|
|
139
144
|
bad_cm = cm_num.isna()
|
|
@@ -256,20 +261,33 @@ def _bed_lookup_int8() -> np.ndarray:
|
|
|
256
261
|
def _read_bed_rows_int8(path: Path, source_rows: np.ndarray, source_num_sample: int) -> np.ndarray:
|
|
257
262
|
path = Path(path)
|
|
258
263
|
source_rows = np.asarray(source_rows, dtype=np.int64).reshape(-1)
|
|
259
|
-
|
|
260
|
-
|
|
264
|
+
n_samples = int(source_num_sample)
|
|
265
|
+
bytes_per_snp = (n_samples + 3) // 4
|
|
266
|
+
if source_rows.size == 0:
|
|
267
|
+
return np.empty((n_samples, 0), dtype=np.int8)
|
|
268
|
+
unique_rows, inverse = np.unique(source_rows, return_inverse=True)
|
|
269
|
+
unique_decoded = np.empty((n_samples, unique_rows.size), dtype=np.int8)
|
|
270
|
+
rows_per_read = unique_rows.size if bytes_per_snp == 0 else max(1, _BED_MAX_READ_BYTES // bytes_per_snp)
|
|
261
271
|
lookup = _bed_lookup_int8()
|
|
262
272
|
with open(path, "rb") as f:
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
)
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
273
|
+
group_starts = np.r_[0, np.flatnonzero(np.diff(unique_rows) != 1) + 1]
|
|
274
|
+
group_stops = np.r_[group_starts[1:], unique_rows.size]
|
|
275
|
+
# PERF: one seek/read per contiguous BED row block; requested order and
|
|
276
|
+
# repeats are restored after block decoding. Blocks are capped to
|
|
277
|
+
# bound transient packed/decode allocations for large requests.
|
|
278
|
+
for start, stop in zip(group_starts, group_stops, strict=True):
|
|
279
|
+
for chunk_start in range(start, stop, rows_per_read):
|
|
280
|
+
chunk_stop = min(chunk_start + rows_per_read, stop)
|
|
281
|
+
block_rows = unique_rows[chunk_start:chunk_stop]
|
|
282
|
+
expected_bytes = int(bytes_per_snp * block_rows.size)
|
|
283
|
+
f.seek(3 + int(block_rows[0]) * bytes_per_snp)
|
|
284
|
+
packed = f.read(expected_bytes)
|
|
285
|
+
if len(packed) != expected_bytes:
|
|
286
|
+
raise ValueError(
|
|
287
|
+
f"{path}: short BED read for source SNP rows {int(block_rows[0])}-{int(block_rows[-1])}: "
|
|
288
|
+
f"expected {expected_bytes} bytes, got {len(packed)}"
|
|
289
|
+
)
|
|
290
|
+
byte_values = np.frombuffer(packed, dtype=np.uint8)
|
|
291
|
+
block_decoded = lookup[byte_values].reshape(block_rows.size, bytes_per_snp * 4)
|
|
292
|
+
unique_decoded[:, chunk_start:chunk_stop] = block_decoded[:, :n_samples].T
|
|
293
|
+
return unique_decoded[:, inverse]
|
|
@@ -8,6 +8,14 @@ _ALLELE_HASH_P = 2_147_483_647
|
|
|
8
8
|
_ALLELE_HASH_BASE1 = 257
|
|
9
9
|
_ALLELE_HASH_BASE2 = 263
|
|
10
10
|
_ALLELE_HASH_MAX_CHARS = 150
|
|
11
|
+
_SINGLE_BASE_BYTES = np.frombuffer(b"ACGT", dtype=np.uint8).astype(np.int64) + 1
|
|
12
|
+
_SINGLE_BASE_HASH64 = (
|
|
13
|
+
(
|
|
14
|
+
((_ALLELE_HASH_BASE1 + _SINGLE_BASE_BYTES) % _ALLELE_HASH_P).astype(np.uint64)
|
|
15
|
+
<< np.uint64(32)
|
|
16
|
+
)
|
|
17
|
+
+ ((_ALLELE_HASH_BASE2 + _SINGLE_BASE_BYTES) % _ALLELE_HASH_P).astype(np.uint64)
|
|
18
|
+
)
|
|
11
19
|
|
|
12
20
|
|
|
13
21
|
def validate_requested_shards(shards, available_labels, where: str) -> list[str]:
|
|
@@ -55,7 +63,38 @@ def allele_hash64(alleles) -> np.ndarray:
|
|
|
55
63
|
strings = [s[:_ALLELE_HASH_MAX_CHARS] for s in strings]
|
|
56
64
|
lengths = np.minimum(lengths, _ALLELE_HASH_MAX_CHARS)
|
|
57
65
|
|
|
58
|
-
|
|
66
|
+
out = np.empty(n, dtype=np.uint64)
|
|
67
|
+
single_base = lengths == 1
|
|
68
|
+
if np.any(single_base):
|
|
69
|
+
single_idx = np.flatnonzero(single_base)
|
|
70
|
+
single_strings = np.array([strings[i] for i in single_idx], dtype=object)
|
|
71
|
+
assigned = np.zeros(single_idx.size, dtype=bool)
|
|
72
|
+
for base, hash_value in zip(("A", "C", "G", "T"), _SINGLE_BASE_HASH64):
|
|
73
|
+
matches = single_strings == base
|
|
74
|
+
if np.any(matches):
|
|
75
|
+
out[single_idx[matches]] = hash_value
|
|
76
|
+
assigned[matches] = True
|
|
77
|
+
|
|
78
|
+
if np.any(~assigned):
|
|
79
|
+
other_idx = single_idx[~assigned]
|
|
80
|
+
encoded = [strings[i].encode("utf-8") for i in other_idx]
|
|
81
|
+
h1, h2 = _allele_hash_lanes_from_utf8(encoded)
|
|
82
|
+
out[other_idx] = (
|
|
83
|
+
h1.astype(np.uint64) << np.uint64(32)
|
|
84
|
+
) + h2.astype(np.uint64)
|
|
85
|
+
|
|
86
|
+
if np.all(single_base):
|
|
87
|
+
return out
|
|
88
|
+
|
|
89
|
+
multi_idx = np.flatnonzero(~single_base)
|
|
90
|
+
encoded = [strings[i].encode("utf-8") for i in multi_idx]
|
|
91
|
+
h1, h2 = _allele_hash_lanes_from_utf8(encoded)
|
|
92
|
+
out[multi_idx] = (h1.astype(np.uint64) << np.uint64(32)) + h2.astype(np.uint64)
|
|
93
|
+
return out
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _allele_hash_lanes_from_utf8(encoded) -> tuple[np.ndarray, np.ndarray]:
|
|
97
|
+
n = len(encoded)
|
|
59
98
|
byte_lengths = np.fromiter((len(b) for b in encoded), dtype=np.int64, count=n)
|
|
60
99
|
max_len = int(byte_lengths.max(initial=0))
|
|
61
100
|
mat = np.zeros((n, max_len), dtype=np.uint8)
|
|
@@ -74,4 +113,4 @@ def allele_hash64(alleles) -> np.ndarray:
|
|
|
74
113
|
h1[active] = (h1[active] * _ALLELE_HASH_BASE1 + x) % _ALLELE_HASH_P
|
|
75
114
|
h2[active] = (h2[active] * _ALLELE_HASH_BASE2 + x) % _ALLELE_HASH_P
|
|
76
115
|
|
|
77
|
-
return
|
|
116
|
+
return h1, h2
|
|
@@ -12,13 +12,13 @@ _CACHE_SCHEMA = "annotations_cache/0.1"
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
|
|
15
|
-
def _as_sparse_binary(mat) -> sparse.csr_matrix:
|
|
15
|
+
def _as_sparse_binary(mat, name: str = "annomat") -> sparse.csr_matrix:
|
|
16
16
|
if sparse.issparse(mat):
|
|
17
17
|
csr = mat.tocsr(copy=True)
|
|
18
18
|
else:
|
|
19
19
|
arr = np.asarray(mat)
|
|
20
20
|
if arr.ndim != 2:
|
|
21
|
-
raise ValueError("
|
|
21
|
+
raise ValueError(f"{name} must be a 2D matrix")
|
|
22
22
|
csr = sparse.csr_matrix(arr)
|
|
23
23
|
|
|
24
24
|
# Guard against CSR matrices carrying explicit stored zeros in .data.
|
|
@@ -30,7 +30,7 @@ def _as_sparse_binary(mat) -> sparse.csr_matrix:
|
|
|
30
30
|
bad = data != 1
|
|
31
31
|
if bad.any():
|
|
32
32
|
idx = int(np.flatnonzero(bad)[0])
|
|
33
|
-
raise ValueError(f"
|
|
33
|
+
raise ValueError(f"{name} contains non-binary value: {data[idx]!r}")
|
|
34
34
|
|
|
35
35
|
csr = csr.astype(np.uint8)
|
|
36
36
|
csr.data[:] = 1
|
|
@@ -336,17 +336,17 @@ class AnnotationPanel:
|
|
|
336
336
|
|
|
337
337
|
return AnnotationPanel(out_shards, lhs_names + rhs_names)
|
|
338
338
|
|
|
339
|
-
def save_cache(self, path
|
|
340
|
-
save_annotations_cache(self, path
|
|
339
|
+
def save_cache(self, path) -> None:
|
|
340
|
+
save_annotations_cache(self, path)
|
|
341
341
|
|
|
342
342
|
|
|
343
|
-
def create_annotations(reference,
|
|
344
|
-
names = _coerce_annonames(
|
|
343
|
+
def create_annotations(reference, annotation_matrix, annotation_names) -> AnnotationPanel:
|
|
344
|
+
names = _coerce_annonames(annotation_names)
|
|
345
345
|
n = int(reference.num_snp)
|
|
346
|
-
mat = _as_sparse_binary(
|
|
346
|
+
mat = _as_sparse_binary(annotation_matrix, "annotation_matrix")
|
|
347
347
|
if mat.shape != (n, len(names)):
|
|
348
348
|
raise ValueError(
|
|
349
|
-
f"
|
|
349
|
+
f"annotation_matrix shape mismatch: expected ({n}, {len(names)}), got {mat.shape}"
|
|
350
350
|
)
|
|
351
351
|
|
|
352
352
|
out_shards = []
|
|
@@ -369,7 +369,7 @@ def create_annotation(reference, annovec, annoname) -> AnnotationPanel:
|
|
|
369
369
|
raise ValueError("annoname must be non-empty")
|
|
370
370
|
vec = _coerce_annovec(annovec, int(reference.num_snp))
|
|
371
371
|
mat = sparse.csr_matrix(vec.reshape(-1, 1))
|
|
372
|
-
return create_annotations(reference, mat, [name])
|
|
372
|
+
return create_annotations(reference, annotation_matrix=mat, annotation_names=[name])
|
|
373
373
|
|
|
374
374
|
|
|
375
375
|
def load_annotations(bed_paths, reference) -> AnnotationPanel:
|
|
@@ -384,10 +384,10 @@ def load_annotations(bed_paths, reference) -> AnnotationPanel:
|
|
|
384
384
|
raise FileNotFoundError(f"BED file not found: {p}")
|
|
385
385
|
|
|
386
386
|
annomat, annonames = _paint_annotations(paths, reference)
|
|
387
|
-
return create_annotations(reference, annomat, annonames)
|
|
387
|
+
return create_annotations(reference, annotation_matrix=annomat, annotation_names=annonames)
|
|
388
388
|
|
|
389
389
|
|
|
390
|
-
def save_annotations_cache(panel: AnnotationPanel, path
|
|
390
|
+
def save_annotations_cache(panel: AnnotationPanel, path) -> None:
|
|
391
391
|
meta = {
|
|
392
392
|
"schema": _CACHE_SCHEMA,
|
|
393
393
|
"shard_labels": [s.label for s in panel.shards],
|
|
@@ -519,8 +519,8 @@ class GenotypePanel:
|
|
|
519
519
|
same_ploidy = ploidy_male.reshape(1, -1)
|
|
520
520
|
geno[np.ix_(is_unknown, cols)] *= same_ploidy / 2.0
|
|
521
521
|
|
|
522
|
-
def save_cache(self, path
|
|
523
|
-
save_genotype_cache(self, path
|
|
522
|
+
def save_cache(self, path) -> None:
|
|
523
|
+
save_genotype_cache(self, path)
|
|
524
524
|
|
|
525
525
|
|
|
526
526
|
def _fam_equal(lhs: pd.DataFrame, rhs: pd.DataFrame) -> bool:
|
|
@@ -642,7 +642,11 @@ def load_genotype(bfile_prefix, reference) -> GenotypePanel:
|
|
|
642
642
|
else:
|
|
643
643
|
source_layout = "non_sharded"
|
|
644
644
|
shared = _load_source_record(prefix, None)
|
|
645
|
-
if
|
|
645
|
+
if (
|
|
646
|
+
"X" in ref_labels
|
|
647
|
+
and (shared.bim["chr"] == "X").any()
|
|
648
|
+
and Path(prefix + ".ploidy").is_file() is False
|
|
649
|
+
):
|
|
646
650
|
warnings.warn(
|
|
647
651
|
f"{prefix}: chrX genotype source has no .ploidy sidecar; "
|
|
648
652
|
"defaulting chrX rows to male/female ploidy (1, 2)",
|
|
@@ -692,7 +696,7 @@ def load_genotype(bfile_prefix, reference) -> GenotypePanel:
|
|
|
692
696
|
)
|
|
693
697
|
|
|
694
698
|
|
|
695
|
-
def save_genotype_cache(panel: GenotypePanel, path
|
|
699
|
+
def save_genotype_cache(panel: GenotypePanel, path) -> None:
|
|
696
700
|
meta = {
|
|
697
701
|
"schema": _CACHE_SCHEMA,
|
|
698
702
|
"n_shards": len(panel.shards),
|
|
@@ -39,17 +39,6 @@ def _is_strand_ambiguous(a1_hash64, a2_hash64) -> np.ndarray:
|
|
|
39
39
|
)
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
def _validate_distinct_alleles(label: str, a1_arr: np.ndarray, a2_arr: np.ndarray) -> None:
|
|
43
|
-
same = np.asarray(a1_arr, dtype=object).reshape(-1) == np.asarray(
|
|
44
|
-
a2_arr, dtype=object
|
|
45
|
-
).reshape(-1)
|
|
46
|
-
if np.any(same):
|
|
47
|
-
idx0 = int(np.flatnonzero(same)[0])
|
|
48
|
-
raise ValueError(
|
|
49
|
-
f"Reference shard {label} variant {idx0 + 1}: a1 and a2 must differ"
|
|
50
|
-
)
|
|
51
|
-
|
|
52
|
-
|
|
53
42
|
def _checksum_from_arrays(
|
|
54
43
|
chr_arr: np.ndarray, bp_arr: np.ndarray, a1_arr: np.ndarray, a2_arr: np.ndarray
|
|
55
44
|
) -> str:
|
|
@@ -92,7 +81,6 @@ class ReferenceShard:
|
|
|
92
81
|
raise ValueError("ReferenceShard vector lengths must match")
|
|
93
82
|
if self._chr.size and not np.all(self._chr == self._label):
|
|
94
83
|
raise ValueError(f"Reference shard {self._label} contains multiple chr labels")
|
|
95
|
-
_validate_distinct_alleles(self._label, self._a1, self._a2)
|
|
96
84
|
self._a1_hash64 = allele_hash64(self._a1)
|
|
97
85
|
self._a2_hash64 = allele_hash64(self._a2)
|
|
98
86
|
self._checksum = _checksum_from_arrays(self._chr, self._bp, self._a1, self._a2)
|
|
@@ -175,7 +163,6 @@ class ReferenceShard:
|
|
|
175
163
|
raise ValueError("Invalid reference cache: panel-wide vector lengths mismatch")
|
|
176
164
|
if obj._chr.size and not np.all(obj._chr == obj._label):
|
|
177
165
|
raise ValueError(f"Invalid reference cache: shard {obj._label} contains multiple chr labels")
|
|
178
|
-
_validate_distinct_alleles(obj._label, obj._a1, obj._a2)
|
|
179
166
|
obj._a1_hash64 = (
|
|
180
167
|
allele_hash64(obj._a1)
|
|
181
168
|
if a1_hash64 is None
|
|
@@ -341,8 +328,8 @@ class ReferencePanel:
|
|
|
341
328
|
ok = False
|
|
342
329
|
return ok
|
|
343
330
|
|
|
344
|
-
def save_cache(self, path
|
|
345
|
-
save_reference_cache(self, path
|
|
331
|
+
def save_cache(self, path) -> None:
|
|
332
|
+
save_reference_cache(self, path)
|
|
346
333
|
|
|
347
334
|
|
|
348
335
|
def load_reference(path, shards=None) -> ReferencePanel:
|
|
@@ -403,7 +390,7 @@ def load_reference(path, shards=None) -> ReferencePanel:
|
|
|
403
390
|
return ReferencePanel(out_shards)
|
|
404
391
|
|
|
405
392
|
|
|
406
|
-
def save_reference_cache(panel: ReferencePanel, path
|
|
393
|
+
def save_reference_cache(panel: ReferencePanel, path) -> None:
|
|
407
394
|
# Metadata (schema, labels, checksums) as a compact JSON blob stored in the npz.
|
|
408
395
|
# SNP-axis numeric vectors (bp) and string arrays are stored as native
|
|
409
396
|
# binary numpy arrays — not JSON — per the performance contract.
|
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
import json
|
|
2
|
+
import re
|
|
2
3
|
import warnings
|
|
4
|
+
import gzip
|
|
3
5
|
from pathlib import Path
|
|
4
6
|
|
|
5
7
|
import numpy as np
|
|
6
8
|
import pandas as pd
|
|
7
|
-
from pandas.errors import ParserError
|
|
9
|
+
from pandas.errors import EmptyDataError, ParserError
|
|
8
10
|
|
|
9
|
-
from ._utils import allele_hash64, validate_requested_shards
|
|
11
|
+
from ._utils import CANONICAL_CHR_ORDER, allele_hash64, validate_requested_shards
|
|
10
12
|
from ._variant_match import match_shard_numeric, numeric_variant_keys
|
|
11
13
|
|
|
12
14
|
_CACHE_SCHEMA = "sumstats_cache/0.1"
|
|
15
|
+
_IGNORED_CHR = {"Y", "MT"}
|
|
16
|
+
_DNA_ALLELE_RE = re.compile(r"^[ACGT]+$")
|
|
13
17
|
_REQUIRED_COLS = ("chr", "bp", "a1", "a2", "p")
|
|
14
18
|
_OPTIONAL_COLS = ("z", "n", "beta", "se", "eaf", "info")
|
|
15
19
|
_OPTIONAL_VECTOR_FIELDS = ("z", "n", "beta", "se", "eaf", "info")
|
|
@@ -18,64 +22,128 @@ _SUMSTATS_COL_MAP = {
|
|
|
18
22
|
"effectallele": "a1",
|
|
19
23
|
"otherallele": "a2",
|
|
20
24
|
}
|
|
25
|
+
_NUMERIC_NA_VALUES = ["", "NaN", "nan", "NA", "N/A", "na", "NULL", "null"]
|
|
21
26
|
|
|
22
27
|
|
|
23
|
-
def
|
|
24
|
-
columns = [_SUMSTATS_COL_MAP.get(str(c).lower(), str(c).lower()) for c in
|
|
28
|
+
def _canonicalize_column_names(columns, path: Path) -> list[str]:
|
|
29
|
+
columns = [_SUMSTATS_COL_MAP.get(str(c).lower(), str(c).lower()) for c in columns]
|
|
25
30
|
duplicates = sorted({c for c in columns if columns.count(c) > 1})
|
|
26
31
|
if duplicates:
|
|
27
32
|
raise ValueError(f"{path}: duplicate columns after column normalization: {', '.join(duplicates)}")
|
|
33
|
+
return columns
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _canonicalize_columns(df: pd.DataFrame, path: Path) -> pd.DataFrame:
|
|
28
37
|
out = df.copy()
|
|
29
|
-
out.columns = columns
|
|
38
|
+
out.columns = _canonicalize_column_names(df.columns, path)
|
|
30
39
|
return out
|
|
31
40
|
|
|
32
41
|
|
|
42
|
+
def _read_sumstats_header(path: Path) -> list[str]:
|
|
43
|
+
opener = gzip.open if str(path).lower().endswith(".gz") else open
|
|
44
|
+
try:
|
|
45
|
+
with opener(path, "rt", newline="") as f:
|
|
46
|
+
header = f.readline()
|
|
47
|
+
except OSError as exc:
|
|
48
|
+
raise ValueError(f"{path}: malformed TSV") from exc
|
|
49
|
+
if header == "":
|
|
50
|
+
raise ValueError(f"{path}: malformed TSV")
|
|
51
|
+
return header.rstrip("\r\n").split("\t")
|
|
52
|
+
|
|
53
|
+
|
|
33
54
|
def _parse_sumstats(path: Path) -> pd.DataFrame:
|
|
55
|
+
raw_columns = _read_sumstats_header(path)
|
|
56
|
+
columns = _canonicalize_column_names(raw_columns, path)
|
|
57
|
+
missing = [c for c in _REQUIRED_COLS if c not in columns]
|
|
58
|
+
if missing:
|
|
59
|
+
raise ValueError(f"{path}: missing required columns: {', '.join(missing)}")
|
|
60
|
+
|
|
61
|
+
selected = [c in (*_REQUIRED_COLS, *_OPTIONAL_COLS) for c in columns]
|
|
62
|
+
selected_raw = [c for c, keep in zip(raw_columns, selected) if keep]
|
|
63
|
+
selected_names = [c for c, keep in zip(columns, selected) if keep]
|
|
64
|
+
char_cols = {
|
|
65
|
+
raw: str
|
|
66
|
+
for raw, canonical in zip(selected_raw, selected_names)
|
|
67
|
+
if canonical in {"chr", "a1", "a2"}
|
|
68
|
+
}
|
|
69
|
+
numeric_cols = {
|
|
70
|
+
raw: float
|
|
71
|
+
for raw, canonical in zip(selected_raw, selected_names)
|
|
72
|
+
if canonical not in {"chr", "a1", "a2"}
|
|
73
|
+
}
|
|
74
|
+
numeric_na_values = {raw: _NUMERIC_NA_VALUES for raw in numeric_cols}
|
|
75
|
+
|
|
34
76
|
try:
|
|
35
77
|
df = pd.read_csv(
|
|
36
78
|
path,
|
|
37
79
|
sep="\t",
|
|
38
80
|
compression="infer",
|
|
39
|
-
|
|
81
|
+
usecols=selected_raw,
|
|
82
|
+
dtype={**char_cols, **numeric_cols},
|
|
40
83
|
keep_default_na=False,
|
|
41
|
-
|
|
84
|
+
na_values=numeric_na_values,
|
|
42
85
|
)
|
|
43
|
-
except ParserError as exc:
|
|
86
|
+
except (EmptyDataError, ParserError, ValueError) as exc:
|
|
44
87
|
raise ValueError(f"{path}: malformed TSV") from exc
|
|
45
88
|
|
|
46
|
-
df =
|
|
89
|
+
df.columns = selected_names
|
|
47
90
|
|
|
48
91
|
missing = [c for c in _REQUIRED_COLS if c not in df.columns]
|
|
49
92
|
if missing:
|
|
50
93
|
raise ValueError(f"{path}: missing required columns: {', '.join(missing)}")
|
|
51
94
|
|
|
52
|
-
out = df
|
|
53
|
-
bp_num = pd.to_numeric(out["bp"], errors="coerce")
|
|
54
|
-
bad_bp = bp_num.isna() | (np.floor(bp_num) != bp_num)
|
|
55
|
-
if bad_bp.any():
|
|
56
|
-
idx = int(bad_bp.idxmax())
|
|
57
|
-
raise ValueError(f"{path}: row {idx + 2}: bp is not an integer: {out['bp'].iat[idx]!r}")
|
|
58
|
-
out["bp"] = bp_num.astype(np.int64)
|
|
59
|
-
|
|
95
|
+
out = df
|
|
60
96
|
for col in ("chr", "a1", "a2"):
|
|
61
97
|
bad_empty = out[col].eq("")
|
|
62
98
|
if bad_empty.any():
|
|
63
99
|
idx = int(bad_empty.idxmax())
|
|
64
100
|
raise ValueError(f"{path}: row {idx + 2}: {col} must be non-empty")
|
|
65
101
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
102
|
+
chr_col = out["chr"]
|
|
103
|
+
chr_style = chr_col.str.lower().str.startswith("chr")
|
|
104
|
+
if chr_style.any():
|
|
105
|
+
idx = int(chr_style.idxmax())
|
|
106
|
+
raise ValueError(f"{path}: row {idx + 2}: chr-style labels (e.g., chr1/chrX) are not allowed")
|
|
107
|
+
known_chr = chr_col.isin(CANONICAL_CHR_ORDER) | chr_col.isin(_IGNORED_CHR)
|
|
108
|
+
if not known_chr.all():
|
|
109
|
+
idx = int((~known_chr).idxmax())
|
|
110
|
+
raise ValueError(
|
|
111
|
+
f"{path}: row {idx + 2}: unsupported chr label {chr_col.iat[idx]!r}; expected 1-22, X (Y/MT are ignored)"
|
|
112
|
+
)
|
|
113
|
+
out = out.loc[chr_col.isin(CANONICAL_CHR_ORDER)].copy()
|
|
114
|
+
|
|
115
|
+
bad_a1_syntax = ~out["a1"].str.fullmatch(_DNA_ALLELE_RE.pattern)
|
|
116
|
+
if bad_a1_syntax.any():
|
|
117
|
+
idx = int(bad_a1_syntax.idxmax())
|
|
118
|
+
raise ValueError(
|
|
119
|
+
f"{path}: row {idx + 2}: a1 must be uppercase DNA bases (A/C/G/T): {out.loc[idx, 'a1']!r}"
|
|
120
|
+
)
|
|
121
|
+
bad_a2_syntax = ~out["a2"].str.fullmatch(_DNA_ALLELE_RE.pattern)
|
|
122
|
+
if bad_a2_syntax.any():
|
|
123
|
+
idx = int(bad_a2_syntax.idxmax())
|
|
124
|
+
raise ValueError(
|
|
125
|
+
f"{path}: row {idx + 2}: a2 must be uppercase DNA bases (A/C/G/T): {out.loc[idx, 'a2']!r}"
|
|
126
|
+
)
|
|
127
|
+
same_allele = out["a1"].eq(out["a2"])
|
|
128
|
+
if same_allele.any():
|
|
129
|
+
idx = int(same_allele.idxmax())
|
|
130
|
+
raise ValueError(f"{path}: row {idx + 2}: a1 and a2 must differ")
|
|
131
|
+
|
|
132
|
+
bad_bp = out["bp"].isna() | (np.floor(out["bp"]) != out["bp"])
|
|
133
|
+
if bad_bp.any():
|
|
134
|
+
idx = int(bad_bp.idxmax())
|
|
135
|
+
raise ValueError(f"{path}: row {idx + 2}: bp is not an integer: {out.loc[idx, 'bp']!r}")
|
|
136
|
+
out["bp"] = out["bp"].astype(np.int64)
|
|
137
|
+
|
|
138
|
+
p_arr = out["p"].to_numpy(dtype=float)
|
|
139
|
+
bad_p = out["p"].isna() | ~np.isfinite(p_arr) | (p_arr < 0.0) | (p_arr > 1.0)
|
|
69
140
|
if bad_p.any():
|
|
70
141
|
idx = int(bad_p.idxmax())
|
|
71
|
-
raise ValueError(f"{path}: row {idx + 2}: p must be finite numeric in [0, 1]: {out['p']
|
|
72
|
-
out["p"] = p_vals.astype(float)
|
|
142
|
+
raise ValueError(f"{path}: row {idx + 2}: p must be finite numeric in [0, 1]: {out.loc[idx, 'p']!r}")
|
|
73
143
|
|
|
74
144
|
for opt in _OPTIONAL_COLS:
|
|
75
145
|
if opt in out.columns:
|
|
76
|
-
|
|
77
|
-
vals = vals.mask(~np.isfinite(vals.to_numpy(dtype=float)))
|
|
78
|
-
out[opt] = vals
|
|
146
|
+
out[opt] = out[opt].mask(~np.isfinite(out[opt].to_numpy(dtype=float)))
|
|
79
147
|
|
|
80
148
|
return out
|
|
81
149
|
|
|
@@ -252,8 +320,8 @@ class Sumstats:
|
|
|
252
320
|
by_label = {s.label: s for s in self._shards}
|
|
253
321
|
return Sumstats([by_label[label] for label in selected])
|
|
254
322
|
|
|
255
|
-
def save_cache(self, path
|
|
256
|
-
save_sumstats_cache(self, path
|
|
323
|
+
def save_cache(self, path) -> None:
|
|
324
|
+
save_sumstats_cache(self, path)
|
|
257
325
|
|
|
258
326
|
|
|
259
327
|
def _derive_logp(p_vals: np.ndarray) -> np.ndarray:
|
|
@@ -393,26 +461,26 @@ def load_sumstats(path, reference) -> Sumstats:
|
|
|
393
461
|
|
|
394
462
|
def create_sumstats(
|
|
395
463
|
reference,
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
464
|
+
p,
|
|
465
|
+
z=None,
|
|
466
|
+
n=None,
|
|
467
|
+
beta=None,
|
|
468
|
+
se=None,
|
|
469
|
+
eaf=None,
|
|
470
|
+
info=None,
|
|
403
471
|
) -> Sumstats:
|
|
404
|
-
|
|
405
|
-
aligned_p = _coerce_aligned_vec("
|
|
406
|
-
_validate_pvec(aligned_p, "
|
|
472
|
+
num_snp_value = int(reference.num_snp)
|
|
473
|
+
aligned_p = _coerce_aligned_vec("p", p, n=num_snp_value, allow_inf=True)
|
|
474
|
+
_validate_pvec(aligned_p, "p")
|
|
407
475
|
aligned_logp = _derive_logp(aligned_p)
|
|
408
476
|
|
|
409
477
|
aligned_optional = {
|
|
410
|
-
"z": None if
|
|
411
|
-
"n": None if
|
|
412
|
-
"beta": None if
|
|
413
|
-
"se": None if
|
|
414
|
-
"eaf": None if
|
|
415
|
-
"info": None if
|
|
478
|
+
"z": None if z is None else _coerce_aligned_vec("z", z, n=num_snp_value, allow_inf=False),
|
|
479
|
+
"n": None if n is None else _coerce_aligned_vec("n", n, n=num_snp_value, allow_inf=False),
|
|
480
|
+
"beta": None if beta is None else _coerce_aligned_vec("beta", beta, n=num_snp_value, allow_inf=False),
|
|
481
|
+
"se": None if se is None else _coerce_aligned_vec("se", se, n=num_snp_value, allow_inf=False),
|
|
482
|
+
"eaf": None if eaf is None else _coerce_aligned_vec("eaf", eaf, n=num_snp_value, allow_inf=False),
|
|
483
|
+
"info": None if info is None else _coerce_aligned_vec("info", info, n=num_snp_value, allow_inf=False),
|
|
416
484
|
}
|
|
417
485
|
|
|
418
486
|
return _build_sumstats_from_aligned(
|
|
@@ -422,7 +490,7 @@ def create_sumstats(
|
|
|
422
490
|
)
|
|
423
491
|
|
|
424
492
|
|
|
425
|
-
def save_sumstats_cache(sumstats: Sumstats, path
|
|
493
|
+
def save_sumstats_cache(sumstats: Sumstats, path) -> None:
|
|
426
494
|
meta = {
|
|
427
495
|
"schema": _CACHE_SCHEMA,
|
|
428
496
|
"shard_labels": [s.label for s in sumstats.shards],
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|