statgen 0.3.3__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {statgen-0.3.3 → statgen-0.4.0}/PKG-INFO +1 -1
- {statgen-0.3.3 → statgen-0.4.0}/pyproject.toml +1 -1
- {statgen-0.3.3 → statgen-0.4.0}/statgen/__init__.py +1 -1
- {statgen-0.3.3 → statgen-0.4.0}/statgen/_bfile_utils.py +3 -1
- {statgen-0.3.3 → statgen-0.4.0}/statgen/_ld_schema.py +2 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen/_utils.py +28 -0
- statgen-0.4.0/statgen/_variant_match.py +105 -0
- statgen-0.4.0/statgen/_verbosity.py +30 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen/annotations.py +146 -55
- {statgen-0.3.3 → statgen-0.4.0}/statgen/genotype.py +53 -17
- {statgen-0.3.3 → statgen-0.4.0}/statgen/reference.py +24 -22
- {statgen-0.3.3 → statgen-0.4.0}/statgen/sumstats.py +34 -20
- {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/PKG-INFO +1 -1
- statgen-0.3.3/statgen/_variant_match.py +0 -76
- statgen-0.3.3/statgen/_verbosity.py +0 -20
- {statgen-0.3.3 → statgen-0.4.0}/README.md +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/setup.cfg +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen/_annotation_painting.py +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen/_ld_npz.py +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen/_ld_reference.py +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen/_ld_writer.py +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen/ld.py +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/SOURCES.txt +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/dependency_links.txt +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/requires.txt +0 -0
- {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/top_level.txt +0 -0
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import re
|
|
2
4
|
import warnings
|
|
3
5
|
from pathlib import Path
|
|
@@ -275,7 +277,7 @@ def _read_bed_rows_int8(path: Path, source_rows: np.ndarray, source_num_sample:
|
|
|
275
277
|
# PERF: one seek/read per contiguous BED row block; requested order and
|
|
276
278
|
# repeats are restored after block decoding. Blocks are capped to
|
|
277
279
|
# bound transient packed/decode allocations for large requests.
|
|
278
|
-
for start, stop in zip(group_starts, group_stops
|
|
280
|
+
for start, stop in zip(group_starts, group_stops):
|
|
279
281
|
for chunk_start in range(start, stop, rows_per_read):
|
|
280
282
|
chunk_stop = min(chunk_start + rows_per_read, stop)
|
|
281
283
|
block_rows = unique_rows[chunk_start:chunk_stop]
|
|
@@ -114,3 +114,31 @@ def _allele_hash_lanes_from_utf8(encoded) -> tuple[np.ndarray, np.ndarray]:
|
|
|
114
114
|
h2[active] = (h2[active] * _ALLELE_HASH_BASE2 + x) % _ALLELE_HASH_P
|
|
115
115
|
|
|
116
116
|
return h1, h2
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
_A_HASH64, _C_HASH64, _G_HASH64, _T_HASH64 = _SINGLE_BASE_HASH64
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _is_single_base(h) -> np.ndarray:
|
|
123
|
+
return np.isin(h, _SINGLE_BASE_HASH64)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _is_single_nucleotide_variant(a1_hash64, a2_hash64) -> np.ndarray:
|
|
127
|
+
return _is_single_base(a1_hash64) & _is_single_base(a2_hash64)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _is_strand_ambiguous(a1_hash64, a2_hash64) -> np.ndarray:
|
|
131
|
+
a1 = np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
|
|
132
|
+
a2 = np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
|
|
133
|
+
return (
|
|
134
|
+
((a1 == _A_HASH64) & (a2 == _T_HASH64))
|
|
135
|
+
| ((a1 == _T_HASH64) & (a2 == _A_HASH64))
|
|
136
|
+
| ((a1 == _C_HASH64) & (a2 == _G_HASH64))
|
|
137
|
+
| ((a1 == _G_HASH64) & (a2 == _C_HASH64))
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _validate_allow_indel_swaps(value) -> bool:
|
|
142
|
+
if not isinstance(value, (bool, np.bool_)):
|
|
143
|
+
raise ValueError("allow_indel_swaps must be a boolean scalar")
|
|
144
|
+
return bool(value)
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from ._utils import _is_single_base, _is_single_nucleotide_variant, _is_strand_ambiguous
|
|
6
|
+
|
|
7
|
+
_KEY_DTYPE = np.dtype([("bp", "<i8"), ("a1_hash64", "<u8"), ("a2_hash64", "<u8")])
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def numeric_variant_keys(bp_arr, a1_hash64, a2_hash64) -> np.ndarray:
|
|
11
|
+
bp = np.asarray(bp_arr, dtype=np.int64).reshape(-1)
|
|
12
|
+
a1h = np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
|
|
13
|
+
a2h = np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
|
|
14
|
+
if not (bp.size == a1h.size == a2h.size):
|
|
15
|
+
raise ValueError("numeric variant key vector lengths mismatch")
|
|
16
|
+
keys = np.empty(bp.size, dtype=_KEY_DTYPE)
|
|
17
|
+
keys["bp"] = bp
|
|
18
|
+
keys["a1_hash64"] = a1h
|
|
19
|
+
keys["a2_hash64"] = a2h
|
|
20
|
+
return keys
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def check_unique_sorted_variant_keys(sorted_keys: np.ndarray, where: str, object_name: str = "source") -> None:
|
|
24
|
+
if sorted_keys.size < 2:
|
|
25
|
+
return
|
|
26
|
+
dup = sorted_keys[1:] == sorted_keys[:-1]
|
|
27
|
+
if np.any(dup):
|
|
28
|
+
pos = int(np.flatnonzero(dup)[0] + 1)
|
|
29
|
+
key = sorted_keys[pos]
|
|
30
|
+
raise ValueError(
|
|
31
|
+
f"Ambiguous duplicate {object_name}/reference matching key in {where}: "
|
|
32
|
+
f"bp={int(key['bp'])}, a1_hash64={int(key['a1_hash64'])}, "
|
|
33
|
+
f"a2_hash64={int(key['a2_hash64'])}"
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class VariantMatch:
|
|
39
|
+
source_rows: np.ndarray
|
|
40
|
+
is_allele_swapped: np.ndarray
|
|
41
|
+
num_strand_ambiguous: int
|
|
42
|
+
num_disabled_indels: int
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _lookup_candidates(ref_keys, src_keys, source_rows, shard_label, source_name, candidate_eligible=None):
|
|
46
|
+
"""Return unique eligible source rows and count hits from ineligible rows."""
|
|
47
|
+
order = np.argsort(src_keys, kind="mergesort", order=("bp", "a1_hash64", "a2_hash64"))
|
|
48
|
+
sorted_keys = src_keys[order]
|
|
49
|
+
left = np.searchsorted(sorted_keys, ref_keys, side="left")
|
|
50
|
+
right = np.searchsorted(sorted_keys, ref_keys, side="right")
|
|
51
|
+
present = right > left
|
|
52
|
+
accepted = present.copy()
|
|
53
|
+
if candidate_eligible is not None:
|
|
54
|
+
# A key fixes both allele hashes, so all candidates for a slot have
|
|
55
|
+
# the same allele class and eligibility, even when they are duplicates.
|
|
56
|
+
accepted[present] = candidate_eligible[order[left[present]]]
|
|
57
|
+
excluded_hits = int(np.sum((right - left)[present & ~accepted]))
|
|
58
|
+
ambiguous = (right - left > 1) & accepted
|
|
59
|
+
if np.any(ambiguous):
|
|
60
|
+
key = ref_keys[np.flatnonzero(ambiguous)[0]]
|
|
61
|
+
raise ValueError(
|
|
62
|
+
f"Ambiguous duplicate {source_name}/reference matching key in shard {shard_label}: "
|
|
63
|
+
f"bp={int(key['bp'])}, a1_hash64={int(key['a1_hash64'])}, "
|
|
64
|
+
f"a2_hash64={int(key['a2_hash64'])}"
|
|
65
|
+
)
|
|
66
|
+
out = np.full(ref_keys.size, -1, dtype=np.int64)
|
|
67
|
+
out[accepted] = source_rows[order[left[accepted]]]
|
|
68
|
+
return out, excluded_hits
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def match_shard_numeric(
|
|
72
|
+
ref_keys: np.ndarray,
|
|
73
|
+
src_keys: np.ndarray,
|
|
74
|
+
shard_label: str,
|
|
75
|
+
*,
|
|
76
|
+
source_name: str = "source",
|
|
77
|
+
allow_indel_swaps: bool = False,
|
|
78
|
+
) -> VariantMatch:
|
|
79
|
+
# Reference validation belongs to load_reference; cached references are trusted.
|
|
80
|
+
rows, _ = _lookup_candidates(
|
|
81
|
+
ref_keys, src_keys, np.arange(src_keys.size), shard_label, source_name
|
|
82
|
+
)
|
|
83
|
+
swapped = np.zeros(ref_keys.size, dtype=bool)
|
|
84
|
+
unused = np.ones(src_keys.size, dtype=bool)
|
|
85
|
+
unused[rows[rows >= 0]] = False
|
|
86
|
+
a1h = src_keys["a1_hash64"]
|
|
87
|
+
a2h = src_keys["a2_hash64"]
|
|
88
|
+
snv = _is_single_nucleotide_variant(a1h, a2h)
|
|
89
|
+
indel = _is_single_base(a1h) ^ _is_single_base(a2h)
|
|
90
|
+
eligible = snv | (indel & allow_indel_swaps)
|
|
91
|
+
source_rows = np.flatnonzero(unused & (snv | indel))
|
|
92
|
+
reversed_keys = numeric_variant_keys(
|
|
93
|
+
src_keys["bp"][source_rows], a2h[source_rows], a1h[source_rows]
|
|
94
|
+
)
|
|
95
|
+
missing = np.flatnonzero(rows < 0)
|
|
96
|
+
fallback, disabled = _lookup_candidates(
|
|
97
|
+
ref_keys[missing], reversed_keys, source_rows, shard_label, source_name,
|
|
98
|
+
candidate_eligible=eligible[source_rows],
|
|
99
|
+
)
|
|
100
|
+
accepted = missing[fallback >= 0]
|
|
101
|
+
rows[accepted] = fallback[fallback >= 0]
|
|
102
|
+
swapped[accepted] = True
|
|
103
|
+
selected = rows[swapped]
|
|
104
|
+
ambiguous = _is_strand_ambiguous(a1h[selected], a2h[selected])
|
|
105
|
+
return VariantMatch(rows, swapped, int(np.count_nonzero(ambiguous)), disabled)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
_LEVELS = ("quiet", "info")
|
|
4
|
+
_VERBOSITY = "info"
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def set_verbosity(level: str) -> None:
|
|
8
|
+
global _VERBOSITY
|
|
9
|
+
if level not in _LEVELS:
|
|
10
|
+
raise ValueError("verbosity must be one of: quiet, info")
|
|
11
|
+
_VERBOSITY = level
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_verbosity() -> str:
|
|
15
|
+
return _VERBOSITY
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def info(message: str) -> None:
|
|
19
|
+
if _VERBOSITY == "info":
|
|
20
|
+
print(message, file=sys.stderr)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def report_variant_swaps(source_path, shard_label, num_swaps, num_ambiguous, num_disabled):
|
|
24
|
+
context = f"{source_path}: shard {shard_label}"
|
|
25
|
+
if num_swaps:
|
|
26
|
+
info(f"{context}: {num_swaps} successful allele swap(s)")
|
|
27
|
+
if num_ambiguous:
|
|
28
|
+
info(f"{context}: {num_ambiguous} strand-ambiguous A/T or C/G swap(s)")
|
|
29
|
+
if num_disabled:
|
|
30
|
+
info(f"{context}: {num_disabled} indel row(s) unmatched because indel swaps are disabled")
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import json
|
|
2
4
|
from pathlib import Path
|
|
3
5
|
|
|
@@ -137,16 +139,27 @@ def _annotation_name_from_path(path: Path) -> str:
|
|
|
137
139
|
return name
|
|
138
140
|
|
|
139
141
|
|
|
140
|
-
def _generated_metadata(
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
142
|
+
def _generated_metadata(
|
|
143
|
+
path: Path,
|
|
144
|
+
has_header: bool,
|
|
145
|
+
num_source_intervals: int,
|
|
146
|
+
*,
|
|
147
|
+
value_column=None,
|
|
148
|
+
group_column=None,
|
|
149
|
+
group_value=None,
|
|
150
|
+
) -> str:
|
|
151
|
+
payload = {
|
|
152
|
+
"source_file": str(path),
|
|
153
|
+
"source_file_has_header": 1 if has_header else 0,
|
|
154
|
+
"num_source_intervals": int(num_source_intervals),
|
|
155
|
+
}
|
|
156
|
+
if value_column is not None:
|
|
157
|
+
payload["value_column"] = value_column
|
|
158
|
+
if group_column is not None:
|
|
159
|
+
payload["group_column"] = group_column
|
|
160
|
+
if group_value is not None:
|
|
161
|
+
payload["group_value"] = group_value
|
|
162
|
+
return json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
|
150
163
|
|
|
151
164
|
|
|
152
165
|
def _read_sidecar_exact(path) -> str:
|
|
@@ -167,7 +180,7 @@ def _read_column_metadata_sidecar(path, num_columns: int) -> list[str]:
|
|
|
167
180
|
return lines
|
|
168
181
|
|
|
169
182
|
|
|
170
|
-
def _read_annotation_table(path: Path,
|
|
183
|
+
def _read_annotation_table(path: Path, has_header: bool) -> tuple[pd.DataFrame, list[str] | None, int]:
|
|
171
184
|
if not path.is_file():
|
|
172
185
|
raise FileNotFoundError(f"annotation file not found: {path}")
|
|
173
186
|
|
|
@@ -175,7 +188,7 @@ def _read_annotation_table(path: Path, header: bool) -> tuple[pd.DataFrame, list
|
|
|
175
188
|
df = pd.read_csv(
|
|
176
189
|
path,
|
|
177
190
|
sep="\t",
|
|
178
|
-
header=0 if
|
|
191
|
+
header=0 if has_header else None,
|
|
179
192
|
dtype=str,
|
|
180
193
|
keep_default_na=False,
|
|
181
194
|
na_filter=False,
|
|
@@ -192,14 +205,14 @@ def _read_annotation_table(path: Path, header: bool) -> tuple[pd.DataFrame, list
|
|
|
192
205
|
|
|
193
206
|
if df.shape[1] < 3:
|
|
194
207
|
raise ValueError(f"{path}: BED must have at least 3 tab-separated columns")
|
|
195
|
-
header_fields = [str(x) for x in df.columns] if
|
|
208
|
+
header_fields = [str(x) for x in df.columns] if has_header else None
|
|
196
209
|
if header_fields is not None:
|
|
197
210
|
if any(x == "" for x in header_fields):
|
|
198
211
|
raise ValueError(f"{path}: header names must be non-empty")
|
|
199
212
|
if len(set(header_fields)) != len(header_fields):
|
|
200
213
|
raise ValueError(f"{path}: header names must be unique")
|
|
201
214
|
|
|
202
|
-
return df, header_fields, 1 if
|
|
215
|
+
return df, header_fields, 1 if has_header else 0
|
|
203
216
|
|
|
204
217
|
|
|
205
218
|
def _validate_interval_columns(
|
|
@@ -251,15 +264,17 @@ def _validate_interval_columns(
|
|
|
251
264
|
return chr_values, start_int, end_int
|
|
252
265
|
|
|
253
266
|
|
|
254
|
-
def _binary_intervals_by_chr(path: Path) -> dict[str, np.ndarray]:
|
|
255
|
-
df, _, row_base0 = _read_annotation_table(path,
|
|
267
|
+
def _binary_intervals_by_chr(path: Path) -> tuple[dict[str, np.ndarray], int]:
|
|
268
|
+
df, _, row_base0 = _read_annotation_table(path, has_header=False)
|
|
256
269
|
if df.shape[1] < 3:
|
|
257
270
|
raise ValueError(f"{path}: BED must have at least 3 tab-separated columns")
|
|
258
271
|
chr_values, starts, ends = _validate_interval_columns(df, path, row_base0=row_base0)
|
|
259
|
-
return binary_intervals_by_chr_from_arrays(chr_values, starts, ends)
|
|
272
|
+
return binary_intervals_by_chr_from_arrays(chr_values, starts, ends), int(df.shape[0])
|
|
260
273
|
|
|
261
274
|
|
|
262
|
-
def _paint_binary_annotations(
|
|
275
|
+
def _paint_binary_annotations(
|
|
276
|
+
bed_paths: list[Path], reference
|
|
277
|
+
) -> tuple[sparse.csr_matrix, list[str], list[int]]:
|
|
263
278
|
annonames = [_annotation_name_from_path(p) for p in bed_paths]
|
|
264
279
|
if len(set(annonames)) != len(annonames):
|
|
265
280
|
raise ValueError("duplicate annotation names derived from BED basenames")
|
|
@@ -267,47 +282,79 @@ def _paint_binary_annotations(bed_paths: list[Path], reference) -> tuple[sparse.
|
|
|
267
282
|
n = int(reference.num_snp)
|
|
268
283
|
k = len(bed_paths)
|
|
269
284
|
if n == 0:
|
|
270
|
-
|
|
285
|
+
counts = []
|
|
286
|
+
for bed_path in bed_paths:
|
|
287
|
+
_, count = _binary_intervals_by_chr(bed_path)
|
|
288
|
+
counts.append(count)
|
|
289
|
+
return sparse.csr_matrix((0, k), dtype=np.float64), annonames, counts
|
|
271
290
|
|
|
272
291
|
columns = []
|
|
292
|
+
counts = []
|
|
273
293
|
for bed_path in bed_paths:
|
|
274
|
-
intervals_by_chr = _binary_intervals_by_chr(bed_path)
|
|
294
|
+
intervals_by_chr, count = _binary_intervals_by_chr(bed_path)
|
|
275
295
|
columns.append(paint_binary_column(intervals_by_chr, reference))
|
|
296
|
+
counts.append(count)
|
|
276
297
|
|
|
277
|
-
return sparse.hstack(columns, format="csr"), annonames
|
|
298
|
+
return sparse.hstack(columns, format="csr"), annonames, counts
|
|
278
299
|
|
|
279
300
|
|
|
280
|
-
def
|
|
281
|
-
if
|
|
301
|
+
def _selector_list(selectors, name: str) -> list | None:
|
|
302
|
+
if selectors is None:
|
|
282
303
|
return None
|
|
283
|
-
if isinstance(
|
|
284
|
-
|
|
304
|
+
if isinstance(selectors, (str, int, np.integer)):
|
|
305
|
+
values = [selectors]
|
|
285
306
|
else:
|
|
286
|
-
|
|
287
|
-
if not
|
|
288
|
-
raise ValueError("
|
|
307
|
+
values = list(selectors)
|
|
308
|
+
if not values:
|
|
309
|
+
raise ValueError(f"{name} must be a non-empty list")
|
|
310
|
+
return values
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _normalize_column_selectors(selectors, header_fields, num_columns: int, path: Path, name: str) -> tuple[list[int], list] | None:
|
|
314
|
+
selector_values = _selector_list(selectors, name)
|
|
315
|
+
if selector_values is None:
|
|
316
|
+
return None
|
|
289
317
|
|
|
290
318
|
out = []
|
|
291
|
-
|
|
319
|
+
metadata_values = []
|
|
320
|
+
singular = name[:-1] if name.endswith("s") else name
|
|
321
|
+
for selector in selector_values:
|
|
292
322
|
if isinstance(selector, str):
|
|
293
323
|
if header_fields is None:
|
|
294
|
-
raise ValueError("named
|
|
324
|
+
raise ValueError(f"named {name} are invalid when has_header=False")
|
|
295
325
|
if selector not in header_fields:
|
|
296
|
-
raise ValueError(f"{path}: unknown
|
|
326
|
+
raise ValueError(f"{path}: unknown {singular} name {selector!r}")
|
|
297
327
|
col0 = header_fields.index(selector)
|
|
328
|
+
metadata_value = selector
|
|
298
329
|
elif isinstance(selector, (int, np.integer)) and not isinstance(selector, bool):
|
|
299
330
|
col0 = int(selector)
|
|
331
|
+
metadata_value = int(selector)
|
|
300
332
|
else:
|
|
301
|
-
raise ValueError("
|
|
333
|
+
raise ValueError(f"{name} must contain column names or integer indices")
|
|
302
334
|
if col0 < 3 or col0 >= num_columns:
|
|
303
335
|
raise ValueError(
|
|
304
|
-
f"{path}:
|
|
336
|
+
f"{path}: {singular} {selector!r} is out of range; selected columns must be physical columns 4 or later"
|
|
305
337
|
)
|
|
306
338
|
out.append(col0)
|
|
339
|
+
metadata_values.append(metadata_value)
|
|
307
340
|
|
|
308
341
|
if len(set(out)) != len(out):
|
|
309
|
-
raise ValueError("
|
|
310
|
-
return out
|
|
342
|
+
raise ValueError(f"{name} must not contain duplicates")
|
|
343
|
+
return out, metadata_values
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _normalize_value_columns(value_columns, header_fields, num_columns: int, path: Path) -> tuple[list[int], list] | None:
|
|
347
|
+
return _normalize_column_selectors(value_columns, header_fields, num_columns, path, "value_columns")
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _normalize_group_column(group_column, header_fields, num_columns: int, path: Path) -> tuple[int, object] | None:
|
|
351
|
+
normalized = _normalize_column_selectors(group_column, header_fields, num_columns, path, "group_column")
|
|
352
|
+
if normalized is None:
|
|
353
|
+
return None
|
|
354
|
+
columns, metadata_values = normalized
|
|
355
|
+
if len(columns) != 1:
|
|
356
|
+
raise ValueError("group_column must identify exactly one source column")
|
|
357
|
+
return columns[0], metadata_values[0]
|
|
311
358
|
|
|
312
359
|
|
|
313
360
|
def _default_annotation_names(
|
|
@@ -323,7 +370,7 @@ def _default_annotation_names(
|
|
|
323
370
|
if header_fields is None:
|
|
324
371
|
return [_annotation_name_from_path(path)]
|
|
325
372
|
return [str(header_fields[3])]
|
|
326
|
-
raise ValueError(f"{path}: input with five or more columns requires explicit value_columns")
|
|
373
|
+
raise ValueError(f"{path}: input with five or more columns requires explicit value_columns or group_column")
|
|
327
374
|
|
|
328
375
|
if header_fields is None and num_columns >= 5:
|
|
329
376
|
raise ValueError(f"{path}: headerless input with five or more columns requires annotation_names")
|
|
@@ -613,7 +660,7 @@ def load_annotations(
|
|
|
613
660
|
if not p.is_file():
|
|
614
661
|
raise FileNotFoundError(f"BED file not found: {p}")
|
|
615
662
|
|
|
616
|
-
annomat, annonames = _paint_binary_annotations(paths, reference)
|
|
663
|
+
annomat, annonames, source_counts = _paint_binary_annotations(paths, reference)
|
|
617
664
|
if annotation_metadata is not None:
|
|
618
665
|
metadata = _coerce_string_vector(annotation_metadata, len(paths), "annotation_metadata")
|
|
619
666
|
elif annotation_metadata_paths is not None:
|
|
@@ -623,14 +670,17 @@ def load_annotations(
|
|
|
623
670
|
f"annotation_metadata_paths length mismatch: expected {len(paths)}, got {len(meta_paths)}"
|
|
624
671
|
)
|
|
625
672
|
values = []
|
|
626
|
-
for bed_path, meta_path in zip(paths, meta_paths):
|
|
673
|
+
for bed_path, meta_path, count in zip(paths, meta_paths, source_counts):
|
|
627
674
|
if meta_path is None or str(meta_path) == "":
|
|
628
|
-
values.append(_generated_metadata(bed_path,
|
|
675
|
+
values.append(_generated_metadata(bed_path, False, count))
|
|
629
676
|
else:
|
|
630
677
|
values.append(_read_sidecar_exact(meta_path))
|
|
631
678
|
metadata = np.asarray(values, dtype=object)
|
|
632
679
|
else:
|
|
633
|
-
metadata = np.asarray(
|
|
680
|
+
metadata = np.asarray(
|
|
681
|
+
[_generated_metadata(p, False, count) for p, count in zip(paths, source_counts)],
|
|
682
|
+
dtype=object,
|
|
683
|
+
)
|
|
634
684
|
|
|
635
685
|
return create_annotations(
|
|
636
686
|
reference,
|
|
@@ -644,26 +694,67 @@ def load_annotations(
|
|
|
644
694
|
def load_annotation(
|
|
645
695
|
path,
|
|
646
696
|
reference,
|
|
647
|
-
|
|
697
|
+
has_header: bool = False,
|
|
648
698
|
value_columns=None,
|
|
699
|
+
group_column=None,
|
|
649
700
|
annotation_names=None,
|
|
650
701
|
annotation_metadata=None,
|
|
651
702
|
annotation_metadata_path=None,
|
|
652
703
|
) -> AnnotationPanel:
|
|
653
704
|
if annotation_metadata is not None and annotation_metadata_path is not None:
|
|
654
705
|
raise ValueError("load_annotation accepts at most one of annotation_metadata and annotation_metadata_path")
|
|
706
|
+
if value_columns is not None and group_column is not None:
|
|
707
|
+
raise ValueError("group_column and value_columns are mutually exclusive")
|
|
708
|
+
if group_column is not None and annotation_names is not None:
|
|
709
|
+
raise ValueError("annotation_names is invalid with group_column")
|
|
710
|
+
if group_column is not None and annotation_metadata is not None:
|
|
711
|
+
raise ValueError("annotation_metadata is invalid with group_column")
|
|
712
|
+
if group_column is not None and annotation_metadata_path is not None:
|
|
713
|
+
raise ValueError("annotation_metadata_path is invalid with group_column")
|
|
655
714
|
|
|
656
715
|
source_path = Path(path)
|
|
657
|
-
df, header_fields, row_base0 = _read_annotation_table(source_path, bool(
|
|
716
|
+
df, header_fields, row_base0 = _read_annotation_table(source_path, bool(has_header))
|
|
658
717
|
num_columns = int(df.shape[1])
|
|
659
718
|
if num_columns < 3:
|
|
660
719
|
raise ValueError(f"{source_path}: annotation input must have at least 3 tab-separated columns")
|
|
661
720
|
|
|
662
|
-
|
|
721
|
+
value_normalized = _normalize_value_columns(value_columns, header_fields, num_columns, source_path)
|
|
722
|
+
value_columns0 = None if value_normalized is None else value_normalized[0]
|
|
723
|
+
value_column_metadata = None if value_normalized is None else value_normalized[1]
|
|
724
|
+
group_normalized = _normalize_group_column(group_column, header_fields, num_columns, source_path)
|
|
725
|
+
group_column0 = None if group_normalized is None else group_normalized[0]
|
|
726
|
+
group_column_metadata = None if group_normalized is None else group_normalized[1]
|
|
663
727
|
chr_values, starts, ends = _validate_interval_columns(df, source_path, row_base0=row_base0)
|
|
664
728
|
n = int(reference.num_snp)
|
|
665
|
-
|
|
666
|
-
|
|
729
|
+
num_source_intervals = int(df.shape[0])
|
|
730
|
+
|
|
731
|
+
if group_column0 is not None:
|
|
732
|
+
group_values = df.iloc[:, group_column0].astype(str).to_numpy(dtype=object)
|
|
733
|
+
empty = group_values == ""
|
|
734
|
+
if empty.any():
|
|
735
|
+
idx = int(np.flatnonzero(empty)[0])
|
|
736
|
+
raise ValueError(f"{source_path}: row {row_base0 + idx + 1}: group_column values must be non-empty")
|
|
737
|
+
names = [str(x) for x in pd.unique(group_values)]
|
|
738
|
+
columns = []
|
|
739
|
+
metadata_values = []
|
|
740
|
+
# PERF: loop retained because grouped binary interval union is defined independently per group.
|
|
741
|
+
for group_value in names:
|
|
742
|
+
rows = group_values == group_value
|
|
743
|
+
intervals_by_chr = binary_intervals_by_chr_from_arrays(chr_values[rows], starts[rows], ends[rows])
|
|
744
|
+
columns.append(paint_binary_column(intervals_by_chr, reference))
|
|
745
|
+
metadata_values.append(
|
|
746
|
+
_generated_metadata(
|
|
747
|
+
source_path,
|
|
748
|
+
bool(has_header),
|
|
749
|
+
int(np.count_nonzero(rows)),
|
|
750
|
+
group_column=group_column_metadata,
|
|
751
|
+
group_value=group_value,
|
|
752
|
+
)
|
|
753
|
+
)
|
|
754
|
+
annomat = sparse.hstack(columns, format="csr") if columns else sparse.csr_matrix((n, 0), dtype=np.float64)
|
|
755
|
+
is_binary = np.ones(len(names), dtype=bool)
|
|
756
|
+
metadata = np.asarray(metadata_values, dtype=object)
|
|
757
|
+
elif value_columns0 is None and num_columns == 3:
|
|
667
758
|
names = (
|
|
668
759
|
_default_annotation_names(source_path, num_columns, header_fields, value_columns0)
|
|
669
760
|
if annotation_names is None
|
|
@@ -674,23 +765,22 @@ def load_annotation(
|
|
|
674
765
|
intervals_by_chr = binary_intervals_by_chr_from_arrays(chr_values, starts, ends)
|
|
675
766
|
annomat = paint_binary_column(intervals_by_chr, reference)
|
|
676
767
|
is_binary = np.array([True], dtype=bool)
|
|
677
|
-
source_columns = [None]
|
|
678
|
-
source_column_names = [None]
|
|
679
768
|
if annotation_metadata_path is not None:
|
|
680
769
|
metadata = np.asarray([_read_sidecar_exact(annotation_metadata_path)], dtype=object)
|
|
681
770
|
elif annotation_metadata is not None:
|
|
682
771
|
metadata = _coerce_string_vector(annotation_metadata, 1, "annotation_metadata")
|
|
683
772
|
else:
|
|
684
773
|
metadata = np.asarray(
|
|
685
|
-
[_generated_metadata(source_path,
|
|
774
|
+
[_generated_metadata(source_path, bool(has_header), num_source_intervals)],
|
|
686
775
|
dtype=object,
|
|
687
776
|
)
|
|
688
777
|
else:
|
|
689
778
|
if value_columns0 is None:
|
|
690
779
|
if num_columns == 4:
|
|
691
780
|
value_columns0 = [3]
|
|
781
|
+
value_column_metadata = [str(header_fields[3])] if header_fields is not None else [3]
|
|
692
782
|
else:
|
|
693
|
-
raise ValueError(f"{source_path}: input with five or more columns requires explicit value_columns")
|
|
783
|
+
raise ValueError(f"{source_path}: input with five or more columns requires explicit value_columns or group_column")
|
|
694
784
|
names = (
|
|
695
785
|
_default_annotation_names(source_path, num_columns, header_fields, value_columns0)
|
|
696
786
|
if annotation_names is None
|
|
@@ -732,10 +822,6 @@ def load_annotation(
|
|
|
732
822
|
blocks[shard_idx] = paint_numeric_sparse(ref_shard.bp, intervals, value_block)
|
|
733
823
|
annomat = sparse.vstack(blocks, format="csr")
|
|
734
824
|
is_binary = np.zeros(len(value_columns0), dtype=bool)
|
|
735
|
-
source_columns = value_columns0
|
|
736
|
-
source_column_names = [
|
|
737
|
-
None if header_fields is None else str(header_fields[i]) for i in value_columns0
|
|
738
|
-
]
|
|
739
825
|
|
|
740
826
|
if annotation_metadata_path is not None:
|
|
741
827
|
sidecar_lines = _read_column_metadata_sidecar(annotation_metadata_path, num_columns)
|
|
@@ -747,8 +833,13 @@ def load_annotation(
|
|
|
747
833
|
else:
|
|
748
834
|
metadata = np.asarray(
|
|
749
835
|
[
|
|
750
|
-
_generated_metadata(
|
|
751
|
-
|
|
836
|
+
_generated_metadata(
|
|
837
|
+
source_path,
|
|
838
|
+
bool(has_header),
|
|
839
|
+
num_source_intervals,
|
|
840
|
+
value_column=value_column,
|
|
841
|
+
)
|
|
842
|
+
for value_column in value_column_metadata
|
|
752
843
|
],
|
|
753
844
|
dtype=object,
|
|
754
845
|
)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import json
|
|
2
4
|
import warnings
|
|
3
5
|
from dataclasses import dataclass
|
|
@@ -15,11 +17,13 @@ from ._bfile_utils import (
|
|
|
15
17
|
_read_bed_rows_int8,
|
|
16
18
|
_validate_bed,
|
|
17
19
|
)
|
|
18
|
-
from ._utils import allele_hash64, validate_requested_shards
|
|
20
|
+
from ._utils import _validate_allow_indel_swaps, allele_hash64, validate_requested_shards
|
|
21
|
+
from ._verbosity import report_variant_swaps
|
|
19
22
|
from ._variant_match import match_shard_numeric, numeric_variant_keys
|
|
20
23
|
|
|
21
|
-
_CACHE_SCHEMA = "genotype_cache/0.
|
|
24
|
+
_CACHE_SCHEMA = "genotype_cache/0.2"
|
|
22
25
|
_BED_MISSING_INT8 = np.int8(-1)
|
|
26
|
+
_SWAP_LUT = np.array([-1, 2, 1, 0], dtype=np.int8)
|
|
23
27
|
_HAPLOID_MODES = {"raw", "ploidy_scaled"}
|
|
24
28
|
_CACHE_ARRAY_FIELDS = (
|
|
25
29
|
"is_present",
|
|
@@ -150,6 +154,7 @@ class GenotypeShard:
|
|
|
150
154
|
subject_present,
|
|
151
155
|
source_subject_row0,
|
|
152
156
|
is_present,
|
|
157
|
+
is_allele_swapped,
|
|
153
158
|
ploidy_male,
|
|
154
159
|
ploidy_female,
|
|
155
160
|
reference_checksum,
|
|
@@ -170,10 +175,12 @@ class GenotypeShard:
|
|
|
170
175
|
|
|
171
176
|
self._source_row0 = np.asarray(source_row0, dtype=np.int64).reshape(-1)
|
|
172
177
|
self._is_present = np.asarray(is_present, dtype=bool).reshape(-1)
|
|
178
|
+
self._is_allele_swapped = np.asarray(is_allele_swapped, dtype=bool).reshape(-1)
|
|
173
179
|
self._ploidy_male = np.asarray(ploidy_male, dtype=float).reshape(-1)
|
|
174
180
|
self._ploidy_female = np.asarray(ploidy_female, dtype=float).reshape(-1)
|
|
175
181
|
if not (
|
|
176
182
|
self._source_row0.size
|
|
183
|
+
== self._is_allele_swapped.size
|
|
177
184
|
== self._is_present.size
|
|
178
185
|
== self._ploidy_male.size
|
|
179
186
|
== self._ploidy_female.size
|
|
@@ -184,6 +191,8 @@ class GenotypeShard:
|
|
|
184
191
|
bad_source = (self._source_row0 < -1) | (self._source_row0 >= self._source_num_snp)
|
|
185
192
|
if bad_source.any():
|
|
186
193
|
raise ValueError(f"GenotypeShard {self._label}: source_row0 out of source BIM bounds")
|
|
194
|
+
if np.any(self._is_allele_swapped & ~self._is_present):
|
|
195
|
+
raise ValueError("is_allele_swapped requires is_present")
|
|
187
196
|
absent = ~self._is_present
|
|
188
197
|
if np.any(~np.isnan(self._ploidy_male[absent])) or np.any(~np.isnan(self._ploidy_female[absent])):
|
|
189
198
|
raise ValueError(f"GenotypeShard {self._label}: absent SNPs must have NaN ploidy")
|
|
@@ -207,6 +216,7 @@ class GenotypeShard:
|
|
|
207
216
|
for name in (
|
|
208
217
|
"_source_row0",
|
|
209
218
|
"_is_present",
|
|
219
|
+
"_is_allele_swapped",
|
|
210
220
|
"_ploidy_male",
|
|
211
221
|
"_ploidy_female",
|
|
212
222
|
"_subject_present",
|
|
@@ -258,6 +268,10 @@ class GenotypeShard:
|
|
|
258
268
|
def is_present(self) -> np.ndarray:
|
|
259
269
|
return self._is_present
|
|
260
270
|
|
|
271
|
+
@property
|
|
272
|
+
def is_allele_swapped(self) -> np.ndarray:
|
|
273
|
+
return self._is_allele_swapped
|
|
274
|
+
|
|
261
275
|
@property
|
|
262
276
|
def ploidy_male(self) -> np.ndarray:
|
|
263
277
|
return self._ploidy_male
|
|
@@ -340,6 +354,12 @@ class GenotypePanel:
|
|
|
340
354
|
return np.array([], dtype=bool)
|
|
341
355
|
return np.concatenate([s.is_present for s in self._shards])
|
|
342
356
|
|
|
357
|
+
@property
|
|
358
|
+
def is_allele_swapped(self) -> np.ndarray:
|
|
359
|
+
if not self._shards:
|
|
360
|
+
return np.array([], dtype=bool)
|
|
361
|
+
return np.concatenate([s.is_allele_swapped for s in self._shards])
|
|
362
|
+
|
|
343
363
|
@property
|
|
344
364
|
def ploidy_male(self) -> np.ndarray:
|
|
345
365
|
if not self._shards:
|
|
@@ -413,7 +433,7 @@ class GenotypePanel:
|
|
|
413
433
|
|
|
414
434
|
def _addressed_shards(self, snp_indices: np.ndarray) -> list[tuple[GenotypeShard, np.ndarray, np.ndarray]]:
|
|
415
435
|
addressed = []
|
|
416
|
-
for shard, offset in zip(self._shards, self._shard_offsets
|
|
436
|
+
for shard, offset in zip(self._shards, self._shard_offsets):
|
|
417
437
|
start = int(offset["start0"])
|
|
418
438
|
stop = int(offset["stop0"])
|
|
419
439
|
cols = np.flatnonzero((snp_indices >= start) & (snp_indices < stop))
|
|
@@ -475,6 +495,9 @@ class GenotypePanel:
|
|
|
475
495
|
source_rows,
|
|
476
496
|
shard.source_num_sample,
|
|
477
497
|
)
|
|
498
|
+
sc = np.flatnonzero(shard.is_allele_swapped[local_indices])
|
|
499
|
+
if sc.size:
|
|
500
|
+
source_geno[:, sc] = _SWAP_LUT[source_geno[:, sc] + 1]
|
|
478
501
|
panel_rows = np.flatnonzero(shard.subject_present)
|
|
479
502
|
if panel_rows.size:
|
|
480
503
|
source_subject_rows = shard.source_subject_row0[panel_rows]
|
|
@@ -572,8 +595,9 @@ def _map_chrx_subjects(chrx_fam: pd.DataFrame, panel_fam: pd.DataFrame, where: s
|
|
|
572
595
|
return subject_present, source_subject_row0
|
|
573
596
|
|
|
574
597
|
|
|
575
|
-
def _build_shard(ref_shard, source: _SourceRecord, subject_present, source_subject_row0) -> GenotypeShard:
|
|
598
|
+
def _build_shard(ref_shard, source: _SourceRecord, subject_present, source_subject_row0, allow_indel_swaps) -> GenotypeShard:
|
|
576
599
|
source_bim = source.bim.loc[source.bim["chr"] == ref_shard.label]
|
|
600
|
+
is_allele_swapped = np.zeros(int(ref_shard.num_snp), dtype=bool)
|
|
577
601
|
if source_bim.empty:
|
|
578
602
|
local_match = np.full(int(ref_shard.num_snp), -1, dtype=np.int64)
|
|
579
603
|
else:
|
|
@@ -583,15 +607,14 @@ def _build_shard(ref_shard, source: _SourceRecord, subject_present, source_subje
|
|
|
583
607
|
source_bim["a1_hash64"].to_numpy(dtype=np.uint64),
|
|
584
608
|
source_bim["a2_hash64"].to_numpy(dtype=np.uint64),
|
|
585
609
|
)
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
)
|
|
610
|
+
match = match_shard_numeric(
|
|
611
|
+
ref_keys, src_keys, ref_shard.label, source_name="genotype",
|
|
612
|
+
allow_indel_swaps=allow_indel_swaps,
|
|
613
|
+
)
|
|
614
|
+
local_match = match.source_rows
|
|
615
|
+
is_allele_swapped = match.is_allele_swapped
|
|
616
|
+
report_variant_swaps(source.bim_path, ref_shard.label, np.count_nonzero(is_allele_swapped),
|
|
617
|
+
match.num_strand_ambiguous, match.num_disabled_indels)
|
|
595
618
|
|
|
596
619
|
is_present = local_match >= 0
|
|
597
620
|
source_row0 = np.full(int(ref_shard.num_snp), -1, dtype=np.int64)
|
|
@@ -613,6 +636,7 @@ def _build_shard(ref_shard, source: _SourceRecord, subject_present, source_subje
|
|
|
613
636
|
subject_present=subject_present,
|
|
614
637
|
source_subject_row0=source_subject_row0,
|
|
615
638
|
is_present=is_present,
|
|
639
|
+
is_allele_swapped=is_allele_swapped,
|
|
616
640
|
ploidy_male=ploidy_male,
|
|
617
641
|
ploidy_female=ploidy_female,
|
|
618
642
|
reference_checksum=ref_shard.checksum,
|
|
@@ -628,7 +652,8 @@ def _validate_reference_for_genotype(reference) -> list:
|
|
|
628
652
|
return ref_shards
|
|
629
653
|
|
|
630
654
|
|
|
631
|
-
def load_genotype(bfile_prefix, reference) -> GenotypePanel:
|
|
655
|
+
def load_genotype(bfile_prefix, reference, allow_indel_swaps=False) -> GenotypePanel:
|
|
656
|
+
allow_indel_swaps = _validate_allow_indel_swaps(allow_indel_swaps)
|
|
632
657
|
ref_shards = _validate_reference_for_genotype(reference)
|
|
633
658
|
prefix = str(bfile_prefix)
|
|
634
659
|
ref_labels = [s.label for s in ref_shards]
|
|
@@ -682,7 +707,7 @@ def load_genotype(bfile_prefix, reference) -> GenotypePanel:
|
|
|
682
707
|
subject_maps["X"] = _all_subjects_present(sources["X"].source_num_sample)
|
|
683
708
|
|
|
684
709
|
shards = [
|
|
685
|
-
_build_shard(ref_shard, sources[ref_shard.label], *subject_maps[ref_shard.label])
|
|
710
|
+
_build_shard(ref_shard, sources[ref_shard.label], *subject_maps[ref_shard.label], allow_indel_swaps)
|
|
686
711
|
for ref_shard in ref_shards
|
|
687
712
|
]
|
|
688
713
|
return GenotypePanel(
|
|
@@ -714,6 +739,7 @@ def save_genotype_cache(panel: GenotypePanel, path) -> None:
|
|
|
714
739
|
arrays = {
|
|
715
740
|
"_meta": np.frombuffer(json.dumps(meta).encode(), dtype=np.uint8),
|
|
716
741
|
"is_present": panel.is_present.astype(bool, copy=False),
|
|
742
|
+
"is_allele_swapped": panel.is_allele_swapped,
|
|
717
743
|
"ploidy_male": panel.ploidy_male.astype(float, copy=False),
|
|
718
744
|
"ploidy_female": panel.ploidy_female.astype(float, copy=False),
|
|
719
745
|
"source_row0": panel.source_row0.astype(np.int64, copy=False),
|
|
@@ -744,9 +770,10 @@ def _load_cache_meta(data) -> dict:
|
|
|
744
770
|
|
|
745
771
|
def _validate_cache_meta(meta: dict, data) -> None:
|
|
746
772
|
schema = meta.get("schema")
|
|
747
|
-
if schema
|
|
773
|
+
if schema not in {_CACHE_SCHEMA, "genotype_cache/0.1"}:
|
|
748
774
|
raise ValueError(f"Unsupported genotype cache schema: {schema!r}")
|
|
749
|
-
|
|
775
|
+
required = _CACHE_ARRAY_FIELDS + (("is_allele_swapped",) if schema == _CACHE_SCHEMA else ())
|
|
776
|
+
missing = [name for name in required if name not in data.files]
|
|
750
777
|
if missing:
|
|
751
778
|
raise ValueError(f"Invalid genotype cache: missing array field {missing[0]!r}")
|
|
752
779
|
n_shards = int(meta.get("n_shards", -1))
|
|
@@ -773,6 +800,12 @@ def _validate_cache_meta(meta: dict, data) -> None:
|
|
|
773
800
|
raise ValueError("Invalid genotype cache: source_layout must be 'non_sharded' or 'sharded'")
|
|
774
801
|
|
|
775
802
|
is_present = data["is_present"]
|
|
803
|
+
if schema == _CACHE_SCHEMA:
|
|
804
|
+
swapped = data["is_allele_swapped"]
|
|
805
|
+
if swapped.dtype != np.dtype(bool) or swapped.shape != is_present.shape:
|
|
806
|
+
raise ValueError("Invalid genotype cache: is_allele_swapped must be a logical SNP-axis vector")
|
|
807
|
+
if np.any(swapped & ~is_present.astype(bool)):
|
|
808
|
+
raise ValueError("Invalid genotype cache: is_allele_swapped requires is_present")
|
|
776
809
|
source_row0 = data["source_row0"]
|
|
777
810
|
if is_present.ndim != 1 or source_row0.ndim != 1 or is_present.shape[0] != source_row0.shape[0]:
|
|
778
811
|
raise ValueError("Invalid genotype cache: SNP-axis vector lengths mismatch")
|
|
@@ -817,6 +850,8 @@ def load_genotype_cache(path, shards=None) -> GenotypePanel:
|
|
|
817
850
|
selected = validate_requested_shards(shards, labels, "load_genotype_cache")
|
|
818
851
|
label_to_index = {label: i for i, label in enumerate(labels)}
|
|
819
852
|
|
|
853
|
+
swap_flags = (data["is_allele_swapped"] if meta["schema"] == _CACHE_SCHEMA
|
|
854
|
+
else np.zeros(data["is_present"].size, dtype=bool))
|
|
820
855
|
shard_objs = []
|
|
821
856
|
for label in selected:
|
|
822
857
|
i = label_to_index[label]
|
|
@@ -833,6 +868,7 @@ def load_genotype_cache(path, shards=None) -> GenotypePanel:
|
|
|
833
868
|
subject_present=data["subject_present"][:, i],
|
|
834
869
|
source_subject_row0=data["source_subject_row0"][:, i],
|
|
835
870
|
is_present=data["is_present"][start:stop],
|
|
871
|
+
is_allele_swapped=swap_flags[start:stop],
|
|
836
872
|
ploidy_male=data["ploidy_male"][start:stop],
|
|
837
873
|
ploidy_female=data["ploidy_female"][start:stop],
|
|
838
874
|
reference_checksum=meta["shard_checksums"][i],
|
|
@@ -9,34 +9,32 @@ import numpy as np
|
|
|
9
9
|
import pandas as pd
|
|
10
10
|
|
|
11
11
|
from ._bfile_utils import _parse_bim
|
|
12
|
-
from ._utils import
|
|
12
|
+
from ._utils import (
|
|
13
|
+
CANONICAL_CHR_ORDER,
|
|
14
|
+
_is_single_nucleotide_variant,
|
|
15
|
+
_is_strand_ambiguous,
|
|
16
|
+
allele_hash64,
|
|
17
|
+
validate_requested_shards,
|
|
18
|
+
)
|
|
13
19
|
|
|
14
20
|
logger = logging.getLogger(__name__)
|
|
15
21
|
|
|
16
22
|
_CACHE_SCHEMA = "reference_cache/0.1"
|
|
17
|
-
_A_HASH64, _C_HASH64, _G_HASH64, _T_HASH64 = allele_hash64(
|
|
18
|
-
np.array(["A", "C", "G", "T"], dtype=object)
|
|
19
|
-
)
|
|
20
|
-
_SINGLE_BASE_HASH64 = np.array(
|
|
21
|
-
[_A_HASH64, _C_HASH64, _G_HASH64, _T_HASH64], dtype=np.uint64
|
|
22
|
-
)
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
def
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
| ((a1 == _G_HASH64) & (a2 == _C_HASH64))
|
|
39
|
-
)
|
|
25
|
+
def _validate_reference_keys(shard) -> None:
|
|
26
|
+
# Ordered-key duplicates have already been rejected by the BIM parser.
|
|
27
|
+
snv = shard.is_single_nucleotide_variant
|
|
28
|
+
bp = shard.bp[snv]
|
|
29
|
+
a1 = shard.a1_hash64[snv]
|
|
30
|
+
a2 = shard.a2_hash64[snv]
|
|
31
|
+
lo, hi = np.minimum(a1, a2), np.maximum(a1, a2)
|
|
32
|
+
order = np.lexsort((hi, lo, bp))
|
|
33
|
+
left, right = order[:-1], order[1:]
|
|
34
|
+
duplicate = (bp[left] == bp[right]) & (lo[left] == lo[right]) & (hi[left] == hi[right])
|
|
35
|
+
if np.any(duplicate):
|
|
36
|
+
pos = right[np.flatnonzero(duplicate)[0]]
|
|
37
|
+
raise ValueError(f"reference shard {shard.label}: reciprocal SNV keys are not allowed at bp={int(bp[pos])}")
|
|
40
38
|
|
|
41
39
|
|
|
42
40
|
def _checksum_from_arrays(
|
|
@@ -410,6 +408,8 @@ def load_reference(path, shards=None) -> ReferencePanel:
|
|
|
410
408
|
bim_df["a2"].to_numpy(dtype=object),
|
|
411
409
|
)
|
|
412
410
|
)
|
|
411
|
+
for shard in shards:
|
|
412
|
+
_validate_reference_keys(shard)
|
|
413
413
|
return ReferencePanel(shards)
|
|
414
414
|
|
|
415
415
|
bim_df = _parse_bim(path_obj)
|
|
@@ -431,6 +431,8 @@ def load_reference(path, shards=None) -> ReferencePanel:
|
|
|
431
431
|
shard_df["a2"].to_numpy(dtype=object),
|
|
432
432
|
)
|
|
433
433
|
)
|
|
434
|
+
for shard in out_shards:
|
|
435
|
+
_validate_reference_keys(shard)
|
|
434
436
|
return ReferencePanel(out_shards)
|
|
435
437
|
|
|
436
438
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import json
|
|
2
4
|
import re
|
|
3
5
|
import warnings
|
|
@@ -8,7 +10,8 @@ import numpy as np
|
|
|
8
10
|
import pandas as pd
|
|
9
11
|
from pandas.errors import EmptyDataError, ParserError
|
|
10
12
|
|
|
11
|
-
from ._utils import CANONICAL_CHR_ORDER, allele_hash64, validate_requested_shards
|
|
13
|
+
from ._utils import CANONICAL_CHR_ORDER, _validate_allow_indel_swaps, allele_hash64, validate_requested_shards
|
|
14
|
+
from ._verbosity import info, report_variant_swaps
|
|
12
15
|
from ._variant_match import match_shard_numeric, numeric_variant_keys
|
|
13
16
|
|
|
14
17
|
_CACHE_SCHEMA = "sumstats_cache/0.1"
|
|
@@ -136,15 +139,19 @@ def _parse_sumstats(path: Path) -> pd.DataFrame:
|
|
|
136
139
|
out["bp"] = out["bp"].astype(np.int64)
|
|
137
140
|
|
|
138
141
|
p_arr = out["p"].to_numpy(dtype=float)
|
|
139
|
-
bad_p =
|
|
142
|
+
bad_p = ~np.isnan(p_arr) & (~np.isfinite(p_arr) | (p_arr < 0.0) | (p_arr > 1.0))
|
|
140
143
|
if bad_p.any():
|
|
141
|
-
idx = int(
|
|
144
|
+
idx = int(out.index[np.flatnonzero(bad_p)[0]])
|
|
142
145
|
raise ValueError(f"{path}: row {idx + 2}: p must be finite numeric in [0, 1]: {out.loc[idx, 'p']!r}")
|
|
143
146
|
|
|
144
147
|
for opt in _OPTIONAL_COLS:
|
|
145
148
|
if opt in out.columns:
|
|
146
149
|
out[opt] = out[opt].mask(~np.isfinite(out[opt].to_numpy(dtype=float)))
|
|
147
150
|
|
|
151
|
+
missing_p = out["p"].isna()
|
|
152
|
+
if missing_p.any():
|
|
153
|
+
info(f"{path}: dropped {int(missing_p.sum())} row(s) with missing p")
|
|
154
|
+
return out.loc[~missing_p].copy()
|
|
148
155
|
return out
|
|
149
156
|
|
|
150
157
|
|
|
@@ -336,7 +343,7 @@ def _derive_logp(p_vals: np.ndarray) -> np.ndarray:
|
|
|
336
343
|
|
|
337
344
|
|
|
338
345
|
def _validate_pvec(p_vals: np.ndarray, name: str) -> None:
|
|
339
|
-
bad = ~np.isfinite(p_vals) | (p_vals < 0.0) | (p_vals > 1.0)
|
|
346
|
+
bad = ~np.isnan(p_vals) & (~np.isfinite(p_vals) | (p_vals < 0.0) | (p_vals > 1.0))
|
|
340
347
|
if bad.any():
|
|
341
348
|
idx = int(np.flatnonzero(bad)[0])
|
|
342
349
|
raise ValueError(f"{name}[{idx}] must be finite numeric in [0, 1]")
|
|
@@ -396,18 +403,7 @@ def _build_sumstats_from_aligned(
|
|
|
396
403
|
return Sumstats(shards)
|
|
397
404
|
|
|
398
405
|
|
|
399
|
-
def
|
|
400
|
-
if count > 0:
|
|
401
|
-
warnings.warn(
|
|
402
|
-
f"{source_path}: shard {shard_label}: {count} unmatched {context} "
|
|
403
|
-
"variant(s) would match the reference if a1/a2 were swapped; "
|
|
404
|
-
"variants remain unmatched",
|
|
405
|
-
RuntimeWarning,
|
|
406
|
-
stacklevel=3,
|
|
407
|
-
)
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
def _build_sumstats_panel(df: pd.DataFrame, reference, source_path) -> Sumstats:
|
|
406
|
+
def _build_sumstats_panel(df: pd.DataFrame, reference, source_path, allow_indel_swaps=False) -> Sumstats:
|
|
411
407
|
n = int(reference.num_snp)
|
|
412
408
|
aligned_optional = {
|
|
413
409
|
col: (np.full(n, np.nan, dtype=float) if col in df.columns else None)
|
|
@@ -427,8 +423,13 @@ def _build_sumstats_panel(df: pd.DataFrame, reference, source_path) -> Sumstats:
|
|
|
427
423
|
src_idx = np.flatnonzero(src_mask)
|
|
428
424
|
ref_keys = numeric_variant_keys(ref_shard.bp, ref_shard.a1_hash64, ref_shard.a2_hash64)
|
|
429
425
|
src_keys = numeric_variant_keys(src_bp[src_idx], src_a1_hash64[src_idx], src_a2_hash64[src_idx])
|
|
430
|
-
|
|
431
|
-
|
|
426
|
+
match = match_shard_numeric(
|
|
427
|
+
ref_keys, src_keys, ref_shard.label, source_name="sumstats",
|
|
428
|
+
allow_indel_swaps=allow_indel_swaps,
|
|
429
|
+
)
|
|
430
|
+
local_match = match.source_rows
|
|
431
|
+
report_variant_swaps(source_path, ref_shard.label, np.count_nonzero(match.is_allele_swapped),
|
|
432
|
+
match.num_strand_ambiguous, match.num_disabled_indels)
|
|
432
433
|
has_match = local_match >= 0
|
|
433
434
|
matched_src = src_idx[local_match[has_match]]
|
|
434
435
|
aligned_ix = np.arange(start, stop, dtype=np.int64)[has_match]
|
|
@@ -437,6 +438,14 @@ def _build_sumstats_panel(df: pd.DataFrame, reference, source_path) -> Sumstats:
|
|
|
437
438
|
for col, out in aligned_optional.items():
|
|
438
439
|
if out is not None:
|
|
439
440
|
out[aligned_ix] = df[col].to_numpy(dtype=float)[matched_src]
|
|
441
|
+
swapped_ix = np.arange(start, stop)[match.is_allele_swapped]
|
|
442
|
+
for col in ("z", "beta", "eaf"):
|
|
443
|
+
out = aligned_optional[col]
|
|
444
|
+
if out is not None:
|
|
445
|
+
out[swapped_ix] = 1 - out[swapped_ix] if col == "eaf" else -out[swapped_ix]
|
|
446
|
+
if not has_match.any():
|
|
447
|
+
warnings.warn(f"{source_path}: no matching source rows for reference shard {ref_shard.label!r}",
|
|
448
|
+
RuntimeWarning, stacklevel=3)
|
|
440
449
|
|
|
441
450
|
aligned_logp = _derive_logp(aligned_p)
|
|
442
451
|
_warn_optional_zn_completeness(
|
|
@@ -454,9 +463,10 @@ def _build_sumstats_panel(df: pd.DataFrame, reference, source_path) -> Sumstats:
|
|
|
454
463
|
)
|
|
455
464
|
|
|
456
465
|
|
|
457
|
-
def load_sumstats(path, reference) -> Sumstats:
|
|
466
|
+
def load_sumstats(path, reference, allow_indel_swaps=False) -> Sumstats:
|
|
467
|
+
allow_indel_swaps = _validate_allow_indel_swaps(allow_indel_swaps)
|
|
458
468
|
path = Path(path)
|
|
459
|
-
return _build_sumstats_panel(_parse_sumstats(path), reference, path)
|
|
469
|
+
return _build_sumstats_panel(_parse_sumstats(path), reference, path, allow_indel_swaps)
|
|
460
470
|
|
|
461
471
|
|
|
462
472
|
def create_sumstats(
|
|
@@ -483,6 +493,10 @@ def create_sumstats(
|
|
|
483
493
|
"info": None if info is None else _coerce_aligned_vec("info", info, n=num_snp_value, allow_inf=False),
|
|
484
494
|
}
|
|
485
495
|
|
|
496
|
+
for name, vec in aligned_optional.items():
|
|
497
|
+
if vec is not None:
|
|
498
|
+
aligned_optional[name] = np.where(np.isnan(aligned_p), np.nan, vec)
|
|
499
|
+
|
|
486
500
|
return _build_sumstats_from_aligned(
|
|
487
501
|
reference,
|
|
488
502
|
aligned_logp=aligned_logp,
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
import numpy as np
|
|
2
|
-
|
|
3
|
-
_KEY_DTYPE = np.dtype([("bp", "<i8"), ("a1_hash64", "<u8"), ("a2_hash64", "<u8")])
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def numeric_variant_keys(bp_arr, a1_hash64, a2_hash64) -> np.ndarray:
|
|
7
|
-
bp = np.asarray(bp_arr, dtype=np.int64).reshape(-1)
|
|
8
|
-
a1h = np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
|
|
9
|
-
a2h = np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
|
|
10
|
-
if not (bp.size == a1h.size == a2h.size):
|
|
11
|
-
raise ValueError("numeric variant key vector lengths mismatch")
|
|
12
|
-
keys = np.empty(bp.size, dtype=_KEY_DTYPE)
|
|
13
|
-
keys["bp"] = bp
|
|
14
|
-
keys["a1_hash64"] = a1h
|
|
15
|
-
keys["a2_hash64"] = a2h
|
|
16
|
-
return keys
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def check_unique_sorted_variant_keys(sorted_keys: np.ndarray, where: str, object_name: str = "source") -> None:
|
|
20
|
-
if sorted_keys.size < 2:
|
|
21
|
-
return
|
|
22
|
-
dup = sorted_keys[1:] == sorted_keys[:-1]
|
|
23
|
-
if np.any(dup):
|
|
24
|
-
pos = int(np.flatnonzero(dup)[0] + 1)
|
|
25
|
-
key = sorted_keys[pos]
|
|
26
|
-
raise ValueError(
|
|
27
|
-
f"Ambiguous duplicate {object_name}/reference matching key in {where}: "
|
|
28
|
-
f"bp={int(key['bp'])}, a1_hash64={int(key['a1_hash64'])}, "
|
|
29
|
-
f"a2_hash64={int(key['a2_hash64'])}"
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def match_shard_numeric(
|
|
34
|
-
ref_keys: np.ndarray,
|
|
35
|
-
src_keys: np.ndarray,
|
|
36
|
-
shard_label: str,
|
|
37
|
-
*,
|
|
38
|
-
source_name: str = "source",
|
|
39
|
-
) -> tuple[np.ndarray, int]:
|
|
40
|
-
ref_order = np.argsort(ref_keys, kind="mergesort", order=("bp", "a1_hash64", "a2_hash64"))
|
|
41
|
-
ref_sorted = ref_keys[ref_order]
|
|
42
|
-
check_unique_sorted_variant_keys(ref_sorted, f"reference shard {shard_label}", source_name)
|
|
43
|
-
|
|
44
|
-
if src_keys.size == 0:
|
|
45
|
-
return np.full(ref_keys.size, -1, dtype=np.int64), 0
|
|
46
|
-
|
|
47
|
-
src_order = np.argsort(src_keys, kind="mergesort", order=("bp", "a1_hash64", "a2_hash64"))
|
|
48
|
-
src_sorted = src_keys[src_order]
|
|
49
|
-
check_unique_sorted_variant_keys(src_sorted, f"{source_name} shard {shard_label}", source_name)
|
|
50
|
-
|
|
51
|
-
pos = np.searchsorted(src_sorted, ref_keys)
|
|
52
|
-
in_range = pos < src_sorted.size
|
|
53
|
-
matched = np.zeros(ref_keys.size, dtype=bool)
|
|
54
|
-
matched[in_range] = src_sorted[pos[in_range]] == ref_keys[in_range]
|
|
55
|
-
|
|
56
|
-
out = np.full(ref_keys.size, -1, dtype=np.int64)
|
|
57
|
-
out[matched] = src_order[pos[matched]]
|
|
58
|
-
|
|
59
|
-
matched_src = np.zeros(src_keys.size, dtype=bool)
|
|
60
|
-
matched_src[out[matched]] = True
|
|
61
|
-
unmatched_src = src_keys[~matched_src]
|
|
62
|
-
if unmatched_src.size == 0 or ref_sorted.size == 0:
|
|
63
|
-
return out, 0
|
|
64
|
-
|
|
65
|
-
swapped_src = np.empty(unmatched_src.size, dtype=_KEY_DTYPE)
|
|
66
|
-
swapped_src["bp"] = unmatched_src["bp"]
|
|
67
|
-
swapped_src["a1_hash64"] = unmatched_src["a2_hash64"]
|
|
68
|
-
swapped_src["a2_hash64"] = unmatched_src["a1_hash64"]
|
|
69
|
-
swapped_pos = np.searchsorted(ref_sorted, swapped_src)
|
|
70
|
-
swapped_in_range = swapped_pos < ref_sorted.size
|
|
71
|
-
swapped_match = np.zeros(swapped_src.size, dtype=bool)
|
|
72
|
-
swapped_match[swapped_in_range] = (
|
|
73
|
-
ref_sorted[swapped_pos[swapped_in_range]]
|
|
74
|
-
== swapped_src[swapped_in_range]
|
|
75
|
-
)
|
|
76
|
-
return out, int(np.count_nonzero(swapped_match))
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
|
|
3
|
-
_LEVELS = ("quiet", "info")
|
|
4
|
-
_VERBOSITY = "info"
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
def set_verbosity(level: str) -> None:
|
|
8
|
-
global _VERBOSITY
|
|
9
|
-
if level not in _LEVELS:
|
|
10
|
-
raise ValueError("verbosity must be one of: quiet, info")
|
|
11
|
-
_VERBOSITY = level
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def get_verbosity() -> str:
|
|
15
|
-
return _VERBOSITY
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def info(message: str) -> None:
|
|
19
|
-
if _VERBOSITY == "info":
|
|
20
|
-
print(message, file=sys.stderr)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|