statgen 0.3.3__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {statgen-0.3.3 → statgen-0.4.0}/PKG-INFO +1 -1
  2. {statgen-0.3.3 → statgen-0.4.0}/pyproject.toml +1 -1
  3. {statgen-0.3.3 → statgen-0.4.0}/statgen/__init__.py +1 -1
  4. {statgen-0.3.3 → statgen-0.4.0}/statgen/_bfile_utils.py +3 -1
  5. {statgen-0.3.3 → statgen-0.4.0}/statgen/_ld_schema.py +2 -0
  6. {statgen-0.3.3 → statgen-0.4.0}/statgen/_utils.py +28 -0
  7. statgen-0.4.0/statgen/_variant_match.py +105 -0
  8. statgen-0.4.0/statgen/_verbosity.py +30 -0
  9. {statgen-0.3.3 → statgen-0.4.0}/statgen/annotations.py +146 -55
  10. {statgen-0.3.3 → statgen-0.4.0}/statgen/genotype.py +53 -17
  11. {statgen-0.3.3 → statgen-0.4.0}/statgen/reference.py +24 -22
  12. {statgen-0.3.3 → statgen-0.4.0}/statgen/sumstats.py +34 -20
  13. {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/PKG-INFO +1 -1
  14. statgen-0.3.3/statgen/_variant_match.py +0 -76
  15. statgen-0.3.3/statgen/_verbosity.py +0 -20
  16. {statgen-0.3.3 → statgen-0.4.0}/README.md +0 -0
  17. {statgen-0.3.3 → statgen-0.4.0}/setup.cfg +0 -0
  18. {statgen-0.3.3 → statgen-0.4.0}/statgen/_annotation_painting.py +0 -0
  19. {statgen-0.3.3 → statgen-0.4.0}/statgen/_ld_npz.py +0 -0
  20. {statgen-0.3.3 → statgen-0.4.0}/statgen/_ld_reference.py +0 -0
  21. {statgen-0.3.3 → statgen-0.4.0}/statgen/_ld_writer.py +0 -0
  22. {statgen-0.3.3 → statgen-0.4.0}/statgen/ld.py +0 -0
  23. {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/SOURCES.txt +0 -0
  24. {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/dependency_links.txt +0 -0
  25. {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/requires.txt +0 -0
  26. {statgen-0.3.3 → statgen-0.4.0}/statgen.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: statgen
3
- Version: 0.3.3
3
+ Version: 0.4.0
4
4
  Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "statgen"
7
- version = "0.3.3"
7
+ version = "0.4.0"
8
8
  description = "Statistical genetics data objects and loaders for Python and MATLAB/Octave."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -1,5 +1,5 @@
1
1
  from ._verbosity import get_verbosity, set_verbosity
2
2
 
3
- __version__ = "0.3.3"
3
+ __version__ = "0.4.0"
4
4
 
5
5
  __all__ = ["__version__", "get_verbosity", "set_verbosity"]
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import re
2
4
  import warnings
3
5
  from pathlib import Path
@@ -275,7 +277,7 @@ def _read_bed_rows_int8(path: Path, source_rows: np.ndarray, source_num_sample:
275
277
  # PERF: one seek/read per contiguous BED row block; requested order and
276
278
  # repeats are restored after block decoding. Blocks are capped to
277
279
  # bound transient packed/decode allocations for large requests.
278
- for start, stop in zip(group_starts, group_stops, strict=True):
280
+ for start, stop in zip(group_starts, group_stops):
279
281
  for chunk_start in range(start, stop, rows_per_read):
280
282
  chunk_stop = min(chunk_start + rows_per_read, stop)
281
283
  block_rows = unique_rows[chunk_start:chunk_stop]
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import hashlib
2
4
  import json
3
5
  from pathlib import Path
@@ -114,3 +114,31 @@ def _allele_hash_lanes_from_utf8(encoded) -> tuple[np.ndarray, np.ndarray]:
114
114
  h2[active] = (h2[active] * _ALLELE_HASH_BASE2 + x) % _ALLELE_HASH_P
115
115
 
116
116
  return h1, h2
117
+
118
+
119
+ _A_HASH64, _C_HASH64, _G_HASH64, _T_HASH64 = _SINGLE_BASE_HASH64
120
+
121
+
122
+ def _is_single_base(h) -> np.ndarray:
123
+ return np.isin(h, _SINGLE_BASE_HASH64)
124
+
125
+
126
+ def _is_single_nucleotide_variant(a1_hash64, a2_hash64) -> np.ndarray:
127
+ return _is_single_base(a1_hash64) & _is_single_base(a2_hash64)
128
+
129
+
130
+ def _is_strand_ambiguous(a1_hash64, a2_hash64) -> np.ndarray:
131
+ a1 = np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
132
+ a2 = np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
133
+ return (
134
+ ((a1 == _A_HASH64) & (a2 == _T_HASH64))
135
+ | ((a1 == _T_HASH64) & (a2 == _A_HASH64))
136
+ | ((a1 == _C_HASH64) & (a2 == _G_HASH64))
137
+ | ((a1 == _G_HASH64) & (a2 == _C_HASH64))
138
+ )
139
+
140
+
141
+ def _validate_allow_indel_swaps(value) -> bool:
142
+ if not isinstance(value, (bool, np.bool_)):
143
+ raise ValueError("allow_indel_swaps must be a boolean scalar")
144
+ return bool(value)
@@ -0,0 +1,105 @@
1
+ from dataclasses import dataclass
2
+
3
+ import numpy as np
4
+
5
+ from ._utils import _is_single_base, _is_single_nucleotide_variant, _is_strand_ambiguous
6
+
7
+ _KEY_DTYPE = np.dtype([("bp", "<i8"), ("a1_hash64", "<u8"), ("a2_hash64", "<u8")])
8
+
9
+
10
+ def numeric_variant_keys(bp_arr, a1_hash64, a2_hash64) -> np.ndarray:
11
+ bp = np.asarray(bp_arr, dtype=np.int64).reshape(-1)
12
+ a1h = np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
13
+ a2h = np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
14
+ if not (bp.size == a1h.size == a2h.size):
15
+ raise ValueError("numeric variant key vector lengths mismatch")
16
+ keys = np.empty(bp.size, dtype=_KEY_DTYPE)
17
+ keys["bp"] = bp
18
+ keys["a1_hash64"] = a1h
19
+ keys["a2_hash64"] = a2h
20
+ return keys
21
+
22
+
23
+ def check_unique_sorted_variant_keys(sorted_keys: np.ndarray, where: str, object_name: str = "source") -> None:
24
+ if sorted_keys.size < 2:
25
+ return
26
+ dup = sorted_keys[1:] == sorted_keys[:-1]
27
+ if np.any(dup):
28
+ pos = int(np.flatnonzero(dup)[0] + 1)
29
+ key = sorted_keys[pos]
30
+ raise ValueError(
31
+ f"Ambiguous duplicate {object_name}/reference matching key in {where}: "
32
+ f"bp={int(key['bp'])}, a1_hash64={int(key['a1_hash64'])}, "
33
+ f"a2_hash64={int(key['a2_hash64'])}"
34
+ )
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class VariantMatch:
39
+ source_rows: np.ndarray
40
+ is_allele_swapped: np.ndarray
41
+ num_strand_ambiguous: int
42
+ num_disabled_indels: int
43
+
44
+
45
+ def _lookup_candidates(ref_keys, src_keys, source_rows, shard_label, source_name, candidate_eligible=None):
46
+ """Return unique eligible source rows and count hits from ineligible rows."""
47
+ order = np.argsort(src_keys, kind="mergesort", order=("bp", "a1_hash64", "a2_hash64"))
48
+ sorted_keys = src_keys[order]
49
+ left = np.searchsorted(sorted_keys, ref_keys, side="left")
50
+ right = np.searchsorted(sorted_keys, ref_keys, side="right")
51
+ present = right > left
52
+ accepted = present.copy()
53
+ if candidate_eligible is not None:
54
+ # A key fixes both allele hashes, so all candidates for a slot have
55
+ # the same allele class and eligibility, even when they are duplicates.
56
+ accepted[present] = candidate_eligible[order[left[present]]]
57
+ excluded_hits = int(np.sum((right - left)[present & ~accepted]))
58
+ ambiguous = (right - left > 1) & accepted
59
+ if np.any(ambiguous):
60
+ key = ref_keys[np.flatnonzero(ambiguous)[0]]
61
+ raise ValueError(
62
+ f"Ambiguous duplicate {source_name}/reference matching key in shard {shard_label}: "
63
+ f"bp={int(key['bp'])}, a1_hash64={int(key['a1_hash64'])}, "
64
+ f"a2_hash64={int(key['a2_hash64'])}"
65
+ )
66
+ out = np.full(ref_keys.size, -1, dtype=np.int64)
67
+ out[accepted] = source_rows[order[left[accepted]]]
68
+ return out, excluded_hits
69
+
70
+
71
+ def match_shard_numeric(
72
+ ref_keys: np.ndarray,
73
+ src_keys: np.ndarray,
74
+ shard_label: str,
75
+ *,
76
+ source_name: str = "source",
77
+ allow_indel_swaps: bool = False,
78
+ ) -> VariantMatch:
79
+ # Reference validation belongs to load_reference; cached references are trusted.
80
+ rows, _ = _lookup_candidates(
81
+ ref_keys, src_keys, np.arange(src_keys.size), shard_label, source_name
82
+ )
83
+ swapped = np.zeros(ref_keys.size, dtype=bool)
84
+ unused = np.ones(src_keys.size, dtype=bool)
85
+ unused[rows[rows >= 0]] = False
86
+ a1h = src_keys["a1_hash64"]
87
+ a2h = src_keys["a2_hash64"]
88
+ snv = _is_single_nucleotide_variant(a1h, a2h)
89
+ indel = _is_single_base(a1h) ^ _is_single_base(a2h)
90
+ eligible = snv | (indel & allow_indel_swaps)
91
+ source_rows = np.flatnonzero(unused & (snv | indel))
92
+ reversed_keys = numeric_variant_keys(
93
+ src_keys["bp"][source_rows], a2h[source_rows], a1h[source_rows]
94
+ )
95
+ missing = np.flatnonzero(rows < 0)
96
+ fallback, disabled = _lookup_candidates(
97
+ ref_keys[missing], reversed_keys, source_rows, shard_label, source_name,
98
+ candidate_eligible=eligible[source_rows],
99
+ )
100
+ accepted = missing[fallback >= 0]
101
+ rows[accepted] = fallback[fallback >= 0]
102
+ swapped[accepted] = True
103
+ selected = rows[swapped]
104
+ ambiguous = _is_strand_ambiguous(a1h[selected], a2h[selected])
105
+ return VariantMatch(rows, swapped, int(np.count_nonzero(ambiguous)), disabled)
@@ -0,0 +1,30 @@
1
+ import sys
2
+
3
+ _LEVELS = ("quiet", "info")
4
+ _VERBOSITY = "info"
5
+
6
+
7
+ def set_verbosity(level: str) -> None:
8
+ global _VERBOSITY
9
+ if level not in _LEVELS:
10
+ raise ValueError("verbosity must be one of: quiet, info")
11
+ _VERBOSITY = level
12
+
13
+
14
+ def get_verbosity() -> str:
15
+ return _VERBOSITY
16
+
17
+
18
+ def info(message: str) -> None:
19
+ if _VERBOSITY == "info":
20
+ print(message, file=sys.stderr)
21
+
22
+
23
+ def report_variant_swaps(source_path, shard_label, num_swaps, num_ambiguous, num_disabled):
24
+ context = f"{source_path}: shard {shard_label}"
25
+ if num_swaps:
26
+ info(f"{context}: {num_swaps} successful allele swap(s)")
27
+ if num_ambiguous:
28
+ info(f"{context}: {num_ambiguous} strand-ambiguous A/T or C/G swap(s)")
29
+ if num_disabled:
30
+ info(f"{context}: {num_disabled} indel row(s) unmatched because indel swaps are disabled")
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import json
2
4
  from pathlib import Path
3
5
 
@@ -137,16 +139,27 @@ def _annotation_name_from_path(path: Path) -> str:
137
139
  return name
138
140
 
139
141
 
140
- def _generated_metadata(path: Path, source_column0, source_column_name) -> str:
141
- return json.dumps(
142
- {
143
- "source_file": str(path),
144
- "source_column0": source_column0,
145
- "source_column_name": source_column_name,
146
- },
147
- sort_keys=True,
148
- separators=(",", ":"),
149
- )
142
+ def _generated_metadata(
143
+ path: Path,
144
+ has_header: bool,
145
+ num_source_intervals: int,
146
+ *,
147
+ value_column=None,
148
+ group_column=None,
149
+ group_value=None,
150
+ ) -> str:
151
+ payload = {
152
+ "source_file": str(path),
153
+ "source_file_has_header": 1 if has_header else 0,
154
+ "num_source_intervals": int(num_source_intervals),
155
+ }
156
+ if value_column is not None:
157
+ payload["value_column"] = value_column
158
+ if group_column is not None:
159
+ payload["group_column"] = group_column
160
+ if group_value is not None:
161
+ payload["group_value"] = group_value
162
+ return json.dumps(payload, sort_keys=True, separators=(",", ":"))
150
163
 
151
164
 
152
165
  def _read_sidecar_exact(path) -> str:
@@ -167,7 +180,7 @@ def _read_column_metadata_sidecar(path, num_columns: int) -> list[str]:
167
180
  return lines
168
181
 
169
182
 
170
- def _read_annotation_table(path: Path, header: bool) -> tuple[pd.DataFrame, list[str] | None, int]:
183
+ def _read_annotation_table(path: Path, has_header: bool) -> tuple[pd.DataFrame, list[str] | None, int]:
171
184
  if not path.is_file():
172
185
  raise FileNotFoundError(f"annotation file not found: {path}")
173
186
 
@@ -175,7 +188,7 @@ def _read_annotation_table(path: Path, header: bool) -> tuple[pd.DataFrame, list
175
188
  df = pd.read_csv(
176
189
  path,
177
190
  sep="\t",
178
- header=0 if header else None,
191
+ header=0 if has_header else None,
179
192
  dtype=str,
180
193
  keep_default_na=False,
181
194
  na_filter=False,
@@ -192,14 +205,14 @@ def _read_annotation_table(path: Path, header: bool) -> tuple[pd.DataFrame, list
192
205
 
193
206
  if df.shape[1] < 3:
194
207
  raise ValueError(f"{path}: BED must have at least 3 tab-separated columns")
195
- header_fields = [str(x) for x in df.columns] if header else None
208
+ header_fields = [str(x) for x in df.columns] if has_header else None
196
209
  if header_fields is not None:
197
210
  if any(x == "" for x in header_fields):
198
211
  raise ValueError(f"{path}: header names must be non-empty")
199
212
  if len(set(header_fields)) != len(header_fields):
200
213
  raise ValueError(f"{path}: header names must be unique")
201
214
 
202
- return df, header_fields, 1 if header else 0
215
+ return df, header_fields, 1 if has_header else 0
203
216
 
204
217
 
205
218
  def _validate_interval_columns(
@@ -251,15 +264,17 @@ def _validate_interval_columns(
251
264
  return chr_values, start_int, end_int
252
265
 
253
266
 
254
- def _binary_intervals_by_chr(path: Path) -> dict[str, np.ndarray]:
255
- df, _, row_base0 = _read_annotation_table(path, header=False)
267
+ def _binary_intervals_by_chr(path: Path) -> tuple[dict[str, np.ndarray], int]:
268
+ df, _, row_base0 = _read_annotation_table(path, has_header=False)
256
269
  if df.shape[1] < 3:
257
270
  raise ValueError(f"{path}: BED must have at least 3 tab-separated columns")
258
271
  chr_values, starts, ends = _validate_interval_columns(df, path, row_base0=row_base0)
259
- return binary_intervals_by_chr_from_arrays(chr_values, starts, ends)
272
+ return binary_intervals_by_chr_from_arrays(chr_values, starts, ends), int(df.shape[0])
260
273
 
261
274
 
262
- def _paint_binary_annotations(bed_paths: list[Path], reference) -> tuple[sparse.csr_matrix, list[str]]:
275
+ def _paint_binary_annotations(
276
+ bed_paths: list[Path], reference
277
+ ) -> tuple[sparse.csr_matrix, list[str], list[int]]:
263
278
  annonames = [_annotation_name_from_path(p) for p in bed_paths]
264
279
  if len(set(annonames)) != len(annonames):
265
280
  raise ValueError("duplicate annotation names derived from BED basenames")
@@ -267,47 +282,79 @@ def _paint_binary_annotations(bed_paths: list[Path], reference) -> tuple[sparse.
267
282
  n = int(reference.num_snp)
268
283
  k = len(bed_paths)
269
284
  if n == 0:
270
- return sparse.csr_matrix((0, k), dtype=np.float64), annonames
285
+ counts = []
286
+ for bed_path in bed_paths:
287
+ _, count = _binary_intervals_by_chr(bed_path)
288
+ counts.append(count)
289
+ return sparse.csr_matrix((0, k), dtype=np.float64), annonames, counts
271
290
 
272
291
  columns = []
292
+ counts = []
273
293
  for bed_path in bed_paths:
274
- intervals_by_chr = _binary_intervals_by_chr(bed_path)
294
+ intervals_by_chr, count = _binary_intervals_by_chr(bed_path)
275
295
  columns.append(paint_binary_column(intervals_by_chr, reference))
296
+ counts.append(count)
276
297
 
277
- return sparse.hstack(columns, format="csr"), annonames
298
+ return sparse.hstack(columns, format="csr"), annonames, counts
278
299
 
279
300
 
280
- def _normalize_value_columns(value_columns, header_fields, num_columns: int, path: Path) -> list[int] | None:
281
- if value_columns is None:
301
+ def _selector_list(selectors, name: str) -> list | None:
302
+ if selectors is None:
282
303
  return None
283
- if isinstance(value_columns, (str, int, np.integer)):
284
- selectors = [value_columns]
304
+ if isinstance(selectors, (str, int, np.integer)):
305
+ values = [selectors]
285
306
  else:
286
- selectors = list(value_columns)
287
- if not selectors:
288
- raise ValueError("value_columns must be a non-empty list")
307
+ values = list(selectors)
308
+ if not values:
309
+ raise ValueError(f"{name} must be a non-empty list")
310
+ return values
311
+
312
+
313
+ def _normalize_column_selectors(selectors, header_fields, num_columns: int, path: Path, name: str) -> tuple[list[int], list] | None:
314
+ selector_values = _selector_list(selectors, name)
315
+ if selector_values is None:
316
+ return None
289
317
 
290
318
  out = []
291
- for selector in selectors:
319
+ metadata_values = []
320
+ singular = name[:-1] if name.endswith("s") else name
321
+ for selector in selector_values:
292
322
  if isinstance(selector, str):
293
323
  if header_fields is None:
294
- raise ValueError("named value_columns are invalid when header=False")
324
+ raise ValueError(f"named {name} are invalid when has_header=False")
295
325
  if selector not in header_fields:
296
- raise ValueError(f"{path}: unknown value column name {selector!r}")
326
+ raise ValueError(f"{path}: unknown {singular} name {selector!r}")
297
327
  col0 = header_fields.index(selector)
328
+ metadata_value = selector
298
329
  elif isinstance(selector, (int, np.integer)) and not isinstance(selector, bool):
299
330
  col0 = int(selector)
331
+ metadata_value = int(selector)
300
332
  else:
301
- raise ValueError("value_columns must contain column names or integer indices")
333
+ raise ValueError(f"{name} must contain column names or integer indices")
302
334
  if col0 < 3 or col0 >= num_columns:
303
335
  raise ValueError(
304
- f"{path}: value column {selector!r} is out of range; selected columns must be physical columns 4 or later"
336
+ f"{path}: {singular} {selector!r} is out of range; selected columns must be physical columns 4 or later"
305
337
  )
306
338
  out.append(col0)
339
+ metadata_values.append(metadata_value)
307
340
 
308
341
  if len(set(out)) != len(out):
309
- raise ValueError("value_columns must not contain duplicates")
310
- return out
342
+ raise ValueError(f"{name} must not contain duplicates")
343
+ return out, metadata_values
344
+
345
+
346
+ def _normalize_value_columns(value_columns, header_fields, num_columns: int, path: Path) -> tuple[list[int], list] | None:
347
+ return _normalize_column_selectors(value_columns, header_fields, num_columns, path, "value_columns")
348
+
349
+
350
+ def _normalize_group_column(group_column, header_fields, num_columns: int, path: Path) -> tuple[int, object] | None:
351
+ normalized = _normalize_column_selectors(group_column, header_fields, num_columns, path, "group_column")
352
+ if normalized is None:
353
+ return None
354
+ columns, metadata_values = normalized
355
+ if len(columns) != 1:
356
+ raise ValueError("group_column must identify exactly one source column")
357
+ return columns[0], metadata_values[0]
311
358
 
312
359
 
313
360
  def _default_annotation_names(
@@ -323,7 +370,7 @@ def _default_annotation_names(
323
370
  if header_fields is None:
324
371
  return [_annotation_name_from_path(path)]
325
372
  return [str(header_fields[3])]
326
- raise ValueError(f"{path}: input with five or more columns requires explicit value_columns")
373
+ raise ValueError(f"{path}: input with five or more columns requires explicit value_columns or group_column")
327
374
 
328
375
  if header_fields is None and num_columns >= 5:
329
376
  raise ValueError(f"{path}: headerless input with five or more columns requires annotation_names")
@@ -613,7 +660,7 @@ def load_annotations(
613
660
  if not p.is_file():
614
661
  raise FileNotFoundError(f"BED file not found: {p}")
615
662
 
616
- annomat, annonames = _paint_binary_annotations(paths, reference)
663
+ annomat, annonames, source_counts = _paint_binary_annotations(paths, reference)
617
664
  if annotation_metadata is not None:
618
665
  metadata = _coerce_string_vector(annotation_metadata, len(paths), "annotation_metadata")
619
666
  elif annotation_metadata_paths is not None:
@@ -623,14 +670,17 @@ def load_annotations(
623
670
  f"annotation_metadata_paths length mismatch: expected {len(paths)}, got {len(meta_paths)}"
624
671
  )
625
672
  values = []
626
- for bed_path, meta_path in zip(paths, meta_paths):
673
+ for bed_path, meta_path, count in zip(paths, meta_paths, source_counts):
627
674
  if meta_path is None or str(meta_path) == "":
628
- values.append(_generated_metadata(bed_path, None, None))
675
+ values.append(_generated_metadata(bed_path, False, count))
629
676
  else:
630
677
  values.append(_read_sidecar_exact(meta_path))
631
678
  metadata = np.asarray(values, dtype=object)
632
679
  else:
633
- metadata = np.asarray([_generated_metadata(p, None, None) for p in paths], dtype=object)
680
+ metadata = np.asarray(
681
+ [_generated_metadata(p, False, count) for p, count in zip(paths, source_counts)],
682
+ dtype=object,
683
+ )
634
684
 
635
685
  return create_annotations(
636
686
  reference,
@@ -644,26 +694,67 @@ def load_annotations(
644
694
  def load_annotation(
645
695
  path,
646
696
  reference,
647
- header: bool = False,
697
+ has_header: bool = False,
648
698
  value_columns=None,
699
+ group_column=None,
649
700
  annotation_names=None,
650
701
  annotation_metadata=None,
651
702
  annotation_metadata_path=None,
652
703
  ) -> AnnotationPanel:
653
704
  if annotation_metadata is not None and annotation_metadata_path is not None:
654
705
  raise ValueError("load_annotation accepts at most one of annotation_metadata and annotation_metadata_path")
706
+ if value_columns is not None and group_column is not None:
707
+ raise ValueError("group_column and value_columns are mutually exclusive")
708
+ if group_column is not None and annotation_names is not None:
709
+ raise ValueError("annotation_names is invalid with group_column")
710
+ if group_column is not None and annotation_metadata is not None:
711
+ raise ValueError("annotation_metadata is invalid with group_column")
712
+ if group_column is not None and annotation_metadata_path is not None:
713
+ raise ValueError("annotation_metadata_path is invalid with group_column")
655
714
 
656
715
  source_path = Path(path)
657
- df, header_fields, row_base0 = _read_annotation_table(source_path, bool(header))
716
+ df, header_fields, row_base0 = _read_annotation_table(source_path, bool(has_header))
658
717
  num_columns = int(df.shape[1])
659
718
  if num_columns < 3:
660
719
  raise ValueError(f"{source_path}: annotation input must have at least 3 tab-separated columns")
661
720
 
662
- value_columns0 = _normalize_value_columns(value_columns, header_fields, num_columns, source_path)
721
+ value_normalized = _normalize_value_columns(value_columns, header_fields, num_columns, source_path)
722
+ value_columns0 = None if value_normalized is None else value_normalized[0]
723
+ value_column_metadata = None if value_normalized is None else value_normalized[1]
724
+ group_normalized = _normalize_group_column(group_column, header_fields, num_columns, source_path)
725
+ group_column0 = None if group_normalized is None else group_normalized[0]
726
+ group_column_metadata = None if group_normalized is None else group_normalized[1]
663
727
  chr_values, starts, ends = _validate_interval_columns(df, source_path, row_base0=row_base0)
664
728
  n = int(reference.num_snp)
665
-
666
- if value_columns0 is None and num_columns == 3:
729
+ num_source_intervals = int(df.shape[0])
730
+
731
+ if group_column0 is not None:
732
+ group_values = df.iloc[:, group_column0].astype(str).to_numpy(dtype=object)
733
+ empty = group_values == ""
734
+ if empty.any():
735
+ idx = int(np.flatnonzero(empty)[0])
736
+ raise ValueError(f"{source_path}: row {row_base0 + idx + 1}: group_column values must be non-empty")
737
+ names = [str(x) for x in pd.unique(group_values)]
738
+ columns = []
739
+ metadata_values = []
740
+ # PERF: loop retained because grouped binary interval union is defined independently per group.
741
+ for group_value in names:
742
+ rows = group_values == group_value
743
+ intervals_by_chr = binary_intervals_by_chr_from_arrays(chr_values[rows], starts[rows], ends[rows])
744
+ columns.append(paint_binary_column(intervals_by_chr, reference))
745
+ metadata_values.append(
746
+ _generated_metadata(
747
+ source_path,
748
+ bool(has_header),
749
+ int(np.count_nonzero(rows)),
750
+ group_column=group_column_metadata,
751
+ group_value=group_value,
752
+ )
753
+ )
754
+ annomat = sparse.hstack(columns, format="csr") if columns else sparse.csr_matrix((n, 0), dtype=np.float64)
755
+ is_binary = np.ones(len(names), dtype=bool)
756
+ metadata = np.asarray(metadata_values, dtype=object)
757
+ elif value_columns0 is None and num_columns == 3:
667
758
  names = (
668
759
  _default_annotation_names(source_path, num_columns, header_fields, value_columns0)
669
760
  if annotation_names is None
@@ -674,23 +765,22 @@ def load_annotation(
674
765
  intervals_by_chr = binary_intervals_by_chr_from_arrays(chr_values, starts, ends)
675
766
  annomat = paint_binary_column(intervals_by_chr, reference)
676
767
  is_binary = np.array([True], dtype=bool)
677
- source_columns = [None]
678
- source_column_names = [None]
679
768
  if annotation_metadata_path is not None:
680
769
  metadata = np.asarray([_read_sidecar_exact(annotation_metadata_path)], dtype=object)
681
770
  elif annotation_metadata is not None:
682
771
  metadata = _coerce_string_vector(annotation_metadata, 1, "annotation_metadata")
683
772
  else:
684
773
  metadata = np.asarray(
685
- [_generated_metadata(source_path, source_columns[0], source_column_names[0])],
774
+ [_generated_metadata(source_path, bool(has_header), num_source_intervals)],
686
775
  dtype=object,
687
776
  )
688
777
  else:
689
778
  if value_columns0 is None:
690
779
  if num_columns == 4:
691
780
  value_columns0 = [3]
781
+ value_column_metadata = [str(header_fields[3])] if header_fields is not None else [3]
692
782
  else:
693
- raise ValueError(f"{source_path}: input with five or more columns requires explicit value_columns")
783
+ raise ValueError(f"{source_path}: input with five or more columns requires explicit value_columns or group_column")
694
784
  names = (
695
785
  _default_annotation_names(source_path, num_columns, header_fields, value_columns0)
696
786
  if annotation_names is None
@@ -732,10 +822,6 @@ def load_annotation(
732
822
  blocks[shard_idx] = paint_numeric_sparse(ref_shard.bp, intervals, value_block)
733
823
  annomat = sparse.vstack(blocks, format="csr")
734
824
  is_binary = np.zeros(len(value_columns0), dtype=bool)
735
- source_columns = value_columns0
736
- source_column_names = [
737
- None if header_fields is None else str(header_fields[i]) for i in value_columns0
738
- ]
739
825
 
740
826
  if annotation_metadata_path is not None:
741
827
  sidecar_lines = _read_column_metadata_sidecar(annotation_metadata_path, num_columns)
@@ -747,8 +833,13 @@ def load_annotation(
747
833
  else:
748
834
  metadata = np.asarray(
749
835
  [
750
- _generated_metadata(source_path, col0, col_name)
751
- for col0, col_name in zip(source_columns, source_column_names)
836
+ _generated_metadata(
837
+ source_path,
838
+ bool(has_header),
839
+ num_source_intervals,
840
+ value_column=value_column,
841
+ )
842
+ for value_column in value_column_metadata
752
843
  ],
753
844
  dtype=object,
754
845
  )
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import json
2
4
  import warnings
3
5
  from dataclasses import dataclass
@@ -15,11 +17,13 @@ from ._bfile_utils import (
15
17
  _read_bed_rows_int8,
16
18
  _validate_bed,
17
19
  )
18
- from ._utils import allele_hash64, validate_requested_shards
20
+ from ._utils import _validate_allow_indel_swaps, allele_hash64, validate_requested_shards
21
+ from ._verbosity import report_variant_swaps
19
22
  from ._variant_match import match_shard_numeric, numeric_variant_keys
20
23
 
21
- _CACHE_SCHEMA = "genotype_cache/0.1"
24
+ _CACHE_SCHEMA = "genotype_cache/0.2"
22
25
  _BED_MISSING_INT8 = np.int8(-1)
26
+ _SWAP_LUT = np.array([-1, 2, 1, 0], dtype=np.int8)
23
27
  _HAPLOID_MODES = {"raw", "ploidy_scaled"}
24
28
  _CACHE_ARRAY_FIELDS = (
25
29
  "is_present",
@@ -150,6 +154,7 @@ class GenotypeShard:
150
154
  subject_present,
151
155
  source_subject_row0,
152
156
  is_present,
157
+ is_allele_swapped,
153
158
  ploidy_male,
154
159
  ploidy_female,
155
160
  reference_checksum,
@@ -170,10 +175,12 @@ class GenotypeShard:
170
175
 
171
176
  self._source_row0 = np.asarray(source_row0, dtype=np.int64).reshape(-1)
172
177
  self._is_present = np.asarray(is_present, dtype=bool).reshape(-1)
178
+ self._is_allele_swapped = np.asarray(is_allele_swapped, dtype=bool).reshape(-1)
173
179
  self._ploidy_male = np.asarray(ploidy_male, dtype=float).reshape(-1)
174
180
  self._ploidy_female = np.asarray(ploidy_female, dtype=float).reshape(-1)
175
181
  if not (
176
182
  self._source_row0.size
183
+ == self._is_allele_swapped.size
177
184
  == self._is_present.size
178
185
  == self._ploidy_male.size
179
186
  == self._ploidy_female.size
@@ -184,6 +191,8 @@ class GenotypeShard:
184
191
  bad_source = (self._source_row0 < -1) | (self._source_row0 >= self._source_num_snp)
185
192
  if bad_source.any():
186
193
  raise ValueError(f"GenotypeShard {self._label}: source_row0 out of source BIM bounds")
194
+ if np.any(self._is_allele_swapped & ~self._is_present):
195
+ raise ValueError("is_allele_swapped requires is_present")
187
196
  absent = ~self._is_present
188
197
  if np.any(~np.isnan(self._ploidy_male[absent])) or np.any(~np.isnan(self._ploidy_female[absent])):
189
198
  raise ValueError(f"GenotypeShard {self._label}: absent SNPs must have NaN ploidy")
@@ -207,6 +216,7 @@ class GenotypeShard:
207
216
  for name in (
208
217
  "_source_row0",
209
218
  "_is_present",
219
+ "_is_allele_swapped",
210
220
  "_ploidy_male",
211
221
  "_ploidy_female",
212
222
  "_subject_present",
@@ -258,6 +268,10 @@ class GenotypeShard:
258
268
  def is_present(self) -> np.ndarray:
259
269
  return self._is_present
260
270
 
271
+ @property
272
+ def is_allele_swapped(self) -> np.ndarray:
273
+ return self._is_allele_swapped
274
+
261
275
  @property
262
276
  def ploidy_male(self) -> np.ndarray:
263
277
  return self._ploidy_male
@@ -340,6 +354,12 @@ class GenotypePanel:
340
354
  return np.array([], dtype=bool)
341
355
  return np.concatenate([s.is_present for s in self._shards])
342
356
 
357
+ @property
358
+ def is_allele_swapped(self) -> np.ndarray:
359
+ if not self._shards:
360
+ return np.array([], dtype=bool)
361
+ return np.concatenate([s.is_allele_swapped for s in self._shards])
362
+
343
363
  @property
344
364
  def ploidy_male(self) -> np.ndarray:
345
365
  if not self._shards:
@@ -413,7 +433,7 @@ class GenotypePanel:
413
433
 
414
434
  def _addressed_shards(self, snp_indices: np.ndarray) -> list[tuple[GenotypeShard, np.ndarray, np.ndarray]]:
415
435
  addressed = []
416
- for shard, offset in zip(self._shards, self._shard_offsets, strict=True):
436
+ for shard, offset in zip(self._shards, self._shard_offsets):
417
437
  start = int(offset["start0"])
418
438
  stop = int(offset["stop0"])
419
439
  cols = np.flatnonzero((snp_indices >= start) & (snp_indices < stop))
@@ -475,6 +495,9 @@ class GenotypePanel:
475
495
  source_rows,
476
496
  shard.source_num_sample,
477
497
  )
498
+ sc = np.flatnonzero(shard.is_allele_swapped[local_indices])
499
+ if sc.size:
500
+ source_geno[:, sc] = _SWAP_LUT[source_geno[:, sc] + 1]
478
501
  panel_rows = np.flatnonzero(shard.subject_present)
479
502
  if panel_rows.size:
480
503
  source_subject_rows = shard.source_subject_row0[panel_rows]
@@ -572,8 +595,9 @@ def _map_chrx_subjects(chrx_fam: pd.DataFrame, panel_fam: pd.DataFrame, where: s
572
595
  return subject_present, source_subject_row0
573
596
 
574
597
 
575
- def _build_shard(ref_shard, source: _SourceRecord, subject_present, source_subject_row0) -> GenotypeShard:
598
+ def _build_shard(ref_shard, source: _SourceRecord, subject_present, source_subject_row0, allow_indel_swaps) -> GenotypeShard:
576
599
  source_bim = source.bim.loc[source.bim["chr"] == ref_shard.label]
600
+ is_allele_swapped = np.zeros(int(ref_shard.num_snp), dtype=bool)
577
601
  if source_bim.empty:
578
602
  local_match = np.full(int(ref_shard.num_snp), -1, dtype=np.int64)
579
603
  else:
@@ -583,15 +607,14 @@ def _build_shard(ref_shard, source: _SourceRecord, subject_present, source_subje
583
607
  source_bim["a1_hash64"].to_numpy(dtype=np.uint64),
584
608
  source_bim["a2_hash64"].to_numpy(dtype=np.uint64),
585
609
  )
586
- local_match, n_swapped = match_shard_numeric(ref_keys, src_keys, ref_shard.label, source_name="genotype")
587
- if n_swapped > 0:
588
- warnings.warn(
589
- f"{source.bim_path}: shard {ref_shard.label}: {n_swapped} unmatched genotype "
590
- "variant(s) would match the reference if a1/a2 were swapped; "
591
- "variants remain unmatched",
592
- RuntimeWarning,
593
- stacklevel=3,
594
- )
610
+ match = match_shard_numeric(
611
+ ref_keys, src_keys, ref_shard.label, source_name="genotype",
612
+ allow_indel_swaps=allow_indel_swaps,
613
+ )
614
+ local_match = match.source_rows
615
+ is_allele_swapped = match.is_allele_swapped
616
+ report_variant_swaps(source.bim_path, ref_shard.label, np.count_nonzero(is_allele_swapped),
617
+ match.num_strand_ambiguous, match.num_disabled_indels)
595
618
 
596
619
  is_present = local_match >= 0
597
620
  source_row0 = np.full(int(ref_shard.num_snp), -1, dtype=np.int64)
@@ -613,6 +636,7 @@ def _build_shard(ref_shard, source: _SourceRecord, subject_present, source_subje
613
636
  subject_present=subject_present,
614
637
  source_subject_row0=source_subject_row0,
615
638
  is_present=is_present,
639
+ is_allele_swapped=is_allele_swapped,
616
640
  ploidy_male=ploidy_male,
617
641
  ploidy_female=ploidy_female,
618
642
  reference_checksum=ref_shard.checksum,
@@ -628,7 +652,8 @@ def _validate_reference_for_genotype(reference) -> list:
628
652
  return ref_shards
629
653
 
630
654
 
631
- def load_genotype(bfile_prefix, reference) -> GenotypePanel:
655
+ def load_genotype(bfile_prefix, reference, allow_indel_swaps=False) -> GenotypePanel:
656
+ allow_indel_swaps = _validate_allow_indel_swaps(allow_indel_swaps)
632
657
  ref_shards = _validate_reference_for_genotype(reference)
633
658
  prefix = str(bfile_prefix)
634
659
  ref_labels = [s.label for s in ref_shards]
@@ -682,7 +707,7 @@ def load_genotype(bfile_prefix, reference) -> GenotypePanel:
682
707
  subject_maps["X"] = _all_subjects_present(sources["X"].source_num_sample)
683
708
 
684
709
  shards = [
685
- _build_shard(ref_shard, sources[ref_shard.label], *subject_maps[ref_shard.label])
710
+ _build_shard(ref_shard, sources[ref_shard.label], *subject_maps[ref_shard.label], allow_indel_swaps)
686
711
  for ref_shard in ref_shards
687
712
  ]
688
713
  return GenotypePanel(
@@ -714,6 +739,7 @@ def save_genotype_cache(panel: GenotypePanel, path) -> None:
714
739
  arrays = {
715
740
  "_meta": np.frombuffer(json.dumps(meta).encode(), dtype=np.uint8),
716
741
  "is_present": panel.is_present.astype(bool, copy=False),
742
+ "is_allele_swapped": panel.is_allele_swapped,
717
743
  "ploidy_male": panel.ploidy_male.astype(float, copy=False),
718
744
  "ploidy_female": panel.ploidy_female.astype(float, copy=False),
719
745
  "source_row0": panel.source_row0.astype(np.int64, copy=False),
@@ -744,9 +770,10 @@ def _load_cache_meta(data) -> dict:
744
770
 
745
771
  def _validate_cache_meta(meta: dict, data) -> None:
746
772
  schema = meta.get("schema")
747
- if schema != _CACHE_SCHEMA:
773
+ if schema not in {_CACHE_SCHEMA, "genotype_cache/0.1"}:
748
774
  raise ValueError(f"Unsupported genotype cache schema: {schema!r}")
749
- missing = [name for name in _CACHE_ARRAY_FIELDS if name not in data.files]
775
+ required = _CACHE_ARRAY_FIELDS + (("is_allele_swapped",) if schema == _CACHE_SCHEMA else ())
776
+ missing = [name for name in required if name not in data.files]
750
777
  if missing:
751
778
  raise ValueError(f"Invalid genotype cache: missing array field {missing[0]!r}")
752
779
  n_shards = int(meta.get("n_shards", -1))
@@ -773,6 +800,12 @@ def _validate_cache_meta(meta: dict, data) -> None:
773
800
  raise ValueError("Invalid genotype cache: source_layout must be 'non_sharded' or 'sharded'")
774
801
 
775
802
  is_present = data["is_present"]
803
+ if schema == _CACHE_SCHEMA:
804
+ swapped = data["is_allele_swapped"]
805
+ if swapped.dtype != np.dtype(bool) or swapped.shape != is_present.shape:
806
+ raise ValueError("Invalid genotype cache: is_allele_swapped must be a logical SNP-axis vector")
807
+ if np.any(swapped & ~is_present.astype(bool)):
808
+ raise ValueError("Invalid genotype cache: is_allele_swapped requires is_present")
776
809
  source_row0 = data["source_row0"]
777
810
  if is_present.ndim != 1 or source_row0.ndim != 1 or is_present.shape[0] != source_row0.shape[0]:
778
811
  raise ValueError("Invalid genotype cache: SNP-axis vector lengths mismatch")
@@ -817,6 +850,8 @@ def load_genotype_cache(path, shards=None) -> GenotypePanel:
817
850
  selected = validate_requested_shards(shards, labels, "load_genotype_cache")
818
851
  label_to_index = {label: i for i, label in enumerate(labels)}
819
852
 
853
+ swap_flags = (data["is_allele_swapped"] if meta["schema"] == _CACHE_SCHEMA
854
+ else np.zeros(data["is_present"].size, dtype=bool))
820
855
  shard_objs = []
821
856
  for label in selected:
822
857
  i = label_to_index[label]
@@ -833,6 +868,7 @@ def load_genotype_cache(path, shards=None) -> GenotypePanel:
833
868
  subject_present=data["subject_present"][:, i],
834
869
  source_subject_row0=data["source_subject_row0"][:, i],
835
870
  is_present=data["is_present"][start:stop],
871
+ is_allele_swapped=swap_flags[start:stop],
836
872
  ploidy_male=data["ploidy_male"][start:stop],
837
873
  ploidy_female=data["ploidy_female"][start:stop],
838
874
  reference_checksum=meta["shard_checksums"][i],
@@ -9,34 +9,32 @@ import numpy as np
9
9
  import pandas as pd
10
10
 
11
11
  from ._bfile_utils import _parse_bim
12
- from ._utils import CANONICAL_CHR_ORDER, allele_hash64, validate_requested_shards
12
+ from ._utils import (
13
+ CANONICAL_CHR_ORDER,
14
+ _is_single_nucleotide_variant,
15
+ _is_strand_ambiguous,
16
+ allele_hash64,
17
+ validate_requested_shards,
18
+ )
13
19
 
14
20
  logger = logging.getLogger(__name__)
15
21
 
16
22
  _CACHE_SCHEMA = "reference_cache/0.1"
17
- _A_HASH64, _C_HASH64, _G_HASH64, _T_HASH64 = allele_hash64(
18
- np.array(["A", "C", "G", "T"], dtype=object)
19
- )
20
- _SINGLE_BASE_HASH64 = np.array(
21
- [_A_HASH64, _C_HASH64, _G_HASH64, _T_HASH64], dtype=np.uint64
22
- )
23
23
 
24
24
 
25
- def _is_single_nucleotide_variant(a1_hash64, a2_hash64) -> np.ndarray:
26
- return np.isin(a1_hash64, _SINGLE_BASE_HASH64) & np.isin(
27
- a2_hash64, _SINGLE_BASE_HASH64
28
- )
29
-
30
-
31
- def _is_strand_ambiguous(a1_hash64, a2_hash64) -> np.ndarray:
32
- a1 = np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
33
- a2 = np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
34
- return (
35
- ((a1 == _A_HASH64) & (a2 == _T_HASH64))
36
- | ((a1 == _T_HASH64) & (a2 == _A_HASH64))
37
- | ((a1 == _C_HASH64) & (a2 == _G_HASH64))
38
- | ((a1 == _G_HASH64) & (a2 == _C_HASH64))
39
- )
25
+ def _validate_reference_keys(shard) -> None:
26
+ # Ordered-key duplicates have already been rejected by the BIM parser.
27
+ snv = shard.is_single_nucleotide_variant
28
+ bp = shard.bp[snv]
29
+ a1 = shard.a1_hash64[snv]
30
+ a2 = shard.a2_hash64[snv]
31
+ lo, hi = np.minimum(a1, a2), np.maximum(a1, a2)
32
+ order = np.lexsort((hi, lo, bp))
33
+ left, right = order[:-1], order[1:]
34
+ duplicate = (bp[left] == bp[right]) & (lo[left] == lo[right]) & (hi[left] == hi[right])
35
+ if np.any(duplicate):
36
+ pos = right[np.flatnonzero(duplicate)[0]]
37
+ raise ValueError(f"reference shard {shard.label}: reciprocal SNV keys are not allowed at bp={int(bp[pos])}")
40
38
 
41
39
 
42
40
  def _checksum_from_arrays(
@@ -410,6 +408,8 @@ def load_reference(path, shards=None) -> ReferencePanel:
410
408
  bim_df["a2"].to_numpy(dtype=object),
411
409
  )
412
410
  )
411
+ for shard in shards:
412
+ _validate_reference_keys(shard)
413
413
  return ReferencePanel(shards)
414
414
 
415
415
  bim_df = _parse_bim(path_obj)
@@ -431,6 +431,8 @@ def load_reference(path, shards=None) -> ReferencePanel:
431
431
  shard_df["a2"].to_numpy(dtype=object),
432
432
  )
433
433
  )
434
+ for shard in out_shards:
435
+ _validate_reference_keys(shard)
434
436
  return ReferencePanel(out_shards)
435
437
 
436
438
 
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import json
2
4
  import re
3
5
  import warnings
@@ -8,7 +10,8 @@ import numpy as np
8
10
  import pandas as pd
9
11
  from pandas.errors import EmptyDataError, ParserError
10
12
 
11
- from ._utils import CANONICAL_CHR_ORDER, allele_hash64, validate_requested_shards
13
+ from ._utils import CANONICAL_CHR_ORDER, _validate_allow_indel_swaps, allele_hash64, validate_requested_shards
14
+ from ._verbosity import info, report_variant_swaps
12
15
  from ._variant_match import match_shard_numeric, numeric_variant_keys
13
16
 
14
17
  _CACHE_SCHEMA = "sumstats_cache/0.1"
@@ -136,15 +139,19 @@ def _parse_sumstats(path: Path) -> pd.DataFrame:
136
139
  out["bp"] = out["bp"].astype(np.int64)
137
140
 
138
141
  p_arr = out["p"].to_numpy(dtype=float)
139
- bad_p = out["p"].isna() | ~np.isfinite(p_arr) | (p_arr < 0.0) | (p_arr > 1.0)
142
+ bad_p = ~np.isnan(p_arr) & (~np.isfinite(p_arr) | (p_arr < 0.0) | (p_arr > 1.0))
140
143
  if bad_p.any():
141
- idx = int(bad_p.idxmax())
144
+ idx = int(out.index[np.flatnonzero(bad_p)[0]])
142
145
  raise ValueError(f"{path}: row {idx + 2}: p must be finite numeric in [0, 1]: {out.loc[idx, 'p']!r}")
143
146
 
144
147
  for opt in _OPTIONAL_COLS:
145
148
  if opt in out.columns:
146
149
  out[opt] = out[opt].mask(~np.isfinite(out[opt].to_numpy(dtype=float)))
147
150
 
151
+ missing_p = out["p"].isna()
152
+ if missing_p.any():
153
+ info(f"{path}: dropped {int(missing_p.sum())} row(s) with missing p")
154
+ return out.loc[~missing_p].copy()
148
155
  return out
149
156
 
150
157
 
@@ -336,7 +343,7 @@ def _derive_logp(p_vals: np.ndarray) -> np.ndarray:
336
343
 
337
344
 
338
345
  def _validate_pvec(p_vals: np.ndarray, name: str) -> None:
339
- bad = ~np.isfinite(p_vals) | (p_vals < 0.0) | (p_vals > 1.0)
346
+ bad = ~np.isnan(p_vals) & (~np.isfinite(p_vals) | (p_vals < 0.0) | (p_vals > 1.0))
340
347
  if bad.any():
341
348
  idx = int(np.flatnonzero(bad)[0])
342
349
  raise ValueError(f"{name}[{idx}] must be finite numeric in [0, 1]")
@@ -396,18 +403,7 @@ def _build_sumstats_from_aligned(
396
403
  return Sumstats(shards)
397
404
 
398
405
 
399
- def _warn_swapped_allele_matches(source_path, shard_label: str, count: int, context: str) -> None:
400
- if count > 0:
401
- warnings.warn(
402
- f"{source_path}: shard {shard_label}: {count} unmatched {context} "
403
- "variant(s) would match the reference if a1/a2 were swapped; "
404
- "variants remain unmatched",
405
- RuntimeWarning,
406
- stacklevel=3,
407
- )
408
-
409
-
410
- def _build_sumstats_panel(df: pd.DataFrame, reference, source_path) -> Sumstats:
406
+ def _build_sumstats_panel(df: pd.DataFrame, reference, source_path, allow_indel_swaps=False) -> Sumstats:
411
407
  n = int(reference.num_snp)
412
408
  aligned_optional = {
413
409
  col: (np.full(n, np.nan, dtype=float) if col in df.columns else None)
@@ -427,8 +423,13 @@ def _build_sumstats_panel(df: pd.DataFrame, reference, source_path) -> Sumstats:
427
423
  src_idx = np.flatnonzero(src_mask)
428
424
  ref_keys = numeric_variant_keys(ref_shard.bp, ref_shard.a1_hash64, ref_shard.a2_hash64)
429
425
  src_keys = numeric_variant_keys(src_bp[src_idx], src_a1_hash64[src_idx], src_a2_hash64[src_idx])
430
- local_match, n_swapped = match_shard_numeric(ref_keys, src_keys, ref_shard.label, source_name="sumstats")
431
- _warn_swapped_allele_matches(source_path, ref_shard.label, n_swapped, "sumstats")
426
+ match = match_shard_numeric(
427
+ ref_keys, src_keys, ref_shard.label, source_name="sumstats",
428
+ allow_indel_swaps=allow_indel_swaps,
429
+ )
430
+ local_match = match.source_rows
431
+ report_variant_swaps(source_path, ref_shard.label, np.count_nonzero(match.is_allele_swapped),
432
+ match.num_strand_ambiguous, match.num_disabled_indels)
432
433
  has_match = local_match >= 0
433
434
  matched_src = src_idx[local_match[has_match]]
434
435
  aligned_ix = np.arange(start, stop, dtype=np.int64)[has_match]
@@ -437,6 +438,14 @@ def _build_sumstats_panel(df: pd.DataFrame, reference, source_path) -> Sumstats:
437
438
  for col, out in aligned_optional.items():
438
439
  if out is not None:
439
440
  out[aligned_ix] = df[col].to_numpy(dtype=float)[matched_src]
441
+ swapped_ix = np.arange(start, stop)[match.is_allele_swapped]
442
+ for col in ("z", "beta", "eaf"):
443
+ out = aligned_optional[col]
444
+ if out is not None:
445
+ out[swapped_ix] = 1 - out[swapped_ix] if col == "eaf" else -out[swapped_ix]
446
+ if not has_match.any():
447
+ warnings.warn(f"{source_path}: no matching source rows for reference shard {ref_shard.label!r}",
448
+ RuntimeWarning, stacklevel=3)
440
449
 
441
450
  aligned_logp = _derive_logp(aligned_p)
442
451
  _warn_optional_zn_completeness(
@@ -454,9 +463,10 @@ def _build_sumstats_panel(df: pd.DataFrame, reference, source_path) -> Sumstats:
454
463
  )
455
464
 
456
465
 
457
- def load_sumstats(path, reference) -> Sumstats:
466
+ def load_sumstats(path, reference, allow_indel_swaps=False) -> Sumstats:
467
+ allow_indel_swaps = _validate_allow_indel_swaps(allow_indel_swaps)
458
468
  path = Path(path)
459
- return _build_sumstats_panel(_parse_sumstats(path), reference, path)
469
+ return _build_sumstats_panel(_parse_sumstats(path), reference, path, allow_indel_swaps)
460
470
 
461
471
 
462
472
  def create_sumstats(
@@ -483,6 +493,10 @@ def create_sumstats(
483
493
  "info": None if info is None else _coerce_aligned_vec("info", info, n=num_snp_value, allow_inf=False),
484
494
  }
485
495
 
496
+ for name, vec in aligned_optional.items():
497
+ if vec is not None:
498
+ aligned_optional[name] = np.where(np.isnan(aligned_p), np.nan, vec)
499
+
486
500
  return _build_sumstats_from_aligned(
487
501
  reference,
488
502
  aligned_logp=aligned_logp,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: statgen
3
- Version: 0.3.3
3
+ Version: 0.4.0
4
4
  Summary: Statistical genetics data objects and loaders for Python and MATLAB/Octave.
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -1,76 +0,0 @@
1
- import numpy as np
2
-
3
- _KEY_DTYPE = np.dtype([("bp", "<i8"), ("a1_hash64", "<u8"), ("a2_hash64", "<u8")])
4
-
5
-
6
- def numeric_variant_keys(bp_arr, a1_hash64, a2_hash64) -> np.ndarray:
7
- bp = np.asarray(bp_arr, dtype=np.int64).reshape(-1)
8
- a1h = np.asarray(a1_hash64, dtype=np.uint64).reshape(-1)
9
- a2h = np.asarray(a2_hash64, dtype=np.uint64).reshape(-1)
10
- if not (bp.size == a1h.size == a2h.size):
11
- raise ValueError("numeric variant key vector lengths mismatch")
12
- keys = np.empty(bp.size, dtype=_KEY_DTYPE)
13
- keys["bp"] = bp
14
- keys["a1_hash64"] = a1h
15
- keys["a2_hash64"] = a2h
16
- return keys
17
-
18
-
19
- def check_unique_sorted_variant_keys(sorted_keys: np.ndarray, where: str, object_name: str = "source") -> None:
20
- if sorted_keys.size < 2:
21
- return
22
- dup = sorted_keys[1:] == sorted_keys[:-1]
23
- if np.any(dup):
24
- pos = int(np.flatnonzero(dup)[0] + 1)
25
- key = sorted_keys[pos]
26
- raise ValueError(
27
- f"Ambiguous duplicate {object_name}/reference matching key in {where}: "
28
- f"bp={int(key['bp'])}, a1_hash64={int(key['a1_hash64'])}, "
29
- f"a2_hash64={int(key['a2_hash64'])}"
30
- )
31
-
32
-
33
- def match_shard_numeric(
34
- ref_keys: np.ndarray,
35
- src_keys: np.ndarray,
36
- shard_label: str,
37
- *,
38
- source_name: str = "source",
39
- ) -> tuple[np.ndarray, int]:
40
- ref_order = np.argsort(ref_keys, kind="mergesort", order=("bp", "a1_hash64", "a2_hash64"))
41
- ref_sorted = ref_keys[ref_order]
42
- check_unique_sorted_variant_keys(ref_sorted, f"reference shard {shard_label}", source_name)
43
-
44
- if src_keys.size == 0:
45
- return np.full(ref_keys.size, -1, dtype=np.int64), 0
46
-
47
- src_order = np.argsort(src_keys, kind="mergesort", order=("bp", "a1_hash64", "a2_hash64"))
48
- src_sorted = src_keys[src_order]
49
- check_unique_sorted_variant_keys(src_sorted, f"{source_name} shard {shard_label}", source_name)
50
-
51
- pos = np.searchsorted(src_sorted, ref_keys)
52
- in_range = pos < src_sorted.size
53
- matched = np.zeros(ref_keys.size, dtype=bool)
54
- matched[in_range] = src_sorted[pos[in_range]] == ref_keys[in_range]
55
-
56
- out = np.full(ref_keys.size, -1, dtype=np.int64)
57
- out[matched] = src_order[pos[matched]]
58
-
59
- matched_src = np.zeros(src_keys.size, dtype=bool)
60
- matched_src[out[matched]] = True
61
- unmatched_src = src_keys[~matched_src]
62
- if unmatched_src.size == 0 or ref_sorted.size == 0:
63
- return out, 0
64
-
65
- swapped_src = np.empty(unmatched_src.size, dtype=_KEY_DTYPE)
66
- swapped_src["bp"] = unmatched_src["bp"]
67
- swapped_src["a1_hash64"] = unmatched_src["a2_hash64"]
68
- swapped_src["a2_hash64"] = unmatched_src["a1_hash64"]
69
- swapped_pos = np.searchsorted(ref_sorted, swapped_src)
70
- swapped_in_range = swapped_pos < ref_sorted.size
71
- swapped_match = np.zeros(swapped_src.size, dtype=bool)
72
- swapped_match[swapped_in_range] = (
73
- ref_sorted[swapped_pos[swapped_in_range]]
74
- == swapped_src[swapped_in_range]
75
- )
76
- return out, int(np.count_nonzero(swapped_match))
@@ -1,20 +0,0 @@
1
- import sys
2
-
3
- _LEVELS = ("quiet", "info")
4
- _VERBOSITY = "info"
5
-
6
-
7
- def set_verbosity(level: str) -> None:
8
- global _VERBOSITY
9
- if level not in _LEVELS:
10
- raise ValueError("verbosity must be one of: quiet, info")
11
- _VERBOSITY = level
12
-
13
-
14
- def get_verbosity() -> str:
15
- return _VERBOSITY
16
-
17
-
18
- def info(message: str) -> None:
19
- if _VERBOSITY == "info":
20
- print(message, file=sys.stderr)
File without changes
File without changes
File without changes
File without changes
File without changes