allelign 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- allelign/__init__.py +69 -0
- allelign/__main__.py +5 -0
- allelign/_common.py +95 -0
- allelign/_numba.py +22 -0
- allelign/align.py +1252 -0
- allelign/alleles.py +90 -0
- allelign/build.py +57 -0
- allelign/catalog.py +88 -0
- allelign/cli.py +323 -0
- allelign/download.py +525 -0
- allelign/genotypes.py +241 -0
- allelign/ldcheck.py +771 -0
- allelign/qc.py +638 -0
- allelign/status.py +130 -0
- allelign/sumstats.py +1207 -0
- allelign/variants.py +155 -0
- allelign/warning_markers.py +15 -0
- allelign/weights.py +614 -0
- allelign-0.1.0.dist-info/METADATA +175 -0
- allelign-0.1.0.dist-info/RECORD +24 -0
- allelign-0.1.0.dist-info/WHEEL +5 -0
- allelign-0.1.0.dist-info/entry_points.txt +2 -0
- allelign-0.1.0.dist-info/licenses/LICENSE +21 -0
- allelign-0.1.0.dist-info/top_level.txt +1 -0
allelign/qc.py
ADDED
|
@@ -0,0 +1,638 @@
|
|
|
1
|
+
"""Quality control for summary statistics.
|
|
2
|
+
|
|
3
|
+
Two kinds of check:
|
|
4
|
+
|
|
5
|
+
* **file-only** -- :func:`qc_sumstats` (validity, duplicates, relative N, MAF,
|
|
6
|
+
INFO, chi-square cap), :func:`genomic_inflation`, :func:`in_long_range_ld`;
|
|
7
|
+
* **against a reference**, after harmonisation -- :func:`sd_consistency_mask`
|
|
8
|
+
compares ``sd_ss ~ 1/sqrt(N se^2 + beta^2)`` with ``sd_ref = sqrt(2f(1-f))``,
|
|
9
|
+
:func:`af_concordance_mask` compares reported and reference frequencies,
|
|
10
|
+
:func:`impute_n_eff` recovers per-variant N (Prive et al. 2022),
|
|
11
|
+
:func:`implied_sample_size` and :func:`genomic_control_factor` diagnose N and
|
|
12
|
+
genomic control. The LD-consistency screen lives in :mod:`allelign.ldcheck`.
|
|
13
|
+
|
|
14
|
+
Reference disagreement can mean bad summary statistics *or* an ancestry
|
|
15
|
+
mismatch, so every check returns a keep mask plus diagnostics, and the AF check
|
|
16
|
+
refuses to drop an implausibly large fraction silently. Every function can
|
|
17
|
+
stamp a caller-allocated :mod:`allelign.status` array with per-row reasons.
|
|
18
|
+
|
|
19
|
+
Ported from ldpred3 (file and reference checks) and bipred (long-range LD).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
from . import status as _st
|
|
27
|
+
from ._common import finite_control, integer_at_least, validate_boolean
|
|
28
|
+
from .alleles import normalize_chrom
|
|
29
|
+
from .warning_markers import AF_SAFETY_LIMIT
|
|
30
|
+
|
|
31
|
+
__all__ = ["qc_sumstats", "sd_consistency_mask", "af_concordance_mask",
|
|
32
|
+
"gwas_af_on_counted_allele", "should_impute_n", "impute_n_eff",
|
|
33
|
+
"genomic_control_factor", "implied_sample_size",
|
|
34
|
+
"genomic_inflation", "in_long_range_ld", "LONG_RANGE_LD_GRCH37",
|
|
35
|
+
"APOE_GRCH37"]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def qc_sumstats(ss, *, min_n_ratio=0.7, min_n_anchor="median",
|
|
39
|
+
min_maf=0.01, min_info=0.7, max_chisq=None,
|
|
40
|
+
drop_duplicates=True, status=None):
|
|
41
|
+
"""File-only QC. Returns ``(keep, log)`` over ``len(ss)`` rows.
|
|
42
|
+
|
|
43
|
+
Parameters
|
|
44
|
+
----------
|
|
45
|
+
ss : Sumstats
|
|
46
|
+
min_n_ratio : float, default 0.7
|
|
47
|
+
Keep rows with ``n_eff >= min_n_ratio * anchor(n_eff)``; in (0, 1].
|
|
48
|
+
min_n_anchor : {"median", "max"}, default "median"
|
|
49
|
+
``"max"`` is the LDpred2 convention; ``"median"`` is robust when a file
|
|
50
|
+
pools cohorts of different sizes.
|
|
51
|
+
min_maf : float, default 0.01
|
|
52
|
+
Applied only where ``eaf`` is present.
|
|
53
|
+
min_info : float, default 0.7
|
|
54
|
+
Applied only where ``info`` is present.
|
|
55
|
+
max_chisq : float, optional
|
|
56
|
+
Drop rows with ``(beta/se)^2 > max_chisq``; ``None`` disables it.
|
|
57
|
+
drop_duplicates : bool, default True
|
|
58
|
+
Drop every row whose id (or chrom:pos:alleles when the id is missing)
|
|
59
|
+
occurs more than once.
|
|
60
|
+
status : int8 array, optional
|
|
61
|
+
Stamped with the first filter that rejected each row. The log's
|
|
62
|
+
``n_drop_nonfinite`` also counts the non-positive-N rows the status
|
|
63
|
+
array attributes to ``nonpositive_n``.
|
|
64
|
+
"""
|
|
65
|
+
min_n_ratio = finite_control("min_n_ratio", min_n_ratio)
|
|
66
|
+
if not 0.0 < min_n_ratio <= 1.0:
|
|
67
|
+
raise ValueError("min_n_ratio must be in (0, 1]")
|
|
68
|
+
min_maf = finite_control("min_maf", min_maf)
|
|
69
|
+
if not 0.0 <= min_maf <= 0.5:
|
|
70
|
+
raise ValueError("min_maf must be in [0, 0.5]")
|
|
71
|
+
min_info = finite_control("min_info", min_info)
|
|
72
|
+
if not 0.0 <= min_info <= 1.0:
|
|
73
|
+
raise ValueError("min_info must be in [0, 1]")
|
|
74
|
+
if min_n_anchor not in ("median", "max"):
|
|
75
|
+
raise ValueError("min_n_anchor must be 'max' or 'median'")
|
|
76
|
+
n = len(ss)
|
|
77
|
+
status = _st.check_status(status, n)
|
|
78
|
+
keep = np.ones(n, dtype=bool)
|
|
79
|
+
log = {"n_input": n, "min_maf": min_maf, "min_info": min_info}
|
|
80
|
+
if getattr(ss, "parse_log", None) is not None:
|
|
81
|
+
log["sumstats_parse"] = dict(ss.parse_log)
|
|
82
|
+
|
|
83
|
+
beta = np.asarray(ss.beta, float)
|
|
84
|
+
se = np.asarray(ss.se, float)
|
|
85
|
+
n_eff = np.asarray(ss.n_eff, float)
|
|
86
|
+
eaf = np.asarray(ss.eaf, float)
|
|
87
|
+
info = np.asarray(ss.info, float)
|
|
88
|
+
|
|
89
|
+
valid_n = np.isfinite(n_eff) & (n_eff > 0)
|
|
90
|
+
valid = np.isfinite(beta) & np.isfinite(se) & (se > 0) & valid_n
|
|
91
|
+
log["n_drop_nonfinite"] = int((keep & ~valid).sum())
|
|
92
|
+
log["n_drop_nonpositive_n"] = int(
|
|
93
|
+
(keep & np.isfinite(n_eff) & (n_eff <= 0)).sum())
|
|
94
|
+
_st.mark(status, _st.NONFINITE,
|
|
95
|
+
~(np.isfinite(beta) & np.isfinite(se) & (se > 0))
|
|
96
|
+
| ~np.isfinite(n_eff))
|
|
97
|
+
_st.mark(status, _st.NONPOSITIVE_N, np.isfinite(n_eff) & (n_eff <= 0))
|
|
98
|
+
keep &= valid
|
|
99
|
+
|
|
100
|
+
if drop_duplicates:
|
|
101
|
+
ids = ss.id
|
|
102
|
+
bad = [i for i, v in enumerate(ids) if not (v and v != ".")]
|
|
103
|
+
if bad:
|
|
104
|
+
keys = list(ids)
|
|
105
|
+
for i in bad:
|
|
106
|
+
keys[i] = (str(ss.chrom[i]), int(ss.pos[i]), ss.ea[i], ss.oa[i])
|
|
107
|
+
else:
|
|
108
|
+
keys = ids
|
|
109
|
+
seen_once, repeated = set(), set()
|
|
110
|
+
for k in keys:
|
|
111
|
+
if k in seen_once:
|
|
112
|
+
repeated.add(k)
|
|
113
|
+
else:
|
|
114
|
+
seen_once.add(k)
|
|
115
|
+
dup = (np.array([k in repeated for k in keys], dtype=bool)
|
|
116
|
+
if repeated else np.zeros(n, dtype=bool))
|
|
117
|
+
log["n_drop_duplicate"] = int((keep & dup).sum())
|
|
118
|
+
_st.mark(status, _st.DUPLICATE, dup)
|
|
119
|
+
keep &= ~dup
|
|
120
|
+
|
|
121
|
+
if np.any(valid_n):
|
|
122
|
+
anchor = (float(np.median(n_eff[valid_n])) if min_n_anchor == "median"
|
|
123
|
+
else float(np.max(n_eff[valid_n])))
|
|
124
|
+
bad = n_eff < (min_n_ratio * anchor)
|
|
125
|
+
log["n_drop_lowN"] = int((keep & bad).sum())
|
|
126
|
+
log["n_anchor"] = min_n_anchor
|
|
127
|
+
log["n_anchor_value"] = anchor
|
|
128
|
+
_st.mark(status, _st.LOW_N, bad)
|
|
129
|
+
keep &= ~bad
|
|
130
|
+
|
|
131
|
+
if np.any(np.isfinite(eaf)):
|
|
132
|
+
maf = np.minimum(eaf, 1 - eaf)
|
|
133
|
+
bad = np.isfinite(eaf) & (maf < min_maf)
|
|
134
|
+
log["n_drop_lowMAF"] = int((keep & bad).sum())
|
|
135
|
+
_st.mark(status, _st.LOW_MAF, bad)
|
|
136
|
+
keep &= ~bad
|
|
137
|
+
|
|
138
|
+
if np.any(np.isfinite(info)):
|
|
139
|
+
bad = np.isfinite(info) & (info < min_info)
|
|
140
|
+
log["n_drop_lowINFO"] = int((keep & bad).sum())
|
|
141
|
+
_st.mark(status, _st.LOW_INFO, bad)
|
|
142
|
+
keep &= ~bad
|
|
143
|
+
|
|
144
|
+
if max_chisq is not None:
|
|
145
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
146
|
+
chisq = (beta / se) ** 2
|
|
147
|
+
bad = np.isfinite(chisq) & (chisq > max_chisq)
|
|
148
|
+
log["n_drop_chisq_outlier"] = int((keep & bad).sum())
|
|
149
|
+
_st.mark(status, _st.CHISQ_OUTLIER, bad)
|
|
150
|
+
keep &= ~bad
|
|
151
|
+
|
|
152
|
+
log["n_kept"] = int(keep.sum())
|
|
153
|
+
return keep, log
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _autosome_mask(chrom, n):
|
|
157
|
+
labels = np.asarray(chrom, dtype=object)
|
|
158
|
+
if labels.shape != (n,):
|
|
159
|
+
raise ValueError("chrom must have one label per variant")
|
|
160
|
+
memo = {}
|
|
161
|
+
out = np.zeros(n, dtype=bool)
|
|
162
|
+
for i, c in enumerate(labels):
|
|
163
|
+
hit = memo.get(c)
|
|
164
|
+
if hit is None:
|
|
165
|
+
s = str(c).strip()
|
|
166
|
+
if s[:3].lower() == "chr":
|
|
167
|
+
s = s[3:]
|
|
168
|
+
s = s.upper()
|
|
169
|
+
hit = memo[c] = s.isdigit() and 1 <= int(s) <= 22
|
|
170
|
+
out[i] = hit
|
|
171
|
+
return out
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def sd_consistency_mask(beta, se, n_eff, af_ref, *, info=None, chrom=None,
|
|
175
|
+
sd_ratio_bounds=(0.5, 2.0), min_sd_ref=0.05,
|
|
176
|
+
rule="ratio", min_sd_ss=0.1, sd_excess=0.1,
|
|
177
|
+
status=None):
|
|
178
|
+
"""SD-consistency check against reference allele frequencies.
|
|
179
|
+
|
|
180
|
+
Compares ``sd_ss ~ 1/sqrt(N se^2 + beta^2)`` with ``sd_ref = sqrt(2f(1-f))``
|
|
181
|
+
after anchoring their median ratio (the phenotype scale is unknown). A
|
|
182
|
+
variant this check cannot test (non-finite ``sd_ss``/``sd_ref`` or
|
|
183
|
+
``sd_ref < min_sd_ref``) is *dropped*.
|
|
184
|
+
|
|
185
|
+
Parameters
|
|
186
|
+
----------
|
|
187
|
+
beta, se, n_eff : array_like
|
|
188
|
+
Harmonised summary statistics.
|
|
189
|
+
af_ref : array_like
|
|
190
|
+
Frequency of the same counted allele in the reference.
|
|
191
|
+
info : array_like, optional
|
|
192
|
+
Divide ``sd_ss`` by ``sqrt(INFO)`` (Prive et al. 2022).
|
|
193
|
+
chrom : array_like, optional
|
|
194
|
+
Anchor autosomes and other chromosomes separately: sex-chromosome
|
|
195
|
+
dosage variance differs from ``2f(1-f)`` by a constant that depends on
|
|
196
|
+
the male fraction and coding. A group with fewer than 50 entries uses
|
|
197
|
+
the other group's (then the global) anchor.
|
|
198
|
+
rule : {"ratio", "prive"}, default "ratio"
|
|
199
|
+
``"ratio"`` keeps ``sd_ratio_bounds[0] <= sd_ss/sd_ref <=
|
|
200
|
+
sd_ratio_bounds[1]``. ``"prive"`` is the published asymmetric rule:
|
|
201
|
+
drop ``sd_ss < 0.5 sd_ref``, ``sd_ss > sd_ref + sd_excess`` or
|
|
202
|
+
``sd_ss < min_sd_ss``.
|
|
203
|
+
|
|
204
|
+
Returns
|
|
205
|
+
-------
|
|
206
|
+
keep, log, diag
|
|
207
|
+
``diag`` holds the ``sd_ss`` and ``sd_ref`` arrays for plotting.
|
|
208
|
+
"""
|
|
209
|
+
beta = np.asarray(beta, float); se = np.asarray(se, float)
|
|
210
|
+
n_eff = np.asarray(n_eff, float); af_ref = np.asarray(af_ref, float)
|
|
211
|
+
|
|
212
|
+
sd_ref = np.sqrt(2.0 * af_ref * (1.0 - af_ref))
|
|
213
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
214
|
+
sd_ss = 1.0 / np.sqrt(n_eff * se ** 2 + beta ** 2)
|
|
215
|
+
if info is not None:
|
|
216
|
+
with np.errstate(invalid="ignore"):
|
|
217
|
+
fac = np.sqrt(np.clip(np.asarray(info, float), 0.0, 1.0))
|
|
218
|
+
fac = np.where(np.isfinite(fac) & (fac > 0.0), fac, 1.0)
|
|
219
|
+
sd_ss = sd_ss / fac
|
|
220
|
+
finite = np.isfinite(sd_ss) & np.isfinite(sd_ref) & (sd_ref > 0)
|
|
221
|
+
n_nonautosomal = 0
|
|
222
|
+
nonauto_scale = None
|
|
223
|
+
if chrom is None:
|
|
224
|
+
if finite.any():
|
|
225
|
+
scale = np.median(sd_ref[finite] / sd_ss[finite])
|
|
226
|
+
sd_ss = sd_ss * scale
|
|
227
|
+
else:
|
|
228
|
+
auto = _autosome_mask(chrom, beta.size)
|
|
229
|
+
n_nonautosomal = int((~auto).sum())
|
|
230
|
+
|
|
231
|
+
def _group_scale(mask):
|
|
232
|
+
sel = finite & mask
|
|
233
|
+
if sel.sum() < 50:
|
|
234
|
+
return None
|
|
235
|
+
return float(np.median(sd_ref[sel] / sd_ss[sel]))
|
|
236
|
+
|
|
237
|
+
global_scale = (float(np.median(sd_ref[finite] / sd_ss[finite]))
|
|
238
|
+
if finite.any() else None)
|
|
239
|
+
auto_scale = _group_scale(auto)
|
|
240
|
+
nonauto_scale = _group_scale(~auto)
|
|
241
|
+
if auto_scale is None:
|
|
242
|
+
auto_scale = nonauto_scale if nonauto_scale is not None else global_scale
|
|
243
|
+
if nonauto_scale is None:
|
|
244
|
+
nonauto_scale = auto_scale if auto_scale is not None else global_scale
|
|
245
|
+
if auto_scale is not None:
|
|
246
|
+
sd_ss = np.where(auto, sd_ss * auto_scale, sd_ss * nonauto_scale)
|
|
247
|
+
|
|
248
|
+
if rule not in ("ratio", "prive"):
|
|
249
|
+
raise ValueError("rule must be 'ratio' or 'prive'")
|
|
250
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
251
|
+
ratio = sd_ss / sd_ref
|
|
252
|
+
testable = finite & (sd_ref >= min_sd_ref)
|
|
253
|
+
if rule == "ratio":
|
|
254
|
+
lo, hi = sd_ratio_bounds
|
|
255
|
+
too_low = ratio < lo
|
|
256
|
+
too_high = ratio > hi
|
|
257
|
+
else:
|
|
258
|
+
too_low = (sd_ss < 0.5 * sd_ref) | (sd_ss < min_sd_ss)
|
|
259
|
+
too_high = sd_ss > sd_ref + sd_excess
|
|
260
|
+
keep = testable & ~too_low & ~too_high
|
|
261
|
+
|
|
262
|
+
dropped = testable & ~keep
|
|
263
|
+
n_low = int((dropped & too_low).sum())
|
|
264
|
+
n_high = int((dropped & too_high & ~too_low).sum())
|
|
265
|
+
status = _st.check_status(status, beta.size)
|
|
266
|
+
_st.mark(status, _st.SD_UNTESTABLE, ~testable)
|
|
267
|
+
_st.mark(status, _st.SD_LOW, dropped & too_low)
|
|
268
|
+
_st.mark(status, _st.SD_HIGH, dropped & too_high & ~too_low)
|
|
269
|
+
if testable.any() and np.sum(sd_ref[testable] ** 2) > 0:
|
|
270
|
+
slope = float(np.sum(sd_ss[testable] * sd_ref[testable])
|
|
271
|
+
/ np.sum(sd_ref[testable] ** 2))
|
|
272
|
+
else:
|
|
273
|
+
slope = float("nan")
|
|
274
|
+
|
|
275
|
+
log = {
|
|
276
|
+
"n_input": int(beta.size),
|
|
277
|
+
"n_drop_sd_inconsistent": int(dropped.sum()),
|
|
278
|
+
"n_drop_sd_low": n_low,
|
|
279
|
+
"n_drop_sd_high": n_high,
|
|
280
|
+
"sd_slope": round(slope, 4),
|
|
281
|
+
"rule": rule,
|
|
282
|
+
"info_corrected": info is not None,
|
|
283
|
+
"n_nonautosomal": n_nonautosomal,
|
|
284
|
+
"nonautosomal_sd_scale": (round(nonauto_scale, 4)
|
|
285
|
+
if nonauto_scale is not None
|
|
286
|
+
and n_nonautosomal > 0 else None),
|
|
287
|
+
"n_drop_nonfinite_or_mono": int((~finite | (sd_ref < min_sd_ref)).sum()),
|
|
288
|
+
"n_kept": int(keep.sum()),
|
|
289
|
+
}
|
|
290
|
+
return keep, log, {"sd_ss": sd_ss, "sd_ref": sd_ref}
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def af_concordance_mask(af_gwas, af_ref, *, max_af_diff=0.1,
|
|
294
|
+
max_drop_frac=0.2, min_testable_for_valve=50,
|
|
295
|
+
mirror_corr=-0.5, status=None):
|
|
296
|
+
"""Drop variants whose reported frequency disagrees with the reference.
|
|
297
|
+
|
|
298
|
+
The Prive et al. (2022) ``qc2`` check. Both inputs must count the same
|
|
299
|
+
allele; use :func:`gwas_af_on_counted_allele` after harmonisation.
|
|
300
|
+
Non-finite entries are untestable and kept. Above ``max_drop_frac``
|
|
301
|
+
failing, nothing is dropped (the signature of an ancestry mismatch or a
|
|
302
|
+
mis-oriented column, not per-variant errors) unless fewer than
|
|
303
|
+
``min_testable_for_valve`` variants were testable.
|
|
304
|
+
|
|
305
|
+
Returns ``(keep, log, diag)``; the log carries orientation diagnostics
|
|
306
|
+
(``af_corr``, ``mirrored``, ``n_mirrored``) whether or not it fired.
|
|
307
|
+
"""
|
|
308
|
+
af_gwas = np.asarray(af_gwas, dtype=float)
|
|
309
|
+
af_ref = np.asarray(af_ref, dtype=float)
|
|
310
|
+
if af_gwas.shape != af_ref.shape:
|
|
311
|
+
raise ValueError("af_gwas and af_ref must have the same shape")
|
|
312
|
+
max_af_diff = float(max_af_diff)
|
|
313
|
+
if not np.isfinite(max_af_diff) or not 0.0 < max_af_diff <= 1.0:
|
|
314
|
+
raise ValueError("max_af_diff must be in (0, 1]")
|
|
315
|
+
max_drop_frac = float(max_drop_frac)
|
|
316
|
+
if not np.isfinite(max_drop_frac) or not 0.0 < max_drop_frac <= 1.0:
|
|
317
|
+
raise ValueError("max_drop_frac must be in (0, 1]")
|
|
318
|
+
status = _st.check_status(status, af_gwas.size)
|
|
319
|
+
|
|
320
|
+
testable = np.isfinite(af_gwas) & np.isfinite(af_ref)
|
|
321
|
+
with np.errstate(invalid="ignore"):
|
|
322
|
+
abs_diff = np.abs(af_gwas - af_ref)
|
|
323
|
+
fail = testable & (abs_diff > max_af_diff)
|
|
324
|
+
n_testable = int(testable.sum())
|
|
325
|
+
frac = float(fail.sum() / n_testable) if n_testable else 0.0
|
|
326
|
+
|
|
327
|
+
if n_testable >= 2 and np.std(af_gwas[testable]) > 0 \
|
|
328
|
+
and np.std(af_ref[testable]) > 0:
|
|
329
|
+
af_corr = float(np.corrcoef(af_gwas[testable], af_ref[testable])[0, 1])
|
|
330
|
+
else:
|
|
331
|
+
af_corr = float("nan")
|
|
332
|
+
n_mirrored = int((testable
|
|
333
|
+
& (np.abs(af_gwas - (1.0 - af_ref)) < abs_diff)).sum())
|
|
334
|
+
|
|
335
|
+
applied = (frac <= max_drop_frac
|
|
336
|
+
or n_testable < int(min_testable_for_valve))
|
|
337
|
+
keep = np.ones(af_gwas.shape, dtype=bool)
|
|
338
|
+
if applied:
|
|
339
|
+
keep &= ~fail
|
|
340
|
+
_st.mark(status, _st.AF_DIFF, fail)
|
|
341
|
+
|
|
342
|
+
log = {
|
|
343
|
+
"n_input": int(af_gwas.size),
|
|
344
|
+
"n_testable": n_testable,
|
|
345
|
+
"max_af_diff": max_af_diff,
|
|
346
|
+
"n_drop_af_diff": int(fail.sum()) if applied else 0,
|
|
347
|
+
"n_flagged_af_diff": int(fail.sum()),
|
|
348
|
+
"frac_failing": round(frac, 6),
|
|
349
|
+
"applied": bool(applied),
|
|
350
|
+
"af_corr": None if np.isnan(af_corr) else round(af_corr, 4),
|
|
351
|
+
"mirrored": bool(np.isfinite(af_corr) and af_corr < mirror_corr),
|
|
352
|
+
"n_mirrored": n_mirrored,
|
|
353
|
+
"median_abs_diff": (round(float(np.median(abs_diff[testable])), 6)
|
|
354
|
+
if n_testable else None),
|
|
355
|
+
"n_kept": int(keep.sum()),
|
|
356
|
+
}
|
|
357
|
+
log["valve_active"] = n_testable >= int(min_testable_for_valve)
|
|
358
|
+
if not applied:
|
|
359
|
+
log["not_applied_reason"] = (
|
|
360
|
+
f"{frac:.1%} of testable variants differ by more than "
|
|
361
|
+
f"{max_af_diff}, above the {max_drop_frac:.0%} {AF_SAFETY_LIMIT}; "
|
|
362
|
+
"this is the signature of an ancestry-mismatched LD panel or a "
|
|
363
|
+
"mis-oriented frequency column, not of per-variant errors. No "
|
|
364
|
+
"variant was dropped on this check.")
|
|
365
|
+
return keep, log, {"af_gwas": af_gwas, "af_ref": af_ref,
|
|
366
|
+
"abs_diff": abs_diff}
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def gwas_af_on_counted_allele(eaf, src_index, flipped):
|
|
370
|
+
"""Align source GWAS frequency to the reference's ``a1``; swaps use ``1-f``."""
|
|
371
|
+
eaf = np.asarray(eaf, dtype=float)
|
|
372
|
+
src_index = np.asarray(src_index, dtype=np.int64)
|
|
373
|
+
flipped = np.asarray(flipped, dtype=bool)
|
|
374
|
+
if src_index.shape != flipped.shape:
|
|
375
|
+
raise ValueError("src_index and flipped must have the same shape")
|
|
376
|
+
af = eaf[src_index]
|
|
377
|
+
return np.where(flipped, 1.0 - af, af)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def should_impute_n(impute_n, n_eff, *, n_from_caller=False):
|
|
381
|
+
"""Resolve a tri-state impute-N policy to ``(apply, reason)``.
|
|
382
|
+
|
|
383
|
+
``True`` always imputes, ``False`` never, ``None`` (auto) imputes only when
|
|
384
|
+
the reported N is one constant -- the case a file-wide N describes -- and
|
|
385
|
+
the constant did not come from the caller (``n_from_caller``), whose own
|
|
386
|
+
arithmetic silently replaced would not be a repair.
|
|
387
|
+
"""
|
|
388
|
+
if impute_n is False:
|
|
389
|
+
return False, "disabled"
|
|
390
|
+
if impute_n is True:
|
|
391
|
+
return True, "requested"
|
|
392
|
+
if impute_n is not None:
|
|
393
|
+
raise ValueError("impute_n must be True, False, or None")
|
|
394
|
+
n = np.asarray(n_eff, dtype=float)
|
|
395
|
+
finite = n[np.isfinite(n)]
|
|
396
|
+
if finite.size == 0:
|
|
397
|
+
return False, "no_finite_n"
|
|
398
|
+
if float(np.ptp(finite)) == 0.0:
|
|
399
|
+
if n_from_caller:
|
|
400
|
+
return False, "constant_n_supplied_by_caller"
|
|
401
|
+
return True, "constant_n"
|
|
402
|
+
return False, "ragged_n"
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def impute_n_eff(se, af_ref, n_total, *, anchor_quantile=0.99,
|
|
406
|
+
anchor_bins=50, info=None):
|
|
407
|
+
"""Recover per-variant effective N from ``se`` and frequency.
|
|
408
|
+
|
|
409
|
+
From Prive et al. (2022), ``N_j ~ 1 / (se_j^2 2 f_j (1-f_j))``. The
|
|
410
|
+
unknown constant is set so that the median over equal-count AF bins of the
|
|
411
|
+
per-bin ``anchor_quantile`` equals ``n_total`` (the best-typed variants in
|
|
412
|
+
each frequency range carry the full sample). Optionally multiplied by
|
|
413
|
+
INFO; non-finite values become ``n_total``; clipped at ``n_total``.
|
|
414
|
+
|
|
415
|
+
Returns ``(n_imp, log)``. ``anchor_quantile`` is in ``[0, 1]`` and
|
|
416
|
+
``anchor_bins`` is an integer of at least 1.
|
|
417
|
+
"""
|
|
418
|
+
se = np.asarray(se, dtype=float)
|
|
419
|
+
af_ref = np.asarray(af_ref, dtype=float)
|
|
420
|
+
n_total = float(n_total)
|
|
421
|
+
anchor_quantile = finite_control("anchor_quantile", anchor_quantile,
|
|
422
|
+
lower=0.0, upper=1.0)
|
|
423
|
+
anchor_bins = integer_at_least("anchor_bins", anchor_bins, 1)
|
|
424
|
+
sd_ref2 = 2.0 * af_ref * (1.0 - af_ref)
|
|
425
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
426
|
+
n_raw = 1.0 / (se ** 2 * sd_ref2)
|
|
427
|
+
finite = np.isfinite(n_raw) & (n_raw > 0)
|
|
428
|
+
if not finite.any():
|
|
429
|
+
raise ValueError("no variants with finite se and polymorphic af_ref to "
|
|
430
|
+
"impute N from")
|
|
431
|
+
idx = np.flatnonzero(finite)
|
|
432
|
+
n_bins = max(1, min(anchor_bins, idx.size // 10))
|
|
433
|
+
if n_bins > 1:
|
|
434
|
+
order = idx[np.argsort(af_ref[idx], kind="stable")]
|
|
435
|
+
per_bin = np.array([np.quantile(n_raw[part], anchor_quantile)
|
|
436
|
+
for part in np.array_split(order, n_bins)])
|
|
437
|
+
ref = float(np.median(per_bin))
|
|
438
|
+
anchor_basis = f"median of per-bin q{anchor_quantile:g} over {n_bins} bins"
|
|
439
|
+
else:
|
|
440
|
+
ref = float(np.quantile(n_raw[idx], anchor_quantile))
|
|
441
|
+
anchor_basis = "raw quantile (too few variants to bin)"
|
|
442
|
+
scale = n_total / ref if ref > 0 else 1.0
|
|
443
|
+
n_imp = n_raw * scale
|
|
444
|
+
if info is not None:
|
|
445
|
+
n_imp = n_imp * np.asarray(info, dtype=float)
|
|
446
|
+
n_imp = np.where(np.isfinite(n_imp) & (n_imp > 0), n_imp, n_total)
|
|
447
|
+
n_imp = np.minimum(n_imp, n_total)
|
|
448
|
+
log = {
|
|
449
|
+
"n_input": int(se.size),
|
|
450
|
+
"n_total": n_total,
|
|
451
|
+
"median_imputed_n": float(np.median(n_imp)),
|
|
452
|
+
"frac_below_half": float(np.mean(n_imp < 0.5 * n_total)),
|
|
453
|
+
"anchor_basis": anchor_basis,
|
|
454
|
+
"anchor_ref": float(ref),
|
|
455
|
+
}
|
|
456
|
+
return n_imp, log
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def genomic_control_factor(intercept, intercept_se, *, min_z=2.0,
|
|
460
|
+
n_variants=None, min_variants=50_000):
|
|
461
|
+
"""SE rescaling that undoes genomic control, or 1.0.
|
|
462
|
+
|
|
463
|
+
Genomic control inflates SE by ``sqrt(lambda)`` while leaving beta fixed, so
|
|
464
|
+
a GC-corrected file regresses to an LDSC intercept near ``1/lambda < 1``.
|
|
465
|
+
When deflation is established (``(1 - intercept)/se >= min_z`` on at least
|
|
466
|
+
``min_variants`` variants) return ``sqrt(intercept)`` to multiply SE by.
|
|
467
|
+
Intercepts above one are never rescaled (possible confounding).
|
|
468
|
+
"""
|
|
469
|
+
intercept = float(intercept)
|
|
470
|
+
intercept_se = float(intercept_se)
|
|
471
|
+
if not (np.isfinite(intercept) and np.isfinite(intercept_se)):
|
|
472
|
+
return 1.0
|
|
473
|
+
if intercept <= 0 or intercept >= 1.0 or intercept_se <= 0:
|
|
474
|
+
return 1.0
|
|
475
|
+
if n_variants is not None and int(n_variants) < int(min_variants):
|
|
476
|
+
return 1.0
|
|
477
|
+
if (1.0 - intercept) / intercept_se < min_z:
|
|
478
|
+
return 1.0
|
|
479
|
+
return float(np.sqrt(intercept))
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def implied_sample_size(beta, se, af, *, binary=False, reported_n=None,
|
|
483
|
+
strict=False):
|
|
484
|
+
"""Effective N the statistics behave as if they had (binary traits).
|
|
485
|
+
|
|
486
|
+
Solving ``sd_ss == sd_ref`` gives ``n = (c^2/(2f(1-f)) - beta^2) / se^2``
|
|
487
|
+
with ``c = 2`` for a case/control trait. The median is the sample size the
|
|
488
|
+
file is internally consistent with. A quantitative trait's phenotype
|
|
489
|
+
scale is unidentified, so ``median``/``ratio`` are NaN there.
|
|
490
|
+
|
|
491
|
+
Returns ``{"median", "ratio", "consistent"}`` (ratio to ``reported_n``,
|
|
492
|
+
consistent when within 0.9-1.1). ``reported_n`` may be a positive scalar
|
|
493
|
+
or a vector matching beta; the ratio uses its median.
|
|
494
|
+
|
|
495
|
+
By default, only finite positive implied estimates enter the median.
|
|
496
|
+
``strict=True`` preserves bipred's diagnostic: reject malformed inputs
|
|
497
|
+
(including for quantitative traits), then take the median of *all*
|
|
498
|
+
implied estimates, including nonpositive ones. This distinguishes input
|
|
499
|
+
validation from deciding which estimates describe a plausible N.
|
|
500
|
+
"""
|
|
501
|
+
validate_boolean(binary=binary, strict=strict)
|
|
502
|
+
beta = np.asarray(beta, dtype=np.float64)
|
|
503
|
+
se = np.asarray(se, dtype=np.float64)
|
|
504
|
+
af = np.asarray(af, dtype=np.float64)
|
|
505
|
+
if (beta.ndim != 1 or se.ndim != 1 or af.ndim != 1 or beta.size == 0
|
|
506
|
+
or se.shape != beta.shape or af.shape != beta.shape):
|
|
507
|
+
raise ValueError("beta, se and af must be non-empty equal-length vectors")
|
|
508
|
+
if strict:
|
|
509
|
+
if not np.all(np.isfinite(beta)):
|
|
510
|
+
raise ValueError("beta contains non-finite values; filter them first")
|
|
511
|
+
if not np.all(np.isfinite(se)) or np.any(se <= 0):
|
|
512
|
+
raise ValueError("se must contain finite positive values")
|
|
513
|
+
if not np.all(np.isfinite(af)) or np.any((af <= 0) | (af >= 1)):
|
|
514
|
+
raise ValueError("af must contain finite values strictly between 0 and 1")
|
|
515
|
+
reported = None
|
|
516
|
+
if reported_n is not None:
|
|
517
|
+
reported = np.asarray(reported_n, dtype=np.float64)
|
|
518
|
+
if reported.ndim != 0 and reported.shape != beta.shape:
|
|
519
|
+
raise ValueError("reported_n must be a scalar or a vector matching beta")
|
|
520
|
+
if not np.all(np.isfinite(reported)) or np.any(reported <= 0):
|
|
521
|
+
if strict:
|
|
522
|
+
raise ValueError("reported_n must contain finite positive values")
|
|
523
|
+
reported = None
|
|
524
|
+
if not binary:
|
|
525
|
+
return {"median": float("nan"), "ratio": float("nan"),
|
|
526
|
+
"consistent": False}
|
|
527
|
+
var = 2.0 * af * (1.0 - af)
|
|
528
|
+
if strict:
|
|
529
|
+
# Preserve the rounding of bipred's historical sd_ref ** 2 formula.
|
|
530
|
+
var = np.sqrt(var) ** 2
|
|
531
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
532
|
+
implied = (4.0 / var - beta ** 2) / se ** 2
|
|
533
|
+
estimates = implied if strict else implied[np.isfinite(implied) & (implied > 0)]
|
|
534
|
+
if estimates.size == 0:
|
|
535
|
+
return {"median": float("nan"), "ratio": float("nan"),
|
|
536
|
+
"consistent": False}
|
|
537
|
+
median = float(np.median(estimates))
|
|
538
|
+
ratio = (median / float(np.median(reported)) if reported is not None
|
|
539
|
+
else float("nan"))
|
|
540
|
+
return {"median": median, "ratio": ratio,
|
|
541
|
+
"consistent": bool(np.isfinite(ratio) and 0.9 <= ratio <= 1.1)}
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
#: Median of a chi-square with one degree of freedom.
|
|
545
|
+
_CHI2_1_MEDIAN = 0.45493642311957283
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def genomic_inflation(z=None, *, chisq=None, pval=None):
|
|
549
|
+
"""Genomic-control lambda: ``median(chi^2) / 0.4549``.
|
|
550
|
+
|
|
551
|
+
Pass z-scores (``beta/se``), chi-square statistics or p-values. Non-finite
|
|
552
|
+
entries are ignored. Lambda mixes polygenic signal with confounding; use
|
|
553
|
+
an LDSC intercept to separate them.
|
|
554
|
+
"""
|
|
555
|
+
given = [x is not None for x in (z, chisq, pval)]
|
|
556
|
+
if sum(given) != 1:
|
|
557
|
+
raise ValueError("pass exactly one of z, chisq or pval")
|
|
558
|
+
if z is not None:
|
|
559
|
+
stat = np.asarray(z, dtype=float) ** 2
|
|
560
|
+
elif chisq is not None:
|
|
561
|
+
stat = np.asarray(chisq, dtype=float)
|
|
562
|
+
else:
|
|
563
|
+
from statistics import NormalDist
|
|
564
|
+
inv = NormalDist().inv_cdf
|
|
565
|
+
p = np.asarray(pval, dtype=float)
|
|
566
|
+
ok = np.isfinite(p) & (p > 0) & (p <= 1)
|
|
567
|
+
stat = np.full(p.shape, np.nan)
|
|
568
|
+
stat[ok] = [inv(v / 2) ** 2 for v in p[ok].tolist()]
|
|
569
|
+
stat = stat[np.isfinite(stat)]
|
|
570
|
+
if stat.size == 0:
|
|
571
|
+
return float("nan")
|
|
572
|
+
return float(np.median(stat) / _CHI2_1_MEDIAN)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
#: Long-range LD regions on GRCh37 as ``(chrom, start, end, label)``: the 24
|
|
576
|
+
#: regions of Price et al. 2008 (Am J Hum Genet 83:132-135), where correlation
|
|
577
|
+
#: extends far beyond the usual few hundred kilobases. Use the mask for
|
|
578
|
+
#: sensitivity analysis, not as a universal exclusion rule.
|
|
579
|
+
LONG_RANGE_LD_GRCH37 = (
|
|
580
|
+
("1", 48_000_000, 52_000_000, "1p13"),
|
|
581
|
+
("2", 86_000_000, 100_500_000, "2p11 / 2q11"),
|
|
582
|
+
("2", 134_500_000, 138_000_000, "2q21"),
|
|
583
|
+
("2", 183_000_000, 190_000_000, "2q31"),
|
|
584
|
+
("3", 47_500_000, 50_000_000, "3p21"),
|
|
585
|
+
("3", 83_500_000, 87_000_000, "3p12"),
|
|
586
|
+
("3", 89_000_000, 97_500_000, "3q11"),
|
|
587
|
+
("5", 44_500_000, 50_500_000, "5p12 / 5q11"),
|
|
588
|
+
("5", 98_000_000, 100_500_000, "5q21"),
|
|
589
|
+
("5", 129_000_000, 132_000_000, "5q31"),
|
|
590
|
+
("5", 135_500_000, 138_500_000, "5q31.2"),
|
|
591
|
+
("6", 25_000_000, 35_000_000, "MHC"),
|
|
592
|
+
("6", 57_000_000, 64_000_000, "6p11 / 6q11"),
|
|
593
|
+
("6", 140_000_000, 142_500_000, "6q24"),
|
|
594
|
+
("7", 55_000_000, 66_000_000, "7p11 / 7q11"),
|
|
595
|
+
("8", 7_000_000, 13_000_000, "8p23 inversion"),
|
|
596
|
+
("8", 43_000_000, 50_000_000, "8p11 / 8q11"),
|
|
597
|
+
("8", 112_000_000, 115_000_000, "8q23"),
|
|
598
|
+
("10", 37_000_000, 43_000_000, "10p11 / 10q11"),
|
|
599
|
+
("11", 46_000_000, 57_000_000, "11p11 / 11q11"),
|
|
600
|
+
("11", 87_500_000, 90_500_000, "11q14"),
|
|
601
|
+
("12", 33_000_000, 40_000_000, "12p11 / 12q11"),
|
|
602
|
+
("12", 109_500_000, 112_000_000, "12q24"),
|
|
603
|
+
("20", 32_000_000, 34_500_000, "20p11 / 20q11"),
|
|
604
|
+
)
|
|
605
|
+
#: APOE on GRCh37: not a long-range LD region, but a single locus whose effect
|
|
606
|
+
#: dominates lipid genetics. Optional.
|
|
607
|
+
APOE_GRCH37 = ("19", 44_912_079, 45_912_079, "APOE")
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def in_long_range_ld(chrom, pos, *, genome_build="GRCh37", include_apoe=False,
|
|
611
|
+
regions=None):
|
|
612
|
+
"""Mask of variants inside a long-range LD region.
|
|
613
|
+
|
|
614
|
+
The built-in table is GRCh37; ``genome_build`` must say so (any other
|
|
615
|
+
build raises rather than silently mis-selecting). Chromosome labels are
|
|
616
|
+
normalised, so ``chr6`` and ``6`` both match. Pass ``regions`` as
|
|
617
|
+
``(chrom, start, end, label)`` tuples to use another list on any build.
|
|
618
|
+
"""
|
|
619
|
+
validate_boolean(include_apoe=include_apoe)
|
|
620
|
+
if regions is None:
|
|
621
|
+
from .build import normalize_build
|
|
622
|
+
if normalize_build(genome_build) != "GRCh37":
|
|
623
|
+
raise ValueError(
|
|
624
|
+
"the built-in long-range LD table is GRCh37; lift positions "
|
|
625
|
+
"over or pass regions= for another build")
|
|
626
|
+
regions = list(LONG_RANGE_LD_GRCH37) + ([APOE_GRCH37]
|
|
627
|
+
if include_apoe else [])
|
|
628
|
+
labels = np.asarray(chrom, dtype=object)
|
|
629
|
+
pos = np.asarray(pos, dtype=np.int64)
|
|
630
|
+
if labels.shape != pos.shape:
|
|
631
|
+
raise ValueError("chrom and pos must have the same shape")
|
|
632
|
+
memo = {c: normalize_chrom(c) for c in set(labels.ravel())}
|
|
633
|
+
norm = np.array([memo[c] for c in labels.ravel()],
|
|
634
|
+
dtype=object).reshape(labels.shape)
|
|
635
|
+
inside = np.zeros(pos.shape, dtype=bool)
|
|
636
|
+
for name, start, end, _label in regions:
|
|
637
|
+
inside |= (norm == normalize_chrom(name)) & (pos >= start) & (pos <= end)
|
|
638
|
+
return inside
|