allelign 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- allelign/__init__.py +69 -0
- allelign/__main__.py +5 -0
- allelign/_common.py +95 -0
- allelign/_numba.py +22 -0
- allelign/align.py +1252 -0
- allelign/alleles.py +90 -0
- allelign/build.py +57 -0
- allelign/catalog.py +88 -0
- allelign/cli.py +323 -0
- allelign/download.py +525 -0
- allelign/genotypes.py +241 -0
- allelign/ldcheck.py +771 -0
- allelign/qc.py +638 -0
- allelign/status.py +130 -0
- allelign/sumstats.py +1207 -0
- allelign/variants.py +155 -0
- allelign/warning_markers.py +15 -0
- allelign/weights.py +614 -0
- allelign-0.1.0.dist-info/METADATA +175 -0
- allelign-0.1.0.dist-info/RECORD +24 -0
- allelign-0.1.0.dist-info/WHEEL +5 -0
- allelign-0.1.0.dist-info/entry_points.txt +2 -0
- allelign-0.1.0.dist-info/licenses/LICENSE +21 -0
- allelign-0.1.0.dist-info/top_level.txt +1 -0
allelign/sumstats.py
ADDED
|
@@ -0,0 +1,1207 @@
|
|
|
1
|
+
"""Read GWAS summary statistics into one canonical table.
|
|
2
|
+
|
|
3
|
+
GWAS arrive as delimited text with wildly inconsistent column names. This
|
|
4
|
+
reader maps a large set of aliases onto canonical fields and lets the caller
|
|
5
|
+
override any column explicitly:
|
|
6
|
+
|
|
7
|
+
============== ===========================================================
|
|
8
|
+
field meaning
|
|
9
|
+
============== ===========================================================
|
|
10
|
+
``id`` variant identifier (rsID), may be empty
|
|
11
|
+
``chrom`` chromosome label as read, may be empty
|
|
12
|
+
``pos`` base-pair position (int), 0 when missing
|
|
13
|
+
``ea`` effect allele: ``beta`` is per copy of this allele
|
|
14
|
+
``oa`` other (non-effect) allele
|
|
15
|
+
``beta`` per-allele effect (log-OR for binary traits)
|
|
16
|
+
``se`` standard error of ``beta``
|
|
17
|
+
``n_eff`` (effective) sample size per variant
|
|
18
|
+
``eaf`` effect-allele frequency (NaN when absent)
|
|
19
|
+
``info`` imputation quality (NaN when absent)
|
|
20
|
+
============== ===========================================================
|
|
21
|
+
|
|
22
|
+
Odds ratios become ``beta = log(OR)``. A missing ``se`` is recovered from
|
|
23
|
+
``beta`` and the p-value (or ``-log10 p``) without underflow. Effective N comes,
|
|
24
|
+
in order of preference, from an explicit effective-N column, from per-variant
|
|
25
|
+
case and control counts (``4 / (1/N_case + 1/N_control)``), or from a total-N
|
|
26
|
+
column; :attr:`Sumstats.parse_log` records which.
|
|
27
|
+
|
|
28
|
+
Format-specific layouts are recognised from the header: SAIGE (effects for
|
|
29
|
+
Allele2), PLINK2 (``A1`` tested against ``REF``/``ALT``, or ``OMITTED``),
|
|
30
|
+
REGENIE (``GENPOS``), GWAS Catalog harmonised files (the ``hm_`` columns as one
|
|
31
|
+
consistent set), PGC ``daner`` files (``FRQ_U_*`` control frequency) and Neale
|
|
32
|
+
UK Biobank files (``chr:pos:ref:alt`` variant strings, effect on ALT).
|
|
33
|
+
|
|
34
|
+
Ported from ldpred3's reader, including its optional Numba block parser, whose
|
|
35
|
+
output is bit-identical to the reference row loop. On ldpred3's standard inputs
|
|
36
|
+
the result is identical to ``ldpred3.sumstats.read_sumstats``; the additions
|
|
37
|
+
above are documented differences.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
from dataclasses import dataclass
|
|
43
|
+
import math
|
|
44
|
+
import os
|
|
45
|
+
import warnings
|
|
46
|
+
|
|
47
|
+
import numpy as np
|
|
48
|
+
|
|
49
|
+
from ._common import atomic_text, open_text
|
|
50
|
+
|
|
51
|
+
__all__ = ["Sumstats", "read_sumstats", "detect_columns", "inspect_columns",
|
|
52
|
+
"write_sumstats", "rescale_n_eff", "anchor_n_eff",
|
|
53
|
+
"n_eff_case_control", "COLUMN_ALIASES", "ColumnMappingError"]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class ColumnMappingError(ValueError):
|
|
57
|
+
"""A column override cannot be resolved, or a layout is inconsistent."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
#: Column aliases (lower-cased) -> canonical field, in priority order. The
|
|
61
|
+
#: first block of every list is ldpred3's, unchanged, so any file ldpred3 maps
|
|
62
|
+
#: maps identically here; later entries only add coverage. One exception: for
|
|
63
|
+
#: ``n_eff`` an effective-N name (``_EFFECTIVE_N_NAMES``) wins over a total-N
|
|
64
|
+
#: alias, so ``Effective_N`` beside ``N`` gives N from ``Effective_N``.
|
|
65
|
+
COLUMN_ALIASES = {
|
|
66
|
+
"id": ["snp", "rsid", "rs", "id", "variant_id", "markername", "snpid",
|
|
67
|
+
"marker", "rs_id",
|
|
68
|
+
"rsids", "hm_rsid", "snp_id", "variant", "snpname"],
|
|
69
|
+
"chrom": ["chr", "chrom", "chromosome", "#chrom", "hg19chr", "chr_name",
|
|
70
|
+
"#chr", "hm_chrom", "hm_chr", "chr_id"],
|
|
71
|
+
"pos": ["bp", "pos", "position", "base_pair_location", "bp_position",
|
|
72
|
+
"pos_b37", "base_pair",
|
|
73
|
+
"chr_position", "hm_pos", "genpos"],
|
|
74
|
+
"ea": ["a1", "effect_allele", "ea", "allele1", "alt", "effectallele",
|
|
75
|
+
"tested_allele", "inc_allele",
|
|
76
|
+
"hm_effect_allele"],
|
|
77
|
+
"oa": ["a2", "other_allele", "oa", "nea", "allele0", "allele2", "ref",
|
|
78
|
+
"noneffect_allele", "dec_allele",
|
|
79
|
+
"hm_other_allele", "omitted"],
|
|
80
|
+
"beta": ["beta", "b", "effect", "effect_size", "effects", "log_odds",
|
|
81
|
+
"hm_beta", "all_inv_var_meta_beta", "inv_var_meta_beta", "logor"],
|
|
82
|
+
"or": ["or", "odds_ratio", "oddsratio",
|
|
83
|
+
"hm_odds_ratio"],
|
|
84
|
+
"se": ["se", "standard_error", "stderr", "standarderror", "sebeta",
|
|
85
|
+
"se_beta", "logor_se", "std_err",
|
|
86
|
+
"log(or)_se", "all_inv_var_meta_sebeta", "inv_var_meta_sebeta",
|
|
87
|
+
"hm_standard_error"],
|
|
88
|
+
"pval": ["p", "pval", "p_value", "pvalue", "p-value", "p.value",
|
|
89
|
+
"p_bolt_lmm", "p_value_association", "p_wald", "pval_nominal",
|
|
90
|
+
"all_inv_var_meta_p", "inv_var_meta_p"],
|
|
91
|
+
"log10p": ["log10p", "mlogp", "neg_log_10_p_value", "neglog10p",
|
|
92
|
+
"log10_p", "-log10p", "neg_log10_p"],
|
|
93
|
+
"n_eff": ["n_eff", "neff", "n", "sample_size", "totalsamplesize",
|
|
94
|
+
"n_samples", "n_total", "obs_ct", "n_complete_samples",
|
|
95
|
+
"n_effective", "effective_n", "all_inv_var_meta_effective",
|
|
96
|
+
"samplesize", "n_sample", "total_sample_size"],
|
|
97
|
+
"n_cases": ["n_cases", "ncases", "n_case", "ncase", "nca", "cases",
|
|
98
|
+
"num_cases", "n_cas"],
|
|
99
|
+
"n_controls": ["n_controls", "ncontrols", "n_control", "ncontrol", "nco",
|
|
100
|
+
"controls", "num_controls", "n_ctrl", "n_con"],
|
|
101
|
+
# MAF is deliberately absent: it does not say which allele is counted, so
|
|
102
|
+
# treating it as EAF can turn a common effect allele (EAF=.9, MAF=.1) into
|
|
103
|
+
# an apparent .8 frequency mismatch. A genuinely effect-allele column with
|
|
104
|
+
# a non-standard name is available through the explicit ``eaf=`` override.
|
|
105
|
+
"eaf": ["eaf", "freq", "frq", "effect_allele_frequency", "a1freq",
|
|
106
|
+
"freq1", "af", "effect_allele_freq",
|
|
107
|
+
"a1_freq", "af_alt", "all_meta_af", "hm_effect_allele_frequency"],
|
|
108
|
+
"info": ["info", "imputation_info", "imp_info", "rsq", "r2", "info_score",
|
|
109
|
+
"imputation_quality", "minimac_r2",
|
|
110
|
+
"imputationinfo"],
|
|
111
|
+
"test": ["test"],
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
# PLINK's additive effect and REGENIE's covariate-adjusted marginal effect.
|
|
115
|
+
# Other ADD-* labels include interactions and gene-level omnibus tests.
|
|
116
|
+
_ADDITIVE_TESTS = frozenset(("ADD", "ADD-CONDTL"))
|
|
117
|
+
|
|
118
|
+
#: N-column names that already are an effective N: preferred over a total-N
|
|
119
|
+
#: column and kept even when case and control counts are present.
|
|
120
|
+
_EFFECTIVE_N_NAMES = frozenset({"n_eff", "neff", "n_effective", "effective_n",
|
|
121
|
+
"all_inv_var_meta_effective"})
|
|
122
|
+
|
|
123
|
+
_SUMSTATS_FIELDS = ("id", "chrom", "pos", "ea", "oa", "beta", "se", "n_eff",
|
|
124
|
+
"eaf", "info")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass
|
|
128
|
+
class Sumstats:
|
|
129
|
+
"""Parsed GWAS summary statistics as parallel arrays.
|
|
130
|
+
|
|
131
|
+
Field order and meaning match ldpred3's ``Sumstats``, so either package's
|
|
132
|
+
functions accept the other's tables.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
id: np.ndarray
|
|
136
|
+
chrom: np.ndarray
|
|
137
|
+
pos: np.ndarray
|
|
138
|
+
ea: np.ndarray
|
|
139
|
+
oa: np.ndarray
|
|
140
|
+
beta: np.ndarray
|
|
141
|
+
se: np.ndarray
|
|
142
|
+
n_eff: np.ndarray
|
|
143
|
+
eaf: np.ndarray
|
|
144
|
+
info: np.ndarray
|
|
145
|
+
parse_log: dict = None # source-file counts, preserved by subsetting
|
|
146
|
+
|
|
147
|
+
def __len__(self):
|
|
148
|
+
return len(self.beta)
|
|
149
|
+
|
|
150
|
+
@property
|
|
151
|
+
def z(self):
|
|
152
|
+
"""``beta / se`` (NaN where undefined)."""
|
|
153
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
154
|
+
return np.asarray(self.beta, float) / np.asarray(self.se, float)
|
|
155
|
+
|
|
156
|
+
def subset(self, mask):
|
|
157
|
+
"""A new table keeping the rows selected by ``mask`` (bool or index)."""
|
|
158
|
+
mask = np.asarray(mask)
|
|
159
|
+
return Sumstats(*(getattr(self, f)[mask] for f in _SUMSTATS_FIELDS),
|
|
160
|
+
parse_log=(None if self.parse_log is None
|
|
161
|
+
else dict(self.parse_log)))
|
|
162
|
+
|
|
163
|
+
def as_variants(self):
|
|
164
|
+
"""These rows as a :class:`~allelign.variants.Variants` table.
|
|
165
|
+
|
|
166
|
+
``a1`` is the effect allele, so harmonising another file against it
|
|
167
|
+
aligns that file's effects to this file's effect alleles.
|
|
168
|
+
"""
|
|
169
|
+
from .variants import Variants
|
|
170
|
+
return Variants(chrom=np.asarray(self.chrom, dtype=object),
|
|
171
|
+
id=np.asarray(self.id, dtype=object),
|
|
172
|
+
pos=np.asarray(self.pos, dtype=np.int64),
|
|
173
|
+
a1=np.asarray(self.ea, dtype=object),
|
|
174
|
+
a2=np.asarray(self.oa, dtype=object))
|
|
175
|
+
|
|
176
|
+
def write(self, path):
|
|
177
|
+
"""Write the canonical table; see :func:`write_sumstats`."""
|
|
178
|
+
return write_sumstats(path, self)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def n_eff_case_control(n_case, n_control):
|
|
182
|
+
"""Effective sample size of a case/control GWAS: ``4/(1/N_case + 1/N_control)``.
|
|
183
|
+
|
|
184
|
+
Equals total N for a balanced study and tends to ``4 N_case`` with many
|
|
185
|
+
more controls than cases. Scalars or arrays.
|
|
186
|
+
|
|
187
|
+
For a meta-analysis the correct effective N is the *sum* of the
|
|
188
|
+
per-cohort values; computing it from summed counts overstates it when
|
|
189
|
+
case fractions differ between cohorts. Prefer a published ``Neff`` column.
|
|
190
|
+
"""
|
|
191
|
+
n_case = np.asarray(n_case, dtype=float)
|
|
192
|
+
n_control = np.asarray(n_control, dtype=float)
|
|
193
|
+
if np.any(n_case <= 0) or np.any(n_control <= 0):
|
|
194
|
+
raise ValueError("n_case and n_control must be positive")
|
|
195
|
+
n = 4.0 / (1.0 / n_case + 1.0 / n_control)
|
|
196
|
+
return float(n) if n.ndim == 0 else n
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _sniff_delimiter(header_line):
|
|
200
|
+
if "\t" in header_line:
|
|
201
|
+
return "\t"
|
|
202
|
+
if "," in header_line:
|
|
203
|
+
return ","
|
|
204
|
+
return None # whitespace; handled with str.split
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _read_header(fh):
|
|
208
|
+
"""``(delimiter, header columns)`` from the first line of ``fh``."""
|
|
209
|
+
first = fh.readline().rstrip("\r\n")
|
|
210
|
+
delim = _sniff_delimiter(first)
|
|
211
|
+
return delim, (first.split(delim) if delim else first.split())
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _build_colmap(header, overrides):
|
|
215
|
+
"""``(colmap, layout)``: canonical field -> column index, plus format rules.
|
|
216
|
+
|
|
217
|
+
``layout`` records format-specific rules the header implied:
|
|
218
|
+
``saige``, ``gwas_catalog_harmonised``, ``plink2_omitted``,
|
|
219
|
+
``plink2_alt`` (index of ALT for per-row other-allele resolution),
|
|
220
|
+
``eaf_from`` (a pattern-matched frequency column) and ``variant_string``
|
|
221
|
+
(index of a ``chr:pos:ref:alt`` column).
|
|
222
|
+
"""
|
|
223
|
+
overrides = dict(overrides or {})
|
|
224
|
+
lower = [h.strip().lower() for h in header]
|
|
225
|
+
colmap = {}
|
|
226
|
+
for field, aliases in COLUMN_ALIASES.items():
|
|
227
|
+
if field == "n_eff":
|
|
228
|
+
# Effective-N names first, so an ``Effective_N`` column is not
|
|
229
|
+
# shadowed by a total ``N`` listed earlier in the aliases.
|
|
230
|
+
aliases = sorted(aliases, key=lambda a: a not in _EFFECTIVE_N_NAMES)
|
|
231
|
+
for a in aliases:
|
|
232
|
+
if a in lower:
|
|
233
|
+
colmap[field] = lower.index(a)
|
|
234
|
+
break
|
|
235
|
+
layout = {}
|
|
236
|
+
# SAIGE reports effects and frequencies for Allele2. Allele1/Allele2 alone
|
|
237
|
+
# are ambiguous across programs, so require a SAIGE-specific column.
|
|
238
|
+
if {"allele1", "allele2"}.issubset(lower) and any(
|
|
239
|
+
c in lower for c in ("af_allele2", "ac_allele2", "is.spa.converge")):
|
|
240
|
+
colmap["ea"] = lower.index("allele2")
|
|
241
|
+
colmap["oa"] = lower.index("allele1")
|
|
242
|
+
if "af_allele2" in lower:
|
|
243
|
+
colmap["eaf"] = lower.index("af_allele2")
|
|
244
|
+
if "imputationinfo" in lower:
|
|
245
|
+
colmap["info"] = lower.index("imputationinfo")
|
|
246
|
+
layout["saige"] = True
|
|
247
|
+
# GWAS Catalog harmonised files carry the submitted columns and the
|
|
248
|
+
# harmonised ``hm_`` columns side by side, possibly on different builds and
|
|
249
|
+
# with the effect allele swapped. Use one set consistently: the hm_ set
|
|
250
|
+
# when it has a harmonised effect, the submitted set otherwise.
|
|
251
|
+
if "hm_effect_allele" in lower and "hm_other_allele" in lower:
|
|
252
|
+
if "hm_beta" in lower or "hm_odds_ratio" in lower:
|
|
253
|
+
for field, name in (("ea", "hm_effect_allele"),
|
|
254
|
+
("oa", "hm_other_allele"), ("id", "hm_rsid"),
|
|
255
|
+
("chrom", "hm_chrom"), ("pos", "hm_pos"),
|
|
256
|
+
("eaf", "hm_effect_allele_frequency")):
|
|
257
|
+
if name in lower:
|
|
258
|
+
colmap[field] = lower.index(name)
|
|
259
|
+
if "hm_beta" in lower:
|
|
260
|
+
colmap["beta"] = lower.index("hm_beta")
|
|
261
|
+
else:
|
|
262
|
+
colmap.pop("beta", None)
|
|
263
|
+
colmap["or"] = lower.index("hm_odds_ratio")
|
|
264
|
+
# The submitted frequency counts the submitted effect allele, which
|
|
265
|
+
# harmonisation may have swapped: without its hm_ counterpart it
|
|
266
|
+
# would describe the other allele on those rows.
|
|
267
|
+
if ("hm_effect_allele_frequency" not in lower
|
|
268
|
+
and colmap.get("eaf") is not None):
|
|
269
|
+
del colmap["eaf"]
|
|
270
|
+
layout["eaf_dropped"] = (
|
|
271
|
+
"hm_ columns without hm_effect_allele_frequency; the "
|
|
272
|
+
"submitted frequency may count the other allele")
|
|
273
|
+
layout["gwas_catalog_harmonised"] = True
|
|
274
|
+
else:
|
|
275
|
+
for field in ("ea", "oa", "id", "chrom", "pos", "eaf"):
|
|
276
|
+
if (colmap.get(field) is not None
|
|
277
|
+
and lower[colmap[field]].startswith("hm_")):
|
|
278
|
+
del colmap[field]
|
|
279
|
+
layout["gwas_catalog_harmonised_ignored"] = (
|
|
280
|
+
"no hm_beta or hm_odds_ratio; the submitted columns are used")
|
|
281
|
+
# PLINK2 --glm: A1 is the tested allele, which may be REF or ALT.
|
|
282
|
+
if (colmap.get("ea") is not None and lower[colmap["ea"]] == "a1"
|
|
283
|
+
and colmap.get("oa") is not None and lower[colmap["oa"]] == "ref"):
|
|
284
|
+
if "omitted" in lower:
|
|
285
|
+
colmap["oa"] = lower.index("omitted")
|
|
286
|
+
layout["plink2_omitted"] = True
|
|
287
|
+
elif "alt" in lower:
|
|
288
|
+
layout["plink2_alt"] = lower.index("alt")
|
|
289
|
+
# PGC daner files: the A1 frequency in controls, FRQ_U_<n>.
|
|
290
|
+
if "eaf" not in colmap:
|
|
291
|
+
for i, h in enumerate(lower):
|
|
292
|
+
if h.startswith("frq_u_"):
|
|
293
|
+
colmap["eaf"] = i
|
|
294
|
+
layout["eaf_from"] = header[i].strip()
|
|
295
|
+
break
|
|
296
|
+
# Neale UK Biobank: variant = chr:pos:ref:alt, effect on ALT.
|
|
297
|
+
if "ea" not in colmap and "oa" not in colmap and "variant" in lower:
|
|
298
|
+
layout["variant_string"] = lower.index("variant")
|
|
299
|
+
|
|
300
|
+
# Explicit overrides win (a column name or a zero-based index).
|
|
301
|
+
for field, col in overrides.items():
|
|
302
|
+
if field not in COLUMN_ALIASES:
|
|
303
|
+
raise ColumnMappingError(
|
|
304
|
+
f"unknown sumstats field {field!r}; choose one of "
|
|
305
|
+
+ ", ".join(COLUMN_ALIASES))
|
|
306
|
+
# A bare digit string (``--column beta=3``) is a zero-based index.
|
|
307
|
+
if isinstance(col, str) and col.strip().isdigit():
|
|
308
|
+
col = int(col.strip())
|
|
309
|
+
if isinstance(col, (int, np.integer)) and not isinstance(col, bool):
|
|
310
|
+
i = int(col)
|
|
311
|
+
if not 0 <= i < len(header):
|
|
312
|
+
raise ColumnMappingError(
|
|
313
|
+
f"sumstats column override {field}={i} is out of range for "
|
|
314
|
+
f"a {len(header)}-column file (indices are zero-based)")
|
|
315
|
+
colmap[field] = i
|
|
316
|
+
else:
|
|
317
|
+
name = str(col).strip().lower()
|
|
318
|
+
if name not in lower:
|
|
319
|
+
raise ColumnMappingError(
|
|
320
|
+
f"sumstats column override {field}={col!r} was not found; "
|
|
321
|
+
"available columns: " + ", ".join(header))
|
|
322
|
+
colmap[field] = lower.index(name)
|
|
323
|
+
if "or" in overrides and "beta" not in overrides:
|
|
324
|
+
colmap.pop("beta", None) # explicit OR semantics override a BETA alias
|
|
325
|
+
if "oa" in overrides or "ea" in overrides:
|
|
326
|
+
layout.pop("plink2_alt", None)
|
|
327
|
+
layout.pop("variant_string", None)
|
|
328
|
+
if colmap.get("ea") is not None and colmap.get("ea") == colmap.get("oa"):
|
|
329
|
+
raise ColumnMappingError(
|
|
330
|
+
"ea and oa resolve to the same column; specify both allele columns")
|
|
331
|
+
return colmap, layout
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _n_arguments(n_eff, n_cases, n_controls, overrides):
|
|
335
|
+
"""Separate scalar N from column overrides, shared by read and inspect."""
|
|
336
|
+
for name, value in (("n_cases", n_cases), ("n_controls", n_controls)):
|
|
337
|
+
if isinstance(value, str):
|
|
338
|
+
overrides[name] = value
|
|
339
|
+
if isinstance(n_eff, str):
|
|
340
|
+
overrides["n_eff"] = n_eff
|
|
341
|
+
n_eff = None
|
|
342
|
+
scalar_counts = [v for v in (n_cases, n_controls)
|
|
343
|
+
if v is not None and not isinstance(v, str)]
|
|
344
|
+
if scalar_counts:
|
|
345
|
+
if len(scalar_counts) != 2:
|
|
346
|
+
raise ValueError("n_cases and n_controls must be given together")
|
|
347
|
+
if n_eff is not None:
|
|
348
|
+
raise ValueError("pass either n_eff or n_cases/n_controls, not both")
|
|
349
|
+
n_eff = n_eff_case_control(_n_eff_scalar_value(n_cases),
|
|
350
|
+
_n_eff_scalar_value(n_controls))
|
|
351
|
+
return _n_eff_scalar_value(n_eff)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _n_plan(colmap, lower, overrides, forced_n):
|
|
355
|
+
"""Where per-variant N comes from: ``(source, detail)``."""
|
|
356
|
+
if forced_n is not None:
|
|
357
|
+
return "scalar", None
|
|
358
|
+
if "n_eff" in overrides:
|
|
359
|
+
return "column", lower[colmap["n_eff"]]
|
|
360
|
+
if "n_cases" in overrides or "n_controls" in overrides:
|
|
361
|
+
if "n_cases" not in colmap or "n_controls" not in colmap:
|
|
362
|
+
raise ColumnMappingError("explicit case/control columns require both counts")
|
|
363
|
+
return "case_control", None
|
|
364
|
+
if "n_eff" in colmap and lower[colmap["n_eff"]] in _EFFECTIVE_N_NAMES:
|
|
365
|
+
return "column", lower[colmap["n_eff"]]
|
|
366
|
+
if "n_cases" in colmap and "n_controls" in colmap:
|
|
367
|
+
return "case_control", None
|
|
368
|
+
if "n_eff" in colmap:
|
|
369
|
+
return "column", lower[colmap["n_eff"]]
|
|
370
|
+
return None, None
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def detect_columns(path, **col_overrides):
|
|
374
|
+
"""Peek at the header and report the detected column mapping.
|
|
375
|
+
|
|
376
|
+
Returns ``(header, mapping)`` with ``mapping = {field: column name}`` for
|
|
377
|
+
every resolved field. Reads only the first line.
|
|
378
|
+
"""
|
|
379
|
+
with open_text(path) as fh:
|
|
380
|
+
_delim, header = _read_header(fh)
|
|
381
|
+
colmap, _layout = _build_colmap(header, col_overrides)
|
|
382
|
+
return header, {field: header[i] for field, i in colmap.items()}
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def inspect_columns(path, *, n_eff=None, n_cases=None, n_controls=None,
|
|
386
|
+
**col_overrides):
|
|
387
|
+
"""Header-only preflight: mapping, recognised layout and N source.
|
|
388
|
+
|
|
389
|
+
Returns a dict with ``header``, ``mapping``, ``layout``, ``n_source`` and
|
|
390
|
+
``missing`` (required fields that could not be resolved, including N).
|
|
391
|
+
Sample-size arguments follow :func:`read_sumstats`: numbers supply a
|
|
392
|
+
constant N; strings name columns or their zero-based indices.
|
|
393
|
+
"""
|
|
394
|
+
forced_n = _n_arguments(n_eff, n_cases, n_controls, col_overrides)
|
|
395
|
+
with open_text(path) as fh:
|
|
396
|
+
delim, header = _read_header(fh)
|
|
397
|
+
colmap, layout = _build_colmap(header, col_overrides)
|
|
398
|
+
lower = [h.strip().lower() for h in header]
|
|
399
|
+
source, detail = _n_plan(colmap, lower, col_overrides, forced_n)
|
|
400
|
+
missing = []
|
|
401
|
+
if "variant_string" not in layout:
|
|
402
|
+
missing += [f for f in ("ea", "oa") if f not in colmap]
|
|
403
|
+
if "beta" not in colmap and "or" not in colmap:
|
|
404
|
+
missing.append("beta/or")
|
|
405
|
+
if "se" not in colmap and "pval" not in colmap and "log10p" not in colmap:
|
|
406
|
+
missing.append("se/pval")
|
|
407
|
+
if source is None:
|
|
408
|
+
missing.append("n_eff")
|
|
409
|
+
return {"header": header, "delimiter": delim or "whitespace",
|
|
410
|
+
"mapping": {field: header[i] for field, i in colmap.items()},
|
|
411
|
+
"layout": {k: (header[v] if isinstance(v, int) and k != "saige"
|
|
412
|
+
and not isinstance(v, bool) else v)
|
|
413
|
+
for k, v in layout.items()},
|
|
414
|
+
"n_source": source if detail is None else f"{source}:{detail}",
|
|
415
|
+
"missing": missing}
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _to_float(s):
|
|
419
|
+
try:
|
|
420
|
+
return float(s)
|
|
421
|
+
except (ValueError, TypeError):
|
|
422
|
+
return math.nan
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _z_from_log_tail(log_tail):
|
|
426
|
+
"""``z`` with ``log Phi(-z) = log_tail``, for tails beyond float range.
|
|
427
|
+
|
|
428
|
+
Newton solve using Mills' expansion; used once ``z > 38``, where the
|
|
429
|
+
neglected term is below 1e-12 in log probability.
|
|
430
|
+
"""
|
|
431
|
+
z = math.sqrt(-2 * log_tail)
|
|
432
|
+
for _ in range(5):
|
|
433
|
+
t = 1 / (z * z)
|
|
434
|
+
correction = math.log1p(-t + 3*t*t - 15*t**3 + 105*t**4)
|
|
435
|
+
log_sf = -.5*z*z - math.log(z) - .5*math.log(2*math.pi) + correction
|
|
436
|
+
z += (log_sf - log_tail) / (z + 1/z)
|
|
437
|
+
return z
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _two_sided_z(token):
|
|
441
|
+
"""Two-sided normal z from a p-value token, without underflow."""
|
|
442
|
+
from statistics import NormalDist
|
|
443
|
+
p = _to_float(token)
|
|
444
|
+
if 0 < p < 1 and p / 2 > 0:
|
|
445
|
+
return -NormalDist().inv_cdf(p / 2)
|
|
446
|
+
if p != 0 and not (0 < p < 1):
|
|
447
|
+
return math.nan
|
|
448
|
+
from decimal import Decimal, InvalidOperation
|
|
449
|
+
try:
|
|
450
|
+
value = Decimal(str(token))
|
|
451
|
+
if not value.is_finite() or not 0 < value < 1:
|
|
452
|
+
return math.nan
|
|
453
|
+
log_tail = float(value.ln()) - math.log(2.)
|
|
454
|
+
except (InvalidOperation, ValueError, OverflowError):
|
|
455
|
+
return math.nan
|
|
456
|
+
return _z_from_log_tail(log_tail)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _z_from_log10p(token):
|
|
460
|
+
"""Two-sided normal z from a ``-log10 p`` token (REGENIE, FinnGen)."""
|
|
461
|
+
from statistics import NormalDist
|
|
462
|
+
x = _to_float(token)
|
|
463
|
+
if not math.isfinite(x) or x <= 0:
|
|
464
|
+
return math.nan
|
|
465
|
+
if x < 300:
|
|
466
|
+
return -NormalDist().inv_cdf(10.0 ** (-x) / 2)
|
|
467
|
+
return _z_from_log_tail(-x * math.log(10.) - math.log(2.))
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def read_sumstats(path, *, n_eff=None, n_cases=None, n_controls=None,
|
|
471
|
+
require_n=True, **col_overrides):
|
|
472
|
+
"""Read a GWAS summary-statistics file into a :class:`Sumstats`.
|
|
473
|
+
|
|
474
|
+
Parameters
|
|
475
|
+
----------
|
|
476
|
+
path : str or path-like
|
|
477
|
+
Delimited text (tab, comma or whitespace, auto-detected), optionally
|
|
478
|
+
gzip/bgzip compressed. Fields are split on that delimiter with no
|
|
479
|
+
CSV quoting: a comma inside a field shifts the later columns, so use
|
|
480
|
+
tabs when a field can contain a comma.
|
|
481
|
+
n_eff : float or str, optional
|
|
482
|
+
A number **overrides** any per-variant N and flattens every row to that
|
|
483
|
+
constant (the low-level escape hatch; :func:`anchor_n_eff` instead keeps
|
|
484
|
+
a column's pattern and rescales its median). A string names the
|
|
485
|
+
per-variant N column (or its zero-based index).
|
|
486
|
+
n_cases, n_controls : float or str, optional
|
|
487
|
+
Numbers give a constant effective N ``4/(1/n_cases + 1/n_controls)``.
|
|
488
|
+
Strings name per-variant case and control count columns.
|
|
489
|
+
require_n : bool, default True
|
|
490
|
+
Raise when no sample size can be found. With ``False`` the ``n_eff``
|
|
491
|
+
column is NaN instead.
|
|
492
|
+
**col_overrides
|
|
493
|
+
Force a field to a column by name or zero-based index, e.g.
|
|
494
|
+
``read_sumstats(path, ea="A1", beta="effect")``.
|
|
495
|
+
|
|
496
|
+
When a ``TEST`` column is present, only ``ADD`` and ``ADD-CONDTL`` rows
|
|
497
|
+
are retained. Covariate, interaction, non-additive and unknown tests are
|
|
498
|
+
dropped with a warning; ``parse_log`` records their count.
|
|
499
|
+
"""
|
|
500
|
+
forced_n = _n_arguments(n_eff, n_cases, n_controls, col_overrides)
|
|
501
|
+
path = os.fspath(path)
|
|
502
|
+
with open_text(path) as fh:
|
|
503
|
+
delim, header = _read_header(fh)
|
|
504
|
+
colmap, layout = _build_colmap(header, col_overrides)
|
|
505
|
+
lower = [h.strip().lower() for h in header]
|
|
506
|
+
|
|
507
|
+
if "variant_string" not in layout:
|
|
508
|
+
for req in ("ea", "oa"):
|
|
509
|
+
if req not in colmap:
|
|
510
|
+
raise ValueError(
|
|
511
|
+
f"could not find a '{req}' allele column in {path}; "
|
|
512
|
+
f"header was {header}. Pass it explicitly, e.g. {req}=...")
|
|
513
|
+
if "beta" not in colmap and "or" not in colmap:
|
|
514
|
+
raise ValueError(
|
|
515
|
+
f"no effect column (beta/OR) found in {path}; header {header}")
|
|
516
|
+
source, detail = _n_plan(colmap, lower, col_overrides, forced_n)
|
|
517
|
+
if source is None and require_n:
|
|
518
|
+
raise ValueError(
|
|
519
|
+
f"no sample-size column found in {path}; pass n_eff=... or "
|
|
520
|
+
"n_cases=/n_controls=")
|
|
521
|
+
plan = {"source": source, "forced": forced_n,
|
|
522
|
+
"i_n": colmap.get("n_eff") if source == "column" else None,
|
|
523
|
+
"i_nca": colmap.get("n_cases") if source == "case_control" else None,
|
|
524
|
+
"i_nco": colmap.get("n_controls") if source == "case_control" else None}
|
|
525
|
+
|
|
526
|
+
def finish(ss):
|
|
527
|
+
log = ss.parse_log
|
|
528
|
+
log["columns"] = {field: header[i] for field, i in colmap.items()}
|
|
529
|
+
if "test" in colmap:
|
|
530
|
+
log["accepted_test_values"] = sorted(_ADDITIVE_TESTS)
|
|
531
|
+
if log["n_dropped_test"]:
|
|
532
|
+
warnings.warn(
|
|
533
|
+
f"ignored {log['n_dropped_test']:,} of "
|
|
534
|
+
f"{log['n_input']:,} rows whose TEST is not ADD or "
|
|
535
|
+
"ADD-CONDTL; only additive association effects are "
|
|
536
|
+
"read", RuntimeWarning, stacklevel=3)
|
|
537
|
+
log["n_eff_source"] = (source if detail is None
|
|
538
|
+
else f"{source}:{detail}") or "missing"
|
|
539
|
+
if layout:
|
|
540
|
+
log["layout"] = {k: (header[v] if isinstance(v, int)
|
|
541
|
+
and not isinstance(v, bool) else v)
|
|
542
|
+
for k, v in layout.items()}
|
|
543
|
+
if source == "case_control":
|
|
544
|
+
warnings.warn(
|
|
545
|
+
"effective N computed per variant from case and control "
|
|
546
|
+
"counts as 4/(1/N_case + 1/N_control). For a meta-analysis "
|
|
547
|
+
"this overstates N_eff when case fractions differ between "
|
|
548
|
+
"cohorts; prefer a published effective-N column.",
|
|
549
|
+
RuntimeWarning, stacklevel=3)
|
|
550
|
+
# A diagnostic, never an inferred transformation: positive effects
|
|
551
|
+
# can be legitimate, and header semantics remain decisive.
|
|
552
|
+
if "beta" in colmap:
|
|
553
|
+
finite = ss.beta[np.isfinite(ss.beta)]
|
|
554
|
+
if (len(finite) >= 20 and np.mean(finite > 0) >= .95
|
|
555
|
+
and .5 < np.median(finite) < 2):
|
|
556
|
+
log["suspected_odds_ratio_in_beta"] = True
|
|
557
|
+
warnings.warn(
|
|
558
|
+
"BETA values are almost all positive and centered near "
|
|
559
|
+
"one; check whether this column contains odds ratios. "
|
|
560
|
+
"Map the OR column explicitly if so; no transformation "
|
|
561
|
+
"has been inferred.", RuntimeWarning, stacklevel=3)
|
|
562
|
+
return ss
|
|
563
|
+
|
|
564
|
+
special = "plink2_alt" in layout or "variant_string" in layout
|
|
565
|
+
kernels = None if special else _jit_kernels()
|
|
566
|
+
# The replay below reopens ``path``, which only a regular file can
|
|
567
|
+
# serve, so pipes keep the single-pass loop.
|
|
568
|
+
if kernels is None or not os.path.isfile(path):
|
|
569
|
+
return finish(_parse_rows_python(fh, delim, colmap, layout, plan))
|
|
570
|
+
try:
|
|
571
|
+
return finish(_parse_rows_jit(fh, delim, colmap, plan, kernels))
|
|
572
|
+
except Exception:
|
|
573
|
+
# Anything the compiled path cannot reproduce exactly replays the
|
|
574
|
+
# whole body through the reference loop, which then returns -- or
|
|
575
|
+
# raises -- what it always does.
|
|
576
|
+
pass
|
|
577
|
+
with open_text(path) as fh:
|
|
578
|
+
fh.readline()
|
|
579
|
+
return finish(_parse_rows_python(fh, delim, colmap, layout, plan))
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _case_control_n(nca, nco):
|
|
583
|
+
"""Per-row ``4/(1/nca + 1/nco)``; NaN unless both counts are positive."""
|
|
584
|
+
nca = np.asarray(nca, dtype=float)
|
|
585
|
+
nco = np.asarray(nco, dtype=float)
|
|
586
|
+
ok = np.isfinite(nca) & np.isfinite(nco) & (nca > 0) & (nco > 0)
|
|
587
|
+
out = np.full(nca.shape, math.nan)
|
|
588
|
+
out[ok] = 4.0 / (1.0 / nca[ok] + 1.0 / nco[ok])
|
|
589
|
+
return out
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
# ---------------------------------------------------------------------------
|
|
593
|
+
# Optional Numba tokenizer (from ldpred3).
|
|
594
|
+
#
|
|
595
|
+
# Without Numba the reference row loop runs. With Numba, ASCII blocks of the
|
|
596
|
+
# file are tokenized and their numeric cells parsed in compiled code, about 3x
|
|
597
|
+
# faster, and anything the kernels cannot reproduce bit for bit goes back to
|
|
598
|
+
# the exact Python rules, cell by cell or block by block.
|
|
599
|
+
#
|
|
600
|
+
# Numeric cells use Clinger's fast path: a strict ``[+-]digits[.digits]
|
|
601
|
+
# [(e|E)[+-]digits]`` token with at most 15 significant digits and a net
|
|
602
|
+
# decimal exponent within +-22 is one correctly rounded IEEE multiply or
|
|
603
|
+
# divide -- the same double CPython's ``float()`` returns. Every other token is
|
|
604
|
+
# flagged and parsed by ``_to_float``.
|
|
605
|
+
# ---------------------------------------------------------------------------
|
|
606
|
+
|
|
607
|
+
_BLOCK_CHARS = 1 << 22
|
|
608
|
+
_JIT_MIN_CHARS = 1 << 21
|
|
609
|
+
_INT64_LIMIT = float(2 ** 63)
|
|
610
|
+
_POW10 = np.array([float(10 ** k) for k in range(23)]) # all exact doubles
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _tokenize(b, delim, starts, ends):
|
|
614
|
+
"""Column-major field offsets of a rectangular block of lines.
|
|
615
|
+
|
|
616
|
+
``delim < 0`` means runs of ASCII whitespace (``str.split()``), else that
|
|
617
|
+
single byte (``str.split(delim)``). Empty lines are skipped. Returns the
|
|
618
|
+
number of lines filled, or -1 as soon as a line has another field count.
|
|
619
|
+
"""
|
|
620
|
+
k = starts.shape[0]
|
|
621
|
+
r = 0
|
|
622
|
+
j = 0
|
|
623
|
+
fs = 0
|
|
624
|
+
ls = 0
|
|
625
|
+
intok = False
|
|
626
|
+
for i in range(len(b)):
|
|
627
|
+
c = b[i]
|
|
628
|
+
if delim >= 0:
|
|
629
|
+
if c == delim or (c == 10 and i > ls):
|
|
630
|
+
if j >= k:
|
|
631
|
+
return -1
|
|
632
|
+
starts[j, r] = fs
|
|
633
|
+
ends[j, r] = i
|
|
634
|
+
j += 1
|
|
635
|
+
if c == delim or c == 10:
|
|
636
|
+
fs = i + 1
|
|
637
|
+
elif c == 32 or 9 <= c <= 13 or 28 <= c <= 31:
|
|
638
|
+
if intok:
|
|
639
|
+
ends[j, r] = i
|
|
640
|
+
j += 1
|
|
641
|
+
intok = False
|
|
642
|
+
elif not intok:
|
|
643
|
+
if j >= k:
|
|
644
|
+
return -1
|
|
645
|
+
starts[j, r] = i
|
|
646
|
+
intok = True
|
|
647
|
+
if c == 10:
|
|
648
|
+
if i > ls:
|
|
649
|
+
if j != k:
|
|
650
|
+
return -1
|
|
651
|
+
r += 1
|
|
652
|
+
j = 0
|
|
653
|
+
ls = i + 1
|
|
654
|
+
return r
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def _parse_decimal(b, starts, ends, pow10, out, ok):
|
|
658
|
+
"""Clinger fast-path parse of each token; ``ok[k]`` False = not handled."""
|
|
659
|
+
for k in range(len(starts)):
|
|
660
|
+
i = starts[k]
|
|
661
|
+
e = ends[k]
|
|
662
|
+
ok[k] = False
|
|
663
|
+
neg = False
|
|
664
|
+
if i < e and (b[i] == 43 or b[i] == 45):
|
|
665
|
+
neg = b[i] == 45
|
|
666
|
+
i += 1
|
|
667
|
+
mant = 0
|
|
668
|
+
nd = 0
|
|
669
|
+
ndig = 0
|
|
670
|
+
frac = 0
|
|
671
|
+
bad = False
|
|
672
|
+
while i < e and 48 <= b[i] <= 57:
|
|
673
|
+
d = b[i] - 48
|
|
674
|
+
ndig += 1
|
|
675
|
+
if mant != 0 or d != 0:
|
|
676
|
+
nd += 1
|
|
677
|
+
mant = mant * 10 + d
|
|
678
|
+
i += 1
|
|
679
|
+
if i < e and b[i] == 46:
|
|
680
|
+
i += 1
|
|
681
|
+
while i < e and 48 <= b[i] <= 57:
|
|
682
|
+
d = b[i] - 48
|
|
683
|
+
ndig += 1
|
|
684
|
+
frac += 1
|
|
685
|
+
if mant != 0 or d != 0:
|
|
686
|
+
nd += 1
|
|
687
|
+
mant = mant * 10 + d
|
|
688
|
+
i += 1
|
|
689
|
+
if ndig == 0 or nd > 15:
|
|
690
|
+
bad = True
|
|
691
|
+
ex = 0
|
|
692
|
+
if not bad and i < e and (b[i] == 101 or b[i] == 69):
|
|
693
|
+
i += 1
|
|
694
|
+
eneg = False
|
|
695
|
+
if i < e and (b[i] == 43 or b[i] == 45):
|
|
696
|
+
eneg = b[i] == 45
|
|
697
|
+
i += 1
|
|
698
|
+
nexp = 0
|
|
699
|
+
while i < e and 48 <= b[i] <= 57:
|
|
700
|
+
if nexp < 6:
|
|
701
|
+
ex = ex * 10 + (b[i] - 48)
|
|
702
|
+
nexp += 1
|
|
703
|
+
i += 1
|
|
704
|
+
if nexp == 0 or nexp > 5:
|
|
705
|
+
bad = True
|
|
706
|
+
if eneg:
|
|
707
|
+
ex = -ex
|
|
708
|
+
if bad or i != e:
|
|
709
|
+
continue
|
|
710
|
+
e10 = ex - frac
|
|
711
|
+
if mant == 0:
|
|
712
|
+
v = 0.0
|
|
713
|
+
elif 0 <= e10 <= 22:
|
|
714
|
+
v = float(mant) * pow10[e10]
|
|
715
|
+
elif -22 <= e10 < 0:
|
|
716
|
+
v = float(mant) / pow10[-e10]
|
|
717
|
+
else:
|
|
718
|
+
continue
|
|
719
|
+
out[k] = -v if neg else v
|
|
720
|
+
ok[k] = True
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _pool_codes(b, starts, ends, codes, first):
|
|
724
|
+
"""Code each token of <= 7 bytes by its distinct value, first-seen order.
|
|
725
|
+
|
|
726
|
+
``codes[k]`` indexes the distinct tokens; a longer token gets -1. Returns
|
|
727
|
+
the number of distinct tokens.
|
|
728
|
+
"""
|
|
729
|
+
n = len(starts)
|
|
730
|
+
size = 64
|
|
731
|
+
while size < 2 * n:
|
|
732
|
+
size *= 2
|
|
733
|
+
table_key = np.full(size, -1, dtype=np.int64)
|
|
734
|
+
table_code = np.empty(size, dtype=np.int64)
|
|
735
|
+
ndistinct = 0
|
|
736
|
+
for k in range(n):
|
|
737
|
+
s = starts[k]
|
|
738
|
+
ln = ends[k] - s
|
|
739
|
+
if ln > 7:
|
|
740
|
+
codes[k] = -1
|
|
741
|
+
continue
|
|
742
|
+
key = np.int64(ln)
|
|
743
|
+
for j in range(ln):
|
|
744
|
+
key |= np.int64(b[s + j]) << (8 * (j + 1))
|
|
745
|
+
h = (key * np.int64(-7046029254386353131)) >> 40
|
|
746
|
+
slot = h & (size - 1)
|
|
747
|
+
while table_key[slot] != -1 and table_key[slot] != key:
|
|
748
|
+
slot = (slot + 1) & (size - 1)
|
|
749
|
+
if table_key[slot] == -1:
|
|
750
|
+
table_key[slot] = key
|
|
751
|
+
table_code[slot] = ndistinct
|
|
752
|
+
first[ndistinct] = k
|
|
753
|
+
ndistinct += 1
|
|
754
|
+
codes[k] = table_code[slot]
|
|
755
|
+
return ndistinct
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
_KERNELS = None
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
def _jit_kernels():
|
|
762
|
+
"""Compiled kernels, or ``None`` when Numba (or its JIT) is unavailable."""
|
|
763
|
+
global _KERNELS
|
|
764
|
+
if _KERNELS is None:
|
|
765
|
+
try:
|
|
766
|
+
from ._numba import jit_available, jit_nogil
|
|
767
|
+
if not jit_available():
|
|
768
|
+
_KERNELS = False
|
|
769
|
+
else:
|
|
770
|
+
_KERNELS = (jit_nogil(_tokenize), jit_nogil(_parse_decimal),
|
|
771
|
+
jit_nogil(_pool_codes))
|
|
772
|
+
except Exception: # a broken Numba install: no JIT
|
|
773
|
+
_KERNELS = False
|
|
774
|
+
return _KERNELS or None
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _empty_log(n, n_se, n_dropped_test=None):
|
|
778
|
+
log = {"n_input": int(n), "n_se_from_pval": int(n_se)}
|
|
779
|
+
if n_dropped_test is not None:
|
|
780
|
+
log.update(n_dropped_test=int(n_dropped_test),
|
|
781
|
+
n_kept=int(n - n_dropped_test))
|
|
782
|
+
return log
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def _parse_rows_jit(fh, delim, colmap, plan, kernels):
|
|
786
|
+
"""Block-wise equivalent of :func:`_parse_rows_python` using ``kernels``.
|
|
787
|
+
|
|
788
|
+
A block goes through the compiled path only when it is ASCII and
|
|
789
|
+
rectangular (every non-empty line has the same field count, at least the
|
|
790
|
+
mapped columns); any other block is parsed by the reference loop itself,
|
|
791
|
+
sharing the string pool.
|
|
792
|
+
"""
|
|
793
|
+
tokenize, parse_decimal, pool_codes = kernels
|
|
794
|
+
from statistics import NormalDist
|
|
795
|
+
_inv_cdf = NormalDist().inv_cdf
|
|
796
|
+
dcode = -1 if delim is None else ord(delim)
|
|
797
|
+
i_id = colmap.get("id")
|
|
798
|
+
i_chrom = colmap.get("chrom")
|
|
799
|
+
i_pos = colmap.get("pos")
|
|
800
|
+
i_beta = colmap.get("beta")
|
|
801
|
+
i_se = colmap.get("se")
|
|
802
|
+
i_pval = colmap.get("pval")
|
|
803
|
+
i_log10p = colmap.get("log10p")
|
|
804
|
+
i_eaf = colmap.get("eaf")
|
|
805
|
+
i_info = colmap.get("info")
|
|
806
|
+
i_test = colmap.get("test")
|
|
807
|
+
nfields = max(colmap.values()) + 1
|
|
808
|
+
pool_dict = {}
|
|
809
|
+
pool = pool_dict.setdefault
|
|
810
|
+
nan = math.nan
|
|
811
|
+
|
|
812
|
+
def reference(text):
|
|
813
|
+
return _parse_rows_python(text.split("\n"), delim, colmap, {}, plan,
|
|
814
|
+
_pool=pool_dict)
|
|
815
|
+
|
|
816
|
+
def block(text):
|
|
817
|
+
if (len(text) < _JIT_MIN_CHARS or not text.endswith("\n")
|
|
818
|
+
or not text.isascii()):
|
|
819
|
+
return reference(text)
|
|
820
|
+
b = np.frombuffer(text.encode("ascii"), dtype=np.uint8)
|
|
821
|
+
first_line = text[:text.index("\n")]
|
|
822
|
+
k0 = len(first_line.split(delim) if delim else first_line.split())
|
|
823
|
+
if not first_line or k0 < nfields:
|
|
824
|
+
return reference(text)
|
|
825
|
+
nlines = text.count("\n")
|
|
826
|
+
starts = np.empty((k0, nlines), dtype=np.int64)
|
|
827
|
+
ends = np.empty((k0, nlines), dtype=np.int64)
|
|
828
|
+
m = tokenize(b, dcode, starts, ends)
|
|
829
|
+
if m < 0:
|
|
830
|
+
return reference(text)
|
|
831
|
+
n_input = m
|
|
832
|
+
if i_test is not None:
|
|
833
|
+
keep = np.fromiter(
|
|
834
|
+
(text[a:z].strip().upper() in _ADDITIVE_TESTS
|
|
835
|
+
for a, z in zip(starts[i_test, :m].tolist(),
|
|
836
|
+
ends[i_test, :m].tolist())),
|
|
837
|
+
dtype=bool, count=m)
|
|
838
|
+
if not keep.all():
|
|
839
|
+
starts, ends = starts[:, :m][:, keep], ends[:, :m][:, keep]
|
|
840
|
+
m = int(keep.sum())
|
|
841
|
+
|
|
842
|
+
def floats(i):
|
|
843
|
+
s = starts[i, :m]
|
|
844
|
+
e = ends[i, :m]
|
|
845
|
+
out = np.empty(m)
|
|
846
|
+
ok = np.empty(m, dtype=np.bool_)
|
|
847
|
+
parse_decimal(b, s, e, _POW10, out, ok)
|
|
848
|
+
bad = np.flatnonzero(~ok)
|
|
849
|
+
if len(bad):
|
|
850
|
+
memo = {}
|
|
851
|
+
for k, a, z in zip(bad.tolist(), s[bad].tolist(),
|
|
852
|
+
e[bad].tolist()):
|
|
853
|
+
tok = text[a:z]
|
|
854
|
+
v = memo.get(tok)
|
|
855
|
+
if v is None:
|
|
856
|
+
v = memo[tok] = _to_float(tok)
|
|
857
|
+
out[k] = v
|
|
858
|
+
return out
|
|
859
|
+
|
|
860
|
+
def tokens(i, rows):
|
|
861
|
+
s = starts[i, rows]
|
|
862
|
+
e = ends[i, rows]
|
|
863
|
+
return [text[a:z] for a, z in zip(s.tolist(), e.tolist())]
|
|
864
|
+
|
|
865
|
+
def strings(i, transform):
|
|
866
|
+
s = starts[i, :m]
|
|
867
|
+
e = ends[i, :m]
|
|
868
|
+
if transform is None:
|
|
869
|
+
return np.array(list(map(text.__getitem__,
|
|
870
|
+
map(slice, s.tolist(), e.tolist()))),
|
|
871
|
+
dtype=object)
|
|
872
|
+
codes = np.empty(m, dtype=np.int64)
|
|
873
|
+
first = np.empty(m, dtype=np.int64)
|
|
874
|
+
u = pool_codes(b, s, e, codes, first)
|
|
875
|
+
vals = np.empty(u + 1, dtype=object) # [-1] -> None
|
|
876
|
+
vals[:u] = [transform(text[a:z]) for a, z in
|
|
877
|
+
zip(s[first[:u]].tolist(), e[first[:u]].tolist())]
|
|
878
|
+
out = vals[codes]
|
|
879
|
+
for k in np.flatnonzero(codes < 0).tolist():
|
|
880
|
+
out[k] = transform(text[s[k]:e[k]])
|
|
881
|
+
return out
|
|
882
|
+
|
|
883
|
+
if i_beta is not None:
|
|
884
|
+
beta = floats(i_beta)
|
|
885
|
+
else:
|
|
886
|
+
orv = floats(colmap["or"])
|
|
887
|
+
beta = np.full(m, nan)
|
|
888
|
+
pos_or = orv > 0
|
|
889
|
+
# math.log, not np.log: NumPy's SIMD log may differ from libm in
|
|
890
|
+
# the last ulp, and the reference uses math.log.
|
|
891
|
+
beta[pos_or] = np.fromiter(map(math.log, orv[pos_or].tolist()),
|
|
892
|
+
dtype=float, count=int(pos_or.sum()))
|
|
893
|
+
se = floats(i_se) if i_se is not None else np.full(m, nan)
|
|
894
|
+
n_se_from_p = 0
|
|
895
|
+
need = np.isnan(se) & ~np.isnan(beta)
|
|
896
|
+
if need.any() and i_pval is not None:
|
|
897
|
+
p = floats(i_pval)
|
|
898
|
+
if np.any(p[need] < np.finfo(float).tiny):
|
|
899
|
+
return reference(text)
|
|
900
|
+
need &= (p > 0) & (p < 1) & (beta != 0)
|
|
901
|
+
for k in np.flatnonzero(need).tolist():
|
|
902
|
+
half = float(p[k]) / 2.0
|
|
903
|
+
if half > 0.0:
|
|
904
|
+
z = -_inv_cdf(half)
|
|
905
|
+
if math.isfinite(z) and z > 0:
|
|
906
|
+
se[k] = abs(float(beta[k])) / z
|
|
907
|
+
n_se_from_p += 1
|
|
908
|
+
elif need.any() and i_log10p is not None:
|
|
909
|
+
rows = np.flatnonzero(need & (beta != 0))
|
|
910
|
+
for k, tok in zip(rows.tolist(), tokens(i_log10p, rows)):
|
|
911
|
+
z = _z_from_log10p(tok)
|
|
912
|
+
if math.isfinite(z) and z > 0:
|
|
913
|
+
se[k] = abs(float(beta[k])) / z
|
|
914
|
+
n_se_from_p += 1
|
|
915
|
+
if i_pos is not None:
|
|
916
|
+
posf = floats(i_pos)
|
|
917
|
+
fin = np.isfinite(posf)
|
|
918
|
+
if ((posf[fin] >= _INT64_LIMIT)
|
|
919
|
+
| (posf[fin] < -_INT64_LIMIT)).any():
|
|
920
|
+
raise OverflowError("position outside int64")
|
|
921
|
+
pos = np.where(fin, posf, 0.0).astype(np.int64)
|
|
922
|
+
else:
|
|
923
|
+
pos = np.zeros(m, dtype=np.int64)
|
|
924
|
+
|
|
925
|
+
if plan["forced"] is not None:
|
|
926
|
+
n_eff = np.full(m, plan["forced"])
|
|
927
|
+
elif plan["i_n"] is not None:
|
|
928
|
+
n_eff = floats(plan["i_n"])
|
|
929
|
+
elif plan["i_nca"] is not None:
|
|
930
|
+
n_eff = _case_control_n(floats(plan["i_nca"]),
|
|
931
|
+
floats(plan["i_nco"]))
|
|
932
|
+
else:
|
|
933
|
+
n_eff = np.full(m, nan)
|
|
934
|
+
|
|
935
|
+
def pooled(t):
|
|
936
|
+
return pool(t, t)
|
|
937
|
+
|
|
938
|
+
def allele(t):
|
|
939
|
+
t = t.upper()
|
|
940
|
+
return pool(t, t)
|
|
941
|
+
|
|
942
|
+
return Sumstats(
|
|
943
|
+
id=(strings(i_id, None) if i_id is not None
|
|
944
|
+
else np.full(m, "", dtype=object)),
|
|
945
|
+
chrom=(strings(i_chrom, pooled) if i_chrom is not None
|
|
946
|
+
else np.full(m, pooled(""), dtype=object)),
|
|
947
|
+
pos=pos,
|
|
948
|
+
ea=strings(colmap["ea"], allele),
|
|
949
|
+
oa=strings(colmap["oa"], allele),
|
|
950
|
+
beta=beta, se=se, n_eff=n_eff,
|
|
951
|
+
eaf=floats(i_eaf) if i_eaf is not None else np.full(m, nan),
|
|
952
|
+
info=floats(i_info) if i_info is not None else np.full(m, nan),
|
|
953
|
+
parse_log=_empty_log(n_input, n_se_from_p,
|
|
954
|
+
n_input - m if i_test is not None else None),
|
|
955
|
+
)
|
|
956
|
+
|
|
957
|
+
pieces = []
|
|
958
|
+
carry = ""
|
|
959
|
+
while True:
|
|
960
|
+
text = fh.read(_BLOCK_CHARS)
|
|
961
|
+
if not text:
|
|
962
|
+
if carry:
|
|
963
|
+
pieces.append(block(carry))
|
|
964
|
+
break
|
|
965
|
+
text = carry + text
|
|
966
|
+
cut = text.rfind("\n") + 1
|
|
967
|
+
carry = text[cut:]
|
|
968
|
+
if cut:
|
|
969
|
+
pieces.append(block(text[:cut]))
|
|
970
|
+
if not pieces:
|
|
971
|
+
return reference("")
|
|
972
|
+
if len(pieces) == 1:
|
|
973
|
+
return pieces[0]
|
|
974
|
+
return Sumstats(*(np.concatenate([getattr(p, f) for p in pieces])
|
|
975
|
+
for f in _SUMSTATS_FIELDS),
|
|
976
|
+
parse_log=_empty_log(
|
|
977
|
+
sum(p.parse_log["n_input"] for p in pieces),
|
|
978
|
+
sum(p.parse_log["n_se_from_pval"] for p in pieces),
|
|
979
|
+
sum(p.parse_log["n_dropped_test"] for p in pieces)
|
|
980
|
+
if i_test is not None else None))
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
def _parse_rows_python(fh, delim, colmap, layout, plan, _pool=None):
|
|
984
|
+
"""Reference row-at-a-time parser for the body lines of ``fh``.
|
|
985
|
+
|
|
986
|
+
The semantic definition of a sumstats row, and the fallback for
|
|
987
|
+
:func:`_parse_rows_jit`. Chromosome labels and alleles are pooled through
|
|
988
|
+
one dict: they come from tiny alphabets, and a fresh ``str`` per cell
|
|
989
|
+
retained 1.88 GB of duplicates at 10 million rows in ldpred3's measurement.
|
|
990
|
+
"""
|
|
991
|
+
if _pool is None:
|
|
992
|
+
_pool = {}
|
|
993
|
+
ids, chroms, poss, eas, oas = [], [], [], [], []
|
|
994
|
+
betas, ses, ns, ncas, ncos, eafs, infos = [], [], [], [], [], [], []
|
|
995
|
+
n_se_from_p = 0
|
|
996
|
+
n_input = n_dropped_test = 0
|
|
997
|
+
|
|
998
|
+
i_id = colmap.get("id")
|
|
999
|
+
i_chrom = colmap.get("chrom")
|
|
1000
|
+
i_pos = colmap.get("pos")
|
|
1001
|
+
i_ea = colmap.get("ea")
|
|
1002
|
+
i_oa = colmap.get("oa")
|
|
1003
|
+
i_beta = colmap.get("beta")
|
|
1004
|
+
i_or = colmap.get("or")
|
|
1005
|
+
i_se = colmap.get("se")
|
|
1006
|
+
i_pval = colmap.get("pval")
|
|
1007
|
+
i_log10p = colmap.get("log10p")
|
|
1008
|
+
i_eaf = colmap.get("eaf")
|
|
1009
|
+
i_info = colmap.get("info")
|
|
1010
|
+
i_test = colmap.get("test")
|
|
1011
|
+
i_n, i_nca, i_nco = plan["i_n"], plan["i_nca"], plan["i_nco"]
|
|
1012
|
+
forced_n = plan["forced"]
|
|
1013
|
+
i_alt = layout.get("plink2_alt")
|
|
1014
|
+
i_var = layout.get("variant_string")
|
|
1015
|
+
used = [i for i in (i_id, i_chrom, i_pos, i_ea, i_oa, i_beta, i_or, i_se,
|
|
1016
|
+
i_pval, i_log10p, i_n, i_nca, i_nco, i_eaf, i_info,
|
|
1017
|
+
i_alt, i_var, i_test) if i is not None]
|
|
1018
|
+
nfields = max(used) + 1
|
|
1019
|
+
|
|
1020
|
+
id_add, chrom_add, pos_add = ids.append, chroms.append, poss.append
|
|
1021
|
+
ea_add, oa_add = eas.append, oas.append
|
|
1022
|
+
beta_add, se_add, n_add = betas.append, ses.append, ns.append
|
|
1023
|
+
eaf_add, info_add = eafs.append, infos.append
|
|
1024
|
+
pool = _pool.setdefault
|
|
1025
|
+
isnan, isfinite, nan = math.isnan, math.isfinite, math.nan
|
|
1026
|
+
|
|
1027
|
+
for raw in fh:
|
|
1028
|
+
raw = raw.rstrip("\r\n")
|
|
1029
|
+
if not raw:
|
|
1030
|
+
continue
|
|
1031
|
+
n_input += 1
|
|
1032
|
+
f = raw.split(delim) if delim else raw.split()
|
|
1033
|
+
if len(f) < nfields: # pad once, not per field
|
|
1034
|
+
f = f + [""] * (nfields - len(f))
|
|
1035
|
+
if i_test is not None and f[i_test].strip().upper() not in _ADDITIVE_TESTS:
|
|
1036
|
+
n_dropped_test += 1
|
|
1037
|
+
continue
|
|
1038
|
+
|
|
1039
|
+
if i_beta is not None:
|
|
1040
|
+
beta = _to_float(f[i_beta])
|
|
1041
|
+
else:
|
|
1042
|
+
orv = _to_float(f[i_or])
|
|
1043
|
+
beta = math.log(orv) if orv > 0 else nan
|
|
1044
|
+
|
|
1045
|
+
se = _to_float(f[i_se]) if i_se is not None else nan
|
|
1046
|
+
if isnan(se) and not isnan(beta) and beta != 0:
|
|
1047
|
+
if i_pval is not None:
|
|
1048
|
+
z = _two_sided_z(f[i_pval])
|
|
1049
|
+
elif i_log10p is not None:
|
|
1050
|
+
z = _z_from_log10p(f[i_log10p])
|
|
1051
|
+
else:
|
|
1052
|
+
z = nan
|
|
1053
|
+
if isfinite(z) and z > 0:
|
|
1054
|
+
se = abs(beta) / z
|
|
1055
|
+
n_se_from_p += 1
|
|
1056
|
+
|
|
1057
|
+
if forced_n is not None:
|
|
1058
|
+
n = forced_n
|
|
1059
|
+
elif i_n is not None:
|
|
1060
|
+
n = _to_float(f[i_n])
|
|
1061
|
+
else:
|
|
1062
|
+
n = nan
|
|
1063
|
+
if i_nca is not None:
|
|
1064
|
+
ncas.append(_to_float(f[i_nca]))
|
|
1065
|
+
ncos.append(_to_float(f[i_nco]))
|
|
1066
|
+
|
|
1067
|
+
if i_var is not None:
|
|
1068
|
+
parts = f[i_var].split(":")
|
|
1069
|
+
if len(parts) >= 4:
|
|
1070
|
+
c, p_tok, ref, alt = parts[0], parts[1], parts[2], parts[3]
|
|
1071
|
+
else:
|
|
1072
|
+
c, p_tok, ref, alt = "", "", "", ""
|
|
1073
|
+
if i_chrom is None:
|
|
1074
|
+
chrom_add(pool(c, c))
|
|
1075
|
+
if i_pos is None:
|
|
1076
|
+
pos_v = _to_float(p_tok)
|
|
1077
|
+
pos_add(int(pos_v) if isfinite(pos_v) else 0)
|
|
1078
|
+
ea = alt.upper()
|
|
1079
|
+
oa = ref.upper()
|
|
1080
|
+
else:
|
|
1081
|
+
ea = f[i_ea].upper()
|
|
1082
|
+
oa = f[i_oa].upper()
|
|
1083
|
+
if i_alt is not None and ea == oa:
|
|
1084
|
+
oa = f[i_alt].upper()
|
|
1085
|
+
if i_var is None or i_chrom is not None:
|
|
1086
|
+
c = str(f[i_chrom]) if i_chrom is not None else ""
|
|
1087
|
+
chrom_add(pool(c, c))
|
|
1088
|
+
if i_var is None or i_pos is not None:
|
|
1089
|
+
# A blank / non-numeric position is missing (0), not an error.
|
|
1090
|
+
pos_v = _to_float(f[i_pos]) if i_pos is not None else 0.0
|
|
1091
|
+
pos_add(int(pos_v) if isfinite(pos_v) else 0)
|
|
1092
|
+
id_add(f[i_id] if i_id is not None else "")
|
|
1093
|
+
ea_add(pool(ea, ea))
|
|
1094
|
+
oa_add(pool(oa, oa))
|
|
1095
|
+
beta_add(beta); se_add(se); n_add(n)
|
|
1096
|
+
eaf_add(_to_float(f[i_eaf]) if i_eaf is not None else nan)
|
|
1097
|
+
info_add(_to_float(f[i_info]) if i_info is not None else nan)
|
|
1098
|
+
|
|
1099
|
+
n_eff = np.array(ns, dtype=float)
|
|
1100
|
+
if i_nca is not None and forced_n is None:
|
|
1101
|
+
n_eff = _case_control_n(ncas, ncos)
|
|
1102
|
+
return Sumstats(
|
|
1103
|
+
id=np.array(ids, dtype=object),
|
|
1104
|
+
chrom=np.array(chroms, dtype=object),
|
|
1105
|
+
pos=np.array(poss, dtype=np.int64),
|
|
1106
|
+
ea=np.array(eas, dtype=object),
|
|
1107
|
+
oa=np.array(oas, dtype=object),
|
|
1108
|
+
beta=np.array(betas, dtype=float),
|
|
1109
|
+
se=np.array(ses, dtype=float),
|
|
1110
|
+
n_eff=n_eff,
|
|
1111
|
+
eaf=np.array(eafs, dtype=float),
|
|
1112
|
+
info=np.array(infos, dtype=float),
|
|
1113
|
+
parse_log=_empty_log(n_input, n_se_from_p,
|
|
1114
|
+
n_dropped_test if i_test is not None else None),
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
def _n_eff_scalar_value(n_eff):
|
|
1119
|
+
"""Float value of a scalar ``n_eff``, or ``None`` (absent or a column name)."""
|
|
1120
|
+
if n_eff is None or isinstance(n_eff, str):
|
|
1121
|
+
return None
|
|
1122
|
+
try:
|
|
1123
|
+
value = float(n_eff)
|
|
1124
|
+
except (TypeError, ValueError, OverflowError):
|
|
1125
|
+
raise ValueError("n_eff must be a finite positive number") from None
|
|
1126
|
+
if (isinstance(n_eff, (bool, np.bool_)) or not math.isfinite(value)
|
|
1127
|
+
or value <= 0):
|
|
1128
|
+
raise ValueError("n_eff must be a finite positive number")
|
|
1129
|
+
return value
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def rescale_n_eff(ss, target):
|
|
1133
|
+
"""Anchor a per-variant N column's median at ``target``, downward only.
|
|
1134
|
+
|
|
1135
|
+
``ss.n_eff`` is rescaled **in place** by ``min(1, target / median)`` over
|
|
1136
|
+
its usable (finite, positive) values: the column's relative pattern is
|
|
1137
|
+
kept, values are never scaled up (a per-variant N below the target is a
|
|
1138
|
+
measurement, not a deficit) and unusable rows stay NaN. Returns the
|
|
1139
|
+
transform record, or ``None`` when no value is usable.
|
|
1140
|
+
"""
|
|
1141
|
+
value = _n_eff_scalar_value(target)
|
|
1142
|
+
if value is None:
|
|
1143
|
+
raise ValueError("n_eff must be a finite positive number")
|
|
1144
|
+
n_col = np.asarray(ss.n_eff, dtype=float)
|
|
1145
|
+
usable = np.isfinite(n_col) & (n_col > 0)
|
|
1146
|
+
n_usable = int(usable.sum())
|
|
1147
|
+
if n_usable == 0:
|
|
1148
|
+
return None
|
|
1149
|
+
median = float(np.median(n_col[usable]))
|
|
1150
|
+
factor = min(1.0, value / median)
|
|
1151
|
+
applied = bool(factor < 1.0 - 1e-12)
|
|
1152
|
+
if applied:
|
|
1153
|
+
n_col[usable] = n_col[usable] * factor
|
|
1154
|
+
ss.n_eff = n_col
|
|
1155
|
+
return {
|
|
1156
|
+
"method": "downward median rescale",
|
|
1157
|
+
"source_median": median,
|
|
1158
|
+
"target_effective_n": value,
|
|
1159
|
+
"factor": factor,
|
|
1160
|
+
"applied": applied,
|
|
1161
|
+
"upward_scaling_refused": bool(value > median and factor == 1.0),
|
|
1162
|
+
"n_usable": n_usable,
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
|
|
1166
|
+
def anchor_n_eff(ss, target):
|
|
1167
|
+
"""Apply a supplied scalar N: rescale the column, or set a constant.
|
|
1168
|
+
|
|
1169
|
+
With at least one usable per-variant value this is :func:`rescale_n_eff`;
|
|
1170
|
+
with none, ``target`` becomes every row's N. Mutates ``ss`` and returns
|
|
1171
|
+
the transform record.
|
|
1172
|
+
"""
|
|
1173
|
+
record = rescale_n_eff(ss, target)
|
|
1174
|
+
if record is not None:
|
|
1175
|
+
return record
|
|
1176
|
+
target = float(target)
|
|
1177
|
+
ss.n_eff = np.full(len(ss), target, dtype=float)
|
|
1178
|
+
return {"method": "constant scalar (no usable per-variant N)",
|
|
1179
|
+
"source_median": None, "target_effective_n": target,
|
|
1180
|
+
"factor": 1.0, "applied": True,
|
|
1181
|
+
"upward_scaling_refused": False, "n_usable": 0}
|
|
1182
|
+
|
|
1183
|
+
|
|
1184
|
+
_WRITE_HEADER = ("ID", "CHR", "POS", "EA", "OA", "BETA", "SE", "N_EFF",
|
|
1185
|
+
"EAF", "INFO")
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
def _fmt(value):
|
|
1189
|
+
# repr round-trips float64 exactly; NaN is written as NA.
|
|
1190
|
+
return "NA" if value != value else repr(float(value))
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
def write_sumstats(path, ss):
|
|
1194
|
+
"""Write ``ss`` as the canonical tab-separated table; returns ``path``.
|
|
1195
|
+
|
|
1196
|
+
Columns ``ID CHR POS EA OA BETA SE N_EFF EAF INFO``, floats written so they
|
|
1197
|
+
read back bit for bit, NaN as ``NA``; a ``.gz`` path is compressed. The
|
|
1198
|
+
file reads back with :func:`read_sumstats` without overrides.
|
|
1199
|
+
"""
|
|
1200
|
+
with atomic_text(path) as fh:
|
|
1201
|
+
fh.write("\t".join(_WRITE_HEADER) + "\n")
|
|
1202
|
+
for row in zip(ss.id, ss.chrom, ss.pos, ss.ea, ss.oa, ss.beta, ss.se,
|
|
1203
|
+
ss.n_eff, ss.eaf, ss.info):
|
|
1204
|
+
vid, c, p, ea, oa, b, s, n, f, q = row
|
|
1205
|
+
fh.write(f"{vid or '.'}\t{c}\t{int(p)}\t{ea}\t{oa}\t{_fmt(b)}\t"
|
|
1206
|
+
f"{_fmt(s)}\t{_fmt(n)}\t{_fmt(f)}\t{_fmt(q)}\n")
|
|
1207
|
+
return path
|