allelign 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
allelign/sumstats.py ADDED
@@ -0,0 +1,1207 @@
1
+ """Read GWAS summary statistics into one canonical table.
2
+
3
+ GWAS arrive as delimited text with wildly inconsistent column names. This
4
+ reader maps a large set of aliases onto canonical fields and lets the caller
5
+ override any column explicitly:
6
+
7
+ ============== ===========================================================
8
+ field meaning
9
+ ============== ===========================================================
10
+ ``id`` variant identifier (rsID), may be empty
11
+ ``chrom`` chromosome label as read, may be empty
12
+ ``pos`` base-pair position (int), 0 when missing
13
+ ``ea`` effect allele: ``beta`` is per copy of this allele
14
+ ``oa`` other (non-effect) allele
15
+ ``beta`` per-allele effect (log-OR for binary traits)
16
+ ``se`` standard error of ``beta``
17
+ ``n_eff`` (effective) sample size per variant
18
+ ``eaf`` effect-allele frequency (NaN when absent)
19
+ ``info`` imputation quality (NaN when absent)
20
+ ============== ===========================================================
21
+
22
+ Odds ratios become ``beta = log(OR)``. A missing ``se`` is recovered from
23
+ ``beta`` and the p-value (or ``-log10 p``) without underflow. Effective N comes,
24
+ in order of preference, from an explicit effective-N column, from per-variant
25
+ case and control counts (``4 / (1/N_case + 1/N_control)``), or from a total-N
26
+ column; :attr:`Sumstats.parse_log` records which.
27
+
28
+ Format-specific layouts are recognised from the header: SAIGE (effects for
29
+ Allele2), PLINK2 (``A1`` tested against ``REF``/``ALT``, or ``OMITTED``),
30
+ REGENIE (``GENPOS``), GWAS Catalog harmonised files (the ``hm_`` columns as one
31
+ consistent set), PGC ``daner`` files (``FRQ_U_*`` control frequency) and Neale
32
+ UK Biobank files (``chr:pos:ref:alt`` variant strings, effect on ALT).
33
+
34
+ Ported from ldpred3's reader, including its optional Numba block parser, whose
35
+ output is bit-identical to the reference row loop. On ldpred3's standard inputs
36
+ the result is identical to ``ldpred3.sumstats.read_sumstats``; the additions
37
+ above are documented differences.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ from dataclasses import dataclass
43
+ import math
44
+ import os
45
+ import warnings
46
+
47
+ import numpy as np
48
+
49
+ from ._common import atomic_text, open_text
50
+
51
+ __all__ = ["Sumstats", "read_sumstats", "detect_columns", "inspect_columns",
52
+ "write_sumstats", "rescale_n_eff", "anchor_n_eff",
53
+ "n_eff_case_control", "COLUMN_ALIASES", "ColumnMappingError"]
54
+
55
+
56
+ class ColumnMappingError(ValueError):
57
+ """A column override cannot be resolved, or a layout is inconsistent."""
58
+
59
+
60
+ #: Column aliases (lower-cased) -> canonical field, in priority order. The
61
+ #: first block of every list is ldpred3's, unchanged, so any file ldpred3 maps
62
+ #: maps identically here; later entries only add coverage. One exception: for
63
+ #: ``n_eff`` an effective-N name (``_EFFECTIVE_N_NAMES``) wins over a total-N
64
+ #: alias, so ``Effective_N`` beside ``N`` gives N from ``Effective_N``.
65
+ COLUMN_ALIASES = {
66
+ "id": ["snp", "rsid", "rs", "id", "variant_id", "markername", "snpid",
67
+ "marker", "rs_id",
68
+ "rsids", "hm_rsid", "snp_id", "variant", "snpname"],
69
+ "chrom": ["chr", "chrom", "chromosome", "#chrom", "hg19chr", "chr_name",
70
+ "#chr", "hm_chrom", "hm_chr", "chr_id"],
71
+ "pos": ["bp", "pos", "position", "base_pair_location", "bp_position",
72
+ "pos_b37", "base_pair",
73
+ "chr_position", "hm_pos", "genpos"],
74
+ "ea": ["a1", "effect_allele", "ea", "allele1", "alt", "effectallele",
75
+ "tested_allele", "inc_allele",
76
+ "hm_effect_allele"],
77
+ "oa": ["a2", "other_allele", "oa", "nea", "allele0", "allele2", "ref",
78
+ "noneffect_allele", "dec_allele",
79
+ "hm_other_allele", "omitted"],
80
+ "beta": ["beta", "b", "effect", "effect_size", "effects", "log_odds",
81
+ "hm_beta", "all_inv_var_meta_beta", "inv_var_meta_beta", "logor"],
82
+ "or": ["or", "odds_ratio", "oddsratio",
83
+ "hm_odds_ratio"],
84
+ "se": ["se", "standard_error", "stderr", "standarderror", "sebeta",
85
+ "se_beta", "logor_se", "std_err",
86
+ "log(or)_se", "all_inv_var_meta_sebeta", "inv_var_meta_sebeta",
87
+ "hm_standard_error"],
88
+ "pval": ["p", "pval", "p_value", "pvalue", "p-value", "p.value",
89
+ "p_bolt_lmm", "p_value_association", "p_wald", "pval_nominal",
90
+ "all_inv_var_meta_p", "inv_var_meta_p"],
91
+ "log10p": ["log10p", "mlogp", "neg_log_10_p_value", "neglog10p",
92
+ "log10_p", "-log10p", "neg_log10_p"],
93
+ "n_eff": ["n_eff", "neff", "n", "sample_size", "totalsamplesize",
94
+ "n_samples", "n_total", "obs_ct", "n_complete_samples",
95
+ "n_effective", "effective_n", "all_inv_var_meta_effective",
96
+ "samplesize", "n_sample", "total_sample_size"],
97
+ "n_cases": ["n_cases", "ncases", "n_case", "ncase", "nca", "cases",
98
+ "num_cases", "n_cas"],
99
+ "n_controls": ["n_controls", "ncontrols", "n_control", "ncontrol", "nco",
100
+ "controls", "num_controls", "n_ctrl", "n_con"],
101
+ # MAF is deliberately absent: it does not say which allele is counted, so
102
+ # treating it as EAF can turn a common effect allele (EAF=.9, MAF=.1) into
103
+ # an apparent .8 frequency mismatch. A genuinely effect-allele column with
104
+ # a non-standard name is available through the explicit ``eaf=`` override.
105
+ "eaf": ["eaf", "freq", "frq", "effect_allele_frequency", "a1freq",
106
+ "freq1", "af", "effect_allele_freq",
107
+ "a1_freq", "af_alt", "all_meta_af", "hm_effect_allele_frequency"],
108
+ "info": ["info", "imputation_info", "imp_info", "rsq", "r2", "info_score",
109
+ "imputation_quality", "minimac_r2",
110
+ "imputationinfo"],
111
+ "test": ["test"],
112
+ }
113
+
114
+ # PLINK's additive effect and REGENIE's covariate-adjusted marginal effect.
115
+ # Other ADD-* labels include interactions and gene-level omnibus tests.
116
+ _ADDITIVE_TESTS = frozenset(("ADD", "ADD-CONDTL"))
117
+
118
+ #: N-column names that already are an effective N: preferred over a total-N
119
+ #: column and kept even when case and control counts are present.
120
+ _EFFECTIVE_N_NAMES = frozenset({"n_eff", "neff", "n_effective", "effective_n",
121
+ "all_inv_var_meta_effective"})
122
+
123
+ _SUMSTATS_FIELDS = ("id", "chrom", "pos", "ea", "oa", "beta", "se", "n_eff",
124
+ "eaf", "info")
125
+
126
+
127
+ @dataclass
128
+ class Sumstats:
129
+ """Parsed GWAS summary statistics as parallel arrays.
130
+
131
+ Field order and meaning match ldpred3's ``Sumstats``, so either package's
132
+ functions accept the other's tables.
133
+ """
134
+
135
+ id: np.ndarray
136
+ chrom: np.ndarray
137
+ pos: np.ndarray
138
+ ea: np.ndarray
139
+ oa: np.ndarray
140
+ beta: np.ndarray
141
+ se: np.ndarray
142
+ n_eff: np.ndarray
143
+ eaf: np.ndarray
144
+ info: np.ndarray
145
+ parse_log: dict = None # source-file counts, preserved by subsetting
146
+
147
+ def __len__(self):
148
+ return len(self.beta)
149
+
150
+ @property
151
+ def z(self):
152
+ """``beta / se`` (NaN where undefined)."""
153
+ with np.errstate(divide="ignore", invalid="ignore"):
154
+ return np.asarray(self.beta, float) / np.asarray(self.se, float)
155
+
156
+ def subset(self, mask):
157
+ """A new table keeping the rows selected by ``mask`` (bool or index)."""
158
+ mask = np.asarray(mask)
159
+ return Sumstats(*(getattr(self, f)[mask] for f in _SUMSTATS_FIELDS),
160
+ parse_log=(None if self.parse_log is None
161
+ else dict(self.parse_log)))
162
+
163
+ def as_variants(self):
164
+ """These rows as a :class:`~allelign.variants.Variants` table.
165
+
166
+ ``a1`` is the effect allele, so harmonising another file against it
167
+ aligns that file's effects to this file's effect alleles.
168
+ """
169
+ from .variants import Variants
170
+ return Variants(chrom=np.asarray(self.chrom, dtype=object),
171
+ id=np.asarray(self.id, dtype=object),
172
+ pos=np.asarray(self.pos, dtype=np.int64),
173
+ a1=np.asarray(self.ea, dtype=object),
174
+ a2=np.asarray(self.oa, dtype=object))
175
+
176
+ def write(self, path):
177
+ """Write the canonical table; see :func:`write_sumstats`."""
178
+ return write_sumstats(path, self)
179
+
180
+
181
+ def n_eff_case_control(n_case, n_control):
182
+ """Effective sample size of a case/control GWAS: ``4/(1/N_case + 1/N_control)``.
183
+
184
+ Equals total N for a balanced study and tends to ``4 N_case`` with many
185
+ more controls than cases. Scalars or arrays.
186
+
187
+ For a meta-analysis the correct effective N is the *sum* of the
188
+ per-cohort values; computing it from summed counts overstates it when
189
+ case fractions differ between cohorts. Prefer a published ``Neff`` column.
190
+ """
191
+ n_case = np.asarray(n_case, dtype=float)
192
+ n_control = np.asarray(n_control, dtype=float)
193
+ if np.any(n_case <= 0) or np.any(n_control <= 0):
194
+ raise ValueError("n_case and n_control must be positive")
195
+ n = 4.0 / (1.0 / n_case + 1.0 / n_control)
196
+ return float(n) if n.ndim == 0 else n
197
+
198
+
199
+ def _sniff_delimiter(header_line):
200
+ if "\t" in header_line:
201
+ return "\t"
202
+ if "," in header_line:
203
+ return ","
204
+ return None # whitespace; handled with str.split
205
+
206
+
207
+ def _read_header(fh):
208
+ """``(delimiter, header columns)`` from the first line of ``fh``."""
209
+ first = fh.readline().rstrip("\r\n")
210
+ delim = _sniff_delimiter(first)
211
+ return delim, (first.split(delim) if delim else first.split())
212
+
213
+
214
+ def _build_colmap(header, overrides):
215
+ """``(colmap, layout)``: canonical field -> column index, plus format rules.
216
+
217
+ ``layout`` records format-specific rules the header implied:
218
+ ``saige``, ``gwas_catalog_harmonised``, ``plink2_omitted``,
219
+ ``plink2_alt`` (index of ALT for per-row other-allele resolution),
220
+ ``eaf_from`` (a pattern-matched frequency column) and ``variant_string``
221
+ (index of a ``chr:pos:ref:alt`` column).
222
+ """
223
+ overrides = dict(overrides or {})
224
+ lower = [h.strip().lower() for h in header]
225
+ colmap = {}
226
+ for field, aliases in COLUMN_ALIASES.items():
227
+ if field == "n_eff":
228
+ # Effective-N names first, so an ``Effective_N`` column is not
229
+ # shadowed by a total ``N`` listed earlier in the aliases.
230
+ aliases = sorted(aliases, key=lambda a: a not in _EFFECTIVE_N_NAMES)
231
+ for a in aliases:
232
+ if a in lower:
233
+ colmap[field] = lower.index(a)
234
+ break
235
+ layout = {}
236
+ # SAIGE reports effects and frequencies for Allele2. Allele1/Allele2 alone
237
+ # are ambiguous across programs, so require a SAIGE-specific column.
238
+ if {"allele1", "allele2"}.issubset(lower) and any(
239
+ c in lower for c in ("af_allele2", "ac_allele2", "is.spa.converge")):
240
+ colmap["ea"] = lower.index("allele2")
241
+ colmap["oa"] = lower.index("allele1")
242
+ if "af_allele2" in lower:
243
+ colmap["eaf"] = lower.index("af_allele2")
244
+ if "imputationinfo" in lower:
245
+ colmap["info"] = lower.index("imputationinfo")
246
+ layout["saige"] = True
247
+ # GWAS Catalog harmonised files carry the submitted columns and the
248
+ # harmonised ``hm_`` columns side by side, possibly on different builds and
249
+ # with the effect allele swapped. Use one set consistently: the hm_ set
250
+ # when it has a harmonised effect, the submitted set otherwise.
251
+ if "hm_effect_allele" in lower and "hm_other_allele" in lower:
252
+ if "hm_beta" in lower or "hm_odds_ratio" in lower:
253
+ for field, name in (("ea", "hm_effect_allele"),
254
+ ("oa", "hm_other_allele"), ("id", "hm_rsid"),
255
+ ("chrom", "hm_chrom"), ("pos", "hm_pos"),
256
+ ("eaf", "hm_effect_allele_frequency")):
257
+ if name in lower:
258
+ colmap[field] = lower.index(name)
259
+ if "hm_beta" in lower:
260
+ colmap["beta"] = lower.index("hm_beta")
261
+ else:
262
+ colmap.pop("beta", None)
263
+ colmap["or"] = lower.index("hm_odds_ratio")
264
+ # The submitted frequency counts the submitted effect allele, which
265
+ # harmonisation may have swapped: without its hm_ counterpart it
266
+ # would describe the other allele on those rows.
267
+ if ("hm_effect_allele_frequency" not in lower
268
+ and colmap.get("eaf") is not None):
269
+ del colmap["eaf"]
270
+ layout["eaf_dropped"] = (
271
+ "hm_ columns without hm_effect_allele_frequency; the "
272
+ "submitted frequency may count the other allele")
273
+ layout["gwas_catalog_harmonised"] = True
274
+ else:
275
+ for field in ("ea", "oa", "id", "chrom", "pos", "eaf"):
276
+ if (colmap.get(field) is not None
277
+ and lower[colmap[field]].startswith("hm_")):
278
+ del colmap[field]
279
+ layout["gwas_catalog_harmonised_ignored"] = (
280
+ "no hm_beta or hm_odds_ratio; the submitted columns are used")
281
+ # PLINK2 --glm: A1 is the tested allele, which may be REF or ALT.
282
+ if (colmap.get("ea") is not None and lower[colmap["ea"]] == "a1"
283
+ and colmap.get("oa") is not None and lower[colmap["oa"]] == "ref"):
284
+ if "omitted" in lower:
285
+ colmap["oa"] = lower.index("omitted")
286
+ layout["plink2_omitted"] = True
287
+ elif "alt" in lower:
288
+ layout["plink2_alt"] = lower.index("alt")
289
+ # PGC daner files: the A1 frequency in controls, FRQ_U_<n>.
290
+ if "eaf" not in colmap:
291
+ for i, h in enumerate(lower):
292
+ if h.startswith("frq_u_"):
293
+ colmap["eaf"] = i
294
+ layout["eaf_from"] = header[i].strip()
295
+ break
296
+ # Neale UK Biobank: variant = chr:pos:ref:alt, effect on ALT.
297
+ if "ea" not in colmap and "oa" not in colmap and "variant" in lower:
298
+ layout["variant_string"] = lower.index("variant")
299
+
300
+ # Explicit overrides win (a column name or a zero-based index).
301
+ for field, col in overrides.items():
302
+ if field not in COLUMN_ALIASES:
303
+ raise ColumnMappingError(
304
+ f"unknown sumstats field {field!r}; choose one of "
305
+ + ", ".join(COLUMN_ALIASES))
306
+ # A bare digit string (``--column beta=3``) is a zero-based index.
307
+ if isinstance(col, str) and col.strip().isdigit():
308
+ col = int(col.strip())
309
+ if isinstance(col, (int, np.integer)) and not isinstance(col, bool):
310
+ i = int(col)
311
+ if not 0 <= i < len(header):
312
+ raise ColumnMappingError(
313
+ f"sumstats column override {field}={i} is out of range for "
314
+ f"a {len(header)}-column file (indices are zero-based)")
315
+ colmap[field] = i
316
+ else:
317
+ name = str(col).strip().lower()
318
+ if name not in lower:
319
+ raise ColumnMappingError(
320
+ f"sumstats column override {field}={col!r} was not found; "
321
+ "available columns: " + ", ".join(header))
322
+ colmap[field] = lower.index(name)
323
+ if "or" in overrides and "beta" not in overrides:
324
+ colmap.pop("beta", None) # explicit OR semantics override a BETA alias
325
+ if "oa" in overrides or "ea" in overrides:
326
+ layout.pop("plink2_alt", None)
327
+ layout.pop("variant_string", None)
328
+ if colmap.get("ea") is not None and colmap.get("ea") == colmap.get("oa"):
329
+ raise ColumnMappingError(
330
+ "ea and oa resolve to the same column; specify both allele columns")
331
+ return colmap, layout
332
+
333
+
334
+ def _n_arguments(n_eff, n_cases, n_controls, overrides):
335
+ """Separate scalar N from column overrides, shared by read and inspect."""
336
+ for name, value in (("n_cases", n_cases), ("n_controls", n_controls)):
337
+ if isinstance(value, str):
338
+ overrides[name] = value
339
+ if isinstance(n_eff, str):
340
+ overrides["n_eff"] = n_eff
341
+ n_eff = None
342
+ scalar_counts = [v for v in (n_cases, n_controls)
343
+ if v is not None and not isinstance(v, str)]
344
+ if scalar_counts:
345
+ if len(scalar_counts) != 2:
346
+ raise ValueError("n_cases and n_controls must be given together")
347
+ if n_eff is not None:
348
+ raise ValueError("pass either n_eff or n_cases/n_controls, not both")
349
+ n_eff = n_eff_case_control(_n_eff_scalar_value(n_cases),
350
+ _n_eff_scalar_value(n_controls))
351
+ return _n_eff_scalar_value(n_eff)
352
+
353
+
354
+ def _n_plan(colmap, lower, overrides, forced_n):
355
+ """Where per-variant N comes from: ``(source, detail)``."""
356
+ if forced_n is not None:
357
+ return "scalar", None
358
+ if "n_eff" in overrides:
359
+ return "column", lower[colmap["n_eff"]]
360
+ if "n_cases" in overrides or "n_controls" in overrides:
361
+ if "n_cases" not in colmap or "n_controls" not in colmap:
362
+ raise ColumnMappingError("explicit case/control columns require both counts")
363
+ return "case_control", None
364
+ if "n_eff" in colmap and lower[colmap["n_eff"]] in _EFFECTIVE_N_NAMES:
365
+ return "column", lower[colmap["n_eff"]]
366
+ if "n_cases" in colmap and "n_controls" in colmap:
367
+ return "case_control", None
368
+ if "n_eff" in colmap:
369
+ return "column", lower[colmap["n_eff"]]
370
+ return None, None
371
+
372
+
373
+ def detect_columns(path, **col_overrides):
374
+ """Peek at the header and report the detected column mapping.
375
+
376
+ Returns ``(header, mapping)`` with ``mapping = {field: column name}`` for
377
+ every resolved field. Reads only the first line.
378
+ """
379
+ with open_text(path) as fh:
380
+ _delim, header = _read_header(fh)
381
+ colmap, _layout = _build_colmap(header, col_overrides)
382
+ return header, {field: header[i] for field, i in colmap.items()}
383
+
384
+
385
+ def inspect_columns(path, *, n_eff=None, n_cases=None, n_controls=None,
386
+ **col_overrides):
387
+ """Header-only preflight: mapping, recognised layout and N source.
388
+
389
+ Returns a dict with ``header``, ``mapping``, ``layout``, ``n_source`` and
390
+ ``missing`` (required fields that could not be resolved, including N).
391
+ Sample-size arguments follow :func:`read_sumstats`: numbers supply a
392
+ constant N; strings name columns or their zero-based indices.
393
+ """
394
+ forced_n = _n_arguments(n_eff, n_cases, n_controls, col_overrides)
395
+ with open_text(path) as fh:
396
+ delim, header = _read_header(fh)
397
+ colmap, layout = _build_colmap(header, col_overrides)
398
+ lower = [h.strip().lower() for h in header]
399
+ source, detail = _n_plan(colmap, lower, col_overrides, forced_n)
400
+ missing = []
401
+ if "variant_string" not in layout:
402
+ missing += [f for f in ("ea", "oa") if f not in colmap]
403
+ if "beta" not in colmap and "or" not in colmap:
404
+ missing.append("beta/or")
405
+ if "se" not in colmap and "pval" not in colmap and "log10p" not in colmap:
406
+ missing.append("se/pval")
407
+ if source is None:
408
+ missing.append("n_eff")
409
+ return {"header": header, "delimiter": delim or "whitespace",
410
+ "mapping": {field: header[i] for field, i in colmap.items()},
411
+ "layout": {k: (header[v] if isinstance(v, int) and k != "saige"
412
+ and not isinstance(v, bool) else v)
413
+ for k, v in layout.items()},
414
+ "n_source": source if detail is None else f"{source}:{detail}",
415
+ "missing": missing}
416
+
417
+
418
+ def _to_float(s):
419
+ try:
420
+ return float(s)
421
+ except (ValueError, TypeError):
422
+ return math.nan
423
+
424
+
425
+ def _z_from_log_tail(log_tail):
426
+ """``z`` with ``log Phi(-z) = log_tail``, for tails beyond float range.
427
+
428
+ Newton solve using Mills' expansion; used once ``z > 38``, where the
429
+ neglected term is below 1e-12 in log probability.
430
+ """
431
+ z = math.sqrt(-2 * log_tail)
432
+ for _ in range(5):
433
+ t = 1 / (z * z)
434
+ correction = math.log1p(-t + 3*t*t - 15*t**3 + 105*t**4)
435
+ log_sf = -.5*z*z - math.log(z) - .5*math.log(2*math.pi) + correction
436
+ z += (log_sf - log_tail) / (z + 1/z)
437
+ return z
438
+
439
+
440
+ def _two_sided_z(token):
441
+ """Two-sided normal z from a p-value token, without underflow."""
442
+ from statistics import NormalDist
443
+ p = _to_float(token)
444
+ if 0 < p < 1 and p / 2 > 0:
445
+ return -NormalDist().inv_cdf(p / 2)
446
+ if p != 0 and not (0 < p < 1):
447
+ return math.nan
448
+ from decimal import Decimal, InvalidOperation
449
+ try:
450
+ value = Decimal(str(token))
451
+ if not value.is_finite() or not 0 < value < 1:
452
+ return math.nan
453
+ log_tail = float(value.ln()) - math.log(2.)
454
+ except (InvalidOperation, ValueError, OverflowError):
455
+ return math.nan
456
+ return _z_from_log_tail(log_tail)
457
+
458
+
459
+ def _z_from_log10p(token):
460
+ """Two-sided normal z from a ``-log10 p`` token (REGENIE, FinnGen)."""
461
+ from statistics import NormalDist
462
+ x = _to_float(token)
463
+ if not math.isfinite(x) or x <= 0:
464
+ return math.nan
465
+ if x < 300:
466
+ return -NormalDist().inv_cdf(10.0 ** (-x) / 2)
467
+ return _z_from_log_tail(-x * math.log(10.) - math.log(2.))
468
+
469
+
470
+ def read_sumstats(path, *, n_eff=None, n_cases=None, n_controls=None,
471
+ require_n=True, **col_overrides):
472
+ """Read a GWAS summary-statistics file into a :class:`Sumstats`.
473
+
474
+ Parameters
475
+ ----------
476
+ path : str or path-like
477
+ Delimited text (tab, comma or whitespace, auto-detected), optionally
478
+ gzip/bgzip compressed. Fields are split on that delimiter with no
479
+ CSV quoting: a comma inside a field shifts the later columns, so use
480
+ tabs when a field can contain a comma.
481
+ n_eff : float or str, optional
482
+ A number **overrides** any per-variant N and flattens every row to that
483
+ constant (the low-level escape hatch; :func:`anchor_n_eff` instead keeps
484
+ a column's pattern and rescales its median). A string names the
485
+ per-variant N column (or its zero-based index).
486
+ n_cases, n_controls : float or str, optional
487
+ Numbers give a constant effective N ``4/(1/n_cases + 1/n_controls)``.
488
+ Strings name per-variant case and control count columns.
489
+ require_n : bool, default True
490
+ Raise when no sample size can be found. With ``False`` the ``n_eff``
491
+ column is NaN instead.
492
+ **col_overrides
493
+ Force a field to a column by name or zero-based index, e.g.
494
+ ``read_sumstats(path, ea="A1", beta="effect")``.
495
+
496
+ When a ``TEST`` column is present, only ``ADD`` and ``ADD-CONDTL`` rows
497
+ are retained. Covariate, interaction, non-additive and unknown tests are
498
+ dropped with a warning; ``parse_log`` records their count.
499
+ """
500
+ forced_n = _n_arguments(n_eff, n_cases, n_controls, col_overrides)
501
+ path = os.fspath(path)
502
+ with open_text(path) as fh:
503
+ delim, header = _read_header(fh)
504
+ colmap, layout = _build_colmap(header, col_overrides)
505
+ lower = [h.strip().lower() for h in header]
506
+
507
+ if "variant_string" not in layout:
508
+ for req in ("ea", "oa"):
509
+ if req not in colmap:
510
+ raise ValueError(
511
+ f"could not find a '{req}' allele column in {path}; "
512
+ f"header was {header}. Pass it explicitly, e.g. {req}=...")
513
+ if "beta" not in colmap and "or" not in colmap:
514
+ raise ValueError(
515
+ f"no effect column (beta/OR) found in {path}; header {header}")
516
+ source, detail = _n_plan(colmap, lower, col_overrides, forced_n)
517
+ if source is None and require_n:
518
+ raise ValueError(
519
+ f"no sample-size column found in {path}; pass n_eff=... or "
520
+ "n_cases=/n_controls=")
521
+ plan = {"source": source, "forced": forced_n,
522
+ "i_n": colmap.get("n_eff") if source == "column" else None,
523
+ "i_nca": colmap.get("n_cases") if source == "case_control" else None,
524
+ "i_nco": colmap.get("n_controls") if source == "case_control" else None}
525
+
526
+ def finish(ss):
527
+ log = ss.parse_log
528
+ log["columns"] = {field: header[i] for field, i in colmap.items()}
529
+ if "test" in colmap:
530
+ log["accepted_test_values"] = sorted(_ADDITIVE_TESTS)
531
+ if log["n_dropped_test"]:
532
+ warnings.warn(
533
+ f"ignored {log['n_dropped_test']:,} of "
534
+ f"{log['n_input']:,} rows whose TEST is not ADD or "
535
+ "ADD-CONDTL; only additive association effects are "
536
+ "read", RuntimeWarning, stacklevel=3)
537
+ log["n_eff_source"] = (source if detail is None
538
+ else f"{source}:{detail}") or "missing"
539
+ if layout:
540
+ log["layout"] = {k: (header[v] if isinstance(v, int)
541
+ and not isinstance(v, bool) else v)
542
+ for k, v in layout.items()}
543
+ if source == "case_control":
544
+ warnings.warn(
545
+ "effective N computed per variant from case and control "
546
+ "counts as 4/(1/N_case + 1/N_control). For a meta-analysis "
547
+ "this overstates N_eff when case fractions differ between "
548
+ "cohorts; prefer a published effective-N column.",
549
+ RuntimeWarning, stacklevel=3)
550
+ # A diagnostic, never an inferred transformation: positive effects
551
+ # can be legitimate, and header semantics remain decisive.
552
+ if "beta" in colmap:
553
+ finite = ss.beta[np.isfinite(ss.beta)]
554
+ if (len(finite) >= 20 and np.mean(finite > 0) >= .95
555
+ and .5 < np.median(finite) < 2):
556
+ log["suspected_odds_ratio_in_beta"] = True
557
+ warnings.warn(
558
+ "BETA values are almost all positive and centered near "
559
+ "one; check whether this column contains odds ratios. "
560
+ "Map the OR column explicitly if so; no transformation "
561
+ "has been inferred.", RuntimeWarning, stacklevel=3)
562
+ return ss
563
+
564
+ special = "plink2_alt" in layout or "variant_string" in layout
565
+ kernels = None if special else _jit_kernels()
566
+ # The replay below reopens ``path``, which only a regular file can
567
+ # serve, so pipes keep the single-pass loop.
568
+ if kernels is None or not os.path.isfile(path):
569
+ return finish(_parse_rows_python(fh, delim, colmap, layout, plan))
570
+ try:
571
+ return finish(_parse_rows_jit(fh, delim, colmap, plan, kernels))
572
+ except Exception:
573
+ # Anything the compiled path cannot reproduce exactly replays the
574
+ # whole body through the reference loop, which then returns -- or
575
+ # raises -- what it always does.
576
+ pass
577
+ with open_text(path) as fh:
578
+ fh.readline()
579
+ return finish(_parse_rows_python(fh, delim, colmap, layout, plan))
580
+
581
+
582
+ def _case_control_n(nca, nco):
583
+ """Per-row ``4/(1/nca + 1/nco)``; NaN unless both counts are positive."""
584
+ nca = np.asarray(nca, dtype=float)
585
+ nco = np.asarray(nco, dtype=float)
586
+ ok = np.isfinite(nca) & np.isfinite(nco) & (nca > 0) & (nco > 0)
587
+ out = np.full(nca.shape, math.nan)
588
+ out[ok] = 4.0 / (1.0 / nca[ok] + 1.0 / nco[ok])
589
+ return out
590
+
591
+
592
+ # ---------------------------------------------------------------------------
593
+ # Optional Numba tokenizer (from ldpred3).
594
+ #
595
+ # Without Numba the reference row loop runs. With Numba, ASCII blocks of the
596
+ # file are tokenized and their numeric cells parsed in compiled code, about 3x
597
+ # faster, and anything the kernels cannot reproduce bit for bit goes back to
598
+ # the exact Python rules, cell by cell or block by block.
599
+ #
600
+ # Numeric cells use Clinger's fast path: a strict ``[+-]digits[.digits]
601
+ # [(e|E)[+-]digits]`` token with at most 15 significant digits and a net
602
+ # decimal exponent within +-22 is one correctly rounded IEEE multiply or
603
+ # divide -- the same double CPython's ``float()`` returns. Every other token is
604
+ # flagged and parsed by ``_to_float``.
605
+ # ---------------------------------------------------------------------------
606
+
607
+ _BLOCK_CHARS = 1 << 22
608
+ _JIT_MIN_CHARS = 1 << 21
609
+ _INT64_LIMIT = float(2 ** 63)
610
+ _POW10 = np.array([float(10 ** k) for k in range(23)]) # all exact doubles
611
+
612
+
613
+ def _tokenize(b, delim, starts, ends):
614
+ """Column-major field offsets of a rectangular block of lines.
615
+
616
+ ``delim < 0`` means runs of ASCII whitespace (``str.split()``), else that
617
+ single byte (``str.split(delim)``). Empty lines are skipped. Returns the
618
+ number of lines filled, or -1 as soon as a line has another field count.
619
+ """
620
+ k = starts.shape[0]
621
+ r = 0
622
+ j = 0
623
+ fs = 0
624
+ ls = 0
625
+ intok = False
626
+ for i in range(len(b)):
627
+ c = b[i]
628
+ if delim >= 0:
629
+ if c == delim or (c == 10 and i > ls):
630
+ if j >= k:
631
+ return -1
632
+ starts[j, r] = fs
633
+ ends[j, r] = i
634
+ j += 1
635
+ if c == delim or c == 10:
636
+ fs = i + 1
637
+ elif c == 32 or 9 <= c <= 13 or 28 <= c <= 31:
638
+ if intok:
639
+ ends[j, r] = i
640
+ j += 1
641
+ intok = False
642
+ elif not intok:
643
+ if j >= k:
644
+ return -1
645
+ starts[j, r] = i
646
+ intok = True
647
+ if c == 10:
648
+ if i > ls:
649
+ if j != k:
650
+ return -1
651
+ r += 1
652
+ j = 0
653
+ ls = i + 1
654
+ return r
655
+
656
+
657
+ def _parse_decimal(b, starts, ends, pow10, out, ok):
658
+ """Clinger fast-path parse of each token; ``ok[k]`` False = not handled."""
659
+ for k in range(len(starts)):
660
+ i = starts[k]
661
+ e = ends[k]
662
+ ok[k] = False
663
+ neg = False
664
+ if i < e and (b[i] == 43 or b[i] == 45):
665
+ neg = b[i] == 45
666
+ i += 1
667
+ mant = 0
668
+ nd = 0
669
+ ndig = 0
670
+ frac = 0
671
+ bad = False
672
+ while i < e and 48 <= b[i] <= 57:
673
+ d = b[i] - 48
674
+ ndig += 1
675
+ if mant != 0 or d != 0:
676
+ nd += 1
677
+ mant = mant * 10 + d
678
+ i += 1
679
+ if i < e and b[i] == 46:
680
+ i += 1
681
+ while i < e and 48 <= b[i] <= 57:
682
+ d = b[i] - 48
683
+ ndig += 1
684
+ frac += 1
685
+ if mant != 0 or d != 0:
686
+ nd += 1
687
+ mant = mant * 10 + d
688
+ i += 1
689
+ if ndig == 0 or nd > 15:
690
+ bad = True
691
+ ex = 0
692
+ if not bad and i < e and (b[i] == 101 or b[i] == 69):
693
+ i += 1
694
+ eneg = False
695
+ if i < e and (b[i] == 43 or b[i] == 45):
696
+ eneg = b[i] == 45
697
+ i += 1
698
+ nexp = 0
699
+ while i < e and 48 <= b[i] <= 57:
700
+ if nexp < 6:
701
+ ex = ex * 10 + (b[i] - 48)
702
+ nexp += 1
703
+ i += 1
704
+ if nexp == 0 or nexp > 5:
705
+ bad = True
706
+ if eneg:
707
+ ex = -ex
708
+ if bad or i != e:
709
+ continue
710
+ e10 = ex - frac
711
+ if mant == 0:
712
+ v = 0.0
713
+ elif 0 <= e10 <= 22:
714
+ v = float(mant) * pow10[e10]
715
+ elif -22 <= e10 < 0:
716
+ v = float(mant) / pow10[-e10]
717
+ else:
718
+ continue
719
+ out[k] = -v if neg else v
720
+ ok[k] = True
721
+
722
+
723
+ def _pool_codes(b, starts, ends, codes, first):
724
+ """Code each token of <= 7 bytes by its distinct value, first-seen order.
725
+
726
+ ``codes[k]`` indexes the distinct tokens; a longer token gets -1. Returns
727
+ the number of distinct tokens.
728
+ """
729
+ n = len(starts)
730
+ size = 64
731
+ while size < 2 * n:
732
+ size *= 2
733
+ table_key = np.full(size, -1, dtype=np.int64)
734
+ table_code = np.empty(size, dtype=np.int64)
735
+ ndistinct = 0
736
+ for k in range(n):
737
+ s = starts[k]
738
+ ln = ends[k] - s
739
+ if ln > 7:
740
+ codes[k] = -1
741
+ continue
742
+ key = np.int64(ln)
743
+ for j in range(ln):
744
+ key |= np.int64(b[s + j]) << (8 * (j + 1))
745
+ h = (key * np.int64(-7046029254386353131)) >> 40
746
+ slot = h & (size - 1)
747
+ while table_key[slot] != -1 and table_key[slot] != key:
748
+ slot = (slot + 1) & (size - 1)
749
+ if table_key[slot] == -1:
750
+ table_key[slot] = key
751
+ table_code[slot] = ndistinct
752
+ first[ndistinct] = k
753
+ ndistinct += 1
754
+ codes[k] = table_code[slot]
755
+ return ndistinct
756
+
757
+
758
+ _KERNELS = None
759
+
760
+
761
+ def _jit_kernels():
762
+ """Compiled kernels, or ``None`` when Numba (or its JIT) is unavailable."""
763
+ global _KERNELS
764
+ if _KERNELS is None:
765
+ try:
766
+ from ._numba import jit_available, jit_nogil
767
+ if not jit_available():
768
+ _KERNELS = False
769
+ else:
770
+ _KERNELS = (jit_nogil(_tokenize), jit_nogil(_parse_decimal),
771
+ jit_nogil(_pool_codes))
772
+ except Exception: # a broken Numba install: no JIT
773
+ _KERNELS = False
774
+ return _KERNELS or None
775
+
776
+
777
+ def _empty_log(n, n_se, n_dropped_test=None):
778
+ log = {"n_input": int(n), "n_se_from_pval": int(n_se)}
779
+ if n_dropped_test is not None:
780
+ log.update(n_dropped_test=int(n_dropped_test),
781
+ n_kept=int(n - n_dropped_test))
782
+ return log
783
+
784
+
785
+ def _parse_rows_jit(fh, delim, colmap, plan, kernels):
786
+ """Block-wise equivalent of :func:`_parse_rows_python` using ``kernels``.
787
+
788
+ A block goes through the compiled path only when it is ASCII and
789
+ rectangular (every non-empty line has the same field count, at least the
790
+ mapped columns); any other block is parsed by the reference loop itself,
791
+ sharing the string pool.
792
+ """
793
+ tokenize, parse_decimal, pool_codes = kernels
794
+ from statistics import NormalDist
795
+ _inv_cdf = NormalDist().inv_cdf
796
+ dcode = -1 if delim is None else ord(delim)
797
+ i_id = colmap.get("id")
798
+ i_chrom = colmap.get("chrom")
799
+ i_pos = colmap.get("pos")
800
+ i_beta = colmap.get("beta")
801
+ i_se = colmap.get("se")
802
+ i_pval = colmap.get("pval")
803
+ i_log10p = colmap.get("log10p")
804
+ i_eaf = colmap.get("eaf")
805
+ i_info = colmap.get("info")
806
+ i_test = colmap.get("test")
807
+ nfields = max(colmap.values()) + 1
808
+ pool_dict = {}
809
+ pool = pool_dict.setdefault
810
+ nan = math.nan
811
+
812
+ def reference(text):
813
+ return _parse_rows_python(text.split("\n"), delim, colmap, {}, plan,
814
+ _pool=pool_dict)
815
+
816
+ def block(text):
817
+ if (len(text) < _JIT_MIN_CHARS or not text.endswith("\n")
818
+ or not text.isascii()):
819
+ return reference(text)
820
+ b = np.frombuffer(text.encode("ascii"), dtype=np.uint8)
821
+ first_line = text[:text.index("\n")]
822
+ k0 = len(first_line.split(delim) if delim else first_line.split())
823
+ if not first_line or k0 < nfields:
824
+ return reference(text)
825
+ nlines = text.count("\n")
826
+ starts = np.empty((k0, nlines), dtype=np.int64)
827
+ ends = np.empty((k0, nlines), dtype=np.int64)
828
+ m = tokenize(b, dcode, starts, ends)
829
+ if m < 0:
830
+ return reference(text)
831
+ n_input = m
832
+ if i_test is not None:
833
+ keep = np.fromiter(
834
+ (text[a:z].strip().upper() in _ADDITIVE_TESTS
835
+ for a, z in zip(starts[i_test, :m].tolist(),
836
+ ends[i_test, :m].tolist())),
837
+ dtype=bool, count=m)
838
+ if not keep.all():
839
+ starts, ends = starts[:, :m][:, keep], ends[:, :m][:, keep]
840
+ m = int(keep.sum())
841
+
842
+ def floats(i):
843
+ s = starts[i, :m]
844
+ e = ends[i, :m]
845
+ out = np.empty(m)
846
+ ok = np.empty(m, dtype=np.bool_)
847
+ parse_decimal(b, s, e, _POW10, out, ok)
848
+ bad = np.flatnonzero(~ok)
849
+ if len(bad):
850
+ memo = {}
851
+ for k, a, z in zip(bad.tolist(), s[bad].tolist(),
852
+ e[bad].tolist()):
853
+ tok = text[a:z]
854
+ v = memo.get(tok)
855
+ if v is None:
856
+ v = memo[tok] = _to_float(tok)
857
+ out[k] = v
858
+ return out
859
+
860
+ def tokens(i, rows):
861
+ s = starts[i, rows]
862
+ e = ends[i, rows]
863
+ return [text[a:z] for a, z in zip(s.tolist(), e.tolist())]
864
+
865
+ def strings(i, transform):
866
+ s = starts[i, :m]
867
+ e = ends[i, :m]
868
+ if transform is None:
869
+ return np.array(list(map(text.__getitem__,
870
+ map(slice, s.tolist(), e.tolist()))),
871
+ dtype=object)
872
+ codes = np.empty(m, dtype=np.int64)
873
+ first = np.empty(m, dtype=np.int64)
874
+ u = pool_codes(b, s, e, codes, first)
875
+ vals = np.empty(u + 1, dtype=object) # [-1] -> None
876
+ vals[:u] = [transform(text[a:z]) for a, z in
877
+ zip(s[first[:u]].tolist(), e[first[:u]].tolist())]
878
+ out = vals[codes]
879
+ for k in np.flatnonzero(codes < 0).tolist():
880
+ out[k] = transform(text[s[k]:e[k]])
881
+ return out
882
+
883
+ if i_beta is not None:
884
+ beta = floats(i_beta)
885
+ else:
886
+ orv = floats(colmap["or"])
887
+ beta = np.full(m, nan)
888
+ pos_or = orv > 0
889
+ # math.log, not np.log: NumPy's SIMD log may differ from libm in
890
+ # the last ulp, and the reference uses math.log.
891
+ beta[pos_or] = np.fromiter(map(math.log, orv[pos_or].tolist()),
892
+ dtype=float, count=int(pos_or.sum()))
893
+ se = floats(i_se) if i_se is not None else np.full(m, nan)
894
+ n_se_from_p = 0
895
+ need = np.isnan(se) & ~np.isnan(beta)
896
+ if need.any() and i_pval is not None:
897
+ p = floats(i_pval)
898
+ if np.any(p[need] < np.finfo(float).tiny):
899
+ return reference(text)
900
+ need &= (p > 0) & (p < 1) & (beta != 0)
901
+ for k in np.flatnonzero(need).tolist():
902
+ half = float(p[k]) / 2.0
903
+ if half > 0.0:
904
+ z = -_inv_cdf(half)
905
+ if math.isfinite(z) and z > 0:
906
+ se[k] = abs(float(beta[k])) / z
907
+ n_se_from_p += 1
908
+ elif need.any() and i_log10p is not None:
909
+ rows = np.flatnonzero(need & (beta != 0))
910
+ for k, tok in zip(rows.tolist(), tokens(i_log10p, rows)):
911
+ z = _z_from_log10p(tok)
912
+ if math.isfinite(z) and z > 0:
913
+ se[k] = abs(float(beta[k])) / z
914
+ n_se_from_p += 1
915
+ if i_pos is not None:
916
+ posf = floats(i_pos)
917
+ fin = np.isfinite(posf)
918
+ if ((posf[fin] >= _INT64_LIMIT)
919
+ | (posf[fin] < -_INT64_LIMIT)).any():
920
+ raise OverflowError("position outside int64")
921
+ pos = np.where(fin, posf, 0.0).astype(np.int64)
922
+ else:
923
+ pos = np.zeros(m, dtype=np.int64)
924
+
925
+ if plan["forced"] is not None:
926
+ n_eff = np.full(m, plan["forced"])
927
+ elif plan["i_n"] is not None:
928
+ n_eff = floats(plan["i_n"])
929
+ elif plan["i_nca"] is not None:
930
+ n_eff = _case_control_n(floats(plan["i_nca"]),
931
+ floats(plan["i_nco"]))
932
+ else:
933
+ n_eff = np.full(m, nan)
934
+
935
+ def pooled(t):
936
+ return pool(t, t)
937
+
938
+ def allele(t):
939
+ t = t.upper()
940
+ return pool(t, t)
941
+
942
+ return Sumstats(
943
+ id=(strings(i_id, None) if i_id is not None
944
+ else np.full(m, "", dtype=object)),
945
+ chrom=(strings(i_chrom, pooled) if i_chrom is not None
946
+ else np.full(m, pooled(""), dtype=object)),
947
+ pos=pos,
948
+ ea=strings(colmap["ea"], allele),
949
+ oa=strings(colmap["oa"], allele),
950
+ beta=beta, se=se, n_eff=n_eff,
951
+ eaf=floats(i_eaf) if i_eaf is not None else np.full(m, nan),
952
+ info=floats(i_info) if i_info is not None else np.full(m, nan),
953
+ parse_log=_empty_log(n_input, n_se_from_p,
954
+ n_input - m if i_test is not None else None),
955
+ )
956
+
957
+ pieces = []
958
+ carry = ""
959
+ while True:
960
+ text = fh.read(_BLOCK_CHARS)
961
+ if not text:
962
+ if carry:
963
+ pieces.append(block(carry))
964
+ break
965
+ text = carry + text
966
+ cut = text.rfind("\n") + 1
967
+ carry = text[cut:]
968
+ if cut:
969
+ pieces.append(block(text[:cut]))
970
+ if not pieces:
971
+ return reference("")
972
+ if len(pieces) == 1:
973
+ return pieces[0]
974
+ return Sumstats(*(np.concatenate([getattr(p, f) for p in pieces])
975
+ for f in _SUMSTATS_FIELDS),
976
+ parse_log=_empty_log(
977
+ sum(p.parse_log["n_input"] for p in pieces),
978
+ sum(p.parse_log["n_se_from_pval"] for p in pieces),
979
+ sum(p.parse_log["n_dropped_test"] for p in pieces)
980
+ if i_test is not None else None))
981
+
982
+
983
+ def _parse_rows_python(fh, delim, colmap, layout, plan, _pool=None):
984
+ """Reference row-at-a-time parser for the body lines of ``fh``.
985
+
986
+ The semantic definition of a sumstats row, and the fallback for
987
+ :func:`_parse_rows_jit`. Chromosome labels and alleles are pooled through
988
+ one dict: they come from tiny alphabets, and a fresh ``str`` per cell
989
+ retained 1.88 GB of duplicates at 10 million rows in ldpred3's measurement.
990
+ """
991
+ if _pool is None:
992
+ _pool = {}
993
+ ids, chroms, poss, eas, oas = [], [], [], [], []
994
+ betas, ses, ns, ncas, ncos, eafs, infos = [], [], [], [], [], [], []
995
+ n_se_from_p = 0
996
+ n_input = n_dropped_test = 0
997
+
998
+ i_id = colmap.get("id")
999
+ i_chrom = colmap.get("chrom")
1000
+ i_pos = colmap.get("pos")
1001
+ i_ea = colmap.get("ea")
1002
+ i_oa = colmap.get("oa")
1003
+ i_beta = colmap.get("beta")
1004
+ i_or = colmap.get("or")
1005
+ i_se = colmap.get("se")
1006
+ i_pval = colmap.get("pval")
1007
+ i_log10p = colmap.get("log10p")
1008
+ i_eaf = colmap.get("eaf")
1009
+ i_info = colmap.get("info")
1010
+ i_test = colmap.get("test")
1011
+ i_n, i_nca, i_nco = plan["i_n"], plan["i_nca"], plan["i_nco"]
1012
+ forced_n = plan["forced"]
1013
+ i_alt = layout.get("plink2_alt")
1014
+ i_var = layout.get("variant_string")
1015
+ used = [i for i in (i_id, i_chrom, i_pos, i_ea, i_oa, i_beta, i_or, i_se,
1016
+ i_pval, i_log10p, i_n, i_nca, i_nco, i_eaf, i_info,
1017
+ i_alt, i_var, i_test) if i is not None]
1018
+ nfields = max(used) + 1
1019
+
1020
+ id_add, chrom_add, pos_add = ids.append, chroms.append, poss.append
1021
+ ea_add, oa_add = eas.append, oas.append
1022
+ beta_add, se_add, n_add = betas.append, ses.append, ns.append
1023
+ eaf_add, info_add = eafs.append, infos.append
1024
+ pool = _pool.setdefault
1025
+ isnan, isfinite, nan = math.isnan, math.isfinite, math.nan
1026
+
1027
+ for raw in fh:
1028
+ raw = raw.rstrip("\r\n")
1029
+ if not raw:
1030
+ continue
1031
+ n_input += 1
1032
+ f = raw.split(delim) if delim else raw.split()
1033
+ if len(f) < nfields: # pad once, not per field
1034
+ f = f + [""] * (nfields - len(f))
1035
+ if i_test is not None and f[i_test].strip().upper() not in _ADDITIVE_TESTS:
1036
+ n_dropped_test += 1
1037
+ continue
1038
+
1039
+ if i_beta is not None:
1040
+ beta = _to_float(f[i_beta])
1041
+ else:
1042
+ orv = _to_float(f[i_or])
1043
+ beta = math.log(orv) if orv > 0 else nan
1044
+
1045
+ se = _to_float(f[i_se]) if i_se is not None else nan
1046
+ if isnan(se) and not isnan(beta) and beta != 0:
1047
+ if i_pval is not None:
1048
+ z = _two_sided_z(f[i_pval])
1049
+ elif i_log10p is not None:
1050
+ z = _z_from_log10p(f[i_log10p])
1051
+ else:
1052
+ z = nan
1053
+ if isfinite(z) and z > 0:
1054
+ se = abs(beta) / z
1055
+ n_se_from_p += 1
1056
+
1057
+ if forced_n is not None:
1058
+ n = forced_n
1059
+ elif i_n is not None:
1060
+ n = _to_float(f[i_n])
1061
+ else:
1062
+ n = nan
1063
+ if i_nca is not None:
1064
+ ncas.append(_to_float(f[i_nca]))
1065
+ ncos.append(_to_float(f[i_nco]))
1066
+
1067
+ if i_var is not None:
1068
+ parts = f[i_var].split(":")
1069
+ if len(parts) >= 4:
1070
+ c, p_tok, ref, alt = parts[0], parts[1], parts[2], parts[3]
1071
+ else:
1072
+ c, p_tok, ref, alt = "", "", "", ""
1073
+ if i_chrom is None:
1074
+ chrom_add(pool(c, c))
1075
+ if i_pos is None:
1076
+ pos_v = _to_float(p_tok)
1077
+ pos_add(int(pos_v) if isfinite(pos_v) else 0)
1078
+ ea = alt.upper()
1079
+ oa = ref.upper()
1080
+ else:
1081
+ ea = f[i_ea].upper()
1082
+ oa = f[i_oa].upper()
1083
+ if i_alt is not None and ea == oa:
1084
+ oa = f[i_alt].upper()
1085
+ if i_var is None or i_chrom is not None:
1086
+ c = str(f[i_chrom]) if i_chrom is not None else ""
1087
+ chrom_add(pool(c, c))
1088
+ if i_var is None or i_pos is not None:
1089
+ # A blank / non-numeric position is missing (0), not an error.
1090
+ pos_v = _to_float(f[i_pos]) if i_pos is not None else 0.0
1091
+ pos_add(int(pos_v) if isfinite(pos_v) else 0)
1092
+ id_add(f[i_id] if i_id is not None else "")
1093
+ ea_add(pool(ea, ea))
1094
+ oa_add(pool(oa, oa))
1095
+ beta_add(beta); se_add(se); n_add(n)
1096
+ eaf_add(_to_float(f[i_eaf]) if i_eaf is not None else nan)
1097
+ info_add(_to_float(f[i_info]) if i_info is not None else nan)
1098
+
1099
+ n_eff = np.array(ns, dtype=float)
1100
+ if i_nca is not None and forced_n is None:
1101
+ n_eff = _case_control_n(ncas, ncos)
1102
+ return Sumstats(
1103
+ id=np.array(ids, dtype=object),
1104
+ chrom=np.array(chroms, dtype=object),
1105
+ pos=np.array(poss, dtype=np.int64),
1106
+ ea=np.array(eas, dtype=object),
1107
+ oa=np.array(oas, dtype=object),
1108
+ beta=np.array(betas, dtype=float),
1109
+ se=np.array(ses, dtype=float),
1110
+ n_eff=n_eff,
1111
+ eaf=np.array(eafs, dtype=float),
1112
+ info=np.array(infos, dtype=float),
1113
+ parse_log=_empty_log(n_input, n_se_from_p,
1114
+ n_dropped_test if i_test is not None else None),
1115
+ )
1116
+
1117
+
1118
+ def _n_eff_scalar_value(n_eff):
1119
+ """Float value of a scalar ``n_eff``, or ``None`` (absent or a column name)."""
1120
+ if n_eff is None or isinstance(n_eff, str):
1121
+ return None
1122
+ try:
1123
+ value = float(n_eff)
1124
+ except (TypeError, ValueError, OverflowError):
1125
+ raise ValueError("n_eff must be a finite positive number") from None
1126
+ if (isinstance(n_eff, (bool, np.bool_)) or not math.isfinite(value)
1127
+ or value <= 0):
1128
+ raise ValueError("n_eff must be a finite positive number")
1129
+ return value
1130
+
1131
+
1132
+ def rescale_n_eff(ss, target):
1133
+ """Anchor a per-variant N column's median at ``target``, downward only.
1134
+
1135
+ ``ss.n_eff`` is rescaled **in place** by ``min(1, target / median)`` over
1136
+ its usable (finite, positive) values: the column's relative pattern is
1137
+ kept, values are never scaled up (a per-variant N below the target is a
1138
+ measurement, not a deficit) and unusable rows stay NaN. Returns the
1139
+ transform record, or ``None`` when no value is usable.
1140
+ """
1141
+ value = _n_eff_scalar_value(target)
1142
+ if value is None:
1143
+ raise ValueError("n_eff must be a finite positive number")
1144
+ n_col = np.asarray(ss.n_eff, dtype=float)
1145
+ usable = np.isfinite(n_col) & (n_col > 0)
1146
+ n_usable = int(usable.sum())
1147
+ if n_usable == 0:
1148
+ return None
1149
+ median = float(np.median(n_col[usable]))
1150
+ factor = min(1.0, value / median)
1151
+ applied = bool(factor < 1.0 - 1e-12)
1152
+ if applied:
1153
+ n_col[usable] = n_col[usable] * factor
1154
+ ss.n_eff = n_col
1155
+ return {
1156
+ "method": "downward median rescale",
1157
+ "source_median": median,
1158
+ "target_effective_n": value,
1159
+ "factor": factor,
1160
+ "applied": applied,
1161
+ "upward_scaling_refused": bool(value > median and factor == 1.0),
1162
+ "n_usable": n_usable,
1163
+ }
1164
+
1165
+
1166
+ def anchor_n_eff(ss, target):
1167
+ """Apply a supplied scalar N: rescale the column, or set a constant.
1168
+
1169
+ With at least one usable per-variant value this is :func:`rescale_n_eff`;
1170
+ with none, ``target`` becomes every row's N. Mutates ``ss`` and returns
1171
+ the transform record.
1172
+ """
1173
+ record = rescale_n_eff(ss, target)
1174
+ if record is not None:
1175
+ return record
1176
+ target = float(target)
1177
+ ss.n_eff = np.full(len(ss), target, dtype=float)
1178
+ return {"method": "constant scalar (no usable per-variant N)",
1179
+ "source_median": None, "target_effective_n": target,
1180
+ "factor": 1.0, "applied": True,
1181
+ "upward_scaling_refused": False, "n_usable": 0}
1182
+
1183
+
1184
+ _WRITE_HEADER = ("ID", "CHR", "POS", "EA", "OA", "BETA", "SE", "N_EFF",
1185
+ "EAF", "INFO")
1186
+
1187
+
1188
+ def _fmt(value):
1189
+ # repr round-trips float64 exactly; NaN is written as NA.
1190
+ return "NA" if value != value else repr(float(value))
1191
+
1192
+
1193
+ def write_sumstats(path, ss):
1194
+ """Write ``ss`` as the canonical tab-separated table; returns ``path``.
1195
+
1196
+ Columns ``ID CHR POS EA OA BETA SE N_EFF EAF INFO``, floats written so they
1197
+ read back bit for bit, NaN as ``NA``; a ``.gz`` path is compressed. The
1198
+ file reads back with :func:`read_sumstats` without overrides.
1199
+ """
1200
+ with atomic_text(path) as fh:
1201
+ fh.write("\t".join(_WRITE_HEADER) + "\n")
1202
+ for row in zip(ss.id, ss.chrom, ss.pos, ss.ea, ss.oa, ss.beta, ss.se,
1203
+ ss.n_eff, ss.eaf, ss.info):
1204
+ vid, c, p, ea, oa, b, s, n, f, q = row
1205
+ fh.write(f"{vid or '.'}\t{c}\t{int(p)}\t{ea}\t{oa}\t{_fmt(b)}\t"
1206
+ f"{_fmt(s)}\t{_fmt(n)}\t{_fmt(f)}\t{_fmt(q)}\n")
1207
+ return path