allelign 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- allelign/__init__.py +69 -0
- allelign/__main__.py +5 -0
- allelign/_common.py +95 -0
- allelign/_numba.py +22 -0
- allelign/align.py +1252 -0
- allelign/alleles.py +90 -0
- allelign/build.py +57 -0
- allelign/catalog.py +88 -0
- allelign/cli.py +323 -0
- allelign/download.py +525 -0
- allelign/genotypes.py +241 -0
- allelign/ldcheck.py +771 -0
- allelign/qc.py +638 -0
- allelign/status.py +130 -0
- allelign/sumstats.py +1207 -0
- allelign/variants.py +155 -0
- allelign/warning_markers.py +15 -0
- allelign/weights.py +614 -0
- allelign-0.1.0.dist-info/METADATA +175 -0
- allelign-0.1.0.dist-info/RECORD +24 -0
- allelign-0.1.0.dist-info/WHEEL +5 -0
- allelign-0.1.0.dist-info/entry_points.txt +2 -0
- allelign-0.1.0.dist-info/licenses/LICENSE +21 -0
- allelign-0.1.0.dist-info/top_level.txt +1 -0
allelign/weights.py
ADDED
|
@@ -0,0 +1,614 @@
|
|
|
1
|
+
"""PGS effect sizes: ldpred3 weight tables and PGS Catalog scoring files.
|
|
2
|
+
|
|
3
|
+
Two on-disk conventions, kept apart because mixing them is a classic error:
|
|
4
|
+
|
|
5
|
+
* :class:`WeightsTable` -- ldpred3's ``ID CHR POS A1 A2 WEIGHT`` table, where
|
|
6
|
+
``WEIGHT`` is a *standardised* effect (per genotype SD), optionally with
|
|
7
|
+
``AF_REF SD_REF SD_SOURCE`` so the same standardisation can be reapplied to
|
|
8
|
+
another cohort ("frozen" scaling).
|
|
9
|
+
* :class:`ScoringFile` -- a `PGS Catalog
|
|
10
|
+
<https://www.pgscatalog.org/downloads/#dl_ftp_scoring>`_ scoring file, where
|
|
11
|
+
``effect_weight`` is per **allele count** and a ``#key=value`` header
|
|
12
|
+
carries the metadata.
|
|
13
|
+
|
|
14
|
+
:func:`write_pgs_scoring_file` converts the first to the second
|
|
15
|
+
(``effect_weight = WEIGHT / SD_REF``). Both align to a reference with
|
|
16
|
+
:func:`allelign.harmonize_weights`. Ported from ldpred3 (tables, export) and
|
|
17
|
+
multipgs (Catalog reading).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import math
|
|
23
|
+
import os
|
|
24
|
+
import warnings
|
|
25
|
+
from array import array
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
import numpy as np
|
|
30
|
+
|
|
31
|
+
from ._common import atomic_text, open_text
|
|
32
|
+
from .alleles import normalize_chrom
|
|
33
|
+
|
|
34
|
+
__all__ = ["WeightsTable", "write_weights", "read_weights", "ScoringFile",
|
|
35
|
+
"read_scoring_file", "write_pgs_scoring_file", "SD_SOURCES",
|
|
36
|
+
"PGS_BUILDS", "SCORING_COLUMNS"]
|
|
37
|
+
|
|
38
|
+
#: Values of ``SD_SOURCE``: observed in the fit cohort, or the HWE value
|
|
39
|
+
#: ``sqrt(2f(1-f))`` used by producers that never see a fit cohort.
|
|
40
|
+
SD_SOURCES = ("empirical", "hwe")
|
|
41
|
+
#: Builds accepted in a scoring file's ``#genome_build=`` header.
|
|
42
|
+
PGS_BUILDS = ("GRCh37", "GRCh38")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _validate_columns(*, id, chrom, pos, effect_allele, other_allele, weight,
|
|
46
|
+
af=None, sd=None):
|
|
47
|
+
"""Validate and normalise the parallel arrays of a weights table."""
|
|
48
|
+
named = {"id": id, "chrom": chrom, "pos": pos,
|
|
49
|
+
"effect_allele": effect_allele, "other_allele": other_allele,
|
|
50
|
+
"weight": weight}
|
|
51
|
+
arrays = {}
|
|
52
|
+
for name, values in named.items():
|
|
53
|
+
arr = np.asarray(values)
|
|
54
|
+
if arr.ndim != 1:
|
|
55
|
+
raise ValueError(f"{name} must be one-dimensional")
|
|
56
|
+
arrays[name] = arr
|
|
57
|
+
lengths = {name: arr.size for name, arr in arrays.items()}
|
|
58
|
+
if len(set(lengths.values())) != 1:
|
|
59
|
+
detail = ", ".join(f"{name}={size}" for name, size in lengths.items())
|
|
60
|
+
raise ValueError(f"weight columns must have equal lengths ({detail})")
|
|
61
|
+
n = arrays["id"].size
|
|
62
|
+
if n == 0:
|
|
63
|
+
raise ValueError("weights must contain at least one variant")
|
|
64
|
+
for name in ("id", "chrom", "effect_allele", "other_allele"):
|
|
65
|
+
values = arrays[name]
|
|
66
|
+
if values.dtype == object and any(value is None for value in values):
|
|
67
|
+
raise ValueError(f"{name} must not contain missing or empty values")
|
|
68
|
+
stripped = np.char.strip(values.astype(str))
|
|
69
|
+
if np.any(stripped == ""):
|
|
70
|
+
raise ValueError(f"{name} must not contain missing or empty values")
|
|
71
|
+
arrays[name] = np.asarray(stripped, dtype=object)
|
|
72
|
+
try:
|
|
73
|
+
pos_float = np.asarray(arrays["pos"], dtype=float)
|
|
74
|
+
except (TypeError, ValueError) as exc:
|
|
75
|
+
raise ValueError("pos must contain finite non-negative integers") from exc
|
|
76
|
+
if (not np.all(np.isfinite(pos_float)) or np.any(pos_float < 0)
|
|
77
|
+
or np.any(pos_float != np.floor(pos_float))):
|
|
78
|
+
raise ValueError("pos must contain finite non-negative integers")
|
|
79
|
+
arrays["pos"] = pos_float.astype(np.int64)
|
|
80
|
+
try:
|
|
81
|
+
arrays["weight"] = np.asarray(arrays["weight"], dtype=float)
|
|
82
|
+
except (TypeError, ValueError) as exc:
|
|
83
|
+
raise ValueError("weight must contain finite numeric values") from exc
|
|
84
|
+
if not np.all(np.isfinite(arrays["weight"])):
|
|
85
|
+
raise ValueError("weight must contain finite numeric values")
|
|
86
|
+
if (af is None) != (sd is None):
|
|
87
|
+
raise ValueError("af and sd must be provided together")
|
|
88
|
+
if af is not None:
|
|
89
|
+
for name, values in (("af", af), ("sd", sd)):
|
|
90
|
+
arr = np.asarray(values)
|
|
91
|
+
if arr.ndim != 1 or arr.size != n:
|
|
92
|
+
raise ValueError(f"{name} must be one-dimensional with length {n}")
|
|
93
|
+
try:
|
|
94
|
+
arr = np.asarray(arr, dtype=float)
|
|
95
|
+
except (TypeError, ValueError) as exc:
|
|
96
|
+
raise ValueError(f"{name} must contain finite numeric values") from exc
|
|
97
|
+
if not np.all(np.isfinite(arr)):
|
|
98
|
+
raise ValueError(f"{name} must contain finite numeric values")
|
|
99
|
+
arrays[name] = arr
|
|
100
|
+
if np.any((arrays["af"] < 0) | (arrays["af"] > 1)):
|
|
101
|
+
raise ValueError("af must be in [0, 1]")
|
|
102
|
+
if np.any(arrays["sd"] < 0):
|
|
103
|
+
raise ValueError("sd must be >= 0")
|
|
104
|
+
else:
|
|
105
|
+
arrays["af"] = arrays["sd"] = None
|
|
106
|
+
return arrays
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@dataclass
|
|
110
|
+
class WeightsTable:
|
|
111
|
+
"""An ldpred3-format weights table (parallel arrays).
|
|
112
|
+
|
|
113
|
+
``weight`` counts ``a1`` on the standardised scale. ``sd_source`` says how
|
|
114
|
+
``sd_ref`` was obtained; ``None`` means the file predates the column.
|
|
115
|
+
"""
|
|
116
|
+
|
|
117
|
+
id: np.ndarray
|
|
118
|
+
chrom: np.ndarray
|
|
119
|
+
pos: np.ndarray
|
|
120
|
+
a1: np.ndarray
|
|
121
|
+
a2: np.ndarray
|
|
122
|
+
weight: np.ndarray
|
|
123
|
+
af_ref: np.ndarray = None
|
|
124
|
+
sd_ref: np.ndarray = None
|
|
125
|
+
sd_source: str = None
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def has_scale(self):
|
|
129
|
+
"""True when ``AF_REF`` and ``SD_REF`` are present (frozen scaling)."""
|
|
130
|
+
return self.af_ref is not None and self.sd_ref is not None
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def sd_is_hwe(self):
|
|
134
|
+
return self.sd_source == "hwe"
|
|
135
|
+
|
|
136
|
+
def __len__(self):
|
|
137
|
+
return len(self.id)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def write_weights(path, *, id, chrom, pos, effect_allele, other_allele, weight,
|
|
141
|
+
af=None, sd=None, sd_source="empirical"):
|
|
142
|
+
"""Write an ldpred3 weights table; returns ``path``.
|
|
143
|
+
|
|
144
|
+
``af``/``sd`` (fit-cohort A1 frequency and dosage SD) are written as
|
|
145
|
+
``AF_REF``/``SD_REF`` with ``SD_SOURCE`` when both are given. Floats use
|
|
146
|
+
``repr``, so :func:`read_weights` reads them back bit for bit. Files
|
|
147
|
+
written at 8 significant digits still parse.
|
|
148
|
+
"""
|
|
149
|
+
if sd_source not in SD_SOURCES:
|
|
150
|
+
raise ValueError(
|
|
151
|
+
f"sd_source must be one of {SD_SOURCES}, got {sd_source!r}")
|
|
152
|
+
cols = _validate_columns(
|
|
153
|
+
id=id, chrom=chrom, pos=pos, effect_allele=effect_allele,
|
|
154
|
+
other_allele=other_allele, weight=weight, af=af, sd=sd)
|
|
155
|
+
have_scale = cols["af"] is not None
|
|
156
|
+
with atomic_text(path) as fh:
|
|
157
|
+
fh.write("ID\tCHR\tPOS\tA1\tA2\tWEIGHT"
|
|
158
|
+
+ ("\tAF_REF\tSD_REF\tSD_SOURCE\n" if have_scale else "\n"))
|
|
159
|
+
out_cols = [cols["id"], cols["chrom"], cols["pos"],
|
|
160
|
+
cols["effect_allele"], cols["other_allele"], cols["weight"]]
|
|
161
|
+
if have_scale:
|
|
162
|
+
out_cols += [cols["af"], cols["sd"]]
|
|
163
|
+
for row in zip(*out_cols):
|
|
164
|
+
if have_scale:
|
|
165
|
+
vid, c, p, a1, a2, w, af_, sd_ = row
|
|
166
|
+
fh.write(f"{vid}\t{c}\t{p}\t{a1}\t{a2}\t{repr(float(w))}"
|
|
167
|
+
f"\t{repr(float(af_))}\t{repr(float(sd_))}\t{sd_source}\n")
|
|
168
|
+
else:
|
|
169
|
+
vid, c, p, a1, a2, w = row
|
|
170
|
+
fh.write(f"{vid}\t{c}\t{p}\t{a1}\t{a2}\t{repr(float(w))}\n")
|
|
171
|
+
return path
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def read_weights(path, *, sd_source=None, warn_missing_sd_source=True):
|
|
175
|
+
"""Parse an ldpred3 weights table (alleles upper-cased).
|
|
176
|
+
|
|
177
|
+
A file with ``AF_REF``/``SD_REF`` but no ``SD_SOURCE`` warns, because
|
|
178
|
+
bipred files written before 2026-08 carry the HWE approximation there;
|
|
179
|
+
pass ``sd_source="hwe"`` to declare such a file. A value contradicting the
|
|
180
|
+
file's own column is an error.
|
|
181
|
+
"""
|
|
182
|
+
ids, chrom, pos, a1, a2, w, af_ref, sd_ref = [], [], [], [], [], [], [], []
|
|
183
|
+
sd_sources = set()
|
|
184
|
+
with open_text(path) as fh:
|
|
185
|
+
header = fh.readline().rstrip("\n")
|
|
186
|
+
if not header:
|
|
187
|
+
raise ValueError("weights file is empty or has no header")
|
|
188
|
+
cols = header.split("\t") if "\t" in header else header.split()
|
|
189
|
+
if len(cols) != len(set(cols)):
|
|
190
|
+
raise ValueError("weights header contains duplicate columns")
|
|
191
|
+
required = ("ID", "CHR", "POS", "A1", "A2", "WEIGHT")
|
|
192
|
+
missing = [name for name in required if name not in cols]
|
|
193
|
+
if missing:
|
|
194
|
+
raise ValueError("weights header is missing required columns: "
|
|
195
|
+
+ ", ".join(missing))
|
|
196
|
+
if ("AF_REF" in cols) != ("SD_REF" in cols):
|
|
197
|
+
raise ValueError("weights header must contain both AF_REF and SD_REF")
|
|
198
|
+
has_scale = "AF_REF" in cols
|
|
199
|
+
index = {name: cols.index(name) for name in required}
|
|
200
|
+
if has_scale:
|
|
201
|
+
index.update(AF_REF=cols.index("AF_REF"), SD_REF=cols.index("SD_REF"))
|
|
202
|
+
has_sd_source = has_scale and "SD_SOURCE" in cols
|
|
203
|
+
if has_sd_source:
|
|
204
|
+
index["SD_SOURCE"] = cols.index("SD_SOURCE")
|
|
205
|
+
needed = max(index.values())
|
|
206
|
+
for line_no, line in enumerate(fh, start=2):
|
|
207
|
+
line = line.rstrip("\n")
|
|
208
|
+
if not line:
|
|
209
|
+
continue
|
|
210
|
+
f = line.split("\t") if "\t" in line else line.split()
|
|
211
|
+
if len(f) <= needed:
|
|
212
|
+
raise ValueError(f"weights row {line_no} has too few fields")
|
|
213
|
+
try:
|
|
214
|
+
ids.append(f[index["ID"]]); chrom.append(str(f[index["CHR"]]))
|
|
215
|
+
pos.append(float(f[index["POS"]]))
|
|
216
|
+
a1.append(f[index["A1"]].upper())
|
|
217
|
+
a2.append(f[index["A2"]].upper())
|
|
218
|
+
w.append(float(f[index["WEIGHT"]]))
|
|
219
|
+
if has_scale:
|
|
220
|
+
af_ref.append(float(f[index["AF_REF"]]))
|
|
221
|
+
sd_ref.append(float(f[index["SD_REF"]]))
|
|
222
|
+
if has_sd_source:
|
|
223
|
+
sd_sources.add(f[index["SD_SOURCE"]].strip().lower())
|
|
224
|
+
except (TypeError, ValueError) as exc:
|
|
225
|
+
raise ValueError(f"invalid value in weights row {line_no}: {exc}") \
|
|
226
|
+
from exc
|
|
227
|
+
if len(sd_sources) > 1:
|
|
228
|
+
raise ValueError(
|
|
229
|
+
"weights file mixes SD_SOURCE values: " + ", ".join(sorted(sd_sources)))
|
|
230
|
+
if sd_sources and not sd_sources <= set(SD_SOURCES):
|
|
231
|
+
raise ValueError(
|
|
232
|
+
f"unknown SD_SOURCE {sd_sources.pop()!r}; expected one of {SD_SOURCES}")
|
|
233
|
+
declared = sd_sources.pop() if sd_sources else None
|
|
234
|
+
if sd_source is not None:
|
|
235
|
+
if sd_source not in SD_SOURCES:
|
|
236
|
+
raise ValueError(
|
|
237
|
+
f"sd_source must be one of {SD_SOURCES}, got {sd_source!r}")
|
|
238
|
+
if declared is not None and declared != sd_source:
|
|
239
|
+
raise ValueError(
|
|
240
|
+
f"sd_source={sd_source!r} contradicts the file's SD_SOURCE="
|
|
241
|
+
f"{declared!r}")
|
|
242
|
+
declared = sd_source
|
|
243
|
+
elif declared is None and has_scale and warn_missing_sd_source:
|
|
244
|
+
warnings.warn(
|
|
245
|
+
"weights file has AF_REF/SD_REF but no SD_SOURCE column; assuming "
|
|
246
|
+
"SD_REF is an observed fit-cohort SD. Files written by bipred "
|
|
247
|
+
"before 2026-08 carry the HWE approximation sqrt(2f(1-f)) instead; "
|
|
248
|
+
"pass sd_source='hwe' when reading those.", UserWarning, stacklevel=2)
|
|
249
|
+
normalized = _validate_columns(
|
|
250
|
+
id=ids, chrom=chrom, pos=pos, effect_allele=a1, other_allele=a2,
|
|
251
|
+
weight=w, af=af_ref if has_scale else None,
|
|
252
|
+
sd=sd_ref if has_scale else None)
|
|
253
|
+
return WeightsTable(
|
|
254
|
+
id=normalized["id"], chrom=normalized["chrom"], pos=normalized["pos"],
|
|
255
|
+
a1=normalized["effect_allele"], a2=normalized["other_allele"],
|
|
256
|
+
weight=normalized["weight"], af_ref=normalized["af"],
|
|
257
|
+
sd_ref=normalized["sd"], sd_source=declared)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# ---------------------------------------------------------------------------
|
|
261
|
+
# PGS Catalog scoring files
|
|
262
|
+
# ---------------------------------------------------------------------------
|
|
263
|
+
|
|
264
|
+
#: Column name -> canonical field. Harmonised names win over the originals.
|
|
265
|
+
SCORING_COLUMNS = {
|
|
266
|
+
"id": ("hm_rsid", "rsid", "rsids", "snpid", "variant_id"),
|
|
267
|
+
"chrom": ("hm_chr", "chr_name", "chromosome", "chr"),
|
|
268
|
+
"pos": ("hm_pos", "chr_position", "position", "bp"),
|
|
269
|
+
"ea": ("effect_allele", "a1"),
|
|
270
|
+
"oa": ("other_allele", "hm_inferotherallele", "a2"),
|
|
271
|
+
"weight": ("effect_weight", "weight", "beta"),
|
|
272
|
+
}
|
|
273
|
+
# Compatibility for early consumers; new code uses SCORING_COLUMNS.
|
|
274
|
+
_SCORING_COLUMNS = SCORING_COLUMNS
|
|
275
|
+
_NON_ADDITIVE = ("is_dominant", "is_recessive", "is_haplotype", "is_diplotype",
|
|
276
|
+
"is_interaction")
|
|
277
|
+
_RATIO_WEIGHTS = ("or", "hr", "odds_ratio", "hazard_ratio")
|
|
278
|
+
_TRUE = ("true", "t", "yes", "y", "1")
|
|
279
|
+
_MISSING_ALLELE = ("", ".", "NA", "N/A", "NULL")
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
@dataclass
|
|
283
|
+
class ScoringFile:
|
|
284
|
+
"""One parsed PGS Catalog scoring file.
|
|
285
|
+
|
|
286
|
+
``weight`` counts ``ea`` per allele copy. ``meta`` holds the
|
|
287
|
+
``#key=value`` header (keys lower-cased); ``log`` records what parsing
|
|
288
|
+
dropped or transformed.
|
|
289
|
+
"""
|
|
290
|
+
|
|
291
|
+
id: np.ndarray
|
|
292
|
+
chrom: np.ndarray
|
|
293
|
+
pos: np.ndarray
|
|
294
|
+
ea: np.ndarray
|
|
295
|
+
oa: np.ndarray
|
|
296
|
+
weight: np.ndarray
|
|
297
|
+
meta: dict = field(default_factory=dict)
|
|
298
|
+
log: dict = field(default_factory=dict)
|
|
299
|
+
path: str = ""
|
|
300
|
+
|
|
301
|
+
def __len__(self):
|
|
302
|
+
return int(self.weight.size)
|
|
303
|
+
|
|
304
|
+
@property
|
|
305
|
+
def pgs_id(self):
|
|
306
|
+
"""``pgs_id`` from the header, falling back to the file name."""
|
|
307
|
+
return self.meta.get("pgs_id") or scoring_file_id(self.path)
|
|
308
|
+
|
|
309
|
+
@property
|
|
310
|
+
def trait(self):
|
|
311
|
+
return self.meta.get("trait_reported", "")
|
|
312
|
+
|
|
313
|
+
@property
|
|
314
|
+
def genome_build(self):
|
|
315
|
+
"""Build of the coordinates used; empty when their provenance is mixed."""
|
|
316
|
+
return self.log.get("build_used", self.meta.get("genome_build", ""))
|
|
317
|
+
|
|
318
|
+
def __str__(self):
|
|
319
|
+
return (f"{self.pgs_id}: {len(self)} variants"
|
|
320
|
+
+ (f", {self.trait}" if self.trait else "")
|
|
321
|
+
+ (f", build {self.genome_build}" if self.genome_build else ""))
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def scoring_file_id(path):
|
|
325
|
+
"""``PGS000123`` from ``.../PGS000123_hmPOS_GRCh37.txt.gz``."""
|
|
326
|
+
base = os.path.basename(str(path))
|
|
327
|
+
for suffix in (".txt.gz", ".tsv.gz", ".txt", ".tsv", ".gz"):
|
|
328
|
+
if base.endswith(suffix):
|
|
329
|
+
base = base[: -len(suffix)]
|
|
330
|
+
break
|
|
331
|
+
return base.split("_")[0] if base.startswith("PGS") else base
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _pick(header, names):
|
|
335
|
+
lower = {name.strip().lower(): i for i, name in enumerate(header)}
|
|
336
|
+
for want in names:
|
|
337
|
+
if want in lower:
|
|
338
|
+
return lower[want], want
|
|
339
|
+
return None, None
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _scoring_rows(fh, meta):
|
|
343
|
+
"""Stream rows; ``#key=value`` lines anywhere go to ``meta``."""
|
|
344
|
+
delimiter, first = None, True
|
|
345
|
+
for line in fh:
|
|
346
|
+
if line.startswith("#"):
|
|
347
|
+
stripped = line.lstrip("#").strip()
|
|
348
|
+
if "=" in stripped:
|
|
349
|
+
key, _, value = stripped.partition("=")
|
|
350
|
+
meta[key.strip().lower()] = value.strip()
|
|
351
|
+
continue
|
|
352
|
+
line = line.rstrip("\n").rstrip("\r")
|
|
353
|
+
if not line:
|
|
354
|
+
continue
|
|
355
|
+
if first:
|
|
356
|
+
delimiter, first = ("\t" if "\t" in line else None), False
|
|
357
|
+
yield line.split(delimiter)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def read_scoring_file(path, *, prefer_harmonized=True, drop_non_additive=True,
|
|
361
|
+
keep=None):
|
|
362
|
+
"""Parse a PGS Catalog scoring file (plain or gzipped).
|
|
363
|
+
|
|
364
|
+
* ``hm_rsID``/``hm_chr``/``hm_pos`` are used when present; coordinate
|
|
365
|
+
build comes from ``HmPOS_build``, independently of the identifier.
|
|
366
|
+
Mixed submitted/harmonised coordinates have an unknown build.
|
|
367
|
+
Where ``other_allele`` is missing,
|
|
368
|
+
``hm_inferOtherAllele`` fills it row by row.
|
|
369
|
+
* ``weight_type`` OR/HR (odds or hazard ratios) are log-transformed; a
|
|
370
|
+
non-positive ratio is an error.
|
|
371
|
+
* Dominant, recessive, haplotype, diplotype and interaction rows cannot be
|
|
372
|
+
represented by an additive dosage score; they are dropped and counted
|
|
373
|
+
(``drop_non_additive=False`` keeps them, which is wrong for every one).
|
|
374
|
+
|
|
375
|
+
``keep(id, position) -> bool`` prefilters rows a reference cannot match,
|
|
376
|
+
so a genome-wide file aligned to a small panel never stores their strings.
|
|
377
|
+
Rejected rows still count in the whole-file checks: the ratio test and
|
|
378
|
+
``log["weight_square_sum_all"]``, the denominator of the matched weight
|
|
379
|
+
mass. ``log["n_reference_prefiltered"]`` counts them.
|
|
380
|
+
"""
|
|
381
|
+
meta = {}
|
|
382
|
+
with open_text(path) as fh:
|
|
383
|
+
rows = _scoring_rows(fh, meta)
|
|
384
|
+
header = next(rows, None)
|
|
385
|
+
if header is None:
|
|
386
|
+
raise ValueError(f"{path}: no column header found (only comment lines)")
|
|
387
|
+
idx, used = {}, {}
|
|
388
|
+
for field_name, names in SCORING_COLUMNS.items():
|
|
389
|
+
candidates = names if prefer_harmonized else \
|
|
390
|
+
tuple(n for n in names if not n.startswith("hm_")) + names
|
|
391
|
+
i, chosen = _pick(header, candidates)
|
|
392
|
+
idx[field_name] = i
|
|
393
|
+
used[field_name] = chosen
|
|
394
|
+
inferred_oa_idx, inferred_oa_name = _pick(header, ("hm_inferotherallele",))
|
|
395
|
+
if idx["weight"] is None:
|
|
396
|
+
raise ValueError(f"{path}: no effect_weight column in {header}")
|
|
397
|
+
if idx["ea"] is None:
|
|
398
|
+
raise ValueError(f"{path}: no effect_allele column in {header}")
|
|
399
|
+
if idx["id"] is None and (idx["chrom"] is None or idx["pos"] is None):
|
|
400
|
+
raise ValueError(f"{path}: needs either an rsID column or both "
|
|
401
|
+
f"chromosome and position, found {header}")
|
|
402
|
+
flag_cols = [i for i, name in enumerate(header)
|
|
403
|
+
if name.strip().lower() in _NON_ADDITIVE]
|
|
404
|
+
n_read = n_flagged = n_bad_weight = n_inferred_oa = n_bad_pos = 0
|
|
405
|
+
ids, chroms, eas, oas = [], [], [], []
|
|
406
|
+
poss, ws, retained = array("q"), array("d"), array("q")
|
|
407
|
+
width = len(header)
|
|
408
|
+
for fields in rows:
|
|
409
|
+
n_read += 1
|
|
410
|
+
if len(fields) < width:
|
|
411
|
+
fields = fields + [""] * (width - len(fields))
|
|
412
|
+
if flag_cols and any(fields[i].strip().lower() in _TRUE
|
|
413
|
+
for i in flag_cols):
|
|
414
|
+
n_flagged += 1
|
|
415
|
+
if drop_non_additive:
|
|
416
|
+
continue
|
|
417
|
+
try:
|
|
418
|
+
w = float(fields[idx["weight"]])
|
|
419
|
+
except ValueError:
|
|
420
|
+
n_bad_weight += 1
|
|
421
|
+
continue
|
|
422
|
+
if not math.isfinite(w):
|
|
423
|
+
n_bad_weight += 1
|
|
424
|
+
continue
|
|
425
|
+
sid = fields[idx["id"]].strip() if idx["id"] is not None else ""
|
|
426
|
+
raw_pos = (fields[idx["pos"]].strip()
|
|
427
|
+
if idx["pos"] is not None else "0")
|
|
428
|
+
if raw_pos.lstrip("-").isdigit():
|
|
429
|
+
position = int(raw_pos)
|
|
430
|
+
else:
|
|
431
|
+
position = 0
|
|
432
|
+
n_bad_pos += 1
|
|
433
|
+
oa = (fields[idx["oa"]].strip().upper()
|
|
434
|
+
if idx["oa"] is not None else "")
|
|
435
|
+
if (oa in _MISSING_ALLELE and inferred_oa_idx is not None
|
|
436
|
+
and inferred_oa_idx != idx["oa"]):
|
|
437
|
+
inferred = fields[inferred_oa_idx].strip().upper()
|
|
438
|
+
if inferred not in _MISSING_ALLELE:
|
|
439
|
+
oa = inferred
|
|
440
|
+
n_inferred_oa += 1
|
|
441
|
+
ws.append(w)
|
|
442
|
+
# Numeric weights are kept for every valid row, so the ratio test
|
|
443
|
+
# and the weight-mass total read the whole file; only the costly
|
|
444
|
+
# strings of rejected rows are skipped.
|
|
445
|
+
if keep is not None:
|
|
446
|
+
if not keep(sid, position):
|
|
447
|
+
continue
|
|
448
|
+
retained.append(len(ws) - 1)
|
|
449
|
+
ids.append(sid)
|
|
450
|
+
chroms.append(fields[idx["chrom"]].strip()
|
|
451
|
+
if idx["chrom"] is not None else "")
|
|
452
|
+
poss.append(position)
|
|
453
|
+
eas.append(fields[idx["ea"]].strip().upper())
|
|
454
|
+
oas.append(oa)
|
|
455
|
+
|
|
456
|
+
pos = np.frombuffer(poss, dtype=np.int64)
|
|
457
|
+
weight = np.frombuffer(ws, dtype=np.float64).copy()
|
|
458
|
+
coordinate_used = [str(used[name]).startswith("hm_")
|
|
459
|
+
for name in ("chrom", "pos") if used[name]]
|
|
460
|
+
harmonized_columns = bool(coordinate_used) and all(coordinate_used)
|
|
461
|
+
log = {"n_rows": n_read, "n_kept": int(weight.size),
|
|
462
|
+
"n_non_additive": n_flagged, "n_unparsable_weight": n_bad_weight,
|
|
463
|
+
"n_inferred_other_allele": n_inferred_oa,
|
|
464
|
+
"n_unparsable_position": n_bad_pos,
|
|
465
|
+
"columns_used": {k: v for k, v in used.items() if v},
|
|
466
|
+
"harmonized_columns": harmonized_columns}
|
|
467
|
+
if any(coordinate_used) and not harmonized_columns:
|
|
468
|
+
log["mixed_coordinate_warning"] = (
|
|
469
|
+
"chromosome and position columns mix submitted and harmonized "
|
|
470
|
+
"coordinates; their genome build is unknown")
|
|
471
|
+
log["build_used"] = ""
|
|
472
|
+
if harmonized_columns:
|
|
473
|
+
log["build_used"] = (meta.get("hmpos_build")
|
|
474
|
+
or meta.get("harmonized_build")
|
|
475
|
+
or "")
|
|
476
|
+
if n_inferred_oa:
|
|
477
|
+
log["columns_used"]["oa_fallback"] = inferred_oa_name
|
|
478
|
+
multi = sum(1 for a in oas if "/" in a)
|
|
479
|
+
if multi:
|
|
480
|
+
log["n_multi_valued_other_allele"] = multi
|
|
481
|
+
weight_type = meta.get("weight_type", "").strip().lower()
|
|
482
|
+
if weight_type in _RATIO_WEIGHTS:
|
|
483
|
+
if np.any(weight <= 0):
|
|
484
|
+
raise ValueError(
|
|
485
|
+
f"{path}: weight_type={weight_type!r} says these are ratios, "
|
|
486
|
+
f"but {int(np.sum(weight <= 0))} are non-positive, so they "
|
|
487
|
+
"cannot be. Fix the header or the file.")
|
|
488
|
+
weight = np.log(weight)
|
|
489
|
+
log["log_transformed"] = True
|
|
490
|
+
if keep is not None:
|
|
491
|
+
log["n_reference_prefiltered"] = int(weight.size - len(retained))
|
|
492
|
+
log["weight_square_sum_all"] = float(weight @ weight)
|
|
493
|
+
weight = weight[np.frombuffer(retained, dtype=np.int64)]
|
|
494
|
+
log["n_kept"] = int(weight.size)
|
|
495
|
+
return ScoringFile(
|
|
496
|
+
id=np.array(ids, dtype=object), chrom=np.array(chroms, dtype=object),
|
|
497
|
+
pos=pos, ea=np.array(eas, dtype=object),
|
|
498
|
+
oa=np.array(oas, dtype=object), weight=weight, meta=meta, log=log,
|
|
499
|
+
path=str(path))
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
#: An SD at or below this is inert in frozen scoring, so it exports as 0.
|
|
503
|
+
_PGS_SD_EPS = 1e-12
|
|
504
|
+
_PGS_CHROM_ALIASES = {"PAR": "X"}
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _pgs_chrom(label):
|
|
508
|
+
"""A chromosome label in the Catalog's ``chr_name`` style (1-22, X, Y, MT)."""
|
|
509
|
+
s = normalize_chrom(label)
|
|
510
|
+
return _PGS_CHROM_ALIASES.get(s, s)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _pgs_default_id(path):
|
|
514
|
+
base = Path(path).name
|
|
515
|
+
for suffix in (".txt.gz", ".tsv.gz", ".txt", ".tsv", ".gz"):
|
|
516
|
+
if base.endswith(suffix):
|
|
517
|
+
base = base[: -len(suffix)]
|
|
518
|
+
break
|
|
519
|
+
return "_".join(base.split()) or "ALLELIGN"
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _pgs_header_value(name, value):
|
|
523
|
+
text = str(value).strip()
|
|
524
|
+
if not text:
|
|
525
|
+
raise ValueError(f"{name} must not be empty")
|
|
526
|
+
if any(ch in text for ch in "\t\n\r#="):
|
|
527
|
+
raise ValueError(f"{name} must not contain tabs, newlines, '#' or '=' "
|
|
528
|
+
f"(got {value!r}); it is embedded in a #key=value "
|
|
529
|
+
"header line")
|
|
530
|
+
return text
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def write_pgs_scoring_file(path, wt, *, genome_build, pgs_id=None,
|
|
534
|
+
trait_reported=None, source=None):
|
|
535
|
+
"""Write an ldpred3 weights table as a PGS Catalog scoring file.
|
|
536
|
+
|
|
537
|
+
``effect_weight = WEIGHT / SD_REF`` on the raw allele-count scale, so a
|
|
538
|
+
plain dosage score ``sum(effect_weight * g)`` equals the frozen score
|
|
539
|
+
``sum((WEIGHT/SD_REF) (g - 2 AF_REF))`` up to the constant
|
|
540
|
+
``sum(effect_weight * 2 AF_REF)``. Rows with ``SD_REF <= 1e-12`` export as
|
|
541
|
+
0. The header records ``#genome_build`` (required: nothing in a weights
|
|
542
|
+
table records the build), ``#variants_number`` (the row count) and
|
|
543
|
+
``#sd_source`` -- ``unspecified`` when the table predates ``SD_SOURCE``,
|
|
544
|
+
rather than claiming ``empirical``. ``#variants_genome_wide`` is omitted:
|
|
545
|
+
a chromosome subset or a hits list would otherwise be labelled as the
|
|
546
|
+
genome-wide score.
|
|
547
|
+
|
|
548
|
+
plink2 ``--score`` reads one header line, not comment blocks: strip the
|
|
549
|
+
``#`` lines first (``grep -v '^#'``). ``pgsc_calc`` ingestion is untested.
|
|
550
|
+
"""
|
|
551
|
+
if not isinstance(wt, WeightsTable):
|
|
552
|
+
raise TypeError("wt must be a WeightsTable")
|
|
553
|
+
if not wt.has_scale:
|
|
554
|
+
raise ValueError(
|
|
555
|
+
"a PGS Catalog export needs AF_REF/SD_REF to divide the weights by; "
|
|
556
|
+
"this table has no scale columns")
|
|
557
|
+
if genome_build not in PGS_BUILDS:
|
|
558
|
+
raise ValueError(f"genome_build must be one of {PGS_BUILDS}, got "
|
|
559
|
+
f"{genome_build!r}")
|
|
560
|
+
pgs_id = _pgs_header_value(
|
|
561
|
+
"pgs_id", pgs_id if pgs_id is not None else _pgs_default_id(path))
|
|
562
|
+
trait = _pgs_header_value(
|
|
563
|
+
"trait_reported",
|
|
564
|
+
trait_reported if trait_reported is not None else "unspecified")
|
|
565
|
+
sd = np.asarray(wt.sd_ref, dtype=float)
|
|
566
|
+
weight = np.asarray(wt.weight, dtype=float)
|
|
567
|
+
inert = ~np.isfinite(sd) | (sd <= _PGS_SD_EPS)
|
|
568
|
+
effect_weight = np.where(inert, 0.0, weight / np.where(inert, 1.0, sd))
|
|
569
|
+
n = len(wt)
|
|
570
|
+
seen, n_dup = set(), 0
|
|
571
|
+
for vid in np.asarray(wt.id):
|
|
572
|
+
key = str(vid)
|
|
573
|
+
n_dup += key in seen
|
|
574
|
+
seen.add(key)
|
|
575
|
+
if n_dup:
|
|
576
|
+
warnings.warn(f"{n_dup} duplicated variant IDs in the PGS export; "
|
|
577
|
+
"plink2 --score refuses them unless passed ignore-dup-ids",
|
|
578
|
+
RuntimeWarning, stacklevel=2)
|
|
579
|
+
if wt.sd_source == "hwe":
|
|
580
|
+
warnings.warn(
|
|
581
|
+
"exporting weights with SD_SOURCE=hwe: effect_weight divides by "
|
|
582
|
+
"the Hardy-Weinberg SD sqrt(2f(1-f)), not an observed dosage SD "
|
|
583
|
+
"(misspecified for sex chromosomes under male hemizygosity).",
|
|
584
|
+
RuntimeWarning, stacklevel=2)
|
|
585
|
+
elif wt.sd_source is None:
|
|
586
|
+
warnings.warn(
|
|
587
|
+
"the weights table records no SD_SOURCE; the export says "
|
|
588
|
+
"#sd_source=unspecified. Read the table with sd_source='hwe' or "
|
|
589
|
+
"'empirical' to record it.", RuntimeWarning, stacklevel=2)
|
|
590
|
+
from . import __version__
|
|
591
|
+
header = [
|
|
592
|
+
"###PGS CATALOG SCORING FILE",
|
|
593
|
+
"#format_version=2",
|
|
594
|
+
f"#pgs_id={pgs_id}",
|
|
595
|
+
f"#trait_reported={trait}",
|
|
596
|
+
f"#genome_build={genome_build}",
|
|
597
|
+
f"#variants_number={n}",
|
|
598
|
+
"#weight_type=beta",
|
|
599
|
+
"#weight_type_description=standardized effect divided by the "
|
|
600
|
+
"fit-cohort dosage SD (WEIGHT/SD_REF), per effect-allele count",
|
|
601
|
+
f"#sd_source={wt.sd_source or 'unspecified'}",
|
|
602
|
+
f"#source={_pgs_header_value('source', source) if source else 'allelign ' + __version__}",
|
|
603
|
+
]
|
|
604
|
+
with atomic_text(path) as fh:
|
|
605
|
+
for line in header:
|
|
606
|
+
fh.write(line + "\n")
|
|
607
|
+
fh.write("rsID\tchr_name\tchr_position\teffect_allele\tother_allele"
|
|
608
|
+
"\teffect_weight\tAF\n")
|
|
609
|
+
for vid, c, p, a1, a2, ew, af_ in zip(
|
|
610
|
+
wt.id, wt.chrom, wt.pos, wt.a1, wt.a2, effect_weight,
|
|
611
|
+
np.asarray(wt.af_ref, dtype=float)):
|
|
612
|
+
fh.write(f"{vid}\t{_pgs_chrom(c)}\t{int(p)}\t{a1}\t{a2}"
|
|
613
|
+
f"\t{ew:.12g}\t{af_:.8g}\n")
|
|
614
|
+
return path
|