ag2kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ag2kit/__init__.py +65 -0
- ag2kit/client.py +940 -0
- ag2kit/evaluate.py +351 -0
- ag2kit/gtex.py +275 -0
- ag2kit/io.py +248 -0
- ag2kit/provenance.py +316 -0
- ag2kit/py.typed +0 -0
- ag2kit/stats.py +375 -0
- ag2kit/variants.py +241 -0
- ag2kit-0.1.0.dist-info/METADATA +211 -0
- ag2kit-0.1.0.dist-info/RECORD +13 -0
- ag2kit-0.1.0.dist-info/WHEEL +4 -0
- ag2kit-0.1.0.dist-info/licenses/LICENSE +21 -0
ag2kit/stats.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"""Directional concordance and rank metrics for predicted and observed effects.
|
|
2
|
+
|
|
3
|
+
Sign agreement with an observed eQTL benchmarks predictions against association data.
|
|
4
|
+
Experimental validation would require an independent measurement of the variant effect.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
from scipy import stats
|
|
14
|
+
|
|
15
|
+
CONCORDANT, DISCORDANT, NO_CALL = "concordant", "discordant", "no_call"
|
|
16
|
+
NO_COMPARATOR = "no_comparator"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def no_call_threshold(null_deltas, quantile: float = 0.90) -> float:
|
|
20
|
+
"""Derives a no-call threshold from absolute scores under a null comparison.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
null_deltas: Null scores, for example from identical-condition contrasts.
|
|
24
|
+
Non-finite scores are discarded.
|
|
25
|
+
quantile: Quantile in [0, 1].
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
Magnitude at or below which classify treats predictions as no-calls.
|
|
29
|
+
|
|
30
|
+
Raises:
|
|
31
|
+
ValueError: If the quantile is outside [0, 1] or no finite scores remain.
|
|
32
|
+
"""
|
|
33
|
+
if not 0 <= quantile <= 1:
|
|
34
|
+
raise ValueError("quantile must be in [0, 1]")
|
|
35
|
+
vals = np.asarray([abs(float(v)) for v in null_deltas if np.isfinite(v)], dtype=float)
|
|
36
|
+
if vals.size == 0:
|
|
37
|
+
raise ValueError("no finite null deltas to derive a threshold from")
|
|
38
|
+
return float(np.quantile(vals, quantile))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def classify(delta: float, beta: float, threshold: float = 0.0) -> str:
|
|
42
|
+
"""Classifies agreement between effects oriented to the same allele.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
delta: Predicted effect.
|
|
46
|
+
beta: Observed effect.
|
|
47
|
+
threshold: Maximum absolute prediction treated as a no-call.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
"no_comparator" for missing, non-finite, or zero beta. Otherwise,
|
|
51
|
+
"no_call" for missing, non-finite, or zero delta, or when its absolute
|
|
52
|
+
value is at or below the threshold. Otherwise, "concordant" or
|
|
53
|
+
"discordant" by sign.
|
|
54
|
+
"""
|
|
55
|
+
if beta is None or not np.isfinite(beta) or beta == 0:
|
|
56
|
+
return NO_COMPARATOR
|
|
57
|
+
if delta is None or not np.isfinite(delta):
|
|
58
|
+
return NO_CALL
|
|
59
|
+
if abs(delta) <= threshold or delta == 0:
|
|
60
|
+
return NO_CALL
|
|
61
|
+
return CONCORDANT if np.sign(delta) == np.sign(beta) else DISCORDANT
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass
|
|
65
|
+
class Concordance:
|
|
66
|
+
"""Summarizes sign agreement with exact binomial inference.
|
|
67
|
+
|
|
68
|
+
Attributes:
|
|
69
|
+
k: Number of concordant calls.
|
|
70
|
+
n_called: Number of concordant and discordant calls.
|
|
71
|
+
n_no_call: Number of comparable pairs with no prediction call.
|
|
72
|
+
n_total: Number of pairs with a comparator.
|
|
73
|
+
threshold: Maximum absolute prediction treated as a no-call.
|
|
74
|
+
n_no_comparator: Number of pairs without a usable observed effect.
|
|
75
|
+
estimate: Concordance among called pairs, or NaN if none were called.
|
|
76
|
+
ci_low: Lower bound of the exact 95% interval for estimate.
|
|
77
|
+
ci_high: Upper bound of the exact 95% interval for estimate.
|
|
78
|
+
p_value: Two-sided exact binomial p-value against 0.5 among calls.
|
|
79
|
+
n_conservative: Number of pairs when no-calls count as discordant.
|
|
80
|
+
p_conservative: Exact p-value with no-calls counted as discordant,
|
|
81
|
+
or NaN if there are no comparable pairs.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
k: int
|
|
85
|
+
n_called: int
|
|
86
|
+
n_no_call: int
|
|
87
|
+
n_total: int
|
|
88
|
+
threshold: float
|
|
89
|
+
n_no_comparator: int = 0
|
|
90
|
+
estimate: float = field(init=False)
|
|
91
|
+
ci_low: float = field(init=False)
|
|
92
|
+
ci_high: float = field(init=False)
|
|
93
|
+
p_value: float = field(init=False)
|
|
94
|
+
n_conservative: int = field(init=False)
|
|
95
|
+
p_conservative: float = field(init=False)
|
|
96
|
+
|
|
97
|
+
def __post_init__(self) -> None:
|
|
98
|
+
if self.n_called > 0:
|
|
99
|
+
test = stats.binomtest(self.k, self.n_called, 0.5, alternative="two-sided")
|
|
100
|
+
ci = test.proportion_ci(confidence_level=0.95, method="exact")
|
|
101
|
+
self.estimate = self.k / self.n_called
|
|
102
|
+
self.ci_low, self.ci_high = float(ci.low), float(ci.high)
|
|
103
|
+
self.p_value = float(test.pvalue)
|
|
104
|
+
else:
|
|
105
|
+
self.estimate = self.ci_low = self.ci_high = self.p_value = float("nan")
|
|
106
|
+
self.n_conservative = self.n_called + self.n_no_call
|
|
107
|
+
self.p_conservative = (
|
|
108
|
+
float(stats.binomtest(self.k, self.n_conservative, 0.5, alternative="two-sided").pvalue)
|
|
109
|
+
if self.n_conservative > 0
|
|
110
|
+
else float("nan")
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
def as_row(self, label: str) -> dict:
|
|
114
|
+
"""Formats concordance and its conservative counterpart for export.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
label: Comparison name.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
A table row with the label, counts, estimates, interval, and p-values.
|
|
121
|
+
"""
|
|
122
|
+
return {
|
|
123
|
+
"comparison": label,
|
|
124
|
+
"k_concordant": self.k,
|
|
125
|
+
"n_called": self.n_called,
|
|
126
|
+
"n_no_call": self.n_no_call,
|
|
127
|
+
"n_no_comparator": self.n_no_comparator,
|
|
128
|
+
"n_total": self.n_total,
|
|
129
|
+
"no_call_threshold": self.threshold,
|
|
130
|
+
"concordance": self.estimate,
|
|
131
|
+
"ci95_low": self.ci_low,
|
|
132
|
+
"ci95_high": self.ci_high,
|
|
133
|
+
"p_binomial_vs_0.5": self.p_value,
|
|
134
|
+
"concordance_conservative": (
|
|
135
|
+
self.k / self.n_conservative if self.n_conservative else float("nan")
|
|
136
|
+
),
|
|
137
|
+
"p_conservative": self.p_conservative,
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def concordance(deltas, betas, threshold: float = 0.0) -> tuple[Concordance, list[str]]:
|
|
142
|
+
"""Tests sign concordance against 0.5 with a two-sided exact binomial test.
|
|
143
|
+
|
|
144
|
+
Args:
|
|
145
|
+
deltas: Predicted effects in pair order.
|
|
146
|
+
betas: Observed effects in the same order and allele orientation.
|
|
147
|
+
threshold: Maximum absolute prediction treated as a no-call.
|
|
148
|
+
|
|
149
|
+
Returns:
|
|
150
|
+
A summary and the classification of each input pair in order.
|
|
151
|
+
Pairs without a comparator are excluded from both tests.
|
|
152
|
+
|
|
153
|
+
Raises:
|
|
154
|
+
ValueError: If the inputs differ in length.
|
|
155
|
+
"""
|
|
156
|
+
deltas = list(deltas)
|
|
157
|
+
betas = list(betas)
|
|
158
|
+
if len(deltas) != len(betas):
|
|
159
|
+
raise ValueError("deltas and betas must have the same length")
|
|
160
|
+
calls = [classify(d, b, threshold) for d, b in zip(deltas, betas, strict=True)]
|
|
161
|
+
k = sum(c == CONCORDANT for c in calls)
|
|
162
|
+
n_called = sum(c in (CONCORDANT, DISCORDANT) for c in calls)
|
|
163
|
+
n_no_comparator = sum(c == NO_COMPARATOR for c in calls)
|
|
164
|
+
return (
|
|
165
|
+
Concordance(
|
|
166
|
+
k=k,
|
|
167
|
+
n_called=n_called,
|
|
168
|
+
n_no_call=sum(c == NO_CALL for c in calls),
|
|
169
|
+
n_total=len(calls) - n_no_comparator,
|
|
170
|
+
threshold=float(threshold),
|
|
171
|
+
n_no_comparator=n_no_comparator,
|
|
172
|
+
),
|
|
173
|
+
calls,
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _pairs(deltas, betas) -> tuple[np.ndarray, np.ndarray]:
|
|
178
|
+
d = np.asarray(deltas, dtype=float)
|
|
179
|
+
b = np.asarray(betas, dtype=float)
|
|
180
|
+
if d.shape != b.shape:
|
|
181
|
+
raise ValueError("deltas and betas must have the same length")
|
|
182
|
+
ok = np.isfinite(d) & np.isfinite(b)
|
|
183
|
+
return d[ok], b[ok]
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def spearman(deltas, betas, absolute: bool = False) -> dict:
|
|
187
|
+
"""Computes Spearman correlation after dropping non-finite pairs.
|
|
188
|
+
|
|
189
|
+
Args:
|
|
190
|
+
deltas: Predicted effects.
|
|
191
|
+
betas: Observed effects aligned to deltas.
|
|
192
|
+
absolute: Whether to correlate magnitudes. The default uses signed
|
|
193
|
+
effects, as in the AlphaGenome eQTL benchmark.
|
|
194
|
+
|
|
195
|
+
Returns:
|
|
196
|
+
The correlation (rho), p_value, and pair count (n). The correlation
|
|
197
|
+
and p-value are NaN with fewer than four finite pairs.
|
|
198
|
+
|
|
199
|
+
Raises:
|
|
200
|
+
ValueError: If the inputs have different shapes.
|
|
201
|
+
"""
|
|
202
|
+
d, b = _pairs(deltas, betas)
|
|
203
|
+
if absolute:
|
|
204
|
+
d, b = np.abs(d), np.abs(b)
|
|
205
|
+
if d.size < 4:
|
|
206
|
+
return {"rho": float("nan"), "p_value": float("nan"), "n": int(d.size)}
|
|
207
|
+
res = stats.spearmanr(d, b)
|
|
208
|
+
return {"rho": float(res.statistic), "p_value": float(res.pvalue), "n": int(d.size)}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def sign_auroc(deltas, betas) -> float:
|
|
212
|
+
"""Computes how well signed predictions rank positive above negative effects.
|
|
213
|
+
|
|
214
|
+
This is the sign metric used in the Borzoi and AlphaGenome eQTL benchmarks.
|
|
215
|
+
|
|
216
|
+
Args:
|
|
217
|
+
deltas: Signed predicted effects.
|
|
218
|
+
betas: Observed effects aligned to deltas.
|
|
219
|
+
|
|
220
|
+
Returns:
|
|
221
|
+
AUROC after excluding non-finite pairs and zero betas, or NaN if
|
|
222
|
+
either sign class is empty.
|
|
223
|
+
|
|
224
|
+
Raises:
|
|
225
|
+
ValueError: If the inputs have different shapes.
|
|
226
|
+
"""
|
|
227
|
+
d, b = _pairs(deltas, betas)
|
|
228
|
+
pos, neg = d[b > 0], d[b < 0]
|
|
229
|
+
if pos.size == 0 or neg.size == 0:
|
|
230
|
+
return float("nan")
|
|
231
|
+
u = stats.mannwhitneyu(pos, neg, alternative="two-sided").statistic
|
|
232
|
+
return float(u / (pos.size * neg.size))
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def recall_at_sign_accuracy(deltas, betas, accuracy: float = 0.9) -> dict:
|
|
236
|
+
"""Finds the longest ranked prefix meeting a target sign accuracy.
|
|
237
|
+
|
|
238
|
+
Pairs are ranked by decreasing absolute prediction, with ties resolved
|
|
239
|
+
in input order. Zero predictions count as incorrect.
|
|
240
|
+
|
|
241
|
+
Args:
|
|
242
|
+
deltas: Predicted effects.
|
|
243
|
+
betas: Observed effects aligned to deltas.
|
|
244
|
+
accuracy: Minimum sign accuracy in (0, 1].
|
|
245
|
+
|
|
246
|
+
Returns:
|
|
247
|
+
A dict with recall (fraction of comparable pairs retained), n_called,
|
|
248
|
+
n (finite pairs with nonzero beta), and min_abs_delta (the last
|
|
249
|
+
retained magnitude, or NaN if none qualify). Recall is NaN when n is zero.
|
|
250
|
+
|
|
251
|
+
Raises:
|
|
252
|
+
ValueError: If accuracy is outside (0, 1] or input shapes differ.
|
|
253
|
+
"""
|
|
254
|
+
if not 0 < accuracy <= 1:
|
|
255
|
+
raise ValueError("accuracy must be in (0, 1]")
|
|
256
|
+
d, b = _pairs(deltas, betas)
|
|
257
|
+
keep = b != 0
|
|
258
|
+
d, b = d[keep], b[keep]
|
|
259
|
+
if d.size == 0:
|
|
260
|
+
return {"recall": float("nan"), "n_called": 0, "n": 0, "min_abs_delta": float("nan")}
|
|
261
|
+
order = np.argsort(-np.abs(d), kind="stable")
|
|
262
|
+
correct = (np.sign(d[order]) == np.sign(b[order])) & (d[order] != 0)
|
|
263
|
+
running = np.cumsum(correct) / np.arange(1, d.size + 1)
|
|
264
|
+
ok = np.flatnonzero(running >= accuracy)
|
|
265
|
+
k = int(ok[-1]) + 1 if ok.size else 0
|
|
266
|
+
cutoff = float(abs(d[order][k - 1])) if k else float("nan")
|
|
267
|
+
return {"recall": k / d.size, "n_called": k, "n": int(d.size), "min_abs_delta": cutoff}
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def by_group(metric, deltas, betas, groups, min_size: int = 2) -> tuple[float, pd.DataFrame]:
|
|
271
|
+
"""Computes a metric per group and averages it by group size.
|
|
272
|
+
|
|
273
|
+
Args:
|
|
274
|
+
metric: Callable taking predicted and observed arrays and returning a float.
|
|
275
|
+
deltas: Predicted effects.
|
|
276
|
+
betas: Observed effects aligned to deltas.
|
|
277
|
+
groups: Group label for each pair, such as a tissue ID.
|
|
278
|
+
min_size: Minimum group size for inclusion in the average.
|
|
279
|
+
|
|
280
|
+
Returns:
|
|
281
|
+
The weighted mean and a table of group, n, and value for each group.
|
|
282
|
+
Small groups and NaN values are excluded from the mean, which is NaN
|
|
283
|
+
if no groups qualify.
|
|
284
|
+
"""
|
|
285
|
+
df = pd.DataFrame({"delta": deltas, "beta": betas, "group": groups})
|
|
286
|
+
rows = []
|
|
287
|
+
for name, g in df.groupby("group", sort=True, observed=False):
|
|
288
|
+
value = float(metric(g["delta"].to_numpy(), g["beta"].to_numpy()))
|
|
289
|
+
rows.append({"group": name, "n": len(g), "value": value})
|
|
290
|
+
table = pd.DataFrame(rows, columns=["group", "n", "value"])
|
|
291
|
+
used = table[(table["n"] >= min_size) & table["value"].notna()]
|
|
292
|
+
weighted = float(np.average(used["value"], weights=used["n"])) if len(used) else float("nan")
|
|
293
|
+
return weighted, table
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def mcnemar(calls_a, calls_b) -> dict:
|
|
297
|
+
"""Compares two sets of calls on the same pairs with an exact McNemar test.
|
|
298
|
+
|
|
299
|
+
Args:
|
|
300
|
+
calls_a: Classifications from the first method.
|
|
301
|
+
calls_b: Classifications from the second method in the same order.
|
|
302
|
+
|
|
303
|
+
Returns:
|
|
304
|
+
Counts a_better and b_better for pairs where one method is concordant
|
|
305
|
+
and the other discordant, their sum (n_discordant_pairs), and the
|
|
306
|
+
two-sided p_exact, which is NaN when there are no disagreements.
|
|
307
|
+
|
|
308
|
+
Raises:
|
|
309
|
+
ValueError: If the lists differ in length.
|
|
310
|
+
"""
|
|
311
|
+
a = list(calls_a)
|
|
312
|
+
b = list(calls_b)
|
|
313
|
+
if len(a) != len(b):
|
|
314
|
+
raise ValueError("call lists must have the same length")
|
|
315
|
+
a_only = sum(x == CONCORDANT and y == DISCORDANT for x, y in zip(a, b, strict=True))
|
|
316
|
+
b_only = sum(x == DISCORDANT and y == CONCORDANT for x, y in zip(a, b, strict=True))
|
|
317
|
+
n = a_only + b_only
|
|
318
|
+
p = (
|
|
319
|
+
float(stats.binomtest(a_only, n, 0.5, alternative="two-sided").pvalue)
|
|
320
|
+
if n > 0
|
|
321
|
+
else float("nan")
|
|
322
|
+
)
|
|
323
|
+
return {"a_better": a_only, "b_better": b_only, "n_discordant_pairs": n, "p_exact": p}
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def monte_carlo_summary(values, seeds, first_half) -> dict:
|
|
327
|
+
"""Summarizes sampling variability across conditioning draws for one variant.
|
|
328
|
+
|
|
329
|
+
A direction is resolved when the absolute mean meets the 95% t margin,
|
|
330
|
+
the mean is nonzero, and both half-means have the same nonzero sign.
|
|
331
|
+
The interval describes Monte Carlo error conditional on the draw procedure.
|
|
332
|
+
|
|
333
|
+
Args:
|
|
334
|
+
values: Scores from at least two finite draws.
|
|
335
|
+
seeds: Seed label for each score in the same order.
|
|
336
|
+
first_half: Seed labels assigned to the first half.
|
|
337
|
+
|
|
338
|
+
Returns:
|
|
339
|
+
Draw count, mean, sample standard deviation, Monte Carlo standard
|
|
340
|
+
error, t interval, half-means, and direction diagnostics.
|
|
341
|
+
|
|
342
|
+
Raises:
|
|
343
|
+
ValueError: If there are fewer than two draws, any score is non-finite,
|
|
344
|
+
or either half is empty.
|
|
345
|
+
"""
|
|
346
|
+
v = np.asarray(values, dtype=float)
|
|
347
|
+
s = np.asarray(seeds)
|
|
348
|
+
if v.size < 2 or not np.all(np.isfinite(v)):
|
|
349
|
+
raise ValueError("need at least two finite draws")
|
|
350
|
+
in_first = np.isin(s, list(first_half))
|
|
351
|
+
if not in_first.any() or in_first.all():
|
|
352
|
+
raise ValueError("both halves need at least one draw")
|
|
353
|
+
n = v.size
|
|
354
|
+
mean = float(v.mean())
|
|
355
|
+
sd = float(v.std(ddof=1))
|
|
356
|
+
mcse = sd / np.sqrt(n)
|
|
357
|
+
t_crit = float(stats.t.ppf(0.975, n - 1))
|
|
358
|
+
half1, half2 = float(v[in_first].mean()), float(v[~in_first].mean())
|
|
359
|
+
excludes_zero = bool(abs(mean) >= t_crit * mcse and mean != 0)
|
|
360
|
+
halves_agree = bool(half1 != 0 and half2 != 0 and np.sign(half1) == np.sign(half2))
|
|
361
|
+
return {
|
|
362
|
+
"n_draws": n,
|
|
363
|
+
"mean": mean,
|
|
364
|
+
"sd_between_draws": sd,
|
|
365
|
+
"mcse": float(mcse),
|
|
366
|
+
"t_critical": t_crit,
|
|
367
|
+
"ci95_low": mean - t_crit * mcse,
|
|
368
|
+
"ci95_high": mean + t_crit * mcse,
|
|
369
|
+
"abs_mean_over_mcse": abs(mean) / mcse if mcse > 0 else float("inf"),
|
|
370
|
+
"half1_mean": half1,
|
|
371
|
+
"half2_mean": half2,
|
|
372
|
+
"ci_excludes_zero": excludes_zero,
|
|
373
|
+
"halves_agree_in_sign": halves_agree,
|
|
374
|
+
"directionally_resolved": excludes_zero and halves_agree,
|
|
375
|
+
}
|
ag2kit/variants.py
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"""Allele harmonization, GRCh38 reference-allele checks and locus clustering.
|
|
2
|
+
|
|
3
|
+
For gene symbols, IDs and coordinates use the SDK's ``alphagenome.data.gene_annotation``
|
|
4
|
+
with a GENCODE GTF, so genes are defined by an annotation release you can pin.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
import math
|
|
12
|
+
import os
|
|
13
|
+
import time
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import pandas as pd
|
|
18
|
+
import requests
|
|
19
|
+
|
|
20
|
+
from ag2kit.provenance import cache_dir
|
|
21
|
+
|
|
22
|
+
log = logging.getLogger("ag2kit")
|
|
23
|
+
|
|
24
|
+
BASES = frozenset("ACGT")
|
|
25
|
+
PALINDROMIC = frozenset({("A", "T"), ("T", "A"), ("C", "G"), ("G", "C")})
|
|
26
|
+
|
|
27
|
+
SEQUENCE_URL = "https://rest.ensembl.org/sequence/region/human/{region}?content-type=text/plain"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class Harmonized:
|
|
32
|
+
"""Stores an allele-oriented effect or its exclusion reason.
|
|
33
|
+
|
|
34
|
+
Attributes:
|
|
35
|
+
estimate: Effect relative to ALT, or None for an excluded record.
|
|
36
|
+
flipped: Whether the original estimate was negated.
|
|
37
|
+
status: "ok" or the reason the record was excluded.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
estimate: float | None
|
|
41
|
+
flipped: bool
|
|
42
|
+
status: str
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def is_snv(ref: str, alt: str) -> bool:
|
|
46
|
+
"""Checks whether both alleles are single A, C, G, or T bases.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
ref: Reference allele.
|
|
50
|
+
alt: Alternate allele.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
Whether both are canonical single bases, ignoring case. Identical
|
|
54
|
+
bases also pass this check.
|
|
55
|
+
"""
|
|
56
|
+
return (
|
|
57
|
+
isinstance(ref, str)
|
|
58
|
+
and isinstance(alt, str)
|
|
59
|
+
and len(ref) == 1
|
|
60
|
+
and len(alt) == 1
|
|
61
|
+
and ref.upper() in BASES
|
|
62
|
+
and alt.upper() in BASES
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def is_palindromic(ref: str, alt: str) -> bool:
|
|
67
|
+
"""Checks for a single-base allele pair with ambiguous strand.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
ref: Reference allele.
|
|
71
|
+
alt: Alternate allele.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
Whether the two alleles are A/T or C/G in either order, ignoring case.
|
|
75
|
+
"""
|
|
76
|
+
return (str(ref).upper(), str(alt).upper()) in PALINDROMIC
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _chrom(chrom: object) -> str:
|
|
80
|
+
return str(chrom).removeprefix("chr")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def harmonize_effect(
|
|
84
|
+
estimate: float | None,
|
|
85
|
+
effect_allele: str,
|
|
86
|
+
ref: str,
|
|
87
|
+
alt: str,
|
|
88
|
+
forward_strand: bool = False,
|
|
89
|
+
) -> Harmonized:
|
|
90
|
+
"""Orients an observed effect to ALT relative to REF.
|
|
91
|
+
|
|
92
|
+
REF and ALT should be checked against the GRCh38 forward strand before
|
|
93
|
+
calling. The function compares allele labels directly without complementing
|
|
94
|
+
bases. With unknown source strand, it excludes indels and palindromic SNVs.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
estimate: Observed effect, or None if missing.
|
|
98
|
+
effect_allele: Allele to which the observed effect refers.
|
|
99
|
+
ref: Forward-strand GRCh38 reference allele.
|
|
100
|
+
alt: Forward-strand GRCh38 alternate allele.
|
|
101
|
+
forward_strand: Whether the source reports GRCh38 forward-strand
|
|
102
|
+
alleles with genomic REF, as GTEx does.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
The original effect if it refers to ALT, the negated effect if it
|
|
106
|
+
refers to REF, or an exclusion status if orientation fails.
|
|
107
|
+
"""
|
|
108
|
+
if estimate is None or pd.isna(estimate):
|
|
109
|
+
return Harmonized(None, False, "missing_statistics")
|
|
110
|
+
estimate = float(estimate)
|
|
111
|
+
if not math.isfinite(estimate):
|
|
112
|
+
return Harmonized(None, False, "missing_statistics")
|
|
113
|
+
ref, alt = str(ref).upper(), str(alt).upper()
|
|
114
|
+
effect = str(effect_allele).upper()
|
|
115
|
+
if ref == alt:
|
|
116
|
+
return Harmonized(None, False, "ref_equals_alt")
|
|
117
|
+
if not forward_strand and not is_snv(ref, alt):
|
|
118
|
+
return Harmonized(None, False, "not_snv")
|
|
119
|
+
if not forward_strand and is_palindromic(ref, alt):
|
|
120
|
+
return Harmonized(None, False, "palindromic")
|
|
121
|
+
if effect == alt:
|
|
122
|
+
return Harmonized(estimate, False, "ok")
|
|
123
|
+
if effect == ref:
|
|
124
|
+
return Harmonized(-estimate, True, "ok")
|
|
125
|
+
return Harmonized(None, False, "effect_allele_not_in_ref_alt")
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _load(path: Path) -> dict:
|
|
129
|
+
try:
|
|
130
|
+
data = json.loads(path.read_text())
|
|
131
|
+
except (OSError, ValueError):
|
|
132
|
+
return {}
|
|
133
|
+
return data if isinstance(data, dict) else {}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _save(path: Path, data: dict) -> None:
|
|
137
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
138
|
+
tmp = path.with_suffix(f".{os.getpid()}.tmp")
|
|
139
|
+
tmp.write_text(json.dumps(data, sort_keys=True))
|
|
140
|
+
tmp.replace(path)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class ReferenceBase:
|
|
144
|
+
"""Looks up GRCh38 reference bases through Ensembl and caches successful lookups.
|
|
145
|
+
|
|
146
|
+
Checking REF before scoring helps prevent swapped REF and ALT labels
|
|
147
|
+
from reversing the interpretation of downstream effects.
|
|
148
|
+
|
|
149
|
+
Attributes:
|
|
150
|
+
cache_path: Path to the local JSON cache.
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
def __init__(self, cache_path: str | Path | None = None):
|
|
154
|
+
"""Loads a reference-base cache.
|
|
155
|
+
|
|
156
|
+
Args:
|
|
157
|
+
cache_path: Cache file, defaulting to derived/grch38_bases.json under
|
|
158
|
+
the ag2kit cache directory.
|
|
159
|
+
"""
|
|
160
|
+
self.cache_path = Path(cache_path or cache_dir() / "derived/grch38_bases.json")
|
|
161
|
+
self._cache: dict[str, str] = _load(self.cache_path)
|
|
162
|
+
|
|
163
|
+
def get(self, chrom: str, pos: int, offline: bool = False) -> str | None:
|
|
164
|
+
"""Looks up the reference base for one position.
|
|
165
|
+
|
|
166
|
+
Args:
|
|
167
|
+
chrom: Chromosome name with or without the chr prefix.
|
|
168
|
+
pos: 1-based GRCh38 position.
|
|
169
|
+
offline: Whether to use only cached results.
|
|
170
|
+
|
|
171
|
+
Returns:
|
|
172
|
+
An uppercase base or None if the lookup fails or the offline cache misses.
|
|
173
|
+
|
|
174
|
+
Raises:
|
|
175
|
+
ValueError: If pos is below one.
|
|
176
|
+
OSError: If a successful lookup cannot be saved to the cache.
|
|
177
|
+
"""
|
|
178
|
+
if int(pos) < 1:
|
|
179
|
+
raise ValueError(f"pos must be 1-based, got {pos}")
|
|
180
|
+
key = f"{_chrom(chrom)}:{int(pos)}"
|
|
181
|
+
if key in self._cache:
|
|
182
|
+
return self._cache[key]
|
|
183
|
+
if offline:
|
|
184
|
+
return None
|
|
185
|
+
url = SEQUENCE_URL.format(region=f"{_chrom(chrom)}:{pos}..{pos}")
|
|
186
|
+
for attempt in range(3):
|
|
187
|
+
try:
|
|
188
|
+
response = requests.get(url, timeout=30)
|
|
189
|
+
except requests.RequestException as exc:
|
|
190
|
+
log.warning("Ensembl sequence lookup %s failed: %s", key, exc)
|
|
191
|
+
else:
|
|
192
|
+
base = response.text.strip().upper() if response.ok else ""
|
|
193
|
+
if len(base) == 1 and base in "ACGTN":
|
|
194
|
+
self._cache[key] = base
|
|
195
|
+
_save(self.cache_path, self._cache)
|
|
196
|
+
return base
|
|
197
|
+
if 400 <= response.status_code < 500 and response.status_code != 429:
|
|
198
|
+
log.warning("Ensembl rejected region %s (HTTP %s)", key, response.status_code)
|
|
199
|
+
return None
|
|
200
|
+
time.sleep(1 + attempt)
|
|
201
|
+
return None
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def cluster_loci(df: pd.DataFrame, window: int = 1_000_000) -> pd.DataFrame:
|
|
205
|
+
"""Labels one lead per locus using distance and gene as proxies for LD.
|
|
206
|
+
|
|
207
|
+
Rows are visited by ascending p-value, breaking ties by variant ID.
|
|
208
|
+
Each unassigned row becomes a lead and absorbs unassigned rows within
|
|
209
|
+
the window on the same chromosome, plus rows with the same gene.
|
|
210
|
+
|
|
211
|
+
Args:
|
|
212
|
+
df: Table with variant_id, chrom, pos, gene, and pval columns.
|
|
213
|
+
window: Maximum distance from the lead in base pairs.
|
|
214
|
+
|
|
215
|
+
Returns:
|
|
216
|
+
The input rows in original order with locus_group (the lead's variant
|
|
217
|
+
ID) and is_lead.
|
|
218
|
+
|
|
219
|
+
Raises:
|
|
220
|
+
ValueError: If a required column is missing.
|
|
221
|
+
"""
|
|
222
|
+
# shortcut: Quadratic scan suits small tables. Index by position if this becomes slow.
|
|
223
|
+
required = {"variant_id", "chrom", "pos", "gene", "pval"}
|
|
224
|
+
missing = required - set(df.columns)
|
|
225
|
+
if missing:
|
|
226
|
+
raise ValueError(f"cluster_loci requires columns {sorted(missing)}")
|
|
227
|
+
work = df.reset_index(drop=True).sort_values(["pval", "variant_id"], kind="mergesort")
|
|
228
|
+
chrom = work["chrom"].map(_chrom)
|
|
229
|
+
pos = work["pos"].astype(float)
|
|
230
|
+
gene = work["gene"].astype(str)
|
|
231
|
+
group = pd.Series(pd.NA, index=work.index, dtype="object")
|
|
232
|
+
lead = pd.Series(False, index=work.index)
|
|
233
|
+
for i in work.index:
|
|
234
|
+
if pd.notna(group[i]):
|
|
235
|
+
continue
|
|
236
|
+
lead[i] = True
|
|
237
|
+
near = (chrom == chrom[i]) & (pos - pos[i]).abs().le(window)
|
|
238
|
+
group[(near | (gene == gene[i])) & group.isna()] = work.at[i, "variant_id"]
|
|
239
|
+
work["locus_group"] = group
|
|
240
|
+
work["is_lead"] = lead
|
|
241
|
+
return df.reset_index(drop=True).join(work[["locus_group", "is_lead"]]).set_axis(df.index)
|