ag2kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ag2kit/__init__.py +65 -0
- ag2kit/client.py +940 -0
- ag2kit/evaluate.py +351 -0
- ag2kit/gtex.py +275 -0
- ag2kit/io.py +248 -0
- ag2kit/provenance.py +316 -0
- ag2kit/py.typed +0 -0
- ag2kit/stats.py +375 -0
- ag2kit/variants.py +241 -0
- ag2kit-0.1.0.dist-info/METADATA +211 -0
- ag2kit-0.1.0.dist-info/RECORD +13 -0
- ag2kit-0.1.0.dist-info/WHEEL +4 -0
- ag2kit-0.1.0.dist-info/licenses/LICENSE +21 -0
ag2kit/evaluate.py
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
"""Evaluates predicted variant effects against observed molecular QTLs.
|
|
2
|
+
|
|
3
|
+
Variants sharing a gene or locus can be correlated through linkage
|
|
4
|
+
disequilibrium. Resampling whole clusters preserves that dependence
|
|
5
|
+
within each bootstrap replicate.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Callable, Mapping
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from ag2kit.provenance import software_versions, utc_now
|
|
19
|
+
from ag2kit.stats import _pairs, by_group, sign_auroc, spearman
|
|
20
|
+
from ag2kit.variants import harmonize_effect
|
|
21
|
+
|
|
22
|
+
Metric = Callable[[np.ndarray, np.ndarray], float]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def sign_accuracy(deltas, betas) -> float:
|
|
26
|
+
"""Computes the fraction of comparable effects with matching signs.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
deltas: Predicted effects.
|
|
30
|
+
betas: Observed effects in the same order and allele orientation.
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
Sign accuracy after excluding non-finite pairs and zero betas,
|
|
34
|
+
or NaN if no pairs remain. Zero predictions count as incorrect.
|
|
35
|
+
|
|
36
|
+
Raises:
|
|
37
|
+
ValueError: If the inputs have different shapes.
|
|
38
|
+
"""
|
|
39
|
+
d, b = _pairs(deltas, betas)
|
|
40
|
+
ok = b != 0
|
|
41
|
+
if not ok.any():
|
|
42
|
+
return float("nan")
|
|
43
|
+
return float(np.mean(np.sign(d[ok]) == np.sign(b[ok])))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _rho(deltas, betas) -> float:
|
|
47
|
+
return spearman(deltas, betas)["rho"]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
METRICS: dict[str, Metric] = {
|
|
51
|
+
"sign_auroc": sign_auroc,
|
|
52
|
+
"spearman_rho": _rho,
|
|
53
|
+
"sign_accuracy": sign_accuracy,
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _key(df: pd.DataFrame) -> pd.DataFrame:
|
|
58
|
+
"""Builds an order-independent allele key using unversioned gene IDs."""
|
|
59
|
+
ref = df["ref"].astype("string").str.upper()
|
|
60
|
+
alt = df["alt"].astype("string").str.upper()
|
|
61
|
+
return pd.DataFrame(
|
|
62
|
+
{
|
|
63
|
+
"_chrom": "chr" + df["chrom"].astype("string").str.removeprefix("chr"),
|
|
64
|
+
"_pos": pd.to_numeric(df["pos"], errors="coerce").astype("Int64"),
|
|
65
|
+
"_alleles": (ref + "/" + alt).where(ref < alt, alt + "/" + ref),
|
|
66
|
+
"_gene": df["gene_id"].astype("string").str.split(".").str[0],
|
|
67
|
+
},
|
|
68
|
+
index=df.index,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class Evaluation:
|
|
74
|
+
"""Stores the result of evaluate.
|
|
75
|
+
|
|
76
|
+
Attributes:
|
|
77
|
+
pairs: Ledger with one row per observed effect and its match status.
|
|
78
|
+
metrics: Overall estimates, confidence intervals, and pair counts.
|
|
79
|
+
per_group: Group estimates when requested, otherwise an empty table.
|
|
80
|
+
settings: Evaluation parameters and input counts.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
pairs: pd.DataFrame
|
|
84
|
+
metrics: pd.DataFrame
|
|
85
|
+
per_group: pd.DataFrame
|
|
86
|
+
settings: dict = field(default_factory=dict)
|
|
87
|
+
|
|
88
|
+
def summary(self) -> pd.DataFrame:
|
|
89
|
+
"""Prints the metrics and counts by pair status.
|
|
90
|
+
|
|
91
|
+
Returns:
|
|
92
|
+
The overall metrics table.
|
|
93
|
+
"""
|
|
94
|
+
counts = self.pairs["pair_status"].value_counts().rename_axis("pair_status")
|
|
95
|
+
print(self.metrics.to_string(index=False))
|
|
96
|
+
print(counts.to_string())
|
|
97
|
+
return self.metrics
|
|
98
|
+
|
|
99
|
+
def save(self, directory: str | Path) -> Path:
|
|
100
|
+
"""Writes pairs.parquet, metrics.tsv, per_group.tsv, and evaluation.json.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
directory: Output directory. Existing files with these names are replaced.
|
|
104
|
+
|
|
105
|
+
Returns:
|
|
106
|
+
Path to the output directory.
|
|
107
|
+
|
|
108
|
+
Raises:
|
|
109
|
+
OSError: If the directory or a file cannot be written.
|
|
110
|
+
"""
|
|
111
|
+
out = Path(directory)
|
|
112
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
113
|
+
self.pairs.to_parquet(out / "pairs.parquet", index=False)
|
|
114
|
+
self.metrics.to_csv(out / "metrics.tsv", sep="\t", index=False)
|
|
115
|
+
self.per_group.to_csv(out / "per_group.tsv", sep="\t", index=False)
|
|
116
|
+
meta = {**self.settings, "software": software_versions(), "written_utc": utc_now()}
|
|
117
|
+
(out / "evaluation.json").write_text(json.dumps(meta, indent=2, default=str))
|
|
118
|
+
return out
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def join(
|
|
122
|
+
predictions: pd.DataFrame,
|
|
123
|
+
observed: pd.DataFrame,
|
|
124
|
+
*,
|
|
125
|
+
match: Mapping[str, str] | None = None,
|
|
126
|
+
effect: str = "raw_effect",
|
|
127
|
+
forward_strand: bool = True,
|
|
128
|
+
) -> pd.DataFrame:
|
|
129
|
+
"""Matches observed effects to predictions and orients them to predicted ALT.
|
|
130
|
+
|
|
131
|
+
Matching uses chromosome, position, unordered REF/ALT, and unversioned
|
|
132
|
+
gene ID. Equivalent predictions with the same REF, score, and status
|
|
133
|
+
count as one candidate.
|
|
134
|
+
|
|
135
|
+
Args:
|
|
136
|
+
predictions: Table with chrom, pos, ref, alt, gene_id, status, and effect.
|
|
137
|
+
observed: Table with chrom, pos, ref, alt, gene_id, beta, and effect_allele.
|
|
138
|
+
match: Optional mapping from observed to prediction column names for
|
|
139
|
+
additional keys, such as {"ontology_id": "condition"}.
|
|
140
|
+
effect: Prediction column containing the signed effect.
|
|
141
|
+
forward_strand: Whether observed alleles are on the GRCh38 forward
|
|
142
|
+
strand. If true, a swapped observed REF is flagged as ref_mismatch.
|
|
143
|
+
|
|
144
|
+
Returns:
|
|
145
|
+
One row per observed record with prediction fields, ALT-oriented
|
|
146
|
+
beta_alt, allele_flipped, and pair_status identifying usable pairs
|
|
147
|
+
("ok"), unmatched or failed predictions, or allele orientation problems.
|
|
148
|
+
|
|
149
|
+
Raises:
|
|
150
|
+
ValueError: If either input lacks a required column.
|
|
151
|
+
"""
|
|
152
|
+
match = dict(match or {})
|
|
153
|
+
need_obs = {"chrom", "pos", "ref", "alt", "gene_id", "beta", "effect_allele", *match}
|
|
154
|
+
need_pred = {"chrom", "pos", "ref", "alt", "gene_id", effect, "status", *match.values()}
|
|
155
|
+
for name, df, need in (
|
|
156
|
+
("observed", observed, need_obs),
|
|
157
|
+
("predictions", predictions, need_pred),
|
|
158
|
+
):
|
|
159
|
+
if missing := need - set(df.columns):
|
|
160
|
+
raise ValueError(f"{name} is missing columns {sorted(missing)}")
|
|
161
|
+
|
|
162
|
+
obs = observed.reset_index(drop=True)
|
|
163
|
+
pred = predictions.reset_index(drop=True)
|
|
164
|
+
keys = ["_chrom", "_pos", "_alleles", "_gene"]
|
|
165
|
+
match_keys = [f"_m_{c}" for c in match]
|
|
166
|
+
o = pd.concat([_key(obs), obs[list(match)].set_axis(match_keys, axis=1)], axis=1)
|
|
167
|
+
p = pd.concat(
|
|
168
|
+
[
|
|
169
|
+
_key(pred),
|
|
170
|
+
pred[list(match.values())].set_axis(match_keys, axis=1),
|
|
171
|
+
],
|
|
172
|
+
axis=1,
|
|
173
|
+
)
|
|
174
|
+
keys += match_keys
|
|
175
|
+
variant_keys = [k for k in keys if k != "_gene"]
|
|
176
|
+
p["_ref"] = pred["ref"].astype("string").str.upper()
|
|
177
|
+
p["_effect"] = pred[effect].to_numpy()
|
|
178
|
+
p["_status"] = pred["status"].to_numpy()
|
|
179
|
+
p = p.dropna(subset=variant_keys).drop_duplicates([*keys, "_ref", "_effect", "_status"])
|
|
180
|
+
p["_n_pred"] = p.groupby(keys, dropna=False)["_gene"].transform("size")
|
|
181
|
+
p = p.drop_duplicates(keys)
|
|
182
|
+
p["_pred_row"] = p.index
|
|
183
|
+
merged = o.merge(p.dropna(subset=keys)[[*keys, "_n_pred", "_pred_row"]], on=keys, how="left")
|
|
184
|
+
# Without gene_column, a request failure applies to every gene for that variant.
|
|
185
|
+
failed = p[(p["_status"] != "completed") & p["_gene"].fillna("").isin(["", "nan", "None"])]
|
|
186
|
+
failed_variants = set(map(tuple, failed[variant_keys].astype(str).to_numpy()))
|
|
187
|
+
merged["_variant_failed"] = [
|
|
188
|
+
tuple(r) in failed_variants for r in merged[variant_keys].astype(str).to_numpy()
|
|
189
|
+
]
|
|
190
|
+
|
|
191
|
+
pred_cols = ["ref", "alt", effect, "status", "error", "source", "condition", "scorer_name"]
|
|
192
|
+
pred_cols = [c for c in pred_cols if c in pred.columns]
|
|
193
|
+
matched = pred.reindex(merged["_pred_row"].fillna(-1).astype(int))[pred_cols]
|
|
194
|
+
matched.index = obs.index
|
|
195
|
+
out = obs.join(matched.add_prefix("pred_"))
|
|
196
|
+
score = out[f"pred_{effect}"]
|
|
197
|
+
out["predicted"] = score.where(score.notna(), np.nan).astype(float)
|
|
198
|
+
|
|
199
|
+
status, beta_alt, flipped = [], [], []
|
|
200
|
+
counts = merged["_n_pred"].fillna(0).to_numpy(dtype=int)
|
|
201
|
+
for (i, row), n in zip(out.iterrows(), counts, strict=True):
|
|
202
|
+
if n == 0:
|
|
203
|
+
gone = "prediction_failed" if merged.at[i, "_variant_failed"] else "no_prediction"
|
|
204
|
+
s, b, f = gone, np.nan, False
|
|
205
|
+
elif n > 1:
|
|
206
|
+
s, b, f = "ambiguous_prediction", np.nan, False
|
|
207
|
+
elif str(row["pred_status"]) != "completed" or not np.isfinite(row["predicted"]):
|
|
208
|
+
s, b, f = "prediction_failed", np.nan, False
|
|
209
|
+
elif forward_strand and str(row["ref"]).upper() != str(row["pred_ref"]).upper():
|
|
210
|
+
# Swapping genomic REF and ALT changes the model input, so negation
|
|
211
|
+
# cannot recover the effect of the intended comparison.
|
|
212
|
+
s, b, f = "ref_mismatch", np.nan, False
|
|
213
|
+
else:
|
|
214
|
+
h = harmonize_effect(
|
|
215
|
+
row["beta"], row["effect_allele"], row["pred_ref"], row["pred_alt"], forward_strand
|
|
216
|
+
)
|
|
217
|
+
s, f = h.status, h.flipped
|
|
218
|
+
b = np.nan if h.estimate is None else h.estimate
|
|
219
|
+
status.append(s)
|
|
220
|
+
beta_alt.append(b)
|
|
221
|
+
flipped.append(f)
|
|
222
|
+
out["beta_alt"] = beta_alt
|
|
223
|
+
out["allele_flipped"] = flipped
|
|
224
|
+
out["pair_status"] = status
|
|
225
|
+
return out
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _bootstrap(
|
|
229
|
+
metric: Metric,
|
|
230
|
+
d: np.ndarray,
|
|
231
|
+
b: np.ndarray,
|
|
232
|
+
clusters: np.ndarray,
|
|
233
|
+
groups: np.ndarray | None,
|
|
234
|
+
n_boot: int,
|
|
235
|
+
rng: np.random.Generator,
|
|
236
|
+
min_group: int,
|
|
237
|
+
) -> np.ndarray:
|
|
238
|
+
labels, inverse = np.unique(clusters, return_inverse=True)
|
|
239
|
+
members = [np.flatnonzero(inverse == k) for k in range(labels.size)]
|
|
240
|
+
stats = np.empty(n_boot)
|
|
241
|
+
for i in range(n_boot):
|
|
242
|
+
idx = np.concatenate([members[k] for k in rng.integers(0, labels.size, labels.size)])
|
|
243
|
+
if groups is None:
|
|
244
|
+
stats[i] = metric(d[idx], b[idx])
|
|
245
|
+
else:
|
|
246
|
+
stats[i] = by_group(metric, d[idx], b[idx], groups[idx], min_group)[0]
|
|
247
|
+
return stats
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def evaluate(
|
|
251
|
+
predictions: pd.DataFrame,
|
|
252
|
+
observed: pd.DataFrame,
|
|
253
|
+
*,
|
|
254
|
+
match: Mapping[str, str] | None = None,
|
|
255
|
+
group: str | None = None,
|
|
256
|
+
cluster: str | None = "gene_id",
|
|
257
|
+
effect: str = "raw_effect",
|
|
258
|
+
forward_strand: bool = True,
|
|
259
|
+
n_boot: int = 1000,
|
|
260
|
+
seed: int = 0,
|
|
261
|
+
min_group: int = 2,
|
|
262
|
+
min_clusters: int = 5,
|
|
263
|
+
) -> Evaluation:
|
|
264
|
+
"""Measures sign AUROC, signed Spearman correlation, and sign accuracy.
|
|
265
|
+
|
|
266
|
+
Metrics use matched pairs with nonzero observed effects. Confidence
|
|
267
|
+
intervals are 95% percentiles over bootstrap replicates. They describe
|
|
268
|
+
resampling of the supplied clusters and do not model GTEx sampling.
|
|
269
|
+
|
|
270
|
+
Args:
|
|
271
|
+
predictions: Prediction table accepted by join.
|
|
272
|
+
observed: Observed-effects table accepted by join.
|
|
273
|
+
match: Additional keys mapping observed columns to prediction columns.
|
|
274
|
+
group: Observed column for group-size-weighted metrics, or None to pool.
|
|
275
|
+
cluster: Observed column defining bootstrap units. None resamples
|
|
276
|
+
individual pairs and can understate uncertainty for related pairs.
|
|
277
|
+
effect: Prediction column containing the signed effect.
|
|
278
|
+
forward_strand: Whether to require forward-strand REF agreement in join.
|
|
279
|
+
n_boot: Number of bootstrap resamples. Zero disables intervals.
|
|
280
|
+
seed: Random seed for bootstrapping.
|
|
281
|
+
min_group: Minimum pair count for a group to contribute to a weighted metric.
|
|
282
|
+
min_clusters: Minimum number of distinct clusters for an interval.
|
|
283
|
+
|
|
284
|
+
Returns:
|
|
285
|
+
Pair ledger, metrics with confidence intervals and counts, per-group
|
|
286
|
+
estimates if requested, and evaluation settings. Intervals are NaN
|
|
287
|
+
when disabled or when there are too few clusters or no finite replicates.
|
|
288
|
+
|
|
289
|
+
Raises:
|
|
290
|
+
ValueError: If a required join or cluster column is missing.
|
|
291
|
+
KeyError: If the requested group column is missing.
|
|
292
|
+
"""
|
|
293
|
+
pairs = join(predictions, observed, match=match, effect=effect, forward_strand=forward_strand)
|
|
294
|
+
usable = pairs[(pairs["pair_status"] == "ok") & (pairs["beta_alt"] != 0)]
|
|
295
|
+
d = usable["predicted"].to_numpy(float)
|
|
296
|
+
b = usable["beta_alt"].to_numpy(float)
|
|
297
|
+
if cluster is not None and cluster not in usable:
|
|
298
|
+
raise ValueError(f"cluster column {cluster!r} not in observed")
|
|
299
|
+
clusters = usable[cluster].astype(str).to_numpy() if cluster else np.arange(len(usable))
|
|
300
|
+
groups = usable[group].astype(str).to_numpy() if group else None
|
|
301
|
+
rng = np.random.default_rng(seed)
|
|
302
|
+
|
|
303
|
+
rows, per_group = [], []
|
|
304
|
+
for name, metric in METRICS.items():
|
|
305
|
+
n_used = len(d)
|
|
306
|
+
if groups is None:
|
|
307
|
+
estimate = metric(d, b)
|
|
308
|
+
else:
|
|
309
|
+
estimate, table = by_group(metric, d, b, groups, min_group)
|
|
310
|
+
per_group.append(table.assign(metric=name))
|
|
311
|
+
n_used = int(table.loc[(table["n"] >= min_group) & table["value"].notna(), "n"].sum())
|
|
312
|
+
enough = len(np.unique(clusters)) >= min_clusters
|
|
313
|
+
boots = (
|
|
314
|
+
_bootstrap(metric, d, b, clusters, groups, n_boot, rng, min_group)
|
|
315
|
+
if len(d) and n_boot and enough
|
|
316
|
+
else np.array([np.nan])
|
|
317
|
+
)
|
|
318
|
+
finite = boots[np.isfinite(boots)]
|
|
319
|
+
lo, hi = np.quantile(finite, [0.025, 0.975]) if finite.size else (np.nan, np.nan)
|
|
320
|
+
rows.append(
|
|
321
|
+
{
|
|
322
|
+
"metric": name,
|
|
323
|
+
"estimate": estimate,
|
|
324
|
+
"ci95_low": float(lo),
|
|
325
|
+
"ci95_high": float(hi),
|
|
326
|
+
"n_pairs": len(d),
|
|
327
|
+
"n_pairs_used": n_used,
|
|
328
|
+
"n_clusters": len(np.unique(clusters)),
|
|
329
|
+
"n_boot_finite": int(finite.size),
|
|
330
|
+
"weighting": f"by {group}" if group else "pooled",
|
|
331
|
+
}
|
|
332
|
+
)
|
|
333
|
+
settings = {
|
|
334
|
+
"match": dict(match or {}),
|
|
335
|
+
"group": group,
|
|
336
|
+
"cluster": cluster,
|
|
337
|
+
"effect": effect,
|
|
338
|
+
"forward_strand": forward_strand,
|
|
339
|
+
"n_boot": n_boot,
|
|
340
|
+
"seed": seed,
|
|
341
|
+
"min_group": min_group,
|
|
342
|
+
"min_clusters": min_clusters,
|
|
343
|
+
"n_observed": len(observed),
|
|
344
|
+
"n_predictions": len(predictions),
|
|
345
|
+
}
|
|
346
|
+
groups_table = (
|
|
347
|
+
pd.concat(per_group, ignore_index=True)
|
|
348
|
+
if per_group
|
|
349
|
+
else pd.DataFrame(columns=["group", "n", "value", "metric"])
|
|
350
|
+
)
|
|
351
|
+
return Evaluation(pairs, pd.DataFrame(rows), groups_table, settings)
|
ag2kit/gtex.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""Loads significant GTEx eQTL pairs from the Portal API or release files.
|
|
2
|
+
|
|
3
|
+
Both sources report normalized expression slopes for ALT dosage on GRCh38.
|
|
4
|
+
Use signs and ranks to compare these slopes with model scores. Restricting
|
|
5
|
+
evaluation to significant pairs favors strong effects (see docs/methods.md).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import time
|
|
13
|
+
from collections.abc import Iterable
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
import requests
|
|
18
|
+
|
|
19
|
+
from ag2kit.provenance import cache_dir, stable_hash, utc_now
|
|
20
|
+
|
|
21
|
+
log = logging.getLogger("ag2kit")
|
|
22
|
+
|
|
23
|
+
API = "https://gtexportal.org/api/v2"
|
|
24
|
+
SOURCE = "GTEx Portal API v2"
|
|
25
|
+
# GENCODE releases come from the Portal metadata for each GRCh38 dataset.
|
|
26
|
+
GENCODE = {"gtex_v8": "v26", "gtex_v10": "v39", "gtex_v11": "v47"}
|
|
27
|
+
COLUMNS = (
|
|
28
|
+
"variant_id",
|
|
29
|
+
"chrom",
|
|
30
|
+
"pos",
|
|
31
|
+
"ref",
|
|
32
|
+
"alt",
|
|
33
|
+
"source_id",
|
|
34
|
+
"rsid",
|
|
35
|
+
"gene_id",
|
|
36
|
+
"gencode_id",
|
|
37
|
+
"gene_symbol",
|
|
38
|
+
"tissue",
|
|
39
|
+
"ontology_id",
|
|
40
|
+
"beta",
|
|
41
|
+
"se",
|
|
42
|
+
"pvalue",
|
|
43
|
+
"effect_allele",
|
|
44
|
+
"effect_scale",
|
|
45
|
+
"assembly",
|
|
46
|
+
"dataset_id",
|
|
47
|
+
"source",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _dataset(dataset: str) -> str:
|
|
52
|
+
if dataset not in GENCODE:
|
|
53
|
+
raise ValueError(f"dataset must be one of {sorted(GENCODE)} (GRCh38 releases)")
|
|
54
|
+
return dataset
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _get(path: str, params: dict, refresh: bool = False) -> dict:
|
|
58
|
+
"""Fetches JSON with disk caching for successful responses.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
path: Endpoint relative to the GTEx API base URL.
|
|
62
|
+
params: Query parameters, also used in the cache key.
|
|
63
|
+
refresh: Whether to bypass the cache read.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
Decoded response body.
|
|
67
|
+
|
|
68
|
+
Raises:
|
|
69
|
+
RuntimeError: If the HTTP request fails after the applicable retries.
|
|
70
|
+
"""
|
|
71
|
+
cache = cache_dir() / "gtex" / f"{stable_hash({'path': path, **params})}.json"
|
|
72
|
+
if not refresh:
|
|
73
|
+
try:
|
|
74
|
+
return json.loads(cache.read_text())["response"]
|
|
75
|
+
except (OSError, ValueError, KeyError):
|
|
76
|
+
pass
|
|
77
|
+
for attempt in range(3):
|
|
78
|
+
try:
|
|
79
|
+
response = requests.get(f"{API}/{path}", params=params, timeout=60)
|
|
80
|
+
except requests.RequestException as exc:
|
|
81
|
+
error: str = str(exc)
|
|
82
|
+
else:
|
|
83
|
+
if response.ok:
|
|
84
|
+
body = response.json()
|
|
85
|
+
cache.parent.mkdir(parents=True, exist_ok=True)
|
|
86
|
+
payload = {"url": response.url, "retrieved_utc": utc_now(), "response": body}
|
|
87
|
+
tmp = cache.with_suffix(".tmp")
|
|
88
|
+
tmp.write_text(json.dumps(payload))
|
|
89
|
+
tmp.replace(cache)
|
|
90
|
+
return body
|
|
91
|
+
error = f"HTTP {response.status_code}: {response.text[:200]}"
|
|
92
|
+
if 400 <= response.status_code < 500 and response.status_code != 429:
|
|
93
|
+
break
|
|
94
|
+
time.sleep(2**attempt)
|
|
95
|
+
raise RuntimeError(f"GTEx API {path} {params} failed: {error}")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _pages(path: str, params: dict, refresh: bool) -> list[dict]:
|
|
99
|
+
rows: list[dict] = []
|
|
100
|
+
page = 0
|
|
101
|
+
while True:
|
|
102
|
+
body = _get(path, {**params, "page": page, "itemsPerPage": 1000}, refresh)
|
|
103
|
+
rows += body.get("data", [])
|
|
104
|
+
info = body.get("paging_info", {})
|
|
105
|
+
page += 1
|
|
106
|
+
if page >= int(info.get("numberOfPages", 1)):
|
|
107
|
+
return rows
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def gencode_id(gene: str, dataset: str = "gtex_v10", refresh: bool = False) -> str:
|
|
111
|
+
"""Resolves a gene to the GENCODE ID used by a GTEx release.
|
|
112
|
+
|
|
113
|
+
The Portal matches versioned IDs exactly, so a suffix from a different
|
|
114
|
+
GENCODE release may prevent a match.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
gene: Gene symbol or Ensembl ID.
|
|
118
|
+
dataset: Supported GTEx dataset ID.
|
|
119
|
+
refresh: Whether to bypass the cached response.
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
The versioned GENCODE ID.
|
|
123
|
+
|
|
124
|
+
Raises:
|
|
125
|
+
ValueError: If the dataset is unsupported or the match is absent or ambiguous.
|
|
126
|
+
RuntimeError: If the Portal request fails.
|
|
127
|
+
"""
|
|
128
|
+
body = _get(
|
|
129
|
+
"reference/gene",
|
|
130
|
+
{
|
|
131
|
+
"geneId": gene,
|
|
132
|
+
"gencodeVersion": GENCODE[_dataset(dataset)],
|
|
133
|
+
"genomeBuild": "GRCh38/hg38",
|
|
134
|
+
},
|
|
135
|
+
refresh,
|
|
136
|
+
)
|
|
137
|
+
hits = body.get("data", [])
|
|
138
|
+
if len(hits) != 1:
|
|
139
|
+
raise ValueError(f"{gene!r} matched {len(hits)} genes in GENCODE {GENCODE[dataset]}")
|
|
140
|
+
return str(hits[0]["gencodeId"])
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def tissues(dataset: str = "gtex_v10", refresh: bool = False) -> pd.DataFrame:
|
|
144
|
+
"""Lists tissue metadata for a GTEx release.
|
|
145
|
+
|
|
146
|
+
Args:
|
|
147
|
+
dataset: Supported GTEx dataset ID.
|
|
148
|
+
refresh: Whether to bypass the cached response.
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
A table of tissue metadata, including UBERON IDs and eGene counts.
|
|
152
|
+
|
|
153
|
+
Raises:
|
|
154
|
+
ValueError: If the dataset is unsupported.
|
|
155
|
+
RuntimeError: If the Portal request fails.
|
|
156
|
+
"""
|
|
157
|
+
body = _get("dataset/tissueSiteDetail", {"datasetId": _dataset(dataset)}, refresh)
|
|
158
|
+
return pd.DataFrame(body.get("data", body if isinstance(body, list) else []))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _split_variant(variant: pd.Series) -> pd.DataFrame:
|
|
162
|
+
parts = variant.astype(str).str.extract(r"^([^_]+)_(0*[1-9][0-9]*)_([^_]+)_([^_]+)_b38\Z")
|
|
163
|
+
invalid = parts[0].isna()
|
|
164
|
+
if invalid.any():
|
|
165
|
+
bad = variant[invalid].iloc[0]
|
|
166
|
+
raise ValueError(f"unexpected GTEx variant id {bad!r} (want chr_pos_ref_alt_b38)")
|
|
167
|
+
return pd.DataFrame(
|
|
168
|
+
{"chrom": parts[0], "pos": parts[1].astype("int64"), "ref": parts[2], "alt": parts[3]}
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _finish(df: pd.DataFrame, dataset: str, source: str) -> pd.DataFrame:
|
|
173
|
+
if df.empty:
|
|
174
|
+
return pd.DataFrame(columns=list(COLUMNS))
|
|
175
|
+
df = df.drop(columns=["chrom", "chromosome", "pos", "ref", "alt"], errors="ignore")
|
|
176
|
+
df = pd.concat([_split_variant(df["source_id"]), df], axis=1)
|
|
177
|
+
df["variant_id"] = df["chrom"] + ":" + df["pos"].astype(str) + ":" + df["ref"] + ">" + df["alt"]
|
|
178
|
+
df["gene_id"] = df["gencode_id"].str.split(".").str[0]
|
|
179
|
+
df["effect_allele"] = df["alt"]
|
|
180
|
+
df["effect_scale"] = "normalized_expression_slope"
|
|
181
|
+
df["assembly"] = "GRCh38"
|
|
182
|
+
df["dataset_id"] = dataset
|
|
183
|
+
df["source"] = source
|
|
184
|
+
for col in COLUMNS:
|
|
185
|
+
if col not in df:
|
|
186
|
+
df[col] = pd.NA
|
|
187
|
+
return df.loc[:, list(COLUMNS)].reset_index(drop=True)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def eqtls(
|
|
191
|
+
genes: Iterable[str],
|
|
192
|
+
tissue: str,
|
|
193
|
+
dataset: str = "gtex_v10",
|
|
194
|
+
refresh: bool = False,
|
|
195
|
+
) -> pd.DataFrame:
|
|
196
|
+
"""Fetches significant single-tissue eQTLs from the GTEx Portal.
|
|
197
|
+
|
|
198
|
+
Args:
|
|
199
|
+
genes: Gene symbols or unversioned Ensembl IDs to resolve. IDs with
|
|
200
|
+
a version suffix are passed through and must match the dataset.
|
|
201
|
+
tissue: GTEx tissue ID, such as "Whole_Blood".
|
|
202
|
+
dataset: Supported GTEx dataset ID.
|
|
203
|
+
refresh: Whether to bypass cached responses.
|
|
204
|
+
|
|
205
|
+
Returns:
|
|
206
|
+
An observed-effects table with the columns in COLUMNS. Beta is the
|
|
207
|
+
ALT-oriented normalized effect size (nes). The API leaves se missing.
|
|
208
|
+
|
|
209
|
+
Raises:
|
|
210
|
+
ValueError: If the dataset is unsupported, a gene cannot be resolved
|
|
211
|
+
uniquely, or returned variant IDs are invalid.
|
|
212
|
+
RuntimeError: If a Portal request fails.
|
|
213
|
+
"""
|
|
214
|
+
_dataset(dataset)
|
|
215
|
+
frames = []
|
|
216
|
+
for gene in genes:
|
|
217
|
+
gid = gene if "." in str(gene) else gencode_id(str(gene), dataset, refresh)
|
|
218
|
+
rows = _pages(
|
|
219
|
+
"association/singleTissueEqtl",
|
|
220
|
+
{"gencodeId": gid, "tissueSiteDetailId": tissue, "datasetId": dataset},
|
|
221
|
+
refresh,
|
|
222
|
+
)
|
|
223
|
+
if not rows:
|
|
224
|
+
log.info("no significant %s eQTLs for %s in %s", dataset, gid, tissue)
|
|
225
|
+
frames.append(pd.DataFrame(rows))
|
|
226
|
+
raw = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
|
|
227
|
+
df = raw.rename(
|
|
228
|
+
columns={
|
|
229
|
+
"variantId": "source_id",
|
|
230
|
+
"snpId": "rsid",
|
|
231
|
+
"gencodeId": "gencode_id",
|
|
232
|
+
"geneSymbol": "gene_symbol",
|
|
233
|
+
"tissueSiteDetailId": "tissue",
|
|
234
|
+
"ontologyId": "ontology_id",
|
|
235
|
+
"nes": "beta",
|
|
236
|
+
"pValue": "pvalue",
|
|
237
|
+
}
|
|
238
|
+
)
|
|
239
|
+
return _finish(df, dataset, SOURCE)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def read_signif_pairs(path: str | Path, tissue: str, dataset: str) -> pd.DataFrame:
|
|
243
|
+
"""Loads significant GTEx variant-gene pairs from a tissue release file.
|
|
244
|
+
|
|
245
|
+
Args:
|
|
246
|
+
path: Per-tissue Parquet file (v10/v11) or tab-delimited file (v8).
|
|
247
|
+
tissue: Tissue ID to assign to every row.
|
|
248
|
+
dataset: Supported GTEx dataset ID.
|
|
249
|
+
|
|
250
|
+
Returns:
|
|
251
|
+
An observed-effects table as in eqtls, with beta from slope, se
|
|
252
|
+
from slope_se when present, and pvalue from pval_nominal.
|
|
253
|
+
|
|
254
|
+
Raises:
|
|
255
|
+
ValueError: If the dataset is unsupported, required columns are missing,
|
|
256
|
+
or variant IDs are invalid.
|
|
257
|
+
OSError: If the file cannot be read.
|
|
258
|
+
"""
|
|
259
|
+
path = Path(path)
|
|
260
|
+
_dataset(dataset)
|
|
261
|
+
raw = pd.read_parquet(path) if path.suffix == ".parquet" else pd.read_csv(path, sep="\t")
|
|
262
|
+
missing = {"variant_id", "gene_id", "slope", "pval_nominal"} - set(raw.columns)
|
|
263
|
+
if missing:
|
|
264
|
+
raise ValueError(f"{path.name} lacks GTEx columns {sorted(missing)}")
|
|
265
|
+
df = raw.rename(
|
|
266
|
+
columns={
|
|
267
|
+
"variant_id": "source_id",
|
|
268
|
+
"gene_id": "gencode_id",
|
|
269
|
+
"slope": "beta",
|
|
270
|
+
"slope_se": "se",
|
|
271
|
+
"pval_nominal": "pvalue",
|
|
272
|
+
}
|
|
273
|
+
)
|
|
274
|
+
df["tissue"] = tissue
|
|
275
|
+
return _finish(df, dataset, f"GTEx {dataset} download: {path.name}")
|