ag2kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ag2kit/__init__.py +65 -0
- ag2kit/client.py +940 -0
- ag2kit/evaluate.py +351 -0
- ag2kit/gtex.py +275 -0
- ag2kit/io.py +248 -0
- ag2kit/provenance.py +316 -0
- ag2kit/py.typed +0 -0
- ag2kit/stats.py +375 -0
- ag2kit/variants.py +241 -0
- ag2kit-0.1.0.dist-info/METADATA +211 -0
- ag2kit-0.1.0.dist-info/RECORD +13 -0
- ag2kit-0.1.0.dist-info/WHEEL +4 -0
- ag2kit-0.1.0.dist-info/licenses/LICENSE +21 -0
ag2kit/client.py
ADDED
|
@@ -0,0 +1,940 @@
|
|
|
1
|
+
"""AlphaGenome scoring with request-level caching and gene-level reduction."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import importlib.metadata
|
|
7
|
+
import importlib.util
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
15
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
16
|
+
from dataclasses import asdict, dataclass, field, replace
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import pandas as pd
|
|
22
|
+
import pyarrow as pa
|
|
23
|
+
import pyarrow.parquet as pq
|
|
24
|
+
|
|
25
|
+
from ag2kit.provenance import cache_dir, stable_hash, utc_now
|
|
26
|
+
|
|
27
|
+
log = logging.getLogger("ag2kit")
|
|
28
|
+
|
|
29
|
+
API_KEY_ENV = "ALPHA_GENOME_API_KEY"
|
|
30
|
+
DEFAULT_SEQUENCE_LENGTH = 2**20
|
|
31
|
+
SUPPORTED_SEQUENCE_LENGTHS = frozenset({2**14, 2**17, 2**19, 2**20})
|
|
32
|
+
MAX_CONDITIONING_CELLS = 256
|
|
33
|
+
SOURCE = "alphagenome"
|
|
34
|
+
_ALLELE = re.compile(r"^[ACGTN]+$")
|
|
35
|
+
_CURIES = re.compile(r"^[A-Za-z]+:[A-Za-z0-9.]+(?:_[A-Za-z]+:[A-Za-z0-9.]+)*$")
|
|
36
|
+
_CACHE_FORMAT = 2
|
|
37
|
+
|
|
38
|
+
_ERROR_TYPES = {
|
|
39
|
+
"RESOURCE_EXHAUSTED": "quota",
|
|
40
|
+
"UNAVAILABLE": "transient",
|
|
41
|
+
"DEADLINE_EXCEEDED": "transient",
|
|
42
|
+
"ABORTED": "transient",
|
|
43
|
+
"INVALID_ARGUMENT": "validation",
|
|
44
|
+
"FAILED_PRECONDITION": "validation",
|
|
45
|
+
"OUT_OF_RANGE": "validation",
|
|
46
|
+
"NOT_FOUND": "validation",
|
|
47
|
+
"UNIMPLEMENTED": "validation",
|
|
48
|
+
"UNAUTHENTICATED": "credentials",
|
|
49
|
+
"PERMISSION_DENIED": "credentials",
|
|
50
|
+
}
|
|
51
|
+
_RETRYABLE = frozenset({"quota", "transient", "network"})
|
|
52
|
+
|
|
53
|
+
Tracks = Mapping[str, str | Sequence[str]]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class AG2KitError(Exception):
|
|
57
|
+
"""Base class for ag2kit errors."""
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class CredentialsError(AG2KitError):
|
|
61
|
+
"""The API key is missing or was rejected."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class ScoringError(AG2KitError):
|
|
65
|
+
"""Reports a failed or blocked scoring request in strict mode.
|
|
66
|
+
|
|
67
|
+
Attributes:
|
|
68
|
+
error_type: Category of the failure.
|
|
69
|
+
record: Scoring record that would otherwise have been returned.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
def __init__(self, message: str, error_type: str, record: dict):
|
|
73
|
+
"""Initializes the exception with its scoring context.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
message: Human-readable error message.
|
|
77
|
+
error_type: Failure category.
|
|
78
|
+
record: Failed or blocked scoring record.
|
|
79
|
+
"""
|
|
80
|
+
super().__init__(message)
|
|
81
|
+
self.error_type = error_type
|
|
82
|
+
self.record = record
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(frozen=True)
|
|
86
|
+
class RequestSpec:
|
|
87
|
+
"""Specifies one GRCh38 variant under one reference or cell condition.
|
|
88
|
+
|
|
89
|
+
Use exactly one condition input. Reference conditions use condition,
|
|
90
|
+
typically with RNA_SEQ. Cell conditioning uses a positive n_cells and
|
|
91
|
+
a condition_label. Pass the actual AnnData to the scoring method.
|
|
92
|
+
Construction raises ValueError for invalid coordinates, alleles, sequence
|
|
93
|
+
length, cell count, organism, or condition settings.
|
|
94
|
+
|
|
95
|
+
Attributes:
|
|
96
|
+
variant_id: Caller-provided variant label.
|
|
97
|
+
chrom: Chromosome name, normalized to include the chr prefix.
|
|
98
|
+
pos: 1-based position of the first REF base.
|
|
99
|
+
ref: Forward-strand reference allele, stored in uppercase.
|
|
100
|
+
alt: Forward-strand alternate allele, stored in uppercase.
|
|
101
|
+
gene_id: Gene to reduce to a single effect. Empty skips gene reduction.
|
|
102
|
+
gene_symbol: Caller-provided gene label.
|
|
103
|
+
condition_label: Cell-condition name used by the SDK to name output tracks.
|
|
104
|
+
conditioning_set_id: Caller-provided label for the conditioning set.
|
|
105
|
+
n_cells: Number of conditioning cells, or zero for reference conditions.
|
|
106
|
+
seed: Caller-provided seed label for the conditioning draw.
|
|
107
|
+
model_version: AlphaGenome model version.
|
|
108
|
+
scorer_name: SDK recommended scorer name.
|
|
109
|
+
sequence_length: Supported input window length in bases.
|
|
110
|
+
organism: SDK organism name. Only HOMO_SAPIENS is supported.
|
|
111
|
+
condition: Reference condition as one or more ontology CURIEs.
|
|
112
|
+
cell_conditioned: Whether the spec uses cell conditioning.
|
|
113
|
+
variant_string: Variant in SDK-compatible chrom:pos:ref>alt form.
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
variant_id: str
|
|
117
|
+
chrom: str
|
|
118
|
+
pos: int
|
|
119
|
+
ref: str
|
|
120
|
+
alt: str
|
|
121
|
+
gene_id: str = ""
|
|
122
|
+
gene_symbol: str = ""
|
|
123
|
+
condition_label: str = ""
|
|
124
|
+
conditioning_set_id: str = ""
|
|
125
|
+
n_cells: int = 0
|
|
126
|
+
seed: int = 0
|
|
127
|
+
model_version: str = "ALL_FOLDS"
|
|
128
|
+
scorer_name: str = "SC_GENE_EXPRESSION_DENOISED"
|
|
129
|
+
sequence_length: int = DEFAULT_SEQUENCE_LENGTH
|
|
130
|
+
organism: str = "HOMO_SAPIENS"
|
|
131
|
+
condition: str = ""
|
|
132
|
+
|
|
133
|
+
def __post_init__(self) -> None:
|
|
134
|
+
object.__setattr__(self, "chrom", "chr" + str(self.chrom).removeprefix("chr"))
|
|
135
|
+
object.__setattr__(self, "ref", str(self.ref).upper())
|
|
136
|
+
object.__setattr__(self, "alt", str(self.alt).upper())
|
|
137
|
+
if isinstance(self.pos, bool) or int(self.pos) != self.pos or self.pos < 1:
|
|
138
|
+
raise ValueError(f"pos must be a 1-based integer position, got {self.pos!r}")
|
|
139
|
+
object.__setattr__(self, "pos", int(self.pos))
|
|
140
|
+
for name in ("ref", "alt"):
|
|
141
|
+
if not _ALLELE.match(getattr(self, name)):
|
|
142
|
+
raise ValueError(f"{name} must be a non-empty A/C/G/T/N string")
|
|
143
|
+
if self.ref == self.alt:
|
|
144
|
+
raise ValueError("ref and alt are identical")
|
|
145
|
+
if self.sequence_length not in SUPPORTED_SEQUENCE_LENGTHS:
|
|
146
|
+
supported = sorted(SUPPORTED_SEQUENCE_LENGTHS)
|
|
147
|
+
raise ValueError(f"sequence_length {self.sequence_length} not in {supported}")
|
|
148
|
+
if not 0 <= self.n_cells <= MAX_CONDITIONING_CELLS:
|
|
149
|
+
raise ValueError(f"n_cells must be between 0 and {MAX_CONDITIONING_CELLS}")
|
|
150
|
+
if self.organism != "HOMO_SAPIENS":
|
|
151
|
+
raise ValueError("only HOMO_SAPIENS is supported")
|
|
152
|
+
if self.condition and not _CURIES.match(self.condition):
|
|
153
|
+
raise ValueError(
|
|
154
|
+
f"condition must be ontology CURIE(s) like UBERON:0013756, got {self.condition!r}"
|
|
155
|
+
)
|
|
156
|
+
if bool(self.condition) == self.cell_conditioned:
|
|
157
|
+
raise ValueError(
|
|
158
|
+
"set exactly one of condition (ontology CURIE) or n_cells > 0 (cell conditioning)"
|
|
159
|
+
)
|
|
160
|
+
if self.cell_conditioned and not self.condition_label:
|
|
161
|
+
raise ValueError("cell conditioning needs a condition_label")
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def cell_conditioned(self) -> bool:
|
|
165
|
+
"""Whether the request uses cell conditioning."""
|
|
166
|
+
return self.n_cells > 0
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def variant_string(self) -> str:
|
|
170
|
+
"""Variant string accepted by the SDK variant parser."""
|
|
171
|
+
return f"{self.chrom}:{self.pos}:{self.ref}>{self.alt}"
|
|
172
|
+
|
|
173
|
+
def interval(self) -> tuple[str, int, int]:
|
|
174
|
+
"""Builds the input window around the reference allele span.
|
|
175
|
+
|
|
176
|
+
Near the chromosome start, the window shifts right to start at zero.
|
|
177
|
+
The API rejects windows that extend beyond the chromosome end.
|
|
178
|
+
|
|
179
|
+
Returns:
|
|
180
|
+
Chromosome, zero-based inclusive start, and exclusive end.
|
|
181
|
+
"""
|
|
182
|
+
centre = self.pos - 1 + (len(self.ref) + 1) // 2
|
|
183
|
+
start = max(0, centre - self.sequence_length // 2)
|
|
184
|
+
return (self.chrom, start, start + self.sequence_length)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def sdk_version() -> str:
|
|
188
|
+
"""Returns the installed AlphaGenome SDK version or "not_installed"."""
|
|
189
|
+
try:
|
|
190
|
+
return importlib.metadata.version("alphagenome")
|
|
191
|
+
except importlib.metadata.PackageNotFoundError:
|
|
192
|
+
return "not_installed"
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def api_key_available() -> bool:
|
|
196
|
+
"""Checks whether the API key environment variable contains a nonblank value."""
|
|
197
|
+
return bool(os.environ.get(API_KEY_ENV, "").strip())
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def sdk_available() -> bool:
|
|
201
|
+
"""Checks whether Python can locate the AlphaGenome SDK."""
|
|
202
|
+
return importlib.util.find_spec("alphagenome") is not None
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def classify_error(exc: BaseException) -> str:
|
|
206
|
+
"""Categorizes scoring errors for records and retry decisions.
|
|
207
|
+
|
|
208
|
+
Args:
|
|
209
|
+
exc: Exception from the SDK or a custom model.
|
|
210
|
+
|
|
211
|
+
Returns:
|
|
212
|
+
An error category, using the RPC status when available and
|
|
213
|
+
"upstream" for unrecognized failures.
|
|
214
|
+
"""
|
|
215
|
+
code = getattr(exc, "code", None)
|
|
216
|
+
if callable(code):
|
|
217
|
+
try:
|
|
218
|
+
name = getattr(code(), "name", "")
|
|
219
|
+
except Exception:
|
|
220
|
+
name = ""
|
|
221
|
+
if name:
|
|
222
|
+
return _ERROR_TYPES.get(name, "upstream")
|
|
223
|
+
if isinstance(exc, PermissionError):
|
|
224
|
+
return "credentials"
|
|
225
|
+
if isinstance(exc, ValueError | KeyError | IndexError | NotImplementedError):
|
|
226
|
+
return "validation"
|
|
227
|
+
if isinstance(exc, ConnectionError | TimeoutError):
|
|
228
|
+
return "network"
|
|
229
|
+
text = str(exc).upper()
|
|
230
|
+
return next((kind for name, kind in _ERROR_TYPES.items() if name in text), "upstream")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _redact(text: str) -> str:
|
|
234
|
+
key = os.environ.get(API_KEY_ENV, "").strip()
|
|
235
|
+
return text.replace(key, "<redacted>") if key else text
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@dataclass
|
|
239
|
+
class _Outcome:
|
|
240
|
+
"""Stores a request result with status and cache metadata."""
|
|
241
|
+
|
|
242
|
+
tidy: pd.DataFrame | None
|
|
243
|
+
status: str
|
|
244
|
+
error: str = ""
|
|
245
|
+
error_type: str = ""
|
|
246
|
+
attempts: int = 0
|
|
247
|
+
from_cache: bool = False
|
|
248
|
+
requested_utc: str = ""
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@dataclass
|
|
252
|
+
class AG2Client:
|
|
253
|
+
"""Scores request specs and caches full scorer outputs on disk.
|
|
254
|
+
|
|
255
|
+
One SDK connection is shared across worker threads. The client retries
|
|
256
|
+
quota, transient, and network failures in addition to the SDK's own retries.
|
|
257
|
+
The API does not report a model revision, so use a new namespace after a
|
|
258
|
+
server-side change. Custom models are identified by class, so give
|
|
259
|
+
differently trained instances of the same class distinct namespaces.
|
|
260
|
+
|
|
261
|
+
Attributes:
|
|
262
|
+
dry_run: Whether to validate and plan requests without live scoring.
|
|
263
|
+
cache_dir: Directory for cached scores, overridden by AG2KIT_AG2_CACHE.
|
|
264
|
+
model_version: SDK model version required by requests on this client.
|
|
265
|
+
max_retries: Maximum attempts per request, including the initial attempt.
|
|
266
|
+
Construction raises ValueError for values below one.
|
|
267
|
+
backoff: Seconds to wait per attempt number before retrying.
|
|
268
|
+
sdk_version: SDK version recorded in request identity and provenance.
|
|
269
|
+
cache_namespace: Label for isolating caches across model revisions.
|
|
270
|
+
model: Optional backend implementing the SDK score_variant interface.
|
|
271
|
+
Live API connections are assigned here on first use.
|
|
272
|
+
"""
|
|
273
|
+
|
|
274
|
+
dry_run: bool = True
|
|
275
|
+
cache_dir: Path = field(default_factory=lambda: cache_dir() / "ag2")
|
|
276
|
+
model_version: str = "ALL_FOLDS"
|
|
277
|
+
max_retries: int = 2
|
|
278
|
+
backoff: float = 30.0
|
|
279
|
+
sdk_version: str = field(default_factory=sdk_version)
|
|
280
|
+
cache_namespace: str = ""
|
|
281
|
+
model: Any = None
|
|
282
|
+
|
|
283
|
+
def __post_init__(self) -> None:
|
|
284
|
+
self.cache_dir = Path(os.environ.get("AG2KIT_AG2_CACHE") or self.cache_dir)
|
|
285
|
+
if self.max_retries < 1:
|
|
286
|
+
raise ValueError("max_retries must be at least 1")
|
|
287
|
+
self._custom_model = self.model is not None
|
|
288
|
+
self._connect_lock = threading.Lock()
|
|
289
|
+
|
|
290
|
+
def connect(self) -> None:
|
|
291
|
+
"""Initializes the live client if needed.
|
|
292
|
+
|
|
293
|
+
Called lazily when scoring. Dry runs and clients with an existing backend
|
|
294
|
+
skip initialization.
|
|
295
|
+
|
|
296
|
+
Raises:
|
|
297
|
+
CredentialsError: If live connection is needed and the key is missing.
|
|
298
|
+
ImportError: If the AlphaGenome SDK is unavailable.
|
|
299
|
+
AttributeError: If model_version is unknown to the SDK.
|
|
300
|
+
"""
|
|
301
|
+
with self._connect_lock:
|
|
302
|
+
if self.model is not None or self.dry_run:
|
|
303
|
+
return
|
|
304
|
+
if not api_key_available():
|
|
305
|
+
raise CredentialsError(f"{API_KEY_ENV} is not set; live scoring is unavailable")
|
|
306
|
+
from alphagenome.models.v2 import dna_client
|
|
307
|
+
|
|
308
|
+
version = getattr(dna_client.ModelVersion, self.model_version)
|
|
309
|
+
self.model = dna_client.create(os.environ[API_KEY_ENV].strip(), model_version=version)
|
|
310
|
+
|
|
311
|
+
def recommended_scorer(self, spec: RequestSpec) -> Any:
|
|
312
|
+
"""Resolves the SDK scorer and checks compatibility with the request.
|
|
313
|
+
|
|
314
|
+
Args:
|
|
315
|
+
spec: Request with the scorer name and conditioning mode.
|
|
316
|
+
|
|
317
|
+
Returns:
|
|
318
|
+
The SDK scorer object.
|
|
319
|
+
|
|
320
|
+
Raises:
|
|
321
|
+
ImportError: If the AlphaGenome SDK is unavailable.
|
|
322
|
+
KeyError: If the scorer name is unknown.
|
|
323
|
+
ValueError: If the scorer is unsigned or requires cell conditioning
|
|
324
|
+
and the request uses a reference condition.
|
|
325
|
+
"""
|
|
326
|
+
from alphagenome.models.v2 import variant_scorers
|
|
327
|
+
|
|
328
|
+
scorers = variant_scorers.RECOMMENDED_VARIANT_SCORERS
|
|
329
|
+
if spec.scorer_name not in scorers:
|
|
330
|
+
raise KeyError(
|
|
331
|
+
f"{spec.scorer_name!r} not in RECOMMENDED_VARIANT_SCORERS: {sorted(scorers)}"
|
|
332
|
+
)
|
|
333
|
+
scorer = scorers[spec.scorer_name]
|
|
334
|
+
if not getattr(scorer, "is_signed", True):
|
|
335
|
+
raise ValueError(f"scorer {spec.scorer_name!r} is unsigned; ag2kit needs ALT-REF")
|
|
336
|
+
cell_only: frozenset = variant_scorers.CELL_CONDITIONING_ONLY_OUTPUT_TYPES
|
|
337
|
+
if not spec.cell_conditioned and getattr(scorer, "requested_output", None) in cell_only:
|
|
338
|
+
raise ValueError(f"scorer {spec.scorer_name!r} needs cell conditioning")
|
|
339
|
+
return scorer
|
|
340
|
+
|
|
341
|
+
def scorer_definition(self, spec: RequestSpec) -> str:
|
|
342
|
+
"""Describes the scorer used for request identity.
|
|
343
|
+
|
|
344
|
+
Args:
|
|
345
|
+
spec: Request whose scorer to describe.
|
|
346
|
+
|
|
347
|
+
Returns:
|
|
348
|
+
The SDK scorer representation, or its name if the SDK or a
|
|
349
|
+
compatible scorer is unavailable.
|
|
350
|
+
"""
|
|
351
|
+
if not sdk_available():
|
|
352
|
+
return spec.scorer_name
|
|
353
|
+
try:
|
|
354
|
+
return repr(self.recommended_scorer(spec))
|
|
355
|
+
except (KeyError, ValueError):
|
|
356
|
+
return spec.scorer_name
|
|
357
|
+
|
|
358
|
+
def _backend(self) -> str:
|
|
359
|
+
if not self._custom_model:
|
|
360
|
+
return SOURCE
|
|
361
|
+
cls = type(self.model)
|
|
362
|
+
return f"custom:{cls.__module__}.{cls.__qualname__}"
|
|
363
|
+
|
|
364
|
+
def request_identity(self, spec: RequestSpec, conditioning: Any = None) -> dict:
|
|
365
|
+
"""Builds the identity shared by requests for the same full scorer output.
|
|
366
|
+
|
|
367
|
+
Gene selection and provenance labels are excluded. The cell-condition
|
|
368
|
+
name participates because it affects output track names.
|
|
369
|
+
|
|
370
|
+
Args:
|
|
371
|
+
spec: Request to identify.
|
|
372
|
+
conditioning: Cell-conditioning AnnData, or None for reference conditions.
|
|
373
|
+
|
|
374
|
+
Returns:
|
|
375
|
+
A dictionary of request inputs and cache provenance.
|
|
376
|
+
"""
|
|
377
|
+
chrom, start, end = spec.interval()
|
|
378
|
+
return {
|
|
379
|
+
"cache_format": _CACHE_FORMAT,
|
|
380
|
+
"variant": spec.variant_string,
|
|
381
|
+
"interval": f"{chrom}:{start}-{end}",
|
|
382
|
+
"model_version": spec.model_version,
|
|
383
|
+
"organism": spec.organism,
|
|
384
|
+
"condition": spec.condition,
|
|
385
|
+
"cell_condition_name": spec.condition_label if spec.cell_conditioned else "",
|
|
386
|
+
"conditioning_sha256": conditioning_fingerprint(conditioning),
|
|
387
|
+
"scorer_definition": self.scorer_definition(spec),
|
|
388
|
+
"backend": self._backend(),
|
|
389
|
+
"sdk_version": self.sdk_version,
|
|
390
|
+
"cache_namespace": self.cache_namespace,
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
def cache_key(self, spec: RequestSpec, conditioning: Any = None) -> str:
|
|
394
|
+
"""Hashes the request identity for cache lookup.
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
spec: Request to identify.
|
|
398
|
+
conditioning: Cell-conditioning AnnData, or None for reference conditions.
|
|
399
|
+
|
|
400
|
+
Returns:
|
|
401
|
+
SHA-256 digest of request_identity.
|
|
402
|
+
"""
|
|
403
|
+
return stable_hash(self.request_identity(spec, conditioning))
|
|
404
|
+
|
|
405
|
+
def _cache_path(self, key: str) -> Path:
|
|
406
|
+
return self.cache_dir / f"{key}.parquet"
|
|
407
|
+
|
|
408
|
+
def _read_cache(self, key: str) -> tuple[pd.DataFrame, dict] | None:
|
|
409
|
+
try:
|
|
410
|
+
table = pq.read_table(self._cache_path(key))
|
|
411
|
+
meta = json.loads((table.schema.metadata or {})[b"ag2kit"])
|
|
412
|
+
except (OSError, KeyError, ValueError, pa.ArrowException):
|
|
413
|
+
return None
|
|
414
|
+
if meta.get("cache_key") != key or table.num_rows == 0:
|
|
415
|
+
return None
|
|
416
|
+
return table.to_pandas(), meta
|
|
417
|
+
|
|
418
|
+
def _write_cache(self, key: str, tidy: pd.DataFrame, meta: dict) -> None:
|
|
419
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
420
|
+
path = self._cache_path(key)
|
|
421
|
+
table = pa.Table.from_pandas(tidy, preserve_index=False)
|
|
422
|
+
table = table.replace_schema_metadata(
|
|
423
|
+
{**(table.schema.metadata or {}), b"ag2kit": json.dumps(meta, sort_keys=True)}
|
|
424
|
+
)
|
|
425
|
+
tmp = path.with_suffix(f".{os.getpid()}.{threading.get_ident()}.tmp")
|
|
426
|
+
pq.write_table(table, tmp)
|
|
427
|
+
tmp.replace(path)
|
|
428
|
+
|
|
429
|
+
def raw_scores(self, spec: RequestSpec, conditioning: Any = None) -> pd.DataFrame | None:
|
|
430
|
+
"""Reads the full cached scorer output without calling the API.
|
|
431
|
+
|
|
432
|
+
Args:
|
|
433
|
+
spec: Request to look up. Gene selection does not affect the lookup.
|
|
434
|
+
conditioning: Cell-conditioning AnnData used by the request, if any.
|
|
435
|
+
|
|
436
|
+
Returns:
|
|
437
|
+
Tidy gene-by-track scores for all strands, including raw_score and
|
|
438
|
+
quantile_score when supplied by the SDK, or None on a cache miss.
|
|
439
|
+
"""
|
|
440
|
+
hit = self._read_cache(self.cache_key(spec, conditioning))
|
|
441
|
+
return None if hit is None else hit[0]
|
|
442
|
+
|
|
443
|
+
def _base(self, spec: RequestSpec, conditioning: Any) -> dict:
|
|
444
|
+
return {
|
|
445
|
+
**asdict(spec),
|
|
446
|
+
"variant": spec.variant_string,
|
|
447
|
+
"source": self._backend(),
|
|
448
|
+
"cache_key": self.cache_key(spec, conditioning),
|
|
449
|
+
"conditioning_sha256": conditioning_fingerprint(conditioning),
|
|
450
|
+
"sdk_version": self.sdk_version,
|
|
451
|
+
"scorer_definition": self.scorer_definition(spec),
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
def plan(self, spec: RequestSpec, conditioning: Any = None) -> dict:
|
|
455
|
+
"""Describes a scoring request without contacting the API.
|
|
456
|
+
|
|
457
|
+
Args:
|
|
458
|
+
spec: Request to describe.
|
|
459
|
+
conditioning: Cell-conditioning AnnData, or None for reference conditions.
|
|
460
|
+
|
|
461
|
+
Returns:
|
|
462
|
+
Request metadata, interval, zero-based offset of the first REF base
|
|
463
|
+
within the interval, and whether the expected cache file exists.
|
|
464
|
+
"""
|
|
465
|
+
chrom, start, end = spec.interval()
|
|
466
|
+
return {
|
|
467
|
+
**self._base(spec, conditioning),
|
|
468
|
+
"interval": f"{chrom}:{start}-{end}",
|
|
469
|
+
"variant_offset_in_interval": spec.pos - 1 - start,
|
|
470
|
+
"cached": self._cache_path(self.cache_key(spec, conditioning)).exists(),
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
def cached(self, spec: RequestSpec, conditioning: Any = None) -> dict | None:
|
|
474
|
+
"""Reads a gene-level scoring record from the cache.
|
|
475
|
+
|
|
476
|
+
Args:
|
|
477
|
+
spec: Request and optional gene selection.
|
|
478
|
+
conditioning: Cell-conditioning AnnData used by the request, if any.
|
|
479
|
+
|
|
480
|
+
Returns:
|
|
481
|
+
A scoring record, or None on a cache miss.
|
|
482
|
+
"""
|
|
483
|
+
hit = self._read_cache(self.cache_key(spec, conditioning))
|
|
484
|
+
if hit is None:
|
|
485
|
+
return None
|
|
486
|
+
tidy, meta = hit
|
|
487
|
+
out = _Outcome(tidy, "completed", from_cache=True, requested_utc=meta["requested_utc"])
|
|
488
|
+
return self._record(spec, conditioning, out)
|
|
489
|
+
|
|
490
|
+
def _check(self, spec: RequestSpec, conditioning: Any) -> str:
|
|
491
|
+
n_obs = getattr(conditioning, "n_obs", None)
|
|
492
|
+
if spec.cell_conditioned and conditioning is None:
|
|
493
|
+
return "cell-conditioned spec but no conditioning AnnData was given"
|
|
494
|
+
if not spec.cell_conditioned and conditioning is not None:
|
|
495
|
+
return "conditioning AnnData given for a spec with a reference condition"
|
|
496
|
+
if n_obs is not None and n_obs != spec.n_cells:
|
|
497
|
+
return f"spec.n_cells={spec.n_cells} but conditioning has {n_obs} cells"
|
|
498
|
+
if sdk_available():
|
|
499
|
+
try:
|
|
500
|
+
self.recommended_scorer(spec)
|
|
501
|
+
except (KeyError, ValueError) as exc:
|
|
502
|
+
return str(exc).strip("'\"")
|
|
503
|
+
if spec.model_version != self.model_version:
|
|
504
|
+
return (
|
|
505
|
+
f"spec.model_version {spec.model_version!r} differs from the client's "
|
|
506
|
+
f"{self.model_version!r}; use one client per model version"
|
|
507
|
+
)
|
|
508
|
+
return ""
|
|
509
|
+
|
|
510
|
+
def _obtain(self, spec: RequestSpec, conditioning: Any) -> _Outcome:
|
|
511
|
+
if problem := self._check(spec, conditioning):
|
|
512
|
+
return _Outcome(None, "failed", problem, "validation")
|
|
513
|
+
if self.dry_run:
|
|
514
|
+
return _Outcome(None, "not_run", "dry_run")
|
|
515
|
+
key = self.cache_key(spec, conditioning)
|
|
516
|
+
if (hit := self._read_cache(key)) is not None:
|
|
517
|
+
return _Outcome(
|
|
518
|
+
hit[0], "completed", from_cache=True, requested_utc=hit[1]["requested_utc"]
|
|
519
|
+
)
|
|
520
|
+
if not sdk_available():
|
|
521
|
+
return _Outcome(
|
|
522
|
+
None,
|
|
523
|
+
"blocked",
|
|
524
|
+
"alphagenome SDK not installed (pip install 'ag2kit[model]')",
|
|
525
|
+
"blocked",
|
|
526
|
+
)
|
|
527
|
+
if not self._custom_model and not api_key_available():
|
|
528
|
+
return _Outcome(None, "blocked", f"{API_KEY_ENV} not set", "blocked")
|
|
529
|
+
|
|
530
|
+
error, error_type, attempt = "", "upstream", 0
|
|
531
|
+
for attempt in range(1, self.max_retries + 1):
|
|
532
|
+
requested = utc_now()
|
|
533
|
+
try:
|
|
534
|
+
tidy = self._call(spec, conditioning)
|
|
535
|
+
except Exception as exc:
|
|
536
|
+
error = _redact(f"{type(exc).__name__}: {exc}")
|
|
537
|
+
error_type = classify_error(exc)
|
|
538
|
+
if error_type not in _RETRYABLE or attempt == self.max_retries:
|
|
539
|
+
break
|
|
540
|
+
log.info(
|
|
541
|
+
"retrying %s after %s (attempt %d)", spec.variant_string, error_type, attempt
|
|
542
|
+
)
|
|
543
|
+
time.sleep(self.backoff * attempt)
|
|
544
|
+
continue
|
|
545
|
+
meta = {
|
|
546
|
+
**self.request_identity(spec, conditioning),
|
|
547
|
+
"cache_key": key,
|
|
548
|
+
"requested_utc": requested,
|
|
549
|
+
}
|
|
550
|
+
try:
|
|
551
|
+
self._write_cache(key, tidy, meta)
|
|
552
|
+
except (OSError, pa.ArrowException) as exc:
|
|
553
|
+
log.warning("could not cache %s: %s", spec.variant_string, exc)
|
|
554
|
+
return _Outcome(tidy, "completed", attempts=attempt, requested_utc=requested)
|
|
555
|
+
return _Outcome(None, "failed", error, error_type, attempts=attempt)
|
|
556
|
+
|
|
557
|
+
def _call(self, spec: RequestSpec, conditioning: Any) -> pd.DataFrame:
|
|
558
|
+
from alphagenome.data import genome
|
|
559
|
+
from alphagenome.models.v2 import variant_scorers
|
|
560
|
+
|
|
561
|
+
if self.model is None:
|
|
562
|
+
self.connect()
|
|
563
|
+
chrom, start, end = spec.interval()
|
|
564
|
+
kwargs: dict[str, Any] = {
|
|
565
|
+
"interval": genome.Interval(chromosome=chrom, start=start, end=end),
|
|
566
|
+
"variant": genome.Variant(
|
|
567
|
+
chromosome=chrom,
|
|
568
|
+
position=spec.pos,
|
|
569
|
+
reference_bases=spec.ref,
|
|
570
|
+
alternate_bases=spec.alt,
|
|
571
|
+
),
|
|
572
|
+
"variant_scorers": [self.recommended_scorer(spec)],
|
|
573
|
+
}
|
|
574
|
+
if spec.cell_conditioned:
|
|
575
|
+
kwargs |= {
|
|
576
|
+
"cell_conditioning": conditioning,
|
|
577
|
+
"cell_condition_name": spec.condition_label,
|
|
578
|
+
}
|
|
579
|
+
else:
|
|
580
|
+
kwargs["condition"] = spec.condition
|
|
581
|
+
outputs = self.model.score_variant(**kwargs)
|
|
582
|
+
if not outputs:
|
|
583
|
+
raise ValueError("AlphaGenome returned no scores")
|
|
584
|
+
tidy = variant_scorers.tidy_anndata(
|
|
585
|
+
outputs[0], match_gene_strand=False, include_extended_metadata=True
|
|
586
|
+
)
|
|
587
|
+
if tidy.empty:
|
|
588
|
+
raise ValueError("AlphaGenome returned no scores")
|
|
589
|
+
# Parquet cannot encode SDK Variant and Interval objects directly.
|
|
590
|
+
for col in tidy.columns[tidy.dtypes == np.dtype("object")]:
|
|
591
|
+
tidy[col] = tidy[col].map(lambda v: v if v is None or isinstance(v, str) else str(v))
|
|
592
|
+
return tidy
|
|
593
|
+
|
|
594
|
+
def _record(
|
|
595
|
+
self, spec: RequestSpec, conditioning: Any, out: _Outcome, tracks: Tracks | None = None
|
|
596
|
+
) -> dict:
|
|
597
|
+
rec = {
|
|
598
|
+
**self._base(spec, conditioning),
|
|
599
|
+
"raw_effect": np.nan,
|
|
600
|
+
"status": out.status,
|
|
601
|
+
"error": out.error,
|
|
602
|
+
"error_type": out.error_type,
|
|
603
|
+
"attempts": out.attempts,
|
|
604
|
+
"from_cache": out.from_cache,
|
|
605
|
+
"requested_utc": out.requested_utc,
|
|
606
|
+
"track_filter": json.dumps(_normalise(tracks), sort_keys=True) if tracks else "",
|
|
607
|
+
}
|
|
608
|
+
if out.tidy is not None and spec.gene_id:
|
|
609
|
+
try:
|
|
610
|
+
effect, meta = reduce_gene_effect(out.tidy, spec.gene_id, tracks)
|
|
611
|
+
except ValueError as exc:
|
|
612
|
+
rec |= {"status": "failed", "error": str(exc), "error_type": "validation"}
|
|
613
|
+
else:
|
|
614
|
+
rec |= {"raw_effect": effect, **meta}
|
|
615
|
+
return rec
|
|
616
|
+
|
|
617
|
+
def score(
|
|
618
|
+
self,
|
|
619
|
+
spec: RequestSpec,
|
|
620
|
+
conditioning: Any = None,
|
|
621
|
+
strict: bool = False,
|
|
622
|
+
tracks: Tracks | None = None,
|
|
623
|
+
) -> dict:
|
|
624
|
+
"""Scores one variant under one condition.
|
|
625
|
+
|
|
626
|
+
The score is a predicted ALT-minus-REF effect. RNA_SEQ uses a log fold
|
|
627
|
+
change, while SC_GENE_EXPRESSION_DENOISED uses a difference in summed
|
|
628
|
+
predicted expression.
|
|
629
|
+
|
|
630
|
+
Args:
|
|
631
|
+
spec: Variant, model inputs, and optional gene selection.
|
|
632
|
+
conditioning: Cell-conditioning AnnData, or None for reference conditions.
|
|
633
|
+
strict: Whether to raise for failed or blocked records.
|
|
634
|
+
tracks: Optional track metadata filters used in gene reduction.
|
|
635
|
+
Filters share the cache entry for the full scorer output.
|
|
636
|
+
|
|
637
|
+
Returns:
|
|
638
|
+
A record with provenance, status, and raw_effect for a successful
|
|
639
|
+
gene reduction. Otherwise raw_effect is NaN. Failed or blocked
|
|
640
|
+
records include error details.
|
|
641
|
+
|
|
642
|
+
Raises:
|
|
643
|
+
ScoringError: If strict is true and scoring fails or is blocked.
|
|
644
|
+
"""
|
|
645
|
+
rec = self._record(spec, conditioning, self._obtain(spec, conditioning), tracks)
|
|
646
|
+
if strict and rec["status"] in ("failed", "blocked"):
|
|
647
|
+
raise ScoringError(f"{rec['variant']}: {rec['error']}", rec["error_type"], rec)
|
|
648
|
+
return rec
|
|
649
|
+
|
|
650
|
+
def score_many(
|
|
651
|
+
self,
|
|
652
|
+
specs: Iterable[RequestSpec],
|
|
653
|
+
conditioning: Any = None,
|
|
654
|
+
max_workers: int = 4,
|
|
655
|
+
tracks: Tracks | None = None,
|
|
656
|
+
) -> list[dict]:
|
|
657
|
+
"""Scores specs concurrently, fetching each distinct request once.
|
|
658
|
+
|
|
659
|
+
Individual request failures are recorded without cancelling the rest.
|
|
660
|
+
Cached requests can be reused after an interruption.
|
|
661
|
+
|
|
662
|
+
Args:
|
|
663
|
+
specs: Requests in the desired output order.
|
|
664
|
+
conditioning: One AnnData for all requests, a mapping from
|
|
665
|
+
condition_label to AnnData, or None for reference conditions.
|
|
666
|
+
max_workers: Maximum number of concurrent workers. Must be positive.
|
|
667
|
+
tracks: Optional track metadata filters used in gene reduction.
|
|
668
|
+
|
|
669
|
+
Returns:
|
|
670
|
+
One scoring record per spec, in input order.
|
|
671
|
+
|
|
672
|
+
Raises:
|
|
673
|
+
ValueError: If max_workers is below one.
|
|
674
|
+
"""
|
|
675
|
+
specs = list(specs)
|
|
676
|
+
|
|
677
|
+
def cells(spec: RequestSpec) -> Any:
|
|
678
|
+
if isinstance(conditioning, Mapping):
|
|
679
|
+
return conditioning.get(spec.condition_label)
|
|
680
|
+
return conditioning
|
|
681
|
+
|
|
682
|
+
pairs = [(s, cells(s)) for s in specs]
|
|
683
|
+
outcomes = self._fetch(pairs, max_workers)
|
|
684
|
+
return [self._record(s, c, o, tracks) for (s, c), o in zip(pairs, outcomes, strict=True)]
|
|
685
|
+
|
|
686
|
+
def _fetch(self, pairs: list[tuple[RequestSpec, Any]], max_workers: int) -> list[_Outcome]:
|
|
687
|
+
"""Validates specs individually before sharing fetches by cache key.
|
|
688
|
+
|
|
689
|
+
Specs can share cached output while still having different validation results.
|
|
690
|
+
"""
|
|
691
|
+
problems = [self._check(s, c) for s, c in pairs]
|
|
692
|
+
keys = [self.cache_key(s, c) for s, c in pairs]
|
|
693
|
+
first: dict[str, tuple[RequestSpec, Any]] = {}
|
|
694
|
+
for key, pair, problem in zip(keys, pairs, problems, strict=True):
|
|
695
|
+
if not problem:
|
|
696
|
+
first.setdefault(key, pair)
|
|
697
|
+
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
|
698
|
+
done = dict(
|
|
699
|
+
zip(first, pool.map(lambda p: self._obtain(*p), first.values()), strict=True)
|
|
700
|
+
)
|
|
701
|
+
return [
|
|
702
|
+
_Outcome(None, "failed", p, "validation") if p else done[k]
|
|
703
|
+
for k, p in zip(keys, problems, strict=True)
|
|
704
|
+
]
|
|
705
|
+
|
|
706
|
+
def score_variants(
|
|
707
|
+
self,
|
|
708
|
+
variants: pd.DataFrame,
|
|
709
|
+
*,
|
|
710
|
+
condition: str = "",
|
|
711
|
+
scorer: str = "RNA_SEQ",
|
|
712
|
+
gene_column: str | None = None,
|
|
713
|
+
conditioning: Any = None,
|
|
714
|
+
condition_label: str = "",
|
|
715
|
+
sequence_length: int = DEFAULT_SEQUENCE_LENGTH,
|
|
716
|
+
max_workers: int = 4,
|
|
717
|
+
tracks: Tracks | None = None,
|
|
718
|
+
) -> pd.DataFrame:
|
|
719
|
+
"""Scores a variant table and reduces each result by gene.
|
|
720
|
+
|
|
721
|
+
Supply either a reference condition or cell conditioning with a label.
|
|
722
|
+
|
|
723
|
+
Args:
|
|
724
|
+
variants: Table with chrom, 1-based pos, ref, and alt. Uses variant_id,
|
|
725
|
+
assembly, and input_status when present.
|
|
726
|
+
condition: Reference condition as an ontology CURIE.
|
|
727
|
+
scorer: SDK recommended scorer name.
|
|
728
|
+
gene_column: Column naming the target gene for each row. None reports
|
|
729
|
+
every gene in the scorer output.
|
|
730
|
+
conditioning: One AnnData for a cell-conditioned batch.
|
|
731
|
+
condition_label: SDK track label for the cell condition.
|
|
732
|
+
sequence_length: Supported input window length in bases.
|
|
733
|
+
max_workers: Maximum number of concurrent workers. Must be positive.
|
|
734
|
+
tracks: Optional track metadata filters used in gene reduction.
|
|
735
|
+
|
|
736
|
+
Returns:
|
|
737
|
+
Scoring rows with a zero-based input row number and input_ columns.
|
|
738
|
+
Failures are represented by status and reason, including variant-level
|
|
739
|
+
failures when gene selection depends on missing scorer output.
|
|
740
|
+
|
|
741
|
+
Raises:
|
|
742
|
+
ValueError: If required variant columns are missing or max_workers
|
|
743
|
+
is below one for a nonempty batch.
|
|
744
|
+
KeyError: If the named gene column is missing from a nonempty batch.
|
|
745
|
+
TypeError: If conditioning is a mapping.
|
|
746
|
+
"""
|
|
747
|
+
missing = {"chrom", "pos", "ref", "alt"} - set(variants.columns)
|
|
748
|
+
if missing:
|
|
749
|
+
raise ValueError(f"variants is missing columns {sorted(missing)}")
|
|
750
|
+
if isinstance(conditioning, Mapping):
|
|
751
|
+
raise TypeError("score_variants takes one AnnData; call it once per condition")
|
|
752
|
+
if variants.empty:
|
|
753
|
+
return pd.DataFrame(columns=["row", "status", "error", "error_type"])
|
|
754
|
+
n_cells = int(getattr(conditioning, "n_obs", 0) or 0)
|
|
755
|
+
pairs: list[tuple[int, RequestSpec | None, str]] = []
|
|
756
|
+
input_rows = variants.to_dict("records")
|
|
757
|
+
for i, row in enumerate(input_rows):
|
|
758
|
+
problem = str(row.get("input_status", "ok") or "ok")
|
|
759
|
+
problem = "" if problem == "ok" else f"input_status={problem}"
|
|
760
|
+
assembly = str(row.get("assembly", "GRCh38") or "GRCh38")
|
|
761
|
+
if not problem and assembly.upper() not in ("GRCH38", "HG38"):
|
|
762
|
+
problem = f"assembly {assembly}: AlphaGenome scores GRCh38; lift over first"
|
|
763
|
+
spec = None
|
|
764
|
+
if not problem:
|
|
765
|
+
try:
|
|
766
|
+
spec = RequestSpec(
|
|
767
|
+
variant_id=str(row.get("variant_id") or ""),
|
|
768
|
+
chrom=row["chrom"],
|
|
769
|
+
pos=row["pos"],
|
|
770
|
+
ref=row["ref"],
|
|
771
|
+
alt=row["alt"],
|
|
772
|
+
gene_id=str(row[gene_column]) if gene_column else "",
|
|
773
|
+
condition=condition,
|
|
774
|
+
condition_label=condition_label,
|
|
775
|
+
n_cells=n_cells,
|
|
776
|
+
model_version=self.model_version,
|
|
777
|
+
scorer_name=scorer,
|
|
778
|
+
sequence_length=sequence_length,
|
|
779
|
+
)
|
|
780
|
+
except (ValueError, TypeError) as exc:
|
|
781
|
+
problem = str(exc)
|
|
782
|
+
pairs.append((i, spec, problem))
|
|
783
|
+
|
|
784
|
+
valid = [(s, conditioning) for _, s, _ in pairs if s is not None]
|
|
785
|
+
outcomes = iter(self._fetch(valid, max_workers))
|
|
786
|
+
records: list[dict] = []
|
|
787
|
+
for i, spec, problem in pairs:
|
|
788
|
+
if spec is None:
|
|
789
|
+
row = input_rows[i]
|
|
790
|
+
records.append(
|
|
791
|
+
{
|
|
792
|
+
"row": i,
|
|
793
|
+
"chrom": "chr" + str(row["chrom"]).removeprefix("chr"),
|
|
794
|
+
"pos": row["pos"],
|
|
795
|
+
"ref": str(row["ref"]).upper(),
|
|
796
|
+
"alt": str(row["alt"]).upper(),
|
|
797
|
+
"gene_id": str(row[gene_column]) if gene_column else "",
|
|
798
|
+
"condition": condition,
|
|
799
|
+
"status": "failed",
|
|
800
|
+
"error": problem,
|
|
801
|
+
"error_type": "validation",
|
|
802
|
+
}
|
|
803
|
+
)
|
|
804
|
+
continue
|
|
805
|
+
out = next(outcomes)
|
|
806
|
+
if gene_column or out.tidy is None:
|
|
807
|
+
records.append({"row": i, **self._record(spec, conditioning, out, tracks)})
|
|
808
|
+
continue
|
|
809
|
+
names = out.tidy.drop_duplicates("gene_id").set_index("gene_id")["gene_name"]
|
|
810
|
+
for gene, name in names.items():
|
|
811
|
+
rec = self._record(replace(spec, gene_id=str(gene)), conditioning, out, tracks)
|
|
812
|
+
records.append({"row": i, **rec, "gene_name": name})
|
|
813
|
+
table = pd.DataFrame.from_records(records)
|
|
814
|
+
inputs = variants.reset_index(drop=True).add_prefix("input_")
|
|
815
|
+
return inputs.join(table.set_index("row"), how="right").rename_axis("row").reset_index()
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
def _normalise(tracks: Tracks | None) -> dict[str, list[str]]:
|
|
819
|
+
return {
|
|
820
|
+
str(k): [v] if isinstance(v, str) else [str(x) for x in v]
|
|
821
|
+
for k, v in (tracks or {}).items()
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def reduce_gene_effect(
|
|
826
|
+
tidy: pd.DataFrame, gene_id: str, tracks: Tracks | None = None
|
|
827
|
+
) -> tuple[float, dict]:
|
|
828
|
+
"""Averages signed scores for a gene over strand-compatible tracks.
|
|
829
|
+
|
|
830
|
+
Unstranded tracks are included, matching the SDK's strand-filtering
|
|
831
|
+
rule. Filter imputation_mode to "observed" to average tracks observed
|
|
832
|
+
in training data separately from imputed tracks.
|
|
833
|
+
|
|
834
|
+
Args:
|
|
835
|
+
tidy: SDK tidy scorer output with all strands.
|
|
836
|
+
gene_id: Ensembl gene ID. Version suffixes are ignored.
|
|
837
|
+
tracks: Optional mapping from metadata columns to allowed values.
|
|
838
|
+
|
|
839
|
+
Returns:
|
|
840
|
+
Mean raw_score and metadata with the number of values, gene strand,
|
|
841
|
+
and up to four distinct track names.
|
|
842
|
+
|
|
843
|
+
Raises:
|
|
844
|
+
ValueError: If the gene or a filter column is absent, no tracks match,
|
|
845
|
+
or any selected score is non-finite.
|
|
846
|
+
"""
|
|
847
|
+
target = str(gene_id).split(".")[0]
|
|
848
|
+
rows = tidy[tidy["gene_id"].astype(str).str.split(".").str[0] == target]
|
|
849
|
+
if rows.empty:
|
|
850
|
+
raise ValueError(f"target gene {gene_id} absent from scorer output")
|
|
851
|
+
for column, allowed in _normalise(tracks).items():
|
|
852
|
+
if column not in rows:
|
|
853
|
+
raise ValueError(f"track filter column {column!r} not in {sorted(tidy.columns)}")
|
|
854
|
+
rows = rows[rows[column].astype(str).isin(allowed)]
|
|
855
|
+
if rows.empty:
|
|
856
|
+
raise ValueError(f"no track of gene {gene_id} has {column} in {allowed}")
|
|
857
|
+
gs, ts = rows["gene_strand"], rows["track_strand"]
|
|
858
|
+
rows = rows[~(((gs == "+") & (ts == "-")) | ((gs == "-") & (ts == "+")))]
|
|
859
|
+
values = rows["raw_score"].to_numpy(dtype=float)
|
|
860
|
+
if values.size == 0:
|
|
861
|
+
raise ValueError(f"no track on the strand of gene {gene_id}")
|
|
862
|
+
if not np.all(np.isfinite(values)):
|
|
863
|
+
raise ValueError(f"non-finite score for gene {gene_id}")
|
|
864
|
+
meta = {
|
|
865
|
+
"n_reduced_values": int(values.size),
|
|
866
|
+
"gene_strand": str(rows["gene_strand"].iloc[0]),
|
|
867
|
+
"track_names": ";".join(map(str, rows["track_name"].unique()[:4])),
|
|
868
|
+
}
|
|
869
|
+
return float(values.mean()), meta
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def extract_gene_effect(adata: Any, gene_id: str) -> tuple[float, dict]:
|
|
873
|
+
"""Reduces a scorer AnnData to one gene's strand-compatible effect.
|
|
874
|
+
|
|
875
|
+
Args:
|
|
876
|
+
adata: SDK scorer result with genes as rows and tracks as columns.
|
|
877
|
+
gene_id: Ensembl ID of the target gene.
|
|
878
|
+
|
|
879
|
+
Returns:
|
|
880
|
+
Mean signed effect and reduction metadata, including the total number
|
|
881
|
+
of tracks in adata before filtering.
|
|
882
|
+
|
|
883
|
+
Raises:
|
|
884
|
+
ImportError: If the AlphaGenome SDK is unavailable.
|
|
885
|
+
ValueError: If the gene is absent, has no compatible tracks, or has
|
|
886
|
+
a non-finite selected score.
|
|
887
|
+
"""
|
|
888
|
+
from alphagenome.models.v2 import variant_scorers
|
|
889
|
+
|
|
890
|
+
tidy = variant_scorers.tidy_anndata(
|
|
891
|
+
adata, match_gene_strand=False, include_extended_metadata=False
|
|
892
|
+
)
|
|
893
|
+
effect, meta = reduce_gene_effect(tidy, gene_id)
|
|
894
|
+
return effect, {**meta, "n_tracks": int(adata.shape[1])}
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
def conditioning_fingerprint(conditioning: Any) -> str:
|
|
898
|
+
"""Hashes cell-conditioning inputs for request identity.
|
|
899
|
+
|
|
900
|
+
Args:
|
|
901
|
+
conditioning: AnnData with a count matrix and ordered cell and gene names,
|
|
902
|
+
or None. Objects without a matrix are identified by their repr.
|
|
903
|
+
|
|
904
|
+
Returns:
|
|
905
|
+
SHA-256 digest, or an empty string for None.
|
|
906
|
+
"""
|
|
907
|
+
if conditioning is None:
|
|
908
|
+
return ""
|
|
909
|
+
import scipy.sparse as sp
|
|
910
|
+
|
|
911
|
+
h = hashlib.sha256()
|
|
912
|
+
x = getattr(conditioning, "X", None)
|
|
913
|
+
if x is None:
|
|
914
|
+
h.update(repr(conditioning).encode())
|
|
915
|
+
return h.hexdigest()
|
|
916
|
+
x = np.ascontiguousarray(x.toarray() if sp.issparse(x) else np.asarray(x))
|
|
917
|
+
h.update(f"{x.dtype}{x.shape}".encode())
|
|
918
|
+
h.update(x.tobytes())
|
|
919
|
+
names = [list(map(str, names)) for names in (conditioning.obs_names, conditioning.var_names)]
|
|
920
|
+
h.update(json.dumps(names).encode())
|
|
921
|
+
return h.hexdigest()
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
def interaction(score_treated: float | None, score_control: float | None) -> float:
|
|
925
|
+
"""Computes a treated-minus-control difference in predicted effects.
|
|
926
|
+
|
|
927
|
+
Args:
|
|
928
|
+
score_treated: Predicted effect under treatment.
|
|
929
|
+
score_control: Predicted effect for the same variant, gene, and scorer
|
|
930
|
+
under control.
|
|
931
|
+
|
|
932
|
+
Returns:
|
|
933
|
+
Difference in predicted effects, or NaN if either score is missing
|
|
934
|
+
or non-finite.
|
|
935
|
+
"""
|
|
936
|
+
if score_treated is None or score_control is None:
|
|
937
|
+
return float("nan")
|
|
938
|
+
if not (np.isfinite(score_treated) and np.isfinite(score_control)):
|
|
939
|
+
return float("nan")
|
|
940
|
+
return float(score_treated) - float(score_control)
|