ag2kit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ag2kit/client.py ADDED
@@ -0,0 +1,940 @@
1
+ """AlphaGenome scoring with request-level caching and gene-level reduction."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import importlib.metadata
7
+ import importlib.util
8
+ import json
9
+ import logging
10
+ import os
11
+ import re
12
+ import threading
13
+ import time
14
+ from collections.abc import Iterable, Mapping, Sequence
15
+ from concurrent.futures import ThreadPoolExecutor
16
+ from dataclasses import asdict, dataclass, field, replace
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ import numpy as np
21
+ import pandas as pd
22
+ import pyarrow as pa
23
+ import pyarrow.parquet as pq
24
+
25
+ from ag2kit.provenance import cache_dir, stable_hash, utc_now
26
+
27
+ log = logging.getLogger("ag2kit")
28
+
29
+ API_KEY_ENV = "ALPHA_GENOME_API_KEY"
30
+ DEFAULT_SEQUENCE_LENGTH = 2**20
31
+ SUPPORTED_SEQUENCE_LENGTHS = frozenset({2**14, 2**17, 2**19, 2**20})
32
+ MAX_CONDITIONING_CELLS = 256
33
+ SOURCE = "alphagenome"
34
+ _ALLELE = re.compile(r"^[ACGTN]+$")
35
+ _CURIES = re.compile(r"^[A-Za-z]+:[A-Za-z0-9.]+(?:_[A-Za-z]+:[A-Za-z0-9.]+)*$")
36
+ _CACHE_FORMAT = 2
37
+
38
+ _ERROR_TYPES = {
39
+ "RESOURCE_EXHAUSTED": "quota",
40
+ "UNAVAILABLE": "transient",
41
+ "DEADLINE_EXCEEDED": "transient",
42
+ "ABORTED": "transient",
43
+ "INVALID_ARGUMENT": "validation",
44
+ "FAILED_PRECONDITION": "validation",
45
+ "OUT_OF_RANGE": "validation",
46
+ "NOT_FOUND": "validation",
47
+ "UNIMPLEMENTED": "validation",
48
+ "UNAUTHENTICATED": "credentials",
49
+ "PERMISSION_DENIED": "credentials",
50
+ }
51
+ _RETRYABLE = frozenset({"quota", "transient", "network"})
52
+
53
+ Tracks = Mapping[str, str | Sequence[str]]
54
+
55
+
56
+ class AG2KitError(Exception):
57
+ """Base class for ag2kit errors."""
58
+
59
+
60
+ class CredentialsError(AG2KitError):
61
+ """The API key is missing or was rejected."""
62
+
63
+
64
+ class ScoringError(AG2KitError):
65
+ """Reports a failed or blocked scoring request in strict mode.
66
+
67
+ Attributes:
68
+ error_type: Category of the failure.
69
+ record: Scoring record that would otherwise have been returned.
70
+ """
71
+
72
+ def __init__(self, message: str, error_type: str, record: dict):
73
+ """Initializes the exception with its scoring context.
74
+
75
+ Args:
76
+ message: Human-readable error message.
77
+ error_type: Failure category.
78
+ record: Failed or blocked scoring record.
79
+ """
80
+ super().__init__(message)
81
+ self.error_type = error_type
82
+ self.record = record
83
+
84
+
85
+ @dataclass(frozen=True)
86
+ class RequestSpec:
87
+ """Specifies one GRCh38 variant under one reference or cell condition.
88
+
89
+ Use exactly one condition input. Reference conditions use condition,
90
+ typically with RNA_SEQ. Cell conditioning uses a positive n_cells and
91
+ a condition_label. Pass the actual AnnData to the scoring method.
92
+ Construction raises ValueError for invalid coordinates, alleles, sequence
93
+ length, cell count, organism, or condition settings.
94
+
95
+ Attributes:
96
+ variant_id: Caller-provided variant label.
97
+ chrom: Chromosome name, normalized to include the chr prefix.
98
+ pos: 1-based position of the first REF base.
99
+ ref: Forward-strand reference allele, stored in uppercase.
100
+ alt: Forward-strand alternate allele, stored in uppercase.
101
+ gene_id: Gene to reduce to a single effect. Empty skips gene reduction.
102
+ gene_symbol: Caller-provided gene label.
103
+ condition_label: Cell-condition name used by the SDK to name output tracks.
104
+ conditioning_set_id: Caller-provided label for the conditioning set.
105
+ n_cells: Number of conditioning cells, or zero for reference conditions.
106
+ seed: Caller-provided seed label for the conditioning draw.
107
+ model_version: AlphaGenome model version.
108
+ scorer_name: SDK recommended scorer name.
109
+ sequence_length: Supported input window length in bases.
110
+ organism: SDK organism name. Only HOMO_SAPIENS is supported.
111
+ condition: Reference condition as one or more ontology CURIEs.
112
+ cell_conditioned: Whether the spec uses cell conditioning.
113
+ variant_string: Variant in SDK-compatible chrom:pos:ref>alt form.
114
+ """
115
+
116
+ variant_id: str
117
+ chrom: str
118
+ pos: int
119
+ ref: str
120
+ alt: str
121
+ gene_id: str = ""
122
+ gene_symbol: str = ""
123
+ condition_label: str = ""
124
+ conditioning_set_id: str = ""
125
+ n_cells: int = 0
126
+ seed: int = 0
127
+ model_version: str = "ALL_FOLDS"
128
+ scorer_name: str = "SC_GENE_EXPRESSION_DENOISED"
129
+ sequence_length: int = DEFAULT_SEQUENCE_LENGTH
130
+ organism: str = "HOMO_SAPIENS"
131
+ condition: str = ""
132
+
133
+ def __post_init__(self) -> None:
134
+ object.__setattr__(self, "chrom", "chr" + str(self.chrom).removeprefix("chr"))
135
+ object.__setattr__(self, "ref", str(self.ref).upper())
136
+ object.__setattr__(self, "alt", str(self.alt).upper())
137
+ if isinstance(self.pos, bool) or int(self.pos) != self.pos or self.pos < 1:
138
+ raise ValueError(f"pos must be a 1-based integer position, got {self.pos!r}")
139
+ object.__setattr__(self, "pos", int(self.pos))
140
+ for name in ("ref", "alt"):
141
+ if not _ALLELE.match(getattr(self, name)):
142
+ raise ValueError(f"{name} must be a non-empty A/C/G/T/N string")
143
+ if self.ref == self.alt:
144
+ raise ValueError("ref and alt are identical")
145
+ if self.sequence_length not in SUPPORTED_SEQUENCE_LENGTHS:
146
+ supported = sorted(SUPPORTED_SEQUENCE_LENGTHS)
147
+ raise ValueError(f"sequence_length {self.sequence_length} not in {supported}")
148
+ if not 0 <= self.n_cells <= MAX_CONDITIONING_CELLS:
149
+ raise ValueError(f"n_cells must be between 0 and {MAX_CONDITIONING_CELLS}")
150
+ if self.organism != "HOMO_SAPIENS":
151
+ raise ValueError("only HOMO_SAPIENS is supported")
152
+ if self.condition and not _CURIES.match(self.condition):
153
+ raise ValueError(
154
+ f"condition must be ontology CURIE(s) like UBERON:0013756, got {self.condition!r}"
155
+ )
156
+ if bool(self.condition) == self.cell_conditioned:
157
+ raise ValueError(
158
+ "set exactly one of condition (ontology CURIE) or n_cells > 0 (cell conditioning)"
159
+ )
160
+ if self.cell_conditioned and not self.condition_label:
161
+ raise ValueError("cell conditioning needs a condition_label")
162
+
163
+ @property
164
+ def cell_conditioned(self) -> bool:
165
+ """Whether the request uses cell conditioning."""
166
+ return self.n_cells > 0
167
+
168
+ @property
169
+ def variant_string(self) -> str:
170
+ """Variant string accepted by the SDK variant parser."""
171
+ return f"{self.chrom}:{self.pos}:{self.ref}>{self.alt}"
172
+
173
+ def interval(self) -> tuple[str, int, int]:
174
+ """Builds the input window around the reference allele span.
175
+
176
+ Near the chromosome start, the window shifts right to start at zero.
177
+ The API rejects windows that extend beyond the chromosome end.
178
+
179
+ Returns:
180
+ Chromosome, zero-based inclusive start, and exclusive end.
181
+ """
182
+ centre = self.pos - 1 + (len(self.ref) + 1) // 2
183
+ start = max(0, centre - self.sequence_length // 2)
184
+ return (self.chrom, start, start + self.sequence_length)
185
+
186
+
187
+ def sdk_version() -> str:
188
+ """Returns the installed AlphaGenome SDK version or "not_installed"."""
189
+ try:
190
+ return importlib.metadata.version("alphagenome")
191
+ except importlib.metadata.PackageNotFoundError:
192
+ return "not_installed"
193
+
194
+
195
+ def api_key_available() -> bool:
196
+ """Checks whether the API key environment variable contains a nonblank value."""
197
+ return bool(os.environ.get(API_KEY_ENV, "").strip())
198
+
199
+
200
+ def sdk_available() -> bool:
201
+ """Checks whether Python can locate the AlphaGenome SDK."""
202
+ return importlib.util.find_spec("alphagenome") is not None
203
+
204
+
205
+ def classify_error(exc: BaseException) -> str:
206
+ """Categorizes scoring errors for records and retry decisions.
207
+
208
+ Args:
209
+ exc: Exception from the SDK or a custom model.
210
+
211
+ Returns:
212
+ An error category, using the RPC status when available and
213
+ "upstream" for unrecognized failures.
214
+ """
215
+ code = getattr(exc, "code", None)
216
+ if callable(code):
217
+ try:
218
+ name = getattr(code(), "name", "")
219
+ except Exception:
220
+ name = ""
221
+ if name:
222
+ return _ERROR_TYPES.get(name, "upstream")
223
+ if isinstance(exc, PermissionError):
224
+ return "credentials"
225
+ if isinstance(exc, ValueError | KeyError | IndexError | NotImplementedError):
226
+ return "validation"
227
+ if isinstance(exc, ConnectionError | TimeoutError):
228
+ return "network"
229
+ text = str(exc).upper()
230
+ return next((kind for name, kind in _ERROR_TYPES.items() if name in text), "upstream")
231
+
232
+
233
+ def _redact(text: str) -> str:
234
+ key = os.environ.get(API_KEY_ENV, "").strip()
235
+ return text.replace(key, "<redacted>") if key else text
236
+
237
+
238
+ @dataclass
239
+ class _Outcome:
240
+ """Stores a request result with status and cache metadata."""
241
+
242
+ tidy: pd.DataFrame | None
243
+ status: str
244
+ error: str = ""
245
+ error_type: str = ""
246
+ attempts: int = 0
247
+ from_cache: bool = False
248
+ requested_utc: str = ""
249
+
250
+
251
+ @dataclass
252
+ class AG2Client:
253
+ """Scores request specs and caches full scorer outputs on disk.
254
+
255
+ One SDK connection is shared across worker threads. The client retries
256
+ quota, transient, and network failures in addition to the SDK's own retries.
257
+ The API does not report a model revision, so use a new namespace after a
258
+ server-side change. Custom models are identified by class, so give
259
+ differently trained instances of the same class distinct namespaces.
260
+
261
+ Attributes:
262
+ dry_run: Whether to validate and plan requests without live scoring.
263
+ cache_dir: Directory for cached scores, overridden by AG2KIT_AG2_CACHE.
264
+ model_version: SDK model version required by requests on this client.
265
+ max_retries: Maximum attempts per request, including the initial attempt.
266
+ Construction raises ValueError for values below one.
267
+ backoff: Seconds to wait per attempt number before retrying.
268
+ sdk_version: SDK version recorded in request identity and provenance.
269
+ cache_namespace: Label for isolating caches across model revisions.
270
+ model: Optional backend implementing the SDK score_variant interface.
271
+ Live API connections are assigned here on first use.
272
+ """
273
+
274
+ dry_run: bool = True
275
+ cache_dir: Path = field(default_factory=lambda: cache_dir() / "ag2")
276
+ model_version: str = "ALL_FOLDS"
277
+ max_retries: int = 2
278
+ backoff: float = 30.0
279
+ sdk_version: str = field(default_factory=sdk_version)
280
+ cache_namespace: str = ""
281
+ model: Any = None
282
+
283
+ def __post_init__(self) -> None:
284
+ self.cache_dir = Path(os.environ.get("AG2KIT_AG2_CACHE") or self.cache_dir)
285
+ if self.max_retries < 1:
286
+ raise ValueError("max_retries must be at least 1")
287
+ self._custom_model = self.model is not None
288
+ self._connect_lock = threading.Lock()
289
+
290
+ def connect(self) -> None:
291
+ """Initializes the live client if needed.
292
+
293
+ Called lazily when scoring. Dry runs and clients with an existing backend
294
+ skip initialization.
295
+
296
+ Raises:
297
+ CredentialsError: If live connection is needed and the key is missing.
298
+ ImportError: If the AlphaGenome SDK is unavailable.
299
+ AttributeError: If model_version is unknown to the SDK.
300
+ """
301
+ with self._connect_lock:
302
+ if self.model is not None or self.dry_run:
303
+ return
304
+ if not api_key_available():
305
+ raise CredentialsError(f"{API_KEY_ENV} is not set; live scoring is unavailable")
306
+ from alphagenome.models.v2 import dna_client
307
+
308
+ version = getattr(dna_client.ModelVersion, self.model_version)
309
+ self.model = dna_client.create(os.environ[API_KEY_ENV].strip(), model_version=version)
310
+
311
+ def recommended_scorer(self, spec: RequestSpec) -> Any:
312
+ """Resolves the SDK scorer and checks compatibility with the request.
313
+
314
+ Args:
315
+ spec: Request with the scorer name and conditioning mode.
316
+
317
+ Returns:
318
+ The SDK scorer object.
319
+
320
+ Raises:
321
+ ImportError: If the AlphaGenome SDK is unavailable.
322
+ KeyError: If the scorer name is unknown.
323
+ ValueError: If the scorer is unsigned or requires cell conditioning
324
+ and the request uses a reference condition.
325
+ """
326
+ from alphagenome.models.v2 import variant_scorers
327
+
328
+ scorers = variant_scorers.RECOMMENDED_VARIANT_SCORERS
329
+ if spec.scorer_name not in scorers:
330
+ raise KeyError(
331
+ f"{spec.scorer_name!r} not in RECOMMENDED_VARIANT_SCORERS: {sorted(scorers)}"
332
+ )
333
+ scorer = scorers[spec.scorer_name]
334
+ if not getattr(scorer, "is_signed", True):
335
+ raise ValueError(f"scorer {spec.scorer_name!r} is unsigned; ag2kit needs ALT-REF")
336
+ cell_only: frozenset = variant_scorers.CELL_CONDITIONING_ONLY_OUTPUT_TYPES
337
+ if not spec.cell_conditioned and getattr(scorer, "requested_output", None) in cell_only:
338
+ raise ValueError(f"scorer {spec.scorer_name!r} needs cell conditioning")
339
+ return scorer
340
+
341
+ def scorer_definition(self, spec: RequestSpec) -> str:
342
+ """Describes the scorer used for request identity.
343
+
344
+ Args:
345
+ spec: Request whose scorer to describe.
346
+
347
+ Returns:
348
+ The SDK scorer representation, or its name if the SDK or a
349
+ compatible scorer is unavailable.
350
+ """
351
+ if not sdk_available():
352
+ return spec.scorer_name
353
+ try:
354
+ return repr(self.recommended_scorer(spec))
355
+ except (KeyError, ValueError):
356
+ return spec.scorer_name
357
+
358
+ def _backend(self) -> str:
359
+ if not self._custom_model:
360
+ return SOURCE
361
+ cls = type(self.model)
362
+ return f"custom:{cls.__module__}.{cls.__qualname__}"
363
+
364
+ def request_identity(self, spec: RequestSpec, conditioning: Any = None) -> dict:
365
+ """Builds the identity shared by requests for the same full scorer output.
366
+
367
+ Gene selection and provenance labels are excluded. The cell-condition
368
+ name participates because it affects output track names.
369
+
370
+ Args:
371
+ spec: Request to identify.
372
+ conditioning: Cell-conditioning AnnData, or None for reference conditions.
373
+
374
+ Returns:
375
+ A dictionary of request inputs and cache provenance.
376
+ """
377
+ chrom, start, end = spec.interval()
378
+ return {
379
+ "cache_format": _CACHE_FORMAT,
380
+ "variant": spec.variant_string,
381
+ "interval": f"{chrom}:{start}-{end}",
382
+ "model_version": spec.model_version,
383
+ "organism": spec.organism,
384
+ "condition": spec.condition,
385
+ "cell_condition_name": spec.condition_label if spec.cell_conditioned else "",
386
+ "conditioning_sha256": conditioning_fingerprint(conditioning),
387
+ "scorer_definition": self.scorer_definition(spec),
388
+ "backend": self._backend(),
389
+ "sdk_version": self.sdk_version,
390
+ "cache_namespace": self.cache_namespace,
391
+ }
392
+
393
+ def cache_key(self, spec: RequestSpec, conditioning: Any = None) -> str:
394
+ """Hashes the request identity for cache lookup.
395
+
396
+ Args:
397
+ spec: Request to identify.
398
+ conditioning: Cell-conditioning AnnData, or None for reference conditions.
399
+
400
+ Returns:
401
+ SHA-256 digest of request_identity.
402
+ """
403
+ return stable_hash(self.request_identity(spec, conditioning))
404
+
405
+ def _cache_path(self, key: str) -> Path:
406
+ return self.cache_dir / f"{key}.parquet"
407
+
408
+ def _read_cache(self, key: str) -> tuple[pd.DataFrame, dict] | None:
409
+ try:
410
+ table = pq.read_table(self._cache_path(key))
411
+ meta = json.loads((table.schema.metadata or {})[b"ag2kit"])
412
+ except (OSError, KeyError, ValueError, pa.ArrowException):
413
+ return None
414
+ if meta.get("cache_key") != key or table.num_rows == 0:
415
+ return None
416
+ return table.to_pandas(), meta
417
+
418
+ def _write_cache(self, key: str, tidy: pd.DataFrame, meta: dict) -> None:
419
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
420
+ path = self._cache_path(key)
421
+ table = pa.Table.from_pandas(tidy, preserve_index=False)
422
+ table = table.replace_schema_metadata(
423
+ {**(table.schema.metadata or {}), b"ag2kit": json.dumps(meta, sort_keys=True)}
424
+ )
425
+ tmp = path.with_suffix(f".{os.getpid()}.{threading.get_ident()}.tmp")
426
+ pq.write_table(table, tmp)
427
+ tmp.replace(path)
428
+
429
+ def raw_scores(self, spec: RequestSpec, conditioning: Any = None) -> pd.DataFrame | None:
430
+ """Reads the full cached scorer output without calling the API.
431
+
432
+ Args:
433
+ spec: Request to look up. Gene selection does not affect the lookup.
434
+ conditioning: Cell-conditioning AnnData used by the request, if any.
435
+
436
+ Returns:
437
+ Tidy gene-by-track scores for all strands, including raw_score and
438
+ quantile_score when supplied by the SDK, or None on a cache miss.
439
+ """
440
+ hit = self._read_cache(self.cache_key(spec, conditioning))
441
+ return None if hit is None else hit[0]
442
+
443
+ def _base(self, spec: RequestSpec, conditioning: Any) -> dict:
444
+ return {
445
+ **asdict(spec),
446
+ "variant": spec.variant_string,
447
+ "source": self._backend(),
448
+ "cache_key": self.cache_key(spec, conditioning),
449
+ "conditioning_sha256": conditioning_fingerprint(conditioning),
450
+ "sdk_version": self.sdk_version,
451
+ "scorer_definition": self.scorer_definition(spec),
452
+ }
453
+
454
+ def plan(self, spec: RequestSpec, conditioning: Any = None) -> dict:
455
+ """Describes a scoring request without contacting the API.
456
+
457
+ Args:
458
+ spec: Request to describe.
459
+ conditioning: Cell-conditioning AnnData, or None for reference conditions.
460
+
461
+ Returns:
462
+ Request metadata, interval, zero-based offset of the first REF base
463
+ within the interval, and whether the expected cache file exists.
464
+ """
465
+ chrom, start, end = spec.interval()
466
+ return {
467
+ **self._base(spec, conditioning),
468
+ "interval": f"{chrom}:{start}-{end}",
469
+ "variant_offset_in_interval": spec.pos - 1 - start,
470
+ "cached": self._cache_path(self.cache_key(spec, conditioning)).exists(),
471
+ }
472
+
473
+ def cached(self, spec: RequestSpec, conditioning: Any = None) -> dict | None:
474
+ """Reads a gene-level scoring record from the cache.
475
+
476
+ Args:
477
+ spec: Request and optional gene selection.
478
+ conditioning: Cell-conditioning AnnData used by the request, if any.
479
+
480
+ Returns:
481
+ A scoring record, or None on a cache miss.
482
+ """
483
+ hit = self._read_cache(self.cache_key(spec, conditioning))
484
+ if hit is None:
485
+ return None
486
+ tidy, meta = hit
487
+ out = _Outcome(tidy, "completed", from_cache=True, requested_utc=meta["requested_utc"])
488
+ return self._record(spec, conditioning, out)
489
+
490
+ def _check(self, spec: RequestSpec, conditioning: Any) -> str:
491
+ n_obs = getattr(conditioning, "n_obs", None)
492
+ if spec.cell_conditioned and conditioning is None:
493
+ return "cell-conditioned spec but no conditioning AnnData was given"
494
+ if not spec.cell_conditioned and conditioning is not None:
495
+ return "conditioning AnnData given for a spec with a reference condition"
496
+ if n_obs is not None and n_obs != spec.n_cells:
497
+ return f"spec.n_cells={spec.n_cells} but conditioning has {n_obs} cells"
498
+ if sdk_available():
499
+ try:
500
+ self.recommended_scorer(spec)
501
+ except (KeyError, ValueError) as exc:
502
+ return str(exc).strip("'\"")
503
+ if spec.model_version != self.model_version:
504
+ return (
505
+ f"spec.model_version {spec.model_version!r} differs from the client's "
506
+ f"{self.model_version!r}; use one client per model version"
507
+ )
508
+ return ""
509
+
510
+ def _obtain(self, spec: RequestSpec, conditioning: Any) -> _Outcome:
511
+ if problem := self._check(spec, conditioning):
512
+ return _Outcome(None, "failed", problem, "validation")
513
+ if self.dry_run:
514
+ return _Outcome(None, "not_run", "dry_run")
515
+ key = self.cache_key(spec, conditioning)
516
+ if (hit := self._read_cache(key)) is not None:
517
+ return _Outcome(
518
+ hit[0], "completed", from_cache=True, requested_utc=hit[1]["requested_utc"]
519
+ )
520
+ if not sdk_available():
521
+ return _Outcome(
522
+ None,
523
+ "blocked",
524
+ "alphagenome SDK not installed (pip install 'ag2kit[model]')",
525
+ "blocked",
526
+ )
527
+ if not self._custom_model and not api_key_available():
528
+ return _Outcome(None, "blocked", f"{API_KEY_ENV} not set", "blocked")
529
+
530
+ error, error_type, attempt = "", "upstream", 0
531
+ for attempt in range(1, self.max_retries + 1):
532
+ requested = utc_now()
533
+ try:
534
+ tidy = self._call(spec, conditioning)
535
+ except Exception as exc:
536
+ error = _redact(f"{type(exc).__name__}: {exc}")
537
+ error_type = classify_error(exc)
538
+ if error_type not in _RETRYABLE or attempt == self.max_retries:
539
+ break
540
+ log.info(
541
+ "retrying %s after %s (attempt %d)", spec.variant_string, error_type, attempt
542
+ )
543
+ time.sleep(self.backoff * attempt)
544
+ continue
545
+ meta = {
546
+ **self.request_identity(spec, conditioning),
547
+ "cache_key": key,
548
+ "requested_utc": requested,
549
+ }
550
+ try:
551
+ self._write_cache(key, tidy, meta)
552
+ except (OSError, pa.ArrowException) as exc:
553
+ log.warning("could not cache %s: %s", spec.variant_string, exc)
554
+ return _Outcome(tidy, "completed", attempts=attempt, requested_utc=requested)
555
+ return _Outcome(None, "failed", error, error_type, attempts=attempt)
556
+
557
+ def _call(self, spec: RequestSpec, conditioning: Any) -> pd.DataFrame:
558
+ from alphagenome.data import genome
559
+ from alphagenome.models.v2 import variant_scorers
560
+
561
+ if self.model is None:
562
+ self.connect()
563
+ chrom, start, end = spec.interval()
564
+ kwargs: dict[str, Any] = {
565
+ "interval": genome.Interval(chromosome=chrom, start=start, end=end),
566
+ "variant": genome.Variant(
567
+ chromosome=chrom,
568
+ position=spec.pos,
569
+ reference_bases=spec.ref,
570
+ alternate_bases=spec.alt,
571
+ ),
572
+ "variant_scorers": [self.recommended_scorer(spec)],
573
+ }
574
+ if spec.cell_conditioned:
575
+ kwargs |= {
576
+ "cell_conditioning": conditioning,
577
+ "cell_condition_name": spec.condition_label,
578
+ }
579
+ else:
580
+ kwargs["condition"] = spec.condition
581
+ outputs = self.model.score_variant(**kwargs)
582
+ if not outputs:
583
+ raise ValueError("AlphaGenome returned no scores")
584
+ tidy = variant_scorers.tidy_anndata(
585
+ outputs[0], match_gene_strand=False, include_extended_metadata=True
586
+ )
587
+ if tidy.empty:
588
+ raise ValueError("AlphaGenome returned no scores")
589
+ # Parquet cannot encode SDK Variant and Interval objects directly.
590
+ for col in tidy.columns[tidy.dtypes == np.dtype("object")]:
591
+ tidy[col] = tidy[col].map(lambda v: v if v is None or isinstance(v, str) else str(v))
592
+ return tidy
593
+
594
+ def _record(
595
+ self, spec: RequestSpec, conditioning: Any, out: _Outcome, tracks: Tracks | None = None
596
+ ) -> dict:
597
+ rec = {
598
+ **self._base(spec, conditioning),
599
+ "raw_effect": np.nan,
600
+ "status": out.status,
601
+ "error": out.error,
602
+ "error_type": out.error_type,
603
+ "attempts": out.attempts,
604
+ "from_cache": out.from_cache,
605
+ "requested_utc": out.requested_utc,
606
+ "track_filter": json.dumps(_normalise(tracks), sort_keys=True) if tracks else "",
607
+ }
608
+ if out.tidy is not None and spec.gene_id:
609
+ try:
610
+ effect, meta = reduce_gene_effect(out.tidy, spec.gene_id, tracks)
611
+ except ValueError as exc:
612
+ rec |= {"status": "failed", "error": str(exc), "error_type": "validation"}
613
+ else:
614
+ rec |= {"raw_effect": effect, **meta}
615
+ return rec
616
+
617
+ def score(
618
+ self,
619
+ spec: RequestSpec,
620
+ conditioning: Any = None,
621
+ strict: bool = False,
622
+ tracks: Tracks | None = None,
623
+ ) -> dict:
624
+ """Scores one variant under one condition.
625
+
626
+ The score is a predicted ALT-minus-REF effect. RNA_SEQ uses a log fold
627
+ change, while SC_GENE_EXPRESSION_DENOISED uses a difference in summed
628
+ predicted expression.
629
+
630
+ Args:
631
+ spec: Variant, model inputs, and optional gene selection.
632
+ conditioning: Cell-conditioning AnnData, or None for reference conditions.
633
+ strict: Whether to raise for failed or blocked records.
634
+ tracks: Optional track metadata filters used in gene reduction.
635
+ Filters share the cache entry for the full scorer output.
636
+
637
+ Returns:
638
+ A record with provenance, status, and raw_effect for a successful
639
+ gene reduction. Otherwise raw_effect is NaN. Failed or blocked
640
+ records include error details.
641
+
642
+ Raises:
643
+ ScoringError: If strict is true and scoring fails or is blocked.
644
+ """
645
+ rec = self._record(spec, conditioning, self._obtain(spec, conditioning), tracks)
646
+ if strict and rec["status"] in ("failed", "blocked"):
647
+ raise ScoringError(f"{rec['variant']}: {rec['error']}", rec["error_type"], rec)
648
+ return rec
649
+
650
+ def score_many(
651
+ self,
652
+ specs: Iterable[RequestSpec],
653
+ conditioning: Any = None,
654
+ max_workers: int = 4,
655
+ tracks: Tracks | None = None,
656
+ ) -> list[dict]:
657
+ """Scores specs concurrently, fetching each distinct request once.
658
+
659
+ Individual request failures are recorded without cancelling the rest.
660
+ Cached requests can be reused after an interruption.
661
+
662
+ Args:
663
+ specs: Requests in the desired output order.
664
+ conditioning: One AnnData for all requests, a mapping from
665
+ condition_label to AnnData, or None for reference conditions.
666
+ max_workers: Maximum number of concurrent workers. Must be positive.
667
+ tracks: Optional track metadata filters used in gene reduction.
668
+
669
+ Returns:
670
+ One scoring record per spec, in input order.
671
+
672
+ Raises:
673
+ ValueError: If max_workers is below one.
674
+ """
675
+ specs = list(specs)
676
+
677
+ def cells(spec: RequestSpec) -> Any:
678
+ if isinstance(conditioning, Mapping):
679
+ return conditioning.get(spec.condition_label)
680
+ return conditioning
681
+
682
+ pairs = [(s, cells(s)) for s in specs]
683
+ outcomes = self._fetch(pairs, max_workers)
684
+ return [self._record(s, c, o, tracks) for (s, c), o in zip(pairs, outcomes, strict=True)]
685
+
686
+ def _fetch(self, pairs: list[tuple[RequestSpec, Any]], max_workers: int) -> list[_Outcome]:
687
+ """Validates specs individually before sharing fetches by cache key.
688
+
689
+ Specs can share cached output while still having different validation results.
690
+ """
691
+ problems = [self._check(s, c) for s, c in pairs]
692
+ keys = [self.cache_key(s, c) for s, c in pairs]
693
+ first: dict[str, tuple[RequestSpec, Any]] = {}
694
+ for key, pair, problem in zip(keys, pairs, problems, strict=True):
695
+ if not problem:
696
+ first.setdefault(key, pair)
697
+ with ThreadPoolExecutor(max_workers=max_workers) as pool:
698
+ done = dict(
699
+ zip(first, pool.map(lambda p: self._obtain(*p), first.values()), strict=True)
700
+ )
701
+ return [
702
+ _Outcome(None, "failed", p, "validation") if p else done[k]
703
+ for k, p in zip(keys, problems, strict=True)
704
+ ]
705
+
706
+ def score_variants(
707
+ self,
708
+ variants: pd.DataFrame,
709
+ *,
710
+ condition: str = "",
711
+ scorer: str = "RNA_SEQ",
712
+ gene_column: str | None = None,
713
+ conditioning: Any = None,
714
+ condition_label: str = "",
715
+ sequence_length: int = DEFAULT_SEQUENCE_LENGTH,
716
+ max_workers: int = 4,
717
+ tracks: Tracks | None = None,
718
+ ) -> pd.DataFrame:
719
+ """Scores a variant table and reduces each result by gene.
720
+
721
+ Supply either a reference condition or cell conditioning with a label.
722
+
723
+ Args:
724
+ variants: Table with chrom, 1-based pos, ref, and alt. Uses variant_id,
725
+ assembly, and input_status when present.
726
+ condition: Reference condition as an ontology CURIE.
727
+ scorer: SDK recommended scorer name.
728
+ gene_column: Column naming the target gene for each row. None reports
729
+ every gene in the scorer output.
730
+ conditioning: One AnnData for a cell-conditioned batch.
731
+ condition_label: SDK track label for the cell condition.
732
+ sequence_length: Supported input window length in bases.
733
+ max_workers: Maximum number of concurrent workers. Must be positive.
734
+ tracks: Optional track metadata filters used in gene reduction.
735
+
736
+ Returns:
737
+ Scoring rows with a zero-based input row number and input_ columns.
738
+ Failures are represented by status and reason, including variant-level
739
+ failures when gene selection depends on missing scorer output.
740
+
741
+ Raises:
742
+ ValueError: If required variant columns are missing or max_workers
743
+ is below one for a nonempty batch.
744
+ KeyError: If the named gene column is missing from a nonempty batch.
745
+ TypeError: If conditioning is a mapping.
746
+ """
747
+ missing = {"chrom", "pos", "ref", "alt"} - set(variants.columns)
748
+ if missing:
749
+ raise ValueError(f"variants is missing columns {sorted(missing)}")
750
+ if isinstance(conditioning, Mapping):
751
+ raise TypeError("score_variants takes one AnnData; call it once per condition")
752
+ if variants.empty:
753
+ return pd.DataFrame(columns=["row", "status", "error", "error_type"])
754
+ n_cells = int(getattr(conditioning, "n_obs", 0) or 0)
755
+ pairs: list[tuple[int, RequestSpec | None, str]] = []
756
+ input_rows = variants.to_dict("records")
757
+ for i, row in enumerate(input_rows):
758
+ problem = str(row.get("input_status", "ok") or "ok")
759
+ problem = "" if problem == "ok" else f"input_status={problem}"
760
+ assembly = str(row.get("assembly", "GRCh38") or "GRCh38")
761
+ if not problem and assembly.upper() not in ("GRCH38", "HG38"):
762
+ problem = f"assembly {assembly}: AlphaGenome scores GRCh38; lift over first"
763
+ spec = None
764
+ if not problem:
765
+ try:
766
+ spec = RequestSpec(
767
+ variant_id=str(row.get("variant_id") or ""),
768
+ chrom=row["chrom"],
769
+ pos=row["pos"],
770
+ ref=row["ref"],
771
+ alt=row["alt"],
772
+ gene_id=str(row[gene_column]) if gene_column else "",
773
+ condition=condition,
774
+ condition_label=condition_label,
775
+ n_cells=n_cells,
776
+ model_version=self.model_version,
777
+ scorer_name=scorer,
778
+ sequence_length=sequence_length,
779
+ )
780
+ except (ValueError, TypeError) as exc:
781
+ problem = str(exc)
782
+ pairs.append((i, spec, problem))
783
+
784
+ valid = [(s, conditioning) for _, s, _ in pairs if s is not None]
785
+ outcomes = iter(self._fetch(valid, max_workers))
786
+ records: list[dict] = []
787
+ for i, spec, problem in pairs:
788
+ if spec is None:
789
+ row = input_rows[i]
790
+ records.append(
791
+ {
792
+ "row": i,
793
+ "chrom": "chr" + str(row["chrom"]).removeprefix("chr"),
794
+ "pos": row["pos"],
795
+ "ref": str(row["ref"]).upper(),
796
+ "alt": str(row["alt"]).upper(),
797
+ "gene_id": str(row[gene_column]) if gene_column else "",
798
+ "condition": condition,
799
+ "status": "failed",
800
+ "error": problem,
801
+ "error_type": "validation",
802
+ }
803
+ )
804
+ continue
805
+ out = next(outcomes)
806
+ if gene_column or out.tidy is None:
807
+ records.append({"row": i, **self._record(spec, conditioning, out, tracks)})
808
+ continue
809
+ names = out.tidy.drop_duplicates("gene_id").set_index("gene_id")["gene_name"]
810
+ for gene, name in names.items():
811
+ rec = self._record(replace(spec, gene_id=str(gene)), conditioning, out, tracks)
812
+ records.append({"row": i, **rec, "gene_name": name})
813
+ table = pd.DataFrame.from_records(records)
814
+ inputs = variants.reset_index(drop=True).add_prefix("input_")
815
+ return inputs.join(table.set_index("row"), how="right").rename_axis("row").reset_index()
816
+
817
+
818
+ def _normalise(tracks: Tracks | None) -> dict[str, list[str]]:
819
+ return {
820
+ str(k): [v] if isinstance(v, str) else [str(x) for x in v]
821
+ for k, v in (tracks or {}).items()
822
+ }
823
+
824
+
825
+ def reduce_gene_effect(
826
+ tidy: pd.DataFrame, gene_id: str, tracks: Tracks | None = None
827
+ ) -> tuple[float, dict]:
828
+ """Averages signed scores for a gene over strand-compatible tracks.
829
+
830
+ Unstranded tracks are included, matching the SDK's strand-filtering
831
+ rule. Filter imputation_mode to "observed" to average tracks observed
832
+ in training data separately from imputed tracks.
833
+
834
+ Args:
835
+ tidy: SDK tidy scorer output with all strands.
836
+ gene_id: Ensembl gene ID. Version suffixes are ignored.
837
+ tracks: Optional mapping from metadata columns to allowed values.
838
+
839
+ Returns:
840
+ Mean raw_score and metadata with the number of values, gene strand,
841
+ and up to four distinct track names.
842
+
843
+ Raises:
844
+ ValueError: If the gene or a filter column is absent, no tracks match,
845
+ or any selected score is non-finite.
846
+ """
847
+ target = str(gene_id).split(".")[0]
848
+ rows = tidy[tidy["gene_id"].astype(str).str.split(".").str[0] == target]
849
+ if rows.empty:
850
+ raise ValueError(f"target gene {gene_id} absent from scorer output")
851
+ for column, allowed in _normalise(tracks).items():
852
+ if column not in rows:
853
+ raise ValueError(f"track filter column {column!r} not in {sorted(tidy.columns)}")
854
+ rows = rows[rows[column].astype(str).isin(allowed)]
855
+ if rows.empty:
856
+ raise ValueError(f"no track of gene {gene_id} has {column} in {allowed}")
857
+ gs, ts = rows["gene_strand"], rows["track_strand"]
858
+ rows = rows[~(((gs == "+") & (ts == "-")) | ((gs == "-") & (ts == "+")))]
859
+ values = rows["raw_score"].to_numpy(dtype=float)
860
+ if values.size == 0:
861
+ raise ValueError(f"no track on the strand of gene {gene_id}")
862
+ if not np.all(np.isfinite(values)):
863
+ raise ValueError(f"non-finite score for gene {gene_id}")
864
+ meta = {
865
+ "n_reduced_values": int(values.size),
866
+ "gene_strand": str(rows["gene_strand"].iloc[0]),
867
+ "track_names": ";".join(map(str, rows["track_name"].unique()[:4])),
868
+ }
869
+ return float(values.mean()), meta
870
+
871
+
872
+ def extract_gene_effect(adata: Any, gene_id: str) -> tuple[float, dict]:
873
+ """Reduces a scorer AnnData to one gene's strand-compatible effect.
874
+
875
+ Args:
876
+ adata: SDK scorer result with genes as rows and tracks as columns.
877
+ gene_id: Ensembl ID of the target gene.
878
+
879
+ Returns:
880
+ Mean signed effect and reduction metadata, including the total number
881
+ of tracks in adata before filtering.
882
+
883
+ Raises:
884
+ ImportError: If the AlphaGenome SDK is unavailable.
885
+ ValueError: If the gene is absent, has no compatible tracks, or has
886
+ a non-finite selected score.
887
+ """
888
+ from alphagenome.models.v2 import variant_scorers
889
+
890
+ tidy = variant_scorers.tidy_anndata(
891
+ adata, match_gene_strand=False, include_extended_metadata=False
892
+ )
893
+ effect, meta = reduce_gene_effect(tidy, gene_id)
894
+ return effect, {**meta, "n_tracks": int(adata.shape[1])}
895
+
896
+
897
+ def conditioning_fingerprint(conditioning: Any) -> str:
898
+ """Hashes cell-conditioning inputs for request identity.
899
+
900
+ Args:
901
+ conditioning: AnnData with a count matrix and ordered cell and gene names,
902
+ or None. Objects without a matrix are identified by their repr.
903
+
904
+ Returns:
905
+ SHA-256 digest, or an empty string for None.
906
+ """
907
+ if conditioning is None:
908
+ return ""
909
+ import scipy.sparse as sp
910
+
911
+ h = hashlib.sha256()
912
+ x = getattr(conditioning, "X", None)
913
+ if x is None:
914
+ h.update(repr(conditioning).encode())
915
+ return h.hexdigest()
916
+ x = np.ascontiguousarray(x.toarray() if sp.issparse(x) else np.asarray(x))
917
+ h.update(f"{x.dtype}{x.shape}".encode())
918
+ h.update(x.tobytes())
919
+ names = [list(map(str, names)) for names in (conditioning.obs_names, conditioning.var_names)]
920
+ h.update(json.dumps(names).encode())
921
+ return h.hexdigest()
922
+
923
+
924
+ def interaction(score_treated: float | None, score_control: float | None) -> float:
925
+ """Computes a treated-minus-control difference in predicted effects.
926
+
927
+ Args:
928
+ score_treated: Predicted effect under treatment.
929
+ score_control: Predicted effect for the same variant, gene, and scorer
930
+ under control.
931
+
932
+ Returns:
933
+ Difference in predicted effects, or NaN if either score is missing
934
+ or non-finite.
935
+ """
936
+ if score_treated is None or score_control is None:
937
+ return float("nan")
938
+ if not (np.isfinite(score_treated) and np.isfinite(score_control)):
939
+ return float("nan")
940
+ return float(score_treated) - float(score_control)