allelix 2.5.0__tar.gz → 2.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {allelix-2.5.0/allelix.egg-info → allelix-2.5.1}/PKG-INFO +24 -3
- {allelix-2.5.0 → allelix-2.5.1}/README.md +22 -2
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/_resolution.py +2 -1
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/clinvar.py +114 -70
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/gwas.py +9 -3
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/pharmgkb.py +43 -4
- {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/_helpers.py +73 -24
- {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/_options.py +5 -2
- {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/db.py +13 -6
- {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/utility.py +208 -34
- {allelix-2.5.0 → allelix-2.5.1}/allelix/compare.py +76 -10
- allelix-2.5.1/allelix/databases/cache_state.py +359 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/loader_utils.py +10 -1
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/manager.py +60 -37
- allelix-2.5.1/allelix/exporters/plink.py +398 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/models.py +5 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/__init__.py +4 -1
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/_helpers.py +93 -16
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/vcf.py +208 -30
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/__init__.py +162 -15
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/_pipeline.py +351 -189
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/diff.py +93 -34
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/html.py +60 -29
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/json_report.py +17 -4
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/terminal.py +18 -1
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/vcf.py +120 -64
- {allelix-2.5.0 → allelix-2.5.1/allelix.egg-info}/PKG-INFO +24 -3
- {allelix-2.5.0 → allelix-2.5.1}/pyproject.toml +2 -1
- allelix-2.5.0/allelix/databases/cache_state.py +0 -161
- allelix-2.5.0/allelix/exporters/plink.py +0 -243
- {allelix-2.5.0 → allelix-2.5.1}/LICENSE +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/MANIFEST.in +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/__init__.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/__init__.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/alphamissense.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/base.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/cadd.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/gnomad.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/rsid_identity.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/snpedia.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/__init__.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/analyze.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/config.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/focused.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/config.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/data/__init__.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/data/clinvar_clnsig_snapshot.yaml +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/data/high_value_snps.yaml +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/__init__.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/_versions.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/alphamissense_loader.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/cadd_loader.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/cpic_loader.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/gnomad_loader.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/gwas_loader.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/pharmgkb_loader.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/rsid_identity_loader.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/schema.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/snpedia_loader.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/snpedia_parser.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/exporters/__init__.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/ancestrydna.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/base.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/decodeme.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/ftdna.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/ftdna_famfinder.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/ftdna_illumina.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/genesforgood.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/livingdna.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/myhappygenes.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/myheritage.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/twentythreeandme.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/py.typed +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/_sources.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/association_text.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/build_text.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/clinvar_text.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/high_value.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/methylation.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/utils/__init__.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/utils/allele.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/utils/build_detect.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix/utils/plausibility.py +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/SOURCES.txt +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/dependency_links.txt +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/entry_points.txt +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/requires.txt +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/top_level.txt +0 -0
- {allelix-2.5.0 → allelix-2.5.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: allelix
|
|
3
|
-
Version: 2.5.
|
|
3
|
+
Version: 2.5.1
|
|
4
4
|
Summary: Open-source genotype analysis toolkit. Format-agnostic ingestion, database-agnostic annotation, offline-first.
|
|
5
5
|
Author: Allelix
|
|
6
6
|
Maintainer-email: dial481 <dial481@users.noreply.github.com>
|
|
@@ -16,6 +16,7 @@ Classifier: Operating System :: OS Independent
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
20
|
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
20
21
|
Requires-Python: >=3.11
|
|
21
22
|
Description-Content-Type: text/markdown
|
|
@@ -87,6 +88,8 @@ allelix analyze your_wgs.vcf.gz --keep-filtered --min-gq 10 --output report.html
|
|
|
87
88
|
allelix analyze your_genotype_file.txt --filter-file my_panel.txt --output report.html
|
|
88
89
|
# Panel coverage puts each requested rsID in one state: annotated, typed (no annotation),
|
|
89
90
|
# no call, or not on chip ("not in file" for a VCF)
|
|
91
|
+
# VCF coverage includes explicit REF/REF calls and explains rejected/excluded records.
|
|
92
|
+
# Gene entries match constituent symbols in lists; hyphens inside symbols stay intact.
|
|
90
93
|
|
|
91
94
|
# Annotated VCF output — VCF-in / VCF-out for pipeline integration
|
|
92
95
|
# Writes a valid VCF with annotations as INFO fields + full provenance in the header.
|
|
@@ -425,11 +428,29 @@ The banner says where the build came from, for example `Build: GRCh38 (from the
|
|
|
425
428
|
|
|
426
429
|
The message names the evidence and asks you to say which build the file uses: `--build grch37` or `--build grch38`, or `--build grch36` for an old GRCh36 / hg18 file, which then runs with position-keyed lookups off (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)). For `compare`, use `--build1` / `--build2`. A stop leaves no report, annotated VCF, diff, PLINK files or extract table behind. A forced build is recorded as such in every report.
|
|
427
430
|
|
|
431
|
+
### Keeping input and output files safe
|
|
432
|
+
|
|
433
|
+
Report `--output`, annotated `--vcf-out` and all three actual PLINK output paths
|
|
434
|
+
must differ from the input, including symlink and hard-link aliases. These
|
|
435
|
+
checks run before cache setup or output creation. Inputs that alias the former
|
|
436
|
+
fixed report or PLINK `.tmp` path are also refused; unrelated old temp files
|
|
437
|
+
are left untouched. Annotated VCF still refuses any existing output at the CLI.
|
|
438
|
+
|
|
439
|
+
Writers use unique temporary files in the destination directory and publish
|
|
440
|
+
only complete, closed content. Failure before publication preserves previous
|
|
441
|
+
outputs. PLINK stages all three files first, then publishes each atomically;
|
|
442
|
+
a later publication failure can leave complete new files next to unchanged old
|
|
443
|
+
ones. Existing ordinary permission bits are retained when replacement is
|
|
444
|
+
permitted; new staging files start private. Ownership, ACLs, extended attributes,
|
|
445
|
+
crash durability and concurrent path swaps are outside this guarantee.
|
|
446
|
+
|
|
428
447
|
### Is my database stale? How fresh is the data?
|
|
429
448
|
|
|
430
449
|
`allelix db status` shows the ingestion date of each cache. The freshness model is signal-driven: `allelix db update` queries an upstream freshness signal (md5 / etag) per source and re-ingests only when the signal changed. There's no time-based staleness — a 6-month-old cache against an unchanged source IS current.
|
|
431
450
|
|
|
432
|
-
If you want to force a refresh: `allelix db update --force` re-downloads every cache regardless of freshness signal.
|
|
451
|
+
If you want to force a refresh: `allelix db update --force` re-downloads every cache regardless of freshness signal. ClinVar is never refreshed unverified: its freshness signal is NCBI's checksum for the submission summary, which also verifies the download. If that checksum cannot be fetched (after one retry), ClinVar is not refreshed, with or without `--force`; the existing caches are kept, and a later `db update` tries again. The GWAS Catalog and ClinPGx publish no checksum, so `--force` refreshes them even when their freshness signal cannot be fetched; it stores no signal, and the next ordinary `db update` refreshes them once more when the signal is available again.
|
|
452
|
+
|
|
453
|
+
A ClinPGx cache built while CPIC's allele-function data could not be fetched is marked as such: `db status` shows "(no CPIC)", and the next `db update` tries CPIC again. A cache from 2.5.0 that claims CPIC data but holds none is repaired the same way by an ordinary `db update`.
|
|
433
454
|
|
|
434
455
|
### Output reports contain real genome annotations
|
|
435
456
|
|
|
@@ -472,7 +493,7 @@ Release history: see [`CHANGELOG.md`](https://github.com/allelix/allelix/blob/ma
|
|
|
472
493
|
|
|
473
494
|
```bash
|
|
474
495
|
source .venv/bin/activate
|
|
475
|
-
pip install -e ".[dev]"
|
|
496
|
+
pip install -e ".[dev,vcf-index]" # the CI install: vcf-index (pysam) runs the indexed-VCF tests
|
|
476
497
|
|
|
477
498
|
# One-time: install pre-commit hooks
|
|
478
499
|
pre-commit install --hook-type pre-commit
|
|
@@ -48,6 +48,8 @@ allelix analyze your_wgs.vcf.gz --keep-filtered --min-gq 10 --output report.html
|
|
|
48
48
|
allelix analyze your_genotype_file.txt --filter-file my_panel.txt --output report.html
|
|
49
49
|
# Panel coverage puts each requested rsID in one state: annotated, typed (no annotation),
|
|
50
50
|
# no call, or not on chip ("not in file" for a VCF)
|
|
51
|
+
# VCF coverage includes explicit REF/REF calls and explains rejected/excluded records.
|
|
52
|
+
# Gene entries match constituent symbols in lists; hyphens inside symbols stay intact.
|
|
51
53
|
|
|
52
54
|
# Annotated VCF output — VCF-in / VCF-out for pipeline integration
|
|
53
55
|
# Writes a valid VCF with annotations as INFO fields + full provenance in the header.
|
|
@@ -386,11 +388,29 @@ The banner says where the build came from, for example `Build: GRCh38 (from the
|
|
|
386
388
|
|
|
387
389
|
The message names the evidence and asks you to say which build the file uses: `--build grch37` or `--build grch38`, or `--build grch36` for an old GRCh36 / hg18 file, which then runs with position-keyed lookups off (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)). For `compare`, use `--build1` / `--build2`. A stop leaves no report, annotated VCF, diff, PLINK files or extract table behind. A forced build is recorded as such in every report.
|
|
388
390
|
|
|
391
|
+
### Keeping input and output files safe
|
|
392
|
+
|
|
393
|
+
Report `--output`, annotated `--vcf-out` and all three actual PLINK output paths
|
|
394
|
+
must differ from the input, including symlink and hard-link aliases. These
|
|
395
|
+
checks run before cache setup or output creation. Inputs that alias the former
|
|
396
|
+
fixed report or PLINK `.tmp` path are also refused; unrelated old temp files
|
|
397
|
+
are left untouched. Annotated VCF still refuses any existing output at the CLI.
|
|
398
|
+
|
|
399
|
+
Writers use unique temporary files in the destination directory and publish
|
|
400
|
+
only complete, closed content. Failure before publication preserves previous
|
|
401
|
+
outputs. PLINK stages all three files first, then publishes each atomically;
|
|
402
|
+
a later publication failure can leave complete new files next to unchanged old
|
|
403
|
+
ones. Existing ordinary permission bits are retained when replacement is
|
|
404
|
+
permitted; new staging files start private. Ownership, ACLs, extended attributes,
|
|
405
|
+
crash durability and concurrent path swaps are outside this guarantee.
|
|
406
|
+
|
|
389
407
|
### Is my database stale? How fresh is the data?
|
|
390
408
|
|
|
391
409
|
`allelix db status` shows the ingestion date of each cache. The freshness model is signal-driven: `allelix db update` queries an upstream freshness signal (md5 / etag) per source and re-ingests only when the signal changed. There's no time-based staleness — a 6-month-old cache against an unchanged source IS current.
|
|
392
410
|
|
|
393
|
-
If you want to force a refresh: `allelix db update --force` re-downloads every cache regardless of freshness signal.
|
|
411
|
+
If you want to force a refresh: `allelix db update --force` re-downloads every cache regardless of freshness signal. ClinVar is never refreshed unverified: its freshness signal is NCBI's checksum for the submission summary, which also verifies the download. If that checksum cannot be fetched (after one retry), ClinVar is not refreshed, with or without `--force`; the existing caches are kept, and a later `db update` tries again. The GWAS Catalog and ClinPGx publish no checksum, so `--force` refreshes them even when their freshness signal cannot be fetched; it stores no signal, and the next ordinary `db update` refreshes them once more when the signal is available again.
|
|
412
|
+
|
|
413
|
+
A ClinPGx cache built while CPIC's allele-function data could not be fetched is marked as such: `db status` shows "(no CPIC)", and the next `db update` tries CPIC again. A cache from 2.5.0 that claims CPIC data but holds none is repaired the same way by an ordinary `db update`.
|
|
394
414
|
|
|
395
415
|
### Output reports contain real genome annotations
|
|
396
416
|
|
|
@@ -433,7 +453,7 @@ Release history: see [`CHANGELOG.md`](https://github.com/allelix/allelix/blob/ma
|
|
|
433
453
|
|
|
434
454
|
```bash
|
|
435
455
|
source .venv/bin/activate
|
|
436
|
-
pip install -e ".[dev]"
|
|
456
|
+
pip install -e ".[dev,vcf-index]" # the CI install: vcf-index (pysam) runs the indexed-VCF tests
|
|
437
457
|
|
|
438
458
|
# One-time: install pre-commit hooks
|
|
439
459
|
pre-commit install --hook-type pre-commit
|
|
@@ -81,7 +81,8 @@ def resolve_batch(
|
|
|
81
81
|
assert identity is not None
|
|
82
82
|
_populate_ref_from_identity(identity, variants)
|
|
83
83
|
if clinvar is not None:
|
|
84
|
-
# bulk_resolve_rsids skips any build whose cache is missing
|
|
84
|
+
# bulk_resolve_rsids skips any build whose cache is missing or not
|
|
85
|
+
# ready for this version.
|
|
85
86
|
rsidless = [v for v in variants if not v.rsid.startswith("rs")]
|
|
86
87
|
if rsidless:
|
|
87
88
|
for (chrom, pos, ref, alt), rsid in clinvar.bulk_resolve_rsids(rsidless).items():
|
|
@@ -14,7 +14,10 @@ and dispatches per-variant by `variant.build`.
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
16
|
import logging
|
|
17
|
+
import os
|
|
17
18
|
import re
|
|
19
|
+
import tempfile
|
|
20
|
+
from pathlib import Path
|
|
18
21
|
from typing import TYPE_CHECKING, ClassVar
|
|
19
22
|
|
|
20
23
|
from allelix.annotators.base import Annotator, LicenseDescriptor, variant_locus
|
|
@@ -50,7 +53,6 @@ from allelix.utils.build_detect import BUILD_GRCH37, BUILD_GRCH38
|
|
|
50
53
|
if TYPE_CHECKING:
|
|
51
54
|
import sqlite3
|
|
52
55
|
from collections.abc import Callable, Iterable, Iterator
|
|
53
|
-
from pathlib import Path
|
|
54
56
|
|
|
55
57
|
from allelix.models import Variant
|
|
56
58
|
|
|
@@ -60,13 +62,29 @@ CLINVAR_SUPPORTED_BUILDS: tuple[str, ...] = ("GRCh37", "GRCh38")
|
|
|
60
62
|
|
|
61
63
|
_BATCH_CHUNK = 500 # SQLite default SQLITE_MAX_VARIABLE_NUMBER is 999
|
|
62
64
|
|
|
63
|
-
# a remote .md5 endpoint can return an HTML error page on a
|
|
64
|
-
# transient blip. The first whitespace-separated token of the body is
|
|
65
|
-
# what we treat as the hash, so without this gate `<!DOCTYPE` would be
|
|
66
|
-
# accepted as the "signal" and later passed to `verify_file_hash`, which
|
|
67
|
-
# would then delete the freshly downloaded VCF. MD5 is exactly 32 hex
|
|
68
|
-
# digits; reject anything else.
|
|
69
65
|
_MD5_HEX_RE = re.compile(r"^[0-9a-fA-F]{32}$")
|
|
66
|
+
_CHECKSUM_MAX_BYTES = 4096
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _fetch_checksum(source_url: str) -> str | None:
|
|
70
|
+
"""Read a bounded MD5 sidecar associated with this source's basename."""
|
|
71
|
+
body = fetch_remote_text(source_url + ".md5", max_bytes=_CHECKSUM_MAX_BYTES)
|
|
72
|
+
if not body or len(body.encode("utf-8")) > _CHECKSUM_MAX_BYTES:
|
|
73
|
+
return None
|
|
74
|
+
expected_name = source_url.rsplit("/", 1)[-1]
|
|
75
|
+
digests: set[str] = set()
|
|
76
|
+
for line in body.splitlines():
|
|
77
|
+
parts = line.split(None, 1)
|
|
78
|
+
if not parts:
|
|
79
|
+
continue
|
|
80
|
+
if not _MD5_HEX_RE.fullmatch(parts[0]):
|
|
81
|
+
return None
|
|
82
|
+
if len(parts) == 2:
|
|
83
|
+
filename = parts[1].strip().removeprefix("*")
|
|
84
|
+
if filename.rsplit("/", 1)[-1] != expected_name:
|
|
85
|
+
return None
|
|
86
|
+
digests.add(parts[0].lower())
|
|
87
|
+
return digests.pop() if len(digests) == 1 else None
|
|
70
88
|
|
|
71
89
|
|
|
72
90
|
def clinvar_db_filename(build: str) -> str:
|
|
@@ -329,6 +347,9 @@ class ClinVarAnnotator(Annotator):
|
|
|
329
347
|
# allele from the other build (the build's reference carries a
|
|
330
348
|
# ClinVar variant allele), with that allele.
|
|
331
349
|
self._normal_alleles: dict[str, dict[str, str]] = {}
|
|
350
|
+
# Per build, whether its cache is ready for this version: rsID
|
|
351
|
+
# resolution outside annotation uses only a ready cache.
|
|
352
|
+
self._ready_builds: dict[str, bool] = {}
|
|
332
353
|
|
|
333
354
|
def serves_build(self, build: str) -> bool:
|
|
334
355
|
"""Only the builds this run manages (the ready caches, when any are)."""
|
|
@@ -375,75 +396,106 @@ class ClinVarAnnotator(Annotator):
|
|
|
375
396
|
endpoint — that file drives the meaningful change cadence
|
|
376
397
|
(per-SCV submissions land daily) and variant_summary turns over
|
|
377
398
|
weekly on a slower clock.
|
|
399
|
+
|
|
400
|
+
Both compressed sources are checked before ingest. All selected
|
|
401
|
+
builds meet fixed/relative row floors and have final metadata
|
|
402
|
+
before publication. Replacements are individually atomic; a later
|
|
403
|
+
replacement failure reports any build already updated.
|
|
404
|
+
|
|
405
|
+
The freshness signal is NCBI's submission_summary checksum, which
|
|
406
|
+
also verifies that download, so a refresh is never done without it.
|
|
407
|
+
A failed fetch is tried once more (a transient failure may clear).
|
|
378
408
|
"""
|
|
379
|
-
signal = self._fetch_remote_signal_for_tsv()
|
|
409
|
+
signal = self._fetch_remote_signal_for_tsv() or self._fetch_remote_signal_for_tsv()
|
|
380
410
|
if signal is None:
|
|
381
|
-
|
|
382
|
-
"clinvar:
|
|
383
|
-
"
|
|
384
|
-
"
|
|
385
|
-
"will re-download to re-establish the signal."
|
|
411
|
+
raise RuntimeError(
|
|
412
|
+
"clinvar: ClinVar not refreshed: NCBI's checksum for submission_summary "
|
|
413
|
+
"is unavailable, so the download cannot be verified. Existing caches "
|
|
414
|
+
"kept; try again later."
|
|
386
415
|
)
|
|
387
|
-
raise RuntimeError(msg)
|
|
388
416
|
|
|
389
417
|
vs_url = _manager_module.CLINVAR_VARIANT_SUMMARY_URL
|
|
390
418
|
ss_url = _manager_module.CLINVAR_SUBMISSION_SUMMARY_URL
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
#
|
|
400
|
-
#
|
|
401
|
-
|
|
419
|
+
previous_counts = {build: self._previous_record_count(build) for build in self._builds}
|
|
420
|
+
variant_md5 = _fetch_checksum(vs_url)
|
|
421
|
+
if variant_md5 is None:
|
|
422
|
+
raise RuntimeError(
|
|
423
|
+
"clinvar: cannot verify variant_summary checksum; refresh aborted, "
|
|
424
|
+
"existing caches kept"
|
|
425
|
+
)
|
|
426
|
+
self.data_dir.mkdir(parents=True, exist_ok=True)
|
|
427
|
+
# All owned downloads and candidates live on the cache filesystem.
|
|
428
|
+
# Validation completes for every selected build before publication.
|
|
429
|
+
with tempfile.TemporaryDirectory(prefix=".clinvar-", dir=self.data_dir) as temporary:
|
|
430
|
+
staging = Path(temporary)
|
|
431
|
+
vs_path = staging / "variant_summary.txt.gz"
|
|
432
|
+
ss_path = staging / "submission_summary.txt.gz"
|
|
433
|
+
download(vs_url, vs_path)
|
|
434
|
+
download(ss_url, ss_path)
|
|
435
|
+
verify_file_hash(vs_path, "md5", variant_md5)
|
|
402
436
|
verify_file_hash(ss_path, "md5", signal.removeprefix("md5:"))
|
|
437
|
+
candidates = {}
|
|
403
438
|
for build in self._builds:
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
439
|
+
candidate = staging / clinvar_db_filename(build)
|
|
440
|
+
self._setup_one_from_tsv(
|
|
441
|
+
vs_path, ss_path, candidate, build, signal, vs_url, previous_counts[build]
|
|
442
|
+
)
|
|
443
|
+
candidates[build] = candidate
|
|
444
|
+
self.close()
|
|
445
|
+
published: list[str] = []
|
|
446
|
+
for build, candidate in candidates.items():
|
|
407
447
|
try:
|
|
408
|
-
|
|
409
|
-
except
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
448
|
+
os.replace(candidate, self._db_paths[build])
|
|
449
|
+
except OSError as exc:
|
|
450
|
+
kept = (
|
|
451
|
+
"old cache kept"
|
|
452
|
+
if self._db_paths[build].exists()
|
|
453
|
+
else "no cache installed"
|
|
454
|
+
)
|
|
455
|
+
raise RuntimeError(
|
|
456
|
+
f"ClinVar {build}: publication failed; {kept}; "
|
|
457
|
+
f"already updated: {', '.join(published) or 'none'}. {exc}"
|
|
458
|
+
) from exc
|
|
459
|
+
published.append(build)
|
|
460
|
+
|
|
461
|
+
def _previous_record_count(self, build: str) -> int | None:
|
|
462
|
+
"""Use existing metadata, refusing an unknown baseline without a full scan."""
|
|
463
|
+
path = self._db_paths[build]
|
|
464
|
+
if not path.exists():
|
|
465
|
+
return None
|
|
466
|
+
info = get_database_info(path, clinvar_record_name(build))
|
|
467
|
+
if info is None:
|
|
468
|
+
info = get_database_info(path, "clinvar")
|
|
469
|
+
count = info["record_count"] if info is not None else None
|
|
470
|
+
if type(count) is not int or count <= 0:
|
|
471
|
+
raise RuntimeError(
|
|
472
|
+
f"ClinVar {build}: existing cache record count is missing or invalid; "
|
|
473
|
+
"cannot establish the relative floor, existing caches kept"
|
|
474
|
+
)
|
|
475
|
+
return count
|
|
413
476
|
|
|
414
477
|
def _setup_one_from_tsv(
|
|
415
478
|
self,
|
|
416
479
|
vs_path: Path,
|
|
417
480
|
ss_path: Path,
|
|
481
|
+
candidate: Path,
|
|
418
482
|
build: str,
|
|
419
483
|
signal: str,
|
|
420
484
|
source_url: str,
|
|
485
|
+
previous_count: int | None,
|
|
421
486
|
) -> None:
|
|
422
|
-
"""
|
|
487
|
+
"""Finish and validate a candidate against the downloaded source pair."""
|
|
423
488
|
load_clinvar_tsv(
|
|
424
489
|
vs_path,
|
|
425
490
|
ss_path,
|
|
426
|
-
|
|
491
|
+
candidate,
|
|
427
492
|
build,
|
|
428
493
|
source_url=source_url,
|
|
429
494
|
remote_signal=signal,
|
|
495
|
+
record_name=clinvar_record_name(build),
|
|
496
|
+
min_rows=_manager_module.CLINVAR_MIN_ROWS[build],
|
|
497
|
+
previous_count=previous_count,
|
|
430
498
|
)
|
|
431
|
-
# load_clinvar_tsv stamps the record name as "clinvar"; for
|
|
432
|
-
# backward compatibility with the per-build record-name scheme
|
|
433
|
-
# (clinvar.GRCh37 / clinvar.GRCh38), rename the row in place.
|
|
434
|
-
# This keeps is_ready() / version() per-build dispatch working
|
|
435
|
-
# against the existing database_versions table layout.
|
|
436
|
-
import contextlib
|
|
437
|
-
import sqlite3
|
|
438
|
-
|
|
439
|
-
per_build_name = clinvar_record_name(build)
|
|
440
|
-
if per_build_name != "clinvar":
|
|
441
|
-
with contextlib.closing(sqlite3.connect(self._db_paths[build])) as conn:
|
|
442
|
-
conn.execute(
|
|
443
|
-
"UPDATE database_versions SET name = ? WHERE name = 'clinvar'",
|
|
444
|
-
(per_build_name,),
|
|
445
|
-
)
|
|
446
|
-
conn.commit()
|
|
447
499
|
|
|
448
500
|
def is_ready(self) -> bool:
|
|
449
501
|
"""True iff EVERY managed build has a populated cache with the current tag."""
|
|
@@ -519,6 +571,7 @@ class ClinVarAnnotator(Annotator):
|
|
|
519
571
|
self._conns.clear()
|
|
520
572
|
self._ref_lookups.clear()
|
|
521
573
|
self._normal_alleles.clear()
|
|
574
|
+
self._ready_builds.clear()
|
|
522
575
|
|
|
523
576
|
def reference_for(self, rsid: str, build: str) -> str | None:
|
|
524
577
|
"""Return ClinVar's single-base REF allele for `rsid` in `build`, or None.
|
|
@@ -742,24 +795,8 @@ class ClinVarAnnotator(Annotator):
|
|
|
742
795
|
|
|
743
796
|
@staticmethod
|
|
744
797
|
def _fetch_remote_signal_for_tsv() -> str | None:
|
|
745
|
-
|
|
746
|
-
if not
|
|
747
|
-
return None
|
|
748
|
-
first_token = body.strip().split(None, 1)[0] if body.strip() else ""
|
|
749
|
-
if not _MD5_HEX_RE.fullmatch(first_token):
|
|
750
|
-
# CDN error page, redirect interstitial, or empty body. Treat
|
|
751
|
-
# as a transient signal failure rather than poisoning the
|
|
752
|
-
# cache: callers handle `None` as "freshness unknown, skip"
|
|
753
|
-
# in `db update`, and `setup()` raises rather than passing
|
|
754
|
-
# garbage to `verify_file_hash` (which would delete the TSV).
|
|
755
|
-
logger.warning(
|
|
756
|
-
"clinvar: submission_summary.md5 endpoint returned a body whose "
|
|
757
|
-
"first token is not a 32-char hex digest (got %r); treating as "
|
|
758
|
-
"no signal",
|
|
759
|
-
first_token[:32],
|
|
760
|
-
)
|
|
761
|
-
return None
|
|
762
|
-
return f"md5:{first_token}"
|
|
798
|
+
digest = _fetch_checksum(_manager_module.CLINVAR_SUBMISSION_SUMMARY_URL)
|
|
799
|
+
return f"md5:{digest}" if digest is not None else None
|
|
763
800
|
|
|
764
801
|
def cached_remote_signal(self) -> str | None:
|
|
765
802
|
"""Return the signal stamped at the last successful setup, or None.
|
|
@@ -826,6 +863,9 @@ class ClinVarAnnotator(Annotator):
|
|
|
826
863
|
per-SCV rows and ``1/2`` calls carrying two alts both resolve).
|
|
827
864
|
|
|
828
865
|
Groups by ``variant.build`` and dispatches per-build (ADR-0021).
|
|
866
|
+
A build whose cache is missing or not ready for this version (an
|
|
867
|
+
older cache lacks columns this version reads) is skipped, as
|
|
868
|
+
annotation skips it; the cache is never changed here.
|
|
829
869
|
|
|
830
870
|
The returned ``{(chrom, pos, ref, alt): rsid}`` mapping carries
|
|
831
871
|
the resolved coordinates so the pipeline's enrichment phase can
|
|
@@ -847,7 +887,11 @@ class ClinVarAnnotator(Annotator):
|
|
|
847
887
|
|
|
848
888
|
resolved: dict[tuple[str, int, str, str], str] = {}
|
|
849
889
|
for build, build_variants in by_build.items():
|
|
850
|
-
|
|
890
|
+
if build not in self._db_paths:
|
|
891
|
+
continue
|
|
892
|
+
if build not in self._ready_builds:
|
|
893
|
+
self._ready_builds[build] = self._build_status(build).ready
|
|
894
|
+
conn = self._connection(build) if self._ready_builds[build] else None
|
|
851
895
|
if conn is None:
|
|
852
896
|
continue
|
|
853
897
|
rows_at = self._rows_at_loci(conn, build_variants)
|
|
@@ -241,11 +241,17 @@ class GWASCatalogAnnotator(Annotator):
|
|
|
241
241
|
self._conn = connect_readonly(self._db_path)
|
|
242
242
|
return self._conn
|
|
243
243
|
|
|
244
|
-
def setup(self) -> None:
|
|
245
|
-
"""Download GWAS Catalog associations ZIP, extract TSV, and ingest.
|
|
244
|
+
def setup(self, *, force: bool = False) -> None:
|
|
245
|
+
"""Download GWAS Catalog associations ZIP, extract TSV, and ingest.
|
|
246
|
+
|
|
247
|
+
``force`` (``db update --force``) refreshes when the freshness signal
|
|
248
|
+
is unavailable. The source publishes no checksum, so the download's
|
|
249
|
+
checks (TLS, Content-Length, row floor) are the same either way; no
|
|
250
|
+
signal is stored, so the next ordinary update refreshes again.
|
|
251
|
+
"""
|
|
246
252
|
url = GWAS_CATALOG_URL
|
|
247
253
|
signal = self.fetch_remote_signal()
|
|
248
|
-
if signal is None:
|
|
254
|
+
if signal is None and not force:
|
|
249
255
|
msg = (
|
|
250
256
|
"gwas: cannot verify remote freshness signal. "
|
|
251
257
|
"Refresh aborted to avoid persisting an incomplete cache stamp. "
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
import contextlib
|
|
7
8
|
import json
|
|
8
9
|
import logging
|
|
9
10
|
import sqlite3
|
|
@@ -67,6 +68,23 @@ def _magnitude(level_of_evidence: str) -> float:
|
|
|
67
68
|
return _LOE_MAGNITUDE.get(level_of_evidence.strip().lower(), 5.0)
|
|
68
69
|
|
|
69
70
|
|
|
71
|
+
def _without_cpic(signal: str) -> str:
|
|
72
|
+
"""The composite signal with its CPIC part marked unavailable; the ClinPGx part kept."""
|
|
73
|
+
return f"{signal.split('|cpic:', 1)[0]}|cpic:unavailable"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _has_cpic_rows(db_path: Path) -> bool:
|
|
77
|
+
"""True when the cache holds at least one CPIC allele-function row (read-only)."""
|
|
78
|
+
try:
|
|
79
|
+
with contextlib.closing(connect_readonly(db_path)) as conn:
|
|
80
|
+
return (
|
|
81
|
+
conn.execute("SELECT 1 FROM pharmgkb_allele_function LIMIT 1").fetchone()
|
|
82
|
+
is not None
|
|
83
|
+
)
|
|
84
|
+
except sqlite3.Error:
|
|
85
|
+
return True # an unreadable table is cache_status's to report, not a CPIC repair
|
|
86
|
+
|
|
87
|
+
|
|
70
88
|
class PharmGKBAnnotator(Annotator):
|
|
71
89
|
"""Annotates variants with ClinPGx's curated drug-gene-variant associations."""
|
|
72
90
|
|
|
@@ -130,7 +148,7 @@ class PharmGKBAnnotator(Annotator):
|
|
|
130
148
|
self._conn = connect_readonly(self._db_path)
|
|
131
149
|
return self._conn
|
|
132
150
|
|
|
133
|
-
def setup(self) -> None:
|
|
151
|
+
def setup(self, *, force: bool = False) -> None:
|
|
134
152
|
"""Download ClinPGx clinical annotations + CPIC allele functions, ingest atomically.
|
|
135
153
|
|
|
136
154
|
Two sources are fetched: ClinPGx's `clinicalAnnotations.zip`
|
|
@@ -143,10 +161,15 @@ class PharmGKBAnnotator(Annotator):
|
|
|
143
161
|
This is also how a cache built by an older interpreter version is
|
|
144
162
|
rebuilt: ``is_ready()`` reports it stale and ``db update`` calls
|
|
145
163
|
this, so the CPIC table is fetched again with the annotations.
|
|
164
|
+
|
|
165
|
+
``force`` (``db update --force``) refreshes when the freshness signal
|
|
166
|
+
is unavailable. ClinPGx publishes no checksum, so the checks are the
|
|
167
|
+
same either way; no signal is stored, so the next ordinary update
|
|
168
|
+
refreshes again.
|
|
146
169
|
"""
|
|
147
170
|
url = PHARMGKB_CLINICAL_URL
|
|
148
171
|
signal = self.fetch_remote_signal()
|
|
149
|
-
if signal is None:
|
|
172
|
+
if signal is None and not force:
|
|
150
173
|
msg = (
|
|
151
174
|
"pharmgkb: cannot verify remote freshness signal. "
|
|
152
175
|
"Refresh aborted to avoid persisting an incomplete cache stamp. "
|
|
@@ -168,6 +191,12 @@ class PharmGKBAnnotator(Annotator):
|
|
|
168
191
|
exc,
|
|
169
192
|
)
|
|
170
193
|
cpic_lookup = {}
|
|
194
|
+
if not cpic_lookup:
|
|
195
|
+
# The stamp says what was built: without CPIC rows the cache is
|
|
196
|
+
# marked "cpic:unavailable", so status says so and the next
|
|
197
|
+
# update retries even when CPIC's probe answered this time. A
|
|
198
|
+
# forced build without a ClinPGx signal keeps only that mark.
|
|
199
|
+
signal = _without_cpic(signal) if signal is not None else "cpic:unavailable"
|
|
171
200
|
load_pharmgkb_tsv(
|
|
172
201
|
zip_path,
|
|
173
202
|
self._db_path,
|
|
@@ -265,11 +294,21 @@ class PharmGKBAnnotator(Annotator):
|
|
|
265
294
|
return f"pgkb:{pgkb_signal}|cpic:{cpic_signal}"
|
|
266
295
|
|
|
267
296
|
def cached_remote_signal(self) -> str | None:
|
|
268
|
-
"""Return the remote signal stored at last successful download.
|
|
297
|
+
"""Return the remote signal stored at last successful download.
|
|
298
|
+
|
|
299
|
+
A cache whose stamp names CPIC data but whose CPIC table is empty
|
|
300
|
+
(2.5.0 stamped the probe's answer even when the data fetch then
|
|
301
|
+
failed) reports ``cpic:unavailable``, so ``db update`` repairs it
|
|
302
|
+
and ``db status`` shows it. One read-only row check; nothing else.
|
|
303
|
+
"""
|
|
269
304
|
info = get_database_info(self._db_path, "pharmgkb")
|
|
270
305
|
if not info or not info["remote_signal"]:
|
|
271
306
|
return None
|
|
272
|
-
|
|
307
|
+
signal = info["remote_signal"]
|
|
308
|
+
claims_cpic = "|cpic:" in signal and not signal.endswith("|cpic:unavailable")
|
|
309
|
+
if claims_cpic and not _has_cpic_rows(self._db_path):
|
|
310
|
+
return _without_cpic(signal)
|
|
311
|
+
return signal
|
|
273
312
|
|
|
274
313
|
def annotate(self, variant: Variant) -> list[Annotation]:
|
|
275
314
|
"""Return ClinPGx annotations for variants the user actually carries.
|