allelix 2.5.0__tar.gz → 2.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {allelix-2.5.0/allelix.egg-info → allelix-2.5.1}/PKG-INFO +24 -3
  2. {allelix-2.5.0 → allelix-2.5.1}/README.md +22 -2
  3. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/_resolution.py +2 -1
  4. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/clinvar.py +114 -70
  5. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/gwas.py +9 -3
  6. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/pharmgkb.py +43 -4
  7. {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/_helpers.py +73 -24
  8. {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/_options.py +5 -2
  9. {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/db.py +13 -6
  10. {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/utility.py +208 -34
  11. {allelix-2.5.0 → allelix-2.5.1}/allelix/compare.py +76 -10
  12. allelix-2.5.1/allelix/databases/cache_state.py +359 -0
  13. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/loader_utils.py +10 -1
  14. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/manager.py +60 -37
  15. allelix-2.5.1/allelix/exporters/plink.py +398 -0
  16. {allelix-2.5.0 → allelix-2.5.1}/allelix/models.py +5 -0
  17. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/__init__.py +4 -1
  18. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/_helpers.py +93 -16
  19. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/vcf.py +208 -30
  20. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/__init__.py +162 -15
  21. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/_pipeline.py +351 -189
  22. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/diff.py +93 -34
  23. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/html.py +60 -29
  24. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/json_report.py +17 -4
  25. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/terminal.py +18 -1
  26. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/vcf.py +120 -64
  27. {allelix-2.5.0 → allelix-2.5.1/allelix.egg-info}/PKG-INFO +24 -3
  28. {allelix-2.5.0 → allelix-2.5.1}/pyproject.toml +2 -1
  29. allelix-2.5.0/allelix/databases/cache_state.py +0 -161
  30. allelix-2.5.0/allelix/exporters/plink.py +0 -243
  31. {allelix-2.5.0 → allelix-2.5.1}/LICENSE +0 -0
  32. {allelix-2.5.0 → allelix-2.5.1}/MANIFEST.in +0 -0
  33. {allelix-2.5.0 → allelix-2.5.1}/allelix/__init__.py +0 -0
  34. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/__init__.py +0 -0
  35. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/alphamissense.py +0 -0
  36. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/base.py +0 -0
  37. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/cadd.py +0 -0
  38. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/gnomad.py +0 -0
  39. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/rsid_identity.py +0 -0
  40. {allelix-2.5.0 → allelix-2.5.1}/allelix/annotators/snpedia.py +0 -0
  41. {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/__init__.py +0 -0
  42. {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/analyze.py +0 -0
  43. {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/config.py +0 -0
  44. {allelix-2.5.0 → allelix-2.5.1}/allelix/cli/focused.py +0 -0
  45. {allelix-2.5.0 → allelix-2.5.1}/allelix/config.py +0 -0
  46. {allelix-2.5.0 → allelix-2.5.1}/allelix/data/__init__.py +0 -0
  47. {allelix-2.5.0 → allelix-2.5.1}/allelix/data/clinvar_clnsig_snapshot.yaml +0 -0
  48. {allelix-2.5.0 → allelix-2.5.1}/allelix/data/high_value_snps.yaml +0 -0
  49. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/__init__.py +0 -0
  50. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/_versions.py +0 -0
  51. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/alphamissense_loader.py +0 -0
  52. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/cadd_loader.py +0 -0
  53. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/cpic_loader.py +0 -0
  54. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/gnomad_loader.py +0 -0
  55. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/gwas_loader.py +0 -0
  56. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/pharmgkb_loader.py +0 -0
  57. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/rsid_identity_loader.py +0 -0
  58. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/schema.py +0 -0
  59. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/snpedia_loader.py +0 -0
  60. {allelix-2.5.0 → allelix-2.5.1}/allelix/databases/snpedia_parser.py +0 -0
  61. {allelix-2.5.0 → allelix-2.5.1}/allelix/exporters/__init__.py +0 -0
  62. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/ancestrydna.py +0 -0
  63. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/base.py +0 -0
  64. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/decodeme.py +0 -0
  65. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/ftdna.py +0 -0
  66. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/ftdna_famfinder.py +0 -0
  67. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/ftdna_illumina.py +0 -0
  68. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/genesforgood.py +0 -0
  69. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/livingdna.py +0 -0
  70. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/myhappygenes.py +0 -0
  71. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/myheritage.py +0 -0
  72. {allelix-2.5.0 → allelix-2.5.1}/allelix/parsers/twentythreeandme.py +0 -0
  73. {allelix-2.5.0 → allelix-2.5.1}/allelix/py.typed +0 -0
  74. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/_sources.py +0 -0
  75. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/association_text.py +0 -0
  76. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/build_text.py +0 -0
  77. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/clinvar_text.py +0 -0
  78. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/high_value.py +0 -0
  79. {allelix-2.5.0 → allelix-2.5.1}/allelix/reports/methylation.py +0 -0
  80. {allelix-2.5.0 → allelix-2.5.1}/allelix/utils/__init__.py +0 -0
  81. {allelix-2.5.0 → allelix-2.5.1}/allelix/utils/allele.py +0 -0
  82. {allelix-2.5.0 → allelix-2.5.1}/allelix/utils/build_detect.py +0 -0
  83. {allelix-2.5.0 → allelix-2.5.1}/allelix/utils/plausibility.py +0 -0
  84. {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/SOURCES.txt +0 -0
  85. {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/dependency_links.txt +0 -0
  86. {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/entry_points.txt +0 -0
  87. {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/requires.txt +0 -0
  88. {allelix-2.5.0 → allelix-2.5.1}/allelix.egg-info/top_level.txt +0 -0
  89. {allelix-2.5.0 → allelix-2.5.1}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: allelix
3
- Version: 2.5.0
3
+ Version: 2.5.1
4
4
  Summary: Open-source genotype analysis toolkit. Format-agnostic ingestion, database-agnostic annotation, offline-first.
5
5
  Author: Allelix
6
6
  Maintainer-email: dial481 <dial481@users.noreply.github.com>
@@ -16,6 +16,7 @@ Classifier: Operating System :: OS Independent
16
16
  Classifier: Programming Language :: Python :: 3
17
17
  Classifier: Programming Language :: Python :: 3.11
18
18
  Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
19
20
  Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
20
21
  Requires-Python: >=3.11
21
22
  Description-Content-Type: text/markdown
@@ -87,6 +88,8 @@ allelix analyze your_wgs.vcf.gz --keep-filtered --min-gq 10 --output report.html
87
88
  allelix analyze your_genotype_file.txt --filter-file my_panel.txt --output report.html
88
89
  # Panel coverage puts each requested rsID in one state: annotated, typed (no annotation),
89
90
  # no call, or not on chip ("not in file" for a VCF)
91
+ # VCF coverage includes explicit REF/REF calls and explains rejected/excluded records.
92
+ # Gene entries match constituent symbols in lists; hyphens inside symbols stay intact.
90
93
 
91
94
  # Annotated VCF output — VCF-in / VCF-out for pipeline integration
92
95
  # Writes a valid VCF with annotations as INFO fields + full provenance in the header.
@@ -425,11 +428,29 @@ The banner says where the build came from, for example `Build: GRCh38 (from the
425
428
 
426
429
  The message names the evidence and asks you to say which build the file uses: `--build grch37` or `--build grch38`, or `--build grch36` for an old GRCh36 / hg18 file, which then runs with position-keyed lookups off (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)). For `compare`, use `--build1` / `--build2`. A stop leaves no report, annotated VCF, diff, PLINK files or extract table behind. A forced build is recorded as such in every report.
427
430
 
431
+ ### Keeping input and output files safe
432
+
433
+ Report `--output`, annotated `--vcf-out` and all three actual PLINK output paths
434
+ must differ from the input, including symlink and hard-link aliases. These
435
+ checks run before cache setup or output creation. Inputs that alias the former
436
+ fixed report or PLINK `.tmp` path are also refused; unrelated old temp files
437
+ are left untouched. Annotated VCF still refuses any existing output at the CLI.
438
+
439
+ Writers use unique temporary files in the destination directory and publish
440
+ only complete, closed content. Failure before publication preserves previous
441
+ outputs. PLINK stages all three files first, then publishes each atomically;
442
+ a later publication failure can leave complete new files next to unchanged old
443
+ ones. Existing ordinary permission bits are retained when replacement is
444
+ permitted; new staging files start private. Ownership, ACLs, extended attributes,
445
+ crash durability and concurrent path swaps are outside this guarantee.
446
+
428
447
  ### Is my database stale? How fresh is the data?
429
448
 
430
449
  `allelix db status` shows the ingestion date of each cache. The freshness model is signal-driven: `allelix db update` queries an upstream freshness signal (md5 / etag) per source and re-ingests only when the signal changed. There's no time-based staleness — a 6-month-old cache against an unchanged source IS current.
431
450
 
432
- If you want to force a refresh: `allelix db update --force` re-downloads every cache regardless of freshness signal.
451
+ If you want to force a refresh: `allelix db update --force` re-downloads every cache regardless of freshness signal. ClinVar is never refreshed unverified: its freshness signal is NCBI's checksum for the submission summary, which also verifies the download. If that checksum cannot be fetched (after one retry), ClinVar is not refreshed, with or without `--force`; the existing caches are kept, and a later `db update` tries again. The GWAS Catalog and ClinPGx publish no checksum, so `--force` refreshes them even when their freshness signal cannot be fetched; it stores no signal, and the next ordinary `db update` refreshes them once more when the signal is available again.
452
+
453
+ A ClinPGx cache built while CPIC's allele-function data could not be fetched is marked as such: `db status` shows "(no CPIC)", and the next `db update` tries CPIC again. A cache from 2.5.0 that claims CPIC data but holds none is repaired the same way by an ordinary `db update`.
433
454
 
434
455
  ### Output reports contain real genome annotations
435
456
 
@@ -472,7 +493,7 @@ Release history: see [`CHANGELOG.md`](https://github.com/allelix/allelix/blob/ma
472
493
 
473
494
  ```bash
474
495
  source .venv/bin/activate
475
- pip install -e ".[dev]"
496
+ pip install -e ".[dev,vcf-index]" # the CI install: vcf-index (pysam) runs the indexed-VCF tests
476
497
 
477
498
  # One-time: install pre-commit hooks
478
499
  pre-commit install --hook-type pre-commit
@@ -48,6 +48,8 @@ allelix analyze your_wgs.vcf.gz --keep-filtered --min-gq 10 --output report.html
48
48
  allelix analyze your_genotype_file.txt --filter-file my_panel.txt --output report.html
49
49
  # Panel coverage puts each requested rsID in one state: annotated, typed (no annotation),
50
50
  # no call, or not on chip ("not in file" for a VCF)
51
+ # VCF coverage includes explicit REF/REF calls and explains rejected/excluded records.
52
+ # Gene entries match constituent symbols in lists; hyphens inside symbols stay intact.
51
53
 
52
54
  # Annotated VCF output — VCF-in / VCF-out for pipeline integration
53
55
  # Writes a valid VCF with annotations as INFO fields + full provenance in the header.
@@ -386,11 +388,29 @@ The banner says where the build came from, for example `Build: GRCh38 (from the
386
388
 
387
389
  The message names the evidence and asks you to say which build the file uses: `--build grch37` or `--build grch38`, or `--build grch36` for an old GRCh36 / hg18 file, which then runs with position-keyed lookups off (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)). For `compare`, use `--build1` / `--build2`. A stop leaves no report, annotated VCF, diff, PLINK files or extract table behind. A forced build is recorded as such in every report.
388
390
 
391
+ ### Keeping input and output files safe
392
+
393
+ Report `--output`, annotated `--vcf-out` and all three actual PLINK output paths
394
+ must differ from the input, including symlink and hard-link aliases. These
395
+ checks run before cache setup or output creation. Inputs that alias the former
396
+ fixed report or PLINK `.tmp` path are also refused; unrelated old temp files
397
+ are left untouched. Annotated VCF still refuses any existing output at the CLI.
398
+
399
+ Writers use unique temporary files in the destination directory and publish
400
+ only complete, closed content. Failure before publication preserves previous
401
+ outputs. PLINK stages all three files first, then publishes each atomically;
402
+ a later publication failure can leave complete new files next to unchanged old
403
+ ones. Existing ordinary permission bits are retained when replacement is
404
+ permitted; new staging files start private. Ownership, ACLs, extended attributes,
405
+ crash durability and concurrent path swaps are outside this guarantee.
406
+
389
407
  ### Is my database stale? How fresh is the data?
390
408
 
391
409
  `allelix db status` shows the ingestion date of each cache. The freshness model is signal-driven: `allelix db update` queries an upstream freshness signal (md5 / etag) per source and re-ingests only when the signal changed. There's no time-based staleness — a 6-month-old cache against an unchanged source IS current.
392
410
 
393
- If you want to force a refresh: `allelix db update --force` re-downloads every cache regardless of freshness signal.
411
+ If you want to force a refresh: `allelix db update --force` re-downloads every cache regardless of freshness signal. ClinVar is never refreshed unverified: its freshness signal is NCBI's checksum for the submission summary, which also verifies the download. If that checksum cannot be fetched (after one retry), ClinVar is not refreshed, with or without `--force`; the existing caches are kept, and a later `db update` tries again. The GWAS Catalog and ClinPGx publish no checksum, so `--force` refreshes them even when their freshness signal cannot be fetched; it stores no signal, and the next ordinary `db update` refreshes them once more when the signal is available again.
412
+
413
+ A ClinPGx cache built while CPIC's allele-function data could not be fetched is marked as such: `db status` shows "(no CPIC)", and the next `db update` tries CPIC again. A cache from 2.5.0 that claims CPIC data but holds none is repaired the same way by an ordinary `db update`.
394
414
 
395
415
  ### Output reports contain real genome annotations
396
416
 
@@ -433,7 +453,7 @@ Release history: see [`CHANGELOG.md`](https://github.com/allelix/allelix/blob/ma
433
453
 
434
454
  ```bash
435
455
  source .venv/bin/activate
436
- pip install -e ".[dev]"
456
+ pip install -e ".[dev,vcf-index]" # the CI install: vcf-index (pysam) runs the indexed-VCF tests
437
457
 
438
458
  # One-time: install pre-commit hooks
439
459
  pre-commit install --hook-type pre-commit
@@ -81,7 +81,8 @@ def resolve_batch(
81
81
  assert identity is not None
82
82
  _populate_ref_from_identity(identity, variants)
83
83
  if clinvar is not None:
84
- # bulk_resolve_rsids skips any build whose cache is missing.
84
+ # bulk_resolve_rsids skips any build whose cache is missing or not
85
+ # ready for this version.
85
86
  rsidless = [v for v in variants if not v.rsid.startswith("rs")]
86
87
  if rsidless:
87
88
  for (chrom, pos, ref, alt), rsid in clinvar.bulk_resolve_rsids(rsidless).items():
@@ -14,7 +14,10 @@ and dispatches per-variant by `variant.build`.
14
14
  from __future__ import annotations
15
15
 
16
16
  import logging
17
+ import os
17
18
  import re
19
+ import tempfile
20
+ from pathlib import Path
18
21
  from typing import TYPE_CHECKING, ClassVar
19
22
 
20
23
  from allelix.annotators.base import Annotator, LicenseDescriptor, variant_locus
@@ -50,7 +53,6 @@ from allelix.utils.build_detect import BUILD_GRCH37, BUILD_GRCH38
50
53
  if TYPE_CHECKING:
51
54
  import sqlite3
52
55
  from collections.abc import Callable, Iterable, Iterator
53
- from pathlib import Path
54
56
 
55
57
  from allelix.models import Variant
56
58
 
@@ -60,13 +62,29 @@ CLINVAR_SUPPORTED_BUILDS: tuple[str, ...] = ("GRCh37", "GRCh38")
60
62
 
61
63
  _BATCH_CHUNK = 500 # SQLite default SQLITE_MAX_VARIABLE_NUMBER is 999
62
64
 
63
- # a remote .md5 endpoint can return an HTML error page on a
64
- # transient blip. The first whitespace-separated token of the body is
65
- # what we treat as the hash, so without this gate `<!DOCTYPE` would be
66
- # accepted as the "signal" and later passed to `verify_file_hash`, which
67
- # would then delete the freshly downloaded VCF. MD5 is exactly 32 hex
68
- # digits; reject anything else.
69
65
  _MD5_HEX_RE = re.compile(r"^[0-9a-fA-F]{32}$")
66
+ _CHECKSUM_MAX_BYTES = 4096
67
+
68
+
69
+ def _fetch_checksum(source_url: str) -> str | None:
70
+ """Read a bounded MD5 sidecar associated with this source's basename."""
71
+ body = fetch_remote_text(source_url + ".md5", max_bytes=_CHECKSUM_MAX_BYTES)
72
+ if not body or len(body.encode("utf-8")) > _CHECKSUM_MAX_BYTES:
73
+ return None
74
+ expected_name = source_url.rsplit("/", 1)[-1]
75
+ digests: set[str] = set()
76
+ for line in body.splitlines():
77
+ parts = line.split(None, 1)
78
+ if not parts:
79
+ continue
80
+ if not _MD5_HEX_RE.fullmatch(parts[0]):
81
+ return None
82
+ if len(parts) == 2:
83
+ filename = parts[1].strip().removeprefix("*")
84
+ if filename.rsplit("/", 1)[-1] != expected_name:
85
+ return None
86
+ digests.add(parts[0].lower())
87
+ return digests.pop() if len(digests) == 1 else None
70
88
 
71
89
 
72
90
  def clinvar_db_filename(build: str) -> str:
@@ -329,6 +347,9 @@ class ClinVarAnnotator(Annotator):
329
347
  # allele from the other build (the build's reference carries a
330
348
  # ClinVar variant allele), with that allele.
331
349
  self._normal_alleles: dict[str, dict[str, str]] = {}
350
+ # Per build, whether its cache is ready for this version: rsID
351
+ # resolution outside annotation uses only a ready cache.
352
+ self._ready_builds: dict[str, bool] = {}
332
353
 
333
354
  def serves_build(self, build: str) -> bool:
334
355
  """Only the builds this run manages (the ready caches, when any are)."""
@@ -375,75 +396,106 @@ class ClinVarAnnotator(Annotator):
375
396
  endpoint — that file drives the meaningful change cadence
376
397
  (per-SCV submissions land daily) and variant_summary turns over
377
398
  weekly on a slower clock.
399
+
400
+ Both compressed sources are checked before ingest. All selected
401
+ builds meet fixed/relative row floors and have final metadata
402
+ before publication. Replacements are individually atomic; a later
403
+ replacement failure reports any build already updated.
404
+
405
+ The freshness signal is NCBI's submission_summary checksum, which
406
+ also verifies that download, so a refresh is never done without it.
407
+ A failed fetch is tried once more (a transient failure may clear).
378
408
  """
379
- signal = self._fetch_remote_signal_for_tsv()
409
+ signal = self._fetch_remote_signal_for_tsv() or self._fetch_remote_signal_for_tsv()
380
410
  if signal is None:
381
- msg = (
382
- "clinvar: cannot verify remote freshness signal. "
383
- "Refresh aborted to avoid persisting an incomplete cache stamp. "
384
- "Retry, or pass --force if you accept that next `db update` "
385
- "will re-download to re-establish the signal."
411
+ raise RuntimeError(
412
+ "clinvar: ClinVar not refreshed: NCBI's checksum for submission_summary "
413
+ "is unavailable, so the download cannot be verified. Existing caches "
414
+ "kept; try again later."
386
415
  )
387
- raise RuntimeError(msg)
388
416
 
389
417
  vs_url = _manager_module.CLINVAR_VARIANT_SUMMARY_URL
390
418
  ss_url = _manager_module.CLINVAR_SUBMISSION_SUMMARY_URL
391
- vs_path = self.data_dir / "variant_summary.txt.gz"
392
- ss_path = self.data_dir / "submission_summary.txt.gz"
393
-
394
- download(vs_url, vs_path)
395
- download(ss_url, ss_path)
396
- # We don't md5-verify variant_summary; signal is calibrated against
397
- # submission_summary (the more frequently-updated file) and that
398
- # endpoint's md5 is what we hashed below. variant_summary integrity
399
- # is asserted structurally by the loader (rows that fail to parse
400
- # are skipped, mismatched assemblies are dropped).
401
- try:
419
+ previous_counts = {build: self._previous_record_count(build) for build in self._builds}
420
+ variant_md5 = _fetch_checksum(vs_url)
421
+ if variant_md5 is None:
422
+ raise RuntimeError(
423
+ "clinvar: cannot verify variant_summary checksum; refresh aborted, "
424
+ "existing caches kept"
425
+ )
426
+ self.data_dir.mkdir(parents=True, exist_ok=True)
427
+ # All owned downloads and candidates live on the cache filesystem.
428
+ # Validation completes for every selected build before publication.
429
+ with tempfile.TemporaryDirectory(prefix=".clinvar-", dir=self.data_dir) as temporary:
430
+ staging = Path(temporary)
431
+ vs_path = staging / "variant_summary.txt.gz"
432
+ ss_path = staging / "submission_summary.txt.gz"
433
+ download(vs_url, vs_path)
434
+ download(ss_url, ss_path)
435
+ verify_file_hash(vs_path, "md5", variant_md5)
402
436
  verify_file_hash(ss_path, "md5", signal.removeprefix("md5:"))
437
+ candidates = {}
403
438
  for build in self._builds:
404
- self._setup_one_from_tsv(vs_path, ss_path, build, signal, vs_url)
405
- finally:
406
- for tmp in (vs_path, ss_path):
439
+ candidate = staging / clinvar_db_filename(build)
440
+ self._setup_one_from_tsv(
441
+ vs_path, ss_path, candidate, build, signal, vs_url, previous_counts[build]
442
+ )
443
+ candidates[build] = candidate
444
+ self.close()
445
+ published: list[str] = []
446
+ for build, candidate in candidates.items():
407
447
  try:
408
- tmp.unlink()
409
- except FileNotFoundError:
410
- pass
411
- except OSError:
412
- logger.warning("Could not remove staged TSV at %s", tmp)
448
+ os.replace(candidate, self._db_paths[build])
449
+ except OSError as exc:
450
+ kept = (
451
+ "old cache kept"
452
+ if self._db_paths[build].exists()
453
+ else "no cache installed"
454
+ )
455
+ raise RuntimeError(
456
+ f"ClinVar {build}: publication failed; {kept}; "
457
+ f"already updated: {', '.join(published) or 'none'}. {exc}"
458
+ ) from exc
459
+ published.append(build)
460
+
461
+ def _previous_record_count(self, build: str) -> int | None:
462
+ """Use existing metadata, refusing an unknown baseline without a full scan."""
463
+ path = self._db_paths[build]
464
+ if not path.exists():
465
+ return None
466
+ info = get_database_info(path, clinvar_record_name(build))
467
+ if info is None:
468
+ info = get_database_info(path, "clinvar")
469
+ count = info["record_count"] if info is not None else None
470
+ if type(count) is not int or count <= 0:
471
+ raise RuntimeError(
472
+ f"ClinVar {build}: existing cache record count is missing or invalid; "
473
+ "cannot establish the relative floor, existing caches kept"
474
+ )
475
+ return count
413
476
 
414
477
  def _setup_one_from_tsv(
415
478
  self,
416
479
  vs_path: Path,
417
480
  ss_path: Path,
481
+ candidate: Path,
418
482
  build: str,
419
483
  signal: str,
420
484
  source_url: str,
485
+ previous_count: int | None,
421
486
  ) -> None:
422
- """Run the per-build cache build against an already-downloaded TSV pair."""
487
+ """Finish and validate a candidate against the downloaded source pair."""
423
488
  load_clinvar_tsv(
424
489
  vs_path,
425
490
  ss_path,
426
- self._db_paths[build],
491
+ candidate,
427
492
  build,
428
493
  source_url=source_url,
429
494
  remote_signal=signal,
495
+ record_name=clinvar_record_name(build),
496
+ min_rows=_manager_module.CLINVAR_MIN_ROWS[build],
497
+ previous_count=previous_count,
430
498
  )
431
- # load_clinvar_tsv stamps the record name as "clinvar"; for
432
- # backward compatibility with the per-build record-name scheme
433
- # (clinvar.GRCh37 / clinvar.GRCh38), rename the row in place.
434
- # This keeps is_ready() / version() per-build dispatch working
435
- # against the existing database_versions table layout.
436
- import contextlib
437
- import sqlite3
438
-
439
- per_build_name = clinvar_record_name(build)
440
- if per_build_name != "clinvar":
441
- with contextlib.closing(sqlite3.connect(self._db_paths[build])) as conn:
442
- conn.execute(
443
- "UPDATE database_versions SET name = ? WHERE name = 'clinvar'",
444
- (per_build_name,),
445
- )
446
- conn.commit()
447
499
 
448
500
  def is_ready(self) -> bool:
449
501
  """True iff EVERY managed build has a populated cache with the current tag."""
@@ -519,6 +571,7 @@ class ClinVarAnnotator(Annotator):
519
571
  self._conns.clear()
520
572
  self._ref_lookups.clear()
521
573
  self._normal_alleles.clear()
574
+ self._ready_builds.clear()
522
575
 
523
576
  def reference_for(self, rsid: str, build: str) -> str | None:
524
577
  """Return ClinVar's single-base REF allele for `rsid` in `build`, or None.
@@ -742,24 +795,8 @@ class ClinVarAnnotator(Annotator):
742
795
 
743
796
  @staticmethod
744
797
  def _fetch_remote_signal_for_tsv() -> str | None:
745
- body = fetch_remote_text(_manager_module.CLINVAR_SUBMISSION_SUMMARY_URL + ".md5")
746
- if not body:
747
- return None
748
- first_token = body.strip().split(None, 1)[0] if body.strip() else ""
749
- if not _MD5_HEX_RE.fullmatch(first_token):
750
- # CDN error page, redirect interstitial, or empty body. Treat
751
- # as a transient signal failure rather than poisoning the
752
- # cache: callers handle `None` as "freshness unknown, skip"
753
- # in `db update`, and `setup()` raises rather than passing
754
- # garbage to `verify_file_hash` (which would delete the TSV).
755
- logger.warning(
756
- "clinvar: submission_summary.md5 endpoint returned a body whose "
757
- "first token is not a 32-char hex digest (got %r); treating as "
758
- "no signal",
759
- first_token[:32],
760
- )
761
- return None
762
- return f"md5:{first_token}"
798
+ digest = _fetch_checksum(_manager_module.CLINVAR_SUBMISSION_SUMMARY_URL)
799
+ return f"md5:{digest}" if digest is not None else None
763
800
 
764
801
  def cached_remote_signal(self) -> str | None:
765
802
  """Return the signal stamped at the last successful setup, or None.
@@ -826,6 +863,9 @@ class ClinVarAnnotator(Annotator):
826
863
  per-SCV rows and ``1/2`` calls carrying two alts both resolve).
827
864
 
828
865
  Groups by ``variant.build`` and dispatches per-build (ADR-0021).
866
+ A build whose cache is missing or not ready for this version (an
867
+ older cache lacks columns this version reads) is skipped, as
868
+ annotation skips it; the cache is never changed here.
829
869
 
830
870
  The returned ``{(chrom, pos, ref, alt): rsid}`` mapping carries
831
871
  the resolved coordinates so the pipeline's enrichment phase can
@@ -847,7 +887,11 @@ class ClinVarAnnotator(Annotator):
847
887
 
848
888
  resolved: dict[tuple[str, int, str, str], str] = {}
849
889
  for build, build_variants in by_build.items():
850
- conn = self._connection(build)
890
+ if build not in self._db_paths:
891
+ continue
892
+ if build not in self._ready_builds:
893
+ self._ready_builds[build] = self._build_status(build).ready
894
+ conn = self._connection(build) if self._ready_builds[build] else None
851
895
  if conn is None:
852
896
  continue
853
897
  rows_at = self._rows_at_loci(conn, build_variants)
@@ -241,11 +241,17 @@ class GWASCatalogAnnotator(Annotator):
241
241
  self._conn = connect_readonly(self._db_path)
242
242
  return self._conn
243
243
 
244
- def setup(self) -> None:
245
- """Download GWAS Catalog associations ZIP, extract TSV, and ingest."""
244
+ def setup(self, *, force: bool = False) -> None:
245
+ """Download GWAS Catalog associations ZIP, extract TSV, and ingest.
246
+
247
+ ``force`` (``db update --force``) refreshes when the freshness signal
248
+ is unavailable. The source publishes no checksum, so the download's
249
+ checks (TLS, Content-Length, row floor) are the same either way; no
250
+ signal is stored, so the next ordinary update refreshes again.
251
+ """
246
252
  url = GWAS_CATALOG_URL
247
253
  signal = self.fetch_remote_signal()
248
- if signal is None:
254
+ if signal is None and not force:
249
255
  msg = (
250
256
  "gwas: cannot verify remote freshness signal. "
251
257
  "Refresh aborted to avoid persisting an incomplete cache stamp. "
@@ -4,6 +4,7 @@
4
4
 
5
5
  from __future__ import annotations
6
6
 
7
+ import contextlib
7
8
  import json
8
9
  import logging
9
10
  import sqlite3
@@ -67,6 +68,23 @@ def _magnitude(level_of_evidence: str) -> float:
67
68
  return _LOE_MAGNITUDE.get(level_of_evidence.strip().lower(), 5.0)
68
69
 
69
70
 
71
+ def _without_cpic(signal: str) -> str:
72
+ """The composite signal with its CPIC part marked unavailable; the ClinPGx part kept."""
73
+ return f"{signal.split('|cpic:', 1)[0]}|cpic:unavailable"
74
+
75
+
76
+ def _has_cpic_rows(db_path: Path) -> bool:
77
+ """True when the cache holds at least one CPIC allele-function row (read-only)."""
78
+ try:
79
+ with contextlib.closing(connect_readonly(db_path)) as conn:
80
+ return (
81
+ conn.execute("SELECT 1 FROM pharmgkb_allele_function LIMIT 1").fetchone()
82
+ is not None
83
+ )
84
+ except sqlite3.Error:
85
+ return True # an unreadable table is cache_status's to report, not a CPIC repair
86
+
87
+
70
88
  class PharmGKBAnnotator(Annotator):
71
89
  """Annotates variants with ClinPGx's curated drug-gene-variant associations."""
72
90
 
@@ -130,7 +148,7 @@ class PharmGKBAnnotator(Annotator):
130
148
  self._conn = connect_readonly(self._db_path)
131
149
  return self._conn
132
150
 
133
- def setup(self) -> None:
151
+ def setup(self, *, force: bool = False) -> None:
134
152
  """Download ClinPGx clinical annotations + CPIC allele functions, ingest atomically.
135
153
 
136
154
  Two sources are fetched: ClinPGx's `clinicalAnnotations.zip`
@@ -143,10 +161,15 @@ class PharmGKBAnnotator(Annotator):
143
161
  This is also how a cache built by an older interpreter version is
144
162
  rebuilt: ``is_ready()`` reports it stale and ``db update`` calls
145
163
  this, so the CPIC table is fetched again with the annotations.
164
+
165
+ ``force`` (``db update --force``) refreshes when the freshness signal
166
+ is unavailable. ClinPGx publishes no checksum, so the checks are the
167
+ same either way; no signal is stored, so the next ordinary update
168
+ refreshes again.
146
169
  """
147
170
  url = PHARMGKB_CLINICAL_URL
148
171
  signal = self.fetch_remote_signal()
149
- if signal is None:
172
+ if signal is None and not force:
150
173
  msg = (
151
174
  "pharmgkb: cannot verify remote freshness signal. "
152
175
  "Refresh aborted to avoid persisting an incomplete cache stamp. "
@@ -168,6 +191,12 @@ class PharmGKBAnnotator(Annotator):
168
191
  exc,
169
192
  )
170
193
  cpic_lookup = {}
194
+ if not cpic_lookup:
195
+ # The stamp says what was built: without CPIC rows the cache is
196
+ # marked "cpic:unavailable", so status says so and the next
197
+ # update retries even when CPIC's probe answered this time. A
198
+ # forced build without a ClinPGx signal keeps only that mark.
199
+ signal = _without_cpic(signal) if signal is not None else "cpic:unavailable"
171
200
  load_pharmgkb_tsv(
172
201
  zip_path,
173
202
  self._db_path,
@@ -265,11 +294,21 @@ class PharmGKBAnnotator(Annotator):
265
294
  return f"pgkb:{pgkb_signal}|cpic:{cpic_signal}"
266
295
 
267
296
  def cached_remote_signal(self) -> str | None:
268
- """Return the remote signal stored at last successful download."""
297
+ """Return the remote signal stored at last successful download.
298
+
299
+ A cache whose stamp names CPIC data but whose CPIC table is empty
300
+ (2.5.0 stamped the probe's answer even when the data fetch then
301
+ failed) reports ``cpic:unavailable``, so ``db update`` repairs it
302
+ and ``db status`` shows it. One read-only row check; nothing else.
303
+ """
269
304
  info = get_database_info(self._db_path, "pharmgkb")
270
305
  if not info or not info["remote_signal"]:
271
306
  return None
272
- return info["remote_signal"] or None
307
+ signal = info["remote_signal"]
308
+ claims_cpic = "|cpic:" in signal and not signal.endswith("|cpic:unavailable")
309
+ if claims_cpic and not _has_cpic_rows(self._db_path):
310
+ return _without_cpic(signal)
311
+ return signal
273
312
 
274
313
  def annotate(self, variant: Variant) -> list[Annotation]:
275
314
  """Return ClinPGx annotations for variants the user actually carries.