allelix 2.1.2__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {allelix-2.1.2 → allelix-2.2.0}/PKG-INFO +28 -9
  2. {allelix-2.1.2 → allelix-2.2.0}/README.md +27 -8
  3. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/clinvar.py +119 -55
  4. {allelix-2.1.2 → allelix-2.2.0}/allelix/cli/_helpers.py +59 -0
  5. {allelix-2.1.2 → allelix-2.2.0}/allelix/cli/_options.py +19 -0
  6. {allelix-2.1.2 → allelix-2.2.0}/allelix/cli/config.py +10 -18
  7. {allelix-2.1.2 → allelix-2.2.0}/allelix/cli/db.py +231 -0
  8. {allelix-2.1.2 → allelix-2.2.0}/allelix/cli/focused.py +3 -3
  9. {allelix-2.1.2 → allelix-2.2.0}/allelix/config.py +31 -13
  10. {allelix-2.1.2 → allelix-2.2.0}/allelix/data/clinvar_clnsig_snapshot.yaml +13 -1
  11. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/_versions.py +7 -1
  12. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/gwas_loader.py +10 -0
  13. allelix-2.2.0/allelix/databases/manager.py +758 -0
  14. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/__init__.py +2 -0
  15. allelix-2.2.0/allelix/parsers/ftdna_famfinder.py +175 -0
  16. {allelix-2.1.2 → allelix-2.2.0}/allelix/reports/__init__.py +3 -3
  17. {allelix-2.1.2 → allelix-2.2.0}/allelix/reports/_pipeline.py +112 -1
  18. {allelix-2.1.2 → allelix-2.2.0}/allelix/reports/diff.py +1 -1
  19. {allelix-2.1.2 → allelix-2.2.0}/allelix/reports/html.py +54 -1
  20. {allelix-2.1.2 → allelix-2.2.0}/allelix/reports/json_report.py +10 -2
  21. {allelix-2.1.2 → allelix-2.2.0}/allelix/reports/terminal.py +38 -0
  22. {allelix-2.1.2 → allelix-2.2.0}/allelix/utils/allele.py +18 -0
  23. {allelix-2.1.2 → allelix-2.2.0}/allelix.egg-info/PKG-INFO +28 -9
  24. {allelix-2.1.2 → allelix-2.2.0}/allelix.egg-info/SOURCES.txt +1 -0
  25. {allelix-2.1.2 → allelix-2.2.0}/pyproject.toml +24 -7
  26. {allelix-2.1.2 → allelix-2.2.0}/tests/test_cli.py +474 -41
  27. {allelix-2.1.2 → allelix-2.2.0}/tests/test_config.py +23 -0
  28. {allelix-2.1.2 → allelix-2.2.0}/tests/test_mock_data_invariants.py +0 -137
  29. allelix-2.1.2/allelix/databases/manager.py +0 -532
  30. {allelix-2.1.2 → allelix-2.2.0}/LICENSE +0 -0
  31. {allelix-2.1.2 → allelix-2.2.0}/allelix/__init__.py +0 -0
  32. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/__init__.py +0 -0
  33. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/alphamissense.py +0 -0
  34. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/base.py +0 -0
  35. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/cadd.py +0 -0
  36. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/gnomad.py +0 -0
  37. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/gwas.py +0 -0
  38. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/pharmgkb.py +0 -0
  39. {allelix-2.1.2 → allelix-2.2.0}/allelix/annotators/snpedia.py +0 -0
  40. {allelix-2.1.2 → allelix-2.2.0}/allelix/cli/__init__.py +0 -0
  41. {allelix-2.1.2 → allelix-2.2.0}/allelix/cli/analyze.py +0 -0
  42. {allelix-2.1.2 → allelix-2.2.0}/allelix/cli/utility.py +0 -0
  43. {allelix-2.1.2 → allelix-2.2.0}/allelix/compare.py +0 -0
  44. {allelix-2.1.2 → allelix-2.2.0}/allelix/data/__init__.py +0 -0
  45. {allelix-2.1.2 → allelix-2.2.0}/allelix/data/high_value_snps.yaml +0 -0
  46. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/__init__.py +0 -0
  47. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/alphamissense_loader.py +0 -0
  48. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/cadd_loader.py +0 -0
  49. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/cpic_loader.py +0 -0
  50. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/gnomad_loader.py +0 -0
  51. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/loader_utils.py +0 -0
  52. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/pharmgkb_loader.py +0 -0
  53. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/schema.py +0 -0
  54. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/snpedia_loader.py +0 -0
  55. {allelix-2.1.2 → allelix-2.2.0}/allelix/databases/snpedia_parser.py +0 -0
  56. {allelix-2.1.2 → allelix-2.2.0}/allelix/exporters/__init__.py +0 -0
  57. {allelix-2.1.2 → allelix-2.2.0}/allelix/exporters/plink.py +0 -0
  58. {allelix-2.1.2 → allelix-2.2.0}/allelix/models.py +0 -0
  59. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/_helpers.py +0 -0
  60. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/ancestrydna.py +0 -0
  61. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/base.py +0 -0
  62. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/ftdna.py +0 -0
  63. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/ftdna_illumina.py +0 -0
  64. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/livingdna.py +0 -0
  65. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/myhappygenes.py +0 -0
  66. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/myheritage.py +0 -0
  67. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/twentythreeandme.py +0 -0
  68. {allelix-2.1.2 → allelix-2.2.0}/allelix/parsers/vcf.py +0 -0
  69. {allelix-2.1.2 → allelix-2.2.0}/allelix/py.typed +0 -0
  70. {allelix-2.1.2 → allelix-2.2.0}/allelix/reports/high_value.py +0 -0
  71. {allelix-2.1.2 → allelix-2.2.0}/allelix/reports/methylation.py +0 -0
  72. {allelix-2.1.2 → allelix-2.2.0}/allelix/utils/__init__.py +0 -0
  73. {allelix-2.1.2 → allelix-2.2.0}/allelix/utils/build_detect.py +0 -0
  74. {allelix-2.1.2 → allelix-2.2.0}/allelix.egg-info/dependency_links.txt +0 -0
  75. {allelix-2.1.2 → allelix-2.2.0}/allelix.egg-info/entry_points.txt +0 -0
  76. {allelix-2.1.2 → allelix-2.2.0}/allelix.egg-info/requires.txt +0 -0
  77. {allelix-2.1.2 → allelix-2.2.0}/allelix.egg-info/top_level.txt +0 -0
  78. {allelix-2.1.2 → allelix-2.2.0}/setup.cfg +0 -0
  79. {allelix-2.1.2 → allelix-2.2.0}/tests/test_cli_helpers.py +0 -0
  80. {allelix-2.1.2 → allelix-2.2.0}/tests/test_compare.py +0 -0
  81. {allelix-2.1.2 → allelix-2.2.0}/tests/test_end_to_end.py +0 -0
  82. {allelix-2.1.2 → allelix-2.2.0}/tests/test_models.py +0 -0
  83. {allelix-2.1.2 → allelix-2.2.0}/tests/test_registry.py +0 -0
  84. {allelix-2.1.2 → allelix-2.2.0}/tests/test_version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: allelix
3
- Version: 2.1.2
3
+ Version: 2.2.0
4
4
  Summary: Open-source genotype analysis toolkit. Format-agnostic ingestion, database-agnostic annotation, offline-first.
5
5
  Author: Allelix
6
6
  Maintainer-email: dial481 <dial481@users.noreply.github.com>
@@ -95,6 +95,7 @@ See [Development](#development) for source installs and running tests, [Managing
95
95
  | MyHeritage DNA | ✓ | CSV, same structure as FTDNA. Detected by "MyHeritage" in comment header. Handles double-double-quoted field variant. |
96
96
  | Living DNA | ✓ | Tab-delimited despite `.csv` extension. Handles AX-, AFFX-prefixed and CHR:POS positional SNP IDs. |
97
97
  | FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default. |
98
+ | FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default. |
98
99
  | VCF / gVCF | ✓ | REF/ALT encoding, `0/1` genotype notation. Plain VCF: absence at a position means reference. gVCF: explicit reference blocks (lines with `<NON_REF>` ALT and `END=` INFO) are skipped — they match nothing in any annotation database. Multi-sample files require `--sample <ID>`. Streams via stdlib; `.vcf.gz` handled transparently. Optional `pip install allelix[vcf-index]` enables pysam-backed tabix random access for fast `extract --snps` on huge VCFs. |
99
100
 
100
101
  Adding a new format means adding one file to `allelix/parsers/` and registering an instance in the `PARSERS` list in `allelix/parsers/__init__.py`.
@@ -253,28 +254,44 @@ Allelix stores reference database caches at `~/.local/share/allelix/` (Linux/mac
253
254
  **The caches are disposable and re-downloadable.** Everything here was fetched by `allelix db update` and can be re-fetched at any time. There's no per-user state in the cache — your genotype files never enter this directory. Safe to delete to reclaim space (or to force a full refresh if you suspect a cache went stale):
254
255
 
255
256
  ```bash
256
- # Reclaim all ~16-22 GB
257
- rm -rf ~/.local/share/allelix/
257
+ # Reclaim all ~16-22 GB, with size report and confirmation prompt
258
+ allelix db clean
258
259
 
259
- # Reclaim everything except CADD (avoids re-downloading the slowest one)
260
- find ~/.local/share/allelix/ -mindepth 1 ! -name 'cadd*.sqlite*' -delete
260
+ # Preserve the CADD cache (avoids re-downloading the slowest one, ~5.8 GB)
261
+ allelix db clean --keep-cadd
262
+
263
+ # Show what would be deleted without acting
264
+ allelix db clean --dry-run
265
+
266
+ # Scripted use — skip the confirmation prompt
267
+ allelix db clean --yes
261
268
 
262
269
  # Re-populate from scratch next time you analyze a file
263
270
  allelix db update
264
271
  ```
265
272
 
266
- **The one thing worth backing up: `config.toml`.** It lives separately (XDG config dir, typically `~/.config/allelix/config.toml`) and stores your license assertions (commercial-mode toggle, CADD opt-in confirmation) and per-source enable/disable settings. It's the only non-reproducible state in the install. The caches will rebuild on the next `db update`; `config.toml` won't.
273
+ `db clean` refuses to act on a directory that doesn't contain any recognized allelix cache files — a typo'd `--data-dir ~/Documents` won't result in `rm -rf` against an unrelated tree, even with `--yes`. Pass `--force` if you've staged caches under a non-standard name and want to delete them anyway.
274
+
275
+ For scripting and backup integration, `allelix db path` prints the resolved data directory:
276
+
277
+ ```bash
278
+ ALLELIX_DATA=$(allelix db path)
279
+ du -sh "$ALLELIX_DATA"
280
+
281
+ # --check additionally verifies the path is writable; exit non-zero if not
282
+ allelix db path --check
283
+ ```
284
+
285
+ **The one thing worth backing up: `config.toml`.** It lives separately (XDG config dir, typically `~/.config/allelix/config.toml`) and stores your license assertions (commercial-mode toggle, CADD opt-in confirmation) and per-source enable/disable settings. It's the only non-reproducible state in the install. The caches will rebuild on the next `db update`; `config.toml` won't. `allelix db clean` preserves `config.toml` even when it lives inside the data directory (older installs).
267
286
 
268
287
  To fully uninstall allelix and reclaim everything:
269
288
 
270
289
  ```bash
271
290
  pip uninstall allelix
272
- rm -rf ~/.local/share/allelix/ # reference database caches
291
+ allelix db clean --yes # reference database caches
273
292
  rm -rf ~/.config/allelix/ # license + source-toggle config
274
293
  ```
275
294
 
276
- A `allelix db clean` subcommand to automate the cache-clear step (with confirmation prompt and size report) is planned for v2.2 alongside `allelix db path` (prints the resolved data directory for scripting / backup integration).
277
-
278
295
  ## Data Sources & Licensing
279
296
 
280
297
  Allelix source code is licensed under the **GNU Affero General Public License v3.0 or later** (AGPL-3.0-or-later). Allelix ships with **zero third-party data**. All reference databases are downloaded by the user at runtime via `allelix db update`. Each database retains its original license on the user's machine:
@@ -383,6 +400,8 @@ pytest
383
400
 
384
401
  The pre-commit hook enforces `ruff check` + `ruff format --check`. If a commit is blocked, fix the underlying problem rather than skipping the hook.
385
402
 
403
+ For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
404
+
386
405
  ## License
387
406
 
388
407
  GNU Affero General Public License v3.0 or later (AGPL-3.0-or-later). See `LICENSE`.
@@ -56,6 +56,7 @@ See [Development](#development) for source installs and running tests, [Managing
56
56
  | MyHeritage DNA | ✓ | CSV, same structure as FTDNA. Detected by "MyHeritage" in comment header. Handles double-double-quoted field variant. |
57
57
  | Living DNA | ✓ | Tab-delimited despite `.csv` extension. Handles AX-, AFFX-prefixed and CHR:POS positional SNP IDs. |
58
58
  | FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default. |
59
+ | FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default. |
59
60
  | VCF / gVCF | ✓ | REF/ALT encoding, `0/1` genotype notation. Plain VCF: absence at a position means reference. gVCF: explicit reference blocks (lines with `<NON_REF>` ALT and `END=` INFO) are skipped — they match nothing in any annotation database. Multi-sample files require `--sample <ID>`. Streams via stdlib; `.vcf.gz` handled transparently. Optional `pip install allelix[vcf-index]` enables pysam-backed tabix random access for fast `extract --snps` on huge VCFs. |
60
61
 
61
62
  Adding a new format means adding one file to `allelix/parsers/` and registering an instance in the `PARSERS` list in `allelix/parsers/__init__.py`.
@@ -214,28 +215,44 @@ Allelix stores reference database caches at `~/.local/share/allelix/` (Linux/mac
214
215
  **The caches are disposable and re-downloadable.** Everything here was fetched by `allelix db update` and can be re-fetched at any time. There's no per-user state in the cache — your genotype files never enter this directory. Safe to delete to reclaim space (or to force a full refresh if you suspect a cache went stale):
215
216
 
216
217
  ```bash
217
- # Reclaim all ~16-22 GB
218
- rm -rf ~/.local/share/allelix/
218
+ # Reclaim all ~16-22 GB, with size report and confirmation prompt
219
+ allelix db clean
219
220
 
220
- # Reclaim everything except CADD (avoids re-downloading the slowest one)
221
- find ~/.local/share/allelix/ -mindepth 1 ! -name 'cadd*.sqlite*' -delete
221
+ # Preserve the CADD cache (avoids re-downloading the slowest one, ~5.8 GB)
222
+ allelix db clean --keep-cadd
223
+
224
+ # Show what would be deleted without acting
225
+ allelix db clean --dry-run
226
+
227
+ # Scripted use — skip the confirmation prompt
228
+ allelix db clean --yes
222
229
 
223
230
  # Re-populate from scratch next time you analyze a file
224
231
  allelix db update
225
232
  ```
226
233
 
227
- **The one thing worth backing up: `config.toml`.** It lives separately (XDG config dir, typically `~/.config/allelix/config.toml`) and stores your license assertions (commercial-mode toggle, CADD opt-in confirmation) and per-source enable/disable settings. It's the only non-reproducible state in the install. The caches will rebuild on the next `db update`; `config.toml` won't.
234
+ `db clean` refuses to act on a directory that doesn't contain any recognized allelix cache files — a typo'd `--data-dir ~/Documents` won't result in `rm -rf` against an unrelated tree, even with `--yes`. Pass `--force` if you've staged caches under a non-standard name and want to delete them anyway.
235
+
236
+ For scripting and backup integration, `allelix db path` prints the resolved data directory:
237
+
238
+ ```bash
239
+ ALLELIX_DATA=$(allelix db path)
240
+ du -sh "$ALLELIX_DATA"
241
+
242
+ # --check additionally verifies the path is writable; exit non-zero if not
243
+ allelix db path --check
244
+ ```
245
+
246
+ **The one thing worth backing up: `config.toml`.** It lives separately (XDG config dir, typically `~/.config/allelix/config.toml`) and stores your license assertions (commercial-mode toggle, CADD opt-in confirmation) and per-source enable/disable settings. It's the only non-reproducible state in the install. The caches will rebuild on the next `db update`; `config.toml` won't. `allelix db clean` preserves `config.toml` even when it lives inside the data directory (older installs).
228
247
 
229
248
  To fully uninstall allelix and reclaim everything:
230
249
 
231
250
  ```bash
232
251
  pip uninstall allelix
233
- rm -rf ~/.local/share/allelix/ # reference database caches
252
+ allelix db clean --yes # reference database caches
234
253
  rm -rf ~/.config/allelix/ # license + source-toggle config
235
254
  ```
236
255
 
237
- A `allelix db clean` subcommand to automate the cache-clear step (with confirmation prompt and size report) is planned for v2.2 alongside `allelix db path` (prints the resolved data directory for scripting / backup integration).
238
-
239
256
  ## Data Sources & Licensing
240
257
 
241
258
  Allelix source code is licensed under the **GNU Affero General Public License v3.0 or later** (AGPL-3.0-or-later). Allelix ships with **zero third-party data**. All reference databases are downloaded by the user at runtime via `allelix db update`. Each database retains its original license on the user's machine:
@@ -344,6 +361,8 @@ pytest
344
361
 
345
362
  The pre-commit hook enforces `ruff check` + `ruff format --check`. If a commit is blocked, fix the underlying problem rather than skipping the hook.
346
363
 
364
+ For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
365
+
347
366
  ## License
348
367
 
349
368
  GNU Affero General Public License v3.0 or later (AGPL-3.0-or-later). See `LICENSE`.
@@ -25,7 +25,7 @@ from allelix.databases.manager import (
25
25
  download,
26
26
  fetch_remote_text,
27
27
  get_database_info,
28
- load_clinvar_vcf,
28
+ load_clinvar_tsv,
29
29
  stamp_existing_clinvar_cache,
30
30
  verify_file_hash,
31
31
  )
@@ -92,11 +92,6 @@ def _magnitude(clnsig: str) -> float:
92
92
  return _CLNSIG_MAGNITUDE.get(_normalize_clnsig(clnsig), 5.0)
93
93
 
94
94
 
95
- def _vcf_filename_for_url(url: str) -> str:
96
- """Pick the right local filename suffix based on the URL."""
97
- return "clinvar.vcf.gz" if url.endswith(".gz") else "clinvar.vcf"
98
-
99
-
100
95
  class ClinVarAnnotator(Annotator):
101
96
  """Annotates variants with ClinVar's clinical significance classifications.
102
97
 
@@ -186,39 +181,88 @@ class ClinVarAnnotator(Annotator):
186
181
  return self._conns[build]
187
182
 
188
183
  def setup(self) -> None:
189
- """Download each managed build's ClinVar VCF and ingest atomically."""
190
- for build in self._builds:
191
- self._setup_one(build)
192
-
193
- def _setup_one(self, build: str) -> None:
194
- url = _manager_module.CLINVAR_URL_BY_BUILD[build]
195
- signal = self._fetch_remote_signal_for(build)
184
+ """Download the per-SCV TSV pair once and ingest both builds from it.
185
+
186
+ v2.2 #42 stage B: the VCF-per-build path is replaced by the
187
+ variant_summary + submission_summary TSV pair. variant_summary
188
+ carries per-(VariationID, Assembly) rows so the same downloaded
189
+ file populates both GRCh37 and GRCh38 caches with no second
190
+ download. submission_summary is build-agnostic; one fetch
191
+ applies across builds.
192
+
193
+ Freshness signal is derived from submission_summary's md5
194
+ endpoint — that file drives the meaningful change cadence
195
+ (per-SCV submissions land daily) and variant_summary turns over
196
+ weekly on a slower clock.
197
+ """
198
+ signal = self._fetch_remote_signal_for_tsv()
196
199
  if signal is None:
197
200
  msg = (
198
- f"clinvar ({build}): cannot verify remote freshness signal. "
201
+ "clinvar: cannot verify remote freshness signal. "
199
202
  "Refresh aborted to avoid persisting an incomplete cache stamp. "
200
203
  "Retry, or pass --force if you accept that next `db update` "
201
204
  "will re-download to re-establish the signal."
202
205
  )
203
206
  raise RuntimeError(msg)
204
- vcf_path = self.data_dir / _vcf_filename_for_url(url)
205
- download(url, vcf_path)
207
+
208
+ vs_url = _manager_module.CLINVAR_VARIANT_SUMMARY_URL
209
+ ss_url = _manager_module.CLINVAR_SUBMISSION_SUMMARY_URL
210
+ vs_path = self.data_dir / "variant_summary.txt.gz"
211
+ ss_path = self.data_dir / "submission_summary.txt.gz"
212
+
213
+ download(vs_url, vs_path)
214
+ download(ss_url, ss_path)
215
+ # We don't md5-verify variant_summary; signal is calibrated against
216
+ # submission_summary (the more frequently-updated file) and that
217
+ # endpoint's md5 is what we hashed below. variant_summary integrity
218
+ # is asserted structurally by the loader (rows that fail to parse
219
+ # are skipped, mismatched assemblies are dropped).
206
220
  try:
207
- verify_file_hash(vcf_path, "md5", signal.removeprefix("md5:"))
208
- load_clinvar_vcf(
209
- vcf_path,
210
- self._db_paths[build],
211
- source_url=url,
212
- remote_signal=signal,
213
- record_name=clinvar_record_name(build),
214
- )
221
+ verify_file_hash(ss_path, "md5", signal.removeprefix("md5:"))
222
+ for build in self._builds:
223
+ self._setup_one_from_tsv(vs_path, ss_path, build, signal, vs_url)
215
224
  finally:
216
- try:
217
- vcf_path.unlink()
218
- except FileNotFoundError:
219
- pass
220
- except OSError:
221
- logger.warning("Could not remove staged VCF at %s", vcf_path)
225
+ for tmp in (vs_path, ss_path):
226
+ try:
227
+ tmp.unlink()
228
+ except FileNotFoundError:
229
+ pass
230
+ except OSError:
231
+ logger.warning("Could not remove staged TSV at %s", tmp)
232
+
233
+ def _setup_one_from_tsv(
234
+ self,
235
+ vs_path: Path,
236
+ ss_path: Path,
237
+ build: str,
238
+ signal: str,
239
+ source_url: str,
240
+ ) -> None:
241
+ """Run the per-build cache build against an already-downloaded TSV pair."""
242
+ load_clinvar_tsv(
243
+ vs_path,
244
+ ss_path,
245
+ self._db_paths[build],
246
+ build,
247
+ source_url=source_url,
248
+ remote_signal=signal,
249
+ )
250
+ # load_clinvar_tsv stamps the record name as "clinvar"; for
251
+ # backward compatibility with the per-build record-name scheme
252
+ # (clinvar.GRCh37 / clinvar.GRCh38), rename the row in place.
253
+ # This keeps is_ready() / version() per-build dispatch working
254
+ # against the existing database_versions table layout.
255
+ import contextlib
256
+ import sqlite3
257
+
258
+ per_build_name = clinvar_record_name(build)
259
+ if per_build_name != "clinvar":
260
+ with contextlib.closing(sqlite3.connect(self._db_paths[build])) as conn:
261
+ conn.execute(
262
+ "UPDATE database_versions SET name = ? WHERE name = 'clinvar'",
263
+ (per_build_name,),
264
+ )
265
+ conn.commit()
222
266
 
223
267
  def is_ready(self) -> bool:
224
268
  """True iff EVERY managed build has a populated, version-stamped cache.
@@ -300,11 +344,15 @@ class ClinVarAnnotator(Annotator):
300
344
  # can't suppress a single-base array readout. The per-build cache
301
345
  # may have BOTH SNV and indel rows for the same rsid; the WHERE
302
346
  # filters those out so we keep only the SNV REF.
303
- rows = conn.execute(
347
+ #
348
+ # GH #28 item 1: iterate the cursor directly instead of fetchall()
349
+ # so peak memory is one row + the dict, not the entire result set
350
+ # twice.
351
+ cursor = conn.execute(
304
352
  "SELECT DISTINCT rsid, ref FROM clinvar_variants WHERE length(ref) = 1"
305
- ).fetchall()
353
+ )
306
354
  out: dict[str, str] = {}
307
- for rsid, ref in rows:
355
+ for rsid, ref in cursor:
308
356
  # If a rsid has multiple single-base REFs (shouldn't happen at
309
357
  # one position but defending against future data shapes), keep
310
358
  # the first.
@@ -313,23 +361,21 @@ class ClinVarAnnotator(Annotator):
313
361
  return out
314
362
 
315
363
  def fetch_remote_signal(self) -> str | None:
316
- r"""Composite freshness signal across managed builds.
317
-
318
- Format: `"GRCh37:md5:<hex>|GRCh38:md5:<hex>"`. Returns None if
319
- ANY managed build's signal probe fails — the CLI then prints
320
- "can't verify" and skips refresh per ADR-0012's policy.
364
+ r"""Freshness signal derived from the per-SCV submission_summary md5.
365
+
366
+ v2.2 #42 stage B: pre-stage-B this returned a composite VCF-md5
367
+ per build. After stage B, the cache is built from a build-agnostic
368
+ TSV pair, and submission_summary's md5 endpoint is the change
369
+ cadence we want to track (per-SCV submissions land daily; the
370
+ slower-moving variant_summary only adds new VariationIDs).
371
+ Returns the signal as ``"md5:<hex>"`` to keep ``setup()``'s
372
+ ``signal.removeprefix("md5:")`` contract unchanged.
321
373
  """
322
- parts: list[str] = []
323
- for build in self._builds:
324
- sig = self._fetch_remote_signal_for(build)
325
- if sig is None:
326
- return None
327
- parts.append(f"{build}:{sig}")
328
- return "|".join(parts) if parts else None
374
+ return self._fetch_remote_signal_for_tsv()
329
375
 
330
376
  @staticmethod
331
- def _fetch_remote_signal_for(build: str) -> str | None:
332
- body = fetch_remote_text(_manager_module.CLINVAR_URL_BY_BUILD[build] + ".md5")
377
+ def _fetch_remote_signal_for_tsv() -> str | None:
378
+ body = fetch_remote_text(_manager_module.CLINVAR_SUBMISSION_SUMMARY_URL + ".md5")
333
379
  if not body:
334
380
  return None
335
381
  first_token = body.strip().split(None, 1)[0] if body.strip() else ""
@@ -338,19 +384,29 @@ class ClinVarAnnotator(Annotator):
338
384
  # as a transient signal failure rather than poisoning the
339
385
  # cache: callers handle `None` as "freshness unknown, skip"
340
386
  # in `db update`, and `setup()` raises rather than passing
341
- # garbage to `verify_file_hash` (which would delete the VCF).
387
+ # garbage to `verify_file_hash` (which would delete the TSV).
342
388
  logger.warning(
343
- "clinvar(%s): .md5 endpoint returned a body whose first token "
344
- "is not a 32-char hex digest (got %r); treating as no signal",
345
- build,
389
+ "clinvar: submission_summary.md5 endpoint returned a body whose "
390
+ "first token is not a 32-char hex digest (got %r); treating as "
391
+ "no signal",
346
392
  first_token[:32],
347
393
  )
348
394
  return None
349
395
  return f"md5:{first_token}"
350
396
 
351
397
  def cached_remote_signal(self) -> str | None:
352
- """Composite cached signal across managed builds. None if any missing."""
353
- parts: list[str] = []
398
+ """Return the signal stamped at the last successful setup, or None.
399
+
400
+ None if any managed build's cache is absent / unstamped or if the
401
+ builds' stamps diverge (an interrupted partial reingest).
402
+
403
+ Stage B: both builds now share the same source TSVs and therefore
404
+ the same remote_signal. We still cross-check that EVERY managed
405
+ build's cache carries the stamp; a missing build means the
406
+ last-setup didn't complete, treat as "no signal" and let
407
+ ``db update`` decide.
408
+ """
409
+ signals: set[str] = set()
354
410
  for build in self._builds:
355
411
  info = get_database_info(self._db_paths[build], clinvar_record_name(build))
356
412
  if info is None or info["remote_signal"] is None:
@@ -358,8 +414,16 @@ class ClinVarAnnotator(Annotator):
358
414
  sig = info["remote_signal"]
359
415
  if not sig:
360
416
  return None
361
- parts.append(f"{build}:{sig}")
362
- return "|".join(parts) if parts else None
417
+ signals.add(sig)
418
+ if not signals:
419
+ return None
420
+ # All builds should carry the same signal (single download, single
421
+ # ingest run). If they diverge — e.g. someone reingested one build
422
+ # against a newer TSV — treat as "no signal" so the next
423
+ # `db update` re-syncs both.
424
+ if len(signals) > 1:
425
+ return None
426
+ return signals.pop()
363
427
 
364
428
  def annotate(self, variant: Variant) -> list[Annotation]:
365
429
  """Return ClinVar annotations whose REF/ALT matches the user's genotype.
@@ -417,6 +417,63 @@ def _emit_build_diagnostics(result: object) -> None:
417
417
  )
418
418
 
419
419
 
420
+ def _emit_runtime_nudges(result: object) -> None:
421
+ """Surface runtime-degradation notes the analyze run can detect.
422
+
423
+ Both are once-per-run, post-pipeline. Neither blocks the report; the
424
+ intent is transparency for cases where the silent default behavior
425
+ used to leave the user uncertain about WHY their count looked low.
426
+
427
+ - **GH #90: strand-aware matching inactive.** When carrier/genotype
428
+ matching ran with ``Variant.ref`` unavailable across the input —
429
+ typically array data with ``--no-gnomad`` or no gnomAD cache —
430
+ the strand-flip path in ``utils/allele.py`` falls back to the
431
+ v2.0.x direct-match-only behavior. Behavior is correct (forward-
432
+ normalized array data matches the direct path regardless), just
433
+ degraded relative to v2.1+'s contract. VCF inputs always carry
434
+ REF from the file, so this never fires for them.
435
+ - **GH #91: GRCh38 rsID-less undercount.** When the effective
436
+ build is GRCh38 AND most variants entered the pipeline without
437
+ rsIDs (the variant-caller ``ID=.`` case), annotation coverage
438
+ is materially lower than the GRCh37 equivalent because rsID
439
+ resolution falls back to ClinVar position lookup — a smaller
440
+ surface than dbSNP. Real fix is #62; this is the interim
441
+ honesty signal until then.
442
+ """
443
+ no_ref = getattr(result, "no_ref_variant_count", 0)
444
+ rsidless = getattr(result, "rsidless_variant_count", 0)
445
+ total = getattr(result, "total_variants", 0)
446
+ parser_name = getattr(result, "parser_name", "")
447
+ diag = getattr(result, "build_diagnostics", None)
448
+ effective_build = diag.effective_build if diag is not None else None
449
+
450
+ # GH #90: array inputs (everything except the VCF parser) running
451
+ # without a usable gnomAD cache leave every variant ref=None. Gate
452
+ # on "the whole input lacked ref" rather than "any variant lacked
453
+ # ref" so a hybrid file (e.g. a few ref-less rows in an otherwise
454
+ # populated VCF) doesn't flap the message on.
455
+ if parser_name != "vcf" and total > 0 and no_ref == total:
456
+ console.print(
457
+ "[dim]strand-aware matching inactive — no reference context "
458
+ "(run `allelix db update` for gnomAD-backed ref resolution, "
459
+ "or this is expected with `--no-gnomad`)[/dim]"
460
+ )
461
+
462
+ # GH #91: GRCh38 + a high rsID-less fraction. The threshold (>50%)
463
+ # distinguishes a variant-caller VCF (mostly ID=. — fires) from a
464
+ # 23andMe-style array (rsID-bearing — no warning) or a mostly-
465
+ # rsID-bearing VCF where a handful of novel calls came in with
466
+ # ID=. (no warning, ID=. for novel sites is expected).
467
+ if effective_build == "GRCh38" and total > 0 and rsidless / total > 0.5:
468
+ console.print(
469
+ "[yellow]GRCh38 input without rsIDs: annotation coverage is "
470
+ "currently lower than GRCh37 because rsID resolution falls "
471
+ "back to ClinVar positions. Full dbSNP resolution is planned "
472
+ "(#62). If you have a GRCh37 build available, it currently "
473
+ "surfaces more annotations.[/yellow]"
474
+ )
475
+
476
+
420
477
  def _run_analysis_command(
421
478
  file_path: Path,
422
479
  fmt: str | None,
@@ -496,11 +553,13 @@ def _run_analysis_command(
496
553
  alphamissense=cast("AlphaMissenseAnnotator | None", am_annotator),
497
554
  cadd=cast("CaddAnnotator | None", cadd_annotator),
498
555
  high_value_rsids=hv_rsids,
556
+ panel_rsids=rsids,
499
557
  )
500
558
  finally:
501
559
  _unwire_parser_logging(counter, stderr_handler, snapshot)
502
560
 
503
561
  _emit_build_diagnostics(result)
562
+ _emit_runtime_nudges(result)
504
563
 
505
564
  hv_warnings = scan_no_calls(result.hv_variants, high_value)
506
565
  if hv_warnings:
@@ -33,6 +33,25 @@ _MIN_MAG_OPT = click.option(
33
33
  show_default=True,
34
34
  help="Filter annotations below this magnitude. Use 0 for the full unfiltered set.",
35
35
  )
36
+ # The focused subcommands (`allelix methylation`, `allelix pharmacogenomics`)
37
+ # default to a lower floor than the general `analyze` report. The intent of
38
+ # a focused report is to surface focused-level signal — ClinPGx LoE 3 hits
39
+ # on COMT / MTHFR / MTR / CBS for `methylation`, similar for the pharma
40
+ # subcommand — which the magnitude-5 floor that protects `analyze` from
41
+ # noise would filter out as if they were noise. Floor 3.0 keeps LoE 3
42
+ # in, keeps ClinVar Benign (mag 1.0) out.
43
+ _FOCUSED_MIN_MAG_OPT = click.option(
44
+ "--min-magnitude",
45
+ type=float,
46
+ default=3.0,
47
+ show_default=True,
48
+ help=(
49
+ "Filter annotations below this magnitude. Focused subcommands default "
50
+ "to 3.0 (vs. analyze's 5.0) so ClinPGx LoE 3 hits on panel genes — "
51
+ "the intended signal of a focused report — aren't filtered out. "
52
+ "Use 0 for the full unfiltered set."
53
+ ),
54
+ )
36
55
  _OUTPUT_OPT = click.option(
37
56
  "--output",
38
57
  type=click.Path(dir_okay=False, path_type=Path),
@@ -29,7 +29,6 @@ def config_show(data_dir: Path | None) -> None:
29
29
  """Display current configuration."""
30
30
  from allelix.annotators import _ANNOTATOR_CLASSES
31
31
  from allelix.annotators.base import Permission
32
- from allelix.annotators.base import permission as check_permission
33
32
  from allelix.config import load_config
34
33
 
35
34
  resolved = resolve_data_dir(data_dir)
@@ -40,24 +39,17 @@ def config_show(data_dir: Path | None) -> None:
40
39
  table.add_column("Enabled", justify="center")
41
40
  table.add_column("Note", style="dim")
42
41
  for name, enabled in sorted(cfg.sources.items()):
43
- cls = _ANNOTATOR_CLASSES.get(name)
42
+ perm = cfg.permission_for(name, _ANNOTATOR_CLASSES)
44
43
  note = ""
45
- if cls is not None:
46
- perm = check_permission(
47
- cls.license,
48
- commercial=cfg.commercial,
49
- license_held=cfg.license_held(name),
50
- )
51
- if perm is Permission.BLOCK_PURCHASABLE:
52
- marker = "[red]no[/red]"
53
- note = f"requires commercial license — purchase: {cls.license.purchase_url}"
54
- elif perm is Permission.BLOCK_FINAL:
55
- marker = "[red]no[/red]"
56
- note = "no commercial license is available"
57
- elif enabled:
58
- marker = "[green]yes[/green]"
59
- else:
60
- marker = "[red]no[/red]"
44
+ if perm is Permission.BLOCK_PURCHASABLE:
45
+ # cls is guaranteed non-None — permission_for only returns
46
+ # BLOCK_PURCHASABLE when an annotator class was resolved.
47
+ cls = _ANNOTATOR_CLASSES[name]
48
+ marker = "[red]no[/red]"
49
+ note = f"requires commercial license — purchase: {cls.license.purchase_url}"
50
+ elif perm is Permission.BLOCK_FINAL:
51
+ marker = "[red]no[/red]"
52
+ note = "no commercial license is available"
61
53
  elif enabled:
62
54
  marker = "[green]yes[/green]"
63
55
  else: