allelix 2.4.0__tar.gz → 2.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. allelix-2.4.1/MANIFEST.in +4 -0
  2. {allelix-2.4.0 → allelix-2.4.1}/PKG-INFO +29 -12
  3. {allelix-2.4.0 → allelix-2.4.1}/README.md +28 -11
  4. {allelix-2.4.0 → allelix-2.4.1}/allelix/__init__.py +2 -2
  5. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/alphamissense.py +7 -3
  6. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/cadd.py +4 -3
  7. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/clinvar.py +7 -7
  8. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/gnomad.py +4 -6
  9. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/gwas.py +9 -3
  10. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/pharmgkb.py +25 -72
  11. {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/_helpers.py +157 -155
  12. {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/_options.py +7 -4
  13. {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/utility.py +206 -50
  14. {allelix-2.4.0 → allelix-2.4.1}/allelix/data/clinvar_clnsig_snapshot.yaml +4 -4
  15. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/_versions.py +5 -2
  16. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/cpic_loader.py +3 -3
  17. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/gwas_loader.py +2 -2
  18. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/manager.py +21 -12
  19. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/pharmgkb_loader.py +1 -1
  20. {allelix-2.4.0 → allelix-2.4.1}/allelix/exporters/plink.py +51 -4
  21. {allelix-2.4.0 → allelix-2.4.1}/allelix/models.py +3 -3
  22. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/ancestrydna.py +6 -3
  23. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/base.py +53 -15
  24. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/ftdna.py +8 -5
  25. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/ftdna_famfinder.py +12 -10
  26. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/ftdna_illumina.py +6 -3
  27. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/livingdna.py +6 -22
  28. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/myhappygenes.py +3 -3
  29. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/myheritage.py +11 -7
  30. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/twentythreeandme.py +4 -17
  31. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/vcf.py +14 -20
  32. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/_pipeline.py +327 -264
  33. allelix-2.4.1/allelix/reports/build_text.py +220 -0
  34. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/diff.py +2 -2
  35. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/html.py +16 -10
  36. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/json_report.py +14 -4
  37. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/terminal.py +2 -2
  38. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/vcf.py +7 -2
  39. {allelix-2.4.0 → allelix-2.4.1}/allelix/utils/allele.py +3 -3
  40. allelix-2.4.1/allelix/utils/build_detect.py +689 -0
  41. {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/PKG-INFO +29 -12
  42. {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/SOURCES.txt +3 -22
  43. {allelix-2.4.0 → allelix-2.4.1}/pyproject.toml +5 -5
  44. allelix-2.4.0/allelix/utils/build_detect.py +0 -262
  45. allelix-2.4.0/tests/test_allele_identity_invariants.py +0 -492
  46. allelix-2.4.0/tests/test_changelog_sanitize.py +0 -179
  47. allelix-2.4.0/tests/test_chip_gvcf_parity.py +0 -591
  48. allelix-2.4.0/tests/test_cli.py +0 -3300
  49. allelix-2.4.0/tests/test_cli_helpers.py +0 -138
  50. allelix-2.4.0/tests/test_cli_vcf_out.py +0 -363
  51. allelix-2.4.0/tests/test_compare.py +0 -160
  52. allelix-2.4.0/tests/test_config.py +0 -400
  53. allelix-2.4.0/tests/test_end_to_end.py +0 -548
  54. allelix-2.4.0/tests/test_extract_rsidless_gh128.py +0 -201
  55. allelix-2.4.0/tests/test_mock_data_invariants.py +0 -327
  56. allelix-2.4.0/tests/test_models.py +0 -221
  57. allelix-2.4.0/tests/test_myheritage_delimiter.py +0 -100
  58. allelix-2.4.0/tests/test_myheritage_lowpass.py +0 -148
  59. allelix-2.4.0/tests/test_original_rsid_never_serializes.py +0 -296
  60. allelix-2.4.0/tests/test_plausibility.py +0 -412
  61. allelix-2.4.0/tests/test_registry.py +0 -41
  62. allelix-2.4.0/tests/test_resolution_hoist.py +0 -306
  63. allelix-2.4.0/tests/test_rsid_identity.py +0 -674
  64. allelix-2.4.0/tests/test_symbolic_vcf_rejection.py +0 -104
  65. allelix-2.4.0/tests/test_version.py +0 -100
  66. {allelix-2.4.0 → allelix-2.4.1}/LICENSE +0 -0
  67. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/__init__.py +0 -0
  68. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/_resolution.py +0 -0
  69. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/base.py +0 -0
  70. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/rsid_identity.py +0 -0
  71. {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/snpedia.py +0 -0
  72. {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/__init__.py +0 -0
  73. {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/analyze.py +0 -0
  74. {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/config.py +0 -0
  75. {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/db.py +0 -0
  76. {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/focused.py +0 -0
  77. {allelix-2.4.0 → allelix-2.4.1}/allelix/compare.py +0 -0
  78. {allelix-2.4.0 → allelix-2.4.1}/allelix/config.py +0 -0
  79. {allelix-2.4.0 → allelix-2.4.1}/allelix/data/__init__.py +0 -0
  80. {allelix-2.4.0 → allelix-2.4.1}/allelix/data/high_value_snps.yaml +0 -0
  81. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/__init__.py +0 -0
  82. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/alphamissense_loader.py +0 -0
  83. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/cadd_loader.py +0 -0
  84. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/gnomad_loader.py +0 -0
  85. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/loader_utils.py +0 -0
  86. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/rsid_identity_loader.py +0 -0
  87. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/schema.py +0 -0
  88. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/snpedia_loader.py +0 -0
  89. {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/snpedia_parser.py +0 -0
  90. {allelix-2.4.0 → allelix-2.4.1}/allelix/exporters/__init__.py +0 -0
  91. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/__init__.py +0 -0
  92. {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/_helpers.py +0 -0
  93. {allelix-2.4.0 → allelix-2.4.1}/allelix/py.typed +0 -0
  94. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/__init__.py +0 -0
  95. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/high_value.py +0 -0
  96. {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/methylation.py +0 -0
  97. {allelix-2.4.0 → allelix-2.4.1}/allelix/utils/__init__.py +0 -0
  98. {allelix-2.4.0 → allelix-2.4.1}/allelix/utils/plausibility.py +0 -0
  99. {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/dependency_links.txt +0 -0
  100. {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/entry_points.txt +0 -0
  101. {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/requires.txt +0 -0
  102. {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/top_level.txt +0 -0
  103. {allelix-2.4.0 → allelix-2.4.1}/setup.cfg +0 -0
@@ -0,0 +1,4 @@
1
+ # The source distribution ships what the package needs to build and install.
2
+ # setuptools adds tests/test*.py by default; the test suite (with its
3
+ # fixtures and conftest) lives in the repository, not in the distribution.
4
+ prune tests
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: allelix
3
- Version: 2.4.0
3
+ Version: 2.4.1
4
4
  Summary: Open-source genotype analysis toolkit. Format-agnostic ingestion, database-agnostic annotation, offline-first.
5
5
  Author: Allelix
6
6
  Maintainer-email: dial481 <dial481@users.noreply.github.com>
@@ -47,7 +47,7 @@ Dynamic: license-file
47
47
 
48
48
  [![python](https://img.shields.io/pypi/pyversions/allelix.svg)](https://www.python.org/downloads/)
49
49
  [![pypi](https://img.shields.io/pypi/v/allelix.svg)](https://pypi.org/project/allelix/)
50
- [![license](https://img.shields.io/pypi/l/allelix.svg)](LICENSE)
50
+ [![license](https://img.shields.io/pypi/l/allelix.svg)](https://github.com/allelix/allelix/blob/main/LICENSE)
51
51
  [![CI](https://github.com/allelix/allelix/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/allelix/allelix/actions/workflows/ci.yml)
52
52
  [![privacy: local-first](https://img.shields.io/badge/privacy-local--first-success)](#privacy)
53
53
 
@@ -55,7 +55,7 @@ Dynamic: license-file
55
55
 
56
56
  # Allelix
57
57
 
58
- > **Status:** Production. Nine parser formats (including VCF + gVCF), four annotators (ClinVar + ClinPGx + GWAS Catalog + SNPedia), three GRCh38 enrichment sources (gnomAD + AlphaMissense + CADD) reached from GRCh37 input through a cross-build rsID identity table, licensable-source gating for commercial users, dual-build ClinVar caches (GRCh37 + GRCh38), HTML/JSON/terminal reports and annotated VCF output for pipeline integration (v2.3.0+), methylation + pharmacogenomics focused commands, report diffing, persistent config with commercial-mode safety switch. Modern-build position detection requires at least three concordant seed SNPs and an 80% winning ratio before it overrides a fallback or header signal; a tentative GRCh36 signal instead fails closed to GRCh36 because no modern position cache is safe for it (ADR-0025, ADR-0037). No regex on prose anywhere in production. The [Changelog](https://github.com/allelix/allelix/blob/main/CHANGELOG.md) tracks every release.
58
+ > **Status:** Production. Nine parser formats (including VCF + gVCF), four annotators (ClinVar + ClinPGx + GWAS Catalog + SNPedia), three GRCh38 enrichment sources (gnomAD + AlphaMissense + CADD) reached from GRCh37 input through a cross-build rsID identity table, licensable-source gating for commercial users, dual-build ClinVar caches (GRCh37 + GRCh38), HTML/JSON/terminal reports and annotated VCF output for pipeline integration (v2.3.0+), methylation + pharmacogenomics focused commands, report diffing, persistent config with commercial-mode safety switch. The genome build is decided before annotation from the evidence the file carries: VCF contig lengths, or known-SNP positions from the whole file. Evidence that contradicts the file's label, or a VCF with no usable evidence, stops the run and asks for `--build` instead of producing a report on the wrong build; a tentative GRCh36 signal fails closed to GRCh36 because no modern position cache is safe for it (ADR-0025, ADR-0037, ADR-0044). No regex on prose anywhere in production. The [Changelog](https://github.com/allelix/allelix/blob/main/CHANGELOG.md) tracks every release.
59
59
 
60
60
  ## Quickstart
61
61
 
@@ -99,14 +99,14 @@ See [Development](#development) for source installs and running tests, [Managing
99
99
 
100
100
  | Format | Status | Notes |
101
101
  |---|---|---|
102
- | MyHappyGenes (Tempus) | ✓ | Tab-delimited, 5 columns. **Build is auto-detected** — real-world MHG exports mislabel the header as "build 37.1" while shipping GRCh38 coordinates. Allelix detects from position data and warns on header/data disagreement (ADR-0021). See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myhappygenes-tempus). |
102
+ | MyHappyGenes (Tempus) | ✓ | Tab-delimited, 5 columns. **Build is auto-detected** — real-world MHG exports mislabel the header as "build 37.1" while shipping GRCh38 coordinates. Allelix decides the build from position data and warns on header/data disagreement (ADR-0021, ADR-0044). See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myhappygenes-tempus). |
103
103
  | 23andMe | ✓ | Tab-delimited, 4 columns, concatenated genotype. Supports build 36/37/38 from header. I-prefixed probe IDs passed through. |
104
104
  | AncestryDNA | ✓ | Tab-delimited, 5 columns. Chromosome mapping: 23→X, 24→Y, 25→X (PAR), 26→MT. V1 and V2 chip layouts. |
105
- | Family Tree DNA | ✓ | CSV, double-quoted fields, concatenated genotype. Build 37 default. |
105
+ | Family Tree DNA | ✓ | CSV, double-quoted fields, concatenated genotype. Build 37 default when known-SNP positions are present but do not decide; with none at all and no build line, the run stops (ADR-0044). |
106
106
  | MyHeritage DNA | ✓ | CSV, same structure as FTDNA. Detected by "MyHeritage" in the file's preamble. Handles double-double-quoted fields and files re-saved through a spreadsheet (padded rows, signature not on line 1, tab-delimited copies). Two exports: the **array** export (pre-2026 kits) and the **low-pass WGS** export (kits from January 2026; genotypes imputed from about 2x coverage), which the report labels as imputed and whose `##reference=build37` line is used as the header build. See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myheritage). |
107
107
  | Living DNA | ✓ | Tab-delimited despite `.csv` extension. Handles AX-, AFFX-prefixed and CHR:POS positional SNP IDs. |
108
- | FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default. |
109
- | FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default. |
108
+ | FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default when known-SNP positions are present but do not decide. |
109
+ | FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default when known-SNP positions are present but do not decide. |
110
110
  | VCF / gVCF | ✓ | REF/ALT encoding, `0/1` genotype notation. Plain VCF: absence at a position means reference. gVCF: explicit reference blocks (lines with `<NON_REF>` ALT and `END=` INFO) are not annotated — they match nothing in any annotation database — but they are read at high-value positions: a block with low genotype quality or no reads there makes the position a no-call, and so does a gVCF position no record covers. High-value no-calls list the reason (for example `low_gq_block`, `no_coverage`, `filtered`, `low_gq_call`). A plain VCF has no reference blocks, so a position without a record is reported as unknown rather than as reference or as a no-call. Multi-sample files require `--sample <ID>`. Rows the variant caller rejected (FILTER other than `PASS` or `.`) are skipped and counted in the report; `--keep-filtered` keeps them. A call with genotype quality below `--min-gq` (default 20) or with no reads (DP 0) is reported as a no-call. The spanning-deletion allele `*` is treated as symbolic. A VCF that holds only structural-variant or copy-number records (such as a provider's separate `sv` / `cnv` file) is rejected with a message naming the file to use instead; see [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#sequencingcom-30x-whole-genome) for per-provider deliverables. Streams via stdlib; `.vcf.gz` handled transparently. Optional `pip install allelix[vcf-index]` enables pysam-backed tabix random access for fast `extract --snps` on huge VCFs. |
111
111
 
112
112
  Adding a new format means adding one file to `allelix/parsers/` and registering an instance in the `PARSERS` list in `allelix/parsers/__init__.py`.
@@ -126,7 +126,7 @@ Issues and milestones are tracked at [github.com/allelix/allelix/issues](https:/
126
126
 
127
127
  | Database | Status | Notes |
128
128
  |---|---|---|
129
- | ClinVar (GRCh37 + GRCh38) | ✓ | Public domain (NCBI). SNVs + indels + multi-allelic sites. **Both builds cached**; `analyze` dispatches by detected build (ADR-0021). Rows are matched by chromosome, position, REF and ALT, never by rsID alone, and reference-allele records (REF = ALT) are never reported (ADR-0039). Carrier rule (ADR-0007) requires the user to carry the ALT allele. Indel-anchor protection (ADR-0011) prevents single-base array readouts from matching anchor-base indels. |
129
+ | ClinVar (GRCh37 + GRCh38) | ✓ | Public domain (NCBI). SNVs + indels + multi-allelic sites. **Both builds cached**; `analyze` dispatches by the decided build (ADR-0044). Rows are matched by chromosome, position, REF and ALT, never by rsID alone, and reference-allele records (REF = ALT) are never reported (ADR-0039). Carrier rule (ADR-0007) requires the user to carry the ALT allele. Indel-anchor protection (ADR-0011) prevents single-base array readouts from matching anchor-base indels. |
130
130
  | ClinPGx (formerly PharmGKB) | ✓ | CC BY-SA 4.0. Clinical annotations only — single-rsid SNVs; star alleles and haplotypes deferred (ADR-0009). **Primary non-finding filter is the ClinVar REF carrier rule (ADR-0023):** if ClinVar publishes a single-base REF for the rsid and the user is homozygous for it, the row is suppressed. CPIC's `(rsid, base) → function_class` join (ADR-0020) survives as a secondary tier for rsids ClinVar doesn't catalog. Earlier prose tiers (ADR-0013, ADR-0017, ADR-0018) are superseded. |
131
131
  | CPIC (per-allele function table) | ✓ | Internal data source for the ClinPGx filter. Fetched from `api.cpicpgx.org` at `db update` time. Used to populate the `pharmgkb_allele_function` table — not surfaced to end users as its own annotator. |
132
132
  | SNPedia | ✓ | CC BY-NC-SA 3.0 US. Pre-built cache downloaded via `db update` (~216K wiki pages, ~105K genotype rows). The cache carries SNPedia's orientation metadata: minus-strand alleles are complemented before matching, while unknown-orientation palindromic genotypes abstain rather than risk a false match. If the database is absent, analysis runs without it. For commercial use, pass `--exclude-snpedia`. The cache can also be rebuilt from source via `scripts/scrape_snpedia.py` + `scripts/parse_snpedia.py`. |
@@ -383,11 +383,27 @@ If the failure is a network timeout on the same source repeatedly, the issue is
383
383
 
384
384
  Almost always a schema or interpreter version bump in a recent allelix release that invalidates the pre-bump cache. The fix is `allelix db update` — it'll re-ingest only the affected annotators (everything else signal-skips). See the relevant `CHANGELOG.md` entry for "Cache invalidation" notes per release.
385
385
 
386
- ### Build mismatch warning when analyzing a known-good file
386
+ ### How the genome build is decided, and why a run can stop
387
387
 
388
- Either: (a) the file's header claims one build and the position data confidently indicates another (real-world MHG / Tempus exports do this — header says 37, positions are 38), or (b) the position signal is too small or mixed to be authoritative. Allelix requires at least three concordant seed SNPs and an 80% winning ratio. A VCF record without an rsID counts as a seed when it sits at that seed's position on a build, so variant-caller output with an empty ID column is detected by position too. Weaker GRCh37/38 evidence is reported as tentative and the header or fallback remains effective. A tentative GRCh36 winner is the safety exception: Allelix uses GRCh36 so it cannot accidentally query a modern position cache. Modern position-keyed caches are unavailable for that build (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)).
388
+ Before anything is annotated, Allelix decides the file's genome build from the evidence the file itself carries (ADR-0044):
389
389
 
390
- If you trust the file's header more than allelix's detection, force the build: `--build grch37` or `--build grch38`. The build banner in the analyze output will reflect what was actually used.
390
+ - **VCF contig lengths.** Every nuclear chromosome has a different length in GRCh37 and GRCh38, so the `##contig` lengths settle the build whatever the contig names look like (`chr1` or `1`). If chrM has the old hg19 length, mitochondrial records are not annotated and a note says so.
391
+ - **Known-SNP positions across the whole file.** A set of well-known SNPs sits at different positions in each build. Allelix reads the whole file for them before annotating (a few seconds on a full chip). At least three concordant positions and an 80% majority make the evidence confident (ADR-0037). A VCF record without an rsID counts when it sits at such a position.
392
+ - **The file's own label** (a build line in an array header, or a VCF `assembly=`) is used only when the evidence does not contradict it.
393
+
394
+ The banner says where the build came from, for example `Build: GRCh38 (from the contig lengths: 24 contig length(s) match GRCh38)`. If confident positions contradict the label (real-world MHG / Tempus exports say 37 and ship 38), Allelix uses the positions and prints a **Build mismatch** notice. An unlabelled chip whose known-SNP positions do not decide uses its format's documented default, with a visible note, provided at least one of those positions matches a build and the build most of them match is the default (a few that disagree do not stop it); the note gives the counts. A tentative GRCh36 winner is the safety exception: Allelix uses GRCh36 so it cannot query a modern position cache (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)).
395
+
396
+ **The run stops** instead of guessing when:
397
+
398
+ - a VCF's contig lengths contradict its `assembly=`, contradict each other, or match no supported assembly;
399
+ - weaker (tentative) position evidence contradicts the file's label, or points an unlabelled chip away from its default;
400
+ - the known-SNP positions are tied between builds and the build that would otherwise decide (the label, the format default, or the contig-length build) is not one of them. When it is one of them, Allelix proceeds on it and prints an "inconclusive" warning, also shown in the HTML report. Without contig lengths, a tie that includes GRCh36 instead takes the GRCh36 safety exception described above, with the same warning;
401
+ - a VCF has no contig lengths, no assembly and too few known-SNP positions;
402
+ - a chip states no build and carries none of the known-SNP positions at all (for example an X-chromosome-only export, since they are all on autosomes);
403
+ - the file has known-SNP positions but none of them sits at any supported build's coordinate, whether or not it states a build (contig lengths, when a VCF declares them, still decide);
404
+ - contig lengths decided, but the known-SNP positions met while annotating point to the other build (for example a lifted-over file that kept its old header). The message says the annotation pass ran; nothing is written.
405
+
406
+ The message names the evidence and asks you to say which build the file uses: `--build grch37` or `--build grch38`, or `--build grch36` for an old GRCh36 / hg18 file, which then runs with position-keyed lookups off (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)). For `compare`, use `--build1` / `--build2`. A stop leaves no report, annotated VCF, diff, PLINK files or extract table behind. A forced build is recorded as such in every report.
391
407
 
392
408
  ### Is my database stale? How fresh is the data?
393
409
 
@@ -419,6 +435,7 @@ Notable load-bearing ADRs:
419
435
  - **ADR-0038 — GWAS effect-modifier disambiguation.** Only positively identified odds ratios receive an effect-size modifier; beta and ambiguous effects abstain.
420
436
  - **ADR-0042 — VCF quality gate.** The variant caller's FILTER and genotype quality are honoured, low-quality reference blocks are no-calls, and every exclusion is counted in the report.
421
437
  - **ADR-0043 — One rsID-recovery step.** Every command that needs rsIDs recovers them for rsID-less input through the same shared step; build detection votes by position instead.
438
+ - **ADR-0044 — Evidence before labels.** The build is decided before annotation from contig lengths or whole-file known-SNP positions; evidence that contradicts the label, or no usable evidence, stops the run instead of producing a report on the wrong build.
422
439
  - **ADR-0041 — Plausibility flag.** Very rare homozygous calls and heterozygous calls with no gnomAD carriers are flagged for review, never suppressed.
423
440
 
424
441
  Release history: see [`CHANGELOG.md`](https://github.com/allelix/allelix/blob/main/CHANGELOG.md).
@@ -439,7 +456,7 @@ pytest
439
456
 
440
457
  The pre-commit hook enforces `ruff check` + `ruff format --check`. If a commit is blocked, fix the underlying problem rather than skipping the hook.
441
458
 
442
- For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
459
+ For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](https://github.com/allelix/allelix/blob/main/test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
443
460
 
444
461
  ## License
445
462
 
@@ -8,7 +8,7 @@
8
8
 
9
9
  [![python](https://img.shields.io/pypi/pyversions/allelix.svg)](https://www.python.org/downloads/)
10
10
  [![pypi](https://img.shields.io/pypi/v/allelix.svg)](https://pypi.org/project/allelix/)
11
- [![license](https://img.shields.io/pypi/l/allelix.svg)](LICENSE)
11
+ [![license](https://img.shields.io/pypi/l/allelix.svg)](https://github.com/allelix/allelix/blob/main/LICENSE)
12
12
  [![CI](https://github.com/allelix/allelix/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/allelix/allelix/actions/workflows/ci.yml)
13
13
  [![privacy: local-first](https://img.shields.io/badge/privacy-local--first-success)](#privacy)
14
14
 
@@ -16,7 +16,7 @@
16
16
 
17
17
  # Allelix
18
18
 
19
- > **Status:** Production. Nine parser formats (including VCF + gVCF), four annotators (ClinVar + ClinPGx + GWAS Catalog + SNPedia), three GRCh38 enrichment sources (gnomAD + AlphaMissense + CADD) reached from GRCh37 input through a cross-build rsID identity table, licensable-source gating for commercial users, dual-build ClinVar caches (GRCh37 + GRCh38), HTML/JSON/terminal reports and annotated VCF output for pipeline integration (v2.3.0+), methylation + pharmacogenomics focused commands, report diffing, persistent config with commercial-mode safety switch. Modern-build position detection requires at least three concordant seed SNPs and an 80% winning ratio before it overrides a fallback or header signal; a tentative GRCh36 signal instead fails closed to GRCh36 because no modern position cache is safe for it (ADR-0025, ADR-0037). No regex on prose anywhere in production. The [Changelog](https://github.com/allelix/allelix/blob/main/CHANGELOG.md) tracks every release.
19
+ > **Status:** Production. Nine parser formats (including VCF + gVCF), four annotators (ClinVar + ClinPGx + GWAS Catalog + SNPedia), three GRCh38 enrichment sources (gnomAD + AlphaMissense + CADD) reached from GRCh37 input through a cross-build rsID identity table, licensable-source gating for commercial users, dual-build ClinVar caches (GRCh37 + GRCh38), HTML/JSON/terminal reports and annotated VCF output for pipeline integration (v2.3.0+), methylation + pharmacogenomics focused commands, report diffing, persistent config with commercial-mode safety switch. The genome build is decided before annotation from the evidence the file carries: VCF contig lengths, or known-SNP positions from the whole file. Evidence that contradicts the file's label, or a VCF with no usable evidence, stops the run and asks for `--build` instead of producing a report on the wrong build; a tentative GRCh36 signal fails closed to GRCh36 because no modern position cache is safe for it (ADR-0025, ADR-0037, ADR-0044). No regex on prose anywhere in production. The [Changelog](https://github.com/allelix/allelix/blob/main/CHANGELOG.md) tracks every release.
20
20
 
21
21
  ## Quickstart
22
22
 
@@ -60,14 +60,14 @@ See [Development](#development) for source installs and running tests, [Managing
60
60
 
61
61
  | Format | Status | Notes |
62
62
  |---|---|---|
63
- | MyHappyGenes (Tempus) | ✓ | Tab-delimited, 5 columns. **Build is auto-detected** — real-world MHG exports mislabel the header as "build 37.1" while shipping GRCh38 coordinates. Allelix detects from position data and warns on header/data disagreement (ADR-0021). See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myhappygenes-tempus). |
63
+ | MyHappyGenes (Tempus) | ✓ | Tab-delimited, 5 columns. **Build is auto-detected** — real-world MHG exports mislabel the header as "build 37.1" while shipping GRCh38 coordinates. Allelix decides the build from position data and warns on header/data disagreement (ADR-0021, ADR-0044). See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myhappygenes-tempus). |
64
64
  | 23andMe | ✓ | Tab-delimited, 4 columns, concatenated genotype. Supports build 36/37/38 from header. I-prefixed probe IDs passed through. |
65
65
  | AncestryDNA | ✓ | Tab-delimited, 5 columns. Chromosome mapping: 23→X, 24→Y, 25→X (PAR), 26→MT. V1 and V2 chip layouts. |
66
- | Family Tree DNA | ✓ | CSV, double-quoted fields, concatenated genotype. Build 37 default. |
66
+ | Family Tree DNA | ✓ | CSV, double-quoted fields, concatenated genotype. Build 37 default when known-SNP positions are present but do not decide; with none at all and no build line, the run stops (ADR-0044). |
67
67
  | MyHeritage DNA | ✓ | CSV, same structure as FTDNA. Detected by "MyHeritage" in the file's preamble. Handles double-double-quoted fields and files re-saved through a spreadsheet (padded rows, signature not on line 1, tab-delimited copies). Two exports: the **array** export (pre-2026 kits) and the **low-pass WGS** export (kits from January 2026; genotypes imputed from about 2x coverage), which the report labels as imputed and whose `##reference=build37` line is used as the header build. See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myheritage). |
68
68
  | Living DNA | ✓ | Tab-delimited despite `.csv` extension. Handles AX-, AFFX-prefixed and CHR:POS positional SNP IDs. |
69
- | FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default. |
70
- | FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default. |
69
+ | FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default when known-SNP positions are present but do not decide. |
70
+ | FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default when known-SNP positions are present but do not decide. |
71
71
  | VCF / gVCF | ✓ | REF/ALT encoding, `0/1` genotype notation. Plain VCF: absence at a position means reference. gVCF: explicit reference blocks (lines with `<NON_REF>` ALT and `END=` INFO) are not annotated — they match nothing in any annotation database — but they are read at high-value positions: a block with low genotype quality or no reads there makes the position a no-call, and so does a gVCF position no record covers. High-value no-calls list the reason (for example `low_gq_block`, `no_coverage`, `filtered`, `low_gq_call`). A plain VCF has no reference blocks, so a position without a record is reported as unknown rather than as reference or as a no-call. Multi-sample files require `--sample <ID>`. Rows the variant caller rejected (FILTER other than `PASS` or `.`) are skipped and counted in the report; `--keep-filtered` keeps them. A call with genotype quality below `--min-gq` (default 20) or with no reads (DP 0) is reported as a no-call. The spanning-deletion allele `*` is treated as symbolic. A VCF that holds only structural-variant or copy-number records (such as a provider's separate `sv` / `cnv` file) is rejected with a message naming the file to use instead; see [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#sequencingcom-30x-whole-genome) for per-provider deliverables. Streams via stdlib; `.vcf.gz` handled transparently. Optional `pip install allelix[vcf-index]` enables pysam-backed tabix random access for fast `extract --snps` on huge VCFs. |
72
72
 
73
73
  Adding a new format means adding one file to `allelix/parsers/` and registering an instance in the `PARSERS` list in `allelix/parsers/__init__.py`.
@@ -87,7 +87,7 @@ Issues and milestones are tracked at [github.com/allelix/allelix/issues](https:/
87
87
 
88
88
  | Database | Status | Notes |
89
89
  |---|---|---|
90
- | ClinVar (GRCh37 + GRCh38) | ✓ | Public domain (NCBI). SNVs + indels + multi-allelic sites. **Both builds cached**; `analyze` dispatches by detected build (ADR-0021). Rows are matched by chromosome, position, REF and ALT, never by rsID alone, and reference-allele records (REF = ALT) are never reported (ADR-0039). Carrier rule (ADR-0007) requires the user to carry the ALT allele. Indel-anchor protection (ADR-0011) prevents single-base array readouts from matching anchor-base indels. |
90
+ | ClinVar (GRCh37 + GRCh38) | ✓ | Public domain (NCBI). SNVs + indels + multi-allelic sites. **Both builds cached**; `analyze` dispatches by the decided build (ADR-0044). Rows are matched by chromosome, position, REF and ALT, never by rsID alone, and reference-allele records (REF = ALT) are never reported (ADR-0039). Carrier rule (ADR-0007) requires the user to carry the ALT allele. Indel-anchor protection (ADR-0011) prevents single-base array readouts from matching anchor-base indels. |
91
91
  | ClinPGx (formerly PharmGKB) | ✓ | CC BY-SA 4.0. Clinical annotations only — single-rsid SNVs; star alleles and haplotypes deferred (ADR-0009). **Primary non-finding filter is the ClinVar REF carrier rule (ADR-0023):** if ClinVar publishes a single-base REF for the rsid and the user is homozygous for it, the row is suppressed. CPIC's `(rsid, base) → function_class` join (ADR-0020) survives as a secondary tier for rsids ClinVar doesn't catalog. Earlier prose tiers (ADR-0013, ADR-0017, ADR-0018) are superseded. |
92
92
  | CPIC (per-allele function table) | ✓ | Internal data source for the ClinPGx filter. Fetched from `api.cpicpgx.org` at `db update` time. Used to populate the `pharmgkb_allele_function` table — not surfaced to end users as its own annotator. |
93
93
  | SNPedia | ✓ | CC BY-NC-SA 3.0 US. Pre-built cache downloaded via `db update` (~216K wiki pages, ~105K genotype rows). The cache carries SNPedia's orientation metadata: minus-strand alleles are complemented before matching, while unknown-orientation palindromic genotypes abstain rather than risk a false match. If the database is absent, analysis runs without it. For commercial use, pass `--exclude-snpedia`. The cache can also be rebuilt from source via `scripts/scrape_snpedia.py` + `scripts/parse_snpedia.py`. |
@@ -344,11 +344,27 @@ If the failure is a network timeout on the same source repeatedly, the issue is
344
344
 
345
345
  Almost always a schema or interpreter version bump in a recent allelix release that invalidates the pre-bump cache. The fix is `allelix db update` — it'll re-ingest only the affected annotators (everything else signal-skips). See the relevant `CHANGELOG.md` entry for "Cache invalidation" notes per release.
346
346
 
347
- ### Build mismatch warning when analyzing a known-good file
347
+ ### How the genome build is decided, and why a run can stop
348
348
 
349
- Either: (a) the file's header claims one build and the position data confidently indicates another (real-world MHG / Tempus exports do this — header says 37, positions are 38), or (b) the position signal is too small or mixed to be authoritative. Allelix requires at least three concordant seed SNPs and an 80% winning ratio. A VCF record without an rsID counts as a seed when it sits at that seed's position on a build, so variant-caller output with an empty ID column is detected by position too. Weaker GRCh37/38 evidence is reported as tentative and the header or fallback remains effective. A tentative GRCh36 winner is the safety exception: Allelix uses GRCh36 so it cannot accidentally query a modern position cache. Modern position-keyed caches are unavailable for that build (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)).
349
+ Before anything is annotated, Allelix decides the file's genome build from the evidence the file itself carries (ADR-0044):
350
350
 
351
- If you trust the file's header more than allelix's detection, force the build: `--build grch37` or `--build grch38`. The build banner in the analyze output will reflect what was actually used.
351
+ - **VCF contig lengths.** Every nuclear chromosome has a different length in GRCh37 and GRCh38, so the `##contig` lengths settle the build whatever the contig names look like (`chr1` or `1`). If chrM has the old hg19 length, mitochondrial records are not annotated and a note says so.
352
+ - **Known-SNP positions across the whole file.** A set of well-known SNPs sits at different positions in each build. Allelix reads the whole file for them before annotating (a few seconds on a full chip). At least three concordant positions and an 80% majority make the evidence confident (ADR-0037). A VCF record without an rsID counts when it sits at such a position.
353
+ - **The file's own label** (a build line in an array header, or a VCF `assembly=`) is used only when the evidence does not contradict it.
354
+
355
+ The banner says where the build came from, for example `Build: GRCh38 (from the contig lengths: 24 contig length(s) match GRCh38)`. If confident positions contradict the label (real-world MHG / Tempus exports say 37 and ship 38), Allelix uses the positions and prints a **Build mismatch** notice. An unlabelled chip whose known-SNP positions do not decide uses its format's documented default, with a visible note, provided at least one of those positions matches a build and the build most of them match is the default (a few that disagree do not stop it); the note gives the counts. A tentative GRCh36 winner is the safety exception: Allelix uses GRCh36 so it cannot query a modern position cache (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)).
356
+
357
+ **The run stops** instead of guessing when:
358
+
359
+ - a VCF's contig lengths contradict its `assembly=`, contradict each other, or match no supported assembly;
360
+ - weaker (tentative) position evidence contradicts the file's label, or points an unlabelled chip away from its default;
361
+ - the known-SNP positions are tied between builds and the build that would otherwise decide (the label, the format default, or the contig-length build) is not one of them. When it is one of them, Allelix proceeds on it and prints an "inconclusive" warning, also shown in the HTML report. Without contig lengths, a tie that includes GRCh36 instead takes the GRCh36 safety exception described above, with the same warning;
362
+ - a VCF has no contig lengths, no assembly and too few known-SNP positions;
363
+ - a chip states no build and carries none of the known-SNP positions at all (for example an X-chromosome-only export, since they are all on autosomes);
364
+ - the file has known-SNP positions but none of them sits at any supported build's coordinate, whether or not it states a build (contig lengths, when a VCF declares them, still decide);
365
+ - contig lengths decided, but the known-SNP positions met while annotating point to the other build (for example a lifted-over file that kept its old header). The message says the annotation pass ran; nothing is written.
366
+
367
+ The message names the evidence and asks you to say which build the file uses: `--build grch37` or `--build grch38`, or `--build grch36` for an old GRCh36 / hg18 file, which then runs with position-keyed lookups off (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)). For `compare`, use `--build1` / `--build2`. A stop leaves no report, annotated VCF, diff, PLINK files or extract table behind. A forced build is recorded as such in every report.
352
368
 
353
369
  ### Is my database stale? How fresh is the data?
354
370
 
@@ -380,6 +396,7 @@ Notable load-bearing ADRs:
380
396
  - **ADR-0038 — GWAS effect-modifier disambiguation.** Only positively identified odds ratios receive an effect-size modifier; beta and ambiguous effects abstain.
381
397
  - **ADR-0042 — VCF quality gate.** The variant caller's FILTER and genotype quality are honoured, low-quality reference blocks are no-calls, and every exclusion is counted in the report.
382
398
  - **ADR-0043 — One rsID-recovery step.** Every command that needs rsIDs recovers them for rsID-less input through the same shared step; build detection votes by position instead.
399
+ - **ADR-0044 — Evidence before labels.** The build is decided before annotation from contig lengths or whole-file known-SNP positions; evidence that contradicts the label, or no usable evidence, stops the run instead of producing a report on the wrong build.
383
400
  - **ADR-0041 — Plausibility flag.** Very rare homozygous calls and heterozygous calls with no gnomAD carriers are flagged for review, never suppressed.
384
401
 
385
402
  Release history: see [`CHANGELOG.md`](https://github.com/allelix/allelix/blob/main/CHANGELOG.md).
@@ -400,7 +417,7 @@ pytest
400
417
 
401
418
  The pre-commit hook enforces `ruff check` + `ruff format --check`. If a commit is blocked, fix the underlying problem rather than skipping the hook.
402
419
 
403
- For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
420
+ For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](https://github.com/allelix/allelix/blob/main/test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
404
421
 
405
422
  ## License
406
423
 
@@ -10,8 +10,8 @@ from importlib.metadata import PackageNotFoundError, version
10
10
  def _read_pyproject_version() -> str | None:
11
11
  """Read the package version from ``pyproject.toml``.
12
12
 
13
- fall back to ``pyproject.toml`` when run from a bare source
14
- checkout (no editable install, no installed package metadata). Keeps
13
+ This is the fallback when run from a bare source
14
+ checkout (no editable install, no installed package metadata). It keeps
15
15
  ``--version`` and the outbound HTTP User-Agent reporting the real
16
16
  version string instead of the ``0.0.0+local`` sentinel that
17
17
  misidentifies our traffic to NCBI / EBI / HuggingFace.
@@ -129,12 +129,16 @@ class AlphaMissenseAnnotator(Annotator):
129
129
  logger.warning("Could not remove staged file at %s", gz_path)
130
130
 
131
131
  def is_ready(self) -> bool:
132
- """True when the AlphaMissense SQLite cache exists with current schema version."""
132
+ """True when the AlphaMissense SQLite cache exists with current schema version.
133
+
134
+ A cache with no ``local_version_tag`` is not ready: its schema version
135
+ is unknown, so it could pass as the current one after a bump. The
136
+ installer stamps the current version, so ``db update`` replaces it.
137
+ """
133
138
  info = get_database_info(self._db_path, "alphamissense")
134
139
  if info is None:
135
140
  return False
136
- tag = info.get("local_version_tag") or ""
137
- return tag == f"sv:{ALPHAMISSENSE_SCHEMA_VERSION}" or not tag
141
+ return info.get("local_version_tag") == f"sv:{ALPHAMISSENSE_SCHEMA_VERSION}"
138
142
 
139
143
  def version(self) -> str | None:
140
144
  """Return the cached database version, or None."""
@@ -158,7 +158,9 @@ class CaddAnnotator(Annotator):
158
158
  """True when the active backend is available.
159
159
 
160
160
  In full mode, checks for the tabix file. In cache mode, checks
161
- the SQLite database.
161
+ the SQLite database, which must carry the current schema stamp: a
162
+ cache with no ``local_version_tag`` has an unknown schema version, so
163
+ it is not ready and ``db update`` replaces it.
162
164
  """
163
165
  if self._full_mode:
164
166
  return (
@@ -168,8 +170,7 @@ class CaddAnnotator(Annotator):
168
170
  info = get_database_info(self._db_path, "cadd")
169
171
  if info is None:
170
172
  return False
171
- tag = info.get("local_version_tag") or ""
172
- return tag == f"sv:{CADD_SCHEMA_VERSION}" or not tag
173
+ return info.get("local_version_tag") == f"sv:{CADD_SCHEMA_VERSION}"
173
174
 
174
175
  def version(self) -> str | None:
175
176
  """Return the cached database version, or None."""
@@ -153,7 +153,7 @@ def _collapse_identical_scvs(annotations: list[Annotation]) -> list[Annotation]:
153
153
  surface, and collapsing them would be the Frankenstein-pair
154
154
  regression the per-SCV loader retired.
155
155
 
156
- cross-PR review (Finding 1): ``alt`` is in the key for the
156
+ ``alt`` is in the key for the
157
157
  wrong-allele safety invariant. Two SCVs at a multi-allelic site
158
158
  can share (significance, condition, gene) but differ in alt — e.g.
159
159
  rs1065852 (G→A drug_response/Codeine_response + G→C drug_response/
@@ -283,7 +283,7 @@ class ClinVarAnnotator(Annotator):
283
283
  def setup(self) -> None:
284
284
  """Download the per-SCV TSV pair once and ingest both builds from it.
285
285
 
286
- v2.2 stage B: the VCF-per-build path is replaced by the
286
+ Since v2.2 the VCF-per-build path is replaced by the
287
287
  variant_summary + submission_summary TSV pair. variant_summary
288
288
  carries per-(VariationID, Assembly) rows so the same downloaded
289
289
  file populates both GRCh37 and GRCh38 caches with no second
@@ -445,7 +445,7 @@ class ClinVarAnnotator(Annotator):
445
445
  # may have BOTH SNV and indel rows for the same rsid; the WHERE
446
446
  # filters those out so we keep only the SNV REF.
447
447
  #
448
- # item 1: iterate the cursor directly instead of fetchall()
448
+ # Iterate the cursor directly instead of fetchall()
449
449
  # so peak memory is one row + the dict, not the entire result set
450
450
  # twice.
451
451
  cursor = conn.execute(
@@ -463,8 +463,8 @@ class ClinVarAnnotator(Annotator):
463
463
  def fetch_remote_signal(self) -> str | None:
464
464
  r"""Freshness signal derived from the per-SCV submission_summary md5.
465
465
 
466
- v2.2 stage B: pre-stage-B this returned a composite VCF-md5
467
- per build. After stage B, the cache is built from a build-agnostic
466
+ Before v2.2 this returned a composite VCF-md5 per build. Since
467
+ v2.2, the cache is built from a build-agnostic
468
468
  TSV pair, and submission_summary's md5 endpoint is the change
469
469
  cadence we want to track (per-SCV submissions land daily; the
470
470
  slower-moving variant_summary only adds new VariationIDs).
@@ -500,7 +500,7 @@ class ClinVarAnnotator(Annotator):
500
500
  None if any managed build's cache is absent / unstamped or if the
501
501
  builds' stamps diverge (an interrupted partial reingest).
502
502
 
503
- Stage B: both builds now share the same source TSVs and therefore
503
+ Since v2.2 both builds share the same source TSVs and therefore
504
504
  the same remote_signal. We still cross-check that EVERY managed
505
505
  build's cache carries the stamp; a missing build means the
506
506
  last-setup didn't complete, treat as "no signal" and let
@@ -699,7 +699,7 @@ class ClinVarAnnotator(Annotator):
699
699
  input). Together with the locus this identifies the variant by
700
700
  ``(chrom, pos, ref, alt)`` rather than by rsID.
701
701
  - ADR-0011: ClinVar indels never match single-base array calls.
702
- - ADR-0035 PR 4 / ADR-0007: the user carries the ALT, directly or
702
+ - ADR-0035 / ADR-0007: the user carries the ALT, directly or
703
703
  via a confirmed strand flip.
704
704
  - Benign rows are suppressed unless ``include_benign``.
705
705
 
@@ -114,12 +114,10 @@ class GnomadAnnotator(Annotator):
114
114
  def is_ready(self) -> bool:
115
115
  """True when the gnomAD SQLite cache exists with current schema version.
116
116
 
117
- a cache with no ``local_version_tag`` used to be accepted
118
- as ready (the previous ``or not tag`` escape). That defeated the
119
- whole point of ``GNOMAD_SCHEMA_VERSION``: if it ever gets bumped,
120
- every tagless legacy cache would silently pass as the new
121
- version. Reject tagless caches so the user is told to re-run
122
- ``db update``.
117
+ A cache with no ``local_version_tag`` is not ready. Accepting it
118
+ would defeat ``GNOMAD_SCHEMA_VERSION``: after a bump, every
119
+ untagged legacy cache would silently pass as the new version.
120
+ Rejecting it tells the user to re-run ``db update``.
123
121
  """
124
122
  info = get_database_info(self._db_path, "gnomad")
125
123
  if info is None:
@@ -26,6 +26,7 @@ from allelix.databases.manager import (
26
26
  download,
27
27
  get_database_info,
28
28
  head_request_headers,
29
+ parse_baked_version,
29
30
  )
30
31
  from allelix.models import Annotation
31
32
 
@@ -57,7 +58,7 @@ logger = logging.getLogger(__name__)
57
58
 
58
59
  _BATCH_CHUNK = 500 # SQLite default SQLITE_MAX_VARIABLE_NUMBER is 999
59
60
 
60
- # ADR-0035 PR 3: PheCode often arrives embedded in the upstream DISEASE/TRAIT
61
+ # ADR-0035: PheCode often arrives embedded in the upstream DISEASE/TRAIT
61
62
  # field (e.g. ``"Type 2 diabetes (PheCode 250.2)"``). The annotator extracts it
62
63
  # at the boundary so downstream code reads ``Annotation.phecode`` as a
63
64
  # structured field instead of regex-parsing prose. The trimmed trait flows into
@@ -504,8 +505,11 @@ def _stamp_existing_gwas_cache(db_path: Path) -> bool:
504
505
  """One-shot migration: stamp ``local_version_tag`` on a GWAS cache.
505
506
 
506
507
  Handles legacy caches with ``|cv:N`` baked into ``remote_signal``
507
- by moving the tag and cleaning the signal. Returns True if the
508
- current categorizer version is now stamped.
508
+ by moving the tag and cleaning the signal, but only when ``N`` is the
509
+ current categorizer version. A cache without the marker, or with
510
+ another ``N``, has unknown or old data: it is left unstamped and the
511
+ caller re-ingests or re-downloads it. Returns True if the current
512
+ categorizer version is now stamped.
509
513
  """
510
514
  if not db_path.exists():
511
515
  return False
@@ -523,6 +527,8 @@ def _stamp_existing_gwas_cache(db_path: Path) -> bool:
523
527
  return True
524
528
  if existing_tag is not None:
525
529
  return False
530
+ if parse_baked_version(sig, "cv") != _CATEGORIZER_VERSION:
531
+ return False
526
532
  clean_signal = (sig or "").split("|cv:")[0]
527
533
  conn.execute(
528
534
  "UPDATE database_versions "
@@ -122,9 +122,9 @@ class PharmGKBAnnotator(Annotator):
122
122
  per-allele function table is the secondary fallback for rsids
123
123
  ClinVar doesn't catalog.
124
124
 
125
- The ZIP is retained on disk so ``is_ready()`` can auto-reingest
126
- when the interpreter version bumps — mirroring the GWAS TSV
127
- retention pattern.
125
+ This is also how a cache built by an older interpreter version is
126
+ rebuilt: ``is_ready()`` reports it stale and ``db update`` calls
127
+ this, so the CPIC table is fetched again with the annotations.
128
128
  """
129
129
  url = PHARMGKB_CLINICAL_URL
130
130
  signal = self.fetch_remote_signal()
@@ -162,13 +162,15 @@ class PharmGKBAnnotator(Annotator):
162
162
  def is_ready(self) -> bool:
163
163
  """True iff a ClinPGx SQLite cache exists with current schema and interpreter stamp.
164
164
 
165
- When the interpreter version has bumped and the raw ZIP is still
166
- on disk (retained since the last ``db update``), auto-reingests
167
- from the cached ZIP using the existing CPIC allele-function data —
168
- mirroring the GWAS auto-reingest pattern.
165
+ Never touches the network, so ``analyze`` stays offline. When the
166
+ interpreter version has bumped, the cache is not ready and
167
+ ``allelix db update`` rebuilds it, fetching the ClinPGx annotations
168
+ and the CPIC allele-function table again (``setup``).
169
169
 
170
- Pre-mechanism caches (tag missing or baked into ``remote_signal``)
171
- are self-healed with a one-shot stamp update.
170
+ Pre-mechanism caches (tag missing, version baked into
171
+ ``remote_signal``) are self-healed with a one-shot stamp update
172
+ when the baked version is the current one; otherwise they are not
173
+ ready either.
172
174
  """
173
175
  info = get_database_info(self._db_path, "pharmgkb")
174
176
  if info is None:
@@ -178,9 +180,7 @@ class PharmGKBAnnotator(Annotator):
178
180
  tag = info.get("local_version_tag") or ""
179
181
  if tag == f"iv:{PHARMGKB_INTERPRETER_VERSION}":
180
182
  return True
181
- if not tag:
182
- return _stamp_existing_pharmgkb_cache(self._db_path)
183
- return _reingest_pharmgkb_from_cached_zip(self._db_path, self.data_dir)
183
+ return not tag and _stamp_existing_pharmgkb_cache(self._db_path)
184
184
 
185
185
  def version(self) -> str | None:
186
186
  """Cached database version (download date, or version supplied to load)."""
@@ -199,7 +199,7 @@ class PharmGKBAnnotator(Annotator):
199
199
  self._conn = None
200
200
 
201
201
  def fetch_remote_signal(self) -> str | None:
202
- """Composite freshness signal for ClinPGx + CPIC (M-2, ADR-0020).
202
+ """Composite freshness signal for ClinPGx + CPIC (ADR-0020).
203
203
 
204
204
  The signal format is `pgkb:<pgkb-signal>|cpic:<cpic-signal>`.
205
205
 
@@ -269,7 +269,7 @@ class PharmGKBAnnotator(Annotator):
269
269
  if is_clinvar_homref(variant, self._clinvar_ref_provider):
270
270
  return []
271
271
 
272
- # ADR-0035 PR 4: fetch all rows for the rsid (typically <5) and apply
272
+ # ADR-0035: fetch all rows for the rsid (typically <5) and apply
273
273
  # strand-aware genotype match in Python. Forward-stranded users hit
274
274
  # the same set of rows as the v0.7-era ``WHERE genotype = ?`` SQL;
275
275
  # reverse-stranded users now also match when Variant.ref confirms
@@ -402,7 +402,7 @@ class PharmGKBAnnotator(Annotator):
402
402
  _score,
403
403
  pgkb_annotation_id,
404
404
  ) = row
405
- # ADR-0035 PR 4: strand-aware genotype match. See annotate() for
405
+ # ADR-0035: strand-aware genotype match. See annotate() for
406
406
  # the safety rationale (palindrome guard + variant.ref orientation).
407
407
  if not strand_aware_genotype_match(
408
408
  variant.allele1,
@@ -458,12 +458,15 @@ def _stamp_existing_pharmgkb_cache(db_path: Path) -> bool:
458
458
  """One-shot migration: stamp ``local_version_tag`` on a ClinPGx cache.
459
459
 
460
460
  Handles legacy caches with ``|iv:N`` baked into ``remote_signal``
461
- by moving the tag and cleaning the signal. Returns True if the
462
- current interpreter version is now stamped.
461
+ by moving the tag and cleaning the signal, but only when ``N`` is the
462
+ current interpreter version. A cache without the marker, or with
463
+ another ``N``, has unknown or old data: it is left unstamped and the
464
+ caller re-ingests or re-downloads it. Returns True if the current
465
+ interpreter version is now stamped.
463
466
  """
464
467
  import contextlib
465
468
 
466
- from allelix.databases.manager import _ensure_local_version_tag_column
469
+ from allelix.databases.manager import _ensure_local_version_tag_column, parse_baked_version
467
470
 
468
471
  if not db_path.exists():
469
472
  return False
@@ -480,6 +483,10 @@ def _stamp_existing_pharmgkb_cache(db_path: Path) -> bool:
480
483
  sig, existing_tag = row
481
484
  if existing_tag == tag:
482
485
  return True
486
+ if existing_tag is not None:
487
+ return False
488
+ if parse_baked_version(sig, "iv") != PHARMGKB_INTERPRETER_VERSION:
489
+ return False
483
490
  clean_signal = (sig or "").split("|iv:")[0]
484
491
  conn.execute(
485
492
  "UPDATE database_versions "
@@ -491,57 +498,3 @@ def _stamp_existing_pharmgkb_cache(db_path: Path) -> bool:
491
498
  return True
492
499
  except (sqlite3.OperationalError, sqlite3.DatabaseError):
493
500
  return False
494
-
495
-
496
- def _read_cached_cpic_lookup(db_path: Path) -> dict[tuple[str, str], str]:
497
- """Extract the CPIC allele-function table from an existing ClinPGx cache."""
498
- import contextlib
499
-
500
- lookup: dict[tuple[str, str], str] = {}
501
- try:
502
- with contextlib.closing(sqlite3.connect(db_path)) as conn:
503
- rows = conn.execute(
504
- "SELECT rsid, allele, function_class FROM pharmgkb_allele_function"
505
- ).fetchall()
506
- for rsid, allele, function_class in rows:
507
- lookup[(rsid, allele)] = function_class
508
- except (sqlite3.OperationalError, sqlite3.DatabaseError):
509
- pass
510
- return lookup
511
-
512
-
513
- def _reingest_pharmgkb_from_cached_zip(db_path: Path, data_dir: Path) -> bool:
514
- """Re-ingest ClinPGx from the retained ZIP when the interpreter version bumps.
515
-
516
- Reads the existing CPIC allele-function data from the current cache
517
- before replacing it — no network required for the reingest. Preserves
518
- the original source URL and version from the previous cache.
519
- """
520
- zip_path = data_dir / "clinicalAnnotations.zip"
521
- if not zip_path.exists():
522
- return False
523
- info = get_database_info(db_path, "pharmgkb")
524
- if info is None:
525
- return False
526
- old_signal = info.get("remote_signal") or ""
527
- old_version = info.get("version") or ""
528
- old_source_url = info.get("source_url") or ""
529
- cpic_lookup = _read_cached_cpic_lookup(db_path)
530
- logger.info("ClinPGx interpreter changed — re-ingesting from cached ZIP...")
531
- try:
532
- load_pharmgkb_tsv(
533
- zip_path,
534
- db_path,
535
- source_url=old_source_url,
536
- version=old_version,
537
- remote_signal=old_signal,
538
- allele_function_lookup=cpic_lookup,
539
- min_rows=PHARMGKB_MIN_ROWS,
540
- )
541
- except Exception:
542
- logger.warning("Auto-reingest from cached ZIP failed", exc_info=True)
543
- return False
544
- new_info = get_database_info(db_path, "pharmgkb")
545
- if new_info is None:
546
- return False
547
- return (new_info.get("local_version_tag") or "") == f"iv:{PHARMGKB_INTERPRETER_VERSION}"