allelix 2.4.0__tar.gz → 2.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- allelix-2.4.1/MANIFEST.in +4 -0
- {allelix-2.4.0 → allelix-2.4.1}/PKG-INFO +29 -12
- {allelix-2.4.0 → allelix-2.4.1}/README.md +28 -11
- {allelix-2.4.0 → allelix-2.4.1}/allelix/__init__.py +2 -2
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/alphamissense.py +7 -3
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/cadd.py +4 -3
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/clinvar.py +7 -7
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/gnomad.py +4 -6
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/gwas.py +9 -3
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/pharmgkb.py +25 -72
- {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/_helpers.py +157 -155
- {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/_options.py +7 -4
- {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/utility.py +206 -50
- {allelix-2.4.0 → allelix-2.4.1}/allelix/data/clinvar_clnsig_snapshot.yaml +4 -4
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/_versions.py +5 -2
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/cpic_loader.py +3 -3
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/gwas_loader.py +2 -2
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/manager.py +21 -12
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/pharmgkb_loader.py +1 -1
- {allelix-2.4.0 → allelix-2.4.1}/allelix/exporters/plink.py +51 -4
- {allelix-2.4.0 → allelix-2.4.1}/allelix/models.py +3 -3
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/ancestrydna.py +6 -3
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/base.py +53 -15
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/ftdna.py +8 -5
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/ftdna_famfinder.py +12 -10
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/ftdna_illumina.py +6 -3
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/livingdna.py +6 -22
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/myhappygenes.py +3 -3
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/myheritage.py +11 -7
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/twentythreeandme.py +4 -17
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/vcf.py +14 -20
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/_pipeline.py +327 -264
- allelix-2.4.1/allelix/reports/build_text.py +220 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/diff.py +2 -2
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/html.py +16 -10
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/json_report.py +14 -4
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/terminal.py +2 -2
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/vcf.py +7 -2
- {allelix-2.4.0 → allelix-2.4.1}/allelix/utils/allele.py +3 -3
- allelix-2.4.1/allelix/utils/build_detect.py +689 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/PKG-INFO +29 -12
- {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/SOURCES.txt +3 -22
- {allelix-2.4.0 → allelix-2.4.1}/pyproject.toml +5 -5
- allelix-2.4.0/allelix/utils/build_detect.py +0 -262
- allelix-2.4.0/tests/test_allele_identity_invariants.py +0 -492
- allelix-2.4.0/tests/test_changelog_sanitize.py +0 -179
- allelix-2.4.0/tests/test_chip_gvcf_parity.py +0 -591
- allelix-2.4.0/tests/test_cli.py +0 -3300
- allelix-2.4.0/tests/test_cli_helpers.py +0 -138
- allelix-2.4.0/tests/test_cli_vcf_out.py +0 -363
- allelix-2.4.0/tests/test_compare.py +0 -160
- allelix-2.4.0/tests/test_config.py +0 -400
- allelix-2.4.0/tests/test_end_to_end.py +0 -548
- allelix-2.4.0/tests/test_extract_rsidless_gh128.py +0 -201
- allelix-2.4.0/tests/test_mock_data_invariants.py +0 -327
- allelix-2.4.0/tests/test_models.py +0 -221
- allelix-2.4.0/tests/test_myheritage_delimiter.py +0 -100
- allelix-2.4.0/tests/test_myheritage_lowpass.py +0 -148
- allelix-2.4.0/tests/test_original_rsid_never_serializes.py +0 -296
- allelix-2.4.0/tests/test_plausibility.py +0 -412
- allelix-2.4.0/tests/test_registry.py +0 -41
- allelix-2.4.0/tests/test_resolution_hoist.py +0 -306
- allelix-2.4.0/tests/test_rsid_identity.py +0 -674
- allelix-2.4.0/tests/test_symbolic_vcf_rejection.py +0 -104
- allelix-2.4.0/tests/test_version.py +0 -100
- {allelix-2.4.0 → allelix-2.4.1}/LICENSE +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/__init__.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/_resolution.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/base.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/rsid_identity.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/annotators/snpedia.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/__init__.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/analyze.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/config.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/db.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/cli/focused.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/compare.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/config.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/data/__init__.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/data/high_value_snps.yaml +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/__init__.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/alphamissense_loader.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/cadd_loader.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/gnomad_loader.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/loader_utils.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/rsid_identity_loader.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/schema.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/snpedia_loader.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/databases/snpedia_parser.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/exporters/__init__.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/__init__.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/parsers/_helpers.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/py.typed +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/__init__.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/high_value.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/reports/methylation.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/utils/__init__.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix/utils/plausibility.py +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/dependency_links.txt +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/entry_points.txt +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/requires.txt +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/allelix.egg-info/top_level.txt +0 -0
- {allelix-2.4.0 → allelix-2.4.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: allelix
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.1
|
|
4
4
|
Summary: Open-source genotype analysis toolkit. Format-agnostic ingestion, database-agnostic annotation, offline-first.
|
|
5
5
|
Author: Allelix
|
|
6
6
|
Maintainer-email: dial481 <dial481@users.noreply.github.com>
|
|
@@ -47,7 +47,7 @@ Dynamic: license-file
|
|
|
47
47
|
|
|
48
48
|
[](https://www.python.org/downloads/)
|
|
49
49
|
[](https://pypi.org/project/allelix/)
|
|
50
|
-
[](LICENSE)
|
|
50
|
+
[](https://github.com/allelix/allelix/blob/main/LICENSE)
|
|
51
51
|
[](https://github.com/allelix/allelix/actions/workflows/ci.yml)
|
|
52
52
|
[](#privacy)
|
|
53
53
|
|
|
@@ -55,7 +55,7 @@ Dynamic: license-file
|
|
|
55
55
|
|
|
56
56
|
# Allelix
|
|
57
57
|
|
|
58
|
-
> **Status:** Production. Nine parser formats (including VCF + gVCF), four annotators (ClinVar + ClinPGx + GWAS Catalog + SNPedia), three GRCh38 enrichment sources (gnomAD + AlphaMissense + CADD) reached from GRCh37 input through a cross-build rsID identity table, licensable-source gating for commercial users, dual-build ClinVar caches (GRCh37 + GRCh38), HTML/JSON/terminal reports and annotated VCF output for pipeline integration (v2.3.0+), methylation + pharmacogenomics focused commands, report diffing, persistent config with commercial-mode safety switch.
|
|
58
|
+
> **Status:** Production. Nine parser formats (including VCF + gVCF), four annotators (ClinVar + ClinPGx + GWAS Catalog + SNPedia), three GRCh38 enrichment sources (gnomAD + AlphaMissense + CADD) reached from GRCh37 input through a cross-build rsID identity table, licensable-source gating for commercial users, dual-build ClinVar caches (GRCh37 + GRCh38), HTML/JSON/terminal reports and annotated VCF output for pipeline integration (v2.3.0+), methylation + pharmacogenomics focused commands, report diffing, persistent config with commercial-mode safety switch. The genome build is decided before annotation from the evidence the file carries: VCF contig lengths, or known-SNP positions from the whole file. Evidence that contradicts the file's label, or a VCF with no usable evidence, stops the run and asks for `--build` instead of producing a report on the wrong build; a tentative GRCh36 signal fails closed to GRCh36 because no modern position cache is safe for it (ADR-0025, ADR-0037, ADR-0044). No regex on prose anywhere in production. The [Changelog](https://github.com/allelix/allelix/blob/main/CHANGELOG.md) tracks every release.
|
|
59
59
|
|
|
60
60
|
## Quickstart
|
|
61
61
|
|
|
@@ -99,14 +99,14 @@ See [Development](#development) for source installs and running tests, [Managing
|
|
|
99
99
|
|
|
100
100
|
| Format | Status | Notes |
|
|
101
101
|
|---|---|---|
|
|
102
|
-
| MyHappyGenes (Tempus) | ✓ | Tab-delimited, 5 columns. **Build is auto-detected** — real-world MHG exports mislabel the header as "build 37.1" while shipping GRCh38 coordinates. Allelix
|
|
102
|
+
| MyHappyGenes (Tempus) | ✓ | Tab-delimited, 5 columns. **Build is auto-detected** — real-world MHG exports mislabel the header as "build 37.1" while shipping GRCh38 coordinates. Allelix decides the build from position data and warns on header/data disagreement (ADR-0021, ADR-0044). See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myhappygenes-tempus). |
|
|
103
103
|
| 23andMe | ✓ | Tab-delimited, 4 columns, concatenated genotype. Supports build 36/37/38 from header. I-prefixed probe IDs passed through. |
|
|
104
104
|
| AncestryDNA | ✓ | Tab-delimited, 5 columns. Chromosome mapping: 23→X, 24→Y, 25→X (PAR), 26→MT. V1 and V2 chip layouts. |
|
|
105
|
-
| Family Tree DNA | ✓ | CSV, double-quoted fields, concatenated genotype. Build 37 default. |
|
|
105
|
+
| Family Tree DNA | ✓ | CSV, double-quoted fields, concatenated genotype. Build 37 default when known-SNP positions are present but do not decide; with none at all and no build line, the run stops (ADR-0044). |
|
|
106
106
|
| MyHeritage DNA | ✓ | CSV, same structure as FTDNA. Detected by "MyHeritage" in the file's preamble. Handles double-double-quoted fields and files re-saved through a spreadsheet (padded rows, signature not on line 1, tab-delimited copies). Two exports: the **array** export (pre-2026 kits) and the **low-pass WGS** export (kits from January 2026; genotypes imputed from about 2x coverage), which the report labels as imputed and whose `##reference=build37` line is used as the header build. See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myheritage). |
|
|
107
107
|
| Living DNA | ✓ | Tab-delimited despite `.csv` extension. Handles AX-, AFFX-prefixed and CHR:POS positional SNP IDs. |
|
|
108
|
-
| FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default. |
|
|
109
|
-
| FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default. |
|
|
108
|
+
| FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default when known-SNP positions are present but do not decide. |
|
|
109
|
+
| FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default when known-SNP positions are present but do not decide. |
|
|
110
110
|
| VCF / gVCF | ✓ | REF/ALT encoding, `0/1` genotype notation. Plain VCF: absence at a position means reference. gVCF: explicit reference blocks (lines with `<NON_REF>` ALT and `END=` INFO) are not annotated — they match nothing in any annotation database — but they are read at high-value positions: a block with low genotype quality or no reads there makes the position a no-call, and so does a gVCF position no record covers. High-value no-calls list the reason (for example `low_gq_block`, `no_coverage`, `filtered`, `low_gq_call`). A plain VCF has no reference blocks, so a position without a record is reported as unknown rather than as reference or as a no-call. Multi-sample files require `--sample <ID>`. Rows the variant caller rejected (FILTER other than `PASS` or `.`) are skipped and counted in the report; `--keep-filtered` keeps them. A call with genotype quality below `--min-gq` (default 20) or with no reads (DP 0) is reported as a no-call. The spanning-deletion allele `*` is treated as symbolic. A VCF that holds only structural-variant or copy-number records (such as a provider's separate `sv` / `cnv` file) is rejected with a message naming the file to use instead; see [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#sequencingcom-30x-whole-genome) for per-provider deliverables. Streams via stdlib; `.vcf.gz` handled transparently. Optional `pip install allelix[vcf-index]` enables pysam-backed tabix random access for fast `extract --snps` on huge VCFs. |
|
|
111
111
|
|
|
112
112
|
Adding a new format means adding one file to `allelix/parsers/` and registering an instance in the `PARSERS` list in `allelix/parsers/__init__.py`.
|
|
@@ -126,7 +126,7 @@ Issues and milestones are tracked at [github.com/allelix/allelix/issues](https:/
|
|
|
126
126
|
|
|
127
127
|
| Database | Status | Notes |
|
|
128
128
|
|---|---|---|
|
|
129
|
-
| ClinVar (GRCh37 + GRCh38) | ✓ | Public domain (NCBI). SNVs + indels + multi-allelic sites. **Both builds cached**; `analyze` dispatches by
|
|
129
|
+
| ClinVar (GRCh37 + GRCh38) | ✓ | Public domain (NCBI). SNVs + indels + multi-allelic sites. **Both builds cached**; `analyze` dispatches by the decided build (ADR-0044). Rows are matched by chromosome, position, REF and ALT, never by rsID alone, and reference-allele records (REF = ALT) are never reported (ADR-0039). Carrier rule (ADR-0007) requires the user to carry the ALT allele. Indel-anchor protection (ADR-0011) prevents single-base array readouts from matching anchor-base indels. |
|
|
130
130
|
| ClinPGx (formerly PharmGKB) | ✓ | CC BY-SA 4.0. Clinical annotations only — single-rsid SNVs; star alleles and haplotypes deferred (ADR-0009). **Primary non-finding filter is the ClinVar REF carrier rule (ADR-0023):** if ClinVar publishes a single-base REF for the rsid and the user is homozygous for it, the row is suppressed. CPIC's `(rsid, base) → function_class` join (ADR-0020) survives as a secondary tier for rsids ClinVar doesn't catalog. Earlier prose tiers (ADR-0013, ADR-0017, ADR-0018) are superseded. |
|
|
131
131
|
| CPIC (per-allele function table) | ✓ | Internal data source for the ClinPGx filter. Fetched from `api.cpicpgx.org` at `db update` time. Used to populate the `pharmgkb_allele_function` table — not surfaced to end users as its own annotator. |
|
|
132
132
|
| SNPedia | ✓ | CC BY-NC-SA 3.0 US. Pre-built cache downloaded via `db update` (~216K wiki pages, ~105K genotype rows). The cache carries SNPedia's orientation metadata: minus-strand alleles are complemented before matching, while unknown-orientation palindromic genotypes abstain rather than risk a false match. If the database is absent, analysis runs without it. For commercial use, pass `--exclude-snpedia`. The cache can also be rebuilt from source via `scripts/scrape_snpedia.py` + `scripts/parse_snpedia.py`. |
|
|
@@ -383,11 +383,27 @@ If the failure is a network timeout on the same source repeatedly, the issue is
|
|
|
383
383
|
|
|
384
384
|
Almost always a schema or interpreter version bump in a recent allelix release that invalidates the pre-bump cache. The fix is `allelix db update` — it'll re-ingest only the affected annotators (everything else signal-skips). See the relevant `CHANGELOG.md` entry for "Cache invalidation" notes per release.
|
|
385
385
|
|
|
386
|
-
###
|
|
386
|
+
### How the genome build is decided, and why a run can stop
|
|
387
387
|
|
|
388
|
-
|
|
388
|
+
Before anything is annotated, Allelix decides the file's genome build from the evidence the file itself carries (ADR-0044):
|
|
389
389
|
|
|
390
|
-
|
|
390
|
+
- **VCF contig lengths.** Every nuclear chromosome has a different length in GRCh37 and GRCh38, so the `##contig` lengths settle the build whatever the contig names look like (`chr1` or `1`). If chrM has the old hg19 length, mitochondrial records are not annotated and a note says so.
|
|
391
|
+
- **Known-SNP positions across the whole file.** A set of well-known SNPs sits at different positions in each build. Allelix reads the whole file for them before annotating (a few seconds on a full chip). At least three concordant positions and an 80% majority make the evidence confident (ADR-0037). A VCF record without an rsID counts when it sits at such a position.
|
|
392
|
+
- **The file's own label** (a build line in an array header, or a VCF `assembly=`) is used only when the evidence does not contradict it.
|
|
393
|
+
|
|
394
|
+
The banner says where the build came from, for example `Build: GRCh38 (from the contig lengths: 24 contig length(s) match GRCh38)`. If confident positions contradict the label (real-world MHG / Tempus exports say 37 and ship 38), Allelix uses the positions and prints a **Build mismatch** notice. An unlabelled chip whose known-SNP positions do not decide uses its format's documented default, with a visible note, provided at least one of those positions matches a build and the build most of them match is the default (a few that disagree do not stop it); the note gives the counts. A tentative GRCh36 winner is the safety exception: Allelix uses GRCh36 so it cannot query a modern position cache (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)).
|
|
395
|
+
|
|
396
|
+
**The run stops** instead of guessing when:
|
|
397
|
+
|
|
398
|
+
- a VCF's contig lengths contradict its `assembly=`, contradict each other, or match no supported assembly;
|
|
399
|
+
- weaker (tentative) position evidence contradicts the file's label, or points an unlabelled chip away from its default;
|
|
400
|
+
- the known-SNP positions are tied between builds and the build that would otherwise decide (the label, the format default, or the contig-length build) is not one of them. When it is one of them, Allelix proceeds on it and prints an "inconclusive" warning, also shown in the HTML report. Without contig lengths, a tie that includes GRCh36 instead takes the GRCh36 safety exception described above, with the same warning;
|
|
401
|
+
- a VCF has no contig lengths, no assembly and too few known-SNP positions;
|
|
402
|
+
- a chip states no build and carries none of the known-SNP positions at all (for example an X-chromosome-only export, since they are all on autosomes);
|
|
403
|
+
- the file has known-SNP positions but none of them sits at any supported build's coordinate, whether or not it states a build (contig lengths, when a VCF declares them, still decide);
|
|
404
|
+
- contig lengths decided, but the known-SNP positions met while annotating point to the other build (for example a lifted-over file that kept its old header). The message says the annotation pass ran; nothing is written.
|
|
405
|
+
|
|
406
|
+
The message names the evidence and asks you to say which build the file uses: `--build grch37` or `--build grch38`, or `--build grch36` for an old GRCh36 / hg18 file, which then runs with position-keyed lookups off (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)). For `compare`, use `--build1` / `--build2`. A stop leaves no report, annotated VCF, diff, PLINK files or extract table behind. A forced build is recorded as such in every report.
|
|
391
407
|
|
|
392
408
|
### Is my database stale? How fresh is the data?
|
|
393
409
|
|
|
@@ -419,6 +435,7 @@ Notable load-bearing ADRs:
|
|
|
419
435
|
- **ADR-0038 — GWAS effect-modifier disambiguation.** Only positively identified odds ratios receive an effect-size modifier; beta and ambiguous effects abstain.
|
|
420
436
|
- **ADR-0042 — VCF quality gate.** The variant caller's FILTER and genotype quality are honoured, low-quality reference blocks are no-calls, and every exclusion is counted in the report.
|
|
421
437
|
- **ADR-0043 — One rsID-recovery step.** Every command that needs rsIDs recovers them for rsID-less input through the same shared step; build detection votes by position instead.
|
|
438
|
+
- **ADR-0044 — Evidence before labels.** The build is decided before annotation from contig lengths or whole-file known-SNP positions; evidence that contradicts the label, or no usable evidence, stops the run instead of producing a report on the wrong build.
|
|
422
439
|
- **ADR-0041 — Plausibility flag.** Very rare homozygous calls and heterozygous calls with no gnomAD carriers are flagged for review, never suppressed.
|
|
423
440
|
|
|
424
441
|
Release history: see [`CHANGELOG.md`](https://github.com/allelix/allelix/blob/main/CHANGELOG.md).
|
|
@@ -439,7 +456,7 @@ pytest
|
|
|
439
456
|
|
|
440
457
|
The pre-commit hook enforces `ruff check` + `ruff format --check`. If a commit is blocked, fix the underlying problem rather than skipping the hook.
|
|
441
458
|
|
|
442
|
-
For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
|
|
459
|
+
For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](https://github.com/allelix/allelix/blob/main/test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
|
|
443
460
|
|
|
444
461
|
## License
|
|
445
462
|
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
[](https://www.python.org/downloads/)
|
|
10
10
|
[](https://pypi.org/project/allelix/)
|
|
11
|
-
[](LICENSE)
|
|
11
|
+
[](https://github.com/allelix/allelix/blob/main/LICENSE)
|
|
12
12
|
[](https://github.com/allelix/allelix/actions/workflows/ci.yml)
|
|
13
13
|
[](#privacy)
|
|
14
14
|
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
# Allelix
|
|
18
18
|
|
|
19
|
-
> **Status:** Production. Nine parser formats (including VCF + gVCF), four annotators (ClinVar + ClinPGx + GWAS Catalog + SNPedia), three GRCh38 enrichment sources (gnomAD + AlphaMissense + CADD) reached from GRCh37 input through a cross-build rsID identity table, licensable-source gating for commercial users, dual-build ClinVar caches (GRCh37 + GRCh38), HTML/JSON/terminal reports and annotated VCF output for pipeline integration (v2.3.0+), methylation + pharmacogenomics focused commands, report diffing, persistent config with commercial-mode safety switch.
|
|
19
|
+
> **Status:** Production. Nine parser formats (including VCF + gVCF), four annotators (ClinVar + ClinPGx + GWAS Catalog + SNPedia), three GRCh38 enrichment sources (gnomAD + AlphaMissense + CADD) reached from GRCh37 input through a cross-build rsID identity table, licensable-source gating for commercial users, dual-build ClinVar caches (GRCh37 + GRCh38), HTML/JSON/terminal reports and annotated VCF output for pipeline integration (v2.3.0+), methylation + pharmacogenomics focused commands, report diffing, persistent config with commercial-mode safety switch. The genome build is decided before annotation from the evidence the file carries: VCF contig lengths, or known-SNP positions from the whole file. Evidence that contradicts the file's label, or a VCF with no usable evidence, stops the run and asks for `--build` instead of producing a report on the wrong build; a tentative GRCh36 signal fails closed to GRCh36 because no modern position cache is safe for it (ADR-0025, ADR-0037, ADR-0044). No regex on prose anywhere in production. The [Changelog](https://github.com/allelix/allelix/blob/main/CHANGELOG.md) tracks every release.
|
|
20
20
|
|
|
21
21
|
## Quickstart
|
|
22
22
|
|
|
@@ -60,14 +60,14 @@ See [Development](#development) for source installs and running tests, [Managing
|
|
|
60
60
|
|
|
61
61
|
| Format | Status | Notes |
|
|
62
62
|
|---|---|---|
|
|
63
|
-
| MyHappyGenes (Tempus) | ✓ | Tab-delimited, 5 columns. **Build is auto-detected** — real-world MHG exports mislabel the header as "build 37.1" while shipping GRCh38 coordinates. Allelix
|
|
63
|
+
| MyHappyGenes (Tempus) | ✓ | Tab-delimited, 5 columns. **Build is auto-detected** — real-world MHG exports mislabel the header as "build 37.1" while shipping GRCh38 coordinates. Allelix decides the build from position data and warns on header/data disagreement (ADR-0021, ADR-0044). See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myhappygenes-tempus). |
|
|
64
64
|
| 23andMe | ✓ | Tab-delimited, 4 columns, concatenated genotype. Supports build 36/37/38 from header. I-prefixed probe IDs passed through. |
|
|
65
65
|
| AncestryDNA | ✓ | Tab-delimited, 5 columns. Chromosome mapping: 23→X, 24→Y, 25→X (PAR), 26→MT. V1 and V2 chip layouts. |
|
|
66
|
-
| Family Tree DNA | ✓ | CSV, double-quoted fields, concatenated genotype. Build 37 default. |
|
|
66
|
+
| Family Tree DNA | ✓ | CSV, double-quoted fields, concatenated genotype. Build 37 default when known-SNP positions are present but do not decide; with none at all and no build line, the run stops (ADR-0044). |
|
|
67
67
|
| MyHeritage DNA | ✓ | CSV, same structure as FTDNA. Detected by "MyHeritage" in the file's preamble. Handles double-double-quoted fields and files re-saved through a spreadsheet (padded rows, signature not on line 1, tab-delimited copies). Two exports: the **array** export (pre-2026 kits) and the **low-pass WGS** export (kits from January 2026; genotypes imputed from about 2x coverage), which the report labels as imputed and whose `##reference=build37` line is used as the header build. See [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#myheritage). |
|
|
68
68
|
| Living DNA | ✓ | Tab-delimited despite `.csv` extension. Handles AX-, AFFX-prefixed and CHR:POS positional SNP IDs. |
|
|
69
|
-
| FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default. |
|
|
70
|
-
| FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default. |
|
|
69
|
+
| FTDNA Illumina raw | ✓ | Tab-delimited variant of the FTDNA export (distinct from the CSV format above). `RSID/CHROMOSOME/POSITION/RESULT` columns. Build 37 default when known-SNP positions are present but do not decide. |
|
|
70
|
+
| FTDNA FamFinder | ✓ | Third FTDNA file shape: tab-delimited with **separate `ALLELE1` / `ALLELE2` columns** instead of a concatenated `RESULT`. Detected by the `famfinder` substring in the file header plus the 5-column canonical header. Build 37 default when known-SNP positions are present but do not decide. |
|
|
71
71
|
| VCF / gVCF | ✓ | REF/ALT encoding, `0/1` genotype notation. Plain VCF: absence at a position means reference. gVCF: explicit reference blocks (lines with `<NON_REF>` ALT and `END=` INFO) are not annotated — they match nothing in any annotation database — but they are read at high-value positions: a block with low genotype quality or no reads there makes the position a no-call, and so does a gVCF position no record covers. High-value no-calls list the reason (for example `low_gq_block`, `no_coverage`, `filtered`, `low_gq_call`). A plain VCF has no reference blocks, so a position without a record is reported as unknown rather than as reference or as a no-call. Multi-sample files require `--sample <ID>`. Rows the variant caller rejected (FILTER other than `PASS` or `.`) are skipped and counted in the report; `--keep-filtered` keeps them. A call with genotype quality below `--min-gq` (default 20) or with no reads (DP 0) is reported as a no-call. The spanning-deletion allele `*` is treated as symbolic. A VCF that holds only structural-variant or copy-number records (such as a provider's separate `sv` / `cnv` file) is rejected with a message naming the file to use instead; see [docs/formats.md](https://github.com/allelix/allelix/blob/main/docs/formats.md#sequencingcom-30x-whole-genome) for per-provider deliverables. Streams via stdlib; `.vcf.gz` handled transparently. Optional `pip install allelix[vcf-index]` enables pysam-backed tabix random access for fast `extract --snps` on huge VCFs. |
|
|
72
72
|
|
|
73
73
|
Adding a new format means adding one file to `allelix/parsers/` and registering an instance in the `PARSERS` list in `allelix/parsers/__init__.py`.
|
|
@@ -87,7 +87,7 @@ Issues and milestones are tracked at [github.com/allelix/allelix/issues](https:/
|
|
|
87
87
|
|
|
88
88
|
| Database | Status | Notes |
|
|
89
89
|
|---|---|---|
|
|
90
|
-
| ClinVar (GRCh37 + GRCh38) | ✓ | Public domain (NCBI). SNVs + indels + multi-allelic sites. **Both builds cached**; `analyze` dispatches by
|
|
90
|
+
| ClinVar (GRCh37 + GRCh38) | ✓ | Public domain (NCBI). SNVs + indels + multi-allelic sites. **Both builds cached**; `analyze` dispatches by the decided build (ADR-0044). Rows are matched by chromosome, position, REF and ALT, never by rsID alone, and reference-allele records (REF = ALT) are never reported (ADR-0039). Carrier rule (ADR-0007) requires the user to carry the ALT allele. Indel-anchor protection (ADR-0011) prevents single-base array readouts from matching anchor-base indels. |
|
|
91
91
|
| ClinPGx (formerly PharmGKB) | ✓ | CC BY-SA 4.0. Clinical annotations only — single-rsid SNVs; star alleles and haplotypes deferred (ADR-0009). **Primary non-finding filter is the ClinVar REF carrier rule (ADR-0023):** if ClinVar publishes a single-base REF for the rsid and the user is homozygous for it, the row is suppressed. CPIC's `(rsid, base) → function_class` join (ADR-0020) survives as a secondary tier for rsids ClinVar doesn't catalog. Earlier prose tiers (ADR-0013, ADR-0017, ADR-0018) are superseded. |
|
|
92
92
|
| CPIC (per-allele function table) | ✓ | Internal data source for the ClinPGx filter. Fetched from `api.cpicpgx.org` at `db update` time. Used to populate the `pharmgkb_allele_function` table — not surfaced to end users as its own annotator. |
|
|
93
93
|
| SNPedia | ✓ | CC BY-NC-SA 3.0 US. Pre-built cache downloaded via `db update` (~216K wiki pages, ~105K genotype rows). The cache carries SNPedia's orientation metadata: minus-strand alleles are complemented before matching, while unknown-orientation palindromic genotypes abstain rather than risk a false match. If the database is absent, analysis runs without it. For commercial use, pass `--exclude-snpedia`. The cache can also be rebuilt from source via `scripts/scrape_snpedia.py` + `scripts/parse_snpedia.py`. |
|
|
@@ -344,11 +344,27 @@ If the failure is a network timeout on the same source repeatedly, the issue is
|
|
|
344
344
|
|
|
345
345
|
Almost always a schema or interpreter version bump in a recent allelix release that invalidates the pre-bump cache. The fix is `allelix db update` — it'll re-ingest only the affected annotators (everything else signal-skips). See the relevant `CHANGELOG.md` entry for "Cache invalidation" notes per release.
|
|
346
346
|
|
|
347
|
-
###
|
|
347
|
+
### How the genome build is decided, and why a run can stop
|
|
348
348
|
|
|
349
|
-
|
|
349
|
+
Before anything is annotated, Allelix decides the file's genome build from the evidence the file itself carries (ADR-0044):
|
|
350
350
|
|
|
351
|
-
|
|
351
|
+
- **VCF contig lengths.** Every nuclear chromosome has a different length in GRCh37 and GRCh38, so the `##contig` lengths settle the build whatever the contig names look like (`chr1` or `1`). If chrM has the old hg19 length, mitochondrial records are not annotated and a note says so.
|
|
352
|
+
- **Known-SNP positions across the whole file.** A set of well-known SNPs sits at different positions in each build. Allelix reads the whole file for them before annotating (a few seconds on a full chip). At least three concordant positions and an 80% majority make the evidence confident (ADR-0037). A VCF record without an rsID counts when it sits at such a position.
|
|
353
|
+
- **The file's own label** (a build line in an array header, or a VCF `assembly=`) is used only when the evidence does not contradict it.
|
|
354
|
+
|
|
355
|
+
The banner says where the build came from, for example `Build: GRCh38 (from the contig lengths: 24 contig length(s) match GRCh38)`. If confident positions contradict the label (real-world MHG / Tempus exports say 37 and ship 38), Allelix uses the positions and prints a **Build mismatch** notice. An unlabelled chip whose known-SNP positions do not decide uses its format's documented default, with a visible note, provided at least one of those positions matches a build and the build most of them match is the default (a few that disagree do not stop it); the note gives the counts. A tentative GRCh36 winner is the safety exception: Allelix uses GRCh36 so it cannot query a modern position cache (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)).
|
|
356
|
+
|
|
357
|
+
**The run stops** instead of guessing when:
|
|
358
|
+
|
|
359
|
+
- a VCF's contig lengths contradict its `assembly=`, contradict each other, or match no supported assembly;
|
|
360
|
+
- weaker (tentative) position evidence contradicts the file's label, or points an unlabelled chip away from its default;
|
|
361
|
+
- the known-SNP positions are tied between builds and the build that would otherwise decide (the label, the format default, or the contig-length build) is not one of them. When it is one of them, Allelix proceeds on it and prints an "inconclusive" warning, also shown in the HTML report. Without contig lengths, a tie that includes GRCh36 instead takes the GRCh36 safety exception described above, with the same warning;
|
|
362
|
+
- a VCF has no contig lengths, no assembly and too few known-SNP positions;
|
|
363
|
+
- a chip states no build and carries none of the known-SNP positions at all (for example an X-chromosome-only export, since they are all on autosomes);
|
|
364
|
+
- the file has known-SNP positions but none of them sits at any supported build's coordinate, whether or not it states a build (contig lengths, when a VCF declares them, still decide);
|
|
365
|
+
- contig lengths decided, but the known-SNP positions met while annotating point to the other build (for example a lifted-over file that kept its old header). The message says the annotation pass ran; nothing is written.
|
|
366
|
+
|
|
367
|
+
The message names the evidence and asks you to say which build the file uses: `--build grch37` or `--build grch38`, or `--build grch36` for an old GRCh36 / hg18 file, which then runs with position-keyed lookups off (see [`docs/grch36-liftover.md`](https://github.com/allelix/allelix/blob/main/docs/grch36-liftover.md)). For `compare`, use `--build1` / `--build2`. A stop leaves no report, annotated VCF, diff, PLINK files or extract table behind. A forced build is recorded as such in every report.
|
|
352
368
|
|
|
353
369
|
### Is my database stale? How fresh is the data?
|
|
354
370
|
|
|
@@ -380,6 +396,7 @@ Notable load-bearing ADRs:
|
|
|
380
396
|
- **ADR-0038 — GWAS effect-modifier disambiguation.** Only positively identified odds ratios receive an effect-size modifier; beta and ambiguous effects abstain.
|
|
381
397
|
- **ADR-0042 — VCF quality gate.** The variant caller's FILTER and genotype quality are honoured, low-quality reference blocks are no-calls, and every exclusion is counted in the report.
|
|
382
398
|
- **ADR-0043 — One rsID-recovery step.** Every command that needs rsIDs recovers them for rsID-less input through the same shared step; build detection votes by position instead.
|
|
399
|
+
- **ADR-0044 — Evidence before labels.** The build is decided before annotation from contig lengths or whole-file known-SNP positions; evidence that contradicts the label, or no usable evidence, stops the run instead of producing a report on the wrong build.
|
|
383
400
|
- **ADR-0041 — Plausibility flag.** Very rare homozygous calls and heterozygous calls with no gnomAD carriers are flagged for review, never suppressed.
|
|
384
401
|
|
|
385
402
|
Release history: see [`CHANGELOG.md`](https://github.com/allelix/allelix/blob/main/CHANGELOG.md).
|
|
@@ -400,7 +417,7 @@ pytest
|
|
|
400
417
|
|
|
401
418
|
The pre-commit hook enforces `ruff check` + `ruff format --check`. If a commit is blocked, fix the underlying problem rather than skipping the hook.
|
|
402
419
|
|
|
403
|
-
For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
|
|
420
|
+
For the full real-data release-validation battery — every parser format, VCF/gVCF, PLINK round-trip, edge cases, the GIAB/1000G corpus — see [`test_data/FULL_TEST_PROTOCOL.md`](https://github.com/allelix/allelix/blob/main/test_data/FULL_TEST_PROTOCOL.md). That document is the project's real-data testing story; running `pytest` covers the fast suite that gates every commit.
|
|
404
421
|
|
|
405
422
|
## License
|
|
406
423
|
|
|
@@ -10,8 +10,8 @@ from importlib.metadata import PackageNotFoundError, version
|
|
|
10
10
|
def _read_pyproject_version() -> str | None:
|
|
11
11
|
"""Read the package version from ``pyproject.toml``.
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
checkout (no editable install, no installed package metadata).
|
|
13
|
+
This is the fallback when run from a bare source
|
|
14
|
+
checkout (no editable install, no installed package metadata). It keeps
|
|
15
15
|
``--version`` and the outbound HTTP User-Agent reporting the real
|
|
16
16
|
version string instead of the ``0.0.0+local`` sentinel that
|
|
17
17
|
misidentifies our traffic to NCBI / EBI / HuggingFace.
|
|
@@ -129,12 +129,16 @@ class AlphaMissenseAnnotator(Annotator):
|
|
|
129
129
|
logger.warning("Could not remove staged file at %s", gz_path)
|
|
130
130
|
|
|
131
131
|
def is_ready(self) -> bool:
|
|
132
|
-
"""True when the AlphaMissense SQLite cache exists with current schema version.
|
|
132
|
+
"""True when the AlphaMissense SQLite cache exists with current schema version.
|
|
133
|
+
|
|
134
|
+
A cache with no ``local_version_tag`` is not ready: its schema version
|
|
135
|
+
is unknown, so it could pass as the current one after a bump. The
|
|
136
|
+
installer stamps the current version, so ``db update`` replaces it.
|
|
137
|
+
"""
|
|
133
138
|
info = get_database_info(self._db_path, "alphamissense")
|
|
134
139
|
if info is None:
|
|
135
140
|
return False
|
|
136
|
-
|
|
137
|
-
return tag == f"sv:{ALPHAMISSENSE_SCHEMA_VERSION}" or not tag
|
|
141
|
+
return info.get("local_version_tag") == f"sv:{ALPHAMISSENSE_SCHEMA_VERSION}"
|
|
138
142
|
|
|
139
143
|
def version(self) -> str | None:
|
|
140
144
|
"""Return the cached database version, or None."""
|
|
@@ -158,7 +158,9 @@ class CaddAnnotator(Annotator):
|
|
|
158
158
|
"""True when the active backend is available.
|
|
159
159
|
|
|
160
160
|
In full mode, checks for the tabix file. In cache mode, checks
|
|
161
|
-
the SQLite database
|
|
161
|
+
the SQLite database, which must carry the current schema stamp: a
|
|
162
|
+
cache with no ``local_version_tag`` has an unknown schema version, so
|
|
163
|
+
it is not ready and ``db update`` replaces it.
|
|
162
164
|
"""
|
|
163
165
|
if self._full_mode:
|
|
164
166
|
return (
|
|
@@ -168,8 +170,7 @@ class CaddAnnotator(Annotator):
|
|
|
168
170
|
info = get_database_info(self._db_path, "cadd")
|
|
169
171
|
if info is None:
|
|
170
172
|
return False
|
|
171
|
-
|
|
172
|
-
return tag == f"sv:{CADD_SCHEMA_VERSION}" or not tag
|
|
173
|
+
return info.get("local_version_tag") == f"sv:{CADD_SCHEMA_VERSION}"
|
|
173
174
|
|
|
174
175
|
def version(self) -> str | None:
|
|
175
176
|
"""Return the cached database version, or None."""
|
|
@@ -153,7 +153,7 @@ def _collapse_identical_scvs(annotations: list[Annotation]) -> list[Annotation]:
|
|
|
153
153
|
surface, and collapsing them would be the Frankenstein-pair
|
|
154
154
|
regression the per-SCV loader retired.
|
|
155
155
|
|
|
156
|
-
|
|
156
|
+
``alt`` is in the key for the
|
|
157
157
|
wrong-allele safety invariant. Two SCVs at a multi-allelic site
|
|
158
158
|
can share (significance, condition, gene) but differ in alt — e.g.
|
|
159
159
|
rs1065852 (G→A drug_response/Codeine_response + G→C drug_response/
|
|
@@ -283,7 +283,7 @@ class ClinVarAnnotator(Annotator):
|
|
|
283
283
|
def setup(self) -> None:
|
|
284
284
|
"""Download the per-SCV TSV pair once and ingest both builds from it.
|
|
285
285
|
|
|
286
|
-
v2.2
|
|
286
|
+
Since v2.2 the VCF-per-build path is replaced by the
|
|
287
287
|
variant_summary + submission_summary TSV pair. variant_summary
|
|
288
288
|
carries per-(VariationID, Assembly) rows so the same downloaded
|
|
289
289
|
file populates both GRCh37 and GRCh38 caches with no second
|
|
@@ -445,7 +445,7 @@ class ClinVarAnnotator(Annotator):
|
|
|
445
445
|
# may have BOTH SNV and indel rows for the same rsid; the WHERE
|
|
446
446
|
# filters those out so we keep only the SNV REF.
|
|
447
447
|
#
|
|
448
|
-
#
|
|
448
|
+
# Iterate the cursor directly instead of fetchall()
|
|
449
449
|
# so peak memory is one row + the dict, not the entire result set
|
|
450
450
|
# twice.
|
|
451
451
|
cursor = conn.execute(
|
|
@@ -463,8 +463,8 @@ class ClinVarAnnotator(Annotator):
|
|
|
463
463
|
def fetch_remote_signal(self) -> str | None:
|
|
464
464
|
r"""Freshness signal derived from the per-SCV submission_summary md5.
|
|
465
465
|
|
|
466
|
-
v2.2
|
|
467
|
-
|
|
466
|
+
Before v2.2 this returned a composite VCF-md5 per build. Since
|
|
467
|
+
v2.2, the cache is built from a build-agnostic
|
|
468
468
|
TSV pair, and submission_summary's md5 endpoint is the change
|
|
469
469
|
cadence we want to track (per-SCV submissions land daily; the
|
|
470
470
|
slower-moving variant_summary only adds new VariationIDs).
|
|
@@ -500,7 +500,7 @@ class ClinVarAnnotator(Annotator):
|
|
|
500
500
|
None if any managed build's cache is absent / unstamped or if the
|
|
501
501
|
builds' stamps diverge (an interrupted partial reingest).
|
|
502
502
|
|
|
503
|
-
|
|
503
|
+
Since v2.2 both builds share the same source TSVs and therefore
|
|
504
504
|
the same remote_signal. We still cross-check that EVERY managed
|
|
505
505
|
build's cache carries the stamp; a missing build means the
|
|
506
506
|
last-setup didn't complete, treat as "no signal" and let
|
|
@@ -699,7 +699,7 @@ class ClinVarAnnotator(Annotator):
|
|
|
699
699
|
input). Together with the locus this identifies the variant by
|
|
700
700
|
``(chrom, pos, ref, alt)`` rather than by rsID.
|
|
701
701
|
- ADR-0011: ClinVar indels never match single-base array calls.
|
|
702
|
-
- ADR-0035
|
|
702
|
+
- ADR-0035 / ADR-0007: the user carries the ALT, directly or
|
|
703
703
|
via a confirmed strand flip.
|
|
704
704
|
- Benign rows are suppressed unless ``include_benign``.
|
|
705
705
|
|
|
@@ -114,12 +114,10 @@ class GnomadAnnotator(Annotator):
|
|
|
114
114
|
def is_ready(self) -> bool:
|
|
115
115
|
"""True when the gnomAD SQLite cache exists with current schema version.
|
|
116
116
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
version. Reject tagless caches so the user is told to re-run
|
|
122
|
-
``db update``.
|
|
117
|
+
A cache with no ``local_version_tag`` is not ready. Accepting it
|
|
118
|
+
would defeat ``GNOMAD_SCHEMA_VERSION``: after a bump, every
|
|
119
|
+
untagged legacy cache would silently pass as the new version.
|
|
120
|
+
Rejecting it tells the user to re-run ``db update``.
|
|
123
121
|
"""
|
|
124
122
|
info = get_database_info(self._db_path, "gnomad")
|
|
125
123
|
if info is None:
|
|
@@ -26,6 +26,7 @@ from allelix.databases.manager import (
|
|
|
26
26
|
download,
|
|
27
27
|
get_database_info,
|
|
28
28
|
head_request_headers,
|
|
29
|
+
parse_baked_version,
|
|
29
30
|
)
|
|
30
31
|
from allelix.models import Annotation
|
|
31
32
|
|
|
@@ -57,7 +58,7 @@ logger = logging.getLogger(__name__)
|
|
|
57
58
|
|
|
58
59
|
_BATCH_CHUNK = 500 # SQLite default SQLITE_MAX_VARIABLE_NUMBER is 999
|
|
59
60
|
|
|
60
|
-
# ADR-0035
|
|
61
|
+
# ADR-0035: PheCode often arrives embedded in the upstream DISEASE/TRAIT
|
|
61
62
|
# field (e.g. ``"Type 2 diabetes (PheCode 250.2)"``). The annotator extracts it
|
|
62
63
|
# at the boundary so downstream code reads ``Annotation.phecode`` as a
|
|
63
64
|
# structured field instead of regex-parsing prose. The trimmed trait flows into
|
|
@@ -504,8 +505,11 @@ def _stamp_existing_gwas_cache(db_path: Path) -> bool:
|
|
|
504
505
|
"""One-shot migration: stamp ``local_version_tag`` on a GWAS cache.
|
|
505
506
|
|
|
506
507
|
Handles legacy caches with ``|cv:N`` baked into ``remote_signal``
|
|
507
|
-
by moving the tag and cleaning the signal
|
|
508
|
-
current categorizer version
|
|
508
|
+
by moving the tag and cleaning the signal, but only when ``N`` is the
|
|
509
|
+
current categorizer version. A cache without the marker, or with
|
|
510
|
+
another ``N``, has unknown or old data: it is left unstamped and the
|
|
511
|
+
caller re-ingests or re-downloads it. Returns True if the current
|
|
512
|
+
categorizer version is now stamped.
|
|
509
513
|
"""
|
|
510
514
|
if not db_path.exists():
|
|
511
515
|
return False
|
|
@@ -523,6 +527,8 @@ def _stamp_existing_gwas_cache(db_path: Path) -> bool:
|
|
|
523
527
|
return True
|
|
524
528
|
if existing_tag is not None:
|
|
525
529
|
return False
|
|
530
|
+
if parse_baked_version(sig, "cv") != _CATEGORIZER_VERSION:
|
|
531
|
+
return False
|
|
526
532
|
clean_signal = (sig or "").split("|cv:")[0]
|
|
527
533
|
conn.execute(
|
|
528
534
|
"UPDATE database_versions "
|
|
@@ -122,9 +122,9 @@ class PharmGKBAnnotator(Annotator):
|
|
|
122
122
|
per-allele function table is the secondary fallback for rsids
|
|
123
123
|
ClinVar doesn't catalog.
|
|
124
124
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
125
|
+
This is also how a cache built by an older interpreter version is
|
|
126
|
+
rebuilt: ``is_ready()`` reports it stale and ``db update`` calls
|
|
127
|
+
this, so the CPIC table is fetched again with the annotations.
|
|
128
128
|
"""
|
|
129
129
|
url = PHARMGKB_CLINICAL_URL
|
|
130
130
|
signal = self.fetch_remote_signal()
|
|
@@ -162,13 +162,15 @@ class PharmGKBAnnotator(Annotator):
|
|
|
162
162
|
def is_ready(self) -> bool:
|
|
163
163
|
"""True iff a ClinPGx SQLite cache exists with current schema and interpreter stamp.
|
|
164
164
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
165
|
+
Never touches the network, so ``analyze`` stays offline. When the
|
|
166
|
+
interpreter version has bumped, the cache is not ready and
|
|
167
|
+
``allelix db update`` rebuilds it, fetching the ClinPGx annotations
|
|
168
|
+
and the CPIC allele-function table again (``setup``).
|
|
169
169
|
|
|
170
|
-
Pre-mechanism caches (tag missing
|
|
171
|
-
are self-healed with a one-shot stamp update
|
|
170
|
+
Pre-mechanism caches (tag missing, version baked into
|
|
171
|
+
``remote_signal``) are self-healed with a one-shot stamp update
|
|
172
|
+
when the baked version is the current one; otherwise they are not
|
|
173
|
+
ready either.
|
|
172
174
|
"""
|
|
173
175
|
info = get_database_info(self._db_path, "pharmgkb")
|
|
174
176
|
if info is None:
|
|
@@ -178,9 +180,7 @@ class PharmGKBAnnotator(Annotator):
|
|
|
178
180
|
tag = info.get("local_version_tag") or ""
|
|
179
181
|
if tag == f"iv:{PHARMGKB_INTERPRETER_VERSION}":
|
|
180
182
|
return True
|
|
181
|
-
|
|
182
|
-
return _stamp_existing_pharmgkb_cache(self._db_path)
|
|
183
|
-
return _reingest_pharmgkb_from_cached_zip(self._db_path, self.data_dir)
|
|
183
|
+
return not tag and _stamp_existing_pharmgkb_cache(self._db_path)
|
|
184
184
|
|
|
185
185
|
def version(self) -> str | None:
|
|
186
186
|
"""Cached database version (download date, or version supplied to load)."""
|
|
@@ -199,7 +199,7 @@ class PharmGKBAnnotator(Annotator):
|
|
|
199
199
|
self._conn = None
|
|
200
200
|
|
|
201
201
|
def fetch_remote_signal(self) -> str | None:
|
|
202
|
-
"""Composite freshness signal for ClinPGx + CPIC (
|
|
202
|
+
"""Composite freshness signal for ClinPGx + CPIC (ADR-0020).
|
|
203
203
|
|
|
204
204
|
The signal format is `pgkb:<pgkb-signal>|cpic:<cpic-signal>`.
|
|
205
205
|
|
|
@@ -269,7 +269,7 @@ class PharmGKBAnnotator(Annotator):
|
|
|
269
269
|
if is_clinvar_homref(variant, self._clinvar_ref_provider):
|
|
270
270
|
return []
|
|
271
271
|
|
|
272
|
-
# ADR-0035
|
|
272
|
+
# ADR-0035: fetch all rows for the rsid (typically <5) and apply
|
|
273
273
|
# strand-aware genotype match in Python. Forward-stranded users hit
|
|
274
274
|
# the same set of rows as the v0.7-era ``WHERE genotype = ?`` SQL;
|
|
275
275
|
# reverse-stranded users now also match when Variant.ref confirms
|
|
@@ -402,7 +402,7 @@ class PharmGKBAnnotator(Annotator):
|
|
|
402
402
|
_score,
|
|
403
403
|
pgkb_annotation_id,
|
|
404
404
|
) = row
|
|
405
|
-
# ADR-0035
|
|
405
|
+
# ADR-0035: strand-aware genotype match. See annotate() for
|
|
406
406
|
# the safety rationale (palindrome guard + variant.ref orientation).
|
|
407
407
|
if not strand_aware_genotype_match(
|
|
408
408
|
variant.allele1,
|
|
@@ -458,12 +458,15 @@ def _stamp_existing_pharmgkb_cache(db_path: Path) -> bool:
|
|
|
458
458
|
"""One-shot migration: stamp ``local_version_tag`` on a ClinPGx cache.
|
|
459
459
|
|
|
460
460
|
Handles legacy caches with ``|iv:N`` baked into ``remote_signal``
|
|
461
|
-
by moving the tag and cleaning the signal
|
|
462
|
-
current interpreter version
|
|
461
|
+
by moving the tag and cleaning the signal, but only when ``N`` is the
|
|
462
|
+
current interpreter version. A cache without the marker, or with
|
|
463
|
+
another ``N``, has unknown or old data: it is left unstamped and the
|
|
464
|
+
caller re-ingests or re-downloads it. Returns True if the current
|
|
465
|
+
interpreter version is now stamped.
|
|
463
466
|
"""
|
|
464
467
|
import contextlib
|
|
465
468
|
|
|
466
|
-
from allelix.databases.manager import _ensure_local_version_tag_column
|
|
469
|
+
from allelix.databases.manager import _ensure_local_version_tag_column, parse_baked_version
|
|
467
470
|
|
|
468
471
|
if not db_path.exists():
|
|
469
472
|
return False
|
|
@@ -480,6 +483,10 @@ def _stamp_existing_pharmgkb_cache(db_path: Path) -> bool:
|
|
|
480
483
|
sig, existing_tag = row
|
|
481
484
|
if existing_tag == tag:
|
|
482
485
|
return True
|
|
486
|
+
if existing_tag is not None:
|
|
487
|
+
return False
|
|
488
|
+
if parse_baked_version(sig, "iv") != PHARMGKB_INTERPRETER_VERSION:
|
|
489
|
+
return False
|
|
483
490
|
clean_signal = (sig or "").split("|iv:")[0]
|
|
484
491
|
conn.execute(
|
|
485
492
|
"UPDATE database_versions "
|
|
@@ -491,57 +498,3 @@ def _stamp_existing_pharmgkb_cache(db_path: Path) -> bool:
|
|
|
491
498
|
return True
|
|
492
499
|
except (sqlite3.OperationalError, sqlite3.DatabaseError):
|
|
493
500
|
return False
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
def _read_cached_cpic_lookup(db_path: Path) -> dict[tuple[str, str], str]:
|
|
497
|
-
"""Extract the CPIC allele-function table from an existing ClinPGx cache."""
|
|
498
|
-
import contextlib
|
|
499
|
-
|
|
500
|
-
lookup: dict[tuple[str, str], str] = {}
|
|
501
|
-
try:
|
|
502
|
-
with contextlib.closing(sqlite3.connect(db_path)) as conn:
|
|
503
|
-
rows = conn.execute(
|
|
504
|
-
"SELECT rsid, allele, function_class FROM pharmgkb_allele_function"
|
|
505
|
-
).fetchall()
|
|
506
|
-
for rsid, allele, function_class in rows:
|
|
507
|
-
lookup[(rsid, allele)] = function_class
|
|
508
|
-
except (sqlite3.OperationalError, sqlite3.DatabaseError):
|
|
509
|
-
pass
|
|
510
|
-
return lookup
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
def _reingest_pharmgkb_from_cached_zip(db_path: Path, data_dir: Path) -> bool:
|
|
514
|
-
"""Re-ingest ClinPGx from the retained ZIP when the interpreter version bumps.
|
|
515
|
-
|
|
516
|
-
Reads the existing CPIC allele-function data from the current cache
|
|
517
|
-
before replacing it — no network required for the reingest. Preserves
|
|
518
|
-
the original source URL and version from the previous cache.
|
|
519
|
-
"""
|
|
520
|
-
zip_path = data_dir / "clinicalAnnotations.zip"
|
|
521
|
-
if not zip_path.exists():
|
|
522
|
-
return False
|
|
523
|
-
info = get_database_info(db_path, "pharmgkb")
|
|
524
|
-
if info is None:
|
|
525
|
-
return False
|
|
526
|
-
old_signal = info.get("remote_signal") or ""
|
|
527
|
-
old_version = info.get("version") or ""
|
|
528
|
-
old_source_url = info.get("source_url") or ""
|
|
529
|
-
cpic_lookup = _read_cached_cpic_lookup(db_path)
|
|
530
|
-
logger.info("ClinPGx interpreter changed — re-ingesting from cached ZIP...")
|
|
531
|
-
try:
|
|
532
|
-
load_pharmgkb_tsv(
|
|
533
|
-
zip_path,
|
|
534
|
-
db_path,
|
|
535
|
-
source_url=old_source_url,
|
|
536
|
-
version=old_version,
|
|
537
|
-
remote_signal=old_signal,
|
|
538
|
-
allele_function_lookup=cpic_lookup,
|
|
539
|
-
min_rows=PHARMGKB_MIN_ROWS,
|
|
540
|
-
)
|
|
541
|
-
except Exception:
|
|
542
|
-
logger.warning("Auto-reingest from cached ZIP failed", exc_info=True)
|
|
543
|
-
return False
|
|
544
|
-
new_info = get_database_info(db_path, "pharmgkb")
|
|
545
|
-
if new_info is None:
|
|
546
|
-
return False
|
|
547
|
-
return (new_info.get("local_version_tag") or "") == f"iv:{PHARMGKB_INTERPRETER_VERSION}"
|