afquery 0.3.3__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {afquery-0.3.3 → afquery-0.4.0}/PKG-INFO +2 -7
- {afquery-0.3.3 → afquery-0.4.0}/README.md +1 -6
- {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/debugging-results.md +2 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/faq.md +5 -1
- {afquery-0.3.3 → afquery-0.4.0}/docs/guides/variant-info.md +12 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/_version.py +2 -2
- afquery-0.4.0/src/afquery/capture.py +216 -0
- afquery-0.4.0/src/afquery/constants.py +59 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/build.py +28 -15
- afquery-0.4.0/src/afquery/preprocess/regions.py +25 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/query.py +20 -11
- afquery-0.4.0/tests/data/beds/wes_kit_nochr.bed +2 -0
- afquery-0.4.0/tests/test_capture.py +302 -0
- afquery-0.4.0/tests/test_constants.py +98 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_preprocess.py +27 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_warnings.py +69 -0
- afquery-0.3.3/src/afquery/capture.py +0 -94
- afquery-0.3.3/src/afquery/constants.py +0 -44
- afquery-0.3.3/src/afquery/preprocess/regions.py +0 -14
- afquery-0.3.3/tests/test_capture.py +0 -96
- afquery-0.3.3/tests/test_constants.py +0 -50
- {afquery-0.3.3 → afquery-0.4.0}/.dockerignore +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/.github/workflows/ci.yml +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/.github/workflows/docs.yml +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/.github/workflows/release.yml +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/.gitignore +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/CONTRIBUTING.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/Dockerfile +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/LICENSE +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/.gitignore +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/INSTALL.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/README.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/Snakefile +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/.gitignore +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/01a_assign_samples.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/01b_write_manifests.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/02_build_databases.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/03_compute_metrics.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/04_classify_acmg.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/05_plot_figures.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/README.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/Snakefile +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/__init__.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/beds/SureSelect_v5.bed +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/beds/SureSelect_v6.bed +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/beds/SureSelect_v7.bed +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/config.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/config.yaml +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/envs/benchmark.yaml +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/.gitignore +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/01_prepare_data.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/02_query_scaling.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/03_build.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/04_annotate.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/05_vs_bcftools.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/06_plot.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/README.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/Snakefile +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_annotate.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_bcftools.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_build_perf.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_prepare.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_query_scaling.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/config.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/config_smoke.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/shared/__init__.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/shared/config.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/shared/rules/download_1kg.smk +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/benchmarks/shared/utils.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/benchmarking.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/coverage-evidence.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/filter-pass-tracking.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/multi-cohort-strategies.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/performance.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/pipeline-integration.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/ploidy-and-sex-chroms.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/assets/img/gap2_mixed_technologies.png +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/concepts.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/installation.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/motivation.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/preprocessing.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/quickstart.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/tutorial.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/understanding-output.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/guides/annotate-vcf.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/guides/create-database.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/guides/dump-export.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/guides/manifest-format.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/guides/query.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/guides/sample-filtering.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/guides/update-database.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/index.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/reference/cli.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/reference/data-model.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/reference/glossary.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/reference/python-api.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/scripts/gen_gap2_figure.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/stylesheets/extra.css +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/troubleshooting.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/acmg-use-cases.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/clinical-prioritization.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/cohort-stratification.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/population-specific-af.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/pseudo-controls.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/sex-specific-af.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/technology-integration.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/examples/demo/README.md +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/examples/demo/create_demo_data.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/mkdocs.yml +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/pyproject.toml +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/recipes/afquery/meta.yaml +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/resources/normalize_vcf.sh +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/__init__.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/annotate.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/benchmark.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/bitmaps.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/cli.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/cli.pyr +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/database.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/dump.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/models.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/ploidy.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/__init__.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/compact.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/ingest.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/manifest.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/synth.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/update.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/src/afquery/variant_info.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/conftest.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/annotate_input.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/annotate_multi_bucket.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/annotate_multi_chrom.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/beds/wes_kit_a.bed +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/beds/wes_kit_b.bed +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/expected_results.json +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/manifest.tsv +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S00.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S01.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S02.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S03.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S04.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S05.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S06.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S07.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S08.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S09.vcf +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_annotate.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_batch.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_benchmark.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_bitmaps.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_cli.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_cli_docs_consistency.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_compact.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_dump.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_haploid_stats.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_info.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_no_coverage.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_pass_filter.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_ploidy.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_query.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_sample_filter.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_synth.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_synthetic_stats.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_update.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_update_metadata.py +0 -0
- {afquery-0.3.3 → afquery-0.4.0}/tests/test_variant_info.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: afquery
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Genomic allele frequency query engine with bitmap-encoded genotypes
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -40,7 +40,7 @@ Description-Content-Type: text/markdown
|
|
|
40
40
|
[](https://pypi.org/project/afquery/)
|
|
41
41
|
[](https://github.com/babelomics/afquery/blob/master/LICENSE)
|
|
42
42
|
|
|
43
|
-
**Fast, capture-aware allele frequency queries on local genomic cohorts
|
|
43
|
+
**Fast, capture-aware allele frequency queries on local genomic cohorts**
|
|
44
44
|
|
|
45
45
|
AFQuery is a bitmap-indexed engine that recomputes AC/AN/AF for arbitrary subcohorts (by phenotype, sex, sequencing technology, or any combination) in tens of milliseconds, independently of cohort size. It accounts for capture-kit heterogeneity, ploidy on sex chromosomes, and FILTER/coverage evidence, and runs locally as a file-based system (Parquet + SQLite) with no server or cluster required.
|
|
46
46
|
|
|
@@ -231,7 +231,6 @@ AFQuery indexes each variant as three Roaring Bitmaps — heterozygous PASS carr
|
|
|
231
231
|
| Workload | AFQuery | bcftools | Speedup |
|
|
232
232
|
|---|---|---|---|
|
|
233
233
|
| Full-chromosome AC/AN/AF export | ~7.0 s | ~3.8 min | **~33×** |
|
|
234
|
-
| AF concordance over 1,106,181 common variants | — | — | **R² > 0.99999** |
|
|
235
234
|
|
|
236
235
|
Point-query latency on AFQuery is ~14 ms and constant from 1K to 50K samples (median over 50 replicates, warm cache).
|
|
237
236
|
|
|
@@ -250,7 +249,3 @@ If you use AFQuery in your work, please cite:
|
|
|
250
249
|
|
|
251
250
|
> AFQuery: fast, capture-aware allele frequency queries on local genomic cohorts.
|
|
252
251
|
> *(manuscript in preparation)*
|
|
253
|
-
|
|
254
|
-
## License
|
|
255
|
-
|
|
256
|
-
[MIT](LICENSE)
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
[](https://pypi.org/project/afquery/)
|
|
11
11
|
[](https://github.com/babelomics/afquery/blob/master/LICENSE)
|
|
12
12
|
|
|
13
|
-
**Fast, capture-aware allele frequency queries on local genomic cohorts
|
|
13
|
+
**Fast, capture-aware allele frequency queries on local genomic cohorts**
|
|
14
14
|
|
|
15
15
|
AFQuery is a bitmap-indexed engine that recomputes AC/AN/AF for arbitrary subcohorts (by phenotype, sex, sequencing technology, or any combination) in tens of milliseconds, independently of cohort size. It accounts for capture-kit heterogeneity, ploidy on sex chromosomes, and FILTER/coverage evidence, and runs locally as a file-based system (Parquet + SQLite) with no server or cluster required.
|
|
16
16
|
|
|
@@ -201,7 +201,6 @@ AFQuery indexes each variant as three Roaring Bitmaps — heterozygous PASS carr
|
|
|
201
201
|
| Workload | AFQuery | bcftools | Speedup |
|
|
202
202
|
|---|---|---|---|
|
|
203
203
|
| Full-chromosome AC/AN/AF export | ~7.0 s | ~3.8 min | **~33×** |
|
|
204
|
-
| AF concordance over 1,106,181 common variants | — | — | **R² > 0.99999** |
|
|
205
204
|
|
|
206
205
|
Point-query latency on AFQuery is ~14 ms and constant from 1K to 50K samples (median over 50 replicates, warm cache).
|
|
207
206
|
|
|
@@ -220,7 +219,3 @@ If you use AFQuery in your work, please cite:
|
|
|
220
219
|
|
|
221
220
|
> AFQuery: fast, capture-aware allele frequency queries on local genomic cohorts.
|
|
222
221
|
> *(manuscript in preparation)*
|
|
223
|
-
|
|
224
|
-
## License
|
|
225
|
-
|
|
226
|
-
[MIT](LICENSE)
|
|
@@ -16,6 +16,8 @@ AN=0 means no eligible samples at the queried position. Work through these check
|
|
|
16
16
|
| Position exists in database | `afquery query --db ./db/ --locus chr1:12345678` | If no result at all, the variant was not observed in any sample during ingestion. |
|
|
17
17
|
| BED coverage (WES) | `afquery info --db ./db/` | If all eligible samples are WES and the position is outside capture regions, AN=0 is correct. |
|
|
18
18
|
| Sample filter too restrictive | Remove `--phenotype` and `--sex` filters | Query with no filters first. If AN>0 without filters, the filter is excluding all samples. |
|
|
19
|
+
| WES samples missing from AN | Compare `variant-info` carriers against `query` counts | If `variant-info` shows many WES carriers but AN reflects only the WGS samples, that technology's capture index is not matching the position. `afquery` warns on open when a capture BED is empty or matches no known chromosome, naming the offending contigs. |
|
|
20
|
+
| AN=0 only on chrM | `afquery query --db ./db/ --locus chrM:3243` | Mitochondrial coverage is easy to lose on its own: a BED naming it `chrMT` still matches on the autosomes, so nothing else looks wrong. Names are normalized (`MT`, `M`, `chrMT` → `chrM`) since v0.4.0 — databases built before that may need the capture BED re-checked. |
|
|
19
21
|
| Technology filter | Remove `--tech` filter | Check if any samples match the requested technology. |
|
|
20
22
|
|
|
21
23
|
### 2. Unexpected AF Value
|
|
@@ -100,7 +100,11 @@ Yes. Both GRCh37 and GRCh38 are supported. Specify at database creation:
|
|
|
100
100
|
afquery create-db --genome-build GRCh37 ...
|
|
101
101
|
```
|
|
102
102
|
|
|
103
|
-
The genome build affects PAR1/PAR2 coordinates on chrX (see [Ploidy & Special Chromosomes](advanced/ploidy-and-sex-chroms.md)). Chromosome names in your VCFs
|
|
103
|
+
The genome build affects PAR1/PAR2 coordinates on chrX (see [Ploidy & Special Chromosomes](advanced/ploidy-and-sex-chroms.md)). Chromosome names in your VCFs and capture BEDs are normalized to a canonical `chr`-prefixed
|
|
104
|
+
form, so the common naming conventions interoperate: `1` and `chr1` both become `chr1`, `X`
|
|
105
|
+
and `x` become `chrX`, and `MT`, `M` and `chrMT` all become `chrM`. Unplaced and alt contigs
|
|
106
|
+
(`GL000209.1`, `chr1_KI270706v1_random`) are left alone and skipped at build time — the build
|
|
107
|
+
log reports which ones it dropped.
|
|
104
108
|
|
|
105
109
|
---
|
|
106
110
|
|
|
@@ -15,6 +15,18 @@ afquery variant-info --db ./db/ --locus chr1:925952
|
|
|
15
15
|
!!! tip
|
|
16
16
|
`variant-info` is the natural next step after `query` — once you find a variant of interest, use it to see which specific samples carry it.
|
|
17
17
|
|
|
18
|
+
!!! note "Carrier counts may exceed the counts reported by `query`"
|
|
19
|
+
`variant-info` lists every carrier present in the source VCFs. `query` additionally
|
|
20
|
+
restricts to samples whose capture regions cover the position, so a WES sample with a
|
|
21
|
+
call just outside its own BED appears here but is not counted in `AN`, `AC` or the
|
|
22
|
+
genotype tallies. The two commands answer different questions — "who carries it?"
|
|
23
|
+
versus "what is the frequency among samples that could have been called?" — so a
|
|
24
|
+
modest difference is expected.
|
|
25
|
+
|
|
26
|
+
A *large* gap is worth investigating: if `variant-info` shows many carriers from a
|
|
27
|
+
technology that contributes nothing to `AN`, check that technology's capture index
|
|
28
|
+
(see [Debugging Results](../advanced/debugging-results.md)).
|
|
29
|
+
|
|
18
30
|
By default all samples are queried and results are printed as an aligned text table:
|
|
19
31
|
|
|
20
32
|
```
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0,
|
|
21
|
+
__version__ = version = '0.4.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 4, 0)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import bisect
|
|
2
|
+
import itertools
|
|
3
|
+
import pickle
|
|
4
|
+
import warnings
|
|
5
|
+
import pyranges as pr
|
|
6
|
+
import pandas as pd
|
|
7
|
+
from .constants import ALL_CHROMS, normalize_chrom
|
|
8
|
+
from .models import Technology
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class CaptureIndex:
|
|
12
|
+
"""Interval index over a capture BED, queried by 1-based position.
|
|
13
|
+
|
|
14
|
+
Chromosome keys in ``_index`` are always normalized (see
|
|
15
|
+
:func:`~afquery.constants.normalize_chrom`), because queries reach
|
|
16
|
+
:meth:`covers` already normalized. BED files in the wild use either
|
|
17
|
+
convention ('1' for GRCh37/hs37d5, 'chr1' for hg38), so normalizing on
|
|
18
|
+
construction is what keeps the two sides in agreement.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
_always_covered: bool
|
|
22
|
+
_pr: "pr.PyRanges | None"
|
|
23
|
+
# Per-chrom sorted interval index, keyed by normalized chrom:
|
|
24
|
+
# {chrom: (sorted_starts, corresponding_ends, running_max_of_ends)}
|
|
25
|
+
# The running max lets covers() answer in O(log n) — see there.
|
|
26
|
+
_index: "dict[str, tuple[list[int], list[int], list[int]]]"
|
|
27
|
+
|
|
28
|
+
def __init__(self, *, always_covered: bool = False, pyranges_obj=None):
|
|
29
|
+
self._always_covered = always_covered
|
|
30
|
+
self._pr = pyranges_obj
|
|
31
|
+
self._index = {}
|
|
32
|
+
if pyranges_obj is not None:
|
|
33
|
+
df = pyranges_obj.df
|
|
34
|
+
by_chrom: dict[str, list[tuple[int, int]]] = {}
|
|
35
|
+
for chrom, group in df.groupby("Chromosome", observed=True):
|
|
36
|
+
# Normalizing can merge two groups ('1' and 'chr1') into one key,
|
|
37
|
+
# so accumulate before sorting rather than assigning per group.
|
|
38
|
+
key = normalize_chrom(str(chrom))
|
|
39
|
+
by_chrom.setdefault(key, []).extend(
|
|
40
|
+
zip(group["Start"].tolist(), group["End"].tolist())
|
|
41
|
+
)
|
|
42
|
+
self._index = {
|
|
43
|
+
chrom: self._pack(pairs) for chrom, pairs in by_chrom.items()
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
@staticmethod
|
|
47
|
+
def _pack(pairs: list[tuple[int, int]]) -> tuple[list[int], list[int], list[int]]:
|
|
48
|
+
"""Sort (start, end) pairs by start and precompute the running max of ends."""
|
|
49
|
+
pairs = sorted(pairs)
|
|
50
|
+
starts = [s for s, _ in pairs]
|
|
51
|
+
ends = [e for _, e in pairs]
|
|
52
|
+
return starts, ends, list(itertools.accumulate(ends, max))
|
|
53
|
+
|
|
54
|
+
@classmethod
|
|
55
|
+
def wgs(cls) -> "CaptureIndex":
|
|
56
|
+
return cls(always_covered=True)
|
|
57
|
+
|
|
58
|
+
@classmethod
|
|
59
|
+
def from_bed(cls, bed_path: str) -> "CaptureIndex":
|
|
60
|
+
with warnings.catch_warnings():
|
|
61
|
+
warnings.filterwarnings("ignore", category=pd.errors.DtypeWarning)
|
|
62
|
+
try:
|
|
63
|
+
ranges = pr.read_bed(bed_path)
|
|
64
|
+
except (IndexError, AssertionError, pd.errors.EmptyDataError):
|
|
65
|
+
# pyranges raises rather than returning an empty frame when a BED
|
|
66
|
+
# holds no intervals: IndexError for a zero-byte file, AssertionError
|
|
67
|
+
# for a comment-only one. An empty index is the honest representation
|
|
68
|
+
# — the caller reports it (see describe_capture_problem) instead of
|
|
69
|
+
# the build dying on an opaque traceback.
|
|
70
|
+
return cls()
|
|
71
|
+
return cls(pyranges_obj=ranges)
|
|
72
|
+
|
|
73
|
+
def covers(self, chrom: str, pos: int) -> bool:
|
|
74
|
+
"""Return True if 1-based pos is within any region for this chrom.
|
|
75
|
+
|
|
76
|
+
BED is 0-based half-open: [Start, End) → 1-based: Start < pos <= End
|
|
77
|
+
"""
|
|
78
|
+
if self._always_covered:
|
|
79
|
+
return True
|
|
80
|
+
entry = self._index.get(chrom)
|
|
81
|
+
if entry is None:
|
|
82
|
+
return False
|
|
83
|
+
starts, _ends, max_ends = entry
|
|
84
|
+
# Find rightmost interval whose Start < pos (0-based start, so Start < pos_1based)
|
|
85
|
+
# bisect_left gives insertion point for pos in starts; all starts[:idx] < pos
|
|
86
|
+
idx = bisect.bisect_left(starts, pos) - 1
|
|
87
|
+
# Intervals may overlap, so any of starts[:idx+1] could reach pos. max_ends[idx]
|
|
88
|
+
# is the largest End among them, so it alone decides — no backwards scan needed.
|
|
89
|
+
return idx >= 0 and max_ends[idx] >= pos
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def is_always_covered(self) -> bool:
|
|
93
|
+
"""True for the WGS sentinel: every position counts as covered."""
|
|
94
|
+
return self._always_covered
|
|
95
|
+
|
|
96
|
+
def indexed_chroms(self) -> set[str]:
|
|
97
|
+
"""Normalized chrom names present in the index, unknown contigs included."""
|
|
98
|
+
return set(self._index)
|
|
99
|
+
|
|
100
|
+
def known_chroms(self) -> set[str]:
|
|
101
|
+
"""Indexed chrom names that are real chromosomes (subset of ALL_CHROMS)."""
|
|
102
|
+
return self.indexed_chroms() & set(ALL_CHROMS)
|
|
103
|
+
|
|
104
|
+
def is_empty(self) -> bool:
|
|
105
|
+
"""True when the index holds no intervals at all.
|
|
106
|
+
|
|
107
|
+
The WGS sentinel is not empty — it covers everything without an index.
|
|
108
|
+
"""
|
|
109
|
+
return not self._always_covered and not self._index
|
|
110
|
+
|
|
111
|
+
def __getstate__(self) -> dict:
|
|
112
|
+
"""Pickle without _pr — nothing outside this class reads the PyRanges object.
|
|
113
|
+
|
|
114
|
+
Keeping it put a pandas DataFrame in every WES pickle and made loading depend
|
|
115
|
+
on the pyranges/pandas versions that wrote it. Worse, __init__ normalizes the
|
|
116
|
+
chrom keys of _index but not of _pr, so a pickled _pr is a stale copy of the
|
|
117
|
+
raw BED names — exactly what a future reader would reintroduce this bug from.
|
|
118
|
+
_index alone answers covers() and is enough to rebuild after a normalization
|
|
119
|
+
change.
|
|
120
|
+
"""
|
|
121
|
+
state = self.__dict__.copy()
|
|
122
|
+
state.pop("_pr", None)
|
|
123
|
+
return state
|
|
124
|
+
|
|
125
|
+
def __setstate__(self, state: dict) -> None:
|
|
126
|
+
"""Migrate pickles written by older versions.
|
|
127
|
+
|
|
128
|
+
Three generations are repaired here, so existing databases self-heal on
|
|
129
|
+
load with no rebuild:
|
|
130
|
+
1. no _index at all (pre binary-search refactor) — rebuilt from _pr;
|
|
131
|
+
2. un-normalized chrom keys ('1' instead of 'chr1'), which made covers()
|
|
132
|
+
miss every WES technology and silently drop them from AN;
|
|
133
|
+
3. 2-tuple entries lacking the running max of ends.
|
|
134
|
+
"""
|
|
135
|
+
self.__dict__.update(state)
|
|
136
|
+
# Pickles written since __getstate__ carry no _pr; keep the attribute present
|
|
137
|
+
# so it stays safe to read on any restored object.
|
|
138
|
+
self.__dict__.setdefault("_pr", None)
|
|
139
|
+
index = getattr(self, "_index", None)
|
|
140
|
+
|
|
141
|
+
if index is None:
|
|
142
|
+
self._index = {}
|
|
143
|
+
if getattr(self, "_pr", None) is not None:
|
|
144
|
+
CaptureIndex.__init__(
|
|
145
|
+
self,
|
|
146
|
+
always_covered=self._always_covered,
|
|
147
|
+
pyranges_obj=self._pr,
|
|
148
|
+
)
|
|
149
|
+
return
|
|
150
|
+
|
|
151
|
+
needs_migration = any(
|
|
152
|
+
chrom != normalize_chrom(chrom) or len(entry) < 3
|
|
153
|
+
for chrom, entry in index.items()
|
|
154
|
+
)
|
|
155
|
+
if not needs_migration:
|
|
156
|
+
return
|
|
157
|
+
|
|
158
|
+
# Rebuild from the stored intervals rather than from _pr: _pr is absent in
|
|
159
|
+
# some pickles, and the intervals themselves are all we need.
|
|
160
|
+
by_chrom: dict[str, list[tuple[int, int]]] = {}
|
|
161
|
+
for chrom, entry in index.items():
|
|
162
|
+
starts, ends = entry[0], entry[1]
|
|
163
|
+
by_chrom.setdefault(normalize_chrom(chrom), []).extend(zip(starts, ends))
|
|
164
|
+
self._index = {chrom: self._pack(pairs) for chrom, pairs in by_chrom.items()}
|
|
165
|
+
|
|
166
|
+
def save(self, path: str) -> None:
|
|
167
|
+
with open(path, "wb") as f:
|
|
168
|
+
pickle.dump(self, f)
|
|
169
|
+
|
|
170
|
+
@classmethod
|
|
171
|
+
def load(cls, path: str) -> "CaptureIndex":
|
|
172
|
+
with open(path, "rb") as f:
|
|
173
|
+
return pickle.load(f)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def describe_capture_problem(index: CaptureIndex, tech_name: str) -> "str | None":
|
|
177
|
+
"""Return a ready-to-emit message if this index cannot cover anything, else None.
|
|
178
|
+
|
|
179
|
+
Build time and load time both need this verdict; sharing one implementation is
|
|
180
|
+
what stops them drifting on what counts as broken. The two failures are reported
|
|
181
|
+
separately because they need different fixes:
|
|
182
|
+
- empty: the BED held no intervals (missing, empty or malformed file);
|
|
183
|
+
- unknown contigs: intervals exist but none is on a real chromosome.
|
|
184
|
+
"""
|
|
185
|
+
if index.is_always_covered or index.known_chroms():
|
|
186
|
+
return None
|
|
187
|
+
consequence = (
|
|
188
|
+
"its samples will be counted as uncovered at every position, "
|
|
189
|
+
"lowering AN and inflating AF."
|
|
190
|
+
)
|
|
191
|
+
if index.is_empty():
|
|
192
|
+
return f"Capture regions for technology {tech_name!r} are empty — {consequence}"
|
|
193
|
+
found = sorted(index.indexed_chroms())
|
|
194
|
+
shown = ", ".join(found[:5]) + (", ..." if len(found) > 5 else "")
|
|
195
|
+
return (
|
|
196
|
+
f"Capture regions for technology {tech_name!r} match no known chromosome "
|
|
197
|
+
f"(found: {shown}) — {consequence}"
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def load_capture_indices(
|
|
202
|
+
technologies: list[Technology], capture_dir: str
|
|
203
|
+
) -> dict[int, "CaptureIndex"]:
|
|
204
|
+
result = {}
|
|
205
|
+
for tech in technologies:
|
|
206
|
+
path = f"{capture_dir}/tech_{tech.tech_id}.pickle"
|
|
207
|
+
try:
|
|
208
|
+
result[tech.tech_id] = CaptureIndex.load(path)
|
|
209
|
+
except FileNotFoundError as exc:
|
|
210
|
+
# Defaulting to WGS here would silently inflate AN, so this must stay fatal
|
|
211
|
+
# — only the message improves.
|
|
212
|
+
raise FileNotFoundError(
|
|
213
|
+
f"Missing capture index for technology {tech.tech_name!r} at {path} — "
|
|
214
|
+
"the database is incomplete; rebuild it or re-run 'afquery update-db'."
|
|
215
|
+
) from exc
|
|
216
|
+
return result
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# PAR1 and PAR2 for GRCh37 and GRCh38
|
|
2
|
+
PAR = {
|
|
3
|
+
"GRCh38": {
|
|
4
|
+
"chrX": [(10_001, 2_781_479), (155_701_383, 156_030_895)],
|
|
5
|
+
"chrY": [(10_001, 2_781_479), (56_887_903, 57_217_415)],
|
|
6
|
+
},
|
|
7
|
+
"GRCh37": {
|
|
8
|
+
"chrX": [(60_001, 2_699_520), (154_931_044, 155_260_560)],
|
|
9
|
+
"chrY": [(10_001, 2_649_520), (59_034_050, 59_363_566)],
|
|
10
|
+
},
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
AUTOSOMES = [f"chr{i}" for i in range(1, 23)]
|
|
14
|
+
SEX_CHROMS = ["chrX", "chrY"]
|
|
15
|
+
MITO_CHROM = "chrM"
|
|
16
|
+
# Mitochondrion aliases seen in the wild (Ensembl 'MT', UCSC 'chrM', hybrid 'chrMT').
|
|
17
|
+
_MITO_ALIASES = frozenset(["m", "mt", "chrm", "chrmt"])
|
|
18
|
+
# Chromosomes whose canonical spelling is uppercase after the 'chr' prefix.
|
|
19
|
+
_UPPERCASE_SUFFIXES = {"x": "X", "y": "Y"}
|
|
20
|
+
ALL_CHROMS = AUTOSOMES + SEX_CHROMS + [MITO_CHROM]
|
|
21
|
+
CHROM_ORDER: dict[str, int] = {c: i for i, c in enumerate(ALL_CHROMS)}
|
|
22
|
+
VALID_GENOME_BUILDS = frozenset(["GRCh37", "GRCh38"])
|
|
23
|
+
VALID_SEX = frozenset(["male", "female"])
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def normalize_chrom(chrom: str) -> str:
|
|
27
|
+
"""Normalize chromosome name to 'chr'-prefixed canonical form.
|
|
28
|
+
|
|
29
|
+
'1' -> 'chr1', 'chr1' -> 'chr1', 'CHR1' -> 'chr1'
|
|
30
|
+
'X' -> 'chrX', 'x' -> 'chrX', 'MT' -> 'chrM', 'chrMT' -> 'chrM'
|
|
31
|
+
|
|
32
|
+
Only the 'chr' prefix and the known chromosome suffixes are case-normalized;
|
|
33
|
+
anything else keeps its body byte-for-byte ('GL000209.1' -> 'chrGL000209.1',
|
|
34
|
+
'chr1_KI270706v1_random' unchanged), so unplaced and alt contigs stay outside
|
|
35
|
+
ALL_CHROMS and are still dropped by the build stage rather than guessed at.
|
|
36
|
+
|
|
37
|
+
Idempotent on every member of ALL_CHROMS: the output is what gets persisted as
|
|
38
|
+
the Parquet partition name, so a name that changed under normalization would
|
|
39
|
+
orphan the partitions of already-built databases.
|
|
40
|
+
"""
|
|
41
|
+
chrom = chrom.strip()
|
|
42
|
+
if chrom.lower() in _MITO_ALIASES:
|
|
43
|
+
return MITO_CHROM
|
|
44
|
+
body = chrom[3:] if chrom[:3].lower() == "chr" else chrom
|
|
45
|
+
if body.lower() in _UPPERCASE_SUFFIXES:
|
|
46
|
+
return "chr" + _UPPERCASE_SUFFIXES[body.lower()]
|
|
47
|
+
return "chr" + body
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_autosome(chrom: str) -> bool:
|
|
51
|
+
return normalize_chrom(chrom) in AUTOSOMES
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def is_sex_chrom(chrom: str) -> bool:
|
|
55
|
+
return normalize_chrom(chrom) in SEX_CHROMS
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def is_mito(chrom: str) -> bool:
|
|
59
|
+
return normalize_chrom(chrom) == MITO_CHROM
|
|
@@ -33,6 +33,24 @@ PARQUET_SCHEMA = pa.schema([
|
|
|
33
33
|
])
|
|
34
34
|
|
|
35
35
|
|
|
36
|
+
def _filter_known_chroms(chroms: list[str], source: str) -> list[str]:
|
|
37
|
+
"""Keep only real chromosomes, reporting the dropped contigs once.
|
|
38
|
+
|
|
39
|
+
Chrom discovery happens in three places and every one of them must drop
|
|
40
|
+
unplaced and alt contigs. Centralizing the filter is what keeps the report
|
|
41
|
+
from either vanishing — it used to live downstream of the filter, where it
|
|
42
|
+
could never fire — or firing once per row.
|
|
43
|
+
"""
|
|
44
|
+
known = [c for c in chroms if c in ALL_CHROMS]
|
|
45
|
+
unknown = sorted({c for c in chroms if c not in ALL_CHROMS})
|
|
46
|
+
if unknown:
|
|
47
|
+
logger.warning(
|
|
48
|
+
"[build] Skipping %d unknown contig(s) from %s: %s",
|
|
49
|
+
len(unknown), source, ", ".join(unknown),
|
|
50
|
+
)
|
|
51
|
+
return known
|
|
52
|
+
|
|
53
|
+
|
|
36
54
|
def _get_chroms_from_file(parquet_path: str) -> list[str]:
|
|
37
55
|
"""SELECT DISTINCT chrom from a single consolidated Parquet file."""
|
|
38
56
|
path = parquet_path.replace("'", "''")
|
|
@@ -46,7 +64,7 @@ def _get_chroms_from_file(parquet_path: str) -> list[str]:
|
|
|
46
64
|
return []
|
|
47
65
|
finally:
|
|
48
66
|
con.close()
|
|
49
|
-
return [r[0] for r in rows
|
|
67
|
+
return _filter_known_chroms([r[0] for r in rows], parquet_path)
|
|
50
68
|
|
|
51
69
|
|
|
52
70
|
def consolidate_temp_files(
|
|
@@ -81,13 +99,12 @@ def _get_chroms_from_consolidated(path: str) -> list[str]:
|
|
|
81
99
|
"""Discover chromosomes from consolidated path (Hive-partitioned directory or single file)."""
|
|
82
100
|
if os.path.isdir(path):
|
|
83
101
|
# Hive-partitioned: scan chrom=X/ subdirs — fast, no SQL needed
|
|
84
|
-
chroms = [
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
return chroms
|
|
102
|
+
chroms = [
|
|
103
|
+
entry.name[6:] # strip "chrom="
|
|
104
|
+
for entry in os.scandir(path)
|
|
105
|
+
if entry.is_dir() and entry.name.startswith("chrom=")
|
|
106
|
+
]
|
|
107
|
+
return _filter_known_chroms(chroms, path)
|
|
91
108
|
else:
|
|
92
109
|
# Single file (legacy) — use SQL
|
|
93
110
|
return _get_chroms_from_file(path)
|
|
@@ -111,7 +128,7 @@ def get_chroms_in_temp_files(tmp_dir: str) -> list[str]:
|
|
|
111
128
|
finally:
|
|
112
129
|
con.close()
|
|
113
130
|
|
|
114
|
-
return [r[0] for r in rows
|
|
131
|
+
return _filter_known_chroms([r[0] for r in rows], tmp_dir)
|
|
115
132
|
|
|
116
133
|
|
|
117
134
|
def _make_table(
|
|
@@ -524,12 +541,8 @@ def build_all_parquets(
|
|
|
524
541
|
chroms = _get_chroms_from_consolidated(consolidated_path)
|
|
525
542
|
else:
|
|
526
543
|
chroms = get_chroms_in_temp_files(tmp_dir)
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
if chrom in ALL_CHROMS:
|
|
530
|
-
valid_chroms.append(chrom)
|
|
531
|
-
else:
|
|
532
|
-
logger.warning("Skipping unknown contig: %s", chrom)
|
|
544
|
+
# Both discovery helpers already filter to ALL_CHROMS and report what they drop.
|
|
545
|
+
valid_chroms = chroms
|
|
533
546
|
|
|
534
547
|
if not valid_chroms:
|
|
535
548
|
return {}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
|
|
3
|
+
from ..capture import CaptureIndex, describe_capture_problem
|
|
4
|
+
from ..models import AfqueryWarning, Technology
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def build_capture_indices(
|
|
8
|
+
technologies: list[Technology],
|
|
9
|
+
capture_dir: str,
|
|
10
|
+
) -> None:
|
|
11
|
+
for tech in technologies:
|
|
12
|
+
if tech.bed_path is None:
|
|
13
|
+
idx = CaptureIndex.wgs()
|
|
14
|
+
else:
|
|
15
|
+
idx = CaptureIndex.from_bed(tech.bed_path)
|
|
16
|
+
problem = describe_capture_problem(idx, tech.tech_name)
|
|
17
|
+
if problem is not None:
|
|
18
|
+
# warnings, not logger: this is also reached from the Python API via
|
|
19
|
+
# update.py, where a logger message is invisible unless the caller
|
|
20
|
+
# configured a handler. "Your allele frequencies will be wrong" has to
|
|
21
|
+
# surface on both paths.
|
|
22
|
+
warnings.warn(
|
|
23
|
+
f"{problem} (BED: {tech.bed_path})", AfqueryWarning, stacklevel=2,
|
|
24
|
+
)
|
|
25
|
+
idx.save(f"{capture_dir}/tech_{tech.tech_id}.pickle")
|
|
@@ -7,7 +7,7 @@ import duckdb
|
|
|
7
7
|
from pyroaring import BitMap
|
|
8
8
|
|
|
9
9
|
from .bitmaps import deserialize
|
|
10
|
-
from .capture import CaptureIndex, load_capture_indices
|
|
10
|
+
from .capture import CaptureIndex, describe_capture_problem, load_capture_indices
|
|
11
11
|
from .constants import normalize_chrom, ALL_CHROMS, CHROM_ORDER
|
|
12
12
|
from .models import AfqueryWarning, QueryParams, QueryResult, SampleCarrier, VariantKey, Sample, Technology, SampleFilter
|
|
13
13
|
from .ploidy import compute_AN, split_ploidy
|
|
@@ -70,6 +70,12 @@ class QueryEngine:
|
|
|
70
70
|
self._male_bm = sex_bms.get("male", BitMap())
|
|
71
71
|
self._female_bm = sex_bms.get("female", BitMap())
|
|
72
72
|
self._capture = load_capture_indices(techs, str(self._db / "capture"))
|
|
73
|
+
for tech_id, capture_idx in self._capture.items():
|
|
74
|
+
problem = describe_capture_problem(
|
|
75
|
+
capture_idx, self._tech_map[tech_id].tech_name
|
|
76
|
+
)
|
|
77
|
+
if problem is not None:
|
|
78
|
+
warnings.warn(problem, AfqueryWarning, stacklevel=2)
|
|
73
79
|
self._tech_bitmaps = self._bitmaps.get("tech", {})
|
|
74
80
|
self._all_samples_bm = BitMap(s.sample_id for s in self._samples)
|
|
75
81
|
self._tech_name_to_id: dict[str, str] = {
|
|
@@ -95,10 +101,19 @@ class QueryEngine:
|
|
|
95
101
|
# For each tech, store its bitmap; also precompute the union of all WGS tech bitmaps
|
|
96
102
|
self._covered_all_bm: BitMap = BitMap()
|
|
97
103
|
for tech_id, capture_idx in self._capture.items():
|
|
98
|
-
if capture_idx.
|
|
104
|
+
if capture_idx.is_always_covered:
|
|
99
105
|
bm = self._tech_bitmaps.get(str(tech_id), BitMap())
|
|
100
106
|
self._covered_all_bm |= bm
|
|
101
107
|
|
|
108
|
+
# Targeted (non-WGS) technologies only. The per-position loops below run once
|
|
109
|
+
# per queried position, so the WGS ones are skipped here rather than re-tested
|
|
110
|
+
# on every iteration — they are already folded into _covered_all_bm.
|
|
111
|
+
self._targeted_capture: dict[int, CaptureIndex] = {
|
|
112
|
+
tech_id: capture_idx
|
|
113
|
+
for tech_id, capture_idx in self._capture.items()
|
|
114
|
+
if not capture_idx.is_always_covered
|
|
115
|
+
}
|
|
116
|
+
|
|
102
117
|
# Cache parquet glob patterns per chrom (avoids filesystem scan on every query)
|
|
103
118
|
self._glob_cache: dict[str, str | None] = {}
|
|
104
119
|
|
|
@@ -213,9 +228,7 @@ class QueryEngine:
|
|
|
213
228
|
|
|
214
229
|
# Phase 1: count-based gate
|
|
215
230
|
if min_pass > 0 or min_observed > 0:
|
|
216
|
-
for tech_id
|
|
217
|
-
if capture_idx._always_covered:
|
|
218
|
-
continue
|
|
231
|
+
for tech_id in self._targeted_capture:
|
|
219
232
|
tech_bm = self._tech_bitmaps.get(str(tech_id), BitMap())
|
|
220
233
|
tech_eligible = eligible & tech_bm
|
|
221
234
|
if len(tech_eligible) == 0:
|
|
@@ -232,9 +245,7 @@ class QueryEngine:
|
|
|
232
245
|
# Phase 2: quality_pass gate (--min-quality-evidence K)
|
|
233
246
|
if quality_pass_bm is not None and min_quality_evidence > 0:
|
|
234
247
|
already_filtered = filtered_bm if filtered_bm is not None else BitMap()
|
|
235
|
-
for tech_id
|
|
236
|
-
if capture_idx._always_covered:
|
|
237
|
-
continue
|
|
248
|
+
for tech_id in self._targeted_capture:
|
|
238
249
|
tech_bm = self._tech_bitmaps.get(str(tech_id), BitMap())
|
|
239
250
|
tech_eligible = eligible & tech_bm
|
|
240
251
|
if len(tech_eligible) == 0:
|
|
@@ -287,9 +298,7 @@ class QueryEngine:
|
|
|
287
298
|
) -> tuple[BitMap, int]:
|
|
288
299
|
"""Return (eligible_bitmap, AN) for a single position."""
|
|
289
300
|
covered = BitMap(self._covered_all_bm) # start with WGS samples (always covered)
|
|
290
|
-
for tech_id, capture_idx in self.
|
|
291
|
-
if capture_idx._always_covered:
|
|
292
|
-
continue # already included in _covered_all_bm
|
|
301
|
+
for tech_id, capture_idx in self._targeted_capture.items():
|
|
293
302
|
if capture_idx.covers(chrom, pos):
|
|
294
303
|
covered |= self._tech_bitmaps.get(str(tech_id), BitMap())
|
|
295
304
|
eligible = sample_bitmap & covered
|