afquery 0.3.3__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. {afquery-0.3.3 → afquery-0.4.0}/PKG-INFO +2 -7
  2. {afquery-0.3.3 → afquery-0.4.0}/README.md +1 -6
  3. {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/debugging-results.md +2 -0
  4. {afquery-0.3.3 → afquery-0.4.0}/docs/faq.md +5 -1
  5. {afquery-0.3.3 → afquery-0.4.0}/docs/guides/variant-info.md +12 -0
  6. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/_version.py +2 -2
  7. afquery-0.4.0/src/afquery/capture.py +216 -0
  8. afquery-0.4.0/src/afquery/constants.py +59 -0
  9. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/build.py +28 -15
  10. afquery-0.4.0/src/afquery/preprocess/regions.py +25 -0
  11. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/query.py +20 -11
  12. afquery-0.4.0/tests/data/beds/wes_kit_nochr.bed +2 -0
  13. afquery-0.4.0/tests/test_capture.py +302 -0
  14. afquery-0.4.0/tests/test_constants.py +98 -0
  15. {afquery-0.3.3 → afquery-0.4.0}/tests/test_preprocess.py +27 -0
  16. {afquery-0.3.3 → afquery-0.4.0}/tests/test_warnings.py +69 -0
  17. afquery-0.3.3/src/afquery/capture.py +0 -94
  18. afquery-0.3.3/src/afquery/constants.py +0 -44
  19. afquery-0.3.3/src/afquery/preprocess/regions.py +0 -14
  20. afquery-0.3.3/tests/test_capture.py +0 -96
  21. afquery-0.3.3/tests/test_constants.py +0 -50
  22. {afquery-0.3.3 → afquery-0.4.0}/.dockerignore +0 -0
  23. {afquery-0.3.3 → afquery-0.4.0}/.github/workflows/ci.yml +0 -0
  24. {afquery-0.3.3 → afquery-0.4.0}/.github/workflows/docs.yml +0 -0
  25. {afquery-0.3.3 → afquery-0.4.0}/.github/workflows/release.yml +0 -0
  26. {afquery-0.3.3 → afquery-0.4.0}/.gitignore +0 -0
  27. {afquery-0.3.3 → afquery-0.4.0}/CONTRIBUTING.md +0 -0
  28. {afquery-0.3.3 → afquery-0.4.0}/Dockerfile +0 -0
  29. {afquery-0.3.3 → afquery-0.4.0}/LICENSE +0 -0
  30. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/.gitignore +0 -0
  31. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/INSTALL.md +0 -0
  32. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/README.md +0 -0
  33. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/Snakefile +0 -0
  34. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/.gitignore +0 -0
  35. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/01a_assign_samples.py +0 -0
  36. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/01b_write_manifests.py +0 -0
  37. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/02_build_databases.py +0 -0
  38. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/03_compute_metrics.py +0 -0
  39. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/04_classify_acmg.py +0 -0
  40. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/05_plot_figures.py +0 -0
  41. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/README.md +0 -0
  42. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/Snakefile +0 -0
  43. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/__init__.py +0 -0
  44. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/beds/SureSelect_v5.bed +0 -0
  45. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/beds/SureSelect_v6.bed +0 -0
  46. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/beds/SureSelect_v7.bed +0 -0
  47. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/capture_kit/config.py +0 -0
  48. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/config.yaml +0 -0
  49. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/envs/benchmark.yaml +0 -0
  50. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/.gitignore +0 -0
  51. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/01_prepare_data.py +0 -0
  52. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/02_query_scaling.py +0 -0
  53. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/03_build.py +0 -0
  54. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/04_annotate.py +0 -0
  55. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/05_vs_bcftools.py +0 -0
  56. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/06_plot.py +0 -0
  57. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/README.md +0 -0
  58. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/Snakefile +0 -0
  59. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_annotate.py +0 -0
  60. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_bcftools.py +0 -0
  61. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_build_perf.py +0 -0
  62. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_prepare.py +0 -0
  63. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/collect_query_scaling.py +0 -0
  64. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/config.py +0 -0
  65. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/performance/config_smoke.py +0 -0
  66. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/shared/__init__.py +0 -0
  67. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/shared/config.py +0 -0
  68. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/shared/rules/download_1kg.smk +0 -0
  69. {afquery-0.3.3 → afquery-0.4.0}/benchmarks/shared/utils.py +0 -0
  70. {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/benchmarking.md +0 -0
  71. {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/coverage-evidence.md +0 -0
  72. {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/filter-pass-tracking.md +0 -0
  73. {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/multi-cohort-strategies.md +0 -0
  74. {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/performance.md +0 -0
  75. {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/pipeline-integration.md +0 -0
  76. {afquery-0.3.3 → afquery-0.4.0}/docs/advanced/ploidy-and-sex-chroms.md +0 -0
  77. {afquery-0.3.3 → afquery-0.4.0}/docs/assets/img/gap2_mixed_technologies.png +0 -0
  78. {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/concepts.md +0 -0
  79. {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/installation.md +0 -0
  80. {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/motivation.md +0 -0
  81. {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/preprocessing.md +0 -0
  82. {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/quickstart.md +0 -0
  83. {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/tutorial.md +0 -0
  84. {afquery-0.3.3 → afquery-0.4.0}/docs/getting-started/understanding-output.md +0 -0
  85. {afquery-0.3.3 → afquery-0.4.0}/docs/guides/annotate-vcf.md +0 -0
  86. {afquery-0.3.3 → afquery-0.4.0}/docs/guides/create-database.md +0 -0
  87. {afquery-0.3.3 → afquery-0.4.0}/docs/guides/dump-export.md +0 -0
  88. {afquery-0.3.3 → afquery-0.4.0}/docs/guides/manifest-format.md +0 -0
  89. {afquery-0.3.3 → afquery-0.4.0}/docs/guides/query.md +0 -0
  90. {afquery-0.3.3 → afquery-0.4.0}/docs/guides/sample-filtering.md +0 -0
  91. {afquery-0.3.3 → afquery-0.4.0}/docs/guides/update-database.md +0 -0
  92. {afquery-0.3.3 → afquery-0.4.0}/docs/index.md +0 -0
  93. {afquery-0.3.3 → afquery-0.4.0}/docs/reference/cli.md +0 -0
  94. {afquery-0.3.3 → afquery-0.4.0}/docs/reference/data-model.md +0 -0
  95. {afquery-0.3.3 → afquery-0.4.0}/docs/reference/glossary.md +0 -0
  96. {afquery-0.3.3 → afquery-0.4.0}/docs/reference/python-api.md +0 -0
  97. {afquery-0.3.3 → afquery-0.4.0}/docs/scripts/gen_gap2_figure.py +0 -0
  98. {afquery-0.3.3 → afquery-0.4.0}/docs/stylesheets/extra.css +0 -0
  99. {afquery-0.3.3 → afquery-0.4.0}/docs/troubleshooting.md +0 -0
  100. {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/acmg-use-cases.md +0 -0
  101. {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/clinical-prioritization.md +0 -0
  102. {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/cohort-stratification.md +0 -0
  103. {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/population-specific-af.md +0 -0
  104. {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/pseudo-controls.md +0 -0
  105. {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/sex-specific-af.md +0 -0
  106. {afquery-0.3.3 → afquery-0.4.0}/docs/use-cases/technology-integration.md +0 -0
  107. {afquery-0.3.3 → afquery-0.4.0}/examples/demo/README.md +0 -0
  108. {afquery-0.3.3 → afquery-0.4.0}/examples/demo/create_demo_data.py +0 -0
  109. {afquery-0.3.3 → afquery-0.4.0}/mkdocs.yml +0 -0
  110. {afquery-0.3.3 → afquery-0.4.0}/pyproject.toml +0 -0
  111. {afquery-0.3.3 → afquery-0.4.0}/recipes/afquery/meta.yaml +0 -0
  112. {afquery-0.3.3 → afquery-0.4.0}/resources/normalize_vcf.sh +0 -0
  113. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/__init__.py +0 -0
  114. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/annotate.py +0 -0
  115. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/benchmark.py +0 -0
  116. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/bitmaps.py +0 -0
  117. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/cli.py +0 -0
  118. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/cli.pyr +0 -0
  119. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/database.py +0 -0
  120. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/dump.py +0 -0
  121. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/models.py +0 -0
  122. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/ploidy.py +0 -0
  123. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/__init__.py +0 -0
  124. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/compact.py +0 -0
  125. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/ingest.py +0 -0
  126. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/manifest.py +0 -0
  127. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/synth.py +0 -0
  128. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/preprocess/update.py +0 -0
  129. {afquery-0.3.3 → afquery-0.4.0}/src/afquery/variant_info.py +0 -0
  130. {afquery-0.3.3 → afquery-0.4.0}/tests/conftest.py +0 -0
  131. {afquery-0.3.3 → afquery-0.4.0}/tests/data/annotate_input.vcf +0 -0
  132. {afquery-0.3.3 → afquery-0.4.0}/tests/data/annotate_multi_bucket.vcf +0 -0
  133. {afquery-0.3.3 → afquery-0.4.0}/tests/data/annotate_multi_chrom.vcf +0 -0
  134. {afquery-0.3.3 → afquery-0.4.0}/tests/data/beds/wes_kit_a.bed +0 -0
  135. {afquery-0.3.3 → afquery-0.4.0}/tests/data/beds/wes_kit_b.bed +0 -0
  136. {afquery-0.3.3 → afquery-0.4.0}/tests/data/expected_results.json +0 -0
  137. {afquery-0.3.3 → afquery-0.4.0}/tests/data/manifest.tsv +0 -0
  138. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S00.vcf +0 -0
  139. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S01.vcf +0 -0
  140. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S02.vcf +0 -0
  141. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S03.vcf +0 -0
  142. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S04.vcf +0 -0
  143. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S05.vcf +0 -0
  144. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S06.vcf +0 -0
  145. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S07.vcf +0 -0
  146. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S08.vcf +0 -0
  147. {afquery-0.3.3 → afquery-0.4.0}/tests/data/vcfs/S09.vcf +0 -0
  148. {afquery-0.3.3 → afquery-0.4.0}/tests/test_annotate.py +0 -0
  149. {afquery-0.3.3 → afquery-0.4.0}/tests/test_batch.py +0 -0
  150. {afquery-0.3.3 → afquery-0.4.0}/tests/test_benchmark.py +0 -0
  151. {afquery-0.3.3 → afquery-0.4.0}/tests/test_bitmaps.py +0 -0
  152. {afquery-0.3.3 → afquery-0.4.0}/tests/test_cli.py +0 -0
  153. {afquery-0.3.3 → afquery-0.4.0}/tests/test_cli_docs_consistency.py +0 -0
  154. {afquery-0.3.3 → afquery-0.4.0}/tests/test_compact.py +0 -0
  155. {afquery-0.3.3 → afquery-0.4.0}/tests/test_dump.py +0 -0
  156. {afquery-0.3.3 → afquery-0.4.0}/tests/test_haploid_stats.py +0 -0
  157. {afquery-0.3.3 → afquery-0.4.0}/tests/test_info.py +0 -0
  158. {afquery-0.3.3 → afquery-0.4.0}/tests/test_no_coverage.py +0 -0
  159. {afquery-0.3.3 → afquery-0.4.0}/tests/test_pass_filter.py +0 -0
  160. {afquery-0.3.3 → afquery-0.4.0}/tests/test_ploidy.py +0 -0
  161. {afquery-0.3.3 → afquery-0.4.0}/tests/test_query.py +0 -0
  162. {afquery-0.3.3 → afquery-0.4.0}/tests/test_sample_filter.py +0 -0
  163. {afquery-0.3.3 → afquery-0.4.0}/tests/test_synth.py +0 -0
  164. {afquery-0.3.3 → afquery-0.4.0}/tests/test_synthetic_stats.py +0 -0
  165. {afquery-0.3.3 → afquery-0.4.0}/tests/test_update.py +0 -0
  166. {afquery-0.3.3 → afquery-0.4.0}/tests/test_update_metadata.py +0 -0
  167. {afquery-0.3.3 → afquery-0.4.0}/tests/test_variant_info.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: afquery
3
- Version: 0.3.3
3
+ Version: 0.4.0
4
4
  Summary: Genomic allele frequency query engine with bitmap-encoded genotypes
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -40,7 +40,7 @@ Description-Content-Type: text/markdown
40
40
  [![Python](https://img.shields.io/pypi/pyversions/afquery.svg)](https://pypi.org/project/afquery/)
41
41
  [![License: MIT](https://img.shields.io/github/license/babelomics/afquery)](https://github.com/babelomics/afquery/blob/master/LICENSE)
42
42
 
43
- **Fast, capture-aware allele frequency queries on local genomic cohorts — without re-scanning VCFs.**
43
+ **Fast, capture-aware allele frequency queries on local genomic cohorts**
44
44
 
45
45
  AFQuery is a bitmap-indexed engine that recomputes AC/AN/AF for arbitrary subcohorts (by phenotype, sex, sequencing technology, or any combination) in tens of milliseconds, independently of cohort size. It accounts for capture-kit heterogeneity, ploidy on sex chromosomes, and FILTER/coverage evidence, and runs locally as a file-based system (Parquet + SQLite) with no server or cluster required.
46
46
 
@@ -231,7 +231,6 @@ AFQuery indexes each variant as three Roaring Bitmaps — heterozygous PASS carr
231
231
  | Workload | AFQuery | bcftools | Speedup |
232
232
  |---|---|---|---|
233
233
  | Full-chromosome AC/AN/AF export | ~7.0 s | ~3.8 min | **~33×** |
234
- | AF concordance over 1,106,181 common variants | — | — | **R² > 0.99999** |
235
234
 
236
235
  Point-query latency on AFQuery is ~14 ms and constant from 1K to 50K samples (median over 50 replicates, warm cache).
237
236
 
@@ -250,7 +249,3 @@ If you use AFQuery in your work, please cite:
250
249
 
251
250
  > AFQuery: fast, capture-aware allele frequency queries on local genomic cohorts.
252
251
  > *(manuscript in preparation)*
253
-
254
- ## License
255
-
256
- [MIT](LICENSE)
@@ -10,7 +10,7 @@
10
10
  [![Python](https://img.shields.io/pypi/pyversions/afquery.svg)](https://pypi.org/project/afquery/)
11
11
  [![License: MIT](https://img.shields.io/github/license/babelomics/afquery)](https://github.com/babelomics/afquery/blob/master/LICENSE)
12
12
 
13
- **Fast, capture-aware allele frequency queries on local genomic cohorts — without re-scanning VCFs.**
13
+ **Fast, capture-aware allele frequency queries on local genomic cohorts**
14
14
 
15
15
  AFQuery is a bitmap-indexed engine that recomputes AC/AN/AF for arbitrary subcohorts (by phenotype, sex, sequencing technology, or any combination) in tens of milliseconds, independently of cohort size. It accounts for capture-kit heterogeneity, ploidy on sex chromosomes, and FILTER/coverage evidence, and runs locally as a file-based system (Parquet + SQLite) with no server or cluster required.
16
16
 
@@ -201,7 +201,6 @@ AFQuery indexes each variant as three Roaring Bitmaps — heterozygous PASS carr
201
201
  | Workload | AFQuery | bcftools | Speedup |
202
202
  |---|---|---|---|
203
203
  | Full-chromosome AC/AN/AF export | ~7.0 s | ~3.8 min | **~33×** |
204
- | AF concordance over 1,106,181 common variants | — | — | **R² > 0.99999** |
205
204
 
206
205
  Point-query latency on AFQuery is ~14 ms and constant from 1K to 50K samples (median over 50 replicates, warm cache).
207
206
 
@@ -220,7 +219,3 @@ If you use AFQuery in your work, please cite:
220
219
 
221
220
  > AFQuery: fast, capture-aware allele frequency queries on local genomic cohorts.
222
221
  > *(manuscript in preparation)*
223
-
224
- ## License
225
-
226
- [MIT](LICENSE)
@@ -16,6 +16,8 @@ AN=0 means no eligible samples at the queried position. Work through these check
16
16
  | Position exists in database | `afquery query --db ./db/ --locus chr1:12345678` | If no result at all, the variant was not observed in any sample during ingestion. |
17
17
  | BED coverage (WES) | `afquery info --db ./db/` | If all eligible samples are WES and the position is outside capture regions, AN=0 is correct. |
18
18
  | Sample filter too restrictive | Remove `--phenotype` and `--sex` filters | Query with no filters first. If AN>0 without filters, the filter is excluding all samples. |
19
+ | WES samples missing from AN | Compare `variant-info` carriers against `query` counts | If `variant-info` shows many WES carriers but AN reflects only the WGS samples, that technology's capture index is not matching the position. `afquery` warns on open when a capture BED is empty or matches no known chromosome, naming the offending contigs. |
20
+ | AN=0 only on chrM | `afquery query --db ./db/ --locus chrM:3243` | Mitochondrial coverage is easy to lose on its own: a BED naming it `chrMT` still matches on the autosomes, so nothing else looks wrong. Names are normalized (`MT`, `M`, `chrMT` → `chrM`) since v0.4.0 — databases built before that may need the capture BED re-checked. |
19
21
  | Technology filter | Remove `--tech` filter | Check if any samples match the requested technology. |
20
22
 
21
23
  ### 2. Unexpected AF Value
@@ -100,7 +100,11 @@ Yes. Both GRCh37 and GRCh38 are supported. Specify at database creation:
100
100
  afquery create-db --genome-build GRCh37 ...
101
101
  ```
102
102
 
103
- The genome build affects PAR1/PAR2 coordinates on chrX (see [Ploidy & Special Chromosomes](advanced/ploidy-and-sex-chroms.md)). Chromosome names in your VCFs should match the chosen build (`chr1`/`1` both work — `normalize_chrom()` handles the `chr` prefix).
103
+ The genome build affects PAR1/PAR2 coordinates on chrX (see [Ploidy & Special Chromosomes](advanced/ploidy-and-sex-chroms.md)). Chromosome names in your VCFs and capture BEDs are normalized to a canonical `chr`-prefixed
104
+ form, so the common naming conventions interoperate: `1` and `chr1` both become `chr1`, `X`
105
+ and `x` become `chrX`, and `MT`, `M` and `chrMT` all become `chrM`. Unplaced and alt contigs
106
+ (`GL000209.1`, `chr1_KI270706v1_random`) are left alone and skipped at build time — the build
107
+ log reports which ones it dropped.
104
108
 
105
109
  ---
106
110
 
@@ -15,6 +15,18 @@ afquery variant-info --db ./db/ --locus chr1:925952
15
15
  !!! tip
16
16
  `variant-info` is the natural next step after `query` — once you find a variant of interest, use it to see which specific samples carry it.
17
17
 
18
+ !!! note "Carrier counts may exceed the counts reported by `query`"
19
+ `variant-info` lists every carrier present in the source VCFs. `query` additionally
20
+ restricts to samples whose capture regions cover the position, so a WES sample with a
21
+ call just outside its own BED appears here but is not counted in `AN`, `AC` or the
22
+ genotype tallies. The two commands answer different questions — "who carries it?"
23
+ versus "what is the frequency among samples that could have been called?" — so a
24
+ modest difference is expected.
25
+
26
+ A *large* gap is worth investigating: if `variant-info` shows many carriers from a
27
+ technology that contributes nothing to `AN`, check that technology's capture index
28
+ (see [Debugging Results](../advanced/debugging-results.md)).
29
+
18
30
  By default all samples are queried and results are printed as an aligned text table:
19
31
 
20
32
  ```
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.3.3'
22
- __version_tuple__ = version_tuple = (0, 3, 3)
21
+ __version__ = version = '0.4.0'
22
+ __version_tuple__ = version_tuple = (0, 4, 0)
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -0,0 +1,216 @@
1
+ import bisect
2
+ import itertools
3
+ import pickle
4
+ import warnings
5
+ import pyranges as pr
6
+ import pandas as pd
7
+ from .constants import ALL_CHROMS, normalize_chrom
8
+ from .models import Technology
9
+
10
+
11
+ class CaptureIndex:
12
+ """Interval index over a capture BED, queried by 1-based position.
13
+
14
+ Chromosome keys in ``_index`` are always normalized (see
15
+ :func:`~afquery.constants.normalize_chrom`), because queries reach
16
+ :meth:`covers` already normalized. BED files in the wild use either
17
+ convention ('1' for GRCh37/hs37d5, 'chr1' for hg38), so normalizing on
18
+ construction is what keeps the two sides in agreement.
19
+ """
20
+
21
+ _always_covered: bool
22
+ _pr: "pr.PyRanges | None"
23
+ # Per-chrom sorted interval index, keyed by normalized chrom:
24
+ # {chrom: (sorted_starts, corresponding_ends, running_max_of_ends)}
25
+ # The running max lets covers() answer in O(log n) — see there.
26
+ _index: "dict[str, tuple[list[int], list[int], list[int]]]"
27
+
28
+ def __init__(self, *, always_covered: bool = False, pyranges_obj=None):
29
+ self._always_covered = always_covered
30
+ self._pr = pyranges_obj
31
+ self._index = {}
32
+ if pyranges_obj is not None:
33
+ df = pyranges_obj.df
34
+ by_chrom: dict[str, list[tuple[int, int]]] = {}
35
+ for chrom, group in df.groupby("Chromosome", observed=True):
36
+ # Normalizing can merge two groups ('1' and 'chr1') into one key,
37
+ # so accumulate before sorting rather than assigning per group.
38
+ key = normalize_chrom(str(chrom))
39
+ by_chrom.setdefault(key, []).extend(
40
+ zip(group["Start"].tolist(), group["End"].tolist())
41
+ )
42
+ self._index = {
43
+ chrom: self._pack(pairs) for chrom, pairs in by_chrom.items()
44
+ }
45
+
46
+ @staticmethod
47
+ def _pack(pairs: list[tuple[int, int]]) -> tuple[list[int], list[int], list[int]]:
48
+ """Sort (start, end) pairs by start and precompute the running max of ends."""
49
+ pairs = sorted(pairs)
50
+ starts = [s for s, _ in pairs]
51
+ ends = [e for _, e in pairs]
52
+ return starts, ends, list(itertools.accumulate(ends, max))
53
+
54
+ @classmethod
55
+ def wgs(cls) -> "CaptureIndex":
56
+ return cls(always_covered=True)
57
+
58
+ @classmethod
59
+ def from_bed(cls, bed_path: str) -> "CaptureIndex":
60
+ with warnings.catch_warnings():
61
+ warnings.filterwarnings("ignore", category=pd.errors.DtypeWarning)
62
+ try:
63
+ ranges = pr.read_bed(bed_path)
64
+ except (IndexError, AssertionError, pd.errors.EmptyDataError):
65
+ # pyranges raises rather than returning an empty frame when a BED
66
+ # holds no intervals: IndexError for a zero-byte file, AssertionError
67
+ # for a comment-only one. An empty index is the honest representation
68
+ # — the caller reports it (see describe_capture_problem) instead of
69
+ # the build dying on an opaque traceback.
70
+ return cls()
71
+ return cls(pyranges_obj=ranges)
72
+
73
+ def covers(self, chrom: str, pos: int) -> bool:
74
+ """Return True if 1-based pos is within any region for this chrom.
75
+
76
+ BED is 0-based half-open: [Start, End) → 1-based: Start < pos <= End
77
+ """
78
+ if self._always_covered:
79
+ return True
80
+ entry = self._index.get(chrom)
81
+ if entry is None:
82
+ return False
83
+ starts, _ends, max_ends = entry
84
+ # Find rightmost interval whose Start < pos (0-based start, so Start < pos_1based)
85
+ # bisect_left gives insertion point for pos in starts; all starts[:idx] < pos
86
+ idx = bisect.bisect_left(starts, pos) - 1
87
+ # Intervals may overlap, so any of starts[:idx+1] could reach pos. max_ends[idx]
88
+ # is the largest End among them, so it alone decides — no backwards scan needed.
89
+ return idx >= 0 and max_ends[idx] >= pos
90
+
91
+ @property
92
+ def is_always_covered(self) -> bool:
93
+ """True for the WGS sentinel: every position counts as covered."""
94
+ return self._always_covered
95
+
96
+ def indexed_chroms(self) -> set[str]:
97
+ """Normalized chrom names present in the index, unknown contigs included."""
98
+ return set(self._index)
99
+
100
+ def known_chroms(self) -> set[str]:
101
+ """Indexed chrom names that are real chromosomes (subset of ALL_CHROMS)."""
102
+ return self.indexed_chroms() & set(ALL_CHROMS)
103
+
104
+ def is_empty(self) -> bool:
105
+ """True when the index holds no intervals at all.
106
+
107
+ The WGS sentinel is not empty — it covers everything without an index.
108
+ """
109
+ return not self._always_covered and not self._index
110
+
111
+ def __getstate__(self) -> dict:
112
+ """Pickle without _pr — nothing outside this class reads the PyRanges object.
113
+
114
+ Keeping it put a pandas DataFrame in every WES pickle and made loading depend
115
+ on the pyranges/pandas versions that wrote it. Worse, __init__ normalizes the
116
+ chrom keys of _index but not of _pr, so a pickled _pr is a stale copy of the
117
+ raw BED names — exactly what a future reader would reintroduce this bug from.
118
+ _index alone answers covers() and is enough to rebuild after a normalization
119
+ change.
120
+ """
121
+ state = self.__dict__.copy()
122
+ state.pop("_pr", None)
123
+ return state
124
+
125
+ def __setstate__(self, state: dict) -> None:
126
+ """Migrate pickles written by older versions.
127
+
128
+ Three generations are repaired here, so existing databases self-heal on
129
+ load with no rebuild:
130
+ 1. no _index at all (pre binary-search refactor) — rebuilt from _pr;
131
+ 2. un-normalized chrom keys ('1' instead of 'chr1'), which made covers()
132
+ miss every WES technology and silently drop them from AN;
133
+ 3. 2-tuple entries lacking the running max of ends.
134
+ """
135
+ self.__dict__.update(state)
136
+ # Pickles written since __getstate__ carry no _pr; keep the attribute present
137
+ # so it stays safe to read on any restored object.
138
+ self.__dict__.setdefault("_pr", None)
139
+ index = getattr(self, "_index", None)
140
+
141
+ if index is None:
142
+ self._index = {}
143
+ if getattr(self, "_pr", None) is not None:
144
+ CaptureIndex.__init__(
145
+ self,
146
+ always_covered=self._always_covered,
147
+ pyranges_obj=self._pr,
148
+ )
149
+ return
150
+
151
+ needs_migration = any(
152
+ chrom != normalize_chrom(chrom) or len(entry) < 3
153
+ for chrom, entry in index.items()
154
+ )
155
+ if not needs_migration:
156
+ return
157
+
158
+ # Rebuild from the stored intervals rather than from _pr: _pr is absent in
159
+ # some pickles, and the intervals themselves are all we need.
160
+ by_chrom: dict[str, list[tuple[int, int]]] = {}
161
+ for chrom, entry in index.items():
162
+ starts, ends = entry[0], entry[1]
163
+ by_chrom.setdefault(normalize_chrom(chrom), []).extend(zip(starts, ends))
164
+ self._index = {chrom: self._pack(pairs) for chrom, pairs in by_chrom.items()}
165
+
166
+ def save(self, path: str) -> None:
167
+ with open(path, "wb") as f:
168
+ pickle.dump(self, f)
169
+
170
+ @classmethod
171
+ def load(cls, path: str) -> "CaptureIndex":
172
+ with open(path, "rb") as f:
173
+ return pickle.load(f)
174
+
175
+
176
+ def describe_capture_problem(index: CaptureIndex, tech_name: str) -> "str | None":
177
+ """Return a ready-to-emit message if this index cannot cover anything, else None.
178
+
179
+ Build time and load time both need this verdict; sharing one implementation is
180
+ what stops them drifting on what counts as broken. The two failures are reported
181
+ separately because they need different fixes:
182
+ - empty: the BED held no intervals (missing, empty or malformed file);
183
+ - unknown contigs: intervals exist but none is on a real chromosome.
184
+ """
185
+ if index.is_always_covered or index.known_chroms():
186
+ return None
187
+ consequence = (
188
+ "its samples will be counted as uncovered at every position, "
189
+ "lowering AN and inflating AF."
190
+ )
191
+ if index.is_empty():
192
+ return f"Capture regions for technology {tech_name!r} are empty — {consequence}"
193
+ found = sorted(index.indexed_chroms())
194
+ shown = ", ".join(found[:5]) + (", ..." if len(found) > 5 else "")
195
+ return (
196
+ f"Capture regions for technology {tech_name!r} match no known chromosome "
197
+ f"(found: {shown}) — {consequence}"
198
+ )
199
+
200
+
201
+ def load_capture_indices(
202
+ technologies: list[Technology], capture_dir: str
203
+ ) -> dict[int, "CaptureIndex"]:
204
+ result = {}
205
+ for tech in technologies:
206
+ path = f"{capture_dir}/tech_{tech.tech_id}.pickle"
207
+ try:
208
+ result[tech.tech_id] = CaptureIndex.load(path)
209
+ except FileNotFoundError as exc:
210
+ # Defaulting to WGS here would silently inflate AN, so this must stay fatal
211
+ # — only the message improves.
212
+ raise FileNotFoundError(
213
+ f"Missing capture index for technology {tech.tech_name!r} at {path} — "
214
+ "the database is incomplete; rebuild it or re-run 'afquery update-db'."
215
+ ) from exc
216
+ return result
@@ -0,0 +1,59 @@
1
+ # PAR1 and PAR2 for GRCh37 and GRCh38
2
+ PAR = {
3
+ "GRCh38": {
4
+ "chrX": [(10_001, 2_781_479), (155_701_383, 156_030_895)],
5
+ "chrY": [(10_001, 2_781_479), (56_887_903, 57_217_415)],
6
+ },
7
+ "GRCh37": {
8
+ "chrX": [(60_001, 2_699_520), (154_931_044, 155_260_560)],
9
+ "chrY": [(10_001, 2_649_520), (59_034_050, 59_363_566)],
10
+ },
11
+ }
12
+
13
+ AUTOSOMES = [f"chr{i}" for i in range(1, 23)]
14
+ SEX_CHROMS = ["chrX", "chrY"]
15
+ MITO_CHROM = "chrM"
16
+ # Mitochondrion aliases seen in the wild (Ensembl 'MT', UCSC 'chrM', hybrid 'chrMT').
17
+ _MITO_ALIASES = frozenset(["m", "mt", "chrm", "chrmt"])
18
+ # Chromosomes whose canonical spelling is uppercase after the 'chr' prefix.
19
+ _UPPERCASE_SUFFIXES = {"x": "X", "y": "Y"}
20
+ ALL_CHROMS = AUTOSOMES + SEX_CHROMS + [MITO_CHROM]
21
+ CHROM_ORDER: dict[str, int] = {c: i for i, c in enumerate(ALL_CHROMS)}
22
+ VALID_GENOME_BUILDS = frozenset(["GRCh37", "GRCh38"])
23
+ VALID_SEX = frozenset(["male", "female"])
24
+
25
+
26
+ def normalize_chrom(chrom: str) -> str:
27
+ """Normalize chromosome name to 'chr'-prefixed canonical form.
28
+
29
+ '1' -> 'chr1', 'chr1' -> 'chr1', 'CHR1' -> 'chr1'
30
+ 'X' -> 'chrX', 'x' -> 'chrX', 'MT' -> 'chrM', 'chrMT' -> 'chrM'
31
+
32
+ Only the 'chr' prefix and the known chromosome suffixes are case-normalized;
33
+ anything else keeps its body byte-for-byte ('GL000209.1' -> 'chrGL000209.1',
34
+ 'chr1_KI270706v1_random' unchanged), so unplaced and alt contigs stay outside
35
+ ALL_CHROMS and are still dropped by the build stage rather than guessed at.
36
+
37
+ Idempotent on every member of ALL_CHROMS: the output is what gets persisted as
38
+ the Parquet partition name, so a name that changed under normalization would
39
+ orphan the partitions of already-built databases.
40
+ """
41
+ chrom = chrom.strip()
42
+ if chrom.lower() in _MITO_ALIASES:
43
+ return MITO_CHROM
44
+ body = chrom[3:] if chrom[:3].lower() == "chr" else chrom
45
+ if body.lower() in _UPPERCASE_SUFFIXES:
46
+ return "chr" + _UPPERCASE_SUFFIXES[body.lower()]
47
+ return "chr" + body
48
+
49
+
50
+ def is_autosome(chrom: str) -> bool:
51
+ return normalize_chrom(chrom) in AUTOSOMES
52
+
53
+
54
+ def is_sex_chrom(chrom: str) -> bool:
55
+ return normalize_chrom(chrom) in SEX_CHROMS
56
+
57
+
58
+ def is_mito(chrom: str) -> bool:
59
+ return normalize_chrom(chrom) == MITO_CHROM
@@ -33,6 +33,24 @@ PARQUET_SCHEMA = pa.schema([
33
33
  ])
34
34
 
35
35
 
36
+ def _filter_known_chroms(chroms: list[str], source: str) -> list[str]:
37
+ """Keep only real chromosomes, reporting the dropped contigs once.
38
+
39
+ Chrom discovery happens in three places and every one of them must drop
40
+ unplaced and alt contigs. Centralizing the filter is what keeps the report
41
+ from either vanishing — it used to live downstream of the filter, where it
42
+ could never fire — or firing once per row.
43
+ """
44
+ known = [c for c in chroms if c in ALL_CHROMS]
45
+ unknown = sorted({c for c in chroms if c not in ALL_CHROMS})
46
+ if unknown:
47
+ logger.warning(
48
+ "[build] Skipping %d unknown contig(s) from %s: %s",
49
+ len(unknown), source, ", ".join(unknown),
50
+ )
51
+ return known
52
+
53
+
36
54
  def _get_chroms_from_file(parquet_path: str) -> list[str]:
37
55
  """SELECT DISTINCT chrom from a single consolidated Parquet file."""
38
56
  path = parquet_path.replace("'", "''")
@@ -46,7 +64,7 @@ def _get_chroms_from_file(parquet_path: str) -> list[str]:
46
64
  return []
47
65
  finally:
48
66
  con.close()
49
- return [r[0] for r in rows if r[0] in ALL_CHROMS]
67
+ return _filter_known_chroms([r[0] for r in rows], parquet_path)
50
68
 
51
69
 
52
70
  def consolidate_temp_files(
@@ -81,13 +99,12 @@ def _get_chroms_from_consolidated(path: str) -> list[str]:
81
99
  """Discover chromosomes from consolidated path (Hive-partitioned directory or single file)."""
82
100
  if os.path.isdir(path):
83
101
  # Hive-partitioned: scan chrom=X/ subdirs — fast, no SQL needed
84
- chroms = []
85
- for entry in os.scandir(path):
86
- if entry.is_dir() and entry.name.startswith("chrom="):
87
- chrom = entry.name[6:] # strip "chrom="
88
- if chrom in ALL_CHROMS:
89
- chroms.append(chrom)
90
- return chroms
102
+ chroms = [
103
+ entry.name[6:] # strip "chrom="
104
+ for entry in os.scandir(path)
105
+ if entry.is_dir() and entry.name.startswith("chrom=")
106
+ ]
107
+ return _filter_known_chroms(chroms, path)
91
108
  else:
92
109
  # Single file (legacy) — use SQL
93
110
  return _get_chroms_from_file(path)
@@ -111,7 +128,7 @@ def get_chroms_in_temp_files(tmp_dir: str) -> list[str]:
111
128
  finally:
112
129
  con.close()
113
130
 
114
- return [r[0] for r in rows if r[0] in ALL_CHROMS]
131
+ return _filter_known_chroms([r[0] for r in rows], tmp_dir)
115
132
 
116
133
 
117
134
  def _make_table(
@@ -524,12 +541,8 @@ def build_all_parquets(
524
541
  chroms = _get_chroms_from_consolidated(consolidated_path)
525
542
  else:
526
543
  chroms = get_chroms_in_temp_files(tmp_dir)
527
- valid_chroms = []
528
- for chrom in chroms:
529
- if chrom in ALL_CHROMS:
530
- valid_chroms.append(chrom)
531
- else:
532
- logger.warning("Skipping unknown contig: %s", chrom)
544
+ # Both discovery helpers already filter to ALL_CHROMS and report what they drop.
545
+ valid_chroms = chroms
533
546
 
534
547
  if not valid_chroms:
535
548
  return {}
@@ -0,0 +1,25 @@
1
+ import warnings
2
+
3
+ from ..capture import CaptureIndex, describe_capture_problem
4
+ from ..models import AfqueryWarning, Technology
5
+
6
+
7
+ def build_capture_indices(
8
+ technologies: list[Technology],
9
+ capture_dir: str,
10
+ ) -> None:
11
+ for tech in technologies:
12
+ if tech.bed_path is None:
13
+ idx = CaptureIndex.wgs()
14
+ else:
15
+ idx = CaptureIndex.from_bed(tech.bed_path)
16
+ problem = describe_capture_problem(idx, tech.tech_name)
17
+ if problem is not None:
18
+ # warnings, not logger: this is also reached from the Python API via
19
+ # update.py, where a logger message is invisible unless the caller
20
+ # configured a handler. "Your allele frequencies will be wrong" has to
21
+ # surface on both paths.
22
+ warnings.warn(
23
+ f"{problem} (BED: {tech.bed_path})", AfqueryWarning, stacklevel=2,
24
+ )
25
+ idx.save(f"{capture_dir}/tech_{tech.tech_id}.pickle")
@@ -7,7 +7,7 @@ import duckdb
7
7
  from pyroaring import BitMap
8
8
 
9
9
  from .bitmaps import deserialize
10
- from .capture import CaptureIndex, load_capture_indices
10
+ from .capture import CaptureIndex, describe_capture_problem, load_capture_indices
11
11
  from .constants import normalize_chrom, ALL_CHROMS, CHROM_ORDER
12
12
  from .models import AfqueryWarning, QueryParams, QueryResult, SampleCarrier, VariantKey, Sample, Technology, SampleFilter
13
13
  from .ploidy import compute_AN, split_ploidy
@@ -70,6 +70,12 @@ class QueryEngine:
70
70
  self._male_bm = sex_bms.get("male", BitMap())
71
71
  self._female_bm = sex_bms.get("female", BitMap())
72
72
  self._capture = load_capture_indices(techs, str(self._db / "capture"))
73
+ for tech_id, capture_idx in self._capture.items():
74
+ problem = describe_capture_problem(
75
+ capture_idx, self._tech_map[tech_id].tech_name
76
+ )
77
+ if problem is not None:
78
+ warnings.warn(problem, AfqueryWarning, stacklevel=2)
73
79
  self._tech_bitmaps = self._bitmaps.get("tech", {})
74
80
  self._all_samples_bm = BitMap(s.sample_id for s in self._samples)
75
81
  self._tech_name_to_id: dict[str, str] = {
@@ -95,10 +101,19 @@ class QueryEngine:
95
101
  # For each tech, store its bitmap; also precompute the union of all WGS tech bitmaps
96
102
  self._covered_all_bm: BitMap = BitMap()
97
103
  for tech_id, capture_idx in self._capture.items():
98
- if capture_idx._always_covered:
104
+ if capture_idx.is_always_covered:
99
105
  bm = self._tech_bitmaps.get(str(tech_id), BitMap())
100
106
  self._covered_all_bm |= bm
101
107
 
108
+ # Targeted (non-WGS) technologies only. The per-position loops below run once
109
+ # per queried position, so the WGS ones are skipped here rather than re-tested
110
+ # on every iteration — they are already folded into _covered_all_bm.
111
+ self._targeted_capture: dict[int, CaptureIndex] = {
112
+ tech_id: capture_idx
113
+ for tech_id, capture_idx in self._capture.items()
114
+ if not capture_idx.is_always_covered
115
+ }
116
+
102
117
  # Cache parquet glob patterns per chrom (avoids filesystem scan on every query)
103
118
  self._glob_cache: dict[str, str | None] = {}
104
119
 
@@ -213,9 +228,7 @@ class QueryEngine:
213
228
 
214
229
  # Phase 1: count-based gate
215
230
  if min_pass > 0 or min_observed > 0:
216
- for tech_id, capture_idx in self._capture.items():
217
- if capture_idx._always_covered:
218
- continue
231
+ for tech_id in self._targeted_capture:
219
232
  tech_bm = self._tech_bitmaps.get(str(tech_id), BitMap())
220
233
  tech_eligible = eligible & tech_bm
221
234
  if len(tech_eligible) == 0:
@@ -232,9 +245,7 @@ class QueryEngine:
232
245
  # Phase 2: quality_pass gate (--min-quality-evidence K)
233
246
  if quality_pass_bm is not None and min_quality_evidence > 0:
234
247
  already_filtered = filtered_bm if filtered_bm is not None else BitMap()
235
- for tech_id, capture_idx in self._capture.items():
236
- if capture_idx._always_covered:
237
- continue
248
+ for tech_id in self._targeted_capture:
238
249
  tech_bm = self._tech_bitmaps.get(str(tech_id), BitMap())
239
250
  tech_eligible = eligible & tech_bm
240
251
  if len(tech_eligible) == 0:
@@ -287,9 +298,7 @@ class QueryEngine:
287
298
  ) -> tuple[BitMap, int]:
288
299
  """Return (eligible_bitmap, AN) for a single position."""
289
300
  covered = BitMap(self._covered_all_bm) # start with WGS samples (always covered)
290
- for tech_id, capture_idx in self._capture.items():
291
- if capture_idx._always_covered:
292
- continue # already included in _covered_all_bm
301
+ for tech_id, capture_idx in self._targeted_capture.items():
293
302
  if capture_idx.covers(chrom, pos):
294
303
  covered |= self._tech_bitmaps.get(str(tech_id), BitMap())
295
304
  eligible = sample_bitmap & covered
@@ -0,0 +1,2 @@
1
+ 1 999 2000
2
+ X 999 2000