afquery 0.2.2__tar.gz → 0.3.0__tar.gz

{afquery-0.2.2 → afquery-0.3.0}/.github/workflows/ci.yml

@@ -34,3 +34,4 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.xml
           fail_ci_if_error: true
+          disable-safe: true

afquery-0.3.0/.github/workflows/docs.yml

@@ -0,0 +1,52 @@
+name: Deploy Documentation
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - 'docs/**'
+      - 'mkdocs.yml'
+      - 'src/**'
+  workflow_dispatch:
+    inputs:
+      bootstrap:
+        description: 'Wipe gh-pages before deploying (one-time when migrating from mkdocs gh-deploy to mike)'
+        type: boolean
+        default: false
+
+permissions:
+  contents: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install docs dependencies
+        run: pip install -e ".[docs]"
+
+      - name: Configure git for mike
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Fetch gh-pages
+        run: git fetch origin gh-pages --depth=1 || true
+
+      - name: Bootstrap gh-pages (wipe old flat-deploy content)
+        if: ${{ inputs.bootstrap == true }}
+        run: mike delete --all --push --allow-empty
+
+      - name: Deploy dev docs with mike
+        run: mike deploy --push --update-aliases dev

{afquery-0.2.2 → afquery-0.3.0}/.github/workflows/release.yml

@@ -127,66 +127,39 @@ jobs:
             ghcr.io/${{ github.repository }}:${{ steps.version.outputs.version }}
             ghcr.io/${{ github.repository }}:latest
 
-  # ── 6. Bioconda PR (final releases only — no "rc" in tag) ────────────
-  bioconda-pr:
-    needs: publish-pypi
+  # ── 6. Versioned docs → gh-pages via mike (final releases only) ──────
+  docs:
+    needs: test
+    if: ${{ !contains(github.ref_name, 'rc') }}
     runs-on: ubuntu-latest
-    if: ${{ !contains(github.ref_name, 'rc') }}   # skip RC tags
+    permissions:
+      contents: write
     steps:
-      - name: Get version from tag
-        run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
-
-      - name: Fetch SHA256 from PyPI
-        run: |
-          SHA256=$(curl -s https://pypi.org/pypi/afquery/${VERSION}/json \
-            | jq -r '.urls[] | select(.packagetype=="sdist") | .digests.sha256')
-          echo "SHA256=${SHA256}" >> $GITHUB_ENV
-
-      - name: Get fork owner
-        env:
-          GH_TOKEN: ${{ secrets.GH_PAT }}
-        run: echo "FORK_USER=$(gh api user --jq .login)" >> $GITHUB_ENV
-
-      - name: Sync fork with upstream
-        env:
-          GH_TOKEN: ${{ secrets.GH_PAT }}
-        run: gh repo sync ${FORK_USER}/bioconda-recipes --source bioconda/bioconda-recipes --branch master
-
-      - name: Checkout afquery repo
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
-          path: afquery-repo
+          fetch-depth: 0
 
-      - name: Checkout fork of bioconda-recipes
-        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          repository: ${{ env.FORK_USER }}/bioconda-recipes
-          token: ${{ secrets.GH_PAT }}
-          path: bioconda-recipes
+          python-version: "3.11"
 
-      - name: Update recipe
-        working-directory: bioconda-recipes
-        run: |
-          mkdir -p recipes/afquery
-          cp ../afquery-repo/recipes/afquery/meta.yaml recipes/afquery/meta.yaml
-          sed -i "s/^{% set version = .* %}/{% set version = \"${VERSION}\" %}/" recipes/afquery/meta.yaml
-          sed -i "s/sha256: .*/sha256: ${SHA256}/" recipes/afquery/meta.yaml
-          sed -i "s/^  number: .*/  number: 0/" recipes/afquery/meta.yaml
-
-      - name: Create pull request
-        working-directory: bioconda-recipes
-        env:
-          GH_TOKEN: ${{ secrets.GH_PAT }}
+      - name: Install docs dependencies
+        run: pip install -e ".[docs]"
+
+      - name: Configure git for mike
         run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git checkout -b afquery-${VERSION}
-          git add recipes/afquery/meta.yaml
-          git commit -m "Update afquery to ${VERSION}"
-          git push origin afquery-${VERSION}
-          gh pr create \
-            --repo bioconda/bioconda-recipes \
-            --title "Update afquery to ${VERSION}" \
-            --body "Automated version bump from https://github.com/dlopez-bioinfo/afquery/releases/tag/v${VERSION}" \
-            --head "${FORK_USER}:afquery-${VERSION}" \
-            --base master
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Fetch gh-pages
+        run: git fetch origin gh-pages --depth=1 || true
+
+      - name: Extract version
+        id: version
+        run: echo "version=${GITHUB_REF_NAME#v}" >> $GITHUB_OUTPUT
+
+      - name: Deploy versioned docs and update latest alias
+        run: mike deploy --push --update-aliases ${{ steps.version.outputs.version }} latest
+
+      - name: Set default to latest
+        run: mike set-default --push latest

{afquery-0.2.2 → afquery-0.3.0}/.gitignore

@@ -60,3 +60,4 @@ src/afquery/_version.py
 *.parquet
 db/*
 site
+CONTEXT.md

afquery-0.3.0/CONTRIBUTING.md

@@ -0,0 +1,35 @@
+# Contributing
+
+## Documentation deployment
+
+The published site at <https://dlopez-bioinfo.github.io/afquery/> is built with [MkDocs](https://www.mkdocs.org/) (Material theme) and versioned with [mike](https://github.com/jimporter/mike). Source content lives under `docs/` and is configured by `mkdocs.yml`.
+
+### CI workflows
+
+| Workflow | Trigger | What it does |
+|---|---|---|
+| `.github/workflows/docs.yml` | push to `master` (paths: `docs/**`, `mkdocs.yml`, `src/**`) and manual dispatch | `mike deploy --push --update-aliases dev` — publishes the working master under the `dev` alias |
+| `.github/workflows/release.yml` (job `docs`) | push of a `v*` tag (non-`rc`) | `mike deploy --push --update-aliases <version> latest` followed by `mike set-default --push latest` — publishes the tagged version, points `latest` at it, and makes `latest` the site root |
+
+### Cutting a release
+
+Tag the commit with a [PEP 440](https://peps.python.org/pep-0440/)-compatible version prefixed with `v` (e.g. `v0.3.0`) and push the tag. The `release.yml` workflow handles PyPI, the GitHub release, the Docker image, and the versioned docs deploy. Pre-release tags (anything containing `rc`) skip Docker and docs publishing.
+
+### Local preview
+
+```bash
+pip install -e ".[docs]"
+
+# Plain build — no versioning, fastest iteration on content
+mkdocs serve
+
+# Versioned layout — only after running mike deploy locally at least once
+mike deploy 0.0.0-test dev      # writes to local gh-pages branch (no push)
+mike serve                      # serves the gh-pages branch with version selector
+```
+
+To discard local mike state: `git branch -D gh-pages`.
+
+### Migrating from a flat `mkdocs gh-deploy` (one-time)
+
+When `gh-pages` still contains content from the previous flat deploy, the first `mike deploy` will leave the old root-level files in place. To wipe `gh-pages` clean before deploying, manually trigger the *Deploy Documentation* workflow with `bootstrap: true`. This runs `mike delete --all --push --allow-empty` before deploying, leaving only the versioned layout (`versions.json` + version subdirectories + redirector at root).

{afquery-0.2.2 → afquery-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: afquery
-Version: 0.2.2
+Version: 0.3.0
 Summary: Genomic allele frequency query engine with bitmap-encoded genotypes
 License: MIT
 License-File: LICENSE

afquery-0.3.0/benchmarks/.gitignore

@@ -0,0 +1,20 @@
+# Benchmark data directories (large, external)
+# Users create these as needed via config.py
+
+# pycache
+__pycache__/
+*.pyc
+.pytest_cache/
+
+# benchmarking results
+results
+.snakemake
+
+# Snakemake log directory
+logs/
+
+# Default data directory
+.results/
+
+# Local test run logs
+smoke_test.log

afquery-0.3.0/benchmarks/INSTALL.md

@@ -0,0 +1,141 @@
+# Installation Guide for AFQuery Benchmarks
+
+This guide covers installing benchmark dependencies in your existing `afquery_bench` micromamba environment.
+See [README.md](./README.md) for the recommended setup using `envs/benchmark.yaml`.
+
+## Option 1: Using conda environment file (Recommended)
+
+The easiest way is to create the pre-configured environment:
+
+```bash
+cd benchmarks/
+micromamba env create -f envs/benchmark.yaml
+micromamba activate afquery_bench
+```
+
+Then skip to [Running Benchmarks](#running-benchmarks).
+
+## Option 2: Manual installation
+
+If you prefer to install into an existing environment, install dependencies into your `snakemake` micromamba environment:
+
+### Prerequisites
+
+You must have:
+- Snakemake ≥ 8 installed in a micromamba environment named `snakemake`
+
+If not, install it:
+```bash
+micromamba install -n snakemake snakemake
+```
+
+### Install all dependencies at once
+
+```bash
+micromamba run -n snakemake mamba install \
+  -c bioconda -c conda-forge \
+  bcftools bedtools matplotlib numpy pandas scipy wget \
+  && micromamba run -n snakemake pip install afquery
+```
+
+### Or step-by-step installation
+
+#### 1. Bioinformatics tools
+
+```bash
+micromamba run -n snakemake mamba install \
+  -c bioconda -c conda-forge \
+  bcftools bedtools wget
+```
+
+#### 2. Python scientific libraries
+
+```bash
+micromamba run -n snakemake mamba install \
+  -c conda-forge \
+  numpy pandas scipy matplotlib
+```
+
+#### 3. AFQuery
+
+```bash
+micromamba run -n snakemake pip install afquery
+```
+
+## Verification
+
+Test that everything is installed:
+
+```bash
+# Test bcftools
+micromamba run -n afquery_bench bcftools --version
+
+# Test Python imports
+micromamba run -n afquery_bench python -c "
+import afquery
+import numpy
+import pandas
+import scipy
+import matplotlib
+print('All dependencies OK!')
+"
+```
+
+Expected output:
+```
+bcftools >=1.17
+All dependencies OK!
+```
+
+## Running Benchmarks
+
+Once dependencies are installed and the environment is active:
+
+```bash
+cd benchmarks/
+micromamba activate afquery_bench
+snakemake --cores 52 all
+```
+
+Or run with conda activation:
+
+```bash
+micromamba run -n afquery_bench snakemake --cores 52 all
+```
+
+See [README.md](./README.md) for more options (smoke test, dry-run, etc.).
+
+## Troubleshooting
+
+### `afquery` import fails
+
+Verify the installation:
+```bash
+micromamba run -n afquery_bench pip show afquery
+```
+
+If not found, reinstall:
+```bash
+micromamba run -n afquery_bench pip install --upgrade afquery
+```
+
+### Missing Python packages
+
+Check installed packages:
+```bash
+micromamba run -n afquery_bench pip list | grep -E "numpy|pandas|scipy|matplotlib"
+```
+
+If any are missing, re-run the mamba install commands above.
+
+### `bcftools` not found
+
+Verify bcftools is installed:
+```bash
+micromamba run -n afquery_bench which bcftools
+```
+
+If not found, install it:
+```bash
+micromamba run -n afquery_bench mamba install -c bioconda bcftools
+```

afquery-0.3.0/benchmarks/README.md

@@ -0,0 +1,172 @@
+# AFQuery Benchmarking Suite
+
+This directory contains benchmarking experiments for AFQuery.
+
+## Benchmark Categories
+
+### 1. [Performance Benchmarks](./performance/README.md)
+
+Core performance characterization across 4 experiments:
+
+- **Experiment 1:** Query latency scaling with sample count
+- **Experiment 2:** Build time scaling with parallelism
+- **Experiment 3:** VCF annotation throughput
+- **Experiment 4:** AFQuery vs. bcftools comparison
+
+Uses real 1000 Genomes data (chr22) and synthetic datasets (1K-50K samples).
+
+### 2. [Capture Kit Benchmark](./capture_kit/README.md)
+
+Capture kit mixing impact on allele frequency classification:
+
+- Sample generation with three Agilent SureSelect kits (v5, v6, v7)
+- Three mixing scenarios (balanced, skewed, extreme)
+- ACMG classification discordance analysis with directional error decomposition (toward-pathogenic vs. toward-benign)
+- Coverage overlap metrics
+
+## Prerequisites
+
+- micromamba with `snakemake` environment already set up (with Snakemake >= 8 and SLURM executor plugin)
+- Benchmark dependencies installed in the `snakemake` environment (see Environment Setup section)
+- `/usr/bin/time` (GNU time) for memory profiling -- should be available on most Linux systems
+- ~200 GB disk space for all 1KG data and databases
+
+## Environment Setup
+
+All benchmark dependencies are installed in the `snakemake` micromamba environment.
+
+### Install dependencies
+
+The benchmark requires:
+- External bioinformatics tools: bcftools >= 1.18, bedtools, bgzip/tabix
+- Python packages: pandas, numpy, scipy, matplotlib
+- AFQuery and its dependencies: pyroaring, pyarrow, duckdb, cyvcf2, pyranges, click, tqdm
+
+Install them in your `snakemake` environment:
+
+```bash
+micromamba run -n snakemake mamba install \
+  -c bioconda -c conda-forge \
+  bcftools bedtools matplotlib numpy pandas scipy wget \
+  && micromamba run -n snakemake \
+  pip install -e <path_to_afquery_repo>
+```
+
+### Verify
+
+```bash
+micromamba run -n snakemake bcftools --version
+micromamba run -n snakemake python -c "import afquery; print('OK')"
+```
+
+## Running the Benchmarks
+
+All commands below are run from the `benchmarks/` directory. Activate the `snakemake` environment first:
+
+```bash
+micromamba activate snakemake
+```
+
+Then use Snakemake with `--cores 52` to utilize all local CPU cores:
+
+```bash
+# Run everything (both benchmarks)
+snakemake --cores 52 all
+
+# Run individual benchmarks
+snakemake --cores 52 performance_all
+snakemake --cores 52 capture_kit_all
+
+# Download 1KG data only (prerequisite for both)
+snakemake --cores 52 download_1kg
+
+# Dry run (preview what will execute)
+snakemake --cores 52 --dry-run all
+
+# Smoke test (fast validation with small parameter scales)
+snakemake --cores 52 --config smoke_test=true all
+```
+
+### Smoke Test
+
+The `smoke_test=true` flag reduces both pipelines to minimal scales for fast validation:
+
+- **Performance:** 1 synthetic scale (1K), 1 1KG subset (500), 2 thread counts (1, 4), 1 rep
+- **Capture kit:** 50 samples, balanced scenario only
+
+This is useful for verifying pipeline logic without waiting for full-scale runs.
+
+### Resuming after a failure
+
+Snakemake uses output files to track completed steps. Re-running any command above will automatically skip steps whose outputs already exist and resume from the first incomplete step.
+
+## Directory Structure
+
+```
+benchmarks/
+├── Snakefile              # Root pipeline (includes both benchmarks)
+├── config.yaml            # Global parameters (data_dir, threads, etc.)
+├── envs/
+│   └── benchmark.yaml     # conda/micromamba environment spec
+├── shared/
+│   ├── __init__.py
+│   ├── config.py          # Common constants: DATA_DIR, 1KG paths, SEED
+│   ├── utils.py           # Common helpers: stats, time_ms, save_figure, WONG_COLORS
+│   └── rules/
+│       └── download_1kg.smk   # Shared Snakemake rules: download + split 1KG
+├── performance/
+│   ├── Snakefile          # Performance-specific rules
+│   ├── config.py          # Performance parameters (scales, reps, thread counts)
+│   ├── config_smoke.py    # Minimal config for quick smoke testing
+│   ├── 01_prepare_data.py
+│   ├── 02_query_scaling.py
+│   ├── 03_build.py
+│   ├── 04_annotate.py
+│   ├── 05_vs_bcftools.py
+│   ├── 06_plot.py
+│   ├── collect_prepare.py        # Aggregation scripts
+│   ├── collect_query_scaling.py
+│   ├── collect_build_perf.py
+│   ├── collect_annotate.py
+│   └── collect_bcftools.py
+└── capture_kit/
+    ├── Snakefile          # Capture kit-specific rules
+    ├── config.py          # Capture kit parameters (scenarios, ACMG thresholds)
+    ├── beds/              # Agilent SureSelect BED files (chr22, committed)
+    │   ├── SureSelect_v5.bed
+    │   ├── SureSelect_v6.bed
+    │   └── SureSelect_v7.bed
+    ├── 01a_assign_samples.py    # Subsample & assign technologies
+    ├── 01b_write_manifests.py   # Write AFQuery manifest TSVs
+    ├── 02_build_databases.py
+    ├── 03_compute_metrics.py
+    ├── 04_classify_acmg.py
+    └── 05_plot_figures.py
+```
+
+## Configuration
+
+Edit `config.yaml` to set the data directory and global parameters:
+
+```yaml
+data_dir: "/path/to/bench_data"   # needs ~200 GB free
+build_memory: "8GB"               # DuckDB memory per worker
+```
+
+The data directory can also be set via the `AFQUERY_BENCH_DATA` environment variable.
+
+## Known Limitations
+
+- **Chromosome scope:** All benchmarks use only chromosome 22 (1000 Genomes Phase 3). Chr22 is one of the smallest autosomes (~51 Mb) and coverage patterns may not be representative of genome-wide behavior. The paper's methods section should discuss generalizability.
+- **Synthetic data for scaling:** Query scaling experiments (1K-50K samples) use synthetic data. Allele frequency spectra and LD patterns differ from real population data.
+- **Capture kits:** The capture kit benchmark uses only Agilent SureSelect kits (v5, v6, v7). Other vendors (Illumina, Roche) and custom panels are not evaluated.
+
+## Reproducibility
+
+All random seeds are fixed in `shared/config.py` (SEED = 42).
+Re-running with the same configuration produces identical results within timing noise.
+
+## References
+
+- Weber LM et al. (2019). Essential guidelines for computational method benchmarking. *Genome Biology* 20:125. DOI: 10.1186/s13059-019-1738-8
+- Auton A et al. (2015). A global reference for human genetic variation. *Nature* 526:68-74. DOI: 10.1038/nature15393

afquery-0.3.0/benchmarks/Snakefile

@@ -0,0 +1,103 @@
+"""Unified benchmark pipeline for AFQuery.
+
+Usage (run from benchmarks/ directory):
+    # All benchmarks
+    snakemake --cores 52 all
+
+    # Only performance benchmark
+    snakemake --cores 52 performance_all
+
+    # Only capture kit benchmark
+    snakemake --cores 52 capture_kit_all
+
+    # Only download 1KG data
+    snakemake --cores 52 download_1kg
+
+    # Dry run (preview rules to execute)
+    snakemake --cores 52 --dry-run all
+
+    # Smoke test (fast validation with small scales)
+    snakemake --cores 52 --config smoke_test=true all
+"""
+
+import sys
+from pathlib import Path
+
+# Make shared/ importable from within Snakemake rules
+_bench = Path(workflow.basedir)
+sys.path.insert(0, str(_bench))
+
+from shared.config import (  # noqa: E402
+    DATA_DIR,
+    ONEKG_DIR,
+    ONEKG_MANIFEST,
+    SEED,
+)
+
+# ---------------------------------------------------------------------------
+# Global settings
+# ---------------------------------------------------------------------------
+configfile: "config.yaml"
+
+# NOTE: conda env management is DISABLED. Dependencies are installed manually
+# in the snakemake environment. Conda path normalization is no longer needed.
+# if config.get("conda_env_file"):
+#     _conda_env = Path(config["conda_env_file"])
+#     if not _conda_env.is_absolute():
+#         config["conda_env_file"] = str((_bench / _conda_env).resolve())
+
+
+
+# Guard: data_dir must be set
+if not config.get("data_dir", "").strip():
+    raise WorkflowError(
+        "config.yaml: 'data_dir' is not set. "
+        "Edit config.yaml or pass --config data_dir=/path/to/data"
+    )
+
+# Override DATA_DIR from config if provided
+if config.get("data_dir"):
+    import shared.config as _sc
+    _sc._set_data_dir(Path(config["data_dir"]))
+    DATA_DIR     = _sc.DATA_DIR
+    ONEKG_DIR    = _sc.ONEKG_DIR
+    ONEKG_MANIFEST = _sc.ONEKG_MANIFEST
+
+# Propagate the resolved DATA_DIR to all child processes via environment.
+# Python scripts import shared.config fresh in each subprocess; without this
+# they fall back to the default Path(".results") and cannot find pipeline outputs.
+import os as _os
+_os.environ["AFQUERY_BENCH_DATA"] = str(DATA_DIR.resolve())
+if config.get("smoke_test"):
+    _os.environ["AFQUERY_BENCH_SMOKE"] = "1"
+
+# Export constants to sub-Snakefiles via globals
+workflow._globals = {}  # not needed; sub-files access via import
+
+# ---------------------------------------------------------------------------
+# Create log directories before any rule runs (Snakemake does not auto-create
+# log dirs, unlike output dirs).
+# ---------------------------------------------------------------------------
+onstart:
+    import os
+    for subdir in ["performance", "capture_kit"]:
+        os.makedirs(f"logs/{subdir}", exist_ok=True)
+
+
+# ---------------------------------------------------------------------------
+# Include sub-pipelines
+# ---------------------------------------------------------------------------
+include: "shared/rules/download_1kg.smk"
+include: "performance/Snakefile"
+include: "capture_kit/Snakefile"
+
+# ---------------------------------------------------------------------------
+# Top-level targets
+# ---------------------------------------------------------------------------
+
+
+rule all:
+    """Run both benchmarks end-to-end."""
+    input:
+        rules.performance_all.output,
+        rules.capture_kit_all.output,

afquery-0.3.0/benchmarks/capture_kit/.gitignore

@@ -0,0 +1,15 @@
+# Benchmark data and results
+/results/
+/figures/
+*.csv
+*.pdf
+*.png
+
+# Cache
+__pycache__/
+*.pyc
+.pytest_cache/
+
+# Temporary
+*.tmp
+