afquery 0.1.0__tar.gz

afquery-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0          # hatch-vcs needs full history to resolve version
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Test
+        run: pytest --tb=short -q

afquery-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,138 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v[0-9]*"      # v1.2.4, v1.2.4rc1, etc.
+
+jobs:
+
+  # ── 1. Test ──────────────────────────────────────────────────────────
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e ".[dev]"
+      - run: pytest --tb=short -q
+
+  # ── 2. Build ─────────────────────────────────────────────────────────
+  build:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0      # required: hatch-vcs reads git tags for version
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install build
+      - run: python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  # ── 3. Publish to PyPI (OIDC — always, RC and final) ─────────────────
+  publish-pypi:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/afquery
+    permissions:
+      id-token: write       # required for OIDC Trusted Publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  # ── 4. GitHub Release ────────────────────────────────────────────────
+  github-release:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true
+
+  # ── 5. Bioconda PR (final releases only — no "rc" in tag) ────────────
+  bioconda-pr:
+    needs: publish-pypi
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.ref_name, 'rc') }}   # skip RC tags
+    steps:
+      - name: Get version from tag
+        run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
+
+      - name: Fetch SHA256 from PyPI
+        run: |
+          SHA256=$(curl -s https://pypi.org/pypi/afquery/${VERSION}/json \
+            | jq -r '.urls[] | select(.packagetype=="sdist") | .digests.sha256')
+          echo "SHA256=${SHA256}" >> $GITHUB_ENV
+
+      - name: Get fork owner
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT }}
+        run: echo "FORK_USER=$(gh api user --jq .login)" >> $GITHUB_ENV
+
+      - name: Sync fork with upstream
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT }}
+        run: gh repo sync ${FORK_USER}/bioconda-recipes --source bioconda/bioconda-recipes --branch master
+
+      - name: Checkout afquery repo
+        uses: actions/checkout@v4
+        with:
+          path: afquery-repo
+
+      - name: Checkout fork of bioconda-recipes
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.FORK_USER }}/bioconda-recipes
+          token: ${{ secrets.GH_PAT }}
+          path: bioconda-recipes
+
+      - name: Update recipe
+        working-directory: bioconda-recipes
+        run: |
+          mkdir -p recipes/afquery
+          cp ../afquery-repo/recipes/afquery/meta.yaml recipes/afquery/meta.yaml
+          sed -i "s/^{% set version = .* %}/{% set version = \"${VERSION}\" %}/" recipes/afquery/meta.yaml
+          sed -i "s/sha256: .*/sha256: ${SHA256}/" recipes/afquery/meta.yaml
+          sed -i "s/^  number: .*/  number: 0/" recipes/afquery/meta.yaml
+
+      - name: Create pull request
+        working-directory: bioconda-recipes
+        env:
+          GH_TOKEN: ${{ secrets.GH_PAT }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b afquery-${VERSION}
+          git add recipes/afquery/meta.yaml
+          git commit -m "Update afquery to ${VERSION}"
+          git push origin afquery-${VERSION}
+          gh pr create \
+            --repo bioconda/bioconda-recipes \
+            --title "Update afquery to ${VERSION}" \
+            --body "Automated version bump from https://github.com/dlopez-bioinfo/afquery/releases/tag/v${VERSION}" \
+            --head "${FORK_USER}:afquery-${VERSION}" \
+            --base master

afquery-0.1.0/.gitignore

@@ -0,0 +1,61 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Virtual environments
+venv/
+ENV/
+env/
+.venv
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.hypothesis/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.DS_Store
+Thumbs.db
+
+# Temporary files
+*.tmp
+*.log
+*.bak
+*~
+
+# Auto-generated by hatch-vcs at build time
+src/afquery/_version.py
+
+# Database files (generated)
+*.duckdb
+*.duckdb.wal
+*.parquet
+db/*

afquery-0.1.0/PKG-INFO

@@ -0,0 +1,15 @@
+Metadata-Version: 2.4
+Name: afquery
+Version: 0.1.0
+Summary: Genomic allele frequency query engine with bitmap-encoded genotypes
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1
+Requires-Dist: cyvcf2>=0.30
+Requires-Dist: duckdb>=0.10
+Requires-Dist: pyarrow>=14.0
+Requires-Dist: pyranges<0.2,>=0.1.2
+Requires-Dist: pyroaring>=0.4.8
+Requires-Dist: tqdm>=4.60
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'

afquery-0.1.0/README.md

@@ -0,0 +1,355 @@
+# afquery
+
+Genomic allele frequency query engine with bitmap-encoded genotypes. Fast, file-based queries over 10K-50K samples at sub-100ms latency.
+
+## Features
+
+- **Fast point queries**: <100ms cold start, ~10ms warm queries on single positions
+- **Batch queries**: Multi-position queries via SQL IN clauses or temporary tables
+- **Region queries**: Genomic range queries with automatic partitioning
+- **VCF annotation**: Annotate VCF files with computed allele frequencies and sample genotypes
+- **Ploidy-aware**: Correct AN/AC computation for autosomes, chrX, chrY, and chrM
+- **Bitmap compression**: Roaring Bitmaps for efficient genotype storage
+- **In-process**: No server process—queries run locally with DuckDB
+- **Incremental updates**: Add new samples to existing databases
+- **Multiple genome builds**: Support for GRCh37 and GRCh38
+
+## Installation
+
+```bash
+pip install -e /home/dan/projects/afquery/ --break-system-packages
+```
+
+Requires Python 3.10+.
+
+## Quick Start
+
+### 1. Create a Database
+
+First, prepare a manifest TSV with sample metadata:
+
+```tsv
+sample_name	sex	tech_name	vcf_path	phenotype_codes
+sample_1	male	wgs	vcfs/sample_1.vcf	E11.9,I10
+sample_2	female	wes_kit_a	vcfs/sample_2.vcf	E11.9
+sample_3	male	wgs	vcfs/sample_3.vcf	I10
+```
+
+**Key points about the manifest:**
+- `tech_name`: Either `wgs` (case-insensitive) for whole genome, or a custom technology name
+- `vcf_path`: Path to the single-sample VCF file (relative to manifest directory, or absolute)
+- For WES/exome technologies, capture regions are loaded from `--bed-dir/{tech_name}.bed`
+  - Example: `tech_name=wes_kit_a` → loads `beds/wes_kit_a.bed`
+
+Organize your files:
+```
+project/
+├── manifest.tsv
+├── vcfs/
+│   ├── sample_1.vcf
+│   ├── sample_2.vcf
+│   └── sample_3.vcf
+└── beds/              # Required for non-WGS technologies
+    └── wes_kit_a.bed  # BED file for WES kit A
+```
+
+Then preprocess your VCF files:
+
+```bash
+afquery preprocess \
+  --manifest manifest.tsv \
+  --bed-dir ./beds/ \
+  --output-dir ./my_db/ \
+  --genome-build GRCh38
+```
+
+This creates:
+- `my_db/manifest.json` — database metadata
+- `my_db/metadata.sqlite` — samples, technologies, phenotype codes, precomputed bitmaps
+- `my_db/variants/{chrom}.parquet` — variant data with encoded genotypes
+- `my_db/capture/` — capture regions for each technology
+
+### 2. Query Allele Frequencies
+
+```bash
+# Point query
+afquery query --db my_db --chrom chr1 --pos 1000 --alt G
+
+# Batch query (100 positions)
+afquery query-batch --db my_db --positions positions.tsv --phenotype E11.9
+
+# Region query
+afquery query --db my_db --chrom chr1 --start 1000 --end 10000 --phenotype E11.9 --sex M
+```
+
+### 3. Annotate VCF Files
+
+```bash
+afquery annotate \
+  --db my_db \
+  --vcf input.vcf \
+  --output annotated.vcf \
+  --phenotype E11.9 \
+  --tech WGS
+```
+
+Adds `AFQUERY_AC`, `AFQUERY_AN`, `AFQUERY_AF` and genotype fields to your VCF.
+
+## Python API
+
+```python
+from afquery import Database
+
+db = Database("/path/to/db")
+
+# Single position query
+# Automatically filters samples by: sex + phenotype codes + capture coverage
+result = db.query(
+    chrom="chr1",
+    pos=1000,
+    alt="G",
+    phenotype_codes=["E11.9"],
+    sex="both"
+)
+print(f"AC={result.ac}, AN={result.an}, AF={result.af}")
+
+# Batch query (multi-variant)
+results = db.query_batch(
+    "chr1",
+    variants=[(1500, "A", "T"), (3500, "G", "C")],
+    phenotype=["E11.9"],
+)
+
+# Region query (genomic range)
+results = db.query_region(
+    chrom="chr1",
+    start=1000,
+    end=10000,
+    phenotype_codes=["E11.9", "I10"]
+)
+
+# Annotate VCF with allele frequencies
+# Note: tech_name filters annotation to samples of that technology
+db.annotate_vcf(
+    vcf_path="input.vcf",
+    output_path="annotated.vcf",
+    phenotype_codes=["E11.9"],
+    tech_name="wgs"  # Only annotate using WGS samples
+)
+```
+
+**How samples are filtered in queries**:
+- Sex filter: `male`, `female`, or `both`
+- phenotype filter: All codes must match
+- Capture filter: Automatic—only samples whose tech's BED covers the position
+
+## Database Structure
+
+```
+my_db/
+├── manifest.json          # Metadata: genome_build, sample_count, schema_version
+├── metadata.sqlite        # SQLite: samples, technologies, phenotype codes, bitmaps
+├── variants/
+│   ├── chr1.parquet
+│   ├── chr2.parquet
+│   └── ...
+└── capture/
+    ├── tech_0.pickle      # WGS capture region (always covered)
+    └── tech_1.pickle      # WES kit capture region
+```
+
+Each variant row contains:
+- `pos` — 1-based genomic position
+- `ref` — reference allele
+- `alt` — alternate allele
+- `het_bitmap` — Roaring Bitmap of heterozygous samples
+- `hom_bitmap` — Roaring Bitmap of homozygous samples
+
+## How Capture BED Files Are Associated with Samples
+
+Samples are linked to capture regions through their **technology**:
+
+1. **Manifest specifies technology**: Each sample lists `tech_name` (e.g., `wgs`, `wes_kit_a`)
+2. **Technology maps to BED file**:
+   - **WGS**: No BED file needed (always fully covered)
+   - **WES/Custom**: BED file loaded from `{bed_dir}/{tech_name}.bed`
+3. **Storage in database**:
+   - `metadata.sqlite::technologies` stores tech_id, tech_name, and bed_path
+   - `metadata.sqlite::samples` stores sample_id, sample_name, and tech_id (foreign key)
+4. **Query-time filtering**: When querying, samples are filtered by:
+   - Sex (male/female/both)
+   - phenotype diagnosis codes
+   - Capture region coverage (via tech's BED file)
+
+**Example**: If you have samples on two exome kits:
+```tsv
+sample_name	sex	tech_name	vcf_path	phenotype_codes
+S001	male	exome_v1	vcfs/S001.vcf	E11.9
+S002	female	exome_v1	vcfs/S002.vcf	E11.9
+S003	male	exome_v2	vcfs/S003.vcf	I10
+```
+
+Then provide:
+```
+beds/
+├── exome_v1.bed    # Coverage for samples S001, S002
+└── exome_v2.bed    # Coverage for sample S003
+```
+
+At query time, each sample's eligible regions are determined by its tech's BED file.
+
+## Advanced Features
+
+### Incremental Updates (add_samples)
+
+```bash
+afquery add-samples \
+  --db my_db \
+  --manifest new_samples.tsv \
+  --vcf-dir ./new_vcfs/
+```
+
+Adds new samples without rebuilding the entire database.
+
+### Compact Database
+
+Remove samples and reclaim disk space:
+
+```bash
+afquery compact --db my_db --samples-to-remove sample_1,sample_2
+```
+
+### Run Benchmarks
+
+```bash
+afquery benchmark --db my_db --n-queries 1000 --query-type point
+```
+
+### Generate Synthetic Data
+
+```bash
+afquery synth --output synthetic_db/ --n-samples 5000 --n-variants 100000
+```
+
+## Ploidy Rules
+
+AF computation respects chromosome-specific ploidy:
+
+| Chromosome | Formula |
+|---|---|
+| Autosomes | `AN = 2 × eligible_samples` |
+| chrM | `AN = 1 × eligible_samples` |
+| chrY | `AN = 1 × eligible_males` |
+| chrX (PAR) | `AN = 2 × eligible_samples` |
+| chrX (non-PAR) | `AN = 2 × eligible_females + 1 × eligible_males` |
+
+Where `eligible` = samples matching sex, phenotype, and technology capture filters.
+
+## Performance Targets
+
+- **Point query (cold)**: <100 ms
+- **Point query (warm)**: ~10 ms
+- **Batch 100 positions**: ~200 ms
+- **VCF annotation (5K variants)**: ~30 s
+- **VCF annotation (5M variants)**: ~30 min
+
+## Command Reference
+
+```
+afquery query                Query single position
+afquery query-batch          Batch query multiple positions
+afquery annotate             Annotate VCF file
+afquery info                 Show database info
+afquery preprocess           Build database from VCFs
+afquery add-samples          Add new samples to database
+afquery compact              Remove samples and reclaim space
+afquery synth                Generate synthetic test database
+afquery benchmark            Run performance benchmarks
+```
+
+Run `afquery --help` for full options.
+
+## Development
+
+### Running Tests
+
+```bash
+# All 190 tests
+python3 -m pytest --tb=short -q
+
+# Specific test module
+python3 -m pytest tests/test_query.py -v
+```
+
+### Key Modules
+
+- `afquery.query` — QueryEngine, point/batch/region queries
+- `afquery.annotate` — VCF annotation pipeline
+- `afquery.database` — Database wrapper (public API)
+- `afquery.preprocess` — Manifest parsing, VCF ingestion, Parquet building
+- `afquery.bitmaps` — Roaring Bitmap encoding/decoding
+- `afquery.ploidy` — Chromosome-specific ploidy rules
+- `afquery.models` — Data classes (QueryResult, ParsedSample, etc.)
+
+### Architecture
+
+See `brain/architecture.md` for detailed system design, data flow, and query algorithm.
+
+## Genome Builds
+
+- **GRCh37** (hg19) — PAR regions: chrX:1-2649520
+- **GRCh38** (hg38) — PAR regions: chrX:1-3099677
+
+## Technologies Supported
+
+- **WGS** — Whole genome sequencing (always fully covered, no BED file needed)
+  - Manifest: `tech_name = wgs` (case-insensitive)
+  - Query-time: All positions in genome considered covered
+
+- **WES** — Whole exome sequencing (coverage defined by BED file)
+  - Manifest: `tech_name = wes_kit_a` (or any custom name)
+  - Preprocessing: Loads `{bed_dir}/wes_kit_a.bed` (0-based half-open BED format)
+  - Query-time: Only positions within BED intervals considered covered
+
+- **Custom** — Any technology with a BED file (e.g., gene panels, targeted sequencing)
+  - Manifest: Use any `tech_name`
+  - Preprocessing: Loads `{bed_dir}/{tech_name}.bed`
+  - Query-time: Respects BED file coverage
+
+## Troubleshooting
+
+### ImportError: `cyvcf2`
+
+`cyvcf2` import happens inside worker processes during preprocessing. Do not import at module level.
+
+### DuckDB Temp Files
+
+Uses Parquet format (not Arrow IPC) for compatibility. Set `DUCKDB_TEMP_DIRECTORY` if needed.
+
+### Sample IDs
+
+Sample IDs are 0-indexed and monotonically increasing. Never reuse removed IDs—use `compact` to reclaim space.
+
+## Contributing
+
+1. Read `brain/project_state.json` for current phase and test count
+2. Read `brain/architecture.md` for system design
+3. Follow code conventions in `CLAUDE.md`
+4. Update `brain/` docs after architectural changes
+5. Run tests before submitting
+
+## License
+
+(Add your license here)
+
+## Citation
+
+If you use afquery in research, please cite:
+
+```
+(Citation format to be determined)
+```
+
+---
+
+**Status**: Phase 5 complete (190 tests passing). Active development.

afquery-0.1.0/docs/webhook_schema.md

@@ -0,0 +1,106 @@
+# Webhook / API JSON Schema
+
+## Batch Query
+
+### Request
+
+```json
+{
+  "chrom":     "chr1",
+  "positions": [1000, 2000, 3000],
+  "phenotype":     ["E11.9", "I10"],
+  "sex":       "both"
+}
+```
+
+| Field       | Type            | Required | Notes                                      |
+|-------------|-----------------|----------|--------------------------------------------|
+| `chrom`     | string          | yes      | Any form accepted: `"1"`, `"chr1"`, `"X"` |
+| `positions` | array of int    | yes      | 1-based positions; duplicates are ignored  |
+| `phenotype`     | array of string | yes      | phenotype codes; union semantics              |
+| `sex`       | string          | no       | `"both"` (default), `"male"`, `"female"`  |
+
+### Response
+
+Array of variant objects, one per variant found (absent variants are omitted). Sorted by `(pos, alt)`.
+
+```json
+[
+  {
+    "chrom":      "chr1",
+    "pos":        1000,
+    "ref":        "A",
+    "alt":        "T",
+    "AC":         5,
+    "AN":         100,
+    "AF":         0.05,
+    "n_eligible": 50
+  }
+]
+```
+
+| Field        | Type   | Notes                                        |
+|--------------|--------|----------------------------------------------|
+| `chrom`      | string | Canonical form (`chr`-prefixed)              |
+| `pos`        | int    | 1-based                                      |
+| `ref`        | string | Reference allele                             |
+| `alt`        | string | Alternate allele (one object per alt)        |
+| `AC`         | int    | Allele count in eligible samples             |
+| `AN`         | int    | Allele number (ploidy-aware)                 |
+| `AF`         | float  | `AC / AN`                                    |
+| `n_eligible` | int    | Number of eligible samples at this position  |
+
+Positions where `AN == 0` (no eligible samples covered) are silently omitted.
+
+---
+
+## Single-Position Query
+
+Same as batch query with a single-element `positions` array.
+
+---
+
+## VCF Annotation
+
+### Request
+
+`POST /annotate`  — multipart form data.
+
+| Part     | Content-Type               | Description                       |
+|----------|----------------------------|-----------------------------------|
+| `vcf`    | `application/octet-stream` | Input VCF file (plain or gzipped) |
+| `params` | `application/json`         | JSON object (see below)           |
+
+`params` object:
+
+```json
+{
+  "phenotype": ["E11.9"],
+  "sex":   "both"
+}
+```
+
+### Response
+
+Annotated VCF file stream (`Content-Type: text/plain`).
+
+Added INFO fields:
+
+| Field      | Number | Type    | Description                          |
+|------------|--------|---------|--------------------------------------|
+| `AFQUERY_AC`  | A      | Integer | Allele count per ALT in eligible set |
+| `AFQUERY_AN`  | 1      | Integer | Allele number (0 if uncovered)       |
+| `AFQUERY_AF`  | A      | Float   | Allele frequency per ALT             |
+
+**Notes**:
+- `AFQUERY_AN=0` means the position was not covered for the given filters; `AFQUERY_AC` and `AFQUERY_AF` are absent (`.`) in that case.
+- `AFQUERY_AC=0`, `AFQUERY_AF=0.0` with `AFQUERY_AN>0` means the position was covered but the specific allele was not observed in the database.
+- For multi-allelic sites, `AFQUERY_AC` and `AFQUERY_AF` are comma-separated lists aligned to the `ALT` column.
+
+### Completion summary (trailing header)
+
+After the VCF body, a summary is written to stderr (or returned in a trailing `X-AFQUERY-Stats` response header):
+
+```json
+{"n_variants": 1000, "n_annotated": 850, "n_uncovered": 12}
+```