ldetect-lite 0.1.0__tar.gz

ldetect_lite-0.1.0/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

ldetect_lite-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,47 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: astral-sh/setup-uv@v8.2.0
+        with:
+          python-version: "3.12"
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --extra dev --extra heatmap
+
+      - name: Lint (ruff)
+        run: uv run ruff check src tests
+
+      - name: Type-check (mypy)
+        run: uv run mypy src
+
+      - name: Run unit tests
+        run: uv run pytest -m "not integration"
+
+  publish:
+    needs: test
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for trusted publishing (OIDC)
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: astral-sh/setup-uv@v8.2.0
+
+      - name: Build distributions
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

ldetect_lite-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,50 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: astral-sh/setup-uv@v8.2.0
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --extra dev --extra heatmap
+
+      - name: Lint (ruff)
+        run: uv run ruff check src tests
+
+      - name: Type-check (mypy)
+        run: uv run mypy src
+
+      - name: Run unit tests
+        run: uv run pytest -m "not integration"
+
+      - name: Run integration tests
+        run: uv run pytest -m integration
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: astral-sh/setup-uv@v8.2.0
+        with:
+          enable-cache: true
+
+      - name: Build sdist and wheel
+        run: uv build

ldetect_lite-0.1.0/.gitignore

@@ -0,0 +1,203 @@
+**/.DS_Store
+**/.vscode
+_reference
+tests/data/
+data/
+**/.snakemake/
+examples/*/results/
+examples/*/data/
+examples/*/ref/
+examples/*/resources/
+examples/*/work/
+
+# Large downloaded inputs and optional legacy toy data for replication examples.
+examples/ldetect_original/**/*.vcf.gz
+examples/ldetect_original/**/*.vcf.gz.tbi
+examples/ldetect_original/**/*.interpolated_genetic_map.gz
+examples/ldetect_original/scripts/legacy_ldetect/ldetect/examples/example_data/
+
+# Downloaded copyrighted supplementary material (Berisa & Pickrell 2016) --
+# useful locally for provenance research, must not be redistributed via git.
+examples/ldetect_original/document-sup.pdf
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor  
+#  Cursor is an AI-powered code editor.`.cursorignore` specifies files/directories to 
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore

ldetect_lite-0.1.0/AGENTS.md

@@ -0,0 +1,117 @@
+# AGENTS.md
+
+This file provides guidance to coding agents when working with code in this repository.
+
+## Project Overview
+
+**ldetect-lite** is a modern reimplementation of [ldetect](https://bitbucket.org/nygcresearch/ldetect), a bioinformatics tool that calculates approximately independent linkage disequilibrium (LD) blocks in the human genome. The algorithm is described in [Berisa & Pickrell, 2016](https://academic.oup.com/bioinformatics/article/32/2/283/1743626).
+
+The complete reference implementation lives in `_reference/ldetect_original/ldetect/` (also `_reference/LDblocks_GRCh38/` for the MacDonald et al. 2022 GRCh38 reproduction). The new implementation is in `src/ldetect_lite/`.
+
+## Commands
+
+```bash
+# Install in editable mode (uv recommended)
+uv sync
+
+# Install with heatmap support (matplotlib, required for --generate-heatmap)
+uv sync --extra heatmap
+
+# Install dev extras (pytest, ruff, mypy, snakemake — needed for examples/)
+uv sync --extra dev
+
+# Run unit tests only (fast)
+uv run pytest -m "not integration"
+
+# Run a single test
+uv run pytest tests/test_filters.py::test_unimodal_single_minimum -v
+
+# Run integration tests (downloads ~5 files from BitBucket on first run, cached to tests/data/)
+uv run pytest -m integration
+
+# Run all tests
+uv run pytest
+
+# Lint / type-check
+uv run ruff check src tests
+uv run mypy src
+
+# CLI entry point
+uv run ldetect --help
+```
+
+## Architecture
+
+The package is under `src/ldetect_lite/`. Key modules:
+
+| Module                        | Role                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `shrinkage.py`                | Steps 1–2: `partition_chromosome` and `calc_covariance` (Wen/Stephens shrinkage LD estimator). The pairwise LD kernel (`_pairwise_ld_impl`) is JIT-compiled with Numba when available (~50x speedup), falling back to pure Python. Writes indexed HDF5 covariance partitions.                                                                                                                                                      |
+| `matrix_analysis.py`          | Step 3: `MatrixAnalysis` class — reduces covariance partitions to a `[position, diagonal_sum]` vector. `calc_diag_lean` (dictionary path) and `calc_diag_array` (array-backed path, via `_util/vector_array.py`) are the two production paths; `calc_diag`/`generate_img` support the `--generate-heatmap` PNG output and require full-schema partitions.                                                                          |
+| `filters.py`                  | Hanning-window convolution (`np.hanning`, symmetric) and `scipy.signal.argrelextrema` minima extraction                                                                                                                                                                                                                                                                                                                            |
+| `find_minima.py`              | Step 4 core: `FlexibleBoundedAccessor`, binary search for optimal filter width, `custom_binary_search_with_trackback`                                                                                                                                                                                                                                                                                                              |
+| `metric.py`                   | `Metric` class — sum of squared correlations across blocks. Float by default; `use_decimal=True` opts into 50-digit `decimal.Decimal` precision (dictionary path) instead of the array-backed path.                                                                                                                                                                                                                                |
+| `local_search.py`             | `LocalSearch` — refines each breakpoint by local search. Array-backed path (`_init_search_array`/`_search_array`) is used for single-partition windows only; multi-partition windows and `--high-precision` fall back to the legacy dictionary/Decimal path (`init_search`/`search`) because the array path does not yet reproduce the legacy multi-partition effective locus list (see `notes/logs/local-search-divergence-asn22.md`). |
+| `pipeline.py`                 | `find_breakpoints` — orchestrates steps 4–5: binary search for filter width, minima extraction, metric computation, local search, JSON output with four subsets (`fourier`, `fourier_ls`, `uniform`, `uniform_ls`). Supports computing only requested subsets plus their local-search dependencies, and reuses an in-memory `ChromosomeCovariance` cache across metric calls where possible.                                       |
+| `interpolate_maps.py`         | `interpolate()` — maps SNP positions to genetic positions via point-based linear interpolation (point-sampled maps). `interpolate_intervals()` — port of MacDonald et al.'s R interval-rate algorithm, for interval-rate maps (e.g. deCODE); selected via `ldetect interpolate-maps --mode {point,interval}` (see `notes/findings/macdonald2022-reproduction.md`)                                                                 |
+| `io/partitions.py`            | `CovarianceStore` dataclass (replaces reference's global `flat_file_consts.py` dict), `read_partitions`, `relevant_subpartitions`, `first_last`, `get_final_partitions`                                                                                                                                                                                                                                                            |
+| `io/covariance.py`            | Legacy dictionary-based covariance matrix I/O: `insert_into_matrix_lean`, `read_partition_into_matrix_lean`, `delete_loci_*`, `write_corr_vector`                                                                                                                                                                                                                                                                                  |
+| `io/covariance_hdf5.py`       | Indexed HDF5 partition I/O: `write_covariance_partition_hdf5` (full schema), `write_compact_covariance_partition_hdf5_chunks`/`_append` (compact, streamed), `HDF5CovariancePartitionReader`, `open_covariance_reader`, `validate_covariance_hdf5`                                                                                                                                                                                 |
+| `io/bed.py`                   | `write_bed`                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| `io/vcf.py`                   | `read_vcf_samples` — sample IDs via `bcftools query -l`                                                                                                                                                                                                                                                                                                                                                                            |
+| `_util/binary_search.py`      | `find_le/ge/lt/gt` and `*_ind` variants wrapping `bisect`                                                                                                                                                                                                                                                                                                                                                                          |
+| `_util/covariance_array.py`   | Array-backed covariance loading for metrics/local search: `ChromosomeCovariance`, `CovariancePartition`, `load_covariance_arrays`, `load_chromosome_covariance`, `metric_from_arrays`, `metric_from_files` (parallel row-pass streaming metric via `--metric-workers`)                                                                                                                                                             |
+| `_util/vector_array.py`       | Array-backed matrix-to-vector implementation used by `calc_diag_array`; streams and chunks HDF5 partitions to bound memory                                                                                                                                                                                                                                                                                                         |
+| `_util/covariance_summary.py` | `summarize_covariance` — per-partition row counts and estimated covariance-array memory, backing `ldetect covariance-summary`                                                                                                                                                                                                                                                                                                     |
+| `_util/memory.py`             | `current_rss_mib`/`max_rss_mib`/`log_memory_checkpoint` — process RSS diagnostics logged at pipeline checkpoints (debug logging)                                                                                                                                                                                                                                                                                                   |
+| `_util/intervals.py`          | Interval helpers                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| `_util/logging.py`            | `log_msg`/`log_debug`/`configure_logging` wrapping stdlib logging (replaces `print_log_msg`)                                                                                                                                                                                                                                                                                                                                       |
+| `_cli/`                       | argparse subcommands: `partition-chromosome`, `calc-covariance`, `covariance-summary`, `matrix-to-vector`, `find-minima`, `extract-bpoints`, `interpolate-maps`, `run`; global `-v/--verbosity {debug,info,warning,error}` flag handled in `main.py`                                                                                                                                                                               |
+
+## Key Design Decisions
+
+- **`CovarianceStore`** (frozen dataclass) replaces the reference's `flat_file_consts.py` global path-config dict. Every function that previously took `input_config: dict` now takes `store: CovarianceStore`.
+- **Numba JIT**: `_pairwise_ld_impl` in `shrinkage.py` is decorated with `@_jit` — a decorator defined as `njit(cache=True)` when Numba is available, or a no-op otherwise. The inner k loop was replaced with `np.sum(a * b)` / `np.sum(a)` / `np.sum(b)` to let LLVM vectorize (~50x speedup over pure Python).
+- **Parallel covariance**: `ldetect run --workers N` uses `concurrent.futures.ProcessPoolExecutor` to calculate covariance matrices for multiple partitions concurrently. Each partition is independent (unique output file, no shared state). The tabix spawn + `calc_covariance` call is extracted into a module-level `_calc_partition` function in `_cli/cmd_run.py` so it is picklable by the process pool. `--workers` also sets the default for the Step 3 (`--matrix-workers`) and Step 4 (`--local-search-workers`, `--metric-workers`) process pools, which run strictly after Step 2 finishes (never concurrently with it or each other), so a single `--workers N` parallelizes the whole pipeline unless a stage-specific override is passed.
+- **Pickle → JSON**: Intermediate breakpoint output uses `.json` (not `.pickle`) for portability.
+- **Flat file → indexed HDF5**: Covariance partitions are stored as indexed HDF5 (`.h5`), not the reference's gzipped space-delimited flat files. `full` schema (naive LD, genetic positions, SNP IDs) supports `--generate-heatmap` and debugging; `compact` schema (canonical `lo`/`hi` pairs, `shrink_ld`, diagonal entries, lookup indexes) is the `ldetect run` default and supports restartable production runs with bounded memory.
+- **Array-backed vs. dictionary paths**: `matrix_analysis.py`, `metric.py`, and `local_search.py` each have a legacy dictionary-based path (ported near-verbatim from the reference, precise but slow) and a newer NumPy array-backed path (faster, vectorized). Float-mode single-partition local search and normal-float metrics use the array path by default; `--high-precision` (Decimal) and multi-partition local-search windows use the dictionary path because the array-backed local search does not yet reproduce the legacy multi-partition effective locus list exactly. Treat `--high-precision` as the correctness oracle when investigating local-search discrepancies.
+- **Float arithmetic by default**: The reference used 50-digit `decimal.Decimal` for all breakpoint metric/local-search arithmetic (~10–30x slower than float, with no practical difference for typical LD data). `float` is now the default; `--high-precision` opts back into Decimal.
+- **Symmetric Hann window**: `filters.py` uses `np.hanning(2*width+1)` — the symmetric variant. `scipy.signal.get_window('hann', N)` (default `fftbins=True`) is a periodic DFT window and gives different results.
+- **Test data**: Downloaded on first run from BitBucket raw URLs into `tests/data/` (gitignored). Session-scoped fixtures in `tests/conftest.py` handle this.
+
+## Input Data Formats
+
+- **Genetic map**: gzipped TSV, columns: `chr  position  genetic_position_cM`
+- **Covariance matrix**: indexed HDF5 partition file (`.h5`); `full` schema also readable/writable as the reference's gzipped space-delimited 8-column flat format (`i_id  j_id  i_pos  j_pos  i_gpos  j_gpos  naive_ld  shrink_ld`) via the legacy `io/covariance.py` path
+- **Partition file**: space-delimited `start end` pairs, one per line
+- **Reference panel**: phased VCF piped via `tabix`
+- **Breakpoint JSON** (from `find-minima`): `{"n_bpoints", "found_width", "computed_subsets", "skipped_subsets", "fourier"/"fourier_ls"/"uniform"/"uniform_ls": {"loci": [...], "metric": {"sum", "N_nonzero", "N_zero"}}}`. `metric.sum`/`N_zero` are stored as strings to preserve precision (relevant when `--high-precision` produces Decimal values).
+
+## Repository Layout Beyond `src/`
+
+- `docs/` — human-facing reference documentation: `docs/pipeline-steps.md` (per-command CLI reference for running each pipeline stage individually) and `docs/optimizations.md` (up-to-date human-readable summary of implemented performance work).
+- `notes/` — internal working notes, split into `notes/findings/` (distilled, current-status summaries of reproduction investigations, meant to be read and mined for the paper writeup) and `notes/logs/` (raw, dated, agent-oriented investigation logs and historical design docs — see `notes/README.md` for the full index). `notes/logs/ldetect-original-*.md` and `notes/logs/local-search-divergence-asn22.md` back the ldetect_original reproduction status; `notes/logs/macdonald2022-*.md` back the MacDonald2022 reproduction status (see Reproduction Status below).
+- `examples/` — Snakemake reproduction workflows, each with its own `README.md`/`config.yaml`:
+  - `ldetect_example/` — toy EUR chr2 example bundled with the original ldetect, used as a correctness fixture.
+  - `ldetect_original/` — full genome-wide reproduction of the published Berisa & Pickrell EUR/AFR/ASN LD blocks from public 1000 Genomes Phase 1 VCFs, plus diagnostic Snakefiles (`Snakefile.diagnostics`, `Snakefile.legacy_diagnostics`, `Snakefile.provenance_diagnostics`) used to isolate divergences from the reference.
+  - `MacDonald2022/` — reproduction of MacDonald et al. (2022) GRCh38 LD blocks (deCODE map + pyrho superpopulation maps).
+  - `r2_zarr_exactness/` — Zarr-based exactness checks.
+- `benchmarks/` — standalone perf benchmarks (e.g. `bench_ld_kernel.py` for the Numba LD kernel, referenced in `docs/optimizations.md`).
+- `_reference/` — vendored reference implementations used for parity testing and porting: `ldetect_original/ldetect/` (original Berisa & Pickrell code) and `LDblocks_GRCh38/` (MacDonald et al. 2022 scripts).
+
+## Reproduction Status
+
+Tracked in detail in `notes/logs/ldetect-original-main-pipeline-audit.md` (log of record) and `notes/findings/ldetect-original-reproduction.md` (current-state snapshot — start here). As of the latest full genome-wide run against the published Bitbucket `ldetect-data` reference BEDs:
+
+- **ASN**: exact match, all 22 autosomes.
+- **AFR**: exact except chr22. (chr11 was previously tracked as divergent too, but is resolved — a data-corruption bug in the published reference BED itself, not a real pipeline divergence.)
+- **EUR**: exact block counts and coverage on every chromosome, but chr8–12 have shifted internal boundaries relative to the published reference (contiguous range bracketed by exact chr7/chr13 matches).
+
+EUR chr8-12 and AFR chr22 are **parked as an accepted, documented residual divergence**, not an active investigation. An extensive diagnostic effort ruled out VCF release-version provenance (including a full pipeline rerun on 4 alternate releases plus one undocumented snapshot), SNP filtering, genetic map family, `Ne` assignment, duplicate/cross-partition handling, sample/panel provenance, and reference-BED structural integrity. The leading remaining hypothesis is an unidentified upstream input/provenance divergence for those chromosomes specifically (precision and legacy-downstream diagnostics rule out arbitrary implementation-choice ambiguity as the cause) — see `notes/findings/ldetect-original-reproduction.md`'s "If this is picked up again" section for the concrete next steps if this is ever revisited.
+
+When investigating reproduction mismatches, read `notes/findings/ldetect-original-reproduction.md` first; it likely already rules out several hypotheses. MacDonald2022 reproduction status is tracked equivalently in `notes/findings/macdonald2022-reproduction.md` (summary) and `notes/logs/macdonald2022-pyrho-handoff.md`/`notes/logs/macdonald2022-interpolation-port.md` (logs of record).
+
+## CLI Subcommands
+
+`partition-chromosome`, `calc-covariance`, `covariance-summary`, `matrix-to-vector`, `find-minima`, `extract-bpoints`, `interpolate-maps`, `run`. Full argument documentation and worked examples for each are in `README.md` — that is the source of truth for CLI usage; keep it in sync with `_cli/cmd_*.py` when changing flags.

ldetect_lite-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Adam Youlin He
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ldetect_lite-0.1.0/PKG-INFO

@@ -0,0 +1,172 @@
+Metadata-Version: 2.4
+Name: ldetect-lite
+Version: 0.1.0
+Summary: Fast calculation of approximately independent LD blocks
+Author: Adam He
+License: MIT License
+        
+        Copyright (c) 2026 Adam Youlin He
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: GWAS,LD,bioinformatics,linkage-disequilibrium
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
+Requires-Python: >=3.11
+Requires-Dist: h5py>=3.10
+Requires-Dist: hdf5plugin>=4.4
+Requires-Dist: numba>=0.57
+Requires-Dist: numpy>=1.24
+Requires-Dist: scipy>=1.10
+Provides-Extra: dev
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: pyyaml>=6.0; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Requires-Dist: snakemake-executor-plugin-slurm; extra == 'dev'
+Requires-Dist: snakemake>=8.0; extra == 'dev'
+Provides-Extra: heatmap
+Requires-Dist: matplotlib>=3.7; extra == 'heatmap'
+Provides-Extra: profiling
+Requires-Dist: psutil>=5.9; extra == 'profiling'
+Description-Content-Type: text/markdown
+
+# LDetect-lite
+
+[![PyPI](https://img.shields.io/pypi/v/ldetect-lite)](https://pypi.org/project/ldetect-lite/)
+[![Tests](https://github.com/adamyhe/ldetect2/actions/workflows/tests.yml/badge.svg)](https://github.com/adamyhe/ldetect2/actions/workflows/tests.yml)
+[![PyPI Downloads](https://static.pepy.tech/personalized-badge/ldetect-lite?period=total&units=INTERNATIONAL_SYSTEM&left_color=BLACK&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/ldetect-lite)
+
+A modern, fast re-implementation of [LDetect](https://bitbucket.org/nygcresearch/ldetect), a method for calculating approximately independent linkage disequilibrium (LD) blocks in the human genome. The algorithm is described in [Berisa & Pickrell, 2016](https://academic.oup.com/bioinformatics/article/32/2/283/1743626).
+
+## Installation
+
+Install from PyPI via:
+
+```bash
+pip install ldetect-lite
+```
+
+Or, with [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv add ldetect-lite
+```
+
+This installs three equivalent CLI entry points — `ldetect-lite`, `ldetect`, and `ldl` — so pick whichever is most convenient; examples below use `ldetect`.
+
+The main `ldetect run` pipeline also requires [htslib](https://www.htslib.org/). Specifically, `tabix` is used to stream VCF files to `ldetect calc-covariance`, and so must be on PATH.
+
+**Optional** (`--generate-heatmap`): install matplotlib with `pip install "ldetect-lite[heatmap]"`, or use `uv sync --extra heatmap` from a source checkout. Generating covariance heatmaps requires a matplotlib install.
+
+### Development
+
+Install from source
+
+```bash
+git clone https://github.com/adamyhe/ldetect-lite.git
+cd ldetect-lite
+uv sync --extra dev
+```
+
+From a development checkout, run CLI commands through `uv run` so they use the managed environment.
+
+## Usage
+
+### End-to-end pipeline
+
+```bash
+uv run ldetect run \
+  --genetic-map chr2.interpolated_genetic_map.gz \
+  --reference-panel 1000G.chr2.vcf.gz \
+  --individuals eurinds.txt \
+  --chromosome chr2 \
+  --output-dir results/chr2/
+```
+
+This writes `results/chr2/chr2-ld-blocks.bed` — a BED file of approximately independent LD blocks.
+
+Global options (before the subcommand):
+
+- `-v / --verbosity {debug,info,warning,error}` — logging verbosity (default: `info`; use `warning` to silence progress messages, `debug` for full detail)
+
+Options:
+
+- `--ne FLOAT` — effective population size Ne used by the Wen & Stephens shrinkage estimator (default: 11418.0, the CEU/HapMap II value; reproduction configs may override this for non-European populations)
+- `--cov-cutoff FLOAT` — LD pairs with absolute shrinkage correlation below this threshold are not written to disk, reducing storage (default: 1e-7)
+- `--covariance-cache {compact,full}` — partition cache schema for `ldetect run` (default: `compact`). Compact caches write only canonical position pairs, `shrink_ld`, diagonals, and lookup indexes, which is enough for restartable matrix-to-vector, metric, and local-search steps. Use `full` when debugging or when later running full-matrix/heatmap readers.
+- `--covariance-compression {lzf,zstd}` — HDF5 compression codec for covariance partitions (default: `zstd`). `zstd` is smaller and faster to read/write than `lzf` at equal precision — see `docs/optimizations.md`.
+- `--n-snps-bw-bpoints N` — target mean number of SNPs between consecutive breakpoints; controls block granularity (default: 10000, following Berisa & Pickrell 2016). The target breakpoint count is `ceil(n_snps / N - 1)`. Mutually exclusive with `--n-bpoints`.
+- `--n-bpoints N` — directly specify the number of breakpoints, bypassing the `--n-snps-bw-bpoints` formula; useful when replicating a published analysis with a known block count
+- `--subset {fourier,fourier_ls,uniform,uniform_ls}` — which of the four breakpoint sets to write to the BED file (default: `fourier_ls`; see `docs/pipeline-steps.md` step 4)
+- `--all-breakpoint-subsets` — compute all four breakpoint sets in the JSON output. By default, `run` computes only the requested `--subset` and its dependencies to avoid unused local-search work.
+- `--workers N` — parallel workers for the pipeline (default: 1); set to the number of available cores to speed up covariance calculation (step 2) and, unless overridden below, matrix-to-vector, local search, and metric scoring as well
+- `--matrix-workers N` — override parallel workers for matrix-to-vector partition processing (default: inherit `--workers`)
+- `--local-search-workers N` — override parallel workers for local search (default: inherit `--workers`). Higher values can multiply RAM use because each worker loads its own covariance window.
+- `--metric-workers N` — override parallel workers for streaming metric row passes during breakpoint scoring (default: inherit `--workers`)
+- `--high-precision` — use 50-digit Decimal arithmetic for local search instead of the default float path (slower; mainly useful for exact reference comparisons)
+
+Each of the five stages (partition, covariance, matrix-to-vector, find-minima, extract-bpoints) can also be run individually, along with a `covariance-summary` inspection utility — see `docs/pipeline-steps.md`.
+
+### Interpolate genetic maps
+
+Convert a recombination rate map (e.g. the [deCODE map](https://www.science.org/doi/10.1126/science.aau1043) or [HapMap-interpolated 1000G maps](https://github.com/joepickrell/1000-genomes-genetic-maps)) to per-SNP genetic positions required by steps 1 and 2:
+
+```bash
+uv run ldetect interpolate-maps \
+  --snp-file snps.bed.gz \
+  --genetic-map recombination_map.gz \
+  --output chr2.interpolated_genetic_map.gz
+```
+
+Arguments:
+- `--snp-file PATH` — bgzipped BED file of SNP positions (columns: `chrom start end rs_id`); typically extracted from a filtered VCF with `bcftools query -f '%CHROM\t%POS0\t%POS\t%ID\n'`
+- `--genetic-map PATH` — gzipped recombination map; interpolation is used to assign a cM value to each SNP position
+- `--output PATH` — gzipped output map in the 3-column format expected by steps 1 and 2 (`rs_id  position  cM`)
+- `--mode {point,interval}` (default: `point`) — interpolation algorithm:
+  - `point` — treats `--genetic-map` as discrete `(position, cM)` points and linearly interpolates between the two points bracketing each SNP. Correct for point-sampled maps (e.g. HapMap-interpolated 1000G maps).
+  - `interval` — treats each map row as the start of a genomic interval with its own recombination rate (`Begin, rate_cM_Mb, cumulative_cM_at_End`), matching MacDonald et al.'s R interpolation scripts ([`interpolate.R`](https://github.com/jmacdon/LDblocks_GRCh38/blob/master/scripts/interpolate.R)/[`interpolate_pyhro.R`](https://github.com/jmacdon/LDblocks_GRCh38/blob/master/scripts/interpolate_pyhro.R)). Required for interval-rate maps such as the deCODE map — feeding those into `point` mode silently uses the *next* interval's rate for SNPs in the *current* interval, an off-by-one bug that produced a ~0.001–0.003 cM error per SNP in earlier testing (see `notes/findings/macdonald2022-reproduction.md`).
+
+## Algorithm
+
+The pipeline detects LD block boundaries by finding local minima in a smoothed diagonal-sum signal derived from the shrinkage LD covariance matrix:
+
+1. **Partition** — chromosome split into ~5000-SNP overlapping windows at low-recombination boundaries
+2. **Covariance** — Wen & Stephens shrinkage estimator applied to phased haplotypes; shrinks sample correlations toward the expected LD decay to reduce finite-sample noise
+3. **Matrix → vector** — each covariance matrix reduced to a `[position, diagonal_sum]` signal; troughs correspond to LD block boundaries
+4. **Find minima** — binary search for optimal Hanning-window filter width; `scipy.signal.argrelextrema` finds local minima; local search refines each breakpoint using sum of squared inter-block correlations as the quality metric
+5. **Extract** — chosen breakpoint set written as BED
+
+The available breakpoint sets are `fourier` and `uniform` (raw minima from Fourier-filtered and uniformly-spaced candidates), plus `fourier_ls` and `uniform_ls` (after local search refinement). `fourier_ls` is the recommended output.
+
+## Known limitations
+
+`ldetect-lite` reproduces the published Berisa & Pickrell (2016) 1000 Genomes LD blocks exactly for ASN (all 22 autosomes) and AFR (all chromosomes except chr22), and matches EUR block counts and coverage exactly but with shifted internal boundaries on chr8–chr12. These two residual divergences (EUR chr8-12, AFR chr22) are understood to stem from an unidentified upstream input/provenance difference from the original authors' pipeline, not a bug in this implementation — an extensive diagnostic effort ruled out VCF release-version provenance, SNP filtering, genetic map family, `Ne` assignment, duplicate/cross-partition handling, and reference-BED integrity as causes. See `notes/findings/ldetect-original-reproduction.md` for the full writeup, and `notes/findings/macdonald2022-reproduction.md` for the equivalent status reproducing MacDonald et al. (2022)'s GRCh38 blocks.
+
+## Pre-computed LD blocks
+
+Pre-computed BED files for 1000 Genomes reference populations are available from in hg19 coordinates from the [original LDetect data repository](https://bitbucket.org/nygcresearch/ldetect-data) and in hg38 coordinates from a more recent effort by [MacDonald et al. (2022)](https://www.biorxiv.org/content/10.1101/2022.03.04.483057v2).
+
+BED files produced by our work will be released once the code base leaves alpha/finish major breaking updates.