migec 2.2.1__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- migec-2.4.0/.github/workflows/benchmark.yml +107 -0
- {migec-2.2.1 → migec-2.4.0}/.github/workflows/publish.yml +6 -1
- {migec-2.2.1 → migec-2.4.0}/.gitignore +4 -0
- {migec-2.2.1 → migec-2.4.0}/CHANGELOG.md +303 -1
- {migec-2.2.1 → migec-2.4.0}/CLAUDE.md +152 -26
- migec-2.4.0/PKG-INFO +249 -0
- migec-2.4.0/README.md +207 -0
- {migec-2.2.1 → migec-2.4.0}/ROADMAP.md +119 -31
- {migec-2.2.1 → migec-2.4.0}/SOURCES.md +122 -2
- migec-2.4.0/assets/collision_split.tsv +6 -0
- migec-2.4.0/assets/ctdna_minreads.tsv +206 -0
- migec-2.4.0/assets/ctdna_panel.bed +6 -0
- migec-2.4.0/assets/ctdna_per_target.tsv +73 -0
- migec-2.4.0/assets/grouping_tools.tsv +22 -0
- migec-2.4.0/assets/migec_v1.tsv +5 -0
- migec-2.4.0/assets/shallow_repertoire.tsv +5 -0
- migec-2.4.0/docs/assays.rst +142 -0
- {migec-2.2.1 → migec-2.4.0}/docs/assemble.rst +53 -1
- {migec-2.2.1 → migec-2.4.0}/docs/barcode_space.rst +5 -0
- {migec-2.2.1 → migec-2.4.0}/docs/checkout.rst +102 -0
- {migec-2.2.1 → migec-2.4.0}/docs/conf.py +1 -1
- migec-2.4.0/docs/detection.rst +633 -0
- {migec-2.2.1 → migec-2.4.0}/docs/examples.rst +13 -6
- {migec-2.2.1 → migec-2.4.0}/docs/formats.rst +23 -3
- migec-2.4.0/docs/grouping.rst +297 -0
- {migec-2.2.1 → migec-2.4.0}/docs/index.rst +15 -2
- {migec-2.2.1 → migec-2.4.0}/docs/layouts.rst +15 -0
- {migec-2.2.1 → migec-2.4.0}/docs/performance.rst +47 -6
- {migec-2.2.1 → migec-2.4.0}/docs/quality_floor.rst +87 -3
- {migec-2.2.1 → migec-2.4.0}/docs/refine.rst +53 -7
- {migec-2.2.1 → migec-2.4.0}/docs/roadmap.rst +3 -2
- migec-2.4.0/docs/validation.rst +248 -0
- {migec-2.2.1 → migec-2.4.0}/docs/variants.rst +102 -40
- {migec-2.2.1 → migec-2.4.0}/include/migec/assemble.hpp +14 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/checkout.hpp +124 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/consensus.hpp +14 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/mig_record.hpp +22 -2
- {migec-2.2.1 → migec-2.4.0}/include/migec/parallel.hpp +20 -1
- {migec-2.2.1 → migec-2.4.0}/include/migec/pattern.hpp +3 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/refine.hpp +40 -6
- {migec-2.2.1 → migec-2.4.0}/include/migec/subsample.hpp +5 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/types.hpp +48 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/umi_stats.hpp +209 -28
- {migec-2.2.1 → migec-2.4.0}/include/migec/version.hpp +1 -1
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/migec/assemble/main.nf +1 -1
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/migec/checkout/main.nf +1 -1
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/migec/plot/main.nf +1 -1
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/migec/refine/main.nf +1 -1
- {migec-2.2.1 → migec-2.4.0}/notebooks/exome_capture.py +26 -0
- {migec-2.2.1 → migec-2.4.0}/pyproject.toml +12 -1
- {migec-2.2.1 → migec-2.4.0}/python/migec/__init__.py +1 -1
- migec-2.4.0/python/migec/assemble.py +500 -0
- migec-2.4.0/python/migec/buckets.py +61 -0
- {migec-2.2.1 → migec-2.4.0}/python/migec/checkout.py +106 -11
- {migec-2.2.1 → migec-2.4.0}/python/migec/cli.py +87 -12
- {migec-2.2.1 → migec-2.4.0}/python/migec/refine.py +24 -5
- {migec-2.2.1 → migec-2.4.0}/python/migec/sheet.py +190 -0
- {migec-2.2.1 → migec-2.4.0}/python/migec/subsample.py +4 -2
- migec-2.4.0/scripts/blind_artifact_filter.py +154 -0
- migec-2.4.0/scripts/collision_split.py +110 -0
- migec-2.4.0/scripts/compare_grouping.py +244 -0
- migec-2.4.0/scripts/compare_migec_v1.py +182 -0
- migec-2.4.0/scripts/ctdna_persite.py +180 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/ctdna_titration.py +15 -0
- migec-2.4.0/scripts/detection_limit.py +205 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/sra_fetch.py +19 -9
- migec-2.4.0/scripts/wheel_smoke.sh +40 -0
- {migec-2.2.1 → migec-2.4.0}/setup.sh +6 -0
- {migec-2.2.1 → migec-2.4.0}/skills/migec/SKILL.md +97 -4
- {migec-2.2.1 → migec-2.4.0}/src/_bindings.cpp +101 -6
- {migec-2.2.1 → migec-2.4.0}/src/assemble.cpp +170 -20
- {migec-2.2.1 → migec-2.4.0}/src/checkout.cpp +369 -22
- {migec-2.2.1 → migec-2.4.0}/src/consensus.cpp +67 -0
- {migec-2.2.1 → migec-2.4.0}/src/fastq.cpp +15 -2
- {migec-2.2.1 → migec-2.4.0}/src/mig_record.cpp +99 -15
- {migec-2.2.1 → migec-2.4.0}/src/refine.cpp +554 -317
- {migec-2.2.1 → migec-2.4.0}/src/subsample.cpp +1 -0
- migec-2.4.0/src/umi_stats.cpp +1652 -0
- {migec-2.2.1 → migec-2.4.0}/tests/benchmark/test_assemble_speed.py +130 -0
- {migec-2.2.1 → migec-2.4.0}/tests/benchmark/test_checkout_speed.py +77 -2
- {migec-2.2.1 → migec-2.4.0}/tests/benchmark/test_refine_speed.py +24 -8
- {migec-2.2.1 → migec-2.4.0}/tests/conftest.py +14 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_mig_record.cpp +46 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_parallel_stages.cpp +66 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_umi_stats.cpp +257 -0
- migec-2.4.0/tests/realworld/_fixture.py +37 -0
- migec-2.4.0/tests/realworld/test_amplicon.py +260 -0
- migec-2.4.0/tests/realworld/test_droplet.py +166 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/_sim.py +35 -1
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_assemble.py +10 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_dual_end.py +10 -0
- migec-2.4.0/tests/synthetic/test_index_hopping.py +199 -0
- migec-2.4.0/tests/synthetic/test_merge_mates.py +156 -0
- migec-2.4.0/tests/synthetic/test_mig_chain.py +116 -0
- migec-2.4.0/tests/synthetic/test_mig_output.py +120 -0
- migec-2.4.0/tests/synthetic/test_paired_pipeline.py +185 -0
- migec-2.4.0/tests/synthetic/test_pre_amp_auto.py +103 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_refine.py +10 -0
- migec-2.4.0/tests/synthetic/test_refine_bucketed.py +106 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_subsample.py +10 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_thread_invariance.py +10 -0
- migec-2.4.0/tests/synthetic/test_umi_spill.py +114 -0
- migec-2.4.0/tests/synthetic/test_whole_pipeline.py +215 -0
- migec-2.4.0/tests/unit/test_blind_artifact_filter.py +82 -0
- migec-2.4.0/tests/unit/test_ctdna_persite.py +81 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/test_ctdna_tools.py +69 -0
- migec-2.4.0/tests/unit/test_detection_limit.py +91 -0
- migec-2.4.0/tests/unit/test_error_paths.py +342 -0
- migec-2.4.0/tests/unit/test_readme_renders_off_github.py +27 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/test_sheet_and_cli.py +63 -0
- migec-2.2.1/PKG-INFO +0 -783
- migec-2.2.1/README.md +0 -749
- migec-2.2.1/docs/grouping.rst +0 -91
- migec-2.2.1/docs/validation.rst +0 -93
- migec-2.2.1/python/migec/assemble.py +0 -259
- migec-2.2.1/src/umi_stats.cpp +0 -698
- {migec-2.2.1 → migec-2.4.0}/.github/workflows/ci.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/.github/workflows/docs.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/.github/workflows/testpypi.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/CMakeLists.txt +0 -0
- {migec-2.2.1 → migec-2.4.0}/LICENSE +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/PBMC.cell_rank.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/PBMC.sizes.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/PBMC.umi_errors.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/SRR1763769.mig.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/assemble.coverage.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/assemble.quality_by_depth.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/assemble_coverage.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/assemble_coverage.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/benchmark_threads.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/benchmark_threads.json +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/benchmark_threads.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/benchmark_threads.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/cell_rank.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/cell_rank.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/consensus_error.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/consensus_error.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/consensus_layout.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/consensus_layout.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/consensus_quality.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/consensus_quality.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/ctdna_titration.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/ctdna_titration_runs.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/mig_size_spectrum.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/mig_size_spectrum.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/mig_size_zipf.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/mig_size_zipf.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/pipeline.dot +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/pipeline.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/umi_error_children.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/umi_error_children.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/umi_error_rate.gp +0 -0
- {migec-2.2.1 → migec-2.4.0}/assets/umi_error_rate.svg +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/Makefile +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/commands.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/downstream.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/fragmented.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/installation.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/method.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/nextflow.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/nulls.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/plots.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/reference.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/subsample.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/suggest.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/umi_errors.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/docs/umi_statistics.rst +0 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/fastq.hpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/resource.hpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/suggest.hpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/include/migec/whitelist.hpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/README.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/assets/NO_FILE +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/main.nf +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/downstream/align/environment.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/downstream/align/main.nf +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/downstream/arda/environment.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/downstream/arda/main.nf +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/downstream/callvariants/environment.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/downstream/callvariants/main.nf +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/migec/assemble/meta.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/migec/checkout/meta.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/migec/environment.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/modules/migec/refine/meta.yml +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/nextflow.config +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/subworkflows/migec/main.nf +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/subworkflows/migec_airr/main.nf +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/nextflow/subworkflows/migec_ctdna/main.nf +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/slurm/README.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/slurm/migec_array.sbatch +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/slurm/migec_sample.sbatch +0 -0
- {migec-2.2.1 → migec-2.4.0}/integrations/slurm/samples.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/notebooks/README.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/notebooks/airr_repertoire.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/notebooks/barcode_space.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/notebooks/ctdna_variants.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/notebooks/platforms.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/notebooks/refine_diagnostics.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/README.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/design-assemble.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/design-benchmarks.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/design-checkout.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/design-io-interop.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/design-refine.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/design-repo.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/review-algorithms.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/project/review-scope.md +0 -0
- {migec-2.2.1 → migec-2.4.0}/python/migec/plot.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/python/migec/suggest.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/benchmark_threads.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/collision_check.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/compare_calib.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/correction_accuracy.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/diagnose.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/example_figures.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/permutation_nulls.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/quality_floor.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/read_start_dispersion.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/scripts/spikein_ratio.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/src/pattern.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/src/resource.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/src/suggest.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/src/types.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/src/whitelist.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/__init__.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/benchmark/__init__.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/doctest.h +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_checkout.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_checkout_run.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_consensus.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_fastq.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_pattern.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_types.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/cpp/test_whitelist.cpp +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/data/migec_spikein_truth.tsv +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/realworld/__init__.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/__init__.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_barcode_space.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_checkout_end_to_end.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_grouping_accuracy.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_nulls.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_quality_floor.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_sim.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_suggest.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/synthetic/test_umi_errors.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/__init__.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/test_bindings.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/test_compare_calib.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/test_core.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/test_downstream.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/test_plot.py +0 -0
- {migec-2.2.1 → migec-2.4.0}/tests/unit/test_spikein_ratio.py +0 -0
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
name: Benchmark
|
|
2
|
+
|
|
3
|
+
# A separate file rather than a job in ci.yml: adding `schedule` to ci.yml's triggers would run
|
|
4
|
+
# cpp, python and lint on the schedule too, and ci.yml's `cancel-in-progress: true` would let a
|
|
5
|
+
# push cancel a benchmark mid-measurement. Keeping the PR path byte-identical is the point.
|
|
6
|
+
#
|
|
7
|
+
# Never on push or pull_request. These are wall-clock and peak-RSS assertions and the runner is a
|
|
8
|
+
# shared VM whose neighbours we do not control, so on every PR they would be a noise generator
|
|
9
|
+
# attached to the merge button.
|
|
10
|
+
#
|
|
11
|
+
# Note: `schedule` only ever fires on the DEFAULT branch. A regression is therefore caught the
|
|
12
|
+
# night after it lands on master, never on the branch that wrote it -- run this by hand from the
|
|
13
|
+
# Actions tab (workflow_dispatch) on any branch that touches a hot path.
|
|
14
|
+
on:
|
|
15
|
+
schedule:
|
|
16
|
+
# 04:17 UTC. Off the hour on purpose: GitHub queues scheduled runs and the round hours are the
|
|
17
|
+
# congested ones.
|
|
18
|
+
- cron: "17 4 * * *"
|
|
19
|
+
workflow_dispatch:
|
|
20
|
+
|
|
21
|
+
permissions:
|
|
22
|
+
contents: read
|
|
23
|
+
|
|
24
|
+
jobs:
|
|
25
|
+
benchmark:
|
|
26
|
+
name: Benchmark and realworld tiers
|
|
27
|
+
# One OS. The assertions are absolute -- reads/s and bytes per distinct UMI -- so a second
|
|
28
|
+
# runner class with a different core count and a different RSS accounting would need a second
|
|
29
|
+
# set of thresholds to mean anything.
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
# A stage that hangs should cost half an hour of runner time, not the six-hour default.
|
|
32
|
+
timeout-minutes: 30
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@v4
|
|
35
|
+
- uses: actions/setup-python@v5
|
|
36
|
+
with:
|
|
37
|
+
python-version: "3.12"
|
|
38
|
+
- uses: astral-sh/setup-uv@v5
|
|
39
|
+
- name: Install
|
|
40
|
+
run: |
|
|
41
|
+
uv pip install --system scikit-build-core "pybind11>=3.0.2,<4" ninja cmake
|
|
42
|
+
uv pip install --system -e ".[test]" --no-build-isolation
|
|
43
|
+
|
|
44
|
+
# -s because the tests print the measured numbers, and the number is the reason to read the
|
|
45
|
+
# log at all -- a green tick says only that nothing fell by 10x.
|
|
46
|
+
#
|
|
47
|
+
# The thresholds these assert are deliberately loose: >20,000 reads/s against a measured
|
|
48
|
+
# 1.5-2.5 M, >2x speedup on 4 threads, <48 bytes per distinct UMI. They exist to catch a 10x
|
|
49
|
+
# regression -- a transcendental back in the scoring loop, zlib level 6 back on the serial
|
|
50
|
+
# path, a hash map back in the UMI counters -- and not to police a 10% one, which on a shared
|
|
51
|
+
# runner is indistinguishable from a busy neighbour.
|
|
52
|
+
- name: Benchmark tier
|
|
53
|
+
env:
|
|
54
|
+
RUN_BENCHMARK: "1"
|
|
55
|
+
run: python -m pytest tests/benchmark -q -s
|
|
56
|
+
|
|
57
|
+
# `isalgo/umi_data` is public, so an anonymous GET on the resolve URL is the whole of it: no
|
|
58
|
+
# token, no huggingface_hub, no git-lfs client on the runner. Verified 2026-08-14 that the
|
|
59
|
+
# resolve URL 302s to the CDN for an unauthenticated client. The three files under ci/ are
|
|
60
|
+
# 1.9 MB together.
|
|
61
|
+
#
|
|
62
|
+
# Note: this and the two steps below run even when the benchmark tier above went red. A
|
|
63
|
+
# wall-clock threshold that flaked on a busy runner must not also cost the night's accuracy
|
|
64
|
+
# signal; `!cancelled()` rather than `always()` so a timeout or a cancel still stops here.
|
|
65
|
+
- name: Fetch the umi_data ci fixtures
|
|
66
|
+
id: fixtures
|
|
67
|
+
if: ${{ !cancelled() }}
|
|
68
|
+
run: |
|
|
69
|
+
base=https://huggingface.co/datasets/isalgo/umi_data/resolve/main/ci
|
|
70
|
+
mkdir -p "$RUNNER_TEMP/umi_data/ci"
|
|
71
|
+
ok=true
|
|
72
|
+
for f in SRR1763769_umi0.5pct.fq.gz SRR1763769_barcodes.txt \
|
|
73
|
+
sc5p_v2_hs_PBMC_1k_t_cells1pct.fq.gz; do
|
|
74
|
+
curl -fsSL --retry 3 --retry-all-errors \
|
|
75
|
+
-o "$RUNNER_TEMP/umi_data/ci/$f" "$base/$f" || ok=false
|
|
76
|
+
done
|
|
77
|
+
# An LFS pointer served in place of the object, or a truncated body, both arrive with a
|
|
78
|
+
# 200 and surface four steps later as an unreadable FASTQ. `gzip -t` names it here.
|
|
79
|
+
for f in "$RUNNER_TEMP"/umi_data/ci/*.fq.gz; do gzip -t "$f" || ok=false; done
|
|
80
|
+
ls -l "$RUNNER_TEMP/umi_data/ci"
|
|
81
|
+
echo "fetched=$ok" >> "$GITHUB_OUTPUT"
|
|
82
|
+
|
|
83
|
+
- name: Realworld tier
|
|
84
|
+
if: ${{ !cancelled() && steps.fixtures.outputs.fetched == 'true' }}
|
|
85
|
+
env:
|
|
86
|
+
UMI_DATA: ${{ runner.temp }}/umi_data
|
|
87
|
+
run: |
|
|
88
|
+
set +e
|
|
89
|
+
python -m pytest tests/realworld -q
|
|
90
|
+
code=$?
|
|
91
|
+
set -e
|
|
92
|
+
# 5 is pytest's "collected nothing". tests/realworld is a tier under construction, and an
|
|
93
|
+
# empty directory has to read as an empty tier rather than as a broken job -- the day a
|
|
94
|
+
# test lands there it is guarded by this step with no change to this file.
|
|
95
|
+
if [ "$code" -eq 5 ]; then
|
|
96
|
+
echo "::warning::tests/realworld collected no tests -- the fixtures fetched and guard nothing yet"
|
|
97
|
+
exit 0
|
|
98
|
+
fi
|
|
99
|
+
exit $code
|
|
100
|
+
|
|
101
|
+
# Loudly, not silently: a realworld tier that quietly stopped running is exactly the failure
|
|
102
|
+
# this workflow exists to end. The job stays green because an HF outage is not a defect in
|
|
103
|
+
# this repo, but the annotation says on the run summary that the tier did not run.
|
|
104
|
+
- name: Report a failed fixture fetch
|
|
105
|
+
if: ${{ !cancelled() && steps.fixtures.outputs.fetched != 'true' }}
|
|
106
|
+
run: |
|
|
107
|
+
echo "::warning::could not fetch isalgo/umi_data ci/ over https -- the realworld tier did NOT run"
|
|
@@ -40,7 +40,12 @@ jobs:
|
|
|
40
40
|
env:
|
|
41
41
|
CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-*"
|
|
42
42
|
CIBW_SKIP: "*-musllinux_* *-manylinux_i686"
|
|
43
|
-
|
|
43
|
+
# Never: an import is not a smoke test. A wheel with a broken console script -- an
|
|
44
|
+
# entry-point typo, typer missing from `dependencies`, a `_core` symbol only touched at
|
|
45
|
+
# runtime -- imports cleanly and publishes green, and the first person to run
|
|
46
|
+
# `migec checkout` finds out. This runs the CLI and one read through the pipeline, which
|
|
47
|
+
# costs seconds and exercises the extension, the console script and the dependency list.
|
|
48
|
+
CIBW_TEST_COMMAND: "bash {project}/scripts/wheel_smoke.sh"
|
|
44
49
|
- uses: actions/upload-artifact@v4
|
|
45
50
|
with:
|
|
46
51
|
name: python-wheels-${{ matrix.os }}
|
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
build/
|
|
7
7
|
# the thread-sanitizer build; see docs/performance.rst for how to make one
|
|
8
8
|
build-tsan/
|
|
9
|
+
# a second ordinary build, for verifying a change without disturbing the installed extension
|
|
10
|
+
build-verify/
|
|
9
11
|
# setup.sh creates this
|
|
10
12
|
.venv/
|
|
11
13
|
# intermediate record files; regenerate with `migec checkout`
|
|
@@ -36,3 +38,5 @@ docs/_build/
|
|
|
36
38
|
*.bam
|
|
37
39
|
*.bai
|
|
38
40
|
*.crai
|
|
41
|
+
build-verify/
|
|
42
|
+
_migec_error.log
|
|
@@ -4,6 +4,308 @@ Hand-written and prose-heavy: each entry says what changed and, where it matters
|
|
|
4
4
|
prevents. Releases before 2.0.0 are the Groovy MIGEC and are described by their git tags on the
|
|
5
5
|
`legacy-v1` branch.
|
|
6
6
|
|
|
7
|
+
## 2.4.0 — 2026-08-14
|
|
8
|
+
|
|
9
|
+
Note: the wheel smoke test lowercased the whole barcode-sheet line, sample id included, so
|
|
10
|
+
`checkout` wrote `co/s1.fq.gz` and the next command asked for `co/S1.fq.gz`. That passes on macOS,
|
|
11
|
+
whose filesystem is case-insensitive, and fails on Linux — and 2.4.0 is the first release whose
|
|
12
|
+
publish run executed the script at all, so it failed there and nowhere else. The adapter is
|
|
13
|
+
lowercased on its own now.
|
|
14
|
+
|
|
15
|
+
**Nothing in the pipeline scales with the library any more**, and the comparisons the version
|
|
16
|
+
number was waiting on have started landing: MIGEC 1.2.9, UMI-tools and fgbio are all run end to
|
|
17
|
+
end and scored.
|
|
18
|
+
|
|
19
|
+
Note: 2.3.0 was written up but never tagged or published — the last wheel on PyPI is 2.2.1 — so
|
|
20
|
+
this release carries its changes too. Its section below still describes them.
|
|
21
|
+
|
|
22
|
+
### MIGEC 1.2.9, head to head
|
|
23
|
+
|
|
24
|
+
`scripts/compare_migec_v1.py` runs the Groovy implementation this repo replaced and migec 2 over
|
|
25
|
+
the same simulated library, same barcode dialect, same sheet, both pipelines end to end.
|
|
26
|
+
**9.4-11.9x the wall clock** against an M5 gate of 3x, 3.5-3.7x less memory, and the molecule count
|
|
27
|
+
is 0.09% over truth against v1's 13.6% -- 99.8-99.99% of our consensuses are exactly a template,
|
|
28
|
+
against 93.9-95.1%. The over-count is barcode errors `--filter-collisions` did not catch, because
|
|
29
|
+
its rule is a count ratio and a count ratio carries nothing below ~3 reads per UMI.
|
|
30
|
+
|
|
31
|
+
`--min-count` is given to both: v1 defaults to 5 and migec 2 to 1, so leaving each at its own
|
|
32
|
+
default would have credited us with recovering molecules v1 was told to throw away.
|
|
33
|
+
`docs/validation.rst`, `assets/migec_v1.tsv`.
|
|
34
|
+
|
|
35
|
+
### UMI-tools and fgbio, measured
|
|
36
|
+
|
|
37
|
+
`scripts/compare_grouping.py` runs both map-first tools end to end and scores all three partitions
|
|
38
|
+
against the simulator's truth with the same adjusted Rand index Calib is scored with. What it
|
|
39
|
+
measures is what the alignment position is worth, because that is the only difference: they group
|
|
40
|
+
on *(position, UMI)*, we group on *(sample, cell, UMI)* and align once, afterwards.
|
|
41
|
+
|
|
42
|
+
On **one reference** — a single amplicon, a clonal control, a targeted ctDNA panel — the position
|
|
43
|
+
carries nothing, and migec wins: ARI 0.9967 against 0.9864 (UMI-tools) and 0.9817 (fgbio), with
|
|
44
|
+
0.65% of reads in clusters that mix molecules against 3.0% and 3.9%. That is 4.6x and 6x fewer
|
|
45
|
+
molecules destroyed, and destroying one is the error nothing downstream can detect. Nothing
|
|
46
|
+
rescues that case: two molecules that collide on one reference hold the same sequence, so neither
|
|
47
|
+
the mapping position nor payload sub-clustering can tell them apart, and the barcode is the only
|
|
48
|
+
evidence there is.
|
|
49
|
+
|
|
50
|
+
On 200 or 20,000 distinct references they win by 0.001 ARI, and that gap is a **depth threshold**
|
|
51
|
+
rather than a limit of the method. Collided molecules there carry *different* sequences, which is
|
|
52
|
+
what makes them separable, and `assemble`'s linkage sub-clustering separates them from the payload
|
|
53
|
+
with no aligner -- but only above the depth the 8.68 threshold fixes for itself, since the strongest
|
|
54
|
+
evidence a 50/50 split can carry is `log10 C(n, n/2)` and that reaches 8.68 at n ~ 32.
|
|
55
|
+
`scripts/collision_split.py` measures it: **0 of 12 collisions separated at 9.1 reads on the
|
|
56
|
+
barcode, 7 of 14 at 40.9, and 10 of 10, 13 of 13, 10 of 10 at 82, 161 and 283.** The comparison
|
|
57
|
+
above was scored at 5 reads per molecule, a third of what the test needs. migec is 8-48x faster
|
|
58
|
+
throughout, including the alignment they cannot skip.
|
|
59
|
+
|
|
60
|
+
A collision is defined on the **true** barcode, never the observed one -- two molecules whose
|
|
61
|
+
observed barcodes coincide because one picked up a sequencing error are what `refine` corrects, and
|
|
62
|
+
counting them makes the collision rate grow with the read count, which it cannot do.
|
|
63
|
+
|
|
64
|
+
`docs/grouping.rst` has the table, `assets/grouping_tools.tsv` the numbers, `SOURCES.md` how to
|
|
65
|
+
install both (fgbio needs a JDK 17+; UMI-tools needs `--no-build-isolation` on Python 3.12+). The
|
|
66
|
+
simulator now writes `clones.fa`, the reference a map-first tool has to align to.
|
|
67
|
+
|
|
68
|
+
### `refine`'s barcode table bounds itself
|
|
69
|
+
|
|
70
|
+
The last thing in the pipeline that scaled with the library. Past 1 GB the table range-partitions
|
|
71
|
+
to disk, correction follows it into the partition, and every table refine writes is streamed one
|
|
72
|
+
bucket at a time. Nothing changes but the wall clock: a partitioned run and a resident one agree on
|
|
73
|
+
every scalar and on every output file byte for byte.
|
|
74
|
+
|
|
75
|
+
Two things this needed that `checkout`'s counters did not. The table carries the **evidence** —
|
|
76
|
+
the barcode's own quality at each position and its payload draft — because a side array indexed
|
|
77
|
+
against the entry list cannot survive a partition, and dropping it would leave the bucketed run
|
|
78
|
+
correcting on the count ratio alone, which reports nothing at 1–3 reads per UMI. And the two passes
|
|
79
|
+
**scan**; they do not merge. A barcode can have a plausible parent on each side of the partition
|
|
80
|
+
boundary, and merging inside a pass takes the first candidate rather than the best: 2 barcodes in
|
|
81
|
+
6,591 landed on a different parent than the resident run gave them, and every table downstream
|
|
82
|
+
moved with them. Both passes now propose and one global apply decides, in the same order the
|
|
83
|
+
resident walk uses.
|
|
84
|
+
|
|
85
|
+
The budget is a `refine.run()` argument, never a CLI flag. Two smaller things came with it: the
|
|
86
|
+
evidence pass folded into the table pass, so refine streams the reads twice rather than three
|
|
87
|
+
times, and the barcode-rank curve is read off the MIG size spectrum rather than off a sorted array
|
|
88
|
+
of every molecule's count — the last allocation in the stage that was one entry per barcode.
|
|
89
|
+
|
|
90
|
+
### `--mate2`: both ends of the fragment are one molecule
|
|
91
|
+
|
|
92
|
+
A read pair carries one barcode because it is one molecule, so `assemble` will take the other mate
|
|
93
|
+
and place it:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
migec assemble out/S1_R1.fq.gz --mate2 out/S1_R2.fq.gz -o cons/
|
|
97
|
+
migec assemble out/S1.000.mig --merge-mates -o cons/ # the pair is already in the record
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Mate 2 is reverse-complemented and placed against mate 1. Overlapping mates give one consensus
|
|
101
|
+
spanning the insert; mates that do not reach each other give two contigs, because the bases between
|
|
102
|
+
them are covered by no read. That is `--contig`'s rule rather than a second one, and the two output
|
|
103
|
+
routes agree byte for byte.
|
|
104
|
+
|
|
105
|
+
The offset is a property of the molecule, not of the pair, so it is voted on once per group over
|
|
106
|
+
up to eight pairs instead of placing every read against every other. The first version did the
|
|
107
|
+
latter and cost 11x the single-end path (119,820 record-pairs/s against 1,356,819); the vote costs
|
|
108
|
+
1,288,686 against 2,115,912, which is the price of the second read's bases and nothing else.
|
|
109
|
+
|
|
110
|
+
The mates are matched by position, which is what checkout's `_R1`/`_R2` output guarantees. A file
|
|
111
|
+
that ends before the other is refused: pairing off what is left attaches one molecule's mate to
|
|
112
|
+
another's, and the consensus that comes out looks perfectly well formed.
|
|
113
|
+
|
|
114
|
+
### `--pre-amp-error auto` fits the floor instead of naming it
|
|
115
|
+
|
|
116
|
+
The cap on every emitted quality is the error that was already in the molecule before
|
|
117
|
+
amplification, and until now it was a named bracket: `rt` 1e-4, `medium` 1e-5, `high` 1e-6. It can
|
|
118
|
+
be measured on the dataset in front of you, and `auto` does it -- X2's estimator, run on migec's
|
|
119
|
+
own consensuses rather than on a bespoke one:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
migec assemble out/S1.fq.gz -o cons/ --pre-amp-error auto
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Molecules at 20 reads or more have suppressed sequencing error to nothing, so what they still get
|
|
126
|
+
wrong against the library's modal sequence is what was there before the first cycle. Injected
|
|
127
|
+
1e-4 comes back as 1.20e-4 [7.94e-5, 1.75e-4] and injected 1e-5 as 1.58e-5 [8.14e-6, 2.76e-5]; with
|
|
128
|
+
no injected error at all the answer is the interval's upper end, 9.56e-6, because a floor of zero
|
|
129
|
+
is a quality of infinity.
|
|
130
|
+
|
|
131
|
+
It refuses more often than it answers, and that is the point. A diverse library has no monomorphic
|
|
132
|
+
position to score, so "disagrees with the modal base" means "is a different molecule" and the
|
|
133
|
+
number that would come out is the library's diversity; a shallow library has no molecule deep
|
|
134
|
+
enough for the curve to have flattened. Both refuse, fall back to the named `rt` class, and say
|
|
135
|
+
why -- in the report and in the new `assemble.pre_amp_error.tsv`, which carries every input the
|
|
136
|
+
fit rested on next to the fit.
|
|
137
|
+
|
|
138
|
+
It costs a second assembly pass, so it is opt-in: the consensus sequences do not depend on the
|
|
139
|
+
floor, only the emitted quality does, so a probe assembly runs, is read back and is deleted.
|
|
140
|
+
|
|
141
|
+
## 2.3.0 — 2026-08-14
|
|
142
|
+
|
|
143
|
+
**The intermediate is a partition now, end to end**, and the `.mig` format is at v2 (a reader takes
|
|
144
|
+
v1 and v2, so an intermediate written by 2.2.1 still reads). Nothing in `checkout` scales with the
|
|
145
|
+
library any more, `refine` and `assemble` both take buckets, and an adversarial audit of all of it
|
|
146
|
+
found three bugs that destroyed or invented data -- fixed, each with the test that reproduces it.
|
|
147
|
+
|
|
148
|
+
Upgrading: nothing changes unless you pass `--mig`. FASTQ is still the default at every stage, the
|
|
149
|
+
CLI has one new flag, and every number a 2.2.1 run produced is the number a 2.3.0 run produces.
|
|
150
|
+
|
|
151
|
+
### `checkout --mig` writes the partition `assemble` was building for itself
|
|
152
|
+
|
|
153
|
+
`assemble`'s first pass range-partitions the reads on the barcode into `.mig` buckets, and that
|
|
154
|
+
pass is most of its wall clock. `checkout --mig` writes those buckets directly -- same key, same
|
|
155
|
+
range partition, `<sample>.<bbb>.mig` -- and `assemble` reads them instead of building them:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
migec checkout reads.fq.gz -b barcodes.txt -o out --mig
|
|
159
|
+
migec assemble out/S1.000.mig -o asm # one bucket names the whole partition
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Measured on 500,000 reads over four samples at `-t 4`: **1.16 s to 0.98 s** end to end for the
|
|
163
|
+
identical 124,878 molecules, and the consensus FASTQ is byte-identical after decompression (the
|
|
164
|
+
gzip framing differs, because bucket boundaries are member boundaries and there are a different
|
|
165
|
+
number of buckets). `checkout` pays 0.06 s of that and `assemble` saves 0.25 s.
|
|
166
|
+
|
|
167
|
+
Opt-in, and FASTQ stays the default: a `.mig` file is a migec intermediate that nothing else
|
|
168
|
+
reads, while every aligner and every pipeline in `docs/downstream.rst` speaks FASTQ.
|
|
169
|
+
|
|
170
|
+
Never: **`-t` still changes nothing but the clock.** One writer per (sample, bucket), owned by one
|
|
171
|
+
thread for the whole run, and every worker walks the chunks in input order -- so ownership decides
|
|
172
|
+
who writes a record and never which file it lands in or where.
|
|
173
|
+
|
|
174
|
+
Note: the open-file budget is for the RUN, not per sample. 256 buckets each on a 96-plex sheet
|
|
175
|
+
would be 24,576 open writers; each sample of such a sheet also holds a 96th of the reads, so a
|
|
176
|
+
couple of buckets each is proportionate rather than a compromise.
|
|
177
|
+
|
|
178
|
+
Two things are refused rather than guessed at: buckets from two samples in one `assemble` (a UMI
|
|
179
|
+
repeats across samples by design, so grouping them together invents molecules), and
|
|
180
|
+
`--limit-read`/`--limit-umi` on an already-partitioned input (a limit is a prefix of the input, and
|
|
181
|
+
a partition has no prefix -- the first records of bucket 0 are one corner of the barcode space).
|
|
182
|
+
|
|
183
|
+
Never: **only buckets `assemble` wrote are deleted after they are consumed.** Pass 2 removes each
|
|
184
|
+
bucket as it finishes with it, which is right for its own temporaries and catastrophic for
|
|
185
|
+
checkout's output -- the first run through `--mig` ate three of the four samples' partitions, and
|
|
186
|
+
a second `assemble` over an eaten one would report a smaller library with nothing to say why.
|
|
187
|
+
|
|
188
|
+
### The i7 x i5 contingency table
|
|
189
|
+
|
|
190
|
+
`checkout` reads the index pair out of the instrument's own read header -- the last field of
|
|
191
|
+
`1:N:0:ATCACG+CGTGAT` -- for every read, matched or not, and writes
|
|
192
|
+
`checkout.index_pairs.tsv`: reads per observed combination, each one's share of its own i7 and i5,
|
|
193
|
+
and whether it looks ordered. Index hopping is the one contamination a per-sample yield cannot see,
|
|
194
|
+
because a hopped read lands in another real sample and looks like one of its reads; the header is
|
|
195
|
+
the only evidence, and it was already in the file.
|
|
196
|
+
|
|
197
|
+
Never: **a single-indexed run is not estimable, and that is not zero.** With one index there are no
|
|
198
|
+
combinations, so nothing can be off-diagonal.
|
|
199
|
+
|
|
200
|
+
Note: it matters most where it is smallest. At 0.1% hopping a 1% variant in a deeply sequenced
|
|
201
|
+
sample contaminates its neighbour at 1e-5, which is the level a rare-variant caller is asked to
|
|
202
|
+
believe.
|
|
203
|
+
|
|
204
|
+
### refine reads and writes buckets, and the format carries the barcode's own quality
|
|
205
|
+
|
|
206
|
+
`checkout --mig` -> `refine` -> `assemble` runs on buckets end to end. refine's output is
|
|
207
|
+
**re-partitioned on the corrected barcode**: a corrected barcode is a different key and a key
|
|
208
|
+
decides its bucket, so copying a bucket through unchanged would stop it being a partition, and the
|
|
209
|
+
reads whose barcode was corrected across a boundary would be grouped with strangers by the next
|
|
210
|
+
stage. Every number matches the FASTQ route -- reads, barcodes, merges, molecules, the estimated
|
|
211
|
+
error to nine digits, the barcode table byte for byte, and the consensus FASTQ byte for byte.
|
|
212
|
+
|
|
213
|
+
`.mig` **v2** adds the BARCODE's own quality, one Phred per base, as two fixed-width columns. v1
|
|
214
|
+
stored the minimum over the barcode, which is not what the correction posterior wants: it weighs
|
|
215
|
+
the reported quality AT THE POSITION THAT DIFFERS, and a minimum says every position is as bad as
|
|
216
|
+
the worst one -- overstating the error everywhere, which makes merges easier, which is the wrong
|
|
217
|
+
direction. A v1 file still reads and falls back to the global rate, exactly as a FASTQ with no
|
|
218
|
+
`QX:Z:` tag does.
|
|
219
|
+
|
|
220
|
+
Note: the audit trail on the bucket route is `<sample>.barcodes.tsv`, every barcode with its
|
|
221
|
+
parent. A `.mig` record has no room for the pre-correction barcode the way a FASTQ comment has
|
|
222
|
+
`OX:Z:`, and two `u64` per READ to carry what is one row per BARCODE is the wrong trade.
|
|
223
|
+
|
|
224
|
+
### Three data-integrity bugs, found by attacking the new code
|
|
225
|
+
|
|
226
|
+
All three reproduced before they were fixed, and all three are in the `.mig` work above.
|
|
227
|
+
|
|
228
|
+
- **refine wrote its output buckets over its input.** Same names on both sides, and `MigWriter`
|
|
229
|
+
opens `"wb"`, which truncates. Pointed at its own input directory the run destroyed the reads it
|
|
230
|
+
was halfway through reading. Refused now, on the resolved path, before a writer is opened.
|
|
231
|
+
- **refine's `.mig` rewrite ignored `--limit-read`/`--limit-umi`.** It wrote reads the table was
|
|
232
|
+
never built from -- uncorrected, beside corrected ones, with nothing saying which was which.
|
|
233
|
+
- **The sample id went into every output path unvalidated**, and in `assemble` and `refine` it comes
|
|
234
|
+
from the DATA (the `BC:Z:` tag of the first read), so a crafted tag wrote outside the output
|
|
235
|
+
directory. `validate_sample_id` refuses path separators, `..`, a leading `-` and control
|
|
236
|
+
characters.
|
|
237
|
+
|
|
238
|
+
### ...and six more from the same audit
|
|
239
|
+
|
|
240
|
+
A truncated spill file was read into an undersized buffer (a 15-byte heap overrun; trigger is a run
|
|
241
|
+
killed mid-spill). A `.mig` header's `bucket_bits`/`bucket_index` went unvalidated into a shift and
|
|
242
|
+
a vector index. A block's `n_records` was `memcpy`'d before it was bounded by the payload. `assemble`
|
|
243
|
+
left its whole temp partition behind on the error path and leaked the per-bucket table handle on a
|
|
244
|
+
throw. `fclose` was unchecked in four writers, so a disk that filled during the final flush was
|
|
245
|
+
reported as a successful run. A failed thread SPAWN unwound over joinable threads, which is
|
|
246
|
+
`std::terminate`.
|
|
247
|
+
|
|
248
|
+
Two usability bugs went with them: a barcode **sheet** could not use the positional spellings that
|
|
249
|
+
work through `--bc-pattern`, and an **empty input** printed statistics computed from no reads and
|
|
250
|
+
exited 0.
|
|
251
|
+
|
|
252
|
+
### Tests and docs
|
|
253
|
+
|
|
254
|
+
`tests/realworld/` exists now -- it was an empty `__init__.py` -- and runs the two public CI
|
|
255
|
+
fixtures through the stages they can support. `tests/unit/test_error_paths.py` covers empty,
|
|
256
|
+
truncated and malformed input into every stage. Paired-end flows past `checkout` for the first time
|
|
257
|
+
in a test, and the four-stage chain is asserted as one invariant. `.github/workflows/benchmark.yml`
|
|
258
|
+
runs the benchmark and realworld tiers nightly and on dispatch, never on the PR path.
|
|
259
|
+
|
|
260
|
+
README.md went from 809 lines to 206 -- install, where the barcode is, the five commands, what comes
|
|
261
|
+
out, and a documentation table -- with every fact moved into a docs page rather than deleted, and
|
|
262
|
+
`docs/assays.rst` is a new home for the per-assay settings table that was buried in a 762-line ctDNA
|
|
263
|
+
page.
|
|
264
|
+
|
|
265
|
+
### The UMI counters bound themselves, and correction follows them
|
|
266
|
+
|
|
267
|
+
This is roadmap items 1 and 2, which turned out to be one item. `checkout`'s UMI counters were the
|
|
268
|
+
last allocation in the pipeline that grew with the library rather than with the chunk: ~22 bytes per
|
|
269
|
+
distinct barcode held in one piece, 8.8 GB at NovaSeq scale, with a warning past 1 GB standing in
|
|
270
|
+
for a fix.
|
|
271
|
+
|
|
272
|
+
They now range-partition to disk past `umi_budget_bytes` (1 GB for the whole run, divided by the
|
|
273
|
+
samples, `0` to keep everything resident). The buckets go in `<out_dir>/.umi_spill` and are removed
|
|
274
|
+
once the summary has been written; `umi_spilled` in the summary says whether it happened. The
|
|
275
|
+
histogram, the composition, the distinct count, the distance-1 census and the correction all stream
|
|
276
|
+
one bucket at a time. Not a CLI flag: 1 GB is where a counter becomes a problem on any machine, and
|
|
277
|
+
nothing below it changes.
|
|
278
|
+
|
|
279
|
+
Never: **a range partition on its own would have bounded the memory and silently stopped
|
|
280
|
+
correcting.** The bucket is the top bits of the key, so a barcode whose error landed in the first
|
|
281
|
+
few positions sits in a different bucket from its parent and the two can never meet — and a third
|
|
282
|
+
of all barcode errors are in those positions. Correction therefore runs twice: once over the
|
|
283
|
+
buckets as they stand, owning the positions the prefix does not touch, and once over a copy whose
|
|
284
|
+
keys are rotated left by the width of the prefix, owning exactly the ones it hides. Every pair is
|
|
285
|
+
weighed in one pass and only one, so `umis_merged` stays a count of barcodes rather than of
|
|
286
|
+
opportunities to look at them. Checked field for field against the resident answer, on a simulated
|
|
287
|
+
library with injected barcode errors and on a 500,000-read corpus: identical, including the
|
|
288
|
+
estimated error rate.
|
|
289
|
+
|
|
290
|
+
Never: a bucketed run reports the **scalars**. `root` and `corrected` are indexed against
|
|
291
|
+
`entries()`, which is the array being bounded, so they come back empty rather than wrong, and
|
|
292
|
+
`BarcodeEvidence` — indexed the same way — is refused rather than ignored. The stage that rewrites
|
|
293
|
+
reads is `refine`, and `refine` does not spill.
|
|
294
|
+
|
|
295
|
+
Two bugs found on the way, both by tests written to state the property rather than the number:
|
|
296
|
+
|
|
297
|
+
- The **first** flush of a counter swaps the append buffer in wholesale and returned before the
|
|
298
|
+
budget check, so a counter whose whole library arrived in one buffer's worth never partitioned at
|
|
299
|
+
all — the bound switched itself off on exactly the small case, and reported a resident answer
|
|
300
|
+
that happened to be right.
|
|
301
|
+
- The bucketed driver reported the census's `0.0` for a clean library where the resident path
|
|
302
|
+
reports the 1e-4 floor it actually corrects at. The rate quoted has to be the rate used; it feeds
|
|
303
|
+
the error budget, which divides by it.
|
|
304
|
+
|
|
305
|
+
Cost: about 2.2x the wall clock when it fires (718,000 reads/s against 333,000 on the benchmark
|
|
306
|
+
corpus), because the partition is read back four times. Nothing when it does not fire — 217,000
|
|
307
|
+
reads/s single-threaded and 23.7 B per distinct UMI, both unchanged.
|
|
308
|
+
|
|
7
309
|
## 2.2.1 — 2026-08-14
|
|
8
310
|
|
|
9
311
|
Documentation, pipelines and tooling. Note: **the wheel is unchanged from 2.2.0** — no command,
|
|
@@ -500,7 +802,7 @@ and the published benchmark comparisons. Install with `pip install --pre migec`.
|
|
|
500
802
|
|
|
501
803
|
Everything below was developed before this tag.
|
|
502
804
|
|
|
503
|
-
##
|
|
805
|
+
## 2.0.0a1 and earlier
|
|
504
806
|
|
|
505
807
|
### Cell barcodes, whitelists, cell calling, dual-end barcodes, and what the Phred is worth
|
|
506
808
|
|