countmut 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- countmut-0.2.5/LICENSE +21 -0
- countmut-0.2.5/MANIFEST.in +2 -0
- countmut-0.2.5/PKG-INFO +184 -0
- countmut-0.2.5/README.md +147 -0
- countmut-0.2.5/backend/Makefile +34 -0
- countmut-0.2.5/backend/bedidx.c +158 -0
- countmut-0.2.5/backend/bgzf.c +714 -0
- countmut-0.2.5/backend/bgzf.h +211 -0
- countmut-0.2.5/backend/countmut_core.c +1246 -0
- countmut-0.2.5/backend/countmut_core.h +100 -0
- countmut-0.2.5/backend/countmut_core_main.c +236 -0
- countmut-0.2.5/backend/countmut_expr.c +1031 -0
- countmut-0.2.5/backend/countmut_expr.h +88 -0
- countmut-0.2.5/backend/faidx.c +445 -0
- countmut-0.2.5/backend/faidx.h +113 -0
- countmut-0.2.5/backend/hts.c +948 -0
- countmut-0.2.5/backend/hts.h +179 -0
- countmut-0.2.5/backend/ketopt.h +123 -0
- countmut-0.2.5/backend/khash.h +614 -0
- countmut-0.2.5/backend/kseq.h +256 -0
- countmut-0.2.5/backend/ksort.h +134 -0
- countmut-0.2.5/backend/kstdint.h +64 -0
- countmut-0.2.5/backend/kstring.h +191 -0
- countmut-0.2.5/backend/razf.c +857 -0
- countmut-0.2.5/backend/razf.h +134 -0
- countmut-0.2.5/backend/sam.c +1279 -0
- countmut-0.2.5/backend/sam.h +230 -0
- countmut-0.2.5/countmut/__init__.py +28 -0
- countmut-0.2.5/countmut/_core/countmut_core +0 -0
- countmut-0.2.5/countmut/backend.py +232 -0
- countmut-0.2.5/countmut/bam_tags.py +384 -0
- countmut-0.2.5/countmut/cli.py +296 -0
- countmut-0.2.5/countmut/core.py +1317 -0
- countmut-0.2.5/countmut/model.py +83 -0
- countmut-0.2.5/countmut/utils.py +103 -0
- countmut-0.2.5/countmut.egg-info/PKG-INFO +184 -0
- countmut-0.2.5/countmut.egg-info/SOURCES.txt +47 -0
- countmut-0.2.5/countmut.egg-info/dependency_links.txt +1 -0
- countmut-0.2.5/countmut.egg-info/entry_points.txt +2 -0
- countmut-0.2.5/countmut.egg-info/requires.txt +15 -0
- countmut-0.2.5/countmut.egg-info/top_level.txt +1 -0
- countmut-0.2.5/pyproject.toml +91 -0
- countmut-0.2.5/setup.cfg +4 -0
- countmut-0.2.5/tests/test_cli.py +174 -0
- countmut-0.2.5/tests/test_core.py +205 -0
- countmut-0.2.5/tests/test_correctness.py +435 -0
- countmut-0.2.5/tests/test_router.py +230 -0
- countmut-0.2.5/tests/test_unified.py +141 -0
- countmut-0.2.5/tests/test_utils.py +145 -0
countmut-0.2.5/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Ye Chang
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
countmut-0.2.5/PKG-INFO
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: countmut
|
|
3
|
+
Version: 0.2.5
|
|
4
|
+
Summary: Unified ultra-fast strand-aware mutation counter (C backend + samtools-style -e/-p filters)
|
|
5
|
+
Author-email: Ye Chang <yech1990@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: bioinformatics,bam,pileup,mutation,bisulfite,sequencing,genomics
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: pysam>=0.21.0
|
|
24
|
+
Requires-Dist: rich>=13.0.0
|
|
25
|
+
Requires-Dist: click>=8.0.0
|
|
26
|
+
Requires-Dist: rich-click>=1.6.0
|
|
27
|
+
Requires-Dist: numpy>=1.21.0
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
30
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff>=0.14.0; extra == "dev"
|
|
32
|
+
Requires-Dist: black>=22.0.0; extra == "dev"
|
|
33
|
+
Provides-Extra: test
|
|
34
|
+
Requires-Dist: pytest>=7.0.0; extra == "test"
|
|
35
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "test"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
# CountMut
|
|
39
|
+
|
|
40
|
+
At each site in a modification assay you want the same small number: how many
|
|
41
|
+
reads still show the reference base, how many show the conversion, and what
|
|
42
|
+
that is as a rate. Existing tools made this hard — a flag for every QC idea,
|
|
43
|
+
two BAM-walking strategies that disagreed on deep overlapping sites, and
|
|
44
|
+
read-level filters priced once per aligned position instead of once per read.
|
|
45
|
+
|
|
46
|
+
CountMut collapses this into one design: a single base counter, QC and trimming
|
|
47
|
+
as expressions (the samtools grammar, in C), and two BAM-walks that provably
|
|
48
|
+
agree. What you *see* is an output format — the per-base table, an allele VCF,
|
|
49
|
+
or a column template you write yourself. On a deep rRNA transcriptome
|
|
50
|
+
(784 k reads / 90 Mb) a genome-wide `mapq >= 20` adds ~0.5 s, and the per-base
|
|
51
|
+
filter was cut ~3×.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install -e .
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Quick start
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
# every base, per strand -> composition table
|
|
61
|
+
countmut -i in.bam -r ref.fa -o depth.tsv
|
|
62
|
+
|
|
63
|
+
# your own columns (e.g. an A/T conversion ratio) -> custom output template
|
|
64
|
+
countmut -i in.bam -r ref.fa -o mine.tsv \
|
|
65
|
+
--output-format "{pos+1}\t{ref}\t{a}\t{t}\t{round(t/(a+t)+0*a, 4)}" \
|
|
66
|
+
--fmt-header "pos\tref\tA\tT\trate"
|
|
67
|
+
|
|
68
|
+
# alleles as VCF -> VCF
|
|
69
|
+
countmut -i in.bam -r ref.fa --vcf -o allele.vcf
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
There is no `--mode`, no `--ref-base`/`--mut-base`: one counter, and the output
|
|
73
|
+
is whatever you choose. Bare runs print the per-strand base composition
|
|
74
|
+
(`ref depth a c g t n`); `--vcf` gives an allele VCF; `--output-format` gives
|
|
75
|
+
your own columns (a conversion ratio is just `{t}/({c}+{t})`). Output is per
|
|
76
|
+
strand by default, and `--strandless` merges the two strands.
|
|
77
|
+
|
|
78
|
+
## Filtering with one expression instead of ten flags
|
|
79
|
+
|
|
80
|
+
In RNA there is no genomic mutation to count: the "converted" base is a
|
|
81
|
+
modification read out through reverse transcription, so the conversion rate
|
|
82
|
+
reports modification level rather than a variant. Whatever your sample, the
|
|
83
|
+
QC and trimming live in one expression language — read-level rules are `-e`
|
|
84
|
+
expressions, site-level rules `-p`, in the samtools `filter=` grammar,
|
|
85
|
+
evaluated inside the C core. The old `--min-mapq` / `--trim-*` flags are gone —
|
|
86
|
+
write them as `-e` expressions.
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
# quality, and not on the error-prone read ends
|
|
90
|
+
countmut -i x -r ref -o out -e "mapq >= 20 and bq >= 20 and dist5 >= 2"
|
|
91
|
+
|
|
92
|
+
# one sample
|
|
93
|
+
countmut -i x -r ref -o out -e "tag('RG') == 'sampleA'"
|
|
94
|
+
|
|
95
|
+
# samtools-style: low mismatch, not a PCR duplicate, read 1 only
|
|
96
|
+
countmut -i x -r ref -o out -e "[NM] <= 3 and not (flag.dup ~= 0) and flag.read1 != 0"
|
|
97
|
+
|
|
98
|
+
# site-level: only well-covered sites with ≥2 G reads
|
|
99
|
+
countmut -i x -r ref -o out -p "depth >= 5 and g >= 2"
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
**`-e` is also a group router.** A bare boolean expression is a filter
|
|
103
|
+
(`true` → count, `nil`/`false` → drop), but an expression that returns an
|
|
104
|
+
integer `0..3` routes each kept base into that **group**; `true` routes to
|
|
105
|
+
group 0. Anything else drops the base (with a stderr warning). The split
|
|
106
|
+
shows up in `--output-format` templates as per-group cells `{a.0}` … `{n.3}`
|
|
107
|
+
(plain `{a}` stays the total over all groups):
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
# bisulfite A->G, 2-group router: group 1 = high-conversion bases, group 0 =
|
|
111
|
+
# everything else that passes the hard NS gate (low quality / read-end trim)
|
|
112
|
+
countmut -i x -r ref -o out \
|
|
113
|
+
-e "([NS] <= 1) and (([Yf] >= 1 and [Zf] <= 3 and bq >= 20 and qpos >= 2 and qlen - qpos > 2) and 1 or 0)" \
|
|
114
|
+
--output-format "{chrom}\t{pos+1}\t{strand}\t{motif}\t{a.0}\t{a.1}\t{g.0}\t{g.1}" \
|
|
115
|
+
--motif-pad 15 --fmt-header "chrom\tpos\tstrand\tmotif\tu0\tu1\tm0\tm1"
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Most filters use roughly ten variables — `mapq`, `bq` (base quality), `flags`,
|
|
119
|
+
`qpos` (position in the read), `dist5`/`dist3` (distance to the read ends),
|
|
120
|
+
`base`/`ref`, `tag('XX')`, and `rname`. A couple of things are worth knowing.
|
|
121
|
+
A missing tag is nothing: comparing `tag('NM')` *errors* and drops that read,
|
|
122
|
+
so guard with `exists('NM')` when tags are optional. Only the six per-base
|
|
123
|
+
values (`qpos`, `bq`, `base`, `ref`, `dist5`, `dist3`) cost anything to
|
|
124
|
+
evaluate; everything else runs once per read and is essentially free. A syntax
|
|
125
|
+
error stops the run (exit code 2), so a typo can never silently change your
|
|
126
|
+
numbers.
|
|
127
|
+
|
|
128
|
+
The full grammar is in
|
|
129
|
+
[`docs/filter_grammar.md`](docs/filter_grammar.md), with an exhaustive
|
|
130
|
+
reference in [`docs/expression_reference.md`](docs/expression_reference.md).
|
|
131
|
+
|
|
132
|
+
## Output format
|
|
133
|
+
|
|
134
|
+
`--output-format` takes a **row template**: literal text plus `{expr}`
|
|
135
|
+
placeholders evaluated per site over the site values (`chrom`, `pos`,
|
|
136
|
+
`strand`, `motif`, `ref`, `depth`, `a c g t n`, `ins del ref_skip fail`, and
|
|
137
|
+
per-group counts `a.0` … `n.3` whenever `-e` routes into groups). Placeholders
|
|
138
|
+
run real Lua, so you can compute cells — a conversion ratio is just
|
|
139
|
+
`{t}/({c}+{t})` — and `round(x, n)` and `int(x)` are helpers for formatting:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
countmut -i x -r ref -o out \
|
|
143
|
+
--output-format "{pos+1}\t{ref}\t{a}\t{t}\t{round(t/(a+t), 4)}"
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
`{{` writes a literal `{`; a placeholder that yields nothing renders as an
|
|
147
|
+
empty cell. `--fmt-header "…"` supplies the header line (`\t`/`\n` expanded);
|
|
148
|
+
without it custom templates print no header. With no `--output-format`, the
|
|
149
|
+
default output is the per-strand `ref depth a c g t n` composition table
|
|
150
|
+
(`--vcf` instead produces an allele VCF).
|
|
151
|
+
|
|
152
|
+
## Engines and options
|
|
153
|
+
|
|
154
|
+
Two BAM-walking strategies live in the C core and emit identical output, so
|
|
155
|
+
the engine choice only affects speed (`--engine auto` uses the pileup walk for
|
|
156
|
+
the per-position counting). The options are few: input/reference/output,
|
|
157
|
+
`--region`, `--threads/-t`, `--engine`, `--strandless`, `--count-indels`,
|
|
158
|
+
`--vcf` (+ `--min-depth`/`--min-allele-support`), `-e`/`-p`, `--motif-pad`,
|
|
159
|
+
and `--output-format`/`--fmt-header`.
|
|
160
|
+
|
|
161
|
+
## Input formats
|
|
162
|
+
|
|
163
|
+
**BAM** (indexed, fast, threaded) and **SAM** (plain or gzipped — auto-transcoded
|
|
164
|
+
to a temp BAM + index, same output as the equivalent BAM) are both supported,
|
|
165
|
+
detected automatically. **CRAM** is not read by this self-contained core; convert
|
|
166
|
+
first: `samtools view -b in.cram -o out.bam` — for a CRAM with an embedded
|
|
167
|
+
reference, that conversion also works without a separate FASTA.
|
|
168
|
+
|
|
169
|
+
## Performance
|
|
170
|
+
|
|
171
|
+

|
|
172
|
+
|
|
173
|
+
Measured on a bimodal benchmark (232 k reads, 23.2 M read-bases, deep rRNA-style
|
|
174
|
+
hotspots): composition counting reads 1.30 s @1 thread → 0.31 s @16 (read-walk)
|
|
175
|
+
and 1.96 → 0.70 s (pileup); both walks are byte-identical (the dynamic work
|
|
176
|
+
queue keeps deep hotspots from serializing). Read-constant filters run once
|
|
177
|
+
per read (≈ free); only true per-base filters
|
|
178
|
+
`qpos/bq/base/ref/dist5/dist3` add cost (~0.3 s @8 threads for `bq and dist5`).
|
|
179
|
+
`tests/make_bench_bam.py` regenerates the fixture; `scripts/plot_perf.py` re-renders
|
|
180
|
+
this figure.
|
|
181
|
+
|
|
182
|
+
## License
|
|
183
|
+
|
|
184
|
+
MIT
|
countmut-0.2.5/README.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# CountMut
|
|
2
|
+
|
|
3
|
+
At each site in a modification assay you want the same small number: how many
|
|
4
|
+
reads still show the reference base, how many show the conversion, and what
|
|
5
|
+
that is as a rate. Existing tools made this hard — a flag for every QC idea,
|
|
6
|
+
two BAM-walking strategies that disagreed on deep overlapping sites, and
|
|
7
|
+
read-level filters priced once per aligned position instead of once per read.
|
|
8
|
+
|
|
9
|
+
CountMut collapses this into one design: a single base counter, QC and trimming
|
|
10
|
+
as expressions (the samtools grammar, in C), and two BAM-walks that provably
|
|
11
|
+
agree. What you *see* is an output format — the per-base table, an allele VCF,
|
|
12
|
+
or a column template you write yourself. On a deep rRNA transcriptome
|
|
13
|
+
(784 k reads / 90 Mb) a genome-wide `mapq >= 20` adds ~0.5 s, and the per-base
|
|
14
|
+
filter was cut ~3×.
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install -e .
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Quick start
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
# every base, per strand -> composition table
|
|
24
|
+
countmut -i in.bam -r ref.fa -o depth.tsv
|
|
25
|
+
|
|
26
|
+
# your own columns (e.g. an A/T conversion ratio) -> custom output template
|
|
27
|
+
countmut -i in.bam -r ref.fa -o mine.tsv \
|
|
28
|
+
--output-format "{pos+1}\t{ref}\t{a}\t{t}\t{round(t/(a+t)+0*a, 4)}" \
|
|
29
|
+
--fmt-header "pos\tref\tA\tT\trate"
|
|
30
|
+
|
|
31
|
+
# alleles as VCF -> VCF
|
|
32
|
+
countmut -i in.bam -r ref.fa --vcf -o allele.vcf
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
There is no `--mode`, no `--ref-base`/`--mut-base`: one counter, and the output
|
|
36
|
+
is whatever you choose. Bare runs print the per-strand base composition
|
|
37
|
+
(`ref depth a c g t n`); `--vcf` gives an allele VCF; `--output-format` gives
|
|
38
|
+
your own columns (a conversion ratio is just `{t}/({c}+{t})`). Output is per
|
|
39
|
+
strand by default, and `--strandless` merges the two strands.
|
|
40
|
+
|
|
41
|
+
## Filtering with one expression instead of ten flags
|
|
42
|
+
|
|
43
|
+
In RNA there is no genomic mutation to count: the "converted" base is a
|
|
44
|
+
modification read out through reverse transcription, so the conversion rate
|
|
45
|
+
reports modification level rather than a variant. Whatever your sample, the
|
|
46
|
+
QC and trimming live in one expression language — read-level rules are `-e`
|
|
47
|
+
expressions, site-level rules `-p`, in the samtools `filter=` grammar,
|
|
48
|
+
evaluated inside the C core. The old `--min-mapq` / `--trim-*` flags are gone —
|
|
49
|
+
write them as `-e` expressions.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
# quality, and not on the error-prone read ends
|
|
53
|
+
countmut -i x -r ref -o out -e "mapq >= 20 and bq >= 20 and dist5 >= 2"
|
|
54
|
+
|
|
55
|
+
# one sample
|
|
56
|
+
countmut -i x -r ref -o out -e "tag('RG') == 'sampleA'"
|
|
57
|
+
|
|
58
|
+
# samtools-style: low mismatch, not a PCR duplicate, read 1 only
|
|
59
|
+
countmut -i x -r ref -o out -e "[NM] <= 3 and not (flag.dup ~= 0) and flag.read1 != 0"
|
|
60
|
+
|
|
61
|
+
# site-level: only well-covered sites with ≥2 G reads
|
|
62
|
+
countmut -i x -r ref -o out -p "depth >= 5 and g >= 2"
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**`-e` is also a group router.** A bare boolean expression is a filter
|
|
66
|
+
(`true` → count, `nil`/`false` → drop), but an expression that returns an
|
|
67
|
+
integer `0..3` routes each kept base into that **group**; `true` routes to
|
|
68
|
+
group 0. Anything else drops the base (with a stderr warning). The split
|
|
69
|
+
shows up in `--output-format` templates as per-group cells `{a.0}` … `{n.3}`
|
|
70
|
+
(plain `{a}` stays the total over all groups):
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
# bisulfite A->G, 2-group router: group 1 = high-conversion bases, group 0 =
|
|
74
|
+
# everything else that passes the hard NS gate (low quality / read-end trim)
|
|
75
|
+
countmut -i x -r ref -o out \
|
|
76
|
+
-e "([NS] <= 1) and (([Yf] >= 1 and [Zf] <= 3 and bq >= 20 and qpos >= 2 and qlen - qpos > 2) and 1 or 0)" \
|
|
77
|
+
--output-format "{chrom}\t{pos+1}\t{strand}\t{motif}\t{a.0}\t{a.1}\t{g.0}\t{g.1}" \
|
|
78
|
+
--motif-pad 15 --fmt-header "chrom\tpos\tstrand\tmotif\tu0\tu1\tm0\tm1"
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Most filters use roughly ten variables — `mapq`, `bq` (base quality), `flags`,
|
|
82
|
+
`qpos` (position in the read), `dist5`/`dist3` (distance to the read ends),
|
|
83
|
+
`base`/`ref`, `tag('XX')`, and `rname`. A couple of things are worth knowing.
|
|
84
|
+
A missing tag is nothing: comparing `tag('NM')` *errors* and drops that read,
|
|
85
|
+
so guard with `exists('NM')` when tags are optional. Only the six per-base
|
|
86
|
+
values (`qpos`, `bq`, `base`, `ref`, `dist5`, `dist3`) cost anything to
|
|
87
|
+
evaluate; everything else runs once per read and is essentially free. A syntax
|
|
88
|
+
error stops the run (exit code 2), so a typo can never silently change your
|
|
89
|
+
numbers.
|
|
90
|
+
|
|
91
|
+
The full grammar is in
|
|
92
|
+
[`docs/filter_grammar.md`](docs/filter_grammar.md), with an exhaustive
|
|
93
|
+
reference in [`docs/expression_reference.md`](docs/expression_reference.md).
|
|
94
|
+
|
|
95
|
+
## Output format
|
|
96
|
+
|
|
97
|
+
`--output-format` takes a **row template**: literal text plus `{expr}`
|
|
98
|
+
placeholders evaluated per site over the site values (`chrom`, `pos`,
|
|
99
|
+
`strand`, `motif`, `ref`, `depth`, `a c g t n`, `ins del ref_skip fail`, and
|
|
100
|
+
per-group counts `a.0` … `n.3` whenever `-e` routes into groups). Placeholders
|
|
101
|
+
run real Lua, so you can compute cells — a conversion ratio is just
|
|
102
|
+
`{t}/({c}+{t})` — and `round(x, n)` and `int(x)` are helpers for formatting:
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
countmut -i x -r ref -o out \
|
|
106
|
+
--output-format "{pos+1}\t{ref}\t{a}\t{t}\t{round(t/(a+t), 4)}"
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`{{` writes a literal `{`; a placeholder that yields nothing renders as an
|
|
110
|
+
empty cell. `--fmt-header "…"` supplies the header line (`\t`/`\n` expanded);
|
|
111
|
+
without it custom templates print no header. With no `--output-format`, the
|
|
112
|
+
default output is the per-strand `ref depth a c g t n` composition table
|
|
113
|
+
(`--vcf` instead produces an allele VCF).
|
|
114
|
+
|
|
115
|
+
## Engines and options
|
|
116
|
+
|
|
117
|
+
Two BAM-walking strategies live in the C core and emit identical output, so
|
|
118
|
+
the engine choice only affects speed (`--engine auto` uses the pileup walk for
|
|
119
|
+
the per-position counting). The options are few: input/reference/output,
|
|
120
|
+
`--region`, `--threads/-t`, `--engine`, `--strandless`, `--count-indels`,
|
|
121
|
+
`--vcf` (+ `--min-depth`/`--min-allele-support`), `-e`/`-p`, `--motif-pad`,
|
|
122
|
+
and `--output-format`/`--fmt-header`.
|
|
123
|
+
|
|
124
|
+
## Input formats
|
|
125
|
+
|
|
126
|
+
**BAM** (indexed, fast, threaded) and **SAM** (plain or gzipped — auto-transcoded
|
|
127
|
+
to a temp BAM + index, same output as the equivalent BAM) are both supported,
|
|
128
|
+
detected automatically. **CRAM** is not read by this self-contained core; convert
|
|
129
|
+
first: `samtools view -b in.cram -o out.bam` — for a CRAM with an embedded
|
|
130
|
+
reference, that conversion also works without a separate FASTA.
|
|
131
|
+
|
|
132
|
+
## Performance
|
|
133
|
+
|
|
134
|
+

|
|
135
|
+
|
|
136
|
+
Measured on a bimodal benchmark (232 k reads, 23.2 M read-bases, deep rRNA-style
|
|
137
|
+
hotspots): composition counting reads 1.30 s @1 thread → 0.31 s @16 (read-walk)
|
|
138
|
+
and 1.96 → 0.70 s (pileup); both walks are byte-identical (the dynamic work
|
|
139
|
+
queue keeps deep hotspots from serializing). Read-constant filters run once
|
|
140
|
+
per read (≈ free); only true per-base filters
|
|
141
|
+
`qpos/bq/base/ref/dist5/dist3` add cost (~0.3 s @8 threads for `bq and dist5`).
|
|
142
|
+
`tests/make_bench_bam.py` regenerates the fixture; `scripts/plot_perf.py` re-renders
|
|
143
|
+
this figure.
|
|
144
|
+
|
|
145
|
+
## License
|
|
146
|
+
|
|
147
|
+
MIT
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Makefile for the countmut C core (htslib-subset backend, from lh3/minipileup).
|
|
2
|
+
#
|
|
3
|
+
# Builds `countmut_core`, a self-contained binary that does all the counting.
|
|
4
|
+
CC ?= cc
|
|
5
|
+
LUA_CFLAGS := $(shell pkg-config --cflags lua5.4 2>/dev/null)
|
|
6
|
+
ifeq ($(LUA_CFLAGS),)
|
|
7
|
+
LUA_CFLAGS := -I/usr/include/lua5.4
|
|
8
|
+
endif
|
|
9
|
+
LUA_LIBS := $(shell pkg-config --libs lua5.4 2>/dev/null)
|
|
10
|
+
ifeq ($(LUA_LIBS),)
|
|
11
|
+
LUA_LIBS := -llua5.4
|
|
12
|
+
endif
|
|
13
|
+
CFLAGS ?= -O3 -Wall -Wno-unused-result -pthread $(LUA_CFLAGS)
|
|
14
|
+
OBJS = razf.o bgzf.o hts.o sam.o faidx.o bedidx.o countmut_expr.o countmut_core.o countmut_core_main.o
|
|
15
|
+
LIBS = -lpthread -lz -lm $(LUA_LIBS)
|
|
16
|
+
PROG = countmut_core
|
|
17
|
+
|
|
18
|
+
.PHONY: all clean asan
|
|
19
|
+
|
|
20
|
+
all: $(PROG)
|
|
21
|
+
|
|
22
|
+
$(PROG): $(OBJS)
|
|
23
|
+
$(CC) $(CFLAGS) -o $@ $^ $(LIBS)
|
|
24
|
+
|
|
25
|
+
# AddressSanitizer build for leak/UB auditing
|
|
26
|
+
asan:
|
|
27
|
+
$(MAKE) clean
|
|
28
|
+
$(MAKE) CFLAGS="-O1 -g -Wall -fsanitize=address -fno-omit-frame-pointer $(LUA_CFLAGS)" LIBS="-lpthread -lz -lm -fsanitize=address $(LUA_LIBS)" $(PROG)
|
|
29
|
+
|
|
30
|
+
.c.o:
|
|
31
|
+
$(CC) $(CFLAGS) -c $< -o $@
|
|
32
|
+
|
|
33
|
+
clean:
|
|
34
|
+
rm -f $(PROG) *.o
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
#include <stdlib.h>
|
|
2
|
+
#include <stdint.h>
|
|
3
|
+
#include <string.h>
|
|
4
|
+
#include <stdio.h>
|
|
5
|
+
#include <zlib.h>
|
|
6
|
+
|
|
7
|
+
#include "ksort.h"
|
|
8
|
+
KSORT_INIT_GENERIC(uint64_t)
|
|
9
|
+
|
|
10
|
+
#include "kseq.h"
|
|
11
|
+
KSTREAM_INIT(gzFile, gzread, 8192)
|
|
12
|
+
|
|
13
|
+
typedef struct {
|
|
14
|
+
int n, m;
|
|
15
|
+
uint64_t *a;
|
|
16
|
+
int *idx;
|
|
17
|
+
} bed_reglist_t;
|
|
18
|
+
|
|
19
|
+
#include "khash.h"
|
|
20
|
+
KHASH_MAP_INIT_STR(reg, bed_reglist_t)
|
|
21
|
+
|
|
22
|
+
#define LIDX_SHIFT 13
|
|
23
|
+
|
|
24
|
+
typedef kh_reg_t reghash_t;
|
|
25
|
+
|
|
26
|
+
int *bed_index_core(int n, uint64_t *a, int *n_idx)
|
|
27
|
+
{
|
|
28
|
+
int i, j, m, *idx;
|
|
29
|
+
m = *n_idx = 0; idx = 0;
|
|
30
|
+
for (i = 0; i < n; ++i) {
|
|
31
|
+
int beg, end;
|
|
32
|
+
beg = a[i]>>32 >> LIDX_SHIFT; end = ((uint32_t)a[i]) >> LIDX_SHIFT;
|
|
33
|
+
if (m < end + 1) {
|
|
34
|
+
int oldm = m;
|
|
35
|
+
m = end + 1;
|
|
36
|
+
kroundup32(m);
|
|
37
|
+
idx = (int*)realloc(idx, m * sizeof(int));
|
|
38
|
+
for (j = oldm; j < m; ++j) idx[j] = -1;
|
|
39
|
+
}
|
|
40
|
+
if (beg == end) {
|
|
41
|
+
if (idx[beg] < 0) idx[beg] = i;
|
|
42
|
+
} else {
|
|
43
|
+
for (j = beg; j <= end; ++j)
|
|
44
|
+
if (idx[j] < 0) idx[j] = i;
|
|
45
|
+
}
|
|
46
|
+
*n_idx = end + 1;
|
|
47
|
+
}
|
|
48
|
+
return idx;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
void bed_index(void *_h)
|
|
52
|
+
{
|
|
53
|
+
reghash_t *h = (reghash_t*)_h;
|
|
54
|
+
khint_t k;
|
|
55
|
+
for (k = 0; k < kh_end(h); ++k) {
|
|
56
|
+
if (kh_exist(h, k)) {
|
|
57
|
+
bed_reglist_t *p = &kh_val(h, k);
|
|
58
|
+
if (p->idx) free(p->idx);
|
|
59
|
+
ks_introsort(uint64_t, p->n, p->a);
|
|
60
|
+
p->idx = bed_index_core(p->n, p->a, &p->m);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
int bed_overlap_core(const bed_reglist_t *p, int beg, int end)
|
|
66
|
+
{
|
|
67
|
+
int i, min_off;
|
|
68
|
+
if (p->n == 0) return 0;
|
|
69
|
+
min_off = (beg>>LIDX_SHIFT >= p->n)? p->idx[p->n-1] : p->idx[beg>>LIDX_SHIFT];
|
|
70
|
+
if (min_off < 0) { // TODO: this block can be improved, but speed should not matter too much here
|
|
71
|
+
int n = beg>>LIDX_SHIFT;
|
|
72
|
+
if (n > p->n) n = p->n;
|
|
73
|
+
for (i = n - 1; i >= 0; --i)
|
|
74
|
+
if (p->idx[i] >= 0) break;
|
|
75
|
+
min_off = i >= 0? p->idx[i] : 0;
|
|
76
|
+
}
|
|
77
|
+
for (i = min_off; i < p->n; ++i) {
|
|
78
|
+
if ((int)(p->a[i]>>32) >= end) break; // out of range; no need to proceed
|
|
79
|
+
if ((int32_t)p->a[i] > beg && (int32_t)(p->a[i]>>32) < end)
|
|
80
|
+
return 1; // find the overlap; return
|
|
81
|
+
}
|
|
82
|
+
return 0;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
int bed_overlap(const void *_h, const char *chr, int beg, int end)
|
|
86
|
+
{
|
|
87
|
+
const reghash_t *h = (const reghash_t*)_h;
|
|
88
|
+
khint_t k;
|
|
89
|
+
if (!h) return 0;
|
|
90
|
+
k = kh_get(reg, h, chr);
|
|
91
|
+
if (k == kh_end(h)) return 0;
|
|
92
|
+
return bed_overlap_core(&kh_val(h, k), beg, end);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
void *bed_read(const char *fn)
|
|
96
|
+
{
|
|
97
|
+
reghash_t *h = kh_init(reg);
|
|
98
|
+
gzFile fp;
|
|
99
|
+
kstream_t *ks;
|
|
100
|
+
int dret;
|
|
101
|
+
kstring_t *str;
|
|
102
|
+
// read the list
|
|
103
|
+
fp = strcmp(fn, "-")? gzopen(fn, "r") : gzdopen(fileno(stdin), "r");
|
|
104
|
+
if (fp == 0) return 0;
|
|
105
|
+
str = (kstring_t*)calloc(1, sizeof(kstring_t));
|
|
106
|
+
ks = ks_init(fp);
|
|
107
|
+
while (ks_getuntil(ks, 0, str, &dret) >= 0) { // read the chr name
|
|
108
|
+
int beg = -1, end = -1;
|
|
109
|
+
bed_reglist_t *p;
|
|
110
|
+
khint_t k = kh_get(reg, h, str->s);
|
|
111
|
+
if (k == kh_end(h)) { // absent from the hash table
|
|
112
|
+
int ret;
|
|
113
|
+
char *s = strdup(str->s);
|
|
114
|
+
k = kh_put(reg, h, s, &ret);
|
|
115
|
+
memset(&kh_val(h, k), 0, sizeof(bed_reglist_t));
|
|
116
|
+
}
|
|
117
|
+
p = &kh_val(h, k);
|
|
118
|
+
if (dret != '\n') { // if the lines has other characters
|
|
119
|
+
if (ks_getuntil(ks, 0, str, &dret) > 0 && isdigit(str->s[0])) {
|
|
120
|
+
beg = atoi(str->s); // begin
|
|
121
|
+
if (dret != '\n') {
|
|
122
|
+
if (ks_getuntil(ks, 0, str, &dret) > 0 && isdigit(str->s[0])) {
|
|
123
|
+
end = atoi(str->s); // end
|
|
124
|
+
if (end < beg) end = -1;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (dret != '\n') while ((dret = ks_getc(ks)) > 0 && dret != '\n'); // skip the rest of the line
|
|
130
|
+
if (end < 0 && beg > 0) end = beg, beg = beg - 1; // if there is only one column
|
|
131
|
+
if (beg >= 0 && end > beg) {
|
|
132
|
+
if (p->n == p->m) {
|
|
133
|
+
p->m = p->m? p->m<<1 : 4;
|
|
134
|
+
p->a = (uint64_t*)realloc(p->a, p->m * 8);
|
|
135
|
+
}
|
|
136
|
+
p->a[p->n++] = (uint64_t)beg<<32 | end;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
ks_destroy(ks);
|
|
140
|
+
gzclose(fp);
|
|
141
|
+
free(str->s); free(str);
|
|
142
|
+
bed_index(h);
|
|
143
|
+
return h;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
void bed_destroy(void *_h)
|
|
147
|
+
{
|
|
148
|
+
reghash_t *h = (reghash_t*)_h;
|
|
149
|
+
khint_t k;
|
|
150
|
+
for (k = 0; k < kh_end(h); ++k) {
|
|
151
|
+
if (kh_exist(h, k)) {
|
|
152
|
+
free(kh_val(h, k).a);
|
|
153
|
+
free(kh_val(h, k).idx);
|
|
154
|
+
free((char*)kh_key(h, k));
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
kh_destroy(reg, h);
|
|
158
|
+
}
|