amplitrim 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. amplitrim-0.1.0/CHANGELOG.md +13 -0
  2. amplitrim-0.1.0/CITATION.cff +21 -0
  3. amplitrim-0.1.0/CONTRIBUTING.md +26 -0
  4. amplitrim-0.1.0/LICENSE +21 -0
  5. amplitrim-0.1.0/MANIFEST.in +4 -0
  6. amplitrim-0.1.0/PKG-INFO +157 -0
  7. amplitrim-0.1.0/README.md +119 -0
  8. amplitrim-0.1.0/amplitrim/__init__.py +1 -0
  9. amplitrim-0.1.0/amplitrim/cli.py +52 -0
  10. amplitrim-0.1.0/amplitrim/config.py +42 -0
  11. amplitrim-0.1.0/amplitrim/merge_wrapper.py +115 -0
  12. amplitrim-0.1.0/amplitrim/pipeline.py +474 -0
  13. amplitrim-0.1.0/amplitrim/ranking.py +125 -0
  14. amplitrim-0.1.0/amplitrim/report.py +610 -0
  15. amplitrim-0.1.0/amplitrim/subsample.py +110 -0
  16. amplitrim-0.1.0/amplitrim/trimmer.py +270 -0
  17. amplitrim-0.1.0/amplitrim.egg-info/PKG-INFO +157 -0
  18. amplitrim-0.1.0/amplitrim.egg-info/SOURCES.txt +41 -0
  19. amplitrim-0.1.0/amplitrim.egg-info/dependency_links.txt +1 -0
  20. amplitrim-0.1.0/amplitrim.egg-info/entry_points.txt +2 -0
  21. amplitrim-0.1.0/amplitrim.egg-info/requires.txt +9 -0
  22. amplitrim-0.1.0/amplitrim.egg-info/top_level.txt +1 -0
  23. amplitrim-0.1.0/docs/.nojekyll +0 -0
  24. amplitrim-0.1.0/docs/index.html +258 -0
  25. amplitrim-0.1.0/docs/usage.md +213 -0
  26. amplitrim-0.1.0/pyproject.toml +53 -0
  27. amplitrim-0.1.0/setup.cfg +4 -0
  28. amplitrim-0.1.0/tests/conftest.py +76 -0
  29. amplitrim-0.1.0/tests/fake_vsearch.py +48 -0
  30. amplitrim-0.1.0/tests/fixtures/vsearch_mergepairs_log.txt +28 -0
  31. amplitrim-0.1.0/tests/simulate.py +140 -0
  32. amplitrim-0.1.0/tests/test_cli.py +75 -0
  33. amplitrim-0.1.0/tests/test_example.py +58 -0
  34. amplitrim-0.1.0/tests/test_filter.py +302 -0
  35. amplitrim-0.1.0/tests/test_input.py +98 -0
  36. amplitrim-0.1.0/tests/test_its.py +121 -0
  37. amplitrim-0.1.0/tests/test_maxee.py +168 -0
  38. amplitrim-0.1.0/tests/test_merge.py +74 -0
  39. amplitrim-0.1.0/tests/test_metadata.py +133 -0
  40. amplitrim-0.1.0/tests/test_ranking.py +98 -0
  41. amplitrim-0.1.0/tests/test_report.py +384 -0
  42. amplitrim-0.1.0/tests/test_search.py +141 -0
  43. amplitrim-0.1.0/tests/test_site.py +243 -0
@@ -0,0 +1,13 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0
4
+
5
+ First release.
6
+
7
+ - Recommends DADA2 `truncLenF`, `truncLenR` and `maxEE` for paired-end amplicon reads. It measures amplicon lengths from the reads, replicates DADA2's quality filter exactly for every forward/reverse split, and tests real merges with vsearch.
8
+ - `maxEE` is recommended, not given: per read, the limit that covers three quarters of the reads at the recommended cut (never below 2, with no upper limit; the report notes a limit above 5); clean reads stay at 2.
9
+ - `--its` for amplicons that vary in length (no fixed truncation length; each read is still trimmed at its first base of quality 2 or below). Its report shows where the pairs go and what the forward reads alone would keep.
10
+ - Three required arguments and two optional ones (`--its`, `--verbose`).
11
+ - One runtime dependency, numpy. matplotlib is optional (`pip install "amplitrim[plot]"`) and only draws `summary.png`: without it the report and `summary.json` are written as usual.
12
+ - A command that makes a small simulated dataset for trying the tool (`tests/simulate.py`; see the README). The evidence for each recommendation is on the [validation page](https://github.com/paytonyau/amplitrim-validation/blob/main/docs/validation.md) of the validation archive.
13
+ - Report (`REPORT.md`, `summary.png`, `summary.json`) with copy-paste snippets for DADA2 and QIIME 2. `summary.json` is meant for scripts and workflow managers: it has a `schema_version`, always-numeric `maxEE_f` and `maxEE_r`, and a `maxEE_rule` that says whether `maxEE` was relaxed.
@@ -0,0 +1,21 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use AmpliTrim, please cite it as below."
3
+ title: "AmpliTrim: choosing DADA2 truncation lengths and maxEE from the reads themselves"
4
+ type: software
5
+ version: "0.1.0"
6
+ license: MIT
7
+ repository-code: "https://github.com/paytonyau/amplitrim"
8
+ abstract: >-
9
+ AmpliTrim recommends DADA2 truncLenF, truncLenR and maxEE for paired-end amplicon reads and tests candidate settings on the user's own reads: it replicates DADA2's
10
+ quality filter from the quality scores, sets a coverage floor from the amplicon lengths, estimates the merge yield with vsearch, and chooses maxEE per read from the reads' expected errors.
11
+ authors:
12
+ - family-names: "Yau"
13
+ given-names: "Payton (Tung On)"
14
+ orcid: "https://orcid.org/0000-0002-3283-0370"
15
+ keywords:
16
+ - DADA2
17
+ - amplicon sequencing
18
+ - 16S rRNA
19
+ - ITS
20
+ - paired-end reads
21
+ # add when releasing: date-released, and doi (from the archived release)
@@ -0,0 +1,26 @@
1
+ # Contributing
2
+
3
+ Bug reports, questions and improvements are welcome. Use the repository's [issues](https://github.com/paytonyau/amplitrim/issues) for all of them.
4
+
5
+ ## Reporting a problem or asking a question
6
+
7
+ Please include:
8
+
9
+ - the output of `amplitrim --version`, your operating system, and the version of vsearch (`vsearch --version`);
10
+ - the command you ran, and what you expected;
11
+ - `REPORT.md` and `summary.json` from the run, if it got that far. They contain no reads. Please do not attach your FASTQ files.
12
+
13
+ If the tool recommends something that looks wrong for your data, that is worth reporting even if it did not fail.
14
+
15
+ ## Changing the code
16
+
17
+ 1. Fork the repository and install it with the test extras: `pip install -e ".[test]"`. vsearch must be on your PATH for the tests that run it; the others run without it.
18
+ 2. Run `python -m pytest tests -q` before and after your change.
19
+ 3. Add a test for what you change, and keep `README.md` and `docs/usage.md` in step with the behaviour.
20
+ 4. Open a pull request that says what it changes and why.
21
+
22
+ People use AmpliTrim's recommendations to choose settings. A change to **what it recommends** (not to how it is run or reported) therefore needs evidence on data, in the way the [validation page](https://github.com/paytonyau/amplitrim-validation/blob/main/docs/validation.md) gives it. Write the criteria down before the run, and report the outcome whether or not it holds. The validation scripts are not in this repository; they are in the [validation archive](https://github.com/paytonyau/amplitrim-validation).
23
+
24
+ ## Conduct
25
+
26
+ Be courteous and assume good faith. Criticism of the code and of the validation is welcome and is what the tests and the written criteria are for.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Payton Yau
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,4 @@
1
+ include LICENSE README.md CHANGELOG.md CITATION.cff CONTRIBUTING.md
2
+ graft docs
3
+ graft tests
4
+ global-exclude __pycache__ *.py[cod] .DS_Store
@@ -0,0 +1,157 @@
1
+ Metadata-Version: 2.4
2
+ Name: amplitrim
3
+ Version: 0.1.0
4
+ Summary: Recommends DADA2 truncLenF, truncLenR and maxEE for paired-end amplicon reads by testing candidate settings on your own reads.
5
+ Author: Payton (Tung On) Yau
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/paytonyau/amplitrim
8
+ Project-URL: Documentation, https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md
9
+ Project-URL: Source, https://github.com/paytonyau/amplitrim
10
+ Project-URL: Issues, https://github.com/paytonyau/amplitrim/issues
11
+ Project-URL: Changelog, https://github.com/paytonyau/amplitrim/blob/main/CHANGELOG.md
12
+ Project-URL: Citation, https://github.com/paytonyau/amplitrim/blob/main/CITATION.cff
13
+ Project-URL: Validation, https://github.com/paytonyau/amplitrim-validation
14
+ Keywords: DADA2,amplicon sequencing,16S rRNA,ITS,paired-end reads,truncLen,maxEE
15
+ Classifier: Development Status :: 4 - Beta
16
+ Classifier: Environment :: Console
17
+ Classifier: Intended Audience :: Science/Research
18
+ Classifier: Natural Language :: English
19
+ Classifier: Operating System :: OS Independent
20
+ Classifier: Programming Language :: Python :: 3
21
+ Classifier: Programming Language :: Python :: 3 :: Only
22
+ Classifier: Programming Language :: Python :: 3.9
23
+ Classifier: Programming Language :: Python :: 3.10
24
+ Classifier: Programming Language :: Python :: 3.11
25
+ Classifier: Programming Language :: Python :: 3.12
26
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
27
+ Requires-Python: >=3.9
28
+ Description-Content-Type: text/markdown
29
+ License-File: LICENSE
30
+ Requires-Dist: numpy>=1.24
31
+ Provides-Extra: plot
32
+ Requires-Dist: matplotlib>=3.7; extra == "plot"
33
+ Provides-Extra: test
34
+ Requires-Dist: pytest; extra == "test"
35
+ Requires-Dist: pandas>=2.0; extra == "test"
36
+ Requires-Dist: matplotlib>=3.7; extra == "test"
37
+ Dynamic: license-file
38
+
39
+ # AmpliTrim
40
+
41
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/)
42
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/paytonyau/amplitrim/blob/main/LICENSE)
43
+ [![Tests](https://github.com/paytonyau/amplitrim/actions/workflows/tests.yml/badge.svg)](https://github.com/paytonyau/amplitrim/actions/workflows/tests.yml)
44
+
45
+ **Choose DADA2 truncation lengths and `maxEE` from your own reads.**
46
+
47
+ AmpliTrim is for **Illumina paired-end amplicon reads** (tested on 16S and ITS data from MiSeq and NovaSeq runs). In DADA2 you pick `truncLenF`, `truncLenR` and `maxEE`. Cut too short, and your longest amplicons cannot merge. Cut too long, and noisy read ends cost you reads. Most people choose by eye from quality plots. AmpliTrim tests candidate settings on your own reads, recommends the best, and shows why. It does not change your files: it only recommends settings.
48
+
49
+ ```
50
+ FASTQ → cutadapt → AmpliTrim → DADA2
51
+ ```
52
+
53
+ Run AmpliTrim after primer removal and before DADA2 (in R or in QIIME 2). It needs paired reads (R1 and R2), not long reads or single-end data.
54
+
55
+ ## Why use it
56
+
57
+ - **It uses every read pair.** After primer removal, reads often differ in length. AmpliTrim does not need them equal, so nothing is thrown away.
58
+ - **It predicts the result.** It replicates DADA2's quality filter exactly and merges a sample of your pairs with vsearch, so you see how many will pass and merge.
59
+ - **It reads your data, not a table.** It measures your amplicon lengths, checks that your longest amplicons can merge, and handles ITS and other regions where lengths vary.
60
+ - **It is ready to use.** You get copy-paste settings for DADA2, in R or in QIIME 2's `dada2 denoise-paired`, and a stable `summary.json` for pipelines.
61
+
62
+ ## How it compares
63
+
64
+ We compared AmpliTrim with an existing automatic tool for choosing truncation lengths, in real DADA2 runs on public datasets and at the same `maxEE`.
65
+
66
+ | | AmpliTrim | Existing tool |
67
+ |---|---|---|
68
+ | Read pairs it can use | all | 8% to 98% (it needs reads of one length) |
69
+ | Pairs merged by DADA2 with its cut | real samples (4 datasets): ahead on 2, tied on 2 (mean +0.6 points)<br>mock communities (5 datasets): ahead on 2, tied on 3 (mean +0.6 points)<br>wastewater (1 dataset): behind by 1.0 point\* | the reference |
70
+ | Error in the predicted filter pass rate | 0.8 points at most (mean 0.2) | 0.2 to 7.7 points |
71
+ | ITS reads | supported | cannot be run |
72
+
73
+ \* Wastewater is shown on its own. Its reads are noisy: it is the only real dataset in this comparison with poor read quality, an estimated 54% of its pairs pass DADA2's default filter, and the loss is to read quality, not to short reads (only 0.6% of its pairs are shorter than the cut). Its reads were also trimmed to mixed lengths, and every pair was in reverse orientation. It is the one case where the other tool was ahead: AmpliTrim chose a cut 6 nt longer, which cost about a point.
74
+
75
+ Merging is a tie: only mock community B (+2.6) is beyond the 2-point margin we set in advance, and the two datasets used in development (mock A and soil V4) both tied. The gains are in the other rows.
76
+
77
+ <details>
78
+ <summary>Per dataset: pairs merged by DADA2, AmpliTrim's cut minus the other tool's (points)</summary>
79
+
80
+ | Type | Dataset (16S) | Samples | Role | Difference |
81
+ |---|---|---|---|---|
82
+ | Real samples | V1-V2 real samples | 33 | held out | +1.1 |
83
+ | Real samples | V4 soil | 36 | development | 0.0 |
84
+ | Real samples | V3-V4 mouse gut with spike-ins | 12 | held out | +1.3 |
85
+ | Real samples | V3-V4 surface swabs | 36 | held out | 0.0 |
86
+ | Mock communities | V1-V3 mock community A | 6 | development | 0.0 |
87
+ | Mock communities | V1-V2 inter-laboratory mock | 6 | held out | 0.0 |
88
+ | Mock communities | V1-V3 mock community B | 8 | held out | +2.6 |
89
+ | Mock communities | V4-V6 inter-laboratory mock | 5 | held out | +0.3 |
90
+ | Mock communities | V3-V4 ZymoBIOMICS mock (D6300) | 10 | held out | 0.0 |
91
+ | Wastewater | V3-V4 wastewater | 9 | held out | -1.0 |
92
+
93
+ </details>
94
+
95
+ What held, what did not, and the limits are on the [validation page](https://github.com/paytonyau/amplitrim-validation/blob/main/docs/validation.md), in the [archive](https://github.com/paytonyau/amplitrim-validation) that also holds the scripts and the criteria written down before each run.
96
+
97
+ ## Quick start
98
+
99
+ ```bash
100
+ pip install amplitrim # Python 3.9 or later; vsearch must also be on your PATH
101
+ amplitrim --input-dir reads/ --work-dir work --out-dir report
102
+ ```
103
+
104
+ Example output (from simulated reads; your numbers will differ):
105
+
106
+ ```
107
+ AmpliTrim 0.1.0
108
+ Analysing reads...
109
+
110
+ Recommended DADA2 settings
111
+ truncLenF = 230 truncLenR = 216 maxEE = 2 minOverlap = 12
112
+
113
+ 89.3% of the 3,000 read pairs analysed should pass filtering and merge.
114
+ Covers amplicons up to 434 nt (your longest: 434 nt).
115
+
116
+ Report: report/REPORT.md
117
+ ```
118
+
119
+ `report/REPORT.md` has the settings with copy-paste snippets for DADA2 in R and in QIIME 2's `dada2 denoise-paired`, what to expect, and anything worth checking. To try it without your own data, make a small simulated dataset (from a clone of this repository; the generator needs numpy):
120
+
121
+ ```bash
122
+ python tests/simulate.py demo --samples 2 --pairs 1500 --seed 11
123
+ amplitrim --input-dir demo/primer_trimmed --work-dir work --out-dir out
124
+ ```
125
+
126
+ These reads are simulated, so use them to see what the report looks like, not to judge the tool. Expect lengths that cover the longest amplicon (434 nt), a `maxEE` of 2, and more than 70% of the pairs passing and merging.
127
+
128
+ ## What you need
129
+
130
+ - **Python 3.9 or later** and **[vsearch](https://github.com/torognes/vsearch)** on your `PATH` ([install steps](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#install), including Windows).
131
+ - **Paired FASTQ files** in one folder, named like `sample_R1.fastq.gz` and `sample_R2.fastq.gz`.
132
+ - **Primers and adapters already removed** (for example with cutadapt), and no quality trimming. R1 must be the forward read ([details](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#input)).
133
+
134
+ ## Options
135
+
136
+ Required: `--input-dir`, `--work-dir` and `--out-dir`. Optional: `--its` (for amplicons that vary a lot in length, such as ITS) and `--verbose`. AmpliTrim chooses everything else, including `maxEE` ([user guide](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#arguments)).
137
+
138
+ ## How the settings are chosen
139
+
140
+ | Setting | How |
141
+ |---|---|
142
+ | `truncLenF`, `truncLenR` | The shortest total that can merge your longest amplicons (the amplicon plus 12), split between the two reads where quality allows. Real merges are tested with vsearch |
143
+ | `maxEE` | 2 (DADA2's default) for clean reads. For noisy reads, a limit for each read that covers three quarters of its reads, never below 2, with a note if it goes above 5 ([how](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#how-maxee-is-chosen)) |
144
+ | `minOverlap` | 12 (DADA2's default) |
145
+
146
+ The outputs are `REPORT.md` (read this one), `summary.json` (for scripts; [every key](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#summaryjson-for-scripts-and-workflow-managers)) and, if matplotlib is installed (`pip install "amplitrim[plot]"`), `summary.png` (a figure of your amplicon lengths and how errors build up).
147
+
148
+ ## Limits
149
+
150
+ - **It is for DADA2.** QIIME 2's other denoiser, Deblur, uses different settings (one trim length for merged reads), which AmpliTrim does not recommend ([guide](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#what-amplitrim-recommends)).
151
+ - **The yield is an optimistic estimate.** It is measured before DADA2 denoises the reads. Use it to compare settings, and run DADA2 once to confirm.
152
+ - **It checks quality and mergeability, not accuracy.**
153
+ - **A tail of much longer amplicons** (for example non-target 18S) can force long, noisy reads. Only the longest 0.2% are set aside, so remove such reads first ([guide](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#limitations)).
154
+
155
+ ## Citation, licence and contributing
156
+
157
+ MIT (see `LICENSE`). Please cite AmpliTrim (`CITATION.cff`) and [vsearch](https://github.com/torognes/vsearch). vsearch (GPLv3 and BSD-2-Clause) runs as a separate program, and its own licence applies. numpy and matplotlib are installed separately. For bug reports, questions and contributions, see [`CONTRIBUTING.md`](https://github.com/paytonyau/amplitrim/blob/main/CONTRIBUTING.md).
@@ -0,0 +1,119 @@
1
+ # AmpliTrim
2
+
3
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/paytonyau/amplitrim/blob/main/LICENSE)
5
+ [![Tests](https://github.com/paytonyau/amplitrim/actions/workflows/tests.yml/badge.svg)](https://github.com/paytonyau/amplitrim/actions/workflows/tests.yml)
6
+
7
+ **Choose DADA2 truncation lengths and `maxEE` from your own reads.**
8
+
9
+ AmpliTrim is for **Illumina paired-end amplicon reads** (tested on 16S and ITS data from MiSeq and NovaSeq runs). In DADA2 you pick `truncLenF`, `truncLenR` and `maxEE`. Cut too short, and your longest amplicons cannot merge. Cut too long, and noisy read ends cost you reads. Most people choose by eye from quality plots. AmpliTrim tests candidate settings on your own reads, recommends the best, and shows why. It does not change your files: it only recommends settings.
10
+
11
+ ```
12
+ FASTQ → cutadapt → AmpliTrim → DADA2
13
+ ```
14
+
15
+ Run AmpliTrim after primer removal and before DADA2 (in R or in QIIME 2). It needs paired reads (R1 and R2), not long reads or single-end data.
16
+
17
+ ## Why use it
18
+
19
+ - **It uses every read pair.** After primer removal, reads often differ in length. AmpliTrim does not need them equal, so nothing is thrown away.
20
+ - **It predicts the result.** It replicates DADA2's quality filter exactly and merges a sample of your pairs with vsearch, so you see how many will pass and merge.
21
+ - **It reads your data, not a table.** It measures your amplicon lengths, checks that your longest amplicons can merge, and handles ITS and other regions where lengths vary.
22
+ - **It is ready to use.** You get copy-paste settings for DADA2, in R or in QIIME 2's `dada2 denoise-paired`, and a stable `summary.json` for pipelines.
23
+
24
+ ## How it compares
25
+
26
+ We compared AmpliTrim with an existing automatic tool for choosing truncation lengths, in real DADA2 runs on public datasets and at the same `maxEE`.
27
+
28
+ | | AmpliTrim | Existing tool |
29
+ |---|---|---|
30
+ | Read pairs it can use | all | 8% to 98% (it needs reads of one length) |
31
+ | Pairs merged by DADA2 with its cut | real samples (4 datasets): ahead on 2, tied on 2 (mean +0.6 points)<br>mock communities (5 datasets): ahead on 2, tied on 3 (mean +0.6 points)<br>wastewater (1 dataset): behind by 1.0 point\* | the reference |
32
+ | Error in the predicted filter pass rate | 0.8 points at most (mean 0.2) | 0.2 to 7.7 points |
33
+ | ITS reads | supported | cannot be run |
34
+
35
+ \* Wastewater is shown on its own. Its reads are noisy: it is the only real dataset in this comparison with poor read quality, an estimated 54% of its pairs pass DADA2's default filter, and the loss is to read quality, not to short reads (only 0.6% of its pairs are shorter than the cut). Its reads were also trimmed to mixed lengths, and every pair was in reverse orientation. It is the one case where the other tool was ahead: AmpliTrim chose a cut 6 nt longer, which cost about a point.
36
+
37
+ Merging is a tie: only mock community B (+2.6) is beyond the 2-point margin we set in advance, and the two datasets used in development (mock A and soil V4) both tied. The gains are in the other rows.
38
+
39
+ <details>
40
+ <summary>Per dataset: pairs merged by DADA2, AmpliTrim's cut minus the other tool's (points)</summary>
41
+
42
+ | Type | Dataset (16S) | Samples | Role | Difference |
43
+ |---|---|---|---|---|
44
+ | Real samples | V1-V2 real samples | 33 | held out | +1.1 |
45
+ | Real samples | V4 soil | 36 | development | 0.0 |
46
+ | Real samples | V3-V4 mouse gut with spike-ins | 12 | held out | +1.3 |
47
+ | Real samples | V3-V4 surface swabs | 36 | held out | 0.0 |
48
+ | Mock communities | V1-V3 mock community A | 6 | development | 0.0 |
49
+ | Mock communities | V1-V2 inter-laboratory mock | 6 | held out | 0.0 |
50
+ | Mock communities | V1-V3 mock community B | 8 | held out | +2.6 |
51
+ | Mock communities | V4-V6 inter-laboratory mock | 5 | held out | +0.3 |
52
+ | Mock communities | V3-V4 ZymoBIOMICS mock (D6300) | 10 | held out | 0.0 |
53
+ | Wastewater | V3-V4 wastewater | 9 | held out | -1.0 |
54
+
55
+ </details>
56
+
57
+ What held, what did not, and the limits are on the [validation page](https://github.com/paytonyau/amplitrim-validation/blob/main/docs/validation.md), in the [archive](https://github.com/paytonyau/amplitrim-validation) that also holds the scripts and the criteria written down before each run.
58
+
59
+ ## Quick start
60
+
61
+ ```bash
62
+ pip install amplitrim # Python 3.9 or later; vsearch must also be on your PATH
63
+ amplitrim --input-dir reads/ --work-dir work --out-dir report
64
+ ```
65
+
66
+ Example output (from simulated reads; your numbers will differ):
67
+
68
+ ```
69
+ AmpliTrim 0.1.0
70
+ Analysing reads...
71
+
72
+ Recommended DADA2 settings
73
+ truncLenF = 230 truncLenR = 216 maxEE = 2 minOverlap = 12
74
+
75
+ 89.3% of the 3,000 read pairs analysed should pass filtering and merge.
76
+ Covers amplicons up to 434 nt (your longest: 434 nt).
77
+
78
+ Report: report/REPORT.md
79
+ ```
80
+
81
+ `report/REPORT.md` has the settings with copy-paste snippets for DADA2 in R and in QIIME 2's `dada2 denoise-paired`, what to expect, and anything worth checking. To try it without your own data, make a small simulated dataset (from a clone of this repository; the generator needs numpy):
82
+
83
+ ```bash
84
+ python tests/simulate.py demo --samples 2 --pairs 1500 --seed 11
85
+ amplitrim --input-dir demo/primer_trimmed --work-dir work --out-dir out
86
+ ```
87
+
88
+ These reads are simulated, so use them to see what the report looks like, not to judge the tool. Expect lengths that cover the longest amplicon (434 nt), a `maxEE` of 2, and more than 70% of the pairs passing and merging.
89
+
90
+ ## What you need
91
+
92
+ - **Python 3.9 or later** and **[vsearch](https://github.com/torognes/vsearch)** on your `PATH` ([install steps](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#install), including Windows).
93
+ - **Paired FASTQ files** in one folder, named like `sample_R1.fastq.gz` and `sample_R2.fastq.gz`.
94
+ - **Primers and adapters already removed** (for example with cutadapt), and no quality trimming. R1 must be the forward read ([details](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#input)).
95
+
96
+ ## Options
97
+
98
+ Required: `--input-dir`, `--work-dir` and `--out-dir`. Optional: `--its` (for amplicons that vary a lot in length, such as ITS) and `--verbose`. AmpliTrim chooses everything else, including `maxEE` ([user guide](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#arguments)).
99
+
100
+ ## How the settings are chosen
101
+
102
+ | Setting | How |
103
+ |---|---|
104
+ | `truncLenF`, `truncLenR` | The shortest total that can merge your longest amplicons (the amplicon plus 12), split between the two reads where quality allows. Real merges are tested with vsearch |
105
+ | `maxEE` | 2 (DADA2's default) for clean reads. For noisy reads, a limit for each read that covers three quarters of its reads, never below 2, with a note if it goes above 5 ([how](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#how-maxee-is-chosen)) |
106
+ | `minOverlap` | 12 (DADA2's default) |
107
+
108
+ The outputs are `REPORT.md` (read this one), `summary.json` (for scripts; [every key](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#summaryjson-for-scripts-and-workflow-managers)) and, if matplotlib is installed (`pip install "amplitrim[plot]"`), `summary.png` (a figure of your amplicon lengths and how errors build up).
109
+
110
+ ## Limits
111
+
112
+ - **It is for DADA2.** QIIME 2's other denoiser, Deblur, uses different settings (one trim length for merged reads), which AmpliTrim does not recommend ([guide](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#what-amplitrim-recommends)).
113
+ - **The yield is an optimistic estimate.** It is measured before DADA2 denoises the reads. Use it to compare settings, and run DADA2 once to confirm.
114
+ - **It checks quality and mergeability, not accuracy.**
115
+ - **A tail of much longer amplicons** (for example non-target 18S) can force long, noisy reads. Only the longest 0.2% are set aside, so remove such reads first ([guide](https://github.com/paytonyau/amplitrim/blob/main/docs/usage.md#limitations)).
116
+
117
+ ## Citation, licence and contributing
118
+
119
+ MIT (see `LICENSE`). Please cite AmpliTrim (`CITATION.cff`) and [vsearch](https://github.com/torognes/vsearch). vsearch (GPLv3 and BSD-2-Clause) runs as a separate program, and its own licence applies. numpy and matplotlib are installed separately. For bug reports, questions and contributions, see [`CONTRIBUTING.md`](https://github.com/paytonyau/amplitrim/blob/main/CONTRIBUTING.md).
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,52 @@
1
+ """Command line interface."""
2
+
3
+ import argparse
4
+ import logging
5
+ import sys
6
+ from pathlib import Path
7
+
8
+ from . import __version__
9
+ from .pipeline import PipelineError, Settings, run
10
+ from .report import screen_summary, write_outputs
11
+
12
+ DESCRIPTION = """Recommend DADA2 truncLenF, truncLenR and maxEE for paired-end amplicon reads by testing candidate
13
+ settings on your own reads: how many pairs pass a DADA2-style quality filter and merge, and whether the
14
+ settings can merge your longest amplicons. The reads should have adapters and primers removed and should
15
+ not be quality-trimmed; their lengths may differ."""
16
+
17
+
18
+ def build_parser() -> argparse.ArgumentParser:
19
+ ap = argparse.ArgumentParser(prog="amplitrim", description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter,
20
+ epilog="example: amplitrim --input-dir reads --work-dir work --out-dir report")
21
+ ap.add_argument("--version", action="version", version=f"amplitrim {__version__}")
22
+ req = ap.add_argument_group("required arguments")
23
+ req.add_argument("--input-dir", required=True, type=Path,
24
+ help="folder of paired FASTQs named sample_R1/_R2.fastq.gz or "
25
+ "sample_S1_L001_R1_001.fastq.gz")
26
+ req.add_argument("--work-dir", required=True, type=Path, help="folder for intermediate files")
27
+ req.add_argument("--out-dir", required=True, type=Path, help="folder for the report")
28
+ opt = ap.add_argument_group("optional arguments")
29
+ opt.add_argument("--its", action="store_true",
30
+ help="the amplicons vary a lot in length (ITS, for example): recommend no fixed truncation length, as DADA2's "
31
+ "ITS workflow does (each read is still trimmed at its first base of quality 2 or below), instead of truncation lengths")
32
+ opt.add_argument("--verbose", action="store_true", help="show progress and detailed messages")
33
+ return ap
34
+
35
+
36
+ def main(argv=None) -> int:
37
+ args = build_parser().parse_args(argv)
38
+ logging.basicConfig(level=logging.INFO if args.verbose else logging.WARNING, format="%(message)s")
39
+ settings = Settings(input_dir=args.input_dir, work_dir=args.work_dir, marker="its" if args.its else "16s")
40
+ print(f"AmpliTrim {__version__}\nAnalysing reads...", flush=True)
41
+ try:
42
+ result = run(settings, say=(lambda msg: print(msg, flush=True)) if args.verbose else (lambda msg: None))
43
+ except (PipelineError, RuntimeError) as err:
44
+ print(f"Error: {err}", file=sys.stderr)
45
+ return 3
46
+ report = write_outputs(result, args.out_dir)
47
+ print("\n".join(screen_summary(result, report)))
48
+ return 0 if result.recommendation else 2
49
+
50
+
51
+ if __name__ == "__main__":
52
+ sys.exit(main())
@@ -0,0 +1,42 @@
1
+ """
2
+ Defaults for the settings that should mirror the DADA2 call you will actually run (for example
3
+ `qiime dada2 denoise-paired --p-max-ee-f 2`), because merging is tested under those rules.
4
+ `min_overlap` and `max_ee` can be set on the command line.
5
+ """
6
+
7
+ import argparse
8
+ from dataclasses import dataclass
9
+
10
+
11
+ def ee_pair(max_ee):
12
+ """(forward, reverse) maxEE from one number (both reads) or a pair, as DADA2's maxEE = c(F, R)."""
13
+ if isinstance(max_ee, (tuple, list)):
14
+ return float(max_ee[0]), float(max_ee[1])
15
+ return float(max_ee), float(max_ee)
16
+
17
+
18
+ def parse_ee(text):
19
+ """Command-line value for maxEE: '2' (both reads) or '2,5' (forward, reverse). Equal values collapse to one number."""
20
+ try:
21
+ parts = [float(x) for x in str(text).replace(";", ",").split(",")]
22
+ except ValueError:
23
+ parts = []
24
+ if len(parts) not in (1, 2) or any(p <= 0 for p in parts):
25
+ raise argparse.ArgumentTypeError(f"'{text}' is not a positive number or a pair such as 2,5 (forward, reverse)")
26
+ return parts[0] if len(parts) == 1 or parts[0] == parts[1] else (parts[0], parts[1])
27
+
28
+
29
+ @dataclass
30
+ class Config:
31
+ dada2_min_overlap: int = 12 # mergePairs minOverlap (DADA2's default)
32
+ dada2_max_mismatch: int = 0 # mergePairs maxMismatch (DADA2's default)
33
+ max_ee: object = 2.0 # one number, or (forward, reverse) as in maxEE = c(F, R)
34
+
35
+ # At most this share of the observed amplicons may be longer than the longest the settings
36
+ # can merge; otherwise long amplicons are systematically lost.
37
+ max_unmergeable_frac: float = 0.002
38
+
39
+ # Settings whose usable-pair yield is within this (absolute) of the best are treated as tied;
40
+ # the one with the smallest overlap then wins. The search also stops once a longer total
41
+ # length improves the yield by less than this.
42
+ yield_tolerance: float = 0.01
@@ -0,0 +1,115 @@
1
+ """
2
+ Stage 2: sequence-based merge testing with vsearch.
3
+
4
+ vsearch exposes the two settings that mirror DADA2's mergePairs (minimum overlap, maximum
5
+ mismatches) and reports the mismatches observed in each overlap.
6
+
7
+ Two vsearch passes per candidate:
8
+ strict - maxdiffs = DADA2's maxMismatch (default 0) applied to RAW reads.
9
+ A LOWER bound on DADA2's merge success: DADA2 merges *denoised*
10
+ sequences, from which sequencing errors have been removed.
11
+ tolerant - vsearch's default diff allowance. Closer proxy for "do these two
12
+ reads geometrically overlap", i.e. what DADA2 can merge after
13
+ denoising. Used for ranking.
14
+
15
+ Both tools are invoked as subprocesses only; no code is vendored.
16
+ """
17
+
18
+ import logging
19
+ import os
20
+ import re
21
+ import subprocess
22
+ from dataclasses import dataclass, field
23
+ from pathlib import Path
24
+ from typing import Dict, List, Optional
25
+
26
+ logger = logging.getLogger(__name__)
27
+ NAN = float("nan")
28
+
29
+
30
+ @dataclass
31
+ class MergeOutcome:
32
+ engine: str
33
+ setting: str
34
+ total_pairs: int
35
+ merged_pairs: int
36
+ mean_fragment_len: float = NAN
37
+ mean_overlap_diffs: float = NAN # mean observed mismatches in merged region
38
+ fail_reasons: Dict[str, int] = field(default_factory=dict)
39
+ merged_path: Optional[Path] = None
40
+
41
+ @property
42
+ def merge_rate(self) -> float:
43
+ return self.merged_pairs / self.total_pairs if self.total_pairs else 0.0
44
+
45
+
46
+ # ---------------------------------------------------------------- vsearch ---
47
+
48
+ def parse_vsearch_log(text: str, setting: str = "") -> MergeOutcome:
49
+ def num(pattern, cast=float, default=NAN):
50
+ m = re.search(pattern, text, re.MULTILINE)
51
+ return cast(m.group(1)) if m else default
52
+
53
+ total = num(r"^\s*(\d+)\s+Pairs", int, 0)
54
+ merged = num(r"^\s*(\d+)\s+Merged", int, 0)
55
+ reasons: Dict[str, int] = {}
56
+ block = re.search(r"failed merging due to various reasons:\n(.*?)\n\s*\n", text, re.DOTALL)
57
+ if block:
58
+ for line in block.group(1).splitlines():
59
+ m = re.match(r"^\s*(\d+)\s+(.+?)\s*$", line)
60
+ if m:
61
+ reasons[m.group(2)] = int(m.group(1))
62
+ return MergeOutcome(
63
+ engine="vsearch", setting=setting, total_pairs=total, merged_pairs=merged,
64
+ mean_fragment_len=num(r"^\s*([\d.]+)\s+Mean fragment length"),
65
+ mean_overlap_diffs=num(r"^\s*([\d.]+)\s+Mean observed errors in merged region\s*$"),
66
+ fail_reasons=reasons,
67
+ )
68
+
69
+
70
+ def run_vsearch(r1: Path, r2: Path, out_dir: Path, tag: str, min_overlap: int,
71
+ max_diffs: Optional[int], setting: str, keep_merged: bool = True,
72
+ threads: int = 1) -> MergeOutcome:
73
+ out_dir.mkdir(parents=True, exist_ok=True)
74
+ merged = out_dir / f"{tag}.{setting}.merged.fasta"
75
+ log = out_dir / f"{tag}.{setting}.vsearch.log"
76
+ cmd = ["vsearch", "--fastq_mergepairs", str(r1), "--reverse", str(r2),
77
+ "--fastaout", str(merged if keep_merged else os.devnull),
78
+ "--fastq_minovlen", str(min_overlap), "--fastq_allowmergestagger",
79
+ "--threads", str(threads), "--log", str(log), "--quiet"]
80
+ if max_diffs is not None:
81
+ cmd += ["--fastq_maxdiffs", str(max_diffs)]
82
+ res = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")
83
+ if res.returncode != 0:
84
+ raise RuntimeError(f"vsearch failed ({' '.join(cmd)}):\n{res.stderr}")
85
+ out = parse_vsearch_log(log.read_bytes().decode("utf-8", errors="replace"), setting) # paths in it may be in the ANSI code page
86
+ out.merged_path = merged if keep_merged else None
87
+ return out
88
+
89
+
90
+ def merged_lengths(fasta: Path) -> List[int]:
91
+ """Lengths of merged sequences (= observed insert/fragment sizes)."""
92
+ lengths, cur = [], 0
93
+ with open(fasta) as fh:
94
+ for line in fh:
95
+ if line.startswith(">"):
96
+ if cur:
97
+ lengths.append(cur)
98
+ cur = 0
99
+ else:
100
+ cur += len(line.strip())
101
+ if cur:
102
+ lengths.append(cur)
103
+ return lengths
104
+
105
+
106
+ # ------------------------------------------------------------------ Stage 0 ---
107
+
108
+ def estimate_insert_sizes(r1: Path, r2: Path, out_dir: Path, min_overlap: int) -> List[int]:
109
+ """Observed insert (fragment) sizes from a tolerant merge of FULL-length
110
+ reads. The upper tail of this distribution is the data-driven answer to
111
+ 'what is the longest amplicon?' - the quantity a fixed amplicon length cannot give
112
+ for itself. Caveat: only pairs that overlap at full length are seen, so
113
+ inserts approaching (len_f + len_r - min_overlap) are under-represented."""
114
+ out = run_vsearch(r1, r2, out_dir, "stage0", min_overlap, None, "tolerant")
115
+ return merged_lengths(out.merged_path)