align_trim 1.2.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- align_trim-2.0.0/CHANGELOG.md +75 -0
- align_trim-2.0.0/PKG-INFO +75 -0
- align_trim-2.0.0/README.md +59 -0
- align_trim-2.0.0/align_trim/console.py +301 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/align_trim/main.py +533 -330
- align_trim-2.0.0/align_trim/qc.py +845 -0
- align_trim-2.0.0/docs/outputs.md +50 -0
- align_trim-2.0.0/docs/qc-reports.md +107 -0
- align_trim-2.0.0/docs/usage.md +124 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/pyproject.toml +1 -1
- align_trim-2.0.0/tests/test_console.py +334 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_integration.py +17 -7
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_normalise.py +2 -2
- align_trim-2.0.0/tests/test_qc.py +1118 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/uv.lock +1 -1
- align_trim-1.2.0/PKG-INFO +0 -148
- align_trim-1.2.0/README.md +0 -132
- {align_trim-1.2.0 → align_trim-2.0.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/.github/workflows/pytest.yml +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/.github/workflows/python-publish.yml +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/.gitignore +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/.pre-commit-config.yaml +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/LICENSE +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/align_trim/__init__.py +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/__init__.py +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_data/primer.bed +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_data/sars-cov-2_v3.0.0_paired.bam +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_data/sars-cov-2_v5.3.2.bam +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_data/v1.0.0.primer.bed +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_data/v3.0.0.primer.bed +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_data/v5.3.2.primer.bed +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_legacy.py +0 -0
- {align_trim-1.2.0 → align_trim-2.0.0}/tests/test_main.py +0 -0
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to align_trim are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [2.0.0]
|
|
8
|
+
|
|
9
|
+
This release changes which reads are kept and how paired-end data is normalised, so trimmed output will differ from 1.2.0 for the same command. See **Changed** before upgrading a pipeline.
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `--qc-prefix PREFIX` writes five amplicon QC reports, alongside the genome coverage report: a run summary, per-amplicon fragment fates, primer pair combinations, split (supplementary) alignments and concordance between overlapping amplicons. A one-line summary of each is printed to stderr. The trimmed output is unchanged by this option. See [QC reports](docs/qc-reports.md).
|
|
14
|
+
- A warning on stderr when the majority base differs between two overlapping amplicons, which can indicate contamination.
|
|
15
|
+
- stderr is grouped under `Run`, `Coverage`, `QC summary` and `Warnings` headings. A completion line reports the records and proper pairs processed, the time taken and the rate. See [terminal output](docs/usage.md#terminal-output).
|
|
16
|
+
- In a terminal: coloured headings, a live progress bar (percentage and time remaining for a BAM file, count and rate for stdin) and the coverage summary as a pre/post-normalisation table. Logs get plain `== Name ==` headings and a `progress:` line every 30 seconds. `NO_COLOR` and `FORCE_COLOR` are respected.
|
|
17
|
+
- `--quiet`/`-q` prints only the coverage summary, QC summary and warnings to stderr.
|
|
18
|
+
- Fragments lying inside two neighbouring amplicons (left primer of amplicon n+1, right primer of amplicon n) are removed as `ambiguous_amplicon` and never counted against either amplicon. `overlaps.tsv` gives the count for each overlap.
|
|
19
|
+
- `--rejected-output FILE` writes the reads that were removed, untrimmed, to a separate SAM/BAM file, each tagged `ZR:Z:<reason>` with the same reasons as the QC summary. See [rejected reads](docs/outputs.md#rejected-reads).
|
|
20
|
+
|
|
21
|
+
### Changed
|
|
22
|
+
|
|
23
|
+
- **Breaking:** `--genome-coverage-report`/`-g` is removed and merged into `--qc-prefix`, which writes the genome coverage and amplicon QC reports from one prefix. Replace `--genome-coverage-report PREFIX` with `--qc-prefix PREFIX`; the coverage file names are unchanged.
|
|
24
|
+
- With `--allow-incorrect-pairs`, fragments lying inside two neighbouring amplicons are now removed. Previously they were kept, trimmed and normalised as amplicon n+1.
|
|
25
|
+
- **Breaking:** the `--primer-match-threshold` default is now 5 (previously 35). Reads with an end more than 5 bases outside their primer's 5' end, i.e. containing genomic sequence beyond the amplicon, are no longer assigned to that primer. On the paired-end test data this removes about 0.4% of read pairs; single-end ONT test output is unchanged.
|
|
26
|
+
- **Breaking:** `--normalise` counts paired-end data in fragments (a read pair counts once where its mates overlap), so paired-end output at the same `-n` is roughly double the size of previous versions.
|
|
27
|
+
- **Breaking:** the `--amp-depth-report` column `mean_depth` is renamed `mean_fragment_depth`.
|
|
28
|
+
- Reads skipped because primer soft masking failed are no longer printed one per line by default. A single warning gives the count, and `--verbose` prints each read. `--verbose` also disables the live progress bar.
|
|
29
|
+
- The README covers installation and a quick start; the full option reference and report descriptions are in [`docs/`](docs/).
|
|
30
|
+
|
|
31
|
+
### Fixed
|
|
32
|
+
|
|
33
|
+
- A read ending exactly `--primer-match-threshold` bases past its right primer's 5' end was given the next amplicon's right primer and removed as mismatched, and a read ending at the last base of a reference matched no right primer. Right ends are now matched within the threshold, as left ends already were.
|
|
34
|
+
- `--amp-depth-report` for paired-end data without `--normalise` counted only one mate of each pair.
|
|
35
|
+
- Supplementary and secondary alignments could be paired in place of the real mate for paired-end data, and secondary alignments were written for single-end data.
|
|
36
|
+
- Reads where primer trimming would produce an invalid CIGAR (e.g. an indel inside the primer) were written with their start position moved but the CIGAR unchanged. They are now dropped.
|
|
37
|
+
- Reads aligned with `=`/`X` CIGAR operations were dropped instead of trimmed.
|
|
38
|
+
- A deletion left directly before the right primer softmask was kept, so reads ended in e.g. `1D22S` and reported a spurious deletion at the last base before the primer. It is now removed, as was already done for the left primer. An insertion next to either softmask (e.g. `2I20S`) is now added to the softmask.
|
|
39
|
+
- A read (or mate) lying entirely inside a primer kept some unmasked primer bases (e.g. `5M15S`), so it was not removed as primer only. It is now fully soft masked and removed as `primer_only`.
|
|
40
|
+
- For paired-end data the right primer was looked up at the end of whichever mate started later, rather than the rightmost mate end, so a pair with one mate contained in the other could be given the wrong right primer or removed by `--require-full-length`.
|
|
41
|
+
|
|
42
|
+
## [1.2.0] - 2026-06-23
|
|
43
|
+
|
|
44
|
+
### Added
|
|
45
|
+
|
|
46
|
+
- `--genome-coverage-report PREFIX` writes per-position genome coverage before and after normalisation, with a coverage summary on stderr ([#23](https://github.com/artic-network/align_trim/pull/23)).
|
|
47
|
+
|
|
48
|
+
## [1.1.0] - 2026-04-09
|
|
49
|
+
|
|
50
|
+
### Fixed
|
|
51
|
+
|
|
52
|
+
- Normalisation logic ([#22](https://github.com/artic-network/align_trim/pull/22)).
|
|
53
|
+
|
|
54
|
+
## [1.0.2] - 2025-08-28
|
|
55
|
+
|
|
56
|
+
### Added
|
|
57
|
+
|
|
58
|
+
- Check for paired reads ([#19](https://github.com/artic-network/align_trim/pull/19)).
|
|
59
|
+
|
|
60
|
+
## [1.0.1] - 2025-08-22
|
|
61
|
+
|
|
62
|
+
### Changed
|
|
63
|
+
|
|
64
|
+
- Unpinned numpy and pysam versions for wider compatibility.
|
|
65
|
+
|
|
66
|
+
## [1.0.0] - 2025-08-22
|
|
67
|
+
|
|
68
|
+
Initial release of align_trim as a standalone package. It was originally developed for primer trimming in [artic-network/fieldbioinformatics](https://github.com/artic-network/fieldbioinformatics).
|
|
69
|
+
|
|
70
|
+
[2.0.0]: https://github.com/artic-network/align_trim/compare/1.2.0...HEAD
|
|
71
|
+
[1.2.0]: https://github.com/artic-network/align_trim/compare/1.1.0...1.2.0
|
|
72
|
+
[1.1.0]: https://github.com/artic-network/align_trim/compare/1.0.2...1.1.0
|
|
73
|
+
[1.0.2]: https://github.com/artic-network/align_trim/compare/1.0.1...1.0.2
|
|
74
|
+
[1.0.1]: https://github.com/artic-network/align_trim/compare/1.0.0...1.0.1
|
|
75
|
+
[1.0.0]: https://github.com/artic-network/align_trim/releases/tag/1.0.0
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: align_trim
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Soft-clip primer sites for SAM/BAM files generated from amplicon sequencing runs
|
|
5
|
+
Project-URL: Repository, https://github.com/artic-network/align_trim.git
|
|
6
|
+
Project-URL: Issues, https://github.com/artic-network/align_trim/issues
|
|
7
|
+
Author-email: Nick Loman <n.j.loman@bham.ac.uk>, Sam Wilkinson <s.a.j.wilkinson@bham.ac.uk>, Chris Kent <c.g.kent@bham.ac.uk>
|
|
8
|
+
Maintainer-email: Sam Wilkinson <s.a.j.wilkinson@bham.ac.uk>, Chris Kent <c.g.kent@bham.ac.uk>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Requires-Dist: numpy
|
|
13
|
+
Requires-Dist: primalbedtools>=0.10.1
|
|
14
|
+
Requires-Dist: pysam
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# align_trim
|
|
18
|
+
|
|
19
|
+
Stand alone version of ARTIC's fieldbioinformatics align_trim.py. It assigns aligned reads from a tiling amplicon scheme to their amplicons, soft-clips the primers and can normalise depth per amplicon.
|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
From conda
|
|
24
|
+
```bash
|
|
25
|
+
conda install bioconda::align_trim
|
|
26
|
+
```
|
|
27
|
+
from pypi
|
|
28
|
+
```bash
|
|
29
|
+
pip install align_trim
|
|
30
|
+
```
|
|
31
|
+
from source
|
|
32
|
+
```bash
|
|
33
|
+
git clone https://github.com/artic-network/align_trim.git
|
|
34
|
+
cd align_trim
|
|
35
|
+
uv sync
|
|
36
|
+
uv run align_trim --help
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Quick start
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
align_trim primer.bed --samfile input.bam --output trimmed.bam
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
- `primer.bed` is the primer scheme in [primer.bed v3](https://doi.org/10.5281/zenodo.16366659) format.
|
|
46
|
+
- `input.bam` is a sorted BAM of reads aligned to the scheme's reference. Leave out `--samfile` to read SAM/BAM from stdin.
|
|
47
|
+
- The output format follows the extension (`.bam` or `.sam`). Leave out `--output` to write SAM to stdout.
|
|
48
|
+
|
|
49
|
+
Normalise to 200x per amplicon and write the genome coverage and amplicon QC reports:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
align_trim primer.bed --samfile input.bam --output trimmed.bam \
|
|
53
|
+
--normalise 200 --qc-prefix sample1
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Common options
|
|
57
|
+
|
|
58
|
+
| option | meaning |
|
|
59
|
+
| --- | --- |
|
|
60
|
+
| `--normalise N`, `-n` | keep reads until each amplicon reaches a depth of N fragments (a read pair counts once). 0 turns it off (default) |
|
|
61
|
+
| `--min-mapq`, `-m` | minimum mapping quality (default: 20) |
|
|
62
|
+
| `--primer-match-threshold`, `-p` | how far, in bases, a read end can be from a primer and still match it (default: 5) |
|
|
63
|
+
| `--allow-incorrect-pairs` | keep reads whose two ends match primers from different amplicons (reads inside two neighbouring amplicons are still removed) |
|
|
64
|
+
| `--require-full-length` | only keep reads spanning a whole amplicon (not for rapid barcoding or other fragmented libraries) |
|
|
65
|
+
| `--no-trim-primers` | assign reads but leave primers untrimmed |
|
|
66
|
+
| `--qc-prefix PREFIX` | write the genome coverage and amplicon QC reports as `PREFIX.<name>.tsv` |
|
|
67
|
+
|
|
68
|
+
Run `align_trim --help` for every option. Progress, coverage and QC summaries and warnings are printed to stderr; see [terminal output](https://github.com/artic-network/align_trim/blob/main/docs/usage.md#terminal-output).
|
|
69
|
+
|
|
70
|
+
## Documentation
|
|
71
|
+
|
|
72
|
+
- [Usage](https://github.com/artic-network/align_trim/blob/main/docs/usage.md): every option, examples and output formats
|
|
73
|
+
- [Report outputs](https://github.com/artic-network/align_trim/blob/main/docs/outputs.md): `--report`, `--amp-depth-report` and the genome coverage reports
|
|
74
|
+
- [QC reports](https://github.com/artic-network/align_trim/blob/main/docs/qc-reports.md): the `--qc-prefix` reports and how to read them
|
|
75
|
+
- [Changelog](https://github.com/artic-network/align_trim/blob/main/CHANGELOG.md): changes in each release, including behaviour changes in 2.0.0
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# align_trim
|
|
2
|
+
|
|
3
|
+
Stand alone version of ARTIC's fieldbioinformatics align_trim.py. It assigns aligned reads from a tiling amplicon scheme to their amplicons, soft-clips the primers and can normalise depth per amplicon.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
From conda
|
|
8
|
+
```bash
|
|
9
|
+
conda install bioconda::align_trim
|
|
10
|
+
```
|
|
11
|
+
from pypi
|
|
12
|
+
```bash
|
|
13
|
+
pip install align_trim
|
|
14
|
+
```
|
|
15
|
+
from source
|
|
16
|
+
```bash
|
|
17
|
+
git clone https://github.com/artic-network/align_trim.git
|
|
18
|
+
cd align_trim
|
|
19
|
+
uv sync
|
|
20
|
+
uv run align_trim --help
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Quick start
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
align_trim primer.bed --samfile input.bam --output trimmed.bam
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
- `primer.bed` is the primer scheme in [primer.bed v3](https://doi.org/10.5281/zenodo.16366659) format.
|
|
30
|
+
- `input.bam` is a sorted BAM of reads aligned to the scheme's reference. Leave out `--samfile` to read SAM/BAM from stdin.
|
|
31
|
+
- The output format follows the extension (`.bam` or `.sam`). Leave out `--output` to write SAM to stdout.
|
|
32
|
+
|
|
33
|
+
Normalise to 200x per amplicon and write the genome coverage and amplicon QC reports:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
align_trim primer.bed --samfile input.bam --output trimmed.bam \
|
|
37
|
+
--normalise 200 --qc-prefix sample1
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Common options
|
|
41
|
+
|
|
42
|
+
| option | meaning |
|
|
43
|
+
| --- | --- |
|
|
44
|
+
| `--normalise N`, `-n` | keep reads until each amplicon reaches a depth of N fragments (a read pair counts once). 0 turns it off (default) |
|
|
45
|
+
| `--min-mapq`, `-m` | minimum mapping quality (default: 20) |
|
|
46
|
+
| `--primer-match-threshold`, `-p` | how far, in bases, a read end can be from a primer and still match it (default: 5) |
|
|
47
|
+
| `--allow-incorrect-pairs` | keep reads whose two ends match primers from different amplicons (reads inside two neighbouring amplicons are still removed) |
|
|
48
|
+
| `--require-full-length` | only keep reads spanning a whole amplicon (not for rapid barcoding or other fragmented libraries) |
|
|
49
|
+
| `--no-trim-primers` | assign reads but leave primers untrimmed |
|
|
50
|
+
| `--qc-prefix PREFIX` | write the genome coverage and amplicon QC reports as `PREFIX.<name>.tsv` |
|
|
51
|
+
|
|
52
|
+
Run `align_trim --help` for every option. Progress, coverage and QC summaries and warnings are printed to stderr; see [terminal output](https://github.com/artic-network/align_trim/blob/main/docs/usage.md#terminal-output).
|
|
53
|
+
|
|
54
|
+
## Documentation
|
|
55
|
+
|
|
56
|
+
- [Usage](https://github.com/artic-network/align_trim/blob/main/docs/usage.md): every option, examples and output formats
|
|
57
|
+
- [Report outputs](https://github.com/artic-network/align_trim/blob/main/docs/outputs.md): `--report`, `--amp-depth-report` and the genome coverage reports
|
|
58
|
+
- [QC reports](https://github.com/artic-network/align_trim/blob/main/docs/qc-reports.md): the `--qc-prefix` reports and how to read them
|
|
59
|
+
- [Changelog](https://github.com/artic-network/align_trim/blob/main/CHANGELOG.md): changes in each release, including behaviour changes in 2.0.0
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""Terminal output for align_trim: section headings, colour and progress on stderr.
|
|
2
|
+
|
|
3
|
+
Everything here writes to the current sys.stderr (looked up at write time, so
|
|
4
|
+
redirect_stderr in tests gets plain output). stdout is never touched. Colour and
|
|
5
|
+
the live progress line are only used when stderr is a terminal; log files get
|
|
6
|
+
plain text with `== Section ==` headings and no escape codes.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import io
|
|
10
|
+
import os
|
|
11
|
+
import shutil
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
BOLD = "1"
|
|
16
|
+
DIM = "2"
|
|
17
|
+
RED = "31"
|
|
18
|
+
GREEN = "32"
|
|
19
|
+
CYAN = "36"
|
|
20
|
+
|
|
21
|
+
# check the clock every 8192 records
|
|
22
|
+
CHECK_MASK = 0x1FFF
|
|
23
|
+
# minimum seconds between live redraws, and between progress lines in a log
|
|
24
|
+
REDRAW_INTERVAL = 0.125
|
|
25
|
+
LOG_INTERVAL = 30.0
|
|
26
|
+
|
|
27
|
+
SPINNER = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
|
|
28
|
+
ASCII_SPINNER = "|/-\\"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _human(n):
|
|
32
|
+
for unit, size in (("M", 1e6), ("k", 1e3)):
|
|
33
|
+
if n >= size:
|
|
34
|
+
return f"{n / size:.2f}{unit}" if n < 10 * size else f"{n / size:.1f}{unit}"
|
|
35
|
+
return str(int(n))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _duration(seconds):
|
|
39
|
+
seconds = int(seconds)
|
|
40
|
+
hours, rest = divmod(seconds, 3600)
|
|
41
|
+
minutes, seconds = divmod(rest, 60)
|
|
42
|
+
return (
|
|
43
|
+
f"{hours}:{minutes:02d}:{seconds:02d}" if hours else f"{minutes}:{seconds:02d}"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def bam_fraction(infile, path):
|
|
48
|
+
"""Return a function giving how far through a BAM file on disk the reader is, or None.
|
|
49
|
+
|
|
50
|
+
Uses the compressed byte offset (the high bits of the BGZF virtual offset) over
|
|
51
|
+
the file size, so it works without an index. Not available for stdin or SAM.
|
|
52
|
+
"""
|
|
53
|
+
if not path or path == "-" or not str(path).endswith(".bam"):
|
|
54
|
+
return None
|
|
55
|
+
try:
|
|
56
|
+
size = os.path.getsize(path)
|
|
57
|
+
infile.tell()
|
|
58
|
+
except (OSError, ValueError, NotImplementedError):
|
|
59
|
+
return None
|
|
60
|
+
if not size:
|
|
61
|
+
return None
|
|
62
|
+
|
|
63
|
+
def fraction():
|
|
64
|
+
try:
|
|
65
|
+
return min(max((infile.tell() >> 16) / size, 0.0), 1.0)
|
|
66
|
+
except (OSError, ValueError):
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
return fraction
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class Console:
|
|
73
|
+
def __init__(self):
|
|
74
|
+
self.reset()
|
|
75
|
+
|
|
76
|
+
def reset(self, quiet=False, verbose=False):
|
|
77
|
+
"""Start a new run: clear warnings and counters."""
|
|
78
|
+
self.quiet = quiet
|
|
79
|
+
self.verbose = verbose
|
|
80
|
+
self.warnings = []
|
|
81
|
+
self.trim_failures = 0
|
|
82
|
+
self._section = None
|
|
83
|
+
self._live = None # the Progress currently drawn on the terminal
|
|
84
|
+
|
|
85
|
+
# terminal detection, evaluated against the current stderr
|
|
86
|
+
@property
|
|
87
|
+
def stream(self):
|
|
88
|
+
return sys.stderr
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def is_tty(self):
|
|
92
|
+
try:
|
|
93
|
+
return self.stream.isatty()
|
|
94
|
+
except (AttributeError, ValueError):
|
|
95
|
+
return False
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def color(self):
|
|
99
|
+
if os.environ.get("FORCE_COLOR"):
|
|
100
|
+
return True
|
|
101
|
+
return (
|
|
102
|
+
self.is_tty
|
|
103
|
+
and not os.environ.get("NO_COLOR")
|
|
104
|
+
and os.environ.get("TERM") != "dumb"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
@property
|
|
108
|
+
def live(self):
|
|
109
|
+
return self.is_tty and os.environ.get("TERM") != "dumb"
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def unicode(self):
|
|
113
|
+
encoding = getattr(self.stream, "encoding", None) or ""
|
|
114
|
+
return encoding.lower().replace("-", "") == "utf8"
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def width(self):
|
|
118
|
+
"""Columns of the terminal stderr is on (stdout is usually redirected), default 80."""
|
|
119
|
+
try:
|
|
120
|
+
columns = os.get_terminal_size(self.stream.fileno()).columns
|
|
121
|
+
except (AttributeError, OSError, ValueError, io.UnsupportedOperation):
|
|
122
|
+
columns = shutil.get_terminal_size((80, 24)).columns
|
|
123
|
+
return columns or 80
|
|
124
|
+
|
|
125
|
+
def style(self, text, *codes):
|
|
126
|
+
if not self.color:
|
|
127
|
+
return text
|
|
128
|
+
return f"\x1b[{';'.join(codes)}m{text}\x1b[0m"
|
|
129
|
+
|
|
130
|
+
# writing
|
|
131
|
+
def _write(self, text):
|
|
132
|
+
try:
|
|
133
|
+
self.stream.write(text)
|
|
134
|
+
self.stream.flush()
|
|
135
|
+
except BrokenPipeError:
|
|
136
|
+
pass
|
|
137
|
+
|
|
138
|
+
def line(self, msg=""):
|
|
139
|
+
"""Write a line, clearing the live progress line first if one is drawn."""
|
|
140
|
+
if self._live is not None:
|
|
141
|
+
self._live.clear()
|
|
142
|
+
self._write(f"{msg}\n")
|
|
143
|
+
|
|
144
|
+
def heading(self, title):
|
|
145
|
+
if title == self._section:
|
|
146
|
+
return
|
|
147
|
+
self._section = title
|
|
148
|
+
if self.live:
|
|
149
|
+
rule = "─" if self.unicode else "-"
|
|
150
|
+
width = min(self.width, 80)
|
|
151
|
+
text = f"{rule * 2} {title} "
|
|
152
|
+
self.line(self.style(text + rule * max(width - len(text), 2), BOLD, CYAN))
|
|
153
|
+
else:
|
|
154
|
+
self.line(f"== {title} ==")
|
|
155
|
+
|
|
156
|
+
def prefixed(self, prefix, msg):
|
|
157
|
+
"""A line with a dimmed prefix such as `QC:`, whose text is unchanged in plain mode."""
|
|
158
|
+
self.line(f"{self.style(prefix, DIM)} {msg}")
|
|
159
|
+
|
|
160
|
+
def warn(self, msg):
|
|
161
|
+
self.line(f"{self.style('WARNING:', BOLD, RED)} {msg}")
|
|
162
|
+
|
|
163
|
+
def error(self, msg):
|
|
164
|
+
self.line(self.style(msg, BOLD, RED))
|
|
165
|
+
|
|
166
|
+
def debug(self, msg):
|
|
167
|
+
self.line(self.style(msg, DIM))
|
|
168
|
+
|
|
169
|
+
def read_warning(self, msg):
|
|
170
|
+
"""A per-read problem: counted for a summary warning, printed only with --verbose."""
|
|
171
|
+
self.trim_failures += 1
|
|
172
|
+
if self.verbose:
|
|
173
|
+
self.line(f"{self.style('warning', RED)} {msg}")
|
|
174
|
+
|
|
175
|
+
def defer_warning(self, msg):
|
|
176
|
+
self.warnings.append(msg)
|
|
177
|
+
|
|
178
|
+
def flush_warnings(self):
|
|
179
|
+
if self.trim_failures:
|
|
180
|
+
self.warnings.insert(
|
|
181
|
+
0,
|
|
182
|
+
f"{self.trim_failures} fragments skipped because primer soft masking failed; "
|
|
183
|
+
"rerun with --verbose for details",
|
|
184
|
+
)
|
|
185
|
+
self.trim_failures = 0
|
|
186
|
+
if self.warnings:
|
|
187
|
+
self.heading("Warnings")
|
|
188
|
+
for msg in self.warnings:
|
|
189
|
+
self.warn(msg)
|
|
190
|
+
self.warnings = []
|
|
191
|
+
|
|
192
|
+
def progress(self, fraction=None, enabled=True, live=True, clock=time.monotonic):
|
|
193
|
+
return Progress(
|
|
194
|
+
self, fraction=fraction, enabled=enabled, live=live, clock=clock
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class Progress:
|
|
199
|
+
"""Count records from an iterator, drawing a live bar on a terminal.
|
|
200
|
+
|
|
201
|
+
In a log (or with --verbose) it prints a plain progress line every LOG_INTERVAL
|
|
202
|
+
seconds instead. `done()` prints a single completion line.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
def __init__(
|
|
206
|
+
self, console, fraction=None, enabled=True, live=True, clock=time.monotonic
|
|
207
|
+
):
|
|
208
|
+
self.console = console
|
|
209
|
+
self.fraction = fraction
|
|
210
|
+
self.enabled = enabled and not console.quiet
|
|
211
|
+
self.live = live and self.enabled and console.live and not console.verbose
|
|
212
|
+
self.clock = clock
|
|
213
|
+
self.count = 0
|
|
214
|
+
self.frame = 0
|
|
215
|
+
self.drawn = False
|
|
216
|
+
self.start = clock()
|
|
217
|
+
|
|
218
|
+
def track(self, iterable):
|
|
219
|
+
if not self.enabled:
|
|
220
|
+
for self.count, item in enumerate(iterable, 1):
|
|
221
|
+
yield item
|
|
222
|
+
return
|
|
223
|
+
|
|
224
|
+
interval = REDRAW_INTERVAL if self.live else LOG_INTERVAL
|
|
225
|
+
next_t = self.start + interval
|
|
226
|
+
n = 0
|
|
227
|
+
try:
|
|
228
|
+
for item in iterable:
|
|
229
|
+
n += 1
|
|
230
|
+
if not n & CHECK_MASK:
|
|
231
|
+
t = self.clock()
|
|
232
|
+
if t >= next_t:
|
|
233
|
+
self.count = n
|
|
234
|
+
self._update(t)
|
|
235
|
+
next_t = t + interval
|
|
236
|
+
yield item
|
|
237
|
+
finally:
|
|
238
|
+
self.count = n
|
|
239
|
+
self.clear()
|
|
240
|
+
|
|
241
|
+
def _rate(self, t):
|
|
242
|
+
elapsed = t - self.start
|
|
243
|
+
return self.count / elapsed if elapsed > 0 else 0.0
|
|
244
|
+
|
|
245
|
+
def _update(self, t):
|
|
246
|
+
if not self.live:
|
|
247
|
+
fraction = self.fraction() if self.fraction else None
|
|
248
|
+
pct = f" ({fraction:.0%})" if fraction is not None else ""
|
|
249
|
+
self.console.line(
|
|
250
|
+
f"progress: {self.count:,} records{pct}, {_human(self._rate(t))} records/s"
|
|
251
|
+
)
|
|
252
|
+
return
|
|
253
|
+
self.console._live = self
|
|
254
|
+
self._write(self._render(t))
|
|
255
|
+
|
|
256
|
+
def _render(self, t):
|
|
257
|
+
c = self.console
|
|
258
|
+
frames = SPINNER if c.unicode else ASCII_SPINNER
|
|
259
|
+
self.frame = (self.frame + 1) % len(frames)
|
|
260
|
+
rate = self._rate(t)
|
|
261
|
+
stats = f"{_human(self.count)} records {_human(rate)} rec/s"
|
|
262
|
+
fraction = self.fraction() if self.fraction else None
|
|
263
|
+
if fraction is None:
|
|
264
|
+
return f"{c.style(frames[self.frame], CYAN)} trimming {stats} {_duration(t - self.start)} elapsed"
|
|
265
|
+
|
|
266
|
+
remaining = (t - self.start) * (1 - fraction) / fraction if fraction else 0
|
|
267
|
+
tail = f" {fraction:4.0%} {stats} ETA {_duration(remaining)}"
|
|
268
|
+
width = c.width
|
|
269
|
+
bar_width = max(10, min(40, width - len(tail) - 14))
|
|
270
|
+
filled = int(bar_width * fraction)
|
|
271
|
+
full, empty = ("━", "─") if c.unicode else ("#", "-")
|
|
272
|
+
bar = c.style(full * filled, GREEN) + c.style(empty * (bar_width - filled), DIM)
|
|
273
|
+
return f"{c.style(frames[self.frame], CYAN)} trimming {bar}{tail}"
|
|
274
|
+
|
|
275
|
+
def _write(self, text):
|
|
276
|
+
self.console._write(f"\r\x1b[K{text}")
|
|
277
|
+
self.drawn = True
|
|
278
|
+
|
|
279
|
+
def clear(self):
|
|
280
|
+
if self.drawn:
|
|
281
|
+
self.console._write("\r\x1b[K")
|
|
282
|
+
self.drawn = False
|
|
283
|
+
if self.console._live is self:
|
|
284
|
+
self.console._live = None
|
|
285
|
+
|
|
286
|
+
def done(self, detail=None):
|
|
287
|
+
"""Print the completion line (not in quiet mode)."""
|
|
288
|
+
self.clear()
|
|
289
|
+
if self.console.quiet:
|
|
290
|
+
return
|
|
291
|
+
t = self.clock()
|
|
292
|
+
c = self.console
|
|
293
|
+
detail = f" ({detail})" if detail else ""
|
|
294
|
+
tick = c.style("✔" if c.unicode else "*", BOLD, GREEN) + " " if c.live else ""
|
|
295
|
+
c.line(
|
|
296
|
+
f"{tick}processed {self.count:,} records{detail} in {t - self.start:.1f}s, "
|
|
297
|
+
f"{_human(self._rate(t))} records/s"
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
console = Console()
|