markadoros 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- markadoros-1.0.0/.github/workflows/pytest.yml +69 -0
- markadoros-1.0.0/.gitignore +15 -0
- markadoros-1.0.0/.python-version +1 -0
- markadoros-1.0.0/CHANGELOG.md +20 -0
- markadoros-1.0.0/LICENSE +21 -0
- markadoros-1.0.0/PKG-INFO +269 -0
- markadoros-1.0.0/README.md +244 -0
- markadoros-1.0.0/pyproject.toml +49 -0
- markadoros-1.0.0/src/markadoros/__init__.py +3 -0
- markadoros-1.0.0/src/markadoros/assembler_runners.py +216 -0
- markadoros-1.0.0/src/markadoros/bold_tsv_processor.py +133 -0
- markadoros-1.0.0/src/markadoros/cli.py +508 -0
- markadoros-1.0.0/src/markadoros/contig_searcher.py +87 -0
- markadoros-1.0.0/src/markadoros/database_creator.py +97 -0
- markadoros-1.0.0/src/markadoros/database_fasta_processor.py +188 -0
- markadoros-1.0.0/src/markadoros/database_index.py +45 -0
- markadoros-1.0.0/src/markadoros/goat_synonym_getter.py +75 -0
- markadoros-1.0.0/src/markadoros/header_processor.py +81 -0
- markadoros-1.0.0/src/markadoros/input_types.py +62 -0
- markadoros-1.0.0/src/markadoros/mmseqs_database_builder.py +139 -0
- markadoros-1.0.0/src/markadoros/read_assembler.py +252 -0
- markadoros-1.0.0/src/markadoros/read_preprocessor.py +120 -0
- markadoros-1.0.0/src/markadoros/results_processor.py +351 -0
- markadoros-1.0.0/src/markadoros/search_pipeline.py +344 -0
- markadoros-1.0.0/src/markadoros/utils.py +172 -0
- markadoros-1.0.0/tests/conftest.py +83 -0
- markadoros-1.0.0/tests/data/BOLD.sub.fa.gz +0 -0
- markadoros-1.0.0/tests/data/BOLD.sub.tsv +4 -0
- markadoros-1.0.0/tests/data/SILVA.sub.fa.gz +0 -0
- markadoros-1.0.0/tests/data/UNITE.sub.fa.gz +0 -0
- markadoros-1.0.0/tests/data/icHalSede1.hic.cram +0 -0
- markadoros-1.0.0/tests/data/icHalSede1.hic.cram.crai +0 -0
- markadoros-1.0.0/tests/data/icHalSede1.pacbio.nohit.fa.gz +0 -0
- markadoros-1.0.0/tests/data/icHalSede1.pacbio.subset.fa.gz +0 -0
- markadoros-1.0.0/tests/data/wolbachia.exclude +1 -0
- markadoros-1.0.0/tests/test_database_creation.py +310 -0
- markadoros-1.0.0/tests/test_search.py +204 -0
- markadoros-1.0.0/uv.lock +1120 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
name: Run pytest
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches:
|
|
5
|
+
- main
|
|
6
|
+
pull_request:
|
|
7
|
+
branches:
|
|
8
|
+
- main
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
uv-example:
|
|
13
|
+
name: python
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
strategy:
|
|
16
|
+
matrix:
|
|
17
|
+
python-version:
|
|
18
|
+
- "3.11"
|
|
19
|
+
- "3.12"
|
|
20
|
+
- "3.13"
|
|
21
|
+
- "3.14"
|
|
22
|
+
defaults:
|
|
23
|
+
run:
|
|
24
|
+
shell: bash -el {0}
|
|
25
|
+
steps:
|
|
26
|
+
- name: Checkout repository code
|
|
27
|
+
uses: actions/checkout@v4
|
|
28
|
+
|
|
29
|
+
- name: Install system dependencies
|
|
30
|
+
run: |
|
|
31
|
+
sudo apt-get update
|
|
32
|
+
sudo apt-get install -y \
|
|
33
|
+
build-essential \
|
|
34
|
+
python3-dev \
|
|
35
|
+
libbz2-dev \
|
|
36
|
+
liblzma-dev \
|
|
37
|
+
libcurl4-gnutls-dev \
|
|
38
|
+
libssl-dev \
|
|
39
|
+
zlib1g-dev
|
|
40
|
+
|
|
41
|
+
- name: Conda setup
|
|
42
|
+
uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3
|
|
43
|
+
with:
|
|
44
|
+
conda-solver: libmamba
|
|
45
|
+
channels: conda-forge,bioconda,nodefaults
|
|
46
|
+
channel-priority: strict
|
|
47
|
+
conda-remove-defaults: true
|
|
48
|
+
|
|
49
|
+
- name: Install bioinformatics tools via conda
|
|
50
|
+
shell: bash -l {0}
|
|
51
|
+
run: |
|
|
52
|
+
conda install -c bioconda -c conda-forge -y mmseqs2 hifiasm spades
|
|
53
|
+
|
|
54
|
+
- name: Install uv
|
|
55
|
+
uses: astral-sh/setup-uv@v8.1.0
|
|
56
|
+
with:
|
|
57
|
+
version: "0.11.7"
|
|
58
|
+
python-version: ${{ matrix.python-version }}
|
|
59
|
+
|
|
60
|
+
- name: Set up Python
|
|
61
|
+
run: uv python install
|
|
62
|
+
|
|
63
|
+
- name: Install the project
|
|
64
|
+
run: uv sync --all-extras --dev
|
|
65
|
+
|
|
66
|
+
- name: Run tests
|
|
67
|
+
run: |
|
|
68
|
+
conda activate test
|
|
69
|
+
uv run pytest tests
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# hifi_trimmer: changelog
|
|
2
|
+
|
|
3
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
|
4
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
5
|
+
|
|
6
|
+
## [1.0.0](https://github.com/sanger-tol/markadoros/releases/tag/v1.0.0)] - [2026-04-23]
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
|
|
10
|
+
- Added options `--find-goat-synonyms` to `markadoros search` to get a list of synonyms from the expected taxon from [GoaT](https://goat.genomehubs.org),
|
|
11
|
+
`--synonyms` to provide a comma-separated list of synonyms.
|
|
12
|
+
- `markadoros search` will now fall back and search the first `--fallback-reads` reads assembly fails but reads containing markers were found.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- If an expected taxon is provided, the results are now correctly sorted by bit score rather than fident and alnlen.
|
|
17
|
+
|
|
18
|
+
## [0.1.0](https://github.com/sanger-tol/markadoros/releases/tag/v0.100)] - [2026-04-01]
|
|
19
|
+
|
|
20
|
+
Initial release of markadoros.
|
markadoros-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Genome Research Ltd.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: markadoros
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Markadoros is a tool for the identification and assembly of barcode genes in raw sequencing data.
|
|
5
|
+
Project-URL: Homepage, https://github.com/sanger-tol/markadoros/
|
|
6
|
+
Project-URL: Issues, https://github.com/sanger-tol/markadoros/issues
|
|
7
|
+
Author-email: Jim Downie <jd42@sanger.ac.uk>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Requires-Dist: biopython>=1.86
|
|
14
|
+
Requires-Dist: click>=8.3.1
|
|
15
|
+
Requires-Dist: isal>=1.8.0
|
|
16
|
+
Requires-Dist: jsonschema>=4.26.0
|
|
17
|
+
Requires-Dist: loguru>=0.7.3
|
|
18
|
+
Requires-Dist: pandas>=2.3.3
|
|
19
|
+
Requires-Dist: polars>=1.39.3
|
|
20
|
+
Requires-Dist: pymmseqs>=1.0.6
|
|
21
|
+
Requires-Dist: pysam>=0.23.3
|
|
22
|
+
Requires-Dist: requests>=2.33.1
|
|
23
|
+
Requires-Dist: scikit-learn>=1.8.0
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# Markadoros - fast barcode assembly and identification from raw sequencing data
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
▗▖ ▗▖▗▞▀▜▌ ▄▄▄ █ ▄ ▗▞▀▜▌▐▌ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄
|
|
30
|
+
▐▛▚▞▜▌▝▚▄▟▌█ █▄▀ ▝▚▄▟▌▐▌█ █ █ █ █ ▀▄▄
|
|
31
|
+
▐▌ ▐▌ █ █ ▀▄ ▗▞▀▜▌▀▄▄▄▀ █ ▀▄▄▄▀ ▄▄▄▀
|
|
32
|
+
▐▌ ▐▌ █ █ ▝▚▄▟▌
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Introduction
|
|
36
|
+
|
|
37
|
+
Markadoros is a Python tool for the identification and assembly of barcode genes in raw sequencing data, using
|
|
38
|
+
[MMSeqs2](https://github.com/soedinglab/MMseqs2) for searching and either [SPAdes](https://github.com/ablab/spades) or [Hifiasm](https://github.com/chhylp123/hifiasm) for assembly. Using an MMSeqs2 database of marker gene sequences, markadoros searches a set of input reads (providable in either FASTX or CRAM format) to quickly pre-filter reads that map to the target marker gene. Reads that match any barcode sequence in the database are extracted and assembled using an appropriate assembler. The resulting assembled contigs are then searched again using the same database, using more sensitive thresholds, to identify the marker genes from the input dataset.
|
|
39
|
+
|
|
40
|
+
The name *markadoros* comes from the Greek word for a marker pen, *μαρκαδόρος*.
|
|
41
|
+
|
|
42
|
+
## Requirements
|
|
43
|
+
|
|
44
|
+
### System Dependencies
|
|
45
|
+
|
|
46
|
+
You will need the following external tools installed to run Markadoros:
|
|
47
|
+
|
|
48
|
+
- [MMSeqs2](https://github.com/soedinglab/MMseqs2)
|
|
49
|
+
- [SPAdes](https://github.com/ablab/spades)
|
|
50
|
+
- [Hifiasm](https://github.com/chhylp123/hifiasm)
|
|
51
|
+
|
|
52
|
+
### Python Dependencies
|
|
53
|
+
|
|
54
|
+
Markadoros requires Python 3.11+ and the following packages:
|
|
55
|
+
|
|
56
|
+
- biopython (≥1.86)
|
|
57
|
+
- click (≥8.3.1)
|
|
58
|
+
- jsonschema (≥4.26.0)
|
|
59
|
+
- pandas (≥2.3.3)
|
|
60
|
+
- pymmseqs (≥1.0.5)
|
|
61
|
+
- pysam (≥0.23.3)
|
|
62
|
+
- scikit-learn (≥1.8.0)
|
|
63
|
+
|
|
64
|
+
These are automatically installed when you install Markadoros.
|
|
65
|
+
|
|
66
|
+
## Installation
|
|
67
|
+
|
|
68
|
+
### From source
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install -e .
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Dependencies
|
|
75
|
+
|
|
76
|
+
Install the required external tools with Conda:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
conda install -c bioconda mmseqs2 spades hifiasm
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Quick Start
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
# 1. Build a database from a BOLD release
|
|
86
|
+
markadoros database -x bold --prefix BOLD --cluster --outdir db/ <bold_release.fasta.gz>
|
|
87
|
+
|
|
88
|
+
# 2. Search your reads
|
|
89
|
+
markadoros search -x illumina -n 100000 --index db/db.json --db BOLD.COI <reads.fq.gz>
|
|
90
|
+
|
|
91
|
+
# 3. Check results
|
|
92
|
+
less reads.COI.summary.json
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Detailed Usage
|
|
96
|
+
|
|
97
|
+
### Database Preparation
|
|
98
|
+
|
|
99
|
+
Use the `database` command to prepare marker gene sequences for searching:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
markadoros database -x bold \
|
|
103
|
+
--prefix BOLD \
|
|
104
|
+
--outdir db/ \
|
|
105
|
+
/path/to/bold/release.fasta.gz
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
**Required options:**
|
|
109
|
+
|
|
110
|
+
- `-x, --header-type <type>` - Use a preset header processor: `bold`, `unite`, `silva_lsu`, `silva_ssu`, `generic`.
|
|
111
|
+
- `--prefix <name>` - Prefix for output database names
|
|
112
|
+
|
|
113
|
+
**Additional options:**
|
|
114
|
+
|
|
115
|
+
- `--marker <name>` - Only when using `--header-type` is `generic` - the name of the marker gene in the `marker` field of the FASTA header.
|
|
116
|
+
- `--min-length <N>` - Minimum sequence length to retain (default: 200)
|
|
117
|
+
- `--deduplicate/--no-deduplicate` - Deduplicate identical sequences. The first record for each identical sequence is kept. (default: false)
|
|
118
|
+
- `--cluster/--no-cluster` - Cluster sequences using MMSeqs2's linear clustering algorithm (default: no cluster)
|
|
119
|
+
- `--cluster_min_seq_id` - Cluster sequences at this percentage identity threshold (default: 0.99)
|
|
120
|
+
- `--cluster_coverage` - Overlap between two sequences required for clustering (default: 0.8)
|
|
121
|
+
- `--create-index/--no-create-index` - Create an MMSeqs2 index for each marker database. This may improve speed for larger databases, but can cause IO issues if multiple processes access the same database. (default: False)
|
|
122
|
+
- `--exclude-file` - New line-separated list of regular expressions. If a header matches a regular expression it will be skipped.
|
|
123
|
+
- `-o, --outdir <path>` - Output directory (default: `./markadoros.db`)
|
|
124
|
+
- `--cleanup/--no-cleanup` - Clean up temporary files (default: cleanup)
|
|
125
|
+
- `-t, --threads <N>` - Number of threads for MMSeqs2 (default: 1)
|
|
126
|
+
|
|
127
|
+
**Header types:**
|
|
128
|
+
|
|
129
|
+
- `bold` - [BOLD Systems](https://www.boldsystems.org/) [general FASTA release](https://bench.boldsystems.org/index.php/datapackages/Latest)
|
|
130
|
+
- `unite` - [UNITE](https://unite.ut.ee/) [general FASTA release](https://unite.ut.ee/repository.php)
|
|
131
|
+
- `silva_lsu` - [SILVA](https://www.arb-silva.de/) [LSU Release](https://www.arb-silva.de/current-release/Exports)
|
|
132
|
+
- `silva_ssu` - [SILVA](https://www.arb-silva.de/) [SSU Release](https://www.arb-silva.de/current-release/Exports)
|
|
133
|
+
|
|
134
|
+
If your FASTA release does not conform to the above presets, set `-x` to `generic`. Your input FASTA headers must then be formatted as:
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
><unique_id>|<marker>|<taxon_name>|<lineage>
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
**Output files:**
|
|
141
|
+
|
|
142
|
+
- `db.json` - Index of available databases and their parameters
|
|
143
|
+
- `<prefix>_<marker>/db*` - MMSeqs2 database files
|
|
144
|
+
- `<prefix>_<marker>/taxon.json.gz` - JSON file counting the number of available sequences per taxon.
|
|
145
|
+
|
|
146
|
+
If you build additional databases pointing to the same output directory, the existing index file will be updated to include the new entries.
|
|
147
|
+
|
|
148
|
+
### Building a database from the BOLD TSV release
|
|
149
|
+
|
|
150
|
+
Alternatively, you can use the `bold-coi-from-tsv` to build a BOLD COI database from the BOLD TSV release. The BOLD TSV release includes information on the BIN, or cluster, of each sequence and it can thus be used to create a database with a single representative sequence per cluster instead of running your own clustering. Currently this appoach is only supported for the COI marker.
|
|
151
|
+
|
|
152
|
+
Run it as follows:
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
markadoros bold-coi-from-tsv --prefix BOLD --threads 16 --outdir . BOLD_Public.27-Mar-2026.tsv.gz
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Options such as `--exclude-file` and `--min-length` work the same as in the standard `database` command.
|
|
159
|
+
|
|
160
|
+
### Searching for Barcodes
|
|
161
|
+
|
|
162
|
+
Use the `search` command to identify barcode genes in a set of reads or pre-assembled contigs:
|
|
163
|
+
|
|
164
|
+
```bash
|
|
165
|
+
markadoros search -x illumina --index db/db.json reads.fq.gz
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
You can supply the binomial name of a taxon that you believe the data should arise from with `--expected-taxon`. In
|
|
169
|
+
this case, markadoros will check the results and tell you how many possible sequences there are to match against and
|
|
170
|
+
how many hits were found for the taxon. If the expected taxon is found, it also reports the top hit for that taxon as the
|
|
171
|
+
best hit instead of the overall top hit.
|
|
172
|
+
|
|
173
|
+
Sometimes, there may be synonyms for the expected taxon that you may wish to check additionally. To accomodate these,
|
|
174
|
+
you can supply either the `--find-goat-synonyms` flag, which will pull the names of all synonyms from
|
|
175
|
+
[GoaT](https://goat.genomehubs.org/), or alternatively supply known synonyms as a comma-separated list with `--synonyms`. If
|
|
176
|
+
a hit is for a synonym, it will be marked as such in the output results.
|
|
177
|
+
|
|
178
|
+
**Required options:**
|
|
179
|
+
|
|
180
|
+
- `-x, --type <type>` - Input data type (see table below)
|
|
181
|
+
- `-i, --index <path>` - Path to database index JSON
|
|
182
|
+
|
|
183
|
+
**Input type aliases:**
|
|
184
|
+
|
|
185
|
+
| Platform | Accepted values |
|
|
186
|
+
|------------------------|--------------------------------------|
|
|
187
|
+
| Illumina / short reads | `sr`, `short`, `illumina` |
|
|
188
|
+
| Short read RNA-seq | `rnaseq` |
|
|
189
|
+
| PacBio HiFi | `pb`, `pacbio`, `pacbio_hifi` |
|
|
190
|
+
| Oxford Nanopore | `ont`, `nanopore`, `oxford_nanopore` |
|
|
191
|
+
| Pre-assembled contigs | `contigs` |
|
|
192
|
+
|
|
193
|
+
**Additional options:**
|
|
194
|
+
|
|
195
|
+
- `--db <name>` - Search a specific database only (default: search all databases in index)
|
|
196
|
+
- `--expected_taxon <name>` - Expected taxon binomial name for validation
|
|
197
|
+
- `-s`, `--find-goat-synonyms` - Get the synonyms for the provided expected taxon from GoaT
|
|
198
|
+
- `--synonyms` - A comma separated list of synonyms. Will override `--find-goat-synonyms` if provided.
|
|
199
|
+
- `-n, --nreads <N>` - Limit to first N reads
|
|
200
|
+
- `-m, --min_seq_id <float>` - Minimum sequence identity for hits (default: 0.96)
|
|
201
|
+
- `-l, --min_aln_len <int>` - Minimum alignment length for hits (default: 450)
|
|
202
|
+
- `--cleanup/--no-cleanup` - Clean up temporary files (default: cleanup)
|
|
203
|
+
- `--db-to-tmpdir/--no-db-to-tmpdir` - Temporarily copy the database to the temporary directory. Can reduce IO and improve speed if multiple processes would access the database simultaneously. Not suggested if the databases include indexes. (default: True)
|
|
204
|
+
- `-t, --threads <N>` - Number of threads (default: 1)
|
|
205
|
+
- `-p, --prefix <name>` - Output file prefix (default: input filename)
|
|
206
|
+
- `-o, --outdir <path>` - Output directory (default: current directory)
|
|
207
|
+
- `--save-contigs` - Write the assembled contigs to disk in the output directory.
|
|
208
|
+
|
|
209
|
+
**Output files:**
|
|
210
|
+
|
|
211
|
+
- `<prefix>.<marker>.summary.json` - Results in JSON format
|
|
212
|
+
|
|
213
|
+
### Output JSON file
|
|
214
|
+
|
|
215
|
+
The search subtool outputs a JSON file summarising the search. It has the following format:
|
|
216
|
+
|
|
217
|
+
```json
|
|
218
|
+
{
|
|
219
|
+
"input": {
|
|
220
|
+
"file": <path>, // input file path
|
|
221
|
+
"n_reads": <int>, // number of reads searched
|
|
222
|
+
"n_aligned_reads": <int>, // number of reads aligned to database
|
|
223
|
+
"marker": <string>, // marker gene in database
|
|
224
|
+
"database": <path>, // path to mmseqs database
|
|
225
|
+
"contig_stats": <dict>, // number, total length and n50 of assembled contigs
|
|
226
|
+
"expected_taxon": <dict>, // name of asserted taxon and number of records present in database, including synonyms
|
|
227
|
+
},
|
|
228
|
+
"summary": {
|
|
229
|
+
"n_contigs_with_hits": <int>, // number of assembled contigs with search hits
|
|
230
|
+
"n_expected_taxon_hits": <int>, // number of hits for the asserted taxon
|
|
231
|
+
"n_synonym_hits": <int>, // number of hits for synonyms
|
|
232
|
+
"top_result": <dict>, // top result (highest bitscore) for asserted taxon if found or overall if not
|
|
233
|
+
"taxon_summary": <dict>, // per-taxon summary of results - number of hits, min and max %ID and alignment lengths, and sequence of top hit
|
|
234
|
+
},
|
|
235
|
+
"results": <list>, // for each contig with results, a list of all hits
|
|
236
|
+
"run_info": <dict> // tool information
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
### Example Workflows
|
|
242
|
+
|
|
243
|
+
**Building a marker database for the BOLD FASTA release:**
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
markadoros database \
|
|
247
|
+
--header-type bold \
|
|
248
|
+
--outdir db/ \
|
|
249
|
+
--prefix BOLD \
|
|
250
|
+
--cluster \
|
|
251
|
+
--threads 16 \
|
|
252
|
+
BOLD_Public.20-Feb-2026.fasta.gz
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
**Searching PacBio HiFi reads with expected taxon:**
|
|
256
|
+
|
|
257
|
+
```bash
|
|
258
|
+
markadoros search -x pb \
|
|
259
|
+
--index db/db.json \
|
|
260
|
+
--db BOLD.COI \
|
|
261
|
+
--expected_taxon "Halyzia sedecimguttata" \
|
|
262
|
+
--threads 16 \
|
|
263
|
+
--nreads 20000 \
|
|
264
|
+
pacbio_reads.fasta.gz
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
## Author
|
|
268
|
+
|
|
269
|
+
- Jim Downie
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
# Markadoros - fast barcode assembly and identification from raw sequencing data
|
|
2
|
+
|
|
3
|
+
```
|
|
4
|
+
▗▖ ▗▖▗▞▀▜▌ ▄▄▄ █ ▄ ▗▞▀▜▌▐▌ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄
|
|
5
|
+
▐▛▚▞▜▌▝▚▄▟▌█ █▄▀ ▝▚▄▟▌▐▌█ █ █ █ █ ▀▄▄
|
|
6
|
+
▐▌ ▐▌ █ █ ▀▄ ▗▞▀▜▌▀▄▄▄▀ █ ▀▄▄▄▀ ▄▄▄▀
|
|
7
|
+
▐▌ ▐▌ █ █ ▝▚▄▟▌
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
## Introduction
|
|
11
|
+
|
|
12
|
+
Markadoros is a Python tool for the identification and assembly of barcode genes in raw sequencing data, using
|
|
13
|
+
[MMSeqs2](https://github.com/soedinglab/MMseqs2) for searching and either [SPAdes](https://github.com/ablab/spades) or [Hifiasm](https://github.com/chhylp123/hifiasm) for assembly. Using an MMSeqs2 database of marker gene sequences, markadoros searches a set of input reads (providable in either FASTX or CRAM format) to quickly pre-filter reads that map to the target marker gene. Reads that match any barcode sequence in the database are extracted and assembled using an appropriate assembler. The resulting assembled contigs are then searched again using the same database, using more sensitive thresholds, to identify the marker genes from the input dataset.
|
|
14
|
+
|
|
15
|
+
The name *markadoros* comes from the Greek word for a marker pen, *μαρκαδόρος*.
|
|
16
|
+
|
|
17
|
+
## Requirements
|
|
18
|
+
|
|
19
|
+
### System Dependencies
|
|
20
|
+
|
|
21
|
+
You will need the following external tools installed to run Markadoros:
|
|
22
|
+
|
|
23
|
+
- [MMSeqs2](https://github.com/soedinglab/MMseqs2)
|
|
24
|
+
- [SPAdes](https://github.com/ablab/spades)
|
|
25
|
+
- [Hifiasm](https://github.com/chhylp123/hifiasm)
|
|
26
|
+
|
|
27
|
+
### Python Dependencies
|
|
28
|
+
|
|
29
|
+
Markadoros requires Python 3.11+ and the following packages:
|
|
30
|
+
|
|
31
|
+
- biopython (≥1.86)
|
|
32
|
+
- click (≥8.3.1)
|
|
33
|
+
- jsonschema (≥4.26.0)
|
|
34
|
+
- pandas (≥2.3.3)
|
|
35
|
+
- pymmseqs (≥1.0.5)
|
|
36
|
+
- pysam (≥0.23.3)
|
|
37
|
+
- scikit-learn (≥1.8.0)
|
|
38
|
+
|
|
39
|
+
These are automatically installed when you install Markadoros.
|
|
40
|
+
|
|
41
|
+
## Installation
|
|
42
|
+
|
|
43
|
+
### From source
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install -e .
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### Dependencies
|
|
50
|
+
|
|
51
|
+
Install the required external tools with Conda:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
conda install -c bioconda mmseqs2 spades hifiasm
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Quick Start
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
# 1. Build a database from a BOLD release
|
|
61
|
+
markadoros database -x bold --prefix BOLD --cluster --outdir db/ <bold_release.fasta.gz>
|
|
62
|
+
|
|
63
|
+
# 2. Search your reads
|
|
64
|
+
markadoros search -x illumina -n 100000 --index db/db.json --db BOLD.COI <reads.fq.gz>
|
|
65
|
+
|
|
66
|
+
# 3. Check results
|
|
67
|
+
less reads.COI.summary.json
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Detailed Usage
|
|
71
|
+
|
|
72
|
+
### Database Preparation
|
|
73
|
+
|
|
74
|
+
Use the `database` command to prepare marker gene sequences for searching:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
markadoros database -x bold \
|
|
78
|
+
--prefix BOLD \
|
|
79
|
+
--outdir db/ \
|
|
80
|
+
/path/to/bold/release.fasta.gz
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
**Required options:**
|
|
84
|
+
|
|
85
|
+
- `-x, --header-type <type>` - Use a preset header processor: `bold`, `unite`, `silva_lsu`, `silva_ssu`, `generic`.
|
|
86
|
+
- `--prefix <name>` - Prefix for output database names
|
|
87
|
+
|
|
88
|
+
**Additional options:**
|
|
89
|
+
|
|
90
|
+
- `--marker <name>` - Only when using `--header-type` is `generic` - the name of the marker gene in the `marker` field of the FASTA header.
|
|
91
|
+
- `--min-length <N>` - Minimum sequence length to retain (default: 200)
|
|
92
|
+
- `--deduplicate/--no-deduplicate` - Deduplicate identical sequences. The first record for each identical sequence is kept. (default: false)
|
|
93
|
+
- `--cluster/--no-cluster` - Cluster sequences using MMSeqs2's linear clustering algorithm (default: no cluster)
|
|
94
|
+
- `--cluster_min_seq_id` - Cluster sequences at this percentage identity threshold (default: 0.99)
|
|
95
|
+
- `--cluster_coverage` - Overlap between two sequences required for clustering (default: 0.8)
|
|
96
|
+
- `--create-index/--no-create-index` - Create an MMSeqs2 index for each marker database. This may improve speed for larger databases, but can cause IO issues if multiple processes access the same database. (default: False)
|
|
97
|
+
- `--exclude-file` - New line-separated list of regular expressions. If a header matches a regular expression it will be skipped.
|
|
98
|
+
- `-o, --outdir <path>` - Output directory (default: `./markadoros.db`)
|
|
99
|
+
- `--cleanup/--no-cleanup` - Clean up temporary files (default: cleanup)
|
|
100
|
+
- `-t, --threads <N>` - Number of threads for MMSeqs2 (default: 1)
|
|
101
|
+
|
|
102
|
+
**Header types:**
|
|
103
|
+
|
|
104
|
+
- `bold` - [BOLD Systems](https://www.boldsystems.org/) [general FASTA release](https://bench.boldsystems.org/index.php/datapackages/Latest)
|
|
105
|
+
- `unite` - [UNITE](https://unite.ut.ee/) [general FASTA release](https://unite.ut.ee/repository.php)
|
|
106
|
+
- `silva_lsu` - [SILVA](https://www.arb-silva.de/) [LSU Release](https://www.arb-silva.de/current-release/Exports)
|
|
107
|
+
- `silva_ssu` - [SILVA](https://www.arb-silva.de/) [SSU Release](https://www.arb-silva.de/current-release/Exports)
|
|
108
|
+
|
|
109
|
+
If your FASTA release does not conform to the above presets, set `-x` to `generic`. Your input FASTA headers must then be formatted as:
|
|
110
|
+
|
|
111
|
+
```
|
|
112
|
+
><unique_id>|<marker>|<taxon_name>|<lineage>
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
**Output files:**
|
|
116
|
+
|
|
117
|
+
- `db.json` - Index of available databases and their parameters
|
|
118
|
+
- `<prefix>_<marker>/db*` - MMSeqs2 database files
|
|
119
|
+
- `<prefix>_<marker>/taxon.json.gz` - JSON file counting the number of available sequences per taxon.
|
|
120
|
+
|
|
121
|
+
If you build additional databases pointing to the same output directory, the existing index file will be updated to include the new entries.
|
|
122
|
+
|
|
123
|
+
### Building a database from the BOLD TSV release
|
|
124
|
+
|
|
125
|
+
Alternatively, you can use the `bold-coi-from-tsv` to build a BOLD COI database from the BOLD TSV release. The BOLD TSV release includes information on the BIN, or cluster, of each sequence and it can thus be used to create a database with a single representative sequence per cluster instead of running your own clustering. Currently this appoach is only supported for the COI marker.
|
|
126
|
+
|
|
127
|
+
Run it as follows:
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
markadoros bold-coi-from-tsv --prefix BOLD --threads 16 --outdir . BOLD_Public.27-Mar-2026.tsv.gz
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Options such as `--exclude-file` and `--min-length` work the same as in the standard `database` command.
|
|
134
|
+
|
|
135
|
+
### Searching for Barcodes
|
|
136
|
+
|
|
137
|
+
Use the `search` command to identify barcode genes in a set of reads or pre-assembled contigs:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
markadoros search -x illumina --index db/db.json reads.fq.gz
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
You can supply the binomial name of a taxon that you believe the data should arise from with `--expected-taxon`. In
|
|
144
|
+
this case, markadoros will check the results and tell you how many possible sequences there are to match against and
|
|
145
|
+
how many hits were found for the taxon. If the expected taxon is found, it also reports the top hit for that taxon as the
|
|
146
|
+
best hit instead of the overall top hit.
|
|
147
|
+
|
|
148
|
+
Sometimes, there may be synonyms for the expected taxon that you may wish to check additionally. To accomodate these,
|
|
149
|
+
you can supply either the `--find-goat-synonyms` flag, which will pull the names of all synonyms from
|
|
150
|
+
[GoaT](https://goat.genomehubs.org/), or alternatively supply known synonyms as a comma-separated list with `--synonyms`. If
|
|
151
|
+
a hit is for a synonym, it will be marked as such in the output results.
|
|
152
|
+
|
|
153
|
+
**Required options:**
|
|
154
|
+
|
|
155
|
+
- `-x, --type <type>` - Input data type (see table below)
|
|
156
|
+
- `-i, --index <path>` - Path to database index JSON
|
|
157
|
+
|
|
158
|
+
**Input type aliases:**
|
|
159
|
+
|
|
160
|
+
| Platform | Accepted values |
|
|
161
|
+
|------------------------|--------------------------------------|
|
|
162
|
+
| Illumina / short reads | `sr`, `short`, `illumina` |
|
|
163
|
+
| Short read RNA-seq | `rnaseq` |
|
|
164
|
+
| PacBio HiFi | `pb`, `pacbio`, `pacbio_hifi` |
|
|
165
|
+
| Oxford Nanopore | `ont`, `nanopore`, `oxford_nanopore` |
|
|
166
|
+
| Pre-assembled contigs | `contigs` |
|
|
167
|
+
|
|
168
|
+
**Additional options:**
|
|
169
|
+
|
|
170
|
+
- `--db <name>` - Search a specific database only (default: search all databases in index)
|
|
171
|
+
- `--expected_taxon <name>` - Expected taxon binomial name for validation
|
|
172
|
+
- `-s`, `--find-goat-synonyms` - Get the synonyms for the provided expected taxon from GoaT
|
|
173
|
+
- `--synonyms` - A comma separated list of synonyms. Will override `--find-goat-synonyms` if provided.
|
|
174
|
+
- `-n, --nreads <N>` - Limit to first N reads
|
|
175
|
+
- `-m, --min_seq_id <float>` - Minimum sequence identity for hits (default: 0.96)
|
|
176
|
+
- `-l, --min_aln_len <int>` - Minimum alignment length for hits (default: 450)
|
|
177
|
+
- `--cleanup/--no-cleanup` - Clean up temporary files (default: cleanup)
|
|
178
|
+
- `--db-to-tmpdir/--no-db-to-tmpdir` - Temporarily copy the database to the temporary directory. Can reduce IO and improve speed if multiple processes would access the database simultaneously. Not suggested if the databases include indexes. (default: True)
|
|
179
|
+
- `-t, --threads <N>` - Number of threads (default: 1)
|
|
180
|
+
- `-p, --prefix <name>` - Output file prefix (default: input filename)
|
|
181
|
+
- `-o, --outdir <path>` - Output directory (default: current directory)
|
|
182
|
+
- `--save-contigs` - Write the assembled contigs to disk in the output directory.
|
|
183
|
+
|
|
184
|
+
**Output files:**
|
|
185
|
+
|
|
186
|
+
- `<prefix>.<marker>.summary.json` - Results in JSON format
|
|
187
|
+
|
|
188
|
+
### Output JSON file
|
|
189
|
+
|
|
190
|
+
The search subtool outputs a JSON file summarising the search. It has the following format:
|
|
191
|
+
|
|
192
|
+
```json
|
|
193
|
+
{
|
|
194
|
+
"input": {
|
|
195
|
+
"file": <path>, // input file path
|
|
196
|
+
"n_reads": <int>, // number of reads searched
|
|
197
|
+
"n_aligned_reads": <int>, // number of reads aligned to database
|
|
198
|
+
"marker": <string>, // marker gene in database
|
|
199
|
+
"database": <path>, // path to mmseqs database
|
|
200
|
+
"contig_stats": <dict>, // number, total length and n50 of assembled contigs
|
|
201
|
+
"expected_taxon": <dict>, // name of asserted taxon and number of records present in database, including synonyms
|
|
202
|
+
},
|
|
203
|
+
"summary": {
|
|
204
|
+
"n_contigs_with_hits": <int>, // number of assembled contigs with search hits
|
|
205
|
+
"n_expected_taxon_hits": <int>, // number of hits for the asserted taxon
|
|
206
|
+
"n_synonym_hits": <int>, // number of hits for synonyms
|
|
207
|
+
"top_result": <dict>, // top result (highest bitscore) for asserted taxon if found or overall if not
|
|
208
|
+
"taxon_summary": <dict>, // per-taxon summary of results - number of hits, min and max %ID and alignment lengths, and sequence of top hit
|
|
209
|
+
},
|
|
210
|
+
"results": <list>, // for each contig with results, a list of all hits
|
|
211
|
+
"run_info": <dict> // tool information
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
### Example Workflows
|
|
217
|
+
|
|
218
|
+
**Building a marker database for the BOLD FASTA release:**
|
|
219
|
+
|
|
220
|
+
```bash
|
|
221
|
+
markadoros database \
|
|
222
|
+
--header-type bold \
|
|
223
|
+
--outdir db/ \
|
|
224
|
+
--prefix BOLD \
|
|
225
|
+
--cluster \
|
|
226
|
+
--threads 16 \
|
|
227
|
+
BOLD_Public.20-Feb-2026.fasta.gz
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
**Searching PacBio HiFi reads with expected taxon:**
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
markadoros search -x pb \
|
|
234
|
+
--index db/db.json \
|
|
235
|
+
--db BOLD.COI \
|
|
236
|
+
--expected_taxon "Halyzia sedecimguttata" \
|
|
237
|
+
--threads 16 \
|
|
238
|
+
--nreads 20000 \
|
|
239
|
+
pacbio_reads.fasta.gz
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
## Author
|
|
243
|
+
|
|
244
|
+
- Jim Downie
|