seqcounter 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seqcounter/__init__.py +3 -0
- seqcounter/__main__.py +6 -0
- seqcounter/cli.py +184 -0
- seqcounter/counts.py +168 -0
- seqcounter/errors.py +5 -0
- seqcounter/references.py +87 -0
- seqcounter/samples.py +123 -0
- seqcounter/seal.py +283 -0
- seqcounter-1.0.0.dist-info/METADATA +140 -0
- seqcounter-1.0.0.dist-info/RECORD +14 -0
- seqcounter-1.0.0.dist-info/WHEEL +5 -0
- seqcounter-1.0.0.dist-info/entry_points.txt +2 -0
- seqcounter-1.0.0.dist-info/licenses/LICENSE +21 -0
- seqcounter-1.0.0.dist-info/top_level.txt +1 -0
seqcounter/__init__.py
ADDED
seqcounter/__main__.py
ADDED
seqcounter/cli.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Command line entry point."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import shlex
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
from argparse import ArgumentParser, ArgumentTypeError, RawDescriptionHelpFormatter
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from . import __version__
|
|
14
|
+
from .counts import count_samples, print_table, run_info, write_reports
|
|
15
|
+
from .errors import SeqCounterError
|
|
16
|
+
from .references import load_references
|
|
17
|
+
from .samples import find_samples, single_sample
|
|
18
|
+
from .seal import DEFAULT_KMER_SIZE, DEFAULT_MISMATCHES, auto_memory, check_seal
|
|
19
|
+
|
|
20
|
+
log = logging.getLogger('seqcounter')
|
|
21
|
+
|
|
22
|
+
# Expected errors are reported without a traceback
|
|
23
|
+
USER_ERRORS = (SeqCounterError, OSError)
|
|
24
|
+
|
|
25
|
+
EPILOG = '''examples:
|
|
26
|
+
seqcounter -ref refs.fasta -r1 sample_R1.fastq.gz -r2 sample_R2.fastq.gz -o results/
|
|
27
|
+
seqcounter -ref refs.fasta -r1 sample.fastq.gz -o results/
|
|
28
|
+
seqcounter -ref refs.fasta -r1 assembly.fasta -o results/
|
|
29
|
+
seqcounter -ref refs.fasta -i folder_of_fastq_and_fasta/ -o results/
|
|
30
|
+
|
|
31
|
+
documentation: https://github.com/duceppemo/seqcounter/wiki'''
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def setup_logging(verbose=False):
|
|
35
|
+
"""Configure the package logger only, so importing seqcounter never alters the root logger."""
|
|
36
|
+
for handler in list(log.handlers):
|
|
37
|
+
log.removeHandler(handler)
|
|
38
|
+
handler = logging.StreamHandler(sys.stderr)
|
|
39
|
+
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)-7s %(message)s', datefmt='%H:%M:%S'))
|
|
40
|
+
log.addHandler(handler)
|
|
41
|
+
log.setLevel(logging.DEBUG if verbose else logging.INFO)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def positive_int(value):
|
|
45
|
+
try:
|
|
46
|
+
value = int(value)
|
|
47
|
+
except ValueError:
|
|
48
|
+
raise ArgumentTypeError('"{}" is not an integer'.format(value)) from None
|
|
49
|
+
if value < 1:
|
|
50
|
+
raise ArgumentTypeError('must be >= 1')
|
|
51
|
+
return value
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def java_memory(value):
|
|
55
|
+
if not value[:-1].isdigit() or value[-1].lower() not in 'mg':
|
|
56
|
+
raise ArgumentTypeError('"{}" is not a Java memory size such as 500m or 2g'.format(value))
|
|
57
|
+
return value
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def available_cpus():
|
|
61
|
+
try:
|
|
62
|
+
return len(os.sched_getaffinity(0)) # Respects cgroup/taskset limits on Linux (e.g. SLURM jobs)
|
|
63
|
+
except AttributeError:
|
|
64
|
+
return os.cpu_count() or 1
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def format_elapsed(seconds):
|
|
68
|
+
"""A readable duration, as the first version of seqcounter printed it: "1h2m3s"."""
|
|
69
|
+
minutes, seconds = divmod(int(round(seconds)), 60)
|
|
70
|
+
hours, minutes = divmod(minutes, 60)
|
|
71
|
+
days, hours = divmod(hours, 24)
|
|
72
|
+
periods = [('d', days), ('h', hours), ('m', minutes), ('s', seconds)]
|
|
73
|
+
return ''.join('{}{}'.format(value, name) for name, value in periods if value) or '0s'
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def build_parser():
|
|
77
|
+
max_cpu = available_cpus()
|
|
78
|
+
parser = ArgumentParser(prog='seqcounter', formatter_class=RawDescriptionHelpFormatter, epilog=EPILOG,
|
|
79
|
+
description='Count the occurrences of reference sequences in sequencing reads (fastq)\n'
|
|
80
|
+
'or genome assemblies (fasta), with Seal from BBTools.')
|
|
81
|
+
inputs = parser.add_argument_group('input (one sample with -r1, or a folder of samples with -i)')
|
|
82
|
+
source = inputs.add_mutually_exclusive_group(required=True)
|
|
83
|
+
source.add_argument('-r1', '--r1', metavar='FILE',
|
|
84
|
+
help='Reads (single-end or R1 of paired-end) or assembly. fastq or fasta, gzipped or not.')
|
|
85
|
+
source.add_argument('-i', '--input', metavar='FOLDER',
|
|
86
|
+
help='Folder of fastq and fasta files, searched recursively. Each fasta file, single-end '
|
|
87
|
+
'fastq file or pair of R1/R2 fastq files is a sample.')
|
|
88
|
+
inputs.add_argument('-r2', '--r2', metavar='FILE',
|
|
89
|
+
help='R2 reads, for paired-end fastq files.')
|
|
90
|
+
inputs.add_argument('-ref', '--ref', metavar='FILE', required=True,
|
|
91
|
+
help='Fasta file of the reference sequences to count, gzipped or not. One row per sequence '
|
|
92
|
+
'is reported, named after the first word of its ">" line.')
|
|
93
|
+
inputs.add_argument('-s', '--sample', metavar='NAME',
|
|
94
|
+
help='With -r1: sample name used in the reports. Default: the input file name, without '
|
|
95
|
+
'extension and read suffix (_R1, _1, ...).')
|
|
96
|
+
|
|
97
|
+
output = parser.add_argument_group('output')
|
|
98
|
+
output.add_argument('-o', '--output', metavar='FOLDER', required=True,
|
|
99
|
+
help='Folder to hold the reports. Created if needed.')
|
|
100
|
+
|
|
101
|
+
counting = parser.add_argument_group('counting')
|
|
102
|
+
counting.add_argument('-k', '--kmer-size', metavar='N', type=positive_int, default=DEFAULT_KMER_SIZE,
|
|
103
|
+
help='K-mer size used to find the reference sequences. A reference shorter than k cannot '
|
|
104
|
+
'be found. Default: {}.'.format(DEFAULT_KMER_SIZE))
|
|
105
|
+
counting.add_argument('-m', '--mismatch', metavar='N', type=int, default=DEFAULT_MISMATCHES,
|
|
106
|
+
help='Mismatches allowed between a reference k-mer and the input (maximum Hamming '
|
|
107
|
+
'distance). Default: {}.'.format(DEFAULT_MISMATCHES))
|
|
108
|
+
|
|
109
|
+
other = parser.add_argument_group('performance and other options')
|
|
110
|
+
other.add_argument('-t', '--threads', metavar='N', type=positive_int, default=max_cpu,
|
|
111
|
+
help='Number of threads. Default: all available ({}).'.format(max_cpu))
|
|
112
|
+
other.add_argument('-p', '-j', '--parallel', '--jobs', metavar='N', type=positive_int, default=1, dest='parallel',
|
|
113
|
+
help='With -i: number of samples counted at the same time, sharing the threads. Default: 1.')
|
|
114
|
+
other.add_argument('--memory', metavar='SIZE', type=java_memory,
|
|
115
|
+
help='Memory for Seal (Java heap size), per job, e.g. 2g. Default: from the size of the '
|
|
116
|
+
'reference sequences, within 85%% of the machine.')
|
|
117
|
+
other.add_argument('--verbose', action='store_true', help='Show debug messages, including the Seal command.')
|
|
118
|
+
other.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)
|
|
119
|
+
return parser
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def check_arguments(parser, args):
|
|
123
|
+
if args.input and (args.r2 or args.sample):
|
|
124
|
+
parser.error('-r2 and --sample are only used with -r1')
|
|
125
|
+
if args.r1 and args.parallel > 1:
|
|
126
|
+
parser.error('--parallel is only used with -i')
|
|
127
|
+
if args.mismatch < 0:
|
|
128
|
+
parser.error('--mismatch cannot be negative')
|
|
129
|
+
if args.mismatch >= args.kmer_size:
|
|
130
|
+
parser.error('--mismatch ({}) must be smaller than --kmer-size ({})'.format(args.mismatch, args.kmer_size))
|
|
131
|
+
if args.sample is not None and (not args.sample.strip() or '/' in args.sample or args.sample in ('.', '..')):
|
|
132
|
+
parser.error('--sample must be a name, not a path: "{}"'.format(args.sample))
|
|
133
|
+
if args.sample is not None and re.search(r'[\t\r\n]', args.sample):
|
|
134
|
+
parser.error('--sample cannot contain tabs or line breaks (they would break the table)')
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def main(argv=None):
|
|
138
|
+
parser = build_parser()
|
|
139
|
+
args = parser.parse_args(argv)
|
|
140
|
+
check_arguments(parser, args)
|
|
141
|
+
setup_logging(args.verbose)
|
|
142
|
+
command = shlex.join([parser.prog, *(sys.argv[1:] if argv is None else argv)])
|
|
143
|
+
log.debug('seqcounter %s: %s', __version__, command)
|
|
144
|
+
started = datetime.now().astimezone()
|
|
145
|
+
start = time.monotonic()
|
|
146
|
+
output = Path(args.output).expanduser()
|
|
147
|
+
try:
|
|
148
|
+
check_seal()
|
|
149
|
+
references = load_references(args.ref, args.kmer_size)
|
|
150
|
+
if args.input:
|
|
151
|
+
samples = find_samples(args.input, exclude=[output])
|
|
152
|
+
log.info('%d sample(s) found in %s', len(samples), args.input)
|
|
153
|
+
else:
|
|
154
|
+
samples = [single_sample(args.r1, args.r2, name=args.sample)]
|
|
155
|
+
if any(s.file_type == 'fasta' for s in samples):
|
|
156
|
+
log.info('fasta input: the count of a reference sequence is the number of sequences (contigs) that '
|
|
157
|
+
'contain it, not the number of copies it has in them.')
|
|
158
|
+
jobs = min(args.parallel, len(samples))
|
|
159
|
+
memory = args.memory or auto_memory(sum(r.length for r in references), jobs=jobs)
|
|
160
|
+
results = count_samples(samples, args.ref, jobs=jobs, threads=args.threads, kmer_size=args.kmer_size,
|
|
161
|
+
mismatch=args.mismatch, memory=memory)
|
|
162
|
+
run = run_info(command, {'Reference': os.path.abspath(args.ref),
|
|
163
|
+
'Input': ' '.join(os.path.abspath(f) for f in (args.input, args.r1, args.r2) if f),
|
|
164
|
+
'Output folder': str(output.resolve()),
|
|
165
|
+
'K-mer size': args.kmer_size, 'Mismatches': args.mismatch,
|
|
166
|
+
'Threads': args.threads, 'Parallel samples': jobs, 'Seal memory': memory}, started)
|
|
167
|
+
rows, reports = write_reports(results, references, output, command=command, run=run)
|
|
168
|
+
except USER_ERRORS as e:
|
|
169
|
+
log.error('%s', e)
|
|
170
|
+
log.debug('Traceback:', exc_info=True) # Shown with --verbose
|
|
171
|
+
sys.exit(1)
|
|
172
|
+
except KeyboardInterrupt:
|
|
173
|
+
log.error('Interrupted')
|
|
174
|
+
sys.exit(130)
|
|
175
|
+
|
|
176
|
+
# The counts go to stdout, the messages to stderr: "seqcounter ... > counts.tsv" works
|
|
177
|
+
print_table(rows)
|
|
178
|
+
for report in reports:
|
|
179
|
+
log.info('Report saved in %s', report)
|
|
180
|
+
log.info('Done in %s', format_elapsed(time.monotonic() - start))
|
|
181
|
+
failed = [r.sample for r in results if r.error]
|
|
182
|
+
if failed:
|
|
183
|
+
log.error('%d of %d sample(s) failed: %s', len(failed), len(results), ', '.join(failed))
|
|
184
|
+
sys.exit(1)
|
seqcounter/counts.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Count the reference sequences in each sample, and write the reports."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import logging
|
|
5
|
+
import platform
|
|
6
|
+
import sys
|
|
7
|
+
from concurrent import futures
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .errors import SeqCounterError
|
|
14
|
+
from .seal import DEFAULT_KMER_SIZE, DEFAULT_MISMATCHES, run_seal, versions
|
|
15
|
+
|
|
16
|
+
log = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
COUNTS_FILE = 'refseqcounts.tsv' # The name the first version of seqcounter wrote, and the same table
|
|
19
|
+
SAMPLES_FILE = 'refseqcounts_samples.tsv'
|
|
20
|
+
JSON_FILE = 'refseqcounts.json'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class Result:
|
|
25
|
+
"""What one sample gave: a count per reference sequence, or an error."""
|
|
26
|
+
sample: str
|
|
27
|
+
file_type: str
|
|
28
|
+
files: list
|
|
29
|
+
counts: dict = field(default_factory=dict)
|
|
30
|
+
reads: int | None = None # Reads, or contigs for a fasta file
|
|
31
|
+
bases: int | None = None
|
|
32
|
+
matched: int | None = None # Reads matching at least one reference sequence
|
|
33
|
+
error: str | None = None
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def matched_percent(self):
|
|
37
|
+
if self.matched is None or not self.reads:
|
|
38
|
+
return None
|
|
39
|
+
return 100.0 * self.matched / self.reads
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def count_sample(sample, ref, kmer_size=DEFAULT_KMER_SIZE, mismatch=DEFAULT_MISMATCHES, threads=1, memory='1g'):
|
|
43
|
+
"""
|
|
44
|
+
Count the reads (or contigs) of one sample matching each reference sequence.
|
|
45
|
+
|
|
46
|
+
An error on one sample is kept in its Result rather than raised, so that a run over a folder reports the
|
|
47
|
+
samples that worked.
|
|
48
|
+
"""
|
|
49
|
+
log.info('Counting %s (%s, %d file(s))', sample.name, sample.file_type, len(sample.files))
|
|
50
|
+
try:
|
|
51
|
+
stats = run_seal(sample.files, ref, threads=threads, memory=memory, k=kmer_size, hdist=mismatch)
|
|
52
|
+
except SeqCounterError as e:
|
|
53
|
+
log.error('%s: %s', sample.name, e)
|
|
54
|
+
return Result(sample.name, sample.file_type, list(sample.files), error=str(e))
|
|
55
|
+
result = Result(sample.name, sample.file_type, list(sample.files), counts=stats.counts,
|
|
56
|
+
reads=stats.reads, bases=stats.bases, matched=stats.matched)
|
|
57
|
+
log.info('%s: %s of %s read(s) matched', sample.name,
|
|
58
|
+
'{:,}'.format(stats.matched) if stats.matched is not None else '?',
|
|
59
|
+
'{:,}'.format(stats.reads) if stats.reads is not None else '?')
|
|
60
|
+
return result
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def count_samples(samples, ref, jobs=1, threads=1, **options):
|
|
64
|
+
"""
|
|
65
|
+
Count the reference sequences in several samples, up to jobs samples at a time.
|
|
66
|
+
|
|
67
|
+
The threads are shared between the jobs, as Seal uses them within one sample.
|
|
68
|
+
|
|
69
|
+
:return: list of Result, in the order of samples
|
|
70
|
+
"""
|
|
71
|
+
jobs = max(1, min(jobs, len(samples)))
|
|
72
|
+
per_job = max(1, threads // jobs)
|
|
73
|
+
if jobs > 1:
|
|
74
|
+
log.info('%d sample(s), %d at a time, %d thread(s) each', len(samples), jobs, per_job)
|
|
75
|
+
with futures.ThreadPoolExecutor(max_workers=jobs) as executor:
|
|
76
|
+
return list(executor.map(lambda s: count_sample(s, ref, threads=per_job, **options), samples))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def counts_table(results, references):
|
|
80
|
+
"""
|
|
81
|
+
The table written to refseqcounts.tsv and printed: one row per reference sequence, one column per sample.
|
|
82
|
+
|
|
83
|
+
Rows are sorted by reference name, as the first version of seqcounter sorted them. A sample that failed has
|
|
84
|
+
empty cells, so that the samples that worked are still readable as numbers.
|
|
85
|
+
|
|
86
|
+
:return: list of rows, the first one being the header
|
|
87
|
+
"""
|
|
88
|
+
names = sorted({r.name for r in references} | {name for result in results for name in result.counts})
|
|
89
|
+
rows = [['Reference'] + [result.sample for result in results]]
|
|
90
|
+
for name in names:
|
|
91
|
+
row = [name]
|
|
92
|
+
for result in results:
|
|
93
|
+
row.append('' if result.error else str(result.counts.get(name, 0)))
|
|
94
|
+
rows.append(row)
|
|
95
|
+
return rows
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def write_counts(results, references, path, command=None):
|
|
99
|
+
"""Write the counts table, keeping the "#command line" first line of the first version of seqcounter."""
|
|
100
|
+
rows = counts_table(results, references)
|
|
101
|
+
with open(path, 'w') as f:
|
|
102
|
+
if command:
|
|
103
|
+
f.write('#{}\n'.format(command))
|
|
104
|
+
for row in rows:
|
|
105
|
+
f.write('\t'.join(row) + '\n')
|
|
106
|
+
return rows
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
SAMPLES_HEADER = ['Sample', 'Type', 'Reads', 'Bases', 'Matched', 'MatchedPct', 'Files', 'Error']
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def write_samples(results, path):
|
|
113
|
+
"""One row per sample: how much was read, how much matched, and the error if it failed."""
|
|
114
|
+
with open(path, 'w') as f:
|
|
115
|
+
f.write('\t'.join(SAMPLES_HEADER) + '\n')
|
|
116
|
+
for r in results:
|
|
117
|
+
percent = r.matched_percent
|
|
118
|
+
f.write('\t'.join([r.sample, r.file_type,
|
|
119
|
+
'' if r.reads is None else str(r.reads),
|
|
120
|
+
'' if r.bases is None else str(r.bases),
|
|
121
|
+
'' if r.matched is None else str(r.matched),
|
|
122
|
+
'' if percent is None else '{:.4f}'.format(percent),
|
|
123
|
+
';'.join(r.files),
|
|
124
|
+
(r.error or '').replace('\n', ' ')]) + '\n')
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def write_json(results, references, path, run=None):
|
|
128
|
+
"""Everything the run produced, for a pipeline: parameters, versions, references and counts."""
|
|
129
|
+
report = {
|
|
130
|
+
'tool': 'seqcounter',
|
|
131
|
+
'version': __version__,
|
|
132
|
+
'references': [{'name': r.name, 'description': r.description, 'length': r.length} for r in references],
|
|
133
|
+
'samples': [{'sample': r.sample, 'type': r.file_type, 'files': r.files, 'reads': r.reads, 'bases': r.bases,
|
|
134
|
+
'matched': r.matched, 'matched_percent': r.matched_percent, 'counts': r.counts,
|
|
135
|
+
'error': r.error} for r in results],
|
|
136
|
+
}
|
|
137
|
+
report.update(run or {})
|
|
138
|
+
with open(path, 'w') as f:
|
|
139
|
+
json.dump(report, f, indent=2, sort_keys=False)
|
|
140
|
+
f.write('\n')
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def run_info(command, parameters, started, finished=None):
|
|
144
|
+
"""The run metadata of the JSON report: what was run, with which versions and parameters."""
|
|
145
|
+
software = {'seqcounter': __version__, 'Python': platform.python_version()}
|
|
146
|
+
software.update(versions())
|
|
147
|
+
return {'command': command, 'started': started.isoformat(timespec='seconds'),
|
|
148
|
+
'finished': (finished or datetime.now().astimezone()).isoformat(timespec='seconds'),
|
|
149
|
+
'software': software, 'parameters': parameters}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def print_table(rows, stream=None):
|
|
153
|
+
"""Print the counts table, tab separated: "seqcounter ... > counts.tsv" works."""
|
|
154
|
+
stream = stream or sys.stdout
|
|
155
|
+
for row in rows:
|
|
156
|
+
print('\t'.join(row), file=stream)
|
|
157
|
+
stream.flush()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def write_reports(results, references, output, command=None, run=None):
|
|
161
|
+
"""Write the three reports in the output folder and return their paths, the counts table first."""
|
|
162
|
+
output = Path(output)
|
|
163
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
164
|
+
counts, samples, report = output / COUNTS_FILE, output / SAMPLES_FILE, output / JSON_FILE
|
|
165
|
+
rows = write_counts(results, references, counts, command=command)
|
|
166
|
+
write_samples(results, samples)
|
|
167
|
+
write_json(results, references, report, run=run)
|
|
168
|
+
return rows, [counts, samples, report]
|
seqcounter/errors.py
ADDED
seqcounter/references.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Read the reference sequences to count, and check that Seal can count them."""
|
|
2
|
+
|
|
3
|
+
import gzip
|
|
4
|
+
import logging
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .errors import SeqCounterError
|
|
10
|
+
|
|
11
|
+
log = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Reference:
|
|
16
|
+
name: str # The name Seal reports: the first word of the fasta header
|
|
17
|
+
description: str # The rest of the header, or ""
|
|
18
|
+
length: int
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def open_text(path):
|
|
22
|
+
"""Open a text file, gzipped or not."""
|
|
23
|
+
with open(path, 'rb') as f:
|
|
24
|
+
gzipped = f.read(2) == b'\x1f\x8b'
|
|
25
|
+
return gzip.open(path, 'rt') if gzipped else open(path)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def read_fasta(path):
|
|
29
|
+
"""[(header, sequence length)] of a fasta file, gzipped or not, in file order."""
|
|
30
|
+
records, header, length = [], None, 0
|
|
31
|
+
with open_text(path) as f:
|
|
32
|
+
for line in f:
|
|
33
|
+
line = line.strip()
|
|
34
|
+
if line.startswith('>'):
|
|
35
|
+
if header is not None:
|
|
36
|
+
records.append((header, length))
|
|
37
|
+
header, length = line[1:].strip(), 0
|
|
38
|
+
elif line and header is not None:
|
|
39
|
+
length += len(re.sub(r'\s', '', line))
|
|
40
|
+
if header is not None:
|
|
41
|
+
records.append((header, length))
|
|
42
|
+
return records
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def load_references(path, kmer_size):
|
|
46
|
+
"""
|
|
47
|
+
The reference sequences of a fasta file, checked against how Seal counts them.
|
|
48
|
+
|
|
49
|
+
Rejected rather than counted wrongly: an empty file, a sequence shorter than the k-mer size (Seal cannot
|
|
50
|
+
find it at all), and two sequences whose first word is the same (their counts would collide in one row).
|
|
51
|
+
|
|
52
|
+
:return: list of Reference, in file order
|
|
53
|
+
"""
|
|
54
|
+
if not Path(path).is_file():
|
|
55
|
+
raise SeqCounterError('Reference file not found: {}'.format(path))
|
|
56
|
+
try:
|
|
57
|
+
records = read_fasta(path)
|
|
58
|
+
except (OSError, EOFError, UnicodeDecodeError) as e:
|
|
59
|
+
raise SeqCounterError('Reference file {} cannot be read: {}'.format(path, e)) from None
|
|
60
|
+
if not records:
|
|
61
|
+
raise SeqCounterError('No sequence found in the reference file {}. It must be in fasta format, with a '
|
|
62
|
+
'">name" line before each sequence.'.format(path))
|
|
63
|
+
|
|
64
|
+
references, seen, duplicates, short = [], {}, [], []
|
|
65
|
+
for header, length in records:
|
|
66
|
+
name, _, description = header.partition(' ')
|
|
67
|
+
name = re.sub(r'[\t\r\n]', '_', name) or 'unnamed'
|
|
68
|
+
if length == 0:
|
|
69
|
+
raise SeqCounterError('The reference sequence "{}" of {} is empty.'.format(name, path))
|
|
70
|
+
if length < kmer_size:
|
|
71
|
+
short.append('"{}" ({} bp)'.format(name, length))
|
|
72
|
+
if name in seen:
|
|
73
|
+
duplicates.append('"{}"'.format(name))
|
|
74
|
+
seen[name] = True
|
|
75
|
+
references.append(Reference(name, description.strip(), length))
|
|
76
|
+
|
|
77
|
+
if duplicates:
|
|
78
|
+
raise SeqCounterError('Several reference sequences of {} have the same name: {}. Seal reports one row per '
|
|
79
|
+
'name, up to the first space, so their counts would be added together. Rename '
|
|
80
|
+
'them.'.format(path, ', '.join(sorted(set(duplicates)))))
|
|
81
|
+
if short:
|
|
82
|
+
raise SeqCounterError('{} reference sequence(s) of {} are shorter than the k-mer size ({} bp): {}. Seal '
|
|
83
|
+
'cannot find them: lower -k to at most the shortest reference, or remove '
|
|
84
|
+
'them.'.format(len(short), path, kmer_size, ', '.join(short)))
|
|
85
|
+
log.info('%d reference sequence(s) read from %s (%d-%d bp)', len(references), path,
|
|
86
|
+
min(r.length for r in references), max(r.length for r in references))
|
|
87
|
+
return references
|
seqcounter/samples.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Find the samples in an input folder: fasta files, and single-end or paired-end fastq files."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from .errors import SeqCounterError
|
|
10
|
+
|
|
11
|
+
log = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
FASTA_EXTENSIONS = ('.fasta', '.fa', '.fna', '.fas', '.fsa')
|
|
14
|
+
FASTQ_EXTENSIONS = ('.fastq', '.fq')
|
|
15
|
+
# Read suffixes of fastq file names, e.g. "S1_R1", "S1_S12_L001_R2_001", "SRR123_1". Group 1 is the mate number.
|
|
16
|
+
READ_SUFFIX = re.compile(r'_R?([12])(?:_001)?$')
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Sample:
|
|
21
|
+
name: str
|
|
22
|
+
file_type: str # "fasta" or "fastq"
|
|
23
|
+
files: list # One file, or R1 and R2
|
|
24
|
+
|
|
25
|
+
@property
|
|
26
|
+
def paired(self):
|
|
27
|
+
return len(self.files) == 2
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def split_extension(filename):
|
|
31
|
+
"""(stem, "fasta" or "fastq") for a sequence file, gzipped or not, or None for any other file."""
|
|
32
|
+
lower = filename.lower()
|
|
33
|
+
if lower.endswith('.gz'):
|
|
34
|
+
lower = lower[:-3]
|
|
35
|
+
for file_type, extensions in (('fasta', FASTA_EXTENSIONS), ('fastq', FASTQ_EXTENSIONS)):
|
|
36
|
+
for extension in extensions:
|
|
37
|
+
if lower.endswith(extension) and len(lower) > len(extension):
|
|
38
|
+
return filename[:len(lower) - len(extension)], file_type
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def sample_name(path, file_type=None):
|
|
43
|
+
"""
|
|
44
|
+
Sample name from a file name: the sequence extension is removed, and for fastq files the read suffix too.
|
|
45
|
+
"S1_R1.fastq.gz" -> "S1", "Iso_R10.fasta" -> "Iso_R10", "E.coli.v2.fasta" -> "E.coli.v2".
|
|
46
|
+
"""
|
|
47
|
+
name = Path(path).name
|
|
48
|
+
parsed = split_extension(name)
|
|
49
|
+
if parsed is not None:
|
|
50
|
+
name, file_type = parsed[0], file_type or parsed[1]
|
|
51
|
+
if file_type == 'fastq':
|
|
52
|
+
name = READ_SUFFIX.sub('', name) or name
|
|
53
|
+
return re.sub(r'[\t\r\n]', '_', name) # Tabs and line breaks would break the table
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def find_samples(folder, exclude=()):
|
|
57
|
+
"""
|
|
58
|
+
Look for fasta and fastq files in a folder and its subfolders, and group R1 and R2 fastq files per sample.
|
|
59
|
+
|
|
60
|
+
Name collisions are errors rather than guesses: two files giving the same sample name would otherwise be
|
|
61
|
+
merged or overwritten silently.
|
|
62
|
+
|
|
63
|
+
:param exclude: folders to skip, e.g. the output folder when it is inside the input folder
|
|
64
|
+
:return: list of Sample, sorted by name
|
|
65
|
+
"""
|
|
66
|
+
if not Path(folder).is_dir():
|
|
67
|
+
raise SeqCounterError('Input folder not found: {}'.format(folder))
|
|
68
|
+
exclude = {Path(p).resolve() for p in exclude}
|
|
69
|
+
found = {} # {sample name: [(file type, mate number or None, path)]}
|
|
70
|
+
visited = set() # Real paths of the folders searched: symbolic links to folders are followed, loops are not
|
|
71
|
+
for root, dirs, filenames in os.walk(folder, followlinks=True):
|
|
72
|
+
real = Path(root).resolve()
|
|
73
|
+
if real in visited:
|
|
74
|
+
dirs[:] = []
|
|
75
|
+
continue
|
|
76
|
+
visited.add(real)
|
|
77
|
+
dirs[:] = sorted(d for d in dirs if Path(root, d).resolve() not in exclude and not d.startswith('.'))
|
|
78
|
+
for filename in sorted(filenames):
|
|
79
|
+
parsed = split_extension(filename)
|
|
80
|
+
if parsed is None or filename.startswith('.'):
|
|
81
|
+
continue
|
|
82
|
+
stem, file_type = parsed
|
|
83
|
+
mate = None
|
|
84
|
+
if file_type == 'fastq':
|
|
85
|
+
match = READ_SUFFIX.search(stem)
|
|
86
|
+
if match and match.start() > 0:
|
|
87
|
+
mate, stem = int(match.group(1)), stem[:match.start()]
|
|
88
|
+
stem = re.sub(r'[\t\r\n]', '_', stem) # Tabs and line breaks would break the table
|
|
89
|
+
found.setdefault(stem, []).append((file_type, mate, os.path.join(root, filename)))
|
|
90
|
+
|
|
91
|
+
samples, errors = [], []
|
|
92
|
+
for name, files in sorted(found.items()):
|
|
93
|
+
paths = ', '.join(f[2] for f in files)
|
|
94
|
+
if len(files) == 1:
|
|
95
|
+
file_type, mate, path = files[0]
|
|
96
|
+
if mate == 2:
|
|
97
|
+
log.warning('%s: R2 file without R1, counted as single-end reads: %s', name, path)
|
|
98
|
+
samples.append(Sample(name, file_type, [path]))
|
|
99
|
+
elif len(files) == 2 and {(f[0], f[1]) for f in files} == {('fastq', 1), ('fastq', 2)}:
|
|
100
|
+
samples.append(Sample(name, 'fastq', [f[2] for f in sorted(files, key=lambda f: f[1])]))
|
|
101
|
+
else:
|
|
102
|
+
errors.append('"{}": {}'.format(name, paths))
|
|
103
|
+
if errors:
|
|
104
|
+
raise SeqCounterError('Several files give the same sample name. Rename, move or remove them:\n '
|
|
105
|
+
+ '\n '.join(errors))
|
|
106
|
+
if not samples:
|
|
107
|
+
raise SeqCounterError('No fasta or fastq file found in {}'.format(folder))
|
|
108
|
+
return samples
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def single_sample(r1, r2=None, name=None):
|
|
112
|
+
"""The Sample for one input given on the command line, with an optional sample name."""
|
|
113
|
+
for path in (r1, r2):
|
|
114
|
+
if path is not None and not Path(path).is_file():
|
|
115
|
+
raise SeqCounterError('Input file not found: {}'.format(path))
|
|
116
|
+
parsed = split_extension(Path(r1).name)
|
|
117
|
+
file_type = parsed[1] if parsed else 'fastq'
|
|
118
|
+
if r2 is not None and file_type == 'fasta':
|
|
119
|
+
raise SeqCounterError('-r2 is for paired-end fastq files; "{}" is a fasta file.'.format(r1))
|
|
120
|
+
files = [os.path.abspath(r1)] + ([os.path.abspath(r2)] if r2 else [])
|
|
121
|
+
if len(files) == 2 and files[0] == files[1]:
|
|
122
|
+
raise SeqCounterError('-r1 and -r2 are the same file: {}'.format(files[0]))
|
|
123
|
+
return Sample(name or sample_name(r1), file_type, files)
|
seqcounter/seal.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
"""Count the reads (or contigs) matching each reference sequence with Seal, from BBTools."""
|
|
2
|
+
|
|
3
|
+
import gzip
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
import shlex
|
|
8
|
+
import shutil
|
|
9
|
+
import subprocess
|
|
10
|
+
import sys
|
|
11
|
+
import tempfile
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from functools import cache
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from .errors import SeqCounterError
|
|
17
|
+
|
|
18
|
+
log = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
DEFAULT_KMER_SIZE = 25 # A reference sequence shorter than k cannot be found
|
|
21
|
+
DEFAULT_MISMATCHES = 1 # Maximum Hamming distance between a reference k-mer and the input
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SealError(SeqCounterError):
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@cache
|
|
29
|
+
def executable():
|
|
30
|
+
"""
|
|
31
|
+
Path of seal.sh: the one in the PATH, otherwise the one installed next to the running Python.
|
|
32
|
+
The fallback makes "/path/to/env/bin/seqcounter" work without activating the conda environment.
|
|
33
|
+
|
|
34
|
+
:return: path to seal.sh, or None if it cannot be found
|
|
35
|
+
"""
|
|
36
|
+
found = shutil.which('seal.sh')
|
|
37
|
+
if found:
|
|
38
|
+
return found
|
|
39
|
+
# Do not resolve symlinks: a virtualenv's python is a symlink to the base interpreter
|
|
40
|
+
candidate = Path(sys.executable).parent / 'seal.sh'
|
|
41
|
+
if candidate.is_file() and os.access(candidate, os.X_OK):
|
|
42
|
+
return str(candidate)
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def check_seal():
|
|
47
|
+
"""Make sure seal.sh is installed and return its path."""
|
|
48
|
+
path = executable()
|
|
49
|
+
if path is None:
|
|
50
|
+
raise SealError('"seal.sh" was not found in your PATH or in "{}". Install BBTools in the environment '
|
|
51
|
+
'with "conda install -c bioconda bbmap".'.format(Path(sys.executable).parent))
|
|
52
|
+
return path
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def seal_command(inputs, refs, stats_file, threads, memory, k=DEFAULT_KMER_SIZE, hdist=DEFAULT_MISMATCHES):
|
|
56
|
+
"""
|
|
57
|
+
:param inputs: one fasta/fastq file, or two paired-end fastq files
|
|
58
|
+
:param refs: reference fasta file(s): a path or a list of paths
|
|
59
|
+
:param k: k-mer size. A reference sequence shorter than k cannot be found
|
|
60
|
+
:param hdist: number of mismatches allowed in a k-mer
|
|
61
|
+
"""
|
|
62
|
+
refs = [refs] if isinstance(refs, (str, os.PathLike)) else list(refs)
|
|
63
|
+
cmd = [check_seal(), '-Xmx{}'.format(memory), 'in={}'.format(inputs[0])]
|
|
64
|
+
if len(inputs) > 1:
|
|
65
|
+
cmd.append('in2={}'.format(inputs[1]))
|
|
66
|
+
else:
|
|
67
|
+
cmd.append('int=f') # Never guess that a single fastq file is interleaved: that would halve the counts
|
|
68
|
+
cmd += ['ref={}'.format(','.join(str(r) for r in refs)),
|
|
69
|
+
'k={}'.format(k),
|
|
70
|
+
'rcomp=t', # Count a match on either strand
|
|
71
|
+
'hdist={}'.format(hdist),
|
|
72
|
+
'maskmiddle=f', # Do not treat the middle base of a k-mer as a wildcard
|
|
73
|
+
'clearzone=999999',
|
|
74
|
+
'ambiguous=all', # Count a read for every reference sequence it matches
|
|
75
|
+
'nzo=f', # Also report reference sequences with no match
|
|
76
|
+
'qin=33', # Force the quality encoding: autodetection fails on some low quality nanopore reads
|
|
77
|
+
'ow=t',
|
|
78
|
+
'stats={}'.format(stats_file),
|
|
79
|
+
'threads={}'.format(threads)]
|
|
80
|
+
return cmd
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# Paths that seal.sh cannot pass to Seal safely: it runs its command line with eval, so any shell character (spaces,
|
|
84
|
+
# quotes, $, `, ;, &, |, parentheses...) is re-interpreted, it splits ref= lists on commas, and BBTools 40 takes any
|
|
85
|
+
# argument containing "xmx" or "xms" (in any case, anywhere) for a Java memory setting. Only plain characters are
|
|
86
|
+
# passed as they are: the other paths are linked under a neutral name.
|
|
87
|
+
UNSAFE = re.compile(r'[^A-Za-z0-9._/+:@-]|xm[xs]', re.IGNORECASE)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def extensions(name):
|
|
91
|
+
"""The extensions Seal uses to recognize the format, e.g. ".fastq.gz" for "S1_R1.fastq.gz"."""
|
|
92
|
+
suffixes = [s for s in Path(name).suffixes[-2:] if re.fullmatch(r'\.[A-Za-z0-9]+', s) and not UNSAFE.search(s)]
|
|
93
|
+
return ''.join(suffixes)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
SEQUENCE_EXTENSION = re.compile(r'\.(fastq|fq|fasta|fa|fna|fas|fsa)(\.gz)?$', re.IGNORECASE)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def content_extension(path):
|
|
100
|
+
"""
|
|
101
|
+
Extension matching the content of a sequence file, e.g. ".fastq.gz". Seal recognizes formats and compression by
|
|
102
|
+
extension only, and Galaxy, for example, names its files "dataset_1.dat".
|
|
103
|
+
"""
|
|
104
|
+
with open(path, 'rb') as f:
|
|
105
|
+
gzipped = f.read(2) == b'\x1f\x8b'
|
|
106
|
+
try:
|
|
107
|
+
with (gzip.open(path, 'rt') if gzipped else open(path)) as f:
|
|
108
|
+
first = next((line for line in f if line.strip()), '>')[0]
|
|
109
|
+
except (OSError, EOFError, UnicodeDecodeError):
|
|
110
|
+
first = '>' # Unreadable: let Seal report the error
|
|
111
|
+
return ('.fastq' if first == '@' else '.fasta') + ('.gz' if gzipped else '')
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def safe_paths(paths, folder, prefix, check_extension=False):
|
|
115
|
+
"""
|
|
116
|
+
Paths that Seal can read: unsafe ones, and with check_extension those whose extension does not match their
|
|
117
|
+
content, are linked into folder under a neutral name (e.g. "in0.fastq.gz").
|
|
118
|
+
"""
|
|
119
|
+
safe = []
|
|
120
|
+
for i, path in enumerate(paths):
|
|
121
|
+
path = str(path)
|
|
122
|
+
extension = extensions(path)
|
|
123
|
+
if check_extension:
|
|
124
|
+
detected = content_extension(path)
|
|
125
|
+
match = SEQUENCE_EXTENSION.search(Path(path).name)
|
|
126
|
+
fastq = match and match.group(1).lower() in ('fastq', 'fq')
|
|
127
|
+
if not match or fastq != detected.startswith('.fastq') or bool(match.group(2)) != detected.endswith('.gz'):
|
|
128
|
+
extension = detected
|
|
129
|
+
if UNSAFE.search(path) or extension != extensions(path):
|
|
130
|
+
link = Path(folder) / '{}{}{}'.format(prefix, i, extension)
|
|
131
|
+
link.symlink_to(Path(path).resolve())
|
|
132
|
+
path = str(link)
|
|
133
|
+
safe.append(path)
|
|
134
|
+
return safe
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def safe_temporary_folder(attempts=20):
|
|
138
|
+
"""A new temporary folder whose path Seal can read: random names can contain "xmx" or "xms", so retry."""
|
|
139
|
+
for _ in range(attempts):
|
|
140
|
+
folder = tempfile.mkdtemp(prefix='seqcounter_')
|
|
141
|
+
if not UNSAFE.search(folder):
|
|
142
|
+
return folder
|
|
143
|
+
os.rmdir(folder)
|
|
144
|
+
raise SealError('The temporary folder "{}" contains a space, a comma, "xmx" or "xms", which Seal cannot handle. '
|
|
145
|
+
'Set TMPDIR to another folder.'.format(tempfile.gettempdir()))
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
GENERIC_ERRORS = re.compile(r'Exception in thread "main"|terminated in an error state')
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def failure_reason(lines):
|
|
152
|
+
"""
|
|
153
|
+
The most informative line of Seal's error output: the first error that is not Seal's generic final message.
|
|
154
|
+
Java exceptions often end with ":" and give their message on the next line, which is then added.
|
|
155
|
+
"""
|
|
156
|
+
candidates = [i for i, line in enumerate(lines) if re.search(r'Exception|Error|error', line)
|
|
157
|
+
and not line.strip().startswith('at ')]
|
|
158
|
+
specific = [i for i in candidates if not GENERIC_ERRORS.search(lines[i])]
|
|
159
|
+
if not candidates:
|
|
160
|
+
return lines[-1].strip()
|
|
161
|
+
i = (specific or candidates)[0]
|
|
162
|
+
reason = lines[i].strip()
|
|
163
|
+
if reason.endswith(':'):
|
|
164
|
+
following = [line.strip() for line in lines[i + 1:] if line.strip() and not line.strip().startswith('at ')]
|
|
165
|
+
if following:
|
|
166
|
+
reason += ' ' + following[0]
|
|
167
|
+
return reason
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@dataclass
|
|
171
|
+
class SealStats:
|
|
172
|
+
counts: dict = field(default_factory=dict) # {reference name: number of matching reads}
|
|
173
|
+
reads: int | None = None # Total number of reads (or contigs) in the input
|
|
174
|
+
bases: int | None = None
|
|
175
|
+
matched: int | None = None # Reads matching at least one reference sequence
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def parse_stats(stats_file):
|
|
179
|
+
"""
|
|
180
|
+
Read Seal's stats file.
|
|
181
|
+
|
|
182
|
+
#File sample_R1.fastq.gz
|
|
183
|
+
#Total 822714 123407100
|
|
184
|
+
#Matched 799 0.09712%
|
|
185
|
+
#Name Reads ReadsPct
|
|
186
|
+
spacer25 62 0.00754%
|
|
187
|
+
"""
|
|
188
|
+
stats = SealStats()
|
|
189
|
+
with open(stats_file) as f:
|
|
190
|
+
for line in f:
|
|
191
|
+
fields = line.rstrip('\n').split('\t')
|
|
192
|
+
if not line.strip():
|
|
193
|
+
continue
|
|
194
|
+
try:
|
|
195
|
+
if fields[0] == '#Total':
|
|
196
|
+
stats.reads = int(fields[1])
|
|
197
|
+
stats.bases = int(fields[2]) if len(fields) > 2 else None
|
|
198
|
+
elif fields[0] == '#Matched':
|
|
199
|
+
stats.matched = int(fields[1])
|
|
200
|
+
elif not line.startswith('#'):
|
|
201
|
+
stats.counts[fields[0].split()[0]] = int(fields[1]) # Name without the description
|
|
202
|
+
except (IndexError, ValueError):
|
|
203
|
+
raise SealError('Unexpected line in Seal stats file {}: {}'.format(stats_file, line.strip())) from None
|
|
204
|
+
return stats
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def run_seal(inputs, refs, threads=1, memory='1g', k=DEFAULT_KMER_SIZE, hdist=DEFAULT_MISMATCHES):
|
|
208
|
+
"""
|
|
209
|
+
Count the reads (or contigs) that contain each reference sequence.
|
|
210
|
+
|
|
211
|
+
:param inputs: one fasta/fastq file, or two paired-end fastq files
|
|
212
|
+
:param refs: reference fasta file(s)
|
|
213
|
+
:param memory: Java heap size given to Seal, e.g. "1g"
|
|
214
|
+
:return: SealStats
|
|
215
|
+
"""
|
|
216
|
+
refs = [refs] if isinstance(refs, (str, os.PathLike)) else list(refs)
|
|
217
|
+
tmp = safe_temporary_folder()
|
|
218
|
+
try:
|
|
219
|
+
stats_file = Path(tmp) / 'stats.tsv'
|
|
220
|
+
cmd = seal_command(safe_paths(inputs, tmp, 'in', check_extension=True), safe_paths(refs, tmp, 'ref'),
|
|
221
|
+
stats_file, threads, memory, k=k, hdist=hdist)
|
|
222
|
+
log.debug('Running: %s', shlex.join(cmd))
|
|
223
|
+
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
224
|
+
if proc.returncode != 0 or not stats_file.exists():
|
|
225
|
+
detail = (proc.stderr or proc.stdout).strip().splitlines() or ['no output']
|
|
226
|
+
raise SealError('Seal failed (exit code {}): {}\nCommand: {}\n{}'.format(
|
|
227
|
+
proc.returncode, failure_reason(detail), shlex.join(cmd), '\n'.join(detail[-10:])))
|
|
228
|
+
log.debug('Seal output:\n%s', proc.stderr.strip())
|
|
229
|
+
return parse_stats(stats_file)
|
|
230
|
+
finally:
|
|
231
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def versions():
|
|
235
|
+
"""{"BBTools": version, "Java": version} of the Seal installation, "unknown" when they cannot be read."""
|
|
236
|
+
found = {'BBTools': 'unknown', 'Java': 'unknown'}
|
|
237
|
+
path = executable()
|
|
238
|
+
if path is None:
|
|
239
|
+
return found
|
|
240
|
+
try: # A small heap: without -Xmx, seal.sh reserves most of the free memory, even for --version
|
|
241
|
+
out = subprocess.run([path, '-Xmx64m', '--version'], capture_output=True, text=True, timeout=60)
|
|
242
|
+
match = re.search(r'(?:BBMap|BBTools) version (\S+)', out.stdout + out.stderr)
|
|
243
|
+
if match:
|
|
244
|
+
found['BBTools'] = match.group(1)
|
|
245
|
+
java = Path(path).parent / 'java'
|
|
246
|
+
java = str(java) if java.exists() else shutil.which('java')
|
|
247
|
+
if java:
|
|
248
|
+
out = subprocess.run([java, '-version'], capture_output=True, text=True, timeout=60)
|
|
249
|
+
first = (out.stderr or out.stdout).strip().splitlines()
|
|
250
|
+
if first:
|
|
251
|
+
found['Java'] = first[0]
|
|
252
|
+
except (OSError, subprocess.SubprocessError):
|
|
253
|
+
pass
|
|
254
|
+
return found
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def system_memory_mb():
|
|
258
|
+
"""Total memory of the machine in MB, or None when it cannot be read."""
|
|
259
|
+
try:
|
|
260
|
+
return os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGE_SIZE') // (1024 * 1024)
|
|
261
|
+
except (AttributeError, ValueError, OSError):
|
|
262
|
+
return None
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def auto_memory(ref_bases, jobs=1):
|
|
266
|
+
"""
|
|
267
|
+
Java heap size for Seal, from the size of the reference sequences: Seal holds every reference k-mer in memory,
|
|
268
|
+
at a few tens of bytes each. Never more than 85% of the machine's memory, shared between the jobs: without
|
|
269
|
+
-Xmx, seal.sh would reserve most of the free memory for each job at once.
|
|
270
|
+
|
|
271
|
+
:return: a size for -Xmx, e.g. "1024m"
|
|
272
|
+
"""
|
|
273
|
+
needed = 512 + ref_bases * 40 / 1e6 # MB
|
|
274
|
+
megabytes = max(1024, -(-int(needed) // 256) * 256) # Rounded up to 256 MB
|
|
275
|
+
total = system_memory_mb()
|
|
276
|
+
if total:
|
|
277
|
+
allowed = max(512, int(total * 0.85) // max(1, jobs))
|
|
278
|
+
if megabytes > allowed:
|
|
279
|
+
log.warning('The reference sequences need about %d MB of memory for Seal, but only %d MB is available '
|
|
280
|
+
'(85%% of the machine, shared between %d job(s)). Seal may run out of memory: use fewer '
|
|
281
|
+
'jobs, or --memory to set the size yourself.', megabytes, allowed, jobs)
|
|
282
|
+
megabytes = allowed
|
|
283
|
+
return '{}m'.format(megabytes)
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: seqcounter
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Count the occurrences of reference sequences in sequencing reads or genome assemblies
|
|
5
|
+
Author-email: Marc-Olivier Duceppe <marc-olivier.duceppe@inspection.gc.ca>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/duceppemo/seqcounter
|
|
8
|
+
Project-URL: Documentation, https://github.com/duceppemo/seqcounter/wiki
|
|
9
|
+
Project-URL: Changelog, https://github.com/duceppemo/seqcounter/wiki/Changelog
|
|
10
|
+
Project-URL: Issues, https://github.com/duceppemo/seqcounter/issues
|
|
11
|
+
Keywords: bioinformatics,kmer,counting,reference sequences,fastq,fasta,wgs,BBTools,Seal
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Provides-Extra: test
|
|
28
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
29
|
+
Requires-Dist: pytest-cov>=5; extra == "test"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
<p align="center">
|
|
33
|
+
<picture>
|
|
34
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/duceppemo/seqcounter/master/assets/logo_dark.svg">
|
|
35
|
+
<img src="https://raw.githubusercontent.com/duceppemo/seqcounter/master/assets/logo.svg" width="520" alt="seqcounter: count reference sequences in reads or assemblies">
|
|
36
|
+
</picture>
|
|
37
|
+
</p>
|
|
38
|
+
|
|
39
|
+
<p align="center">
|
|
40
|
+
<a href="https://github.com/duceppemo/seqcounter/actions/workflows/tests.yml"><img src="https://github.com/duceppemo/seqcounter/actions/workflows/tests.yml/badge.svg" alt="Tests"></a>
|
|
41
|
+
<a href="https://codecov.io/gh/duceppemo/seqcounter"><img src="https://codecov.io/gh/duceppemo/seqcounter/graph/badge.svg" alt="Coverage"></a>
|
|
42
|
+
<a href="https://github.com/duceppemo/seqcounter/releases/latest"><img src="https://img.shields.io/github/v/release/duceppemo/seqcounter?cacheSeconds=3600" alt="Release"></a>
|
|
43
|
+
<a href="https://pypi.org/project/seqcounter/"><img src="https://img.shields.io/pypi/v/seqcounter?cacheSeconds=3600" alt="PyPI"></a>
|
|
44
|
+
<img src="https://img.shields.io/badge/python-3.10%E2%80%933.14-blue" alt="Python 3.10–3.14">
|
|
45
|
+
<a href="LICENSE"><img src="https://img.shields.io/github/license/duceppemo/seqcounter" alt="License"></a>
|
|
46
|
+
<a href="https://github.com/duceppemo/seqcounter/wiki"><img src="https://img.shields.io/badge/docs-wiki-informational" alt="Documentation"></a>
|
|
47
|
+
</p>
|
|
48
|
+
|
|
49
|
+
Count how many sequencing reads (fastq) or contigs (fasta) contain each sequence of a reference file, with
|
|
50
|
+
[Seal](https://sourceforge.net/projects/bbmap/) from BBTools. Point it at a reference fasta file and a sample, or a
|
|
51
|
+
folder of samples, and it writes one table of counts per reference sequence and per sample.
|
|
52
|
+
|
|
53
|
+
Any set of short sequences works: spoligotype spacers, marker or resistance genes, primer or probe sequences,
|
|
54
|
+
plasmid or phage fragments, barcodes, spike-ins. seqcounter does not interpret them; it counts them, reproducibly
|
|
55
|
+
and in the same way for reads and for assemblies.
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
$ seqcounter -ref refs.fasta -i samples/ -o results/
|
|
59
|
+
Reference assembly no_match paired single_end
|
|
60
|
+
absent 0 0 0 0
|
|
61
|
+
present 1 0 86 34
|
|
62
|
+
twice 1 0 154 65
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Features
|
|
66
|
+
* **Reads or assemblies**: single-end or paired-end fastq, or fasta, gzipped or not. Both mates of a pair are
|
|
67
|
+
counted, once each.
|
|
68
|
+
* **Batch mode**: point it at a folder; fastq and fasta files are found recursively and R1/R2 files paired
|
|
69
|
+
automatically. Several samples can be counted at the same time.
|
|
70
|
+
* **Both strands, and mismatches**: a reference is found on either strand, and with up to `-m` mismatches per k-mer
|
|
71
|
+
(one by default).
|
|
72
|
+
* **Three reports**: the counts table (`refseqcounts.tsv`, the table the first version of seqcounter wrote, with the
|
|
73
|
+
command that produced it on its first line), a per-sample summary of how much was read and how much matched, and a
|
|
74
|
+
JSON report with the parameters, the software versions and the counts.
|
|
75
|
+
* **Nothing is hidden**: references Seal cannot find (shorter than the k-mer size), names that would collide, and
|
|
76
|
+
samples that failed are reported, not silently dropped. A failed sample leaves the others readable.
|
|
77
|
+
* **Nothing to configure**: no database, no index, no reference genome. Only the Python standard library, plus
|
|
78
|
+
BBTools.
|
|
79
|
+
* **Tested**: 105 tests, 100% coverage, end to end on synthetic references, genomes and reads with known counts.
|
|
80
|
+
|
|
81
|
+
## Installation
|
|
82
|
+
```
|
|
83
|
+
conda create -n seqcounter -c conda-forge -c bioconda bbmap python">=3.10"
|
|
84
|
+
conda activate seqcounter
|
|
85
|
+
pip install seqcounter
|
|
86
|
+
```
|
|
87
|
+
BBTools (which provides `seal.sh`) is not on PyPI, so it comes from bioconda; seqcounter itself needs nothing else.
|
|
88
|
+
See [Installation](https://github.com/duceppemo/seqcounter/wiki/Installation) for other options, including running
|
|
89
|
+
it from a clone with `python seqcounter.py`, as the first version did.
|
|
90
|
+
|
|
91
|
+
## Quick start
|
|
92
|
+
```
|
|
93
|
+
seqcounter -ref refs.fasta -r1 sample_R1.fastq.gz -r2 sample_R2.fastq.gz -o results/ # Paired-end reads
|
|
94
|
+
seqcounter -ref refs.fasta -r1 sample.fastq.gz -o results/ # Single-end reads
|
|
95
|
+
seqcounter -ref refs.fasta -r1 assembly.fasta -o results/ # An assembly
|
|
96
|
+
seqcounter -ref refs.fasta -i samples/ -o results/ -p 4 # A folder, 4 at a time
|
|
97
|
+
```
|
|
98
|
+
`-ref` is a fasta file of the sequences to count: each one gives a row, named after the first word of its `>` line.
|
|
99
|
+
The counts are printed and saved in `results/`. To see it work on data with known counts, run
|
|
100
|
+
`bash examples/run_example.sh`, which generates its own references and samples.
|
|
101
|
+
|
|
102
|
+
**What a count is**: the number of reads, or contigs, that contain the reference sequence, not the number of copies
|
|
103
|
+
it has in them. A reference present twice in a single-contig assembly is counted once; in reads, about twice as many
|
|
104
|
+
reads match it. See [How it works](https://github.com/duceppemo/seqcounter/wiki/How-it-works).
|
|
105
|
+
|
|
106
|
+
## Documentation
|
|
107
|
+
The **[wiki](https://github.com/duceppemo/seqcounter/wiki)** covers
|
|
108
|
+
[installation](https://github.com/duceppemo/seqcounter/wiki/Installation),
|
|
109
|
+
[usage and options](https://github.com/duceppemo/seqcounter/wiki/Usage),
|
|
110
|
+
[output files](https://github.com/duceppemo/seqcounter/wiki/Output-files),
|
|
111
|
+
[how it works](https://github.com/duceppemo/seqcounter/wiki/How-it-works),
|
|
112
|
+
[troubleshooting](https://github.com/duceppemo/seqcounter/wiki/Troubleshooting) and the
|
|
113
|
+
[FAQ](https://github.com/duceppemo/seqcounter/wiki/FAQ).
|
|
114
|
+
|
|
115
|
+
## Citing
|
|
116
|
+
If you use seqcounter, please cite it and BBTools, which does the counting:
|
|
117
|
+
|
|
118
|
+
> Duceppe M-O. seqcounter: count the occurrences of reference sequences in sequencing reads or genome assemblies.
|
|
119
|
+
> https://github.com/duceppemo/seqcounter
|
|
120
|
+
|
|
121
|
+
> Bushnell B. BBTools. https://sourceforge.net/projects/bbmap/
|
|
122
|
+
|
|
123
|
+
GitHub's **"Cite this repository"** button gives the citation in APA and BibTeX formats, from
|
|
124
|
+
[`CITATION.cff`](CITATION.cff).
|
|
125
|
+
|
|
126
|
+
## Related
|
|
127
|
+
* [spoligotyper](https://github.com/duceppemo/spoligotyper) — *in silico* spoligotyping of the *M. tuberculosis*
|
|
128
|
+
complex. It counts the 43 spacers the way seqcounter counts any reference, and then reads them as a spoligotype,
|
|
129
|
+
with the species, the lineage and a PDF report.
|
|
130
|
+
* [brucetyper](https://github.com/duceppemo/brucetyper) — *in silico* Bruce-ladder, Suis-ladder, AMOS and RB51
|
|
131
|
+
junction PCR for *Brucella*, from assemblies.
|
|
132
|
+
|
|
133
|
+
## Contributing
|
|
134
|
+
Bug reports, questions and pull requests are welcome: see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
135
|
+
|
|
136
|
+
## Author
|
|
137
|
+
Marc-Olivier Duceppe, Canadian Food Inspection Agency (CFIA): marc-olivier.duceppe@inspection.gc.ca
|
|
138
|
+
|
|
139
|
+
## License
|
|
140
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
seqcounter/__init__.py,sha256=KYpNYEbG86FKu52mmSQ5vWqjinL6kh1RR_bY_OPSLjM,112
|
|
2
|
+
seqcounter/__main__.py,sha256=hAZmm9T7RoDlCXwO5VSXl3ySu-_GssGBfw39BLS9eGM,123
|
|
3
|
+
seqcounter/cli.py,sha256=FMgq-BQPWajmHlNK9AK8-4E-3e1o5anxGmjdUgToIoM,9434
|
|
4
|
+
seqcounter/counts.py,sha256=ISCaj18FEZly0egZH41_u7FuBlZ50If78tc4PfgjQBA,7081
|
|
5
|
+
seqcounter/errors.py,sha256=lPP1G7DyN2JYp3VxeqU1okTn3cgI4SJofAgFxjyLW3Y,129
|
|
6
|
+
seqcounter/references.py,sha256=ob9qjrFDTqDeEib1h2r-c1JXsi7hbTK6Y69ydGNhKP4,3607
|
|
7
|
+
seqcounter/samples.py,sha256=eAQRmGYhxHVwrlpHt4VJntKfJVSz16yBcvDpbN8g_wI,5314
|
|
8
|
+
seqcounter/seal.py,sha256=LFsblV0ghPBDqivXWgoWeLXoaRsSzUAluhqJU_25HSY,12021
|
|
9
|
+
seqcounter-1.0.0.dist-info/licenses/LICENSE,sha256=xKXtFTyegSo7EWwkmNW4r0_tuKmmeO_9t5Ogbh2okYY,1077
|
|
10
|
+
seqcounter-1.0.0.dist-info/METADATA,sha256=KWENv-_jC1C59BL8kPXIljO9ppNF2cDYn_TUd0MZc7E,8038
|
|
11
|
+
seqcounter-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
12
|
+
seqcounter-1.0.0.dist-info/entry_points.txt,sha256=6hN0y7taCJNOWtWJP-BP0Ic4wC1EQbcoI5soTVK3fWk,51
|
|
13
|
+
seqcounter-1.0.0.dist-info/top_level.txt,sha256=Uarq7-22b5aTo0JsmBA7A84AyW71MSEp7JxNtzld7Cc,11
|
|
14
|
+
seqcounter-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Marc-Olivier Duceppe
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
seqcounter
|