cutseq 0.0.3__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: cutseq
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: Automatically cut adapter / barcode / UMI from NGS data
|
|
5
5
|
Home-page: https://github.com/y9c/cutseq
|
|
6
6
|
License: MIT
|
|
@@ -19,9 +19,20 @@ Requires-Dist: cutadapt (>=4.8,<5.0)
|
|
|
19
19
|
Project-URL: Repository, https://github.com/y9c/cutseq
|
|
20
20
|
Description-Content-Type: text/markdown
|
|
21
21
|
|
|
22
|
-
|
|
22
|
+
# ✂️ CutSeq
|
|
23
|
+
|
|
24
|
+
[](https://pypi.python.org/pypi/cutseq)
|
|
25
|
+
[](https://pepy.tech/project/cutseq)
|
|
26
|
+
|
|
27
|
+
# How to install?
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install cutseq
|
|
31
|
+
```
|
|
23
32
|
|
|
24
33
|
## How to use?
|
|
25
34
|
|
|
26
|
-
|
|
35
|
+
```bash
|
|
36
|
+
cutseq -a "ACACGACGCTCTTCCGATCTX<XXXAGATCGGAAGAGCACACGTC" test_R1.fq.gz test_R2.fq.gz
|
|
37
|
+
```
|
|
27
38
|
|
cutseq-0.0.4/README.md
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# ✂️ CutSeq
|
|
2
|
+
|
|
3
|
+
[](https://pypi.python.org/pypi/cutseq)
|
|
4
|
+
[](https://pepy.tech/project/cutseq)
|
|
5
|
+
|
|
6
|
+
# How to install?
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install cutseq
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## How to use?
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
cutseq -a "ACACGACGCTCTTCCGATCTX<XXXAGATCGGAAGAGCACACGTC" test_R1.fq.gz test_R2.fq.gz
|
|
16
|
+
```
|
|
@@ -0,0 +1,599 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
#
|
|
4
|
+
# Copyright © 2024 Ye Chang yech1990@gmail.com
|
|
5
|
+
# Distributed under terms of the GNU license.
|
|
6
|
+
#
|
|
7
|
+
# Created: 2024-04-19 18:57
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import logging
|
|
11
|
+
import re
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
from cutadapt.adapters import (
|
|
16
|
+
BackAdapter,
|
|
17
|
+
NonInternalBackAdapter,
|
|
18
|
+
NonInternalFrontAdapter,
|
|
19
|
+
PrefixAdapter,
|
|
20
|
+
RightmostFrontAdapter,
|
|
21
|
+
)
|
|
22
|
+
from cutadapt.files import InputPaths, OutputFiles
|
|
23
|
+
from cutadapt.modifiers import (
|
|
24
|
+
AdapterCutter,
|
|
25
|
+
PairedEndRenamer,
|
|
26
|
+
QualityTrimmer,
|
|
27
|
+
Renamer,
|
|
28
|
+
SuffixRemover,
|
|
29
|
+
UnconditionalCutter,
|
|
30
|
+
)
|
|
31
|
+
from cutadapt.pipeline import PairedEndPipeline, SingleEndPipeline
|
|
32
|
+
from cutadapt.predicates import IsUntrimmed, TooShort
|
|
33
|
+
from cutadapt.runners import make_runner
|
|
34
|
+
from cutadapt.steps import (
|
|
35
|
+
InfoFileWriter,
|
|
36
|
+
PairedEndFilter,
|
|
37
|
+
PairedEndSink,
|
|
38
|
+
PairedSingleEndStep,
|
|
39
|
+
SingleEndFilter,
|
|
40
|
+
SingleEndSink,
|
|
41
|
+
)
|
|
42
|
+
from cutadapt.utils import Progress
|
|
43
|
+
|
|
44
|
+
logging.basicConfig(
|
|
45
|
+
level=logging.INFO,
|
|
46
|
+
format="%(asctime)s - %(levelname)s - %(message)s",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def reverse_complement(b):
|
|
51
|
+
return "".join(
|
|
52
|
+
[dict(zip("ATGCNatgcn", "TACGNtacgn"))[x] for x in b[::-1] if x in "ATGCNatgcn"]
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def remove_fq_suffix(f):
|
|
57
|
+
suffixes_base = ["_R1_001", "_R2_001", "_R1", "_R2", ""]
|
|
58
|
+
suffixes = [
|
|
59
|
+
y + "." + x for x in ["fastq.gz", "fq.gz", "fastq", "fq"] for y in suffixes_base
|
|
60
|
+
]
|
|
61
|
+
print(suffixes)
|
|
62
|
+
for suffix in suffixes:
|
|
63
|
+
if f.endswith(suffix):
|
|
64
|
+
return f.removesuffix(suffix)
|
|
65
|
+
return f
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class BarcodeSeq:
|
|
69
|
+
def __init__(self, seq):
|
|
70
|
+
self.fw = seq
|
|
71
|
+
self.rc = reverse_complement(seq)
|
|
72
|
+
self.len = len(seq)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class BarcodeConfig:
|
|
76
|
+
"""
|
|
77
|
+
Adapter scheme:
|
|
78
|
+
(p5)(inline5)(umi5)(mask5)(strand)(mask3)(umi3)(inline3)(p7)
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
def __init__(self, adapter=None):
|
|
82
|
+
self.strand = None
|
|
83
|
+
self.p5 = BarcodeSeq("")
|
|
84
|
+
self.p7 = BarcodeSeq("")
|
|
85
|
+
self.inline5 = BarcodeSeq("")
|
|
86
|
+
self.inline3 = BarcodeSeq("")
|
|
87
|
+
self.umi5 = BarcodeSeq("")
|
|
88
|
+
self.umi3 = BarcodeSeq("")
|
|
89
|
+
self.mask5 = BarcodeSeq("")
|
|
90
|
+
self.mask3 = BarcodeSeq("")
|
|
91
|
+
if adapter is not None:
|
|
92
|
+
self._parse_barcode(adapter)
|
|
93
|
+
|
|
94
|
+
def _parse_barcode(self, b):
|
|
95
|
+
m = re.match(
|
|
96
|
+
r"(?P<p5>[ATGCatgc]+)(\((?P<inline5>[ATGCatgc]+)\))?(?P<umi5>N*)(?P<mask5>X*)(?P<strand>-|>|<)(?P<mask3>X*)(?P<umi3>N*)(\((?P<inline3>[ATGCatgc]+)\))?(?P<p7>[ATGCatgc]+)",
|
|
97
|
+
b,
|
|
98
|
+
)
|
|
99
|
+
if m is None:
|
|
100
|
+
logging.error(f"barcode {b} is not valid")
|
|
101
|
+
sys.exit(1)
|
|
102
|
+
d = m.groupdict()
|
|
103
|
+
if d["inline5"] is None:
|
|
104
|
+
d["inline5"] = ""
|
|
105
|
+
if d["inline3"] is None:
|
|
106
|
+
d["inline3"] = ""
|
|
107
|
+
self.strand = "+" if d["strand"] == ">" else "-" if d["strand"] == "<" else None
|
|
108
|
+
self.p5 = BarcodeSeq(d["p5"])
|
|
109
|
+
self.p7 = BarcodeSeq(d["p7"])
|
|
110
|
+
self.inline5 = BarcodeSeq(d["inline5"])
|
|
111
|
+
self.inline3 = BarcodeSeq(d["inline3"])
|
|
112
|
+
self.umi5 = BarcodeSeq(d["umi5"])
|
|
113
|
+
self.umi3 = BarcodeSeq(d["umi3"])
|
|
114
|
+
self.mask5 = BarcodeSeq(d["mask5"])
|
|
115
|
+
self.mask3 = BarcodeSeq(d["mask3"])
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class CutadaptConfig:
|
|
119
|
+
def __init__(self):
|
|
120
|
+
self.rname_suffix = False
|
|
121
|
+
self.discarded_untrimmed = False
|
|
122
|
+
self.trim_polyA = False
|
|
123
|
+
self.min_length = 20
|
|
124
|
+
self.min_quality = 20
|
|
125
|
+
self.dry_run = False
|
|
126
|
+
self.threads = 1
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def run_steps(steps, dry_run=False):
|
|
130
|
+
if dry_run:
|
|
131
|
+
print(
|
|
132
|
+
" |\\\n".join([(" " + s if i > 0 else s) for i, s in enumerate(steps)])
|
|
133
|
+
)
|
|
134
|
+
process = subprocess.run("true", shell=True, capture_output=True)
|
|
135
|
+
else:
|
|
136
|
+
cmd = " | ".join(steps)
|
|
137
|
+
process = subprocess.run(cmd, shell=True, capture_output=True)
|
|
138
|
+
return process.stdout.decode(), process.stderr.decode()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def pipeline_single(input1, output1, short1, untrimed1, barcode, settings):
|
|
142
|
+
modifiers = []
|
|
143
|
+
# step 1: remove suffix in the read name
|
|
144
|
+
modifiers.extend([SuffixRemover(".1"), SuffixRemover("/1")])
|
|
145
|
+
# step 2: remove adapter on the 5' end, artifact of template switching
|
|
146
|
+
modifiers.append(
|
|
147
|
+
AdapterCutter(
|
|
148
|
+
[
|
|
149
|
+
RightmostFrontAdapter(
|
|
150
|
+
sequence=barcode.p5.fw, max_errors=0.25, min_overlap=10
|
|
151
|
+
)
|
|
152
|
+
],
|
|
153
|
+
times=1,
|
|
154
|
+
)
|
|
155
|
+
)
|
|
156
|
+
# step 3: remove adapter on the 3' end, read though in the sequencing
|
|
157
|
+
modifiers.append(
|
|
158
|
+
AdapterCutter(
|
|
159
|
+
[BackAdapter(sequence=barcode.p7.fw, max_errors=0.2, min_overlap=3)],
|
|
160
|
+
times=2,
|
|
161
|
+
),
|
|
162
|
+
)
|
|
163
|
+
# step 4: trim inline barcode
|
|
164
|
+
if barcode.inline5.len > 0:
|
|
165
|
+
modifiers.append(
|
|
166
|
+
AdapterCutter(
|
|
167
|
+
[PrefixAdapter(sequence=barcode.inline5.fw, max_errors=0.2)],
|
|
168
|
+
times=1,
|
|
169
|
+
)
|
|
170
|
+
)
|
|
171
|
+
if barcode.inline3.len > 0:
|
|
172
|
+
modifiers.append(UnconditionalCutter(-barcode.inline3.len))
|
|
173
|
+
|
|
174
|
+
# step 5: extract UMI
|
|
175
|
+
if barcode.umi5.len > 0:
|
|
176
|
+
modifiers.append(UnconditionalCutter(barcode.umi5.len))
|
|
177
|
+
if barcode.umi3.len > 0:
|
|
178
|
+
modifiers.append(UnconditionalCutter(-barcode.umi3.len))
|
|
179
|
+
if barcode.umi5.len + barcode.umi3.len > 0:
|
|
180
|
+
modifiers.append(Renamer("{id}_{cut_prefix}{cut_suffix}"))
|
|
181
|
+
else:
|
|
182
|
+
modifiers.append(Renamer("{id}"))
|
|
183
|
+
|
|
184
|
+
# step 6: mask tail in the RNA, which might be artifact of RT
|
|
185
|
+
if barcode.mask5.len > 0:
|
|
186
|
+
modifiers.append(UnconditionalCutter(barcode.mask5.len))
|
|
187
|
+
if barcode.mask3.len > 0:
|
|
188
|
+
modifiers.append(UnconditionalCutter(-barcode.mask3.len))
|
|
189
|
+
# step 7: trim polyA
|
|
190
|
+
if settings.trim_polyA:
|
|
191
|
+
if barcode.strand == "+":
|
|
192
|
+
modifiers.append(
|
|
193
|
+
AdapterCutter(
|
|
194
|
+
[NonInternalBackAdapter(sequence="A" * 100, max_errors=0.15)]
|
|
195
|
+
)
|
|
196
|
+
)
|
|
197
|
+
elif barcode.strand == "-":
|
|
198
|
+
modifiers.append(
|
|
199
|
+
AdapterCutter(
|
|
200
|
+
[NonInternalFrontAdapter(sequence="T" * 100, max_errors=0.15)]
|
|
201
|
+
)
|
|
202
|
+
)
|
|
203
|
+
else:
|
|
204
|
+
logging.info("No strand information provided, skip polyA trimming.")
|
|
205
|
+
# step 8: quality control, remove short reads
|
|
206
|
+
modifiers.append(
|
|
207
|
+
QualityTrimmer(cutoff_front=0, cutoff_back=settings.min_quality),
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
inpaths = InputPaths(input1)
|
|
211
|
+
|
|
212
|
+
with make_runner(inpaths, cores=settings.threads) as runner:
|
|
213
|
+
outfiles = OutputFiles(
|
|
214
|
+
proxied=settings.threads > 1,
|
|
215
|
+
qualities=runner.input_file_format().has_qualities(),
|
|
216
|
+
interleaved=False,
|
|
217
|
+
)
|
|
218
|
+
steps = [
|
|
219
|
+
# --info-file=info.txt
|
|
220
|
+
# PairedSingleEndStep(InfoFileWriter(outfiles.open_text("info.txt"))),
|
|
221
|
+
# -m 10
|
|
222
|
+
SingleEndFilter(
|
|
223
|
+
TooShort(settings.min_length), outfiles.open_record_writer(short1)
|
|
224
|
+
),
|
|
225
|
+
# TODO: --max-n=0 support
|
|
226
|
+
# --discard-untrimmed
|
|
227
|
+
# SingleEndFilter( IsUntrimmed(), IsUntrimmed(), pair_filter_mode="any"),
|
|
228
|
+
# -o
|
|
229
|
+
SingleEndSink(outfiles.open_record_writer(output1, interleaved=False)),
|
|
230
|
+
]
|
|
231
|
+
pipeline = SingleEndPipeline(modifiers, steps)
|
|
232
|
+
_stats = runner.run(pipeline, Progress(), outfiles)
|
|
233
|
+
# _ = stats.as_json()
|
|
234
|
+
outfiles.close()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def pipeline_paired(
|
|
238
|
+
input1,
|
|
239
|
+
input2,
|
|
240
|
+
output1,
|
|
241
|
+
output2,
|
|
242
|
+
short1,
|
|
243
|
+
short2,
|
|
244
|
+
untrimed1,
|
|
245
|
+
untrimmed2,
|
|
246
|
+
barcode,
|
|
247
|
+
settings,
|
|
248
|
+
):
|
|
249
|
+
modifiers = []
|
|
250
|
+
# step 1: remove suffix in the read name
|
|
251
|
+
modifiers.extend(
|
|
252
|
+
[
|
|
253
|
+
(SuffixRemover(".1"), SuffixRemover(".2")),
|
|
254
|
+
(SuffixRemover("/1"), SuffixRemover("/2")),
|
|
255
|
+
]
|
|
256
|
+
)
|
|
257
|
+
# step 2: remove adapter on the 5' end, artifact of template switching
|
|
258
|
+
modifiers.append(
|
|
259
|
+
(
|
|
260
|
+
AdapterCutter(
|
|
261
|
+
[
|
|
262
|
+
RightmostFrontAdapter(
|
|
263
|
+
sequence=barcode.p5.fw, max_errors=0.25, min_overlap=10
|
|
264
|
+
)
|
|
265
|
+
],
|
|
266
|
+
times=1,
|
|
267
|
+
),
|
|
268
|
+
AdapterCutter(
|
|
269
|
+
[
|
|
270
|
+
RightmostFrontAdapter(
|
|
271
|
+
sequence=barcode.p7.rc, max_errors=0.25, min_overlap=10
|
|
272
|
+
)
|
|
273
|
+
],
|
|
274
|
+
times=1,
|
|
275
|
+
),
|
|
276
|
+
),
|
|
277
|
+
)
|
|
278
|
+
# step 3: remove adapter on the 3' end, read though in the sequencing
|
|
279
|
+
modifiers.append(
|
|
280
|
+
(
|
|
281
|
+
AdapterCutter(
|
|
282
|
+
[BackAdapter(sequence=barcode.p7.fw, max_errors=0.2, min_overlap=3)],
|
|
283
|
+
times=2,
|
|
284
|
+
),
|
|
285
|
+
AdapterCutter(
|
|
286
|
+
[BackAdapter(sequence=barcode.p5.rc, max_errors=0.2, min_overlap=3)],
|
|
287
|
+
times=2,
|
|
288
|
+
),
|
|
289
|
+
),
|
|
290
|
+
)
|
|
291
|
+
# step 4: trim inline barcode
|
|
292
|
+
if barcode.inline5.len > 0:
|
|
293
|
+
modifiers.append(
|
|
294
|
+
(
|
|
295
|
+
AdapterCutter(
|
|
296
|
+
[PrefixAdapter(sequence=barcode.inline5.fw, max_errors=0.2)],
|
|
297
|
+
times=1,
|
|
298
|
+
),
|
|
299
|
+
UnconditionalCutter(-barcode.inline5.len),
|
|
300
|
+
)
|
|
301
|
+
)
|
|
302
|
+
if barcode.inline3.len > 0:
|
|
303
|
+
modifiers.append(
|
|
304
|
+
(
|
|
305
|
+
UnconditionalCutter(-barcode.inline3.len),
|
|
306
|
+
AdapterCutter(
|
|
307
|
+
[BackAdapter(sequence=barcode.inline3.rc, max_errors=0.2)],
|
|
308
|
+
times=1,
|
|
309
|
+
),
|
|
310
|
+
)
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
# step 5: extract UMI
|
|
314
|
+
if barcode.umi5.len > 0:
|
|
315
|
+
modifiers.append(
|
|
316
|
+
(
|
|
317
|
+
UnconditionalCutter(barcode.umi5.len),
|
|
318
|
+
UnconditionalCutter(-barcode.umi5.len),
|
|
319
|
+
),
|
|
320
|
+
)
|
|
321
|
+
if barcode.umi3.len > 0:
|
|
322
|
+
modifiers.append(
|
|
323
|
+
(
|
|
324
|
+
UnconditionalCutter(-barcode.umi3.len),
|
|
325
|
+
UnconditionalCutter(barcode.umi3.len),
|
|
326
|
+
)
|
|
327
|
+
)
|
|
328
|
+
if barcode.umi5.len + barcode.umi3.len > 0:
|
|
329
|
+
modifiers.append(PairedEndRenamer("{id}_{r1.cut_prefix}{r2.cut_prefix}"))
|
|
330
|
+
else:
|
|
331
|
+
modifiers.append(PairedEndRenamer("{id}"))
|
|
332
|
+
|
|
333
|
+
# step 6: mask tail in the RNA, which might be artifact of RT
|
|
334
|
+
if barcode.mask5.len > 0:
|
|
335
|
+
modifiers.append(
|
|
336
|
+
(
|
|
337
|
+
UnconditionalCutter(barcode.mask5.len),
|
|
338
|
+
UnconditionalCutter(-barcode.mask5.len),
|
|
339
|
+
)
|
|
340
|
+
)
|
|
341
|
+
if barcode.mask3.len > 0:
|
|
342
|
+
modifiers.append(
|
|
343
|
+
(
|
|
344
|
+
UnconditionalCutter(-barcode.mask3.len),
|
|
345
|
+
UnconditionalCutter(barcode.mask3.len),
|
|
346
|
+
)
|
|
347
|
+
)
|
|
348
|
+
# step 7: trim polyA
|
|
349
|
+
if settings.trim_polyA:
|
|
350
|
+
if barcode.strand == "+":
|
|
351
|
+
modifiers.append(
|
|
352
|
+
(
|
|
353
|
+
AdapterCutter(
|
|
354
|
+
[NonInternalBackAdapter(sequence="A" * 100, max_errors=0.15)]
|
|
355
|
+
),
|
|
356
|
+
AdapterCutter(
|
|
357
|
+
[NonInternalFrontAdapter(sequence="T" * 100, max_errors=0.15)]
|
|
358
|
+
),
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
elif barcode.strand == "-":
|
|
362
|
+
modifiers.append(
|
|
363
|
+
(
|
|
364
|
+
AdapterCutter(
|
|
365
|
+
[NonInternalFrontAdapter(sequence="T" * 100, max_errors=0.15)]
|
|
366
|
+
),
|
|
367
|
+
AdapterCutter(
|
|
368
|
+
[NonInternalBackAdapter(sequence="A" * 100, max_errors=0.15)]
|
|
369
|
+
),
|
|
370
|
+
)
|
|
371
|
+
)
|
|
372
|
+
else:
|
|
373
|
+
logging.info("No strand information provided, skip polyA trimming.")
|
|
374
|
+
# step 8: quality control, remove short reads
|
|
375
|
+
modifiers.append(
|
|
376
|
+
(
|
|
377
|
+
QualityTrimmer(cutoff_front=0, cutoff_back=settings.min_quality),
|
|
378
|
+
QualityTrimmer(cutoff_front=0, cutoff_back=settings.min_quality),
|
|
379
|
+
)
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
inpaths = InputPaths(input1, input2)
|
|
383
|
+
|
|
384
|
+
with make_runner(inpaths, cores=settings.threads) as runner:
|
|
385
|
+
outfiles = OutputFiles(
|
|
386
|
+
proxied=settings.threads > 1,
|
|
387
|
+
qualities=runner.input_file_format().has_qualities(),
|
|
388
|
+
interleaved=False,
|
|
389
|
+
)
|
|
390
|
+
steps = [
|
|
391
|
+
# --info-file=info.txt
|
|
392
|
+
# PairedSingleEndStep(InfoFileWriter(outfiles.open_text("info.txt"))),
|
|
393
|
+
# -m 10:10
|
|
394
|
+
PairedEndFilter(
|
|
395
|
+
TooShort(settings.min_length),
|
|
396
|
+
TooShort(settings.min_length),
|
|
397
|
+
outfiles.open_record_writer(short1, short2, interleaved=False),
|
|
398
|
+
),
|
|
399
|
+
# TODO: --max-n=0 support
|
|
400
|
+
# --discard-untrimmed
|
|
401
|
+
# PairedEndFilter( IsUntrimmed(), IsUntrimmed(), pair_filter_mode="any"),
|
|
402
|
+
# -o ... -p ...
|
|
403
|
+
PairedEndSink(outfiles.open_record_writer(output1, output2)),
|
|
404
|
+
]
|
|
405
|
+
pipeline = PairedEndPipeline(modifiers, steps)
|
|
406
|
+
_stats = runner.run(pipeline, Progress(), outfiles)
|
|
407
|
+
# _ = stats.as_json()
|
|
408
|
+
outfiles.close()
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def run_cutseq(args):
|
|
412
|
+
barcode_config = BarcodeConfig(args.adapter_scheme.upper())
|
|
413
|
+
settings = CutadaptConfig()
|
|
414
|
+
if args.with_rname_suffix:
|
|
415
|
+
settings.rname_suffix = True
|
|
416
|
+
if args.untrimmed_file is None:
|
|
417
|
+
settings.discarded_untrimmed = True
|
|
418
|
+
if args.trim_polyA:
|
|
419
|
+
settings.trim_polyA = True
|
|
420
|
+
settings.threads = args.threads
|
|
421
|
+
settings.min_length = args.min_length
|
|
422
|
+
settings.dry_run = args.dry_run
|
|
423
|
+
if len(args.input_file) == 1:
|
|
424
|
+
pipeline_single(
|
|
425
|
+
args.input_file[0],
|
|
426
|
+
args.output_file[0],
|
|
427
|
+
args.short_file[0],
|
|
428
|
+
args.untrimmed_file[0],
|
|
429
|
+
barcode_config,
|
|
430
|
+
settings,
|
|
431
|
+
)
|
|
432
|
+
else:
|
|
433
|
+
pipeline_paired(
|
|
434
|
+
args.input_file[0],
|
|
435
|
+
args.input_file[1],
|
|
436
|
+
args.output_file[0],
|
|
437
|
+
args.output_file[1],
|
|
438
|
+
args.short_file[0],
|
|
439
|
+
args.short_file[1],
|
|
440
|
+
args.untrimmed_file[0],
|
|
441
|
+
args.untrimmed_file[1],
|
|
442
|
+
barcode_config,
|
|
443
|
+
settings,
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def main():
|
|
448
|
+
parser = argparse.ArgumentParser(
|
|
449
|
+
description="Trim sequencing adapters from NGS data automatically."
|
|
450
|
+
)
|
|
451
|
+
# input file can be one or two for single or paired-end reads, but can not be more than two
|
|
452
|
+
parser.add_argument(
|
|
453
|
+
"input_file",
|
|
454
|
+
type=str,
|
|
455
|
+
nargs="+",
|
|
456
|
+
help="Input file path for NGS data, one or two files.",
|
|
457
|
+
)
|
|
458
|
+
# output file can be number of files matching the input files, if not provided it will generate based on the output prefix,
|
|
459
|
+
# if no output prefix provided it will generate based on the input file name
|
|
460
|
+
parser.add_argument(
|
|
461
|
+
"-a",
|
|
462
|
+
"--adapter-scheme",
|
|
463
|
+
type=str,
|
|
464
|
+
help="Adapter sequence configuration.",
|
|
465
|
+
)
|
|
466
|
+
parser.add_argument("-A", "--adapter-name", type=str, help="Built-in adapter name.")
|
|
467
|
+
parser.add_argument(
|
|
468
|
+
"-O",
|
|
469
|
+
"--output-prefix",
|
|
470
|
+
type=str,
|
|
471
|
+
help="Output file prefix for keep trimmed data.",
|
|
472
|
+
)
|
|
473
|
+
parser.add_argument(
|
|
474
|
+
"-o",
|
|
475
|
+
"--output-file",
|
|
476
|
+
type=str,
|
|
477
|
+
nargs="+",
|
|
478
|
+
help="Output file path for keep trimmed data.",
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
parser.add_argument(
|
|
482
|
+
"-q",
|
|
483
|
+
"--min-quality",
|
|
484
|
+
type=int,
|
|
485
|
+
default=20,
|
|
486
|
+
help="Minimum quality of the read tails in the reads to keep.",
|
|
487
|
+
)
|
|
488
|
+
# discard short reads
|
|
489
|
+
parser.add_argument(
|
|
490
|
+
"-S",
|
|
491
|
+
"--short-file",
|
|
492
|
+
type=str,
|
|
493
|
+
nargs="+",
|
|
494
|
+
help="Output file path for discarded too short data.",
|
|
495
|
+
)
|
|
496
|
+
parser.add_argument(
|
|
497
|
+
"-m",
|
|
498
|
+
"--min-length",
|
|
499
|
+
type=int,
|
|
500
|
+
default=20,
|
|
501
|
+
help="Minimum length of the reads to keep.",
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
parser.add_argument(
|
|
505
|
+
"--with-rname-suffix",
|
|
506
|
+
action="store_true",
|
|
507
|
+
help="R1 and R2 suffix cotains suffix. MGI platform.",
|
|
508
|
+
)
|
|
509
|
+
parser.add_argument(
|
|
510
|
+
"-U",
|
|
511
|
+
"--untrimmed-file",
|
|
512
|
+
type=str,
|
|
513
|
+
nargs="+",
|
|
514
|
+
help="Output file path for discarded reads without inline barcode.",
|
|
515
|
+
)
|
|
516
|
+
|
|
517
|
+
parser.add_argument("--trim-polyA", action="store_true", help="Trim polyA tail.")
|
|
518
|
+
|
|
519
|
+
parser.add_argument(
|
|
520
|
+
"-t",
|
|
521
|
+
"--threads",
|
|
522
|
+
type=int,
|
|
523
|
+
default=1,
|
|
524
|
+
help="Number of threads to use for trimming.",
|
|
525
|
+
)
|
|
526
|
+
parser.add_argument(
|
|
527
|
+
"-n",
|
|
528
|
+
"--dry-run",
|
|
529
|
+
action="store_true",
|
|
530
|
+
help="Print command instead of running it.",
|
|
531
|
+
)
|
|
532
|
+
args = parser.parse_args()
|
|
533
|
+
|
|
534
|
+
if args.adapter_name is not None:
|
|
535
|
+
if args.adapter_scheme is not None:
|
|
536
|
+
logging.info("Adapter scheme is provided, ignore adapter name.")
|
|
537
|
+
else:
|
|
538
|
+
if args.adapter_name.upper() == "TAKARAV2":
|
|
539
|
+
args.adapter_scheme = "ACACGACGCTCTTCCGATCTX<XXXAGATCGGAAGAGCACACGTC"
|
|
540
|
+
elif args.adapter_name.upper() == "STRANDED":
|
|
541
|
+
args.adapter_scheme = "ACACGACGCTCTTCCGATCTX<XXXAGATCGGAAGAGCACACGTC"
|
|
542
|
+
elif args.adapter_name.upper() == "TAKARAV3":
|
|
543
|
+
args.adapter_scheme = (
|
|
544
|
+
"ACACGACGCTCTTCCGATCTXXX<XXXXXXNNNNNNNNAGATCGGAAGAGCACACGTC"
|
|
545
|
+
)
|
|
546
|
+
else:
|
|
547
|
+
logging.error("Adapter name is not valid.")
|
|
548
|
+
sys.exit(1)
|
|
549
|
+
elif args.adapter_scheme is None:
|
|
550
|
+
logging.error("Adapter scheme or name is required.")
|
|
551
|
+
sys.exit(1)
|
|
552
|
+
|
|
553
|
+
if len(args.input_file) > 2:
|
|
554
|
+
logging.error("Input file can not be more than two.")
|
|
555
|
+
sys.exit(1)
|
|
556
|
+
|
|
557
|
+
def validate_output_file(output_files, input_files, output_prefix, output_suffix):
|
|
558
|
+
default_format = ".fastq.gz"
|
|
559
|
+
r1 = "_" + output_suffix + "_R1" + default_format
|
|
560
|
+
r2 = "_" + output_suffix + "_R2" + default_format
|
|
561
|
+
if output_files:
|
|
562
|
+
if len(output_files) != len(input_files):
|
|
563
|
+
logging.error("Output file should be same as input file.")
|
|
564
|
+
sys.exit(1)
|
|
565
|
+
return output_files
|
|
566
|
+
elif output_prefix is not None:
|
|
567
|
+
if len(input_files) == 1:
|
|
568
|
+
return [output_prefix + r1]
|
|
569
|
+
else:
|
|
570
|
+
return [output_prefix + r1, output_prefix + r2]
|
|
571
|
+
else:
|
|
572
|
+
if len(input_files) == 1:
|
|
573
|
+
return [remove_fq_suffix(input_files[0]) + r1]
|
|
574
|
+
else:
|
|
575
|
+
return [
|
|
576
|
+
remove_fq_suffix(input_files[0]) + r1,
|
|
577
|
+
remove_fq_suffix(input_files[1]) + r2,
|
|
578
|
+
]
|
|
579
|
+
return output_files
|
|
580
|
+
|
|
581
|
+
args.output_file = validate_output_file(
|
|
582
|
+
args.output_file, args.input_file, args.output_prefix, "trimmed"
|
|
583
|
+
)
|
|
584
|
+
args.short_file = validate_output_file(
|
|
585
|
+
args.short_file, args.input_file, args.output_prefix, "short"
|
|
586
|
+
)
|
|
587
|
+
args.untrimmed_file = (
|
|
588
|
+
validate_output_file(
|
|
589
|
+
args.untrimmed_file, args.input_file, args.output_prefix, "untrimmed"
|
|
590
|
+
)
|
|
591
|
+
if args.untrimmed_file is not None
|
|
592
|
+
else [None] * len(args.input_file)
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
run_cutseq(args)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
if __name__ == "__main__":
|
|
599
|
+
main()
|
|
@@ -40,23 +40,29 @@ def remove_fq_suffix(f):
|
|
|
40
40
|
return f
|
|
41
41
|
|
|
42
42
|
|
|
43
|
+
class BarcodeSeq:
|
|
44
|
+
def __init__(self, seq):
|
|
45
|
+
self.fw = seq
|
|
46
|
+
self.rc = reverse_complement(seq)
|
|
47
|
+
self.len = len(seq)
|
|
48
|
+
|
|
49
|
+
|
|
43
50
|
class BarcodeConfig:
|
|
51
|
+
"""
|
|
52
|
+
Adapter scheme:
|
|
53
|
+
(p5)(inline5)(umi5)(mask5)(strand)(mask3)(umi3)(inline3)(p7)
|
|
54
|
+
"""
|
|
55
|
+
|
|
44
56
|
def __init__(self, adapter=None):
|
|
45
57
|
self.strand = None
|
|
46
|
-
self.
|
|
47
|
-
self.
|
|
48
|
-
self.
|
|
49
|
-
self.
|
|
50
|
-
self.
|
|
51
|
-
self.
|
|
52
|
-
self.
|
|
53
|
-
self.
|
|
54
|
-
self.inline3_rc = ""
|
|
55
|
-
self.inline3 = 0
|
|
56
|
-
self.umi5 = 0
|
|
57
|
-
self.umi3 = 0
|
|
58
|
-
self.mask5 = 0
|
|
59
|
-
self.mask3 = 0
|
|
58
|
+
self.p5 = BarcodeSeq("")
|
|
59
|
+
self.p7 = BarcodeSeq("")
|
|
60
|
+
self.inline5 = BarcodeSeq("")
|
|
61
|
+
self.inline3 = BarcodeSeq("")
|
|
62
|
+
self.umi5 = BarcodeSeq("")
|
|
63
|
+
self.umi3 = BarcodeSeq("")
|
|
64
|
+
self.mask5 = BarcodeSeq("")
|
|
65
|
+
self.mask3 = BarcodeSeq("")
|
|
60
66
|
if adapter is not None:
|
|
61
67
|
self._parse_barcode(adapter)
|
|
62
68
|
|
|
@@ -74,20 +80,14 @@ class BarcodeConfig:
|
|
|
74
80
|
if d["inline3"] is None:
|
|
75
81
|
d["inline3"] = ""
|
|
76
82
|
self.strand = "+" if d["strand"] == ">" else "-" if d["strand"] == "<" else None
|
|
77
|
-
self.
|
|
78
|
-
self.
|
|
79
|
-
self.
|
|
80
|
-
self.
|
|
81
|
-
self.
|
|
82
|
-
self.
|
|
83
|
-
self.
|
|
84
|
-
self.
|
|
85
|
-
self.inline3_rc = reverse_complement(d["inline3"]) if d["inline3"] else ""
|
|
86
|
-
self.inline3 = len(d["inline3"])
|
|
87
|
-
self.umi5 = len(d["umi5"])
|
|
88
|
-
self.umi3 = len(d["umi3"])
|
|
89
|
-
self.mask5 = len(d["mask5"])
|
|
90
|
-
self.mask3 = len(d["mask3"])
|
|
83
|
+
self.p5 = BarcodeSeq(d["p5"])
|
|
84
|
+
self.p7 = BarcodeSeq(d["p7"])
|
|
85
|
+
self.inline5 = BarcodeSeq(d["inline5"])
|
|
86
|
+
self.inline3 = BarcodeSeq(d["inline3"])
|
|
87
|
+
self.umi5 = BarcodeSeq(d["umi5"])
|
|
88
|
+
self.umi3 = BarcodeSeq(d["umi3"])
|
|
89
|
+
self.mask5 = BarcodeSeq(d["mask5"])
|
|
90
|
+
self.mask3 = BarcodeSeq(d["mask3"])
|
|
91
91
|
|
|
92
92
|
|
|
93
93
|
class CutadaptConfig:
|
|
@@ -112,19 +112,19 @@ def run_cutadapt_PE(
|
|
|
112
112
|
else:
|
|
113
113
|
config_rname = ""
|
|
114
114
|
steps.append(
|
|
115
|
-
f"{cutadapt}{config_rname} -e 0.25 -n 2 -O 10 -g '{barcode.
|
|
115
|
+
f"{cutadapt}{config_rname} -e 0.25 -n 2 -O 10 -g '{barcode.p5.fw};rightmost' -G '{barcode.p7.rc};rightmost' --interleaved {input1} {input2}"
|
|
116
116
|
)
|
|
117
117
|
# step 2: remove adapter on the 3' end, read though in the sequencing
|
|
118
118
|
steps.append(
|
|
119
|
-
f"{cutadapt} -e 0.2 -n 2 -O 3 -a '{barcode.
|
|
119
|
+
f"{cutadapt} -e 0.2 -n 2 -O 3 -a '{barcode.p7.fw}' -A '{barcode.p5.rc}' --interleaved -"
|
|
120
120
|
)
|
|
121
121
|
# step 3: trim inline barcode
|
|
122
122
|
config_inline_args = []
|
|
123
|
-
if barcode.inline5 > 0:
|
|
124
|
-
config_inline_args.append(f"-g ^{barcode.
|
|
125
|
-
if barcode.inline3 > 0:
|
|
126
|
-
config_inline_args.append(f"-G ^{barcode.
|
|
127
|
-
if barcode.inline5 + barcode.inline3 > 0:
|
|
123
|
+
if barcode.inline5.len > 0:
|
|
124
|
+
config_inline_args.append(f"-g ^{barcode.inline5.fw} -U -{barcode.inline5.len}")
|
|
125
|
+
if barcode.inline3.len > 0:
|
|
126
|
+
config_inline_args.append(f"-G ^{barcode.inline3.rc} -u -{barcode.inline3.len}")
|
|
127
|
+
if barcode.inline5.len + barcode.inline3.len > 0:
|
|
128
128
|
if settings.discarded_untrimmed:
|
|
129
129
|
config_inline_args.append(
|
|
130
130
|
f"--untrimmed-output={discard1} --untrimmed-paired-output={discard2}"
|
|
@@ -132,16 +132,16 @@ def run_cutadapt_PE(
|
|
|
132
132
|
config_inline = " ".join(config_inline_args)
|
|
133
133
|
steps.append(f"{cutadapt} {config_inline} --interleaved -")
|
|
134
134
|
# step 4: extract UMI
|
|
135
|
-
if barcode.umi5 + barcode.umi3 > 0:
|
|
135
|
+
if barcode.umi5.len + barcode.umi3.len > 0:
|
|
136
136
|
steps.append(
|
|
137
|
-
f"{cutadapt} -u {barcode.umi5} -u -{barcode.umi3} -U {barcode.umi3} -U -{barcode.umi5} --rename='{{id}}_{{r1.cut_prefix}}{{r2.cut_prefix}}' --interleaved -"
|
|
137
|
+
f"{cutadapt} -u {barcode.umi5.len} -u -{barcode.umi3.len} -U {barcode.umi3.len} -U -{barcode.umi5.len} --rename='{{id}}_{{r1.cut_prefix}}{{r2.cut_prefix}}' --interleaved -"
|
|
138
138
|
)
|
|
139
139
|
else:
|
|
140
140
|
steps.append(f"{cutadapt} --rename='{{id}}' --interleaved -")
|
|
141
141
|
# step 5: mask tail in the RNA, which might be artifact of RT
|
|
142
|
-
if barcode.mask5 + barcode.mask3 > 0:
|
|
142
|
+
if barcode.mask5.len + barcode.mask3.len > 0:
|
|
143
143
|
steps.append(
|
|
144
|
-
f"{cutadapt} -u {barcode.mask5} -u -{barcode.mask3} -U {barcode.mask3} -U -{barcode.mask5} --interleaved -"
|
|
144
|
+
f"{cutadapt} -u {barcode.mask5.len} -u -{barcode.mask3.len} -U {barcode.mask3.len} -U -{barcode.mask5.len} --interleaved -"
|
|
145
145
|
)
|
|
146
146
|
# step 6: trim polyA
|
|
147
147
|
if settings.trim_polyA:
|
|
@@ -171,29 +171,29 @@ def run_cutadapt_SE(input1, output1, discard1, barcode, settings):
|
|
|
171
171
|
else:
|
|
172
172
|
config_rname = ""
|
|
173
173
|
steps.append(
|
|
174
|
-
f"{cutadapt}{config_rname} -e 0.25 -n 2 -O 10 -g '{barcode.
|
|
174
|
+
f"{cutadapt}{config_rname} -e 0.25 -n 2 -O 10 -g '{barcode.p5.fw};rightmost' {input1}"
|
|
175
175
|
)
|
|
176
176
|
# step 2: remove adapter on the 3' end, read though in the sequencing
|
|
177
|
-
steps.append(f"{cutadapt} -e 0.2 -n 2 -O 3 -a '{barcode.
|
|
177
|
+
steps.append(f"{cutadapt} -e 0.2 -n 2 -O 3 -a '{barcode.p7.fw}' -")
|
|
178
178
|
# step 3: trim inline barcode
|
|
179
179
|
config_inline_args = []
|
|
180
|
-
if barcode.inline3 > 0:
|
|
181
|
-
config_inline_args.append(f"-a {barcode.
|
|
182
|
-
if barcode.inline5 > 0:
|
|
183
|
-
config_inline_args.append(f"-g ^{barcode.
|
|
184
|
-
if barcode.inline5 + barcode.inline3 > 0:
|
|
180
|
+
if barcode.inline3.len > 0:
|
|
181
|
+
config_inline_args.append(f"-a {barcode.inline3.fw}$")
|
|
182
|
+
if barcode.inline5.len > 0:
|
|
183
|
+
config_inline_args.append(f"-g ^{barcode.inline5.fw}")
|
|
184
|
+
if barcode.inline5.len + barcode.inline3.len > 0:
|
|
185
185
|
if settings.discarded_untrimmed:
|
|
186
186
|
config_inline_args.append(f"--untrimmed-output={discard1}")
|
|
187
187
|
|
|
188
188
|
config_inline = " ".join(config_inline_args)
|
|
189
189
|
steps.append(f"{cutadapt} {config_inline} -")
|
|
190
190
|
# step 4: extract UMI
|
|
191
|
-
if barcode.umi5 + barcode.umi3 > 0:
|
|
191
|
+
if barcode.umi5.len + barcode.umi3.len > 0:
|
|
192
192
|
steps.append(
|
|
193
|
-
f"{cutadapt} -u {barcode.umi5} -u -{barcode.umi3} --rename='{{id}}_{{cut_prefix}}{{cut_suffix}}' -"
|
|
193
|
+
f"{cutadapt} -u {barcode.umi5.len} -u -{barcode.umi3.len} --rename='{{id}}_{{cut_prefix}}{{cut_suffix}}' -"
|
|
194
194
|
)
|
|
195
195
|
# step 5: mask tail in the RNA, which might be artifact of RT
|
|
196
|
-
steps.append(f"{cutadapt} -u {barcode.mask5} -u -{barcode.mask3} -")
|
|
196
|
+
steps.append(f"{cutadapt} -u {barcode.mask5.len} -u -{barcode.mask3.len} -")
|
|
197
197
|
# step 6: trim polyA
|
|
198
198
|
if settings.trim_polyA:
|
|
199
199
|
if barcode.strand == "+":
|
|
@@ -233,7 +233,6 @@ def run_cutseq(args):
|
|
|
233
233
|
settings.threads = args.threads
|
|
234
234
|
settings.min_length = args.min_length
|
|
235
235
|
settings.dry_run = args.dry_run
|
|
236
|
-
# Example command setup, you'll need to expand this based on your actual requirements
|
|
237
236
|
if len(args.input_file) == 1:
|
|
238
237
|
steps = run_cutadapt_SE(
|
|
239
238
|
args.input_file[0],
|