HTSeq 2.1.2__cp313-cp313-macosx_10_15_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- HTSeq/StepVector.py +629 -0
- HTSeq/StretchVector.py +491 -0
- HTSeq/_HTSeq.cpython-313-darwin.so +0 -0
- HTSeq/_HTSeq_internal.py +85 -0
- HTSeq/_StepVector.cpython-313-darwin.so +0 -0
- HTSeq/__init__.py +1249 -0
- HTSeq/features.py +489 -0
- HTSeq/scripts/__init__.py +0 -0
- HTSeq/scripts/count.py +528 -0
- HTSeq/scripts/count_features/__init__.py +0 -0
- HTSeq/scripts/count_features/count_features_per_file.py +465 -0
- HTSeq/scripts/count_features/reads_io_processor.py +187 -0
- HTSeq/scripts/count_features/reads_stats.py +92 -0
- HTSeq/scripts/count_with_barcodes.py +746 -0
- HTSeq/scripts/qa.py +336 -0
- HTSeq/scripts/utils.py +372 -0
- HTSeq/utils.py +92 -0
- htseq-2.1.2.dist-info/METADATA +813 -0
- htseq-2.1.2.dist-info/RECORD +23 -0
- htseq-2.1.2.dist-info/WHEEL +5 -0
- htseq-2.1.2.dist-info/entry_points.txt +4 -0
- htseq-2.1.2.dist-info/licenses/LICENSE +674 -0
- htseq-2.1.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,746 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import argparse
|
|
3
|
+
from collections import Counter, defaultdict
|
|
4
|
+
import operator
|
|
5
|
+
import itertools
|
|
6
|
+
import warnings
|
|
7
|
+
import traceback
|
|
8
|
+
import os.path
|
|
9
|
+
import multiprocessing
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pysam
|
|
12
|
+
|
|
13
|
+
import HTSeq
|
|
14
|
+
from HTSeq.scripts.utils import (
|
|
15
|
+
UnknownChrom,
|
|
16
|
+
my_showwarning,
|
|
17
|
+
invert_strand,
|
|
18
|
+
_write_output,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def correct_barcodes(counts, hamming=1):
|
|
23
|
+
'''Correct barcodes, usually UMIs.
|
|
24
|
+
|
|
25
|
+
Notice: This function does not use recursive correction. Recursion sounds
|
|
26
|
+
great in theory, but due to experimental corner cases it can lead to
|
|
27
|
+
overcorrection and loss of signal.
|
|
28
|
+
'''
|
|
29
|
+
if hamming == 0:
|
|
30
|
+
return
|
|
31
|
+
|
|
32
|
+
# Count reads from all feature per barcode, and prepare to oder
|
|
33
|
+
n_reads = Counter({key: sum(val.values()) for key, val in counts.items()})
|
|
34
|
+
|
|
35
|
+
# Order by counts, from most to least
|
|
36
|
+
order = [umi for (umi, _) in n_reads.most_common()]
|
|
37
|
+
|
|
38
|
+
# Get close Hamming distances, and aggregate them into higher count ones
|
|
39
|
+
umi_vectors = np.array([list(x) for x in order])
|
|
40
|
+
idx_left = list(range(len(order)))
|
|
41
|
+
while idx_left:
|
|
42
|
+
i = idx_left.pop(0)
|
|
43
|
+
vi = ''.join(umi_vectors[i])
|
|
44
|
+
# Distance from all remaining barcodes
|
|
45
|
+
dis = (umi_vectors[i] != umi_vectors[idx_left]).sum(axis=1)
|
|
46
|
+
# Get indices of barcodes within reach
|
|
47
|
+
idx = (dis <= hamming).nonzero()[0]
|
|
48
|
+
js = [idx_left[idxi] for idxi in idx]
|
|
49
|
+
for j in js:
|
|
50
|
+
# Merge barcode counts into higher count one
|
|
51
|
+
vj = ''.join(umi_vectors[j])
|
|
52
|
+
counts[vi].update(counts.pop(vj))
|
|
53
|
+
# Shorten list of remaining UMIs
|
|
54
|
+
idx_left = [j for j in idx_left if j not in js]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def count_reads_with_barcodes(
|
|
58
|
+
sam_filename,
|
|
59
|
+
features,
|
|
60
|
+
feature_attr,
|
|
61
|
+
order,
|
|
62
|
+
max_buffer_size,
|
|
63
|
+
stranded,
|
|
64
|
+
overlap_mode,
|
|
65
|
+
multimapped_mode,
|
|
66
|
+
secondary_alignment_mode,
|
|
67
|
+
supplementary_alignment_mode,
|
|
68
|
+
feature_type,
|
|
69
|
+
id_attribute,
|
|
70
|
+
additional_attributes,
|
|
71
|
+
quiet,
|
|
72
|
+
minaqual,
|
|
73
|
+
samout_format,
|
|
74
|
+
samout_filename,
|
|
75
|
+
cb_tag,
|
|
76
|
+
ub_tag,
|
|
77
|
+
correct_ub_distance,
|
|
78
|
+
):
|
|
79
|
+
|
|
80
|
+
def write_to_samout(r, assignment, samoutfile, template=None):
|
|
81
|
+
if samoutfile is None:
|
|
82
|
+
return
|
|
83
|
+
if not pe_mode:
|
|
84
|
+
r = (r,)
|
|
85
|
+
for read in r:
|
|
86
|
+
if read is not None:
|
|
87
|
+
read.optional_fields.append(('XF', assignment))
|
|
88
|
+
if template is not None:
|
|
89
|
+
samoutfile.write(read.to_pysam_AlignedSegment(template))
|
|
90
|
+
elif samout_format in ('SAM', 'sam'):
|
|
91
|
+
samoutfile.write(read.get_sam_line() + "\n")
|
|
92
|
+
else:
|
|
93
|
+
raise ValueError(
|
|
94
|
+
'BAM/SAM output: no template and not a test SAM file',
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
def identify_barcodes(r):
|
|
98
|
+
'''Identify barcode from the read or pair (both must have the same)'''
|
|
99
|
+
if not pe_mode:
|
|
100
|
+
r = (r,)
|
|
101
|
+
|
|
102
|
+
# If either cell or UMI barcode doesn't exist, just raise exception
|
|
103
|
+
has_cb_tag = False
|
|
104
|
+
has_ub_tag = False
|
|
105
|
+
for read in r:
|
|
106
|
+
if read is not None:
|
|
107
|
+
# If tags have not been found, then try to find it
|
|
108
|
+
if not has_cb_tag:
|
|
109
|
+
has_cb_tag = read.has_optional_field(cb_tag)
|
|
110
|
+
if not has_ub_tag:
|
|
111
|
+
has_ub_tag = read.has_optional_field(ub_tag)
|
|
112
|
+
if not has_cb_tag or not has_ub_tag:
|
|
113
|
+
raise Exception("Missing cell or UMI barcode")
|
|
114
|
+
|
|
115
|
+
# cell, UMI
|
|
116
|
+
barcodes = [None, None]
|
|
117
|
+
nbar = 0
|
|
118
|
+
for read in r:
|
|
119
|
+
if read is not None:
|
|
120
|
+
for tag, val in read.optional_fields:
|
|
121
|
+
if tag == cb_tag:
|
|
122
|
+
barcodes[0] = val
|
|
123
|
+
nbar += 1
|
|
124
|
+
if nbar == 2:
|
|
125
|
+
return barcodes
|
|
126
|
+
elif tag == ub_tag:
|
|
127
|
+
barcodes[1] = val
|
|
128
|
+
nbar += 1
|
|
129
|
+
if nbar == 2:
|
|
130
|
+
return barcodes
|
|
131
|
+
return barcodes
|
|
132
|
+
|
|
133
|
+
try:
|
|
134
|
+
if sam_filename == "-":
|
|
135
|
+
read_seq_file = HTSeq.BAM_Reader(sys.stdin)
|
|
136
|
+
else:
|
|
137
|
+
read_seq_file = HTSeq.BAM_Reader(sam_filename)
|
|
138
|
+
|
|
139
|
+
# Get template for output BAM
|
|
140
|
+
if samout_filename is None:
|
|
141
|
+
template = None
|
|
142
|
+
samoutfile = None
|
|
143
|
+
elif samout_format in ('bam', 'BAM'):
|
|
144
|
+
template = read_seq_file.get_template()
|
|
145
|
+
samoutfile = pysam.AlignmentFile(
|
|
146
|
+
samout_filename, 'wb',
|
|
147
|
+
template=template,
|
|
148
|
+
)
|
|
149
|
+
elif (samout_format in ('sam', 'SAM')) and \
|
|
150
|
+
hasattr(read_seq_file, 'get_template'):
|
|
151
|
+
template = read_seq_file.get_template()
|
|
152
|
+
samoutfile = pysam.AlignmentFile(
|
|
153
|
+
samout_filename, 'w',
|
|
154
|
+
template=template,
|
|
155
|
+
)
|
|
156
|
+
else:
|
|
157
|
+
template = None
|
|
158
|
+
samoutfile = open(samout_filename, 'w')
|
|
159
|
+
|
|
160
|
+
read_seq_iter = iter(read_seq_file)
|
|
161
|
+
# Catch empty BAM files
|
|
162
|
+
try:
|
|
163
|
+
first_read = next(read_seq_iter)
|
|
164
|
+
pe_mode = first_read.paired_end
|
|
165
|
+
# FIXME: catchall can hide subtle bugs
|
|
166
|
+
except:
|
|
167
|
+
first_read = None
|
|
168
|
+
pe_mode = False
|
|
169
|
+
if first_read is not None:
|
|
170
|
+
read_seq = itertools.chain([first_read], read_seq_iter)
|
|
171
|
+
else:
|
|
172
|
+
read_seq = []
|
|
173
|
+
except:
|
|
174
|
+
sys.stderr.write(
|
|
175
|
+
"Error occured when reading beginning of SAM/BAM file.\n")
|
|
176
|
+
raise
|
|
177
|
+
|
|
178
|
+
# CIGAR match characters (including alignment match, sequence match, and
|
|
179
|
+
# sequence mismatch
|
|
180
|
+
com = ('M', '=', 'X')
|
|
181
|
+
|
|
182
|
+
try:
|
|
183
|
+
if pe_mode:
|
|
184
|
+
if ((supplementary_alignment_mode == 'ignore') and
|
|
185
|
+
(secondary_alignment_mode == 'ignore')):
|
|
186
|
+
primary_only = True
|
|
187
|
+
else:
|
|
188
|
+
primary_only = False
|
|
189
|
+
if order == "name":
|
|
190
|
+
read_seq = HTSeq.pair_SAM_alignments(
|
|
191
|
+
read_seq,
|
|
192
|
+
primary_only=primary_only)
|
|
193
|
+
elif order == "pos":
|
|
194
|
+
read_seq = HTSeq.pair_SAM_alignments_with_buffer(
|
|
195
|
+
read_seq,
|
|
196
|
+
max_buffer_size=max_buffer_size,
|
|
197
|
+
primary_only=primary_only)
|
|
198
|
+
else:
|
|
199
|
+
raise ValueError("Illegal order specified.")
|
|
200
|
+
|
|
201
|
+
# The nesting is cell barcode, UMI, feature
|
|
202
|
+
counts = defaultdict(lambda: defaultdict(Counter))
|
|
203
|
+
i = 0
|
|
204
|
+
for r in read_seq:
|
|
205
|
+
if i > 0 and i % 100000 == 0 and not quiet:
|
|
206
|
+
sys.stderr.write(
|
|
207
|
+
"%d alignment record%s processed.\n" %
|
|
208
|
+
(i, "s" if not pe_mode else " pairs"))
|
|
209
|
+
sys.stderr.flush()
|
|
210
|
+
|
|
211
|
+
i += 1
|
|
212
|
+
|
|
213
|
+
try:
|
|
214
|
+
cb, ub = identify_barcodes(r)
|
|
215
|
+
except:
|
|
216
|
+
# Happens when cb or ub is not found
|
|
217
|
+
write_to_samout(
|
|
218
|
+
r, "__too_low_aQual", samoutfile,
|
|
219
|
+
template)
|
|
220
|
+
continue
|
|
221
|
+
|
|
222
|
+
if not pe_mode:
|
|
223
|
+
if not r.aligned:
|
|
224
|
+
counts[cb][ub]['__not_aligned'] += 1
|
|
225
|
+
write_to_samout(
|
|
226
|
+
r, "__not_aligned", samoutfile,
|
|
227
|
+
template)
|
|
228
|
+
continue
|
|
229
|
+
if ((secondary_alignment_mode == 'ignore') and
|
|
230
|
+
r.not_primary_alignment):
|
|
231
|
+
continue
|
|
232
|
+
if ((supplementary_alignment_mode == 'ignore') and
|
|
233
|
+
r.supplementary):
|
|
234
|
+
continue
|
|
235
|
+
try:
|
|
236
|
+
if r.optional_field("NH") > 1:
|
|
237
|
+
counts[cb][ub]['__alignment_not_unique'] += 1
|
|
238
|
+
write_to_samout(
|
|
239
|
+
r,
|
|
240
|
+
"__alignment_not_unique",
|
|
241
|
+
samoutfile,
|
|
242
|
+
template)
|
|
243
|
+
if multimapped_mode == 'none':
|
|
244
|
+
continue
|
|
245
|
+
except KeyError:
|
|
246
|
+
pass
|
|
247
|
+
if r.aQual < minaqual:
|
|
248
|
+
counts[cb][ub]['__too_low_aQual'] += 1
|
|
249
|
+
write_to_samout(
|
|
250
|
+
r, "__too_low_aQual", samoutfile,
|
|
251
|
+
template)
|
|
252
|
+
continue
|
|
253
|
+
if stranded != "reverse":
|
|
254
|
+
iv_seq = (co.ref_iv for co in r.cigar if co.type in com
|
|
255
|
+
and co.size > 0)
|
|
256
|
+
else:
|
|
257
|
+
iv_seq = (invert_strand(co.ref_iv)
|
|
258
|
+
for co in r.cigar if (co.type in com and
|
|
259
|
+
co.size > 0))
|
|
260
|
+
else:
|
|
261
|
+
if r[0] is not None and r[0].aligned:
|
|
262
|
+
if stranded != "reverse":
|
|
263
|
+
iv_seq = (co.ref_iv for co in r[0].cigar
|
|
264
|
+
if co.type in com and co.size > 0)
|
|
265
|
+
else:
|
|
266
|
+
iv_seq = (invert_strand(co.ref_iv) for co in r[0].cigar
|
|
267
|
+
if co.type in com and co.size > 0)
|
|
268
|
+
else:
|
|
269
|
+
iv_seq = tuple()
|
|
270
|
+
if r[1] is not None and r[1].aligned:
|
|
271
|
+
if stranded != "reverse":
|
|
272
|
+
iv_seq = itertools.chain(
|
|
273
|
+
iv_seq,
|
|
274
|
+
(invert_strand(co.ref_iv) for co in r[1].cigar
|
|
275
|
+
if co.type in com and co.size > 0))
|
|
276
|
+
else:
|
|
277
|
+
iv_seq = itertools.chain(
|
|
278
|
+
iv_seq,
|
|
279
|
+
(co.ref_iv for co in r[1].cigar
|
|
280
|
+
if co.type in com and co.size > 0))
|
|
281
|
+
else:
|
|
282
|
+
if (r[0] is None) or not (r[0].aligned):
|
|
283
|
+
write_to_samout(
|
|
284
|
+
r, "__not_aligned", samoutfile,
|
|
285
|
+
template)
|
|
286
|
+
counts[cb][ub]['__not_aligned'] += 1
|
|
287
|
+
continue
|
|
288
|
+
if secondary_alignment_mode == 'ignore':
|
|
289
|
+
if (r[0] is not None) and r[0].not_primary_alignment:
|
|
290
|
+
continue
|
|
291
|
+
elif (r[1] is not None) and r[1].not_primary_alignment:
|
|
292
|
+
continue
|
|
293
|
+
if supplementary_alignment_mode == 'ignore':
|
|
294
|
+
if (r[0] is not None) and r[0].supplementary:
|
|
295
|
+
continue
|
|
296
|
+
elif (r[1] is not None) and r[1].supplementary:
|
|
297
|
+
continue
|
|
298
|
+
try:
|
|
299
|
+
if ((r[0] is not None and r[0].optional_field("NH") > 1) or
|
|
300
|
+
(r[1] is not None and r[1].optional_field("NH") > 1)):
|
|
301
|
+
write_to_samout(
|
|
302
|
+
r, "__alignment_not_unique", samoutfile,
|
|
303
|
+
template)
|
|
304
|
+
counts[cb][ub]['__alignment_not_unique'] += 1
|
|
305
|
+
if multimapped_mode == 'none':
|
|
306
|
+
continue
|
|
307
|
+
except KeyError:
|
|
308
|
+
pass
|
|
309
|
+
if ((r[0] and r[0].aQual < minaqual) or
|
|
310
|
+
(r[1] and r[1].aQual < minaqual)):
|
|
311
|
+
write_to_samout(
|
|
312
|
+
r, "__too_low_aQual", samoutfile,
|
|
313
|
+
template)
|
|
314
|
+
counts[cb][ub]['__too_low_aQual'] += 1
|
|
315
|
+
continue
|
|
316
|
+
|
|
317
|
+
try:
|
|
318
|
+
if overlap_mode == "union":
|
|
319
|
+
fs = set()
|
|
320
|
+
for iv in iv_seq:
|
|
321
|
+
if iv.chrom not in features.chrom_vectors:
|
|
322
|
+
raise UnknownChrom
|
|
323
|
+
for iv2, fs2 in features[iv].steps():
|
|
324
|
+
fs = fs.union(fs2)
|
|
325
|
+
elif overlap_mode in ("intersection-strict",
|
|
326
|
+
"intersection-nonempty"):
|
|
327
|
+
fs = None
|
|
328
|
+
for iv in iv_seq:
|
|
329
|
+
if iv.chrom not in features.chrom_vectors:
|
|
330
|
+
raise UnknownChrom
|
|
331
|
+
for iv2, fs2 in features[iv].steps():
|
|
332
|
+
if ((len(fs2) > 0) or
|
|
333
|
+
(overlap_mode == "intersection-strict")):
|
|
334
|
+
if fs is None:
|
|
335
|
+
fs = fs2.copy()
|
|
336
|
+
else:
|
|
337
|
+
fs = fs.intersection(fs2)
|
|
338
|
+
else:
|
|
339
|
+
sys.exit("Illegal overlap mode.")
|
|
340
|
+
|
|
341
|
+
if fs is None or len(fs) == 0:
|
|
342
|
+
write_to_samout(
|
|
343
|
+
r, "__no_feature", samoutfile,
|
|
344
|
+
template)
|
|
345
|
+
counts[cb][ub]['__no_feature'] += 1
|
|
346
|
+
elif len(fs) > 1:
|
|
347
|
+
write_to_samout(
|
|
348
|
+
r, "__ambiguous[" + '+'.join(fs) + "]",
|
|
349
|
+
samoutfile,
|
|
350
|
+
template)
|
|
351
|
+
counts[cb][ub]['__ambiguous'] += 1
|
|
352
|
+
else:
|
|
353
|
+
write_to_samout(
|
|
354
|
+
r, list(fs)[0], samoutfile,
|
|
355
|
+
template)
|
|
356
|
+
|
|
357
|
+
if fs is not None and len(fs) > 0:
|
|
358
|
+
if multimapped_mode == 'none':
|
|
359
|
+
if len(fs) == 1:
|
|
360
|
+
counts[cb][ub][list(fs)[0]] += 1
|
|
361
|
+
elif multimapped_mode == 'all':
|
|
362
|
+
for fsi in list(fs):
|
|
363
|
+
counts[cb][ub][fsi] += 1
|
|
364
|
+
else:
|
|
365
|
+
sys.exit("Illegal multimap mode.")
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
except UnknownChrom:
|
|
369
|
+
write_to_samout(
|
|
370
|
+
r, "__no_feature", samoutfile,
|
|
371
|
+
template)
|
|
372
|
+
counts[cb][ub]['__no_feature'] += 1
|
|
373
|
+
|
|
374
|
+
except:
|
|
375
|
+
sys.stderr.write(
|
|
376
|
+
"Error occured when processing input (%s):\n" %
|
|
377
|
+
(read_seq_file.get_line_number_string()))
|
|
378
|
+
raise
|
|
379
|
+
|
|
380
|
+
if not quiet:
|
|
381
|
+
sys.stderr.write(
|
|
382
|
+
"%d %s processed.\n" %
|
|
383
|
+
(i, "alignments " if not pe_mode else "alignment pairs"))
|
|
384
|
+
sys.stderr.flush()
|
|
385
|
+
|
|
386
|
+
if samoutfile is not None:
|
|
387
|
+
samoutfile.close()
|
|
388
|
+
|
|
389
|
+
# A UMI could be mapped to more than one feature. We count the feature
|
|
390
|
+
# with the highest number of reads. In case of ties, we discard the whole
|
|
391
|
+
# UMI to be on the safe side (it should not happen anyway).
|
|
392
|
+
cbs = sorted(counts.keys())
|
|
393
|
+
counts_noumi = {}
|
|
394
|
+
for cb in cbs:
|
|
395
|
+
counts_cell = Counter()
|
|
396
|
+
|
|
397
|
+
# Correct barcodes within a certain Hamming distance
|
|
398
|
+
correct_barcodes(counts[cb], hamming=correct_ub_distance)
|
|
399
|
+
|
|
400
|
+
for ub, udic in counts.pop(cb).items():
|
|
401
|
+
# In case of a tie, do not increment either feature
|
|
402
|
+
top = udic.most_common(2)
|
|
403
|
+
if (len(top) == 2) and (top[0][1] == top[1][1]):
|
|
404
|
+
continue
|
|
405
|
+
counts_cell[top[0][0]] += 1
|
|
406
|
+
counts_noumi[cb] = counts_cell
|
|
407
|
+
|
|
408
|
+
return {
|
|
409
|
+
'cell_barcodes': cbs,
|
|
410
|
+
'counts': counts_noumi,
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def count_reads_in_features(
|
|
415
|
+
sam_filename,
|
|
416
|
+
gff_filename,
|
|
417
|
+
order,
|
|
418
|
+
max_buffer_size,
|
|
419
|
+
stranded,
|
|
420
|
+
overlap_mode,
|
|
421
|
+
multimapped_mode,
|
|
422
|
+
secondary_alignment_mode,
|
|
423
|
+
supplementary_alignment_mode,
|
|
424
|
+
feature_type,
|
|
425
|
+
id_attribute,
|
|
426
|
+
additional_attributes,
|
|
427
|
+
add_chromosome_info,
|
|
428
|
+
quiet,
|
|
429
|
+
minaqual,
|
|
430
|
+
samout,
|
|
431
|
+
samout_format,
|
|
432
|
+
output_delimiter,
|
|
433
|
+
output_filename,
|
|
434
|
+
counts_output_sparse,
|
|
435
|
+
cb_tag,
|
|
436
|
+
ub_tag,
|
|
437
|
+
correct_ub_distance,
|
|
438
|
+
):
|
|
439
|
+
'''Count reads in features, parallelizing by file'''
|
|
440
|
+
|
|
441
|
+
if samout is not None:
|
|
442
|
+
# Try to open samout file early in case any of them has issues
|
|
443
|
+
if samout_format in ('SAM', 'sam'):
|
|
444
|
+
with open(samout, 'w'):
|
|
445
|
+
pass
|
|
446
|
+
else:
|
|
447
|
+
# We don't have a template if the input is stdin
|
|
448
|
+
if sam_filename != '-':
|
|
449
|
+
with pysam.AlignmentFile(sam_filename, 'r') as sf:
|
|
450
|
+
with pysam.AlignmentFile(samout, 'w', template=sf):
|
|
451
|
+
pass
|
|
452
|
+
|
|
453
|
+
# Try to open samfiles to fail early in case any of them is not there
|
|
454
|
+
if sam_filename != '-':
|
|
455
|
+
with pysam.AlignmentFile(sam_filename, 'r') as sf:
|
|
456
|
+
pass
|
|
457
|
+
|
|
458
|
+
# Prepare features
|
|
459
|
+
gff = HTSeq.GFF_Reader(gff_filename)
|
|
460
|
+
feature_scan = HTSeq.make_feature_genomicarrayofsets(
|
|
461
|
+
gff,
|
|
462
|
+
id_attribute,
|
|
463
|
+
feature_type=feature_type,
|
|
464
|
+
additional_attributes=additional_attributes,
|
|
465
|
+
stranded=stranded != 'no',
|
|
466
|
+
verbose=not quiet,
|
|
467
|
+
add_chromosome_info=add_chromosome_info,
|
|
468
|
+
)
|
|
469
|
+
features = feature_scan['features']
|
|
470
|
+
attributes = feature_scan['attributes']
|
|
471
|
+
feature_attr = sorted(attributes.keys())
|
|
472
|
+
|
|
473
|
+
if len(feature_attr) == 0:
|
|
474
|
+
sys.stderr.write(
|
|
475
|
+
"Warning: No features of type '%s' found.\n" % feature_type)
|
|
476
|
+
|
|
477
|
+
# Count reads
|
|
478
|
+
results = count_reads_with_barcodes(
|
|
479
|
+
sam_filename,
|
|
480
|
+
features,
|
|
481
|
+
feature_attr,
|
|
482
|
+
order,
|
|
483
|
+
max_buffer_size,
|
|
484
|
+
stranded,
|
|
485
|
+
overlap_mode,
|
|
486
|
+
multimapped_mode,
|
|
487
|
+
secondary_alignment_mode,
|
|
488
|
+
supplementary_alignment_mode,
|
|
489
|
+
feature_type,
|
|
490
|
+
id_attribute,
|
|
491
|
+
additional_attributes,
|
|
492
|
+
quiet,
|
|
493
|
+
minaqual,
|
|
494
|
+
samout_format,
|
|
495
|
+
samout,
|
|
496
|
+
cb_tag,
|
|
497
|
+
ub_tag,
|
|
498
|
+
correct_ub_distance,
|
|
499
|
+
)
|
|
500
|
+
|
|
501
|
+
# Write output
|
|
502
|
+
_write_output(
|
|
503
|
+
results,
|
|
504
|
+
results['cell_barcodes'],
|
|
505
|
+
attributes,
|
|
506
|
+
additional_attributes,
|
|
507
|
+
output_filename,
|
|
508
|
+
output_delimiter,
|
|
509
|
+
False,
|
|
510
|
+
sparse=counts_output_sparse,
|
|
511
|
+
dtype=np.float32,
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def main():
|
|
516
|
+
|
|
517
|
+
pa = argparse.ArgumentParser(
|
|
518
|
+
add_help=False,
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
pa.add_argument(
|
|
522
|
+
"--version", action="store_true",
|
|
523
|
+
help='Show software version and exit')
|
|
524
|
+
args, argv = pa.parse_known_args()
|
|
525
|
+
|
|
526
|
+
# Version is the only case where the BAM and GTF files are optional
|
|
527
|
+
if args.version:
|
|
528
|
+
print(HTSeq.__version__)
|
|
529
|
+
sys.exit()
|
|
530
|
+
|
|
531
|
+
pa = argparse.ArgumentParser(
|
|
532
|
+
parents=[pa],
|
|
533
|
+
description="This script takes one alignment file in SAM/BAM " +
|
|
534
|
+
"format and a feature file in GFF format and calculates for each feature " +
|
|
535
|
+
"the number of reads mapping to it, accounting for barcodes. See " +
|
|
536
|
+
"http://htseq.readthedocs.io/en/master/count.html for details.",
|
|
537
|
+
epilog="Written by Simon Anders (sanders@fs.tum.de), " +
|
|
538
|
+
"European Molecular Biology Laboratory (EMBL) and Fabio Zanini " +
|
|
539
|
+
"(fabio.zanini@unsw.edu.au), UNSW Sydney. (c) 2010-2020. " +
|
|
540
|
+
"Released under the terms of the GNU General Public License v3. " +
|
|
541
|
+
"Please cite the following paper if you use this script: \n" +
|
|
542
|
+
" G. Putri et al. Analysing high-throughput sequencing data in " +
|
|
543
|
+
"Python with HTSeq 2.0. Bioinformatics (2022). " +
|
|
544
|
+
"https://doi.org/10.1093/bioinformatics/btac166.\n" +
|
|
545
|
+
"Part of the 'HTSeq' framework, version %s." % HTSeq.__version__,
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
pa.add_argument(
|
|
549
|
+
"samfilename", type=str,
|
|
550
|
+
help="Path to the SAM/BAM file containing the barcoded, mapped " +
|
|
551
|
+
"reads. If '-' is selected, read from standard input")
|
|
552
|
+
|
|
553
|
+
pa.add_argument(
|
|
554
|
+
"featuresfilename", type=str,
|
|
555
|
+
help="Path to the GTF file containing the features")
|
|
556
|
+
|
|
557
|
+
pa.add_argument(
|
|
558
|
+
"-f", "--format", dest="samtype",
|
|
559
|
+
choices=("sam", "bam", "auto"), default="auto",
|
|
560
|
+
help="Type of <alignment_file> data. DEPRECATED: " +
|
|
561
|
+
"file format is detected automatically. This option is ignored.")
|
|
562
|
+
|
|
563
|
+
pa.add_argument(
|
|
564
|
+
"-r", "--order", dest="order",
|
|
565
|
+
choices=("pos", "name"), default="name",
|
|
566
|
+
help="'pos' or 'name'. Sorting order of <alignment_file> (default: name). Paired-end sequencing " +
|
|
567
|
+
"data must be sorted either by position or by read name, and the sorting order " +
|
|
568
|
+
"must be specified. Ignored for single-end data.")
|
|
569
|
+
|
|
570
|
+
pa.add_argument(
|
|
571
|
+
"--max-reads-in-buffer", dest="max_buffer_size", type=int,
|
|
572
|
+
default=30000000,
|
|
573
|
+
help="When <alignment_file> is paired end sorted by position, " +
|
|
574
|
+
"allow only so many reads to stay in memory until the mates are " +
|
|
575
|
+
"found (raising this number will use more memory). Has no effect " +
|
|
576
|
+
"for single end or paired end sorted by name")
|
|
577
|
+
|
|
578
|
+
pa.add_argument(
|
|
579
|
+
"-s", "--stranded", dest="stranded",
|
|
580
|
+
choices=("yes", "no", "reverse"), default="yes",
|
|
581
|
+
help="Whether the data is from a strand-specific assay. Specify 'yes', " +
|
|
582
|
+
"'no', or 'reverse' (default: yes). " +
|
|
583
|
+
"'reverse' means 'yes' with reversed strand interpretation")
|
|
584
|
+
|
|
585
|
+
pa.add_argument(
|
|
586
|
+
"-a", "--minaqual", type=int, dest="minaqual",
|
|
587
|
+
default=10,
|
|
588
|
+
help="Skip all reads with MAPQ alignment quality lower than the given " +
|
|
589
|
+
"minimum value (default: 10). MAPQ is the 5th column of a SAM/BAM " +
|
|
590
|
+
"file and its usage depends on the software used to map the reads.")
|
|
591
|
+
|
|
592
|
+
pa.add_argument(
|
|
593
|
+
"-t", "--type", type=str, dest="featuretype",
|
|
594
|
+
action="append", default=["exon"],
|
|
595
|
+
help="Feature type (3rd column in GTF file) to be used, all "
|
|
596
|
+
+ "features of other type are ignored (default, suitable for"
|
|
597
|
+
+ "Ensembl GTF files: exon). You can call this option multiple "
|
|
598
|
+
+ "times. Features of all specified types will be included. "
|
|
599
|
+
+ "E.g. to include both genes and pseudogenes you might use "
|
|
600
|
+
+ "-t gene -t pseudogene")
|
|
601
|
+
|
|
602
|
+
pa.add_argument(
|
|
603
|
+
"-i", "--idattr", type=str, dest="idattr",
|
|
604
|
+
default="gene_id",
|
|
605
|
+
help="GTF attribute to be used as feature ID (default, " +
|
|
606
|
+
"suitable for Ensembl GTF files: gene_id)")
|
|
607
|
+
|
|
608
|
+
pa.add_argument(
|
|
609
|
+
"--additional-attr", type=str,
|
|
610
|
+
action='append',
|
|
611
|
+
default=[],
|
|
612
|
+
help="Additional feature attributes (default: none, " +
|
|
613
|
+
"suitable for Ensembl GTF files: gene_name). Use multiple times " +
|
|
614
|
+
"for each different attribute")
|
|
615
|
+
|
|
616
|
+
pa.add_argument(
|
|
617
|
+
"--add-chromosome-info", action='store_true',
|
|
618
|
+
help="Store information about the chromosome of each feature as " +
|
|
619
|
+
"an additional attribute (e.g. colunm in the TSV output file).",
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
pa.add_argument(
|
|
623
|
+
"-m", "--mode", dest="mode",
|
|
624
|
+
choices=("union", "intersection-strict", "intersection-nonempty"),
|
|
625
|
+
default="union",
|
|
626
|
+
help="Mode to handle reads overlapping more than one feature " +
|
|
627
|
+
"(choices: union, intersection-strict, intersection-nonempty; default: union)")
|
|
628
|
+
|
|
629
|
+
pa.add_argument(
|
|
630
|
+
"--nonunique", dest="nonunique", type=str,
|
|
631
|
+
choices=("none", "all"), default="none",
|
|
632
|
+
help="Whether to score reads that are not uniquely aligned " +
|
|
633
|
+
"or ambiguously assigned to features")
|
|
634
|
+
|
|
635
|
+
pa.add_argument(
|
|
636
|
+
"--secondary-alignments", dest="secondary_alignments", type=str,
|
|
637
|
+
choices=("score", "ignore"), default="ignore",
|
|
638
|
+
help="Whether to score secondary alignments (0x100 flag)")
|
|
639
|
+
|
|
640
|
+
pa.add_argument(
|
|
641
|
+
"--supplementary-alignments", dest="supplementary_alignments", type=str,
|
|
642
|
+
choices=("score", "ignore"), default="ignore",
|
|
643
|
+
help="Whether to score supplementary alignments (0x800 flag)")
|
|
644
|
+
|
|
645
|
+
pa.add_argument(
|
|
646
|
+
"-o", "--samout", type=str, dest="samout",
|
|
647
|
+
default=None,
|
|
648
|
+
help="Write out all SAM alignment records into a" +
|
|
649
|
+
"SAM/BAM file, annotating each line " +
|
|
650
|
+
"with its feature assignment (as an optional field with tag 'XF')" +
|
|
651
|
+
". See the -p option to use BAM instead of SAM.")
|
|
652
|
+
|
|
653
|
+
pa.add_argument(
|
|
654
|
+
"-p", '--samout-format', type=str, dest='samout_format',
|
|
655
|
+
choices=('SAM', 'BAM', 'sam', 'bam'), default='SAM',
|
|
656
|
+
help="Format to use with the --samout option."
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
pa.add_argument(
|
|
660
|
+
"-d", '--delimiter', type=str, dest='output_delimiter',
|
|
661
|
+
default='\t',
|
|
662
|
+
help="Column delimiter in output (default: TAB)."
|
|
663
|
+
)
|
|
664
|
+
pa.add_argument(
|
|
665
|
+
"-c", '--counts_output', type=str, dest='output_filename',
|
|
666
|
+
default='',
|
|
667
|
+
help="TSV/CSV filename to output the counts to instead of stdout."
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
pa.add_argument(
|
|
671
|
+
"--counts_output_sparse", action='store_true',
|
|
672
|
+
help="Store the counts as a sparse matrix (mtx, h5ad, loom)."
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
pa.add_argument(
|
|
676
|
+
'--cell-barcode', type=str, dest='cb_tag',
|
|
677
|
+
default='CB',
|
|
678
|
+
help='BAM tag used for the cell barcode (default compatible ' +
|
|
679
|
+
'with 10X Genomics Chromium is CB).',
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
pa.add_argument(
|
|
683
|
+
'--UMI', type=str, dest='ub_tag',
|
|
684
|
+
default='UB',
|
|
685
|
+
help='BAM tag used for the unique molecular identifier, also ' +
|
|
686
|
+
'known as molecular barcode (default compatible ' +
|
|
687
|
+
'with 10X Genomics Chromium is UB).',
|
|
688
|
+
)
|
|
689
|
+
|
|
690
|
+
pa.add_argument(
|
|
691
|
+
'--correct-UMI-distance',
|
|
692
|
+
type=int,
|
|
693
|
+
choices=[0, 1, 2],
|
|
694
|
+
dest='correct_ub_distance',
|
|
695
|
+
default=0,
|
|
696
|
+
help='Correct for sequencing errors in the UMI tag, based on ' +
|
|
697
|
+
'Hamming distance. For each UMI, if another UMI with more reads ' +
|
|
698
|
+
'within 1 or 2 mutations is found, merge this UMI\'s reads into ' +
|
|
699
|
+
'the more popular one. The default is to not correct UMIs.',
|
|
700
|
+
)
|
|
701
|
+
|
|
702
|
+
pa.add_argument(
|
|
703
|
+
"-q", "--quiet", action="store_true", dest="quiet",
|
|
704
|
+
help="Suppress progress report") # and warnings" )
|
|
705
|
+
|
|
706
|
+
args = pa.parse_args()
|
|
707
|
+
|
|
708
|
+
warnings.showwarning = my_showwarning
|
|
709
|
+
try:
|
|
710
|
+
count_reads_in_features(
|
|
711
|
+
args.samfilename,
|
|
712
|
+
args.featuresfilename,
|
|
713
|
+
args.order,
|
|
714
|
+
args.max_buffer_size,
|
|
715
|
+
args.stranded,
|
|
716
|
+
args.mode,
|
|
717
|
+
args.nonunique,
|
|
718
|
+
args.secondary_alignments,
|
|
719
|
+
args.supplementary_alignments,
|
|
720
|
+
args.featuretype,
|
|
721
|
+
args.idattr,
|
|
722
|
+
args.additional_attr,
|
|
723
|
+
args.add_chromosome_info,
|
|
724
|
+
args.quiet,
|
|
725
|
+
args.minaqual,
|
|
726
|
+
args.samout,
|
|
727
|
+
args.samout_format,
|
|
728
|
+
args.output_delimiter,
|
|
729
|
+
args.output_filename,
|
|
730
|
+
args.counts_output_sparse,
|
|
731
|
+
args.cb_tag,
|
|
732
|
+
args.ub_tag,
|
|
733
|
+
args.correct_ub_distance,
|
|
734
|
+
)
|
|
735
|
+
except:
|
|
736
|
+
sys.stderr.write(" %s\n" % str(sys.exc_info()[1]))
|
|
737
|
+
sys.stderr.write(" [Exception type: %s, raised in %s:%d]\n" %
|
|
738
|
+
(sys.exc_info()[1].__class__.__name__,
|
|
739
|
+
os.path.basename(traceback.extract_tb(
|
|
740
|
+
sys.exc_info()[2])[-1][0]),
|
|
741
|
+
traceback.extract_tb(sys.exc_info()[2])[-1][1]))
|
|
742
|
+
sys.exit(1)
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
if __name__ == "__main__":
|
|
746
|
+
main()
|