xengsort 2.0.9__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xengsort-2.0.9/xengsort.egg-info → xengsort-2.1.0}/PKG-INFO +16 -1
- {xengsort-2.0.9 → xengsort-2.1.0}/README.md +15 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/pyproject.toml +1 -1
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/cptasks_kmers.py +40 -7
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/dnaencode.py +8 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/dnaencode_fast.py +4 -4
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/fastcash_main.py +7 -13
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/hash_s3c_fbcbvb.py +16 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/fastqio.py +32 -14
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/generalio.py +23 -11
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/hashio.py +12 -4
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/kmers.py +123 -1
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/bitarray.py +61 -7
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/debug.py +2 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/llvm.py +179 -2
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/srhash.py +91 -4
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/config/classify.yaml +16 -1
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_classify.py +105 -7
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_index.py +5 -5
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_main.py +4 -0
- {xengsort-2.0.9 → xengsort-2.1.0/xengsort.egg-info}/PKG-INFO +16 -1
- {xengsort-2.0.9 → xengsort-2.1.0}/LICENSE +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/MANIFEST.in +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/setup.cfg +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/tests/test_trimmed_reads.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/fastcash_info.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/fastcash_optimize.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/hash_minimizer.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/hash_srh_fbcbvb.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/binaryio.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/fastaio.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/seqio.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/textio.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/xorio.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/aligned_arrays.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/conpro.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/intbitarray.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/libc.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/numbautils.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/packedarray.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/mask.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/mathutils.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/parameters.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/subtable_hashfunctions.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/values/xenograft.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/config/index.yaml +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_info.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_load.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_remove.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_weak.py +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/SOURCES.txt +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/dependency_links.txt +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/entry_points.txt +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/requires.txt +0 -0
- {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: xengsort
|
|
3
|
-
Version: 2.0
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: xenograft sorting of FASTQ reads into five categories: host, graft, both, neither, ambiguous.
|
|
5
5
|
Author-email: Jens Zentgraf <zentgraf@cs.uni-saarland.de>, Sven Rahmann <rahmann@cs.uni-saarland.de>
|
|
6
6
|
License: MIT License
|
|
@@ -287,6 +287,21 @@ Further parameters and options are:
|
|
|
287
287
|
- `--filter`: With this flag, only the graft reads are output.
|
|
288
288
|
- `--count`: With this flag, only the number of reads in each category is counted, but the reads are not sorted.
|
|
289
289
|
|
|
290
|
+
### How to filter bisulfite sequencing data
|
|
291
|
+
In bisulfite sequencing data we have a conversion from C to T and from G to A in the reverse complement. The standard xengsort index classifies most of the reads as `neither`.
|
|
292
|
+
To address this issue, we have created a `--bisulfite` mode.
|
|
293
|
+
This parameter must be set during the index building step.
|
|
294
|
+
If set, all Cs are translated to Ts in the reference and add the $k$-mers are added to the index, the same for G to A.
|
|
295
|
+
This nearly doubles the number of $k$-mers in the index (`-n`)
|
|
296
|
+
|
|
297
|
+
```
|
|
298
|
+
xengsort index --index myindex -H host.fa.gz -G graft.fa.gz -n 9_000_000_000 --bisulfite [OPTIONS]
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
If an index is build using the bisulfite parameter, also the read is converted in the classification step.
|
|
302
|
+
It is not necessary to change the parameters of the classification step.
|
|
303
|
+
Currently, it only supports the count-based classification.
|
|
304
|
+
|
|
290
305
|
|
|
291
306
|
Please report a [GitLab issue](https://gitlab.com/genomeinformatics/xengsort/-/issues) in case of problems.
|
|
292
307
|
|
|
@@ -240,6 +240,21 @@ Further parameters and options are:
|
|
|
240
240
|
- `--filter`: With this flag, only the graft reads are output.
|
|
241
241
|
- `--count`: With this flag, only the number of reads in each category is counted, but the reads are not sorted.
|
|
242
242
|
|
|
243
|
+
### How to filter bisulfite sequencing data
|
|
244
|
+
In bisulfite sequencing data we have a conversion from C to T and from G to A in the reverse complement. The standard xengsort index classifies most of the reads as `neither`.
|
|
245
|
+
To address this issue, we have created a `--bisulfite` mode.
|
|
246
|
+
This parameter must be set during the index building step.
|
|
247
|
+
If set, all Cs are translated to Ts in the reference and add the $k$-mers are added to the index, the same for G to A.
|
|
248
|
+
This nearly doubles the number of $k$-mers in the index (`-n`)
|
|
249
|
+
|
|
250
|
+
```
|
|
251
|
+
xengsort index --index myindex -H host.fa.gz -G graft.fa.gz -n 9_000_000_000 --bisulfite [OPTIONS]
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
If an index is build using the bisulfite parameter, also the read is converted in the classification step.
|
|
255
|
+
It is not necessary to change the parameters of the classification step.
|
|
256
|
+
Currently, it only supports the count-based classification.
|
|
257
|
+
|
|
243
258
|
|
|
244
259
|
Please report a [GitLab issue](https://gitlab.com/genomeinformatics/xengsort/-/issues) in case of problems.
|
|
245
260
|
|
|
@@ -4,9 +4,10 @@ Functions for compiling ConsumerProducerTasks for k-mers
|
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
import numpy as np
|
|
7
|
-
from numba import njit, int32, int64, uint64
|
|
7
|
+
from numba import njit, uint8, int32, int64, uint64
|
|
8
8
|
|
|
9
9
|
from .dnaencode import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_old
|
|
10
|
+
from .dnaencode import _TABLE_DNA_TO_2BITS, compile_quick_dna_to_2bits
|
|
10
11
|
from .dnaencode import quick_dna_to_2bits as quick_dna_to_2bits_old
|
|
11
12
|
from .dnaencode_fast import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_new
|
|
12
13
|
from .dnaencode_fast import quick_dna_to_2bits as quick_dna_to_2bits_new
|
|
@@ -30,7 +31,9 @@ compile_cptask_scatter_kmers_from_buffers():
|
|
|
30
31
|
|
|
31
32
|
def compile_cptask_scatter_kmers_from_linemarked(
|
|
32
33
|
mask, rcmode, hf0,
|
|
33
|
-
nsubtables, noutbuffers_per_subtable, outbufsize,
|
|
34
|
+
nsubtables, noutbuffers_per_subtable, outbufsize,
|
|
35
|
+
new_encoding=False,
|
|
36
|
+
bisulfite=False):
|
|
34
37
|
"""
|
|
35
38
|
Compile a cptask function that
|
|
36
39
|
creates (gapped) k-mers from linemarked buffers.
|
|
@@ -40,10 +43,26 @@ def compile_cptask_scatter_kmers_from_linemarked(
|
|
|
40
43
|
- nsubtables: the number of subtables
|
|
41
44
|
- noutbuffers_per_subtable: number of output buffers per subtable
|
|
42
45
|
- outbufsize: the output buffer size for which to compile this function
|
|
46
|
+
-
|
|
43
47
|
"""
|
|
44
48
|
debugprint0, debugprint1, debugprint2 = debug.debugprint
|
|
45
49
|
both = (rcmode[0].lower() == "b")
|
|
46
|
-
|
|
50
|
+
if bisulfite:
|
|
51
|
+
if new_encoding:
|
|
52
|
+
raise NotImplementedError('WGBS with new encoding not supported')
|
|
53
|
+
|
|
54
|
+
else:
|
|
55
|
+
ct_table = _TABLE_DNA_TO_2BITS.copy()
|
|
56
|
+
ct_table[99] = 3
|
|
57
|
+
ct_table[67] = 3
|
|
58
|
+
quick_dna_to_2bits_ct = compile_quick_dna_to_2bits(ct_table)
|
|
59
|
+
|
|
60
|
+
ga_table = _TABLE_DNA_TO_2BITS.copy()
|
|
61
|
+
ga_table[103] = 0
|
|
62
|
+
ga_table[71] = 0
|
|
63
|
+
quick_dna_to_2bits_ga = compile_quick_dna_to_2bits(ga_table)
|
|
64
|
+
else:
|
|
65
|
+
quick_dna_to_2bits = quick_dna_to_2bits_new if new_encoding else quick_dna_to_2bits_old
|
|
47
66
|
|
|
48
67
|
k, w, tpl = mask.k, mask.w, mask.tuple
|
|
49
68
|
assert len(tpl) == k
|
|
@@ -183,6 +202,8 @@ def compile_cptask_scatter_kmers_from_linemarked(
|
|
|
183
202
|
nactive = -1 # active input buffer
|
|
184
203
|
outidx = np.full(nsubtables, (-1), dtype=np.int32)
|
|
185
204
|
outpos = np.full(nsubtables, outbufsize, dtype=np.int32)
|
|
205
|
+
if bisulfite:
|
|
206
|
+
seq_buffer = np.empty(inbuffers[0].size, dtype=np.uint8)
|
|
186
207
|
while True:
|
|
187
208
|
nactive, wait = find_buffer_for_reading(incontrol, nactive)
|
|
188
209
|
wait_read += wait
|
|
@@ -196,10 +217,22 @@ def compile_cptask_scatter_kmers_from_linemarked(
|
|
|
196
217
|
errorcode = 0
|
|
197
218
|
linemarks = linemarkbuffers[nactive]
|
|
198
219
|
for i in range(nseqs):
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
220
|
+
if bisulfite:
|
|
221
|
+
seq_len = linemarks[i, 1] - linemarks[i, 0]
|
|
222
|
+
seq_buffer[0:seq_len] = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
|
|
223
|
+
quick_dna_to_2bits_ct(seq_buffer[0:seq_len])
|
|
224
|
+
(wait, _errorcode) = _process(
|
|
225
|
+
seq_buffer[:seq_len], outidx, outpos, outbuffers, outcontrol)
|
|
226
|
+
|
|
227
|
+
seq_buffer[0:seq_len] = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
|
|
228
|
+
quick_dna_to_2bits_ga(seq_buffer[0:seq_len])
|
|
229
|
+
(wait, _errorcode) = _process(
|
|
230
|
+
seq_buffer[:seq_len], outidx, outpos, outbuffers, outcontrol)
|
|
231
|
+
else:
|
|
232
|
+
sq = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
|
|
233
|
+
quick_dna_to_2bits(sq)
|
|
234
|
+
(wait, _errorcode) = _process(
|
|
235
|
+
sq, outidx, outpos, outbuffers, outcontrol)
|
|
203
236
|
wait_write += wait
|
|
204
237
|
errorcode |= _errorcode
|
|
205
238
|
if errorcode:
|
|
@@ -34,6 +34,14 @@ def _dna_to_2bits(x, table):
|
|
|
34
34
|
x[i] = table[x[i]]
|
|
35
35
|
|
|
36
36
|
|
|
37
|
+
def compile_quick_dna_to_2bits(table):
|
|
38
|
+
@njit(nogil=True)
|
|
39
|
+
def _quick_dna_to_2bits(x):
|
|
40
|
+
for i in range(len(x)):
|
|
41
|
+
x[i] = table[x[i]]
|
|
42
|
+
|
|
43
|
+
return _quick_dna_to_2bits
|
|
44
|
+
|
|
37
45
|
# this is the one to use!
|
|
38
46
|
@njit(nogil=True)
|
|
39
47
|
def quick_dna_to_2bits(x):
|
|
@@ -10,13 +10,13 @@ from .mask import Mask
|
|
|
10
10
|
U64_MINUSONE = uint64(np.iinfo(np.uint64).max)
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
@njit
|
|
13
|
+
@njit(nogil=True)
|
|
14
14
|
def quick_dna_to_2bits(x):
|
|
15
15
|
for i in range(len(x)):
|
|
16
16
|
x[i] = (x[i] >> 1) & 7
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
@njit
|
|
19
|
+
@njit(nogil=True)
|
|
20
20
|
def dna_to_2bits(x, y):
|
|
21
21
|
for i in range(len(x)):
|
|
22
22
|
y[i] = (x[i] >> 1) & 7
|
|
@@ -63,7 +63,7 @@ def compile_revcomp_and_canonical_code(q, rcmode):
|
|
|
63
63
|
mask = uint64(int(q * '10', 2))
|
|
64
64
|
shift = uint64(64 - 2 * q)
|
|
65
65
|
|
|
66
|
-
@njit(locals=dict(value=uint64))
|
|
66
|
+
@njit(nogil=True, locals=dict(value=uint64))
|
|
67
67
|
def reverse_in_pairs(value):
|
|
68
68
|
value = ((value & 0x3333333333333333) << 2) | ((value & 0xCCCCCCCCCCCCCCCC) >> 2)
|
|
69
69
|
value = ((value & 0x0F0F0F0F0F0F0F0F) << 4) | ((value >> 4) & 0x0F0F0F0F0F0F0F0F)
|
|
@@ -72,7 +72,7 @@ def compile_revcomp_and_canonical_code(q, rcmode):
|
|
|
72
72
|
value = (value << 32) | (value >> 32)
|
|
73
73
|
return value
|
|
74
74
|
|
|
75
|
-
@njit(locals=dict(value=uint64))
|
|
75
|
+
@njit(nogil=True, locals=dict(value=uint64))
|
|
76
76
|
def _rc(value):
|
|
77
77
|
value ^= mask
|
|
78
78
|
value = reverse_in_pairs(value)
|
|
@@ -204,38 +204,32 @@ def get_argument_parser():
|
|
|
204
204
|
"get information about a hash table; export its (filtered) data",
|
|
205
205
|
info,
|
|
206
206
|
"fastcash_info", "main",
|
|
207
|
-
),
|
|
208
|
-
|
|
209
|
-
"export",
|
|
207
|
+
), (
|
|
208
|
+
"export", # this is a synonym for info. Both commands do both.
|
|
210
209
|
"get information about a hash table; export its (filtered) data",
|
|
211
210
|
info,
|
|
212
211
|
"fastcash_info", "main",
|
|
213
|
-
),
|
|
214
|
-
(
|
|
212
|
+
), (
|
|
215
213
|
"lookup",
|
|
216
214
|
"look up the values of a few k-mers in the given hash table",
|
|
217
215
|
lookup,
|
|
218
216
|
"fastcash_lookup", "main",
|
|
219
|
-
),
|
|
220
|
-
(
|
|
217
|
+
), (
|
|
221
218
|
"optimize",
|
|
222
219
|
"optimize the assignment of an existing hash table from exported data",
|
|
223
220
|
optimize,
|
|
224
221
|
"fastcash_optimize", "main",
|
|
225
|
-
),
|
|
226
|
-
(
|
|
222
|
+
), (
|
|
227
223
|
"deploy",
|
|
228
224
|
"deploy a fastcash-based tool (like xengsort, hackgap, etc.)",
|
|
229
225
|
deploy,
|
|
230
226
|
"fastcash_deploy", "main",
|
|
231
|
-
),
|
|
232
|
-
(
|
|
227
|
+
), (
|
|
233
228
|
"mphf",
|
|
234
229
|
"compute a minimal perfect hash function on a key set",
|
|
235
230
|
mphf,
|
|
236
231
|
"fastcash_mphf", "main",
|
|
237
|
-
),
|
|
238
|
-
(
|
|
232
|
+
), (
|
|
239
233
|
"xorfilter",
|
|
240
234
|
"build a XOR filter from a hash table",
|
|
241
235
|
xorfilter,
|
|
@@ -135,6 +135,22 @@ def build_hash(universe, n, subtables, bucketsize,
|
|
|
135
135
|
startbit = subtable * subtablebits + bucket * bucketsizebits
|
|
136
136
|
hprefetch(table, startbit)
|
|
137
137
|
|
|
138
|
+
@njit(nogil=True, locals=dict(
|
|
139
|
+
bucket=int64, startbit=uint64))
|
|
140
|
+
def prefetch_for_key(table, key):
|
|
141
|
+
st, sk = get_subtable_subkey_from_key(key)
|
|
142
|
+
bucket, _ = get_bs1(sk)
|
|
143
|
+
startbit = st * subtablebits + bucket * bucketsizebits
|
|
144
|
+
hprefetch(table, startbit)
|
|
145
|
+
|
|
146
|
+
bucket, _ = get_bs2(sk)
|
|
147
|
+
startbit = st * subtablebits + bucket * bucketsizebits
|
|
148
|
+
hprefetch(table, startbit)
|
|
149
|
+
|
|
150
|
+
bucket, _ = get_bs3(sk)
|
|
151
|
+
startbit = st * subtablebits + bucket * bucketsizebits
|
|
152
|
+
hprefetch(table, startbit)
|
|
153
|
+
|
|
138
154
|
# Define private low-level hash table accssor methods
|
|
139
155
|
@njit(nogil=True, locals=dict(
|
|
140
156
|
bucket=int64, startbit=int64, v=uint64))
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import io
|
|
2
1
|
from argparse import ArgumentParser
|
|
3
2
|
|
|
4
3
|
import numpy as np
|
|
@@ -7,9 +6,11 @@ from numba import njit, int32, int64
|
|
|
7
6
|
from .seqio import FormatError, _universal_reads
|
|
8
7
|
from ..lowlevel.libc import read_block
|
|
9
8
|
from ..lowlevel.conpro import \
|
|
10
|
-
find_buffer_for_writing
|
|
9
|
+
find_buffer_for_writing, \
|
|
11
10
|
mark_buffer_for_reading, mark_buffer_for_writing
|
|
11
|
+
from ..lowlevel import debug
|
|
12
12
|
|
|
13
|
+
debugprint0, debugprint1, debugprint2 = debug.debugprint
|
|
13
14
|
|
|
14
15
|
# Python FASTQ/gz handling ######################################
|
|
15
16
|
|
|
@@ -33,7 +34,7 @@ def fastq_reads(files, sequences_only=False, dirty=False):
|
|
|
33
34
|
func = _fastq_seqs_from_filelike
|
|
34
35
|
if dirty:
|
|
35
36
|
func = _fastq_seqs_dirty_from_filelike
|
|
36
|
-
if
|
|
37
|
+
if isinstance(files, list) or isinstance(files, tuple):
|
|
37
38
|
# multiple files
|
|
38
39
|
for f in files:
|
|
39
40
|
yield from _universal_reads(f, func)
|
|
@@ -137,6 +138,9 @@ def cptask_read_fastq_into_linemarked_buffers(fd, inbuffers, incontrol, ininfos,
|
|
|
137
138
|
if ntodo <= 0:
|
|
138
139
|
if ntodo < 0 or offset == 0:
|
|
139
140
|
errorcode = ntodo
|
|
141
|
+
if errorcode == -99_999:
|
|
142
|
+
debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
|
|
143
|
+
debugprint0("- Please increase buffersize using --buffersize.")
|
|
140
144
|
break
|
|
141
145
|
ntodo = offset
|
|
142
146
|
nseqs, nxt = _find_fastq_seqmarks(active_buffer[:ntodo], linemarkbuffers[nactive])
|
|
@@ -165,6 +169,7 @@ def cptask_read_pairedfastq_into_linemarked_buffers(fd, inbuffers, incontrol, in
|
|
|
165
169
|
linemarkbuffers = outinfos.reshape(M, N // 4, 4)
|
|
166
170
|
|
|
167
171
|
offset1 = offset2 = ntodo1 = ntodo2 = wait_read = wait_write = 0
|
|
172
|
+
nseqs1, nseqs2 = 0, 0
|
|
168
173
|
nactive = -1
|
|
169
174
|
active_buffer = outbuffers[0] # irrelevant
|
|
170
175
|
buffersize = active_buffer.shape[0] // 2
|
|
@@ -200,25 +205,37 @@ def cptask_read_pairedfastq_into_linemarked_buffers(fd, inbuffers, incontrol, in
|
|
|
200
205
|
break
|
|
201
206
|
|
|
202
207
|
# only read if the buffer is filled less than 80%
|
|
203
|
-
if offset1 < active_buffer1.size * 4 // 5:
|
|
208
|
+
if offset1 < active_buffer1.size * 4 // 5 or nseqs1 == 0:
|
|
204
209
|
inner_offset1 = offset1
|
|
205
|
-
while inner_offset1 < active_buffer1.size * 4 // 5:
|
|
210
|
+
while inner_offset1 < active_buffer1.size * 4 // 5 or nseqs1 == 0:
|
|
206
211
|
ntodo1 = read_block(int32(fd1), active_buffer1, inner_offset1)
|
|
207
212
|
if ntodo1 == 0:
|
|
208
213
|
ntodo1 = inner_offset1
|
|
209
214
|
break
|
|
215
|
+
if ntodo1 == -99_999:
|
|
216
|
+
debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
|
|
217
|
+
debugprint0("- Please increase buffersize using --buffersize.")
|
|
218
|
+
break
|
|
210
219
|
inner_offset1 = ntodo1
|
|
220
|
+
if inner_offset1 >= active_buffer1.size * 4 // 5:
|
|
221
|
+
break
|
|
211
222
|
else:
|
|
212
223
|
ntodo1 = offset1
|
|
213
224
|
|
|
214
|
-
if offset2 < active_buffer2.size * 4 // 5:
|
|
225
|
+
if offset2 < active_buffer2.size * 4 // 5 or nseqs2 == 0:
|
|
215
226
|
inner_offset2 = offset2
|
|
216
|
-
while inner_offset2 < active_buffer2.size * 4 // 5:
|
|
227
|
+
while inner_offset2 < active_buffer2.size * 4 // 5 or nseqs2 == 0:
|
|
217
228
|
ntodo2 = read_block(int32(fd2), active_buffer2, inner_offset2)
|
|
218
229
|
if ntodo2 == 0:
|
|
219
230
|
ntodo2 = inner_offset2
|
|
220
231
|
break
|
|
232
|
+
if ntodo2 == -99_999:
|
|
233
|
+
debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
|
|
234
|
+
debugprint0("- Please increase buffersize using --buffersize.")
|
|
235
|
+
break
|
|
221
236
|
inner_offset2 = ntodo2
|
|
237
|
+
if inner_offset2 >= active_buffer2.size * 4 // 5:
|
|
238
|
+
break
|
|
222
239
|
else:
|
|
223
240
|
ntodo2 = offset2
|
|
224
241
|
|
|
@@ -320,18 +337,18 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
|
|
|
320
337
|
"""
|
|
321
338
|
Find start and end positions of lines in byte buffer 'buf'.
|
|
322
339
|
Store the information in 'linemarks', such that
|
|
323
|
-
linemarks[i, 0:4] has information about record number i.
|
|
324
|
-
If we need to store the sequence id and the position in the read, linemarks has
|
|
340
|
+
linemarks[i, 0:4] (int32) has information about record number i.
|
|
341
|
+
If we need to store the sequence id and the position in the read, linemarks has 6 entries.
|
|
325
342
|
|
|
326
343
|
Return pair (m, nxt), where:
|
|
327
344
|
m: number of sequences processed
|
|
328
345
|
nxt: position at which to continue processing
|
|
329
346
|
in the next iteration (start of new entry, '@')
|
|
347
|
+
For sequence i, we have
|
|
330
348
|
linemarks[i, 0] = start of sequence line
|
|
331
349
|
linemarks[i, 1] = end of sequence line
|
|
332
350
|
linemarks[i, 2] = start of record (all lines)
|
|
333
351
|
linemarks[i, 3] = end of record (all lines)
|
|
334
|
-
---------------------------------------------
|
|
335
352
|
linemarks[i, 4] = sequence id
|
|
336
353
|
linemarks[i, 5] = position in the sequence (currently always 0 since we expect that a sequence fits in a buffer)
|
|
337
354
|
"""
|
|
@@ -346,6 +363,7 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
|
|
|
346
363
|
# find start of current line
|
|
347
364
|
while True:
|
|
348
365
|
if buf[i] == 10:
|
|
366
|
+
# assert m >= 0 # DEBUG 28.Aug.2025
|
|
349
367
|
i += 1
|
|
350
368
|
if line == 0:
|
|
351
369
|
linemarks[m, 3] = i
|
|
@@ -354,7 +372,7 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
|
|
|
354
372
|
linemarks[m, 5] = 0
|
|
355
373
|
sequence_id += 1
|
|
356
374
|
if i >= n:
|
|
357
|
-
# we did not find valid record start
|
|
375
|
+
# we did not find a valid record start
|
|
358
376
|
if line == 0:
|
|
359
377
|
m += 1
|
|
360
378
|
nxt = n
|
|
@@ -362,9 +380,9 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
|
|
|
362
380
|
if line == 0:
|
|
363
381
|
m += 1
|
|
364
382
|
nxt = i
|
|
365
|
-
linemarks[m, 2] = i
|
|
366
383
|
if m >= M:
|
|
367
384
|
return (M, nxt)
|
|
385
|
+
linemarks[m, 2] = i
|
|
368
386
|
elif line == 1:
|
|
369
387
|
linemarks[m, 0] = i
|
|
370
388
|
# find end of current line
|
|
@@ -397,7 +415,7 @@ def fastq_chunks(files,
|
|
|
397
415
|
if not (isinstance(files, tuple) or isinstance(files, list)):
|
|
398
416
|
files = (files,) # single file?
|
|
399
417
|
if bufsize / 2**30 > 1.5:
|
|
400
|
-
raise ValueError(f"ERROR: bufsize={bufsize/2**30} GiB must be <= 1.5 GiB (2**30 bytes); use smaller chunksize of fewer threads.")
|
|
418
|
+
raise ValueError(f"ERROR: bufsize={bufsize / 2**30} GiB must be <= 1.5 GiB (2**30 bytes); use smaller chunksize of fewer threads.")
|
|
401
419
|
linemarks = np.empty((maxreads, 4), dtype=np.int32)
|
|
402
420
|
buf = np.empty(bufsize, dtype=np.uint8)
|
|
403
421
|
for file in files:
|
|
@@ -406,7 +424,7 @@ def fastq_chunks(files,
|
|
|
406
424
|
prev = 0
|
|
407
425
|
while True:
|
|
408
426
|
read = f.readinto(buf[prev:])
|
|
409
|
-
if read == 0:
|
|
427
|
+
if read == 0 and prev == 0:
|
|
410
428
|
break
|
|
411
429
|
available = prev + read # number of bytes available
|
|
412
430
|
m, cont = _find_fastq_seqmarks(buf[:available], linemarks)
|
|
@@ -22,16 +22,19 @@ from .binaryio import \
|
|
|
22
22
|
_in_suffixes = {
|
|
23
23
|
".gz": "bgzip -cd",
|
|
24
24
|
".bz2": "bzip2 -cd",
|
|
25
|
-
".xz": "xz -cd"
|
|
25
|
+
".xz": "xz -cd",
|
|
26
|
+
".zst": "zstd -cd",
|
|
26
27
|
}
|
|
27
28
|
_out_suffixes = {
|
|
28
29
|
".gz": "bgzip",
|
|
29
30
|
".bz2": "bzip2",
|
|
30
|
-
".xz": "xz"
|
|
31
|
+
".xz": "xz",
|
|
32
|
+
".zst": "zstd",
|
|
31
33
|
}
|
|
32
|
-
|
|
34
|
+
_threads = {
|
|
33
35
|
"bgzip": "@",
|
|
34
|
-
"xz": "T"
|
|
36
|
+
"xz": "T",
|
|
37
|
+
"zstd": "T",
|
|
35
38
|
}
|
|
36
39
|
_out_compression = defaultdict(str)
|
|
37
40
|
_out_compression['bgzip'] = 'l'
|
|
@@ -41,13 +44,18 @@ _extensions = dict(
|
|
|
41
44
|
fastq=[".fastq", ".fq"],
|
|
42
45
|
vcf=[".vcf"],
|
|
43
46
|
numpy=[".np", ".npy"],
|
|
44
|
-
data=[".data"]
|
|
47
|
+
data=[".data"],
|
|
48
|
+
gff=[".gff3"],
|
|
49
|
+
gtf=[".gtf"],
|
|
50
|
+
txt=[".txt"],
|
|
51
|
+
tsv=[".tsv"],
|
|
52
|
+
mex=[".mex", '.mtx'], # sparse matrix format
|
|
45
53
|
)
|
|
46
54
|
_filetypes = {ex: typ for typ, L in _extensions.items() for ex in L}
|
|
47
55
|
|
|
48
56
|
|
|
49
57
|
class InputFileHandler:
|
|
50
|
-
def __init__(self, filename, bufsize=2**16):
|
|
58
|
+
def __init__(self, filename, bufsize=2**16, compression_threads=1):
|
|
51
59
|
"""
|
|
52
60
|
Context manager class representing an input file or stream.
|
|
53
61
|
|
|
@@ -77,6 +85,7 @@ class InputFileHandler:
|
|
|
77
85
|
if is_compressed:
|
|
78
86
|
if which(tool.split()[0]) is None:
|
|
79
87
|
raise RuntimeError(f"Cannot decompress input file {filename} because {tool} is not installed.")
|
|
88
|
+
tool += f' -{_threads[tool.split()[0]]} {compression_threads}' if tool.split()[0] in _threads and tool.split()[0] != "zstd" else ''
|
|
80
89
|
debugprint2(f"- creating Popen command for {filename} of type {file_type} using '{tool} {filename}'")
|
|
81
90
|
self.cmd = cmdsplit(f"{tool} {filename}")
|
|
82
91
|
else:
|
|
@@ -136,8 +145,11 @@ class OutputFileHandler:
|
|
|
136
145
|
if is_compressed:
|
|
137
146
|
if which(tool) is None:
|
|
138
147
|
raise RuntimeError(f"Cannot compress output file {filename}. {tool} is not installed")
|
|
139
|
-
|
|
140
|
-
|
|
148
|
+
if tool == 'zstd':
|
|
149
|
+
threads = f' -{_threads[tool]}{compression_threads}'
|
|
150
|
+
else:
|
|
151
|
+
threads = f' -{_threads[tool]} {compression_threads}' if tool in _threads else ''
|
|
152
|
+
tool += f' {threads} -{_out_compression[tool]}{compression_level}'
|
|
141
153
|
debugprint2(f"- creating Popen command for {filename} of type {file_type} using '{tool} {filename}'")
|
|
142
154
|
self.cmd = cmdsplit(f"{tool}")
|
|
143
155
|
else:
|
|
@@ -168,7 +180,7 @@ class OutputFileHandler:
|
|
|
168
180
|
self.fd = None
|
|
169
181
|
|
|
170
182
|
|
|
171
|
-
def cptask_read_file(fname, valid_types, w, *allbuffers):
|
|
183
|
+
def cptask_read_file(fname, valid_types, w, *allbuffers, compression_threads=2):
|
|
172
184
|
"""
|
|
173
185
|
ConsumerProducer task that consumes a file and writes sequences into buffers.
|
|
174
186
|
If the file is compressed using any of {gzip/pigz, bzip, xz},
|
|
@@ -183,8 +195,8 @@ def cptask_read_file(fname, valid_types, w, *allbuffers):
|
|
|
183
195
|
paired = True
|
|
184
196
|
fname, fname2 = fname
|
|
185
197
|
|
|
186
|
-
with InputFileHandler(fname) as infile, \
|
|
187
|
-
InputFileHandler(fname2) if paired else nullcontext() as infile2:
|
|
198
|
+
with InputFileHandler(fname, compression_threads=compression_threads) as infile, \
|
|
199
|
+
InputFileHandler(fname2, compression_threads=compression_threads) if paired else nullcontext() as infile2:
|
|
188
200
|
ft = infile.file_type
|
|
189
201
|
if (valid_types is not None) and (ft not in valid_types):
|
|
190
202
|
debugprint0(f"Error: file type '{ft}' not among {valid_types=} for file '{fname}'.")
|
|
@@ -212,10 +212,18 @@ def load_hash(filename, *, shared_memory=False, info_only=False, quiet=False):
|
|
|
212
212
|
assert shm_buf.shape[0] % 8 == 0
|
|
213
213
|
assert shm_buf.itemsize == 1
|
|
214
214
|
init = np.ndarray(shm_buf.shape[0] // 8, dtype=np.uint64, buffer=shm_buf)
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
215
|
+
|
|
216
|
+
if hashtype == 'new':
|
|
217
|
+
choices = hashinfo['choices']
|
|
218
|
+
h = m.build_hash(universe, n, subtables, choices,
|
|
219
|
+
bucketsize, hashfuncs, nvalues, update_value,
|
|
220
|
+
aligned=aligned, nfingerprints=nfingerprints,
|
|
221
|
+
init=init, maxwalk=maxwalk, shm=shm)
|
|
222
|
+
else:
|
|
223
|
+
h = m.build_hash(universe, n, subtables,
|
|
224
|
+
bucketsize, hashfuncs, nvalues, update_value,
|
|
225
|
+
aligned=aligned, nfingerprints=nfingerprints,
|
|
226
|
+
init=init, maxwalk=maxwalk, shortcutbits=shortcutbits, shm=shm)
|
|
219
227
|
if not shared_memory:
|
|
220
228
|
debugprint1(f"- loading '{filename}.hash' into array")
|
|
221
229
|
checked = load_array_into(f"{filename}.hash", h.hashtable, check=checksum)
|
|
@@ -5,11 +5,13 @@ Compiler functions for k-mer processors
|
|
|
5
5
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
from numba import njit, int64, uint64
|
|
8
|
+
from numba import njit, int64, uint64, uint8
|
|
9
|
+
import numpy as np
|
|
9
10
|
|
|
10
11
|
from .dnaencode import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_old
|
|
11
12
|
from .dnaencode_fast import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_new
|
|
12
13
|
from .lowlevel import debug
|
|
14
|
+
from .lowlevel.llvm import compile_popcount, compile_cttz
|
|
13
15
|
|
|
14
16
|
|
|
15
17
|
def compile_kmer_iterator(shp, rcmode="f", new_encoding=False):
|
|
@@ -472,3 +474,123 @@ def compile_positional_translated_kmer_processor(shp, func, rcmode="f", new_enco
|
|
|
472
474
|
pass # all done here
|
|
473
475
|
|
|
474
476
|
return k, processor
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
# Iterative spaced seed hashing (doi: 10.1007/978-3-030-20242-2_18, reversed order compared to paper)
|
|
480
|
+
|
|
481
|
+
def compile_positional_kmer_processor_issh(shp, func, rcmode="f", new_encoding=False):
|
|
482
|
+
"""
|
|
483
|
+
like compile_kmer_processor, but also uses the current k-mer start position
|
|
484
|
+
as an additional argument to func:
|
|
485
|
+
func(hashtable, kmercode, position, *parameters)
|
|
486
|
+
"""
|
|
487
|
+
both = (rcmode == "both")
|
|
488
|
+
k = len(shp)
|
|
489
|
+
if k < 1 or k > 32:
|
|
490
|
+
raise ValueError(f"only 1<=k<=32 is supported, but k={k}.")
|
|
491
|
+
revcomp, ccode = compile_revcomp_and_canonical_code_new(k, rcmode) if new_encoding else compile_revcomp_and_canonical_code_old(k, rcmode)
|
|
492
|
+
popcnt = compile_popcount('uint64')
|
|
493
|
+
cttz = compile_cttz('uint64')
|
|
494
|
+
|
|
495
|
+
weight, width = uint64(len(shp)), uint64(max(shp) + 1)
|
|
496
|
+
sig_mask = 0
|
|
497
|
+
m = np.zeros(width, dtype=np.uint64)
|
|
498
|
+
for i in shp:
|
|
499
|
+
sig_mask |= 1 << (width - i - 1)
|
|
500
|
+
sig_mask = uint64(sig_mask)
|
|
501
|
+
|
|
502
|
+
for i in range(width - 1):
|
|
503
|
+
m[i + 1] = m[i] + (sig_mask >> (width - i - 1) & 1)
|
|
504
|
+
|
|
505
|
+
C = np.zeros((width, width), dtype=np.uint64)
|
|
506
|
+
masks = np.zeros((width, width), dtype=np.uint64)
|
|
507
|
+
for g in range(width):
|
|
508
|
+
for j in range(width - 1):
|
|
509
|
+
for k in shp:
|
|
510
|
+
if k + j < width and (sig_mask >> (width - (k + j) - 1) & 1) == 1 and m[k] == m[k + j] - m[j] + m[g]:
|
|
511
|
+
C[g, j] |= 1 << (width - k - 1)
|
|
512
|
+
masks[g, j] |= (3 << 2 * (weight - m[k + j] - 1))
|
|
513
|
+
|
|
514
|
+
@njit(nogil=True, locals=dict(best=uint64, best_i=uint64, best_j=uint64, use=uint64))
|
|
515
|
+
def find_best_prev(s, to_cover):
|
|
516
|
+
best = 0
|
|
517
|
+
best_i, best_j = 0, 1
|
|
518
|
+
for z in range(0, s - 1):
|
|
519
|
+
for k in range(1, s):
|
|
520
|
+
use = popcnt(to_cover & C[z, k])
|
|
521
|
+
if use > best:
|
|
522
|
+
best = use
|
|
523
|
+
best_i, best_j = z, k
|
|
524
|
+
return best_i, best_j
|
|
525
|
+
|
|
526
|
+
@njit(nogil=True, locals=dict(mask=uint64, n_ones=uint64, positions=uint64, n_ones_new=uint64))
|
|
527
|
+
def compute_shift(g, k):
|
|
528
|
+
mask = masks[g, k]
|
|
529
|
+
n_ones = cttz(mask) // 2
|
|
530
|
+
positions = width - shp[weight - n_ones - 1] - 1 + k
|
|
531
|
+
n_ones_new = popcnt(sig_mask & (2**positions - 1))
|
|
532
|
+
return 2 * (n_ones_new - n_ones)
|
|
533
|
+
|
|
534
|
+
steps = []
|
|
535
|
+
last = 0
|
|
536
|
+
# new position does not need to be covered
|
|
537
|
+
to_cover = sig_mask & ~uint64(1)
|
|
538
|
+
while to_cover != 0:
|
|
539
|
+
g, k = find_best_prev(width - 1, to_cover)
|
|
540
|
+
shift = compute_shift(g, k)
|
|
541
|
+
remove = to_cover & C[g, k]
|
|
542
|
+
to_cover &= ~remove
|
|
543
|
+
steps.append([k, shift, masks[g, k]])
|
|
544
|
+
last = max(last, k)
|
|
545
|
+
steps = np.array(steps, dtype=np.uint64)
|
|
546
|
+
|
|
547
|
+
@njit(nogil=True, locals=dict(code=uint64, shift=uint64, k=uint64, mask=uint64, contained_ns=uint64, c=uint8))
|
|
548
|
+
def issh_processor_fast(ht, seq, start, end, *parameters):
|
|
549
|
+
if (end - start) < (width - 1):
|
|
550
|
+
return
|
|
551
|
+
hashes = np.zeros(last, dtype=np.uint64)
|
|
552
|
+
# compute history
|
|
553
|
+
end_naive = min(start + last, end)
|
|
554
|
+
for i in range(start, end_naive):
|
|
555
|
+
contained_ns = 0
|
|
556
|
+
code = 0
|
|
557
|
+
for j in shp:
|
|
558
|
+
c = seq[i + j]
|
|
559
|
+
contained_ns |= (c > 3)
|
|
560
|
+
code = (code << 2) | min(uint64(c), uint64(3))
|
|
561
|
+
hashes[i] = code
|
|
562
|
+
if not contained_ns:
|
|
563
|
+
if both:
|
|
564
|
+
failed = func(ht, code, i, *parameters)
|
|
565
|
+
failed |= func(ht, revcomp(code), i, *parameters)
|
|
566
|
+
else:
|
|
567
|
+
failed = func(ht, ccode(code), i, *parameters)
|
|
568
|
+
if failed is True:
|
|
569
|
+
return
|
|
570
|
+
|
|
571
|
+
# necessary to check for Ns in first window
|
|
572
|
+
contained_ns = 0
|
|
573
|
+
for i in range(width):
|
|
574
|
+
contained_ns = (contained_ns << 1) | (seq[start + last + i] > 3)
|
|
575
|
+
contained_ns = contained_ns >> 1
|
|
576
|
+
|
|
577
|
+
# compute gapped k-mers iteratively based on previous gapped k-mers
|
|
578
|
+
for i in range(start + last, (end - start) - (width - 1)):
|
|
579
|
+
contained_ns = contained_ns << 1
|
|
580
|
+
code = 0
|
|
581
|
+
for k, shift, mask in steps:
|
|
582
|
+
code |= (hashes[(i - k) % last] & mask) << shift
|
|
583
|
+
c = seq[i + width - 1]
|
|
584
|
+
contained_ns |= c > 3
|
|
585
|
+
code |= min(uint64(c), uint64(3))
|
|
586
|
+
|
|
587
|
+
hashes[i % last] = code
|
|
588
|
+
if not contained_ns & sig_mask:
|
|
589
|
+
if both:
|
|
590
|
+
failed = func(ht, code, i, *parameters)
|
|
591
|
+
failed |= func(ht, revcomp(code), i, *parameters)
|
|
592
|
+
else:
|
|
593
|
+
failed = func(ht, ccode(code), i, *parameters)
|
|
594
|
+
if failed is True:
|
|
595
|
+
return
|
|
596
|
+
return k, issh_processor_fast
|