xengsort 2.0.9__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. {xengsort-2.0.9/xengsort.egg-info → xengsort-2.1.0}/PKG-INFO +16 -1
  2. {xengsort-2.0.9 → xengsort-2.1.0}/README.md +15 -0
  3. {xengsort-2.0.9 → xengsort-2.1.0}/pyproject.toml +1 -1
  4. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/cptasks_kmers.py +40 -7
  5. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/dnaencode.py +8 -0
  6. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/dnaencode_fast.py +4 -4
  7. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/fastcash_main.py +7 -13
  8. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/hash_s3c_fbcbvb.py +16 -0
  9. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/fastqio.py +32 -14
  10. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/generalio.py +23 -11
  11. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/hashio.py +12 -4
  12. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/kmers.py +123 -1
  13. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/bitarray.py +61 -7
  14. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/debug.py +2 -0
  15. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/llvm.py +179 -2
  16. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/srhash.py +91 -4
  17. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/config/classify.yaml +16 -1
  18. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_classify.py +105 -7
  19. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_index.py +5 -5
  20. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_main.py +4 -0
  21. {xengsort-2.0.9 → xengsort-2.1.0/xengsort.egg-info}/PKG-INFO +16 -1
  22. {xengsort-2.0.9 → xengsort-2.1.0}/LICENSE +0 -0
  23. {xengsort-2.0.9 → xengsort-2.1.0}/MANIFEST.in +0 -0
  24. {xengsort-2.0.9 → xengsort-2.1.0}/setup.cfg +0 -0
  25. {xengsort-2.0.9 → xengsort-2.1.0}/tests/test_trimmed_reads.py +0 -0
  26. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/fastcash_info.py +0 -0
  27. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/fastcash_optimize.py +0 -0
  28. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/hash_minimizer.py +0 -0
  29. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/hash_srh_fbcbvb.py +0 -0
  30. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/binaryio.py +0 -0
  31. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/fastaio.py +0 -0
  32. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/seqio.py +0 -0
  33. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/textio.py +0 -0
  34. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/io/xorio.py +0 -0
  35. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/aligned_arrays.py +0 -0
  36. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/conpro.py +0 -0
  37. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/intbitarray.py +0 -0
  38. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/libc.py +0 -0
  39. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/numbautils.py +0 -0
  40. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/lowlevel/packedarray.py +0 -0
  41. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/mask.py +0 -0
  42. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/mathutils.py +0 -0
  43. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/parameters.py +0 -0
  44. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/subtable_hashfunctions.py +0 -0
  45. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/values/xenograft.py +0 -0
  46. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/config/index.yaml +0 -0
  47. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_info.py +0 -0
  48. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_load.py +0 -0
  49. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_remove.py +0 -0
  50. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort/xengsort/xengsort_weak.py +0 -0
  51. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/SOURCES.txt +0 -0
  52. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/dependency_links.txt +0 -0
  53. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/entry_points.txt +0 -0
  54. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/requires.txt +0 -0
  55. {xengsort-2.0.9 → xengsort-2.1.0}/xengsort.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: xengsort
3
- Version: 2.0.9
3
+ Version: 2.1.0
4
4
  Summary: xenograft sorting of FASTQ reads into five categories: host, graft, both, neither, ambiguous.
5
5
  Author-email: Jens Zentgraf <zentgraf@cs.uni-saarland.de>, Sven Rahmann <rahmann@cs.uni-saarland.de>
6
6
  License: MIT License
@@ -287,6 +287,21 @@ Further parameters and options are:
287
287
  - `--filter`: With this flag, only the graft reads are output.
288
288
  - `--count`: With this flag, only the number of reads in each category is counted, but the reads are not sorted.
289
289
 
290
+ ### How to filter bisulfite sequencing data
291
+ In bisulfite sequencing data we have a conversion from C to T and from G to A in the reverse complement. The standard xengsort index classifies most of the reads as `neither`.
292
+ To address this issue, we have created a `--bisulfite` mode.
293
+ This parameter must be set during the index building step.
294
+ If set, all Cs are translated to Ts in the reference and add the $k$-mers are added to the index, the same for G to A.
295
+ This nearly doubles the number of $k$-mers in the index (`-n`)
296
+
297
+ ```
298
+ xengsort index --index myindex -H host.fa.gz -G graft.fa.gz -n 9_000_000_000 --bisulfite [OPTIONS]
299
+ ```
300
+
301
+ If an index is build using the bisulfite parameter, also the read is converted in the classification step.
302
+ It is not necessary to change the parameters of the classification step.
303
+ Currently, it only supports the count-based classification.
304
+
290
305
 
291
306
  Please report a [GitLab issue](https://gitlab.com/genomeinformatics/xengsort/-/issues) in case of problems.
292
307
 
@@ -240,6 +240,21 @@ Further parameters and options are:
240
240
  - `--filter`: With this flag, only the graft reads are output.
241
241
  - `--count`: With this flag, only the number of reads in each category is counted, but the reads are not sorted.
242
242
 
243
+ ### How to filter bisulfite sequencing data
244
+ In bisulfite sequencing data we have a conversion from C to T and from G to A in the reverse complement. The standard xengsort index classifies most of the reads as `neither`.
245
+ To address this issue, we have created a `--bisulfite` mode.
246
+ This parameter must be set during the index building step.
247
+ If set, all Cs are translated to Ts in the reference and add the $k$-mers are added to the index, the same for G to A.
248
+ This nearly doubles the number of $k$-mers in the index (`-n`)
249
+
250
+ ```
251
+ xengsort index --index myindex -H host.fa.gz -G graft.fa.gz -n 9_000_000_000 --bisulfite [OPTIONS]
252
+ ```
253
+
254
+ If an index is build using the bisulfite parameter, also the read is converted in the classification step.
255
+ It is not necessary to change the parameters of the classification step.
256
+ Currently, it only supports the count-based classification.
257
+
243
258
 
244
259
  Please report a [GitLab issue](https://gitlab.com/genomeinformatics/xengsort/-/issues) in case of problems.
245
260
 
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "xengsort"
8
- version = "2.0.9"
8
+ version = "2.1.0"
9
9
 
10
10
  license = {file = "LICENSE"}
11
11
 
@@ -4,9 +4,10 @@ Functions for compiling ConsumerProducerTasks for k-mers
4
4
  """
5
5
 
6
6
  import numpy as np
7
- from numba import njit, int32, int64, uint64
7
+ from numba import njit, uint8, int32, int64, uint64
8
8
 
9
9
  from .dnaencode import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_old
10
+ from .dnaencode import _TABLE_DNA_TO_2BITS, compile_quick_dna_to_2bits
10
11
  from .dnaencode import quick_dna_to_2bits as quick_dna_to_2bits_old
11
12
  from .dnaencode_fast import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_new
12
13
  from .dnaencode_fast import quick_dna_to_2bits as quick_dna_to_2bits_new
@@ -30,7 +31,9 @@ compile_cptask_scatter_kmers_from_buffers():
30
31
 
31
32
  def compile_cptask_scatter_kmers_from_linemarked(
32
33
  mask, rcmode, hf0,
33
- nsubtables, noutbuffers_per_subtable, outbufsize, new_encoding=False):
34
+ nsubtables, noutbuffers_per_subtable, outbufsize,
35
+ new_encoding=False,
36
+ bisulfite=False):
34
37
  """
35
38
  Compile a cptask function that
36
39
  creates (gapped) k-mers from linemarked buffers.
@@ -40,10 +43,26 @@ def compile_cptask_scatter_kmers_from_linemarked(
40
43
  - nsubtables: the number of subtables
41
44
  - noutbuffers_per_subtable: number of output buffers per subtable
42
45
  - outbufsize: the output buffer size for which to compile this function
46
+ -
43
47
  """
44
48
  debugprint0, debugprint1, debugprint2 = debug.debugprint
45
49
  both = (rcmode[0].lower() == "b")
46
- quick_dna_to_2bits = quick_dna_to_2bits_new if new_encoding else quick_dna_to_2bits_old
50
+ if bisulfite:
51
+ if new_encoding:
52
+ raise NotImplementedError('WGBS with new encoding not supported')
53
+
54
+ else:
55
+ ct_table = _TABLE_DNA_TO_2BITS.copy()
56
+ ct_table[99] = 3
57
+ ct_table[67] = 3
58
+ quick_dna_to_2bits_ct = compile_quick_dna_to_2bits(ct_table)
59
+
60
+ ga_table = _TABLE_DNA_TO_2BITS.copy()
61
+ ga_table[103] = 0
62
+ ga_table[71] = 0
63
+ quick_dna_to_2bits_ga = compile_quick_dna_to_2bits(ga_table)
64
+ else:
65
+ quick_dna_to_2bits = quick_dna_to_2bits_new if new_encoding else quick_dna_to_2bits_old
47
66
 
48
67
  k, w, tpl = mask.k, mask.w, mask.tuple
49
68
  assert len(tpl) == k
@@ -183,6 +202,8 @@ def compile_cptask_scatter_kmers_from_linemarked(
183
202
  nactive = -1 # active input buffer
184
203
  outidx = np.full(nsubtables, (-1), dtype=np.int32)
185
204
  outpos = np.full(nsubtables, outbufsize, dtype=np.int32)
205
+ if bisulfite:
206
+ seq_buffer = np.empty(inbuffers[0].size, dtype=np.uint8)
186
207
  while True:
187
208
  nactive, wait = find_buffer_for_reading(incontrol, nactive)
188
209
  wait_read += wait
@@ -196,10 +217,22 @@ def compile_cptask_scatter_kmers_from_linemarked(
196
217
  errorcode = 0
197
218
  linemarks = linemarkbuffers[nactive]
198
219
  for i in range(nseqs):
199
- sq = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
200
- quick_dna_to_2bits(sq)
201
- (wait, _errorcode) = _process(
202
- sq, outidx, outpos, outbuffers, outcontrol)
220
+ if bisulfite:
221
+ seq_len = linemarks[i, 1] - linemarks[i, 0]
222
+ seq_buffer[0:seq_len] = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
223
+ quick_dna_to_2bits_ct(seq_buffer[0:seq_len])
224
+ (wait, _errorcode) = _process(
225
+ seq_buffer[:seq_len], outidx, outpos, outbuffers, outcontrol)
226
+
227
+ seq_buffer[0:seq_len] = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
228
+ quick_dna_to_2bits_ga(seq_buffer[0:seq_len])
229
+ (wait, _errorcode) = _process(
230
+ seq_buffer[:seq_len], outidx, outpos, outbuffers, outcontrol)
231
+ else:
232
+ sq = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
233
+ quick_dna_to_2bits(sq)
234
+ (wait, _errorcode) = _process(
235
+ sq, outidx, outpos, outbuffers, outcontrol)
203
236
  wait_write += wait
204
237
  errorcode |= _errorcode
205
238
  if errorcode:
@@ -34,6 +34,14 @@ def _dna_to_2bits(x, table):
34
34
  x[i] = table[x[i]]
35
35
 
36
36
 
37
+ def compile_quick_dna_to_2bits(table):
38
+ @njit(nogil=True)
39
+ def _quick_dna_to_2bits(x):
40
+ for i in range(len(x)):
41
+ x[i] = table[x[i]]
42
+
43
+ return _quick_dna_to_2bits
44
+
37
45
  # this is the one to use!
38
46
  @njit(nogil=True)
39
47
  def quick_dna_to_2bits(x):
@@ -10,13 +10,13 @@ from .mask import Mask
10
10
  U64_MINUSONE = uint64(np.iinfo(np.uint64).max)
11
11
 
12
12
 
13
- @njit
13
+ @njit(nogil=True)
14
14
  def quick_dna_to_2bits(x):
15
15
  for i in range(len(x)):
16
16
  x[i] = (x[i] >> 1) & 7
17
17
 
18
18
 
19
- @njit
19
+ @njit(nogil=True)
20
20
  def dna_to_2bits(x, y):
21
21
  for i in range(len(x)):
22
22
  y[i] = (x[i] >> 1) & 7
@@ -63,7 +63,7 @@ def compile_revcomp_and_canonical_code(q, rcmode):
63
63
  mask = uint64(int(q * '10', 2))
64
64
  shift = uint64(64 - 2 * q)
65
65
 
66
- @njit(locals=dict(value=uint64))
66
+ @njit(nogil=True, locals=dict(value=uint64))
67
67
  def reverse_in_pairs(value):
68
68
  value = ((value & 0x3333333333333333) << 2) | ((value & 0xCCCCCCCCCCCCCCCC) >> 2)
69
69
  value = ((value & 0x0F0F0F0F0F0F0F0F) << 4) | ((value >> 4) & 0x0F0F0F0F0F0F0F0F)
@@ -72,7 +72,7 @@ def compile_revcomp_and_canonical_code(q, rcmode):
72
72
  value = (value << 32) | (value >> 32)
73
73
  return value
74
74
 
75
- @njit(locals=dict(value=uint64))
75
+ @njit(nogil=True, locals=dict(value=uint64))
76
76
  def _rc(value):
77
77
  value ^= mask
78
78
  value = reverse_in_pairs(value)
@@ -204,38 +204,32 @@ def get_argument_parser():
204
204
  "get information about a hash table; export its (filtered) data",
205
205
  info,
206
206
  "fastcash_info", "main",
207
- ),
208
- (
209
- "export",
207
+ ), (
208
+ "export", # this is a synonym for info. Both commands do both.
210
209
  "get information about a hash table; export its (filtered) data",
211
210
  info,
212
211
  "fastcash_info", "main",
213
- ),
214
- (
212
+ ), (
215
213
  "lookup",
216
214
  "look up the values of a few k-mers in the given hash table",
217
215
  lookup,
218
216
  "fastcash_lookup", "main",
219
- ),
220
- (
217
+ ), (
221
218
  "optimize",
222
219
  "optimize the assignment of an existing hash table from exported data",
223
220
  optimize,
224
221
  "fastcash_optimize", "main",
225
- ),
226
- (
222
+ ), (
227
223
  "deploy",
228
224
  "deploy a fastcash-based tool (like xengsort, hackgap, etc.)",
229
225
  deploy,
230
226
  "fastcash_deploy", "main",
231
- ),
232
- (
227
+ ), (
233
228
  "mphf",
234
229
  "compute a minimal perfect hash function on a key set",
235
230
  mphf,
236
231
  "fastcash_mphf", "main",
237
- ),
238
- (
232
+ ), (
239
233
  "xorfilter",
240
234
  "build a XOR filter from a hash table",
241
235
  xorfilter,
@@ -135,6 +135,22 @@ def build_hash(universe, n, subtables, bucketsize,
135
135
  startbit = subtable * subtablebits + bucket * bucketsizebits
136
136
  hprefetch(table, startbit)
137
137
 
138
+ @njit(nogil=True, locals=dict(
139
+ bucket=int64, startbit=uint64))
140
+ def prefetch_for_key(table, key):
141
+ st, sk = get_subtable_subkey_from_key(key)
142
+ bucket, _ = get_bs1(sk)
143
+ startbit = st * subtablebits + bucket * bucketsizebits
144
+ hprefetch(table, startbit)
145
+
146
+ bucket, _ = get_bs2(sk)
147
+ startbit = st * subtablebits + bucket * bucketsizebits
148
+ hprefetch(table, startbit)
149
+
150
+ bucket, _ = get_bs3(sk)
151
+ startbit = st * subtablebits + bucket * bucketsizebits
152
+ hprefetch(table, startbit)
153
+
138
154
  # Define private low-level hash table accssor methods
139
155
  @njit(nogil=True, locals=dict(
140
156
  bucket=int64, startbit=int64, v=uint64))
@@ -1,4 +1,3 @@
1
- import io
2
1
  from argparse import ArgumentParser
3
2
 
4
3
  import numpy as np
@@ -7,9 +6,11 @@ from numba import njit, int32, int64
7
6
  from .seqio import FormatError, _universal_reads
8
7
  from ..lowlevel.libc import read_block
9
8
  from ..lowlevel.conpro import \
10
- find_buffer_for_writing,\
9
+ find_buffer_for_writing, \
11
10
  mark_buffer_for_reading, mark_buffer_for_writing
11
+ from ..lowlevel import debug
12
12
 
13
+ debugprint0, debugprint1, debugprint2 = debug.debugprint
13
14
 
14
15
  # Python FASTQ/gz handling ######################################
15
16
 
@@ -33,7 +34,7 @@ def fastq_reads(files, sequences_only=False, dirty=False):
33
34
  func = _fastq_seqs_from_filelike
34
35
  if dirty:
35
36
  func = _fastq_seqs_dirty_from_filelike
36
- if type(files) == list or type(files) == tuple:
37
+ if isinstance(files, list) or isinstance(files, tuple):
37
38
  # multiple files
38
39
  for f in files:
39
40
  yield from _universal_reads(f, func)
@@ -137,6 +138,9 @@ def cptask_read_fastq_into_linemarked_buffers(fd, inbuffers, incontrol, ininfos,
137
138
  if ntodo <= 0:
138
139
  if ntodo < 0 or offset == 0:
139
140
  errorcode = ntodo
141
+ if errorcode == -99_999:
142
+ debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
143
+ debugprint0("- Please increase buffersize using --buffersize.")
140
144
  break
141
145
  ntodo = offset
142
146
  nseqs, nxt = _find_fastq_seqmarks(active_buffer[:ntodo], linemarkbuffers[nactive])
@@ -165,6 +169,7 @@ def cptask_read_pairedfastq_into_linemarked_buffers(fd, inbuffers, incontrol, in
165
169
  linemarkbuffers = outinfos.reshape(M, N // 4, 4)
166
170
 
167
171
  offset1 = offset2 = ntodo1 = ntodo2 = wait_read = wait_write = 0
172
+ nseqs1, nseqs2 = 0, 0
168
173
  nactive = -1
169
174
  active_buffer = outbuffers[0] # irrelevant
170
175
  buffersize = active_buffer.shape[0] // 2
@@ -200,25 +205,37 @@ def cptask_read_pairedfastq_into_linemarked_buffers(fd, inbuffers, incontrol, in
200
205
  break
201
206
 
202
207
  # only read if the buffer is filled less than 80%
203
- if offset1 < active_buffer1.size * 4 // 5:
208
+ if offset1 < active_buffer1.size * 4 // 5 or nseqs1 == 0:
204
209
  inner_offset1 = offset1
205
- while inner_offset1 < active_buffer1.size * 4 // 5:
210
+ while inner_offset1 < active_buffer1.size * 4 // 5 or nseqs1 == 0:
206
211
  ntodo1 = read_block(int32(fd1), active_buffer1, inner_offset1)
207
212
  if ntodo1 == 0:
208
213
  ntodo1 = inner_offset1
209
214
  break
215
+ if ntodo1 == -99_999:
216
+ debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
217
+ debugprint0("- Please increase buffersize using --buffersize.")
218
+ break
210
219
  inner_offset1 = ntodo1
220
+ if inner_offset1 >= active_buffer1.size * 4 // 5:
221
+ break
211
222
  else:
212
223
  ntodo1 = offset1
213
224
 
214
- if offset2 < active_buffer2.size * 4 // 5:
225
+ if offset2 < active_buffer2.size * 4 // 5 or nseqs2 == 0:
215
226
  inner_offset2 = offset2
216
- while inner_offset2 < active_buffer2.size * 4 // 5:
227
+ while inner_offset2 < active_buffer2.size * 4 // 5 or nseqs2 == 0:
217
228
  ntodo2 = read_block(int32(fd2), active_buffer2, inner_offset2)
218
229
  if ntodo2 == 0:
219
230
  ntodo2 = inner_offset2
220
231
  break
232
+ if ntodo2 == -99_999:
233
+ debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
234
+ debugprint0("- Please increase buffersize using --buffersize.")
235
+ break
221
236
  inner_offset2 = ntodo2
237
+ if inner_offset2 >= active_buffer2.size * 4 // 5:
238
+ break
222
239
  else:
223
240
  ntodo2 = offset2
224
241
 
@@ -320,18 +337,18 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
320
337
  """
321
338
  Find start and end positions of lines in byte buffer 'buf'.
322
339
  Store the information in 'linemarks', such that
323
- linemarks[i, 0:4] has information about record number i.
324
- If we need to store the sequence id and the position in the read, linemarks has 4 entries.
340
+ linemarks[i, 0:4] (int32) has information about record number i.
341
+ If we need to store the sequence id and the position in the read, linemarks has 6 entries.
325
342
 
326
343
  Return pair (m, nxt), where:
327
344
  m: number of sequences processed
328
345
  nxt: position at which to continue processing
329
346
  in the next iteration (start of new entry, '@')
347
+ For sequence i, we have
330
348
  linemarks[i, 0] = start of sequence line
331
349
  linemarks[i, 1] = end of sequence line
332
350
  linemarks[i, 2] = start of record (all lines)
333
351
  linemarks[i, 3] = end of record (all lines)
334
- ---------------------------------------------
335
352
  linemarks[i, 4] = sequence id
336
353
  linemarks[i, 5] = position in the sequence (currently always 0 since we expect that a sequence fits in a buffer)
337
354
  """
@@ -346,6 +363,7 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
346
363
  # find start of current line
347
364
  while True:
348
365
  if buf[i] == 10:
366
+ # assert m >= 0 # DEBUG 28.Aug.2025
349
367
  i += 1
350
368
  if line == 0:
351
369
  linemarks[m, 3] = i
@@ -354,7 +372,7 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
354
372
  linemarks[m, 5] = 0
355
373
  sequence_id += 1
356
374
  if i >= n:
357
- # we did not find valid record start
375
+ # we did not find a valid record start
358
376
  if line == 0:
359
377
  m += 1
360
378
  nxt = n
@@ -362,9 +380,9 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
362
380
  if line == 0:
363
381
  m += 1
364
382
  nxt = i
365
- linemarks[m, 2] = i
366
383
  if m >= M:
367
384
  return (M, nxt)
385
+ linemarks[m, 2] = i
368
386
  elif line == 1:
369
387
  linemarks[m, 0] = i
370
388
  # find end of current line
@@ -397,7 +415,7 @@ def fastq_chunks(files,
397
415
  if not (isinstance(files, tuple) or isinstance(files, list)):
398
416
  files = (files,) # single file?
399
417
  if bufsize / 2**30 > 1.5:
400
- raise ValueError(f"ERROR: bufsize={bufsize/2**30} GiB must be <= 1.5 GiB (2**30 bytes); use smaller chunksize of fewer threads.")
418
+ raise ValueError(f"ERROR: bufsize={bufsize / 2**30} GiB must be <= 1.5 GiB (2**30 bytes); use smaller chunksize of fewer threads.")
401
419
  linemarks = np.empty((maxreads, 4), dtype=np.int32)
402
420
  buf = np.empty(bufsize, dtype=np.uint8)
403
421
  for file in files:
@@ -406,7 +424,7 @@ def fastq_chunks(files,
406
424
  prev = 0
407
425
  while True:
408
426
  read = f.readinto(buf[prev:])
409
- if read == 0:
427
+ if read == 0 and prev == 0:
410
428
  break
411
429
  available = prev + read # number of bytes available
412
430
  m, cont = _find_fastq_seqmarks(buf[:available], linemarks)
@@ -22,16 +22,19 @@ from .binaryio import \
22
22
  _in_suffixes = {
23
23
  ".gz": "bgzip -cd",
24
24
  ".bz2": "bzip2 -cd",
25
- ".xz": "xz -cd"
25
+ ".xz": "xz -cd",
26
+ ".zst": "zstd -cd",
26
27
  }
27
28
  _out_suffixes = {
28
29
  ".gz": "bgzip",
29
30
  ".bz2": "bzip2",
30
- ".xz": "xz"
31
+ ".xz": "xz",
32
+ ".zst": "zstd",
31
33
  }
32
- _out_threads = {
34
+ _threads = {
33
35
  "bgzip": "@",
34
- "xz": "T"
36
+ "xz": "T",
37
+ "zstd": "T",
35
38
  }
36
39
  _out_compression = defaultdict(str)
37
40
  _out_compression['bgzip'] = 'l'
@@ -41,13 +44,18 @@ _extensions = dict(
41
44
  fastq=[".fastq", ".fq"],
42
45
  vcf=[".vcf"],
43
46
  numpy=[".np", ".npy"],
44
- data=[".data"]
47
+ data=[".data"],
48
+ gff=[".gff3"],
49
+ gtf=[".gtf"],
50
+ txt=[".txt"],
51
+ tsv=[".tsv"],
52
+ mex=[".mex", '.mtx'], # sparse matrix format
45
53
  )
46
54
  _filetypes = {ex: typ for typ, L in _extensions.items() for ex in L}
47
55
 
48
56
 
49
57
  class InputFileHandler:
50
- def __init__(self, filename, bufsize=2**16):
58
+ def __init__(self, filename, bufsize=2**16, compression_threads=1):
51
59
  """
52
60
  Context manager class representing an input file or stream.
53
61
 
@@ -77,6 +85,7 @@ class InputFileHandler:
77
85
  if is_compressed:
78
86
  if which(tool.split()[0]) is None:
79
87
  raise RuntimeError(f"Cannot decompress input file {filename} because {tool} is not installed.")
88
+ tool += f' -{_threads[tool.split()[0]]} {compression_threads}' if tool.split()[0] in _threads and tool.split()[0] != "zstd" else ''
80
89
  debugprint2(f"- creating Popen command for {filename} of type {file_type} using '{tool} {filename}'")
81
90
  self.cmd = cmdsplit(f"{tool} {filename}")
82
91
  else:
@@ -136,8 +145,11 @@ class OutputFileHandler:
136
145
  if is_compressed:
137
146
  if which(tool) is None:
138
147
  raise RuntimeError(f"Cannot compress output file {filename}. {tool} is not installed")
139
- _threads = f' -{_out_threads[tool]} {compression_threads}' if tool in _out_threads else ''
140
- tool += f' {_threads} -{_out_compression[tool]}{compression_level}'
148
+ if tool == 'zstd':
149
+ threads = f' -{_threads[tool]}{compression_threads}'
150
+ else:
151
+ threads = f' -{_threads[tool]} {compression_threads}' if tool in _threads else ''
152
+ tool += f' {threads} -{_out_compression[tool]}{compression_level}'
141
153
  debugprint2(f"- creating Popen command for {filename} of type {file_type} using '{tool} {filename}'")
142
154
  self.cmd = cmdsplit(f"{tool}")
143
155
  else:
@@ -168,7 +180,7 @@ class OutputFileHandler:
168
180
  self.fd = None
169
181
 
170
182
 
171
- def cptask_read_file(fname, valid_types, w, *allbuffers):
183
+ def cptask_read_file(fname, valid_types, w, *allbuffers, compression_threads=2):
172
184
  """
173
185
  ConsumerProducer task that consumes a file and writes sequences into buffers.
174
186
  If the file is compressed using any of {gzip/pigz, bzip, xz},
@@ -183,8 +195,8 @@ def cptask_read_file(fname, valid_types, w, *allbuffers):
183
195
  paired = True
184
196
  fname, fname2 = fname
185
197
 
186
- with InputFileHandler(fname) as infile, \
187
- InputFileHandler(fname2) if paired else nullcontext() as infile2:
198
+ with InputFileHandler(fname, compression_threads=compression_threads) as infile, \
199
+ InputFileHandler(fname2, compression_threads=compression_threads) if paired else nullcontext() as infile2:
188
200
  ft = infile.file_type
189
201
  if (valid_types is not None) and (ft not in valid_types):
190
202
  debugprint0(f"Error: file type '{ft}' not among {valid_types=} for file '{fname}'.")
@@ -212,10 +212,18 @@ def load_hash(filename, *, shared_memory=False, info_only=False, quiet=False):
212
212
  assert shm_buf.shape[0] % 8 == 0
213
213
  assert shm_buf.itemsize == 1
214
214
  init = np.ndarray(shm_buf.shape[0] // 8, dtype=np.uint64, buffer=shm_buf)
215
- h = m.build_hash(universe, n, subtables,
216
- bucketsize, hashfuncs, nvalues, update_value,
217
- aligned=aligned, nfingerprints=nfingerprints,
218
- init=init, maxwalk=maxwalk, shortcutbits=shortcutbits, shm=shm)
215
+
216
+ if hashtype == 'new':
217
+ choices = hashinfo['choices']
218
+ h = m.build_hash(universe, n, subtables, choices,
219
+ bucketsize, hashfuncs, nvalues, update_value,
220
+ aligned=aligned, nfingerprints=nfingerprints,
221
+ init=init, maxwalk=maxwalk, shm=shm)
222
+ else:
223
+ h = m.build_hash(universe, n, subtables,
224
+ bucketsize, hashfuncs, nvalues, update_value,
225
+ aligned=aligned, nfingerprints=nfingerprints,
226
+ init=init, maxwalk=maxwalk, shortcutbits=shortcutbits, shm=shm)
219
227
  if not shared_memory:
220
228
  debugprint1(f"- loading '{filename}.hash' into array")
221
229
  checked = load_array_into(f"{filename}.hash", h.hashtable, check=checksum)
@@ -5,11 +5,13 @@ Compiler functions for k-mer processors
5
5
 
6
6
  """
7
7
 
8
- from numba import njit, int64, uint64
8
+ from numba import njit, int64, uint64, uint8
9
+ import numpy as np
9
10
 
10
11
  from .dnaencode import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_old
11
12
  from .dnaencode_fast import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_new
12
13
  from .lowlevel import debug
14
+ from .lowlevel.llvm import compile_popcount, compile_cttz
13
15
 
14
16
 
15
17
  def compile_kmer_iterator(shp, rcmode="f", new_encoding=False):
@@ -472,3 +474,123 @@ def compile_positional_translated_kmer_processor(shp, func, rcmode="f", new_enco
472
474
  pass # all done here
473
475
 
474
476
  return k, processor
477
+
478
+
479
+ # Iterative spaced seed hashing (doi: 10.1007/978-3-030-20242-2_18, reversed order compared to paper)
480
+
481
+ def compile_positional_kmer_processor_issh(shp, func, rcmode="f", new_encoding=False):
482
+ """
483
+ like compile_kmer_processor, but also uses the current k-mer start position
484
+ as an additional argument to func:
485
+ func(hashtable, kmercode, position, *parameters)
486
+ """
487
+ both = (rcmode == "both")
488
+ k = len(shp)
489
+ if k < 1 or k > 32:
490
+ raise ValueError(f"only 1<=k<=32 is supported, but k={k}.")
491
+ revcomp, ccode = compile_revcomp_and_canonical_code_new(k, rcmode) if new_encoding else compile_revcomp_and_canonical_code_old(k, rcmode)
492
+ popcnt = compile_popcount('uint64')
493
+ cttz = compile_cttz('uint64')
494
+
495
+ weight, width = uint64(len(shp)), uint64(max(shp) + 1)
496
+ sig_mask = 0
497
+ m = np.zeros(width, dtype=np.uint64)
498
+ for i in shp:
499
+ sig_mask |= 1 << (width - i - 1)
500
+ sig_mask = uint64(sig_mask)
501
+
502
+ for i in range(width - 1):
503
+ m[i + 1] = m[i] + (sig_mask >> (width - i - 1) & 1)
504
+
505
+ C = np.zeros((width, width), dtype=np.uint64)
506
+ masks = np.zeros((width, width), dtype=np.uint64)
507
+ for g in range(width):
508
+ for j in range(width - 1):
509
+ for k in shp:
510
+ if k + j < width and (sig_mask >> (width - (k + j) - 1) & 1) == 1 and m[k] == m[k + j] - m[j] + m[g]:
511
+ C[g, j] |= 1 << (width - k - 1)
512
+ masks[g, j] |= (3 << 2 * (weight - m[k + j] - 1))
513
+
514
+ @njit(nogil=True, locals=dict(best=uint64, best_i=uint64, best_j=uint64, use=uint64))
515
+ def find_best_prev(s, to_cover):
516
+ best = 0
517
+ best_i, best_j = 0, 1
518
+ for z in range(0, s - 1):
519
+ for k in range(1, s):
520
+ use = popcnt(to_cover & C[z, k])
521
+ if use > best:
522
+ best = use
523
+ best_i, best_j = z, k
524
+ return best_i, best_j
525
+
526
+ @njit(nogil=True, locals=dict(mask=uint64, n_ones=uint64, positions=uint64, n_ones_new=uint64))
527
+ def compute_shift(g, k):
528
+ mask = masks[g, k]
529
+ n_ones = cttz(mask) // 2
530
+ positions = width - shp[weight - n_ones - 1] - 1 + k
531
+ n_ones_new = popcnt(sig_mask & (2**positions - 1))
532
+ return 2 * (n_ones_new - n_ones)
533
+
534
+ steps = []
535
+ last = 0
536
+ # new position does not need to be covered
537
+ to_cover = sig_mask & ~uint64(1)
538
+ while to_cover != 0:
539
+ g, k = find_best_prev(width - 1, to_cover)
540
+ shift = compute_shift(g, k)
541
+ remove = to_cover & C[g, k]
542
+ to_cover &= ~remove
543
+ steps.append([k, shift, masks[g, k]])
544
+ last = max(last, k)
545
+ steps = np.array(steps, dtype=np.uint64)
546
+
547
+ @njit(nogil=True, locals=dict(code=uint64, shift=uint64, k=uint64, mask=uint64, contained_ns=uint64, c=uint8))
548
+ def issh_processor_fast(ht, seq, start, end, *parameters):
549
+ if (end - start) < (width - 1):
550
+ return
551
+ hashes = np.zeros(last, dtype=np.uint64)
552
+ # compute history
553
+ end_naive = min(start + last, end)
554
+ for i in range(start, end_naive):
555
+ contained_ns = 0
556
+ code = 0
557
+ for j in shp:
558
+ c = seq[i + j]
559
+ contained_ns |= (c > 3)
560
+ code = (code << 2) | min(uint64(c), uint64(3))
561
+ hashes[i] = code
562
+ if not contained_ns:
563
+ if both:
564
+ failed = func(ht, code, i, *parameters)
565
+ failed |= func(ht, revcomp(code), i, *parameters)
566
+ else:
567
+ failed = func(ht, ccode(code), i, *parameters)
568
+ if failed is True:
569
+ return
570
+
571
+ # necessary to check for Ns in first window
572
+ contained_ns = 0
573
+ for i in range(width):
574
+ contained_ns = (contained_ns << 1) | (seq[start + last + i] > 3)
575
+ contained_ns = contained_ns >> 1
576
+
577
+ # compute gapped k-mers iteratively based on previous gapped k-mers
578
+ for i in range(start + last, (end - start) - (width - 1)):
579
+ contained_ns = contained_ns << 1
580
+ code = 0
581
+ for k, shift, mask in steps:
582
+ code |= (hashes[(i - k) % last] & mask) << shift
583
+ c = seq[i + width - 1]
584
+ contained_ns |= c > 3
585
+ code |= min(uint64(c), uint64(3))
586
+
587
+ hashes[i % last] = code
588
+ if not contained_ns & sig_mask:
589
+ if both:
590
+ failed = func(ht, code, i, *parameters)
591
+ failed |= func(ht, revcomp(code), i, *parameters)
592
+ else:
593
+ failed = func(ht, ccode(code), i, *parameters)
594
+ if failed is True:
595
+ return
596
+ return k, issh_processor_fast