xengsort 2.1.1__tar.gz → 2.2.1__tar.gz

{xengsort-2.1.1/xengsort.egg-info → xengsort-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xengsort
-Version: 2.1.1
+Version: 2.2.1
 Summary: xenograft sorting of FASTQ reads into five categories: host, graft, both, neither, ambiguous.
 Author-email: Jens Zentgraf <zentgraf@cs.uni-saarland.de>, Sven Rahmann <rahmann@cs.uni-saarland.de>
 License: MIT License

{xengsort-2.1.1 → xengsort-2.2.1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xengsort"
-version = "2.1.1"
+version = "2.2.1"
 
 license = {file = "LICENSE"}
 

{xengsort-2.1.1 → xengsort-2.2.1}/xengsort/cptasks_kmers.py

@@ -33,7 +33,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
         mask, rcmode, hf0,
         nsubtables, noutbuffers_per_subtable, outbufsize,
         new_encoding=False,
-        bisulfite=False):
+        methylation=False):
     """
     Compile a cptask function that
     creates (gapped) k-mers from linemarked buffers.
@@ -47,7 +47,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
     """
     debugprint0, debugprint1, debugprint2 = debug.debugprint
     both = (rcmode[0].lower() == "b")
-    if bisulfite:
+    if methylation:
         if new_encoding:
             raise NotImplementedError('WGBS with new encoding not supported')
 
@@ -202,7 +202,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
         nactive = -1  # active input buffer
         outidx = np.full(nsubtables, (-1), dtype=np.int32)
         outpos = np.full(nsubtables, outbufsize, dtype=np.int32)
-        if bisulfite:
+        if methylation:
             seq_buffer = np.empty(inbuffers[0].size, dtype=np.uint8)
         while True:
             nactive, wait = find_buffer_for_reading(incontrol, nactive)
@@ -217,7 +217,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
                 errorcode = 0
                 linemarks = linemarkbuffers[nactive]
                 for i in range(nseqs):
-                    if bisulfite:
+                    if methylation:
                         seq_len = linemarks[i, 1] - linemarks[i, 0]
                         seq_buffer[0:seq_len] = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
                         quick_dna_to_2bits_ct(seq_buffer[0:seq_len])

{xengsort-2.1.1 → xengsort-2.2.1}/xengsort/fastcash_main.py

@@ -7,6 +7,7 @@ from argparse import ArgumentParser
 from importlib import import_module  # dynamically import subcommand
 from importlib.metadata import metadata
 from os.path import split as splitpath, splitext
+from collections import defaultdict
 
 from .lowlevel.debug import set_debugfunctions
 
@@ -28,7 +29,7 @@ def add_hash_table_arguments(p: ArgumentParser):
         help="bucket size, i.e. number of elements on a bucket")
     p.add_argument("--fill", type=float, default="0.9",
         help="desired fill rate of the hash table")
-    p.add_argument("--subtables", type=int, default=5, 
+    p.add_argument("--subtables", type=int, default=5,
         help="number of subtables used; subtables+1 threads are used")
 
 
@@ -51,20 +52,20 @@ def add_hash_arguments(p: ArgumentParser):
         help="number of objects to be stored")
     p.add_argument("--type", default="default",  # default="3c_fbcbvb",
         help="hash type (e.g. [s]3c_fbcbvb, 2v_fbcbvb), implemented in hash_{TYPE}.py")
-    p.add_argument("--unaligned", action="store_const", 
+    p.add_argument("--unaligned", action="store_const",
         const=False, dest="aligned", default=None,
         help="use unaligned buckets (smaller, slightly slower; default)")
     p.add_argument("--aligned", action="store_const",
         const=True, dest="aligned", default=None,
         help="use power-of-two aligned buckets (faster, but larger)")
-    p.add_argument("--hashfunctions", "--functions", 
+    p.add_argument("--hashfunctions", "--functions",
         help="hash functions: 'default', 'random', or func1:func2[:func3]")
     # less important hash options
     p.add_argument("--nostatistics", "--nostats", action="store_true",
         help="do not compute or show index statistics at the end")
     p.add_argument("--maxwalk", metavar="INT", type=int, default=500,
         help="maximum length of random walk through hash table before failing [500]")
-    p.add_argument("--maxfailures", metavar="INT", type=int, default=0, 
+    p.add_argument("--maxfailures", metavar="INT", type=int, default=0,
         help="continue even after this many failures [default:0; forever:-1]")
     p.add_argument("--walkseed", type=int, default=7,
         help="seed for random walks while inserting elements [7]")

{xengsort-2.1.1 → xengsort-2.2.1}/xengsort/hash_s3c_fbcbvb.py

@@ -137,7 +137,7 @@ def build_hash(universe, n, subtables, bucketsize,
 
     @njit(nogil=True, locals=dict(
         bucket=int64, startbit=uint64))
-    def prefetch_for_key(table, key):
+    def prefetch_key(table, key):
         st, sk = get_subtable_subkey_from_key(key)
         bucket, _ = get_bs1(sk)
         startbit = st * subtablebits + bucket * bucketsizebits

{xengsort-2.1.1 → xengsort-2.2.1}/xengsort/io/fastqio.py

@@ -1,7 +1,7 @@
 from argparse import ArgumentParser
 
 import numpy as np
-from numba import njit, int32, int64
+from numba import njit, int32, int64, uint64
 
 from .seqio import FormatError, _universal_reads
 from ..lowlevel.libc import read_block
@@ -14,6 +14,7 @@ debugprint0, debugprint1, debugprint2 = debug.debugprint
 
 # Python FASTQ/gz handling ######################################
 
+
 def fastq_reads(files, sequences_only=False, dirty=False):
     """
     For the given 'files':
@@ -274,6 +275,253 @@ def cptask_read_pairedfastq_into_linemarked_buffers(fd, inbuffers, incontrol, in
     return (wait_read, wait_write, errorcode)
 
 
+# read fastq without computing linemarks
+
+@njit(nogil=True, locals=dict(
+    ntodo1=int32, ntodo2=int32, errorcode=int32, offset1=int32, offset2=int32,
+    skip=int32, nxt1=int32, nxt2=int32, nseqs=int32, nseqs1=int32, nseqs2=int32, nlines1=int32, nlines2=int32,
+    wait=int64, wait_read=int64, wait_write=int64, buffersize=int64))
+def cptask_read_pairedfastq_into_buffers(fd, inbuffers, incontrol, ininfos, outbuffers, outcontrol, outinfos):
+    """
+    Has to be run as a thread within a consumer producer task.
+    Keep reading bytes into one of the outbuffers (cycling),
+    and compute linemarks in one of the outinfos (same index, cycling),
+    until EOF is reached or an error occurs.
+    Return 0 (EOF), or an error code (negative), corresponding to -os.errno.
+    """
+    fd1, fd2 = fd
+    M, N = outinfos.shape
+    assert N % 4 == 0
+
+    offset1 = offset2 = ntodo1 = ntodo2 = wait_read = wait_write = 0
+    nseqs1 = nseqs2 = 0
+    nactive = -1
+    active_buffer = outbuffers[0]  # irrelevant
+    buffersize = active_buffer.shape[0] // 2
+    active_buffer1 = active_buffer[:len(active_buffer) // 2]
+    active_buffer2 = active_buffer[len(active_buffer) // 2:]
+
+    while True:
+        old = nactive
+        old_buffer1 = active_buffer1
+        old_buffer2 = active_buffer2
+        nactive, wait = find_buffer_for_writing(outcontrol, old)
+        if nactive < 0:
+            break
+        wait_write += wait
+        active_buffer = outbuffers[nactive]
+        active_buffer1 = active_buffer[:buffersize]
+        active_buffer2 = active_buffer[buffersize:buffersize * 2]
+
+        if old >= 0:
+            assert nactive != old
+            if offset1 > 0:
+                for i in range(offset1):
+                    active_buffer1[i] = old_buffer1[ntodo1 - offset1 + i]
+            if offset2 > 0:
+                for i in range(offset2):
+                    active_buffer2[i] = old_buffer2[ntodo2 - offset2 + i]
+            mark_buffer_for_reading(outcontrol, old)
+
+        errorcode = outcontrol[nactive, 1]
+        if errorcode != 0:
+            errorcode = -errorcode
+            break
+
+        # only read if the buffer is filled less than 80%
+        if offset1 < active_buffer1.size * 4 // 5 or nseqs1 == 0:
+            inner_offset1 = offset1
+            while inner_offset1 < active_buffer1.size * 4 // 5 or nseqs1 == 0:
+                ntodo1 = read_block(int32(fd1), active_buffer1, inner_offset1)
+                if ntodo1 == 0:
+                    ntodo1 = inner_offset1
+                    break
+                if ntodo1 == -99_999:
+                    debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
+                    debugprint0("- Please increase buffersize using --buffersize.")
+                    break
+                inner_offset1 = ntodo1
+                if inner_offset1 >= active_buffer1.size * 4 // 5:
+                    break
+        else:
+            ntodo1 = offset1
+
+        if offset2 < active_buffer2.size * 4 // 5 or nseqs2 == 0:
+            inner_offset2 = offset2
+            while inner_offset2 < active_buffer2.size * 4 // 5 or nseqs2 == 0:
+                ntodo2 = read_block(int32(fd2), active_buffer2, inner_offset2)
+                if ntodo2 == 0:
+                    ntodo2 = inner_offset2
+                    break
+                if ntodo2 == -99_999:
+                    debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
+                    debugprint0("- Please increase buffersize using --buffersize.")
+                    break
+                inner_offset2 = ntodo2
+                if inner_offset2 >= active_buffer2.size * 4 // 5:
+                    break
+        else:
+            ntodo2 = offset2
+
+        if ntodo1 <= 0 or ntodo2 <= 0:
+            # Error reading first file
+            if ntodo1 < 0:
+                errorcode = ntodo1
+                break
+            # Error reading second file
+            if ntodo2 < 0:
+                errorcode = ntodo2
+                break
+            # ntodo1=ntodo2=0 nothing read and no offset left; we are done
+            if ntodo1 == ntodo2 == offset1 == offset2 == 0:
+                errorcode = ntodo1
+                break
+            if ntodo1 == 0:
+                ntodo1 = offset1
+            elif ntodo2 == 0:
+                ntodo2 = offset2
+            else:
+                assert False
+
+        # count new lines to determine last complete record pair in both buffers
+        nlines1, nlines2 = 0, 0
+        for i in range(ntodo1):
+            nlines1 += active_buffer1[i] == 10
+        for i in range(ntodo2):
+            nlines2 += active_buffer2[i] == 10
+        nseqs1 = nlines1 // 4
+        nseqs2 = nlines2 // 4
+        nseqs = min(nseqs1, nseqs2)
+
+        rm_lines = nlines1 - nseqs * 4
+        assert rm_lines >= 0
+        if rm_lines == 0 and active_buffer1[ntodo1 - 1] == 10:
+            offset1 = 0
+        else:
+            nlines1 = offset1 = 0
+            for i in range(ntodo1):
+                nlines1 += active_buffer1[ntodo1 - i - 1] == 10
+                if nlines1 == rm_lines + 1:
+                    break
+                offset1 += 1
+
+        rm_lines = nlines2 - nseqs * 4
+        assert rm_lines >= 0
+        if rm_lines == 0 and active_buffer2[ntodo2 - 1] == 10:
+            offset2 = 0
+        else:
+            nlines2 = offset2 = 0
+            for i in range(ntodo2):
+                nlines2 += active_buffer2[ntodo2 - i - 1] == 10
+                if nlines2 == rm_lines + 1:
+                    break
+                offset2 += 1
+
+        # update outcontrol
+        outcontrol[nactive, 7] = nseqs
+        outcontrol[nactive, 4] = ntodo1 - offset1
+        outcontrol[nactive, 5] = ntodo2 - offset2
+        outcontrol[nactive, 6] = 0
+
+    mark_buffer_for_writing(outcontrol, nactive, force=True)  # nothing left to read; re-use buffer
+    return (wait_read, wait_write, errorcode)
+
+
+@njit(nogil=True, locals=dict(
+    ntodo=int32, errorcode=int32, offset=int32, last_record_end=int32,
+    wait=int64, wait_read=int64, wait_write=int64, buffersize=int64, nactive=int32))
+def cptask_read_fastq_into_buffers(fd, inbuffers, incontrol, ininfos, outbuffers, outcontrol, outinfos):
+    """
+    Has to be run as a thread within a consumer producer task.
+    Keep reading bytes into one of the outbuffers (cycling),
+    and compute linemarks in one of the outinfos (same index, cycling),
+    until EOF is reached or an error occurs.
+    Return 0 (EOF), or an error code (negative), corresponding to -os.errno.
+    """
+    M, N = outinfos.shape
+    assert N % 4 == 0
+
+    offset = ntodo = wait_read = wait_write = 0
+    nactive = -1
+    active_buffer = outbuffers[0]  # irrelevant
+    line_starts = np.zeros(4, dtype=np.int8)
+    line_start_positions = np.zeros(4, np.int32)
+
+    while True:
+        old = nactive
+        old_buffer = active_buffer
+        nactive, wait = find_buffer_for_writing(outcontrol, old)
+        if nactive < 0:
+            break
+        wait_write += wait
+        active_buffer = outbuffers[nactive]
+
+        if old >= 0:
+            assert nactive != old
+            if offset > 0:
+                for i in range(offset):
+                    active_buffer[i] = old_buffer[ntodo - offset + i]
+            mark_buffer_for_reading(outcontrol, old)
+
+        errorcode = outcontrol[nactive, 1]
+        if errorcode != 0:
+            errorcode = -errorcode
+            break
+
+        ntodo = read_block(int32(fd), active_buffer, offset)
+        if ntodo <= 0:
+            if ntodo < 0 or offset == 0:
+                errorcode = ntodo
+                if errorcode == -99_999:
+                    debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
+                    debugprint0("- Please increase buffersize using --buffersize.")
+                break
+            ntodo = offset
+
+        # search for last complete entry
+        record_end = 0
+        if active_buffer[ntodo - 1] == 10:
+            record_end = ntodo
+        else:
+            for i in range(ntodo):
+                if active_buffer[ntodo - i - 1] == 10:
+                    record_end = ntodo - i
+                    break
+
+        idx = 0
+        for i in range(record_end - 1):
+            if active_buffer[record_end - i - 1] == 10:
+                line_starts[4 - idx] = active_buffer[record_end - i]
+                line_start_positions[4 - idx] = record_end - i
+                idx += 1
+            if idx == 5:
+                break
+
+        last_record_end = 0
+        if idx < 4:
+            last_record_end = 0
+        elif idx == 4 and line_starts[2] == 43 and active_buffer[0] == 64:
+            last_record_end = record_end
+        elif idx > 4 and line_starts[0] == 64 and line_starts[2] == 43:
+            last_record_end = record_end
+        elif idx > 4 and line_starts[1] == 43 and line_starts[3] == 64:
+            last_record_end = line_start_positions[3]
+        elif idx > 4 and line_starts[0] == 43 and line_starts[2] == 64:
+            last_record_end = line_start_positions[2]
+        elif idx > 4 and line_starts[1] == 64 and line_starts[3] == 43:
+            last_record_end = line_start_positions[1]
+        else:
+            debugprint0("- Invalid FASTQ file, each record must have four lines.")
+
+        # add bytes up to last complete record to outcontrol
+        offset = ntodo - last_record_end
+        outcontrol[nactive, 4] = last_record_end
+        outcontrol[nactive, 6] = 0
+
+    mark_buffer_for_writing(outcontrol, nactive, force=True)  # nothing left to read; re-use buffer
+    return (wait_read, wait_write, errorcode)
+
+
 @njit(nogil=True, locals=dict(
     ntodo=int32, errorcode=int32, offset=int32, wait=int64,
     wait_write=int64, sequence_id=int64, nseq=int64))
@@ -332,7 +580,7 @@ def cptask_read_fastq_seqid_pos_into_buffers(fd, inbuffers, incontrol, ininfos,
 
 # numba FASTQ handling ######################################
 
-@njit(nogil=True)
+@njit(nogil=True, locals=dict(M=int64, i=uint64, m=int64, nxt=int64, line=uint64))
 def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
     """
     Find start and end positions of lines in byte buffer 'buf'.
@@ -342,7 +590,7 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
 
     Return pair (m, nxt), where:
         m: number of sequences processed
-        nxt: position at which to continue processing 
+        nxt: position at which to continue processing
              in the next iteration (start of new entry, '@')
     For sequence i, we have
     linemarks[i, 0] = start of sequence line
@@ -404,7 +652,7 @@ def fastq_chunks(files,
     Yield all chunks from a list or tuple of FASTQ 'files'.
     A chunk is a pair (buffer, linemarks), where
     - buffer is readable/writable byte buffer
-    - buffer[linemarks[i,0]:linemarks[i,1]] contains the i-th sequence 
+    - buffer[linemarks[i,0]:linemarks[i,1]] contains the i-th sequence
       of the chunk; other line marks are not stored.
     - The part of linemarks that is returned is such that
       linemarks.shape[0] is at most maxreads and equals the number of reads.
@@ -452,7 +700,7 @@ def fastq_chunks_paired(pair, bufsize=2**23, maxreads=2**23 // 200):
     Yield all chunks from a pair of InputFileHandler objects.
     A chunk is a pair (buffer, linemarks), where
     - buffer is readable/writable byte buffer
-    - buffer[linemarks[i,0]:linemarks[i,1]] contains the i-th sequence 
+    - buffer[linemarks[i,0]:linemarks[i,1]] contains the i-th sequence
       of the chunk; other line marks are not stored.
     - The part of linemarks that is returned is such that
       linemarks.shape[0] is at most maxreads and equals the number of reads.
@@ -487,7 +735,8 @@ def fastq_chunks_paired(pair, bufsize=2**23, maxreads=2**23 // 200):
             available2 = prev2 + read2
 
             if read1 == 0 and read2 == 0:
-                break
+                if available1 == 0 and available2 == 0:
+                    break
 
             m1, cont1 = _find_fastq_seqmarks(buf1[:available1], linemarks1)
             m2, cont2 = _find_fastq_seqmarks(buf2[:available2], linemarks2)
@@ -503,6 +752,7 @@ def fastq_chunks_paired(pair, bufsize=2**23, maxreads=2**23 // 200):
             else:
                 chunk = (buf1, linemarks1[:m2], buf2, linemarks2[:m2])
                 cont1 = linemarks1[m2, 2]
+
             yield chunk
             prev1 = available1 - cont1
             prev2 = available2 - cont2
@@ -512,8 +762,8 @@ def fastq_chunks_paired(pair, bufsize=2**23, maxreads=2**23 // 200):
             if prev2 > 0:
                 buf2[:prev2] = buf2[cont2:available2]
                 buf2[prev2:] = 0
-            # assert prev1 < cont1
-            # assert prev2 < cont2
+            assert prev1 < cont1
+            assert prev2 < cont2
 
 
 # FASTQ checking ####################################################
@@ -564,7 +814,7 @@ def fastqcheck_single(files, args):
 
 def guess_pairs(filenames, replacer):
     """guess_pairs(filenames, replacer):
-    Return a list of file names where the read number (1/2) 
+    Return a list of file names where the read number (1/2)
     has been replaced by the 'replacer'.
     Example: guess_pairs(['abc_R1.fq', 'xyz_R1.fq'], 2)
         returns ['abc_R2.fq', 'xyz_R2.fq'].

{xengsort-2.1.1 → xengsort-2.2.1}/xengsort/kmers.py

@@ -328,6 +328,7 @@ def compile_positional_kmer_processor(shp, func, rcmode="f", new_encoding=False)
 
     return k, processor
 
+
 def compile_positional_translated_kmer_processor(shp, func, rcmode="f", new_encoding=False):
     """
     like compile_positional_kmer_processor, but also translates C->T and G->A and
@@ -480,9 +481,7 @@ def compile_positional_translated_kmer_processor(shp, func, rcmode="f", new_enco
 
 def compile_positional_kmer_processor_issh(shp, func, rcmode="f", new_encoding=False):
     """
-    like compile_kmer_processor, but also uses the current k-mer start position
-    as an additional argument to func:
-        func(hashtable, kmercode, position, *parameters)
+    computes all gapped k-mers of a sequences and applies func on them
     """
     both = (rcmode == "both")
     k = len(shp)
@@ -594,3 +593,134 @@ def compile_positional_kmer_processor_issh(shp, func, rcmode="f", new_encoding=F
                 if failed is True:
                     return
     return k, issh_processor_fast
+
+
+def compile_get_gapped_kmers_issh(mask, rcmode='max', new_encoding=True, sampled=False):
+    """
+    computes all gapped k-mers of a sequences and writes them to an numpy array
+
+    """
+    k = mask.k
+    shp = mask.tuple
+    w = mask.w
+    if k < 1 or k > 32:
+        raise ValueError(f"only 1<=k<=32 is supported, but k={k}.")
+    revcomp, ccode = compile_revcomp_and_canonical_code_new(k, rcmode) if new_encoding else compile_revcomp_and_canonical_code_old(k, rcmode)
+    invalid = uint64(np.iinfo(np.uint64).max)
+    popcnt = compile_popcount('uint64')
+    cttz = compile_cttz('uint64')
+
+    weight, width = uint64(len(shp)), uint64(max(shp) + 1)
+    sig_mask = 0
+    m = np.zeros(width, dtype=np.uint64)
+    for i in shp:
+        sig_mask |= 1 << (width - i - 1)
+    sig_mask = uint64(sig_mask)
+
+    for i in range(width - 1):
+        m[i + 1] = m[i] + (sig_mask >> (width - i - 1) & 1)
+
+    C = np.zeros((width, width), dtype=np.uint64)
+    masks = np.zeros((width, width), dtype=np.uint64)
+    for g in range(width):
+        for j in range(width - 1):
+            for k in shp:
+                if k + j < width and (sig_mask >> (width - (k + j) - 1) & 1) == 1 and m[k] == m[k + j] - m[j] + m[g]:
+                    C[g, j] |= 1 << (width - k - 1)
+                    masks[g, j] |= (3 << 2 * (weight - m[k + j] - 1))
+
+    @njit(nogil=True, locals=dict(best=uint64, best_i=uint64, best_j=uint64, use=uint64))
+    def find_best_prev(s, to_cover):
+        best = 0
+        best_i, best_j = 0, 1
+        for z in range(0, s - 1):
+            for k in range(1, s):
+                use = popcnt(to_cover & C[z, k])
+                if use > best:
+                    best = use
+                    best_i, best_j = z, k
+        return best_i, best_j
+
+    @njit(nogil=True, locals=dict(mask=uint64, n_ones=uint64, positions=uint64, n_ones_new=uint64))
+    def compute_shift(g, k):
+        mask = masks[g, k]
+        n_ones = cttz(mask) // 2
+        positions = width - shp[weight - n_ones - 1] - 1 + k
+        n_ones_new = popcnt(sig_mask & (2**positions - 1))
+        return 2 * (n_ones_new - n_ones)
+
+    steps = []
+    last = 0
+    # new position does not need to be covered
+    to_cover = sig_mask & ~uint64(1)
+    while to_cover != 0:
+        g, k = find_best_prev(width - 1, to_cover)
+        shift = compute_shift(g, k)
+        remove = to_cover & C[g, k]
+        to_cover &= ~remove
+        steps.append([k, shift, masks[g, k]])
+        last = max(last, k)
+    steps = np.array(steps, dtype=np.uint64)
+
+    @njit(nogil=True, locals=dict(code=uint64, shift=uint64, k=uint64, mask=uint64, contained_ns=uint64, c=uint8, end_naive=uint64))
+    def issh_processor_fast(ht, seq, start, end, *parameters):
+        if (end - start) < (width - 1):
+            return
+        hashes = np.zeros(last, dtype=np.uint64)
+        # compute history
+        end_naive = min(start + last, end)
+        for i in range(start, end_naive):
+            contained_ns = 0
+            code = 0
+            for j in shp:
+                c = seq[i + j]
+                contained_ns |= (c > 3)
+                code = (code << 2) | min(uint64(c), uint64(3))
+            hashes[i] = code
+            ht[i] = code if not (contained_ns & sig_mask) else invalid
+
+        # necessary to check for Ns in first window
+        contained_ns = 0
+        for i in range(width):
+            contained_ns = (contained_ns << 1) | (seq[start + last + i] > 3)
+        contained_ns = contained_ns >> 1
+
+        # compute gapped k-mers iteratively based on previous gapped k-mers
+        for i in range(start + last, (end - start) - (width - 1)):
+            contained_ns = contained_ns << 1
+            code = 0
+            for k, shift, mask in steps:
+                code |= (hashes[(i - k) % last] & mask) << shift
+            c = seq[i + width - 1]
+            contained_ns |= c > 3
+            code |= min(uint64(c), uint64(3))
+
+            hashes[i % last] = code
+            ht[i] = code if not (contained_ns & sig_mask) else invalid
+
+    @njit(nogil=True)
+    def canonical_codes_inplace(arr, nhashes):
+        for i in range(nhashes):
+            arr[i] = ccode(arr[i])
+        return nhashes
+
+    @njit(nogil=True)
+    def canonical_codes_sampled(arr, nhashes):
+        for idx, hash_idx in enumerate(range(0, nhashes, w // 2)):
+            arr[idx] = ccode(arr[hash_idx])
+        return idx + 1
+
+    canonical_codes = canonical_codes_sampled if sampled else canonical_codes_inplace
+
+    @njit(nogil=True)
+    def get_gapped_kmers(seq, hashes):
+        hashes[:] = 0
+        issh_processor_fast(hashes, seq, 0, len(seq))
+        return canonical_codes(hashes, len(seq) - w + 1)
+
+    @njit(nogil=True)
+    def get_gapped_kmers_forward(seq, hashes):
+        issh_processor_fast(hashes, seq, 0, len(seq))
+        return max(0, len(seq) - w + 1)
+
+    return get_gapped_kmers_forward if rcmode == 'f' and not sampled else get_gapped_kmers

{xengsort-2.1.1 → xengsort-2.2.1}/xengsort/lowlevel/libc.py

@@ -87,16 +87,17 @@ def read_block(fd, buf, offset=0, ATTEMPTS=5):
     if space <= 0:
         return -99_999  # no space means overflow
 
+    ptr = buf.ctypes.data
     # Wrap this in a loop to handle EAGAIN errors
     for attempt in range(ATTEMPTS):
-        res = _read(fd, buf.ctypes.data + offset, space)
-        err = errno()
+        res = _read(fd, ptr + offset, space)
         if res > 0:
             # print(res)
             fill = offset + res
             return fill if fill <= buflen else (-99_999)  # -99_999 means overflow
         elif res == 0:
             return 0
+        err = errno()
         if err == 11:
             continue  # EAGAIN: try again, no data
         return (-err) if err > 0 else (-99_998)  # negative errno on error
@@ -109,18 +110,19 @@ def read_block(fd, buf, offset=0, ATTEMPTS=5):
 def write_block(fd, buf, n, ATTEMPTS=5):
     """
     Write n bytes from buf to the file with file descriptor fd (int32).
-    Return number of ele
+    Return number of written bytes
     """
     N = n
     written = 0
     attempts = 0
+    ptr = buf.ctypes.data
     while n > 0:
-        res = _write(fd, buf.ctypes.data, n)
-        err = errno()
+        res = _write(fd, ptr + written, n)
         if res > 0:
             written += res
             n -= res
         elif res < 0:  # indicates error; error is in err
+            err = errno()
             return (-err) if err > 0 else (-99_998)
         else:  # res == 0: nothing done
             attempts += 1