xengsort 2.2.0__tar.gz → 2.2.1__tar.gz

{xengsort-2.2.0/xengsort.egg-info → xengsort-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xengsort
-Version: 2.2.0
+Version: 2.2.1
 Summary: xenograft sorting of FASTQ reads into five categories: host, graft, both, neither, ambiguous.
 Author-email: Jens Zentgraf <zentgraf@cs.uni-saarland.de>, Sven Rahmann <rahmann@cs.uni-saarland.de>
 License: MIT License

{xengsort-2.2.0 → xengsort-2.2.1}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xengsort"
-version = "2.2.0"
+version = "2.2.1"
 
 license = {file = "LICENSE"}
 

{xengsort-2.2.0 → xengsort-2.2.1}/xengsort/fastcash_main.py

@@ -7,6 +7,7 @@ from argparse import ArgumentParser
 from importlib import import_module  # dynamically import subcommand
 from importlib.metadata import metadata
 from os.path import split as splitpath, splitext
+from collections import defaultdict
 
 from .lowlevel.debug import set_debugfunctions
 
@@ -28,7 +29,7 @@ def add_hash_table_arguments(p: ArgumentParser):
         help="bucket size, i.e. number of elements on a bucket")
     p.add_argument("--fill", type=float, default="0.9",
         help="desired fill rate of the hash table")
-    p.add_argument("--subtables", type=int, default=5, 
+    p.add_argument("--subtables", type=int, default=5,
         help="number of subtables used; subtables+1 threads are used")
 
 
@@ -51,20 +52,20 @@ def add_hash_arguments(p: ArgumentParser):
         help="number of objects to be stored")
     p.add_argument("--type", default="default",  # default="3c_fbcbvb",
         help="hash type (e.g. [s]3c_fbcbvb, 2v_fbcbvb), implemented in hash_{TYPE}.py")
-    p.add_argument("--unaligned", action="store_const", 
+    p.add_argument("--unaligned", action="store_const",
         const=False, dest="aligned", default=None,
         help="use unaligned buckets (smaller, slightly slower; default)")
     p.add_argument("--aligned", action="store_const",
         const=True, dest="aligned", default=None,
         help="use power-of-two aligned buckets (faster, but larger)")
-    p.add_argument("--hashfunctions", "--functions", 
+    p.add_argument("--hashfunctions", "--functions",
         help="hash functions: 'default', 'random', or func1:func2[:func3]")
     # less important hash options
     p.add_argument("--nostatistics", "--nostats", action="store_true",
         help="do not compute or show index statistics at the end")
     p.add_argument("--maxwalk", metavar="INT", type=int, default=500,
         help="maximum length of random walk through hash table before failing [500]")
-    p.add_argument("--maxfailures", metavar="INT", type=int, default=0, 
+    p.add_argument("--maxfailures", metavar="INT", type=int, default=0,
         help="continue even after this many failures [default:0; forever:-1]")
     p.add_argument("--walkseed", type=int, default=7,
         help="seed for random walks while inserting elements [7]")

{xengsort-2.2.0 → xengsort-2.2.1}/xengsort/io/fastqio.py

@@ -383,6 +383,7 @@ def cptask_read_pairedfastq_into_buffers(fd, inbuffers, incontrol, ininfos, outb
             else:
                 assert False
 
+        # count new lines to determine last complete record pair in both buffers
         nlines1, nlines2 = 0, 0
         for i in range(ntodo1):
             nlines1 += active_buffer1[i] == 10
@@ -392,9 +393,6 @@ def cptask_read_pairedfastq_into_buffers(fd, inbuffers, incontrol, ininfos, outb
         nseqs2 = nlines2 // 4
         nseqs = min(nseqs1, nseqs2)
 
-        outcontrol[nactive, 7] = nseqs
-        outcontrol[nactive, 6] = 0
-
         rm_lines = nlines1 - nseqs * 4
         assert rm_lines >= 0
         if rm_lines == 0 and active_buffer1[ntodo1 - 1] == 10:
@@ -419,14 +417,19 @@ def cptask_read_pairedfastq_into_buffers(fd, inbuffers, incontrol, ininfos, outb
                     break
                 offset2 += 1
 
+        # update outcontrol
+        outcontrol[nactive, 7] = nseqs
+        outcontrol[nactive, 4] = ntodo1 - offset1
+        outcontrol[nactive, 5] = ntodo2 - offset2
+        outcontrol[nactive, 6] = 0
+
     mark_buffer_for_writing(outcontrol, nactive, force=True)  # nothing left to read; re-use buffer
     return (wait_read, wait_write, errorcode)
 
 
 @njit(nogil=True, locals=dict(
-    ntodo1=int32, ntodo2=int32, errorcode=int32, offset1=int32, offset2=int32,
-    skip=int32, nxt1=int32, nxt2=int32, nseqs=int32, nseqs1=int32, nseqs2=int32, nlines1=int32, nlines2=int32,
-    wait=int64, wait_read=int64, wait_write=int64, buffersize=int64))
+    ntodo=int32, errorcode=int32, offset=int32, last_record_end=int32,
+    wait=int64, wait_read=int64, wait_write=int64, buffersize=int64, nactive=int32))
 def cptask_read_fastq_into_buffers(fd, inbuffers, incontrol, ininfos, outbuffers, outcontrol, outinfos):
     """
     Has to be run as a thread within a consumer producer task.
@@ -439,9 +442,10 @@ def cptask_read_fastq_into_buffers(fd, inbuffers, incontrol, ininfos, outbuffers
     assert N % 4 == 0
 
     offset = ntodo = wait_read = wait_write = 0
-    nseqs = 0
     nactive = -1
     active_buffer = outbuffers[0]  # irrelevant
+    line_starts = np.zeros(4, dtype=np.int8)
+    line_start_positions = np.zeros(4, np.int32)
 
     while True:
         old = nactive
@@ -474,25 +478,45 @@ def cptask_read_fastq_into_buffers(fd, inbuffers, incontrol, ininfos, outbuffers
                 break
             ntodo = offset
 
-        nlines = 0
-        for i in range(ntodo):
-            nlines += active_buffer[i] == 10
-        nseqs = nlines // 4
-
-        outcontrol[nactive, 7] = nseqs
-        outcontrol[nactive, 6] = 0
-
-        rm_lines = nlines - nseqs * 4
-        assert rm_lines >= 0
-        if rm_lines == 0 and active_buffer[ntodo - 1] == 10:
-            offset = 0
+        # search for last complete entry
+        record_end = 0
+        if active_buffer[ntodo - 1] == 10:
+            record_end = ntodo
         else:
-            nlines = offset = 0
             for i in range(ntodo):
-                nlines += active_buffer[ntodo - i - 1] == 10
-                if nlines == rm_lines + 1:
+                if active_buffer[ntodo - i - 1] == 10:
+                    record_end = ntodo - i
                     break
-                offset += 1
+
+        idx = 0
+        for i in range(record_end - 1):
+            if active_buffer[record_end - i - 1] == 10:
+                line_starts[4 - idx] = active_buffer[record_end - i]
+                line_start_positions[4 - idx] = record_end - i
+                idx += 1
+            if idx == 5:
+                break
+
+        last_record_end = 0
+        if idx < 4:
+            last_record_end = 0
+        elif idx == 4 and line_starts[2] == 43 and active_buffer[0] == 64:
+            last_record_end = record_end
+        elif idx > 4 and line_starts[0] == 64 and line_starts[2] == 43:
+            last_record_end = record_end
+        elif idx > 4 and line_starts[1] == 43 and line_starts[3] == 64:
+            last_record_end = line_start_positions[3]
+        elif idx > 4 and line_starts[0] == 43 and line_starts[2] == 64:
+            last_record_end = line_start_positions[2]
+        elif idx > 4 and line_starts[1] == 64 and line_starts[3] == 43:
+            last_record_end = line_start_positions[1]
+        else:
+            debugprint0("- Invalid FASTQ file, each record must have four lines.")
+
+        # add bytes up to last complete record to outcontrol
+        offset = ntodo - last_record_end
+        outcontrol[nactive, 4] = last_record_end
+        outcontrol[nactive, 6] = 0
 
     mark_buffer_for_writing(outcontrol, nactive, force=True)  # nothing left to read; re-use buffer
     return (wait_read, wait_write, errorcode)
@@ -566,7 +590,7 @@ def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
 
     Return pair (m, nxt), where:
         m: number of sequences processed
-        nxt: position at which to continue processing 
+        nxt: position at which to continue processing
              in the next iteration (start of new entry, '@')
     For sequence i, we have
     linemarks[i, 0] = start of sequence line
@@ -628,7 +652,7 @@ def fastq_chunks(files,
     Yield all chunks from a list or tuple of FASTQ 'files'.
     A chunk is a pair (buffer, linemarks), where
     - buffer is readable/writable byte buffer
-    - buffer[linemarks[i,0]:linemarks[i,1]] contains the i-th sequence 
+    - buffer[linemarks[i,0]:linemarks[i,1]] contains the i-th sequence
       of the chunk; other line marks are not stored.
     - The part of linemarks that is returned is such that
       linemarks.shape[0] is at most maxreads and equals the number of reads.
@@ -676,7 +700,7 @@ def fastq_chunks_paired(pair, bufsize=2**23, maxreads=2**23 // 200):
     Yield all chunks from a pair of InputFileHandler objects.
     A chunk is a pair (buffer, linemarks), where
     - buffer is readable/writable byte buffer
-    - buffer[linemarks[i,0]:linemarks[i,1]] contains the i-th sequence 
+    - buffer[linemarks[i,0]:linemarks[i,1]] contains the i-th sequence
       of the chunk; other line marks are not stored.
     - The part of linemarks that is returned is such that
       linemarks.shape[0] is at most maxreads and equals the number of reads.
@@ -711,7 +735,8 @@ def fastq_chunks_paired(pair, bufsize=2**23, maxreads=2**23 // 200):
             available2 = prev2 + read2
 
             if read1 == 0 and read2 == 0:
-                break
+                if available1 == 0 and available2 == 0:
+                    break
 
             m1, cont1 = _find_fastq_seqmarks(buf1[:available1], linemarks1)
             m2, cont2 = _find_fastq_seqmarks(buf2[:available2], linemarks2)
@@ -727,6 +752,7 @@ def fastq_chunks_paired(pair, bufsize=2**23, maxreads=2**23 // 200):
             else:
                 chunk = (buf1, linemarks1[:m2], buf2, linemarks2[:m2])
                 cont1 = linemarks1[m2, 2]
+
             yield chunk
             prev1 = available1 - cont1
             prev2 = available2 - cont2
@@ -736,8 +762,8 @@ def fastq_chunks_paired(pair, bufsize=2**23, maxreads=2**23 // 200):
             if prev2 > 0:
                 buf2[:prev2] = buf2[cont2:available2]
                 buf2[prev2:] = 0
-            # assert prev1 < cont1
-            # assert prev2 < cont2
+            assert prev1 < cont1
+            assert prev2 < cont2
 
 
 # FASTQ checking ####################################################
@@ -788,7 +814,7 @@ def fastqcheck_single(files, args):
 
 def guess_pairs(filenames, replacer):
     """guess_pairs(filenames, replacer):
-    Return a list of file names where the read number (1/2) 
+    Return a list of file names where the read number (1/2)
     has been replaced by the 'replacer'.
     Example: guess_pairs(['abc_R1.fq', 'xyz_R1.fq'], 2)
         returns ['abc_R2.fq', 'xyz_R2.fq'].

{xengsort-2.2.0 → xengsort-2.2.1}/xengsort/kmers.py

@@ -328,6 +328,7 @@ def compile_positional_kmer_processor(shp, func, rcmode="f", new_encoding=False)
 
     return k, processor
 
+
 def compile_positional_translated_kmer_processor(shp, func, rcmode="f", new_encoding=False):
     """
     like compile_positional_kmer_processor, but also translates C->T and G->A and
@@ -480,9 +481,7 @@ def compile_positional_translated_kmer_processor(shp, func, rcmode="f", new_enco
 
 def compile_positional_kmer_processor_issh(shp, func, rcmode="f", new_encoding=False):
     """
-    like compile_kmer_processor, but also uses the current k-mer start position
-    as an additional argument to func:
-        func(hashtable, kmercode, position, *parameters)
+    computes all gapped k-mers of a sequences and applies func on them
     """
     both = (rcmode == "both")
     k = len(shp)
@@ -594,3 +593,134 @@ def compile_positional_kmer_processor_issh(shp, func, rcmode="f", new_encoding=F
                 if failed is True:
                     return
     return k, issh_processor_fast
+
+
+def compile_get_gapped_kmers_issh(mask, rcmode='max', new_encoding=True, sampled=False):
+    """
+    computes all gapped k-mers of a sequences and writes them to an numpy array
+
+    """
+    k = mask.k
+    shp = mask.tuple
+    w = mask.w
+    if k < 1 or k > 32:
+        raise ValueError(f"only 1<=k<=32 is supported, but k={k}.")
+    revcomp, ccode = compile_revcomp_and_canonical_code_new(k, rcmode) if new_encoding else compile_revcomp_and_canonical_code_old(k, rcmode)
+    invalid = uint64(np.iinfo(np.uint64).max)
+    popcnt = compile_popcount('uint64')
+    cttz = compile_cttz('uint64')
+
+    weight, width = uint64(len(shp)), uint64(max(shp) + 1)
+    sig_mask = 0
+    m = np.zeros(width, dtype=np.uint64)
+    for i in shp:
+        sig_mask |= 1 << (width - i - 1)
+    sig_mask = uint64(sig_mask)
+
+    for i in range(width - 1):
+        m[i + 1] = m[i] + (sig_mask >> (width - i - 1) & 1)
+
+    C = np.zeros((width, width), dtype=np.uint64)
+    masks = np.zeros((width, width), dtype=np.uint64)
+    for g in range(width):
+        for j in range(width - 1):
+            for k in shp:
+                if k + j < width and (sig_mask >> (width - (k + j) - 1) & 1) == 1 and m[k] == m[k + j] - m[j] + m[g]:
+                    C[g, j] |= 1 << (width - k - 1)
+                    masks[g, j] |= (3 << 2 * (weight - m[k + j] - 1))
+
+    @njit(nogil=True, locals=dict(best=uint64, best_i=uint64, best_j=uint64, use=uint64))
+    def find_best_prev(s, to_cover):
+        best = 0
+        best_i, best_j = 0, 1
+        for z in range(0, s - 1):
+            for k in range(1, s):
+                use = popcnt(to_cover & C[z, k])
+                if use > best:
+                    best = use
+                    best_i, best_j = z, k
+        return best_i, best_j
+
+    @njit(nogil=True, locals=dict(mask=uint64, n_ones=uint64, positions=uint64, n_ones_new=uint64))
+    def compute_shift(g, k):
+        mask = masks[g, k]
+        n_ones = cttz(mask) // 2
+        positions = width - shp[weight - n_ones - 1] - 1 + k
+        n_ones_new = popcnt(sig_mask & (2**positions - 1))
+        return 2 * (n_ones_new - n_ones)
+
+    steps = []
+    last = 0
+    # new position does not need to be covered
+    to_cover = sig_mask & ~uint64(1)
+    while to_cover != 0:
+        g, k = find_best_prev(width - 1, to_cover)
+        shift = compute_shift(g, k)
+        remove = to_cover & C[g, k]
+        to_cover &= ~remove
+        steps.append([k, shift, masks[g, k]])
+        last = max(last, k)
+    steps = np.array(steps, dtype=np.uint64)
+
+    @njit(nogil=True, locals=dict(code=uint64, shift=uint64, k=uint64, mask=uint64, contained_ns=uint64, c=uint8, end_naive=uint64))
+    def issh_processor_fast(ht, seq, start, end, *parameters):
+        if (end - start) < (width - 1):
+            return
+        hashes = np.zeros(last, dtype=np.uint64)
+        # compute history
+        end_naive = min(start + last, end)
+        for i in range(start, end_naive):
+            contained_ns = 0
+            code = 0
+            for j in shp:
+                c = seq[i + j]
+                contained_ns |= (c > 3)
+                code = (code << 2) | min(uint64(c), uint64(3))
+            hashes[i] = code
+            ht[i] = code if not (contained_ns & sig_mask) else invalid
+
+        # necessary to check for Ns in first window
+        contained_ns = 0
+        for i in range(width):
+            contained_ns = (contained_ns << 1) | (seq[start + last + i] > 3)
+        contained_ns = contained_ns >> 1
+
+        # compute gapped k-mers iteratively based on previous gapped k-mers
+        for i in range(start + last, (end - start) - (width - 1)):
+            contained_ns = contained_ns << 1
+            code = 0
+            for k, shift, mask in steps:
+                code |= (hashes[(i - k) % last] & mask) << shift
+            c = seq[i + width - 1]
+            contained_ns |= c > 3
+            code |= min(uint64(c), uint64(3))
+
+            hashes[i % last] = code
+            ht[i] = code if not (contained_ns & sig_mask) else invalid
+
+    @njit(nogil=True)
+    def canonical_codes_inplace(arr, nhashes):
+        for i in range(nhashes):
+            arr[i] = ccode(arr[i])
+        return nhashes
+
+    @njit(nogil=True)
+    def canonical_codes_sampled(arr, nhashes):
+        for idx, hash_idx in enumerate(range(0, nhashes, w // 2)):
+            arr[idx] = ccode(arr[hash_idx])
+        return idx + 1
+
+    canonical_codes = canonical_codes_sampled if sampled else canonical_codes_inplace
+
+    @njit(nogil=True)
+    def get_gapped_kmers(seq, hashes):
+        hashes[:] = 0
+        issh_processor_fast(hashes, seq, 0, len(seq))
+        return canonical_codes(hashes, len(seq) - w + 1)
+
+    @njit(nogil=True)
+    def get_gapped_kmers_forward(seq, hashes):
+        issh_processor_fast(hashes, seq, 0, len(seq))
+        return max(0, len(seq) - w + 1)
+
+    return get_gapped_kmers_forward if rcmode == 'f' and not sampled else get_gapped_kmers

{xengsort-2.2.0 → xengsort-2.2.1}/xengsort/lowlevel/libc.py

@@ -87,16 +87,17 @@ def read_block(fd, buf, offset=0, ATTEMPTS=5):
     if space <= 0:
         return -99_999  # no space means overflow
 
+    ptr = buf.ctypes.data
     # Wrap this in a loop to handle EAGAIN errors
     for attempt in range(ATTEMPTS):
-        res = _read(fd, buf.ctypes.data + offset, space)
-        err = errno()
+        res = _read(fd, ptr + offset, space)
         if res > 0:
             # print(res)
             fill = offset + res
             return fill if fill <= buflen else (-99_999)  # -99_999 means overflow
         elif res == 0:
             return 0
+        err = errno()
         if err == 11:
             continue  # EAGAIN: try again, no data
         return (-err) if err > 0 else (-99_998)  # negative errno on error
@@ -109,18 +110,19 @@ def read_block(fd, buf, offset=0, ATTEMPTS=5):
 def write_block(fd, buf, n, ATTEMPTS=5):
     """
     Write n bytes from buf to the file with file descriptor fd (int32).
-    Return number of ele
+    Return number of written bytes
     """
     N = n
     written = 0
     attempts = 0
+    ptr = buf.ctypes.data
     while n > 0:
-        res = _write(fd, buf.ctypes.data, n)
-        err = errno()
+        res = _write(fd, ptr + written, n)
         if res > 0:
             written += res
             n -= res
         elif res < 0:  # indicates error; error is in err
+            err = errno()
             return (-err) if err > 0 else (-99_998)
         else:  # res == 0: nothing done
             attempts += 1

{xengsort-2.2.0 → xengsort-2.2.1}/xengsort/lowlevel/llvm.py

@@ -47,7 +47,7 @@ from warnings import warn
 
 import numpy as np
 import numba
-from numba import njit, uint64
+from numba import njit, uint64, types
 import llvmlite
 from llvmlite import ir
 
@@ -155,7 +155,7 @@ def compile_memmove():
         def codegen(context, builder, sig, args):
             fty = ir.FunctionType(ir.VoidType(), (ir.IntType(8).as_pointer(), ir.IntType(8).as_pointer(), ir.IntType(64), ir.IntType(1)))
             _memmove = builder.module.declare_intrinsic(
-                f"llvm.memmove.p0.p0.i64", fnty=fty)
+                "llvm.memmove.p0.p0.i64", fnty=fty)
             volatile = ir.Constant(ir.IntType(1), 1)
             return builder.call(_memmove, (builder.inttoptr(args[0], ir.IntType(8).as_pointer()), builder.inttoptr(args[1], ir.IntType(8).as_pointer()), args[2], volatile,))
         return sig, codegen
@@ -171,7 +171,32 @@ def compile_memmove():
     return memmove
 
 
+# MEMCOPY
 
+_I1 = ir.IntType(1)
+_I64 = ir.IntType(64)
+_VOIDPTR = ir.IntType(8).as_pointer()
+
+
+@numba.extending.intrinsic
+def memcpy(typingctx, dst, dst_off, src, src_off, nbytes):
+    """Emit llvm.memcpy(dst+dst_off, src+src_off, nbytes)."""
+    if not isinstance(dst, types.Array) or not isinstance(src, types.Array):
+        return None
+    sig = types.void(dst, dst_off, src, src_off, nbytes)
+
+    def codegen(context, builder, signature, args):
+        dst_a = context.make_array(dst)(context, builder, value=args[0])
+        src_a = context.make_array(src)(context, builder, value=args[2])
+        dst_ptr = builder.gep(builder.bitcast(dst_a.data, _VOIDPTR), [args[1]])
+        src_ptr = builder.gep(builder.bitcast(src_a.data, _VOIDPTR), [args[3]])
+        # llvm.memcpy.p0i8.p0i8.i64(dst, src, n, isvolatile)
+        memcpy = builder.module.declare_intrinsic(
+            "llvm.memcpy", [_VOIDPTR, _VOIDPTR, _I64])
+        builder.call(memcpy, [dst_ptr, src_ptr, args[4], ir.Constant(_I1, 0)])
+        return context.get_dummy_value()
+
+    return sig, codegen
 
 # POPCOUNT #############################################
 

{xengsort-2.2.0 → xengsort-2.2.1/xengsort.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xengsort
-Version: 2.2.0
+Version: 2.2.1
 Summary: xenograft sorting of FASTQ reads into five categories: host, graft, both, neither, ambiguous.
 Author-email: Jens Zentgraf <zentgraf@cs.uni-saarland.de>, Sven Rahmann <rahmann@cs.uni-saarland.de>
 License: MIT License