xengsort 2.1.0__tar.gz → 2.2.0__tar.gz

{xengsort-2.1.0/xengsort.egg-info → xengsort-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xengsort
-Version: 2.1.0
+Version: 2.2.0
 Summary: xenograft sorting of FASTQ reads into five categories: host, graft, both, neither, ambiguous.
 Author-email: Jens Zentgraf <zentgraf@cs.uni-saarland.de>, Sven Rahmann <rahmann@cs.uni-saarland.de>
 License: MIT License

{xengsort-2.1.0 → xengsort-2.2.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xengsort"
-version = "2.1.0"
+version = "2.2.0"
 
 license = {file = "LICENSE"}
 

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/cptasks_kmers.py

@@ -33,7 +33,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
         mask, rcmode, hf0,
         nsubtables, noutbuffers_per_subtable, outbufsize,
         new_encoding=False,
-        bisulfite=False):
+        methylation=False):
     """
     Compile a cptask function that
     creates (gapped) k-mers from linemarked buffers.
@@ -47,7 +47,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
     """
     debugprint0, debugprint1, debugprint2 = debug.debugprint
     both = (rcmode[0].lower() == "b")
-    if bisulfite:
+    if methylation:
         if new_encoding:
             raise NotImplementedError('WGBS with new encoding not supported')
 
@@ -202,7 +202,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
         nactive = -1  # active input buffer
         outidx = np.full(nsubtables, (-1), dtype=np.int32)
         outpos = np.full(nsubtables, outbufsize, dtype=np.int32)
-        if bisulfite:
+        if methylation:
             seq_buffer = np.empty(inbuffers[0].size, dtype=np.uint8)
         while True:
             nactive, wait = find_buffer_for_reading(incontrol, nactive)
@@ -217,7 +217,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
                 errorcode = 0
                 linemarks = linemarkbuffers[nactive]
                 for i in range(nseqs):
-                    if bisulfite:
+                    if methylation:
                         seq_len = linemarks[i, 1] - linemarks[i, 0]
                         seq_buffer[0:seq_len] = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
                         quick_dna_to_2bits_ct(seq_buffer[0:seq_len])

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/hash_s3c_fbcbvb.py

@@ -137,7 +137,7 @@ def build_hash(universe, n, subtables, bucketsize,
 
     @njit(nogil=True, locals=dict(
         bucket=int64, startbit=uint64))
-    def prefetch_for_key(table, key):
+    def prefetch_key(table, key):
         st, sk = get_subtable_subkey_from_key(key)
         bucket, _ = get_bs1(sk)
         startbit = st * subtablebits + bucket * bucketsizebits

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/io/fastqio.py

@@ -1,7 +1,7 @@
 from argparse import ArgumentParser
 
 import numpy as np
-from numba import njit, int32, int64
+from numba import njit, int32, int64, uint64
 
 from .seqio import FormatError, _universal_reads
 from ..lowlevel.libc import read_block
@@ -14,6 +14,7 @@ debugprint0, debugprint1, debugprint2 = debug.debugprint
 
 # Python FASTQ/gz handling ######################################
 
+
 def fastq_reads(files, sequences_only=False, dirty=False):
     """
     For the given 'files':
@@ -274,6 +275,229 @@ def cptask_read_pairedfastq_into_linemarked_buffers(fd, inbuffers, incontrol, in
     return (wait_read, wait_write, errorcode)
 
 
+# read fastq without computing linemarks
+
+@njit(nogil=True, locals=dict(
+    ntodo1=int32, ntodo2=int32, errorcode=int32, offset1=int32, offset2=int32,
+    skip=int32, nxt1=int32, nxt2=int32, nseqs=int32, nseqs1=int32, nseqs2=int32, nlines1=int32, nlines2=int32,
+    wait=int64, wait_read=int64, wait_write=int64, buffersize=int64))
+def cptask_read_pairedfastq_into_buffers(fd, inbuffers, incontrol, ininfos, outbuffers, outcontrol, outinfos):
+    """
+    Has to be run as a thread within a consumer producer task.
+    Keep reading bytes into one of the outbuffers (cycling),
+    and compute linemarks in one of the outinfos (same index, cycling),
+    until EOF is reached or an error occurs.
+    Return 0 (EOF), or an error code (negative), corresponding to -os.errno.
+    """
+    fd1, fd2 = fd
+    M, N = outinfos.shape
+    assert N % 4 == 0
+
+    offset1 = offset2 = ntodo1 = ntodo2 = wait_read = wait_write = 0
+    nseqs1 = nseqs2 = 0
+    nactive = -1
+    active_buffer = outbuffers[0]  # irrelevant
+    buffersize = active_buffer.shape[0] // 2
+    active_buffer1 = active_buffer[:len(active_buffer) // 2]
+    active_buffer2 = active_buffer[len(active_buffer) // 2:]
+
+    while True:
+        old = nactive
+        old_buffer1 = active_buffer1
+        old_buffer2 = active_buffer2
+        nactive, wait = find_buffer_for_writing(outcontrol, old)
+        if nactive < 0:
+            break
+        wait_write += wait
+        active_buffer = outbuffers[nactive]
+        active_buffer1 = active_buffer[:buffersize]
+        active_buffer2 = active_buffer[buffersize:buffersize * 2]
+
+        if old >= 0:
+            assert nactive != old
+            if offset1 > 0:
+                for i in range(offset1):
+                    active_buffer1[i] = old_buffer1[ntodo1 - offset1 + i]
+            if offset2 > 0:
+                for i in range(offset2):
+                    active_buffer2[i] = old_buffer2[ntodo2 - offset2 + i]
+            mark_buffer_for_reading(outcontrol, old)
+
+        errorcode = outcontrol[nactive, 1]
+        if errorcode != 0:
+            errorcode = -errorcode
+            break
+
+        # only read if the buffer is filled less than 80%
+        if offset1 < active_buffer1.size * 4 // 5 or nseqs1 == 0:
+            inner_offset1 = offset1
+            while inner_offset1 < active_buffer1.size * 4 // 5 or nseqs1 == 0:
+                ntodo1 = read_block(int32(fd1), active_buffer1, inner_offset1)
+                if ntodo1 == 0:
+                    ntodo1 = inner_offset1
+                    break
+                if ntodo1 == -99_999:
+                    debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
+                    debugprint0("- Please increase buffersize using --buffersize.")
+                    break
+                inner_offset1 = ntodo1
+                if inner_offset1 >= active_buffer1.size * 4 // 5:
+                    break
+        else:
+            ntodo1 = offset1
+
+        if offset2 < active_buffer2.size * 4 // 5 or nseqs2 == 0:
+            inner_offset2 = offset2
+            while inner_offset2 < active_buffer2.size * 4 // 5 or nseqs2 == 0:
+                ntodo2 = read_block(int32(fd2), active_buffer2, inner_offset2)
+                if ntodo2 == 0:
+                    ntodo2 = inner_offset2
+                    break
+                if ntodo2 == -99_999:
+                    debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
+                    debugprint0("- Please increase buffersize using --buffersize.")
+                    break
+                inner_offset2 = ntodo2
+                if inner_offset2 >= active_buffer2.size * 4 // 5:
+                    break
+        else:
+            ntodo2 = offset2
+
+        if ntodo1 <= 0 or ntodo2 <= 0:
+            # Error reading first file
+            if ntodo1 < 0:
+                errorcode = ntodo1
+                break
+            # Error reading second file
+            if ntodo2 < 0:
+                errorcode = ntodo2
+                break
+            # ntodo1=ntodo2=0 nothing read and no offset left; we are done
+            if ntodo1 == ntodo2 == offset1 == offset2 == 0:
+                errorcode = ntodo1
+                break
+            if ntodo1 == 0:
+                ntodo1 = offset1
+            elif ntodo2 == 0:
+                ntodo2 = offset2
+            else:
+                assert False
+
+        nlines1, nlines2 = 0, 0
+        for i in range(ntodo1):
+            nlines1 += active_buffer1[i] == 10
+        for i in range(ntodo2):
+            nlines2 += active_buffer2[i] == 10
+        nseqs1 = nlines1 // 4
+        nseqs2 = nlines2 // 4
+        nseqs = min(nseqs1, nseqs2)
+
+        outcontrol[nactive, 7] = nseqs
+        outcontrol[nactive, 6] = 0
+
+        rm_lines = nlines1 - nseqs * 4
+        assert rm_lines >= 0
+        if rm_lines == 0 and active_buffer1[ntodo1 - 1] == 10:
+            offset1 = 0
+        else:
+            nlines1 = offset1 = 0
+            for i in range(ntodo1):
+                nlines1 += active_buffer1[ntodo1 - i - 1] == 10
+                if nlines1 == rm_lines + 1:
+                    break
+                offset1 += 1
+
+        rm_lines = nlines2 - nseqs * 4
+        assert rm_lines >= 0
+        if rm_lines == 0 and active_buffer2[ntodo2 - 1] == 10:
+            offset2 = 0
+        else:
+            nlines2 = offset2 = 0
+            for i in range(ntodo2):
+                nlines2 += active_buffer2[ntodo2 - i - 1] == 10
+                if nlines2 == rm_lines + 1:
+                    break
+                offset2 += 1
+
+    mark_buffer_for_writing(outcontrol, nactive, force=True)  # nothing left to read; re-use buffer
+    return (wait_read, wait_write, errorcode)
+
+
+@njit(nogil=True, locals=dict(
+    ntodo1=int32, ntodo2=int32, errorcode=int32, offset1=int32, offset2=int32,
+    skip=int32, nxt1=int32, nxt2=int32, nseqs=int32, nseqs1=int32, nseqs2=int32, nlines1=int32, nlines2=int32,
+    wait=int64, wait_read=int64, wait_write=int64, buffersize=int64))
+def cptask_read_fastq_into_buffers(fd, inbuffers, incontrol, ininfos, outbuffers, outcontrol, outinfos):
+    """
+    Has to be run as a thread within a consumer producer task.
+    Keep reading bytes into one of the outbuffers (cycling),
+    and compute linemarks in one of the outinfos (same index, cycling),
+    until EOF is reached or an error occurs.
+    Return 0 (EOF), or an error code (negative), corresponding to -os.errno.
+    """
+    M, N = outinfos.shape
+    assert N % 4 == 0
+
+    offset = ntodo = wait_read = wait_write = 0
+    nseqs = 0
+    nactive = -1
+    active_buffer = outbuffers[0]  # irrelevant
+
+    while True:
+        old = nactive
+        old_buffer = active_buffer
+        nactive, wait = find_buffer_for_writing(outcontrol, old)
+        if nactive < 0:
+            break
+        wait_write += wait
+        active_buffer = outbuffers[nactive]
+
+        if old >= 0:
+            assert nactive != old
+            if offset > 0:
+                for i in range(offset):
+                    active_buffer[i] = old_buffer[ntodo - offset + i]
+            mark_buffer_for_reading(outcontrol, old)
+
+        errorcode = outcontrol[nactive, 1]
+        if errorcode != 0:
+            errorcode = -errorcode
+            break
+
+        ntodo = read_block(int32(fd), active_buffer, offset)
+        if ntodo <= 0:
+            if ntodo < 0 or offset == 0:
+                errorcode = ntodo
+                if errorcode == -99_999:
+                    debugprint0("- Failed to read file into buffer (probably due to too long reads, len(header+seq+quality) must be smaller than buffersize, default 65kB).")
+                    debugprint0("- Please increase buffersize using --buffersize.")
+                break
+            ntodo = offset
+
+        nlines = 0
+        for i in range(ntodo):
+            nlines += active_buffer[i] == 10
+        nseqs = nlines // 4
+
+        outcontrol[nactive, 7] = nseqs
+        outcontrol[nactive, 6] = 0
+
+        rm_lines = nlines - nseqs * 4
+        assert rm_lines >= 0
+        if rm_lines == 0 and active_buffer[ntodo - 1] == 10:
+            offset = 0
+        else:
+            nlines = offset = 0
+            for i in range(ntodo):
+                nlines += active_buffer[ntodo - i - 1] == 10
+                if nlines == rm_lines + 1:
+                    break
+                offset += 1
+
+    mark_buffer_for_writing(outcontrol, nactive, force=True)  # nothing left to read; re-use buffer
+    return (wait_read, wait_write, errorcode)
+
+
 @njit(nogil=True, locals=dict(
     ntodo=int32, errorcode=int32, offset=int32, wait=int64,
     wait_write=int64, sequence_id=int64, nseq=int64))
@@ -332,7 +556,7 @@ def cptask_read_fastq_seqid_pos_into_buffers(fd, inbuffers, incontrol, ininfos,
 
 # numba FASTQ handling ######################################
 
-@njit(nogil=True)
+@njit(nogil=True, locals=dict(M=int64, i=uint64, m=int64, nxt=int64, line=uint64))
 def _find_fastq_seqmarks(buf, linemarks, sequence_id=-1):
     """
     Find start and end positions of lines in byte buffer 'buf'.

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/io/generalio.py

@@ -1,5 +1,6 @@
 import sys
 from os.path import splitext
+from pathlib import Path
 from shlex import split as cmdsplit
 from subprocess import Popen, PIPE
 from shutil import which
@@ -79,8 +80,12 @@ class InputFileHandler:
             file_extension = splitext(fname)[1]
         file_type = _filetypes.get(file_extension, None)
         if file_type is None:
-            debugprint0(f"Error: InputFileHandler: Unknown file type for '{filename}'")
-            sys.exit(1)
+            if Path(filename).is_fifo():
+                file_type = 'fifo'
+                debugprint0(f'Warning: InputFileHandler: {filename} is a stream. No file type set.')
+            else:
+                debugprint0(f"Error: InputFileHandler: Unknown file type for '{filename}'")
+                sys.exit(1)
 
         if is_compressed:
             if which(tool.split()[0]) is None:

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/lowlevel/llvm.py

@@ -298,11 +298,13 @@ def compile_prefetch_array_element(nbits=None):
             sig = numba.void(numba.types.uintp)
             return sig, codegen
 
+    # ToDo make consistent to either get position or startbit!!!
+    # prefetch_array_element_ptr gets starbit and prefetch_array_element position
     if nbits is not None:
         @numba.njit(nogil=True)
         def prefetch_array_element_ptr(array_ptr, i):
-            """prefetch array element a[i]"""
-            return prefetch_address(array_ptr + (nbits * i) // 8)
+            """prefetch array element"""
+            return prefetch_address(array_ptr + i // 8)
 
         return prefetch_array_element_ptr
 

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/srhash.py

@@ -92,7 +92,7 @@ SRHash_private = namedtuple("SRHash_private", [
     "get_subkey_from_bucket_signature",
     "get_subkey_choice_from_bucket_signature",
     "prefetch_bucket",
-    "prefetch_for_key",
+    "prefetch_key",
 
     # Copies of the public functions but with subtable/subkey interface
     # (only defined for hashes with subtables; otherwise None)

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/xengsort/config/classify.yaml

@@ -16,7 +16,7 @@ params_count:
     Mn_factor: 4
     Mn_min: 3
 
-params_bisulfite:
+params_methylation:
     nothing: 0
     few: 6
     Ag: 3

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/xengsort/xengsort_classify.py

@@ -28,6 +28,9 @@ from ..lowlevel import debug
 from ..mask import create_mask
 
 
+MAXUINT64 = np.iinfo(np.uint64).max
+
+
 # ####### Classification methods ######
 def compile_count_based_classification(params):
     Mh_factor = params.get("Mh", 4)
@@ -327,8 +330,7 @@ def compile_classify_read_from_fastq(
         mode, mask, rcmode, bits, path, h, threads, pairs,
         bufsize=2**23, chunkreads=(2**23) // 200, quick=False,
         filt=False, count=False, prefetchlevel=0, params=dict(),
-        compression="gz", show_progress=False):
-
+        compression="gz", show_progress=False, use_start_sites=False):
 
     if compression != "none":
         compression = "." + compression
@@ -339,6 +341,10 @@ def compile_classify_read_from_fastq(
     ba = None
     count_kmers = True
 
+    if mode != "methylation" and use_start_sites:
+        debugprint0('Using start_sites and without a methylation index is not supported')
+        exit(1)
+
     if mode == "count":
         debugprint0("- Using count based classification mode")
         classify = compile_count_based_classification(params)
@@ -351,8 +357,8 @@ def compile_classify_read_from_fastq(
     elif mode == "quick":
         debugprint0("- using quick classification mode")
         assert False
-    elif mode == 'bisulfite':
-        debugprint0("- Using count based classification mode for bisulfite sequencing data")
+    elif mode == 'methylation':
+        debugprint0("- Using count based classification mode for methylation sequencing data")
         classify = compile_count_based_classification(params)
         ct_table = _TABLE_DNA_TO_2BITS.copy()
         ct_table[99] = 3
@@ -363,13 +369,39 @@ def compile_classify_read_from_fastq(
         ga_table[103] = 0
         ga_table[71] = 0
         quick_dna_to_2bits_ga = compile_quick_dna_to_2bits(ga_table)
+
+        # Classification matrix for pairs
+        # H: Host G: Graft A: Ambiguous B: Both N: Neither
+        # Given the two classifications of the read pair, we return the value in the matrix
+        #     H G A B N
+        #     0 1 2 3 4
+        # H 0 0 2 0 0 0
+        # G 1 2 1 1 1 1
+        # A 2 0 1 2 2 2
+        # B 3 0 1 2 3 2
+        # N 4 0 1 2 2 4
+        matrix = [[0, 2, 0, 0, 0], [2, 1, 1, 1, 1], [0, 1, 2, 2, 2], [0, 1, 2, 3, 2], [0, 1, 2, 2, 4]]
+
+        methylation_paired = np.array(matrix, dtype=np.uint8)
+
+        if use_start_sites:
+            start_sites = np.frombuffer(b"CGGTGG", dtype=np.uint8)
     else:
         raise ValueError(f"Unknown mode '{mode}'")
 
+    @njit(nogil=True, locals=dict(code=uint64))
+    def _collect_code(_, code, pos, codes):
+        """Store the k-mer code at its sequence position; hash lookup deferred."""
+        codes[pos] = code
+
+    _, collect_kmers = compile_positional_kmer_processor(mask.tuple, _collect_code, rcmode=rcmode, new_encoding=False)
+
     get_kmer_values = compile_get_kmer_values(mask, rcmode, h, prefetchlevel, count_kmers, ba)
     _, twobit_to_code = compile_twobit_to_codes(mask, rcmode)
 
     get_value = h.get_value
+    prefetch_key = h.private.prefetch_key
+    LOOKAHEAD = 16
     k, w = mask.k, mask.w
 
 
@@ -422,7 +454,7 @@ def compile_classify_read_from_fastq(
     @njit(nogil=True, locals=dict(
         third_kmer=uint64, third=uint64,
         thirdlast_kmer=uint64, thirdlast=uint64))
-    def get_bisulfite_classification(ht, sq, kcount):
+    def get_methylation_classification(ht, sq, kcount):
         # if quick:
         #     # get 2 k-mers (3rd and 3rd-last)
         #     third_kmer = twobit_to_code(sq, 2)
@@ -490,33 +522,132 @@ def compile_classify_read_from_fastq(
                 sq = buf[linemarks[i, 0]:linemarks[i, 1]]
                 classifications[i] = get_classification(ht, sq, counts)
                 twobits_to_dna_inplace(buf, linemarks[i, 0], linemarks[i, 1])
-        elif mode == "bisulfite":
+        elif mode == "methylation":
             local_seq = np.empty(250, dtype=np.uint8)
-            for i in range(n):
-                seq_len = linemarks[i, 1] - linemarks[i, 0]
-
+            counts2 = np.zeros(8, dtype=np.uint32)
+            max_kmers = buf.size // 2  # Upper limit for number of k-mers in a buffer. Buffer size divided by two for qualities
+            codes_ct = np.empty(max_kmers, dtype=np.uint64)
+            codes_ga = np.empty(max_kmers, dtype=np.uint64)
+            start_codes = 0
+            # pass 1.
+
+            # Does it make sense to process the buffer twice for ct and ga so the codes_ arrays are linear?
+            for readnum in range(n):
+
+                if use_start_sites:
+                    if not ((buf[linemarks[readnum, 0]:linemarks[readnum, 0] + 3] == start_sites[:3]).all() or \
+                       (buf[linemarks[readnum, 0]:linemarks[readnum, 0] + 3] == start_sites[3:]).all()):
+                        continue
+                seq_len = linemarks[readnum, 1] - linemarks[readnum, 0]
+                nk = seq_len - w + 1
                 if seq_len > local_seq.size:
                     debugprint2('WGBS, local sequence length increased to ', seq_len * 2)
                     local_seq = np.empty(seq_len * 2, dtype=np.uint8)
-
-                counts[:] = 0
-                # C -> T
-                local_seq[:seq_len] = buf[linemarks[i, 0]:linemarks[i, 1]]
+                local_seq[:seq_len] = buf[linemarks[readnum, 0]:linemarks[readnum, 1]]
+                end_codes = start_codes + nk
                 quick_dna_to_2bits_ct(local_seq[:seq_len])
-                get_bisulfite_classification(ht, local_seq[:seq_len], counts)
+                codes_ct[start_codes:end_codes] = MAXUINT64
+                collect_kmers(None, local_seq[:seq_len], 0, seq_len, codes_ct[start_codes:end_codes])
 
-                # G -> A
-                local_seq[:seq_len] = buf[linemarks[i, 0]:linemarks[i, 1]]
+                local_seq[:seq_len] = buf[linemarks[readnum, 0]:linemarks[readnum, 1]]
                 quick_dna_to_2bits_ga(local_seq[:seq_len])
-                get_bisulfite_classification(ht, local_seq[:seq_len], counts)
-                r1_clasification = classify(counts)
-                classifications[i] = r1_clasification
+                codes_ga[start_codes:end_codes] = MAXUINT64
+                collect_kmers(None, local_seq[:seq_len], 0, seq_len, codes_ga[start_codes:end_codes])
+                start_codes = end_codes
+
+            total_kmers = start_codes
+            # 2. Pass: classify all reads
+            start_codes = 0
+            found = 0
+            not_found = 0
+            for readnum, (start_seq, end_seq, start_record, end_record) in enumerate(linemarks):
+                if use_start_sites:
+                    if not ((buf[linemarks[readnum, 0]:linemarks[readnum, 0] + 3] == start_sites[:3]).all() or \
+                       (buf[linemarks[readnum, 0]:linemarks[readnum, 0] + 3] == start_sites[3:]).all()):
+                        classifications[readnum] = 5  # no start site
+                        continue
+                counts[:] = 0
+                counts2[:] = 0
+                nk = end_seq - start_seq - w + 1
+
+                for pos in range(nk):
+                    code_pos = pos + start_codes
+                    if code_pos + LOOKAHEAD < total_kmers:
+                        ahead_code = codes_ct[code_pos + LOOKAHEAD]
+                        if ahead_code != MAXUINT64:
+                            prefetch_key(ht, ahead_code)
+                        ahead_code = codes_ga[code_pos + LOOKAHEAD]
+                        if ahead_code != MAXUINT64:
+                            prefetch_key(ht, ahead_code)
+
+                    code = codes_ct[code_pos]
+                    if code != MAXUINT64:
+                        found += 1
+                        v = get_value(ht, code)
+                        counts[v] += 1
+                    else:
+                        not_found += 1
+
+                    code = codes_ga[code_pos]
+                    if code != MAXUINT64:
+                        found += 1
+                        v = get_value(ht, code)
+                        counts2[v] += 1
+                    else:
+                        not_found += 1
+                if counts[0] < counts2[0] // 2:
+                    classification = classify(counts)
+                elif counts2[0] < counts[0] // 2:
+                    classification = classify(counts2)
+                else:
+                    counts += counts2
+                    classification = classify(counts)
+
+                classifications[readnum] = classification
+                start_codes += nk
 
         else:
             raise ValueError(f"Classification method {mode} is not supported.")
 
         return threadid, (classifications, linemarks)
 
+    @njit(nogil=True)
+    def code_ct(buf, seq_len, local_seq, codes, start_codes, end_codes):
+        codes[start_codes:end_codes] = MAXUINT64
+        local_seq[:seq_len] = buf
+        quick_dna_to_2bits_ct(local_seq[:seq_len])
+        codes[start_codes:end_codes] = MAXUINT64
+        collect_kmers(None, local_seq[:seq_len], 0, seq_len, codes[start_codes:end_codes])
+
+    @njit(nogil=True)
+    def code_ga(buf, seq_len, local_seq, codes, start_codes, end_codes):
+        codes[start_codes:end_codes] = MAXUINT64
+        local_seq[:seq_len] = buf
+        quick_dna_to_2bits_ga(local_seq[:seq_len])
+        codes[start_codes:end_codes] = MAXUINT64
+        collect_kmers(None, local_seq[:seq_len], 0, seq_len, codes[start_codes:end_codes])
+
+    @njit(nogil=True)
+    def prefetch_index(ht, arr, index):
+        ahead_code = arr[index]
+        if ahead_code != MAXUINT64:
+            prefetch_key(ht, ahead_code)
+
+    @njit(nogil=True)
+    def get_v(ht, code, counts):
+        if code != MAXUINT64:
+            v = get_value(ht, code)
+            counts[v] += 1
+
+    @njit(nogil=True)
+    def meth_classify(counts_ct, counts_ga):
+        if counts_ct[0] < counts_ga[0] // 2:
+            return classify(counts_ct)
+        if counts_ga[0] < counts_ct[0] // 2:
+            return classify(counts_ga)
+        counts_ct += counts_ga
+        return classify(counts_ct)
+
     @njit(nogil=True)
     def classify_paired_kmers_chunkwise(threadid, buf, linemarks, buf1, linemarks1, ht,
          ba_neither=None, ba_stronghost=None, ba_stronggraft=None, ba_both=None,
@@ -580,47 +711,86 @@ def compile_classify_read_from_fastq(
                 classifications[i] = get_paired_classification(ht, sq1, sq2, counts)
                 twobits_to_dna_inplace(buf, linemarks[i, 0], linemarks[i, 1])
                 twobits_to_dna_inplace(buf1, linemarks1[i, 0], linemarks1[i, 1])
-        elif mode == "bisulfite":
-
+        elif mode == "methylation":
+
+            counts_ct_r1 = np.zeros(8, dtype=np.uint32)
+            counts_ga_r1 = np.zeros(8, dtype=np.uint32)
+            counts_ct_r2 = np.zeros(8, dtype=np.uint32)
+            counts_ga_r2 = np.zeros(8, dtype=np.uint32)
+
+            max_kmers = buf.size // 2  # Upper limit for number of k-mers in a buffer. Buffer size divided by two for qualities
+            codes_ct_r1 = np.empty(max_kmers, dtype=np.uint64)
+            codes_ct_r2 = np.empty(max_kmers, dtype=np.uint64)
+            codes_ga_r1 = np.empty(max_kmers, dtype=np.uint64)
+            codes_ga_r2 = np.empty(max_kmers, dtype=np.uint64)
             local_seq = np.empty(250, dtype=np.uint8)
-            for i in range(n):
-                seq_len = linemarks[i, 1] - linemarks[i, 0]
-                seq_len1 = linemarks1[i, 1] - linemarks1[i, 0]
 
-                max_seq_len = max(seq_len, seq_len1)
-                if max_seq_len > local_seq.size:
-                    debugprint2('WGBS, local sequence length increased to ', max_seq_len * 2)
-                    local_seq = np.empty(max_seq_len * 2, dtype=np.uint8)
+            start_codes = 0
+            # pass 1.
 
-                # R1
-                counts[:] = 0
-                # C -> T
-                local_seq[:seq_len] = buf[linemarks[i, 0]:linemarks[i, 1]]
-                quick_dna_to_2bits_ct(local_seq[:seq_len])
-                q1 = get_bisulfite_classification(ht, local_seq[:seq_len], counts)
-                # G -> A
-                local_seq[:seq_len] = buf[linemarks[i, 0]:linemarks[i, 1]]
-                quick_dna_to_2bits_ct(local_seq[:seq_len])
-                q2 = get_bisulfite_classification(ht, local_seq[:seq_len], counts)
-                r1_clasification = classify(counts)
+            for readnum in range(n):
+                seq_len_r1 = linemarks[readnum, 1] - linemarks[readnum, 0]
+                seq_len_r2 = linemarks1[readnum, 1] - linemarks1[readnum, 0]
+                seq_len = max(seq_len_r1, seq_len_r2)
+                nk = seq_len - w + 1
+
+                if use_start_sites:
+                    if not ((buf[linemarks[readnum, 0]:linemarks[readnum, 0] + 3] == start_sites[:3]).all() or \
+                       (buf[linemarks[readnum, 0]:linemarks[readnum, 0] + 3] == start_sites[3:]).all()):
+                        continue
+
+                if seq_len > local_seq.size:
+                    debugprint2('WGBS, local sequence length increased to ', seq_len * 2)
+                    local_seq = np.empty(seq_len * 2, dtype=np.uint8)
+
+                end_codes = start_codes + nk
+
+                code_ct(buf[linemarks[readnum, 0]:linemarks[readnum, 1]], seq_len_r1, local_seq, codes_ct_r1, start_codes, end_codes)
+                code_ga(buf[linemarks[readnum, 0]:linemarks[readnum, 1]], seq_len_r1, local_seq, codes_ga_r1, start_codes, end_codes)
+
+                code_ct(buf1[linemarks1[readnum, 0]:linemarks1[readnum, 1]], seq_len_r2, local_seq, codes_ct_r2, start_codes, end_codes)
+                code_ga(buf1[linemarks1[readnum, 0]:linemarks1[readnum, 1]], seq_len_r2, local_seq, codes_ga_r2, start_codes, end_codes)
+
+                start_codes = end_codes
+
+            total_kmers = start_codes
+            # 2. Pass: classify all reads
+            start_codes = 0
+            for readnum in range(n):
+
+                if use_start_sites:
+                    if not ((buf[linemarks[readnum, 0]:linemarks[readnum, 0] + 3] == start_sites[:3]).all() or \
+                       (buf[linemarks[readnum, 0]:linemarks[readnum, 0] + 3] == start_sites[3:]).all()):
+                        classifications[readnum] = 5  # no start site
+                        continue
+                seq_len_r1 = linemarks[readnum, 1] - linemarks[readnum, 0]
+                seq_len_r2 = linemarks1[readnum, 1] - linemarks1[readnum, 0]
+                seq_len = max(seq_len_r1, seq_len_r2)
+                nk = seq_len - w + 1
+
+                counts_ct_r1[:] = 0
+                counts_ga_r1[:] = 0
+                counts_ct_r2[:] = 0
+                counts_ga_r2[:] = 0
+
+                for pos in range(nk):
+                    code_pos = pos + start_codes
+                    if code_pos + LOOKAHEAD < total_kmers:
+                        prefetch_index(ht, codes_ct_r1, code_pos + LOOKAHEAD)
+                        prefetch_index(ht, codes_ga_r1, code_pos + LOOKAHEAD)
+                        prefetch_index(ht, codes_ct_r2, code_pos + LOOKAHEAD)
+                        prefetch_index(ht, codes_ga_r2, code_pos + LOOKAHEAD)
+
+                    get_v(ht, codes_ct_r1[code_pos], counts_ct_r1)
+                    get_v(ht, codes_ga_r1[code_pos], counts_ga_r1)
+                    get_v(ht, codes_ct_r2[code_pos], counts_ct_r2)
+                    get_v(ht, codes_ga_r2[code_pos], counts_ga_r2)
+                start_codes += nk
+                r1_classification = meth_classify(counts_ct_r1, counts_ga_r1)
+                r2_classification = meth_classify(counts_ct_r2, counts_ga_r2)
+
+                classifications[readnum] = methylation_paired[r1_classification, r2_classification]
 
-                # R2
-                counts[:] = 0
-                # C -> T
-                local_seq[:seq_len1] = buf1[linemarks1[i, 0]:linemarks1[i, 1]]
-                quick_dna_to_2bits_ga(local_seq[:seq_len1])
-                get_bisulfite_classification(ht, local_seq[:seq_len1], counts)
-
-                # G -> A
-                local_seq[:seq_len1] = buf1[linemarks1[i, 0]:linemarks1[i, 1]]
-                quick_dna_to_2bits_ga(local_seq[:seq_len1])
-                get_bisulfite_classification(ht, local_seq[:seq_len1], counts)
-                r2_clasification = classify(counts)
-
-                if r1_clasification == r2_clasification:
-                    classifications[i] = r1_clasification
-                else:
-                    classifications[i] = 2  # ambiguous
         else:
             raise ValueError(f"{mode} is not a supported classification mode")
 
@@ -667,8 +837,10 @@ def compile_classify_read_from_fastq(
                 yield outfile.file
 
     def classify_read_from_fastq_paired(fastqs, pairs, ht):
-        counts = [0, 0, 0, 0, 0]  # host, graft, amb., both, neither
+        counts = [0, 0, 0, 0, 0, 0]  # host, graft, amb., both, neither
         nprocessed = 0
+
+        site_count = count and not use_start_sites
         with cond_contextmgr(path, f"-host.1.fq{compression}", count, filt) as host1, \
              cond_contextmgr(path, f"-host.2.fq{compression}", count, filt) as host2, \
              cond_contextmgr(path, f"-graft.1.fq{compression}", count, filt) as graft1, \
@@ -678,8 +850,10 @@ def compile_classify_read_from_fastq(
              cond_contextmgr(path, f"-both.1.fq{compression}", count, filt) as both1, \
              cond_contextmgr(path, f"-both.2.fq{compression}", count, filt) as both2, \
              cond_contextmgr(path, f"-neither.1.fq{compression}", count, filt) as neither1, \
-             cond_contextmgr(path, f"-neither.2.fq{compression}", count, filt) as neither2:
-            streams = ((host1, host2), (graft1, graft2), (ambiguous1, ambiguous2), (both1, both2), (neither1, neither2))
+             cond_contextmgr(path, f"-neither.2.fq{compression}", count, filt) as neither2, \
+             cond_contextmgr(path, f"-sites.1.fq{compression}", site_count, filt) as sites1, \
+             cond_contextmgr(path, f"-sites.2.fq{compression}", site_count, filt) as sites2:
+            streams = ((host1, host2), (graft1, graft2), (ambiguous1, ambiguous2), (both1, both2), (neither1, neither2), (sites1, sites2))
 
             with ThreadPoolExecutor(max_workers=threads) as executor:
                 for fastq1, fastq2 in zip(fastqs, pairs):
@@ -734,14 +908,16 @@ def compile_classify_read_from_fastq(
             return counts
 
     def classify_read_from_fastq_single(fastqs, ht):
-        counts = [0, 0, 0, 0, 0]
+        counts = [0, 0, 0, 0, 0, 0]
         nprocessed = 0
+        site_count = count and not use_start_sites
         with cond_contextmgr(path, f"-host.fq{compression}", count, filt) as host, \
              cond_contextmgr(path, f"-graft.fq{compression}", count, filt) as graft, \
              cond_contextmgr(path, f"-ambiguous.fq{compression}", count, filt) as ambiguous, \
              cond_contextmgr(path, f"-both.fq{compression}", count, filt) as both, \
-             cond_contextmgr(path, f"-neither.fq{compression}", count, filt) as neither:
-            streams = (host, graft, ambiguous, both, neither)
+             cond_contextmgr(path, f"-neither.fq{compression}", count, filt) as neither, \
+             cond_contextmgr(path, f"-sites.fq{compression}", site_count, filt) as sites:
+            streams = (host, graft, ambiguous, both, neither, sites)
 
             running_jobs = []
             with ThreadPoolExecutor(max_workers=threads) as executor:
@@ -794,24 +970,26 @@ def compile_classify_read_from_fastq(
     return classify_read_from_fastq
 
 
-def print_class_stats(prefix, stats):
-    classes = ["host", "graft", "ambiguous", "both", "neither"]
+def print_class_stats(prefix, stats, sites):
+    classes = ["host", "graft", "ambiguous", "both", "neither", 'sites']
 
+    nstats = len(stats) - (not sites)
+    stats = stats[:nstats]
     percentages = [i / sum(stats) * 100 for i in stats]
     str_counts = "\t".join(str(i) for i in stats)
     ndigits = max(map(lambda x: len(str(x)), stats))
 
-    print("\n## Classification Statistics")
-    print("\n```")
-    print("prefix\thost\tgraft\tambiguous\tboth\tneither")
-    print(f"{prefix}\t{str_counts}")
-    print("```\n")
-    print("```")
-    print(f"| prefix    | {prefix} ")
-    for i in range(len(classes)):
-        print(f"| {classes[i]:9s} | {stats[i]:{ndigits}d} | {percentages[i]:5.2f}% |")
-    print("```")
-    print()
+    debugprint0("\n## Classification Statistics")
+    debugprint0("\n```")
+    debugprint0("prefix\thost\tgraft\tambiguous\tboth\tneither")
+    debugprint0(f"{prefix}\t{str_counts}")
+    debugprint0("```\n")
+    debugprint0("```")
+    debugprint0(f"| prefix    | {prefix} ")
+    for i in range(len(classes) - (not sites)):
+        debugprint0(f"| {classes[i]:9s} | {stats[i]:{ndigits}d} | {percentages[i]:5.2f}% |")
+    debugprint0("```")
+    debugprint0()
 
 
 def main(args):
@@ -850,13 +1028,13 @@ def main(args):
     if mode == "quick":
         mode = "count"
         quick = True
-    if 'bisulfite' in appinfo:
-        if appinfo['bisulfite']:
-            debugprint0('- The provided index is build for WGBS analysis.\n The classification mode will be switched to bisulfite.')
-            mode = 'bisulfite'
-    if mode == 'bisulfite':
+    if 'methylation' in appinfo:
+        if appinfo['methylation']:
+            debugprint0('- The provided index is build for WGBS analysis.\n The classification mode will be switched to methylation.')
+            mode = 'methylation'
+    if mode == 'methylation':
         params = args["params_" + 'count']
-        params = args["params_" + 'bisulfite']
+        params = args["params_" + 'methylation']
     else:
         params = args["params_" + mode]
     if not args.fastq:
@@ -893,13 +1071,13 @@ def main(args):
         bufsize=chunksize, chunkreads=chunkreads, quick=quick,
         filt=args.filter, count=args.count, prefetchlevel=args.prefetchlevel,
         params=params, compression=args.compression,
-        show_progress=args.progress)
+        show_progress=args.progress, use_start_sites=args.use_start_sites)
 
     if paired:
         counts = classify_read_from_fastq(args.fastq, args.pairs, h.hashtable)
     else:
         counts = classify_read_from_fastq(args.fastq, h.hashtable)
-    print_class_stats(args.prefix, counts)
+    print_class_stats(args.prefix, counts, args.use_start_sites)
 
     debugprint0("## Running time statistics\n")
     timestamp0(starttime, msg="- Running time")

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/xengsort/xengsort_index.py

@@ -104,7 +104,7 @@ def create_new_index(nsubtables, args):
     return (h, values, valueset, mask, rcmode)
 
 
-def process_files(constant_value, h, fnames, mask, rcmode, bisulfite, *,
+def process_files(constant_value, h, fnames, mask, rcmode, methylation, *,
         maxfailures=0, maxwalk=1000,
         threads_read=1, threads_split=1):
 
@@ -134,7 +134,7 @@ def process_files(constant_value, h, fnames, mask, rcmode, bisulfite, *,
     outbufsize = 2**16
     cptask_split = compile_cptask_scatter_kmers_from_linemarked(
         mask, rcmode, hashfunc0,
-        nsubtables, nbuffers_per_subtable, outbufsize, bisulfite=bisulfite)
+        nsubtables, nbuffers_per_subtable, outbufsize, methylation=methylation)
 
     splitter_jobs = ConsumerProducer(
         name='kmer_splitter',
@@ -189,7 +189,7 @@ def main(args):
 
     # process all host files
     if args.host:
-        success_host = process_files(1, h, args.host, mask, rcmode, args.bisulfite,
+        success_host = process_files(1, h, args.host, mask, rcmode, args.methylation,
             maxfailures=0, maxwalk=args.maxwalk,
             threads_read=threads_read, threads_split=threads_split)
 
@@ -198,7 +198,7 @@ def main(args):
             exit(1)
 
     # process all graft files
-    success_graft = process_files(2, h, args.graft, mask, rcmode, args.bisulfite,
+    success_graft = process_files(2, h, args.graft, mask, rcmode, args.methylation,
         maxfailures=0, maxwalk=args.maxwalk,
         threads_read=threads_read, threads_split=threads_split)
 
@@ -220,7 +220,7 @@ def main(args):
     # #################### Save Hash ####################
 
     optinfo = dict(walkseed=args.walkseed, maxwalk=args.maxwalk, maxfailures=0)
-    appinfo = dict(rcmode=rcmode, mask=mask.tuple, k=mask.k, bisulfite=args.bisulfite)
+    appinfo = dict(rcmode=rcmode, mask=mask.tuple, k=mask.k, methylation=args.methylation)
     save_hash(args.index, h, valuetup, optinfo, appinfo)
 
     print_statistics(h, level=args.statistics)

{xengsort-2.1.0 → xengsort-2.2.0}/xengsort/xengsort/xengsort_main.py

@@ -40,7 +40,7 @@ def classify(p):
         choices=CL_CHOICES, help=f"classification mode {CL_CHOICES}")
     p.add_argument("--params_count", type=dict, help=SUPPRESS)
         # help="All paramteres for classification based on counts as a dictionary. Update in config file.")
-    p.add_argument("--params_bisulfite", type=dict, help=SUPPRESS)
+    p.add_argument("--params_methylation", type=dict, help=SUPPRESS)
         # help="All paramteres for classification based on counts as a dictionary. Update in config file.")
     p.add_argument("--params_coverage", type=dict, help=SUPPRESS)
         # help="All paramteres for classification based on coverage as a dictionary. Update in config file.")
@@ -60,6 +60,8 @@ def classify(p):
         help="maximum number of reads per chunk per thread [SIZE_MB*(2**20) // 200]")
     p.add_argument("--progress", "-P", action="store_true",
         help="Show progress")
+    p.add_argument("--use-start-sites", action="store_true",
+        help="Filter read that do not start with the expected start sites.")
 
 
 def index(p):
@@ -118,7 +120,7 @@ def index(p):
         help="continue even after this many failures; forever: -1]")
     p.add_argument("--walkseed", type=int, metavar="INT",
         help="seed for random walks while inserting elements")
-    p.add_argument("--bisulfite", action='store_true',
+    p.add_argument("--bisulfite", "--methylation", action='store_true', dest="methylation",
         help="filtering WGBS data with translated index (EXPERIMENTAL)")
 
 

{xengsort-2.1.0 → xengsort-2.2.0/xengsort.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xengsort
-Version: 2.1.0
+Version: 2.2.0
 Summary: xenograft sorting of FASTQ reads into five categories: host, graft, both, neither, ambiguous.
 Author-email: Jens Zentgraf <zentgraf@cs.uni-saarland.de>, Sven Rahmann <rahmann@cs.uni-saarland.de>
 License: MIT License