xengsort 2.0.8__tar.gz → 2.1.0__tar.gz

{xengsort-2.0.8/xengsort.egg-info → xengsort-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: xengsort
-Version: 2.0.8
+Version: 2.1.0
 Summary: xenograft sorting of FASTQ reads into five categories: host, graft, both, neither, ambiguous.
 Author-email: Jens Zentgraf <zentgraf@cs.uni-saarland.de>, Sven Rahmann <rahmann@cs.uni-saarland.de>
 License: MIT License
@@ -43,6 +43,7 @@ Requires-Dist: numba
 Requires-Dist: pytest
 Requires-Dist: jsonargparse>=4.29.0
 Requires-Dist: pip
+Dynamic: license-file
 
 # xengsort: Fast lightweight accurate xenograft sorting
 
@@ -286,6 +287,21 @@ Further parameters and options are:
 - `--filter`: With this flag, only the graft reads are output.
 - `--count`: With this flag, only the number of reads in each category is counted, but the reads are not sorted.
 
+### How to filter bisulfite sequencing data
+In bisulfite sequencing data we have a conversion from C to T and from G to A in the reverse complement. The standard xengsort index classifies most of the reads as `neither`.
+To address this issue, we have created a `--bisulfite` mode.
+This parameter must be set during the index building step.
+If set, all Cs are translated to Ts in the reference and add the $k$-mers are added to the index, the same for G to A.
+This nearly doubles the number of $k$-mers in the index (`-n`)
+
+```
+xengsort index --index myindex -H host.fa.gz -G graft.fa.gz -n 9_000_000_000 --bisulfite [OPTIONS]
+```
+
+If an index is build using the bisulfite parameter, also the read is converted in the classification step.
+It is not necessary to change the parameters of the classification step.
+Currently, it only supports the count-based classification.
+
 
 Please report a [GitLab issue](https://gitlab.com/genomeinformatics/xengsort/-/issues) in case of problems.
 

{xengsort-2.0.8 → xengsort-2.1.0}/README.md

@@ -240,6 +240,21 @@ Further parameters and options are:
 - `--filter`: With this flag, only the graft reads are output.
 - `--count`: With this flag, only the number of reads in each category is counted, but the reads are not sorted.
 
+### How to filter bisulfite sequencing data
+In bisulfite sequencing data we have a conversion from C to T and from G to A in the reverse complement. The standard xengsort index classifies most of the reads as `neither`.
+To address this issue, we have created a `--bisulfite` mode.
+This parameter must be set during the index building step.
+If set, all Cs are translated to Ts in the reference and add the $k$-mers are added to the index, the same for G to A.
+This nearly doubles the number of $k$-mers in the index (`-n`)
+
+```
+xengsort index --index myindex -H host.fa.gz -G graft.fa.gz -n 9_000_000_000 --bisulfite [OPTIONS]
+```
+
+If an index is build using the bisulfite parameter, also the read is converted in the classification step.
+It is not necessary to change the parameters of the classification step.
+Currently, it only supports the count-based classification.
+
 
 Please report a [GitLab issue](https://gitlab.com/genomeinformatics/xengsort/-/issues) in case of problems.
 

{xengsort-2.0.8 → xengsort-2.1.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "xengsort"
-version = "2.0.8"
+version = "2.1.0"
 
 license = {file = "LICENSE"}
 

{xengsort-2.0.8 → xengsort-2.1.0}/xengsort/cptasks_kmers.py

@@ -4,15 +4,18 @@ Functions for compiling ConsumerProducerTasks for k-mers
 """
 
 import numpy as np
-from numba import njit, int32, int64, uint64
+from numba import njit, uint8, int32, int64, uint64
 
-from .dnaencode import compile_revcomp_and_canonical_code
+from .dnaencode import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_old
+from .dnaencode import _TABLE_DNA_TO_2BITS, compile_quick_dna_to_2bits
+from .dnaencode import quick_dna_to_2bits as quick_dna_to_2bits_old
+from .dnaencode_fast import compile_revcomp_and_canonical_code as compile_revcomp_and_canonical_code_new
+from .dnaencode_fast import quick_dna_to_2bits as quick_dna_to_2bits_new
 from .lowlevel import debug
 from .lowlevel.conpro import \
     find_buffer_for_reading, find_buffer_for_writing, \
     mark_buffer_for_reading, mark_buffer_for_writing, \
     mark_my_buffers_failed
-from .dnaencode import quick_dna_to_2bits
 
 
 """
@@ -28,7 +31,9 @@ compile_cptask_scatter_kmers_from_buffers():
 
 def compile_cptask_scatter_kmers_from_linemarked(
         mask, rcmode, hf0,
-        nsubtables, noutbuffers_per_subtable, outbufsize):
+        nsubtables, noutbuffers_per_subtable, outbufsize,
+        new_encoding=False,
+        bisulfite=False):
     """
     Compile a cptask function that
     creates (gapped) k-mers from linemarked buffers.
@@ -38,16 +43,33 @@ def compile_cptask_scatter_kmers_from_linemarked(
     - nsubtables: the number of subtables
     - noutbuffers_per_subtable: number of output buffers per subtable
     - outbufsize: the output buffer size for which to compile this function
+    -
     """
     debugprint0, debugprint1, debugprint2 = debug.debugprint
     both = (rcmode[0].lower() == "b")
+    if bisulfite:
+        if new_encoding:
+            raise NotImplementedError('WGBS with new encoding not supported')
+
+        else:
+            ct_table = _TABLE_DNA_TO_2BITS.copy()
+            ct_table[99] = 3
+            ct_table[67] = 3
+            quick_dna_to_2bits_ct = compile_quick_dna_to_2bits(ct_table)
+
+            ga_table = _TABLE_DNA_TO_2BITS.copy()
+            ga_table[103] = 0
+            ga_table[71] = 0
+            quick_dna_to_2bits_ga = compile_quick_dna_to_2bits(ga_table)
+    else:
+        quick_dna_to_2bits = quick_dna_to_2bits_new if new_encoding else quick_dna_to_2bits_old
 
     k, w, tpl = mask.k, mask.w, mask.tuple
     assert len(tpl) == k
     if k < 1 or k > 32:
         raise ValueError(f"Only 1 <= k <= 32 is supported, but {k=}.")
     codemask = uint64(4**(k - 1) - 1)
-    revcomp, ccode = compile_revcomp_and_canonical_code(k, rcmode)
+    revcomp, ccode = compile_revcomp_and_canonical_code_new(k, rcmode) if new_encoding else compile_revcomp_and_canonical_code_old(k, rcmode)
     if mask.is_contiguous:
         debugprint1(f"- compiling cptask_scatter_kmers for contiguous {k}-mers.")
     else:
@@ -55,7 +77,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
 
     @njit(nogil=True, locals=dict(
         code=uint64, st=uint64, sk=uint64,
-        j=int32, stj=int32, p=int32, wait=int64))
+        j=int32, stj=int32, p=int32, wait=int64, error=int32))
     def _store(code, outidx, outpos, outbuffers, outcontrol):
         st, sk = hf0(code)
         j = outidx[st]
@@ -180,6 +202,8 @@ def compile_cptask_scatter_kmers_from_linemarked(
         nactive = -1  # active input buffer
         outidx = np.full(nsubtables, (-1), dtype=np.int32)
         outpos = np.full(nsubtables, outbufsize, dtype=np.int32)
+        if bisulfite:
+            seq_buffer = np.empty(inbuffers[0].size, dtype=np.uint8)
         while True:
             nactive, wait = find_buffer_for_reading(incontrol, nactive)
             wait_read += wait
@@ -193,10 +217,22 @@ def compile_cptask_scatter_kmers_from_linemarked(
                 errorcode = 0
                 linemarks = linemarkbuffers[nactive]
                 for i in range(nseqs):
-                    sq = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
-                    quick_dna_to_2bits(sq)
-                    (wait, _errorcode) = _process(
-                        sq, outidx, outpos, outbuffers, outcontrol)
+                    if bisulfite:
+                        seq_len = linemarks[i, 1] - linemarks[i, 0]
+                        seq_buffer[0:seq_len] = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
+                        quick_dna_to_2bits_ct(seq_buffer[0:seq_len])
+                        (wait, _errorcode) = _process(
+                            seq_buffer[:seq_len], outidx, outpos, outbuffers, outcontrol)
+
+                        seq_buffer[0:seq_len] = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
+                        quick_dna_to_2bits_ga(seq_buffer[0:seq_len])
+                        (wait, _errorcode) = _process(
+                            seq_buffer[:seq_len], outidx, outpos, outbuffers, outcontrol)
+                    else:
+                        sq = active_buffer[linemarks[i, 0]:linemarks[i, 1]]
+                        quick_dna_to_2bits(sq)
+                        (wait, _errorcode) = _process(
+                            sq, outidx, outpos, outbuffers, outcontrol)
                     wait_write += wait
                     errorcode |= _errorcode
                 if errorcode:
@@ -229,6 +265,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
                 if j >= 0:
                     # assert outpos[st] > 0
                     outcontrol[j, 7] = outpos[st]
+                    outcontrol[j, 6] = 1  # buffer type k-mer codes
                     mark_buffer_for_reading(outcontrol, j)
             debugprint2("- ending: cptask_scatter_kmers", errorcode)
         return (wait_read, wait_write, -errorcode)
@@ -238,7 +275,7 @@ def compile_cptask_scatter_kmers_from_linemarked(
 
 def compile_cptask_scatter_kmers_position_from_linemarked(
         mask, values, rcmode, hf0,
-        nsubtables, noutbuffers_per_subtable, outbufsize):
+        nsubtables, noutbuffers_per_subtable, outbufsize, new_encoding=False):
     """
     Compile a cptask function that
     creates (gapped) k-mers and value from linemarked buffers.
@@ -251,14 +288,16 @@ def compile_cptask_scatter_kmers_position_from_linemarked(
     - outbufsize: the output buffer size for which to compile this function
     """
     debugprint0, debugprint1, debugprint2 = debug.debugprint
+    dassert = debug.dassert
     both = (rcmode[0].lower() == "b")
+    quick_dna_to_2bits = quick_dna_to_2bits_new if new_encoding else quick_dna_to_2bits_old
 
     k, w, tpl = mask.k, mask.w, mask.tuple
     assert len(tpl) == k
     if k < 1 or k > 32:
         raise ValueError(f"Only 1 <= k <= 32 is supported, but {k=}.")
     codemask = uint64(4**(k - 1) - 1)
-    revcomp, ccode = compile_revcomp_and_canonical_code(k, rcmode)
+    revcomp, ccode = compile_revcomp_and_canonical_code_new(k, rcmode) if new_encoding else compile_revcomp_and_canonical_code_old(k, rcmode)
     if mask.is_contiguous:
         debugprint1(f"- compiling cptask_scatter_kmers for contiguous {k}-mers.")
     else:
@@ -281,9 +320,9 @@ def compile_cptask_scatter_kmers_position_from_linemarked(
                 outcontrol[j, 6] = 1  # buffer type k-mer codes
                 mark_buffer_for_reading(outcontrol, j)
                 stj = j - st * noutbuffers_per_subtable  # the "local" j for subtable st
-                assert 0 <= stj < noutbuffers_per_subtable
+                dassert(0 <= stj < noutbuffers_per_subtable, __name__, '0 <= stj < noutbuffers_per_subtable', (stj, noutbuffers_per_subtable))
             else:
-                assert j == -1
+                dassert(j == -1, __name__, 'j == -1', j)
                 stj = -1
             stoutcontrol = outcontrol[(st * noutbuffers_per_subtable):(st + 1) * noutbuffers_per_subtable]
             stj, wait = find_buffer_for_writing(stoutcontrol, stj)
@@ -382,14 +421,15 @@ def compile_cptask_scatter_kmers_position_from_linemarked(
         # ininfos[i]: linemarks for input buffer i
         # outcontrol[B, 7]: k-mers in buffer B
         debugprint2("- running: cptask_scatter_kmers; outbuffers shape =", outbuffers.shape)
-        assert inbuffers.ndim == 2
-        assert outbuffers.ndim == 2
+        dassert(inbuffers.ndim == 2, __name__, 'inbuffers.ndim == 2', inbuffers.ndim, message='check input buffer dimensions')
+        dassert(outbuffers.ndim == 2, __name__, 'outbuffers.ndim == 2', outbuffers.ndim, message='check output buffer dimensions')
+
         M, N = ininfos.shape
-        assert N % 6 == 0  # We have 6 entries per linemark. Start and end of record and read and chromosome and positions
+        dassert(N % 6 == 0, __name__, 'N % 6 == 0', N, message='We have 6 entries per linemark. Start and end of record and read and chromosome and positions')
         linemarkbuffers = ininfos.reshape(M, N // 6, 6)
-        assert outbuffers.shape[0] == nsubtables * noutbuffers_per_subtable
-        assert outbuffers.shape[1] >= outbufsize
-        assert outinfos.shape[1] == 0
+        dassert(outbuffers.shape[0] == nsubtables * noutbuffers_per_subtable, __name__, 'outbuffers.shape[0] == nsubtables * noutbuffers_per_subtable', (outbuffers.shape[0], nsubtables, noutbuffers_per_subtable), message='Check number of output buffer per subtable')
+        dassert(outbuffers.shape[1] >= outbufsize, __name__, 'outbuffers.shape[1] >= outbufsize', (outbuffers.shape[1], outbufsize))
+        dassert(outinfos.shape[1] == 0, __name__, 'outinfos.shape[1] == 0', outinfos.shape[1])
 
         wait_read = wait_write = 0
         nactive = -1  # active input buffer
@@ -404,7 +444,7 @@ def compile_cptask_scatter_kmers_position_from_linemarked(
             active_buffer = inbuffers[nactive]
             nseqs = incontrol[nactive, 7]
             buffer_type = incontrol[nactive, 6]
-            assert buffer_type == 0  # 0 means sequence buffer
+            dassert(buffer_type == 0, __name__, 'buffer_type == 0', buffer_type, message='0 means sequence buffer')
 
             errorcode = 0
             linemarks = linemarkbuffers[nactive]
@@ -430,6 +470,7 @@ def compile_cptask_scatter_kmers_position_from_linemarked(
                 if j >= 0:
                     # assert outpos[st] > 0
                     outcontrol[j, 7] = outpos[st]
+                    outcontrol[j, 6] = 1  # buffer type k-mer codes
                     mark_buffer_for_reading(outcontrol, j)
             debugprint2("- ending: cptask_scatter_kmers", errorcode)
         return (wait_read, wait_write, -errorcode)
@@ -461,7 +502,6 @@ def compile_cptask_insert_subkey_value(
         assert outbuffers.shape[1] >= maxwalk + 1
         assert outinfos.shape[1] == 0
         debugprint2("- running: cptask_insert_subkey_value; subtable", st)
-
         wait_read = wait_write = 0
         nactive = -1  # active input buffer
         active_buffer = inbuffers[0]  # irrelevant
@@ -506,6 +546,7 @@ def compile_cptask_insert_filtered_subkeys(
         maxfailures=0,
         maxwalk=1000):
     debugprint0, debugprint1, debugprint2 = debug.debugprint
+    dassert = debug.dassert
     if myfilter is not None:
         wwo = "with"
         lookup_and_insert_in_subfilter = myfilter.private.lookup_and_insert_in_subfilter
@@ -552,7 +593,10 @@ def compile_cptask_insert_filtered_subkeys(
             active_buffer = inbuffers[nactive]
             nsubkeys = incontrol[nactive, 7]
             buffer_type = incontrol[nactive, 6]
-            assert buffer_type == 1  # 1 means k-mer buffer (or subkeys)
+            if buffer_type != 1:
+                print(buffer_type, nsubkeys)
+            assert buffer_type == 1
+            # dassert(buffer_type == 1, __name__, 'buffer_type == 1', buffer_type, message='0 means sequence buffer')
 
             # consume the buffer
             for subkey in active_buffer[:nsubkeys]:

{xengsort-2.0.8 → xengsort-2.1.0}/xengsort/dnaencode.py

@@ -34,6 +34,15 @@ def _dna_to_2bits(x, table):
         x[i] = table[x[i]]
 
 
+def compile_quick_dna_to_2bits(table):
+    @njit(nogil=True)
+    def _quick_dna_to_2bits(x):
+        for i in range(len(x)):
+            x[i] = table[x[i]]
+
+    return _quick_dna_to_2bits
+
+# this is the one to use!
 @njit(nogil=True)
 def quick_dna_to_2bits(x):
     for i in range(len(x)):
@@ -51,7 +60,7 @@ def parallel_dna_to_2bits(x):
 def dna_to_2bits(seq, table=_TABLE_DNA_TO_2BITS):
     # we expect seq to be a bytearray
     # xx = np.array(seq, dtype=np.uint8)
-    if type(seq) == bytes:
+    if isinstance(seq, bytes):
         xx = np.frombuffer(bytearray(seq), dtype=np.uint8)
     else:
         xx = np.frombuffer(seq, dtype=np.uint8)
@@ -70,7 +79,7 @@ def qcode_to_dnastr(qcode, q, table=_TABLE_BITS_TO_DNASTR):
 @njit(nogil=True, locals=dict(base=uint64))
 def write_qcode_to_buffer(qcode, q, buf, start):
     for i in range(q):
-        base = qcode >> (2 * (q - i - 1)) & 3
+        base = (qcode >> (2 * (q - i - 1))) & 3
         buf[start + i] = uint8(base)
 
 
@@ -245,6 +254,7 @@ def compile_twobit_to_codes(tmask, rcmode, invalid=U64_MINUSONE):
 
     @njit(nogil=True, locals=dict(code=uint64))
     def twobit_to_codes(seq, out, start=0, n=-1):
+        """write n (or all) canonical k-mer codes from seq[start...] into out buffer"""
         if n == -1:
             n = len(seq) - w + 1 - start
         for i in range(start, start + n):
@@ -260,7 +270,8 @@ def compile_twobit_to_codes(tmask, rcmode, invalid=U64_MINUSONE):
                 out[i - start] = code
 
     @njit(nogil=True, locals=dict(code=uint64))
-    def twobit_to_code(seq, start):
+    def twobit_to_code(seq, start=0):
+        """return a single canonical code at seq[start...]"""
         code = 0
         for j in tmask:
             c = seq[start + j]

xengsort-2.1.0/xengsort/dnaencode_fast.py

@@ -0,0 +1,156 @@
+"""
+Fast DNA encoding and computation of the reverse complement A=00, C=01, T=10, G=11, U=10
+Other characters are not treated correctly, except N=7>3 and n = 6>3
+"""
+
+from numba import njit, uint64
+import numpy as np
+from .mask import Mask
+
+U64_MINUSONE = uint64(np.iinfo(np.uint64).max)
+
+
+@njit(nogil=True)
+def quick_dna_to_2bits(x):
+    for i in range(len(x)):
+        x[i] = (x[i] >> 1) & 7
+
+
+@njit(nogil=True)
+def dna_to_2bits(x, y):
+    for i in range(len(x)):
+        y[i] = (x[i] >> 1) & 7
+
+
+# no need to njit!
+def _get_table_2bits_to_dna():
+    b = np.full(256, 35, dtype=np.uint8)  # fill with b'#'
+    b[0] = 65
+    b[1] = 67
+    b[2] = 84
+    b[3] = 71
+    b[6] = 78
+    b[7] = 78
+    return b
+
+
+_TABLE_2BITS_TO_DNA = _get_table_2bits_to_dna()
+
+
+@njit(nogil=True)
+def twobits_to_dna_inplace(buf, start=0, end=0):
+    if end <= 0:
+        end = len(buf) - end
+    for i in range(start, end):
+        buf[i] = _TABLE_2BITS_TO_DNA[buf[i]]
+
+
+_TABLE_BITS_TO_DNASTR = ["A", "C", "T", "G"]
+
+
+def qcode_to_dnastr(qcode, q, table=_TABLE_BITS_TO_DNASTR):
+    qc = int(qcode)
+    return "".join([table[((qc >> (2 * (q - i - 1))) & 3)] for i in range(q)])
+
+
+def compile_revcomp_and_canonical_code(q, rcmode):
+    """
+    return pair of functions (revcomp_code_q, canonical_code_q)
+    specialized for q-gram codes for the given value of q.
+    It is expected that LLVM optimization does loop unrolling.
+    """
+
+    mask = uint64(int(q * '10', 2))
+    shift = uint64(64 - 2 * q)
+
+    @njit(nogil=True, locals=dict(value=uint64))
+    def reverse_in_pairs(value):
+        value = ((value & 0x3333333333333333) << 2) | ((value & 0xCCCCCCCCCCCCCCCC) >> 2)
+        value = ((value & 0x0F0F0F0F0F0F0F0F) << 4) | ((value >> 4) & 0x0F0F0F0F0F0F0F0F)
+        value = ((value & 0x00FF00FF00FF00FF) << 8) | ((value >> 8) & 0x00FF00FF00FF00FF)
+        value = ((value & 0x0000FFFF0000FFFF) << 16) | ((value >> 16) & 0x0000FFFF0000FFFF)
+        value = (value << 32) | (value >> 32)
+        return value
+
+    @njit(nogil=True, locals=dict(value=uint64))
+    def _rc(value):
+        value ^= mask
+        value = reverse_in_pairs(value)
+        return value >> shift
+
+    if rcmode == "min":
+        @njit(nogil=True, locals=dict(
+            code=uint64, rc=uint64))
+        def _cc(code):
+            rc = _rc(code)
+            return code if code <= rc else rc
+    elif rcmode == "max":
+        @njit(nogil=True, locals=dict(
+            code=uint64, rc=uint64))
+        def _cc(code):
+            rc = _rc(code)
+            return code if code >= rc else rc
+    elif rcmode == "r":
+        @njit(nogil=True, locals=dict(
+            code=uint64, rc=uint64))
+        def _cc(code):
+            rc = _rc(code)
+            return rc
+    else:  # 'f', 'both', ...
+        @njit(nogil=True, locals=dict(
+            code=uint64, rc=uint64))
+        def _cc(code):
+            return code
+
+    return _rc, _cc
+
+
+# translation of a DNA buffer into codes
+
+
+def compile_twobit_to_codes(tmask, rcmode, invalid=U64_MINUSONE):
+    # tmask should be a mask tuple, but might be an int in some cases
+    if isinstance(tmask, int):
+        k = w = tmask  # be safe here
+        tmask = tuple(range(k))
+    elif isinstance(tmask, Mask):
+        k = tmask.k
+        w = tmask.w
+        tmask = tmask.tuple
+    elif type(tmask) is tuple:
+        k, w = len(tmask), tmask[-1] + 1
+    else:
+        raise ValueError(f"mask type {type(tmask)} is not supported.")
+    _, ccc = compile_revcomp_and_canonical_code(k, rcmode)
+
+    @njit(nogil=True, locals=dict(code=uint64))
+    def twobit_to_codes(seq, out, start=0, n=-1):
+        """write n (or all) canonical k-mer codes from seq[start...] into out buffer"""
+        if n == -1:
+            n = len(seq) - w + 1 - start
+        for i in range(start, start + n):
+            code = 0
+            for j in tmask:
+                c = seq[i + j]
+                if c >= 4:
+                    out[i - start] = uint64(invalid)
+                    break
+                code = (code << 2) | c
+            else:
+                code = ccc(code)
+                out[i - start] = code
+
+    @njit(nogil=True, locals=dict(code=uint64))
+    def twobit_to_code(seq, start=0):
+        """return a single canonical code at seq[start...]"""
+        code = 0
+        for j in tmask:
+            c = seq[start + j]
+            if c >= 4:
+                return uint64(invalid)
+            code = (code << 2) | c
+        else:
+            code = ccc(code)
+            return code
+
+    return twobit_to_codes, twobit_to_code

xengsort-2.1.0/xengsort/fastcash_info.py

@@ -0,0 +1,156 @@
+"""
+fastcash_info.py:
+Collect and print information on a hash table stored on disk.
+Select elements according to a filter functions;
+export selected or all elements (keys, values, choices, info).
+"""
+
+from os import sep as SLASH
+from concurrent.futures import ThreadPoolExecutor, wait
+from importlib.util import spec_from_file_location, module_from_spec
+
+from numba import njit
+
+from .lowlevel import debug
+from .srhash import print_statistics
+from .io.hashio import load_hash, export_hash
+
+
+def print_dict(d, title=None):
+    if title is not None:
+        print(f"\n### {title}")
+    for key, value in d.items():
+        print(f"- {key}: {value}")
+
+
+def print_values(values, valuetup, title=None):
+    if title is not None:
+        print(f"\n### {title}")
+    print(f"- value set specification: {valuetup}")
+    print(f"- value set name: {values.NAME}")
+    print(f"- value set nvalues: {values.NVALUES}")
+
+
+def compile_filter_function(
+        expr=None, compile=None,
+        valueset=None, appinfo=None):
+    # any filter given?
+    if expr is None and compile is None:
+        f = (lambda key, choice, value: True)
+        ff = njit(nogil=True)(f)
+        return ff, False
+    # filter by expression in key, choice, value
+    elif expr is not None:
+        # expr must be an expression using 'key', 'choice', 'value' and constants.
+        f = eval(f'lambda key, choice, value: ({expr})')
+        ff = njit(nogil=True)(f)
+        return ff, True
+    # filter by compiling a function, given by name='modulepath::funcname'.
+    # compile must be a list: [name, *params] (all strings).
+    # name must be of the form "module.function"
+    # This function receives the valueset and appinfo as constants,
+    # and must return a compiled function f(key, choice, value).
+    name, params = compile[0], compile[1:]
+    module_path, func = map(str.strip, name.split("::"))
+    *_, module_name = module_path.split(SLASH)
+    module_path += ".py"
+    _spec = spec_from_file_location(module_name, module_path)
+    module = module_from_spec(_spec)
+    _spec.loader.exec_module(module)
+    cf = getattr(module, func)
+    ff = cf(valueset=valueset, appinfo=appinfo, params=params)
+    return ff, True
+
+
+def compile_count_filtered(h, filterfunc):
+    hp = h.private
+    is_slot_empty_at = hp.is_slot_empty_at
+    get_item_at = hp.get_item_at
+    get_subkey_choice_from_bucket_signature = hp.get_subkey_choice_from_bucket_signature
+    get_key_from_subtable_subkey = hp.get_key_from_subtable_subkey
+    nbuckets, bucketsize = h.nbuckets, h.bucketsize
+
+    @njit(nogil=True)
+    def count_filtered(ht, st):
+        n = total = 0
+        for bucket in range(nbuckets):
+            for slot in range(bucketsize):
+                if is_slot_empty_at(ht, st, bucket, slot):
+                    break  # go to next bucket
+                sig, val = get_item_at(ht, st, bucket, slot)
+                sbk, chc = get_subkey_choice_from_bucket_signature(bucket, sig)
+                key = get_key_from_subtable_subkey(st, sbk)
+                n += filterfunc(key, chc, val)
+                total += 1
+        return n, total
+
+    return count_filtered
+
+
+def run_filter(h, filterfunc):
+    count_filtered = compile_count_filtered(h, filterfunc)
+    nsubtables = h.nsubtables if hasattr(h, "nsubtables") else h.subtables
+    with ThreadPoolExecutor(max_workers=nsubtables) as executor:
+        futures = [
+            executor.submit(count_filtered, h.hashtable, st)
+            for st in range(nsubtables)
+        ]
+    wait(futures)
+    filtered = total = 0
+    for st, f in enumerate(futures):
+        sgood, stotal = f.result()  # this may raise errors
+        filtered += sgood
+        total += stotal
+    return filtered, total
+
+
+def main(args):
+    global debugprint0, debugprint1, debugprint2
+    global timestamp0, timestamp1, timestamp2
+    debugprint0, debugprint1, debugprint2 = debug.debugprint
+    timestamp0, timestamp1, timestamp2 = debug.timestamp
+    starttime = timestamp0(msg="\n# fastcash info/export : information and export of fastcash hash tables")
+
+    # load hash table information
+    _, values, infotup = load_hash(args.hashname, info_only=True)
+    (hashinfo, valueinfo, optinfo, appinfo) = infotup
+    print_dict(hashinfo, "Hash table properties")
+    print_dict(optinfo)
+    print_dict(appinfo)
+    print_values(values, valueinfo, "Value set properties")
+
+    # count number of k-mers passing the filter
+    filterfunc, runfilter = compile_filter_function(
+        expr=args.filterexpression,
+        compile=args.compilefilter,
+        valueset=values, appinfo=appinfo)
+
+    # load the hash table and print statistics
+    h, values, infotup = load_hash(args.hashname)
+    nvalues = values.NVALUES
+    SV = {'all': nvalues, 'none': False}
+    sv = args.showvalues
+    sv = SV[sv] if sv in SV else int(sv)
+    print_statistics(h, level=args.statistics, show_values=sv)
+
+    # run a compiled filter
+    if runfilter:
+        filtered, total = run_filter(h, filterfunc)
+        percent = filtered / total if total > 0 else 0.0
+        debugprint0(f"\n- k-mers passing filter: {filtered} of {total}, or {percent:.2%}")
+
+    # export data
+    fmt = args.format  # native, packed, text/txt, dna
+    anyformat = (fmt is not None)
+    prefix = args.outprefix
+    if (prefix is None) and anyformat:
+        prefix = args.hashname
+    if prefix is not None:
+        startout = timestamp0(msg="\n## Data Output")
+        fmt = {None: 'native', 'txt': 'text'}.get(fmt, fmt)
+        written = export_hash(h, nvalues, infotup, prefix, fmt, filterfunc)
+        debugprint0(f"- wrote {written} items")
+        timestamp0(startout, msg="- done writing data; writing time")
+
+    # that's it; successfully exit the program
+    timestamp0(starttime, msg="- DONE; total time")