Demultiplex 1.2.3__py2.py3-none-any.whl

demultiplex/__init__.py

@@ -0,0 +1,28 @@
+from importlib.metadata import PackageNotFoundError, metadata
+from re import split
+from typing import Callable
+
+from .demultiplex import Extractor, count, demultiplex
+
+
+def _extract(key: str, delim: str = r'[^\s\S]', index: int = 0) -> str:
+    try:
+        value = metadata(__package__).get(key, '')
+    except PackageNotFoundError:
+        return '<NO DATA>'
+    return split(delim, value)[index]
+
+
+def doc_split(func: Callable) -> str:
+    return func.__doc__.split('\n\n')[0]
+
+
+_project = _extract('Name')
+_version = _extract('Version')
+_year = '2013-2026'
+_author = _extract('Author')
+_email = _extract('Author-email')
+_description = _extract('Summary')
+_copyright = f'Copyright (c) {_year} by {_author} <{_email}>'
+_url = _extract('Project-URL')
+_info = f'{_project} version {_version}\n\n{_copyright}\nHomepage: {_url}'

demultiplex/cli.py

@@ -0,0 +1,150 @@
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from sys import stdin
+
+from fastools import Peeker
+
+from . import _copyright, _description, _info, doc_split
+from .demultiplex import (
+    _get_barcode, _type_handler, Extractor, count, demultiplex, match)
+
+
+def _file_type(*args, **kwargs):
+    """Argparse FileType replacement."""
+    def _open(name):
+        return _type_handler[name.split('.')[-1]](name, *args, **kwargs)
+
+    return _open
+
+
+def guess(
+        input_handle, output_handle, in_read, fmt, start, end, sample_size,
+        threshold, use_freq):
+    """Retrieve the most frequent barcodes."""
+    extractor = Extractor(input_handle, in_read, fmt, start, end)
+    barcodes = count(input_handle, extractor, sample_size, threshold, use_freq)
+
+    for i, barcode in enumerate(sorted(barcodes)):
+        output_handle.write('{} {}\n'.format(i + 1, barcode))
+
+
+def demux(
+        input_handles, barcodes_handle, in_read, fmt, start, end, mismatch,
+        use_edit, path='.'):
+    """Demultiplex any number of files given a list of barcodes."""
+    extractor = Extractor(input_handles[0], in_read, fmt, start, end)
+    demultiplex(
+        input_handles, barcodes_handle, extractor, mismatch, use_edit, path)
+
+
+def bcmatch(
+        input_handles, barcodes_handle, mismatch, use_edit, path='.',
+        filter_multiple=False, directional=False):
+    """Demultiplex one file given a list of barcode tuples."""
+    match(
+        input_handles, barcodes_handle, mismatch, use_edit, path,
+        filter_multiple, directional)
+
+
+def _arg_parser() -> object:
+    """Command line argument parsing."""
+    common_parser = ArgumentParser(add_help=False)
+    common_parser.add_argument(
+        '-r', dest='in_read', action='store_true',
+        help='extract the barcodes from the read')
+    common_parser.add_argument(
+        '--format', dest='fmt', default=None, choices=_get_barcode.keys(),
+        help='provdide the header format')
+    common_parser.add_argument(
+        '-s', dest='start', type=int, default=None,
+        help='start of the selection')
+    common_parser.add_argument(
+        '-e', dest='end', type=int, default=None, help='end of the selection')
+
+    common_options_parser = ArgumentParser(add_help=False)
+    common_options_parser.add_argument(
+        '-m', dest='mismatch', type=int, default=1,
+        help='number of mismatches')
+    common_options_parser.add_argument(
+        '-d', dest='use_edit', action='store_true',
+        help='use Levenshtein distance')
+    common_options_parser.add_argument(
+        '-p', dest='path', type=str, default='.', help='output directory')
+
+    input_parser = ArgumentParser(add_help=False)
+    input_parser.add_argument(
+        'barcodes_handle', metavar='BARCODES', type=_file_type('rt'),
+        help='barcodes file')
+    input_parser.add_argument(
+        'input_handles', metavar='INPUT', nargs='+', type=_file_type('rt'),
+        help='input files')
+
+    parser = ArgumentParser(
+        formatter_class=ArgumentDefaultsHelpFormatter, description=_description,
+        epilog=_copyright)
+    parser.add_argument('-v', action='version', version=_info)
+    subparsers = parser.add_subparsers(dest='subcommand')
+    subparsers.required = True
+
+    subparser = subparsers.add_parser(
+        'guess', formatter_class=ArgumentDefaultsHelpFormatter,
+        parents=[common_parser], description=doc_split(guess))
+    subparser.add_argument(
+        'input_handle', metavar='INPUT', type=_file_type('rt'),
+        help='input file')
+    subparser.add_argument(
+        '-o', dest='output_handle', metavar='OUTPUT', type=_file_type('wt'),
+        default='-', help='output file')
+    subparser.add_argument(
+        '-n', dest='sample_size', type=int, default=1000000,
+        help='sample size')
+    subparser.add_argument(
+        '-f', dest='use_freq', action='store_true',
+        help='select on frequency instead of a fixed amount')
+    subparser.add_argument(
+        '-t', dest='threshold', type=int, default=12,
+        help='threshold for the selection method')
+    subparser.set_defaults(func=guess)
+
+    subparser = subparsers.add_parser(
+        'demux', formatter_class=ArgumentDefaultsHelpFormatter,
+        parents=[common_parser, common_options_parser, input_parser],
+        description=doc_split(demux))
+    subparser.set_defaults(func=demux)
+
+    subparser = subparsers.add_parser(
+        'match', formatter_class=ArgumentDefaultsHelpFormatter,
+        parents=[common_options_parser, input_parser],
+        description=doc_split(bcmatch))
+    subparser.add_argument(
+        '-f', dest='filter_multiple', default=False, action='store_true',
+        help='write multiple matches to separate files')
+    subparser.add_argument(
+        '-D', dest='directional', default=False, action='store_true',
+        help='directional input data')
+    subparser.set_defaults(func=bcmatch)
+
+    return parser
+
+
+def main():
+    """Main entry point."""
+    parser = _arg_parser()
+
+    global stdin
+    stdin = Peeker(stdin)
+
+    try:
+        args = parser.parse_args()
+    except IOError as error:
+        parser.error(error)
+
+    try:
+        args.func(
+            **{k: v for k, v in vars(args).items()
+                if k not in ('func', 'subcommand')})
+    except (ValueError, OSError) as error:
+        parser.error(error)
+
+
+if __name__ == '__main__':
+    main()

demultiplex/demultiplex.py

@@ -0,0 +1,222 @@
+from bz2 import open as bz2_open
+from collections import defaultdict
+from gzip import open as gzip_open
+from os import mkdir
+from os.path import basename, exists
+
+from Bio import SeqIO
+from Bio.Seq import reverse_complement
+from dict_trie import Trie
+from fastools import guess_file_format, guess_header_format
+from jit_open import Handle, Queue
+
+from .match import multi_align
+
+
+_get_barcode = {
+    'normal': lambda record: record.id.split('#')[1].split('/')[0],
+    'x': lambda record: record.description.split(':')[-1],
+    'umi': lambda record: record.description.split(' ')[0].split(':')[-1],
+    'unknown': lambda record: str(record.seq)}
+
+_type_handler = defaultdict(lambda: open, {
+    'bz2': bz2_open,
+    'bzip2': bz2_open,
+    'gz': gzip_open,
+    'gzip': gzip_open})
+
+
+def _name(handle):
+    if hasattr(handle.buffer, '_fp'):
+        return handle.buffer._fp.name
+    return handle.name
+
+
+class Extractor(object):
+    def __init__(self, handle, in_read=False, fmt=None, start=None, end=None):
+        """Configure a barcode extractor.
+
+        :arg stream handle: Handle to an NGS data file.
+        :arg bool in_read: Inspect the read instead of the header.
+        :arg str fmt: Header format.
+        :arg int start: Start of the barcode.
+        :arg int end: End of the barcode.
+        """
+        self._start = start
+        self._end = end
+
+        if self._start:
+            self._start -= 1
+
+        if not fmt:
+            if not in_read:
+                self._get_barcode = _get_barcode[guess_header_format(handle)]
+            else:
+                self._get_barcode = _get_barcode['unknown']
+        else:
+            self._get_barcode = _get_barcode[fmt]
+
+    def get(self, record):
+        return self._get_barcode(record)[self._start:self._end]
+
+
+def count(handle, extractor, sample_size, threshold, use_freq=False):
+    """Get the most frequent barcodes from an NGS data file.
+
+    :arg stream handle: Handle to an NGS data file.
+    :arg Extractor extractor: A barcode extractor.
+    :arg int sample_size: Number of records to probe.
+    :arg int threshold: Threshold for the selection method.
+    :arg bool use_freq: Select frequent barcodes instead of a fixed amount.
+
+    :returns list: A list of barcodes.
+    """
+    barcodes = defaultdict(int)
+
+    for i, record in enumerate(SeqIO.parse(handle, guess_file_format(handle))):
+        if i > sample_size:
+            break
+        barcodes[extractor.get(record)] += 1
+
+    if use_freq:
+        return filter(lambda x: barcodes[x] >= threshold, barcodes)
+    return sorted(barcodes, key=barcodes.get, reverse=True)[:threshold]
+
+
+def _open_files(path, filenames, barcode, queue):
+    """For a list of input files, open the corresponding output files.
+
+    :arg str path: Output directory.
+    :arg list filename: List of input filenames.
+    :arg str barcode: Name of the barcode.
+    :arg Queue queue: Queue for open files.
+
+    :returns list: List of handles of output files.
+    """
+    if not exists(path):
+        mkdir(path)
+
+    handles = []
+
+    for filename in filenames:
+        base, ext = basename(filename).split('.', True)
+        handles.append(
+            Handle('{}/{}_{}.{}'.format(path, base, barcode, ext), queue,
+                   f_open=_type_handler[ext.split('.')[-1]]))
+
+    return handles
+
+
+def _write(handles, records, file_format):
+    for i, record in enumerate(records):
+        SeqIO.write(record, handles[i], file_format)
+
+
+def demultiplex(
+        input_handles, barcodes_handle, extractor, mismatch, use_edit,
+        path='.'):
+    """Demultiplex a list of NGS data files.
+
+    :arg list input_handles: List of handles to NGS data files.
+    :arg stream barcodes_handle: Handle to a file containing barcodes.
+    :arg Extractor extractor: A barcode extractor.
+    :arg int mismatch: Number of allowed mismatches.
+    :arg bool use_edit: Use Levenshtein distance instead of Hamming distance.
+    :arg str path: Output directory.
+    """
+    filenames = list(map(lambda x: _name(x), input_handles))
+    queue = Queue()
+    default_handles = _open_files(path, filenames, 'UNKNOWN', queue)
+
+    barcodes = {}
+    for line in barcodes_handle.readlines():
+        try:
+            name, barcode = line.strip().split()
+        except ValueError:
+            raise ValueError('invalid barcodes file format')
+        barcodes[barcode] = _open_files(path, filenames, name, queue)
+
+    trie = Trie(barcodes.keys())
+    distance_function = trie.best_hamming
+    if use_edit:
+        distance_function = trie.best_levenshtein
+
+    file_format = guess_file_format(input_handles[0])
+    readers = list(map(
+        lambda x: SeqIO.parse(x, file_format), input_handles))
+
+    while True:
+        records = list(map(lambda x: next(x), readers))
+        if not records:
+            break
+
+        barcode = distance_function(extractor.get(records[0]), mismatch)
+        if barcode:
+            _write(barcodes[barcode], records, file_format)
+        else:
+            _write(default_handles, records, file_format)
+
+    queue.flush()
+
+
+def match(
+        input_handles, barcodes_handle, mismatch, use_edit, path='.',
+        filter_multiple=False, directional=False):
+    """Demultiplex a list of NGS data files.
+
+    :arg list input_handles: List of handles to NGS data files.
+    :arg stream barcodes_handle: Handle to a file containing barcodes.
+    :arg int mismatch: Number of allowed mismatches.
+    :arg bool use_edit: Use Levenshtein distance instead of Hamming distance.
+    :arg str path: Output directory.
+    :arg bool filter_multiple: Write multiple matches to separate files.
+    :arg bool directional: Directional input data.
+    """
+    filenames = list(map(lambda x: _name(x), input_handles))
+    queue = Queue()
+    default_handles = _open_files(path, filenames, 'UNKNOWN', queue)
+    multiple_handles = _open_files(path, filenames, 'MULTIPLE', queue)
+
+    indel_score = 1
+    if not use_edit:
+        indel_score = 1000
+
+    barcodes = []
+    for line in map(lambda x: x.strip().split(), barcodes_handle.readlines()):
+        try:
+            name = line.pop(0)
+        except (IndexError, ValueError):
+            raise ValueError('invalid barcodes file format')
+        barcodes.append((_open_files(path, filenames, name, queue), line))
+
+    file_format = guess_file_format(input_handles[0])
+    readers = list(map(
+        lambda x: SeqIO.parse(x, file_format), input_handles))
+
+    while True:
+        records = list(map(lambda x: next(x), readers))
+        if not records:
+            break
+
+        reference = str(records[0].seq)
+        if directional:
+            reference_rc = reverse_complement(reference)
+
+        found_handles = []
+        for handles, barcode in barcodes:
+            if multi_align(reference, barcode, mismatch, indel_score):
+                found_handles.append(handles)
+            elif directional and multi_align(
+                    reference_rc, barcode, mismatch, indel_score):
+                found_handles.append(handles)
+
+        if found_handles:
+            if not filter_multiple or len(found_handles) == 1:
+                for handles in found_handles:
+                    _write(handles, records, file_format)
+            else:
+                _write(multiple_handles, records, file_format)
+        else:
+            _write(default_handles, records, file_format)
+
+    queue.flush()

demultiplex/match.py

@@ -0,0 +1,24 @@
+from tssv import align
+
+
+def multi_align(reference, barcodes, distance, indel_score):
+    """Align multiple barcodes in order to a reference.
+
+    :arg str reference:
+    :arg list barcodes:
+    :arg int distance:
+    :arg int indel_score:
+
+    :returns bool: True if all barcodes align in order, False otherwise.
+    """
+    _reference = reference
+
+    for barcode in barcodes:
+        alignment = align(_reference, barcode, indel_score)
+
+        if alignment['distance'] > distance:
+            return False
+
+        _reference = _reference[alignment['position']:]
+
+    return True

demultiplex-1.2.3.dist-info/METADATA

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: Demultiplex
+Version: 1.2.3
+Summary: Demultiplex any number of FASTA or a FASTQ files based on a list of barcodes.
+Project-URL: homepage, https://github.com/jfjlaros/demultiplex
+Author-email: "Jeroen F.J. Laros" <jlaros@fixedpoint.nl>
+License-Expression: MIT
+License-File: LICENSE.md
+Keywords: FASTA,FASTQ,barcode,bioinformatics,demultiplex
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Requires-Dist: biopython
+Requires-Dist: dict-trie
+Requires-Dist: fastools
+Requires-Dist: jit-open
+Requires-Dist: tssv
+Description-Content-Type: text/x-rst
+
+Demultiplex: FASTA/FASTQ demultiplexer
+======================================
+
+.. image:: https://img.shields.io/github/last-commit/jfjlaros/demultiplex.svg
+   :target: https://github.com/jfjlaros/demultiplex/graphs/commit-activity
+.. image:: https://github.com/jfjlaros/demultiplex/actions/workflows/test.yml/badge.svg
+   :target: https://github.com/jfjlaros/demultiplex/actions/workflows/test.yml
+.. image:: https://readthedocs.org/projects/demultiplex/badge/?version=latest
+   :target: https://demultiplex.readthedocs.io/en/latest
+.. image:: https://img.shields.io/github/release-date/jfjlaros/demultiplex.svg
+   :target: https://github.com/jfjlaros/demultiplex/releases
+.. image:: https://img.shields.io/github/release/jfjlaros/demultiplex.svg
+   :target: https://github.com/jfjlaros/demultiplex/releases
+.. image:: https://img.shields.io/pypi/v/demultiplex.svg
+   :target: https://pypi.org/project/demultiplex/
+.. image:: https://img.shields.io/github/languages/code-size/jfjlaros/demultiplex.svg
+   :target: https://github.com/jfjlaros/demultiplex
+.. image:: https://img.shields.io/github/languages/count/jfjlaros/demultiplex.svg
+   :target: https://github.com/jfjlaros/demultiplex
+.. image:: https://img.shields.io/github/languages/top/jfjlaros/demultiplex.svg
+   :target: https://github.com/jfjlaros/demultiplex
+.. image:: https://img.shields.io/github/license/jfjlaros/demultiplex.svg
+   :target: https://raw.githubusercontent.com/jfjlaros/demultiplex/master/LICENSE.md
+.. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.8362959.svg
+   :target: https://zenodo.org/record/8362959
+
+----
+
+Versatile NGS demultiplexer with the following features:
+
+- Support for FASTA and FASTQ files.
+- Support for gzip and bzip2 compressed files.
+- Support for multiple reads per fragment, e.g., paired-end.
+- Handles barcodes in the header and in the reads.
+- Handles barcodes at *unknown* locations in reads (e.g., PacBio or Nanopore
+  barcodes).
+- Support for selection of part of a barcode.
+- Allows for mismatches, insertions and deletions.
+- Barcode guessing by frequency or fixed amount.
+- Handles large numbers (over one million) of barcodes.
+
+Please see ReadTheDocs_ for the latest documentation.
+
+
+.. _ReadTheDocs: https://demultiplex.readthedocs.io/en/latest/index.html

demultiplex-1.2.3.dist-info/RECORD

@@ -0,0 +1,9 @@
+demultiplex/__init__.py,sha256=ibYYpUFoquxlchobG9SBPEjvHLHkOn8qPP2n_0BT7FQ,827
+demultiplex/cli.py,sha256=E9Hdz-JuxVP8brMHMMd1dQ-OEj2q-xUZRMWkRrtgCzU,5365
+demultiplex/demultiplex.py,sha256=SsyYjNhYrGjHfYEO_Fc4nxUMuWqrXq2qkCGmUSe2fQw,7352
+demultiplex/match.py,sha256=GwvngjAotvkm5tSDgTGVgfd9Y7ZDsEKMVThchGuS6Cg,584
+demultiplex-1.2.3.dist-info/METADATA,sha256=UBX4zlH2flCZNjCWWmUvrDCRDxiHrQ--pI1u9TAmiHw,3032
+demultiplex-1.2.3.dist-info/WHEEL,sha256=VX-VJ7c6dw9Ge3EqJIbA6W3pOUbz24SnnGGFNr55jY4,105
+demultiplex-1.2.3.dist-info/entry_points.txt,sha256=kg7ARrQaxKCFhUPPE-_EDOje-y3iXuZAtSuW_MptWUo,53
+demultiplex-1.2.3.dist-info/licenses/LICENSE.md,sha256=kSdtuXPyVgLRqkNJH1nLyEy4jm8VHh0MyCp1VWPOAZU,1023
+demultiplex-1.2.3.dist-info/RECORD,,

demultiplex-1.2.3.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any

demultiplex-1.2.3.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+demultiplex = demultiplex.cli:main

demultiplex-1.2.3.dist-info/licenses/LICENSE.md

@@ -0,0 +1,17 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.