phantombuster 0.12.0__tar.gz

phantombuster-0.12.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: phantombuster
+Version: 0.12.0
+Summary: 
+Author: Simon Haendeler
+Author-email: simon.emanuel.haendeler@univie.ac.at
+Requires-Python: >=3.9,<3.13
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: click (>=8.1.3,<9.0.0)
+Requires-Dist: pandas (>=2.0,<3.0)
+Requires-Dist: polars (>=0.19.12,<0.20.0)
+Requires-Dist: pyarrow (>=15,<16)
+Requires-Dist: pysam (>=0.20.0,<0.21.0)
+Requires-Dist: regex (>=2022.10.31,<2023.0.0)
+Requires-Dist: scipy (>=1.10.1,<2.0.0)
+Requires-Dist: trio (>=0.22.0,<0.23.0)
+Requires-Dist: zmq (>=0.0.0,<0.0.1)
+Description-Content-Type: text/markdown
+
+

phantombuster-0.12.0/build.py

@@ -0,0 +1,39 @@
+import numpy as np
+import pyarrow as pa
+
+import os
+
+from setuptools.extension import Extension
+from Cython.Build import cythonize
+
+def build(setup_kwargs):
+    """
+    This is a callback for poetry used to hook in our extensions.
+    """
+    ext_modules = cythonize("src/phantombuster/merge_cython.pyx")
+
+    import pyarrow
+    pyarrow.create_library_symlinks()
+
+    for ext in ext_modules:
+        # The Numpy C headers are currently required
+        ext.include_dirs.append(np.get_include())
+        ext.include_dirs.append(pa.get_include())
+        ext.libraries.extend(pa.get_libraries())
+        ext.library_dirs.extend(pa.get_library_dirs())
+
+        if os.name == 'posix':
+            ext.extra_compile_args.append('-std=c++17')
+        ext.runtime_library_dirs.append("$ORIGIN/../pyarrow")
+
+        # Try uncommenting the following line on Linux
+        # if you get weird linker errors or runtime crashes
+        ext.define_macros.append(("_GLIBCXX_USE_CXX11_ABI", "0"))
+
+
+    setup_kwargs.update(
+        {
+            # declare the extension so that setuptools will compile it
+            "ext_modules": ext_modules,
+        }
+    )

phantombuster-0.12.0/pyproject.toml

@@ -0,0 +1,50 @@
+[tool.poetry]
+name = "phantombuster"
+version = "0.12.0"
+description = ""
+authors = ["Simon Haendeler <simon.emanuel.haendeler@univie.ac.at>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.13"
+pandas = "^2.0"
+pysam = "^0.20.0"
+regex = "^2022.10.31"
+pyarrow = "^15"
+click = "^8.1.3"
+zmq = "^0.0.0"
+trio = "^0.22.0"
+scipy = "^1.10.1"
+polars = "^0.19.12"
+
+[tool.poetry.dev-dependencies]
+mypy = "^0.990"
+pytest = "^7.2.0"
+Cython = "^0.29.32"
+cibuildwheel = "^2.12.0"
+
+[tool.poetry.group.dev.dependencies]
+auditwheel = "^5.3.0"
+
+[tool.pytest.ini_options]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+]
+
+[[tool.poetry.source]]
+name = "syntonym"
+url = "http://localhost:8080/"
+priority = 'supplemental'
+
+[build-system]
+requires = ["poetry-core", "cython", "numpy", "pyarrow==15.0.2", "setuptools"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.build]
+script = "build.py"
+generate-setup-file = true
+
+[tool.poetry.scripts]
+phantombuster = 'phantombuster.cli:phantombuster'
+
+

phantombuster-0.12.0/setup.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+from setuptools import setup
+
+package_dir = \
+{'': 'src'}
+
+packages = \
+['phantombuster', 'phantombuster.remoter']
+
+package_data = \
+{'': ['*']}
+
+install_requires = \
+['click>=8.1.3,<9.0.0',
+ 'pandas>=2.0,<3.0',
+ 'polars>=0.19.12,<0.20.0',
+ 'pyarrow>=15,<16',
+ 'pysam>=0.20.0,<0.21.0',
+ 'regex>=2022.10.31,<2023.0.0',
+ 'scipy>=1.10.1,<2.0.0',
+ 'trio>=0.22.0,<0.23.0',
+ 'zmq>=0.0.0,<0.0.1']
+
+entry_points = \
+{'console_scripts': ['phantombuster = phantombuster.cli:phantombuster']}
+
+setup_kwargs = {
+    'name': 'phantombuster',
+    'version': '0.12.0',
+    'description': '',
+    'long_description': '',
+    'author': 'Simon Haendeler',
+    'author_email': 'simon.emanuel.haendeler@univie.ac.at',
+    'maintainer': 'None',
+    'maintainer_email': 'None',
+    'url': 'None',
+    'package_dir': package_dir,
+    'packages': packages,
+    'package_data': package_data,
+    'install_requires': install_requires,
+    'entry_points': entry_points,
+    'python_requires': '>=3.9,<3.13',
+}
+from build import *
+build(setup_kwargs)
+
+setup(**setup_kwargs)

phantombuster-0.12.0/src/phantombuster/__init__.py

@@ -0,0 +1,3 @@
+import importlib.metadata
+
+__version__ = importlib.metadata.version("phantombuster")

phantombuster-0.12.0/src/phantombuster/bamindexer.py

@@ -0,0 +1,101 @@
+import zlib
+import re
+import numpy as np
+
+
+class EndOfFile(Exception):
+    pass
+
+def read_header(f):
+    header = f.read(12)
+    if header == b"":
+        raise EndOfFile()
+    xlen = parse_xlen(header)
+
+    subfields = f.read(xlen)
+    if subfields == b"":
+        raise EndOfFile()
+
+    bsize = parse_bsize(subfields, xlen)
+    return xlen, bsize
+
+def read_block(f, bsize, xlen):
+    """Assumes you called read_header before"""
+    rest = f.read(bsize - 12 - xlen)
+    return zlib.decompress(rest, -15)
+
+def parse_xlen(header):
+    return int.from_bytes(header[-2:], "little")
+
+def parse_bsize(subfields, xlen):
+    offset = 0
+    block_size = None
+    while offset <= xlen:
+        if subfields[offset:offset+2] == b"BC":
+            block_size = int.from_bytes(subfields[offset+4:offset+6], "little") +1
+            break
+        else:
+            offset += int.from_bytes(subfields[offset+2:offset+4], "little") + 4
+    if block_size is None:
+        raise Exception("Could not find BSIZE")
+
+    return block_size
+
+def parse_read_start(data):
+    if data == b"":
+        raise EndOfFile()
+
+    found = None
+    i = 0
+    for m in re.finditer(8*b"\xff", data):
+        if found:
+            offset = m.start() - found 
+            if offset == 20 and (found-4 >= 0):
+                return found-4 
+            else:
+                found = m.start()
+        else:
+            found = m.start()
+        i += 1
+        if i == 10:
+            raise Exception("Could not find the beginning of the read, either read contains FF bytes or assumptions are violated")
+    raise Exception("Could not find the beginning of the read, assumptions are violated")
+
+def index(f, every):
+    coffset = 0
+    index = []
+    global_i = 0
+    i = 0
+    first_block = True
+
+    while True:
+        i += 1
+        global_i += 1
+        f.seek(coffset)
+        try:
+            xlen, bsize = read_header(f)
+        except EndOfFile:
+            break
+
+        #if True:
+        if i % every == 0 or global_i <= 2:
+            block_data = read_block(f, bsize, xlen)
+            if first_block:
+                uoffset = 0
+            else:
+                try:
+                    uoffset = parse_read_start(block_data)
+                except EndOfFile:
+                    break
+            idx = (coffset << 16) | uoffset
+            if idx >= 0:
+                index.append(idx)
+
+        coffset += bsize
+        first_block = False
+
+    index = index[1:] + [np.inf]
+
+    return index
+
+

phantombuster-0.12.0/src/phantombuster/cli.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+
+import logging
+import logging.config
+import os
+import os.path
+import sys
+import multiprocessing as mp
+import glob
+
+import ntpath
+from dataclasses import dataclass
+import json
+
+from phantombuster import porcelain, plumbing, stores
+from phantombuster import core
+from phantombuster.stores import deduplicator_to_pyarrow_table
+from phantombuster.remoter import Worker
+from phantombuster.io_ import write_parquet
+from phantombuster.project import Project
+import click
+from typing import Optional, List
+import pyarrow.parquet
+import pyarrow.csv
+
+from pathlib import Path
+
+def configure_logging(outputlog, verbose):
+    logging_config = {
+                      'version': 1,
+                      'formatters':{'default': {'format': "%(asctime)s %(levelname)-8s %(name)-15s %(message)s",
+                                                'datefmt': "%Y-%m-%d %H:%M:%S"}},
+                      'handlers': {'console': {'class': 'logging.StreamHandler', 'formatter': 'default', 'stream': 'ext://sys.stdout'}},
+                      'loggers': {'remoter': {'level': 'WARNING'}},
+                      'root': {'handlers': ['console'], 'level': 'INFO'}
+                      }
+    if outputlog:
+        logging_config['handlers']['file'] = {'class': 'logging.FileHandler', 'formatter': 'default', 'filename': outputlog}
+        logging_config['root']['handlers'].append('file')
+    if verbose:
+        logging_config['root']['level'] = 'DEBUG'
+
+    logging.config.dictConfig(logging_config)
+    logging.info('Logging configured')
+
+
+def log_call(function, **kwargs):
+    """Log a CLI call with all arguments"""
+    logging.info(f"PhantomBuster {function} was called with the following arguments: {kwargs}")
+
+
+@click.group()
+@click.version_option(package_name='phantombuster')
+@click.option("--verbose/--silent", default=False, help="Enable verbose debugging")
+@click.option("-o", "--outputlog", type=click.Path(), help="Output file for logs")
+@click.option("--save-results/--no-save-results", type=bool, default=True, help="DEBUG OPTION set no-save-results to not save which stages were already done")
+def phantombuster(verbose: bool, outputlog: str, save_results: bool) -> None:
+    print('configuring logging')
+    configure_logging(outputlog, verbose)
+    print('configured logging')
+
+
+# -- Main Commands -- #
+
+
+@phantombuster.command()
+@click.argument("input", type=click.Path(exists=True))
+@click.option("--outdir", required=True)
+@click.option("--regex-file", required=True)
+@click.option("--barcode-hierarchy-file", type=click.Path(exists=True), required=True)
+@click.option("--debug/--production", default=False)
+@click.option("--show-qc/--no-qc", default=False)
+@click.option("--force/--no-force", default=False)
+def demultiplex(input, regex_file, debug, outdir, show_qc, force, barcode_hierarchy_file):
+    print('start demultiplex CLI command')
+    log_call("demultiplex", input=input, regex_file=regex_file, debug=debug,
+             outdir=outdir, show_qc=show_qc, barcode_hierarchy_file=barcode_hierarchy_file)
+    project = Project(outdir)
+
+    print('logged call, on to work')
+    try:
+        core.demultiplex(input, regex_file, barcode_hierarchy_file, project, debug=debug, show_qc=show_qc)
+    except Exception as e:
+        logging.exception("Pipeline encountered an error. Aborting.")
+        raise click.Abort()
+    return
+
+
+@phantombuster.command()
+@click.option("--outdir", required=True)
+@click.option("--error-threshold", default=1)
+@click.option("--barcode-hierarchy-file", required=True)
+def error_correct(outdir, error_threshold, barcode_hierarchy_file):
+    log_call("error-correct", outdir=outdir, error_threshold=error_threshold)
+    project = Project(outdir)
+    core.error_correct(project, error_threshold, barcode_hierarchy_file)
+
+
+@phantombuster.command()
+@click.argument('hopping-barcodes', nargs=-1)
+@click.option("--outdir", required=True)
+@click.option("--threshold", default=0.05, type=float)
+def hopping_removal(outdir, threshold, hopping_barcodes):
+    log_call("hopping-removal", outdir=outdir, threshold=threshold, hopping_barcodes=hopping_barcodes)
+    project = Project(outdir)
+    hopping_barcodes = [bc.split(',') for bc in hopping_barcodes]
+    core.hopping_removal(project, hopping_barcodes, threshold)
+
+@phantombuster.command()
+@click.option("--outdir", required=True)
+@click.option("--prefix")
+@click.option("--threshold-file", required=True)
+def threshold(outdir, prefix, threshold_file):
+    log_call("threshold", outdir=outdir, prefix=prefix, threshold_file=threshold_file)
+    core.threshold(outdir, prefix, threshold_file)
+
+# -- Helper Commands -- #
+
+@phantombuster.command()
+@click.argument("prefixes", nargs=-1)
+@click.option("--outdir", required=True)
+@click.option("--prefix")
+@click.option("--barcode-hierarchy-file", type=click.Path(exists=True), required=True)
+def merge(prefixes, outdir, prefix, barcode_hierarchy_file):
+    """
+    Merge multiple prefixes under one prefix
+    """
+    # Log the call of this function with all parameters to the logfile
+    log_call("merge", prefixes=prefixes, outdir=outdir, prefix=prefix, barcode_hierarchy_file=barcode_hierarchy_file)
+
+    master_paths = PathsAndFiles(outdir, prefix, None)
+    master_paths.create()
+
+    try:
+        barcode_hierarchy = plumbing.read_barcode_hierarchy_file(barcode_hierarchy_file)
+    except Exception:
+        raise Exception("Could not read barcode hierarchy file correctly")
+
+    to_merge = [PathsAndFiles(outdir, prefix, None) for prefix in prefixes]
+
+    results = [stores.load(('deduplication', True), paths.stage_path('deduplication')) for paths in to_merge]
+    out = plumbing.combine(results, barcode_hierarchy)
+
+    stores.save(out, master_paths.stage_path('deduplication'), id='deduplication')
+
+
+
+@phantombuster.command()
+@click.argument("parquetfile")
+@click.argument("outfile", default=None, required=False)
+def to_csv(parquetfile, outfile):
+    log_call("to_csv", sample=parquetfile, outdir=outfile)
+    table = pyarrow.parquet.read_table(parquetfile)
+    if outfile is None:
+        outfile = parquetfile.replace(".parquet", ".csv")
+    pyarrow.csv.write_csv(table, outfile)
+
+
+@phantombuster.command()
+@click.argument("csvfile")
+@click.argument("outfile", default=None, required=False)
+def to_parquet(csvfile, outfile):
+    log_call("to_parquet", csvfile=csvfile, outfile=outfile)
+    table = pyarrow.csv.read_csv(csvfile)
+    if outfile is None:
+        outfile = csvfile.replace(".csv", ".parquet")
+    write_parquet(table, outfile)
+
+
+@phantombuster.command()
+@click.option("--outdir", default=None, required=True)
+@click.option("--name", default=None)
+def worker(outdir, name):
+    import phantombuster as phantombuster
+    import phantombuster.plumbing
+
+    project = Project(outdir)
+    project.create()
+    path = project._get_server_path()
+
+    print(f"Connecting to {path}")
+    worker = Worker(path, name=name)
+    worker.start_async()
+
+if __name__ == "__main__":
+    phantombuster()

phantombuster-0.12.0/src/phantombuster/config_files.py

@@ -0,0 +1,179 @@
+from dataclasses import dataclass
+import logging
+import pyarrow
+import pyarrow.csv
+import regex
+from phantombuster.plumbing import calculate_threshold
+
+VALID_TAGS = ['b2', 'query', 'bc', 'name', 'seq']
+
+
+def read_file(name: str) -> pyarrow.Table:
+    if name.endswith('tsv'):
+        options = pyarrow.csv.ParseOptions(delimiter="\t")
+    else:
+        options = pyarrow.csv.ParseOptions()
+    table = pyarrow.csv.read_csv(name, parse_options=options)
+    return table
+
+
+@dataclass
+class InputFile:
+    path: str
+    group: str
+    prefix: str
+
+    def _to_json_e(self):
+        return {'path': self.path, 'group': self.group, 'prefix': self.prefix}
+
+    @classmethod
+    def _from_json_e(cls, d: dict):
+        return InputFile(**d)
+
+
+@dataclass
+class InputGroup:
+    files: list[InputFile]
+    group: str
+
+    def _to_json_e(self) -> dict:
+        return {'files': [f._to_json_e() for f in self.files], 'group': self.group}
+
+    @classmethod
+    def _from_json_e(cls, d: dict) -> 'InputGroup':
+        ig = InputGroup(files=[InputFile._from_json_e(f) for f in d['files']], group=d['group'])
+        return ig
+
+
+class RegexDictionary:
+
+    def __init__(self, d=None):
+        if d is None:
+            d = {}
+        self._groups = d
+
+    def __eq__(self, o):
+        if isinstance(o, RegexDictionary):
+            return self._groups == o._groups
+        else:
+            return NotImplemented()
+
+    def add_regex(self, tag, rex, prefix='', group='*'):
+        d = self._groups.get(group, {})
+        d[prefix+tag] = rex
+        self._groups[group] = d
+
+    def get_regexes_for_group(self, group):
+        d1 = self._groups.get(group, {})
+        d2 = self._groups.get('*', {})
+        d = d1|d2
+        assert len(d) > 0
+        return {key: regex.compile(re) for key, re in d.items()}
+
+    def _to_json_e(self):
+        return self._groups
+
+    @classmethod
+    def _from_json_e(cls, d):
+        r = RegexDictionary(d)
+        return r
+
+
+def read_regex_file(path):
+    table = read_file(path)
+    column_names = ['group', 'prefix', 'tag', 'regex']
+
+    tags = table['tag'].to_pylist()
+
+    invalid_tags = [tag for tag in tags if tag not in VALID_TAGS]
+    if len(invalid_tags) > 0:
+        raise KeyError(f'Invalid tags: {", ".join(invalid_tags)}')
+
+    regexes = table['regex'].to_pylist()
+
+    if 'group' not in table.column_names:
+        groups = ['*'] * len(tags)
+    else:
+        groups = table['group'].to_pylist()
+
+    if 'prefix' not in table.column_names:
+        prefixs = [''] * len(tags)
+    else:
+        prefixs = table['prefix'].to_pylist()
+
+    regex_dict = RegexDictionary()
+
+    for group, tag, re, prefix in zip(groups, tags, regexes, prefixs):
+        regex_dict.add_regex(tag, re, prefix, group)
+    return regex_dict
+
+
+def read_barcode_hierarchy_file(name):
+    table = read_file(name)
+    column_names = ["barcode", "type", "referencefile", "threshold", "min_length", "max_length"]
+    missing_columns = [col for col in column_names if col not in table.column_names]
+    if len(missing_columns) > 0 :
+        raise Exception(f"Barcode Hierarchy File Incorrect, missing columns: {missing_columns}") # TODO better error reporting
+
+    def parse_length(value):
+        if value == "-":
+            return None
+        else:
+            try:
+                return int(value)
+            except Exception:
+                raise Exception("Can not parse min or max length correctly in barcode hierarchy file, must be either '-' or an integer")
+
+    def parse_type(value):
+        if value == "reference":
+            return "reference"
+        else:
+            return "random"
+
+    barcodes = [{"name": name, "type": parse_type(type), "referencefile": reference, "threshold": threshold,
+                 "min_length": parse_length(min_length), "max_length": parse_length(max_length)}
+                 for name, type, reference, threshold, min_length, max_length in zip(*[table[name].to_pylist() for name in column_names])]
+
+    for bc in barcodes:
+        if bc["type"] == "reference":
+            table = read_file(bc["referencefile"])
+            bc["reference"] = {bc: name for bc, name in zip([str(bc) for bc in table["barcode"].to_pylist()], [str(name) for name in table["name"].to_pylist()])}
+            if bc["threshold"] == "auto":
+                bc["threshold"] = calculate_threshold(bc["reference"])
+            else:
+                try:
+                    bc["threshold"] = int(bc["threshold"])
+                except Exception:
+                    raise Exception("Can not properly read threshold column of barcode hierarchy file, needs to be 'auto' or an integer")
+    return barcodes
+
+
+def read_input_files_file(path):
+    table = read_file(path)
+
+    column_names = ['file', 'group', 'prefix']
+
+    files = table['file'].to_pylist()
+
+    if 'group' not in table.column_names:
+        groups = table["file"].to_pylist()
+    else:
+        groups = table['group'].to_pylist()
+
+    if 'prefix' not in table.column_names:
+        prefixs = [''] * len(files)
+    else:
+        prefixs = table['prefix'].to_pylist()
+
+    input_files = [InputFile(file, group if group is not None else "", prefix if prefix is not None else "") for file, group, prefix in zip(files, groups, prefixs)]
+    groups = {}
+    for f in input_files:
+        l = groups.get(f.group, [])
+        l.append(f)
+        groups[f.group] = l
+
+    igs = []
+    for group_name, files in groups.items():
+        igs.append(InputGroup(files, group_name))
+
+    return igs