chipstream 0.6.0a5__py3-none-any.whl

chipstream/__init__.py

@@ -0,0 +1,20 @@
+# flake8: noqa: F401
+import os
+
+from ._version import __version__, __version_tuple__
+
+# Disable multithreading for all major tools that could somehow
+# parallelize our code. We don't want that, because we have linear
+# code and rely on multiprocessing for parallelization. This has
+# to be done before importing numpy or any other library affected.
+# If someone uses chipstream in a script, they either have to import
+# chipstream first, or set the environment variables manually.
+# We use `setdefault` to honor any other variables set by the
+# user for whatever reason.
+os.environ.setdefault("MKL_NUM_THREADS", "1")
+os.environ.setdefault("NUMBA_NUM_THREADS", "1")
+os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
+os.environ.setdefault("NUMPY_NUM_THREADS", "1")
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
+os.environ.setdefault("OMP_NUM_THREADS", "1")
+os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1")

chipstream/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.6.0a5'
+__version_tuple__ = version_tuple = (0, 6, 0)

chipstream/cli/__init__.py

@@ -0,0 +1 @@
+from ._main import main  # noqa: F401

chipstream/cli/__main__.py

@@ -0,0 +1,4 @@
+from _main import main
+
+if __name__ == '__main__':
+    main()

chipstream/cli/_main.py

@@ -0,0 +1,10 @@
+try:
+    import click
+except ImportError:
+    click = None
+
+if click is None:
+    def main(*args, **kwargs):
+        print("Please install the 'click' Python package to access the CLI!")
+else:
+    from .cli_main import chipstream_cli as main  # noqa: F401

chipstream/cli/cli_common.py

@@ -0,0 +1,95 @@
+import logging
+
+from dcnum.feat.feat_background import get_available_background_methods
+from dcnum.feat import Gate, QueueEventExtractor  # noqa: F401
+from dcnum.meta.ppid import get_class_method_info
+from dcnum.segm import get_available_segmenters
+
+
+seg_methods = get_available_segmenters()
+bg_methods = get_available_background_methods()
+
+
+class PrettyFormatter(logging.Formatter):
+    grey = "\x1b[38;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    format = "%(levelname)s %(name)s: %(message)s"
+
+    FORMATS = {
+        logging.DEBUG: grey + format + reset,
+        logging.INFO: grey + format + reset,
+        logging.WARNING: yellow + format + reset,
+        logging.ERROR: red + format + reset,
+        logging.CRITICAL: bold_red + format + reset
+    }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+
+
+def format_help_class_method_info(info_list):
+    """Format a list of `info` dictionaries from `get_class_method_info`"""
+    choices = ["\b"]  # "\b" means paragraph is not formatted in help string
+    for info in info_list:
+        code = info["code"]
+        title = info["title"]
+        choices.append(f" - '{code}': {title}")
+        for mname in info.get("defaults", []):
+            # We are flattening through all methods, because the CLI
+            # makes them transparently available. There must not be
+            # any name clashes, the developer is responsible for
+            # that.
+            kwds = info["defaults"][mname]
+            if kwds:
+                for kw in kwds:
+                    choices.append(f"   - {kw}={kwds[kw]}")
+    return "\n".join(choices)
+
+
+def get_choices_help_string(class_dict, static_kw_methods=None):
+    """Return a chipstream help string for this class
+
+    Parameters
+    ----------
+    class_dict: dictionary of objects
+        The dictionary holds the classes to inspect
+    static_kw_methods: list of callable
+        The methods to inspect; all kwargs-only keyword arguments
+        are extracted.
+    """
+    info_list = []
+    for key in class_dict:
+        class_obj = class_dict[key]
+        info_list.append(get_class_method_info(class_obj, static_kw_methods))
+    return format_help_class_method_info(info_list)
+
+
+def get_choices_help_string_segmenter(class_dict):
+    """Return a chipstream help string for a segmenter class
+
+    This is a custom function, analogous to get_choices_help_string,
+    but for segmenters, which might or might not allow mask
+    post-processing (depending on whether `mask_postprocessing` is set).
+
+    Parameters
+    ----------
+    class_dict: dictionary of objects
+        The dictionary holds the classes to inspect
+    """
+    info_list = []
+    for key in class_dict:
+        class_obj = class_dict[key]
+        static_kw_methods = ["segment_algorithm"]
+        static_kw_defaults = {}
+        if class_obj.mask_postprocessing:
+            static_kw_methods.append("process_mask")
+            static_kw_defaults["process_mask"] = class_obj.mask_default_kwargs
+        info_list.append(get_class_method_info(class_obj,
+                                               static_kw_methods,
+                                               static_kw_defaults))
+    return format_help_class_method_info(info_list)

chipstream/cli/cli_main.py

@@ -0,0 +1,228 @@
+import logging
+import multiprocessing as mp
+import pathlib
+import sys
+
+import click
+
+from .._version import version
+
+from . import cli_common as cm
+from .cli_proc import process_dataset
+
+
+@click.command(name="chipstream-cli",
+               help=f"""
+Segmentation and feature extraction for deformability cytometry data.
+
+Read the image data from an input file, perform segmentation, feature
+extraction, and gating.
+If ``PATH_OUT`` is not specified, then the data are written to a new file
+with the suffix "_dcn.rtdc". In recursive mode (``-r``), a target
+directory may be specified.
+
+You can specify which segmenter and feature extractor should be used.
+You can also specify additional keyword arguments for
+segmentation (``-ks``), feature extraction (``-kf``), and gating (``-kg``).
+
+Available segmenters with keyword arguments:
+
+{cm.get_choices_help_string_segmenter(cm.seg_methods)}
+
+Available feature extractors:
+
+{cm.get_choices_help_string({"legacy": cm.QueueEventExtractor},
+                            ["get_events_from_masks"])}
+
+Available gating options:
+
+{cm.get_choices_help_string({"": cm.Gate}, ["__init__"])}
+
+
+Examples:
+
+Process an .rtdc measurement, without computing the Haralick texture features::
+
+ chipstream-cli process -kf haralick=False M001_data.rtdc
+
+Recursively analyze a directory containing .rtdc files::
+
+ chipstream-cli process --recursive directory_name
+
+""")
+@click.argument("path_in",
+                type=click.Path(exists=True,
+                                dir_okay=True,
+                                resolve_path=True,
+                                path_type=pathlib.Path))
+@click.argument("path_out",
+                required=False,
+                type=click.Path(dir_okay=True,
+                                writable=True,
+                                resolve_path=True,
+                                path_type=pathlib.Path),
+                )
+@click.option("-b", "--background-method",
+              type=click.Choice(sorted(cm.bg_methods.keys()),
+                                case_sensitive=False),
+              default="sparsemed",
+              help="Background computation method to use.")
+@click.option("-kb", "background_kwargs",
+              multiple=True,
+              help="Optional ``KEY=VALUE`` argument for the specified "
+                   "background method",
+              metavar="KEY=VALUE",
+              )
+@click.option("-s", "--segmentation-method",
+              type=click.Choice(sorted(cm.seg_methods.keys()),
+                                case_sensitive=False),
+              default="thresh",
+              help="Segmentation method to use.")
+@click.option("-ks", "segmentation_kwargs",
+              multiple=True,
+              help="Optional ``KEY=VALUE`` argument for the specified "
+                   "segmenter.",
+              metavar="KEY=VALUE",
+              )
+@click.option("-kf", "feature_kwargs",
+              multiple=True,
+              help="Optional ``KEY=VALUE`` argument for the specified "
+                   "feature extractor.",
+              metavar="KEY=VALUE",
+              )
+@click.option("-kg", "gate_kwargs",
+              multiple=True,
+              help="Optional ``KEY=VALUE`` argument for event gating.",
+              metavar="KEY=VALUE",
+              )
+@click.option("-p", "--pixel-size", type=float, default=0,
+              help="Set/override the pixel size for feature "
+                   "extraction [µm].")
+@click.option("--limit-events", type=str, default="0",
+              help="Limit events of events to analyze. This can be either "
+                   "a number (e.g. '5000') or a range (e.g. '5000-7000'). "
+                   "You can also specify a step size (e.g. '5000-7000-2' for "
+                   "every second event). The convention follows Python slices "
+                   "with 'n' substituting for 'None'.")
+@click.option("--drain-basins", type=str, is_flag=True,
+              help="Write all basin features from input to output file. This "
+                   "option trades computation time and small file size for "
+                   "an output file that contains all available features.")
+@click.option("-r", "--recursive", is_flag=True,
+              help="Recurse into subdirectories.")
+@click.option("--num-cpus",
+              type=click.IntRange(min=1, max=mp.cpu_count(), clamp=True),
+              help="Number of processes to create."
+              )
+@click.option("--dry-run", is_flag=True,
+              help="Only print the pipeline identifiers and exit.")
+@click.option("--verbose", is_flag=True,
+              help="Yield a more verbose output.")
+@click.option("--debug", is_flag=True,
+              help="Run chipstream in debugging mode. This disables "
+                   "multiprocessing and yields a more verbose output.")
+@click.version_option(version)
+def chipstream_cli(
+    path_in,
+    path_out=None,
+    background_method="sparsemed",
+    background_kwargs=None,
+    segmentation_method="thresh",
+    segmentation_kwargs=None,
+    feature_kwargs=None,
+    gate_kwargs=None,
+    pixel_size=0,
+    limit_events="0",
+    drain_basins=False,
+    recursive=False,
+    num_cpus=None,
+    dry_run=False,
+    verbose=False,
+    debug=False,
+):
+
+    if debug:
+        click.secho("Running in debug mode (this will be slow)",
+                    fg="yellow")
+        verbose = True
+
+    # Parse limit_frames to get the HDF5Data index_mapping
+    if limit_events == "0":
+        index_mapping = None
+    elif limit_events.count("-"):
+        vals = limit_events.split("-")
+        assert len(vals) in [2, 3], "slice definition must have length 2 or 3"
+        start = None if vals[0] == "n" else int(vals[0])
+        stop = None if vals[1] == "n" else int(vals[1])
+        if len(vals) == 3:
+            step = None if vals[2] == "n" else int(vals[2])
+        else:
+            step = None
+        index_mapping = slice(start, stop, step)
+    else:
+        index_mapping = int(limit_events)
+
+    # Tell the root logger to pretty-print logs
+    root_logger = logging.getLogger()
+    handler = logging.StreamHandler()
+    handler.setFormatter(cm.PrettyFormatter())
+    root_logger.addHandler(handler)
+    handler.setLevel(logging.DEBUG if verbose else logging.WARNING)
+
+    mp.freeze_support()
+
+    process_kwargs = dict(
+        background_method=background_method,
+        background_kwargs=background_kwargs,
+        segmentation_method=segmentation_method,
+        segmentation_kwargs=segmentation_kwargs,
+        feature_kwargs=feature_kwargs,
+        gate_kwargs=gate_kwargs,
+        pixel_size=pixel_size,
+        index_mapping=index_mapping,
+        # Below this line are arguments that do not define the pipeline ID
+        basin_strategy="drain" if drain_basins else "tap",
+        num_cpus=num_cpus or mp.cpu_count(),
+        dry_run=dry_run,
+        debug=debug,
+        )
+
+    if recursive:
+        failed = 0  # keeps track of files that failed to process
+        for pi in sorted(path_in.rglob("*.rtdc")):
+            if pi.name.endswith("_dcn.rtdc"):
+                continue
+            if path_out is not None:
+                poi = path_out / pi.relative_to(path_in)
+                po = poi.with_name(poi.stem + "_dcn.rtdc")
+                po.parent.mkdir(parents=True, exist_ok=True)
+            else:
+                po = None
+            click.secho(f"\nProcessing {pi}")
+            failed += process_dataset(path_in=pi,
+                                      path_out=po,
+                                      **process_kwargs)
+            if dry_run:
+                click.secho("Stopping dry run after one iteration")
+                break
+        if failed:
+            click.secho(f"Could not process {failed} files", fg="red")
+        exit_code = bool(failed)
+    else:
+        if path_in.is_dir():
+            click.secho(f"PATH_IN must be a file, but '{path_in}' "
+                        f"is a directory. Did you forget to specify "
+                        f"the `--recursive` flag?", fg="red")
+            exit_code = 1
+        elif path_out and path_out.is_dir():
+            click.secho(f"PATH_OUT must be a path to a file, but "
+                        f"'{path_out}' is a directory")
+            exit_code = 1
+        else:
+            # everything ok
+            exit_code = process_dataset(path_in=path_in,
+                                        path_out=path_out,
+                                        **process_kwargs)
+    if exit_code:
+        click.secho("Encountered problems during processing", fg="red")
+    sys.exit(exit_code)

chipstream/cli/cli_proc.py

@@ -0,0 +1,141 @@
+import pathlib
+import time
+from typing import List, Literal
+
+import click
+import dcnum.logic
+from dcnum.meta import ppid
+import dcnum.read
+import dcnum.segm
+
+
+from . import cli_common as cm
+from .cli_valid import (
+    validate_background_kwargs, validate_feature_kwargs, validate_gate_kwargs,
+    validate_pixel_size, validate_segmentation_kwargs
+)
+
+
+def process_dataset(
+    path_in: pathlib.Path,
+    path_out: pathlib.Path,
+    background_method: str,
+    background_kwargs: List[str],
+    segmentation_method: str,
+    segmentation_kwargs: List[str],
+    feature_kwargs: List[str],
+    gate_kwargs: List[str],
+    pixel_size: float,
+    index_mapping: int | slice | None,
+    # Below this line are arguments that do not affect the pipeline ID
+    basin_strategy: Literal["drain", "tap"],
+    num_cpus: int,
+    dry_run: bool,
+    debug: bool,
+):
+    # Make sure the pixel size makes sense
+    if pixel_size == 0:
+        pixel_size = validate_pixel_size(data_path=path_in)
+
+    if path_out is None:
+        path_out = path_in.with_name(path_in.stem + "_dcn.rtdc")
+    path_out.parent.mkdir(parents=True, exist_ok=True)
+
+    # data keyword arguments
+    data_kwargs = {"pixel_size": pixel_size,
+                   "index_mapping": index_mapping}
+    with dcnum.read.HDF5Data(path_in, **data_kwargs) as data:
+        dat_id = data.get_ppid()
+    click.echo(f"Data ID:\t{dat_id}")
+
+    # background keyword arguments
+    bg_kwargs = validate_background_kwargs(background_method,
+                                           background_kwargs)
+    bg_cls = cm.bg_methods[background_method]
+    bg_id = bg_cls.get_ppid_from_ppkw(bg_kwargs)
+    click.echo(f"Background ID:\t{bg_id}")
+
+    # segmenter keyword arguments
+    seg_kwargs = validate_segmentation_kwargs(segmentation_method,
+                                              segmentation_kwargs)
+    seg_cls = cm.seg_methods[segmentation_method]
+    seg_id = seg_cls.get_ppid_from_ppkw(seg_kwargs)
+    click.echo(f"Segmenter ID:\t{seg_id}")
+
+    # feature keyword arguments
+    feat_kwargs = validate_feature_kwargs(feature_kwargs)
+    feat_cls = cm.QueueEventExtractor
+    feat_id = feat_cls.get_ppid_from_ppkw(feat_kwargs)
+    click.echo(f"Feature ID:\t{feat_id}")
+
+    # gate keyword arguments
+    gate_cls = cm.Gate
+    gate_kwargs = validate_gate_kwargs(gate_kwargs)
+    gate_id = gate_cls.get_ppid_from_ppkw(gate_kwargs)
+    click.echo(f"Gate ID:\t{gate_id}")
+
+    # compute pipeline hash
+    pph = ppid.compute_pipeline_hash(
+        gen_id=ppid.DCNUM_PPID_GENERATION,
+        dat_id=dat_id,
+        bg_id=bg_id,
+        seg_id=seg_id,
+        feat_id=feat_id,
+        gate_id=gate_id)
+    click.secho(f"Pipeline hash:\t{pph}")
+
+    if dry_run:
+        click.echo("Dry run complete")
+        return 0
+
+    job = dcnum.logic.DCNumPipelineJob(
+        path_in=path_in,
+        path_out=path_out,
+        data_code="hdf",
+        data_kwargs=data_kwargs,
+        background_code=bg_cls.get_ppid_code(),
+        background_kwargs=bg_kwargs,
+        segmenter_code=seg_cls.get_ppid_code(),
+        segmenter_kwargs=seg_kwargs,
+        feature_code=feat_cls.get_ppid_code(),
+        feature_kwargs=feat_kwargs,
+        gate_code=gate_cls.get_ppid_code(),
+        gate_kwargs=gate_kwargs,
+        basin_strategy=basin_strategy,
+        num_procs=num_cpus,
+        debug=debug,
+    )
+
+    try:
+        job.validate()
+    except dcnum.segm.SegmenterNotApplicableError as e:
+        click.secho(f"Segmenter '{segmentation_method}' cannot be applied "
+                    f"to '{path_in}': {', '.join(e.reasons_list)}",
+                    fg="red")
+        return 1
+
+    runner = dcnum.logic.DCNumJobRunner(job)
+    runner.start()
+    strlen = 0
+    prev_str = ""
+    while True:
+        status = runner.get_status()
+        progress = status["progress"]
+        state = status["state"]
+        print_str = f"Processing {progress:.0%} ({state})"
+        if print_str != prev_str:  # don't clutter stdout
+            strlen = max(strlen, len(print_str))
+            print(print_str.ljust(strlen), end="\r", flush=True)
+            prev_str = print_str
+        if status["state"] in ["done", "error"]:
+            break
+        time.sleep(.3)  # don't use 100% CPU
+    print("")  # new line
+
+    if status["state"] == "error":
+        click.secho(runner.error_tb, fg="red")
+        runner.join(delete_temporary_files=False)
+        return 1  # "exit code" > 0 means error
+    else:
+        runner.join(delete_temporary_files=True)
+        return 0

chipstream/cli/cli_valid.py

@@ -0,0 +1,127 @@
+import inspect
+import warnings
+
+from dcnum.meta import ppid
+import dcnum.read
+import h5py
+
+from . import cli_common as cm
+
+
+def validate_background_kwargs(bg_method, args):
+    """Parse background keyword arguments"""
+    # Get list of valid keyword arguments
+    bg_cls = cm.bg_methods[bg_method]
+    spec = inspect.getfullargspec(bg_cls.check_user_kwargs)
+    valid_kw = spec.kwonlyargs
+    annot = spec.annotations
+    # Convert the input args to key-value pairs
+    kwargs = {}
+    for key, value in [a.split("=") for a in args]:
+        if key not in valid_kw:
+            raise ValueError(f"Invalid keyword '{key}' for {bg_method}. "
+                             + f"Allowed keywords are {valid_kw}!")
+        # Convert to correct dtype (default to string)
+        kwargs[key] = ppid.convert_to_dtype(value, annot[key])
+    return kwargs
+
+
+def validate_feature_kwargs(args):
+    # Get list of valid keyword arguments
+    feat_cls = cm.QueueEventExtractor
+    feat_code = feat_cls.get_ppid_code()
+    # extract_approach
+    spec_appr = inspect.getfullargspec(feat_cls.get_events_from_masks)
+    valid_kw_appr = spec_appr.kwonlyargs
+    annot_appr = spec_appr.annotations
+    # Convert the input args to key-value pairs
+    kwargs = {}
+    for key, value in [a.split("=") for a in args]:
+        if key in valid_kw_appr:
+            kwargs[key] = ppid.convert_to_dtype(value, annot_appr[key])
+        else:
+            raise ValueError(
+                f"Invalid keyword '{key}' for '{feat_code}'. "
+                f"Allowed keywords are {valid_kw_appr}!")
+    return kwargs
+
+
+def validate_gate_kwargs(args):
+    spec = inspect.getfullargspec(cm.Gate.__init__)
+    valid_kw_appr = spec.kwonlyargs
+    annot_appr = spec.annotations
+    # Convert the input args to key-value pairs
+    kwargs = {}
+    for key, value in [a.split("=") for a in args]:
+        if key in valid_kw_appr:
+            kwargs[key] = ppid.convert_to_dtype(value, annot_appr[key])
+        else:
+            raise ValueError(
+                f"Invalid keyword '{key}' for gating. "
+                f"Allowed keywords are {valid_kw_appr} (and box filters)!")
+    return kwargs
+
+
+def validate_pixel_size(data_path):
+    with h5py.File(data_path) as h5:
+        did = h5.attrs.get("setup:identifier", "EMPTY")
+        pixel_size = h5.attrs.get("imaging:pixel size", 0)
+        if (did.startswith("RC-")
+                and (pixel_size < 0.255 or pixel_size > 0.275)):
+            hd = dcnum.read.HDF5Data(h5, pixel_size=0.260)  # placeholder
+            warnings.warn(
+                f"Correcting for invalid pixel size in '{hd.path}'!")
+            # Set default pixel size for Rivercyte devices
+            # Check the logs for the device name used.
+            logdat = hd.logs.get("cytoshot-acquisition", [])
+            for line in logdat:
+                line = line.strip().lower()
+                if line.startswith("device name:"):
+                    dev_name = line.split(":")[1].strip()
+                    break
+            else:
+                # fall-back to old camera
+                dev_name = "naiad 1.0"
+            if dev_name == "naiad 1.0":
+                # Naiad v1.0 camera VLXT06MI
+                pixel_size = 0.2645
+            elif dev_name == "naiad 1.1":
+                # Naiad v1.1 camera VCXU213M
+                pixel_size = 0.2675
+            else:
+                warnings.warn(f"Unknown device name: '{dev_name}'; "
+                              f"Not changing pixel size {pixel_size}")
+        return pixel_size
+
+
+def validate_segmentation_kwargs(seg_method, args):
+    """Parse segmenter keyword arguments"""
+    # Get list of valid keyword arguments
+    seg_cls = cm.seg_methods[seg_method]
+    # segment_algorithm
+    spec_appr = inspect.getfullargspec(seg_cls.segment_algorithm)
+    valid_kw_appr = spec_appr.kwonlyargs
+    annot_appr = spec_appr.annotations
+    # process_mask
+    if seg_cls.mask_postprocessing:
+        spec_mask = inspect.getfullargspec(seg_cls.process_mask)
+        valid_kw_mask = spec_mask.kwonlyargs
+        annot_mask = spec_mask.annotations
+    else:
+        valid_kw_mask = []
+        annot_mask = {}
+
+    # Convert the input args to key-value pairs
+    kwargs = {}
+    kwargs_mask = {}
+    for key, value in [a.split("=") for a in args]:
+        if key in valid_kw_mask:
+            kwargs_mask[key] = ppid.convert_to_dtype(value, annot_mask[key])
+        elif key in valid_kw_appr:
+            kwargs[key] = ppid.convert_to_dtype(value, annot_appr[key])
+        else:
+            raise ValueError(
+                f"Invalid keyword '{key}' for {seg_method}. "
+                f"Allowed keywords are {valid_kw_appr + valid_kw_mask}!")
+    kwargs["kwargs_mask"] = kwargs_mask
+    return kwargs

chipstream/gui/__init__.py

@@ -0,0 +1 @@
+from ._main import main  # noqa: F401

chipstream/gui/__main__.py

@@ -0,0 +1,4 @@
+from ._main import main
+
+if __name__ == '__main__':
+    main()