dclab 0.67.0__cp314-cp314t-macosx_10_13_x86_64.whl

dclab/cli/task_split.py

@@ -0,0 +1,154 @@
+"""Split an .rtdc file into smaller .rtdc files"""
+from __future__ import annotations
+
+import argparse
+import pathlib
+import warnings
+
+import hdf5plugin
+
+from ..rtdc_dataset import fmt_tdms, new_dataset, RTDCWriter
+from .._version import version
+
+from . import common
+
+
+def split(
+        path_in: str | pathlib.Path = None,
+        path_out: str | pathlib.Path = None,
+        split_events: int = 10000,
+        skip_initial_empty_image: bool = True,
+        skip_final_empty_image: bool = True,
+        ret_out_paths: bool = False,
+        verbose: bool = False):
+    """Split a measurement file
+
+    Parameters
+    ----------
+    path_in: str or pathlib.Path
+        path of input measurement file
+    path_out: str or pathlib.Path
+        path to output directory (optional)
+    split_events: int
+        maximum number of events in each output file
+    skip_initial_empty_image: bool
+        remove the first event of the dataset if the image is zero
+    skip_final_empty_image: bool
+        remove the final event of the dataset if the image is zero
+    ret_out_paths:
+        if True, return the list of output file paths
+    verbose: bool
+        if True, print messages to stdout
+
+    Returns
+    -------
+    [out_paths]: list of pathlib.Path
+        List of generated files (only if `ret_out_paths` is specified)
+    """
+    cmp_kw = hdf5plugin.Zstd(clevel=5)
+    if path_in is None:
+        parser = split_parser()
+        args = parser.parse_args()
+
+        path_in = pathlib.Path(args.path_in).resolve()
+        path_out = args.path_out
+        split_events = args.split_events
+        skip_initial_empty_image = not args.include_empty_boundary_images
+        skip_final_empty_image = not args.include_empty_boundary_images
+        verbose = True
+
+    if path_out in ["SAME", None]:  # default to input directory
+        path_out = path_in.parent
+
+    path_in = pathlib.Path(path_in)
+    path_out = pathlib.Path(path_out)
+
+    logs = {"dclab-split": common.get_command_log(paths=[path_in])}
+
+    paths_gen = []
+    paths_temp = []
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        # ignore ResourceWarning: unclosed file <_io.BufferedReader...>
+        warnings.simplefilter("ignore", ResourceWarning)  # noqa: F821
+        if fmt_tdms.NPTDMS_AVAILABLE:  # tdms-related warning filters
+            # ignore SlowVideoWarning
+            warnings.simplefilter("ignore",
+                                  fmt_tdms.event_image.SlowVideoWarning)
+            if skip_initial_empty_image:
+                # If the initial frame is skipped when empty,
+                # suppress any related warning messages.
+                warnings.simplefilter(
+                    "ignore",
+                    fmt_tdms.event_image.InitialFrameMissingWarning)
+
+        with new_dataset(path_in) as ds:
+            num_files = len(ds) // split_events
+            if len(ds) % split_events:
+                num_files += 1
+            for ii in range(num_files):
+                pp = path_out / f"{path_in.stem}_{ii+1:04d}.rtdc"
+                pt = pp.with_suffix(".rtdc~")
+                paths_gen.append(pp)
+                paths_temp.append(pt)
+                if verbose:
+                    print(f"Generating {ii+1:d}/{num_files:d}: {pt}")
+                ds.filter.manual[:] = False  # reset filter
+                ds.filter.manual[ii*split_events:(ii+1)*split_events] = True
+                common.skip_empty_image_events(
+                    ds=ds,
+                    initial=skip_initial_empty_image,
+                    final=skip_final_empty_image)
+                ds.apply_filter()
+                ds.export.hdf5(path=pt,
+                               features=ds.features_innate,
+                               logs=True,
+                               tables=True,
+                               basins=True,
+                               filtered=True,
+                               compression_kwargs=cmp_kw,
+                               )
+
+        if w:
+            logs["dclab-split-warnings"] = common.assemble_warnings(w)
+        sample_name = ds.config["experiment"]["sample"]
+
+    # Add the logs and update sample name
+    for ii, pt in enumerate(paths_temp):
+        meta = {"experiment": {"sample": f"{sample_name} {ii+1}/{num_files}"}}
+        with RTDCWriter(pt, compression_kwargs=cmp_kw) as hw:
+            for name in logs:
+                hw.store_log(name, logs[name])
+            hw.store_metadata(meta)
+
+    for pt, pp in zip(paths_temp, paths_gen):
+        pt.rename(pp)
+
+    if ret_out_paths:
+        return paths_gen
+
+
+def split_parser():
+    descr = "Split an RT-DC measurement file (.tdms or .rtdc) into multiple " \
+            + "smaller .rtdc files."
+    parser = argparse.ArgumentParser(description=descr)
+    parser.add_argument('path_in', metavar="PATH_IN", type=str,
+                        help='Input path (.tdms or .rtdc file)')
+    parser.add_argument('--path_out', metavar="PATH_OUT", type=str,
+                        default="SAME",
+                        help='Output directory (defaults to same directory)')
+    parser.add_argument('--split-events', type=int, default=10000,
+                        help='Maximum number of events in each output file')
+    parser.add_argument('--include-empty-boundary-images',
+                        dest='include_empty_boundary_images',
+                        action='store_true',
+                        help='In old versions of Shape-In, the first or last '
+                             + 'images were sometimes not stored in the '
+                             + 'resulting .avi file. In dclab, such images '
+                             + 'are represented as zero-valued images. Set '
+                             + 'this option, if you wish to include these '
+                             + 'events with empty image data.')
+    parser.set_defaults(include_empty_boundary_images=False)
+    parser.add_argument('--version', action='version',
+                        version=f'dclab-split {version}')
+    return parser

dclab/cli/task_tdms2rtdc.py

@@ -0,0 +1,186 @@
+"""Convert .tdms to .rtdc files"""
+import argparse
+import pathlib
+import warnings
+
+import hdf5plugin
+
+from ..rtdc_dataset import fmt_tdms, new_dataset, RTDCWriter
+from .._version import version
+
+from . import common
+
+
+def tdms2rtdc(path_tdms=None, path_rtdc=None, compute_features=False,
+              skip_initial_empty_image=True, skip_final_empty_image=True,
+              verbose=False):
+    """Convert .tdms datasets to the hdf5-based .rtdc file format
+
+    Parameters
+    ----------
+    path_tdms: str or pathlib.Path
+        Path to input .tdms file
+    path_rtdc: str or pathlib.Path
+        Path to output .rtdc file
+    compute_features: bool
+        If `True`, compute all ancillary features and store them in the
+        output file
+    skip_initial_empty_image: bool
+        In old versions of Shape-In, the first image was sometimes
+        not stored in the resulting .avi file. In dclab, such images
+        are represented as zero-valued images. If `True` (default),
+        this first image is not included in the resulting .rtdc file.
+    skip_final_empty_image: bool
+        In other versions of Shape-In, the final image is sometimes
+        also not stored in the .avi file. If `True` (default), this
+        final image is not included in the resulting .rtdc file.
+    verbose: bool
+        If `True`, print messages to stdout
+    """
+    cmp_kw = hdf5plugin.Zstd(clevel=5)
+    if path_tdms is None or path_rtdc is None:
+        parser = tdms2rtdc_parser()
+        args = parser.parse_args()
+
+        path_tdms = pathlib.Path(args.tdms_path).resolve()
+        path_rtdc = pathlib.Path(args.rtdc_path)
+        compute_features = args.compute_features
+        skip_initial_empty_image = not args.include_empty_boundary_images
+        skip_final_empty_image = not args.include_empty_boundary_images
+        verbose = True
+
+    # Determine whether input path is a tdms file or a directory
+    if path_tdms.is_dir():
+        # we have a directory to search
+        files_tdms = fmt_tdms.get_tdms_files(path_tdms)
+        if path_rtdc.is_file():
+            raise ValueError(
+                f"Output path is a file, expected folder: '{path_rtdc}'!")
+        files_rtdc = []
+        for path_in in files_tdms:
+            path_in = pathlib.Path(path_in)
+            rp = path_in.relative_to(path_tdms)
+            # determine output file name (same relative path)
+            rpr = path_rtdc / rp.with_suffix(".rtdc")
+            files_rtdc.append(rpr)
+    else:
+        # we have a single file or a non-existent path
+        files_tdms = [path_tdms]
+        files_rtdc = [path_rtdc]
+
+    files_tdms, files_rtdc, files_temp = common.setup_task_paths(
+        paths_in=files_tdms,
+        paths_out=files_rtdc,
+        allowed_input_suffixes=[".tdms"]
+    )
+
+    for ii in range(len(files_tdms)):
+        path_in = files_tdms[ii]
+        path_out = files_rtdc[ii]
+        path_temp = files_temp[ii]
+
+        if verbose:
+            common.print_info(
+                f"Converting {ii+1:d}/{len(files_tdms):d}: {path_in}")
+        # create directory
+        path_out.parent.mkdir(parents=True, exist_ok=True)
+        # load and export dataset
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            # ignore ResourceWarning: unclosed file <_io.BufferedReader...>
+            warnings.simplefilter("ignore", ResourceWarning)  # noqa: F821
+            # ignore SlowVideoWarning
+            warnings.simplefilter("ignore",
+                                  fmt_tdms.event_image.SlowVideoWarning)
+            if skip_initial_empty_image:
+                # If the initial frame is skipped when empty,
+                # suppress any related warning messages.
+                warnings.simplefilter(
+                    "ignore",
+                    fmt_tdms.event_image.InitialFrameMissingWarning)
+
+            with new_dataset(path_in) as ds:
+                # determine features to export
+                if compute_features:
+                    features = ds.features
+                else:
+                    # consider special case for "image", "trace", and "contour"
+                    # (This will export both "mask" and "contour".
+                    # The "mask" is computed from "contour" and it is needed
+                    # by dclab for other ancillary features. We still keep
+                    # "contour" because it is original data.
+                    features = ds.features_innate
+
+                common.skip_empty_image_events(
+                    ds=ds,
+                    initial=skip_initial_empty_image,
+                    final=skip_final_empty_image)
+                # export as hdf5
+                ds.export.hdf5(path=path_temp,
+                               features=features,
+                               filtered=True,
+                               override=True,
+                               compression_kwargs=cmp_kw)
+
+                # write logs
+                custom_dict = {}
+                # computed features
+                cfeats = list(set(features) - set(ds.features_innate))
+                if "mask" in features:
+                    # Mask is always computed from contour data
+                    cfeats.append("mask")
+                custom_dict["ancillary features"] = sorted(cfeats)
+
+                # command log
+                logs = {"dclab-tdms2rtdc": common.get_command_log(
+                    paths=[path_in], custom_dict=custom_dict)}
+                # warnings log
+                if w:
+                    logs["dclab-tdms2rtdc-warnings"] = \
+                        common.assemble_warnings(w)
+                logs.update(ds.logs)
+                with RTDCWriter(path_temp, compression_kwargs=cmp_kw) as hw:
+                    for name in logs:
+                        hw.store_log(name, logs[name])
+
+                # Finally, rename temp to out
+                path_temp.rename(path_out)
+
+
+def tdms2rtdc_parser():
+    descr = "Convert RT-DC .tdms files to the hdf5-based .rtdc file format. " \
+            + "Note: Do not delete original .tdms files after conversion. " \
+            + "The conversion might be incomplete."
+    parser = argparse.ArgumentParser(description=descr)
+    parser.add_argument('--compute-ancillary-features',
+                        dest='compute_features',
+                        action='store_true',
+                        help='Compute features, such as volume or emodulus, '
+                             + 'that are otherwise computed on-the-fly. '
+                             + 'Use this if you want to minimize analysis '
+                             + 'time in e.g. DCscope. CAUTION: ancillary '
+                             + 'feature recipes might be subject to change '
+                             + '(e.g. if an error is found in the recipe). '
+                             + 'Disabling this option maximizes '
+                             + 'compatibility with future versions and '
+                             + 'allows to isolate the original data.')
+    parser.set_defaults(compute_features=False)
+    parser.add_argument('--include-empty-boundary-images',
+                        dest='include_empty_boundary_images',
+                        action='store_true',
+                        help='In old versions of Shape-In, the first or last '
+                             + 'images were sometimes not stored in the '
+                             + 'resulting .avi file. In dclab, such images '
+                             + 'are represented as zero-valued images. Set '
+                             + 'this option, if you wish to include these '
+                             + 'events with empty image data.')
+    parser.set_defaults(include_empty_boundary_images=False)
+    parser.add_argument('tdms_path', metavar="TDMS_PATH", type=str,
+                        help='Input path (tdms file or folder containing '
+                             + 'tdms files)')
+    parser.add_argument('rtdc_path', metavar="RTDC_PATH", type=str,
+                        help='Output path (file or folder), existing data '
+                             + 'will be overridden')
+    parser.add_argument('--version', action='version',
+                        version=f'dclab-tdms2rtdc {version}')
+    return parser

dclab/cli/task_verify_dataset.py

@@ -0,0 +1,75 @@
+"""Verify dataset integrity"""
+import argparse
+import pathlib
+import sys
+
+from ..rtdc_dataset import check_dataset, fmt_tdms
+from .._version import version
+
+from . import common
+
+
+def verify_dataset(path_in=None):
+    """Perform checks on experimental datasets"""
+    if path_in is None:
+        parser = verify_dataset_parser()
+        args = parser.parse_args()
+        path_in = pathlib.Path(args.path).resolve()
+    if not path_in.exists():
+        common.print_violation(f"File not found: {path_in}")
+        return sys.exit(4)
+    else:
+        common.print_info(f"Checking {path_in}")
+    # The exit status of this script. Non-zero exit status means:
+    # 1: alerts
+    # 2: violations
+    # 3: alerts and violations
+    # 4: other error
+    exit_status = 4
+    try:
+        viol, aler, info = check_dataset(path_in)
+    except fmt_tdms.InvalidTDMSFileFormatError:
+        common.print_violation("Invalid tdms file format!")
+    except fmt_tdms.IncompleteTDMSFileFormatError:
+        common.print_violation("Incomplete dataset!")
+    except fmt_tdms.ContourIndexingError:
+        common.print_violation("Invalid contour data!")
+    except fmt_tdms.InvalidVideoFileError:
+        common.print_violation("Invalid image data!")
+    except BaseException as e:
+        common.print_violation(f"{e.__class__.__name__}: {', '.join(e.args)}")
+    else:
+        for inf in info:
+            common.print_info(inf)
+        for ale in aler:
+            common.print_alert(ale)
+        for vio in viol:
+            common.print_violation(vio)
+        common.print_info(f"Check Complete: {len(viol)} violations and "
+                          + f"{len(aler)} alerts")
+        if aler and viol:
+            exit_status = 3
+        elif aler:
+            exit_status = 1
+        elif viol:
+            exit_status = 2
+        else:
+            # everything is ok
+            exit_status = 0
+    finally:
+        # return sys.exit for testing (monkeypatched)
+        return sys.exit(exit_status)
+
+
+def verify_dataset_parser():
+    descr = "Check experimental datasets for completeness. This command " \
+            + "is used e.g. to enforce data integrity with Shape-In. The " \
+            + "following exit codes are defined: ``0: valid dataset``, " \
+            + "``1: alerts encountered``, ``2: violations encountered``, " \
+            + "``3: alerts and violations``, ``4: other error``."
+    parser = argparse.ArgumentParser(description=descr)
+    parser.add_argument('path', metavar='PATH', type=str,
+                        help='Path to experimental dataset')
+    parser.add_argument('--version', action='version',
+                        version=f'dclab-verify-dataset {version}')
+    return parser

dclab/definitions/__init__.py

@@ -0,0 +1,79 @@
+# flake8: noqa: F401
+import warnings
+
+from .feat_const import (
+    FEATURES_SCALAR, FEATURES_NON_SCALAR, FLUOR_TRACES,
+    # these should not be used
+    feature_names, feature_labels, feature_name2label,
+    # this one should also not be used, but we wait with deprecation,
+    # because DCscope heavily relies on it (it shouldn't)
+    scalar_feature_names
+    )
+from .feat_logic import (
+    check_feature_shape, feature_exists, get_feature_label,
+    scalar_feature_exists
+    )
+from .meta_const import (
+    CFG_ANALYSIS, CFG_METADATA, config_keys,
+    # these should not be used (in contrast, `config_keys` is quite
+    # important for iterating over all internal configuration keys)
+    config_descr, config_funcs, config_types,
+    )
+from .meta_logic import (
+    config_key_exists,
+    get_config_value_descr, get_config_value_func, get_config_value_type
+    )
+
+
+class DeprecatedObject:
+    def __init__(self, anobject, name, new):
+        self.anobject = anobject
+        self.message = f"dclab.dfn.{name} is deprecated, please use " \
+            + f"dclab.dfn.{new} instead."
+
+    def __getattr__(self, item):
+        warnings.warn(self.message, DeprecationWarning)
+        return self.anobject.__getattr__(item)
+
+    def __getitem__(self, item):
+        warnings.warn(self.message, DeprecationWarning)
+        return self.anobject.__getitem__(item)
+
+    def __iter__(self):
+        warnings.warn(self.message, DeprecationWarning)
+        return self.anobject.__iter__()
+
+    def __contains__(self, item):
+        warnings.warn(self.message, DeprecationWarning)
+        return self.anobject.__contains__(item)
+
+
+config_descr = DeprecatedObject(config_descr,
+                                "config_descr",
+                                "get_config_value_descr",
+                                )
+
+config_funcs = DeprecatedObject(config_funcs,
+                                "config_funcs",
+                                "get_config_value_func",
+                                )
+
+config_types = DeprecatedObject(config_types,
+                                "config_types",
+                                "get_config_value_type",
+                                )
+
+feature_names = DeprecatedObject(feature_names,
+                                 "feature_names",
+                                 "feature_exists",
+                                 )
+
+feature_labels = DeprecatedObject(feature_labels,
+                                  "feature_labels",
+                                  "get_feature_label",
+                                  )
+
+feature_name2label = DeprecatedObject(feature_name2label,
+                                      "feature_name2label",
+                                      "get_feature_label",
+                                      )