dcnum 0.25.7__py3-none-any.whl → 0.25.9__py3-none-any.whl

dcnum/__init__.py

@@ -1,2 +1,25 @@
+"""Base library for deformability cytometry postprocessing
+MIT License
+
+Copyright (c) 2023 Paul Müller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
 # flake8: noqa: F401
 from ._version import __version__, __version_tuple__

dcnum/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.25.7'
-__version_tuple__ = version_tuple = (0, 25, 7)
+__version__ = version = '0.25.9'
+__version_tuple__ = version_tuple = (0, 25, 9)

dcnum/feat/feat_background/base.py

@@ -9,7 +9,7 @@ import time
 import h5py
 
 from ...meta import ppid
-from ...read import HDF5Data
+from ...read import HDF5Data, md5sum
 from ...write import HDF5Writer, create_with_basins, set_default_filter_kwargs
 
 
@@ -87,6 +87,9 @@ class Background(abc.ABC):
 
         # Check whether user passed an array or a path
         if isinstance(input_data, pathlib.Path):
+            # Compute MD5 sum before opening the file so that we don't
+            # get a file-locking issue (PermissionError) on Windows.
+            md5_5m = md5sum(input_data, blocksize=65536, count=80)
             if str(input_data.resolve()) == str(output_path.resolve()):
                 self.h5in = h5py.File(input_data, "a", libver="latest")
                 self.h5out = self.h5in
@@ -98,7 +101,7 @@ class Background(abc.ABC):
             #       the ImageCache. We have to go via the ImageCache route,
             #       because HDF5Data properly resolves basins and the image
             #       feature might be in a basin.
-            self.hdin = HDF5Data(self.h5in)
+            self.hdin = HDF5Data(self.h5in, md5_5m=md5_5m)
             self.input_data = self.hdin.image.h5ds
         else:
             self.input_data = input_data

dcnum/read/__init__.py

@@ -2,5 +2,6 @@
 from .cache import md5sum
 from .const import PROTECTED_FEATURES
 from .detect_flicker import detect_flickering
-from .hdf5_data import HDF5Data, HDF5ImageCache, concatenated_hdf5_data
+from .hdf5_data import HDF5Data, HDF5ImageCache
+from .hdf5_concat import concatenated_hdf5_data
 from .mapped import get_mapping_indices, get_mapped_object

dcnum/read/hdf5_concat.py

@@ -0,0 +1,145 @@
+import io
+import pathlib
+import tempfile
+import warnings
+
+import h5py
+import numpy as np
+
+
+from .hdf5_data import HDF5Data
+
+
+def concatenated_hdf5_data(paths: list[pathlib.Path],
+                           path_out: bool | pathlib.Path | None = True,
+                           compute_frame: bool = True,
+                           features: list[str] | None = None):
+    """Return a virtual dataset concatenating all the input paths
+
+    Parameters
+    ----------
+    paths:
+        Path of the input HDF5 files that will be concatenated along
+        the feature axis. The metadata will be taken from the first
+        file.
+    path_out:
+        If `None`, then the dataset is created in memory. If `True`
+        (default), create a file on disk. If a pathlib.Path is specified,
+        the dataset is written to that file. Note that datasets in memory
+        are likely not pickable (so don't use them for multiprocessing).
+    compute_frame:
+        Whether to compute the "events/frame" feature, taking the frame
+        data from the input files and properly incrementing them along
+        the file index.
+    features:
+        List of features to take from the input files.
+
+    Notes
+    -----
+    - If one of the input files does not contain a feature from the first
+      input `paths`, then a `ValueError` is raised. Use the `features`
+      argument to specify which features you need instead.
+    - Basins are not considered.
+    """
+    h5kwargs = {"mode": "w", "libver": "latest"}
+    if isinstance(path_out, (pathlib.Path, str)):
+        h5kwargs["name"] = path_out
+    elif path_out is True:
+        tf = tempfile.NamedTemporaryFile(prefix="dcnum_vc_",
+                                         suffix=".hdf5",
+                                         delete=False)
+        tf.write(b"dummy")
+        h5kwargs["name"] = tf.name
+        tf.close()
+    elif path_out is None:
+        h5kwargs["name"] = io.BytesIO()
+    else:
+        raise ValueError(
+            f"Invalid type for `path_out`: {type(path_out)} ({path_out}")
+
+    if len(paths) == 0:
+        raise ValueError("Please specify at least one file in `paths`!")
+    elif len(paths) == 1:
+        warnings.warn("Only one file passed to `concatenated_hdf5_data`; this "
+                      "is equivalent to using `HDF5Data`, but slower.")
+
+    frames = []
+
+    with h5py.File(**h5kwargs) as hv:
+        # determine the sizes of the input files
+        shapes = {}
+        dtypes = {}
+        size = 0
+        for ii, pp in enumerate(paths):
+            pp = pathlib.Path(pp).resolve()
+            with h5py.File(pp, libver="latest") as h5:
+                # get all feature keys
+                featsi = sorted(h5["events"].keys())
+                # get metadata
+                if ii == 0:
+                    meta = dict(h5.attrs)
+                    if not features:
+                        features = featsi
+                # make sure number of features are consistent
+                if not set(features) <= set(featsi):
+                    raise ValueError(
+                        f"File {pp} contains more features than {paths[0]}!")
+                # populate shapes for all features
+                for feat in features:
+                    if not isinstance(h5["events"][feat], h5py.Dataset):
+                        warnings.warn(
+                            f"Ignoring {feat}; not implemented yet!")
+                        continue
+                    if feat in ["frame", "time"]:
+                        continue
+                    shapes.setdefault(feat, []).append(
+                        h5["events"][feat].shape)
+                    if ii == 0:
+                        dtypes[feat] = h5["events"][feat].dtype
+                # increment size
+                size += h5["events"][features[0]].shape[0]
+                # remember the frame feature if requested
+                if compute_frame:
+                    frames.append(h5["events/frame"][:])
+
+        # write metadata
+        hv.attrs.update(meta)
+
+        # Create the virtual datasets
+        for feat in shapes:
+            if len(shapes[feat][0]) == 1:
+                # scalar feature
+                shape = (sum([sh[0] for sh in shapes[feat]]))
+            else:
+                # non-scalar feature
+                length = (sum([sh[0] for sh in shapes[feat]]))
+                shape = list(shapes[feat][0])
+                shape[0] = length
+                shape = tuple(shape)
+            layout = h5py.VirtualLayout(shape=shape, dtype=dtypes[feat])
+            loc = 0
+            for jj, pp in enumerate(paths):
+                vsource = h5py.VirtualSource(pp, f"events/{feat}",
+                                             shape=shapes[feat][jj])
+                cursize = shapes[feat][jj][0]
+                layout[loc:loc+cursize] = vsource
+                loc += cursize
+            hv.create_virtual_dataset(f"/events/{feat}", layout, fillvalue=0)
+
+        if compute_frame:
+            # concatenate frames and store in dataset
+            frame_concat = np.zeros(size, dtype=np.uint64)
+            locf = 0  # indexing location
+            prevmax = 0  # maximum frame number stored so far in array
+            for fr in frames:
+                offset = prevmax + 1 - fr[0]
+                frame_concat[locf:locf+fr.size] = fr + offset
+                locf += fr.size
+                prevmax = fr[-1] + offset
+            hv.create_dataset("/events/frame", data=frame_concat)
+
+        # write metadata
+        hv.attrs["experiment:event count"] = size
+
+    data = HDF5Data(h5kwargs["name"])
+    return data

dcnum/read/hdf5_data.py

@@ -1,11 +1,9 @@
 from __future__ import annotations
 
 import hashlib
-import io
 import json
 import numbers
 import pathlib
-import tempfile
 from typing import Dict, BinaryIO, List
 import uuid
 import warnings
@@ -160,7 +158,10 @@ class HDF5Data:
         if not hasattr(self, "h5"):
             self.h5 = None
 
-        self.path = state["path"]
+        path = state["path"]
+        if isinstance(path, str):
+            path = pathlib.Path(path)
+        self.path = path
 
         self.md5_5m = state["md5_5m"]
         if self.md5_5m is None:
@@ -552,136 +553,10 @@ class HDF5Data:
         return self._keys
 
 
-def concatenated_hdf5_data(paths: List[pathlib.Path],
-                           path_out: True | pathlib.Path | None = True,
-                           compute_frame: bool = True,
-                           features: List[str] | None = None):
-    """Return a virtual dataset concatenating all the input paths
-
-    Parameters
-    ----------
-    paths:
-        Path of the input HDF5 files that will be concatenated along
-        the feature axis. The metadata will be taken from the first
-        file.
-    path_out:
-        If `None`, then the dataset is created in memory. If `True`
-        (default), create a file on disk. If a pathlib.Path is specified,
-        the dataset is written to that file. Note that datasets in memory
-        are likely not pickable (so don't use them for multiprocessing).
-    compute_frame:
-        Whether to compute the "events/frame" feature, taking the frame
-        data from the input files and properly incrementing them along
-        the file index.
-    features:
-        List of features to take from the input files.
-
-    Notes
-    -----
-    - If one of the input files does not contain a feature from the first
-      input `paths`, then a `ValueError` is raised. Use the `features`
-      argument to specify which features you need instead.
-    - Basins are not considered.
-    """
-    h5kwargs = {"mode": "w", "libver": "latest"}
-    if isinstance(path_out, (pathlib.Path, str)):
-        h5kwargs["name"] = path_out
-    elif path_out is True:
-        tf = tempfile.NamedTemporaryFile(prefix="dcnum_vc_",
-                                         suffix=".hdf5",
-                                         delete=False)
-        tf.write(b"dummy")
-        h5kwargs["name"] = tf.name
-        tf.close()
-    elif path_out is None:
-        h5kwargs["name"] = io.BytesIO()
-    else:
-        raise ValueError(
-            f"Invalid type for `path_out`: {type(path_out)} ({path_out}")
-
-    if len(paths) == 0:
-        raise ValueError("Please specify at least one file in `paths`!")
-    elif len(paths) == 1:
-        warnings.warn("Only one file passed to `concatenated_hdf5_data`; this "
-                      "is equivalent to using `HDF5Data`, but slower.")
-
-    frames = []
-
-    with h5py.File(**h5kwargs) as hv:
-        # determine the sizes of the input files
-        shapes = {}
-        dtypes = {}
-        size = 0
-        for ii, pp in enumerate(paths):
-            pp = pathlib.Path(pp).resolve()
-            with h5py.File(pp, libver="latest") as h5:
-                # get all feature keys
-                featsi = sorted(h5["events"].keys())
-                # get metadata
-                if ii == 0:
-                    meta = dict(h5.attrs)
-                    if not features:
-                        features = featsi
-                # make sure number of features are consistent
-                if not set(features) <= set(featsi):
-                    raise ValueError(
-                        f"File {pp} contains more features than {paths[0]}!")
-                # populate shapes for all features
-                for feat in features:
-                    if not isinstance(h5["events"][feat], h5py.Dataset):
-                        warnings.warn(
-                            f"Ignoring {feat}; not implemented yet!")
-                        continue
-                    if feat in ["frame", "time"]:
-                        continue
-                    shapes.setdefault(feat, []).append(
-                        h5["events"][feat].shape)
-                    if ii == 0:
-                        dtypes[feat] = h5["events"][feat].dtype
-                # increment size
-                size += h5["events"][features[0]].shape[0]
-                # remember the frame feature if requested
-                if compute_frame:
-                    frames.append(h5["events/frame"][:])
-
-        # write metadata
-        hv.attrs.update(meta)
-
-        # Create the virtual datasets
-        for feat in shapes:
-            if len(shapes[feat][0]) == 1:
-                # scalar feature
-                shape = (sum([sh[0] for sh in shapes[feat]]))
-            else:
-                # non-scalar feature
-                length = (sum([sh[0] for sh in shapes[feat]]))
-                shape = list(shapes[feat][0])
-                shape[0] = length
-                shape = tuple(shape)
-            layout = h5py.VirtualLayout(shape=shape, dtype=dtypes[feat])
-            loc = 0
-            for jj, pp in enumerate(paths):
-                vsource = h5py.VirtualSource(pp, f"events/{feat}",
-                                             shape=shapes[feat][jj])
-                cursize = shapes[feat][jj][0]
-                layout[loc:loc+cursize] = vsource
-                loc += cursize
-            hv.create_virtual_dataset(f"/events/{feat}", layout, fillvalue=0)
-
-        if compute_frame:
-            # concatenate frames and store in dataset
-            frame_concat = np.zeros(size, dtype=np.uint64)
-            locf = 0  # indexing location
-            prevmax = 0  # maximum frame number stored so far in array
-            for fr in frames:
-                offset = prevmax + 1 - fr[0]
-                frame_concat[locf:locf+fr.size] = fr + offset
-                locf += fr.size
-                prevmax = fr[-1] + offset
-            hv.create_dataset("/events/frame", data=frame_concat)
-
-        # write metadata
-        hv.attrs["experiment:event count"] = size
-
-    data = HDF5Data(h5kwargs["name"])
-    return data
+def concatenated_hdf5_data(*args, **kwargs):
+    warnings.warn(
+        "Please use `dcnum.read.hdf5_concat.concatenated_hdf5_data`. "
+        "Accessing this method via `dcnum.read.hdf5_data` is deprecated.",
+        DeprecationWarning)
+    from . import hdf5_concat
+    return hdf5_concat.concatenated_hdf5_data(*args, **kwargs)

{dcnum-0.25.7.dist-info → dcnum-0.25.9.dist-info}/METADATA

@@ -1,10 +1,10 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dcnum
-Version: 0.25.7
+Version: 0.25.9
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
-License: MIT
+License-Expression: MIT
 Project-URL: source, https://github.com/DC-Analysis/dcnum
 Project-URL: tracker, https://github.com/DC-Analysis/dcnum/issues
 Project-URL: documentation, https://dcnum.readthedocs.io/en/stable/
@@ -27,6 +27,7 @@ Requires-Dist: scikit-image<1,>=0.24
 Requires-Dist: scipy<1.15.0,>=1.8.0
 Provides-Extra: torch
 Requires-Dist: torch>=2.2; extra == "torch"
+Dynamic: license-file
 
 |dcnum|
 =======

{dcnum-0.25.7.dist-info → dcnum-0.25.9.dist-info}/RECORD

@@ -1,12 +1,12 @@
-dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=hjm1pvHN0ZX7nlIod7PG7Fba0cyr_wIaCY9stsERSag,513
+dcnum/__init__.py,sha256=p0mYg01FQ6nsERYmx_FfVxqqHvYcSMEyIAMBIivAmO8,1206
+dcnum/_version.py,sha256=lcAUa6uFmnm8xHO8GS8Xd2LE1e4-BcAMDWwWwgiDo-I,513
 dcnum/os_env_st.py,sha256=4psq-gPuWTTQ118kCiTx0Mhoyads4Irn6JSUzZk8gyc,3052
 dcnum/feat/__init__.py,sha256=jUJYWTD3VIoDNKrmryXbjHb1rGwYtK4b7VPWihYgUoo,325
 dcnum/feat/event_extractor_manager_thread.py,sha256=6D3RVYBuH7gOoGZ4Kz74n6fhq7MtlTY26kpSwZRqg3M,7972
 dcnum/feat/gate.py,sha256=EOleB83sOlBjc8bjaZfWwGuxfCcEO5mknuFunlw_j7o,7252
 dcnum/feat/queue_event_extractor.py,sha256=s-sC7Inkly_HYbqT3OPB6zWRvKm4TNOewujt1EuZF4M,16183
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
-dcnum/feat/feat_background/base.py,sha256=N2_CqA1LJwvsfhr3xMPbicVs3DVDXtvlNDZ-uPrJi5w,8749
+dcnum/feat/feat_background/base.py,sha256=pWxMCBg2NbO6uGMPl3b-Ia1eXlfHNFsNG5JjWqAFZKI,8979
 dcnum/feat/feat_background/bg_copy.py,sha256=Xfju4DqbrtPyaUf3x-ybrWci5ip_cEbeiEg7Jh56YIY,1182
 dcnum/feat/feat_background/bg_roll_median.py,sha256=rb3JnFzm4aXVLMKdP9wJlRXwTwmKUz2R1RBS9GXPepg,13622
 dcnum/feat/feat_background/bg_sparse_median.py,sha256=PbcTehk46b9l03ZA1mBdyeMPHmGgGN4-59eLy2TzwX8,22634
@@ -27,11 +27,12 @@ dcnum/logic/json_encoder.py,sha256=wb6uk6EeTkXyrvwtLm9uWe0cfmiBannzcsKLsDLHuQo,8
 dcnum/meta/__init__.py,sha256=AVqRgyKXO1orKnE305h88IBvoZ1oz6X11HN1WP5nGvg,60
 dcnum/meta/paths.py,sha256=aIG39JYbZpOlCbPQIlp0SqGumjbGINYhL2AAoznJt5o,1113
 dcnum/meta/ppid.py,sha256=JInGtwSCsO9nr1E1aishm0k9iQIFB-essBKvv5aBE98,8510
-dcnum/read/__init__.py,sha256=LYHyZHgiNTpjV5oEcty-7Kh5topLpHT_cFlNl-QX8gg,262
+dcnum/read/__init__.py,sha256=vhriJFlJ3DlqkAnRPQsOfUQWKYSzLNNp_NZeZ5eBvmo,286
 dcnum/read/cache.py,sha256=ChxokVuMaTfi6N6ZbOTWpNYkPgAAYi1lR8nD7JbzjPQ,6497
 dcnum/read/const.py,sha256=x6LfRwWvIxm6nDWlSADVWqDuzMX6bLzy5kQprwLPzA4,496
 dcnum/read/detect_flicker.py,sha256=XVf7nqaHx6weRTtS7KPa5_WRU2flDQIZTbKspeguqdU,1829
-dcnum/read/hdf5_data.py,sha256=Q4sFT1HBrkrKCX1TUaOpibvz8VFj0ETMa9lw_xIF6tw,26360
+dcnum/read/hdf5_concat.py,sha256=A4Ah_NLxa1ESapEWJcUhdglzi7_E3qKNd81ES7A-_2o,5589
+dcnum/read/hdf5_data.py,sha256=KGMQJYtirBSjnen7FWwfMJB4sr_eOuT8qPGkLZwuMN0,21293
 dcnum/read/mapped.py,sha256=zU2fYdZfLNHn0rKHxDzBhNFMu4--WWa8nSeE2likyZA,3637
 dcnum/segm/__init__.py,sha256=9cLEAd3JWE8IGqDHV-eSDIYOGBfOepd8OcebtNs8Omk,309
 dcnum/segm/segm_thresh.py,sha256=iVhvIhzO0Gw0t3rXOgH71rOI0CNjJJQq4Gg6BulUhK8,948
@@ -50,8 +51,8 @@ dcnum/write/__init__.py,sha256=sK79IlvCFIqf2oFABVeyYedMnHOsEIQpxAauEeNO-Tw,273
 dcnum/write/deque_writer_thread.py,sha256=ao7F1yrVKyufgC4rC0Y2_Vt7snuT6KpI7W2qVxcjdhk,1994
 dcnum/write/queue_collector_thread.py,sha256=-p5vrk9cDhtaIMFIu_cCmvlZJafrFkW68uONonMURYo,11617
 dcnum/write/writer.py,sha256=JkVb4KDBV3oo9r3p2yy9wECO1REx7FG0PRBmVWTxJdk,20577
-dcnum-0.25.7.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.25.7.dist-info/METADATA,sha256=X3578YE2gN-g5mMPHH8bMnFyU9E64PP_ivRsIHPKcYc,2321
-dcnum-0.25.7.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-dcnum-0.25.7.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.25.7.dist-info/RECORD,,
+dcnum-0.25.9.dist-info/licenses/LICENSE,sha256=rX7tNSxP-EhLz-yYUyoBGwjJheA2fiZpT1Iw0LXnJ2M,1069
+dcnum-0.25.9.dist-info/METADATA,sha256=ZIlsz6SBct_bYtVxfz5gaNrKRRVNPUe7zC5LM3rNSfM,2354
+dcnum-0.25.9.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dcnum-0.25.9.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.25.9.dist-info/RECORD,,

{dcnum-0.25.7.dist-info → dcnum-0.25.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.1.0)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{dcnum-0.25.7.dist-info → dcnum-0.25.9.dist-info/licenses}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Deformability Cytometry Analysis
+Copyright (c) 2023 Paul Müller
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal