dcnum 0.25.7__tar.gz → 0.25.9__tar.gz

{dcnum-0.25.7 → dcnum-0.25.9}/CHANGELOG

@@ -1,3 +1,10 @@
+0.25.9
+ - fix: md5 sum caused PermissionError on Windows for background computation
+0.25.8
+ - fix: invalid type definition for `concatenated_hdf5_data`
+ - enh: make sure `HDF5Data.path` is converted to `pathlib.Path` if it is `str`
+ - docs: properly employ license MIT
+ - ref: move `concatenated_hdf5_data` to `read.hdf5_concat` submodule
 0.25.7
  - enh: `HDF5Writer.store_log` returns created dataset
  - docs: add code reference using apidoc

{dcnum-0.25.7 → dcnum-0.25.9}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Deformability Cytometry Analysis
+Copyright (c) 2023 Paul Müller
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{dcnum-0.25.7 → dcnum-0.25.9}/PKG-INFO

@@ -1,10 +1,10 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dcnum
-Version: 0.25.7
+Version: 0.25.9
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
-License: MIT
+License-Expression: MIT
 Project-URL: source, https://github.com/DC-Analysis/dcnum
 Project-URL: tracker, https://github.com/DC-Analysis/dcnum/issues
 Project-URL: documentation, https://dcnum.readthedocs.io/en/stable/
@@ -27,6 +27,7 @@ Requires-Dist: scikit-image<1,>=0.24
 Requires-Dist: scipy<1.15.0,>=1.8.0
 Provides-Extra: torch
 Requires-Dist: torch>=2.2; extra == "torch"
+Dynamic: license-file
 
 |dcnum|
 =======

{dcnum-0.25.7 → dcnum-0.25.9}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
     'Topic :: Scientific/Engineering :: Visualization',
     'Intended Audience :: Science/Research',
 ]
-license = {text = "MIT"}
+license = "MIT"
 dependencies = [
     "h5py>=3.0.0, <4",  # BSD
     "hdf5plugin>=3.3.1, <6",  # MIT and others (per plugin)

dcnum-0.25.9/src/dcnum/__init__.py

@@ -0,0 +1,25 @@
+"""Base library for deformability cytometry postprocessing
+MIT License
+
+Copyright (c) 2023 Paul Müller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+# flake8: noqa: F401
+from ._version import __version__, __version_tuple__

{dcnum-0.25.7 → dcnum-0.25.9}/src/dcnum/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.25.7'
-__version_tuple__ = version_tuple = (0, 25, 7)
+__version__ = version = '0.25.9'
+__version_tuple__ = version_tuple = (0, 25, 9)

{dcnum-0.25.7 → dcnum-0.25.9}/src/dcnum/feat/feat_background/base.py

@@ -9,7 +9,7 @@ import time
 import h5py
 
 from ...meta import ppid
-from ...read import HDF5Data
+from ...read import HDF5Data, md5sum
 from ...write import HDF5Writer, create_with_basins, set_default_filter_kwargs
 
 
@@ -87,6 +87,9 @@ class Background(abc.ABC):
 
         # Check whether user passed an array or a path
         if isinstance(input_data, pathlib.Path):
+            # Compute MD5 sum before opening the file so that we don't
+            # get a file-locking issue (PermissionError) on Windows.
+            md5_5m = md5sum(input_data, blocksize=65536, count=80)
             if str(input_data.resolve()) == str(output_path.resolve()):
                 self.h5in = h5py.File(input_data, "a", libver="latest")
                 self.h5out = self.h5in
@@ -98,7 +101,7 @@ class Background(abc.ABC):
             #       the ImageCache. We have to go via the ImageCache route,
             #       because HDF5Data properly resolves basins and the image
             #       feature might be in a basin.
-            self.hdin = HDF5Data(self.h5in)
+            self.hdin = HDF5Data(self.h5in, md5_5m=md5_5m)
             self.input_data = self.hdin.image.h5ds
         else:
             self.input_data = input_data

{dcnum-0.25.7 → dcnum-0.25.9}/src/dcnum/read/__init__.py

@@ -2,5 +2,6 @@
 from .cache import md5sum
 from .const import PROTECTED_FEATURES
 from .detect_flicker import detect_flickering
-from .hdf5_data import HDF5Data, HDF5ImageCache, concatenated_hdf5_data
+from .hdf5_data import HDF5Data, HDF5ImageCache
+from .hdf5_concat import concatenated_hdf5_data
 from .mapped import get_mapping_indices, get_mapped_object

dcnum-0.25.9/src/dcnum/read/hdf5_concat.py

@@ -0,0 +1,145 @@
+import io
+import pathlib
+import tempfile
+import warnings
+
+import h5py
+import numpy as np
+
+
+from .hdf5_data import HDF5Data
+
+
+def concatenated_hdf5_data(paths: list[pathlib.Path],
+                           path_out: bool | pathlib.Path | None = True,
+                           compute_frame: bool = True,
+                           features: list[str] | None = None):
+    """Return a virtual dataset concatenating all the input paths
+
+    Parameters
+    ----------
+    paths:
+        Path of the input HDF5 files that will be concatenated along
+        the feature axis. The metadata will be taken from the first
+        file.
+    path_out:
+        If `None`, then the dataset is created in memory. If `True`
+        (default), create a file on disk. If a pathlib.Path is specified,
+        the dataset is written to that file. Note that datasets in memory
+        are likely not pickable (so don't use them for multiprocessing).
+    compute_frame:
+        Whether to compute the "events/frame" feature, taking the frame
+        data from the input files and properly incrementing them along
+        the file index.
+    features:
+        List of features to take from the input files.
+
+    Notes
+    -----
+    - If one of the input files does not contain a feature from the first
+      input `paths`, then a `ValueError` is raised. Use the `features`
+      argument to specify which features you need instead.
+    - Basins are not considered.
+    """
+    h5kwargs = {"mode": "w", "libver": "latest"}
+    if isinstance(path_out, (pathlib.Path, str)):
+        h5kwargs["name"] = path_out
+    elif path_out is True:
+        tf = tempfile.NamedTemporaryFile(prefix="dcnum_vc_",
+                                         suffix=".hdf5",
+                                         delete=False)
+        tf.write(b"dummy")
+        h5kwargs["name"] = tf.name
+        tf.close()
+    elif path_out is None:
+        h5kwargs["name"] = io.BytesIO()
+    else:
+        raise ValueError(
+            f"Invalid type for `path_out`: {type(path_out)} ({path_out}")
+
+    if len(paths) == 0:
+        raise ValueError("Please specify at least one file in `paths`!")
+    elif len(paths) == 1:
+        warnings.warn("Only one file passed to `concatenated_hdf5_data`; this "
+                      "is equivalent to using `HDF5Data`, but slower.")
+
+    frames = []
+
+    with h5py.File(**h5kwargs) as hv:
+        # determine the sizes of the input files
+        shapes = {}
+        dtypes = {}
+        size = 0
+        for ii, pp in enumerate(paths):
+            pp = pathlib.Path(pp).resolve()
+            with h5py.File(pp, libver="latest") as h5:
+                # get all feature keys
+                featsi = sorted(h5["events"].keys())
+                # get metadata
+                if ii == 0:
+                    meta = dict(h5.attrs)
+                    if not features:
+                        features = featsi
+                # make sure number of features are consistent
+                if not set(features) <= set(featsi):
+                    raise ValueError(
+                        f"File {pp} contains more features than {paths[0]}!")
+                # populate shapes for all features
+                for feat in features:
+                    if not isinstance(h5["events"][feat], h5py.Dataset):
+                        warnings.warn(
+                            f"Ignoring {feat}; not implemented yet!")
+                        continue
+                    if feat in ["frame", "time"]:
+                        continue
+                    shapes.setdefault(feat, []).append(
+                        h5["events"][feat].shape)
+                    if ii == 0:
+                        dtypes[feat] = h5["events"][feat].dtype
+                # increment size
+                size += h5["events"][features[0]].shape[0]
+                # remember the frame feature if requested
+                if compute_frame:
+                    frames.append(h5["events/frame"][:])
+
+        # write metadata
+        hv.attrs.update(meta)
+
+        # Create the virtual datasets
+        for feat in shapes:
+            if len(shapes[feat][0]) == 1:
+                # scalar feature
+                shape = (sum([sh[0] for sh in shapes[feat]]))
+            else:
+                # non-scalar feature
+                length = (sum([sh[0] for sh in shapes[feat]]))
+                shape = list(shapes[feat][0])
+                shape[0] = length
+                shape = tuple(shape)
+            layout = h5py.VirtualLayout(shape=shape, dtype=dtypes[feat])
+            loc = 0
+            for jj, pp in enumerate(paths):
+                vsource = h5py.VirtualSource(pp, f"events/{feat}",
+                                             shape=shapes[feat][jj])
+                cursize = shapes[feat][jj][0]
+                layout[loc:loc+cursize] = vsource
+                loc += cursize
+            hv.create_virtual_dataset(f"/events/{feat}", layout, fillvalue=0)
+
+        if compute_frame:
+            # concatenate frames and store in dataset
+            frame_concat = np.zeros(size, dtype=np.uint64)
+            locf = 0  # indexing location
+            prevmax = 0  # maximum frame number stored so far in array
+            for fr in frames:
+                offset = prevmax + 1 - fr[0]
+                frame_concat[locf:locf+fr.size] = fr + offset
+                locf += fr.size
+                prevmax = fr[-1] + offset
+            hv.create_dataset("/events/frame", data=frame_concat)
+
+        # write metadata
+        hv.attrs["experiment:event count"] = size
+
+    data = HDF5Data(h5kwargs["name"])
+    return data

{dcnum-0.25.7 → dcnum-0.25.9}/src/dcnum/read/hdf5_data.py

@@ -1,11 +1,9 @@
 from __future__ import annotations
 
 import hashlib
-import io
 import json
 import numbers
 import pathlib
-import tempfile
 from typing import Dict, BinaryIO, List
 import uuid
 import warnings
@@ -160,7 +158,10 @@ class HDF5Data:
         if not hasattr(self, "h5"):
             self.h5 = None
 
-        self.path = state["path"]
+        path = state["path"]
+        if isinstance(path, str):
+            path = pathlib.Path(path)
+        self.path = path
 
         self.md5_5m = state["md5_5m"]
         if self.md5_5m is None:
@@ -552,136 +553,10 @@ class HDF5Data:
         return self._keys
 
 
-def concatenated_hdf5_data(paths: List[pathlib.Path],
-                           path_out: True | pathlib.Path | None = True,
-                           compute_frame: bool = True,
-                           features: List[str] | None = None):
-    """Return a virtual dataset concatenating all the input paths
-
-    Parameters
-    ----------
-    paths:
-        Path of the input HDF5 files that will be concatenated along
-        the feature axis. The metadata will be taken from the first
-        file.
-    path_out:
-        If `None`, then the dataset is created in memory. If `True`
-        (default), create a file on disk. If a pathlib.Path is specified,
-        the dataset is written to that file. Note that datasets in memory
-        are likely not pickable (so don't use them for multiprocessing).
-    compute_frame:
-        Whether to compute the "events/frame" feature, taking the frame
-        data from the input files and properly incrementing them along
-        the file index.
-    features:
-        List of features to take from the input files.
-
-    Notes
-    -----
-    - If one of the input files does not contain a feature from the first
-      input `paths`, then a `ValueError` is raised. Use the `features`
-      argument to specify which features you need instead.
-    - Basins are not considered.
-    """
-    h5kwargs = {"mode": "w", "libver": "latest"}
-    if isinstance(path_out, (pathlib.Path, str)):
-        h5kwargs["name"] = path_out
-    elif path_out is True:
-        tf = tempfile.NamedTemporaryFile(prefix="dcnum_vc_",
-                                         suffix=".hdf5",
-                                         delete=False)
-        tf.write(b"dummy")
-        h5kwargs["name"] = tf.name
-        tf.close()
-    elif path_out is None:
-        h5kwargs["name"] = io.BytesIO()
-    else:
-        raise ValueError(
-            f"Invalid type for `path_out`: {type(path_out)} ({path_out}")
-
-    if len(paths) == 0:
-        raise ValueError("Please specify at least one file in `paths`!")
-    elif len(paths) == 1:
-        warnings.warn("Only one file passed to `concatenated_hdf5_data`; this "
-                      "is equivalent to using `HDF5Data`, but slower.")
-
-    frames = []
-
-    with h5py.File(**h5kwargs) as hv:
-        # determine the sizes of the input files
-        shapes = {}
-        dtypes = {}
-        size = 0
-        for ii, pp in enumerate(paths):
-            pp = pathlib.Path(pp).resolve()
-            with h5py.File(pp, libver="latest") as h5:
-                # get all feature keys
-                featsi = sorted(h5["events"].keys())
-                # get metadata
-                if ii == 0:
-                    meta = dict(h5.attrs)
-                    if not features:
-                        features = featsi
-                # make sure number of features are consistent
-                if not set(features) <= set(featsi):
-                    raise ValueError(
-                        f"File {pp} contains more features than {paths[0]}!")
-                # populate shapes for all features
-                for feat in features:
-                    if not isinstance(h5["events"][feat], h5py.Dataset):
-                        warnings.warn(
-                            f"Ignoring {feat}; not implemented yet!")
-                        continue
-                    if feat in ["frame", "time"]:
-                        continue
-                    shapes.setdefault(feat, []).append(
-                        h5["events"][feat].shape)
-                    if ii == 0:
-                        dtypes[feat] = h5["events"][feat].dtype
-                # increment size
-                size += h5["events"][features[0]].shape[0]
-                # remember the frame feature if requested
-                if compute_frame:
-                    frames.append(h5["events/frame"][:])
-
-        # write metadata
-        hv.attrs.update(meta)
-
-        # Create the virtual datasets
-        for feat in shapes:
-            if len(shapes[feat][0]) == 1:
-                # scalar feature
-                shape = (sum([sh[0] for sh in shapes[feat]]))
-            else:
-                # non-scalar feature
-                length = (sum([sh[0] for sh in shapes[feat]]))
-                shape = list(shapes[feat][0])
-                shape[0] = length
-                shape = tuple(shape)
-            layout = h5py.VirtualLayout(shape=shape, dtype=dtypes[feat])
-            loc = 0
-            for jj, pp in enumerate(paths):
-                vsource = h5py.VirtualSource(pp, f"events/{feat}",
-                                             shape=shapes[feat][jj])
-                cursize = shapes[feat][jj][0]
-                layout[loc:loc+cursize] = vsource
-                loc += cursize
-            hv.create_virtual_dataset(f"/events/{feat}", layout, fillvalue=0)
-
-        if compute_frame:
-            # concatenate frames and store in dataset
-            frame_concat = np.zeros(size, dtype=np.uint64)
-            locf = 0  # indexing location
-            prevmax = 0  # maximum frame number stored so far in array
-            for fr in frames:
-                offset = prevmax + 1 - fr[0]
-                frame_concat[locf:locf+fr.size] = fr + offset
-                locf += fr.size
-                prevmax = fr[-1] + offset
-            hv.create_dataset("/events/frame", data=frame_concat)
-
-        # write metadata
-        hv.attrs["experiment:event count"] = size
-
-    data = HDF5Data(h5kwargs["name"])
-    return data
+def concatenated_hdf5_data(*args, **kwargs):
+    warnings.warn(
+        "Please use `dcnum.read.hdf5_concat.concatenated_hdf5_data`. "
+        "Accessing this method via `dcnum.read.hdf5_data` is deprecated.",
+        DeprecationWarning)
+    from . import hdf5_concat
+    return hdf5_concat.concatenated_hdf5_data(*args, **kwargs)

{dcnum-0.25.7 → dcnum-0.25.9}/src/dcnum.egg-info/PKG-INFO

@@ -1,10 +1,10 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: dcnum
-Version: 0.25.7
+Version: 0.25.9
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
-License: MIT
+License-Expression: MIT
 Project-URL: source, https://github.com/DC-Analysis/dcnum
 Project-URL: tracker, https://github.com/DC-Analysis/dcnum/issues
 Project-URL: documentation, https://dcnum.readthedocs.io/en/stable/
@@ -27,6 +27,7 @@ Requires-Dist: scikit-image<1,>=0.24
 Requires-Dist: scipy<1.15.0,>=1.8.0
 Provides-Extra: torch
 Requires-Dist: torch>=2.2; extra == "torch"
+Dynamic: license-file
 
 |dcnum|
 =======

{dcnum-0.25.7 → dcnum-0.25.9}/src/dcnum.egg-info/SOURCES.txt

@@ -55,6 +55,7 @@ src/dcnum/read/__init__.py
 src/dcnum/read/cache.py
 src/dcnum/read/const.py
 src/dcnum/read/detect_flicker.py
+src/dcnum/read/hdf5_concat.py
 src/dcnum/read/hdf5_data.py
 src/dcnum/read/mapped.py
 src/dcnum/segm/__init__.py
@@ -105,6 +106,7 @@ tests/test_read_concat_hdf5.py
 tests/test_read_detect_flicker.py
 tests/test_read_hdf5.py
 tests/test_read_hdf5_basins.py
+tests/test_read_hdf5_concat.py
 tests/test_read_hdf5_index_mapping.py
 tests/test_segm_base.py
 tests/test_segm_mpo.py

dcnum-0.25.9/tests/test_init.py

@@ -0,0 +1,7 @@
+import dcnum
+
+
+def test_init():
+    # Checks if the object `dcnum` has an attribute named `__version__`.
+    # If not, an AssertionError is raised.
+    assert hasattr(dcnum, "__version__")

dcnum-0.25.9/tests/test_read_hdf5_concat.py

@@ -0,0 +1,29 @@
+import pathlib
+
+from dcnum import read
+
+from helper_methods import retrieve_data
+
+
+def test_concatenated_hdf5_as_file(tmp_path):
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
+    path_concat = tmp_path / "concatenated.rtdc"
+    with read.concatenated_hdf5_data([path] * 10, path_out=path_concat):
+        pass
+    assert path_concat.exists()
+
+
+def test_concatenated_hdf5_new_file():
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
+    with read.concatenated_hdf5_data([path] * 10, path_out=True) as hd:
+        pass
+        path_out = hd.path
+    assert path_out.exists()
+
+
+def test_concatenated_hdf5_in_memory():
+    path = retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip")
+    with read.concatenated_hdf5_data([path] * 10, path_out=None) as hd:
+        pass
+        path_out = hd.path
+    assert not isinstance(path_out, (pathlib.Path, str))

dcnum-0.25.7/src/dcnum/__init__.py

@@ -1,2 +0,0 @@
-# flake8: noqa: F401
-from ._version import __version__, __version_tuple__

dcnum-0.25.7/tests/test_init.py

@@ -1,5 +0,0 @@
-import dcnum
-
-
-def test_init():
-    assert hasattr(dcnum, "__version__")