dcnum 0.25.0__tar.gz → 0.25.1__tar.gz

{dcnum-0.25.0 → dcnum-0.25.1}/.github/workflows/check.yml

@@ -19,7 +19,7 @@ jobs:
       with:
         fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@main
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -29,8 +29,6 @@ jobs:
         python -m pip install coverage flake8 pytest
     - name: Install dcnum
       run: |
-        # mahotas 1.4.15 does not yet support numpy 2.0
-        pip install "numpy<2"
         pip install .[torch]
     - name: List installed packages
       run: |
@@ -42,4 +40,4 @@ jobs:
       run: |
         coverage run --source=dcnum -m pytest tests
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4

{dcnum-0.25.0 → dcnum-0.25.1}/CHANGELOG

@@ -1,3 +1,8 @@
+0.25.1
+ - fix: catch ValueError when computing relative path on different anchors
+ - ref: replace np.string_ with np.bytes_
+ - ref: remove unused `data` argument to QueueCollectorThread
+ - ref: minor speed-up for QueueCollectorThread
 0.25.0
  - feat: identify flickering in raw data via dcnum.read.detect_flickering
  - fix: handle out-of-bounds slice indexing for BaseImageChunkCache

{dcnum-0.25.0 → dcnum-0.25.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.25.0
+Version: 0.25.1
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

dcnum-0.25.1/benchmark/.gitignore

@@ -0,0 +1,2 @@
+bm_*.txt
+*.lprof

dcnum-0.25.1/benchmark/Readme.md

@@ -0,0 +1,11 @@
+This directory contains benchmarking scripts used for optimizing dcnum performance.
+To run all benchmarks, execute `python benchmark.py`. You can also specify
+individual benchmarks or a list of benchmarks (path to `bm_*.py` file)
+as arguments to `benchmark.py`.
+
+The benchmarks are also ideal use cases for identifying bottlenecks with
+tools such as [line profiler](https://kernprof.readthedocs.io/en/latest/),
+since benchmarks can be designed to run in single threads.
+
+    pip install line_profiler
+    kernprof -lv benchmark.py

dcnum-0.25.1/benchmark/benchmark.py

@@ -0,0 +1,54 @@
+import importlib
+import os
+import pathlib
+import sys
+import time
+import timeit
+
+import numpy as np
+
+
+here = pathlib.Path(__file__).parent
+
+
+def print_underline(msg):
+    print(msg)
+    print("-" * len(msg))
+
+
+def run_benchmark(bm_path, repeats=5):
+    print_underline(f"Running {bm_path}")
+    bm_path = pathlib.Path(bm_path).resolve()
+    os.chdir(f"{bm_path.parent}")
+    bm_mod = importlib.import_module(f"{bm_path.stem}")
+
+    reps = []
+    print("Running...", end="\r")
+    for ii in range(repeats):
+        t = timeit.timeit(bm_mod.main, setup=bm_mod.setup, number=1)
+        reps.append(t)
+        print(f"Running {ii + 1}/{repeats}", end="\r")
+    print(reps)
+    print(f"best={min(reps):.3g}, mean={np.mean(reps):.3g}")
+    return reps
+
+
+if __name__ == "__main__":
+    benchmark_paths = []
+    for arg in sys.argv[1:]:
+        if arg.startswith("bm_"):
+            benchmark_paths.append(arg)
+
+    if not benchmark_paths:
+        print("No benchmarking script specified, running all benchmarks.")
+        benchmark_paths = here.glob("bm_*.py")
+
+    results = {}
+
+    for bmp in sorted(benchmark_paths):
+        bmp = pathlib.Path(bmp)
+        print("")
+        res = run_benchmark(bmp)
+        with bmp.with_suffix(".txt").open("a") as fd:
+            fd.write(time.strftime(f"%Y-%m-%d_%H.%M.%S {res}\n"))
+        print("")

dcnum-0.25.1/benchmark/bm_write_deque_writer_thread.py

@@ -0,0 +1,46 @@
+import atexit
+from collections import deque
+import pathlib
+import shutil
+import tempfile
+
+import multiprocessing as mp
+
+import numpy as np
+
+from dcnum import write
+
+
+mp_spawn = mp.get_context('spawn')
+
+
+def setup():
+    global path_out
+    global writer_dq
+    total_frames = 3000
+    batch_size = 500
+    num_batches = 6
+    assert batch_size * num_batches == total_frames
+
+    writer_dq = deque()
+    # Create 1000 events with at most two repetitions in a frame
+    np.random.seed(42)
+    rng = np.random.default_rng()
+
+    # create a sample event
+    for ii in range(num_batches):
+        writer_dq.append(("mask", rng.random((batch_size, 80, 320)) > .5))
+        writer_dq.append(("temp", rng.normal(23, size=batch_size)))
+
+    temp_dir = tempfile.mkdtemp(prefix=pathlib.Path(__file__).name)
+    atexit.register(shutil.rmtree, temp_dir, ignore_errors=True, onerror=None)
+    path_out = pathlib.Path(temp_dir) / "out.rtdc"
+
+
+def main():
+    thr_drw = write.DequeWriterThread(
+        path_out=path_out,
+        dq=writer_dq,
+    )
+    thr_drw.may_stop_loop = True
+    thr_drw.run()

dcnum-0.25.1/benchmark/bm_write_queue_collector_thread.py

@@ -0,0 +1,45 @@
+from collections import deque
+import multiprocessing as mp
+
+import numpy as np
+
+from dcnum import write
+
+
+mp_spawn = mp.get_context('spawn')
+
+
+def setup():
+    global event_queue
+    global writer_dq
+    global feat_nevents
+    batch_size = 1000
+    num_batches = 3
+    num_events = batch_size * num_batches
+    event_queue = mp.Queue()
+    writer_dq = deque()
+    feat_nevents = mp_spawn.Array("i", num_events)
+
+    # Create 1000 events with at most two repetitions in a frame
+    np.random.seed(42)
+    rng = np.random.default_rng()
+    number_order = rng.choice(batch_size, size=batch_size, replace=False)
+
+    # create a sample event
+    event = {
+        "temp": np.atleast_1d(rng.normal(23)),
+        "mask": rng.random((1, 80, 320)) > .5,
+    }
+    for ii in range(num_batches):
+        for idx in number_order:
+            event_queue.put((ii*batch_size + idx, event))
+
+
+def main():
+    thr_coll = write.QueueCollectorThread(
+        event_queue=event_queue,
+        writer_dq=writer_dq,
+        feat_nevents=feat_nevents,
+        write_threshold=500,
+    )
+    thr_coll.run()

{dcnum-0.25.0 → dcnum-0.25.1}/src/dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.25.0'
-__version_tuple__ = version_tuple = (0, 25, 0)
+__version__ = version = '0.25.1'
+__version_tuple__ = version_tuple = (0, 25, 1)

{dcnum-0.25.0 → dcnum-0.25.1}/src/dcnum/logic/ctrl.py

@@ -679,8 +679,14 @@ class DCNumJobRunner(threading.Thread):
                     self.logger.debug(f"Creating basin for {feats}")
                     # Relative and absolute paths.
                     pin = pathlib.Path(hin.filename).resolve()
+                    paths = [pin]
                     pout = pathlib.Path(hout.filename).resolve().parent
-                    paths = [pin, os.path.relpath(pin, pout)]
+                    try:
+                        paths.append(os.path.relpath(pin, pout))
+                    except ValueError:
+                        # This means it is impossible to compute a relative
+                        # path (e.g. different drive letter on Windows).
+                        pass
                     hw.store_basin(name="dcnum basin",
                                    features=feats,
                                    mapping=basinmap0,
@@ -776,7 +782,6 @@ class DCNumJobRunner(threading.Thread):
 
         # Start the data collection thread
         thr_coll = QueueCollectorThread(
-            data=self.dtin,
             event_queue=fe_kwargs["event_queue"],
             writer_dq=writer_dq,
             feat_nevents=fe_kwargs["feat_nevents"],

{dcnum-0.25.0 → dcnum-0.25.1}/src/dcnum/write/queue_collector_thread.py

@@ -8,8 +8,6 @@ from typing import List
 
 import numpy as np
 
-from ..read import HDF5Data
-
 
 class EventStash:
     def __init__(self,
@@ -61,11 +59,10 @@ class EventStash:
             Event dictionary
         """
         idx_loc = index - self.index_offset
-        idx_stop = self.nev_idx[idx_loc]
-        self._tracker[idx_loc] = True
 
         if events:
             slice_loc = None
+            idx_stop = self.nev_idx[idx_loc]
             for feat in events:
                 dev = events[feat]
                 if dev.size:
@@ -76,6 +73,8 @@ class EventStash:
             if slice_loc:
                 self.indices_for_data[slice_loc] = index
 
+        self._tracker[idx_loc] = True
+
     def require_feature(self, feat, sample_data):
         """Create a new empty feature array in `self.events` and return it
 
@@ -87,10 +86,10 @@ class EventStash:
             Sample data for one event of the feature (used to determine
             shape and dtype of the feature array)
         """
-        sample_data = np.array(sample_data)
-        event_shape = sample_data.shape
-        dtype = sample_data.dtype
         if feat not in self.events:
+            sample_data = np.array(sample_data)
+            event_shape = sample_data.shape
+            dtype = sample_data.dtype
             darr = np.zeros((self.size,) + tuple(event_shape),
                             dtype=dtype)
             self.events[feat] = darr
@@ -99,7 +98,6 @@ class EventStash:
 
 class QueueCollectorThread(threading.Thread):
     def __init__(self,
-                 data: HDF5Data,
                  event_queue: mp.Queue,
                  writer_dq: deque,
                  feat_nevents: mp.Array,
@@ -115,9 +113,6 @@ class QueueCollectorThread(threading.Thread):
 
         Parameters
         ----------
-        data:
-            Data source object. This is used for appending additional
-            information
         event_queue:
             A queue object to which other processes or threads write
             events as tuples `(frame_index, events_dict)`.
@@ -146,8 +141,6 @@ class QueueCollectorThread(threading.Thread):
         super(QueueCollectorThread, self).__init__(
               name="QueueCollector", *args, **kwargs)
         self.logger = logging.getLogger("dcnum.write.QueueCollector")
-        #: HDF5 data instance
-        self.data = data
         #: Event queue from which to collect event data
         self.event_queue = event_queue
         #: Writer deque to which event arrays are appended
@@ -169,7 +162,7 @@ class QueueCollectorThread(threading.Thread):
         # We are not writing to `event_queue` so we can safely cancel
         # our queue thread if we are told to stop.
         self.event_queue.cancel_join_thread()
-        # Indexes the current frame in `self.data`.
+        # Indexes the current frame in the input HDF5Data instance.
         last_idx = 0
         self.logger.debug("Started collector thread")
         while True:

{dcnum-0.25.0 → dcnum-0.25.1}/src/dcnum/write/writer.py

@@ -140,10 +140,10 @@ class HDF5Writer:
                                                feat_dtype=feat_dtype),
                 **ds_kwds)
             if len(item_shape) == 2:
-                dset.attrs.create('CLASS', np.string_('IMAGE'))
-                dset.attrs.create('IMAGE_VERSION', np.string_('1.2'))
+                dset.attrs.create('CLASS', np.bytes_('IMAGE'))
+                dset.attrs.create('IMAGE_VERSION', np.bytes_('1.2'))
                 dset.attrs.create('IMAGE_SUBCLASS',
-                                  np.string_('IMAGE_GRAYSCALE'))
+                                  np.bytes_('IMAGE_GRAYSCALE'))
             offset = 0
         else:
             dset = egroup[feat]

{dcnum-0.25.0 → dcnum-0.25.1}/src/dcnum.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.25.0
+Version: 0.25.1
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.25.0 → dcnum-0.25.1}/src/dcnum.egg-info/SOURCES.txt

@@ -6,6 +6,11 @@ README.rst
 pyproject.toml
 .github/workflows/check.yml
 .github/workflows/deploy_pypi.yml
+benchmark/.gitignore
+benchmark/Readme.md
+benchmark/benchmark.py
+benchmark/bm_write_deque_writer_thread.py
+benchmark/bm_write_queue_collector_thread.py
 docs/conf.py
 docs/index.rst
 docs/requirements.txt

{dcnum-0.25.0 → dcnum-0.25.1}/tests/test_write_queue_collector_thread.py

@@ -4,9 +4,8 @@ import pathlib
 
 import numpy as np
 
-from dcnum import read, write
+from dcnum import write
 
-from helper_methods import retrieve_data
 
 data_path = pathlib.Path(__file__).parent / "data"
 
@@ -44,15 +43,12 @@ def test_event_stash():
 
 def test_queue_collector_thread():
     # keyword arguments
-    data = read.HDF5Data(
-        retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip"))
     event_queue = mp.Queue()
     writer_dq = collections.deque()
     feat_nevents = np.array([1, 3, 1, 5])
     write_threshold = 2
     # queue collector thread
     qct = write.QueueCollectorThread(
-        data=data,
         event_queue=event_queue,
         writer_dq=writer_dq,
         feat_nevents=feat_nevents,
@@ -99,15 +95,12 @@ def test_queue_collector_thread():
 
 def test_queue_collector_thread_with_full_stash():
     # keyword arguments
-    data = read.HDF5Data(
-        retrieve_data("fmt-hdf5_cytoshot_full-features_2023.zip"))
     event_queue = mp.Queue()
     writer_dq = collections.deque()
     feat_nevents = np.array([1, 3, 1, 5])
     write_threshold = 2
     # queue collector thread
     qct = write.QueueCollectorThread(
-        data=data,
         event_queue=event_queue,
         writer_dq=writer_dq,
         feat_nevents=feat_nevents,