dcnum 0.25.2__py3-none-any.whl → 0.25.4__py3-none-any.whl

dcnum/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.25.2'
-__version_tuple__ = version_tuple = (0, 25, 2)
+__version__ = version = '0.25.4'
+__version_tuple__ = version_tuple = (0, 25, 4)

dcnum/feat/feat_background/bg_roll_median.py

@@ -4,6 +4,8 @@ import time
 import numpy as np
 from scipy import ndimage
 
+from ...os_env_st import RequestSingleThreaded, confirm_single_threaded
+
 from .base import mp_spawn, Background
 
 
@@ -241,6 +243,8 @@ class WorkerRollMed(mp_spawn.Process):
 
     def run(self):
         """Main loop of worker process (breaks when `self.counter` <0)"""
+        # confirm single-threadedness (prints to log)
+        confirm_single_threaded()
         # Create the ctypes arrays here instead of during __init__, because
         # for some reason they are copied in __init__ and not mapped.
         shared_input = np.ctypeslib.as_array(
@@ -261,6 +265,12 @@ class WorkerRollMed(mp_spawn.Process):
                 with self.counter.get_lock():
                     self.counter.value += 1
 
+    def start(self):
+        # Set all relevant os environment variables such libraries in the
+        # new process only use single-threaded computation.
+        with RequestSingleThreaded():
+            mp_spawn.Process.start(self)
+
 
 def compute_median_for_slice(shared_input, shared_output, kernel_size,
                              output_size, job_slice):

dcnum/feat/feat_background/bg_sparse_median.py

@@ -6,6 +6,8 @@ from scipy import ndimage
 
 from ...read import HDF5Data
 
+from ...os_env_st import RequestSingleThreaded, confirm_single_threaded
+
 from .base import mp_spawn, Background
 
 
@@ -436,6 +438,8 @@ class WorkerSparseMed(mp_spawn.Process):
 
     def run(self):
         """Main loop of worker process (breaks when `self.counter` <0)"""
+        # confirm single-threadedness (prints to log)
+        confirm_single_threaded()
         # Create the ctypes arrays here instead of during __init__, because
         # for some reason they are copied in __init__ and not mapped.
         shared_input = np.ctypeslib.as_array(
@@ -468,3 +472,9 @@ class WorkerSparseMed(mp_spawn.Process):
                 #     overwrite_input=False)
                 with self.counter.get_lock():
                     self.counter.value += 1
+
+    def start(self):
+        # Set all relevant os environment variables such libraries in the
+        # new process only use single-threaded computation.
+        with RequestSingleThreaded():
+            mp_spawn.Process.start(self)

dcnum/feat/queue_event_extractor.py

@@ -10,6 +10,7 @@ import traceback
 
 import numpy as np
 
+from ..os_env_st import RequestSingleThreaded, confirm_single_threaded
 from ..meta.ppid import kwargs_to_ppid, ppid_to_kwargs
 from ..read import HDF5Data
 
@@ -325,6 +326,7 @@ class QueueEventExtractor:
 
     def run(self):
         """Main loop of worker process"""
+        confirm_single_threaded()
         self.worker_monitor[self.worker_index] = 0
         # Don't wait for these two queues when joining workers
         self.raw_queue.cancel_join_thread()
@@ -399,6 +401,12 @@ class EventExtractorProcess(QueueEventExtractor, mp_spawn.Process):
         super(EventExtractorProcess, self).__init__(
             name="EventExtractorProcess", *args, **kwargs)
 
+    def start(self):
+        # Set all relevant os environment variables such libraries in the
+        # new process only use single-threaded computation.
+        with RequestSingleThreaded():
+            mp_spawn.Process.start(self)
+
 
 class EventExtractorThread(QueueEventExtractor, threading.Thread):
     """Threading worker for debugging (only one single thread)"""

dcnum/logic/ctrl.py

@@ -128,33 +128,6 @@ class DCNumJobRunner(threading.Thread):
 
         self.logger = logging.getLogger(f"dcnum.Runner-{self.pphash[:2]}")
 
-        # Sanity checks
-        for os_env in ["MKL_NUM_THREADS", "NUMBA_NUM_THREADS",
-                       "NUMEXPR_NUM_THREADS", "NUMPY_NUM_THREADS",
-                       "OPENBLAS_NUM_THREADS", "OMP_NUM_THREADS",
-                       "VECLIB_MAXIMUM_THREADS"]:
-            # You should disable multithreading for all major tools that
-            # use dcnum.logic. We don't want multithreading, because dcnum
-            # uses linear code and relies on multiprocessing for
-            # parallelization. This has to be done before importing numpy
-            # or any other library affected. In your scripts, you can use:
-            #
-            #    os.environ.setdefault("MKL_NUM_THREADS", "1")
-            #    os.environ.setdefault("NUMBA_NUM_THREADS", "1")
-            #    os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
-            #    os.environ.setdefault("NUMPY_NUM_THREADS", "1")
-            #    os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
-            #    os.environ.setdefault("OMP_NUM_THREADS", "1")
-            #    os.environ.setdefault("VECLIB_MAXIMUM_THREADS", "1")
-            #
-            val_act = os.environ.get(os_env)
-            if val_act != "1":
-                self.logger.warning(
-                    f"Make sure to set the environment variable {os_env} to "
-                    f"'1' (disables multithreading)! Other values will reduce "
-                    f"performance and your system may become inresponsive. "
-                    f"The current value is '{val_act}'.")
-
     def __enter__(self):
         return self
 

dcnum/os_env_st.py

@@ -0,0 +1,85 @@
+import logging
+import os
+
+
+logger = logging.getLogger(__name__)
+
+#: environment variables that set number of threads
+os_env_threading = [
+    "MKL_NUM_THREADS",
+    "NUMBA_NUM_THREADS",
+    "NUMEXPR_NUM_THREADS",
+    "NUMPY_NUM_THREADS",
+    "OMP_NUM_THREADS",
+    "OPENBLAS_NUM_THREADS",
+    "VECLIB_MAXIMUM_THREADS",
+]
+
+
+class RequestSingleThreaded:
+    """Context manager for starting a process with specific environment
+
+    When entering the context, the environment variables defined in
+    `os_env_threading` are all set to "1", telling the relevant libraries
+    that they should work in single-threaded mode.
+    When exiting the context, these environment variables are reset to
+    their original values (or unset if applicable).
+
+    Note that it makes only sense to use this context manager when
+    starting new multiprocessing processes. When the process spawns,
+    the environment from the current thread is copied. Setting the
+    environment variable after e.g. importing numpy has no effect
+    on how many threads numpy will use.
+    """
+    def __init__(self):
+        self.previous_env = {}
+
+    def __enter__(self):
+        """Ask nicely for single-threaded computation using `os.environ`
+
+        Note that this only affects new processes in which the
+        relevant libraries have not yet been imported.
+        """
+        for key in os_env_threading:
+            if key in os.environ:
+                self.previous_env[key] = os.environ[key]
+            os.environ[key] = "1"
+        return self
+
+    def __exit__(self, type, value, traceback):
+        """Restore the previous environment"""
+        for key in os_env_threading:
+            if key not in self.previous_env:
+                os.environ.pop(key)
+            else:
+                os.environ[key] = self.previous_env[key]
+
+
+def confirm_single_threaded():
+    """Warn via logs when environment variables are not set to single thread"""
+    # Sanity checks
+    for os_env in os_env_threading:
+        # You should disable multithreading for all major tools that
+        # use dcnum.logic. We don't want multithreading, because dcnum
+        # uses linear code and relies on multiprocessing for
+        # parallelization. This has to be done before importing numpy
+        # or any other library affected. In your scripts, you can use:
+        #
+        val_act = os.environ.get(os_env)
+        if val_act != "1":
+            logger.warning(
+                f"Make sure to set the environment variable {os_env} to "
+                f"'1' (disables multithreading)! Other values will reduce "
+                f"performance and your system may become unresponsive. "
+                f"The current value is '{val_act}'.")
+
+
+def request_single_threaded():
+    """Set the environment variable to single thread
+
+    This function must be called before importing the multithreaded
+    libraries (such as numpy) in order for them to pick up the
+    environment variables.
+    """
+    for key in os_env_threading:
+        os.environ[key] = "1"

dcnum/read/hdf5_data.py

@@ -593,8 +593,11 @@ def concatenated_hdf5_data(paths: List[pathlib.Path],
         raise ValueError(
             f"Invalid type for `path_out`: {type(path_out)} ({path_out}")
 
-    if len(paths) <= 1:
-        raise ValueError("Please specify at least two files in `paths`!")
+    if len(paths) == 0:
+        raise ValueError("Please specify at least one file in `paths`!")
+    elif len(paths) == 1:
+        warnings.warn("Only one file passed to `concatenated_hdf5_data`; this "
+                      "is equivalent to using `HDF5Data`, but slower.")
 
     frames = []
 

dcnum/segm/segmenter_mpo.py

@@ -7,6 +7,8 @@ from typing import Dict
 import numpy as np
 import scipy.ndimage as ndi
 
+from ..os_env_st import RequestSingleThreaded, confirm_single_threaded
+
 from .segmenter import Segmenter
 
 
@@ -325,7 +327,8 @@ class MPOSegmenterWorker:
         self.sl_stop = sl_stop
 
     def run(self):
-        # print(f"Running {self} in PID {os.getpid()}")
+        # confirm single-threadedness (prints to log)
+        confirm_single_threaded()
         # We have to create the numpy-versions of the mp.RawArrays here,
         # otherwise we only get some kind of copy in the new process
         # when we use "spawn" instead of "fork".
@@ -369,6 +372,12 @@ class MPOSegmenterWorkerProcess(MPOSegmenterWorker, mp_spawn.Process):
     def __init__(self, *args, **kwargs):
         super(MPOSegmenterWorkerProcess, self).__init__(*args, **kwargs)
 
+    def start(self):
+        # Set all relevant os environment variables such libraries in the
+        # new process only use single-threaded computation.
+        with RequestSingleThreaded():
+            mp_spawn.Process.start(self)
+
 
 class MPOSegmenterWorkerThread(MPOSegmenterWorker, threading.Thread):
     def __init__(self, *args, **kwargs):

{dcnum-0.25.2.dist-info → dcnum-0.25.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcnum
-Version: 0.25.2
+Version: 0.25.4
 Summary: numerics toolbox for imaging deformability cytometry
 Author: Maximilian Schlögel, Paul Müller, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dcnum-0.25.2.dist-info → dcnum-0.25.4.dist-info}/RECORD

@@ -1,14 +1,15 @@
 dcnum/__init__.py,sha256=hcawIKS7utYiOyVhOAX9t7K3xYzP1b9862VV0b6qSrQ,74
-dcnum/_version.py,sha256=O16YsGa9nckhVhnz7jJ3nC69Kk6pQkIag-qsDHQCsBY,413
+dcnum/_version.py,sha256=2KpEHoGK3-a2p8saB4NXVEuWwhXBRfqcz_ebPkLNw7A,413
+dcnum/os_env_st.py,sha256=ujEVzn1G5sxZfJSITOMw48e2O_oMFu2VD6oj5QCUFSU,3025
 dcnum/feat/__init__.py,sha256=jUJYWTD3VIoDNKrmryXbjHb1rGwYtK4b7VPWihYgUoo,325
 dcnum/feat/event_extractor_manager_thread.py,sha256=FAxSyRfaNAuBWNplxHngp5h-44s0qIP24XX_oETdfMk,7836
 dcnum/feat/gate.py,sha256=Yhxq80JoRMmQzBxl35C8NT91c9QcmQa-EIKLuxK6WvE,7221
-dcnum/feat/queue_event_extractor.py,sha256=0ncTQleT1sfc98zYkFuZWxU-akecfTrW6-OOU3z-d8o,15698
+dcnum/feat/queue_event_extractor.py,sha256=KA7K7fOUbLYRfHjdfmN6mVuPPjCJbWMCHv0-b41lBMs,16038
 dcnum/feat/feat_background/__init__.py,sha256=OTmMuazHNaSrZb2XW4cnJ6PlgJLbKrPbaidpEixYa0A,341
 dcnum/feat/feat_background/base.py,sha256=bQBPvztrku-8YSVk8YBUUNh7MaYcnztgyD2-dQHxpzw,8674
 dcnum/feat/feat_background/bg_copy.py,sha256=PK8x4_Uph-_A6uszZC5uhe1gD1dSRdHnDMEsN0HSGHA,1034
-dcnum/feat/feat_background/bg_roll_median.py,sha256=EyjstMDXFBYuJB1lN6g4Uw7tPm434X3hXQxKSqvcoJ4,13175
-dcnum/feat/feat_background/bg_sparse_median.py,sha256=wt7lvPSiZkdaerRErgezd_YDVxHA2kAqO1LMX9PuHJk,22053
+dcnum/feat/feat_background/bg_roll_median.py,sha256=NqdgVYm-iUhgDZEonZxQrDvh5e26NoryQKCge8pNGoM,13571
+dcnum/feat/feat_background/bg_sparse_median.py,sha256=QrerkPENHkC9PBgivamu-N1Od6-0b61Ohy_oZYL4www,22449
 dcnum/feat/feat_brightness/__init__.py,sha256=o6AebVlmydwNgVF5kW6ITqJyFreoKrU3Ki_3EC8If-s,155
 dcnum/feat/feat_brightness/bright_all.py,sha256=vf8xaYBdKD24hHUXdkI0_S7nbr7m49KW6gvuWvbHDVg,4545
 dcnum/feat/feat_brightness/common.py,sha256=JX49EszYDmnvoOKXFVV1CalEIWRmOuY5EryNbqGbdac,156
@@ -20,7 +21,7 @@ dcnum/feat/feat_texture/__init__.py,sha256=6StM9S540UVtdFFR3bHa7nfCTomeVdoo7Uy9C
 dcnum/feat/feat_texture/common.py,sha256=COXHpXS-7DMouGu3WF83I76L02Sr7P9re4lxajh6g0E,439
 dcnum/feat/feat_texture/tex_all.py,sha256=_5H3sXYRN0Uq2eUHn3XUyEHkU_tncEqbqJTC-HZcnGY,5198
 dcnum/logic/__init__.py,sha256=7J3GrwJInNQbrLk61HRIV7X7p69TAIbMYpR34hh6u14,177
-dcnum/logic/ctrl.py,sha256=FwzrCou4YBPBaaRRTqJkPt6jenFev9JfnxxjUMS0B-w,39042
+dcnum/logic/ctrl.py,sha256=RZ2xl0496Iv91cdCdbkIg2W1fClqsTUDq7YlHHwWQfk,37514
 dcnum/logic/job.py,sha256=9BN2WjYqjjJuLnfNZAtQ2Nn47Glo2jVrivDodGJoqlQ,7713
 dcnum/logic/json_encoder.py,sha256=cxMnqisbKEVf-rVcw6rK2BBAb6iz_hKFaGl81kK36lQ,571
 dcnum/meta/__init__.py,sha256=AVqRgyKXO1orKnE305h88IBvoZ1oz6X11HN1WP5nGvg,60
@@ -30,13 +31,13 @@ dcnum/read/__init__.py,sha256=LYHyZHgiNTpjV5oEcty-7Kh5topLpHT_cFlNl-QX8gg,262
 dcnum/read/cache.py,sha256=LNA5nnDyrw8Nj07E7XfG2GcHEoWm6vA6Qo_8N-n-sGw,6492
 dcnum/read/const.py,sha256=GG9iyXDtEldvJYOBnhZjlimzIeBMAt4bSr2-xn2gzzc,464
 dcnum/read/detect_flicker.py,sha256=CeUyxI6LaX_lCNvBPm_yzsiWmiNcZYqbNZCtvKPdkcU,1827
-dcnum/read/hdf5_data.py,sha256=JVk9YWw1rPgTPxaMZsw2ehk4FJq9UqhmB1SW7yhPw50,25867
+dcnum/read/hdf5_data.py,sha256=jo3LbwMqrr-qjoj8Fkv1eOdEE-ZZCZTjcGddzSx--MY,26044
 dcnum/read/mapped.py,sha256=zU2fYdZfLNHn0rKHxDzBhNFMu4--WWa8nSeE2likyZA,3637
 dcnum/segm/__init__.py,sha256=9cLEAd3JWE8IGqDHV-eSDIYOGBfOepd8OcebtNs8Omk,309
 dcnum/segm/segm_thresh.py,sha256=iVhvIhzO0Gw0t3rXOgH71rOI0CNjJJQq4Gg6BulUhK8,948
 dcnum/segm/segmenter.py,sha256=FWLFDBR-x_85ku2rObA2F-QBrM4IUaUL-YHChLagVvM,14902
 dcnum/segm/segmenter_manager_thread.py,sha256=frM0sMxC7f7TQiFjmpRxuwG2kUBFpW1inV8dtpADHiI,5924
-dcnum/segm/segmenter_mpo.py,sha256=o6mQlITHgEWvQt9v6oCWwAcZUvxE7MOeLE9DFManzpY,13757
+dcnum/segm/segmenter_mpo.py,sha256=XcYMKTnCu6-D-TJ62V18S3OE9DhaPhqFkhGhUaDWJFg,14096
 dcnum/segm/segmenter_sto.py,sha256=e6MtN_RWusA0wTExV-FLGpDXNJs1CbSyXcSdWUPBMvM,3959
 dcnum/segm/segm_torch/__init__.py,sha256=DtUqJTbj7ybrTbXlwHq1Y4SCzi22rMW9Cus6wX-iU-A,822
 dcnum/segm/segm_torch/segm_torch_base.py,sha256=G9AhVyD6LkAmk0tkbYnJUSpvcj3_HYf0uqfILZQsyus,4479
@@ -49,8 +50,8 @@ dcnum/write/__init__.py,sha256=sK79IlvCFIqf2oFABVeyYedMnHOsEIQpxAauEeNO-Tw,273
 dcnum/write/deque_writer_thread.py,sha256=ao7F1yrVKyufgC4rC0Y2_Vt7snuT6KpI7W2qVxcjdhk,1994
 dcnum/write/queue_collector_thread.py,sha256=KwwNIDFEF2DU83woKES5K05MxxOhDxPMZLLeyPugfDo,11542
 dcnum/write/writer.py,sha256=oHlq4bDHQxb33-3Fw1xnzJwACecLyH-6koGK8SN0cSk,20528
-dcnum-0.25.2.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
-dcnum-0.25.2.dist-info/METADATA,sha256=8x1L3Y9upiMz-WC0GmOFYSkdcA9OMByyi2z88HRlX94,2326
-dcnum-0.25.2.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-dcnum-0.25.2.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
-dcnum-0.25.2.dist-info/RECORD,,
+dcnum-0.25.4.dist-info/LICENSE,sha256=YRChA1C8A2E-amJbudwMcbTCZy_HzmeY0hMIvduh1MM,1089
+dcnum-0.25.4.dist-info/METADATA,sha256=uqKK3Br_6ILIASamWdczQsgdqKG8rJcOwCAV_1JEN78,2326
+dcnum-0.25.4.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+dcnum-0.25.4.dist-info/top_level.txt,sha256=Hmh38rgG_MFTVDpUDGuO2HWTSq80P585Het4COQzFTg,6
+dcnum-0.25.4.dist-info/RECORD,,

{dcnum-0.25.2.dist-info → dcnum-0.25.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any