dclab 0.62.17__cp313-cp313-macosx_10_13_x86_64.whl → 0.63.1__cp313-cp313-macosx_10_13_x86_64.whl

dclab/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.62.17'
-__version_tuple__ = version_tuple = (0, 62, 17)
+__version__ = version = '0.63.1'
+__version_tuple__ = version_tuple = (0, 63, 1)

dclab/rtdc_dataset/export.py

@@ -51,6 +51,7 @@ class Export(object):
             pixel_format: str = "yuv420p",
             codec: str = "rawvideo",
             codec_options: dict[str, str] = None,
+            progress_callback: callable = None,
             ):
         """Exports filtered event images to a video file
 
@@ -72,6 +73,10 @@ class Export(object):
         codec_options:
             Additional arguments to give to the codec using ffmpeg,
             e.g. `{'preset': 'slow', 'crf': '0'}` for "libx264" codec.
+        progress_callback: callable
+            Function that takes at least two arguments: float between 0 and
+            1 for monitoring progress and a string describing what is being
+            done.
 
         Notes
         -----
@@ -103,6 +108,10 @@ class Export(object):
 
                 # write the filtered frames to the video file
                 for evid in np.arange(len(ds)):
+
+                    if progress_callback is not None and evid % 10_000 == 0:
+                        progress_callback(evid / len(ds), "exporting video")
+
                     # skip frames that were filtered out
                     if filtered and not ds.filter.all[evid]:
                         continue
@@ -116,12 +125,22 @@ class Export(object):
 
                     for packet in stream.encode(av_frame):
                         container.mux(packet)
+
+            if progress_callback is not None:
+                progress_callback(1.0, "video export complete")
+
         else:
             msg = "No image data to export: dataset {} !".format(ds.title)
             raise OSError(msg)
 
-    def fcs(self, path, features, meta_data=None, filtered=True,
-            override=False):
+    def fcs(self,
+            path: pathlib.Path | str,
+            features: list[str],
+            meta_data: dict = None,
+            filtered: bool = True,
+            override: bool = False,
+            progress_callback: callable = None,
+            ):
         """Export the data of an RT-DC dataset to an .fcs file
 
         Parameters
@@ -142,6 +161,10 @@ class Export(object):
         override: bool
             If set to `True`, an existing file ``path`` will be overridden.
             If set to `False`, raises `OSError` if ``path`` exists.
+        progress_callback: callable
+            Function that takes at least two arguments: float between 0 and
+            1 for monitoring progress and a string describing what is being
+            done.
 
         Notes
         -----
@@ -175,12 +198,18 @@ class Export(object):
         # Collect the header
         chn_names = [dfn.get_feature_label(c, rtdc_ds=ds) for c in features]
 
+        if progress_callback is not None:
+            progress_callback(0.0, "collecting data")
+
         # Collect the data
         if filtered:
             data = [ds[c][ds.filter.all] for c in features]
         else:
             data = [ds[c] for c in features]
 
+        if progress_callback is not None:
+            progress_callback(0.5, "exporting data")
+
         data = np.array(data).transpose()
         meta_data["dclab version"] = version
         fcswrite.write_fcs(filename=str(path),
@@ -189,6 +218,9 @@ class Export(object):
                            text_kw_pr=meta_data,
                            )
 
+        if progress_callback is not None:
+            progress_callback(1.0, "export complete")
+
     def hdf5(self,
              path: str | pathlib.Path,
              features: List[str] = None,
@@ -200,7 +232,9 @@ class Export(object):
              override: bool = False,
              compression_kwargs: Dict = None,
              compression: str = "deprecated",
-             skip_checks: bool = False):
+             skip_checks: bool = False,
+             progress_callback: callable = None,
+             ):
         """Export the data of the current instance to an HDF5 file
 
         Parameters
@@ -244,7 +278,10 @@ class Export(object):
                 Use `compression_kwargs` instead.
         skip_checks: bool
             Disable checking whether all features have the same length.
-
+        progress_callback: callable
+            Function that takes at least two arguments: float between 0 and
+            1 for monitoring progress and a string describing what is being
+            done.
 
         .. versionchanged:: 0.58.0
 
@@ -335,6 +372,8 @@ class Export(object):
         with RTDCWriter(path,
                         mode="append",
                         compression_kwargs=compression_kwargs) as hw:
+            if progress_callback is not None:
+                progress_callback(0.0, "writing metadata")
             # write meta data
             hw.store_metadata(meta)
 
@@ -369,7 +408,10 @@ class Export(object):
                                    ds.tables[tab])
 
             # write each feature individually
-            for feat in features:
+            for ii, feat in enumerate(features):
+                if progress_callback is not None:
+                    progress_callback(ii / len(features), f"exporting {feat}")
+
                 if (filter_arr is None or
                         # This does not work for the .tdms file format
                         # (and probably also not for DCOR).
@@ -393,6 +435,10 @@ class Export(object):
                                            filtarr=filter_arr)
 
             if basins:
+                if progress_callback:
+                    progress_callback(1 - 1 / (len(features) or 1),
+                                      "writing basins")
+
                 # We have to store basins. There are three options:
                 # - filtering disabled: just copy basins
                 # - filtering enabled
@@ -472,9 +518,17 @@ class Export(object):
 
                     # Do not verify basins, it takes too long.
                     hw.store_basin(**bn_dict, verify=False)
+        if progress_callback is not None:
+            progress_callback(1.0, "export complete")
 
-    def tsv(self, path, features, meta_data=None, filtered=True,
-            override=False):
+    def tsv(self,
+            path: pathlib.Path | str,
+            features: list[str],
+            meta_data: dict = None,
+            filtered: bool = True,
+            override: bool = False,
+            progress_callback: callable = None,
+            ):
         """Export the data of the current instance to a .tsv file
 
         Parameters
@@ -496,6 +550,10 @@ class Export(object):
         override: bool
             If set to `True`, an existing file ``path`` will be overridden.
             If set to `False`, raises `OSError` if ``path`` exists.
+        progress_callback: callable
+            Function that takes at least two arguments: float between 0 and
+            1 for monitoring progress and a string describing what is being
+            done.
         """
         if meta_data is None:
             meta_data = {}
@@ -516,6 +574,10 @@ class Export(object):
             if c not in ds.features_scalar:
                 raise ValueError("Invalid feature name {}".format(c))
         meta_data["dclab version"] = version
+
+        if progress_callback is not None:
+            progress_callback(0.0, "writing metadata")
+
         # Write BOM header
         with path.open("wb") as fd:
             fd.write(codecs.BOM_UTF8)
@@ -539,17 +601,26 @@ class Export(object):
             fd.write("# "+header2+"\n")
 
         with path.open("ab") as fd:
-            # write data
+            if progress_callback is not None:
+                progress_callback(0.1, "collecting data")
+
+            # collect data
             if filtered:
                 data = [ds[c][ds.filter.all] for c in features]
             else:
                 data = [ds[c] for c in features]
 
+            if progress_callback is not None:
+                progress_callback(0.5, "writing data")
+
             np.savetxt(fd,
                        np.array(data).transpose(),
                        fmt=str("%.10e"),
                        delimiter="\t")
 
+        if progress_callback is not None:
+            progress_callback(1.0, "export complete")
+
 
 def yield_filtered_array_stacks(data, indices):
     """Generator returning chunks with the filtered feature data

dclab/rtdc_dataset/fmt_dcor/api.py

@@ -12,7 +12,7 @@ class DCORAccessError(BaseException):
 class APIHandler:
     """Handles the DCOR api with caching for simple queries"""
     #: these are cached to minimize network usage
-    cache_queries = ["metadata", "size", "feature_list", "valid"]
+    cache_queries = ["basins", "metadata", "size", "feature_list", "valid"]
     #: DCOR API Keys/Tokens in the current session
     api_keys = []
 

dclab/rtdc_dataset/fmt_s3.py

@@ -42,6 +42,27 @@ S3_ACCESS_KEY_ID = os.environ.get("DCLAB_S3_ACCESS_KEY_ID")
 S3_SECRET_ACCESS_KEY = os.environ.get("DCLAB_S3_SECRET_ACCESS_KEY")
 
 
+@functools.lru_cache(maxsize=1000)
+def get_s3_session_client(access_key_id: str,
+                          secret_access_key: str,
+                          use_ssl: bool,
+                          verify_ssl: bool,
+                          endpoint_url: str
+                          ):
+    botocore_session = botocore.session.get_session()
+    s3_session = boto3.Session(
+        aws_access_key_id=access_key_id,
+        aws_secret_access_key=secret_access_key,
+        botocore_session=botocore_session)
+    s3_client = s3_session.client(
+        service_name='s3',
+        use_ssl=use_ssl,
+        verify=verify_ssl,
+        endpoint_url=endpoint_url,
+    )
+    return botocore_session, s3_session, s3_client
+
+
 class S3File(HTTPFile):
     """Monkeypatched `HTTPFile` to support authenticated access to S3"""
     def __init__(self,
@@ -74,17 +95,15 @@ class S3File(HTTPFile):
                 "not specify the full S3 URL or that you forgot to set "
                 "the `S3_ENDPOINT_URL` environment variable.")
         endpoint_url = endpoint_url.strip().rstrip("/")
-        self.botocore_session = botocore.session.get_session()
-        self.s3_session = boto3.Session(
-            aws_access_key_id=access_key_id,
-            aws_secret_access_key=secret_access_key,
-            botocore_session=self.botocore_session)
-        self.s3_client = self.s3_session.client(
-            service_name='s3',
-            use_ssl=use_ssl,
-            verify=verify_ssl,
-            endpoint_url=endpoint_url,
+        self.botocore_session, self.s3_session, self.s3_client = \
+            get_s3_session_client(
+                access_key_id=access_key_id,
+                secret_access_key=secret_access_key,
+                use_ssl=use_ssl,
+                verify_ssl=verify_ssl,
+                endpoint_url=endpoint_url,
             )
+
         # Use a configuration that allows anonymous access
         # https://stackoverflow.com/a/34866092
         if not secret_access_key:

{dclab-0.62.17.dist-info → dclab-0.63.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dclab
-Version: 0.62.17
+Version: 0.63.1
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>

{dclab-0.62.17.dist-info → dclab-0.63.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 dclab/kde_methods.py,sha256=f0-zDN7ETintvGB3gSzxwgBb53YtT9jZtzI70EAX50g,365
-dclab/_version.py,sha256=GEa7QH3NrBgvyLi3LNdBdNGUvveY9cblWLa2mkR4bCI,515
+dclab/_version.py,sha256=qDZMrEPFSKYNbMiOoBOEXqq2P0z4X69tZSzimIIP7H0,513
 dclab/util.py,sha256=HFT5ZQV6AW8GIIruVMldukbVVdlMyKH50GUfOogAcxI,5860
 dclab/downsampling.pyx,sha256=OK7zbgGLl5gVyoU8ZBHo9EWwb8C9ChavmLNEvQvC9T0,7258
 dclab/__init__.py,sha256=wyJWhElQRPcq09vUqUnuquTU_KHgHxv6wQxuxQ988Iw,1583
@@ -7,7 +7,7 @@ dclab/warn.py,sha256=MjJvyQeuvIXFQ2-fHDzbmXJ0scnHqqRJlIxfuLI_utE,523
 dclab/cached.py,sha256=eWTYBiI-HQM7JuPH-oxa5LLnhAX32GpRwlYg2kQ3sTA,2917
 dclab/http_utils.py,sha256=YtZHEwB-BBBo2fCvwhlJvlnWvfWFMHclqol3OIJ7atM,10910
 dclab/polygon_filter.py,sha256=qexmo-rXe06CUPZhN6EMJy4y4B5gXZeqejdvIB2arOE,13480
-dclab/downsampling.cpython-313-darwin.so,sha256=B4q14jsN0fa_JvcAndaBk_G3WgjwC0FP01VH0KjNrPQ,258720
+dclab/downsampling.cpython-313-darwin.so,sha256=dsZFUONlPg1TA2v6HEKX6y3bccBCL9NUUFL6mA7rDZQ,258720
 dclab/statistics.py,sha256=tJDqPlY_Jw2Hhl-s7ugMBSZAxcRuPu4LQuBAZBXz7t8,6355
 dclab/kde_contours.py,sha256=UlU64lrzMQUZH11oZndW7xf7NFCzwP3FcVujwuqXDCI,278
 dclab/isoelastics/iso_LE-2D-FEM-19-volume-deform.txt,sha256=vTcazOlOXo3BQ0NQtGB_IdHKA0neOLXZ_d3JuMU--RE,83358
@@ -66,16 +66,16 @@ dclab/external/statsmodels/nonparametric/__init__.py,sha256=-GEWgwsF27ems5WTBvR1
 dclab/external/statsmodels/nonparametric/kernels.py,sha256=fuy4kStFz2ZA9pqgfUb4cly-YBpXLu4TJ9-ZujayuIw,1075
 dclab/external/skimage/measure.py,sha256=y1idCqD9TUxp3-QnOiWR_d674OKaeqBJ4MN2-gVP6ro,247
 dclab/external/skimage/LICENSE,sha256=ivsSBvn3c0R9mOctWRRdza7C7wdZSRYgCVxlVqUdlB8,1452
-dclab/external/skimage/_find_contours_cy.cpython-313-darwin.so,sha256=9dNAVtMe-yEzJqHNAcUa3YB-XNlvN1RV3T5q43PiIck,204096
+dclab/external/skimage/_find_contours_cy.cpython-313-darwin.so,sha256=aIxiN0VgjnFiuK48hs0ntPfdzUR2MS1X8cRN7kOH2yg,204096
 dclab/external/skimage/pnpoly.py,sha256=r8hFNiTz5XlUoNZjosqA0iyv1FPn0l7ewbplgFgkdaw,1347
 dclab/external/skimage/_find_contours.py,sha256=16v5eeTZBmevG8SSuXtJ6yUpVPhwfSmtc8pDD0nuuOU,9340
-dclab/external/skimage/_pnpoly.cpython-313-darwin.so,sha256=4JtzvHdTSCniVqN0ra0ooKH2RxlHQNjn48JEUyWVJyc,205960
+dclab/external/skimage/_pnpoly.cpython-313-darwin.so,sha256=u-dwNKwrbRzA3k6AialvCvVSeFmNH_d0aB_y0O7-8Pg,205960
 dclab/external/skimage/__init__.py,sha256=-B2QUKHAFzQuBWuuKvPDC5JIl0Zb-x3OGmbwPaE9VwQ,72
 dclab/external/skimage/_pnpoly.pyx,sha256=Qdn6xPazDschBqbr46DzB75MB2MnqvdnoTSBMK7kUGE,2504
 dclab/external/skimage/_find_contours_cy.pyx,sha256=pZJOBhMHzYEMkcz4WQVyjn7jDNrdjCfet47FU1hRAxk,7161
 dclab/external/skimage/_shared/geometry.pxd,sha256=kRsu9ifv_rL3kbRIgSLf86p0hn2oTMp6s013lZ9bBZM,346
 dclab/external/skimage/_shared/__init__.py,sha256=2sHZwTtJSlMTa3Q2YSvQW7jrPLMUSqDJQa-ROe5zfcw,37
-dclab/external/skimage/_shared/geometry.cpython-313-darwin.so,sha256=k4REBBCIbh2pjE7OAgQxMQOaQvOua2KHa0g4yTb7v8Q,38656
+dclab/external/skimage/_shared/geometry.cpython-313-darwin.so,sha256=xK656WCujc4Rzy8YaKF25C1RN1PoNvmYyzzLnJenYM0,38656
 dclab/external/skimage/_shared/geometry.pyx,sha256=miCHUh6mBDbRRIoaF_0xAER1MRzsCAzFdlYQZhV7RmE,1667
 dclab/definitions/feat_logic.py,sha256=SXsSlAusgtE3uXcPu84dQwYZ07zxmV37DmPednA3_dM,5823
 dclab/definitions/meta_parse.py,sha256=YdaTdM8DAMsIFn5ITH9OHYGTXeAOBGWtx22oVjxXcWk,2393
@@ -87,11 +87,11 @@ dclab/rtdc_dataset/config.py,sha256=MvBteFya3R6Ch3U6UgTakCsJoBgVykTxS_Z25STWPHU,
 dclab/rtdc_dataset/check.py,sha256=lJNaz4QTe2WNlxik6zSohRHTiAYuP_bKOzSDjPGTUS0,35006
 dclab/rtdc_dataset/meta_table.py,sha256=ucqBNrgI6rDAuQFuMRckY8lp1LpnYAoRgEsLObWTJCE,648
 dclab/rtdc_dataset/feat_basin.py,sha256=ViKdvJcwFM8joysnrBYdZbA5t_wZix-6xn_FsvzpYsQ,21072
-dclab/rtdc_dataset/fmt_s3.py,sha256=FVw0q3CwiPwDKmz37EsjK2T5CLr4MsH3pvscu-gToVM,11278
+dclab/rtdc_dataset/fmt_s3.py,sha256=bU3V_WGyqJhxPCH80X_nlNqq-jXcgoZKv_aUBIqwaL8,11877
 dclab/rtdc_dataset/feat_temp.py,sha256=XbDIS1iUUkRH0Zp9uVlwvK_untJ7hkOnKshK1Drsnt8,3694
 dclab/rtdc_dataset/__init__.py,sha256=MUHSGVQJ4Zc0IyU2lf01dpDWyOyNveHip-UjSkmPNvQ,486
 dclab/rtdc_dataset/core.py,sha256=EjNWk9SV-2xBTRtf34XosLCOS164vGWKP5dKKLSOSq4,34441
-dclab/rtdc_dataset/export.py,sha256=RPnWNDAPW1m3vPcStjqIX-YnxDOKyWLtQ1HrA38S3Uo,30384
+dclab/rtdc_dataset/export.py,sha256=NarfazGkPasyRpnhQ2H_riUP-oUq5QL7lG9WwvLCL2o,33121
 dclab/rtdc_dataset/fmt_dict.py,sha256=gumVQOiVVDFUKow_483PY7cxInqo-NiBBnBhIU8s4lg,3009
 dclab/rtdc_dataset/writer.py,sha256=jc6ADyxGoujXpoXu1vF2nfZjGFMaO5LbRmoYJZ83JVo,41418
 dclab/rtdc_dataset/filter.py,sha256=AFPUBzOIi3pqXgUdMQ5CIi9ZeGOKC71rfSZKLMLgtog,10023
@@ -123,7 +123,7 @@ dclab/rtdc_dataset/feat_anc_ml/__init__.py,sha256=99jZlz17aBCAxI5xx42XINomMy20Hi
 dclab/rtdc_dataset/fmt_dcor/access_token.py,sha256=jotLQay138RUlv8wbdF2ishRnyE9N0KwGGBlbCL0wRI,2028
 dclab/rtdc_dataset/fmt_dcor/basin.py,sha256=tQZ4GumqURjS3eppRrSyUq1zBPD0y_8rwznMRDXiDUs,2526
 dclab/rtdc_dataset/fmt_dcor/__init__.py,sha256=WjO1uM_Vlof15Y7HkhkV5Xv75q9TDIdOBIuS_I38qps,210
-dclab/rtdc_dataset/fmt_dcor/api.py,sha256=COPRnPfPBcxbQGxHFEbGxp2CjK-Mgnt3cIu20-Zz04M,4245
+dclab/rtdc_dataset/fmt_dcor/api.py,sha256=GfQdKhkGXN-e9-e5YxgzL81tDeWB2M-UoyouVfXXbVg,4255
 dclab/rtdc_dataset/fmt_dcor/logs.py,sha256=1JsMr_4r5j8rkfrrUsiN42_l92GcvDjapYxopZKimnw,583
 dclab/rtdc_dataset/fmt_dcor/tables.py,sha256=NaVEwLKmOg7Mz5iAMe2S8C4xRVC_YO3zeT7g5EbQE1M,1682
 dclab/rtdc_dataset/fmt_dcor/base.py,sha256=wD127W5LvvhkUy8SvFVVwAR6EEYtzgoWJ4booh45rfA,6588
@@ -134,9 +134,9 @@ dclab/rtdc_dataset/fmt_tdms/event_image.py,sha256=-jp7Z-N91e4ieumYQ1huMicj7PMJqw
 dclab/rtdc_dataset/fmt_tdms/event_trace.py,sha256=Vkym0QKSw2mq1XZl5n8wDkgHXmaZwQGiMAV5AuRSJkE,5215
 dclab/rtdc_dataset/fmt_tdms/exc.py,sha256=WzrMqnyrzp8gsT8Pf7JKqGGv43ewx7d_qgtirURppRI,813
 dclab/rtdc_dataset/fmt_tdms/event_contour.py,sha256=kjo0wJx9F0gmmOOyR0NoLw6VEtSl3h63WXXkcbfnoS8,9627
-dclab-0.62.17.dist-info/RECORD,,
-dclab-0.62.17.dist-info/WHEEL,sha256=D5oYZcu7xsHPQ1lvRcEfimRZkYr_RmvD7J5S2QmS8s0,138
-dclab-0.62.17.dist-info/entry_points.txt,sha256=eOpjgznu-eW-9utUpLU-77O5098YyUEgGF3ksGMdtec,273
-dclab-0.62.17.dist-info/top_level.txt,sha256=irvwZMgs1edY1Zj60ZFk7Almb9Zhk4k6E6aC4YPFnnM,6
-dclab-0.62.17.dist-info/METADATA,sha256=sxoMpN4ygAI9x12l_IsEKUqi32wGJ3o2Tg2OKEx8cKQ,4756
-dclab-0.62.17.dist-info/licenses/LICENSE,sha256=gLDaVZWRrlnLdyfOrR0qfWjLbOVcjvoJ-kCLUK0fyXA,15360
+dclab-0.63.1.dist-info/RECORD,,
+dclab-0.63.1.dist-info/WHEEL,sha256=0zT6bEe6ExT6o7m_6mGvkzRfdTnnYpeEzXPUtdm-wYw,138
+dclab-0.63.1.dist-info/entry_points.txt,sha256=eOpjgznu-eW-9utUpLU-77O5098YyUEgGF3ksGMdtec,273
+dclab-0.63.1.dist-info/top_level.txt,sha256=irvwZMgs1edY1Zj60ZFk7Almb9Zhk4k6E6aC4YPFnnM,6
+dclab-0.63.1.dist-info/METADATA,sha256=kaDiBVjVDUqc77B4nSoNNQhk-IE6Rfb7WI9yAV1UABM,4755
+dclab-0.63.1.dist-info/licenses/LICENSE,sha256=gLDaVZWRrlnLdyfOrR0qfWjLbOVcjvoJ-kCLUK0fyXA,15360

{dclab-0.62.17.dist-info → dclab-0.63.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: false
 Tag: cp313-cp313-macosx_10_13_x86_64
 Generator: delocate 0.13.0