dclab 0.63.1__cp310-cp310-macosx_11_0_arm64.whl → 0.64.0__cp310-cp310-macosx_11_0_arm64.whl

dclab/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.63.1'
-__version_tuple__ = version_tuple = (0, 63, 1)
+__version__ = version = '0.64.0'
+__version_tuple__ = version_tuple = (0, 64, 0)

dclab/http_utils.py

@@ -254,7 +254,7 @@ class ResoluteRequestsSession(requests.Session):
                 break
         else:
             raise requests.exceptions.ReadTimeout(
-                f"Resolut sesion failed for {args} and {kwargs}!")
+                f"Resolute session failed for {args} and {kwargs}!")
         return resp
 
 

dclab/kde/base.py

@@ -2,7 +2,12 @@ import warnings
 
 import numpy as np
 
-from .methods import bin_width_doane, get_bad_vals, methods
+from .methods import bin_width_doane_div5, get_bad_vals, methods
+from .contours import find_contours_level, get_quantile_levels
+
+
+class ContourSpacingTooLarge(UserWarning):
+    pass
 
 
 class KernelDensityEstimator:
@@ -107,6 +112,139 @@ class KernelDensityEstimator:
         yscale: str
             See `xscale`.
 
+        Returns
+        -------
+        X, Y, Z : coordinates
+            The kernel density Z evaluated on a rectangular grid (X,Y).
+        """
+        warnings.warn("`get_contour` is deprecated; please use "
+                      "`get_raster` instead", DeprecationWarning)
+        return self.get_raster(
+            xax=xax, yax=yax, xacc=xacc, yacc=yacc,
+            kde_type=kde_type, kde_kwargs=kde_kwargs,
+            xscale=xscale, yscale=yscale
+        )
+
+    def get_contour_lines(self, quantiles=None, xax="area_um", yax="deform",
+                          xacc=None, yacc=None, kde_type="histogram",
+                          kde_kwargs=None, xscale="linear", yscale="linear",
+                          ret_levels=False):
+        """Compute contour lines for a given kernel kensity estimate.
+
+        Parameters
+        ----------
+        quantiles: list or array of floats
+            KDE Quantiles for which contour levels are computed. The
+            values must be between 0 and 1. If set to None, use
+            [0.5, 0.95] as default.
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        yacc: float
+            Contour accuracy in y direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`
+        ret_levels: bool
+            If set to True, return the levels of the contours
+            (default: False)
+
+        Returns
+        -------
+        contour_lines: list of lists (of lists)
+            For every number in `quantiles`, this list contains a list of
+            corresponding contour lines. Each contour line is a 2D
+            array of shape (N, 2), where N is the number of points in the
+            contour line.
+        levels: list of floats
+            The density levels corresponding to each number in `quantiles`.
+            Only returned if `ret_levels` is set to True.
+        """
+        if not quantiles:
+            quantiles = [0.5, 0.95]
+        try:
+            x, y, density = self.get_raster(
+                xax=xax,
+                yax=yax,
+                xacc=xacc,
+                yacc=yacc,
+                xscale=xscale,
+                yscale=yscale,
+                kde_type=kde_type,
+                kde_kwargs=kde_kwargs,
+            )
+        except ValueError:
+            # most-likely there is nothing to compute a contour for
+            return []
+        if density.shape[0] < 3 or density.shape[1] < 3:
+            warnings.warn("Contour not possible; spacing may be too large!",
+                          ContourSpacingTooLarge)
+            return []
+        levels = get_quantile_levels(
+            density=density,
+            x=x,
+            y=y,
+            xp=self.rtdc_ds[xax][self.rtdc_ds.filter.all],
+            yp=self.rtdc_ds[yax][self.rtdc_ds.filter.all],
+            q=np.array(quantiles),
+            normalize=False)
+        contours = []
+        # Normalize levels to [0, 1]
+        nlevels = np.array(levels) / density.max()
+        for nlev in nlevels:
+            # make sure that the contour levels are not at the boundaries
+            if not (np.allclose(nlev, 0, atol=1e-12, rtol=0)
+                    or np.allclose(nlev, 1, atol=1e-12, rtol=0)):
+                cc = find_contours_level(
+                    density, x=x, y=y, level=nlev)
+                contours.append(cc)
+            else:
+                contours.append([])
+        if ret_levels:
+            return contours, levels
+        else:
+            return contours
+
+    def get_raster(self, xax="area_um", yax="deform", xacc=None, yacc=None,
+                   kde_type="histogram", kde_kwargs=None, xscale="linear",
+                   yscale="linear"):
+        """Evaluate the kernel density estimate on a grid
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        yacc: float
+            Contour accuracy in y direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
         Returns
         -------
         X, Y, Z : coordinates
@@ -128,21 +266,21 @@ class KernelDensityEstimator:
             a=x,
             feat=xax,
             scale=xscale,
-            method=bin_width_doane,
+            method=bin_width_doane_div5,
             ret_scaled=True)
 
         yacc_sc, ys = self.get_spacing(
             a=y,
             feat=yax,
             scale=yscale,
-            method=bin_width_doane,
+            method=bin_width_doane_div5,
             ret_scaled=True)
 
         if xacc is None or xacc == 0:
-            xacc = xacc_sc / 5
+            xacc = xacc_sc
 
         if yacc is None or yacc == 0:
-            yacc = yacc_sc / 5
+            yacc = yacc_sc
 
         # Ignore infs and nans
         bad = get_bad_vals(xs, ys)

dclab/kde/methods.py

@@ -56,6 +56,16 @@ def bin_width_doane(a):
     return acc
 
 
+def bin_width_doane_div5(a):
+    """Compute contour spacing based on Doane's formula divided by five
+
+    See Also
+    --------
+    bin_width_doane: method used to compute the bin width
+    """
+    return bin_width_doane(a) / 5
+
+
 def bin_width_percentile(a):
     """Compute contour spacing based on data percentiles
 

dclab/rtdc_dataset/core.py

@@ -635,7 +635,7 @@ class RTDCBase(abc.ABC):
             The kernel density Z evaluated on a rectangular grid (X,Y).
         """
         kde_instance = KernelDensityEstimator(rtdc_ds=self)
-        xmesh, ymesh, density = kde_instance.get_contour(
+        xmesh, ymesh, density = kde_instance.get_raster(
             xax=xax, yax=yax, xacc=xacc, yacc=yacc, kde_type=kde_type,
             kde_kwargs=kde_kwargs, xscale=xscale, yscale=yscale
         )
@@ -746,6 +746,8 @@ class RTDCBase(abc.ABC):
                 "ignored_basins": bd_keys,
                 # basin key
                 "key": bdict["key"],
+                # whether the basin is perishable or not
+                "perishable": bdict.get("perishable", False),
             }
 
             # Check whether this basin is supported and exists
@@ -783,12 +785,19 @@ class RTDCBase(abc.ABC):
                     b_cls = bc[bdict["format"]]
                     # Try absolute path
                     bna = b_cls(pp, **kwargs)
-                    if bna.verify_basin():
-                        basins.append(bna)
-                        break
+
+                    try:
+                        absolute_exists = bna.verify_basin()
+                    except BaseException:
+                        pass
+                    else:
+                        if absolute_exists:
+                            basins.append(bna)
+                            break
                     # Try relative path
                     this_path = pathlib.Path(self.path)
                     if this_path.exists():
+
                         # Insert relative path
                         bnr = b_cls(this_path.parent / pp, **kwargs)
                         if bnr.verify_basin():

dclab/rtdc_dataset/export.py

@@ -268,8 +268,8 @@ class Export(object):
         compression_kwargs: dict
             Dictionary with the keys "compression" and "compression_opts"
             which are passed to :func:`h5py.H5File.create_dataset`. The
-            default is Zstandard compression with the lowest compression
-            level `hdf5plugin.Zstd(clevel=1)`.
+            default is Zstandard compression with the compression
+            level 5 `hdf5plugin.Zstd(clevel=5)`.
         compression: str or None
             Compression method used for data storage;
             one of [None, "lzf", "gzip", "szip"].
@@ -300,7 +300,7 @@ class Export(object):
             # be backwards-compatible
             compression_kwargs = {"compression": compression}
         if compression_kwargs is None:
-            compression_kwargs = hdf5plugin.Zstd(clevel=1)
+            compression_kwargs = hdf5plugin.Zstd(clevel=5)
         path = pathlib.Path(path)
         # Make sure that path ends with .rtdc
         if path.suffix not in [".rtdc", ".rtdc~"]:
@@ -505,6 +505,11 @@ class Export(object):
                         # defined in. Since we are exporting, it does not
                         # make sense to store these basins in the output file.
                         continue
+                    elif bn_dict.get("perishable"):
+                        # Perishable basins require secret keys or complicated
+                        # logic to execute in order to refresh them. We do not
+                        # store them in the output file.
+                        continue
                     basinmap_orig = bn_dict.get("basin_map")
                     if not filtered:
                         # filtering disabled: just copy basins

dclab/rtdc_dataset/feat_basin.py

@@ -6,9 +6,10 @@ which, when opened in dclab, can access features stored in the input file
 from __future__ import annotations
 
 import abc
+import logging
 import numbers
 import threading
-from typing import Dict, List, Literal
+from typing import Callable, Dict, List, Literal, Union
 import uuid
 import warnings
 import weakref
@@ -18,6 +19,9 @@ import numpy as np
 from ..util import copy_if_needed
 
 
+logger = logging.getLogger(__name__)
+
+
 class BasinFeatureMissingWarning(UserWarning):
     """Used when a badin feature is defined but not stored"""
 
@@ -26,6 +30,10 @@ class CyclicBasinDependencyFoundWarning(UserWarning):
     """Used when a basin is defined in one of its sub-basins"""
 
 
+class IgnoringPerishableBasinTTL(UserWarning):
+    """Used when refreshing a basin does not support TTL"""
+
+
 class BasinmapFeatureMissingError(KeyError):
     """Used when one of the `basinmap` features is not defined"""
     pass
@@ -47,6 +55,114 @@ class BasinAvailabilityChecker(threading.Thread):
         self.basin.is_available()
 
 
+class PerishableRecord:
+    """A class containing information about perishable basins
+
+    Perishable basins are basins than may discontinue to work after
+    e.g. a specific amount of time (e.g. presigned S3 URLs). With the
+    `PerishableRecord`, these basins may be "refreshed" (made
+    available again).
+    """
+    def __init__(self,
+                 basin,
+                 expiration_func: Callable = None,
+                 expiration_kwargs: Dict = None,
+                 refresh_func: Callable = None,
+                 refresh_kwargs: Dict = None,
+                 ):
+        """
+        Parameters
+        ----------
+        basin: Basin
+            Instance of the perishable basin
+        expiration_func: callable
+            A function that determines whether the basin has perished.
+            It must accept `basin` as the first argument. Calling this
+            function should be fast, as it is called every time a feature
+            is accessed.
+            Note that if you are implementing this in the time domain, then
+            you should use `time.time()` (TSE), because you need an absolute
+            time measure. `time.monotonic()` for instance does not count up
+            when the system goes to sleep. However, keep in mind that if
+            a remote machine dictates the expiration time, then that
+            remote machine should also transmit the creation time (in case
+            there are time offsets).
+        expiration_kwargs: dict
+            Additional kwargs for `expiration_func`.
+        refresh_func: callable
+            The function used to refresh the `basin`. It must accept
+            `basin` as the first argument.
+        refresh_kwargs: dict
+            Additional kwargs for `refresh_func`
+        """
+        if not isinstance(basin, weakref.ProxyType):
+            basin = weakref.proxy(basin)
+        self.basin = basin
+        self.expiration_func = expiration_func
+        self.expiration_kwargs = expiration_kwargs or {}
+        self.refresh_func = refresh_func
+        self.refresh_kwargs = refresh_kwargs or {}
+
+    def __repr__(self):
+        state = "perished" if self.perished() else "valid"
+        return f"<PerishableRecord ({state}) at {hex(id(self))}>"
+
+    def perished(self) -> Union[bool, None]:
+        """Determine whether the basin has perished
+
+        Returns
+        -------
+        state: bool or None
+            True means the basin has perished, False means the basin
+            has not perished, and `None` means we don't know
+        """
+        if self.expiration_func is None:
+            return None
+        else:
+            return self.expiration_func(self.basin, **self.expiration_kwargs)
+
+    def refresh(self, extend_by: float = None) -> None:
+        """Extend the lifetime of the associated perishable basin
+
+        Parameters
+        ----------
+        extend_by: float
+            Custom argument for extending the life of the basin.
+            Normally, this would be a lifetime.
+
+        Returns
+        -------
+        basin: dict | None
+            Dictionary for instantiating a new basin
+        """
+        if self.refresh_func is None:
+            # The basin is a perishable basin, but we have no way of
+            # refreshing it.
+            logger.error(f"Cannot refresh basin '{self.basin}'")
+            return
+
+        if extend_by and "extend_by" not in self.refresh_kwargs:
+            warnings.warn(
+                "Parameter 'extend_by' ignored, because the basin "
+                "source does not support it",
+                IgnoringPerishableBasinTTL)
+            extend_by = None
+
+        rkw = {}
+        rkw.update(self.refresh_kwargs)
+
+        if extend_by is not None:
+            rkw["extend_by"] = extend_by
+
+        self.refresh_func(self.basin, **rkw)
+        logger.info(f"Refreshed basin '{self.basin}'")
+
+        # If everything went well, reset the current dataset of the basin
+        if self.basin._ds is not None:
+            self.basin._ds.close()
+            self.basin._ds = None
+
+
 class Basin(abc.ABC):
     """A basin represents data from an external source
 
@@ -76,6 +192,7 @@ class Basin(abc.ABC):
                  mapping_referrer: Dict = None,
                  ignored_basins: List[str] = None,
                  key: str = None,
+                 perishable=False,
                  **kwargs):
         """
 
@@ -115,6 +232,10 @@ class Basin(abc.ABC):
             Unique key to identify this basin; normally computed from
             a JSON dump of the basin definition. A random string is used
             if None is specified.
+        perishable: bool or PerishableRecord
+            If this is not False, then it must be a :class:`.PerishableRecord`
+            that holds the information about the expiration time, and that
+            comes with a method `refresh` to extend the lifetime of the basin.
         kwargs:
             Additional keyword arguments passed to the `load_dataset`
             method of the `Basin` subclass.
@@ -130,7 +251,12 @@ class Basin(abc.ABC):
         self.name = name
         #: lengthy description of the basin
         self.description = description
-        # defining key of the basin
+        # perishable record
+        if isinstance(perishable, bool) and perishable:
+            # Create an empty perishable record
+            perishable = PerishableRecord(self)
+        self.perishable = perishable
+        # define key of the basin
         self.key = key or str(uuid.uuid4())
         # features this basin provides
         self._features = features
@@ -164,10 +290,14 @@ class Basin(abc.ABC):
         self._av_check.start()
 
     def __repr__(self):
+        try:
+            feature_info = len(self.features)
+        except BaseException:
+            feature_info = "unknown"
         options = [
             self.name,
             f"mapped {self.mapping}" if self.mapping != "same" else "",
-            f"features {self._features}" if self.features else "full-featured",
+            f"{feature_info} features",
             f"location {self.location}",
         ]
         opt_str = ", ".join([o for o in options if o])
@@ -220,6 +350,10 @@ class Basin(abc.ABC):
     @property
     def ds(self):
         """The :class:`.RTDCBase` instance represented by the basin"""
+        if self.perishable and self.perishable.perished():
+            # We have perished. Ask the PerishableRecord to refresh this
+            # basin so we can access it again.
+            self.perishable.refresh()
         if self._ds is None:
             if not self.is_available():
                 raise BasinNotAvailableError(f"Basin {self} is not available!")
@@ -265,6 +399,7 @@ class Basin(abc.ABC):
             "basin_descr": self.description,
             "basin_feats": self.features,
             "basin_map": self.basinmap,
+            "perishable": bool(self.perishable),
         }
 
     def close(self):

dclab/rtdc_dataset/fmt_dcor/api.py

@@ -11,8 +11,10 @@ class DCORAccessError(BaseException):
 
 class APIHandler:
     """Handles the DCOR api with caching for simple queries"""
-    #: these are cached to minimize network usage
-    cache_queries = ["basins", "metadata", "size", "feature_list", "valid"]
+    #: These are cached to minimize network usage
+    #: Note that we are not caching basins, since they may contain
+    #: expiring URLs.
+    cache_queries = ["metadata", "size", "feature_list", "valid"]
     #: DCOR API Keys/Tokens in the current session
     api_keys = []
 
@@ -52,8 +54,36 @@ class APIHandler:
         if api_key.strip() and api_key not in APIHandler.api_keys:
             APIHandler.api_keys.append(api_key)
 
-    def _get(self, query, feat=None, trace=None, event=None, api_key="",
-             retries=13):
+    def _get(self,
+             query: str,
+             feat: str = None,
+             trace: str = None,
+             event: str = None,
+             api_key: str = "",
+             timeout: float = None,
+             retries: int = 5):
+        """Fetch information via the DCOR API
+
+        Parameters
+        ----------
+        query: str
+            API route
+        feat: str
+            DEPRECATED (use basins instead), adds f"&feature={feat}" to query
+        trace: str
+            DEPRECATED (use basins instead), adds f"&trace={trace}" to query
+        event: str
+            DEPRECATED (use basins instead), adds f"&event={event}" to query
+        api_key: str
+            DCOR API token to use
+        timeout: float
+            Request timeout
+        retries: int
+            Number of retries to fetch the request. For every retry, the
+            timeout is increased by two seconds.
+        """
+        if timeout is None:
+            timeout = 1
         # "version=2" introduced in dclab 0.54.3
         # (supported since ckanext.dc_serve 0.13.2)
         qstr = f"&version={self.dcserv_api_version}&query={query}"
@@ -65,13 +95,13 @@ class APIHandler:
             qstr += f"&event={event}"
         apicall = self.url + qstr
         fail_reasons = []
-        for _ in range(retries):
+        for ii in range(retries):
             try:
                 # try-except both requests and json conversion
                 req = self.session.get(apicall,
                                        headers={"Authorization": api_key},
                                        verify=self.verify,
-                                       timeout=1,
+                                       timeout=timeout + ii * 2,
                                        )
                 jreq = req.json()
             except requests.urllib3.exceptions.ConnectionError:  # requests
@@ -92,13 +122,45 @@ class APIHandler:
                                   f"Messages: {fail_reasons}")
         return jreq
 
-    def get(self, query, feat=None, trace=None, event=None):
+    def get(self,
+            query: str,
+            feat: str = None,
+            trace: str = None,
+            event: str = None,
+            timeout: float = None,
+            retries: int = 5,
+            ):
+        """Fetch information from DCOR
+
+        Parameters
+        ----------
+        query: str
+            API route
+        feat: str
+            DEPRECATED (use basins instead), adds f"&feature={feat}" to query
+        trace: str
+            DEPRECATED (use basins instead), adds f"&trace={trace}" to query
+        event: str
+            DEPRECATED (use basins instead), adds f"&event={event}" to query
+        timeout: float
+            Request timeout
+        retries: int
+            Number of retries to fetch the request. For every retry, the
+            timeout is increased by two seconds.
+        """
         if query in APIHandler.cache_queries and query in self._cache:
             result = self._cache[query]
         else:
             req = {"error": {"message": "No access to API (api key?)"}}
             for api_key in [self.api_key] + APIHandler.api_keys:
-                req = self._get(query, feat, trace, event, api_key)
+                req = self._get(query=query,
+                                feat=feat,
+                                trace=trace,
+                                event=event,
+                                api_key=api_key,
+                                timeout=timeout,
+                                retries=retries,
+                                )
                 if req["success"]:
                     self.api_key = api_key  # remember working key
                     break

dclab/rtdc_dataset/fmt_dcor/base.py

@@ -1,17 +1,23 @@
 """DCOR client interface"""
+import logging
 import pathlib
 import re
+import time
 
 from ...util import hashobj
 
 from ..config import Configuration
 from ..core import RTDCBase
+from ..feat_basin import PerishableRecord
 
 from . import api
 from .logs import DCORLogs
 from .tables import DCORTables
 
 
+logger = logging.getLogger(__name__)
+
+
 #: Append directories here where dclab should look for certificate bundles
 #: for a specific host. The directory should contain files named after the
 #: hostname, e.g. "dcor.mpl.mpg.de.cert".
@@ -73,6 +79,8 @@ class RTDC_DCOR(RTDCBase):
         super(RTDC_DCOR, self).__init__(*args, **kwargs)
 
         self._hash = None
+        self._cache_basin_dict = None
+        self.cache_basin_dict_time = 600
         self.path = RTDC_DCOR.get_full_url(url, use_ssl, host)
 
         if cert_path is None:
@@ -161,15 +169,106 @@ class RTDC_DCOR(RTDCBase):
         new_url = f"{scheme}://{netloc}/{api_path}"
         return new_url
 
-    def basins_get_dicts(self):
-        """Return list of dicts for all basins defined in `self.h5file`"""
+    def _basin_refresh(self, basin):
+        """Refresh the specified basin"""
+        # Retrieve the basin dictionary from DCOR
+        basin_dicts = self.basins_get_dicts()
+        for bn_dict in basin_dicts:
+            if bn_dict.get("name") == basin.name:
+                break
+        else:
+            raise ValueError(f"Basin '{basin.name}' not found in {self}")
+
+        tre = bn_dict["time_request"]
+        ttl = bn_dict["time_expiration"]
+        # remember time relative to time.time, subtract 30s to be on safe side
+        tex = bn_dict["time_local_request"] + (ttl - tre) - 30
+
+        if isinstance(basin.perishable, bool):
+            logger.debug("Initializing basin perishable %s", basin.name)
+            # create a perishable record
+            basin.perishable = PerishableRecord(
+                basin=basin,
+                expiration_func=self._basin_expiration,
+                expiration_kwargs={"time_local_expiration": tex},
+                refresh_func=self._basin_refresh,
+            )
+        else:
+            logger.debug("Refreshing basin perishable %s", basin.name)
+            # only update (this also works with weakref.ProxyType)
+            basin.perishable.expiration_kwargs = {"time_local_expiration": tex}
+
+        if len(bn_dict["urls"]) > 1:
+            logger.warning(f"Basin {basin.name} has multiple URLs. I am not "
+                           f"checking their availability: {bn_dict}")
+        basin.location = bn_dict["urls"][0]
+
+    def _basin_expiration(self, basin, time_local_expiration):
+        """Check whether the basin has perished"""
+        return time_local_expiration < time.time()
+
+    def _basins_get_dicts(self):
         try:
-            basins = self.api.get(query="basins")
+            basin_dicts = self.api.get(query="basins")
+            # Fill in missing timing information
+            for bn_dict in basin_dicts:
+                if (bn_dict.get("format") == "http"
+                        and "perishable" not in bn_dict):
+                    # We are communicating with an older version of
+                    # ckanext-dc_serve. Take a look at the URL and check
+                    # whether we have a perishable (~1 hour) URL or whether
+                    # this is a public resource.
+                    expires_regexp = re.compile(".*expires=([0-9]*)$")
+                    for url in bn_dict.get("urls", []):
+                        if match := expires_regexp.match(url.lower()):
+                            logger.debug("Detected perishable basin: %s",
+                                         bn_dict["name"])
+                            bn_dict["perishable"] = True
+                            bn_dict["time_request"] = time.time()
+                            bn_dict["time_expiration"] = int(match.group(1))
+                            # add part of the resource ID to the name
+                            infourl = url.split(bn_dict["name"], 1)[-1]
+                            infourl = infourl.replace("/", "")
+                            bn_dict["name"] += f"-{infourl[:5]}"
+                            break
+                    else:
+                        bn_dict["perishable"] = False
+                # If we have a perishable basin, add the local request time
+                if bn_dict.get("perishable"):
+                    bn_dict["time_local_request"] = time.time()
         except api.DCORAccessError:
             # TODO: Do not catch this exception when all DCOR instances
             #       implement the 'basins' query.
             # This means that the server does not implement the 'basins' query.
-            basins = []
+            basin_dicts = []
+        return basin_dicts
+
+    def basins_get_dicts(self):
+        """Return list of dicts for all basins defined on DCOR
+
+        The return value of this method is cached for 10 minutes
+        (cache time defined in the `cache_basin_dict_time` [s] property).
+        """
+        if (self._cache_basin_dict is None
+            or time.time() > (self._cache_basin_dict[1]
+                              + self.cache_basin_dict_time)):
+            self._cache_basin_dict = (self._basins_get_dicts(), time.time())
+        return self._cache_basin_dict[0]
+
+    def basins_retrieve(self):
+        """Same as superclass, but add perishable information"""
+        basin_dicts = self.basins_get_dicts()
+        basins = super(RTDC_DCOR, self).basins_retrieve()
+        for bn in basins:
+            for bn_dict in basin_dicts:
+                if bn.name == bn_dict.get("name"):
+                    # Determine whether we have to set a perishable record.
+                    if bn_dict.get("perishable"):
+                        # required for `_basin_refresh` to create a record
+                        bn.perishable = True
+                        # create the actual record
+                        self._basin_refresh(bn)
+                    break
         return basins
 
 

dclab/rtdc_dataset/fmt_dcor/logs.py

@@ -22,5 +22,5 @@ class DCORLogs:
     @property
     def _logs(self):
         if self._logs_cache is None:
-            self._logs_cache = self.api.get(query="logs")
+            self._logs_cache = self.api.get(query="logs", timeout=5)
         return self._logs_cache

dclab/rtdc_dataset/fmt_dcor/tables.py

@@ -27,7 +27,7 @@ class DCORTables:
     @property
     def _tables(self):
         if self._tables_cache is None:
-            table_data = self.api.get(query="tables")
+            table_data = self.api.get(query="tables", timeout=13)
             # assemble the tables
             tables = {}
             for key in table_data:

dclab/rtdc_dataset/writer.py

@@ -66,8 +66,8 @@ class RTDCWriter:
         compression_kwargs: dict-like
             Dictionary with the keys "compression" and "compression_opts"
             which are passed to :func:`h5py.H5File.create_dataset`. The
-            default is Zstandard compression with the lowest compression
-            level `hdf5plugin.Zstd(clevel=1)`. To disable compression, use
+            default is Zstandard compression with the compression
+            level 5 `hdf5plugin.Zstd(clevel=5)`. To disable compression, use
             `{"compression": None}`.
         compression: str or None
             Compression method used for data storage;
@@ -88,7 +88,7 @@ class RTDCWriter:
             # be backwards-compatible
             compression_kwargs = {"compression": compression}
         if compression_kwargs is None:
-            compression_kwargs = hdf5plugin.Zstd(clevel=1)
+            compression_kwargs = hdf5plugin.Zstd(clevel=5)
 
         self.mode = mode
         self.compression_kwargs = compression_kwargs
@@ -209,6 +209,7 @@ class RTDCWriter:
                     basin_map: np.ndarray | Tuple[str, np.ndarray] = None,
                     internal_data: Dict | h5py.Group = None,
                     verify: bool = True,
+                    perishable: bool = False,
                     ):
         """Write basin information
 
@@ -249,9 +250,13 @@ class RTDCWriter:
             This must be specified when storing internal basins, and it
             must not be specified for any other basin type.
         verify: bool
-            whether to verify the basin before storing it; You might have
+            Whether to verify the basin before storing it; You might have
             set this to False if you would like to write a basin that is
             e.g. temporarily not available
+        perishable: bool
+            Whether the basin is perishable. If this is True, then a
+            warning will be issued, because perishable basins may not be
+            accessed (e.g. time-based URL for private S3 data).
 
         Returns
         -------
@@ -261,6 +266,8 @@ class RTDCWriter:
 
             .. versionadded:: 0.58.0
         """
+        if perishable:
+            warnings.warn(f"Storing perishable basin {basin_name}")
         if basin_type == "internal":
             if internal_data is None:
                 raise ValueError(
@@ -381,6 +388,7 @@ class RTDCWriter:
             "type": basin_type,
             "features": None if basin_feats is None else sorted(basin_feats),
             "mapping": basin_map_name,
+            "perishable": perishable,
         }
         if basin_type == "file":
             flocs = []

{dclab-0.63.1.dist-info → dclab-0.64.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dclab
-Version: 0.63.1
+Version: 0.64.0
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -14,7 +14,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Visualization
 Classifier: Intended Audience :: Science/Research
-Requires-Python: <4,>=3.8
+Requires-Python: <4,>=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: h5py<4,>=3.0.0

{dclab-0.63.1.dist-info → dclab-0.64.0.dist-info}/RECORD

@@ -1,12 +1,12 @@
 dclab/kde_methods.py,sha256=f0-zDN7ETintvGB3gSzxwgBb53YtT9jZtzI70EAX50g,365
-dclab/_version.py,sha256=qDZMrEPFSKYNbMiOoBOEXqq2P0z4X69tZSzimIIP7H0,513
+dclab/_version.py,sha256=zVb-mNfv7Q80VP6gLIonMXNXR1BAme5zhf4Ww25kC90,513
 dclab/util.py,sha256=HFT5ZQV6AW8GIIruVMldukbVVdlMyKH50GUfOogAcxI,5860
 dclab/downsampling.pyx,sha256=OK7zbgGLl5gVyoU8ZBHo9EWwb8C9ChavmLNEvQvC9T0,7258
 dclab/__init__.py,sha256=wyJWhElQRPcq09vUqUnuquTU_KHgHxv6wQxuxQ988Iw,1583
 dclab/warn.py,sha256=MjJvyQeuvIXFQ2-fHDzbmXJ0scnHqqRJlIxfuLI_utE,523
-dclab/downsampling.cpython-310-darwin.so,sha256=bEuZI1oCLS1NIGtfJynFgrYJNc8o0ItpZ5W7YlL1mY4,241840
+dclab/downsampling.cpython-310-darwin.so,sha256=46G33A6W0loraETqfSHk-fGHzSxdeQlFUIm2BdLaHH0,243392
 dclab/cached.py,sha256=eWTYBiI-HQM7JuPH-oxa5LLnhAX32GpRwlYg2kQ3sTA,2917
-dclab/http_utils.py,sha256=YtZHEwB-BBBo2fCvwhlJvlnWvfWFMHclqol3OIJ7atM,10910
+dclab/http_utils.py,sha256=XHicbHQts5LY3zSNmYqNgAZpKTktotEiwQgJ8d2sBlk,10912
 dclab/polygon_filter.py,sha256=qexmo-rXe06CUPZhN6EMJy4y4B5gXZeqejdvIB2arOE,13480
 dclab/statistics.py,sha256=tJDqPlY_Jw2Hhl-s7ugMBSZAxcRuPu4LQuBAZBXz7t8,6355
 dclab/kde_contours.py,sha256=UlU64lrzMQUZH11oZndW7xf7NFCzwP3FcVujwuqXDCI,278
@@ -48,9 +48,9 @@ dclab/cli/task_join.py,sha256=R04ZmXG2-O82E8le0ywZpy9NzOYoErw8hLx8m4q76ms,9464
 dclab/cli/task_split.py,sha256=dqW3GWkwi4ceDqRVrQx8ADGk4--KSsD3Q1knkD3ZXI4,6060
 dclab/cli/task_compress.py,sha256=LH3j41aKq5regAH-ZFdOgnXImf5DrReDJ7ITwUObUSQ,4100
 dclab/kde/contours.py,sha256=WoRqBj_xK-23FZjtaYly7E2Q8sGZ16q2ILq-DmrlmC8,6742
-dclab/kde/methods.py,sha256=SYlAjoST66hEZnRmsdZ6izMmgfebxQxTfBR5PHhzDkE,9208
+dclab/kde/methods.py,sha256=8g4lYUKYqt2pdA9efHVRBDCUUzmePmWPp6rljtJ0XD8,9438
 dclab/kde/__init__.py,sha256=_WSLPMfxE2su6tmO5mJxUE_9ON16-pqQUQCUlzRtyKI,55
-dclab/kde/base.py,sha256=i6bNYYeQSFlq03Z5CZjhBE-V3xpY6YLWivL3guscsnE,7683
+dclab/kde/base.py,sha256=KuV_9_AJ4Sn14C4WAH4eF63URAVYCnaFOcLgsVCaAjo,12961
 dclab/external/__init__.py,sha256=gb0UdzoMymEURPsTXRqVTT1ZJedJ2ubH3jApBIRkwjk,93
 dclab/external/packaging/version.py,sha256=9MLL6_EYHvGA1yCGndwL5ZmmDA_wqQsW15GyKrI6siQ,14685
 dclab/external/packaging/LICENSE,sha256=ytHvW9NA1z4HS6YU0m996spceUDD2MNIUuZcSQlobEg,197
@@ -70,12 +70,12 @@ dclab/external/skimage/pnpoly.py,sha256=r8hFNiTz5XlUoNZjosqA0iyv1FPn0l7ewbplgFgk
 dclab/external/skimage/_find_contours.py,sha256=16v5eeTZBmevG8SSuXtJ6yUpVPhwfSmtc8pDD0nuuOU,9340
 dclab/external/skimage/__init__.py,sha256=-B2QUKHAFzQuBWuuKvPDC5JIl0Zb-x3OGmbwPaE9VwQ,72
 dclab/external/skimage/_pnpoly.pyx,sha256=Qdn6xPazDschBqbr46DzB75MB2MnqvdnoTSBMK7kUGE,2504
-dclab/external/skimage/_pnpoly.cpython-310-darwin.so,sha256=PUhL3bN-JMD-oTRJ_-RWFw27oG4JaUYH10bHQOOaFgk,221808
+dclab/external/skimage/_pnpoly.cpython-310-darwin.so,sha256=hTaCIjJEYLRAaX_Xjq-glzk_9nLVBWaBA7hY7zhscC8,206656
 dclab/external/skimage/_find_contours_cy.pyx,sha256=pZJOBhMHzYEMkcz4WQVyjn7jDNrdjCfet47FU1hRAxk,7161
-dclab/external/skimage/_find_contours_cy.cpython-310-darwin.so,sha256=ov3bakWSSMfG5BPrnnGuMgevCEpNeAnaZcBDIDJzFyg,204104
+dclab/external/skimage/_find_contours_cy.cpython-310-darwin.so,sha256=nORyRpRTav0kfJaYADOTMrlY-1IR0N-3I_HGQ1RFg_Q,188328
 dclab/external/skimage/_shared/geometry.pxd,sha256=kRsu9ifv_rL3kbRIgSLf86p0hn2oTMp6s013lZ9bBZM,346
 dclab/external/skimage/_shared/__init__.py,sha256=2sHZwTtJSlMTa3Q2YSvQW7jrPLMUSqDJQa-ROe5zfcw,37
-dclab/external/skimage/_shared/geometry.cpython-310-darwin.so,sha256=owTJ2WwrO0g7RKgckweQI9xOWFwDhsZq_mTQP0VCkN0,55872
+dclab/external/skimage/_shared/geometry.cpython-310-darwin.so,sha256=QxFADGAMur2-nGu0OtFZREUIlS63Q0ELIRiT7iB21XY,55200
 dclab/external/skimage/_shared/geometry.pyx,sha256=miCHUh6mBDbRRIoaF_0xAER1MRzsCAzFdlYQZhV7RmE,1667
 dclab/definitions/feat_logic.py,sha256=SXsSlAusgtE3uXcPu84dQwYZ07zxmV37DmPednA3_dM,5823
 dclab/definitions/meta_parse.py,sha256=YdaTdM8DAMsIFn5ITH9OHYGTXeAOBGWtx22oVjxXcWk,2393
@@ -86,14 +86,14 @@ dclab/definitions/meta_logic.py,sha256=wCgb7DPRHaR8DCWw_VbwNkpslUnazzWfgX0iS8oEe
 dclab/rtdc_dataset/config.py,sha256=MvBteFya3R6Ch3U6UgTakCsJoBgVykTxS_Z25STWPHU,17432
 dclab/rtdc_dataset/check.py,sha256=lJNaz4QTe2WNlxik6zSohRHTiAYuP_bKOzSDjPGTUS0,35006
 dclab/rtdc_dataset/meta_table.py,sha256=ucqBNrgI6rDAuQFuMRckY8lp1LpnYAoRgEsLObWTJCE,648
-dclab/rtdc_dataset/feat_basin.py,sha256=ViKdvJcwFM8joysnrBYdZbA5t_wZix-6xn_FsvzpYsQ,21072
+dclab/rtdc_dataset/feat_basin.py,sha256=i1J6iKQQEJXdi3u-TB9y_OdxbkFPBK6_G2jeaGcBsLY,26172
 dclab/rtdc_dataset/fmt_s3.py,sha256=bU3V_WGyqJhxPCH80X_nlNqq-jXcgoZKv_aUBIqwaL8,11877
 dclab/rtdc_dataset/feat_temp.py,sha256=XbDIS1iUUkRH0Zp9uVlwvK_untJ7hkOnKshK1Drsnt8,3694
 dclab/rtdc_dataset/__init__.py,sha256=MUHSGVQJ4Zc0IyU2lf01dpDWyOyNveHip-UjSkmPNvQ,486
-dclab/rtdc_dataset/core.py,sha256=EjNWk9SV-2xBTRtf34XosLCOS164vGWKP5dKKLSOSq4,34441
-dclab/rtdc_dataset/export.py,sha256=NarfazGkPasyRpnhQ2H_riUP-oUq5QL7lG9WwvLCL2o,33121
+dclab/rtdc_dataset/core.py,sha256=p1Wy9Dq1Ny4i-cfvgPfCZov_lQAhGz0--x59j-g4mTs,34753
+dclab/rtdc_dataset/export.py,sha256=Ukmdz-Mm9iei6vhfp1lh7oNQiAbXvA2o3tRx1XlBWCM,33416
 dclab/rtdc_dataset/fmt_dict.py,sha256=gumVQOiVVDFUKow_483PY7cxInqo-NiBBnBhIU8s4lg,3009
-dclab/rtdc_dataset/writer.py,sha256=jc6ADyxGoujXpoXu1vF2nfZjGFMaO5LbRmoYJZ83JVo,41418
+dclab/rtdc_dataset/writer.py,sha256=BqP4QSzHxCSxp9z7_cJF2v04taO3ZXE3Wg6zJY6fMao,41819
 dclab/rtdc_dataset/filter.py,sha256=AFPUBzOIi3pqXgUdMQ5CIi9ZeGOKC71rfSZKLMLgtog,10023
 dclab/rtdc_dataset/fmt_http.py,sha256=vXVxRLXZp2_V1v3xk4lu4VUHYXfNHJdsRkVt3trC1RU,3374
 dclab/rtdc_dataset/load.py,sha256=5_xGw2P8Mjs0gW-vGr2Kn28j6Qv3BvvMvguMibC_mM4,2761
@@ -123,10 +123,10 @@ dclab/rtdc_dataset/feat_anc_ml/__init__.py,sha256=99jZlz17aBCAxI5xx42XINomMy20Hi
 dclab/rtdc_dataset/fmt_dcor/access_token.py,sha256=jotLQay138RUlv8wbdF2ishRnyE9N0KwGGBlbCL0wRI,2028
 dclab/rtdc_dataset/fmt_dcor/basin.py,sha256=tQZ4GumqURjS3eppRrSyUq1zBPD0y_8rwznMRDXiDUs,2526
 dclab/rtdc_dataset/fmt_dcor/__init__.py,sha256=WjO1uM_Vlof15Y7HkhkV5Xv75q9TDIdOBIuS_I38qps,210
-dclab/rtdc_dataset/fmt_dcor/api.py,sha256=GfQdKhkGXN-e9-e5YxgzL81tDeWB2M-UoyouVfXXbVg,4255
-dclab/rtdc_dataset/fmt_dcor/logs.py,sha256=1JsMr_4r5j8rkfrrUsiN42_l92GcvDjapYxopZKimnw,583
-dclab/rtdc_dataset/fmt_dcor/tables.py,sha256=NaVEwLKmOg7Mz5iAMe2S8C4xRVC_YO3zeT7g5EbQE1M,1682
-dclab/rtdc_dataset/fmt_dcor/base.py,sha256=wD127W5LvvhkUy8SvFVVwAR6EEYtzgoWJ4booh45rfA,6588
+dclab/rtdc_dataset/fmt_dcor/api.py,sha256=IhmNpEdVSGHdJUepCkmuyTVOp3fNn1WASPVohAzwhu8,6274
+dclab/rtdc_dataset/fmt_dcor/logs.py,sha256=FHaDGjh4wMd18S2FFr6IVdd23l21S6s3fwZnFbPG83E,594
+dclab/rtdc_dataset/fmt_dcor/tables.py,sha256=a4gMIjH0TgFdz36l_F-xriHTT4LMI64ur-atHyvAp80,1694
+dclab/rtdc_dataset/fmt_dcor/base.py,sha256=cFiZcWG43jOGTY3oNl-E_vdzbm5QZU1CEB7ucrLZP_U,11167
 dclab/rtdc_dataset/fmt_tdms/event_mask.py,sha256=eZiDHAGG3MCVckEMHsV-YBbL-pETVLomGk1gmpjc8-k,2175
 dclab/rtdc_dataset/fmt_tdms/naming.py,sha256=biI9l1EO6BuSYgwZG0deacj4i1fMHQcW78AKXEcm5Wc,5373
 dclab/rtdc_dataset/fmt_tdms/__init__.py,sha256=3Hc6ASVHEmZBW0pCdFiLZcBtNPcEjG6c7eddqeAZRK4,18923
@@ -134,9 +134,9 @@ dclab/rtdc_dataset/fmt_tdms/event_image.py,sha256=-jp7Z-N91e4ieumYQ1huMicj7PMJqw
 dclab/rtdc_dataset/fmt_tdms/event_trace.py,sha256=Vkym0QKSw2mq1XZl5n8wDkgHXmaZwQGiMAV5AuRSJkE,5215
 dclab/rtdc_dataset/fmt_tdms/exc.py,sha256=WzrMqnyrzp8gsT8Pf7JKqGGv43ewx7d_qgtirURppRI,813
 dclab/rtdc_dataset/fmt_tdms/event_contour.py,sha256=kjo0wJx9F0gmmOOyR0NoLw6VEtSl3h63WXXkcbfnoS8,9627
-dclab-0.63.1.dist-info/RECORD,,
-dclab-0.63.1.dist-info/WHEEL,sha256=IXYzzLeGhzM4nRz5U4mzre-uliJyF_FEfdbcpUEVBrU,136
-dclab-0.63.1.dist-info/entry_points.txt,sha256=eOpjgznu-eW-9utUpLU-77O5098YyUEgGF3ksGMdtec,273
-dclab-0.63.1.dist-info/top_level.txt,sha256=irvwZMgs1edY1Zj60ZFk7Almb9Zhk4k6E6aC4YPFnnM,6
-dclab-0.63.1.dist-info/METADATA,sha256=kaDiBVjVDUqc77B4nSoNNQhk-IE6Rfb7WI9yAV1UABM,4755
-dclab-0.63.1.dist-info/licenses/LICENSE,sha256=gLDaVZWRrlnLdyfOrR0qfWjLbOVcjvoJ-kCLUK0fyXA,15360
+dclab-0.64.0.dist-info/RECORD,,
+dclab-0.64.0.dist-info/WHEEL,sha256=6h6JDR7KEjDmDmYh9AYbh3iumxbuiwKCwb1O-T24foc,136
+dclab-0.64.0.dist-info/entry_points.txt,sha256=eOpjgznu-eW-9utUpLU-77O5098YyUEgGF3ksGMdtec,273
+dclab-0.64.0.dist-info/top_level.txt,sha256=irvwZMgs1edY1Zj60ZFk7Almb9Zhk4k6E6aC4YPFnnM,6
+dclab-0.64.0.dist-info/METADATA,sha256=zfrjZaLGPiE_ZPYkVtoUGvGI-CPkjgY-eiKJshd-3bo,4755
+dclab-0.64.0.dist-info/licenses/LICENSE,sha256=gLDaVZWRrlnLdyfOrR0qfWjLbOVcjvoJ-kCLUK0fyXA,15360

{dclab-0.63.1.dist-info → dclab-0.64.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: false
 Tag: cp310-cp310-macosx_11_0_arm64
 Generator: delocate 0.13.0