dclab 0.63.1__tar.gz → 0.64.0__tar.gz

{dclab-0.63.1 → dclab-0.64.0}/CHANGELOG

@@ -1,3 +1,18 @@
+0.64.0
+ - feat: introduce concept of perishable (e.g. temporally expiring) basins
+ - feat: add `PerishableRecord` to presigned URLs for DCOR basins (#244)
+ - fix: do not export perishable basins (#254)
+ - fix: allow `Basin.verify_basin` to fail
+ - fix: `Basin.__repr__` failed when location did not exist
+ - fix: fetching tables information from DCOR dataset could time out
+ - fix: default to compression level 5 when exporting or writing (#272)
+ - enh: increase timeout every time fetching data from DCOR API times out
+ - docs: update docs `RTDCWriter` I/O
+ - setup: drop support for Python 3.8
+ - tests: add tests for fetching tables from a DCOR dataset (#229)
+ - ref: add `get_contour_lines` method to `KernelDensityEstimator` class
+ - ref: deprecate get_contour and use bin_width_doane_div5 in kde submodule
+ - ref: cache "basins" in `RTDC_DCOR` instead of in `RTDC_DCOR.api`
 0.63.1
  - docs: update documentation and code references for kde submodule
  - enh: cache S3 sessions and clients (test suite duration reduced by 30%)

{dclab-0.63.1 → dclab-0.64.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dclab
-Version: 0.63.1
+Version: 0.64.0
 Summary: Library for real-time deformability cytometry (RT-DC)
 Author: Benedikt Hartmann, Eoghan O'Connell, Maik Herbig, Maximilian Schlögel, Nadia Sbaa, Paul Müller, Philipp Rosendahl, Raghava Alajangi
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -14,7 +14,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Visualization
 Classifier: Intended Audience :: Science/Research
-Requires-Python: <4,>=3.8
+Requires-Python: <4,>=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: h5py<4,>=3.0.0

{dclab-0.63.1 → dclab-0.64.0}/dclab/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.63.1'
-__version_tuple__ = version_tuple = (0, 63, 1)
+__version__ = version = '0.64.0'
+__version_tuple__ = version_tuple = (0, 64, 0)

{dclab-0.63.1 → dclab-0.64.0}/dclab/http_utils.py

@@ -254,7 +254,7 @@ class ResoluteRequestsSession(requests.Session):
                 break
         else:
             raise requests.exceptions.ReadTimeout(
-                f"Resolut sesion failed for {args} and {kwargs}!")
+                f"Resolute session failed for {args} and {kwargs}!")
         return resp
 
 

{dclab-0.63.1 → dclab-0.64.0}/dclab/kde/base.py

@@ -2,7 +2,12 @@ import warnings
 
 import numpy as np
 
-from .methods import bin_width_doane, get_bad_vals, methods
+from .methods import bin_width_doane_div5, get_bad_vals, methods
+from .contours import find_contours_level, get_quantile_levels
+
+
+class ContourSpacingTooLarge(UserWarning):
+    pass
 
 
 class KernelDensityEstimator:
@@ -107,6 +112,139 @@ class KernelDensityEstimator:
         yscale: str
             See `xscale`.
 
+        Returns
+        -------
+        X, Y, Z : coordinates
+            The kernel density Z evaluated on a rectangular grid (X,Y).
+        """
+        warnings.warn("`get_contour` is deprecated; please use "
+                      "`get_raster` instead", DeprecationWarning)
+        return self.get_raster(
+            xax=xax, yax=yax, xacc=xacc, yacc=yacc,
+            kde_type=kde_type, kde_kwargs=kde_kwargs,
+            xscale=xscale, yscale=yscale
+        )
+
+    def get_contour_lines(self, quantiles=None, xax="area_um", yax="deform",
+                          xacc=None, yacc=None, kde_type="histogram",
+                          kde_kwargs=None, xscale="linear", yscale="linear",
+                          ret_levels=False):
+        """Compute contour lines for a given kernel kensity estimate.
+
+        Parameters
+        ----------
+        quantiles: list or array of floats
+            KDE Quantiles for which contour levels are computed. The
+            values must be between 0 and 1. If set to None, use
+            [0.5, 0.95] as default.
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        yacc: float
+            Contour accuracy in y direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`
+        ret_levels: bool
+            If set to True, return the levels of the contours
+            (default: False)
+
+        Returns
+        -------
+        contour_lines: list of lists (of lists)
+            For every number in `quantiles`, this list contains a list of
+            corresponding contour lines. Each contour line is a 2D
+            array of shape (N, 2), where N is the number of points in the
+            contour line.
+        levels: list of floats
+            The density levels corresponding to each number in `quantiles`.
+            Only returned if `ret_levels` is set to True.
+        """
+        if not quantiles:
+            quantiles = [0.5, 0.95]
+        try:
+            x, y, density = self.get_raster(
+                xax=xax,
+                yax=yax,
+                xacc=xacc,
+                yacc=yacc,
+                xscale=xscale,
+                yscale=yscale,
+                kde_type=kde_type,
+                kde_kwargs=kde_kwargs,
+            )
+        except ValueError:
+            # most-likely there is nothing to compute a contour for
+            return []
+        if density.shape[0] < 3 or density.shape[1] < 3:
+            warnings.warn("Contour not possible; spacing may be too large!",
+                          ContourSpacingTooLarge)
+            return []
+        levels = get_quantile_levels(
+            density=density,
+            x=x,
+            y=y,
+            xp=self.rtdc_ds[xax][self.rtdc_ds.filter.all],
+            yp=self.rtdc_ds[yax][self.rtdc_ds.filter.all],
+            q=np.array(quantiles),
+            normalize=False)
+        contours = []
+        # Normalize levels to [0, 1]
+        nlevels = np.array(levels) / density.max()
+        for nlev in nlevels:
+            # make sure that the contour levels are not at the boundaries
+            if not (np.allclose(nlev, 0, atol=1e-12, rtol=0)
+                    or np.allclose(nlev, 1, atol=1e-12, rtol=0)):
+                cc = find_contours_level(
+                    density, x=x, y=y, level=nlev)
+                contours.append(cc)
+            else:
+                contours.append([])
+        if ret_levels:
+            return contours, levels
+        else:
+            return contours
+
+    def get_raster(self, xax="area_um", yax="deform", xacc=None, yacc=None,
+                   kde_type="histogram", kde_kwargs=None, xscale="linear",
+                   yscale="linear"):
+        """Evaluate the kernel density estimate on a grid
+
+        Parameters
+        ----------
+        xax: str
+            Identifier for X axis (e.g. "area_um", "aspect", "deform")
+        yax: str
+            Identifier for Y axis
+        xacc: float
+            Contour accuracy in x direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        yacc: float
+            Contour accuracy in y direction
+            if set to None, will use :func:`bin_width_doane_div5`
+        kde_type: str
+            The KDE method to use
+        kde_kwargs: dict
+            Additional keyword arguments to the KDE method
+        xscale: str
+            If set to "log", take the logarithm of the x-values before
+            computing the KDE. This is useful when data are
+            displayed on a log-scale. Defaults to "linear".
+        yscale: str
+            See `xscale`.
+
         Returns
         -------
         X, Y, Z : coordinates
@@ -128,21 +266,21 @@ class KernelDensityEstimator:
             a=x,
             feat=xax,
             scale=xscale,
-            method=bin_width_doane,
+            method=bin_width_doane_div5,
             ret_scaled=True)
 
         yacc_sc, ys = self.get_spacing(
             a=y,
             feat=yax,
             scale=yscale,
-            method=bin_width_doane,
+            method=bin_width_doane_div5,
             ret_scaled=True)
 
         if xacc is None or xacc == 0:
-            xacc = xacc_sc / 5
+            xacc = xacc_sc
 
         if yacc is None or yacc == 0:
-            yacc = yacc_sc / 5
+            yacc = yacc_sc
 
         # Ignore infs and nans
         bad = get_bad_vals(xs, ys)

{dclab-0.63.1 → dclab-0.64.0}/dclab/kde/methods.py

@@ -56,6 +56,16 @@ def bin_width_doane(a):
     return acc
 
 
+def bin_width_doane_div5(a):
+    """Compute contour spacing based on Doane's formula divided by five
+
+    See Also
+    --------
+    bin_width_doane: method used to compute the bin width
+    """
+    return bin_width_doane(a) / 5
+
+
 def bin_width_percentile(a):
     """Compute contour spacing based on data percentiles
 

{dclab-0.63.1 → dclab-0.64.0}/dclab/rtdc_dataset/core.py

@@ -635,7 +635,7 @@ class RTDCBase(abc.ABC):
             The kernel density Z evaluated on a rectangular grid (X,Y).
         """
         kde_instance = KernelDensityEstimator(rtdc_ds=self)
-        xmesh, ymesh, density = kde_instance.get_contour(
+        xmesh, ymesh, density = kde_instance.get_raster(
             xax=xax, yax=yax, xacc=xacc, yacc=yacc, kde_type=kde_type,
             kde_kwargs=kde_kwargs, xscale=xscale, yscale=yscale
         )
@@ -746,6 +746,8 @@ class RTDCBase(abc.ABC):
                 "ignored_basins": bd_keys,
                 # basin key
                 "key": bdict["key"],
+                # whether the basin is perishable or not
+                "perishable": bdict.get("perishable", False),
             }
 
             # Check whether this basin is supported and exists
@@ -783,12 +785,19 @@ class RTDCBase(abc.ABC):
                     b_cls = bc[bdict["format"]]
                     # Try absolute path
                     bna = b_cls(pp, **kwargs)
-                    if bna.verify_basin():
-                        basins.append(bna)
-                        break
+
+                    try:
+                        absolute_exists = bna.verify_basin()
+                    except BaseException:
+                        pass
+                    else:
+                        if absolute_exists:
+                            basins.append(bna)
+                            break
                     # Try relative path
                     this_path = pathlib.Path(self.path)
                     if this_path.exists():
+
                         # Insert relative path
                         bnr = b_cls(this_path.parent / pp, **kwargs)
                         if bnr.verify_basin():

{dclab-0.63.1 → dclab-0.64.0}/dclab/rtdc_dataset/export.py

@@ -268,8 +268,8 @@ class Export(object):
         compression_kwargs: dict
             Dictionary with the keys "compression" and "compression_opts"
             which are passed to :func:`h5py.H5File.create_dataset`. The
-            default is Zstandard compression with the lowest compression
-            level `hdf5plugin.Zstd(clevel=1)`.
+            default is Zstandard compression with the compression
+            level 5 `hdf5plugin.Zstd(clevel=5)`.
         compression: str or None
             Compression method used for data storage;
             one of [None, "lzf", "gzip", "szip"].
@@ -300,7 +300,7 @@ class Export(object):
             # be backwards-compatible
             compression_kwargs = {"compression": compression}
         if compression_kwargs is None:
-            compression_kwargs = hdf5plugin.Zstd(clevel=1)
+            compression_kwargs = hdf5plugin.Zstd(clevel=5)
         path = pathlib.Path(path)
         # Make sure that path ends with .rtdc
         if path.suffix not in [".rtdc", ".rtdc~"]:
@@ -505,6 +505,11 @@ class Export(object):
                         # defined in. Since we are exporting, it does not
                         # make sense to store these basins in the output file.
                         continue
+                    elif bn_dict.get("perishable"):
+                        # Perishable basins require secret keys or complicated
+                        # logic to execute in order to refresh them. We do not
+                        # store them in the output file.
+                        continue
                     basinmap_orig = bn_dict.get("basin_map")
                     if not filtered:
                         # filtering disabled: just copy basins

{dclab-0.63.1 → dclab-0.64.0}/dclab/rtdc_dataset/feat_basin.py

@@ -6,9 +6,10 @@ which, when opened in dclab, can access features stored in the input file
 from __future__ import annotations
 
 import abc
+import logging
 import numbers
 import threading
-from typing import Dict, List, Literal
+from typing import Callable, Dict, List, Literal, Union
 import uuid
 import warnings
 import weakref
@@ -18,6 +19,9 @@ import numpy as np
 from ..util import copy_if_needed
 
 
+logger = logging.getLogger(__name__)
+
+
 class BasinFeatureMissingWarning(UserWarning):
     """Used when a badin feature is defined but not stored"""
 
@@ -26,6 +30,10 @@ class CyclicBasinDependencyFoundWarning(UserWarning):
     """Used when a basin is defined in one of its sub-basins"""
 
 
+class IgnoringPerishableBasinTTL(UserWarning):
+    """Used when refreshing a basin does not support TTL"""
+
+
 class BasinmapFeatureMissingError(KeyError):
     """Used when one of the `basinmap` features is not defined"""
     pass
@@ -47,6 +55,114 @@ class BasinAvailabilityChecker(threading.Thread):
         self.basin.is_available()
 
 
+class PerishableRecord:
+    """A class containing information about perishable basins
+
+    Perishable basins are basins than may discontinue to work after
+    e.g. a specific amount of time (e.g. presigned S3 URLs). With the
+    `PerishableRecord`, these basins may be "refreshed" (made
+    available again).
+    """
+    def __init__(self,
+                 basin,
+                 expiration_func: Callable = None,
+                 expiration_kwargs: Dict = None,
+                 refresh_func: Callable = None,
+                 refresh_kwargs: Dict = None,
+                 ):
+        """
+        Parameters
+        ----------
+        basin: Basin
+            Instance of the perishable basin
+        expiration_func: callable
+            A function that determines whether the basin has perished.
+            It must accept `basin` as the first argument. Calling this
+            function should be fast, as it is called every time a feature
+            is accessed.
+            Note that if you are implementing this in the time domain, then
+            you should use `time.time()` (TSE), because you need an absolute
+            time measure. `time.monotonic()` for instance does not count up
+            when the system goes to sleep. However, keep in mind that if
+            a remote machine dictates the expiration time, then that
+            remote machine should also transmit the creation time (in case
+            there are time offsets).
+        expiration_kwargs: dict
+            Additional kwargs for `expiration_func`.
+        refresh_func: callable
+            The function used to refresh the `basin`. It must accept
+            `basin` as the first argument.
+        refresh_kwargs: dict
+            Additional kwargs for `refresh_func`
+        """
+        if not isinstance(basin, weakref.ProxyType):
+            basin = weakref.proxy(basin)
+        self.basin = basin
+        self.expiration_func = expiration_func
+        self.expiration_kwargs = expiration_kwargs or {}
+        self.refresh_func = refresh_func
+        self.refresh_kwargs = refresh_kwargs or {}
+
+    def __repr__(self):
+        state = "perished" if self.perished() else "valid"
+        return f"<PerishableRecord ({state}) at {hex(id(self))}>"
+
+    def perished(self) -> Union[bool, None]:
+        """Determine whether the basin has perished
+
+        Returns
+        -------
+        state: bool or None
+            True means the basin has perished, False means the basin
+            has not perished, and `None` means we don't know
+        """
+        if self.expiration_func is None:
+            return None
+        else:
+            return self.expiration_func(self.basin, **self.expiration_kwargs)
+
+    def refresh(self, extend_by: float = None) -> None:
+        """Extend the lifetime of the associated perishable basin
+
+        Parameters
+        ----------
+        extend_by: float
+            Custom argument for extending the life of the basin.
+            Normally, this would be a lifetime.
+
+        Returns
+        -------
+        basin: dict | None
+            Dictionary for instantiating a new basin
+        """
+        if self.refresh_func is None:
+            # The basin is a perishable basin, but we have no way of
+            # refreshing it.
+            logger.error(f"Cannot refresh basin '{self.basin}'")
+            return
+
+        if extend_by and "extend_by" not in self.refresh_kwargs:
+            warnings.warn(
+                "Parameter 'extend_by' ignored, because the basin "
+                "source does not support it",
+                IgnoringPerishableBasinTTL)
+            extend_by = None
+
+        rkw = {}
+        rkw.update(self.refresh_kwargs)
+
+        if extend_by is not None:
+            rkw["extend_by"] = extend_by
+
+        self.refresh_func(self.basin, **rkw)
+        logger.info(f"Refreshed basin '{self.basin}'")
+
+        # If everything went well, reset the current dataset of the basin
+        if self.basin._ds is not None:
+            self.basin._ds.close()
+            self.basin._ds = None
+
+
 class Basin(abc.ABC):
     """A basin represents data from an external source
 
@@ -76,6 +192,7 @@ class Basin(abc.ABC):
                  mapping_referrer: Dict = None,
                  ignored_basins: List[str] = None,
                  key: str = None,
+                 perishable=False,
                  **kwargs):
         """
 
@@ -115,6 +232,10 @@ class Basin(abc.ABC):
             Unique key to identify this basin; normally computed from
             a JSON dump of the basin definition. A random string is used
             if None is specified.
+        perishable: bool or PerishableRecord
+            If this is not False, then it must be a :class:`.PerishableRecord`
+            that holds the information about the expiration time, and that
+            comes with a method `refresh` to extend the lifetime of the basin.
         kwargs:
             Additional keyword arguments passed to the `load_dataset`
             method of the `Basin` subclass.
@@ -130,7 +251,12 @@ class Basin(abc.ABC):
         self.name = name
         #: lengthy description of the basin
         self.description = description
-        # defining key of the basin
+        # perishable record
+        if isinstance(perishable, bool) and perishable:
+            # Create an empty perishable record
+            perishable = PerishableRecord(self)
+        self.perishable = perishable
+        # define key of the basin
         self.key = key or str(uuid.uuid4())
         # features this basin provides
         self._features = features
@@ -164,10 +290,14 @@ class Basin(abc.ABC):
         self._av_check.start()
 
     def __repr__(self):
+        try:
+            feature_info = len(self.features)
+        except BaseException:
+            feature_info = "unknown"
         options = [
             self.name,
             f"mapped {self.mapping}" if self.mapping != "same" else "",
-            f"features {self._features}" if self.features else "full-featured",
+            f"{feature_info} features",
             f"location {self.location}",
         ]
         opt_str = ", ".join([o for o in options if o])
@@ -220,6 +350,10 @@ class Basin(abc.ABC):
     @property
     def ds(self):
         """The :class:`.RTDCBase` instance represented by the basin"""
+        if self.perishable and self.perishable.perished():
+            # We have perished. Ask the PerishableRecord to refresh this
+            # basin so we can access it again.
+            self.perishable.refresh()
         if self._ds is None:
             if not self.is_available():
                 raise BasinNotAvailableError(f"Basin {self} is not available!")
@@ -265,6 +399,7 @@ class Basin(abc.ABC):
             "basin_descr": self.description,
             "basin_feats": self.features,
             "basin_map": self.basinmap,
+            "perishable": bool(self.perishable),
         }
 
     def close(self):

{dclab-0.63.1 → dclab-0.64.0}/dclab/rtdc_dataset/fmt_dcor/api.py

@@ -11,8 +11,10 @@ class DCORAccessError(BaseException):
 
 class APIHandler:
     """Handles the DCOR api with caching for simple queries"""
-    #: these are cached to minimize network usage
-    cache_queries = ["basins", "metadata", "size", "feature_list", "valid"]
+    #: These are cached to minimize network usage
+    #: Note that we are not caching basins, since they may contain
+    #: expiring URLs.
+    cache_queries = ["metadata", "size", "feature_list", "valid"]
     #: DCOR API Keys/Tokens in the current session
     api_keys = []
 
@@ -52,8 +54,36 @@ class APIHandler:
         if api_key.strip() and api_key not in APIHandler.api_keys:
             APIHandler.api_keys.append(api_key)
 
-    def _get(self, query, feat=None, trace=None, event=None, api_key="",
-             retries=13):
+    def _get(self,
+             query: str,
+             feat: str = None,
+             trace: str = None,
+             event: str = None,
+             api_key: str = "",
+             timeout: float = None,
+             retries: int = 5):
+        """Fetch information via the DCOR API
+
+        Parameters
+        ----------
+        query: str
+            API route
+        feat: str
+            DEPRECATED (use basins instead), adds f"&feature={feat}" to query
+        trace: str
+            DEPRECATED (use basins instead), adds f"&trace={trace}" to query
+        event: str
+            DEPRECATED (use basins instead), adds f"&event={event}" to query
+        api_key: str
+            DCOR API token to use
+        timeout: float
+            Request timeout
+        retries: int
+            Number of retries to fetch the request. For every retry, the
+            timeout is increased by two seconds.
+        """
+        if timeout is None:
+            timeout = 1
         # "version=2" introduced in dclab 0.54.3
         # (supported since ckanext.dc_serve 0.13.2)
         qstr = f"&version={self.dcserv_api_version}&query={query}"
@@ -65,13 +95,13 @@ class APIHandler:
             qstr += f"&event={event}"
         apicall = self.url + qstr
         fail_reasons = []
-        for _ in range(retries):
+        for ii in range(retries):
             try:
                 # try-except both requests and json conversion
                 req = self.session.get(apicall,
                                        headers={"Authorization": api_key},
                                        verify=self.verify,
-                                       timeout=1,
+                                       timeout=timeout + ii * 2,
                                        )
                 jreq = req.json()
             except requests.urllib3.exceptions.ConnectionError:  # requests
@@ -92,13 +122,45 @@ class APIHandler:
                                   f"Messages: {fail_reasons}")
         return jreq
 
-    def get(self, query, feat=None, trace=None, event=None):
+    def get(self,
+            query: str,
+            feat: str = None,
+            trace: str = None,
+            event: str = None,
+            timeout: float = None,
+            retries: int = 5,
+            ):
+        """Fetch information from DCOR
+
+        Parameters
+        ----------
+        query: str
+            API route
+        feat: str
+            DEPRECATED (use basins instead), adds f"&feature={feat}" to query
+        trace: str
+            DEPRECATED (use basins instead), adds f"&trace={trace}" to query
+        event: str
+            DEPRECATED (use basins instead), adds f"&event={event}" to query
+        timeout: float
+            Request timeout
+        retries: int
+            Number of retries to fetch the request. For every retry, the
+            timeout is increased by two seconds.
+        """
         if query in APIHandler.cache_queries and query in self._cache:
             result = self._cache[query]
         else:
             req = {"error": {"message": "No access to API (api key?)"}}
             for api_key in [self.api_key] + APIHandler.api_keys:
-                req = self._get(query, feat, trace, event, api_key)
+                req = self._get(query=query,
+                                feat=feat,
+                                trace=trace,
+                                event=event,
+                                api_key=api_key,
+                                timeout=timeout,
+                                retries=retries,
+                                )
                 if req["success"]:
                     self.api_key = api_key  # remember working key
                     break