dclab 0.62.17__cp39-cp39-macosx_11_0_arm64.whl → 0.67.3__cp39-cp39-macosx_11_0_arm64.whl

dclab/rtdc_dataset/feat_basin.py

@@ -6,9 +6,10 @@ which, when opened in dclab, can access features stored in the input file
 from __future__ import annotations
 
 import abc
+import logging
 import numbers
 import threading
-from typing import Dict, List, Literal
+from typing import Callable, Dict, List, Literal, Union
 import uuid
 import warnings
 import weakref
@@ -18,14 +19,25 @@ import numpy as np
 from ..util import copy_if_needed
 
 
+logger = logging.getLogger(__name__)
+
+
 class BasinFeatureMissingWarning(UserWarning):
     """Used when a badin feature is defined but not stored"""
 
 
+class BasinIdentifierMismatchError(BaseException):
+    """Used when the identifier of a basin does not match the definition"""
+
+
 class CyclicBasinDependencyFoundWarning(UserWarning):
     """Used when a basin is defined in one of its sub-basins"""
 
 
+class IgnoringPerishableBasinTTL(UserWarning):
+    """Used when refreshing a basin does not support TTL"""
+
+
 class BasinmapFeatureMissingError(KeyError):
     """Used when one of the `basinmap` features is not defined"""
     pass
@@ -47,6 +59,114 @@ class BasinAvailabilityChecker(threading.Thread):
         self.basin.is_available()
 
 
+class PerishableRecord:
+    """A class containing information about perishable basins
+
+    Perishable basins are basins that may discontinue to work after
+    e.g. a specific amount of time (e.g. presigned S3 URLs). With the
+    `PerishableRecord`, these basins may be "refreshed" (made
+    available again).
+    """
+    def __init__(self,
+                 basin,
+                 expiration_func: Callable = None,
+                 expiration_kwargs: Dict = None,
+                 refresh_func: Callable = None,
+                 refresh_kwargs: Dict = None,
+                 ):
+        """
+        Parameters
+        ----------
+        basin: Basin
+            Instance of the perishable basin
+        expiration_func: callable
+            A function that determines whether the basin has perished.
+            It must accept `basin` as the first argument. Calling this
+            function should be fast, as it is called every time a feature
+            is accessed.
+            Note that if you are implementing this in the time domain, then
+            you should use `time.time()` (TSE), because you need an absolute
+            time measure. `time.monotonic()` for instance does not count up
+            when the system goes to sleep. However, keep in mind that if
+            a remote machine dictates the expiration time, then that
+            remote machine should also transmit the creation time (in case
+            there are time offsets).
+        expiration_kwargs: dict
+            Additional kwargs for `expiration_func`.
+        refresh_func: callable
+            The function used to refresh the `basin`. It must accept
+            `basin` as the first argument.
+        refresh_kwargs: dict
+            Additional kwargs for `refresh_func`
+        """
+        if not isinstance(basin, weakref.ProxyType):
+            basin = weakref.proxy(basin)
+        self.basin = basin
+        self.expiration_func = expiration_func
+        self.expiration_kwargs = expiration_kwargs or {}
+        self.refresh_func = refresh_func
+        self.refresh_kwargs = refresh_kwargs or {}
+
+    def __repr__(self):
+        state = "perished" if self.perished() else "valid"
+        return f"<PerishableRecord ({state}) at {hex(id(self))}>"
+
+    def perished(self) -> Union[bool, None]:
+        """Determine whether the basin has perished
+
+        Returns
+        -------
+        state: bool or None
+            True means the basin has perished, False means the basin
+            has not perished, and `None` means we don't know
+        """
+        if self.expiration_func is None:
+            return None
+        else:
+            return self.expiration_func(self.basin, **self.expiration_kwargs)
+
+    def refresh(self, extend_by: float = None) -> None:
+        """Extend the lifetime of the associated perishable basin
+
+        Parameters
+        ----------
+        extend_by: float
+            Custom argument for extending the life of the basin.
+            Normally, this would be a lifetime.
+
+        Returns
+        -------
+        basin: dict | None
+            Dictionary for instantiating a new basin
+        """
+        if self.refresh_func is None:
+            # The basin is a perishable basin, but we have no way of
+            # refreshing it.
+            logger.error(f"Cannot refresh basin '{self.basin}'")
+            return
+
+        if extend_by and "extend_by" not in self.refresh_kwargs:
+            warnings.warn(
+                "Parameter 'extend_by' ignored, because the basin "
+                "source does not support it",
+                IgnoringPerishableBasinTTL)
+            extend_by = None
+
+        rkw = {}
+        rkw.update(self.refresh_kwargs)
+
+        if extend_by is not None:
+            rkw["extend_by"] = extend_by
+
+        self.refresh_func(self.basin, **rkw)
+        logger.info(f"Refreshed basin '{self.basin}'")
+
+        # If everything went well, reset the current dataset of the basin
+        if self.basin._ds is not None:
+            self.basin._ds.close()
+            self.basin._ds = None
+
+
 class Basin(abc.ABC):
     """A basin represents data from an external source
 
@@ -60,7 +180,8 @@ class Basin(abc.ABC):
                  name: str = None,
                  description: str = None,
                  features: List[str] = None,
-                 measurement_identifier: str = None,
+                 referrer_identifier: str = None,
+                 basin_identifier: str = None,
                  mapping: Literal["same",
                                   "basinmap0",
                                   "basinmap1",
@@ -76,6 +197,7 @@ class Basin(abc.ABC):
                  mapping_referrer: Dict = None,
                  ignored_basins: List[str] = None,
                  key: str = None,
+                 perishable=False,
                  **kwargs):
         """
 
@@ -91,10 +213,18 @@ class Basin(abc.ABC):
         features: list of str
             List of features this basin provides; This list is enforced,
             even if the basin actually contains more features.
-        measurement_identifier: str
+        referrer_identifier: str
             A measurement identifier against which to check the basin.
+            If the basin mapping is "same", then this must match the
+            identifier of the basin exactly, otherwise it must start
+            with the basin identifier (e.g. "basin-id_referrer-sub-id").
             If this is set to None (default), there is no certainty
             that the downstream dataset is from the same measurement.
+        basin_identifier: str
+            A measurement identifier that must match the basin exactly.
+            In contrast to `referrer_identifier`, the basin identifier is
+            the identifier of the basin file. If `basin_identifier` is
+            specified, the identifier of the basin must be identical to it.
         mapping: str
             Which type of mapping to use. This can be either "same"
             when the event list of the basin is identical to that
@@ -115,6 +245,10 @@ class Basin(abc.ABC):
             Unique key to identify this basin; normally computed from
             a JSON dump of the basin definition. A random string is used
             if None is specified.
+        perishable: bool or PerishableRecord
+            If this is not False, then it must be a :class:`.PerishableRecord`
+            that holds the information about the expiration time, and that
+            comes with a method `refresh` to extend the lifetime of the basin.
         kwargs:
             Additional keyword arguments passed to the `load_dataset`
             method of the `Basin` subclass.
@@ -130,13 +264,19 @@ class Basin(abc.ABC):
         self.name = name
         #: lengthy description of the basin
         self.description = description
-        # defining key of the basin
+        # perishable record
+        if isinstance(perishable, bool) and perishable:
+            # Create an empty perishable record
+            perishable = PerishableRecord(self)
+        self.perishable = perishable
+        # define key of the basin
         self.key = key or str(uuid.uuid4())
         # features this basin provides
         self._features = features
         #: measurement identifier of the referencing dataset
-        self.measurement_identifier = measurement_identifier
-        self._measurement_identifier_verified = False
+        self.referrer_identifier = referrer_identifier
+        self.basin_identifier = basin_identifier or None
+        self._identifiers_verification = None
         #: ignored basins
         self.ignored_basins = ignored_basins or []
         #: additional keyword arguments passed to the basin
@@ -164,23 +304,28 @@ class Basin(abc.ABC):
         self._av_check.start()
 
     def __repr__(self):
+        try:
+            feature_info = len(self.features)
+        except BaseException:
+            feature_info = "unknown"
         options = [
             self.name,
             f"mapped {self.mapping}" if self.mapping != "same" else "",
-            f"features {self._features}" if self.features else "full-featured",
+            f"{feature_info} features",
             f"location {self.location}",
         ]
         opt_str = ", ".join([o for o in options if o])
 
         return f"<{self.__class__.__name__} ({opt_str}) at {hex(id(self))}>"
 
-    def _assert_measurement_identifier(self):
+    def _assert_referrer_identifier(self):
         """Make sure the basin matches the measurement identifier
         """
         if not self.verify_basin(run_identifier=True):
-            raise KeyError(f"Measurement identifier of basin {self.ds} "
-                           f"({self.get_measurement_identifier()}) does "
-                           f"not match {self.measurement_identifier}!")
+            raise BasinIdentifierMismatchError(
+                f"Measurement identifier of basin {self.ds} "
+                f"({self.get_measurement_identifier()}) does "
+                f"not match {self.referrer_identifier}!")
 
     @property
     def basinmap(self):
@@ -220,6 +365,10 @@ class Basin(abc.ABC):
     @property
     def ds(self):
         """The :class:`.RTDCBase` instance represented by the basin"""
+        if self.perishable and self.perishable.perished():
+            # We have perished. Ask the PerishableRecord to refresh this
+            # basin so we can access it again.
+            self.perishable.refresh()
         if self._ds is None:
             if not self.is_available():
                 raise BasinNotAvailableError(f"Basin {self} is not available!")
@@ -265,6 +414,7 @@ class Basin(abc.ABC):
             "basin_descr": self.description,
             "basin_feats": self.features,
             "basin_map": self.basinmap,
+            "perishable": bool(self.perishable),
         }
 
     def close(self):
@@ -275,7 +425,7 @@ class Basin(abc.ABC):
 
     def get_feature_data(self, feat):
         """Return an object representing feature data of the basin"""
-        self._assert_measurement_identifier()
+        self._assert_referrer_identifier()
         return self.ds[feat]
 
     def get_measurement_identifier(self):
@@ -320,26 +470,44 @@ class Basin(abc.ABC):
         # Only check for run identifier if requested and if the availability
         # check did not fail.
         if run_identifier and check_avail:
-            if not self._measurement_identifier_verified:
-                if self.measurement_identifier is None:
+            if self._identifiers_verification is None:
+                # This is the measurement identifier of the basin.
+                basin_identifier = self.get_measurement_identifier()
+
+                # Perform a sanity check for the basin identifier.
+                if (self.basin_identifier
+                        and self.basin_identifier != basin_identifier):
+                    # We should not proceed any further with this basin.
+                    self._identifiers_verification = False
+                    warnings.warn(
+                        f"Basin identifier mismatch for {self}. Expected "
+                        f"'{self.basin_identifier}', got '{basin_identifier}'")
+
+                if self.referrer_identifier is None:
                     # No measurement identifier was presented by the
-                    # referencing dataset. Don't perform any checks.
-                    self._measurement_identifier_verified = True
+                    # referencing dataset. We are in the dark.
+                    # Don't perform any checks.
+                    self._identifiers_verification = True
                 else:
-                    if self.mapping == "same":
-                        # When we have identical mapping, then the measurement
-                        # identifier has to match exactly.
-                        verifier = str.__eq__
+                    if basin_identifier is None:
+                        # Again, we are in the dark, because the basin dataset
+                        # does not have an identifier. This is an undesirable
+                        # situation, but there is nothing we can do about it.
+                        self._identifiers_verification = True
                     else:
-                        # When we have non-identical mapping (e.g. exported
-                        # data), then the measurement identifier has to
-                        # partially match.
-                        verifier = str.startswith
-                    self._measurement_identifier_verified = verifier(
-                        self.measurement_identifier,
-                        self.get_measurement_identifier()
-                    )
-            check_rid = self._measurement_identifier_verified
+                        if self.mapping == "same":
+                            # When we have identical mapping, then the
+                            # measurement identifier has to match exactly.
+                            verifier = str.__eq__
+                        else:
+                            # When we have non-identical mapping (e.g. exported
+                            # data), then the measurement identifier has to
+                            # partially match.
+                            verifier = str.startswith
+                        self._identifiers_verification = verifier(
+                            self.referrer_identifier, basin_identifier)
+
+            check_rid = self._identifiers_verification
         else:
             check_rid = True
 
@@ -403,8 +571,12 @@ class BasinProxy:
 
     def __getitem__(self, feat):
         if feat not in self._features:
-            feat_obj = BasinProxyFeature(feat_obj=self.ds[feat],
-                                         basinmap=self.basinmap)
+            if feat == "contour":
+                feat_obj = BasinProxyContour(feat_obj=self.ds[feat],
+                                             basinmap=self.basinmap)
+            else:
+                feat_obj = BasinProxyFeature(feat_obj=self.ds[feat],
+                                             basinmap=self.basinmap)
             self._features[feat] = feat_obj
         return self._features[feat]
 
@@ -412,14 +584,61 @@ class BasinProxy:
         return len(self.basinmap)
 
 
+class BasinProxyContour:
+    def __init__(self, feat_obj, basinmap):
+        """Wrap around a contour, mapping it upon data access, no caching"""
+        self.feat_obj = feat_obj
+        self.basinmap = basinmap
+        self.is_scalar = False
+        self.shape = (len(self.basinmap), np.nan, 2)
+        self.identifier = feat_obj.identifier
+
+    def __getattr__(self, item):
+        if item in [
+            "dtype",
+        ]:
+            return getattr(self.feat_obj, item)
+        else:
+            raise AttributeError(
+                f"BasinProxyContour does not implement {item}")
+
+    def __getitem__(self, index):
+        if isinstance(index, numbers.Integral):
+            # single index, cheap operation
+            return self.feat_obj[self.basinmap[index]]
+        else:
+            raise NotImplementedError(
+                "Cannot index contours without anything else than integers.")
+
+    def __len__(self):
+        return self.shape[0]
+
+
 class BasinProxyFeature(np.lib.mixins.NDArrayOperatorsMixin):
     def __init__(self, feat_obj, basinmap):
         """Wrap around a feature object, mapping it upon data access"""
         self.feat_obj = feat_obj
         self.basinmap = basinmap
         self._cache = None
+        self._shape = None
+        self._size = None
         self.is_scalar = bool(len(self.feat_obj.shape) == 1)
 
+    @property
+    def shape(self):
+        if self._shape is None:
+            if self.is_scalar:
+                self._shape = self.basinmap.shape
+            else:
+                self._shape = (self.basinmap.size,) + self.feat_obj.shape[1:]
+        return self._shape
+
+    @property
+    def size(self):
+        if self._size is None:
+            self._size = np.prod(self.shape)
+        return self._size
+
     def __array__(self, dtype=None, copy=copy_if_needed, *args, **kwargs):
         if self._cache is None and self.is_scalar:
             self._cache = self.feat_obj[:][self.basinmap]
@@ -436,8 +655,6 @@ class BasinProxyFeature(np.lib.mixins.NDArrayOperatorsMixin):
     def __getattr__(self, item):
         if item in [
             "dtype",
-            "shape",
-            "size",
         ]:
             return getattr(self.feat_obj, item)
         else:

dclab/rtdc_dataset/fmt_dcor/api.py

@@ -11,7 +11,9 @@ class DCORAccessError(BaseException):
 
 class APIHandler:
     """Handles the DCOR api with caching for simple queries"""
-    #: these are cached to minimize network usage
+    #: These are cached to minimize network usage
+    #: Note that we are not caching basins, since they may contain
+    #: expiring URLs.
     cache_queries = ["metadata", "size", "feature_list", "valid"]
     #: DCOR API Keys/Tokens in the current session
     api_keys = []
@@ -52,8 +54,36 @@ class APIHandler:
         if api_key.strip() and api_key not in APIHandler.api_keys:
             APIHandler.api_keys.append(api_key)
 
-    def _get(self, query, feat=None, trace=None, event=None, api_key="",
-             retries=13):
+    def _get(self,
+             query: str,
+             feat: str = None,
+             trace: str = None,
+             event: str = None,
+             api_key: str = "",
+             timeout: float = None,
+             retries: int = 5):
+        """Fetch information via the DCOR API
+
+        Parameters
+        ----------
+        query: str
+            API route
+        feat: str
+            DEPRECATED (use basins instead), adds f"&feature={feat}" to query
+        trace: str
+            DEPRECATED (use basins instead), adds f"&trace={trace}" to query
+        event: str
+            DEPRECATED (use basins instead), adds f"&event={event}" to query
+        api_key: str
+            DCOR API token to use
+        timeout: float
+            Request timeout
+        retries: int
+            Number of retries to fetch the request. For every retry, the
+            timeout is increased by two seconds.
+        """
+        if timeout is None:
+            timeout = 1
         # "version=2" introduced in dclab 0.54.3
         # (supported since ckanext.dc_serve 0.13.2)
         qstr = f"&version={self.dcserv_api_version}&query={query}"
@@ -65,13 +95,13 @@ class APIHandler:
             qstr += f"&event={event}"
         apicall = self.url + qstr
         fail_reasons = []
-        for _ in range(retries):
+        for ii in range(retries):
             try:
                 # try-except both requests and json conversion
                 req = self.session.get(apicall,
                                        headers={"Authorization": api_key},
                                        verify=self.verify,
-                                       timeout=1,
+                                       timeout=timeout + ii * 2,
                                        )
                 jreq = req.json()
             except requests.urllib3.exceptions.ConnectionError:  # requests
@@ -92,13 +122,45 @@ class APIHandler:
                                   f"Messages: {fail_reasons}")
         return jreq
 
-    def get(self, query, feat=None, trace=None, event=None):
+    def get(self,
+            query: str,
+            feat: str = None,
+            trace: str = None,
+            event: str = None,
+            timeout: float = None,
+            retries: int = 5,
+            ):
+        """Fetch information from DCOR
+
+        Parameters
+        ----------
+        query: str
+            API route
+        feat: str
+            DEPRECATED (use basins instead), adds f"&feature={feat}" to query
+        trace: str
+            DEPRECATED (use basins instead), adds f"&trace={trace}" to query
+        event: str
+            DEPRECATED (use basins instead), adds f"&event={event}" to query
+        timeout: float
+            Request timeout
+        retries: int
+            Number of retries to fetch the request. For every retry, the
+            timeout is increased by two seconds.
+        """
         if query in APIHandler.cache_queries and query in self._cache:
             result = self._cache[query]
         else:
             req = {"error": {"message": "No access to API (api key?)"}}
             for api_key in [self.api_key] + APIHandler.api_keys:
-                req = self._get(query, feat, trace, event, api_key)
+                req = self._get(query=query,
+                                feat=feat,
+                                trace=trace,
+                                event=event,
+                                api_key=api_key,
+                                timeout=timeout,
+                                retries=retries,
+                                )
                 if req["success"]:
                     self.api_key = api_key  # remember working key
                     break

dclab/rtdc_dataset/fmt_dcor/base.py

@@ -1,17 +1,23 @@
 """DCOR client interface"""
+import logging
 import pathlib
 import re
+import time
 
 from ...util import hashobj
 
 from ..config import Configuration
 from ..core import RTDCBase
+from ..feat_basin import PerishableRecord
 
 from . import api
 from .logs import DCORLogs
 from .tables import DCORTables
 
 
+logger = logging.getLogger(__name__)
+
+
 #: Append directories here where dclab should look for certificate bundles
 #: for a specific host. The directory should contain files named after the
 #: hostname, e.g. "dcor.mpl.mpg.de.cert".
@@ -73,6 +79,8 @@ class RTDC_DCOR(RTDCBase):
         super(RTDC_DCOR, self).__init__(*args, **kwargs)
 
         self._hash = None
+        self._cache_basin_dict = None
+        self.cache_basin_dict_time = 600
         self.path = RTDC_DCOR.get_full_url(url, use_ssl, host)
 
         if cert_path is None:
@@ -161,15 +169,106 @@ class RTDC_DCOR(RTDCBase):
         new_url = f"{scheme}://{netloc}/{api_path}"
         return new_url
 
-    def basins_get_dicts(self):
-        """Return list of dicts for all basins defined in `self.h5file`"""
+    def _basin_refresh(self, basin):
+        """Refresh the specified basin"""
+        # Retrieve the basin dictionary from DCOR
+        basin_dicts = self.basins_get_dicts()
+        for bn_dict in basin_dicts:
+            if bn_dict.get("name") == basin.name:
+                break
+        else:
+            raise ValueError(f"Basin '{basin.name}' not found in {self}")
+
+        tre = bn_dict["time_request"]
+        ttl = bn_dict["time_expiration"]
+        # remember time relative to time.time, subtract 30s to be on safe side
+        tex = bn_dict["time_local_request"] + (ttl - tre) - 30
+
+        if isinstance(basin.perishable, bool):
+            logger.debug("Initializing basin perishable %s", basin.name)
+            # create a perishable record
+            basin.perishable = PerishableRecord(
+                basin=basin,
+                expiration_func=self._basin_expiration,
+                expiration_kwargs={"time_local_expiration": tex},
+                refresh_func=self._basin_refresh,
+            )
+        else:
+            logger.debug("Refreshing basin perishable %s", basin.name)
+            # only update (this also works with weakref.ProxyType)
+            basin.perishable.expiration_kwargs = {"time_local_expiration": tex}
+
+        if len(bn_dict["urls"]) > 1:
+            logger.warning(f"Basin {basin.name} has multiple URLs. I am not "
+                           f"checking their availability: {bn_dict}")
+        basin.location = bn_dict["urls"][0]
+
+    def _basin_expiration(self, basin, time_local_expiration):
+        """Check whether the basin has perished"""
+        return time_local_expiration < time.time()
+
+    def _basins_get_dicts(self):
         try:
-            basins = self.api.get(query="basins")
+            basin_dicts = self.api.get(query="basins")
+            # Fill in missing timing information
+            for bn_dict in basin_dicts:
+                if (bn_dict.get("format") == "http"
+                        and "perishable" not in bn_dict):
+                    # We are communicating with an older version of
+                    # ckanext-dc_serve. Take a look at the URL and check
+                    # whether we have a perishable (~1 hour) URL or whether
+                    # this is a public resource.
+                    expires_regexp = re.compile(".*expires=([0-9]*)$")
+                    for url in bn_dict.get("urls", []):
+                        if match := expires_regexp.match(url.lower()):
+                            logger.debug("Detected perishable basin: %s",
+                                         bn_dict["name"])
+                            bn_dict["perishable"] = True
+                            bn_dict["time_request"] = time.time()
+                            bn_dict["time_expiration"] = int(match.group(1))
+                            # add part of the resource ID to the name
+                            infourl = url.split(bn_dict["name"], 1)[-1]
+                            infourl = infourl.replace("/", "")
+                            bn_dict["name"] += f"-{infourl[:5]}"
+                            break
+                    else:
+                        bn_dict["perishable"] = False
+                # If we have a perishable basin, add the local request time
+                if bn_dict.get("perishable"):
+                    bn_dict["time_local_request"] = time.time()
         except api.DCORAccessError:
             # TODO: Do not catch this exception when all DCOR instances
             #       implement the 'basins' query.
             # This means that the server does not implement the 'basins' query.
-            basins = []
+            basin_dicts = []
+        return basin_dicts
+
+    def basins_get_dicts(self):
+        """Return list of dicts for all basins defined on DCOR
+
+        The return value of this method is cached for 10 minutes
+        (cache time defined in the `cache_basin_dict_time` [s] property).
+        """
+        if (self._cache_basin_dict is None
+            or time.time() > (self._cache_basin_dict[1]
+                              + self.cache_basin_dict_time)):
+            self._cache_basin_dict = (self._basins_get_dicts(), time.time())
+        return self._cache_basin_dict[0]
+
+    def basins_retrieve(self):
+        """Same as superclass, but add perishable information"""
+        basin_dicts = self.basins_get_dicts()
+        basins = super(RTDC_DCOR, self).basins_retrieve()
+        for bn in basins:
+            for bn_dict in basin_dicts:
+                if bn.name == bn_dict.get("name"):
+                    # Determine whether we have to set a perishable record.
+                    if bn_dict.get("perishable"):
+                        # required for `_basin_refresh` to create a record
+                        bn.perishable = True
+                        # create the actual record
+                        self._basin_refresh(bn)
+                    break
         return basins