dclab 0.67.0__cp314-cp314t-macosx_11_0_arm64.whl

dclab/rtdc_dataset/feat_temp.py

@@ -0,0 +1,102 @@
+"""
+.. versionadded:: 0.33.0
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+import numpy as np
+
+from ..definitions import feat_logic
+
+from .core import RTDCBase
+from .fmt_hierarchy import RTDC_Hierarchy, map_indices_child2root
+
+
+_registered_temporary_features = []
+
+
+def deregister_all():
+    """Deregisters all temporary features"""
+    for feat in list(_registered_temporary_features):
+        deregister_temporary_feature(feat)
+
+
+def deregister_temporary_feature(feature: str):
+    """Convenience function for deregistering a temporary feature
+
+    This method is mostly used during testing. It does not
+    remove the actual feature data from any dataset; the data
+    will stay in memory but is not accessible anymore through
+    the public methods of the :class:`RTDCBase` user interface.
+    """
+    if feature in _registered_temporary_features:
+        _registered_temporary_features.remove(feature)
+        feat_logic.feature_deregister(feature)
+
+
+def register_temporary_feature(feature: str,
+                               label: Optional[str] = None,
+                               is_scalar: bool = True):
+    """Register a new temporary feature
+
+    Temporary features are custom features that can be defined ad hoc
+    by the user. Temporary features are helpful when the integral
+    features are not enough, e.g. for prototyping, testing, or
+    collating with other data. Temporary features allow you to
+    leverage the full functionality of :class:`RTDCBase` with
+    your custom features (no need to go for a custom `pandas.Dataframe`).
+
+    Parameters
+    ----------
+    feature: str
+        Feature name; allowed characters are lower-case letters,
+        digits, and underscores
+    label: str
+        Feature label used e.g. for plotting
+    is_scalar: bool
+        Whether or not the feature is a scalar feature
+    """
+    feat_logic.feature_register(feature, label, is_scalar)
+    _registered_temporary_features.append(feature)
+
+
+def set_temporary_feature(rtdc_ds: RTDCBase,
+                          feature: str,
+                          data: np.ndarray):
+    """Set temporary feature data for a dataset
+
+    Parameters
+    ----------
+    rtdc_ds: dclab.RTDCBase
+        Dataset for which to set the feature. Note that the
+        length of the feature `data` must match the number of events
+        in `rtdc_ds`. If the dataset is a hierarchy child, the data will also
+        be set in the parent dataset, but only for those events that are part
+        of the child. For all events in the parent dataset that are not part
+        of the child dataset, the temporary feature is set to np.nan.
+    feature: str
+        Feature name
+    data: np.ndarray
+        The data
+    """
+    if not feat_logic.feature_exists(feature):
+        raise ValueError(
+            f"Temporary feature '{feature}' has not been registered!")
+    if len(data) != len(rtdc_ds):
+        raise ValueError(f"The temporary feature {feature} must have same "
+                         f"length as the dataset. Expected length "
+                         f"{len(rtdc_ds)}, got length {len(data)}!")
+    if isinstance(rtdc_ds, RTDC_Hierarchy):
+        root_ids = map_indices_child2root(rtdc_ds, np.arange(len(rtdc_ds)))
+        root_parent = rtdc_ds.get_root_parent()
+        root_feat_data = np.empty((len(root_parent)))
+        root_feat_data[:] = np.nan
+        root_feat_data[root_ids] = data
+        set_temporary_feature(root_parent, feature, root_feat_data)
+        rtdc_ds.rejuvenate()
+    else:
+        feat_logic.check_feature_shape(feature, data)
+        data_ro = data.view()
+        data_ro.setflags(write=False)
+        rtdc_ds._usertemp[feature] = data_ro

dclab/rtdc_dataset/filter.py

@@ -0,0 +1,263 @@
+"""RT-DC dataset core classes and methods"""
+
+import warnings
+
+import numpy as np
+
+from dclab import definitions as dfn
+
+from .. import downsampling
+from ..polygon_filter import PolygonFilter
+
+
+class NanWarning(UserWarning):
+    pass
+
+
+class Filter(object):
+    def __init__(self, rtdc_ds):
+        """Boolean filter arrays for RT-DC measurements
+
+        Parameters
+        ----------
+        rtdc_ds: instance of RTDCBase
+            The RT-DC dataset the filter applies to
+        """
+        # dictionary of boolean array for box filters
+        self._box_filters = {}
+        # dictionary of (hash, boolean array) for polygon filters
+        self._poly_filters = {}
+        # dictionary of all internal property filters
+        self._array_props = {}
+        # initialize important parameters
+        self._init_rtdc_ds(rtdc_ds)
+        # initialize properties
+        self.reset()
+
+    def __getitem__(self, key):
+        """Return the filter for a feature in `self.features`"""
+        if key in self.features and dfn.scalar_feature_exists(key):
+            if key not in self._box_filters:
+                # Generate filters on-the-fly
+                self._box_filters[key] = np.ones(self.size, dtype=bool)
+        else:
+            raise KeyError("Feature not available: '{}'".format(key))
+        return self._box_filters[key]
+
+    @property
+    def all(self):
+        """All filters combined (see :func:`Filter.update`)
+
+        Use this property to filter the features of
+        :class:`dclab.rtdc_dataset.RTDCBase` instances
+        """
+        return self._get_ro_array("all")
+
+    @property
+    def box(self):
+        """All box filters"""
+        return self._get_ro_array("box")
+
+    @property
+    def polygon(self):
+        """Polygon filters"""
+        return self._get_ro_array("polygon")
+
+    @property
+    def invalid(self):
+        """Invalid (nan/inf) events"""
+        return self._get_ro_array("invalid")
+
+    def _get_ro_array(self, key):
+        view = self._get_rw_array(key).view()
+        view.flags.writeable = False
+        return view
+
+    def _get_rw_array(self, key):
+        if key not in self._array_props:
+            self._array_props[key] = np.ones(self.size, dtype=bool)
+        return self._array_props[key]
+
+    def _init_rtdc_ds(self, rtdc_ds):
+        #: Available feature names
+        self.features = rtdc_ds.features_scalar
+        if hasattr(self, "size") and self.size != len(rtdc_ds):
+            raise ValueError("Change of RTDCBase size not supported!")
+        self.size = len(rtdc_ds)
+        # determine box filters that have been removed
+        for key in list(self._box_filters.keys()):
+            if key not in self.features:
+                self._box_filters.pop(key)
+        # determine polygon filters that have been removed
+        for pf_id in list(self._poly_filters.keys()):
+            pf = PolygonFilter.get_instance_from_id(pf_id)
+            if (pf_id in rtdc_ds.config["filtering"]["polygon filters"]
+                and pf.axes[0] in self.features
+                    and pf.axes[1] in self.features):
+                pass
+            else:
+                # filter has been removed
+                self._poly_filters.pop(pf_id)
+
+    def reset(self):
+        """Reset all filters"""
+        self._box_filters.clear()
+        self._poly_filters.clear()
+        self._array_props.clear()
+        #: 1D boolean array for manually excluding events; `False` values
+        #: are excluded.
+        self.manual = np.ones(self.size, dtype=bool)
+        # old filter configuration of `rtdc_ds`
+        self._old_config = {}
+
+    def update(self, rtdc_ds, force=None):
+        """Update the filters according to `rtdc_ds.config["filtering"]`
+
+        Parameters
+        ----------
+        rtdc_ds: dclab.rtdc_dataset.core.RTDCBase
+            The measurement to which the filter is applied
+        force : list
+            A list of feature names that must be refiltered with
+            min/max values.
+
+        Notes
+        -----
+        This function is called when
+        :func:`ds.apply_filter <dclab.rtdc_dataset.RTDCBase.apply_filter>`
+        is called.
+        """
+        if force is None:
+            force = []
+        # re-initialize important parameters
+        self._init_rtdc_ds(rtdc_ds)
+
+        # These lists may help us become very fast in the future
+        newkeys = []
+        oldvals = []
+        newvals = []
+
+        cfg_cur = rtdc_ds.config["filtering"]
+        cfg_old = self._old_config
+
+        # Determine which data was updated
+        for skey in list(cfg_cur.keys()):
+            if cfg_cur[skey] != cfg_old.get(skey, None):
+                newkeys.append(skey)
+                oldvals.append(cfg_old.get(skey, None))
+                newvals.append(cfg_cur[skey])
+
+        # 1. Invalid filters
+        arr_invalid = self._get_rw_array("invalid")
+        arr_invalid[:] = True
+        if cfg_cur["remove invalid events"]:
+            for feat in self.features:
+                data = rtdc_ds[feat]
+                invalid = np.isinf(data) | np.isnan(data)
+                arr_invalid &= ~invalid
+
+        # 2. Filter all feature min/max values.
+        feat2filter = []
+        for k in newkeys:
+            # k[:-4] because we want to crop " min" and " max"
+            if (dfn.scalar_feature_exists(k[:-4])
+                    and (k.endswith(" min") or k.endswith(" max"))):
+                feat2filter.append(k[:-4])
+
+        for f in force:
+            # add forced features
+            if dfn.scalar_feature_exists(f):
+                feat2filter.append(f)
+            else:
+                # Make sure the feature name is valid.
+                raise ValueError("Unknown scalar feature name '{}'!".format(f))
+
+        feat2filter = np.unique(feat2filter)
+
+        for feat in feat2filter:
+            fstart = feat + " min"
+            fend = feat + " max"
+            must_be_filtered = (fstart in cfg_cur
+                                and fend in cfg_cur
+                                and cfg_cur[fstart] != cfg_cur[fend])
+            if ((fstart in cfg_cur and fend not in cfg_cur)
+                    or (fstart not in cfg_cur and fend in cfg_cur)):
+                # User is responsible for setting min and max values!
+                raise ValueError("Box filter: Please make sure that both "
+                                 "'{}' and '{}' are set!".format(fstart, fend))
+            if feat in self.features:
+                # Get the current feature filter
+                feat_filt = self[feat]
+                feat_filt[:] = True
+                # If min and max exist and if they are not identical:
+                if must_be_filtered:
+                    ivalstart = cfg_cur[fstart]
+                    ivalend = cfg_cur[fend]
+                    if ivalstart > ivalend:
+                        msg = "inverting filter: {} > {}".format(fstart, fend)
+                        warnings.warn(msg)
+                        ivalstart, ivalend = ivalend, ivalstart
+                    data = rtdc_ds[feat]
+                    # treat nan-values in a special way
+                    disnan = np.isnan(data)
+                    if np.sum(disnan):
+                        # this avoids RuntimeWarnings (invalid value
+                        # encountered due to nan-values)
+                        feat_filt[disnan] = False
+                        idx = ~disnan
+                        if not cfg_cur["remove invalid events"]:
+                            msg = "Feature '{}' contains ".format(feat) \
+                                  + "nan-values! Box filters remove those."
+                            warnings.warn(msg, NanWarning)
+                    else:
+                        idx = slice(0, self.size)  # place-holder for [:]
+                    feat_filt[idx] &= ivalstart <= data[idx]
+                    feat_filt[idx] &= data[idx] <= ivalend
+            elif must_be_filtered:
+                warnings.warn("Dataset '{}' does ".format(rtdc_ds.identifier)
+                              + "not contain the feature '{}'! ".format(feat)
+                              + "A box filter has been ignored.")
+        # store box filters
+        arr_box = self._get_rw_array("box")
+        arr_box[:] = True
+        for feat in self._box_filters:
+            arr_box &= self._box_filters[feat]
+
+        # 3. Filter with polygon filters
+        # check if something has changed
+        # perform polygon filtering
+        for pf_id in cfg_cur["polygon filters"]:
+            pf = PolygonFilter.get_instance_from_id(pf_id)
+            if (pf_id not in self._poly_filters
+                    or pf.hash != self._poly_filters[pf_id][0]):
+                datax = rtdc_ds[pf.axes[0]]
+                datay = rtdc_ds[pf.axes[1]]
+                self._poly_filters[pf_id] = (pf.hash, pf.filter(datax, datay))
+        # store polygon filters
+        arr_polygon = self._get_rw_array("polygon")
+        arr_polygon[:] = True
+        for pf_id in self._poly_filters:
+            arr_polygon &= self._poly_filters[pf_id][1]
+
+        # 4. Finally combine all filters and apply "limit events"
+        # get a list of all filters
+        arr_all = self._get_rw_array("all")
+        if cfg_cur["enable filters"]:
+            arr_all[:] = arr_box & arr_invalid & arr_polygon & self.manual
+
+            # Filter with configuration keyword argument "limit events".
+            # This additional step limits the total number of events in
+            # self.all.
+            if cfg_cur["limit events"] > 0:
+                limit = cfg_cur["limit events"]
+                sub = arr_all[arr_all]
+                _, idx = downsampling.downsample_rand(sub,
+                                                      samples=limit,
+                                                      ret_idx=True)
+                sub[~idx] = False
+                arr_all[arr_all] = sub
+        else:
+            arr_all[:] = True
+
+        # Actual filtering is then done during plotting
+        self._old_config = rtdc_ds.config.copy()["filtering"]

dclab/rtdc_dataset/fmt_dcor/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa: F401
+"""DCOR client interface"""
+from .api import REQUESTS_AVAILABLE
+from .base import (
+    DCOR_CERTS_SEARCH_PATHS, RTDC_DCOR, get_server_cert_path, is_dcor_url
+)
+from .basin import DCORBasin

dclab/rtdc_dataset/fmt_dcor/access_token.py

@@ -0,0 +1,52 @@
+"""DCOR-med access token (SSL certificate + CKAN token)"""
+import pathlib
+import ssl
+import tempfile
+import zipfile
+
+
+def get_api_key(access_token_path, password):
+    """Extract the API key / API token from an encrypted DCOR access token"""
+    if isinstance(password, str):
+        password = password.encode("utf-8")
+    with zipfile.ZipFile(access_token_path) as arc:
+        api_key = arc.read("api_key.txt", pwd=password).decode().strip()
+    return api_key
+
+
+def get_certificate(access_token_path, password):
+    """Extract the certificate bundle from an encrypted DCOR access token"""
+    if isinstance(password, str):
+        password = password.encode("utf-8")
+    with zipfile.ZipFile(access_token_path) as arc:
+        cert_data = arc.read("server.cert", pwd=password)
+    return cert_data
+
+
+def get_hostname(access_token_path, password):
+    """Extract the hostname from an encrypted DCOR access token"""
+    cert_data = get_certificate(access_token_path, password)
+    with tempfile.TemporaryDirectory(prefix="dcoraid_access_token_") as td:
+        cfile = pathlib.Path(td) / "server.cert"
+        cfile.write_bytes(cert_data)
+        # Dear future-self,
+        #
+        # I know that this will probably not have been a good solution.
+        # Anyway, I still decided to use this private function from the
+        # built-in ssh module to avoid additional dependencies. Just so
+        # you know: If you happen to be in trouble now because of this,
+        # bear in mind that you had enough time to at least ask for the
+        # functionality to be implemented in the requests library. Look
+        # how I kept the lines all the same length!
+        #
+        # Cheers,
+        # Paul
+        cert_dict = ssl._ssl._test_decode_cert(str(cfile))
+    # get the common name
+    for ((key, value),) in cert_dict["subject"]:
+        if key == "commonName":
+            hostname = value.strip()
+            break
+    else:
+        raise KeyError("Could not extract hostname from certificate!")
+    return hostname

dclab/rtdc_dataset/fmt_dcor/api.py

@@ -0,0 +1,173 @@
+import json
+import time
+
+from ...http_utils import REQUESTS_AVAILABLE  # noqa: F401
+from ...http_utils import requests, session_cache
+
+
+class DCORAccessError(BaseException):
+    pass
+
+
+class APIHandler:
+    """Handles the DCOR api with caching for simple queries"""
+    #: These are cached to minimize network usage
+    #: Note that we are not caching basins, since they may contain
+    #: expiring URLs.
+    cache_queries = ["metadata", "size", "feature_list", "valid"]
+    #: DCOR API Keys/Tokens in the current session
+    api_keys = []
+
+    def __init__(self, url, api_key="", cert_path=None, dcserv_api_version=2):
+        """
+
+        Parameters
+        ----------
+        url: str
+            URL to DCOR API
+        api_key: str
+            DCOR API token
+        cert_path: pathlib.Path
+            the path to the server's CA bundle; by default this
+            will use the default certificates (which depends on
+            from where you obtained certifi/requests)
+        """
+        #: DCOR API URL
+        self.url = url
+        #: keyword argument to :func:`requests.request`
+        self.verify = cert_path or True
+        #: DCOR API Token
+        self.api_key = api_key
+        #: ckanext-dc_serve dcserv API version
+        self.dcserv_api_version = dcserv_api_version
+        #: create a session
+        self.session = session_cache.get_session(url)
+        self._cache = {}
+
+    @classmethod
+    def add_api_key(cls, api_key):
+        """Add an API Key/Token to the base class
+
+        When accessing the DCOR API, all available API Keys/Tokens are
+        used to access a resource (trial and error).
+        """
+        if api_key.strip() and api_key not in APIHandler.api_keys:
+            APIHandler.api_keys.append(api_key)
+
+    def _get(self,
+             query: str,
+             feat: str = None,
+             trace: str = None,
+             event: str = None,
+             api_key: str = "",
+             timeout: float = None,
+             retries: int = 5):
+        """Fetch information via the DCOR API
+
+        Parameters
+        ----------
+        query: str
+            API route
+        feat: str
+            DEPRECATED (use basins instead), adds f"&feature={feat}" to query
+        trace: str
+            DEPRECATED (use basins instead), adds f"&trace={trace}" to query
+        event: str
+            DEPRECATED (use basins instead), adds f"&event={event}" to query
+        api_key: str
+            DCOR API token to use
+        timeout: float
+            Request timeout
+        retries: int
+            Number of retries to fetch the request. For every retry, the
+            timeout is increased by two seconds.
+        """
+        if timeout is None:
+            timeout = 1
+        # "version=2" introduced in dclab 0.54.3
+        # (supported since ckanext.dc_serve 0.13.2)
+        qstr = f"&version={self.dcserv_api_version}&query={query}"
+        if feat is not None:
+            qstr += f"&feature={feat}"
+        if trace is not None:
+            qstr += f"&trace={trace}"
+        if event is not None:
+            qstr += f"&event={event}"
+        apicall = self.url + qstr
+        fail_reasons = []
+        for ii in range(retries):
+            try:
+                # try-except both requests and json conversion
+                req = self.session.get(apicall,
+                                       headers={"Authorization": api_key},
+                                       verify=self.verify,
+                                       timeout=timeout + ii * 2,
+                                       )
+                jreq = req.json()
+            except requests.urllib3.exceptions.ConnectionError:  # requests
+                fail_reasons.append("connection problem")
+                continue
+            except (requests.urllib3.exceptions.ReadTimeoutError,
+                    requests.exceptions.ConnectTimeout):  # requests
+                fail_reasons.append("timeout")
+            except json.decoder.JSONDecodeError:  # json
+                fail_reasons.append("invalid json")
+                time.sleep(1)  # wait a bit, maybe the server is overloaded
+                continue
+            else:
+                break
+        else:
+            raise DCORAccessError(f"Could not complete query '{apicall}'. "
+                                  f"I retried {retries} times. "
+                                  f"Messages: {fail_reasons}")
+        return jreq
+
+    def get(self,
+            query: str,
+            feat: str = None,
+            trace: str = None,
+            event: str = None,
+            timeout: float = None,
+            retries: int = 5,
+            ):
+        """Fetch information from DCOR
+
+        Parameters
+        ----------
+        query: str
+            API route
+        feat: str
+            DEPRECATED (use basins instead), adds f"&feature={feat}" to query
+        trace: str
+            DEPRECATED (use basins instead), adds f"&trace={trace}" to query
+        event: str
+            DEPRECATED (use basins instead), adds f"&event={event}" to query
+        timeout: float
+            Request timeout
+        retries: int
+            Number of retries to fetch the request. For every retry, the
+            timeout is increased by two seconds.
+        """
+        if query in APIHandler.cache_queries and query in self._cache:
+            result = self._cache[query]
+        else:
+            req = {"error": {"message": "No access to API (api key?)"}}
+            for api_key in [self.api_key] + APIHandler.api_keys:
+                req = self._get(query=query,
+                                feat=feat,
+                                trace=trace,
+                                event=event,
+                                api_key=api_key,
+                                timeout=timeout,
+                                retries=retries,
+                                )
+                if req["success"]:
+                    self.api_key = api_key  # remember working key
+                    break
+            else:
+                raise DCORAccessError(
+                    f"Cannot access {query}: {req['error']['message']}")
+            result = req["result"]
+            if query in APIHandler.cache_queries:
+                self._cache[query] = result
+        return result