dclab 0.62.11__cp313-cp313-win_amd64.whl

dclab/rtdc_dataset/filter.py

@@ -0,0 +1,263 @@
+"""RT-DC dataset core classes and methods"""
+
+import warnings
+
+import numpy as np
+
+from dclab import definitions as dfn
+
+from .. import downsampling
+from ..polygon_filter import PolygonFilter
+
+
+class NanWarning(UserWarning):
+    pass
+
+
+class Filter(object):
+    def __init__(self, rtdc_ds):
+        """Boolean filter arrays for RT-DC measurements
+
+        Parameters
+        ----------
+        rtdc_ds: instance of RTDCBase
+            The RT-DC dataset the filter applies to
+        """
+        # dictionary of boolean array for box filters
+        self._box_filters = {}
+        # dictionary of (hash, boolean array) for polygon filters
+        self._poly_filters = {}
+        # dictionary of all internal property filters
+        self._array_props = {}
+        # initialize important parameters
+        self._init_rtdc_ds(rtdc_ds)
+        # initialize properties
+        self.reset()
+
+    def __getitem__(self, key):
+        """Return the filter for a feature in `self.features`"""
+        if key in self.features and dfn.scalar_feature_exists(key):
+            if key not in self._box_filters:
+                # Generate filters on-the-fly
+                self._box_filters[key] = np.ones(self.size, dtype=bool)
+        else:
+            raise KeyError("Feature not available: '{}'".format(key))
+        return self._box_filters[key]
+
+    @property
+    def all(self):
+        """All filters combined (see :func:`Filter.update`)
+
+        Use this property to filter the features of
+        :class:`dclab.rtdc_dataset.RTDCBase` instances
+        """
+        return self._get_ro_array("all")
+
+    @property
+    def box(self):
+        """All box filters"""
+        return self._get_ro_array("box")
+
+    @property
+    def polygon(self):
+        """Polygon filters"""
+        return self._get_ro_array("polygon")
+
+    @property
+    def invalid(self):
+        """Invalid (nan/inf) events"""
+        return self._get_ro_array("invalid")
+
+    def _get_ro_array(self, key):
+        view = self._get_rw_array(key).view()
+        view.flags.writeable = False
+        return view
+
+    def _get_rw_array(self, key):
+        if key not in self._array_props:
+            self._array_props[key] = np.ones(self.size, dtype=bool)
+        return self._array_props[key]
+
+    def _init_rtdc_ds(self, rtdc_ds):
+        #: Available feature names
+        self.features = rtdc_ds.features_scalar
+        if hasattr(self, "size") and self.size != len(rtdc_ds):
+            raise ValueError("Change of RTDCBase size not supported!")
+        self.size = len(rtdc_ds)
+        # determine box filters that have been removed
+        for key in list(self._box_filters.keys()):
+            if key not in self.features:
+                self._box_filters.pop(key)
+        # determine polygon filters that have been removed
+        for pf_id in list(self._poly_filters.keys()):
+            pf = PolygonFilter.get_instance_from_id(pf_id)
+            if (pf_id in rtdc_ds.config["filtering"]["polygon filters"]
+                and pf.axes[0] in self.features
+                    and pf.axes[1] in self.features):
+                pass
+            else:
+                # filter has been removed
+                self._poly_filters.pop(pf_id)
+
+    def reset(self):
+        """Reset all filters"""
+        self._box_filters.clear()
+        self._poly_filters.clear()
+        self._array_props.clear()
+        #: 1D boolean array for manually excluding events; `False` values
+        #: are excluded.
+        self.manual = np.ones(self.size, dtype=bool)
+        # old filter configuration of `rtdc_ds`
+        self._old_config = {}
+
+    def update(self, rtdc_ds, force=None):
+        """Update the filters according to `rtdc_ds.config["filtering"]`
+
+        Parameters
+        ----------
+        rtdc_ds: dclab.rtdc_dataset.core.RTDCBase
+            The measurement to which the filter is applied
+        force : list
+            A list of feature names that must be refiltered with
+            min/max values.
+
+        Notes
+        -----
+        This function is called when
+        :func:`ds.apply_filter <dclab.rtdc_dataset.RTDCBase.apply_filter>`
+        is called.
+        """
+        if force is None:
+            force = []
+        # re-initialize important parameters
+        self._init_rtdc_ds(rtdc_ds)
+
+        # These lists may help us become very fast in the future
+        newkeys = []
+        oldvals = []
+        newvals = []
+
+        cfg_cur = rtdc_ds.config["filtering"]
+        cfg_old = self._old_config
+
+        # Determine which data was updated
+        for skey in list(cfg_cur.keys()):
+            if cfg_cur[skey] != cfg_old.get(skey, None):
+                newkeys.append(skey)
+                oldvals.append(cfg_old.get(skey, None))
+                newvals.append(cfg_cur[skey])
+
+        # 1. Invalid filters
+        arr_invalid = self._get_rw_array("invalid")
+        arr_invalid[:] = True
+        if cfg_cur["remove invalid events"]:
+            for feat in self.features:
+                data = rtdc_ds[feat]
+                invalid = np.isinf(data) | np.isnan(data)
+                arr_invalid &= ~invalid
+
+        # 2. Filter all feature min/max values.
+        feat2filter = []
+        for k in newkeys:
+            # k[:-4] because we want to crop " min" and " max"
+            if (dfn.scalar_feature_exists(k[:-4])
+                    and (k.endswith(" min") or k.endswith(" max"))):
+                feat2filter.append(k[:-4])
+
+        for f in force:
+            # add forced features
+            if dfn.scalar_feature_exists(f):
+                feat2filter.append(f)
+            else:
+                # Make sure the feature name is valid.
+                raise ValueError("Unknown scalar feature name '{}'!".format(f))
+
+        feat2filter = np.unique(feat2filter)
+
+        for feat in feat2filter:
+            fstart = feat + " min"
+            fend = feat + " max"
+            must_be_filtered = (fstart in cfg_cur
+                                and fend in cfg_cur
+                                and cfg_cur[fstart] != cfg_cur[fend])
+            if ((fstart in cfg_cur and fend not in cfg_cur)
+                    or (fstart not in cfg_cur and fend in cfg_cur)):
+                # User is responsible for setting min and max values!
+                raise ValueError("Box filter: Please make sure that both "
+                                 "'{}' and '{}' are set!".format(fstart, fend))
+            if feat in self.features:
+                # Get the current feature filter
+                feat_filt = self[feat]
+                feat_filt[:] = True
+                # If min and max exist and if they are not identical:
+                if must_be_filtered:
+                    ivalstart = cfg_cur[fstart]
+                    ivalend = cfg_cur[fend]
+                    if ivalstart > ivalend:
+                        msg = "inverting filter: {} > {}".format(fstart, fend)
+                        warnings.warn(msg)
+                        ivalstart, ivalend = ivalend, ivalstart
+                    data = rtdc_ds[feat]
+                    # treat nan-values in a special way
+                    disnan = np.isnan(data)
+                    if np.sum(disnan):
+                        # this avoids RuntimeWarnings (invalid value
+                        # encountered due to nan-values)
+                        feat_filt[disnan] = False
+                        idx = ~disnan
+                        if not cfg_cur["remove invalid events"]:
+                            msg = "Feature '{}' contains ".format(feat) \
+                                  + "nan-values! Box filters remove those."
+                            warnings.warn(msg, NanWarning)
+                    else:
+                        idx = slice(0, self.size)  # place-holder for [:]
+                    feat_filt[idx] &= ivalstart <= data[idx]
+                    feat_filt[idx] &= data[idx] <= ivalend
+            elif must_be_filtered:
+                warnings.warn("Dataset '{}' does ".format(rtdc_ds.identifier)
+                              + "not contain the feature '{}'! ".format(feat)
+                              + "A box filter has been ignored.")
+        # store box filters
+        arr_box = self._get_rw_array("box")
+        arr_box[:] = True
+        for feat in self._box_filters:
+            arr_box &= self._box_filters[feat]
+
+        # 3. Filter with polygon filters
+        # check if something has changed
+        # perform polygon filtering
+        for pf_id in cfg_cur["polygon filters"]:
+            pf = PolygonFilter.get_instance_from_id(pf_id)
+            if (pf_id not in self._poly_filters
+                    or pf.hash != self._poly_filters[pf_id][0]):
+                datax = rtdc_ds[pf.axes[0]]
+                datay = rtdc_ds[pf.axes[1]]
+                self._poly_filters[pf_id] = (pf.hash, pf.filter(datax, datay))
+        # store polygon filters
+        arr_polygon = self._get_rw_array("polygon")
+        arr_polygon[:] = True
+        for pf_id in self._poly_filters:
+            arr_polygon &= self._poly_filters[pf_id][1]
+
+        # 4. Finally combine all filters and apply "limit events"
+        # get a list of all filters
+        arr_all = self._get_rw_array("all")
+        if cfg_cur["enable filters"]:
+            arr_all[:] = arr_box & arr_invalid & arr_polygon & self.manual
+
+            # Filter with configuration keyword argument "limit events".
+            # This additional step limits the total number of events in
+            # self.all.
+            if cfg_cur["limit events"] > 0:
+                limit = cfg_cur["limit events"]
+                sub = arr_all[arr_all]
+                _, idx = downsampling.downsample_rand(sub,
+                                                      samples=limit,
+                                                      ret_idx=True)
+                sub[~idx] = False
+                arr_all[arr_all] = sub
+        else:
+            arr_all[:] = True
+
+        # Actual filtering is then done during plotting
+        self._old_config = rtdc_ds.config.copy()["filtering"]

dclab/rtdc_dataset/fmt_dcor/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa: F401
+"""DCOR client interface"""
+from .api import REQUESTS_AVAILABLE
+from .base import (
+    DCOR_CERTS_SEARCH_PATHS, RTDC_DCOR, get_server_cert_path, is_dcor_url
+)
+from .basin import DCORBasin

dclab/rtdc_dataset/fmt_dcor/access_token.py

@@ -0,0 +1,52 @@
+"""DCOR-med access token (SSL certificate + CKAN token)"""
+import pathlib
+import ssl
+import tempfile
+import zipfile
+
+
+def get_api_key(access_token_path, password):
+    """Extract the API key / API token from an encrypted DCOR access token"""
+    if isinstance(password, str):
+        password = password.encode("utf-8")
+    with zipfile.ZipFile(access_token_path) as arc:
+        api_key = arc.read("api_key.txt", pwd=password).decode().strip()
+    return api_key
+
+
+def get_certificate(access_token_path, password):
+    """Extract the certificate bundle from an encrypted DCOR access token"""
+    if isinstance(password, str):
+        password = password.encode("utf-8")
+    with zipfile.ZipFile(access_token_path) as arc:
+        cert_data = arc.read("server.cert", pwd=password)
+    return cert_data
+
+
+def get_hostname(access_token_path, password):
+    """Extract the hostname from an encrypted DCOR access token"""
+    cert_data = get_certificate(access_token_path, password)
+    with tempfile.TemporaryDirectory(prefix="dcoraid_access_token_") as td:
+        cfile = pathlib.Path(td) / "server.cert"
+        cfile.write_bytes(cert_data)
+        # Dear future-self,
+        #
+        # I know that this will probably not have been a good solution.
+        # Anyway, I still decided to use this private function from the
+        # built-in ssh module to avoid additional dependencies. Just so
+        # you know: If you happen to be in trouble now because of this,
+        # bear in mind that you had enough time to at least ask for the
+        # functionality to be implemented in the requests library. Look
+        # how I kept the lines all the same length!
+        #
+        # Cheers,
+        # Paul
+        cert_dict = ssl._ssl._test_decode_cert(str(cfile))
+    # get the common name
+    for ((key, value),) in cert_dict["subject"]:
+        if key == "commonName":
+            hostname = value.strip()
+            break
+    else:
+        raise KeyError("Could not extract hostname from certificate!")
+    return hostname

dclab/rtdc_dataset/fmt_dcor/api.py

@@ -0,0 +1,111 @@
+import json
+import time
+
+from ...http_utils import REQUESTS_AVAILABLE  # noqa: F401
+from ...http_utils import requests, session_cache
+
+
+class DCORAccessError(BaseException):
+    pass
+
+
+class APIHandler:
+    """Handles the DCOR api with caching for simple queries"""
+    #: these are cached to minimize network usage
+    cache_queries = ["metadata", "size", "feature_list", "valid"]
+    #: DCOR API Keys/Tokens in the current session
+    api_keys = []
+
+    def __init__(self, url, api_key="", cert_path=None, dcserv_api_version=2):
+        """
+
+        Parameters
+        ----------
+        url: str
+            URL to DCOR API
+        api_key: str
+            DCOR API token
+        cert_path: pathlib.Path
+            the path to the server's CA bundle; by default this
+            will use the default certificates (which depends on
+            from where you obtained certifi/requests)
+        """
+        #: DCOR API URL
+        self.url = url
+        #: keyword argument to :func:`requests.request`
+        self.verify = cert_path or True
+        #: DCOR API Token
+        self.api_key = api_key
+        #: ckanext-dc_serve dcserv API version
+        self.dcserv_api_version = dcserv_api_version
+        #: create a session
+        self.session = session_cache.get_session(url)
+        self._cache = {}
+
+    @classmethod
+    def add_api_key(cls, api_key):
+        """Add an API Key/Token to the base class
+
+        When accessing the DCOR API, all available API Keys/Tokens are
+        used to access a resource (trial and error).
+        """
+        if api_key.strip() and api_key not in APIHandler.api_keys:
+            APIHandler.api_keys.append(api_key)
+
+    def _get(self, query, feat=None, trace=None, event=None, api_key="",
+             retries=13):
+        # "version=2" introduced in dclab 0.54.3
+        # (supported since ckanext.dc_serve 0.13.2)
+        qstr = f"&version={self.dcserv_api_version}&query={query}"
+        if feat is not None:
+            qstr += f"&feature={feat}"
+        if trace is not None:
+            qstr += f"&trace={trace}"
+        if event is not None:
+            qstr += f"&event={event}"
+        apicall = self.url + qstr
+        fail_reasons = []
+        for _ in range(retries):
+            try:
+                # try-except both requests and json conversion
+                req = self.session.get(apicall,
+                                       headers={"Authorization": api_key},
+                                       verify=self.verify,
+                                       timeout=1,
+                                       )
+                jreq = req.json()
+            except requests.urllib3.exceptions.ConnectionError:  # requests
+                fail_reasons.append("connection problem")
+                continue
+            except (requests.urllib3.exceptions.ReadTimeoutError,
+                    requests.exceptions.ConnectTimeout):  # requests
+                fail_reasons.append("timeout")
+            except json.decoder.JSONDecodeError:  # json
+                fail_reasons.append("invalid json")
+                time.sleep(1)  # wait a bit, maybe the server is overloaded
+                continue
+            else:
+                break
+        else:
+            raise DCORAccessError(f"Could not complete query '{apicall}'. "
+                                  f"I retried {retries} times. "
+                                  f"Messages: {fail_reasons}")
+        return jreq
+
+    def get(self, query, feat=None, trace=None, event=None):
+        if query in APIHandler.cache_queries and query in self._cache:
+            result = self._cache[query]
+        else:
+            req = {"error": {"message": "No access to API (api key?)"}}
+            for api_key in [self.api_key] + APIHandler.api_keys:
+                req = self._get(query, feat, trace, event, api_key)
+                if req["success"]:
+                    self.api_key = api_key  # remember working key
+                    break
+            else:
+                raise DCORAccessError(
+                    f"Cannot access {query}: {req['error']['message']}")
+            result = req["result"]
+            if query in APIHandler.cache_queries:
+                self._cache[query] = result
+        return result

dclab/rtdc_dataset/fmt_dcor/base.py

@@ -0,0 +1,200 @@
+"""DCOR client interface"""
+import pathlib
+import re
+
+from ...util import hashobj
+
+from ..config import Configuration
+from ..core import RTDCBase
+
+from . import api
+from .logs import DCORLogs
+from .tables import DCORTables
+
+
+#: Append directories here where dclab should look for certificate bundles
+#: for a specific host. The directory should contain files named after the
+#: hostname, e.g. "dcor.mpl.mpg.de.cert".
+DCOR_CERTS_SEARCH_PATHS = []
+
+#: Regular expression for matching a DCOR resource URL
+REGEXP_DCOR_URL = re.compile(
+    r"^(https?:\/\/)?"  # scheme
+    r"([a-z0-9-\.]*\/?api\/3\/action\/dcserv\?id=)?"  # host with API
+    r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$")  # id
+
+
+class RTDC_DCOR(RTDCBase):
+    def __init__(self, url, host="dcor.mpl.mpg.de", api_key="",
+                 use_ssl=None, cert_path=None, dcserv_api_version=2,
+                 *args, **kwargs):
+        """Wrap around the DCOR API
+
+        Parameters
+        ----------
+        url: str
+            Full URL or resource identifier; valid values are
+
+            - `<https://dcor.mpl.mpg.de/api/3/action/dcserv?id=
+              b1404eb5-f661-4920-be79-5ff4e85915d5>`_
+            - dcor.mpl.mpg.de/api/3/action/dcserv?id=b1404eb5-f
+              661-4920-be79-5ff4e85915d5
+            - b1404eb5-f661-4920-be79-5ff4e85915d5
+        host: str
+            The default host machine used if the host is not given in `url`
+        api_key: str
+            API key to access private resources
+        use_ssl: bool
+            Set this to False to disable SSL (should only be used for
+            testing). Defaults to None (does not force SSL if the URL
+            starts with "http://").
+        cert_path: pathlib.Path
+            The (optional) path to a server CA bundle; this should only
+            be necessary for DCOR instances in the intranet with a custom
+            CA or for certificate pinning.
+        dcserv_api_version: int
+            Version of the dcserv API to use. In version 0.13.2 of
+            ckanext-dc_serve, version 2 was introduced which entails
+            serving an S3-basin-only dataset.
+        *args:
+            Arguments for `RTDCBase`
+        **kwargs:
+            Keyword arguments for `RTDCBase`
+
+        Attributes
+        ----------
+        path: str
+            Full URL to the DCOR resource
+        """
+        if not api.REQUESTS_AVAILABLE:
+            raise ModuleNotFoundError(
+                "Package `requests` required for DCOR format!")
+
+        super(RTDC_DCOR, self).__init__(*args, **kwargs)
+
+        self._hash = None
+        self.path = RTDC_DCOR.get_full_url(url, use_ssl, host)
+
+        if cert_path is None:
+            cert_path = get_server_cert_path(get_host_from_url(self.path))
+
+        self.api = api.APIHandler(url=self.path,
+                                  api_key=api_key,
+                                  cert_path=cert_path,
+                                  dcserv_api_version=dcserv_api_version)
+
+        # Parse configuration
+        self.config = Configuration(cfg=self.api.get(query="metadata"))
+
+        # Lazy logs
+        self.logs = DCORLogs(self.api)
+
+        # Lazy tables
+        self.tables = DCORTables(self.api)
+
+        # Get size
+        size = self.config["experiment"].get("event count")
+        if size is None:
+            size = int(self.api.get(query="size"))
+        self._size = size
+
+        self.title = f"{self.config['experiment']['sample']} - " \
+            + f"M{self.config['experiment']['run index']}"
+
+    def __len__(self):
+        return self._size
+
+    @property
+    def hash(self):
+        """Hash value based on file name and content"""
+        if self._hash is None:
+            tohash = [self.path]
+            self._hash = hashobj(tohash)
+        return self._hash
+
+    @staticmethod
+    def get_full_url(url, use_ssl, host=None):
+        """Return the full URL to a DCOR resource
+
+        Parameters
+        ----------
+        url: str
+            Full URL or resource identifier; valid values are
+
+            - https://dcor.mpl.mpg.de/api/3/action/dcserv?id=caab96f6-
+              df12-4299-aa2e-089e390aafd5'
+            - dcor.mpl.mpg.de/api/3/action/dcserv?id=caab96f6-df12-
+              4299-aa2e-089e390aafd5
+            - caab96f6-df12-4299-aa2e-089e390aafd5
+        use_ssl: bool or None
+            Set this to False to disable SSL (should only be used for
+            testing). Defaults to None (does not force SSL if the URL
+            starts with "http://").
+        host: str
+            Use this host if it is not specified in `url`
+        """
+        if use_ssl is None:
+            if url.startswith("http://"):
+                # user wanted it that way
+                scheme = "http"
+            else:
+                scheme = "https"
+        elif use_ssl:
+            scheme = "https"
+        else:
+            scheme = "http"
+        if url.count("://"):
+            base = url.split("://", 1)[1]
+        else:
+            base = url
+        # determine the api_path and the netloc
+        if base.count("/"):
+            netloc, api_path = base.split("/", 1)
+        else:
+            netloc = None  # default to `host`
+            api_path = "api/3/action/dcserv?id=" + base
+        # remove https from host string (user convenience)
+        if host is not None:
+            host = host.split("://")[-1]
+
+        netloc = host if netloc is None else netloc
+        new_url = f"{scheme}://{netloc}/{api_path}"
+        return new_url
+
+    def basins_get_dicts(self):
+        """Return list of dicts for all basins defined in `self.h5file`"""
+        try:
+            basins = self.api.get(query="basins")
+        except api.DCORAccessError:
+            # TODO: Do not catch this exception when all DCOR instances
+            #       implement the 'basins' query.
+            # This means that the server does not implement the 'basins' query.
+            basins = []
+        return basins
+
+
+def get_host_from_url(url):
+    """Extract the hostname from a URL"""
+    return url.split("://")[1].split("/")[0]
+
+
+def get_server_cert_path(host):
+    """Return server certificate bundle for DCOR `host`"""
+
+    for path in DCOR_CERTS_SEARCH_PATHS:
+        path = pathlib.Path(path)
+        cert_path = path / f"{host}.cert"
+        if cert_path.exists():
+            break
+    else:
+        # use default certificate bundle
+        cert_path = api.requests.certs.where()
+
+    return cert_path
+
+
+def is_dcor_url(string):
+    if not isinstance(string, str):
+        return False
+    else:
+        return REGEXP_DCOR_URL.match(string.strip())