dclab 0.62.11__cp313-cp313-win_amd64.whl

dclab/rtdc_dataset/fmt_dcor/basin.py

@@ -0,0 +1,73 @@
+import re
+
+from ..feat_basin import Basin
+
+from .api import REQUESTS_AVAILABLE, APIHandler, DCORAccessError
+from .base import RTDC_DCOR
+
+
+REGEXP_FULL_DCOR_URL = re.compile(
+    r"^https?:\/\/"  # scheme
+    r"[a-z0-9-\.]*\.[a-z0-9-\.]*\/?api\/3\/action\/dcserv\?id="  # host and API
+    r"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$")  # id
+
+
+class DCORBasin(Basin):
+    basin_format = "dcor"
+    basin_type = "remote"
+
+    def __init__(self, *args, **kwargs):
+        """Access to private and public DCOR resources
+
+        Since version 2 of the DCOR data API, all feature data are
+        accessed via :class:`.HTTPBasin`s on S3. The DCOR basin is just
+        a wrapper around those `HTTPBasin`s.
+
+        For private resources, the DCOR format facilitates authentication
+        via access tokens. Behind the scenes, DCOR creates a pre-signed
+        URL to access private data on an S3 object storage provider.
+        Note that you must let dclab know your DCOR access
+        token via :func:`.APIHandler.add_api_key` for this to work.
+
+        The `location` must be a full DCOR URL, including the scheme
+        and netloc, e.g:
+
+            https://dcor.mpl.mpg.de/api/3/action/dcserv?
+            id=b1404eb5-f661-4920-be79-5ff4e85915d5
+        """
+        self._available_verified = None
+        super(DCORBasin, self).__init__(*args, **kwargs)
+
+    def _load_dataset(self, location, **kwargs):
+        return RTDC_DCOR(location, **kwargs)
+
+    def is_available(self):
+        """Check whether a DCOR resource is available
+
+        Notes
+        -----
+        - Make sure that your DCOR access token is stored in
+          :class:`.APIHandler`. You can add tokens with
+          :func:`.APIHandler.add_api_key`.
+        """
+        with self._av_check_lock:
+            if not REQUESTS_AVAILABLE:
+                # don't even bother
+                self._available_verified = False
+            elif not is_full_dcor_url(self.location):
+                # not a full DCOR URL
+                self._available_verified = False
+            if self._available_verified is None:
+                api = APIHandler(self.location)
+                try:
+                    self._available_verified = api.get("valid")
+                except DCORAccessError:
+                    self._available_verified = False
+        return self._available_verified
+
+
+def is_full_dcor_url(string):
+    if not isinstance(string, str):
+        return False
+    else:
+        return REGEXP_FULL_DCOR_URL.match(string.strip())

dclab/rtdc_dataset/fmt_dcor/logs.py

@@ -0,0 +1,26 @@
+class DCORLogs:
+    def __init__(self, api):
+        self.api = api
+        self._logs_cache = None
+
+    def __contains__(self, key):
+        return key in self.keys()
+
+    def __getitem__(self, key):
+        return self._logs[key]
+
+    def __iter__(self):
+        for key in self.keys():
+            yield key
+
+    def __len__(self):
+        return len(self._logs)
+
+    def keys(self):
+        return self._logs.keys()
+
+    @property
+    def _logs(self):
+        if self._logs_cache is None:
+            self._logs_cache = self.api.get(query="logs")
+        return self._logs_cache

dclab/rtdc_dataset/fmt_dcor/tables.py

@@ -0,0 +1,42 @@
+import numpy as np
+
+
+class DCORTables:
+    def __init__(self, api):
+        self.api = api
+        self._tables_cache = None
+
+    def __contains__(self, key):
+        return key in self.keys()
+
+    def __getitem__(self, key):
+        return self._tables[key]
+
+    def __iter__(self):
+        for key in self.keys():
+            yield key
+
+    def __len__(self):
+        return len(self._tables)
+
+    def keys(self):
+        return self._tables.keys()
+
+    @property
+    def _tables(self):
+        if self._tables_cache is None:
+            table_data = self.api.get(query="tables")
+            # assemble the tables
+            tables = {}
+            for key in table_data:
+                columns, data = table_data[key]
+                tab_data = np.asarray(data)
+                if columns is not None:
+                    # We have a rec-array (named columns)
+                    ds_dt = np.dtype({'names': columns,
+                                      'formats': [np.float64] * len(columns)})
+                    tab_data = np.rec.array(tab_data, dtype=ds_dt)
+                tables[key] = tab_data
+
+            self._tables_cache = tables
+        return self._tables_cache

dclab/rtdc_dataset/fmt_dict.py

@@ -0,0 +1,103 @@
+"""RT-DC dictionary format"""
+import collections
+import time
+
+import numpy as np
+
+from .. import definitions as dfn
+from ..util import hashobj
+
+from .config import Configuration
+from .core import RTDCBase
+
+
+class DictContourEvent:
+    def __init__(self, contours):
+        assert contours[0].shape[1] == 2
+        self.shape = (len(contours), np.nan, 2)
+        self.contours = contours
+
+    def __iter__(self):
+        return iter(self.contours)
+
+    def __getitem__(self, item):
+        return self.contours[item]
+
+    def __len__(self):
+        return len(self.contours)
+
+
+class DictTraceEvent(collections.UserDict):
+
+    @property
+    def shape(self):
+        key0 = sorted(self.keys())[0]
+        return len(self), len(self[key0]), len(self[key0][0])
+
+
+class RTDC_Dict(RTDCBase):
+    _local_basins_allowed = True
+
+    def __init__(self, ddict, *args, **kwargs):
+        """Dictionary-based RT-DC dataset
+
+        Parameters
+        ----------
+        ddict: dict
+            Dictionary with features as keys (valid features like
+            "area_cvx", "deform", "image" are defined by
+            `dclab.definitions.feature_exists`) with which the class
+            will be instantiated. The configuration is set to the
+            default configuration of dclab.
+
+            .. versionchanged:: 0.27.0
+                Scalar features are automatically converted to arrays.
+        *args:
+            Arguments for `RTDCBase`
+        **kwargs:
+            Keyword arguments for `RTDCBase`
+        """
+        assert ddict
+
+        super(RTDC_Dict, self).__init__(*args, **kwargs)
+
+        t = time.localtime()
+
+        # Get an identifying string
+        keys = list(ddict.keys())
+        keys.sort()
+        ids = hashobj(ddict[keys[0]])
+        self._ids = ids
+        self.path = "none"
+        self.title = "{}_{:02d}_{:02d}/{}.dict".format(t[0], t[1], t[2], ids)
+
+        # Populate events
+        for feat in ddict:
+            if dfn.feature_exists(feat):
+                if dfn.scalar_feature_exists(feat):
+                    data = np.array(ddict[feat])
+                elif feat == "contour":
+                    data = DictContourEvent(ddict[feat])
+                elif feat == "trace":
+                    data = DictTraceEvent(ddict[feat])
+                elif isinstance(ddict[feat], list):
+                    # convert e.g. image data to arrays
+                    data = np.array(ddict[feat])
+                else:
+                    data = ddict[feat]
+            else:
+                raise ValueError("Invalid feature name '{}'".format(feat))
+            if isinstance(data, np.ndarray):
+                # Convert numpy array to read-only array
+                data = data.view()
+                data.setflags(write=False)
+            self._events[feat] = data
+
+        event_count = len(ddict[list(ddict.keys())[0]])
+
+        self.config = Configuration()
+        self.config["experiment"]["event count"] = event_count
+
+    @property
+    def hash(self):
+        return self._ids

dclab/rtdc_dataset/fmt_hdf5/__init__.py

@@ -0,0 +1,6 @@
+# flake8: noqa: F401
+from .base import RTDC_HDF5, MIN_DCLAB_EXPORT_VERSION
+from .basin import HDF5Basin  # import means registering
+from .events import (
+    H5Events, H5MaskEvent, H5TraceEvent, H5ScalarEvent, H5ContourEvent)
+from .feat_defect import DEFECTIVE_FEATURES

dclab/rtdc_dataset/fmt_hdf5/base.py

@@ -0,0 +1,192 @@
+"""RT-DC hdf5 format"""
+from __future__ import annotations
+
+import io
+import json
+import pathlib
+from typing import Any, BinaryIO, Dict
+import warnings
+
+import h5py
+
+from ...external.packaging import parse as parse_version
+from ...util import hashobj, hashfile
+
+from ..config import Configuration
+from ..core import RTDCBase
+
+from . import events
+from . import logs
+from . import tables
+
+#: rtdc files exported with dclab prior to this version are not supported
+MIN_DCLAB_EXPORT_VERSION = "0.3.3.dev2"
+
+
+class OldFormatNotSupportedError(BaseException):
+    pass
+
+
+class UnknownKeyWarning(UserWarning):
+    pass
+
+
+class RTDC_HDF5(RTDCBase):
+    def __init__(self,
+                 h5path: str | pathlib.Path | BinaryIO | io.IOBase,
+                 h5kwargs: Dict[str, Any] = None,
+                 *args,
+                 **kwargs):
+        """HDF5 file format for RT-DC measurements
+
+        Parameters
+        ----------
+        h5path: str or pathlib.Path or file-like object
+            Path to an '.rtdc' measurement file or a file-like object
+        h5kwargs: dict
+            Additional keyword arguments given to :class:`h5py.File`
+        *args:
+            Arguments for `RTDCBase`
+        **kwargs:
+            Keyword arguments for `RTDCBase`
+
+        Attributes
+        ----------
+        path: pathlib.Path
+            Path to the experimental HDF5 (.rtdc) file
+        """
+        super(RTDC_HDF5, self).__init__(*args, **kwargs)
+
+        # Any subclass from RTDC_HDF5 is probably a remote-type and should
+        # not be able to access local basins. If you do not agree, please
+        # enable this in the definition of the subclass.
+        self._local_basins_allowed = True if self.format == "hdf5" else False
+
+        if isinstance(h5path, (str, pathlib.Path)):
+            h5path = pathlib.Path(h5path)
+        else:
+            h5path = h5path
+
+        self._hash = None
+        self.path = h5path
+
+        # Increase the read cache (which defaults to 1MiB), since
+        # normally we have around 2.5MiB image chunks.
+        if h5kwargs is None:
+            h5kwargs = {}
+        h5kwargs.setdefault("rdcc_nbytes", 10 * 1024 ** 2)
+        h5kwargs.setdefault("rdcc_w0", 0)
+
+        self.h5kwargs = h5kwargs
+        self.h5file = h5py.File(h5path, **h5kwargs)
+
+        self._events = events.H5Events(self.h5file)
+
+        # Parse configuration
+        self.config = RTDC_HDF5.parse_config(self.h5file)
+
+        # Override logs property with HDF5 data
+        self.logs = logs.H5Logs(self.h5file)
+
+        # Override the tables property with HDF5 data
+        self.tables = tables.H5Tables(self.h5file)
+
+        # check version
+        rtdc_soft = self.config["setup"].get("software version", "unknown")
+        if rtdc_soft.startswith("dclab "):
+            rtdc_ver = parse_version(rtdc_soft.split(" ")[1])
+            if rtdc_ver < parse_version(MIN_DCLAB_EXPORT_VERSION):
+                msg = "The file {} was created ".format(self.path) \
+                      + "with dclab {} which is ".format(rtdc_ver) \
+                      + "not supported anymore! Please rerun " \
+                      + "dclab-tdms2rtdc / export the data again."
+                raise OldFormatNotSupportedError(msg)
+
+        self.title = "{} - M{}".format(
+            self.config["experiment"].get("sample", "undefined sample"),
+            self.config["experiment"].get("run index", "0"))
+
+    def close(self):
+        """Close the underlying HDF5 file"""
+        super(RTDC_HDF5, self).close()
+        self.h5file.close()
+
+    @property
+    def _h5(self):
+        warnings.warn("Access to the underlying HDF5 file is now public. "
+                      "Please use the `h5file` attribute instead of `_h5`!",
+                      DeprecationWarning)
+        return self.h5file
+
+    @staticmethod
+    def can_open(h5path):
+        """Check whether a given file is in the .rtdc file format"""
+        h5path = pathlib.Path(h5path)
+        if h5path.suffix == ".rtdc":
+            return True
+        else:
+            # we don't know the extension; check for the "events" group
+            canopen = False
+            try:
+                # This is a workaround for Python2 where h5py cannot handle
+                # unicode file names.
+                with h5path.open("rb") as fd:
+                    h5 = h5py.File(fd, "r")
+                    if "events" in h5:
+                        canopen = True
+            except IOError:
+                # not an HDF5 file
+                pass
+            return canopen
+
+    @staticmethod
+    def parse_config(h5path):
+        """Parse the RT-DC configuration of an HDF5 file
+
+        `h5path` may be a h5py.File object or an actual path
+        """
+        if not isinstance(h5path, h5py.File):
+            with h5py.File(h5path, mode="r") as fh5:
+                h5attrs = dict(fh5.attrs)
+        else:
+            h5attrs = dict(h5path.attrs)
+
+        # Convert byte strings to unicode strings
+        # https://github.com/h5py/h5py/issues/379
+        for key in h5attrs:
+            if isinstance(h5attrs[key], bytes):
+                h5attrs[key] = h5attrs[key].decode("utf-8")
+
+        config = Configuration()
+        for key in h5attrs:
+            section, pname = key.split(":")
+            config[section][pname] = h5attrs[key]
+        return config
+
+    @property
+    def hash(self):
+        """Hash value based on file name and content"""
+        if self._hash is None:
+            tohash = [self.path.name,
+                      # Hash a maximum of ~1MB of the hdf5 file
+                      hashfile(self.path, blocksize=65536, count=20)]
+            self._hash = hashobj(tohash)
+        return self._hash
+
+    def basins_get_dicts(self):
+        """Return list of dicts for all basins defined in `self.h5file`"""
+        return self.basin_get_dicts_from_h5file(self.h5file)
+
+    @staticmethod
+    def basin_get_dicts_from_h5file(h5file):
+        """Return list of dicts for all basins defined in `h5file`"""
+        basins = []
+        # Do not sort anything here, sorting is done in `RTDCBase`.
+        for bk in h5file.get("basins", []):
+            bdat = list(h5file["basins"][bk])
+            if isinstance(bdat[0], bytes):
+                bdat = [bi.decode("utf") for bi in bdat]
+            bdict = json.loads(" ".join(bdat))
+            bdict["key"] = bk
+            basins.append(bdict)
+        return basins

dclab/rtdc_dataset/fmt_hdf5/basin.py

@@ -0,0 +1,30 @@
+"""RT-DC hdf5 format"""
+from __future__ import annotations
+
+import pathlib
+
+from .. import feat_basin
+
+from .base import RTDC_HDF5
+
+
+class HDF5Basin(feat_basin.Basin):
+    basin_format = "hdf5"
+    basin_type = "file"
+
+    def __init__(self, *args, **kwargs):
+        self._available_verified = None
+        super(HDF5Basin, self).__init__(*args, **kwargs)
+
+    def _load_dataset(self, location, **kwargs):
+        return RTDC_HDF5(location, **kwargs)
+
+    def is_available(self):
+        if self._available_verified is None:
+            with self._av_check_lock:
+                try:
+                    self._available_verified = \
+                        pathlib.Path(self.location).exists()
+                except OSError:
+                    pass
+        return self._available_verified