dclab 0.67.0__cp314-cp314-macosx_11_0_arm64.whl

dclab/features/volume.py

@@ -0,0 +1,242 @@
+"""Volume computation based on contour revolution"""
+import numpy as np
+
+
+def get_volume(cont, pos_x, pos_y, pix, fix_orientation=False):
+    """Calculate the volume of a polygon revolved around an axis
+
+    The volume estimation assumes rotational symmetry.
+
+    Parameters
+    ----------
+    cont: ndarray or list of ndarrays of shape (N,2)
+        A 2D array that holds the contour of an event [px]
+        e.g. obtained using `mm.contour` where  `mm` is an instance
+        of `RTDCBase`. The first and second columns of `cont`
+        correspond to the x- and y-coordinates of the contour.
+    pos_x: float or ndarray of length N
+        The x coordinate(s) of the centroid of the event(s) [µm]
+        e.g. obtained using `mm.pos_x`
+    pos_y: float or ndarray of length N
+        The y coordinate(s) of the centroid of the event(s) [µm]
+        e.g. obtained using `mm.pos_y`
+    pix: float
+        The detector pixel size in µm.
+        e.g. obtained using: `mm.config["imaging"]["pixel size"]`
+    fix_orientation: bool
+        If set to True, make sure that the orientation of the
+        contour is counter-clockwise in the r-z plane
+        (see :func:`vol_revolve`). This is False by default, because
+        (1) Shape-In always stores the contours in the correct
+        orientation and (2) there may be events with high porosity
+        where "fixing" the orientation makes things worse and a
+        negative volume is returned.
+
+    Returns
+    -------
+    volume: float or ndarray
+        volume in um^3
+
+    Notes
+    -----
+    The computation of the volume is based on a full rotation of the
+    upper and the lower halves of the contour from which the
+    average is then used.
+
+    The volume is computed radially from the the center position
+    given by (`pos_x`, `pos_y`). For sufficiently smooth contours,
+    such as densely sampled ellipses, the center position does not
+    play an important role. For contours that are given on a coarse
+    grid, as is the case for RT-DC, the center position must be
+    given.
+
+    References
+    ----------
+    - https://de.wikipedia.org/wiki/Kegelstumpf#Formeln
+    - Yields identical results to the Matlab script by Geoff Olynyk
+      <https://de.mathworks.com/matlabcentral/fileexchange/36525-volrevolve>`_
+    """
+    if np.isscalar(pos_x):
+        cont = [cont]
+        ret_list = False
+    else:
+        ret_list = True
+
+    # Convert input to 1D arrays
+    pos_x = np.atleast_1d(pos_x)
+    pos_y = np.atleast_1d(pos_y)
+
+    if pos_x.size != pos_y.size:
+        raise ValueError("Size of `pos_x` and `pos_y` must match!")
+
+    if pos_x.size > 1 and len(cont) <= 1:
+        raise ValueError("Number of given contours too small!")
+
+    # results are stored in a separate array initialized with nans
+    v_avg = np.zeros_like(pos_x, dtype=np.float64) * np.nan
+
+    # v_avg has the shape of `pos_x`. We are iterating over the smallest
+    # length for `cont` and `pos_x`.
+    for ii in range(min(len(cont), pos_x.shape[0])):
+        # If the contour has less than 4 pixels, the computation will fail.
+        # In that case, the value np.nan is already assigned.
+        cc = cont[ii]
+        if cc.shape[0] >= 4:
+            # Center contour coordinates with given centroid
+            contour_x = cc[:, 0] - pos_x[ii] / pix
+            contour_y = cc[:, 1] - pos_y[ii] / pix
+            # Switch to r and z to follow notation of vol_revolve
+            # (In RT-DC the axis of rotation is x, but for vol_revolve
+            # we need the axis vertically)
+            contour_r = contour_y
+            contour_z = contour_x
+            if fix_orientation:
+                # Make sure the contour is counter-clockwise
+                contour_r, contour_z = counter_clockwise(contour_r, contour_z)
+
+            # Compute right volume
+            # Which points are at negative r-values (r<0)?
+            inx_neg = np.where(contour_r < 0)
+            # These points will be shifted up to r=0 directly on the z-axis
+            contour_right = np.copy(contour_r)
+            contour_right[inx_neg] = 0
+            vol_right = vol_revolve(contour_right, contour_x, pix)
+
+            # Compute left volume
+            # Which points are at positive r-values? (r>0)?
+            idx_pos = np.where(contour_r > 0)
+            # These points will be shifted down to y=0 to build an x-axis
+            contour_left = np.copy(contour_r)
+            contour_left[idx_pos] = 0
+            # Now we still have negative r values, but vol_revolve needs
+            # positive values, so we flip the sign...
+            contour_left[:] *= -1
+            # ... but in doing so, we have switched to clockwise rotation
+            # and we need to pass the array in reverse order
+            vol_left = vol_revolve(contour_left[::-1], contour_x[::-1], pix)
+
+            # Compute the average
+            v_avg[ii] = (vol_right + vol_left) / 2
+
+    if not ret_list:
+        # Do not return a list if the input contour was not in a list
+        v_avg = v_avg[0]
+
+    return v_avg
+
+
+def counter_clockwise(cx, cy):
+    """Put contour coordinates into counter-clockwise order
+
+    Parameters
+    ----------
+    cx, cy: 1d ndarrays
+        The x- and y-coordinates of the contour
+
+    Returns
+    -------
+    cx_cc, cy_cc:
+        The x- and y-coordinates of the contour in
+        counter-clockwise orientation.
+
+    Notes
+    -----
+    The contour must be centered around (0, 0).
+    """
+    # test orientation
+    angles = np.unwrap(np.arctan2(cy, cx))
+    grad = np.diff(angles)
+    if np.average(grad) < 0:
+        return cx[::-1], cy[::-1]
+    else:
+        return cx, cy
+
+
+def vol_revolve(r, z, point_scale=1.):
+    r"""Calculate the volume of a polygon revolved around the Z-axis
+
+    This implementation yields the same results as the volRevolve
+    Matlab function by Geoff Olynyk (from 2012-05-03)
+    https://de.mathworks.com/matlabcentral/fileexchange/36525-volrevolve.
+
+    The difference here is that the volume is computed using (a much
+    more approachable) implementation using the volume of a truncated
+    cone (https://de.wikipedia.org/wiki/Kegelstumpf).
+
+    .. math::
+
+      V = \frac{h \cdot \pi}{3} \cdot (R^2 + R \cdot r + r^2)
+
+    Where :math:`h` is the height of the cone and :math:`r` and
+    `R` are the smaller and larger radii of the truncated cone.
+
+    Each line segment of the contour resembles one truncated cone. If
+    the z-step is positive (counter-clockwise contour), then the
+    truncated cone volume is added to the total volume. If the z-step
+    is negative (e.g. inclusion), then the truncated cone volume is
+    removed from the total volume.
+
+    .. versionchanged:: 0.37.0
+
+      The volume in previous versions was overestimated by on average
+      2µm³.
+
+    Parameters
+    ----------
+    r: 1d np.ndarray
+        radial coordinates (perpendicular to the z axis)
+    z: 1d np.ndarray
+        coordinate along the axis of rotation
+    point_scale: float
+        point size in your preferred units; The volume is multiplied
+        by a factor of `point_scale**3`.
+
+    Notes
+    -----
+    The coordinates must be given in counter-clockwise order,
+    otherwise the volume will be negative.
+    """
+    r = np.array(r).flatten()
+    z = np.array(z).flatten()
+
+    # sanity checks
+    assert len(r) == len(z)
+    assert len(r) >= 3
+    assert len(r.shape) == len(z.shape) == 1
+    assert np.all(r >= 0)
+
+    # make sure we have a closed contour
+    if (r[-1] != r[0]) or (z[-1] != z[0]):
+        # We have an open contour - close it.
+        r = np.resize(r, len(r) + 1)
+        z = np.resize(z, len(z) + 1)
+
+    rp = r[:-1]
+
+    # array of radii differences: R - r
+    dr = np.diff(r)
+    # array of height differences: h
+    dz = np.diff(z)
+
+    # If we expand the function in the doc string with
+    # dr = R - r and dz = h, then we get three terms for the volume
+    # (as opposed to four terms in Olynyk's script). Those three terms
+    # all resemble area slices multiplied by the z-distance dz.
+    a1 = 3 * rp**2
+    a2 = 3 * rp*dr
+    a3 = dr**2
+
+    # Note that the formula for computing the volume is symmetric
+    # with respect to r and R. This means that it does not matter
+    # which sign dr has (R and r are always positive). Since this
+    # algorithm assumes that the contour is ordered counter-clockwise,
+    # positive dz means adding to the contour while negative dz means
+    # subtracting from the contour (see test functions for more details).
+    # Conveniently so, dz only appears one time in this formula, so
+    # we can take the sign of dz as it is (Otherwise, we would have
+    # to take the absolute value of every truncated cone volume and
+    # multiply it by np.sign(dz)).
+    v = np.pi / 3 * dz * np.abs(a1 + a2 + a3)
+    vol = np.sum(v) * point_scale ** 3
+
+    return vol

dclab/http_utils.py

@@ -0,0 +1,322 @@
+import functools
+import io
+import os
+import re
+import socket
+from unittest import mock
+from urllib.parse import urlparse
+import warnings
+
+import numpy as np
+
+
+try:
+    import requests
+except ModuleNotFoundError:
+    requests = mock.Mock()
+    REQUESTS_AVAILABLE = False
+else:
+    REQUESTS_AVAILABLE = True
+
+
+#: Regular expression for matching a regular HTTP URL
+REGEXP_HTTP_URL = re.compile(
+    r"^(https?:\/\/)"  # protocol (http or https or omitted)
+    r"([a-z0-9-\.]*)(\:[0-9]*)?\/"  # host:port
+    r".+"  # path
+)
+
+
+class ETagNotInResponseHeaderWarning(UserWarning):
+    """Used for cases where the requests.Response does not contain an ETag"""
+
+
+class ConnectionTimeoutWarning(UserWarning):
+    """Used when a connection fails or times out"""
+
+
+class HTTPFile(io.IOBase):
+    def __init__(self, url, chunk_size=2**18, keep_chunks=200):
+        """Chunk-cached access to a URL supporting range requests
+
+        Range requests (https://en.wikipedia.org/wiki/Byte_serving)
+        allow clients to access specific parts of a file via HTTP
+        without downloading the entire file.
+
+        This class creates a file-like object from a URL that can
+        then be passed on to e.g. h5py for reading. In addition, this
+        class keeps a chunk cache of the URL, making it (A) fast to
+        access frequently used parts of the file and (B) fast to slice
+        through large files since the ratio of data downloaded versus
+        (time-consuming) HTTP requests is very large.
+
+        Parameters
+        ----------
+        url: str
+            Path to the URL
+        chunk_size: int
+            Download chunk size. The entire file is split up into
+            equally-sized (and thus indexable) chunks.
+        keep_chunks: int
+            Number of downloaded chunks to keep in memory. For a
+            `chunk_size` of 2**18 bytes, a `keep_chunks` of 200
+            impliese a chunk cache size of 50 MiB.
+        """
+        self.url = url
+        self._chunk_size = chunk_size
+        self._keep_chunks = keep_chunks
+        self.session = session_cache.get_session(url)
+        self._len = None
+        self._etag = None
+        self._pos = 0
+        self.cache = {}
+
+    def _parse_header(self):
+        """parse the header sent by the server, populates length and etag"""
+        if self._len is None:
+            # Do not use `self.session.head`, because it might return a
+            # wrong content-length for pre-signed S3 URLS.
+            resp = self.session.get(self.url, stream=True, timeout=0.5)
+            if resp.status_code != 200:
+                raise ValueError(
+                    f"Server replied with status code {resp.status_code} "
+                    f"{resp.reason} for '{self.url}'")
+            self._len = int(resp.headers["content-length"])
+            # Try to determine the etag of the file.
+            etag = resp.headers.get("etag", "").strip("'").strip('"')
+            if len(etag) < 5:
+                etag = None
+                warnings.warn(f"Got empty ETag header for {self.url}",
+                              ETagNotInResponseHeaderWarning)
+            self._etag = etag
+
+    @property
+    def etag(self):
+        """Unique identifier for this resource (version) by the web server"""
+        self._parse_header()
+        return self._etag
+
+    @property
+    def length(self):
+        self._parse_header()
+        return self._len
+
+    @property
+    def max_cache_size(self):
+        """The maximum cache size allowed by `chunk_size` and `keep_chunks`"""
+        return self._chunk_size * self._keep_chunks
+
+    def close(self):
+        """Close the file
+
+        This closes the requests session and then calls `close` on
+        the super class.
+        """
+        self.session.close()
+        super(HTTPFile, self).close()
+
+    def download_range(self, start, stop):
+        """Download bytes given by the range (`start`, `stop`)
+
+        `stop` is not inclusive (In the HTTP range request it normally is).
+        """
+        resp = self.session.get(self.url,
+                                headers={"Range": f"bytes={start}-{stop-1}"}
+                                )
+        return resp.content
+
+    def get_cache_chunk(self, index):
+        """Return the cache chunk defined by `index`
+
+        If the chunk is not in `self.cache`, it is downloaded.
+        """
+        if index not in self.cache:
+            start = index*self._chunk_size
+            stop = min((index+1)*self._chunk_size, self.length)
+            self.cache[index] = self.download_range(start, stop)
+        if len(self.cache) > self._keep_chunks:
+            for kk in self.cache.keys():
+                if kk != 0:  # always keep the first chunk
+                    self.cache.pop(kk)
+                    break
+        return self.cache[index]
+
+    def read(self, size=-1, /):
+        """Cache-supported read operation (file object)"""
+        data = self.read_range_cached(self._pos, self._pos + size)
+        if size > 0:
+            self._pos += size
+        else:
+            self._pos = self.length
+        return data
+
+    def read_range_cached(self, start, stop):
+        """Concatenate the requested bytes from the cached chunks
+
+        This calls `get_cache_chunk` and thus downloads cache
+        chunks when necessary.
+        """
+        toread = stop - start
+        # compute the chunk indices between start and stop
+        chunk_start = np.int64(start // self._chunk_size)
+        chunk_stop = np.int64(stop // self._chunk_size + 1)
+        data = b""
+        pos = start
+        for chunk_index in range(chunk_start, chunk_stop):
+            chunk = self.get_cache_chunk(chunk_index)
+            chunk_start = pos % self._chunk_size
+            if toread == 0:
+                break
+            elif chunk_start + toread >= self._chunk_size:
+                data += chunk[chunk_start:]
+                chunks_read = self._chunk_size - chunk_start
+            else:
+                chunk_end = stop % self._chunk_size
+                data += chunk[chunk_start:chunk_end]
+                chunks_read = chunk_end - chunk_start
+            toread -= chunks_read
+            pos += chunks_read
+
+        return data
+
+    def seek(self, offset, whence=os.SEEK_SET):
+        """Seek to a position (file object)"""
+        if whence == os.SEEK_SET:
+            self._pos = offset
+        elif whence == os.SEEK_CUR:
+            self._pos += offset
+        elif whence == os.SEEK_END:
+            self._pos = self.length + offset
+
+    def seekable(self):
+        """The HTTP file is seekable"""
+        return True
+
+    def tell(self):
+        """Tell the position (file object)"""
+        return self._pos
+
+
+class ResoluteRequestsSessionCache:
+    def __init__(self):
+        """A multiprocessing-safe cache for requests session objects
+
+        This class implements empty `__getstate__` and `__setstate__`
+        methods, so that when used in a multiprocessing context, sessions
+        are never mirrored to the subprocesses. Each subprocess creates
+        its own sessions.
+
+        Note that only :class:`ResoluteRequestsSession` objects are used,
+        which is ideal for the use-case of unstable internet connections.
+        """
+        #: This dictionary holds all sessions in use by the current process.
+        #: Sessions are stored with the host name / netloc as the key.
+        self.sessions = {}
+
+    def __getstate__(self):
+        """Returns None, so sessions are not pickled into subrpocesses"""
+        pass
+
+    def __setstate__(self, state):
+        """Does nothing (see `__getstate__`)"""
+        pass
+
+    def get_session(self, url: str):
+        """Return a requests session for the specified URL
+
+        For each hostname, a different session is returned,
+        but for identical hostnames, cached sessions are used.
+        """
+        urlp = urlparse(url)
+        key = urlp.netloc
+        if key not in self.sessions:
+            self.sessions[key] = ResoluteRequestsSession()
+        return self.sessions[key]
+
+
+class ResoluteRequestsSession(requests.Session):
+    """A session with built-in retry for `get`"""
+    def get(self, *args, **kwargs):
+        kwargs.setdefault("timeout", 0.5)
+        for ii in range(100):
+            try:
+                resp = super(ResoluteRequestsSession,
+                             self).get(*args, **kwargs)
+            except (requests.exceptions.ConnectionError,
+                    requests.exceptions.ReadTimeout,
+                    requests.exceptions.ConnectTimeout,
+                    requests.urllib3.exceptions.ConnectionError,
+                    requests.urllib3.exceptions.ReadTimeoutError) as e:
+                warnings.warn(f"Encountered {e} for {args} {kwargs}",
+                              ConnectionTimeoutWarning)
+                continue
+            else:
+                break
+        else:
+            raise requests.exceptions.ReadTimeout(
+                f"Resolute session failed for {args} and {kwargs}!")
+        return resp
+
+
+def is_url_available(url: str, ret_reason=False):
+    """Check whether a URL is available
+
+    Parameters
+    ----------
+    url: str
+        full URL to the object
+    ret_reason: bool
+        whether to return reason for unavailability
+
+    Returns
+    -------
+    available: bool
+        whether the URL is available
+    reason: str
+        reason for the URL not being available is `available` is False
+    """
+    avail = False
+    reason = "none"
+    if is_http_url(url):
+        urlp = urlparse(url)
+        # default to https if no scheme or port is specified
+        port = urlp.port or (80 if urlp.scheme == "http" else 443)
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.settimeout(1)
+            # Try to connect to the host
+            try:
+                # Use `hostname`, not `netloc`, because `netloc` contains
+                # the port number which we do not want here.
+                s.connect((urlp.hostname, port))
+            except (socket.gaierror, OSError):
+                reason = "no connection"
+            else:
+                # Try to access the url
+                try:
+                    ses = session_cache.get_session(url)
+                    req = ses.get(url, stream=True, timeout=1)
+                    avail = req.ok
+                    if not avail:
+                        reason = req.reason.lower()
+                except OSError:
+                    reason = "oserror"
+                    pass
+    else:
+        reason = "invalid"
+    if ret_reason:
+        return avail, reason
+    else:
+        return avail
+
+
+@functools.lru_cache()
+def is_http_url(string):
+    """Check whether `string` is a valid URL using regexp"""
+    if not isinstance(string, str):
+        return False
+    else:
+        return REGEXP_HTTP_URL.match(string.strip())
+
+
+#: cache of requests sessions for current process
+session_cache = ResoluteRequestsSessionCache()