blissdata 0.3.4__py3-none-any.whl

blissdata/__init__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+"""Blissdata package
+
+.. autosummary::
+    :toctree:
+
+    common
+    data
+"""
+
+__version__ = "0.3.4"

blissdata/beacon/__init__.py

@@ -0,0 +1 @@
+"""Beacon communication"""

blissdata/beacon/_base.py

@@ -0,0 +1,106 @@
+"""Base client to communicate with Beacon."""
+
+import platform
+import socket
+import struct
+import threading
+from typing import Optional
+from . import config
+
+
+class IncompleteBeaconMessage(Exception):
+    """Raised when a received message is incomplete"""
+
+
+class BeaconClient:
+    """Synchronous blocking Beacon client.
+
+    It takes a host and port to a beacon server to be instantiated or
+    uses the BEACON_HOST environment variable when when missing.
+    """
+
+    HEADER_SIZE = struct.calcsize("<ii")
+
+    def __init__(
+        self, host: Optional[str] = None, port: Optional[int] = None, timeout=3.0
+    ):
+        if host is None or port is None:
+            self._address = config.get_beacon_address()
+        else:
+            self._address = host, port
+        connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if platform.system() != "Windows":
+            connection.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10)
+        connection.connect(self._address)
+        connection.settimeout(timeout)
+        self._connection = connection
+        self._cursor_id = 0
+        self._lock = threading.Lock()
+
+    def __repr__(self) -> str:
+        return (
+            f"{type(self).__name__}(host='{self._address[0]}', port={self._address[1]})"
+        )
+
+    def close(self):
+        """Close the connection to Beacon."""
+        self._connection.close()
+        self._connection = None
+
+    def _request(self, message_id, param1):
+        """Send a request and returns a response object"""
+        message_key = self._gen_message_key()
+        content = f"{message_key}|{param1}".encode()
+        header = struct.pack("<ii", message_id, len(content))
+        msg = b"%s%s" % (header, content)
+        self._connection.sendall(msg)
+        client = self
+
+        class Response:
+            def read(self):
+                return client._read(message_key)
+
+        return Response()
+
+    def _gen_message_key(self):
+        """Generate a unique message key.
+
+        This is not really needed for a synchronous service.
+        It could be a fixed value.
+        """
+        self._cursor_id = (self._cursor_id + 1) % 100000
+        return "%s" % self._cursor_id
+
+    def _unpack_message(self, s):
+        header_size = self.HEADER_SIZE
+        if len(s) < header_size:
+            raise IncompleteBeaconMessage
+        message_type, message_len = struct.unpack("<ii", s[:header_size])
+        if len(s) < header_size + message_len:
+            raise IncompleteBeaconMessage
+        message = s[header_size : header_size + message_len]
+        remaining = s[header_size + message_len :]
+        return message_type, message, remaining
+
+    def _read(self, expected_message_key):
+        data = b""
+        while True:
+            raw_data = self._connection.recv(16 * 1024)
+            if not raw_data:
+                break
+            data = b"%s%s" % (data, raw_data)
+            try:
+                message_type, message, data = self._unpack_message(data)
+            except IncompleteBeaconMessage:
+                continue
+            break
+        message_key, data = self._get_msg_key(message)
+        if message_key != expected_message_key:
+            raise RuntimeError(f"Unexpected message key '{message_key}'")
+        return message_type, data
+
+    def _get_msg_key(self, message):
+        pos = message.find(b"|")
+        if pos < 0:
+            return message.decode(), None
+        return message[:pos].decode(), message[pos + 1 :]

blissdata/beacon/config.py

@@ -0,0 +1,24 @@
+"""Beacon configuration"""
+
+import os
+from typing import Tuple
+
+
+def get_beacon_address() -> Tuple[str, int]:
+    """Beacon address from the environment var `BEACON_HOST`.
+
+    For example `('foobar', 25000)`.
+
+    Raises:
+        ValueError: If BEACON_HOST is missing or not properly set
+    """
+    beacon_host = os.environ.get("BEACON_HOST")
+    if beacon_host is None:
+        raise ValueError("BEACON_HOST is not specified")
+    try:
+        host, port = beacon_host.split(":")
+        return host, int(port)
+    except Exception:
+        raise ValueError(
+            f"BEACON_HOST variable not properly set. Expected: 'hostname:port'. Found: from '{beacon_host}'."
+        )

blissdata/beacon/data.py

@@ -0,0 +1,45 @@
+"""Get Bliss data information from Beacon."""
+
+import struct
+from ._base import BeaconClient
+from ._base import IncompleteBeaconMessage
+
+
+class BeaconData(BeaconClient):
+    """Provides the API to read the redis databases urls."""
+
+    REDIS_QUERY = 30
+    REDIS_QUERY_ANSWER = 31
+
+    REDIS_DATA_SERVER_QUERY = 32
+    REDIS_DATA_SERVER_FAILED = 33
+    REDIS_DATA_SERVER_OK = 34
+
+    def get_redis_db(self) -> str:
+        """Returns the URL of the Redis database that contains the Bliss settings. For example 'foobar:25001'."""
+        msg = b"%s%s" % (struct.pack("<ii", self.REDIS_QUERY, 0), b"")
+        self._connection.sendall(msg)
+        data = b""
+        while True:
+            raw_data = self._connection.recv(16 * 1024)
+            if not raw_data:
+                break
+            data = b"%s%s" % (data, raw_data)
+            try:
+                message_type, message, data = self._unpack_message(data)
+            except IncompleteBeaconMessage:
+                continue
+            break
+        if message_type != self.REDIS_QUERY_ANSWER:
+            raise RuntimeError(f"Unexpected message type '{message_type}'")
+        return message.decode()
+
+    def get_redis_data_db(self) -> str:
+        """Returns the URL of the Redis database that contains the Bliss scan data. For example 'foobar:25002'."""
+        response = self._request(self.REDIS_DATA_SERVER_QUERY, "")
+        response_type, data = response.read()
+        if response_type == self.REDIS_DATA_SERVER_OK:
+            return data.decode().replace("|", ":", 1)
+        elif response_type == self.REDIS_DATA_SERVER_FAILED:
+            raise RuntimeError(data.decode())
+        raise RuntimeError(f"Unexpected Beacon response type {response_type}")

blissdata/beacon/files.py

@@ -0,0 +1,141 @@
+"""Get files from Beacon."""
+
+import sys
+import json
+from typing import Any
+from urllib.parse import urlparse, ParseResult
+from ._base import BeaconClient
+
+try:
+    import ruamel.yaml
+
+    yaml_load = ruamel.yaml.YAML().load
+except ImportError:
+    try:
+        from yaml import safe_load as yaml_load
+    except ImportError:
+        yaml_load = None
+
+
+def read_config(url: str) -> Any:
+    """
+    Read configuration from a url.
+
+    In case of a Beacon url with missing host and port, the Beacon
+    server will be found from environment variable `BEACON_HOST`.
+
+    Arguments:
+        url: This can be a local yaml file (for example `/path/to/file.yaml`, `file:///path/to/file.yaml`)
+             or a Beacon URL (for example `beacon:///path/to/file.yml`, `beacon://id00:25000/path/to/file.yml`).
+    Returns:
+        A Python dict/list structure
+    """
+    url = _parse_config_url(url)
+    if url.scheme == "beacon":
+        return _read_config_beacon(url)
+    elif url.scheme in ("file", ""):
+        return _read_config_yaml(url)
+    else:
+        raise ValueError(
+            f"Configuration URL scheme '{url.scheme}' is not supported (Full URL: {url})"
+        )
+
+
+def _parse_config_url(url: str) -> ParseResult:
+    presult = urlparse(url)
+    if presult.scheme == "beacon":
+        # beacon:///path/to/file.yml
+        # beacon://id00:25000/path/to/file.yml
+        return presult
+    elif presult.scheme in ("file", ""):
+        # /path/to/file.yaml
+        # file:///path/to/file.yaml
+        return presult
+    elif sys.platform == "win32" and len(presult.scheme) == 1:
+        # c:\\path\\to\\file.yaml
+        return urlparse(f"file://{url}")
+    else:
+        return presult
+
+
+def _url_to_filename(url: ParseResult) -> str:
+    if url.netloc and url.path:
+        # urlparse("file://c:/a/b")
+        return url.netloc + url.path
+    elif url.netloc:
+        # urlparse("file://c:\\a\\b")
+        return url.netloc
+    else:
+        return url.path
+
+
+def _read_config_beacon(url: ParseResult) -> Any:
+    if url.netloc:
+        host, port = url.netloc.split(":")
+        port = int(port)
+    else:
+        host = None
+        port = None
+
+    # Bliss < 1.11: Beacon cannot handle leading slashes
+    file_path = url.path
+    while file_path.startswith("/"):
+        file_path = file_path[1:]
+
+    beacon = BeaconFiles(host=host, port=port)
+    try:
+        config = beacon.get_file(file_path)
+        return yaml_load(config)
+    finally:
+        beacon.close()
+
+
+def _read_config_yaml(url: ParseResult) -> Any:
+    if yaml_load is None:
+        raise ImportError(
+            "No yaml parser available. Try to install 'ruamel.yaml' or 'pyyaml'"
+        )
+    filename = _url_to_filename(url)
+    with open(filename, "r") as f:
+        return yaml_load(f)
+
+
+class BeaconFiles(BeaconClient):
+    """Provides the API to read files managed by Beacon."""
+
+    CONFIG_GET_FILE = 50
+    CONFIG_GET_FILE_FAILED = 51
+    CONFIG_GET_FILE_OK = 52
+
+    CONFIG_GET_DB_TREE = 86
+    CONFIG_GET_DB_TREE_FAILED = 87
+    CONFIG_GET_DB_TREE_OK = 88
+
+    def get_file(self, file_path: str) -> bytes:
+        """Returns the binary content of a file from the Beacon configuration
+        file repository."""
+        with self._lock:
+            response = self._request(self.CONFIG_GET_FILE, file_path)
+            response_type, data = response.read()
+            if response_type == self.CONFIG_GET_FILE_OK:
+                return data
+            elif response_type == self.CONFIG_GET_FILE_FAILED:
+                raise RuntimeError(data.decode())
+            raise RuntimeError(f"Unexpected Beacon response type {response_type}")
+
+    def get_tree(self, base_path: str = "") -> dict:
+        """Returns the file tree from a base path of the Beacon configuration
+        file repository.
+
+        Return: A nested dictionary structure, where a file is a mapping
+                `filename: None`, an a directory is mapping of a dirname and a
+                nested dictionary.
+        """
+        with self._lock:
+            response = self._request(self.CONFIG_GET_DB_TREE, base_path)
+            response_type, data = response.read()
+            if response_type == self.CONFIG_GET_DB_TREE_OK:
+                return json.loads(data)
+            elif response_type == self.CONFIG_GET_DB_TREE_FAILED:
+                raise RuntimeError(data.decode())
+            raise RuntimeError(f"Unexpected Beacon response type {response_type}")

blissdata/client.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+from typing import Optional
+from blissdata.beacon.data import BeaconData
+from blissdata.redis.manager import RedisConnectionManager, RedisAddress
+
+
+_default_redis_connection_manager_callback = None
+
+
+def set_default_redis_connection_manager_callback(cb):
+    global _default_redis_connection_manager_callback
+    _default_redis_connection_manager_callback = cb
+
+
+def configure_with_beacon_address(
+    host: Optional[str] = None, port: Optional[int] = None
+):
+    beacon_client = BeaconData(host=host, port=port)
+    addresses = {
+        0: RedisAddress.factory(beacon_client.get_redis_db()),
+        1: RedisAddress.factory(beacon_client.get_redis_data_db()),
+    }
+    redis_connection_manager = RedisConnectionManager(addresses)
+
+    def redis_connection_manager_cb():
+        return redis_connection_manager
+
+    set_default_redis_connection_manager_callback(redis_connection_manager_cb)
+
+
+def _get_default_redis_connection_manager():
+    global _default_redis_connection_manager_callback
+    if _default_redis_connection_manager_callback is None:
+        try:
+            configure_with_beacon_address()
+        except Exception as e:
+            raise RuntimeError("Blissdata configuration from BEACON_HOST failed") from e
+    return _default_redis_connection_manager_callback()
+
+
+def get_redis_proxy(db: int = 0, caching: bool = False, shared: bool = True):
+    """Get a greenlet-safe proxy to a Redis database.
+
+    :param int db: Redis database too which we need a proxy
+    :param bool caching: client-side caching
+    :param bool shared: use a shared proxy held by the Beacon connection
+    """
+    return _get_default_redis_connection_manager().get_db_proxy(
+        db=db, caching=caching, shared=shared
+    )
+
+
+def close_all_redis_connections():
+    default_redis_connection_manager = _get_default_redis_connection_manager()
+    if _get_default_redis_connection_manager() is not None:
+        default_redis_connection_manager.close_all_connections()

blissdata/common/__init__.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+
+"""Common function section
+
+.. autosummary::
+    :toctree:
+
+    utils
+"""

blissdata/common/utils.py

@@ -0,0 +1,159 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+import fnmatch
+import re
+from collections.abc import Mapping
+
+
+class UndefinedType:
+    __slots__ = []
+
+
+Undefined = UndefinedType()
+"""Can be used as default function argument when a `None` value is a valid
+and optional input. For example for a default value from a `dict.get` method."""
+
+
+def deep_update(d, u):
+    """Do a deep merge of one dict into another.
+
+    This will update d with values in u, but will not delete keys in d
+    not found in u at some arbitrary depth of d. That is, u is deeply
+    merged into d.
+
+    Args -
+      d, u: dicts
+
+    Note: this is destructive to d, but not u.
+
+    Returns: None
+    """
+    stack = [(d, u)]
+    while stack:
+        d, u = stack.pop(0)
+        for k, v in u.items():
+            if not isinstance(v, Mapping):
+                # u[k] is not a dict, nothing to merge, so just set it,
+                # regardless if d[k] *was* a dict
+                d[k] = v
+            else:
+                # note: u[k] is a dict
+
+                # get d[k], defaulting to a dict, if it doesn't previously
+                # exist
+                dv = d.setdefault(k, {})
+
+                if not isinstance(dv, Mapping):
+                    # d[k] is not a dict, so just set it to u[k],
+                    # overriding whatever it was
+                    d[k] = v
+                else:
+                    # both d[k] and u[k] are dicts, push them on the stack
+                    # to merge
+                    stack.append((dv, v))
+
+
+def grouped(iterable, n):
+    """
+    Group elements of an iterable n by n.
+    Return a zip object.
+    s -> (s0,s1,s2,...sn-1), (sn,sn+1,sn+2,...s2n-1), (s2n,s2n+1,s2n+2,...s3n-1), ...
+    Excedentary elements are discarded.
+    Example:
+    DEMO [5]: list(grouped([1,2,3,4,5], 2))
+    Out  [5]: [(1, 2), (3, 4)]
+    """
+    return zip(*[iter(iterable)] * n)
+
+
+def update_node_info(node, d):
+    """Updates the BaseHashSetting of a DataNode and does a deep update if needed.
+    parameters: node: DataNode or DataNodeContainer; d: dict"""
+    assert isinstance(d, Mapping)
+    for key, value in d.items():
+        tmp = node.info.get(key)
+        if tmp and isinstance(value, Mapping) and isinstance(tmp, Mapping):
+            deep_update(tmp, value)
+            node.info[key] = tmp
+        else:
+            node.info[key] = value
+
+
+def get_matching_names(patterns, names, strict_pattern_as_short_name=False):
+    """Search a pattern into a list of names (unix pattern style).
+
+    .. list-table::
+       :header-rows: 1
+
+       * - Pattern
+         - Meaning
+       * - `*`
+         - matches everything
+       * - `?`
+         - matches any single character
+       * - `[seq]`
+         - matches any character in seq
+       * - `[!seq]`
+         - matches any character not in seq
+
+    Arguments:
+        patterns: a list of patterns
+        names: a list of names
+        strict_pattern_as_short_name: if True patterns without special character,
+                                      are transformed like this: `'pattern' -> '*:pattern'`
+                                      (as the 'short name' part of a 'fullname')
+
+    Return: dict { pattern : matching names }
+    """
+
+    special_char = ["*", ":"]
+
+    if not isinstance(patterns, (list, tuple)):
+        patterns = [patterns]
+
+    matches = {}
+    for pat in patterns:
+
+        if not isinstance(pat, str):
+            pat = str(pat)
+
+        sub_pat = [pat]
+
+        if strict_pattern_as_short_name:
+            if all([sc not in pat for sc in special_char]):
+                sub_pat = [f"*:{pat}", f"*:{pat}:*", f"{pat}:*"]
+
+        # store the fullname of matching counters
+        matching_names = []
+        for _pat in sub_pat:
+
+            for name in names:
+                if fnmatch.fnmatch(name, _pat):
+                    matching_names.append(name)
+
+            if matching_names:
+                break
+
+        matches[pat] = matching_names
+
+    return matches
+
+
+def natural_sort(words):
+    def convert(text):
+        return int(text) if text.isdigit() else text.lower()
+
+    def alphanum_key(key):
+        return [
+            convert(c)
+            for c in re.split(
+                "([0-9]+)", key.decode() if isinstance(key, bytes) else key
+            )
+        ]
+
+    return sorted(words, key=alphanum_key)

blissdata/data/__init__.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2023 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+"""Data management
+
+.. autosummary::
+    :toctree:
+
+    nodes
+    scan
+"""

blissdata/data/events/__init__.py

@@ -0,0 +1,22 @@
+"""
+Raw Redis stream event decoding/encoding for all data nodes
+Importing this module will "register" all stream events.
+"""
+
+from . import node
+from .node import *  # noqa: F401,F403
+from . import channel
+from .channel import *  # noqa: F401,F403
+from . import lima
+from .lima import *  # noqa: F401,F403
+from . import scan
+from .scan import *  # noqa: F401,F403
+from . import walk
+from .walk import *  # noqa: F401,F403
+
+__all__ = []
+__all__.extend(node.__all__)
+__all__.extend(channel.__all__)
+__all__.extend(lima.__all__)
+__all__.extend(scan.__all__)
+__all__.extend(walk.__all__)