blisswriter 1.0.0b0__tar.gz

blisswriter-1.0.0b0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.1
+Name: blisswriter
+Version: 1.0.0b0
+Summary: Bliss NeXus-compliant HDF5 file writer
+Home-page: https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blisswriter
+Author: BCU (ESRF)
+License: LGPLv3
+Project-URL: Source, https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blisswriter
+Project-URL: Documentation, https://bliss.gitlab-pages.esrf.fr/bliss/
+Project-URL: Tracker, https://gitlab.esrf.fr/bliss/bliss/issues/
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: blissdata>=1
+Requires-Dist: pyicat-plus
+Provides-Extra: tango
+Requires-Dist: blissdata[tango]; extra == "tango"
+Provides-Extra: test
+Requires-Dist: blissdata[tango]; extra == "test"
+Requires-Dist: pytest>=7; extra == "test"
+Provides-Extra: dev
+Requires-Dist: blissdata[tango]; extra == "dev"
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: black>=22; extra == "dev"
+Requires-Dist: flake8>=4; extra == "dev"
+
+Bliss Writer
+============
+
+The `blisswriter` project provides NeXus-compliant HDF5 file writing based on
+data published by `blissdata`. It can be used in three ways
+
+* writer service for a single `bliss` session as a *process*
+  * command line interface: `NexusSessionWriter`
+  * consumer of Redis data published by `blissdata`
+* writer service for a single `bliss` session as a *Tango device*
+  * command line interface: `NexusWriterService`
+  * consumer of Redis data published by `blissdata`
+* writer python API
+  * python API: `from blisswriter.writer.main import ScanWriterWithState`
+  * same API used by the writer services
+
+`blisswriter` is developed and maintained by the [Software group](https://www.esrf.fr/Instrumentation/software)
+of the [European Synchrotron](https://www.esrf.fr/).

blisswriter-1.0.0b0/README.md

@@ -0,0 +1,18 @@
+Bliss Writer
+============
+
+The `blisswriter` project provides NeXus-compliant HDF5 file writing based on
+data published by `blissdata`. It can be used in three ways
+
+* writer service for a single `bliss` session as a *process*
+  * command line interface: `NexusSessionWriter`
+  * consumer of Redis data published by `blissdata`
+* writer service for a single `bliss` session as a *Tango device*
+  * command line interface: `NexusWriterService`
+  * consumer of Redis data published by `blissdata`
+* writer python API
+  * python API: `from blisswriter.writer.main import ScanWriterWithState`
+  * same API used by the writer services
+
+`blisswriter` is developed and maintained by the [Software group](https://www.esrf.fr/Instrumentation/software)
+of the [European Synchrotron](https://www.esrf.fr/).

blisswriter-1.0.0b0/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["setuptools>=46.4", "wheel"]

blisswriter-1.0.0b0/setup.cfg

@@ -0,0 +1,62 @@
+[metadata]
+name = blisswriter
+version = attr: blisswriter.__version__
+author = BCU (ESRF)
+description = Bliss NeXus-compliant HDF5 file writer
+long_description = file: README.md
+long_description_content_type = text/markdown
+license = LGPLv3
+url = https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blisswriter
+project_urls = 
+	Source =  https://gitlab.esrf.fr/bliss/bliss/-/tree/master/blisswriter
+	Documentation = https://bliss.gitlab-pages.esrf.fr/bliss/
+	Tracker = https://gitlab.esrf.fr/bliss/bliss/issues/
+classifiers = 
+	Intended Audience :: Science/Research
+	License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
+	Programming Language :: Python :: 3
+
+[options]
+package_dir = 
+	=src
+packages = find:
+python_requires = >=3.7
+install_requires = 
+	blissdata >=1
+	pyicat-plus
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+console_scripts = 
+	NexusWriterService = blisswriter.app.service:main
+	NexusSessionWriter = blisswriter.app.process:main
+	RegisterNexusWriter = blisswriter.app.register_service:main
+
+[options.extras_require]
+tango = 
+	blissdata[tango]
+test = 
+	%(tango)s
+	pytest >=7
+dev = 
+	%(test)s
+	black >=22
+	flake8 >=4
+
+[flake8]
+ignore = E501, E203, W503
+max-line-length = 88
+exclude = 
+	.eggs
+
+[coverage:run]
+omit = 
+	setup.py
+	*/tests/*
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

blisswriter-1.0.0b0/setup.py

@@ -0,0 +1,4 @@
+import setuptools
+
+if __name__ == "__main__":
+    setuptools.setup()

blisswriter-1.0.0b0/src/blisswriter/__init__.py

@@ -0,0 +1,4 @@
+"""NeXus-compliant HDF5 writer for Bliss
+"""
+
+__version__ = "1.0.0b0"

blisswriter-1.0.0b0/src/blisswriter/app/process.py

@@ -0,0 +1,44 @@
+import sys
+from blisswriter.parameters import all_cli_saveoptions
+from blisswriter.subscribers.session_subscriber import NexusSessionSubscriber
+from blisswriter.utils import logging_utils
+from .utils import config_root_logger
+
+
+def main(argv=None):
+    import argparse
+
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(
+        description="Start python a process for saving Bliss data in HDF5"
+    )
+    parser.add_argument("session_name", type=str, help="Session name")
+    cli_saveoptions = all_cli_saveoptions()
+    for attr, okwargs in cli_saveoptions.items():
+        parser.add_argument("--" + attr, **okwargs)
+    logging_utils.add_cli_args(parser)
+
+    # Parse CLI arguments
+    args = parser.parse_args(argv[1:])
+    config_root_logger(args)
+    kwargs = {}
+    cli_saveoptions = all_cli_saveoptions(configurable=args.configurable)
+    for attr, okwargs in cli_saveoptions.items():
+        option = okwargs["dest"]
+        try:
+            kwargs[option] = getattr(args, option)
+        except AttributeError:
+            continue
+
+    # Launch the session writer
+    subscriber = NexusSessionSubscriber(
+        args.session_name, writer_name="external", **kwargs
+    )
+    subscriber.start()
+    subscriber.join()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

blisswriter-1.0.0b0/src/blisswriter/app/register_service.py

@@ -0,0 +1,210 @@
+import os
+import sys
+import logging
+
+from tango import DeviceProxy, DevFailed, Database, DbDevInfo
+
+from ..utils import logging_utils
+from .utils import config_root_logger
+
+logger = logging.getLogger(__name__)
+
+
+WRITER_CLASS = "NexusWriter"
+
+
+def beamline():
+    """
+    :returns str:
+    """
+    name = "id00"
+    for k in "BEAMLINENAME", "BEAMLINE":
+        name = os.environ.get(k, name)
+    return name.lower()
+
+
+def find_session_writers(session_name, db=None):
+    """
+    Find all TANGO devices of class NexusWriter listening to a particular BLISS session.
+
+    :param std session_name:
+    :param Database db:
+    :returns list(str): device names
+    """
+    if db is None:
+        db = Database()
+    writers = []
+    for obj_name in db.get_device_name("*", WRITER_CLASS):
+        prop = db.get_device_property(obj_name, "session")["session"]
+        if prop:
+            if session_name == prop[0]:
+                writers.append(obj_name)
+    logger.info(
+        "Registered writers for session {}: {}".format(repr(session_name), writers)
+    )
+    return writers
+
+
+def find_session_writer(session_name, db=None):
+    """
+    Find TANGO device of class NexusWriter listening to a particular BLISS session.
+
+    :param std session_name:
+    :param Database db:
+    :returns str or None: device name
+    :raises RuntimeError: more than one writer listening to session
+    """
+    writers = find_session_writers(session_name, db=db)
+    if not writers:
+        return None
+    if len(writers) > 1:
+        raise ValueError(
+            "Found more than one writer for session {}: {}".format(
+                repr(session_name), writers
+            )
+        )
+    return writers[0]
+
+
+def get_uri(p):
+    return "tango://{}:{}/{}".format(p.get_db_host(), p.get_db_port(), p.dev_name())
+
+
+def ensure_existence(
+    session_name,
+    server="nexuswriter",
+    instance=None,
+    domain=None,
+    family="bliss_nxwriter",
+    member=None,
+    use_existing=True,
+):
+    """
+    Find or register TANGO device of class NexusWriter
+
+    :param str session_name:
+    :param str server: device server name
+    :param str instance: device server instance (Default: `session_name`)
+    :param str domain: location of device
+    :param str family: type of device
+    :param str member: device name (Default: `session_name`)
+    :param bool use_existing: use an existing writer for this session (if any)
+    :returns DeviceProxy:
+    """
+    db = Database()
+    if not member:
+        member = session_name
+    if not domain:
+        domain = beamline()
+    if not instance:
+        instance = session_name
+    dev_name = "/".join([domain, family, member])
+    if use_existing:
+        pdev_name = find_session_writer(session_name, db=db)
+        if pdev_name:
+            proxy = DeviceProxy(pdev_name)
+            msg = "{} already registered".format(repr(get_uri(proxy)))
+            if dev_name == pdev_name:
+                logger.info(msg)
+            else:
+                logger.warning(msg)
+            return proxy
+    return register(session_name, dev_name, server=server, instance=instance)
+
+
+def register(session_name, dev_name, server="nexuswriter", instance="nexuswriters"):
+    """
+    Register TANGO device of class NexusWriter
+
+    :param str session_name:
+    :param str dev_name: for example id00/bliss_nxwriter/test_session
+    :param str instance: device server instance
+    :returns DeviceProxy:
+    """
+    # Register proxy is not already registered
+    try:
+        proxy = DeviceProxy(dev_name)
+        logger.info("{} already registered".format(repr(get_uri(proxy))))
+    except DevFailed:
+        db = Database()
+        dev_info = DbDevInfo()
+        dev_info.name = dev_name
+        dev_info._class = WRITER_CLASS
+        server = "/".join([server, instance])
+        dev_info.server = server
+        db.add_device(dev_info)
+        proxy = DeviceProxy(dev_name)
+        logger.info("{} registered".format(repr(get_uri(proxy))))
+        proxy.put_property({"session": session_name})
+    try:
+        session = proxy.get_property("session")["session"][0]
+    except (IndexError, KeyError):
+        session = ""
+    if session != session_name:
+        raise RuntimeError(
+            "{} is listening to Bliss session {} instead of {}".format(
+                repr(get_uri(proxy)), repr(session), repr(session_name)
+            )
+        )
+    return proxy
+
+
+def main(argv=None):
+    import argparse
+
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(
+        description="Register a Tango device saving Bliss data in HDF5"
+    )
+    parser.add_argument("session", type=str, help="Bliss session name")
+    parser.add_argument(
+        "--server",
+        type=str,
+        default="nexuswriter",
+        help="Server name ('nexuswriter' by default)",
+    )
+    parser.add_argument(
+        "--instance",
+        type=str,
+        default="",
+        help="Server instance name (session name by default)",
+    )
+    parser.add_argument(
+        "--domain",
+        type=str,
+        default="",
+        help="Device domain name (checks environment or 'id00' by default)",
+    )
+    parser.add_argument(
+        "--family",
+        type=str,
+        default="bliss_nxwriter",
+        help="Device family name ('bliss_nxwriter' by default)",
+    )
+    parser.add_argument(
+        "--member", type=str, default="", help="Device name (session name by default)"
+    )
+    parser.add_argument(
+        "--ignore_existing",
+        action="store_false",
+        dest="use_existing",
+        help="Ignore existing writer for this session",
+    )
+    logging_utils.add_cli_args(parser)
+    args = parser.parse_args(argv[1:])
+    config_root_logger(args)
+    ensure_existence(
+        args.session,
+        server=args.server,
+        instance=args.instance,
+        domain=args.domain,
+        family=args.family,
+        member=args.member,
+        use_existing=args.use_existing,
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main())

blisswriter-1.0.0b0/src/blisswriter/app/service.py

@@ -0,0 +1,54 @@
+import sys
+from blisswriter.tango.servers import NexusWriter
+from blisswriter.utils.logging_utils import add_cli_args
+from blisswriter.utils.log_levels import tango_cli_slog_level
+from blisswriter.utils.log_levels import add_tango_cli_args
+from .utils import config_root_logger
+
+
+def run(server, instance, log_level):
+    """
+    :param str server: device server name
+    :param str instance: device server instance name
+    :param str log_level:
+    :returns Util:
+    """
+    verbose = tango_cli_slog_level.get(log_level, 0)
+    if verbose:
+        verbose = "-v{:d}".format(verbose)
+        serverargs = [server, instance, verbose]
+    else:
+        serverargs = [server, instance]
+    return NexusWriter.main(args=serverargs)
+
+
+def main(argv=None):
+    import argparse
+
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(
+        description="Start a Tango server for saving Bliss data in HDF5"
+    )
+    parser.add_argument(
+        "instance",
+        type=str,
+        default="nexuswriters",
+        help="Server instance name ('nexuswriters' by default)",
+    )
+    parser.add_argument(
+        "--server",
+        type=str,
+        default="nexuswriter",
+        help="Server name ('nexuswriter' by default)",
+    )
+    add_cli_args(parser)
+    add_tango_cli_args(parser)
+    args = parser.parse_args(argv[1:])
+    config_root_logger(args)
+    run(args.server, args.instance, args.log_tango)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

blisswriter-1.0.0b0/src/blisswriter/app/utils.py

@@ -0,0 +1,7 @@
+import logging
+from blisswriter.utils import logging_utils
+
+
+def config_root_logger(args):
+    logger = logging.getLogger("blisswriter")
+    logging_utils.cliconfig(logger, args)

blisswriter-1.0.0b0/src/blisswriter/io/__init__.py

@@ -0,0 +1,2 @@
+"""Nexus writer I/O utilities
+"""

blisswriter-1.0.0b0/src/blisswriter/io/h5_config.py

@@ -0,0 +1,175 @@
+"""HDF5 configuration of optimal data storage (IO speed, compression, ...)
+"""
+
+from typing import Optional, Tuple
+from numbers import Integral
+from collections.abc import Mapping
+
+import numpy
+from numpy.typing import DTypeLike
+
+try:
+    import hdf5plugin
+except ImportError:
+    hdf5plugin = None
+
+from ..utils.array_size import ShapeType
+from ..utils import array_size
+
+VarShapeType = Tuple[array_size.Integral]  # <=0 marks a variable dimension
+VarH5pyShapeType = Tuple[
+    Optional[array_size.StrictPositiveIntegral]
+]  # None marks a variable dimension
+
+DEFAULT_CHUNK_NBYTES = 1 << 20
+DEFAULT_COMPRESSION_LIMIT_NBYTES = 1 << 20
+DEFAULT_CHUNK_SPLIT = 4
+DEFAULT_COMPRESSION_SCHEME = "gzip-byteshuffle"
+
+# Default data size is 2 MB for a 1D detector
+# and 2 KB for a 0D detector:
+DEFAULT_SCAN_DIM_SIZE = 512
+DEFAULT_DETECTOR_DIM_SIZE = 1024
+DEFAULT_DTYPE = numpy.int32
+
+
+def guess_data_shape(
+    scan_shape: VarShapeType,
+    detector_shape: VarShapeType,
+    max_shape: Optional[VarH5pyShapeType],
+) -> ShapeType:
+    scan_shape = tuple(n if n > 0 else DEFAULT_SCAN_DIM_SIZE for n in scan_shape)
+    detector_shape = tuple(
+        n if n > 0 else DEFAULT_SCAN_DIM_SIZE for n in detector_shape
+    )
+    data_shape = scan_shape + detector_shape
+    if max_shape:
+        assert len(max_shape) == len(
+            data_shape
+        ), "HDF5 dataset shape must have the same dimensions as maxshape"
+        data_shape = tuple(
+            n1 if not n2 else max(n1, n2) for n1, n2 in zip(data_shape, max_shape)
+        )
+    return data_shape
+
+
+def guess_chunk_shape(
+    data_shape: ShapeType,
+    dtype: DTypeLike,
+    chunk_split: Optional[Integral] = None,
+    chunk_nbytes: Optional[Integral] = None,
+) -> Optional[ShapeType]:
+    """Try to guess the optimal chunk shape with these constraints:
+    * Split any dimension for partial access
+    * Below the maximal chunk size (1 MB by default, uncompressed)
+
+    The inner-most dimensions are split in `chunk_split` parts
+    until chunk_nbytes is reached. The chunk size in the outer
+    dimensions will be 1, unless the data size is to small.
+    """
+    if chunk_nbytes is None:
+        chunk_nbytes = DEFAULT_CHUNK_NBYTES
+    if chunk_split is None:
+        chunk_split = DEFAULT_CHUNK_SPLIT
+    itemsize = array_size.dtype_nbytes(dtype)
+    size = array_size.shape_to_size(data_shape)
+    nbytes = size * itemsize
+    if nbytes <= chunk_nbytes:
+        return None
+
+    max_size = chunk_nbytes // itemsize
+    current_size = 1
+    chunk_shape = []
+    for n_i in data_shape[-1::-1]:
+        if current_size >= max_size:
+            c_i = 1
+        else:
+            a = int(numpy.ceil(n_i / chunk_split))
+            b = int(numpy.ceil(max_size / current_size))
+            c_i = min(a, b)
+        chunk_shape.append(c_i)
+        current_size *= c_i
+    chunk_shape = tuple(chunk_shape[::-1])
+    if chunk_shape == data_shape:
+        return None
+    return chunk_shape
+
+
+def guess_compression(
+    data_shape: ShapeType,
+    dtype: DTypeLike,
+    compression_limit_nbytes: Optional[Integral] = None,
+) -> bool:
+    """Compression is needed when the total data size exceeds the limits (1 MB by default)."""
+    if compression_limit_nbytes is None:
+        compression_limit_nbytes = DEFAULT_COMPRESSION_LIMIT_NBYTES
+    nbytes = array_size.shape_to_nbytes(data_shape, dtype)
+    return nbytes > compression_limit_nbytes
+
+
+def get_compression_arguments(compression_scheme: Optional[str] = None) -> Mapping:
+    if compression_scheme:
+        compression_scheme = compression_scheme.lower()
+    if compression_scheme is None:
+        compression_scheme = DEFAULT_COMPRESSION_SCHEME
+    if compression_scheme == "none":
+        return dict()
+    elif compression_scheme == "gzip":
+        return {"compression": "gzip"}
+    elif compression_scheme == "byteshuffle":
+        return {"shuffle": True}
+    elif compression_scheme == "gzip-byteshuffle":
+        return {"compression": "gzip", "shuffle": True}
+    elif compression_scheme == "bitshuffle":
+        if hdf5plugin is None:
+            raise RuntimeError(
+                "Writer does not support HDF5 'bitshuffle' compression. Install the hdf5plugin library"
+            )
+        return hdf5plugin.Bitshuffle(nelems=0, lz4=False)
+    elif compression_scheme == "lz4-bitshuffle":
+        if hdf5plugin is None:
+            raise RuntimeError(
+                "Writer does not support HDF5 'bitshuffle' compression. Install the hdf5plugin library"
+            )
+        return hdf5plugin.Bitshuffle(nelems=0, lz4=True)
+    else:
+        raise ValueError(f"Unknown HDF5 compression '{compression_scheme}'")
+
+
+def guess_dataset_config(
+    scan_shape: VarShapeType,
+    detector_shape: VarShapeType,
+    dtype: Optional[DTypeLike] = None,
+    chunk_split: Optional[Integral] = None,
+    chunk_nbytes: Optional[Integral] = None,
+    compression_limit_nbytes: Optional[Integral] = None,
+    compression_scheme: Optional[str] = None,
+    max_shape: Optional[VarH5pyShapeType] = None,
+) -> dict:
+    """Dataset configuration passed the `h5py.Group.create_dataset` for optimal
+    storage (IO speed, compression, ...)
+    """
+    data_shape = guess_data_shape(
+        scan_shape=scan_shape, detector_shape=detector_shape, max_shape=max_shape
+    )
+    if dtype is None:
+        dtype = DEFAULT_DTYPE
+    chunk_shape = guess_chunk_shape(
+        data_shape=data_shape,
+        dtype=dtype,
+        chunk_split=chunk_split,
+        chunk_nbytes=chunk_nbytes,
+    )
+    config = {"chunks": chunk_shape}
+    compression = guess_compression(
+        data_shape=data_shape,
+        dtype=dtype,
+        compression_limit_nbytes=compression_limit_nbytes,
+    )
+    if compression:
+        config.update(get_compression_arguments(compression_scheme=compression_scheme))
+    chunking_required = compression or max_shape is not None
+    if chunking_required and chunk_shape is None:
+        # Do not let h5py guess the chunk size
+        config["chunks"] = data_shape
+    return config