ewoksdata 1.2.0__tar.gz → 1.3.0__tar.gz

{ewoksdata-1.2.0/src/ewoksdata.egg-info → ewoksdata-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 1.2.0
+Version: 1.3.0
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License
@@ -43,6 +43,7 @@ Requires-Dist: blissdata>=0.3.4
 Requires-Dist: redis
 Requires-Dist: redis!=5.3.0b1,!=5.3.0b3
 Requires-Dist: redis-om!=1.0.0b0; python_version < "3.9"
+Requires-Dist: redis-om!=1.0.3b0; python_version < "3.9"
 Provides-Extra: online
 Requires-Dist: blissdata[tango]; extra == "online"
 Provides-Extra: test

{ewoksdata-1.2.0 → ewoksdata-1.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ewoksdata"
-version = "1.2.0"
+version = "1.3.0"
 authors = [{ name = "ESRF", email = "dau-pydev@esrf.fr" }]
 description = "Ewoks tasks for data access"
 readme = { file = "README.md", content-type = "text/markdown" }
@@ -24,6 +24,7 @@ dependencies = [
     "redis",                     # missing dependency in blissdata<1
     "redis !=5.3.0b1,!=5.3.0b3",
     "redis-om !=1.0.0b0;python_version < '3.9'",  # https://github.com/redis/redis-om-python/issues/693
+    "redis-om !=1.0.3b0;python_version < '3.9'",  # https://github.com/redis/redis-om-python/issues/706
 ]
 
 [project.urls]

{ewoksdata-1.2.0 → ewoksdata-1.3.0}/src/ewoksdata/data/bliss.py

@@ -139,4 +139,10 @@ def iter_bliss_data(
 
 
 def _is_bliss_file(h5item: Union[h5py.Dataset, h5py.Group]) -> bool:
-    return h5item.file.attrs.get("creator", "").lower() == "bliss"
+    return (
+        h5item.file.attrs.get("creator", "").lower() in _BLISS_PUBLISHERS
+        or h5item.file.attrs.get("publisher", "").lower() in _BLISS_PUBLISHERS
+    )
+
+
+_BLISS_PUBLISHERS = ("bliss", "blissdata")

{ewoksdata-1.2.0 → ewoksdata-1.3.0}/src/ewoksdata/data/blissdata/blissdatav2.py

@@ -51,15 +51,31 @@ def iter_bliss_scan_data_from_memory(
         except EndOfStream:
             break
 
-        for stream, view in views.items():
-            ctr_name = ctr_names[stream.name]
-            buffers[ctr_name].extend(view.get_data())
-
-        nyield = min(len(v) for v in buffers.values())
-        if nyield:
-            for i in range(nyield):
-                yield {name: values[i] for name, values in buffers.items()}
-            buffers = {name: values[nyield:] for name, values in buffers.items()}
+        views = {ctr_names[stream.name]: view for stream, view in views.items()}
+
+        nyield = min(
+            len(buffer) + len(views.get(ctr_name, ()))
+            for ctr_name, buffer in buffers.items()
+        )
+        view_positions = {ctr_name: 0 for ctr_name in buffers.keys()}
+        for _ in range(nyield):
+            data = {}
+            for ctr_name, buffer in buffers.items():
+                if buffer:
+                    data_point = buffer.pop(0)
+                else:
+                    view = views[ctr_name]
+                    start = view_positions[ctr_name]
+                    data_points = view.get_data(start=start, stop=start + 1)
+                    data_point = data_points[0]
+                    view_positions[ctr_name] += 1
+                data[ctr_name] = data_point
+            yield data
+
+        for ctr_name, view in views.items():
+            start = view_positions[ctr_name]
+            data_points = view.get_data(start=start, stop=None)
+            buffers[ctr_name].extend(data_points)
 
 
 def last_lima_image(scan_key: str, lima_name: str) -> ArrayLike:

ewoksdata-1.3.0/src/ewoksdata/data/nexus.py

@@ -0,0 +1,182 @@
+import os
+from contextlib import contextmanager
+from typing import Union, Iterator, Optional, Tuple, Sequence
+
+import h5py
+from silx.io import h5py_utils
+from .url import DataUrl
+from .url import as_dataurl
+
+
+def ensure_nxclass(group: h5py.Group) -> None:
+    """
+    Ensure the HDF5 group has an appropriate ``NX_class`` attribute.
+
+    The attribute is assigned based on the group's depth in the hierarchy:
+      * Root group (``/``): ``NXroot``
+      * First-level group: ``NXentry``
+      * Deeper groups: ``NXcollection``
+
+    :param group: The HDF5 group to update.
+    :type group: h5py.Group
+    """
+    if group.attrs.get("NX_class"):
+        return
+    groups = [s for s in group.name.split("/") if s]
+    n = len(groups)
+    if n == 0:
+        group.attrs["NX_class"] = "NXroot"
+    elif n == 1:
+        group.attrs["NX_class"] = "NXentry"
+    else:
+        group.attrs["NX_class"] = "NXcollection"
+
+
+def select_default_plot(nxdata: h5py.Group) -> None:
+    """
+    Set the default NXdata group for visualization in the parent hierarchy.
+
+    The function walks up the group tree and sets the ``default`` attribute
+    in each parent to point to the given NXdata group.
+
+    :param nxdata: An HDF5 group representing an NXdata node.
+    :type nxdata: h5py.Group
+    """
+    parent = nxdata.parent
+    for name in nxdata.name.split("/")[::-1]:
+        if not name:
+            continue
+        parent.attrs["default"] = name
+        parent = parent.parent
+
+
+def create_url(url: str, overwrite: bool = False, **open_options) -> DataUrl:
+    """
+    Create (or ensure the existence of) an HDF5 group specified by a URL.
+
+    :param url: The HDF5 URL (e.g. ``path/to/file.h5::/entry/data``).
+    :param overwrite: If True, overwrite the final group if it exists.
+                      If False, reuse existing groups. Default is False.
+    :param open_options: Additional options passed to ``h5py.File``.
+    :return: A DataUrl object pointing to the created or existing group.
+    """
+    url = as_dataurl(url)
+    filename = url.file_path()
+    data_path = url.data_path() or "/"
+    dirname = os.path.dirname(filename)
+    if dirname:
+        os.makedirs(dirname, exist_ok=True)
+    _ = open_options.setdefault("mode", "a")
+    with h5py_utils.open_item(filename, "/", **open_options) as parent:
+        h5item, _ = _create_h5group(parent, data_path, overwrite=overwrite)
+        return as_dataurl(f"{filename}::{h5item.name}")
+
+
+def get_nxentry(h5item: Union[h5py.Dataset, h5py.Group]) -> h5py.Group:
+    """
+    Return the top-level NXentry group containing the given HDF5 item.
+
+    :param h5item: An HDF5 dataset or group.
+    :return: The NXentry group containing ``h5item``.
+    :raises ValueError: If the item does not belong to an NXentry hierarchy.
+    """
+    parts = [s for s in h5item.name.split("/") if s]
+    if parts:
+        return h5item.file[parts[0]]
+    raise ValueError("HDF5 item must be part of an NXentry")
+
+
+@contextmanager
+def create_nexus_group(
+    url: Union[str, DataUrl],
+    retry_timeout=None,
+    retry_period=None,
+    default_levels: Optional[Sequence[str]] = None,
+    **open_options,
+) -> Iterator[Tuple[h5py.Group, bool]]:
+    """
+    Context manager to create or access an HDF5 group specified by a URL.
+
+    The data path inside HDF5 is ensured to have at least ``default_levels`` levels.
+
+    :param url: The HDF5 URL (e.g. ``path/to/file.h5::/entry/data``).
+    :param retry_timeout: Maximum time in seconds to retry opening the file.
+    :param retry_period: Time in seconds between retries.
+    :param default_levels: Minimum path hierarchy touse when missing. Default is ``["/", "results"]``.
+    :param open_options: Additional options passed to ``h5py.File``.
+    :yields: Tuple of (HDF5 group, already_existed flag).
+    """
+    url = as_dataurl(url)
+    filename = url.file_path()
+    itemname = url.data_path() or "/"
+    dirname = os.path.dirname(filename)
+    if dirname:
+        os.makedirs(dirname, exist_ok=True)
+    open_options.setdefault("mode", "a")
+    with h5py_utils.open_item(
+        filename,
+        "/",
+        retry_timeout=retry_timeout,
+        retry_period=retry_period,
+        **open_options,
+    ) as root:
+        yield _create_h5group(root, itemname, default_levels=default_levels)
+
+
+def _create_h5group(
+    parent: h5py.Group,
+    data_path: str,
+    default_levels: Optional[Sequence[str]] = None,
+    overwrite: bool = False,
+) -> Tuple[h5py.Group, bool]:
+    """
+    Create or retrieve an HDF5 group under a given parent.
+
+    Ensures that intermediate groups exist, optionally enforcing a minimum
+    depth using ``default_levels``.
+
+    Missing ``NX_class`` attributes are automatically created on all levels.
+    The final group can be replaced if ``overwrite=True``.
+
+    :param parent: Parent group under which the path should be created.
+    :param data_path: Group path to create relative to ``parent``.
+    :param default_levels: Minimum hierarchy to enforce.
+                           If ``data_path`` is shorter, extra levels are added.
+    :param overwrite: If True, delete and recreate the final group if it exists.
+                      Default is False.
+    :return: Tuple of (HDF5 group, already_existed flag).
+    """
+    # Ensure the data path has at least N levels as defined by ``default_levels``
+    groups = [s for s in data_path.split("/") if s]
+    if default_levels:
+        default_levels = list(default_levels)
+    else:
+        default_levels = ["results"]
+    if len(groups) < len(default_levels):
+        groups += default_levels[len(groups) :]
+
+    # Ensure parent groups exist + ensure NX_class
+    create = False
+    ensure_nxclass(parent)
+    for group in groups[:-1]:
+        if group in parent:
+            parent = parent[group]
+        else:
+            parent = parent.create_group(group)
+            create = True
+        ensure_nxclass(parent)
+
+    # Final group: create, overwrite or confirm existence + ensure NX_class
+    group = groups[-1]
+    if group in parent:
+        if overwrite:
+            del parent[group]
+            parent = parent.create_group(group)
+        else:
+            parent = parent[group]
+    else:
+        parent = parent.create_group(group)
+        create = True
+    ensure_nxclass(parent)
+
+    return parent, not create

{ewoksdata-1.2.0 → ewoksdata-1.3.0}/src/ewoksdata/data/url.py

@@ -11,12 +11,17 @@ def as_dataurl(url: Union[str, DataUrl]) -> DataUrl:
     return url
 
 
-def h5dataset_url_parse(url: Union[str, DataUrl]) -> Tuple[str, str, Tuple]:
+def as_h5url(url: Union[str, DataUrl]) -> DataUrl:
     url = as_dataurl(url)
+    if not url.data_path():
+        return DataUrl(f"{url.file_path()}::/")
+    return url
+
+
+def h5dataset_url_parse(url: Union[str, DataUrl]) -> Tuple[str, str, Tuple]:
+    url = as_h5url(url)
     filename = str(url.file_path())
     h5path = url.data_path()
-    if h5path is None:
-        h5path = "/"
     idx = url.data_slice()
     if idx is None:
         idx = tuple()

{ewoksdata-1.2.0 → ewoksdata-1.3.0}/src/ewoksdata/tests/test_data_nexus.py

@@ -1,8 +1,10 @@
 import time
 
+import pytest
 from silx.io import h5py_utils
 from silx.io.url import DataUrl
 
+from ..data import url
 from ..data import nexus
 
 
@@ -38,11 +40,23 @@ def test_create_nexus_group(tmp_path):
         assert h5item.name == "/c/b"
 
 
-def test_create_url(tmp_path):
-    input_url = DataUrl(f"{tmp_path / 'file.h5'}::/1.1/measurement/integrated").path()
+@pytest.mark.parametrize("as_string", [True, False])
+@pytest.mark.parametrize("data_path", ["", "::/1.1/measurement/integrated"])
+def test_create_url(tmp_path, data_path, as_string):
+    tmp_url = DataUrl(f"{tmp_path / 'file.h5'}{data_path}")
+
+    if data_path:
+        expected_url = url.as_dataurl(tmp_url.path())
+    else:
+        expected_url = url.as_dataurl(f"{tmp_url.path()}::/results")
+
+    if as_string:
+        input_url = tmp_url.path()
+    else:
+        input_url = tmp_url
 
     created_url = nexus.create_url(input_url)
-    assert created_url == nexus.as_dataurl(input_url)
+    assert created_url == expected_url
 
     with h5py_utils.open_item(
         created_url.file_path(), created_url.data_path(), mode="a"

{ewoksdata-1.2.0 → ewoksdata-1.3.0/src/ewoksdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 1.2.0
+Version: 1.3.0
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License
@@ -43,6 +43,7 @@ Requires-Dist: blissdata>=0.3.4
 Requires-Dist: redis
 Requires-Dist: redis!=5.3.0b1,!=5.3.0b3
 Requires-Dist: redis-om!=1.0.0b0; python_version < "3.9"
+Requires-Dist: redis-om!=1.0.3b0; python_version < "3.9"
 Provides-Extra: online
 Requires-Dist: blissdata[tango]; extra == "online"
 Provides-Extra: test

{ewoksdata-1.2.0 → ewoksdata-1.3.0}/src/ewoksdata.egg-info/requires.txt

@@ -8,6 +8,7 @@ redis!=5.3.0b1,!=5.3.0b3
 
 [:python_version < "3.9"]
 redis-om!=1.0.0b0
+redis-om!=1.0.3b0
 
 [dev]
 ewoksdata[test]

ewoksdata-1.2.0/src/ewoksdata/data/nexus.py

@@ -1,111 +0,0 @@
-import os
-from contextlib import contextmanager
-from typing import Union, Iterator, Optional, Tuple
-
-import h5py
-from silx.io import h5py_utils
-from .url import DataUrl
-from .url import as_dataurl
-
-
-def ensure_nxclass(group: h5py.Group) -> None:
-    if group.attrs.get("NX_class"):
-        return
-    groups = [s for s in group.name.split("/") if s]
-    n = len(groups)
-    if n == 0:
-        group.attrs["NX_class"] = "NXroot"
-    elif n == 1:
-        group.attrs["NX_class"] = "NXentry"
-    else:
-        group.attrs["NX_class"] = "NXcollection"
-
-
-def select_default_plot(nxdata: h5py.Group) -> None:
-    parent = nxdata.parent
-    for name in nxdata.name.split("/")[::-1]:
-        if not name:
-            continue
-        parent.attrs["default"] = name
-        parent = parent.parent
-
-
-def create_url(url: str, overwrite: bool = False, **open_options) -> DataUrl:
-    url = as_dataurl(url)
-    filename = url.file_path()
-    dirname = os.path.dirname(filename)
-    if dirname:
-        os.makedirs(dirname, exist_ok=True)
-    open_options.setdefault("mode", "a")
-    with h5py_utils.open_item(filename, "/", **open_options) as parent:
-        if overwrite and url.data_path() in parent:
-            del parent[url.data_path()]
-        h5item, _ = _create_h5group(parent, url.data_path())
-        return as_dataurl(f"{filename}::{h5item.name}")
-
-
-def get_nxentry(h5item: Union[h5py.Dataset, h5py.Group]):
-    parts = [s for s in h5item.name.split("/") if s]
-    if parts:
-        return h5item.file[parts[0]]
-    raise ValueError("HDF5 item must be part of an NXentry")
-
-
-@contextmanager
-def create_nexus_group(
-    url: Union[str, DataUrl],
-    retry_timeout=None,
-    retry_period=None,
-    default_levels: Optional[Tuple[str]] = None,
-    **open_options,
-) -> Iterator[Tuple[h5py.Group, bool]]:
-    """
-    :yields: (h5group, already_existed)
-    """
-    url = as_dataurl(url)
-    filename = url.file_path()
-    itemname = url.data_path()
-    if not itemname:
-        itemname = "/"
-    dirname = os.path.dirname(filename)
-    if dirname:
-        os.makedirs(dirname, exist_ok=True)
-    open_options.setdefault("mode", "a")
-    with h5py_utils.open_item(
-        filename,
-        "/",
-        retry_timeout=retry_timeout,
-        retry_period=retry_period,
-        **open_options,
-    ) as root:
-        yield _create_h5group(root, itemname, default_levels=default_levels)
-
-
-def _create_h5group(
-    parent, data_path: Optional[str], default_levels: Optional[Tuple[str]] = None
-) -> Tuple[h5py.Group, bool]:
-    """
-    :yields: (h5group, already_existed)
-    """
-    if not data_path:
-        data_path = ""
-    groups = [""] + [s for s in data_path.split("/") if s]
-    if default_levels:
-        default_levels = [""] + list(default_levels)
-    else:
-        default_levels = ["", "results"]
-    if len(groups) < len(default_levels):
-        groups += default_levels[len(groups) :]
-    data_path = "/".join(groups)
-    groups[0] = "/"
-
-    create = False
-    ensure_nxclass(parent)
-    for group in groups:
-        if group in parent:
-            parent = parent[group]
-        else:
-            parent = parent.create_group(group)
-            create = True
-        ensure_nxclass(parent)
-    return parent, not create