ewoksdata 0.4.4__tar.gz → 0.5.0__tar.gz

{ewoksdata-0.4.4 → ewoksdata-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ewoksdata
-Version: 0.4.4
+Version: 0.5.0
 Summary: Ewoks tasks for data access
 Home-page: https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
 Author: ESRF
@@ -36,19 +36,12 @@ Requires-Dist: blissdata[tango]; extra == "doc"
 Requires-Dist: pytest>=7; extra == "doc"
 Requires-Dist: sphinx>=4.5; extra == "doc"
 Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
+Requires-Dist: pydata-sphinx-theme<0.15; extra == "doc"
 
 # ewoksdata
 
 Data access utilities for [ewoks](https://ewoks.readthedocs.io/) workflows.
 
-## Installation
-
-Until `silx>1.0.0` is released:
-
-```bash
-python -m pip install git+https://github.com/silx-kit/silx.git
-```
-
 ## Documentation
 
 https://ewoksdata.readthedocs.io

{ewoksdata-0.4.4 → ewoksdata-0.5.0}/README.md

@@ -2,14 +2,6 @@
 
 Data access utilities for [ewoks](https://ewoks.readthedocs.io/) workflows.
 
-## Installation
-
-Until `silx>1.0.0` is released:
-
-```bash
-python -m pip install git+https://github.com/silx-kit/silx.git
-```
-
 ## Documentation
 
 https://ewoksdata.readthedocs.io

{ewoksdata-0.4.4 → ewoksdata-0.5.0}/setup.cfg

@@ -47,6 +47,7 @@ doc =
 	%(test)s
 	sphinx >=4.5
 	sphinx-autodoc-typehints >=1.16
+	pydata-sphinx-theme < 0.15
 
 [flake8]
 ignore = E501, E203, W503

ewoksdata-0.5.0/src/ewoksdata/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.0"

{ewoksdata-0.4.4 → ewoksdata-0.5.0}/src/ewoksdata/data/bliss.py

@@ -15,6 +15,7 @@ from .blissdata import iter_bliss_scan_data_from_memory  # noqa F401
 from .blissdata import last_lima_image  # noqa F401
 from .blissdata import dynamic_hdf5
 
+from .contextiterator import contextiterator
 from . import hdf5
 from . import nexus
 from . import url
@@ -59,6 +60,7 @@ def _get_hdf5_data(filename: str, h5path: str, idx=None, **options) -> numpy.nda
         return dset[idx]
 
 
+@contextiterator
 def iter_bliss_scan_data(
     filename: str,
     scan_nr: Integral,
@@ -96,6 +98,7 @@ def iter_bliss_scan_data(
             yield dict(zip(names, values))
 
 
+@contextiterator
 def iter_bliss_data(
     filename: str,
     scan_nr: Integral,

ewoksdata-0.5.0/src/ewoksdata/data/contextiterator.py

@@ -0,0 +1,35 @@
+from collections.abc import Generator
+
+
+class ContextIterator(Generator):
+    def __init__(self, it):
+        self._it = it
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return next(self._it)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        self._it.close()
+
+    def send(self):
+        return next(self._it)
+
+    def throw(self):
+        raise StopIteration
+
+
+def contextiterator(iterator):
+    """Decorator that allows to use an iterator as a context manager.
+    This ensures that the iterator is closed when exiting the context manager.
+    """
+
+    def wrapper(*args, **kw):
+        return ContextIterator(iterator(*args, **kw))
+
+    return wrapper

{ewoksdata-0.4.4 → ewoksdata-0.5.0}/src/ewoksdata/data/hdf5/dataset_writer.py

@@ -55,6 +55,21 @@ class _DatasetWriterBase:
 
 
 class DatasetWriter(_DatasetWriterBase):
+    """Append arrays of the same shape to a new HDF5 dataset in a sequential manner.
+
+    .. code-block::python
+
+        with DatasetWriter(parent, "mydataset") as writer:
+            writer.add_point([1,2,3])
+            writer.add_points([[4,5,6], [7,8,9]])
+
+    This is equivalent to
+
+    .. code-block::python
+
+        parent["mydataset"] = [[1,2,3], [4,5,6], [7,8,9]]
+    """
+
     def __init__(
         self,
         parent: h5py.Group,
@@ -103,6 +118,7 @@ class DatasetWriter(_DatasetWriterBase):
         return self._npoints_added
 
     def add_point(self, data: ArrayLike) -> bool:
+        """Append one array to the dataset."""
         if self._dataset is None:
             self._dataset = self._create_dataset(data)
         self._buffer.append(data)
@@ -110,6 +126,7 @@ class DatasetWriter(_DatasetWriterBase):
         return self.flush_buffer(align=True)
 
     def add_points(self, data: ArrayLike) -> bool:
+        """Append several arrays at once to the dataset."""
         if self._dataset is None:
             self._dataset = self._create_dataset(data[0])
         self._buffer.extend(data)
@@ -147,6 +164,27 @@ class DatasetWriter(_DatasetWriterBase):
 
 
 class StackDatasetWriter(_DatasetWriterBase):
+    """Append arrays of the same shape to each item of a new HDF5 dataset
+    in a sequential manner per item. So each item of the HDF5 dataset is a
+    stack to which we can append data in a sequential manner.
+
+    .. code-block::python
+
+        with DatasetWriter(parent, "mydataset") as writer:
+            writer.add_point([1,2,3], 0)
+            writer.add_point([10,11,12], 1)
+            writer.add_points([[4,5,6], [7,8,9]], 0)
+            writer.add_points([[13,14,15], [16,17,18]], 1)
+
+    This is equivalent to
+
+    .. code-block::python
+
+        stack0 = [[1,2,3], [4,5,6], [7,8,9]]
+        stack1 = [[10,11,12], [13,14,15], [16,17,18]]
+        parent["mydataset"] = [stack0, stack1]
+    """
+
     def __init__(
         self,
         parent: h5py.Group,
@@ -194,12 +232,14 @@ class StackDatasetWriter(_DatasetWriterBase):
         return dset
 
     def _get_buffer(self, stack_index: int) -> List[ArrayLike]:
+        # Add stack buffers when needed
         for _ in range(max(stack_index - len(self._buffers) + 1, 0)):
             self._buffers.append(list())
             self._nflushed = numpy.append(self._nflushed, 0)
         return self._buffers[stack_index]
 
     def add_point(self, data: ArrayLike, stack_index: int) -> bool:
+        """Append one array to one stack of the dataset."""
         if self._dataset is None:
             self._dataset = self._create_dataset(data, stack_index)
         buffer = self._get_buffer(stack_index)
@@ -207,6 +247,7 @@ class StackDatasetWriter(_DatasetWriterBase):
         return self.flush_buffer(align=True)
 
     def add_points(self, data: ArrayLike, stack_index: int) -> bool:
+        """Append several arrays at once to one stack of the dataset."""
         if self._dataset is None:
             self._dataset = self._create_dataset(data[0], stack_index)
         buffer = self._get_buffer(stack_index)

ewoksdata-0.5.0/src/ewoksdata/tests/conftest.py

@@ -0,0 +1,29 @@
+import numpy
+import pytest
+from . import data
+
+
+@pytest.fixture(scope="session")
+def bliss_scan(tmpdir_factory):
+    tmpdir = tmpdir_factory.mktemp("sample_dataset")
+    npoints_per_file = 3
+    npoints = 31
+    scannr = 2
+    image = numpy.zeros((10, 12))
+    return (
+        str(
+            data.save_bliss_scan(
+                tmpdir,
+                image,
+                npoints_per_file,
+                npoints,
+                scannr,
+                subscannr=1,
+                lima_names=("p3", "p4"),
+                counter_names=("diode1", "diode2"),
+                sequence="add",
+            )
+        ),
+        npoints,
+        scannr,
+    )

ewoksdata-0.5.0/src/ewoksdata/tests/test_data_bliss.py

@@ -0,0 +1,66 @@
+import numpy
+import h5py
+from fabio.edfimage import edfimage
+import pytest
+from ..data import bliss
+
+
+def test_get_data_edf(tmpdir):
+    filename = str(tmpdir / "data.edf")
+    img1 = numpy.random.uniform(((10, 12)))
+    edf = edfimage(data=img1)
+    edf.write(filename)
+    img2 = bliss.get_data(filename)
+    numpy.testing.assert_array_equal(img1, img2)
+
+
+def test_get_data_hdf5(tmpdir):
+    filename = str(tmpdir / "data.h5")
+    img1 = numpy.random.uniform(((10, 12)))
+    with h5py.File(filename, "w") as f:
+        f["img"] = img1
+    img2 = bliss.get_data(f"{filename}::/img")
+    numpy.testing.assert_array_equal(img1, img2)
+    img3 = bliss.get_data(f"{filename}?path=/img")
+    numpy.testing.assert_array_equal(img1, img3)
+
+
+@pytest.mark.parametrize("lima_names", [(), ("p3",), ("p3", "p4")])
+@pytest.mark.parametrize("counter_names", [(), ("diode1",), ("diode1", "diode2")])
+def test_iter_bliss_data(lima_names, counter_names, bliss_scan):
+    nexpected = len(lima_names) + len(counter_names)
+    index = None
+    filename, npoints, scan_nb = bliss_scan
+    with bliss.iter_bliss_data(
+        filename, scan_nb, lima_names=lima_names, counter_names=counter_names
+    ) as bliss_data_iterator:
+        for index, data in bliss_data_iterator:
+            assert len(data) == nexpected
+            if "diode1" in counter_names:
+                assert data["diode1"] == index
+            if "diode2" in counter_names:
+                assert data["diode2"] == index
+            if "p3" in lima_names:
+                assert (data["p3"] == index).all()
+            if "p4" in lima_names:
+                assert (data["p4"] == index).all()
+
+    if nexpected:
+        assert index == npoints - 1
+    else:
+        assert index is None
+
+
+def test_closure_of_iter_bliss_data(bliss_scan):
+    filename, npoints, scan_nb = bliss_scan
+
+    # This opens filename with `locking=False`
+    with bliss.iter_bliss_data(
+        filename, scan_nb, counter_names=[], lima_names=("p3",)
+    ) as lima_iterator:
+        for i in range(npoints):
+            next(lima_iterator)
+
+    # Make sure that the file is closed and can be reopened with a different locking flag
+    with h5py.File(filename, "r", locking=True):
+        pass

ewoksdata-0.5.0/src/ewoksdata/tests/test_data_hdf5.py

@@ -0,0 +1,14 @@
+from ..data import hdf5
+
+
+def test_bliss_data(bliss_scan):
+    filename, nb_points, scan_nb = bliss_scan
+    with hdf5.h5context(filename) as f:
+        values = f[f"/{scan_nb}.1/measurement/diode1"][()]
+        assert values.tolist() == list(range(nb_points))
+        values = f[f"/{scan_nb}.1/measurement/diode2"][()]
+        assert values.tolist() == list(range(nb_points))
+        values = f[f"/{scan_nb}.1/measurement/p3"][:, 0, 0]
+        assert values.tolist() == list(range(nb_points))
+        values = f[f"/{scan_nb}.1/measurement/p3"][:, -1, -1]
+        assert values.tolist() == list(range(nb_points))

ewoksdata-0.5.0/src/ewoksdata/tests/test_dataset_writer.py

@@ -0,0 +1,89 @@
+import time
+import itertools
+
+import h5py
+import numpy
+import pytest
+from ..data.hdf5 import dataset_writer
+
+
+@pytest.mark.parametrize("npoints", (1, 3, 1000))
+@pytest.mark.parametrize("flush_period", (None, 0.1))
+@pytest.mark.parametrize("known_npoints", (True, False))
+def test_dataset_writer(tmpdir, npoints, flush_period, known_npoints):
+    expected = list()
+    filename = str(tmpdir / "test.h5")
+    if flush_period is None:
+        sleep_time = None
+    else:
+        sleep_time = flush_period + 0.1
+    isleep = npoints // 3
+
+    kwargs = {"flush_period": flush_period}
+    if known_npoints:
+        kwargs["npoints"] = npoints
+
+    with h5py.File(filename, mode="w") as f:
+        with dataset_writer.DatasetWriter(f, "data", **kwargs) as writer:
+            for ipoint in range(npoints):
+                data = numpy.random.random((10, 20))
+                writer.add_point(data)
+                expected.append(data)
+                if sleep_time and ipoint == isleep:
+                    time.sleep(sleep_time)
+
+    with h5py.File(filename, mode="r") as f:
+        data = f["data"][()]
+    numpy.testing.assert_allclose(data, expected)
+
+
+@pytest.mark.parametrize("nstack", (1, 4))
+@pytest.mark.parametrize("npoints", (1, 3, 1000))
+@pytest.mark.parametrize("flush_period", (None, 0.1))
+@pytest.mark.parametrize("known_npoints", (True, False))
+@pytest.mark.parametrize("known_nstack", (True, False))
+@pytest.mark.parametrize("append_stacks_in_parallel", (True, False))
+def test_stack_dataset_writer(
+    tmpdir,
+    nstack,
+    npoints,
+    flush_period,
+    known_npoints,
+    known_nstack,
+    append_stacks_in_parallel,
+):
+    expected = [list() for _ in range(nstack)]
+    filename = str(tmpdir / "test.h5")
+    if flush_period is None:
+        sleep_time = None
+    else:
+        sleep_time = flush_period + 0.1
+    isleep = (nstack * npoints) // 3
+
+    kwargs = {"flush_period": flush_period}
+    if known_npoints:
+        kwargs["npoints"] = npoints
+    if known_nstack:
+        kwargs["nstack"] = nstack
+
+    if append_stacks_in_parallel:
+        itpoints = itertools.product(range(npoints), range(nstack))
+    else:
+        itpoints = itertools.product(range(nstack), range(npoints))
+
+    with h5py.File(filename, mode="w") as f:
+        with dataset_writer.StackDatasetWriter(f, "data", **kwargs) as writer:
+            for tpl in itpoints:
+                if append_stacks_in_parallel:
+                    ipoint, istack = tpl
+                else:
+                    istack, ipoint = tpl
+                data = numpy.random.random((10, 20))
+                writer.add_point(data, istack)
+                expected[istack].append(data)
+                if sleep_time and (ipoint * nstack + istack) == isleep:
+                    time.sleep(sleep_time)
+
+    with h5py.File(filename, mode="r") as f:
+        data = f["data"][()]
+    numpy.testing.assert_allclose(data, expected)

{ewoksdata-0.4.4 → ewoksdata-0.5.0}/src/ewoksdata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ewoksdata
-Version: 0.4.4
+Version: 0.5.0
 Summary: Ewoks tasks for data access
 Home-page: https://gitlab.esrf.fr/workflow/ewoksapps/ewoksdata/
 Author: ESRF
@@ -36,19 +36,12 @@ Requires-Dist: blissdata[tango]; extra == "doc"
 Requires-Dist: pytest>=7; extra == "doc"
 Requires-Dist: sphinx>=4.5; extra == "doc"
 Requires-Dist: sphinx-autodoc-typehints>=1.16; extra == "doc"
+Requires-Dist: pydata-sphinx-theme<0.15; extra == "doc"
 
 # ewoksdata
 
 Data access utilities for [ewoks](https://ewoks.readthedocs.io/) workflows.
 
-## Installation
-
-Until `silx>1.0.0` is released:
-
-```bash
-python -m pip install git+https://github.com/silx-kit/silx.git
-```
-
 ## Documentation
 
 https://ewoksdata.readthedocs.io

{ewoksdata-0.4.4 → ewoksdata-0.5.0}/src/ewoksdata.egg-info/SOURCES.txt

@@ -11,6 +11,7 @@ src/ewoksdata.egg-info/requires.txt
 src/ewoksdata.egg-info/top_level.txt
 src/ewoksdata/data/__init__.py
 src/ewoksdata/data/bliss.py
+src/ewoksdata/data/contextiterator.py
 src/ewoksdata/data/nexus.py
 src/ewoksdata/data/url.py
 src/ewoksdata/data/utils.py

{ewoksdata-0.4.4 → ewoksdata-0.5.0}/src/ewoksdata.egg-info/requires.txt

@@ -20,6 +20,7 @@ blissdata[tango]
 pytest>=7
 sphinx>=4.5
 sphinx-autodoc-typehints>=1.16
+pydata-sphinx-theme<0.15
 
 [online]
 blissdata[tango]

ewoksdata-0.4.4/src/ewoksdata/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.4.4"

ewoksdata-0.4.4/src/ewoksdata/tests/conftest.py

@@ -1,23 +0,0 @@
-import numpy
-import pytest
-from . import data
-
-
-@pytest.fixture(scope="session")
-def bliss_scan(tmpdir_factory):
-    tmpdir = tmpdir_factory.mktemp("sample_dataset")
-    npoints_per_file = 3
-    npoints = 31
-    scannr = 2
-    image = numpy.zeros((10, 12))
-    return data.save_bliss_scan(
-        tmpdir,
-        image,
-        npoints_per_file,
-        npoints,
-        scannr,
-        subscannr=1,
-        lima_names=("p3", "p4"),
-        counter_names=("diode1", "diode2"),
-        sequence="add",
-    )

ewoksdata-0.4.4/src/ewoksdata/tests/test_data_bliss.py

@@ -1,49 +0,0 @@
-import numpy
-import h5py
-from fabio.edfimage import edfimage
-import pytest
-from ..data import bliss
-
-
-def test_get_data_edf(tmpdir):
-    filename = str(tmpdir / "data.edf")
-    img1 = numpy.random.uniform(((10, 12)))
-    edf = edfimage(data=img1)
-    edf.write(filename)
-    img2 = bliss.get_data(filename)
-    numpy.testing.assert_array_equal(img1, img2)
-
-
-def test_get_data_hdf5(tmpdir):
-    filename = str(tmpdir / "data.h5")
-    img1 = numpy.random.uniform(((10, 12)))
-    with h5py.File(filename, "w") as f:
-        f["img"] = img1
-    img2 = bliss.get_data(f"{filename}::/img")
-    numpy.testing.assert_array_equal(img1, img2)
-    img3 = bliss.get_data(f"{filename}?path=/img")
-    numpy.testing.assert_array_equal(img1, img3)
-
-
-@pytest.mark.parametrize("lima_names", [(), ("p3",), ("p3", "p4")])
-@pytest.mark.parametrize("counter_names", [(), ("diode1",), ("diode1", "diode2")])
-def test_iter_bliss_data(lima_names, counter_names, bliss_scan):
-    nexpected = len(lima_names) + len(counter_names)
-    index = None
-    for index, data in bliss.iter_bliss_data(
-        str(bliss_scan), 2, lima_names=lima_names, counter_names=counter_names
-    ):
-        assert len(data) == nexpected
-        if "diode1" in counter_names:
-            assert data["diode1"] == index
-        if "diode2" in counter_names:
-            assert data["diode2"] == index
-        if "p3" in counter_names:
-            assert (data["p3"] == index).all()
-        if "p4" in counter_names:
-            assert (data["p4"] == index).all()
-
-    if nexpected:
-        assert index == 30
-    else:
-        assert index is None

ewoksdata-0.4.4/src/ewoksdata/tests/test_data_hdf5.py

@@ -1,13 +0,0 @@
-from ..data import hdf5
-
-
-def test_bliss_data(bliss_scan):
-    with hdf5.h5context(bliss_scan) as f:
-        values = f["/2.1/measurement/diode1"][()]
-        assert values.tolist() == list(range(31))
-        values = f["/2.1/measurement/diode2"][()]
-        assert values.tolist() == list(range(31))
-        values = f["/2.1/measurement/p3"][:, 0, 0]
-        assert values.tolist() == list(range(31))
-        values = f["/2.1/measurement/p3"][:, -1, -1]
-        assert values.tolist() == list(range(31))

ewoksdata-0.4.4/src/ewoksdata/tests/test_dataset_writer.py

@@ -1,113 +0,0 @@
-import time
-import h5py
-import numpy
-import pytest
-from ..data.hdf5 import dataset_writer
-
-
-@pytest.mark.parametrize("npoints", (1, 3, 1000))
-@pytest.mark.parametrize("flush_period", (None, 0.1))
-def test_dataset_writer_variable_points(tmpdir, npoints, flush_period):
-    expected = list()
-    filename = str(tmpdir / "test.h5")
-    if flush_period is None:
-        sleep_time = None
-    else:
-        sleep_time = flush_period + 0.1
-    isleep = npoints // 3
-
-    with h5py.File(filename, mode="w") as f:
-        with dataset_writer.DatasetWriter(
-            f, "data", flush_period=flush_period
-        ) as writer:
-            for ipoint in range(npoints):
-                data = numpy.random.random((10, 20))
-                writer.add_point(data)
-                expected.append(data)
-                if sleep_time and ipoint == isleep:
-                    time.sleep(sleep_time)
-    with h5py.File(filename, mode="r") as f:
-        data = f["data"][()]
-    numpy.testing.assert_allclose(data, expected)
-
-
-@pytest.mark.parametrize("npoints", (1, 3, 1000))
-@pytest.mark.parametrize("flush_period", (None, 0.1))
-def test_dataset_writer_fixed_points(tmpdir, npoints, flush_period):
-    expected = list()
-    filename = str(tmpdir / "test.h5")
-    if flush_period is None:
-        sleep_time = None
-    else:
-        sleep_time = flush_period + 0.1
-    isleep = npoints // 3
-
-    with h5py.File(filename, mode="w") as f:
-        with dataset_writer.DatasetWriter(
-            f, "data", flush_period=flush_period, npoints=npoints
-        ) as writer:
-            for ipoint in range(npoints):
-                data = numpy.random.random((10, 20))
-                writer.add_point(data)
-                expected.append(data)
-                if sleep_time and ipoint == isleep:
-                    time.sleep(sleep_time)
-    with h5py.File(filename, mode="r") as f:
-        data = f["data"][()]
-    numpy.testing.assert_allclose(data, expected)
-
-
-@pytest.mark.parametrize("nstack", (1, 4))
-@pytest.mark.parametrize("npoints", (1, 3, 1000))
-@pytest.mark.parametrize("flush_period", (None, 0.1))
-def test_stack_dataset_writer_variable_points(tmpdir, nstack, npoints, flush_period):
-    expected = [list() for _ in range(nstack)]
-    filename = str(tmpdir / "test.h5")
-    if flush_period is None:
-        sleep_time = None
-    else:
-        sleep_time = flush_period + 0.1
-    isleep = (nstack * npoints) // 3
-
-    with h5py.File(filename, mode="w") as f:
-        with dataset_writer.StackDatasetWriter(
-            f, "data", flush_period=flush_period
-        ) as writer:
-            for ipoint in range(npoints):
-                for istack in range(nstack):
-                    data = numpy.random.random((10, 20))
-                    writer.add_point(data, istack)
-                    expected[istack].append(data)
-                    if sleep_time and (ipoint * nstack + istack) == isleep:
-                        time.sleep(sleep_time)
-    with h5py.File(filename, mode="r") as f:
-        data = f["data"][()]
-    numpy.testing.assert_allclose(data, expected)
-
-
-@pytest.mark.parametrize("nstack", (1, 4))
-@pytest.mark.parametrize("npoints", (1, 3, 1000))
-@pytest.mark.parametrize("flush_period", (None, 0.1))
-def test_stack_dataset_writer_fixed_points(tmpdir, nstack, npoints, flush_period):
-    expected = [list() for _ in range(nstack)]
-    filename = str(tmpdir / "test.h5")
-    if flush_period is None:
-        sleep_time = None
-    else:
-        sleep_time = flush_period + 0.1
-    isleep = npoints // 3
-
-    with h5py.File(filename, mode="w") as f:
-        with dataset_writer.StackDatasetWriter(
-            f, "data", flush_period=flush_period, npoints=npoints, nstack=nstack
-        ) as writer:
-            for ipoint in range(npoints):
-                for istack in range(nstack):
-                    data = numpy.random.random((10, 20))
-                    writer.add_point(data, istack)
-                    expected[istack].append(data)
-                    if sleep_time and (ipoint * nstack + istack) == isleep:
-                        time.sleep(sleep_time)
-    with h5py.File(filename, mode="r") as f:
-        data = f["data"][()]
-    numpy.testing.assert_allclose(data, expected)