ewoksdata 2.3.0rc3__tar.gz → 2.3.0rc4__tar.gz

{ewoksdata-2.3.0rc3/src/ewoksdata.egg-info → ewoksdata-2.3.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 2.3.0rc3
+Version: 2.3.0rc4
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License

{ewoksdata-2.3.0rc3 → ewoksdata-2.3.0rc4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ewoksdata"
-version = "2.3.0rc3"
+version = "2.3.0rc4"
 authors = [{ name = "ESRF", email = "dau-pydev@esrf.fr" }]
 description = "Ewoks tasks for data access"
 readme = { file = "README.md", content-type = "text/markdown" }

ewoksdata-2.3.0rc4/src/ewoksdata/data/hdf5/overwrite.py

@@ -0,0 +1,102 @@
+import os
+import shutil
+import tempfile
+import uuid
+from collections import defaultdict
+from typing import Dict
+from typing import List
+from typing import Union
+
+from silx.io import h5py_utils
+from silx.io.url import DataUrl
+
+from ..url import as_dataurl
+
+
+def ensure_items_do_not_exist(
+    *urls: Union[str, DataUrl], delete_existing: bool = False
+) -> None:
+    """Make sure the HDF5 items (groups, datasets or links) do not exist.
+
+    Existing items are deleted when :code:`delete_existing` is :code:`True`.
+    A file is removed when its root group is deleted or nothing remains,
+    and repacked otherwise.
+
+    :raises FileExistsError: when items exist and :code:`delete_existing` is :code:`False`.
+    :raises ValueError: when an item to delete is in another file (external link).
+    """
+    data_paths: Dict[str, List[str]] = defaultdict(list)
+    for url in urls:
+        url = as_dataurl(url)
+        filename = url.file_path()
+        parts = [s for s in (url.data_path() or "").split("/") if s]
+        data_paths[filename].append("/".join(parts) or "/")
+    for filename, file_data_paths in data_paths.items():
+        _ensure_items_do_not_exist(filename, file_data_paths, delete_existing)
+
+
+def _ensure_items_do_not_exist(
+    filename: str, data_paths: List[str], delete_existing: bool
+) -> None:
+    """:code:`ensure_items_do_not_exist` for the items of one file."""
+    if not os.path.exists(filename):
+        return
+    if "/" in data_paths:
+        if not delete_existing:
+            raise FileExistsError(f"HDF5 file {filename!r} already exists")
+        os.remove(filename)
+        return
+
+    mode = "a" if delete_existing else "r"
+    with h5py_utils.File(filename, mode=mode) as root:
+        existing = [data_path for data_path in data_paths if data_path in root]
+        if not existing:
+            return
+        if not delete_existing:
+            raise FileExistsError(
+                f"HDF5 items already exist in {filename!r}: {existing}"
+            )
+
+        real_filename = os.path.realpath(root.filename)
+        for data_path in existing:
+            parent_path = data_path.rpartition("/")[0]
+            if not parent_path:
+                continue
+            parent_filename = os.path.realpath(root[parent_path].file.filename)
+            if parent_filename != real_filename:
+                raise ValueError(
+                    f"HDF5 item {data_path!r} of {filename!r} is in another file"
+                )
+
+        for data_path in existing:
+            # Can be gone already when a parent was deleted
+            if data_path in root:
+                del root[data_path]
+        remove_file = len(root) == 0
+
+    if remove_file:
+        os.remove(filename)
+    else:
+        # Deleting does not free the space in the file
+        _repack(filename)
+
+
+def _repack(filename: str) -> None:
+    """Rewrite the file without its unused space."""
+    dirname = os.path.dirname(os.path.abspath(filename))
+    with tempfile.TemporaryDirectory(dir=dirname) as tmp_dirname:
+        tmp_filename = os.path.join(tmp_dirname, os.path.basename(filename))
+        with h5py_utils.File(filename, mode="r") as src:
+            with h5py_utils.File(tmp_filename, mode="w") as dst:
+                # The root cannot be copied onto the root of another file. Copy it
+                # in one call to a temporary group so that hard links between items
+                # stay shared. Deleting that group leaves its header and link index
+                # unused: about 1 KB plus 9 bytes per root item, not data.
+                tmp_name = f"_repack_{uuid.uuid4().hex}"
+                src.copy(src, dst, name=tmp_name)
+                for name in list(dst[tmp_name]):
+                    dst.move(f"{tmp_name}/{name}", name)
+                del dst[tmp_name]
+                dst.attrs.update(src.attrs)
+        shutil.copymode(filename, tmp_filename)
+        os.replace(tmp_filename, filename)

{ewoksdata-2.3.0rc3 → ewoksdata-2.3.0rc4}/src/ewoksdata/data/utils.py

@@ -1,3 +1,4 @@
+import warnings
 from numbers import Number
 from typing import List
 from typing import Mapping
@@ -16,6 +17,18 @@ def is_data(data):
 
 
 def data_from_storage(data, remove_numpy=True):
+    """Deprecated: values restored from ewoks persistent storage no longer need to be
+    converted."""
+    warnings.warn(
+        "'data_from_storage' is deprecated. Values restored from ewoks persistent "
+        "storage keep their python type since ewokscore 5.1.0rc4.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return _data_from_storage(data, remove_numpy=remove_numpy)
+
+
+def _data_from_storage(data, remove_numpy=True):
     if isinstance(data, numpy.ndarray):
         if not remove_numpy:
             return data
@@ -25,7 +38,7 @@ def data_from_storage(data, remove_numpy=True):
             return data.tolist()
     elif isinstance(data, Mapping):
         return {
-            k: data_from_storage(v, remove_numpy=remove_numpy)
+            k: _data_from_storage(v, remove_numpy=remove_numpy)
             for k, v in data.items()
             if not k.startswith("@")
         }

{ewoksdata-2.3.0rc3 → ewoksdata-2.3.0rc4}/src/ewoksdata/tests/unit/test_dataset_writer.py

@@ -191,9 +191,19 @@ def test_concurrent_reader(tmp_path):
 
     mp_context = multiprocessing.get_context("spawn")
     start_event = mp_context.Event()
+    done_event = mp_context.Event()
+    nread = mp_context.Value("i", 0)
     reader_process = mp_context.Process(
         target=_read_hdf5_file,
-        args=(hdf5_filename, dataset_name, npoints, read_timestamps, start_event),
+        args=(
+            hdf5_filename,
+            dataset_name,
+            npoints,
+            read_timestamps,
+            start_event,
+            done_event,
+            nread,
+        ),
     )
     reader_process.start()
 
@@ -201,80 +211,100 @@ def test_concurrent_reader(tmp_path):
 
     # Run write loop
     try:
-        _write_hdf5_file(hdf5_filename, dataset_name, npoints, write_timestamps)
+        _write_hdf5_file(hdf5_filename, dataset_name, npoints, write_timestamps, nread)
     except Exception:
         reader_process.terminate()
         raise
 
-    # Wait for read loop to finish
+    # Do not rely on the exit code: with gevent monkey-patching (bliss
+    # integration tests in the same session), once any subprocess was started,
+    # gevent collects the exit code of every child process that terminates.
+    # The exit code of the reader is then no longer available to
+    # multiprocessing, so `reader_process.exitcode` stays `None`.
     try:
-        reader_process.join(timeout=20)
-        if reader_process.exitcode is None:
-            raise TimeoutError("Reader process did not terminate within the timeout.")
-        elif reader_process.exitcode != 0:
-            raise RuntimeError(
-                f"Reader process terminated with exit code {reader_process.exitcode}."
-            )
-    except Exception:
-        reader_process.terminate()
-        raise
+        assert done_event.wait(timeout=20), "Reader did not finish within the timeout."
+    finally:
+        reader_process.join(timeout=5)
+        if reader_process.is_alive():
+            reader_process.terminate()
 
-    # Check that reading and writing happened concurrently
-    with open(read_timestamps, "r") as eventfile:
-        read_events = [int(line.strip()) for line in eventfile.readlines()]
-    with open(write_timestamps, "r") as eventfile:
-        write_events = [int(line.strip()) for line in eventfile.readlines()]
+    read_events = _load_events(read_timestamps)
+    write_events = _load_events(write_timestamps)
 
-    assert len(read_events) == npoints
-    assert len(write_events) == npoints
+    assert [value for _, value in read_events] == list(range(npoints))
+    assert [value for _, value in write_events] == list(range(npoints))
 
-    all_events = sorted(read_events + write_events)
-    assert read_events != all_events[: len(read_events)], (
+    # The reader got data before the writer finished
+    assert read_events[0][0] < write_events[-1][0], (
         "reads and writes are not interleaved"
     )
 
 
 class _DatasetWriter(dataset_writer.DatasetWriter):
-    """Testing concurrent reading and writing always ends up
-    being flaky. So we introduce a sleep after flushing
-    to give the reader plenty of time to fetch new data.
-    """
+    """Waits after each flush until the reader has read all flushed points."""
+
+    def __init__(self, *args, nread, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._nread = nread
 
     def _flush_hdf5(self) -> None:
         print("flush", flush=True)
         super()._flush_hdf5()
-        time.sleep(0.2)
+        # Timing alone does not guarantee interleaving on a loaded machine
+        nflushed = self.dataset.shape[0]
+        t0 = time.monotonic()
+        while self._nread.value < nflushed and time.monotonic() - t0 < 10:
+            time.sleep(0.01)
 
 
-def _write_hdf5_file(hdf5_filename, dataset_name, npoints, event_filename):
-    data = numpy.zeros((500, 500), dtype=numpy.int64)
+def _write_hdf5_file(hdf5_filename, dataset_name, npoints, event_filename, nread):
     with open(event_filename, "a") as eventfile:
         with h5py.File(hdf5_filename, mode="w") as h5file:
-            with _DatasetWriter(h5file, dataset_name) as writer:
+            with _DatasetWriter(
+                h5file, dataset_name, flush_period=0.1, nread=nread
+            ) as writer:
                 for i in range(npoints):
-                    writer.add_point(data)
-                    eventfile.write(f"{time.perf_counter_ns()}\n")
+                    writer.add_point(numpy.full((500, 500), i, dtype=numpy.int64))
+                    eventfile.write(f"{time.perf_counter_ns()} {i}\n")
                     eventfile.flush()
                     print(f"write {i}", flush=True)
+                    # Ensure flushes happen before the writer exits
+                    time.sleep(0.02)
 
 
-def _read_hdf5_file(hdf5_filename, dataset_name, npoints, event_filename, start_event):
+def _read_hdf5_file(
+    hdf5_filename,
+    dataset_name,
+    npoints,
+    event_filename,
+    start_event,
+    done_event,
+    nread,
+):
     i = 0
     with open(event_filename, "a") as eventfile:
         start_event.set()
         while i < npoints:
-            for _ in _iter_data(hdf5_filename, dataset_name):
-                eventfile.write(f"{time.perf_counter_ns()}\n")
+            for value in _iter_data(hdf5_filename, dataset_name, start_index=i):
+                eventfile.write(f"{time.perf_counter_ns()} {value}\n")
                 eventfile.flush()
                 print(f"read {i}", flush=True)
                 i += 1
+                nread.value = i
+    print("reader finished", flush=True)
+    done_event.set()
 
 
 @h5py_utils.retry()
 def _iter_data(hdf5_filename, dataset_name, start_index=0):
     with h5py_utils.File(hdf5_filename) as h5file:
-        for _ in h5file[dataset_name][start_index:]:
-            yield None
+        for value in h5file[dataset_name][start_index:, 0, 0]:
+            yield int(value)
+
+
+def _load_events(filename):
+    with open(filename, "r") as eventfile:
+        return [tuple(map(int, line.split())) for line in eventfile]
 
 
 def test_stack_dataset_writer_multi_chunk_dim0(tmp_path):

ewoksdata-2.3.0rc4/src/ewoksdata/tests/unit/test_overwrite_hdf5.py

@@ -0,0 +1,143 @@
+import os
+import stat
+
+import h5py
+import numpy
+import pytest
+
+from ...data.hdf5.overwrite import ensure_items_do_not_exist
+
+
+def test_ensure_items_do_not_exist_nested(tmp_path):
+    filename = tmp_path / "test.h5"
+    with h5py.File(filename, "w") as f:
+        f["1.1/fit/data"] = numpy.zeros((1000, 1000))
+        f.create_group("1.1/norm")
+    size_before = os.path.getsize(filename)
+
+    ensure_items_do_not_exist(
+        f"{filename}::/1.1/fit", f"{filename}::/1.1/missing", delete_existing=True
+    )
+
+    with h5py.File(filename, "r") as f:
+        assert set(f["1.1"]) == {"norm"}
+    assert os.path.getsize(filename) < size_before / 10
+
+
+def test_ensure_items_do_not_exist_some_top_level(tmp_path):
+    filename = tmp_path / "test.h5"
+    with h5py.File(filename, "w") as f:
+        f.attrs["creator"] = "test"
+        f["1.1/data"] = numpy.zeros((1000, 1000))
+        f["2.1/data"] = numpy.zeros(10)
+        f["2.1/hard"] = f["2.1/data"]
+        f["3.1/hard"] = f["2.1/data"]
+        f["3.1/soft"] = h5py.SoftLink("/2.1/data")
+        f["3.1/external"] = h5py.ExternalLink("other.h5", "/data")
+    os.chmod(filename, 0o644)
+    size_before = os.path.getsize(filename)
+
+    ensure_items_do_not_exist(f"{filename}::/1.1", delete_existing=True)
+
+    with h5py.File(filename, "r") as f:
+        assert set(f) == {"2.1", "3.1"}
+        assert f.attrs["creator"] == "test"
+        assert f["2.1/hard"].id == f["2.1/data"].id
+        assert f["3.1/hard"].id == f["2.1/data"].id
+        assert f.get("3.1/soft", getlink=True).path == "/2.1/data"
+        assert f.get("3.1/external", getlink=True).filename == "other.h5"
+    assert os.path.getsize(filename) < size_before / 10
+    assert stat.S_IMODE(os.stat(filename).st_mode) == 0o644
+    assert os.listdir(tmp_path) == ["test.h5"]
+
+
+def test_ensure_items_do_not_exist_all_top_level(tmp_path):
+    filename = tmp_path / "test.h5"
+    with h5py.File(filename, "w") as f:
+        f.create_group("1.1")
+        f.create_group("2.1")
+
+    ensure_items_do_not_exist(
+        f"{filename}::/1.1", f"{filename}::/2.1", delete_existing=True
+    )
+
+    assert not filename.exists()
+
+
+def test_ensure_items_do_not_exist_missing(tmp_path):
+    filename = tmp_path / "test.h5"
+    ensure_items_do_not_exist(f"{filename}::/1.1")
+    ensure_items_do_not_exist(f"{filename}::/1.1", delete_existing=True)
+    assert not filename.exists()
+
+
+def test_ensure_items_do_not_exist_root(tmp_path):
+    filename = tmp_path / "test.h5"
+    with h5py.File(filename, "w") as f:
+        f.create_group("1.1")
+
+    with pytest.raises(FileExistsError):
+        ensure_items_do_not_exist(f"{filename}::/")
+    assert filename.exists()
+
+    ensure_items_do_not_exist(
+        f"{filename}::/1.1", f"{filename}::/", delete_existing=True
+    )
+    assert not filename.exists()
+
+
+def test_ensure_items_do_not_exist_raise(tmp_path):
+    filename = tmp_path / "test.h5"
+    with h5py.File(filename, "w") as f:
+        f["1.1/data"] = numpy.zeros(10)
+        f.create_group("2.1")
+
+    ensure_items_do_not_exist(f"{filename}::/1.1/missing")
+    with pytest.raises(FileExistsError):
+        ensure_items_do_not_exist(f"{filename}::/1.1/missing", f"{filename}::/2.1")
+
+    with h5py.File(filename, "r") as f:
+        assert set(f) == {"1.1", "2.1"}
+
+
+def test_ensure_items_do_not_exist_datasets_and_links(tmp_path):
+    filename = tmp_path / "test.h5"
+    with h5py.File(filename, "w") as f:
+        f["1.1/data"] = numpy.zeros(10)
+        f["1.1/other"] = numpy.zeros(10)
+        f["1.1/dangling"] = h5py.SoftLink("/missing")
+
+    ensure_items_do_not_exist(
+        f"{filename}::/1.1/data", f"{filename}::/1.1/dangling", delete_existing=True
+    )
+
+    with h5py.File(filename, "r") as f:
+        assert set(f["1.1"]) == {"other"}
+
+
+def test_ensure_items_do_not_exist_external(tmp_path):
+    filename = tmp_path / "test.h5"
+    external_filename = tmp_path / "external.h5"
+    with h5py.File(external_filename, "w") as f:
+        f["group/data"] = numpy.zeros(10)
+    with h5py.File(filename, "w") as f:
+        f["1.1/data"] = numpy.zeros(10)
+        f["1.1/external"] = h5py.ExternalLink(str(external_filename), "/group")
+
+    with pytest.raises(ValueError):
+        ensure_items_do_not_exist(
+            f"{filename}::/1.1/data",
+            f"{filename}::/1.1/external/data",
+            delete_existing=True,
+        )
+
+    with h5py.File(external_filename, "r") as f:
+        assert "group/data" in f
+    with h5py.File(filename, "r") as f:
+        assert "1.1/data" in f
+
+    ensure_items_do_not_exist(f"{filename}::/1.1/external", delete_existing=True)
+    with h5py.File(filename, "r") as f:
+        assert set(f["1.1"]) == {"data"}
+    with h5py.File(external_filename, "r") as f:
+        assert "group/data" in f

{ewoksdata-2.3.0rc3 → ewoksdata-2.3.0rc4/src/ewoksdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 2.3.0rc3
+Version: 2.3.0rc4
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License

{ewoksdata-2.3.0rc3 → ewoksdata-2.3.0rc4}/src/ewoksdata.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ src/ewoksdata/data/blissdata/blissdatav2.py
 src/ewoksdata/data/hdf5/__init__.py
 src/ewoksdata/data/hdf5/config.py
 src/ewoksdata/data/hdf5/mirror.py
+src/ewoksdata/data/hdf5/overwrite.py
 src/ewoksdata/data/hdf5/types.py
 src/ewoksdata/data/hdf5/dataset_writer/__init__.py
 src/ewoksdata/data/hdf5/dataset_writer/_base.py
@@ -41,5 +42,6 @@ src/ewoksdata/tests/unit/test_data_nexus.py
 src/ewoksdata/tests/unit/test_dataset_writer.py
 src/ewoksdata/tests/unit/test_find_lima_scans.py
 src/ewoksdata/tests/unit/test_mirror_hdf5.py
+src/ewoksdata/tests/unit/test_overwrite_hdf5.py
 src/ewoksdata/tests/unit/data/__init__.py
 src/ewoksdata/tests/unit/data/bliss_scans.py