ewoksdata 2.3.0rc2__tar.gz → 2.3.0rc3__tar.gz

{ewoksdata-2.3.0rc2/src/ewoksdata.egg-info → ewoksdata-2.3.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ewoksdata
-Version: 2.3.0rc2
+Version: 2.3.0rc3
 Summary: Ewoks tasks for data access
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: # MIT License

{ewoksdata-2.3.0rc2 → ewoksdata-2.3.0rc3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ewoksdata"
-version = "2.3.0rc2"
+version = "2.3.0rc3"
 authors = [{ name = "ESRF", email = "dau-pydev@esrf.fr" }]
 description = "Ewoks tasks for data access"
 readme = { file = "README.md", content-type = "text/markdown" }

{ewoksdata-2.3.0rc2 → ewoksdata-2.3.0rc3}/src/ewoksdata/data/hdf5/mirror.py

@@ -1,11 +1,14 @@
 import logging
 import os
+from typing import Any
 from typing import NamedTuple
+from typing import Optional
 from typing import Sequence
 from typing import Set
 from typing import Union
 
 import h5py
+import numpy
 from silx.io.url import DataUrl
 
 # For type annotations only. Code works for any h5py-like objects.
@@ -44,7 +47,8 @@ def mirror_hdf5_content(
 
     HDF5 links are soft links, external links and virtual datasets.
 
-    An HDF5 link with target(s) within :code:`original` is always kept as a link.
+    An HDF5 link with target(s) within :code:`original` is always kept as a link,
+    re-targeted to the mirror of those target(s).
 
     An HDF5 link with target(s) outside :code:`original` is called out-of-scope.
     An out-of-scope link is kept as a link to wherever the data already is
@@ -78,10 +82,11 @@ def mirror_hdf5_content(
     expand_paths = _normalize_relative_paths(expand)
     recursive_paths = _normalize_relative_paths(recursive)
 
-    home = DataUrl(
+    home_original = DataUrl(
         file_path=os.path.abspath(_get_hdf5_filename(original.file)),
         data_path=original.name,
     )
+    home_destination = _normalize_hdf5_item_name(parent.name, name)
 
     _mirror(
         parent,
@@ -94,7 +99,8 @@ def mirror_hdf5_content(
         expand=expand_paths,
         recursive=recursive_paths,
         copy=copy,
-        home=home,
+        home_original=home_original,
+        home_destination=home_destination,
         overwrite=overwrite,
         allow_cross_file=allow_cross_file,
     )
@@ -103,6 +109,7 @@ def mirror_hdf5_content(
 class _ReachedBy(NamedTuple):
     link: Union[None, h5py.HardLink, h5py.SoftLink, h5py.ExternalLink]
     in_scope: bool
+    soft_link_target: Optional[str] = None
 
     @property
     def is_soft_link_in_scope(self) -> bool:
@@ -115,17 +122,21 @@ class _ReachedBy(NamedTuple):
         )
 
     @classmethod
-    def from_h5py(cls, parent: GroupType, name: str, home: DataUrl) -> "_ReachedBy":
+    def from_h5py(
+        cls, parent: GroupType, name: str, home_original: DataUrl
+    ) -> "_ReachedBy":
         link = _get_hdf5_link(parent, name)
+        soft_link_target = None
         if isinstance(link, h5py.SoftLink):
-            in_scope = _soft_link_in_scope(parent, link, home)
+            soft_link_target = _absolute_soft_link_target(parent, link)
+            in_scope = _soft_link_in_scope(parent, soft_link_target, home_original)
         elif isinstance(link, h5py.ExternalLink):
             in_scope = False
         elif isinstance(link, h5py.HardLink):
             in_scope = True
         else:
             raise NotImplementedError(f"Unknown HDF5 link type: {type(link)}")
-        return _ReachedBy(link, in_scope)
+        return _ReachedBy(link, in_scope, soft_link_target)
 
 
 def _mirror(
@@ -139,7 +150,8 @@ def _mirror(
     expand: Set[str],
     recursive: Set[str],
     copy: bool,
-    home: DataUrl,
+    home_original: DataUrl,
+    home_destination: str,
     overwrite: bool,
     allow_cross_file: bool,
 ) -> None:
@@ -164,12 +176,16 @@ def _mirror(
     :param copy: copy instead of link by default at this level (already
                  accounts for any :code:`invert` entry applied by an
                  ancestor).
-    :param home: file and HDF5 path of the root original; used to never
-                physically copy data reached through a link into another
-                file (unless :code:`allow_cross_file`), and to tell
-                whether a soft link or virtual dataset source refers to a
-                path guaranteed to exist in :code:`parent` (within
-                :code:`home`'s tree) or not.
+    :param home_original: file and HDF5 path of the root original; used to
+                          never physically copy data reached through a link
+                          into another file (unless :code:`allow_cross_file`),
+                          and to tell whether a soft link or virtual dataset
+                          source refers to a path guaranteed to exist in
+                          :code:`parent` (within :code:`home_original`'s tree)
+                          or not.
+    :param home_destination: absolute HDF5 path of the mirror of
+                             :code:`home_original`; used to re-target a link
+                             to a path within :code:`home_original`'s tree.
     :param overwrite: delete a conflicting existing name instead of warning
                       and leaving it untouched.
     :param allow_cross_file: allow crossing into a file other than the
@@ -219,7 +235,7 @@ def _mirror(
     # than the one being mirrored: never allowed unless `allow_cross_file`,
     # regardless of what `expand`/`recursive` ask for (see `traverse` below).
     crosses_file = (
-        os.path.abspath(_get_hdf5_filename(original.file)) != home.file_path()
+        os.path.abspath(_get_hdf5_filename(original.file)) != home_original.file_path()
     )
 
     # Open `original` as a group instead of mirroring it as a single link or
@@ -243,7 +259,7 @@ def _mirror(
             destination = parent.create_group(name)
         _update_attrs(destination, original, overwrite)
         for child_name in original:
-            child_reached_by = _ReachedBy.from_h5py(original, child_name, home)
+            child_reached_by = _ReachedBy.from_h5py(original, child_name, home_original)
             try:
                 original_child = _resolve_hdf5_group_child(original, child_name)
             except KeyError:
@@ -262,7 +278,8 @@ def _mirror(
                 expand=expand,
                 recursive=recursive,
                 copy=do_copy,
-                home=home,
+                home_original=home_original,
+                home_destination=home_destination,
                 overwrite=overwrite,
                 allow_cross_file=allow_cross_file,
             )
@@ -274,23 +291,35 @@ def _mirror(
     # `name` is ensured to not exist in `parent`
 
     if not do_copy:
+        if reached_by.is_soft_link_in_scope and _create_mirrored_soft_link(
+            parent, name, reached_by, home_original, home_destination, exclude
+        ):
+            # Preserve the link faithfully instead of linking to what it
+            # points to in the file being mirrored.
+            return
         _create_hdf5_link(parent, name, original, overwrite=overwrite)
         return
 
     # The content is being copied. First try preserving links faithfully.
 
-    if reached_by.is_soft_link_in_scope:
-        # Preserve the link faithfully instead of copying the data it
-        # points to. Same file, so there is no path to re-adjust.
-        parent[name] = reached_by.link
+    if reached_by.is_soft_link_in_scope and _create_mirrored_soft_link(
+        parent, name, reached_by, home_original, home_destination, exclude
+    ):
+        # Preserve the link faithfully instead of copying the data it points to.
         return
 
-    is_vds = not is_group and original.is_virtual
-    if is_vds and _virtual_dataset_in_scope(original, home):
+    # Virtual datasets and copies are handled with the h5py dataset
+    # underneath, when there is one.
+    h5py_original = None if is_group else _as_h5py_item(original)
+
+    is_vds = h5py_original is not None and h5py_original.is_virtual
+    if is_vds and _virtual_dataset_in_scope(h5py_original, home_original):
         # Every source of this VDS resolves within the tree being mirrored,
         # which is guaranteed to also exist in `parent`'s file: preserve it
         # faithfully as a VDS.
-        if _copy_virtual_dataset(parent, name, original, home.data_path()):
+        if _copy_virtual_dataset(
+            parent, name, h5py_original, home_original, home_destination
+        ):
             return
 
     if allow_cross_file:
@@ -298,8 +327,8 @@ def _mirror(
             # Materialize instead of preserving the VDS: the whole point of
             # allow_cross_file is to no longer depend on the file being
             # mirrored, which is the only place this source resolves in.
-            dataset = parent.create_dataset(name, data=original[()])
-            dataset.attrs.update(original.attrs)
+            dataset = parent.create_dataset(name, data=h5py_original[()])
+            dataset.attrs.update(h5py_original.attrs)
             return
     else:
         # When possible, preserve links faithfully.
@@ -314,13 +343,15 @@ def _mirror(
             return
         if is_vds:
             # Re-create the VDS (no always possible) to recomputed relative file path.
-            if _copy_virtual_dataset(parent, name, original, home.data_path()):
+            if _copy_virtual_dataset(
+                parent, name, h5py_original, home_original, home_destination
+            ):
                 return
 
     # Copy the actual content
 
     if not is_group:
-        parent.copy(original, name)
+        _copy_hdf5_dataset(parent, name, original, h5py_original)
         return
 
     # Recreate the group instead of `h5py.Group.copy` to handle links.
@@ -337,14 +368,19 @@ def _mirror(
         # {""} forces the whole subtree to be traversed (see `in_recursive`).
         recursive={""},
         copy=True,
-        home=home,
+        home_original=home_original,
+        home_destination=home_destination,
         overwrite=overwrite,
         allow_cross_file=allow_cross_file,
     )
 
 
 def _copy_virtual_dataset(
-    parent: GroupType, name: str, original: DatasetType, home_path: str
+    parent: GroupType,
+    name: str,
+    original: h5py.Dataset,
+    home_original: DataUrl,
+    home_destination: str,
 ) -> bool:
     """Try to faithfully copy virtual dataset :code:`original` under :code:`name`
     in :code:`parent`, by reconstructing its virtual sources with paths
@@ -352,8 +388,10 @@ def _copy_virtual_dataset(
     mapping verbatim (which does not adjust source paths when :code:`parent`
     is not in the same directory as :code:`original`).
 
-    :param home_path: absolute HDF5 path of the root original being mirrored;
-                      see :code:`_mirror`.
+    :param home_original: file and HDF5 path of the root original being
+                          mirrored; see :code:`_mirror`.
+    :param home_destination: absolute HDF5 path of the mirror of :code:`home_original`;
+                             see :code:`_mirror`.
     :returns: :code:`False` (nothing created) when a source mapping is not a
               single, whole-source hyperslab, the only shape this can
               faithfully reconstruct; the caller should fall back to a plain
@@ -369,6 +407,7 @@ def _copy_virtual_dataset(
             return False
         mappings.append(mapping)
 
+    home_original_path = home_original.data_path()
     layout = h5py.VirtualLayout(shape=original.shape, dtype=original.dtype)
     original_filename = _get_hdf5_filename(original.file)
     parent_filename = _get_hdf5_filename(parent.file)
@@ -376,11 +415,16 @@ def _copy_virtual_dataset(
         start, stop = mapping.vspace.get_select_bounds()
         region = tuple(slice(a, b + 1) for a, b in zip(start, stop))
         region_shape = tuple(b - a + 1 for a, b in zip(start, stop))
+        vsource_dset_name = mapping.dset_name
         if mapping.file_name == ".":
-            if _in_scope(mapping.dset_name, home_path):
-                # Keep it as-is, since `mapping.dset_name` is guaranteed
-                # to also exist in `parent`'s file.
+            if _in_scope(mapping.dset_name, home_original_path):
+                # Point to the mirror of the source, which is guaranteed to
+                # exist in `parent`'s file.
                 vsource_filename = "."
+                vsource_dset_name = _normalize_hdf5_item_name(
+                    home_destination,
+                    _relative_to_home_original(mapping.dset_name, home_original),
+                )
             else:
                 # "." pointing outside the tree being mirrored: that path is
                 # only guaranteed to exist in `original`'s own file, so point
@@ -395,7 +439,7 @@ def _copy_virtual_dataset(
                 parent_filename, original_filename, mapping.file_name
             )
         source = h5py.VirtualSource(
-            vsource_filename, mapping.dset_name, shape=region_shape
+            vsource_filename, vsource_dset_name, shape=region_shape
         )
         layout[region] = source
     parent.create_virtual_dataset(name, layout, fillvalue=original.fillvalue)
@@ -418,6 +462,21 @@ def _relative_virtual_source_filename(
     return os.path.relpath(abs_vsource_filename, abs_parent_dir)
 
 
+def _copy_hdf5_dataset(
+    parent: GroupType,
+    name: str,
+    original: DatasetType,
+    h5py_original: Optional[h5py.Dataset],
+) -> None:
+    """Copy dataset :code:`original` under :code:`name` in :code:`parent`."""
+    if h5py_original is not None:
+        parent.copy(h5py_original, name)
+    else:
+        # `h5py.Group.copy` only accepts h5py objects.
+        dataset = parent.create_dataset(name, data=original[()])
+        dataset.attrs.update(original.attrs)
+
+
 def _resolve_hdf5_group_child(
     parent: GroupType, name: str
 ) -> Union[DatasetType, GroupType]:
@@ -427,51 +486,115 @@ def _resolve_hdf5_group_child(
     after resolving has the correct `:code:.file` but not the correct `:code:.name`.
     """
     link = _get_hdf5_link(parent, name)
-    if not isinstance(link, h5py.SoftLink):
-        return parent[name]
 
-    # Resolve the target path of the soft link.
-    target_path = link.path
-    if not target_path.startswith("/"):
-        target_path = _normalize_hdf5_item_name(parent.name, target_path)
+    if isinstance(link, h5py.SoftLink):
+        # Resolve the target of the soft link instead of resolving the soft
+        # link, as the target may be an external link itself.
+        target_path = _absolute_soft_link_target(parent, link)
+        return _resolve_hdf5_item(parent.file, target_path)
 
-    # Resolve the target of the soft link instead of resolving the soft link.
-    return parent.file[target_path]
+    if isinstance(link, h5py.ExternalLink):
+        # An h5py-like object behind an external link has no way of telling
+        # which file it belongs to: use the h5py object underneath, which does.
+        h5py_parent = _as_h5py_item(parent)
+        if h5py_parent is not None:
+            return h5py_parent[name]
 
+    return parent[name]
 
-def _soft_link_in_scope(
+
+def _resolve_hdf5_item(root: GroupType, path: str) -> Union[DatasetType, GroupType]:
+    """Resolve the absolute HDF5 path :code:`path` in :code:`root`, one link at
+    a time.
+    """
+    item = root
+    for name in path.split("/"):
+        if name:
+            item = _resolve_hdf5_group_child(item, name)
+    return item
+
+
+def _create_mirrored_soft_link(
     parent: GroupType,
-    link: Union[h5py.HardLink, h5py.SoftLink, h5py.ExternalLink],
-    home: DataUrl,
+    name: str,
+    reached_by: "_ReachedBy",
+    home_original: DataUrl,
+    home_destination: str,
+    exclude: Set[str],
 ) -> bool:
-    """Does :code:`link` refer to a path within :code:`home`'s path?"""
-    if not isinstance(link, h5py.SoftLink):
+    """Create a soft link under :code:`name` in :code:`parent`, pointing to the
+    mirror of the target of the in-scope soft link :code:`reached_by`.
+
+    :returns: :code:`False` when the target is not mirrored, in which case
+              nothing is created and the caller has to fall back to
+              referring to the target itself.
+    """
+    relative_target = _relative_to_home_original(
+        reached_by.soft_link_target, home_original
+    )
+    if any(
+        relative_target == path or relative_target.startswith(path + "/")
+        for path in exclude
+    ):
         return False
-    if os.path.abspath(_get_hdf5_filename(parent.file)) != home.file_path():
+    filename = _get_hdf5_filename(parent.file)
+    link = _hdf5_link_object(
+        parent_filename=filename,
+        parent_name=parent.name,
+        link_name=name,
+        target_filename=filename,
+        target_name=_normalize_hdf5_item_name(home_destination, relative_target),
+    )
+    if link is not None:
+        parent[name] = link
+    return True
+
+
+def _relative_to_home_original(absolute_path: str, home_original: DataUrl) -> str:
+    """HDF5 path of :code:`absolute_path` relative to :code:`home_original`'s path,
+    empty for :code:`home_original` itself.
+    """
+    relative_path = os.path.relpath(absolute_path, home_original.data_path())
+    if relative_path == ".":
+        return ""
+    return relative_path
+
+
+def _absolute_soft_link_target(parent: GroupType, link: h5py.SoftLink) -> str:
+    """Absolute HDF5 path of the target of soft link :code:`link` in :code:`parent`."""
+    if link.path.startswith("/"):
+        return _normalize_hdf5_item_name(link.path)
+    return _normalize_hdf5_item_name(parent.name, link.path)
+
+
+def _soft_link_in_scope(
+    parent: GroupType, target_path: str, home_original: DataUrl
+) -> bool:
+    """Does the soft link target :code:`target_path` refer to a path within
+    :code:`home_original`'s path?
+    """
+    if os.path.abspath(_get_hdf5_filename(parent.file)) != home_original.file_path():
         return False
-    target_path = link.path
-    if not target_path.startswith("/"):
-        target_path = _normalize_hdf5_item_name(parent.name, target_path)
-    return _in_scope(target_path, home.data_path())
+    return _in_scope(target_path, home_original.data_path())
 
 
-def _virtual_dataset_in_scope(original: DatasetType, home: DataUrl) -> bool:
+def _virtual_dataset_in_scope(original: h5py.Dataset, home_original: DataUrl) -> bool:
     """Does every virtual source of :code:`original` refer to a path
-    within :code:`home`'s path?
+    within :code:`home_original`'s path?
     """
-    home_path = home.data_path()
+    home_original_path = home_original.data_path()
     return all(
-        mapping.file_name == "." and _in_scope(mapping.dset_name, home_path)
+        mapping.file_name == "." and _in_scope(mapping.dset_name, home_original_path)
         for mapping in original.virtual_sources()
     )
 
 
-def _in_scope(absolute_path: str, home_path: str) -> bool:
+def _in_scope(absolute_path: str, home_original_path: str) -> bool:
     """Is :code:`absolute_path` (an absolute HDF5 path) equal to
-    :code:`home_path` or a descendant of it?
+    :code:`home_original_path` or a descendant of it?
     """
-    prefix = home_path.rstrip("/") + "/"
-    return absolute_path == home_path or absolute_path.startswith(prefix)
+    prefix = home_original_path.rstrip("/") + "/"
+    return absolute_path == home_original_path or absolute_path.startswith(prefix)
 
 
 def _create_hdf5_link(
@@ -596,12 +719,16 @@ def _item_conflict(parent: GroupType, name: str, overwrite: bool) -> bool:
 
 
 def _attr_conflict(
-    destination: Union[DatasetType, GroupType], key: str, overwrite: bool
+    destination: Union[DatasetType, GroupType], key: str, value: Any, overwrite: bool
 ) -> bool:
     """Returns :code:`True` when these is a conflict and it cannot be resolved."""
     if key not in destination.attrs:
         return False
 
+    if numpy.array_equal(destination.attrs[key], value):
+        # Nothing to overwrite
+        return True
+
     if not overwrite:
         logger.warning(
             "attribute %s already exists on %s: not overwritten",
@@ -620,7 +747,7 @@ def _update_attrs(
 ) -> None:
     """Copy :code:`original`'s attributes onto :code:`destination`."""
     for key, value in original.attrs.items():
-        if key in destination.attrs and _attr_conflict(destination, key, overwrite):
+        if _attr_conflict(destination, key, value, overwrite):
             continue
         destination.attrs[key] = value
 
@@ -654,6 +781,20 @@ def _get_hdf5_link(
     return h5py.HardLink()
 
 
+def _as_h5py_item(
+    item: Union[DatasetType, GroupType],
+) -> Optional[Union[h5py.Dataset, h5py.Group]]:
+    """The h5py object underlying :code:`item`, :code:`None` when there is none."""
+    # Duck-typed so it works for any h5py-like objects.
+    if isinstance(item, (h5py.Dataset, h5py.Group)):
+        return item
+    try:
+        # to be fixed in blissdata
+        return item._retry_handler.file_obj[item.name]
+    except (AttributeError, KeyError):
+        return None
+
+
 def _is_group(item: Union[DatasetType, GroupType]) -> bool:
     # Duck-typed so it works for any h5py-like objects.
     return hasattr(item, "keys")