legend-pydataobj 1.12.0a3__tar.gz → 1.13.0__tar.gz

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.12.0a3
+Version: 1.13.0
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/legend_pydataobj.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.12.0a3
+Version: 1.13.0
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/legend_pydataobj.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ src/lgdo/lh5/core.py
 src/lgdo/lh5/datatype.py
 src/lgdo/lh5/exceptions.py
 src/lgdo/lh5/iterator.py
+src/lgdo/lh5/settings.py
 src/lgdo/lh5/store.py
 src/lgdo/lh5/tools.py
 src/lgdo/lh5/utils.py

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.12.0a3'
-__version_tuple__ = version_tuple = (1, 12, 0, 'a3')
+__version__ = version = '1.13.0'
+__version_tuple__ = version_tuple = (1, 13, 0)

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/lh5/__init__.py

@@ -7,7 +7,6 @@ browsed easily in python like any `HDF5 <https://www.hdfgroup.org>`_ file using
 
 from __future__ import annotations
 
-from ._serializers.write.array import DEFAULT_HDF5_SETTINGS
 from .core import read, read_as, write
 from .iterator import LH5Iterator
 from .store import LH5Store
@@ -15,14 +14,15 @@ from .tools import ls, show
 from .utils import read_n_rows
 
 __all__ = [
-    "DEFAULT_HDF5_SETTINGS",
     "LH5Iterator",
     "LH5Store",
     "concat",
+    "default_hdf5_settings",
     "ls",
     "read",
     "read_as",
     "read_n_rows",
+    "reset_default_hdf5_settings",
     "show",
     "write",
 ]

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/lh5/_serializers/write/array.py

@@ -6,12 +6,11 @@ import h5py
 import numpy as np
 
 from .... import types
+from ... import settings
 from ...exceptions import LH5EncodeError
 
 log = logging.getLogger(__name__)
 
-DEFAULT_HDF5_SETTINGS: dict[str, ...] = {"shuffle": True, "compression": "gzip"}
-
 
 def _h5_write_array(
     obj,
@@ -49,7 +48,7 @@ def _h5_write_array(
             del group[name]
 
         # set default compression options
-        for k, v in DEFAULT_HDF5_SETTINGS.items():
+        for k, v in settings.DEFAULT_HDF5_SETTINGS.items():
             h5py_kwargs.setdefault(k, v)
 
         # compress using the 'compression' LGDO attribute, if available

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/lh5/_serializers/write/composite.py

@@ -64,6 +64,59 @@ def _h5_write_lgdo(
 
     group = utils.get_h5_group(group, lh5_file)
 
+    # name already in file
+    if name in group or (
+        ("datatype" in group.attrs or group == "/")
+        and (len(name) <= 2 or "/" not in name[1:-1])
+    ):
+        pass
+    # group is in file but not struct or need to create nesting
+    else:
+        # check if name is nested
+        # if name is nested, iterate up from parent
+        # otherwise we just need to iterate the group
+        if len(name) > 2 and "/" in name[1:-1]:
+            group = utils.get_h5_group(
+                name[:-1].rsplit("/", 1)[0],
+                group,
+            )
+            curr_name = (
+                name.rsplit("/", 1)[1]
+                if name[-1] != "/"
+                else name[:-1].rsplit("/", 1)[1]
+            )
+        else:
+            curr_name = name
+        # initialize the object to be written
+        obj = types.Struct({curr_name.replace("/", ""): obj})
+
+        # if base group already has a child we just append
+        if len(group) >= 1:
+            wo_mode = "ac"
+        else:
+            # iterate up the group hierarchy until we reach the root or a group with more than one child
+            while group.name != "/":
+                if len(group) > 1:
+                    break
+                curr_name = group.name
+                group = group.parent
+                if group.name != "/":
+                    obj = types.Struct({curr_name[len(group.name) + 1 :]: obj})
+                else:
+                    obj = types.Struct({curr_name[1:]: obj})
+            # if the group has more than one child, we need to append else we can overwrite
+            wo_mode = "ac" if len(group) > 1 else "o"
+
+        # set the new name
+        if group.name == "/":
+            name = "/"
+        elif group.parent.name == "/":
+            name = group.name[1:]
+        else:
+            name = group.name[len(group.parent.name) + 1 :]
+        # get the new group
+        group = utils.get_h5_group(group.parent if group.name != "/" else "/", lh5_file)
+
     if wo_mode == "w" and name in group:
         msg = f"can't overwrite '{name}' in wo_mode 'write_safe'"
         raise LH5EncodeError(msg, lh5_file, group, name)
@@ -87,7 +140,7 @@ def _h5_write_lgdo(
             lh5_file,
             group=group,
             start_row=start_row,
-            n_rows=n_rows,
+            n_rows=n_rows,  # if isinstance(obj, types.Table | types.Histogram) else None,
             wo_mode=wo_mode,
             write_start=write_start,
             **h5py_kwargs,
@@ -186,19 +239,31 @@ def _h5_write_struct(
     write_start=0,
     **h5py_kwargs,
 ):
+    # this works for structs and derived (tables)
     assert isinstance(obj, types.Struct)
 
     # In order to append a column, we need to update the
-    # `table{old_fields}` value in `group.attrs['datatype"]` to include
-    # the new fields.  One way to do this is to override
-    # `obj.attrs["datatype"]` to include old and new fields. Then we
-    # can write the fields to the table as normal.
+    # `struct/table{old_fields}` value in `group.attrs['datatype"]` to include
+    # the new fields. One way to do this is to override `obj.attrs["datatype"]`
+    # to include old and new fields. Then we can write the fields to the
+    # struct/table as normal.
     if wo_mode == "ac":
+        if name not in group:
+            msg = "Cannot append column to non-existing struct on disk"
+            raise LH5EncodeError(msg, lh5_file, group, name)
+
         old_group = utils.get_h5_group(name, group)
+        if "datatype" not in old_group.attrs:
+            msg = "Cannot append column to an existing  non-LGDO object on disk"
+            raise LH5EncodeError(msg, lh5_file, group, name)
+
         lgdotype = datatype.datatype(old_group.attrs["datatype"])
         fields = datatype.get_struct_fields(old_group.attrs["datatype"])
-        if not issubclass(lgdotype, types.Struct):
-            msg = f"Trying to append columns to an object of type {lgdotype.__name__}"
+        if lgdotype is not type(obj):
+            msg = (
+                "Trying to append columns to an object of different "
+                f"type {lgdotype.__name__}!={type(obj)}"
+            )
             raise LH5EncodeError(msg, lh5_file, group, name)
 
         # If the mode is `append_column`, make sure we aren't appending
@@ -211,8 +276,14 @@ def _h5_write_struct(
                 "column(s) to a table with the same field(s)"
             )
             raise LH5EncodeError(msg, lh5_file, group, name)
+
         # It doesn't matter what key we access, as all fields in the old table have the same size
-        if old_group[next(iter(old_group.keys()))].size != obj.size:
+        if (
+            isinstance(obj, types.Table)
+            and old_group.attrs["datatype"][:6]
+            != "struct"  # structs dont care about size
+            and old_group[next(iter(old_group.keys()))].size != obj.size
+        ):
             msg = (
                 f"Table sizes don't match. Trying to append column of size {obj.size} "
                 f"to a table of size {old_group[next(iter(old_group.keys()))].size}."
@@ -222,16 +293,27 @@ def _h5_write_struct(
         # Now we can append the obj.keys() to the old fields, and then update obj.attrs.
         fields.extend(list(obj.keys()))
         obj.attrs.pop("datatype")
-        obj.attrs["datatype"] = "table" + "{" + ",".join(fields) + "}"
+
+        obj.attrs["datatype"] = (
+            obj.datatype_name() + "{" + ",".join(sorted(fields)) + "}"
+        )
+
+        # propagating wo_mode="ac" to nested LGDOs does not make any sense
+        wo_mode = "append"
+
+        # overwrite attributes of the existing struct
+        attrs_overwrite = True
+    else:
+        attrs_overwrite = wo_mode == "o"
 
     group = utils.get_h5_group(
         name,
         group,
         grp_attrs=obj.attrs,
-        overwrite=(wo_mode in ["o", "ac"]),
+        overwrite=attrs_overwrite,
     )
     # If the mode is overwrite, then we need to peek into the file's
-    # table's existing fields.  If we are writing a new table to the
+    # table's existing fields. If we are writing a new table to the
     # group that does not contain an old field, we should delete that
     # old field from the file
     if wo_mode == "o":
@@ -260,11 +342,9 @@ def _h5_write_struct(
         else:
             obj_fld = obj[field]
 
-        # Convert keys to string for dataset names
-        f = str(field)
         _h5_write_lgdo(
             obj_fld,
-            f,
+            str(field),
             lh5_file,
             group=group,
             start_row=start_row,

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/lh5/core.py

@@ -5,6 +5,7 @@ import inspect
 import sys
 from collections.abc import Mapping, Sequence
 from contextlib import suppress
+from pathlib import Path
 from typing import Any
 
 import h5py
@@ -268,11 +269,13 @@ def write(
           end of array is the same as ``append``.
         - ``overwrite_file`` or ``of``: delete file if present prior to
           writing to it. `write_start` should be 0 (its ignored).
-        - ``append_column`` or ``ac``: append columns from an
-          :class:`~.lgdo.table.Table` `obj` only if there is an existing
-          :class:`~.lgdo.table.Table` in the `lh5_file` with the same
-          `name` and :class:`~.lgdo.table.Table.size`. If the sizes don't
-          match, or if there are matching fields, it errors out.
+        - ``append_column`` or ``ac``: append fields/columns from an
+          :class:`~.lgdo.struct.Struct` `obj` (and derived types such as
+          :class:`~.lgdo.table.Table`) only if there is an existing
+          :class:`~.lgdo.struct.Struct` in the `lh5_file` with the same `name`.
+          If there are matching fields, it errors out. If appending to a
+          ``Table`` and the size of the new column is different from the size
+          of the existing table, it errors out.
     write_start
         row in the output file (if already existing) to start overwriting
         from.
@@ -288,7 +291,12 @@ def write(
         datasets. **Note: `compression` Ignored if compression is specified
         as an `obj` attribute.**
     """
-    if wo_mode in ("w", "write", "of", "overwrite_file"):
+
+    if (
+        isinstance(lh5_file, str)
+        and not Path(lh5_file).is_file()
+        and wo_mode in ("w", "write_safe", "of", "overwrite_file")
+    ):
         h5py_kwargs.update(
             {
                 "fs_strategy": "page",

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/lh5/datatype.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import re
 from collections import OrderedDict
+from itertools import permutations as perm
 
 from .. import types as lgdo
 
@@ -14,7 +15,10 @@ _lgdo_datatype_map: dict[str, lgdo.LGDO] = OrderedDict(
             lgdo.ArrayOfEncodedEqualSizedArrays,
             r"^array_of_encoded_equalsized_arrays<1,1>\{.+\}$",
         ),
-        (lgdo.Histogram, r"^struct\{binning,weights,isdensity\}$"),
+        (
+            lgdo.Histogram,
+            rf"^struct\{{(?:{'|'.join([','.join(p) for p in perm(['binning', 'weights', 'isdensity'])])})\}}$",
+        ),
         (lgdo.Struct, r"^struct\{.*\}$"),
         (lgdo.Table, r"^table\{.*\}$"),
         (lgdo.FixedSizeArray, r"^fixedsize_array<\d+>\{.+\}$"),

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/lh5/iterator.py

@@ -17,48 +17,43 @@ LGDO = typing.Union[Array, Scalar, Struct, VectorOfVectors]
 
 
 class LH5Iterator(typing.Iterator):
-    """
-    A class for iterating through one or more LH5 files, one block of entries
-    at a time. This also accepts an entry list/mask to enable event selection,
-    and a field mask.
-
-    This can be used as an iterator:
-
-
-    >>> for lh5_obj in LH5Iterator(...):
-    >>>    # do the thing!
-
-    This is intended for if you are reading a large quantity of data. This
-    will ensure that you traverse files efficiently to minimize caching time
-    and will limit your memory usage (particularly when reading in waveforms!).
-    The ``lh5_obj`` that is read by this class is reused in order to avoid
-    reallocation of memory; this means that if you want to hold on to data
-    between reads, you will have to copy it somewhere!
-
-    When defining an LH5Iterator, you must give it a list of files and the
-    hdf5 groups containing the data tables you are reading. You may also
-    provide a field mask, and an entry list or mask, specifying which entries
-    to read from the files. You may also pair it with a friend iterator, which
-    contains a parallel group of files which will be simultaneously read.
-    In addition to accessing requested data via ``lh5_obj``, several
-    properties exist to tell you where that data came from:
-
-    - lh5_it.current_i_entry: get the index within the entry list of the
-      first entry that is currently read
-    - lh5_it.current_local_entries: get the entry numbers relative to the
-      file the data came from
-    - lh5_it.current_global_entries: get the entry number relative to the
-      full dataset
-    - lh5_it.current_files: get the file name corresponding to each entry
-    - lh5_it.current_groups: get the group name corresponding to each entry
-
-    This class can also be used for random access:
-
-    >>> lh5_obj = lh5_it.read(i_entry)
-
-    to read the block of entries starting at i_entry. In case of multiple files
-    or the use of an event selection, i_entry refers to a global event index
-    across files and does not count events that are excluded by the selection.
+    """Iterate over chunks of entries from LH5 files.
+
+    The iterator reads ``buffer_len`` entries at a time from one or more
+    files.  The LGDO instance returned at each iteration is reused to avoid
+    reallocations, so copy the data if it should be preserved.
+
+    Examples
+    --------
+    Iterate through a table one chunk at a time::
+
+        from lgdo.lh5 import LH5Iterator
+
+        for table in LH5Iterator("data.lh5", "geds/raw/energy", buffer_len=100):
+            process(table)
+
+    ``LH5Iterator`` can also be used for random access::
+
+        it = LH5Iterator(files, groups)
+        table = it.read(i_entry)
+
+    In case of multiple files or an entry selection, ``i_entry`` refers to the
+    global event index across all files.
+
+    When instantiating an iterator you must provide a list of files and the
+    HDF5 groups to read.  Optional parameters allow field masking, event
+    selection and pairing the iterator with a "friend" iterator that is read in
+    parallel.  Several properties are available to obtain the provenance of the
+    data currently loaded:
+
+    - ``current_i_entry`` -- index within the entry list of the first entry in
+      the buffer
+    - ``current_local_entries`` -- entry numbers relative to the file the data
+      came from
+    - ``current_global_entries`` -- entry number relative to the full dataset
+    - ``current_files`` -- file name corresponding to each entry in the buffer
+    - ``current_groups`` -- group name corresponding to each entry in the
+      buffer
     """
 
     def __init__(
@@ -505,7 +500,7 @@ class LH5Iterator(typing.Iterator):
         return self
 
     def __next__(self) -> tuple[LGDO, int, int]:
-        """Read next buffer_len entries and return lh5_table and iterator entry."""
+        """Read the next chunk of entries and return the buffer."""
         n_entries = self.n_entries
         if n_entries is not None:
             n_entries = min(

legend_pydataobj-1.13.0/src/lgdo/lh5/settings.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def default_hdf5_settings() -> dict[str, Any]:
+    """Returns the HDF5 settings for writing data to disk to the pydataobj defaults.
+
+    Examples
+    --------
+    >>> from lgdo import lh5
+    >>> lh5.DEFAULT_HDF5_SETTINGS["compression"] = "lzf"
+    >>> lh5.write(data, "data", "file.lh5")  # compressed with LZF
+    >>> lh5.DEFAULT_HDF5_SETTINGS = lh5.default_hdf5_settings()
+    >>> lh5.write(data, "data", "file.lh5", "of")  # compressed with default settings (GZIP)
+    """
+
+    return {
+        "shuffle": True,
+        "compression": "gzip",
+    }
+
+
+DEFAULT_HDF5_SETTINGS: dict[str, ...] = default_hdf5_settings()
+"""Global dictionary storing the default HDF5 settings for writing data to disk.
+
+Modify this global variable before writing data to disk with this package.
+
+Examples
+--------
+>>> from lgdo import lh5
+>>> lh5.DEFAULT_HDF5_SETTINGS["compression"] = "lzf"
+>>> lh5.write(data, "data", "file.lh5")  # compressed with LZF
+"""

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/lh5/utils.py

@@ -111,7 +111,7 @@ def get_h5_group(
         grp_attrs is not None
         and len(set(grp_attrs.items()) ^ set(group.attrs.items())) > 0
     ):
-        if not overwrite:
+        if not overwrite and len(group.attrs) != 0:
             msg = (
                 f"Provided {grp_attrs=} are different from "
                 f"existing ones {dict(group.attrs)=} but overwrite flag is not set"
@@ -154,7 +154,7 @@ def expand_vars(expr: str, substitute: dict[str, str] | None = None) -> str:
 
     # use provided mapping
     # then expand env variables
-    return os.path.expandvars(string.Template(expr).safe_substitute(substitute))
+    return os.path.expandvars(string.Template(str(expr)).safe_substitute(substitute))
 
 
 def expand_path(

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/types/struct.py

@@ -5,7 +5,9 @@ utilities.
 
 from __future__ import annotations
 
+import copy
 import logging
+import re
 from collections.abc import Mapping
 from typing import Any
 
@@ -56,7 +58,21 @@ class Struct(LGDO, dict):
                     # assign
                     super().update({k: v})
 
-        # call LGDO constructor to setup attributes
+        # check the datatype attribute passed by the user and sort the fields
+        # to ensure consistent behavior
+        if attrs is not None and "datatype" in attrs:
+            _attrs = copy.copy(dict(attrs))
+
+            if not _is_struct_datatype(self.datatype_name(), _attrs["datatype"]):
+                msg = (
+                    f"datatype attribute ({self.attrs['datatype']}) is not "
+                    f"compatible with class datatype!"
+                )
+                raise ValueError(msg)
+
+            _attrs["datatype"] = _sort_datatype_fields(_attrs["datatype"])
+            attrs = _attrs
+
         super().__init__(attrs)
 
     def datatype_name(self) -> str:
@@ -64,7 +80,10 @@ class Struct(LGDO, dict):
 
     def form_datatype(self) -> str:
         return (
-            self.datatype_name() + "{" + ",".join([str(k) for k in self.keys()]) + "}"
+            self.datatype_name()
+            + "{"
+            + ",".join(sorted([str(k) for k in self.keys()]))
+            + "}"
         )
 
     def update_datatype(self) -> None:
@@ -157,3 +176,34 @@ class Struct(LGDO, dict):
             "not possible. Call view_as() on the fields instead."
         )
         raise NotImplementedError(msg)
+
+
+def _is_struct_datatype(dt_name, expr):
+    return re.search("^" + dt_name + r"\{(.*)\}$", expr) is not None
+
+
+def _get_struct_fields(expr: str) -> list[str]:
+    assert _is_struct_datatype(".*", expr)
+
+    arr = re.search(r"\{(.*)\}$", expr).group(1).split(",")
+    if arr == [""]:
+        arr = []
+
+    return sorted(arr)
+
+
+def _struct_datatype_equal(dt_name, dt1, dt2):
+    if any(not _is_struct_datatype(dt_name, dt) for dt in (dt1, dt2)):
+        return False
+
+    return _get_struct_fields(dt1) == _get_struct_fields(dt2)
+
+
+def _sort_datatype_fields(expr):
+    assert _is_struct_datatype(".*", expr)
+
+    match = re.search(r"^(.*)\{.*\}$", expr)
+    struct_type = match.group(1)
+    fields = _get_struct_fields(expr)
+
+    return struct_type + "{" + ",".join(sorted([str(k) for k in fields])) + "}"

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/src/lgdo/types/table.py

@@ -81,8 +81,9 @@ class Table(Struct, LGDOCollection):
             col_dict = _ak_to_lgdo_or_col_dict(col_dict)
 
         # call Struct constructor
-        Struct.__init__(self, obj_dict=col_dict)
-        LGDOCollection.__init__(self, attrs=attrs)
+        Struct.__init__(self, obj_dict=col_dict, attrs=attrs)
+        # no need to call the LGDOCollection constructor, as we are calling the
+        # Struct constructor already
 
         # if col_dict is not empty, set size according to it
         # if size is also supplied, resize all fields to match it

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/tests/lh5/test_core.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import numpy as np
+import pytest
 
 import lgdo
 from lgdo import lh5
@@ -28,6 +29,51 @@ def test_write(tmptestdir):
     assert lh5.ls(f"{tmptestdir}/tmp-pygama-lgdo-types2.lh5")
 
 
+def test_write_safe(tmptestdir):
+    # write_safe should create new file
+    struct = lgdo.Struct()
+    struct.add_field("scalar", lgdo.Scalar(value=10, attrs={"sth": 1}))
+    lh5.write(
+        struct,
+        "struct",
+        f"{tmptestdir}/tmp-pygama-write_safe.lh5",
+        group="/data",
+        start_row=1,
+        n_rows=3,
+        wo_mode="w",
+    )
+    assert lh5.ls(f"{tmptestdir}/tmp-pygama-write_safe.lh5")
+
+    # write_safe should add a new group to an existing file
+    struct = lgdo.Struct()
+    struct.add_field("scalar", lgdo.Scalar(value=10, attrs={"sth": 1}))
+    lh5.write(
+        struct,
+        "struct2",
+        f"{tmptestdir}/tmp-pygama-write_safe.lh5",
+        group="/data",
+        start_row=1,
+        n_rows=3,
+        wo_mode="w",
+    )
+    assert lh5.ls(f"{tmptestdir}/tmp-pygama-write_safe.lh5", "data/") == [
+        "data/struct",
+        "data/struct2",
+    ]
+
+    # write_safe should not allow writing to existing dataset
+    with pytest.raises(lh5.exceptions.LH5EncodeError):
+        lh5.write(
+            struct,
+            "struct",
+            f"{tmptestdir}/tmp-pygama-write_safe.lh5",
+            group="/data",
+            start_row=1,
+            n_rows=3,
+            wo_mode="w",
+        )
+
+
 def test_read_as(lh5_file):
     store = lh5.LH5Store()
     obj1 = store.read("/data/struct/table", lh5_file, start_row=1)

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/tests/lh5/test_lh5_store.py

@@ -9,7 +9,6 @@ import pytest
 
 import lgdo
 from lgdo import lh5, types
-from lgdo.lh5 import DEFAULT_HDF5_SETTINGS
 
 
 def test_init():
@@ -42,6 +41,54 @@ def test_write_objects(lh5_file):
     pass
 
 
+def test_write_safe(tmptestdir):
+    # write_safe should create new file
+    lh5_st = lh5.LH5Store()
+    struct = lgdo.Struct()
+    struct.add_field("scalar", lgdo.Scalar(value=10, attrs={"sth": 1}))
+    lh5_st.write(
+        struct,
+        "struct",
+        f"{tmptestdir}/tmp-pygama-write_safe_store.lh5",
+        group="/data",
+        start_row=1,
+        n_rows=3,
+        wo_mode="w",
+    )
+    assert lh5.ls(f"{tmptestdir}/tmp-pygama-write_safe_store.lh5")
+
+    # write_safe should add a new group to an existing file
+    lh5_st = lh5.LH5Store()
+    struct = lgdo.Struct()
+    struct.add_field("scalar", lgdo.Scalar(value=10, attrs={"sth": 1}))
+    lh5_st.write(
+        struct,
+        "struct2",
+        f"{tmptestdir}/tmp-pygama-write_safe_store.lh5",
+        group="/data",
+        start_row=1,
+        n_rows=3,
+        wo_mode="w",
+    )
+    assert lh5.ls(f"{tmptestdir}/tmp-pygama-write_safe_store.lh5", "data/") == [
+        "data/struct",
+        "data/struct2",
+    ]
+
+    # write_safe should not allow writing to existing dataset
+    lh5_st = lh5.LH5Store()
+    with pytest.raises(lh5.exceptions.LH5EncodeError):
+        lh5_st.write(
+            struct,
+            "struct",
+            f"{tmptestdir}/tmp-pygama-write_safe_store.lh5",
+            group="/data",
+            start_row=1,
+            n_rows=3,
+            wo_mode="w",
+        )
+
+
 def test_read_n_rows(lh5_file):
     store = lh5.LH5Store()
     assert store.read_n_rows("/data/struct_full/aoesa", lh5_file) == 5
@@ -96,7 +143,7 @@ def test_read_array(lh5_file):
     with h5py.File(lh5_file) as h5f:
         assert (
             h5f["/data/struct/array"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
 
     lh5_obj = store.read("/data/struct_full/array2d", lh5_file)
@@ -170,11 +217,11 @@ def test_read_vov(lh5_file):
     with h5py.File(lh5_file) as h5f:
         assert (
             h5f["/data/struct/vov/cumulative_length"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
         assert (
             h5f["/data/struct/vov/flattened_data"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
 
     lh5_obj = store.read("/data/struct/vov3d", lh5_file)
@@ -229,11 +276,11 @@ def test_read_voev(lh5_file):
         assert h5f["/data/struct/voev/encoded_data/flattened_data"].compression is None
         assert (
             h5f["/data/struct/voev/encoded_data/cumulative_length"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
         assert (
             h5f["/data/struct/voev/decoded_size"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
 
 
@@ -312,7 +359,7 @@ def test_read_hdf5_compressed_data(lh5_file):
     with h5py.File(lh5_file) as h5f:
         assert (
             h5f["/data/struct/table/a"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
         assert h5f["/data/struct/table/b"].compression == "gzip"
         assert h5f["/data/struct/table/c"].compression == "gzip"
@@ -333,15 +380,15 @@ def test_read_wftable(lh5_file):
     with h5py.File(lh5_file) as h5f:
         assert (
             h5f["/data/struct/wftable/values"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
         assert (
             h5f["/data/struct/wftable/t0"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
         assert (
             h5f["/data/struct/wftable/dt"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
 
 
@@ -387,11 +434,11 @@ def test_read_wftable_encoded(lh5_file):
         assert h5f["/data/struct/wftable_enc/values/decoded_size"].compression is None
         assert (
             h5f["/data/struct/wftable_enc/t0"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
         assert (
             h5f["/data/struct/wftable_enc/dt"].compression
-            is DEFAULT_HDF5_SETTINGS["compression"]
+            is lh5.settings.DEFAULT_HDF5_SETTINGS["compression"]
         )
 
 

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/tests/lh5/test_lh5_utils.py

@@ -51,3 +51,10 @@ def test_expand_path(lgnd_test_data):
 
     # Check if it finds a list of files correctly
     assert sorted(utils.expand_path(f"{base_dir}/*.lh5", list=True)) == sorted(files)
+
+    # check with base_path specified
+    base_path = base_dir.parent
+    assert (
+        utils.expand_path(f"{base_dir.name}/*20230318T012144Z*", base_path=base_path)
+        == Path(files[0]).relative_to(base_path).as_posix()
+    )

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/tests/lh5/test_lh5_write.py

@@ -15,6 +15,30 @@ def test_write_compressed_lgnd_waveform_table(enc_lgnd_file):
     pass
 
 
+def test_write_with_hdf5_compression_global(tmptestdir):
+    data = types.Table(
+        size=1000,
+        col_dict={
+            "col1": types.Array(np.arange(0, 100, 0.1)),
+            "col2": types.Array(np.random.default_rng().random(1000)),
+        },
+    )
+    outfile = f"{tmptestdir}/write_hdf5_data_global_var.lh5"
+
+    lh5.settings.DEFAULT_HDF5_SETTINGS["shuffle"] = False
+    lh5.settings.DEFAULT_HDF5_SETTINGS["compression"] = "lzf"
+
+    lh5.write(data, "data", outfile, wo_mode="of")
+
+    with h5py.File(outfile) as h5f:
+        assert h5f["/data/col1"].shuffle is False
+        assert h5f["/data/col1"].compression == "lzf"
+
+    lh5.settings.DEFAULT_HDF5_SETTINGS = lh5.settings.default_hdf5_settings()
+    assert lh5.settings.DEFAULT_HDF5_SETTINGS["shuffle"] is True
+    assert lh5.settings.DEFAULT_HDF5_SETTINGS["compression"] == "gzip"
+
+
 def test_write_with_hdf5_compression(lgnd_file, tmptestdir):
     store = lh5.LH5Store()
     wft = store.read("/geds/raw/waveform", lgnd_file)
@@ -452,6 +476,7 @@ def test_write_histogram(caplog, tmptestdir):
 
     # Now, check that the data were overwritten
     h3 = store.read("my_group/my_histogram", f"{tmptestdir}/write_histogram_test.lh5")
+    assert isinstance(h3, types.Histogram)
     assert np.array_equal(h3.weights.nda, np.array([[10, 10], [10, 10]]))
     assert h3.binning[0].edges[0] == 2
     assert h3.binning[1].edges[-1] == 7
@@ -518,6 +543,7 @@ def test_write_histogram_variable(caplog, tmptestdir):
 
     # Now, check that the data were overwritten
     h3 = store.read("my_group/my_histogram", f"{tmptestdir}/write_histogram_test.lh5")
+    assert isinstance(h3, types.Histogram)
     assert np.array_equal(h3.weights.nda, np.array([[10, 10], [10, 10]]))
     assert np.array_equal(h3.binning[0].edges, np.array([2, 3.5, 4]))
     with pytest.raises(TypeError):
@@ -531,3 +557,86 @@ def test_write_histogram_variable(caplog, tmptestdir):
     assert h3.attrs["testattr"] == "test"
     assert h3["weights"].attrs["weightattr"] == "testweight"
     assert h3.binning[0].attrs["binningattr"] == "testbinning"
+
+
+def test_write_append_struct(tmptestdir):
+    outfile = str(tmptestdir / "test-write-append-struct.lh5")
+    st = types.Struct({"arr1": types.Table({"a": types.Array([1, 2, 3])})})
+    lh5.write(st, "struct", outfile, wo_mode="of")
+    st2 = types.Struct({"arr2": types.Table({"a": types.Array([1, 2, 3])})})
+    lh5.write(st2, "struct", outfile, wo_mode="ac")
+
+    result = lh5.read("struct", outfile)
+    assert list(result.keys()) == ["arr1", "arr2"]
+    assert len(result.arr1) == len(st.arr1)
+    assert len(result.arr2) == len(st2.arr2)
+
+    # test error when appending existing field
+    with pytest.raises(lh5.exceptions.LH5EncodeError):
+        lh5.write(
+            types.Struct({"arr2": types.Array([4, 5, 6])}),
+            "struct",
+            outfile,
+            wo_mode="ac",
+        )
+
+    # error if appending to object of different type
+    with pytest.raises(lh5.exceptions.LH5EncodeError):
+        lh5.write(
+            types.Struct({"arr2": types.Array([4, 5, 6])}),
+            "struct",
+            outfile,
+            wo_mode="ac",
+        )
+
+    outfile = str(tmptestdir / "test-write-append-struct.lh5")
+    lh5.write(
+        types.Table({"arr1": types.Array([1, 2, 3])}), "struct", outfile, wo_mode="of"
+    )
+
+    # error if appending to object of different type
+    with pytest.raises(lh5.exceptions.LH5EncodeError):
+        lh5.write(
+            types.Table({"arr2": types.Array([4, 5, 6, 7])}),
+            "struct",
+            outfile,
+            wo_mode="ac",
+        )
+
+    # append to empty struct
+    outfile = str(tmptestdir / "test-write-append-struct.lh5")
+    lh5.write(types.Struct({}), "struct", outfile, wo_mode="of")
+    st2 = types.Struct({"arr2": types.Table({"a": types.Array([1, 2, 3])})})
+    lh5.write(st2, "struct", outfile, wo_mode="ac")
+
+    result = lh5.read("struct", outfile)
+    assert list(result.keys()) == ["arr2"]
+    assert len(result.arr2) == len(st2.arr2)
+
+
+def test_write_structs_not_groups(tmptestdir):
+    outfile = str(tmptestdir / "test-write-structs-not-groups2.lh5")
+
+    scalar = types.Scalar("made with legend-pydataobj!")
+    array = types.Array([1, 2, 3])
+    array2 = types.Array([4, 5, 6])
+    lh5.write(scalar, name="message", lh5_file=outfile, wo_mode="overwrite_file")
+    lh5.write(array, name="numbers", group="closet", lh5_file=outfile)
+    lh5.write(array2, name="numbers2", group="closet", lh5_file=outfile)
+    result = lh5.read("/", outfile)
+    assert isinstance(result, types.Struct)
+    assert result.attrs["datatype"] == "struct{closet,message}"
+
+    outfile = str(tmptestdir / "test-write-structs-not-groups.lh5")
+    tb = types.Table({"a": types.Array([1, 2, 3])})
+    lh5.write(tb, "test/table", outfile)
+    print(lh5.show(outfile))
+    tb2 = types.Table({"a": types.Array([4, 5, 6])})
+    lh5.write(tb2, "test/table2", outfile)
+    print(lh5.show(outfile))
+
+    result = lh5.read("test", outfile)
+    assert isinstance(result, types.Struct)
+    assert result.attrs["datatype"] == "struct{table,table2}"
+    assert result.table.a == tb.a
+    assert result.table2.a == tb2.a

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/tests/types/test_histogram.py

@@ -98,7 +98,7 @@ def test_init_np():
 def test_datatype_name():
     h = Histogram(np.array([1, 1]), (np.array([0, 1, 2]),))
     assert h.datatype_name() == "struct"
-    assert h.form_datatype() == "struct{binning,weights,isdensity}"
+    assert h.form_datatype() == "struct{binning,isdensity,weights}"
 
 
 def test_axes():
@@ -266,7 +266,7 @@ def test_view_as_np():
 
 def test_not_like_table():
     h = Histogram(np.array([1, 1]), (np.array([0, 1, 2]),))
-    assert h.form_datatype() == "struct{binning,weights,isdensity}"
+    assert h.form_datatype() == "struct{binning,isdensity,weights}"
     with pytest.raises(AttributeError):
         x = h.x  # noqa: F841
     with pytest.raises(AttributeError):

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/tests/types/test_struct.py

@@ -58,10 +58,10 @@ def test_add_field():
     assert struct["scalar1"].__class__.__name__ == "Scalar"
 
     struct.add_field("array1", lgdo.Array(shape=(700, 21), dtype="f", fill_val=2))
-    assert struct.attrs["datatype"] == "struct{scalar1,array1}"
+    assert struct.attrs["datatype"] == "struct{array1,scalar1}"
 
     struct["array2"] = lgdo.Array(shape=(700, 21), dtype="f", fill_val=2)
-    assert struct.attrs["datatype"] == "struct{scalar1,array1,array2}"
+    assert struct.attrs["datatype"] == "struct{array1,array2,scalar1}"
 
 
 def test_getattr():

{legend_pydataobj-1.12.0a3 → legend_pydataobj-1.13.0}/tests/types/test_waveformtable.py

@@ -12,7 +12,7 @@ def test_init():
     assert (wft.dt.nda == np.full(1024, fill_value=1)).all()
     assert isinstance(wft.values, lgdo.VectorOfVectors)
     assert len(wft.values) == 1024
-    assert wft.attrs == {"datatype": "table{t0,dt,values}"}
+    assert wft.attrs == {"datatype": "table{dt,t0,values}"}
 
     wft = WaveformTable(dt_units="ns", values_units="adc")
     assert wft.dt.attrs["units"] == "ns"