legend-pydataobj 1.13.0__py3-none-any.whl → 1.14.1__py3-none-any.whl

{legend_pydataobj-1.13.0.dist-info → legend_pydataobj-1.14.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.13.0
+Version: 1.14.1
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.13.0.dist-info → legend_pydataobj-1.14.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
-legend_pydataobj-1.13.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+legend_pydataobj-1.14.1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
 lgdo/__init__.py,sha256=fkRv79kdtBasw31gPVK9SdLQ2vEEajTV2t3UPDvFg9o,3206
-lgdo/_version.py,sha256=tH8KJgTzCjiwedQXB_0jJIzqyRjqKXz40ci9OEj6wNA,513
+lgdo/_version.py,sha256=dJRV9pQgpOiwiRbOfIEKw29vLhc6CuMVTXwqoHaRZvM,513
 lgdo/cli.py,sha256=s_EWTBWW76l7zWb6gaTSTjiT-0RzzcYEmjeFEQCVxfk,4647
 lgdo/lgdo_utils.py,sha256=6a2YWEwpyEMXlAyTHZMO01aqxy6SxJzPZkGNWKNWuS0,2567
 lgdo/logging.py,sha256=82wIOj7l7xr3WYyeHdpSXbbjzHJsy-uRyKYUYx2vMfQ,1003
@@ -9,48 +9,48 @@ lgdo/utils.py,sha256=WRTmXnaQ-h2hVxwJ27qiOigdsD3DHcaDrdDjvupCuZU,3940
 lgdo/compression/__init__.py,sha256=xHt_8Th0LxxNwj9iYHf5uGNTm3A_4qyW7zEVdAX3NwI,1127
 lgdo/compression/base.py,sha256=82cQJujfvoAOKBFx761dEcx_xM02TBCBBuBo6i78tuI,838
 lgdo/compression/generic.py,sha256=tF3UhLJbUDcovLxpIzgQRxFSjZ5Fz3uDRy9kI4mFntQ,2515
-lgdo/compression/radware.py,sha256=GcNTtjuyL7VBBqziUBmSqNXuhqy1bJJgvcyvyumPtrc,23839
+lgdo/compression/radware.py,sha256=-W7LgvkSVzdVJ6qNn7Ts3O9EcRcl8mUiApTLqR4dtIo,23836
 lgdo/compression/utils.py,sha256=W2RkBrxPpXlat84dnU9Ad7d_tTws0irtGl7O1dNWjnk,1140
-lgdo/compression/varlen.py,sha256=6ZZUItyoOfygDdE0DyoISeFZfqdbH6xl7T0eclfarzg,15127
+lgdo/compression/varlen.py,sha256=bjyxhHzfpi6PIPy-Uc47W8_LrRbFoJLJ2kVeD5nhyqo,15125
 lgdo/lh5/__init__.py,sha256=smHTawINIiogHNfYJq3aPvtxleTnBMdPADRCdc1wea8,748
 lgdo/lh5/concat.py,sha256=BZCgK7TWPKK8fMmha8K83d3bC31FVO1b5LOW7x-Ru1s,6186
-lgdo/lh5/core.py,sha256=U0ZZk6EmojRRYFBEo_bMy7jZ3SKBU41MIsSulyFxZIU,13752
+lgdo/lh5/core.py,sha256=ER5tAEevD5--xEm_kNHpeHC9hcSaSBFHW3qohyvuvmM,14120
 lgdo/lh5/datatype.py,sha256=ry3twFaosuBoskiTKqtBYRMk9PQAf403593xKaItfog,1827
-lgdo/lh5/exceptions.py,sha256=3kj8avXl4eBGvebl3LG12gJEmw91W0T8PYR0AfvUAyM,1211
-lgdo/lh5/iterator.py,sha256=DOx6tDaJE1NajA7klTVWeblL_k6M0JWjmBiHYhpSri0,19970
+lgdo/lh5/exceptions.py,sha256=Q374YeqajpptVCYfxJYrThiPZSnfpdbGV3qVwoUuEFo,1697
+lgdo/lh5/iterator.py,sha256=vuN98pa-xHDWXM2GMxvMxFEJGfHatMX6ajqnaP55PuY,20680
 lgdo/lh5/settings.py,sha256=cmPd6ZvneAF5sFMA1qf-9g_YSSygJcQSRmZDp1_sBEU,1001
-lgdo/lh5/store.py,sha256=qkBm3gPbr1R2UlQpUuDR5sGRMzpYJBWFL8fDIry6tmQ,8474
-lgdo/lh5/tools.py,sha256=drtJWHY82wCFuFr6LVVnm2AQgs_wZuFmAvyOB4tcOHs,6431
-lgdo/lh5/utils.py,sha256=D3Cn1U7NLEkUVGpKdUzTjoNIhaMBFZPbjGsjnxpeu6A,6362
+lgdo/lh5/store.py,sha256=HJuDjWQ8ztrKDoyWW3cplhtWDnz3J4a-Fe2WF4fzOY4,8676
+lgdo/lh5/tools.py,sha256=EZTCj3TMMp4Rnocq1F0QeO1yYHzx4yMR7l_Em4G7sC4,6503
+lgdo/lh5/utils.py,sha256=hxPoaG25MOhuu7emrw2xzx3zerl-GzeMWdlfoQmLiYo,6667
 lgdo/lh5/_serializers/__init__.py,sha256=eZzxMp1SeZWG0PkEXUiCz3XyprQ8EmelHUmJogC8xYE,1263
 lgdo/lh5/_serializers/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lgdo/lh5/_serializers/read/array.py,sha256=uWfMCihfAmW2DE2ewip2qCK_kvQC_mb2zvOv26uzijc,1000
 lgdo/lh5/_serializers/read/composite.py,sha256=UvkZHEhf0V7SFLxzF52eyP68hU0guGOLqosrfmIfeys,11729
 lgdo/lh5/_serializers/read/encoded.py,sha256=Q98c08d8LkZq2AlY4rThYECVaEqwbv4T2Urn7TGnsyE,4130
-lgdo/lh5/_serializers/read/ndarray.py,sha256=lFCXD6bSzmMOH7cVmvRYXakkfMCI8EoqTPNONRJ1F0s,3690
+lgdo/lh5/_serializers/read/ndarray.py,sha256=cxzZ7esT5BzxyoXfITBG_EDTtCVxSeSu6dVZrohOdOY,3685
 lgdo/lh5/_serializers/read/scalar.py,sha256=kwhWm1T91pXf86CqtUUD8_qheSR92gXZrQVtssV5YCg,922
 lgdo/lh5/_serializers/read/utils.py,sha256=YfSqPO-83A1XvhhuULxQ0Qz2A5ODa3sb7ApNxQVJXd0,7581
 lgdo/lh5/_serializers/read/vector_of_vectors.py,sha256=765P8mElGArAaEPkHTAUXFQ47t1_3-3BQAete0LckBQ,7207
 lgdo/lh5/_serializers/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lgdo/lh5/_serializers/write/array.py,sha256=gAB1EjPa9YojPqohVwY_VPeuY7_bLN-lttCmfgty-qk,3175
-lgdo/lh5/_serializers/write/composite.py,sha256=sZfV8aGZCH0mvMZ2dGDKt-MoepgL4PlR9ZWbT_JNIjQ,12171
+lgdo/lh5/_serializers/write/array.py,sha256=qzRNPQ4mtvc7HYPE3vUcM6bi7lWYnolNStdJVcDfzPU,3174
+lgdo/lh5/_serializers/write/composite.py,sha256=Cm9q0xVk27fmmrAeAp1q18E7VP7UwHako01115XdFHg,12964
 lgdo/lh5/_serializers/write/scalar.py,sha256=JPt_fcdTKOSFp5hfJdcKIfK4hxhcD8vhOlvDF-7btQ8,763
 lgdo/lh5/_serializers/write/vector_of_vectors.py,sha256=puGQX9XF5P_5DVbm_Cc6TvPrsDywgBLSYtkqFNltbB4,3493
 lgdo/types/__init__.py,sha256=DNfOErPiAZg-7Gygkp6ZKAi20Yrm1mfderZHvKo1Y4s,821
-lgdo/types/array.py,sha256=vxViJScqKw4zGUrrIOuuU_9Y0oTfOkEEhs0TOyUYjwI,9284
+lgdo/types/array.py,sha256=TpZINHgGIptslwr5mwKYWU_PrYAk8bH1ECJ4XfLkWxg,9338
 lgdo/types/arrayofequalsizedarrays.py,sha256=DOGJiTmc1QCdm7vLbE6uIRXoMPtt8uuCfmwQawgWf5s,4949
-lgdo/types/encoded.py,sha256=_e8u_BPfpjJbLnEdyTo9QG3kbNsGj0BN4gjdj3L1ndw,15640
+lgdo/types/encoded.py,sha256=8DJHb3kxz6RrmjkeLWS6iyjvIJqx86mDInWqqjpMON0,15752
 lgdo/types/fixedsizearray.py,sha256=7RjUwTz1bW0pcrdy27JlfrXPAuOU89Kj7pOuSUCojK8,1527
 lgdo/types/histogram.py,sha256=Jz1lLH56BfYnmcUhxUHK1h2wLDQ0Abgyd-6LznU-3-k,19979
 lgdo/types/lgdo.py,sha256=21YNtJCHnSO3M60rjsAdbMO5crDjL_0BtuFpudZ2xvU,4500
-lgdo/types/scalar.py,sha256=c5Es2vyDqyWTPV6mujzfIzMpC1jNWkEIcvYyWQUxH3Q,1933
+lgdo/types/scalar.py,sha256=nBPiqX4g3GrPavEbG6nCt2Jel7Mj0IchXqwxB6ei_rg,1989
 lgdo/types/struct.py,sha256=m3pYfGfKptV8ti3wb4n1nsPKMvhjdWCFoRdR5YooZBM,6353
-lgdo/types/table.py,sha256=hvOwhFkm-_CkNhGmD8SJoeepZcwFY6ItYOS76LztKtA,20158
-lgdo/types/vectorofvectors.py,sha256=GbAKV_ehXN4XdWSwnmKS_ErCiudRetcH_3wo7iDrVjw,26854
+lgdo/types/table.py,sha256=huhgpzdAUx0bRaEaitwnb-Ve7oAu5B6zxPK5EXPUfg0,20233
+lgdo/types/vectorofvectors.py,sha256=k1LwNnX3TcRAhOujj85kNkfZN0MXZYL9aaMUbr82JlE,26910
 lgdo/types/vovutils.py,sha256=LW3ZcwECxVYxxcFadAtY3nnK-9-rk8Xbg_m8hY30lo4,10708
 lgdo/types/waveformtable.py,sha256=9S_NMg894NZTGt2pLuskwH4-zQ5EbLnzWI6FVui6fXE,9827
-legend_pydataobj-1.13.0.dist-info/METADATA,sha256=j_3ElXkBp__BND0nYsfGPEEwVgB_-_P4LNcffIFiv70,44443
-legend_pydataobj-1.13.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-legend_pydataobj-1.13.0.dist-info/entry_points.txt,sha256=0KWfnwbuwhNn0vPUqARukjp04Ca6lzfZBSirouRmk7I,76
-legend_pydataobj-1.13.0.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
-legend_pydataobj-1.13.0.dist-info/RECORD,,
+legend_pydataobj-1.14.1.dist-info/METADATA,sha256=zkJj_GxpFmtHsxmgZOxN2hjUxEZduPKpswZoIS_Kq54,44443
+legend_pydataobj-1.14.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+legend_pydataobj-1.14.1.dist-info/entry_points.txt,sha256=0KWfnwbuwhNn0vPUqARukjp04Ca6lzfZBSirouRmk7I,76
+legend_pydataobj-1.14.1.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
+legend_pydataobj-1.14.1.dist-info/RECORD,,

lgdo/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.13.0'
-__version_tuple__ = version_tuple = (1, 13, 0)
+__version__ = version = '1.14.1'
+__version_tuple__ = version_tuple = (1, 14, 1)

lgdo/compression/radware.py

@@ -95,13 +95,13 @@ def encode(
     if isinstance(sig_in, np.ndarray):
         s = sig_in.shape
         if len(sig_in) == 0:
-            return np.empty(s[:-1] + (0,), dtype=ubyte), np.empty(0, dtype=uint32)
+            return np.empty((*s[:-1], 0), dtype=ubyte), np.empty(0, dtype=uint32)
 
         if sig_out is None:
             # the encoded signal is an array of bytes
             # -> twice as long as a uint16
             # pre-allocate ubyte (uint8) array, expand last dimension
-            sig_out = np.empty(s[:-1] + (s[-1] * 2,), dtype=ubyte)
+            sig_out = np.empty((*s[:-1], s[-1] * 2), dtype=ubyte)
 
         if sig_out.dtype != ubyte:
             msg = "sig_out must be of type ubyte"
@@ -226,7 +226,7 @@ def decode(
             # allocate output array with lasd dim as large as the longest
             # uncompressed wf
             maxs = np.max(_get_hton_u16(sig_in[0], 0))
-            sig_out = np.empty(s[:-1] + (maxs,), dtype=int32)
+            sig_out = np.empty((*s[:-1], maxs), dtype=int32)
 
         # siglen has one dimension less (the last)
         siglen = np.empty(s[:-1], dtype=uint32)

lgdo/compression/varlen.py

@@ -74,14 +74,14 @@ def encode(
     if isinstance(sig_in, np.ndarray):
         s = sig_in.shape
         if len(sig_in) == 0:
-            return np.empty(s[:-1] + (0,), dtype=ubyte), np.empty(0, dtype=uint32)
+            return np.empty((*s[:-1], 0), dtype=ubyte), np.empty(0, dtype=uint32)
 
         if sig_out is None:
             # the encoded signal is an array of bytes
             # pre-allocate ubyte (uint8) array with a generous (but safe) size
             max_b = int(np.ceil(np.iinfo(sig_in.dtype).bits / 16) * 5)
             # expand last dimension
-            sig_out = np.empty(s[:-1] + (s[-1] * max_b,), dtype=ubyte)
+            sig_out = np.empty((*s[:-1], s[-1] * max_b), dtype=ubyte)
 
         if sig_out.dtype != ubyte:
             msg = "sig_out must be of type ubyte"

lgdo/lh5/_serializers/read/ndarray.py

@@ -57,7 +57,7 @@ def _h5_read_ndarray(
             (start_row,) + (0,) * (h5d.rank - 1),
             (1,) * h5d.rank,
             None,
-            (n_rows_to_read,) + fspace.shape[1:],
+            (n_rows_to_read, *fspace.shape[1:]),
         )
     elif use_h5idx:
         # Note that h5s will automatically merge adjacent elements into a range
@@ -67,7 +67,7 @@ def _h5_read_ndarray(
                 (i,) + (0,) * (h5d.rank - 1),
                 (1,) * h5d.rank,
                 None,
-                (1,) + fspace.shape[1:],
+                (1, *fspace.shape[1:]),
                 h5py.h5s.SELECT_OR,
             )
 
@@ -84,7 +84,7 @@ def _h5_read_ndarray(
                 (obj_buf_start,) + (0,) * (h5d.rank - 1),
                 (1,) * h5d.rank,
                 None,
-                (n_rows_to_read,) + fspace.shape[1:],
+                (n_rows_to_read, *fspace.shape[1:]),
             )
             h5d.read(mspace, fspace, obj_buf.nda)
         else:
@@ -93,10 +93,10 @@ def _h5_read_ndarray(
             obj_buf.nda[dest_sel, ...] = tmp[idx, ...]
         nda = obj_buf.nda
     elif n_rows == 0:
-        tmp_shape = (0,) + h5d.shape[1:]
+        tmp_shape = (0, *h5d.shape[1:])
         nda = np.empty(tmp_shape, h5d.dtype)
     else:
-        mspace = h5py.h5s.create_simple((n_rows_to_read,) + fspace.shape[1:])
+        mspace = h5py.h5s.create_simple((n_rows_to_read, *fspace.shape[1:]))
         nda = np.empty(mspace.shape, h5d.dtype)
         if idx is None or use_h5idx:
             h5d.read(mspace, fspace, nda)

lgdo/lh5/_serializers/write/array.py

@@ -40,7 +40,7 @@ def _h5_write_array(
         # this is needed in order to have a resizable (in the first
         # axis) data set, i.e. rows can be appended later
         # NOTE: this automatically turns chunking on!
-        maxshape = (None,) + nda.shape[1:]
+        maxshape = (None, *nda.shape[1:])
         h5py_kwargs.setdefault("maxshape", maxshape)
 
         if wo_mode == "o" and name in group:

lgdo/lh5/_serializers/write/composite.py

@@ -52,140 +52,166 @@ def _h5_write_lgdo(
     # In hdf5, 'a' is really "modify" -- in addition to appending, you can
     # change any object in the file. So we use file:append for
     # write_object:overwrite.
+    opened_here = False
     if not isinstance(lh5_file, h5py.File):
         mode = "w" if wo_mode == "of" or not Path(lh5_file).exists() else "a"
-        lh5_file = h5py.File(lh5_file, mode=mode, **file_kwargs)
 
-    log.debug(
-        f"writing {obj!r}[{start_row}:{n_rows}] as "
-        f"{lh5_file.filename}:{group}/{name}[{write_start}:], "
-        f"mode = {wo_mode}, h5py_kwargs = {h5py_kwargs}"
-    )
-
-    group = utils.get_h5_group(group, lh5_file)
+        try:
+            fh = h5py.File(lh5_file, mode=mode, **file_kwargs)
+        except OSError as oe:
+            raise LH5EncodeError(str(oe), lh5_file, None) from oe
 
-    # name already in file
-    if name in group or (
-        ("datatype" in group.attrs or group == "/")
-        and (len(name) <= 2 or "/" not in name[1:-1])
-    ):
-        pass
-    # group is in file but not struct or need to create nesting
+        opened_here = True
     else:
-        # check if name is nested
-        # if name is nested, iterate up from parent
-        # otherwise we just need to iterate the group
-        if len(name) > 2 and "/" in name[1:-1]:
-            group = utils.get_h5_group(
-                name[:-1].rsplit("/", 1)[0],
-                group,
-            )
-            curr_name = (
-                name.rsplit("/", 1)[1]
-                if name[-1] != "/"
-                else name[:-1].rsplit("/", 1)[1]
-            )
-        else:
-            curr_name = name
-        # initialize the object to be written
-        obj = types.Struct({curr_name.replace("/", ""): obj})
+        fh = lh5_file
 
-        # if base group already has a child we just append
-        if len(group) >= 1:
-            wo_mode = "ac"
-        else:
-            # iterate up the group hierarchy until we reach the root or a group with more than one child
-            while group.name != "/":
-                if len(group) > 1:
-                    break
-                curr_name = group.name
-                group = group.parent
-                if group.name != "/":
-                    obj = types.Struct({curr_name[len(group.name) + 1 :]: obj})
-                else:
-                    obj = types.Struct({curr_name[1:]: obj})
-            # if the group has more than one child, we need to append else we can overwrite
-            wo_mode = "ac" if len(group) > 1 else "o"
-
-        # set the new name
-        if group.name == "/":
-            name = "/"
-        elif group.parent.name == "/":
-            name = group.name[1:]
-        else:
-            name = group.name[len(group.parent.name) + 1 :]
-        # get the new group
-        group = utils.get_h5_group(group.parent if group.name != "/" else "/", lh5_file)
+    try:
+        log.debug(
+            f"writing {obj!r}[{start_row}:{n_rows}] as "
+            f"{fh.filename}:{group}/{name}[{write_start}:], "
+            f"mode = {wo_mode}, h5py_kwargs = {h5py_kwargs}"
+        )
 
-    if wo_mode == "w" and name in group:
-        msg = f"can't overwrite '{name}' in wo_mode 'write_safe'"
-        raise LH5EncodeError(msg, lh5_file, group, name)
+        group = utils.get_h5_group(group, fh)
 
-    # struct, table, waveform table or histogram.
-    if isinstance(obj, types.Struct):
-        if (
-            isinstance(obj, types.Histogram)
-            and wo_mode not in ["w", "o", "of"]
-            and name in group
+        # name already in file
+        if name in group or (
+            ("datatype" in group.attrs or group == "/")
+            and (len(name) <= 2 or "/" not in name[1:-1])
         ):
-            msg = f"can't append-write to histogram in wo_mode '{wo_mode}'"
-            raise LH5EncodeError(msg, lh5_file, group, name)
-        if isinstance(obj, types.Histogram) and write_start != 0:
-            msg = f"can't write histogram in wo_mode '{wo_mode}' with write_start != 0"
-            raise LH5EncodeError(msg, lh5_file, group, name)
-
-        return _h5_write_struct(
-            obj,
-            name,
-            lh5_file,
-            group=group,
-            start_row=start_row,
-            n_rows=n_rows,  # if isinstance(obj, types.Table | types.Histogram) else None,
-            wo_mode=wo_mode,
-            write_start=write_start,
-            **h5py_kwargs,
-        )
-
-    # scalars
-    if isinstance(obj, types.Scalar):
-        return _h5_write_scalar(obj, name, lh5_file, group, wo_mode)
+            pass
+        # group is in file but not struct or need to create nesting
+        else:
+            # check if name is nested
+            # if name is nested, iterate up from parent
+            # otherwise we just need to iterate the group
+            if len(name) > 2 and "/" in name[1:-1]:
+                group = utils.get_h5_group(
+                    name[:-1].rsplit("/", 1)[0],
+                    group,
+                )
+                curr_name = (
+                    name.rsplit("/", 1)[1]
+                    if name[-1] != "/"
+                    else name[:-1].rsplit("/", 1)[1]
+                )
+            else:
+                curr_name = name
+            # initialize the object to be written
+            obj = types.Struct({curr_name.replace("/", ""): obj})
+
+            # if base group already has a child we just append
+            if len(group) >= 1:
+                wo_mode = "ac"
+            else:
+                # iterate up the group hierarchy until we reach the root or a group with more than one child
+                while group.name != "/":
+                    if len(group) > 1:
+                        break
+                    curr_name = group.name
+                    group = group.parent
+                    if group.name != "/":
+                        obj = types.Struct({curr_name[len(group.name) + 1 :]: obj})
+                    else:
+                        obj = types.Struct({curr_name[1:]: obj})
+                # if the group has more than one child, we need to append else we can overwrite
+                wo_mode = "ac" if len(group) > 1 else "o"
+
+            # set the new name
+            if group.name == "/":
+                name = "/"
+            elif group.parent.name == "/":
+                name = group.name[1:]
+            else:
+                name = group.name[len(group.parent.name) + 1 :]
+            # get the new group
+            group = utils.get_h5_group(group.parent if group.name != "/" else "/", fh)
+
+        if wo_mode == "w" and name in group:
+            msg = f"can't overwrite '{name}' in wo_mode 'write_safe'"
+            raise LH5EncodeError(msg, fh, group, name)
+
+        # struct, table, waveform table or histogram.
+        if isinstance(obj, types.Struct):
+            if (
+                isinstance(obj, types.Histogram)
+                and wo_mode not in ["w", "o", "of"]
+                and name in group
+            ):
+                msg = f"can't append-write to histogram in wo_mode '{wo_mode}'"
+                raise LH5EncodeError(msg, fh, group, name)
+            if isinstance(obj, types.Histogram) and write_start != 0:
+                msg = f"can't write histogram in wo_mode '{wo_mode}' with write_start != 0"
+                raise LH5EncodeError(msg, fh, group, name)
+
+            return _h5_write_struct(
+                obj,
+                name,
+                fh,
+                group=group,
+                start_row=start_row,
+                n_rows=n_rows,  # if isinstance(obj, types.Table | types.Histogram) else None,
+                wo_mode=wo_mode,
+                write_start=write_start,
+                **h5py_kwargs,
+            )
 
-    # vector of encoded vectors
-    if isinstance(
-        obj, (types.VectorOfEncodedVectors, types.ArrayOfEncodedEqualSizedArrays)
-    ):
-        group = utils.get_h5_group(
-            name, group, grp_attrs=obj.attrs, overwrite=(wo_mode == "o")
-        )
+        # scalars
+        if isinstance(obj, types.Scalar):
+            return _h5_write_scalar(obj, name, fh, group, wo_mode)
 
-        # ask not to further compress flattened_data, it is already compressed!
-        obj.encoded_data.flattened_data.attrs["compression"] = None
+        # vector of encoded vectors
+        if isinstance(
+            obj, (types.VectorOfEncodedVectors, types.ArrayOfEncodedEqualSizedArrays)
+        ):
+            group = utils.get_h5_group(
+                name, group, grp_attrs=obj.attrs, overwrite=(wo_mode == "o")
+            )
 
-        _h5_write_vector_of_vectors(
-            obj.encoded_data,
-            "encoded_data",
-            lh5_file,
-            group=group,
-            start_row=start_row,
-            n_rows=n_rows,
-            wo_mode=wo_mode,
-            write_start=write_start,
-            **h5py_kwargs,
-        )
+            # ask not to further compress flattened_data, it is already compressed!
+            obj.encoded_data.flattened_data.attrs["compression"] = None
 
-        if isinstance(obj.decoded_size, types.Scalar):
-            _h5_write_scalar(
-                obj.decoded_size,
-                "decoded_size",
-                lh5_file,
+            _h5_write_vector_of_vectors(
+                obj.encoded_data,
+                "encoded_data",
+                fh,
                 group=group,
+                start_row=start_row,
+                n_rows=n_rows,
                 wo_mode=wo_mode,
+                write_start=write_start,
+                **h5py_kwargs,
             )
-        else:
-            _h5_write_array(
-                obj.decoded_size,
-                "decoded_size",
-                lh5_file,
+
+            if isinstance(obj.decoded_size, types.Scalar):
+                _h5_write_scalar(
+                    obj.decoded_size,
+                    "decoded_size",
+                    fh,
+                    group=group,
+                    wo_mode=wo_mode,
+                )
+            else:
+                _h5_write_array(
+                    obj.decoded_size,
+                    "decoded_size",
+                    fh,
+                    group=group,
+                    start_row=start_row,
+                    n_rows=n_rows,
+                    wo_mode=wo_mode,
+                    write_start=write_start,
+                    **h5py_kwargs,
+                )
+
+            return None
+
+        # vector of vectors
+        if isinstance(obj, types.VectorOfVectors):
+            return _h5_write_vector_of_vectors(
+                obj,
+                name,
+                fh,
                 group=group,
                 start_row=start_row,
                 n_rows=n_rows,
@@ -194,38 +220,25 @@ def _h5_write_lgdo(
                 **h5py_kwargs,
             )
 
-        return None
-
-    # vector of vectors
-    if isinstance(obj, types.VectorOfVectors):
-        return _h5_write_vector_of_vectors(
-            obj,
-            name,
-            lh5_file,
-            group=group,
-            start_row=start_row,
-            n_rows=n_rows,
-            wo_mode=wo_mode,
-            write_start=write_start,
-            **h5py_kwargs,
-        )
-
-    # if we get this far, must be one of the Array types
-    if isinstance(obj, types.Array):
-        return _h5_write_array(
-            obj,
-            name,
-            lh5_file,
-            group=group,
-            start_row=start_row,
-            n_rows=n_rows,
-            wo_mode=wo_mode,
-            write_start=write_start,
-            **h5py_kwargs,
-        )
+        # if we get this far, must be one of the Array types
+        if isinstance(obj, types.Array):
+            return _h5_write_array(
+                obj,
+                name,
+                fh,
+                group=group,
+                start_row=start_row,
+                n_rows=n_rows,
+                wo_mode=wo_mode,
+                write_start=write_start,
+                **h5py_kwargs,
+            )
 
-    msg = f"do not know how to write '{name}' of type '{type(obj).__name__}'"
-    raise LH5EncodeError(msg, lh5_file, group, name)
+        msg = f"do not know how to write '{name}' of type '{type(obj).__name__}'"
+        raise LH5EncodeError(msg, fh, group, name)
+    finally:
+        if opened_here:
+            fh.close()
 
 
 def _h5_write_struct(

lgdo/lh5/core.py

@@ -14,12 +14,13 @@ from numpy.typing import ArrayLike
 
 from .. import types
 from . import _serializers
+from .exceptions import LH5DecodeError
 from .utils import read_n_rows
 
 
 def read(
     name: str,
-    lh5_file: str | h5py.File | Sequence[str | h5py.File],
+    lh5_file: str | Path | h5py.File | Sequence[str | Path | h5py.File],
     start_row: int = 0,
     n_rows: int = sys.maxsize,
     idx: ArrayLike = None,
@@ -110,15 +111,20 @@ def read(
     object
         the read-out object
     """
+    close_after = False
     if isinstance(lh5_file, h5py.File):
         lh5_obj = lh5_file[name]
-    elif isinstance(lh5_file, str):
-        lh5_file = h5py.File(lh5_file, mode="r", locking=locking)
+    elif isinstance(lh5_file, (str, Path)):
+        try:
+            lh5_file = h5py.File(str(Path(lh5_file)), mode="r", locking=locking)
+        except (OSError, FileExistsError) as oe:
+            raise LH5DecodeError(str(oe), lh5_file, None) from oe
+
+        close_after = True
         try:
             lh5_obj = lh5_file[name]
         except KeyError as ke:
-            err = f"Object {name} not found in file {lh5_file.filename}"
-            raise KeyError(err) from ke
+            raise LH5DecodeError(str(ke), lh5_file, name) from ke
     else:
         if obj_buf is not None:
             obj_buf.resize(obj_buf_start)
@@ -173,29 +179,32 @@ def read(
     if isinstance(idx, np.ndarray) and idx.dtype == np.dtype("?"):
         idx = np.where(idx)[0]
 
-    obj, n_rows_read = _serializers._h5_read_lgdo(
-        lh5_obj.id,
-        lh5_obj.file.filename,
-        lh5_obj.name,
-        start_row=start_row,
-        n_rows=n_rows,
-        idx=idx,
-        use_h5idx=use_h5idx,
-        field_mask=field_mask,
-        obj_buf=obj_buf,
-        obj_buf_start=obj_buf_start,
-        decompress=decompress,
-    )
-    with suppress(AttributeError):
-        obj.resize(obj_buf_start + n_rows_read)
-
-    return obj
+    try:
+        obj, n_rows_read = _serializers._h5_read_lgdo(
+            lh5_obj.id,
+            lh5_obj.file.filename,
+            lh5_obj.name,
+            start_row=start_row,
+            n_rows=n_rows,
+            idx=idx,
+            use_h5idx=use_h5idx,
+            field_mask=field_mask,
+            obj_buf=obj_buf,
+            obj_buf_start=obj_buf_start,
+            decompress=decompress,
+        )
+        with suppress(AttributeError):
+            obj.resize(obj_buf_start + n_rows_read)
+        return obj
+    finally:
+        if close_after:
+            lh5_file.close()
 
 
 def write(
     obj: types.LGDO,
     name: str,
-    lh5_file: str | h5py.File,
+    lh5_file: str | Path | h5py.File,
     group: str | h5py.Group = "/",
     start_row: int = 0,
     n_rows: int | None = None,
@@ -318,7 +327,7 @@ def write(
 
 def read_as(
     name: str,
-    lh5_file: str | h5py.File | Sequence[str | h5py.File],
+    lh5_file: str | Path | h5py.File | Sequence[str | Path | h5py.File],
     library: str,
     **kwargs,
 ) -> Any:

lgdo/lh5/exceptions.py

@@ -4,17 +4,21 @@ import h5py
 
 
 class LH5DecodeError(Exception):
-    def __init__(self, message: str, fname: str, oname: str) -> None:
+    def __init__(
+        self, message: str, file: str | h5py.File, oname: str | None = None
+    ) -> None:
         super().__init__(message)
 
-        self.file = fname
+        self.file = file.filename if isinstance(file, h5py.File) else file
         self.obj = oname
 
     def __str__(self) -> str:
-        return (
-            f"while reading object '{self.obj}' in file {self.file}: "
-            + super().__str__()
-        )
+        if self.obj is None:
+            msg = f"while opening file {self.file} for decoding: "
+        else:
+            msg = f"while decoding object '{self.obj}' in file {self.file}: "
+
+        return msg + super().__str__()
 
     def __reduce__(self) -> tuple:  # for pickling.
         return self.__class__, (*self.args, self.file, self.obj)
@@ -22,19 +26,30 @@ class LH5DecodeError(Exception):
 
 class LH5EncodeError(Exception):
     def __init__(
-        self, message: str, file: str | h5py.File, group: str | h5py.Group, name: str
+        self,
+        message: str,
+        file: str | h5py.File,
+        group: str | h5py.Group | None = None,
+        name: str | None = None,
     ) -> None:
         super().__init__(message)
 
         self.file = file.filename if isinstance(file, h5py.File) else file
-        self.group = (group.name if isinstance(file, h5py.File) else group).rstrip("/")
-        self.name = name.lstrip("/")
+        self.group = (
+            (group.name if isinstance(file, h5py.File) else group).rstrip("/")
+            if group is not None
+            else None
+        )
+        self.name = name.lstrip("/") if name is not None else None
 
     def __str__(self) -> str:
-        return (
-            f"while writing object {self.group}/{self.name} to file {self.file}: "
-            + super().__str__()
-        )
+        if self.name is None:
+            msg = f"while opening file {self.file} for encoding: "
+        else:
+            msg = (
+                f"while encoding object {self.group}/{self.name} to file {self.file}: "
+            )
+        return msg + super().__str__()
 
     def __reduce__(self) -> tuple:  # for pickling.
         return self.__class__, (*self.args, self.file, self.group, self.name)

lgdo/lh5/iterator.py

@@ -60,6 +60,7 @@ class LH5Iterator(typing.Iterator):
         self,
         lh5_files: str | list[str],
         groups: str | list[str] | list[list[str]],
+        *,
         base_path: str = "",
         entry_list: list[int] | list[list[int]] | None = None,
         entry_mask: list[bool] | list[list[bool]] | None = None,
@@ -70,6 +71,7 @@ class LH5Iterator(typing.Iterator):
         file_cache: int = 10,
         file_map: NDArray[int] = None,
         friend: typing.Iterator | None = None,
+        h5py_open_mode: str = "r",
     ) -> None:
         """
         Parameters
@@ -110,9 +112,21 @@ class LH5Iterator(typing.Iterator):
             The friend should have the same length and entry list. A single
             LH5 table containing columns from both iterators will be returned.
             Note that buffer_len will be set to the minimum of the two.
+        h5py_open_mode
+            file open mode used when acquiring file handles. ``r`` (default)
+            opens files read-only while ``a`` allow opening files for
+            write-appending as well.
         """
         self.lh5_st = LH5Store(base_path=base_path, keep_open=file_cache)
 
+        if h5py_open_mode == "read":
+            h5py_open_mode = "r"
+        if h5py_open_mode == "append":
+            h5py_open_mode = "a"
+        if h5py_open_mode not in ["r", "a"]:
+            msg = f"unknown h5py_open_mode '{h5py_open_mode}'"
+            raise ValueError(msg)
+
         # List of files, with wildcards and env vars expanded
         if isinstance(lh5_files, str):
             lh5_files = [lh5_files]
@@ -147,6 +161,10 @@ class LH5Iterator(typing.Iterator):
                 self.lh5_files += [f_exp] * len(g)
                 self.groups += list(g)
 
+        # open files in the requested mode so they are writable if needed
+        for f in set(self.lh5_files):
+            self.lh5_st.gimme_file(f, mode=h5py_open_mode)
+
         if entry_list is not None and entry_mask is not None:
             msg = "entry_list and entry_mask arguments are mutually exclusive"
             raise ValueError(msg)

lgdo/lh5/store.py

@@ -38,7 +38,10 @@ class LH5Store:
     """
 
     def __init__(
-        self, base_path: str = "", keep_open: bool = False, locking: bool = False
+        self,
+        base_path: str | Path = "",
+        keep_open: bool = False,
+        locking: bool = False,
     ) -> None:
         """
         Parameters
@@ -52,6 +55,7 @@ class LH5Store:
         locking
             whether to lock files when reading
         """
+        base_path = str(Path(base_path)) if base_path != "" else ""
         self.base_path = "" if base_path == "" else utils.expand_path(base_path)
         self.keep_open = keep_open
         self.locking = locking
@@ -59,7 +63,7 @@ class LH5Store:
 
     def gimme_file(
         self,
-        lh5_file: str | h5py.File,
+        lh5_file: str | Path | h5py.File,
         mode: str = "r",
         page_buffer: int = 0,
         **file_kwargs,
@@ -83,6 +87,8 @@ class LH5Store:
         if isinstance(lh5_file, h5py.File):
             return lh5_file
 
+        lh5_file = str(Path(lh5_file))
+
         if mode == "r":
             lh5_file = utils.expand_path(lh5_file, base_path=self.base_path)
             file_kwargs["locking"] = self.locking
@@ -147,7 +153,7 @@ class LH5Store:
     def get_buffer(
         self,
         name: str,
-        lh5_file: str | h5py.File | Sequence[str | h5py.File],
+        lh5_file: str | Path | h5py.File | Sequence[str | Path | h5py.File],
         size: int | None = None,
         field_mask: Mapping[str, bool] | Sequence[str] | None = None,
     ) -> types.LGDO:
@@ -162,7 +168,7 @@ class LH5Store:
     def read(
         self,
         name: str,
-        lh5_file: str | h5py.File | Sequence[str | h5py.File],
+        lh5_file: str | Path | h5py.File | Sequence[str | Path | h5py.File],
         start_row: int = 0,
         n_rows: int = sys.maxsize,
         idx: ArrayLike = None,
@@ -180,7 +186,7 @@ class LH5Store:
         .lh5.core.read
         """
         # grab files from store
-        if isinstance(lh5_file, (str, h5py.File)):
+        if isinstance(lh5_file, (str, Path, h5py.File)):
             h5f = self.gimme_file(lh5_file, "r", **file_kwargs)
         else:
             h5f = [self.gimme_file(f, "r", **file_kwargs) for f in lh5_file]
@@ -201,7 +207,7 @@ class LH5Store:
         self,
         obj: types.LGDO,
         name: str,
-        lh5_file: str | h5py.File,
+        lh5_file: str | Path | h5py.File,
         group: str | h5py.Group = "/",
         start_row: int = 0,
         n_rows: int | None = None,
@@ -256,14 +262,14 @@ class LH5Store:
             **h5py_kwargs,
         )
 
-    def read_n_rows(self, name: str, lh5_file: str | h5py.File) -> int | None:
+    def read_n_rows(self, name: str, lh5_file: str | Path | h5py.File) -> int | None:
         """Look up the number of rows in an Array-like object called `name` in `lh5_file`.
 
         Return ``None`` if it is a :class:`.Scalar` or a :class:`.Struct`.
         """
         return utils.read_n_rows(name, self.gimme_file(lh5_file, "r"))
 
-    def read_size_in_bytes(self, name: str, lh5_file: str | h5py.File) -> int:
+    def read_size_in_bytes(self, name: str, lh5_file: str | Path | h5py.File) -> int:
         """Look up the size (in B) of the object in memory. Will recursively
         crawl through all objects in a Struct or Table
         """

lgdo/lh5/tools.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import fnmatch
 import logging
 from copy import copy
+from pathlib import Path
 
 import h5py
 
@@ -13,7 +14,7 @@ log = logging.getLogger(__name__)
 
 
 def ls(
-    lh5_file: str | h5py.Group,
+    lh5_file: str | Path | h5py.Group,
     lh5_group: str = "",
     recursive: bool = False,
 ) -> list[str]:
@@ -39,8 +40,8 @@ def ls(
 
     lh5_st = LH5Store()
     # To use recursively, make lh5_file a h5group instead of a string
-    if isinstance(lh5_file, str):
-        lh5_file = lh5_st.gimme_file(lh5_file, "r")
+    if isinstance(lh5_file, (str, Path)):
+        lh5_file = lh5_st.gimme_file(str(Path(lh5_file)), "r")
         if lh5_group.startswith("/"):
             lh5_group = lh5_group[1:]
 
@@ -75,7 +76,7 @@ def ls(
 
 
 def show(
-    lh5_file: str | h5py.Group,
+    lh5_file: str | Path | h5py.Group,
     lh5_group: str = "/",
     attrs: bool = False,
     indent: str = "",
@@ -121,8 +122,8 @@ def show(
         return
 
     # open file
-    if isinstance(lh5_file, str):
-        lh5_file = h5py.File(utils.expand_path(lh5_file), "r", locking=False)
+    if isinstance(lh5_file, (str, Path)):
+        lh5_file = h5py.File(utils.expand_path(Path(lh5_file)), "r", locking=False)
 
     # go to group
     if lh5_group != "/":

lgdo/lh5/utils.py

@@ -21,7 +21,7 @@ log = logging.getLogger(__name__)
 
 def get_buffer(
     name: str,
-    lh5_file: str | h5py.File | Sequence[str | h5py.File],
+    lh5_file: str | Path | h5py.File | Sequence[str | Path | h5py.File],
     size: int | None = None,
     field_mask: Mapping[str, bool] | Sequence[str] | None = None,
 ) -> types.LGDO:
@@ -39,7 +39,7 @@ def get_buffer(
     return obj
 
 
-def read_n_rows(name: str, h5f: str | h5py.File) -> int | None:
+def read_n_rows(name: str, h5f: str | Path | h5py.File) -> int | None:
     """Look up the number of rows in an Array-like LGDO object on disk.
 
     Return ``None`` if `name` is a :class:`.Scalar` or a :class:`.Struct`.
@@ -56,7 +56,7 @@ def read_n_rows(name: str, h5f: str | h5py.File) -> int | None:
     return _serializers.read.utils.read_n_rows(h5o, h5f.name, name)
 
 
-def read_size_in_bytes(name: str, h5f: str | h5py.File) -> int | None:
+def read_size_in_bytes(name: str, h5f: str | Path | h5py.File) -> int | None:
     """Look up the size (in B) in an LGDO object in memory. Will crawl
     recursively through members of a Struct or Table
     """
@@ -158,10 +158,10 @@ def expand_vars(expr: str, substitute: dict[str, str] | None = None) -> str:
 
 
 def expand_path(
-    path: str,
+    path: str | Path,
     substitute: dict[str, str] | None = None,
     list: bool = False,
-    base_path: str | None = None,
+    base_path: str | Path | None = None,
 ) -> str | list:
     """Expand (environment) variables and wildcards to return absolute paths.
 
@@ -184,18 +184,26 @@ def expand_path(
         Unique absolute path, or list of all absolute paths
     """
     if base_path is not None and base_path != "":
-        base_path = Path(os.path.expandvars(base_path)).expanduser()
-        path = base_path / path
+        base_path = Path(expand_vars(str(base_path))).expanduser()
+        if not Path(path).expanduser().is_absolute():
+            path = base_path / path
 
     # first expand variables
-    _path = expand_vars(path, substitute)
+    _path = expand_vars(str(path), substitute)
 
     # then expand wildcards
     # pathlib glob works differently so use glob for now
     paths = sorted(glob.glob(str(Path(_path).expanduser())))  # noqa: PTH207
 
     if base_path is not None and base_path != "":
-        paths = [os.path.relpath(p, base_path) for p in paths]
+        rel_paths = []
+        for p in paths:
+            p_path = Path(p)
+            try:
+                rel_paths.append(str(p_path.relative_to(base_path)))
+            except ValueError:
+                rel_paths.append(str(p_path))
+        paths = rel_paths
 
     if not list:
         if len(paths) == 0:

lgdo/types/array.py

@@ -109,14 +109,14 @@ class Array(LGDOCollection):
 
     @property
     def shape(self):
-        return (len(self),) + self._nda.shape[1:]
+        return (len(self), *self._nda.shape[1:])
 
     def reserve_capacity(self, capacity: int) -> None:
         "Set size (number of rows) of internal memory buffer"
         if capacity < len(self):
             msg = "Cannot reduce capacity below Array length"
             raise ValueError(msg)
-        self._nda.resize((capacity,) + self._nda.shape[1:], refcheck=False)
+        self._nda.resize((capacity, *self._nda.shape[1:]), refcheck=False)
 
     def get_capacity(self) -> int:
         "Get capacity (i.e. max size before memory must be re-allocated)"
@@ -190,6 +190,9 @@ class Array(LGDOCollection):
 
         return False
 
+    def __hash__(self):
+        return hash(self.name)
+
     def __iter__(self) -> Iterator:
         yield from self.nda
 

lgdo/types/encoded.py

@@ -92,6 +92,9 @@ class VectorOfEncodedVectors(LGDOCollection):
 
         return False
 
+    def __hash__(self):
+        return hash(self.name)
+
     def reserve_capacity(self, *capacity: int) -> None:
         self.encoded_data.reserve_capacity(*capacity)
         self.decoded_size.reserve_capacity(capacity[0])
@@ -345,6 +348,9 @@ class ArrayOfEncodedEqualSizedArrays(LGDOCollection):
 
         return False
 
+    def __hash__(self):
+        return hash(self.name)
+
     def reserve_capacity(self, *capacity: int) -> None:
         self.encoded_data.reserve_capacity(capacity)
 

lgdo/types/scalar.py

@@ -63,6 +63,9 @@ class Scalar(LGDO):
 
         return False
 
+    def __hash__(self):
+        return hash(self.name)
+
     def __str__(self) -> str:
         attrs = self.getattrs()
         return f"{self.value!s} with attrs={attrs!r}"

lgdo/types/table.py

@@ -330,9 +330,10 @@ class Table(Struct, LGDOCollection):
             :func:`numexpr.evaluate`` as `local_dict` argument or to
             :func:`eval` as `locals` argument.
         modules
-            a dictionary of additional modules used by the expression. If this is not `None`
-            then :func:`eval`is used and the expression can depend on any modules from this dictionary in
-            addition to awkward and numpy. These are passed to :func:`eval` as `globals` argument.
+            a dictionary of additional modules used by the expression. If this
+            is not `None` then :func:`eval`is used and the expression can
+            depend on any modules from this dictionary in addition to awkward
+            and numpy. These are passed to :func:`eval` as `globals` argument.
 
         Examples
         --------
@@ -403,7 +404,10 @@ class Table(Struct, LGDOCollection):
                 return _make_lgdo(out_data)
 
             except Exception:
-                msg = f"Warning {expr} could not be evaluated with numexpr probably due to some not allowed characters, trying with eval()."
+                msg = (
+                    f"Warning {expr} could not be evaluated with numexpr probably "
+                    "due to some not allowed characters, trying with eval()."
+                )
                 log.debug(msg)
 
         # resort to good ol' eval()

lgdo/types/vectorofvectors.py

@@ -284,6 +284,9 @@ class VectorOfVectors(LGDOCollection):
 
         return False
 
+    def __hash__(self):
+        return hash(self.name)
+
     def __getitem__(self, i: int) -> NDArray:
         """Return a view of the vector at index `i` along the first axis."""
         if self.ndim == 2: