legend-pydataobj 1.9.0__tar.gz → 1.10.1__tar.gz

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: legend_pydataobj
-Version: 1.9.0
+Version: 1.10.1
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration
@@ -700,7 +700,7 @@ License-File: LICENSE
 Requires-Dist: awkward>=2
 Requires-Dist: awkward-pandas
 Requires-Dist: colorlog
-Requires-Dist: h5py>=3.2
+Requires-Dist: h5py>=3.10
 Requires-Dist: hdf5plugin
 Requires-Dist: hist
 Requires-Dist: numba!=0.53.*,!=0.54.*

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.1}/pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "awkward>=2",
     "awkward-pandas",
     "colorlog",
-    "h5py>=3.2",
+    "h5py>=3.10",
     "hdf5plugin",
     "hist",
     "numba!=0.53.*,!=0.54.*",
@@ -137,6 +137,7 @@ ignore = [
   "PLR2004",  # Magic value used in comparison
   "ISC001",   # Conflicts with formatter
   "PT011",
+  "RUF013",   # complains if you default to None for an asinine reason
 ]
 isort.required-imports = ["from __future__ import annotations"]
 # Uncomment if using a _compat.typing backport
@@ -145,6 +146,7 @@ isort.required-imports = ["from __future__ import annotations"]
 [tool.ruff.lint.per-file-ignores]
 "tests/**" = ["T20"]
 "noxfile.py" = ["T20"]
+"docs/source/notebooks/*" = ["T201", "E402"]
 
 [tool.pylint]
 py-version = "3.8"

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.1}/src/legend_pydataobj.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: legend_pydataobj
-Version: 1.9.0
+Version: 1.10.1
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration
@@ -700,7 +700,7 @@ License-File: LICENSE
 Requires-Dist: awkward>=2
 Requires-Dist: awkward-pandas
 Requires-Dist: colorlog
-Requires-Dist: h5py>=3.2
+Requires-Dist: h5py>=3.10
 Requires-Dist: hdf5plugin
 Requires-Dist: hist
 Requires-Dist: numba!=0.53.*,!=0.54.*

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.1}/src/legend_pydataobj.egg-info/requires.txt

@@ -1,7 +1,7 @@
 awkward>=2
 awkward-pandas
 colorlog
-h5py>=3.2
+h5py>=3.10
 hdf5plugin
 hist
 numba!=0.53.*,!=0.54.*

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.1}/src/lgdo/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.9.0'
-__version_tuple__ = version_tuple = (1, 9, 0)
+__version__ = version = '1.10.1'
+__version_tuple__ = version_tuple = (1, 10, 1)

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.1}/src/lgdo/compression/radware.py

@@ -441,15 +441,11 @@ def _radware_sigcompress_encode(
         while (i < sig_in.size) and (i < j + 48):
             si_i = int16(sig_in[i] + shift)
             si_im1 = int16(sig_in[i - 1] + shift)
-            if max1 < si_i:
-                max1 = si_i
-            if min1 > si_i:
-                min1 = si_i
+            max1 = max(max1, si_i)
+            min1 = min(min1, si_i)
             ds = si_i - si_im1
-            if max2 < ds:
-                max2 = ds
-            if min2 > ds:
-                min2 = ds
+            max2 = max(max2, ds)
+            min2 = min(min2, ds)
             nw += 1
             i += 1
         if max1 - min1 <= max2 - min2:  # use absolute values
@@ -460,15 +456,13 @@ def _radware_sigcompress_encode(
                 i < j + 128
             ):  # FIXME: 128 could be tuned better?
                 si_i = int16(sig_in[i] + shift)
-                if max1 < si_i:
-                    max1 = si_i
+                max1 = max(max1, si_i)
                 dd1 = max1 - min1
                 if min1 > si_i:
                     dd1 = max1 - si_i
                 if dd1 > mask[nb1]:
                     break
-                if min1 > si_i:
-                    min1 = si_i
+                min1 = min(min1, si_i)
                 nw += 1
                 i += 1
         else:  # use difference values
@@ -481,15 +475,13 @@ def _radware_sigcompress_encode(
                 si_i = int16(sig_in[i] + shift)
                 si_im1 = int16(sig_in[i - 1] + shift)
                 ds = si_i - si_im1
-                if max2 < ds:
-                    max2 = ds
+                max2 = max(max2, ds)
                 dd2 = max2 - min2
                 if min2 > ds:
                     dd2 = max2 - ds
                 if dd2 > mask[nb2]:
                     break
-                if min2 > ds:
-                    min2 = ds
+                min2 = min(min2, ds)
                 nw += 1
                 i += 1
 

legend_pydataobj-1.10.1/src/lgdo/lh5/_serializers/read/array.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import logging
+
+from ....types import Array, ArrayOfEqualSizedArrays, FixedSizeArray
+from . import utils
+from .ndarray import _h5_read_ndarray
+
+log = logging.getLogger(__name__)
+
+
+def _h5_read_array_generic(type_, h5d, fname, oname, **kwargs):
+    nda, attrs, n_rows_to_read = _h5_read_ndarray(h5d, fname, oname, **kwargs)
+
+    obj_buf = kwargs["obj_buf"]
+
+    if obj_buf is None:
+        return type_(nda=nda, attrs=attrs), n_rows_to_read
+
+    utils.check_obj_buf_attrs(obj_buf.attrs, attrs, fname, oname)
+
+    return obj_buf, n_rows_to_read
+
+
+def _h5_read_array(h5d, fname, oname, **kwargs):
+    return _h5_read_array_generic(Array, h5d, fname, oname, **kwargs)
+
+
+def _h5_read_fixedsize_array(h5d, fname, oname, **kwargs):
+    return _h5_read_array_generic(FixedSizeArray, h5d, fname, oname, **kwargs)
+
+
+def _h5_read_array_of_equalsized_arrays(h5d, fname, oname, **kwargs):
+    return _h5_read_array_generic(ArrayOfEqualSizedArrays, h5d, fname, oname, **kwargs)

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.1}/src/lgdo/lh5/_serializers/read/composite.py

@@ -23,7 +23,6 @@ from ....types import (
 )
 from ... import datatype as dtypeutils
 from ...exceptions import LH5DecodeError
-from ...utils import read_n_rows
 from . import utils
 from .array import (
     _h5_read_array,
@@ -42,6 +41,8 @@ log = logging.getLogger(__name__)
 
 def _h5_read_lgdo(
     h5o,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -51,69 +52,23 @@ def _h5_read_lgdo(
     obj_buf_start=0,
     decompress=True,
 ):
-    # Handle list-of-files recursively
-    if not isinstance(h5o, (h5py.Group, h5py.Dataset)):
-        lh5_objs = list(h5o)
-        n_rows_read = 0
-
-        for i, _h5o in enumerate(lh5_objs):
-            if isinstance(idx, list) and len(idx) > 0 and not np.isscalar(idx[0]):
-                # a list of lists: must be one per file
-                idx_i = idx[i]
-            elif idx is not None:
-                # make idx a proper tuple if it's not one already
-                if not (isinstance(idx, tuple) and len(idx) == 1):
-                    idx = (idx,)
-                # idx is a long continuous array
-                n_rows_i = read_n_rows(_h5o)
-                # find the length of the subset of idx that contains indices
-                # that are less than n_rows_i
-                n_rows_to_read_i = bisect.bisect_left(idx[0], n_rows_i)
-                # now split idx into idx_i and the remainder
-                idx_i = (idx[0][:n_rows_to_read_i],)
-                idx = (idx[0][n_rows_to_read_i:] - n_rows_i,)
-            else:
-                idx_i = None
-            n_rows_i = n_rows - n_rows_read
-
-            obj_buf, n_rows_read_i = _h5_read_lgdo(
-                _h5o,
-                start_row=start_row,
-                n_rows=n_rows_i,
-                idx=idx_i,
-                use_h5idx=use_h5idx,
-                field_mask=field_mask,
-                obj_buf=obj_buf,
-                obj_buf_start=obj_buf_start,
-                decompress=decompress,
-            )
-
-            n_rows_read += n_rows_read_i
-            if n_rows_read >= n_rows or obj_buf is None:
-                return obj_buf, n_rows_read
-            start_row = 0
-            obj_buf_start += n_rows_read_i
-
-        return obj_buf, n_rows_read
-
     log.debug(
-        f"reading {h5o.file.filename}:{h5o.name}[{start_row}:{n_rows}], decompress = {decompress}, "
+        f"reading {fname}:{oname}[{start_row}:{n_rows}], decompress = {decompress}, "
         + (f" with field mask {field_mask}" if field_mask else "")
     )
 
-    # make idx a proper tuple if it's not one already
-    if not (isinstance(idx, tuple) and len(idx) == 1) and idx is not None:
-        idx = (idx,)
-
+    attrs = utils.read_attrs(h5o, fname, oname)
     try:
-        lgdotype = dtypeutils.datatype(h5o.attrs["datatype"])
+        lgdotype = dtypeutils.datatype(attrs["datatype"])
     except KeyError as e:
         msg = "dataset not in file or missing 'datatype' attribute"
-        raise LH5DecodeError(msg, h5o) from e
+        raise LH5DecodeError(msg, fname, oname) from e
 
     if lgdotype is Scalar:
         return _h5_read_scalar(
             h5o,
+            fname,
+            oname,
             obj_buf=obj_buf,
         )
 
@@ -125,7 +80,7 @@ def _h5_read_lgdo(
         if len(field_mask) > 0:
             default = not field_mask[next(iter(field_mask.keys()))]
         field_mask = defaultdict(lambda: default, field_mask)
-    elif isinstance(field_mask, (list, tuple)):
+    elif isinstance(field_mask, (list, tuple, set)):
         field_mask = defaultdict(bool, {field: True for field in field_mask})
     elif not isinstance(field_mask, defaultdict):
         msg = "bad field_mask type"
@@ -134,6 +89,8 @@ def _h5_read_lgdo(
     if lgdotype is Struct:
         return _h5_read_struct(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -145,20 +102,21 @@ def _h5_read_lgdo(
     # Below here is all array-like types. So trim idx if needed
     if idx is not None:
         # check if idx is just an ordered list of the integers if so can ignore
-        if (idx[0] == np.arange(0, len(idx[0]), 1)).all():
-            if n_rows > len(idx[0]):
-                n_rows = len(idx[0])
+        if (idx == np.arange(0, len(idx), 1)).all():
+            n_rows = min(n_rows, len(idx))
             idx = None
         else:
             # chop off indices < start_row
-            i_first_valid = bisect.bisect_left(idx[0], start_row)
-            idxa = idx[0][i_first_valid:]
+            i_first_valid = bisect.bisect_left(idx, start_row)
+            idxa = idx[i_first_valid:]
             # don't readout more than n_rows indices
-            idx = (idxa[:n_rows],)  # works even if n_rows > len(idxa)
+            idx = idxa[:n_rows]  # works even if n_rows > len(idxa)
 
     if lgdotype is Table:
         return _h5_read_table(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -172,6 +130,8 @@ def _h5_read_lgdo(
     if lgdotype is Histogram:
         return _h5_read_histogram(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -185,6 +145,8 @@ def _h5_read_lgdo(
     if lgdotype is ArrayOfEncodedEqualSizedArrays:
         return _h5_read_array_of_encoded_equalsized_arrays(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -197,6 +159,8 @@ def _h5_read_lgdo(
     if lgdotype is VectorOfEncodedVectors:
         return _h5_read_vector_of_encoded_vectors(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -209,6 +173,8 @@ def _h5_read_lgdo(
     if lgdotype is VectorOfVectors:
         return _h5_read_vector_of_vectors(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -220,6 +186,8 @@ def _h5_read_lgdo(
     if lgdotype is FixedSizeArray:
         return _h5_read_fixedsize_array(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -231,6 +199,8 @@ def _h5_read_lgdo(
     if lgdotype is ArrayOfEqualSizedArrays:
         return _h5_read_array_of_equalsized_arrays(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -242,6 +212,8 @@ def _h5_read_lgdo(
     if lgdotype is Array:
         return _h5_read_array(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -251,11 +223,13 @@ def _h5_read_lgdo(
         )
 
     msg = f"no rule to decode {lgdotype.__name__} from LH5"
-    raise LH5DecodeError(msg, h5o)
+    raise LH5DecodeError(msg, fname, oname)
 
 
 def _h5_read_struct(
     h5g,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -268,7 +242,7 @@ def _h5_read_struct(
     # table... Maybe should emit a warning? Or allow them to be
     # dicts keyed by field name?
 
-    attrs = dict(h5g.attrs)
+    attrs = utils.read_attrs(h5g, fname, oname)
 
     # determine fields to be read out
     all_fields = dtypeutils.get_struct_fields(attrs["datatype"])
@@ -286,20 +260,26 @@ def _h5_read_struct(
     for field in selected_fields:
         # support for integer keys
         field_key = int(field) if attrs.get("int_keys") else str(field)
+        h5o = h5py.h5o.open(h5g, field.encode("utf-8"))
         obj_dict[field_key], _ = _h5_read_lgdo(
-            h5g[field],
+            h5o,
+            fname,
+            f"{oname}/{field}",
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
             use_h5idx=use_h5idx,
             decompress=decompress,
         )
+        h5o.close()
 
     return Struct(obj_dict=obj_dict, attrs=attrs), 1
 
 
 def _h5_read_table(
     h5g,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -311,9 +291,9 @@ def _h5_read_table(
 ):
     if obj_buf is not None and not isinstance(obj_buf, Table):
         msg = "provided object buffer is not a Table"
-        raise LH5DecodeError(msg, h5g)
+        raise LH5DecodeError(msg, fname, oname)
 
-    attrs = dict(h5g.attrs)
+    attrs = utils.read_attrs(h5g, fname, oname)
 
     # determine fields to be read out
     all_fields = dtypeutils.get_struct_fields(attrs["datatype"])
@@ -334,12 +314,15 @@ def _h5_read_table(
         if obj_buf is not None:
             if not isinstance(obj_buf, Table) or field not in obj_buf:
                 msg = "provided object buffer is not a Table or columns are missing"
-                raise LH5DecodeError(msg, h5g)
+                raise LH5DecodeError(msg, fname, oname)
 
             fld_buf = obj_buf[field]
 
+        h5o = h5py.h5o.open(h5g, field.encode("utf-8"))
         col_dict[field], n_rows_read = _h5_read_lgdo(
-            h5g[field],
+            h5o,
+            fname,
+            f"{oname}/{field}",
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -348,6 +331,7 @@ def _h5_read_table(
             obj_buf_start=obj_buf_start,
             decompress=decompress,
         )
+        h5o.close()
 
         if obj_buf is not None and obj_buf_start + n_rows_read > len(obj_buf):
             obj_buf.resize(obj_buf_start + n_rows_read)
@@ -359,12 +343,12 @@ def _h5_read_table(
         n_rows_read = rows_read[0]
     else:
         n_rows_read = 0
-        log.warning(f"Table '{h5g.name}' has no fields specified by {field_mask=}")
+        log.warning(f"Table '{oname}' has no fields specified by {field_mask=}")
 
     for n in rows_read[1:]:
         if n != n_rows_read:
             log.warning(
-                f"Table '{h5g.name}' got strange n_rows_read = {n}, "
+                f"Table '{oname}' got strange n_rows_read = {n}, "
                 "{n_rows_read} was expected ({rows_read})"
             )
 
@@ -396,13 +380,15 @@ def _h5_read_table(
     obj_buf.loc = obj_buf_start + n_rows_read
 
     # check attributes
-    utils.check_obj_buf_attrs(obj_buf.attrs, attrs, h5g)
+    utils.check_obj_buf_attrs(obj_buf.attrs, attrs, fname, oname)
 
     return obj_buf, n_rows_read
 
 
 def _h5_read_histogram(
     h5g,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -414,17 +400,20 @@ def _h5_read_histogram(
 ):
     if obj_buf is not None or obj_buf_start != 0:
         msg = "reading a histogram into an existing object buffer is not supported"
-        raise LH5DecodeError(msg, h5g)
+        raise LH5DecodeError(msg, fname, oname)
 
     struct, n_rows_read = _h5_read_struct(
         h5g,
-        start_row,
-        n_rows,
-        idx,
-        use_h5idx,
-        field_mask,
-        decompress,
+        fname,
+        oname,
+        start_row=start_row,
+        n_rows=n_rows,
+        idx=idx,
+        use_h5idx=use_h5idx,
+        field_mask=field_mask,
+        decompress=decompress,
     )
+
     binning = []
     for _, a in struct.binning.items():
         be = a.binedges
@@ -434,7 +423,7 @@ def _h5_read_histogram(
             b = (be, None, None, None, a.closedleft.value)
         else:
             msg = "unexpected binning of histogram"
-            raise LH5DecodeError(msg, h5g)
+            raise LH5DecodeError(msg, fname, oname)
         ax = Histogram.Axis(*b)
         # copy attrs to "clone" the "whole" struct.
         ax.attrs = a.getattrs(datatype=True)

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.1}/src/lgdo/lh5/_serializers/read/encoded.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 import logging
 import sys
 
+import h5py
+
 from .... import compression as compress
 from ....types import (
     ArrayOfEncodedEqualSizedArrays,
@@ -13,6 +15,7 @@ from .array import (
     _h5_read_array,
 )
 from .scalar import _h5_read_scalar
+from .utils import read_attrs
 from .vector_of_vectors import _h5_read_vector_of_vectors
 
 log = logging.getLogger(__name__)
@@ -20,21 +23,29 @@ log = logging.getLogger(__name__)
 
 def _h5_read_array_of_encoded_equalsized_arrays(
     h5g,
+    fname,
+    oname,
     **kwargs,
 ):
-    return _h5_read_encoded_array(ArrayOfEncodedEqualSizedArrays, h5g, **kwargs)
+    return _h5_read_encoded_array(
+        ArrayOfEncodedEqualSizedArrays, h5g, fname, oname, **kwargs
+    )
 
 
 def _h5_read_vector_of_encoded_vectors(
     h5g,
+    fname,
+    oname,
     **kwargs,
 ):
-    return _h5_read_encoded_array(VectorOfEncodedVectors, h5g, **kwargs)
+    return _h5_read_encoded_array(VectorOfEncodedVectors, h5g, fname, oname, **kwargs)
 
 
 def _h5_read_encoded_array(
     lgdotype,
     h5g,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -45,11 +56,11 @@ def _h5_read_encoded_array(
 ):
     if lgdotype not in (ArrayOfEncodedEqualSizedArrays, VectorOfEncodedVectors):
         msg = f"unsupported read of encoded type {lgdotype.__name__}"
-        raise LH5DecodeError(msg, h5g)
+        raise LH5DecodeError(msg, fname, oname)
 
     if not decompress and obj_buf is not None and not isinstance(obj_buf, lgdotype):
         msg = f"object buffer is not a {lgdotype.__name__}"
-        raise LH5DecodeError(msg, h5g)
+        raise LH5DecodeError(msg, fname, oname)
 
     # read out decoded_size, either a Scalar or an Array
     decoded_size_buf = encoded_data_buf = None
@@ -58,8 +69,11 @@ def _h5_read_encoded_array(
         encoded_data_buf = obj_buf.encoded_data
 
     if lgdotype is VectorOfEncodedVectors:
+        h5o = h5py.h5o.open(h5g, b"decoded_size")
         decoded_size, _ = _h5_read_array(
-            h5g["decoded_size"],
+            h5o,
+            fname,
+            f"{oname}/decoded_size",
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -67,16 +81,23 @@ def _h5_read_encoded_array(
             obj_buf=None if decompress else decoded_size_buf,
             obj_buf_start=0 if decompress else obj_buf_start,
         )
-
+        h5o.close()
     else:
+        h5o = h5py.h5o.open(h5g, b"decoded_size")
         decoded_size, _ = _h5_read_scalar(
-            h5g["decoded_size"],
+            h5o,
+            fname,
+            f"{oname}/decoded_size",
             obj_buf=None if decompress else decoded_size_buf,
         )
+        h5o.close()
 
     # read out encoded_data, a VectorOfVectors
+    h5o = h5py.h5o.open(h5g, b"encoded_data")
     encoded_data, n_rows_read = _h5_read_vector_of_vectors(
-        h5g["encoded_data"],
+        h5o,
+        fname,
+        f"{oname}/encoded_data",
         start_row=start_row,
         n_rows=n_rows,
         idx=idx,
@@ -84,6 +105,7 @@ def _h5_read_encoded_array(
         obj_buf=None if decompress else encoded_data_buf,
         obj_buf_start=0 if decompress else obj_buf_start,
     )
+    h5o.close()
 
     # return the still encoded data in the buffer object, if there
     if obj_buf is not None and not decompress:
@@ -93,7 +115,7 @@ def _h5_read_encoded_array(
     rawdata = lgdotype(
         encoded_data=encoded_data,
         decoded_size=decoded_size,
-        attrs=dict(h5g.attrs),
+        attrs=read_attrs(h5g, fname, oname),
     )
 
     # already return if no decompression is requested