legend-pydataobj 1.10.1__py3-none-any.whl → 1.11.0__py3-none-any.whl

{legend_pydataobj-1.10.1.dist-info → legend_pydataobj-1.11.0.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: legend_pydataobj
-Version: 1.10.1
+Version: 1.11.0
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration
-License: GNU GENERAL PUBLIC LICENSE
+License:                     GNU GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
         
          Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
@@ -697,34 +697,34 @@ Classifier: Topic :: Scientific/Engineering
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: awkward >=2
+Requires-Dist: awkward>=2
 Requires-Dist: awkward-pandas
 Requires-Dist: colorlog
-Requires-Dist: h5py >=3.10
+Requires-Dist: h5py>=3.10
 Requires-Dist: hdf5plugin
 Requires-Dist: hist
-Requires-Dist: numba !=0.53.*,!=0.54.*
+Requires-Dist: numba!=0.53.*,!=0.54.*
 Requires-Dist: numexpr
-Requires-Dist: numpy >=1.21
-Requires-Dist: pandas >=1.4.4
+Requires-Dist: numpy>=1.21
+Requires-Dist: pandas>=1.4.4
 Requires-Dist: parse
-Requires-Dist: pint >0.24
+Requires-Dist: pint>0.24
 Requires-Dist: pint-pandas
 Provides-Extra: all
-Requires-Dist: legend-pydataobj[docs,test] ; extra == 'all'
+Requires-Dist: legend-pydataobj[docs,test]; extra == "all"
 Provides-Extra: docs
-Requires-Dist: furo ; extra == 'docs'
-Requires-Dist: jupyter ; extra == 'docs'
-Requires-Dist: myst-parser ; extra == 'docs'
-Requires-Dist: nbsphinx ; extra == 'docs'
-Requires-Dist: sphinx ; extra == 'docs'
-Requires-Dist: sphinx-copybutton ; extra == 'docs'
-Requires-Dist: sphinx-inline-tabs ; extra == 'docs'
+Requires-Dist: furo; extra == "docs"
+Requires-Dist: jupyter; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: nbsphinx; extra == "docs"
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-copybutton; extra == "docs"
+Requires-Dist: sphinx-inline-tabs; extra == "docs"
 Provides-Extra: test
-Requires-Dist: pre-commit ; extra == 'test'
-Requires-Dist: pylegendtestdata ; extra == 'test'
-Requires-Dist: pytest >=6.0 ; extra == 'test'
-Requires-Dist: pytest-cov ; extra == 'test'
+Requires-Dist: pre-commit; extra == "test"
+Requires-Dist: pylegendtestdata; extra == "test"
+Requires-Dist: pytest>=6.0; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
 
 # legend-pydataobj
 

{legend_pydataobj-1.10.1.dist-info → legend_pydataobj-1.11.0.dist-info}/RECORD

@@ -1,6 +1,6 @@
 lgdo/__init__.py,sha256=1YUuAFQHNrOOkr3ZfrtEJOpYqgzbHRYA81ssbQZitQE,3196
-lgdo/_version.py,sha256=8G9z72uuzZV_GnX2AJUyUhUAHl8bmD6KQXou4HB100U,413
-lgdo/cli.py,sha256=vB1Oj6kZ5gWaY9HBPBRRRyiepp72hm3bFvQeUUWeMYg,8214
+lgdo/_version.py,sha256=rZqhcUFwPMyj_mTWUN2A6qcFr8Ptv08CSbXbruC3jR4,413
+lgdo/cli.py,sha256=Qm2EPmoIVxENAR8BeW7oWpTdHT4GbV-owfzM5NkgjvM,9353
 lgdo/lgdo_utils.py,sha256=6a2YWEwpyEMXlAyTHZMO01aqxy6SxJzPZkGNWKNWuS0,2567
 lgdo/lh5_store.py,sha256=5BzbJA9sLcqjp8bJDc2olwOiw0VS6rmfg3cfh1kQkRY,8512
 lgdo/logging.py,sha256=82wIOj7l7xr3WYyeHdpSXbbjzHJsy-uRyKYUYx2vMfQ,1003
@@ -13,7 +13,7 @@ lgdo/compression/radware.py,sha256=GcNTtjuyL7VBBqziUBmSqNXuhqy1bJJgvcyvyumPtrc,2
 lgdo/compression/utils.py,sha256=W2RkBrxPpXlat84dnU9Ad7d_tTws0irtGl7O1dNWjnk,1140
 lgdo/compression/varlen.py,sha256=6ZZUItyoOfygDdE0DyoISeFZfqdbH6xl7T0eclfarzg,15127
 lgdo/lh5/__init__.py,sha256=KzWF6HI-6N1NqQUm8LAxMmDbg0rgRY4DAaJ2s7w2tLM,811
-lgdo/lh5/core.py,sha256=YVtkTaU3SISDoLqR9UE_BDzsPApEW6_h_ac2NwSZ9zg,13868
+lgdo/lh5/core.py,sha256=__-A6Abctzfwfo4-xJi68xs2e4vfzONEQTJVrUCOw-I,13922
 lgdo/lh5/datatype.py,sha256=O_7BqOlX8PFMyG0ppkfUT5aps5HEqX0bpuKcJO3jhu0,1691
 lgdo/lh5/exceptions.py,sha256=43fQ8MnAsylY4aG6GF6hsRclagYaMkUv8957c1uTjWE,962
 lgdo/lh5/iterator.py,sha256=ZaBBnmuNIjinwO0JUY55wLxX8Om9rVRRzXBC5uHmSKM,19772
@@ -23,12 +23,12 @@ lgdo/lh5/utils.py,sha256=ioz8DlyXZsejwnU2qYdIccdHcF12H62jgLkZsiDOLSM,6243
 lgdo/lh5/_serializers/__init__.py,sha256=NSH8uOVY3r_Wn3t0nQHhEHhkHT7-GJYlxuS3YTDJa5Y,1263
 lgdo/lh5/_serializers/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lgdo/lh5/_serializers/read/array.py,sha256=uWfMCihfAmW2DE2ewip2qCK_kvQC_mb2zvOv26uzijc,1000
-lgdo/lh5/_serializers/read/composite.py,sha256=vQGh6nUQdSuHl0NTF2FeU7QC9UAA-E7XvNvrRJi4uw8,12384
+lgdo/lh5/_serializers/read/composite.py,sha256=yTm5dfTgkIL7eG9iZXxhdiRhG04cQLd_hybP4wmxCJE,11809
 lgdo/lh5/_serializers/read/encoded.py,sha256=Q98c08d8LkZq2AlY4rThYECVaEqwbv4T2Urn7TGnsyE,4130
 lgdo/lh5/_serializers/read/ndarray.py,sha256=lFCXD6bSzmMOH7cVmvRYXakkfMCI8EoqTPNONRJ1F0s,3690
 lgdo/lh5/_serializers/read/scalar.py,sha256=kwhWm1T91pXf86CqtUUD8_qheSR92gXZrQVtssV5YCg,922
-lgdo/lh5/_serializers/read/utils.py,sha256=0kYUFKiaQ3JUbjhP7tuKas_s80Kou6DhPlVXc40NHRE,5945
-lgdo/lh5/_serializers/read/vector_of_vectors.py,sha256=aCWTMbym7dF2yrhEfQs_GotcDqOKALRxgdJm4CA-bYs,7189
+lgdo/lh5/_serializers/read/utils.py,sha256=USacxDA0eY-u9lDOZDuJHcScoSVMNeAYljmRvW0T1Jk,7587
+lgdo/lh5/_serializers/read/vector_of_vectors.py,sha256=98P_XoXE8QWLQeSyBm9QHBF_5WGHKrfpNppLhj3QrbE,7169
 lgdo/lh5/_serializers/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lgdo/lh5/_serializers/write/array.py,sha256=eyVPwwddVOR9TNNyliCNYlS-XYXsdTEA8QoTnnOYJbw,2943
 lgdo/lh5/_serializers/write/composite.py,sha256=I6lH0nWFIpAfZyG4-0rLxzg3mfazZ_FEhQVp1FZ0aA4,9254
@@ -44,12 +44,12 @@ lgdo/types/lgdo.py,sha256=UnJDi1emQYVgH_H29Vipfs4LelPopxG5pgZUu1eKOlw,2761
 lgdo/types/scalar.py,sha256=c5Es2vyDqyWTPV6mujzfIzMpC1jNWkEIcvYyWQUxH3Q,1933
 lgdo/types/struct.py,sha256=Q0OWLVd4B0ciLb8t6VsxU3MPbmGLZ7WfQNno1lSQS0Q,4918
 lgdo/types/table.py,sha256=lB_jj6C0C5w8jbo17Lp0P8_uY8jy7opkTJc1OrbCGEI,17956
-lgdo/types/vectorofvectors.py,sha256=fBLI8P0HDe12Ib95eFUJObLa--gxz6wfAmOs_mDsokg,24390
+lgdo/types/vectorofvectors.py,sha256=cic9PsZ5EptQ6RMsykYeVHA8T7fh_KBZCcqeTP4i1wU,24395
 lgdo/types/vovutils.py,sha256=7BWPP0BSj-92ifbCIUBcfqxG5-TS8uxujTyJJuDFI04,10302
 lgdo/types/waveformtable.py,sha256=f2tS4f1OEoYaTM5ldCX9zmw8iSISCT3t3wS1SrPdu_o,9901
-legend_pydataobj-1.10.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-legend_pydataobj-1.10.1.dist-info/METADATA,sha256=gY35ifo00rptHQjVQ3BqkGeInupz20DGh1VIjeNEGlY,44381
-legend_pydataobj-1.10.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-legend_pydataobj-1.10.1.dist-info/entry_points.txt,sha256=Uu5MTlppBZxB4QGlLv-oX8FqACWjAZDNii__TBDJwLQ,72
-legend_pydataobj-1.10.1.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
-legend_pydataobj-1.10.1.dist-info/RECORD,,
+legend_pydataobj-1.11.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+legend_pydataobj-1.11.0.dist-info/METADATA,sha256=d4lBYc7gKmmSdW59x4tUxcfUSlETmAknC1f2TchfgNc,44382
+legend_pydataobj-1.11.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+legend_pydataobj-1.11.0.dist-info/entry_points.txt,sha256=Uu5MTlppBZxB4QGlLv-oX8FqACWjAZDNii__TBDJwLQ,72
+legend_pydataobj-1.11.0.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
+legend_pydataobj-1.11.0.dist-info/RECORD,,

{legend_pydataobj-1.10.1.dist-info → legend_pydataobj-1.11.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

lgdo/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.10.1'
-__version_tuple__ = version_tuple = (1, 10, 1)
+__version__ = version = '1.11.0'
+__version_tuple__ = version_tuple = (1, 11, 0)

lgdo/cli.py

@@ -7,7 +7,7 @@ import fnmatch
 import logging
 import sys
 
-from . import Array, Table, VectorOfVectors, __version__, lh5
+from . import Array, Scalar, Struct, Table, VectorOfVectors, __version__, lh5
 from . import logging as lgdogging  # eheheh
 
 log = logging.getLogger(__name__)
@@ -212,6 +212,7 @@ Exclude the /data/table1/col1 Table column:
     store = lh5.LH5Store()
     h5f0 = store.gimme_file(file0)
     lgdos = {}
+    lgdo_structs = {}
     # loop over object list in the first file
     for name in obj_list:
         # now loop over groups starting from root
@@ -222,7 +223,7 @@ Exclude the /data/table1/col1 Table column:
             if current in lgdos:
                 break
 
-            # not even an LGDO!
+            # not even an LGDO (i.e. a plain HDF5 group)!
             if "datatype" not in h5f0[current].attrs:
                 continue
 
@@ -232,14 +233,30 @@ Exclude the /data/table1/col1 Table column:
                 # read all!
                 obj, _ = store.read(current, h5f0)
                 lgdos[current] = obj
+            elif isinstance(obj, Struct):
+                # structs might be used in a "group-like" fashion (i.e. they might only
+                # contain array-like objects).
+                # note: handle after handling tables, as tables also satisfy this check.
+                lgdo_structs[current] = obj.attrs["datatype"]
+                continue
+            elif isinstance(obj, Scalar):
+                msg = f"cannot concat scalar field {current}"
+                log.warning(msg)
 
             break
 
     msg = f"first-level, array-like objects: {lgdos.keys()}"
     log.debug(msg)
+    msg = f"nested structs: {lgdo_structs.keys()}"
+    log.debug(msg)
 
     h5f0.close()
 
+    if lgdos == {}:
+        msg = "did not find any field to concatenate, exit"
+        log.error(msg)
+        return
+
     # 2. remove (nested) table fields based on obj_list
 
     def _inplace_table_filter(name, table, obj_list):
@@ -298,3 +315,14 @@ Exclude the /data/table1/col1 Table column:
                 _inplace_table_filter(name, obj, obj_list)
 
             store.write(obj, name, args.output, wo_mode="append")
+
+    # 5. reset datatypes of the "group-like" structs
+
+    if lgdo_structs != {}:
+        output_file = store.gimme_file(args.output, mode="a")
+        for struct, struct_dtype in lgdo_structs.items():
+            msg = f"reset datatype of struct {struct} to {struct_dtype}"
+            log.debug(msg)
+
+            output_file[struct].attrs["datatype"] = struct_dtype
+        output_file.close()

lgdo/lh5/_serializers/read/composite.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import bisect
 import logging
 import sys
-from collections import defaultdict
 
 import h5py
 import numpy as np
@@ -72,19 +71,8 @@ def _h5_read_lgdo(
             obj_buf=obj_buf,
         )
 
-    # check field_mask and make it a default dict
-    if field_mask is None:
-        field_mask = defaultdict(lambda: True)
-    elif isinstance(field_mask, dict):
-        default = True
-        if len(field_mask) > 0:
-            default = not field_mask[next(iter(field_mask.keys()))]
-        field_mask = defaultdict(lambda: default, field_mask)
-    elif isinstance(field_mask, (list, tuple, set)):
-        field_mask = defaultdict(bool, {field: True for field in field_mask})
-    elif not isinstance(field_mask, defaultdict):
-        msg = "bad field_mask type"
-        raise ValueError(msg, type(field_mask).__name__)
+    # Convert whatever we input into a defaultdict
+    field_mask = utils.build_field_mask(field_mask)
 
     if lgdotype is Struct:
         return _h5_read_struct(
@@ -246,18 +234,16 @@ def _h5_read_struct(
 
     # determine fields to be read out
     all_fields = dtypeutils.get_struct_fields(attrs["datatype"])
-    selected_fields = (
-        [field for field in all_fields if field_mask[field]]
-        if field_mask is not None
-        else all_fields
-    )
+    selected_fields = utils.eval_field_mask(field_mask, all_fields)
 
     # modify datatype in attrs if a field_mask was used
-    attrs["datatype"] = "struct{" + ",".join(selected_fields) + "}"
+    attrs["datatype"] = (
+        "struct{" + ",".join(field for field, _ in selected_fields) + "}"
+    )
 
     # loop over fields and read
     obj_dict = {}
-    for field in selected_fields:
+    for field, submask in selected_fields:
         # support for integer keys
         field_key = int(field) if attrs.get("int_keys") else str(field)
         h5o = h5py.h5o.open(h5g, field.encode("utf-8"))
@@ -269,6 +255,7 @@ def _h5_read_struct(
             n_rows=n_rows,
             idx=idx,
             use_h5idx=use_h5idx,
+            field_mask=submask,
             decompress=decompress,
         )
         h5o.close()
@@ -297,19 +284,15 @@ def _h5_read_table(
 
     # determine fields to be read out
     all_fields = dtypeutils.get_struct_fields(attrs["datatype"])
-    selected_fields = (
-        [field for field in all_fields if field_mask[field]]
-        if field_mask is not None
-        else all_fields
-    )
+    selected_fields = utils.eval_field_mask(field_mask, all_fields)
 
     # modify datatype in attrs if a field_mask was used
-    attrs["datatype"] = "table{" + ",".join(selected_fields) + "}"
+    attrs["datatype"] = "table{" + ",".join(field for field, _ in selected_fields) + "}"
 
     # read out each of the fields
     col_dict = {}
     rows_read = []
-    for field in selected_fields:
+    for field, submask in selected_fields:
         fld_buf = None
         if obj_buf is not None:
             if not isinstance(obj_buf, Table) or field not in obj_buf:
@@ -329,6 +312,7 @@ def _h5_read_table(
             use_h5idx=use_h5idx,
             obj_buf=fld_buf,
             obj_buf_start=obj_buf_start,
+            field_mask=submask,
             decompress=decompress,
         )
         h5o.close()

lgdo/lh5/_serializers/read/utils.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import logging
+from collections import defaultdict
+from collections.abc import Collection, Mapping
 
 import h5py
 import numpy as np
@@ -22,6 +24,56 @@ def check_obj_buf_attrs(attrs, new_attrs, fname, oname):
         raise LH5DecodeError(msg, fname, oname)
 
 
+def build_field_mask(field_mask: Mapping[str, bool] | Collection[str]) -> defaultdict:
+    # check field_mask and make it a default dict
+    if field_mask is None:
+        return defaultdict(lambda: True)
+    if isinstance(field_mask, dict):
+        default = True
+        if len(field_mask) > 0:
+            default = not field_mask[next(iter(field_mask.keys()))]
+        return defaultdict(lambda: default, field_mask)
+    if isinstance(field_mask, (list, tuple, set)):
+        return defaultdict(bool, {field: True for field in field_mask})
+    if isinstance(field_mask, defaultdict):
+        return field_mask
+    msg = "bad field_mask type"
+    raise ValueError(msg, type(field_mask).__name__)
+
+
+def eval_field_mask(
+    field_mask: defaultdict, all_fields: list[str]
+) -> list[tuple(str, defaultdict)]:
+    """Get list of fields that need to be loaded along with a sub-field-mask
+    in case we have a nested Table"""
+
+    if field_mask is None:
+        return all_fields
+
+    this_field_mask = defaultdict(field_mask.default_factory)
+    sub_field_masks = {}
+
+    for key, val in field_mask.items():
+        field = key.strip("/")
+        pos = field.find("/")
+        if pos < 0:
+            this_field_mask[field] = val
+        else:
+            sub_field = field[pos + 1 :]
+            field = field[:pos]
+            this_field_mask[field] = True
+            sub_mask = sub_field_masks.setdefault(
+                field, defaultdict(field_mask.default_factory)
+            )
+            sub_mask[sub_field] = val
+
+    return [
+        (field, sub_field_masks.get(field))
+        for field in all_fields
+        if this_field_mask[field]
+    ]
+
+
 def read_attrs(h5o, fname, oname):
     """Read all attributes for an hdf5 dataset or group using low level API
     and return them as a dict. Assume all are strings or scalar types."""
@@ -114,6 +166,7 @@ def read_size_in_bytes(h5o, fname, oname, field_mask=None):
     h5a.read(type_attr)
     type_attr = type_attr.item().decode()
     lgdotype = datatype.datatype(type_attr)
+    field_mask = build_field_mask(field_mask)
 
     # scalars are dim-0 datasets
     if lgdotype in (
@@ -124,24 +177,21 @@ def read_size_in_bytes(h5o, fname, oname, field_mask=None):
     ):
         return int(np.prod(h5o.shape) * h5o.dtype.itemsize)
 
-    # structs don't have rows
-    if lgdotype in (types.Struct, types.Histogram, types.Histogram.Axis):
-        size = 0
-        for key in h5o:
-            obj = h5py.h5o.open(h5o, key)
-            size += read_size_in_bytes(obj, fname, oname, field_mask)
-            obj.close()
-        return size
-
     # tables should have elements with all the same length
-    if lgdotype in (types.Table, types.WaveformTable):
+    if lgdotype in (
+        types.Struct,
+        types.Histogram,
+        types.Histogram.Axis,
+        types.Table,
+        types.WaveformTable,
+    ):
         # read out each of the fields
         size = 0
-        if not field_mask:
-            field_mask = datatype.get_struct_fields(type_attr)
-        for field in field_mask:
+        all_fields = datatype.get_struct_fields(type_attr)
+        selected_fields = eval_field_mask(field_mask, all_fields)
+        for field, submask in selected_fields:
             obj = h5py.h5o.open(h5o, field.encode())
-            size += read_size_in_bytes(obj, fname, field)
+            size += read_size_in_bytes(obj, fname, field, submask)
             obj.close()
         return size
 

lgdo/lh5/_serializers/read/vector_of_vectors.py

@@ -156,7 +156,7 @@ def _h5_read_vector_of_vectors(
         # grow fd_buf if necessary to hold the data
         fdb_size = fd_buf_start + fd_n_rows
         if len(fd_buf) < fdb_size:
-            fd_buf.nda.resize(fdb_size, refcheck=False)
+            fd_buf.resize(fdb_size)
 
     # now read
     h5o = h5py.h5o.open(h5g, b"flattened_data")

lgdo/lh5/core.py

@@ -120,6 +120,7 @@ def read(
         lh5_obj = lh5_file[name]
     else:
         lh5_files = list(lh5_file)
+
         n_rows_read = 0
         obj_buf_is_new = False
 
@@ -353,5 +354,8 @@ def read_as(
     # NOTE: providing a buffer does not make much sense
     obj = read(name, lh5_file, **kwargs1)
 
+    if isinstance(obj, tuple):
+        obj = obj[0]
+
     # and finally return a view
     return obj.view_as(library, **kwargs2)

lgdo/types/vectorofvectors.py

@@ -503,7 +503,7 @@ class VectorOfVectors(LGDO):
     def __str__(self) -> str:
         string = self.view_as("ak").show(stream=None)
 
-        string = string.strip().removesuffix("]")
+        string = str(string).strip().removesuffix("]")
         string += "\n]"
 
         tmp_attrs = self.attrs.copy()