legend-pydataobj 1.11.6__py3-none-any.whl → 1.12.0a1__py3-none-any.whl

{legend_pydataobj-1.11.6.dist-info → legend_pydataobj-1.12.0a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.11.6
+Version: 1.12.0a1
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration
@@ -726,6 +726,7 @@ Requires-Dist: pylegendtestdata; extra == "test"
 Requires-Dist: pytest>=6.0; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: dbetto; extra == "test"
+Dynamic: license-file
 
 # legend-pydataobj
 

{legend_pydataobj-1.11.6.dist-info → legend_pydataobj-1.12.0a1.dist-info}/RECORD

@@ -1,6 +1,7 @@
+legend_pydataobj-1.12.0a1.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
 lgdo/__init__.py,sha256=QMYK9HhoMi0pbahPN8mPD18gyTxscFgo7QKfCxVhy-0,3196
-lgdo/_version.py,sha256=Uoj5vQkNSraBmH8UOZgP0kMxvcHSlKlg1J7QJpv8_BM,413
-lgdo/cli.py,sha256=Qm2EPmoIVxENAR8BeW7oWpTdHT4GbV-owfzM5NkgjvM,9353
+lgdo/_version.py,sha256=kTYHwRhTzZEJHpwJeVgXBi4yFTeQDpnR6MYkvCMA06Q,515
+lgdo/cli.py,sha256=s_EWTBWW76l7zWb6gaTSTjiT-0RzzcYEmjeFEQCVxfk,4647
 lgdo/lgdo_utils.py,sha256=6a2YWEwpyEMXlAyTHZMO01aqxy6SxJzPZkGNWKNWuS0,2567
 lgdo/lh5_store.py,sha256=5BzbJA9sLcqjp8bJDc2olwOiw0VS6rmfg3cfh1kQkRY,8512
 lgdo/logging.py,sha256=82wIOj7l7xr3WYyeHdpSXbbjzHJsy-uRyKYUYx2vMfQ,1003
@@ -12,44 +13,44 @@ lgdo/compression/generic.py,sha256=tF3UhLJbUDcovLxpIzgQRxFSjZ5Fz3uDRy9kI4mFntQ,2
 lgdo/compression/radware.py,sha256=GcNTtjuyL7VBBqziUBmSqNXuhqy1bJJgvcyvyumPtrc,23839
 lgdo/compression/utils.py,sha256=W2RkBrxPpXlat84dnU9Ad7d_tTws0irtGl7O1dNWjnk,1140
 lgdo/compression/varlen.py,sha256=6ZZUItyoOfygDdE0DyoISeFZfqdbH6xl7T0eclfarzg,15127
-lgdo/lh5/__init__.py,sha256=FflSA1LF3RTom1YvlGmTleJcl1ytxwez8B_Hn_o10wU,811
-lgdo/lh5/core.py,sha256=__-A6Abctzfwfo4-xJi68xs2e4vfzONEQTJVrUCOw-I,13922
+lgdo/lh5/__init__.py,sha256=y1XE_mpFWwamrl7WVjAVSVB25X4PrEfdVXSneSQEmlQ,825
+lgdo/lh5/concat.py,sha256=BZCgK7TWPKK8fMmha8K83d3bC31FVO1b5LOW7x-Ru1s,6186
+lgdo/lh5/core.py,sha256=GjosZGUp4GSO5FtWV9eXUt_6DGU_OwJXODlj5K1j93M,13320
 lgdo/lh5/datatype.py,sha256=O_7BqOlX8PFMyG0ppkfUT5aps5HEqX0bpuKcJO3jhu0,1691
 lgdo/lh5/exceptions.py,sha256=3kj8avXl4eBGvebl3LG12gJEmw91W0T8PYR0AfvUAyM,1211
-lgdo/lh5/iterator.py,sha256=ZaBBnmuNIjinwO0JUY55wLxX8Om9rVRRzXBC5uHmSKM,19772
-lgdo/lh5/store.py,sha256=3wAaQDd1Zmo0_bQ9DbB-FbKS4Uy_Tb642qKHXtZpSw4,10643
+lgdo/lh5/iterator.py,sha256=1ob9B7Bf3ioGCtZkUZoL6ibTxAwLf4ld8_33ghVVEa4,20498
+lgdo/lh5/store.py,sha256=MYbMt-Mc7izELxuyLlSrrYrylCIzxc2CLzZYIVbZ33w,8455
 lgdo/lh5/tools.py,sha256=T9CgHA8A3_tVBMtiNJ6hATQKhdqI61m3cX4p2wGKc6c,9937
 lgdo/lh5/utils.py,sha256=ioz8DlyXZsejwnU2qYdIccdHcF12H62jgLkZsiDOLSM,6243
 lgdo/lh5/_serializers/__init__.py,sha256=eZzxMp1SeZWG0PkEXUiCz3XyprQ8EmelHUmJogC8xYE,1263
 lgdo/lh5/_serializers/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lgdo/lh5/_serializers/read/array.py,sha256=uWfMCihfAmW2DE2ewip2qCK_kvQC_mb2zvOv26uzijc,1000
-lgdo/lh5/_serializers/read/composite.py,sha256=yTm5dfTgkIL7eG9iZXxhdiRhG04cQLd_hybP4wmxCJE,11809
+lgdo/lh5/_serializers/read/composite.py,sha256=UvkZHEhf0V7SFLxzF52eyP68hU0guGOLqosrfmIfeys,11729
 lgdo/lh5/_serializers/read/encoded.py,sha256=Q98c08d8LkZq2AlY4rThYECVaEqwbv4T2Urn7TGnsyE,4130
 lgdo/lh5/_serializers/read/ndarray.py,sha256=lFCXD6bSzmMOH7cVmvRYXakkfMCI8EoqTPNONRJ1F0s,3690
 lgdo/lh5/_serializers/read/scalar.py,sha256=kwhWm1T91pXf86CqtUUD8_qheSR92gXZrQVtssV5YCg,922
-lgdo/lh5/_serializers/read/utils.py,sha256=USacxDA0eY-u9lDOZDuJHcScoSVMNeAYljmRvW0T1Jk,7587
-lgdo/lh5/_serializers/read/vector_of_vectors.py,sha256=Fqh2gXFqeR2y0ofQn4GoSdSAATPvHiuBzzcgL16e6ss,7205
+lgdo/lh5/_serializers/read/utils.py,sha256=YfSqPO-83A1XvhhuULxQ0Qz2A5ODa3sb7ApNxQVJXd0,7581
+lgdo/lh5/_serializers/read/vector_of_vectors.py,sha256=765P8mElGArAaEPkHTAUXFQ47t1_3-3BQAete0LckBQ,7207
 lgdo/lh5/_serializers/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lgdo/lh5/_serializers/write/array.py,sha256=66DKnW2yqIBlUGNBPWcE-m4W0B2-nTKusDHGX9m6GY0,3223
 lgdo/lh5/_serializers/write/composite.py,sha256=I6lH0nWFIpAfZyG4-0rLxzg3mfazZ_FEhQVp1FZ0aA4,9254
 lgdo/lh5/_serializers/write/scalar.py,sha256=JPt_fcdTKOSFp5hfJdcKIfK4hxhcD8vhOlvDF-7btQ8,763
 lgdo/lh5/_serializers/write/vector_of_vectors.py,sha256=puGQX9XF5P_5DVbm_Cc6TvPrsDywgBLSYtkqFNltbB4,3493
 lgdo/types/__init__.py,sha256=DNfOErPiAZg-7Gygkp6ZKAi20Yrm1mfderZHvKo1Y4s,821
-lgdo/types/array.py,sha256=sUxh1CNCaefrnybt5qdjmmMpVQa_RqFxUv1tJ_pyBbc,6537
+lgdo/types/array.py,sha256=e3p93yrfzSmyBgWdGqqtETcKpM7_FxENaAErru15rvo,8904
 lgdo/types/arrayofequalsizedarrays.py,sha256=DOGJiTmc1QCdm7vLbE6uIRXoMPtt8uuCfmwQawgWf5s,4949
-lgdo/types/encoded.py,sha256=JW4U5ow7KLMzhKnmhdnxbC3SZJAs4bOEDZWKG4KY1uU,15293
+lgdo/types/encoded.py,sha256=_e8u_BPfpjJbLnEdyTo9QG3kbNsGj0BN4gjdj3L1ndw,15640
 lgdo/types/fixedsizearray.py,sha256=7RjUwTz1bW0pcrdy27JlfrXPAuOU89Kj7pOuSUCojK8,1527
-lgdo/types/histogram.py,sha256=y6j2VDuGYYnLy7WI4J90ApS0PAwic4kCpouZPX09Nus,19974
-lgdo/types/lgdo.py,sha256=RQ2P70N7IWMBDnLLuJI3sm6zQTIKyOMSsKZtBNzmE90,2928
+lgdo/types/histogram.py,sha256=Jz1lLH56BfYnmcUhxUHK1h2wLDQ0Abgyd-6LznU-3-k,19979
+lgdo/types/lgdo.py,sha256=21YNtJCHnSO3M60rjsAdbMO5crDjL_0BtuFpudZ2xvU,4500
 lgdo/types/scalar.py,sha256=c5Es2vyDqyWTPV6mujzfIzMpC1jNWkEIcvYyWQUxH3Q,1933
 lgdo/types/struct.py,sha256=Q0OWLVd4B0ciLb8t6VsxU3MPbmGLZ7WfQNno1lSQS0Q,4918
-lgdo/types/table.py,sha256=VIHQOPXJHJgiCjMMb_p7EdbcCqLFSObHMdHSxC1Dm5Y,19212
-lgdo/types/vectorofvectors.py,sha256=Al9FmY44M-vnzhPdQlFOzwm06LNGBI7RSLSdU5pl9us,24663
-lgdo/types/vovutils.py,sha256=7BWPP0BSj-92ifbCIUBcfqxG5-TS8uxujTyJJuDFI04,10302
+lgdo/types/table.py,sha256=FkWesoEA9bmGGSW8Ewig1Zs77ffUoR_nggfYSmkWpjU,20079
+lgdo/types/vectorofvectors.py,sha256=CtPR2WDBmJmzzfXwH4aUcNMB5LvTiGWmL_qRbFah3to,24756
+lgdo/types/vovutils.py,sha256=WjvPLEJrRNjktnbyfypfgxZX-K_aOvcwPygfzoknsyA,10701
 lgdo/types/waveformtable.py,sha256=f2tS4f1OEoYaTM5ldCX9zmw8iSISCT3t3wS1SrPdu_o,9901
-legend_pydataobj-1.11.6.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-legend_pydataobj-1.11.6.dist-info/METADATA,sha256=tZZ9ocWZZR9ECF5Hg8DcQCL6uzCthV8L1ApxIbOu6UY,44421
-legend_pydataobj-1.11.6.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-legend_pydataobj-1.11.6.dist-info/entry_points.txt,sha256=Uu5MTlppBZxB4QGlLv-oX8FqACWjAZDNii__TBDJwLQ,72
-legend_pydataobj-1.11.6.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
-legend_pydataobj-1.11.6.dist-info/RECORD,,
+legend_pydataobj-1.12.0a1.dist-info/METADATA,sha256=55pMph32j8h4LKGnoVEdvHX27bHr8k__sdT4L9O5dIA,44445
+legend_pydataobj-1.12.0a1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+legend_pydataobj-1.12.0a1.dist-info/entry_points.txt,sha256=0KWfnwbuwhNn0vPUqARukjp04Ca6lzfZBSirouRmk7I,76
+legend_pydataobj-1.12.0a1.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
+legend_pydataobj-1.12.0a1.dist-info/RECORD,,

{legend_pydataobj-1.11.6.dist-info → legend_pydataobj-1.12.0a1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{legend_pydataobj-1.11.6.dist-info → legend_pydataobj-1.12.0a1.dist-info}/entry_points.txt

@@ -1,3 +1,3 @@
 [console_scripts]
-lh5concat = lgdo.cli:lh5concat
+lh5concat = lgdo.cli:lh5concat_cli
 lh5ls = lgdo.cli:lh5ls

lgdo/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.11.6'
-__version_tuple__ = version_tuple = (1, 11, 6)
+__version__ = version = '1.12.0a1'
+__version_tuple__ = version_tuple = (1, 12, 0)

lgdo/cli.py

@@ -3,12 +3,12 @@
 from __future__ import annotations
 
 import argparse
-import fnmatch
 import logging
 import sys
 
-from . import Array, Scalar, Struct, Table, VectorOfVectors, __version__, lh5
+from . import __version__, lh5
 from . import logging as lgdogging  # eheheh
+from .lh5.concat import lh5concat
 
 log = logging.getLogger(__name__)
 
@@ -80,7 +80,7 @@ def lh5ls(args=None):
     )
 
 
-def lh5concat(args=None):
+def lh5concat_cli(args=None):
     """Command line interface for concatenating array-like LGDOs in LH5 files."""
     parser = argparse.ArgumentParser(
         prog="lh5concat",
@@ -174,155 +174,10 @@ Exclude the /data/table1/col1 Table column:
         print(__version__)  # noqa: T201
         sys.exit()
 
-    if len(args.lh5_file) < 2:
-        msg = "you must provide at least two input files"
-        raise RuntimeError(msg)
-
-    # determine list of objects by recursively ls'ing first file
-    file0 = args.lh5_file[0]
-    obj_list_full = set(lh5.ls(file0, recursive=True))
-
-    # let's remove objects with nested LGDOs inside
-    to_remove = set()
-    for name in obj_list_full:
-        if len(fnmatch.filter(obj_list_full, f"{name}/*")) > 1:
-            to_remove.add(name)
-    obj_list_full -= to_remove
-
-    obj_list = set()
-    # now first remove excluded stuff
-    if args.exclude is not None:
-        for exc in args.exclude:
-            obj_list_full -= set(fnmatch.filter(obj_list_full, exc.strip("/")))
-
-    # then make list of included, based on latest list
-    if args.include is not None:
-        for inc in args.include:
-            obj_list |= set(fnmatch.filter(obj_list_full, inc.strip("/")))
-    else:
-        obj_list = obj_list_full
-
-    # sort
-    obj_list = sorted(obj_list)
-
-    msg = f"objects matching include patterns {args.include} in {file0}: {obj_list}"
-    log.debug(msg)
-
-    # 1. read first valid lgdo from left to right
-    store = lh5.LH5Store()
-    h5f0 = store.gimme_file(file0)
-    lgdos = {}
-    lgdo_structs = {}
-    # loop over object list in the first file
-    for name in obj_list:
-        # now loop over groups starting from root
-        current = ""
-        for item in name.split("/"):
-            current = f"{current}/{item}".strip("/")
-
-            if current in lgdos:
-                break
-
-            # not even an LGDO (i.e. a plain HDF5 group)!
-            if "datatype" not in h5f0[current].attrs:
-                continue
-
-            # read as little as possible
-            obj, _ = store.read(current, h5f0, n_rows=1)
-            if isinstance(obj, (Table, Array, VectorOfVectors)):
-                # read all!
-                obj, _ = store.read(current, h5f0)
-                lgdos[current] = obj
-            elif isinstance(obj, Struct):
-                # structs might be used in a "group-like" fashion (i.e. they might only
-                # contain array-like objects).
-                # note: handle after handling tables, as tables also satisfy this check.
-                lgdo_structs[current] = obj.attrs["datatype"]
-                continue
-            elif isinstance(obj, Scalar):
-                msg = f"cannot concat scalar field {current}"
-                log.warning(msg)
-
-            break
-
-    msg = f"first-level, array-like objects: {lgdos.keys()}"
-    log.debug(msg)
-    msg = f"nested structs: {lgdo_structs.keys()}"
-    log.debug(msg)
-
-    h5f0.close()
-
-    if lgdos == {}:
-        msg = "did not find any field to concatenate, exit"
-        log.error(msg)
-        return
-
-    # 2. remove (nested) table fields based on obj_list
-
-    def _inplace_table_filter(name, table, obj_list):
-        # filter objects nested in this LGDO
-        skm = fnmatch.filter(obj_list, f"{name}/*")
-        kept = {it.removeprefix(name).strip("/").split("/")[0] for it in skm}
-
-        # now remove fields
-        for k in list(table.keys()):
-            if k not in kept:
-                table.remove_column(k)
-
-        msg = f"fields left in table '{name}': {table.keys()}"
-        log.debug(msg)
-
-        # recurse!
-        for k2, v2 in table.items():
-            if not isinstance(v2, Table):
-                continue
-
-            _inplace_table_filter(f"{name}/{k2}", v2, obj_list)
-
-    for key, val in lgdos.items():
-        if not isinstance(val, Table):
-            continue
-
-        _inplace_table_filter(key, val, obj_list)
-
-    # 3. write to output file
-    msg = f"creating output file {args.output}"
-    log.info(msg)
-
-    first_done = False
-    for name, obj in lgdos.items():
-        store.write(
-            obj,
-            name,
-            args.output,
-            wo_mode="overwrite_file"
-            if (args.overwrite and not first_done)
-            else "write_safe",
-        )
-
-        first_done = True
-
-    # 4. loop over rest of files/names and write-append
-
-    for file in args.lh5_file[1:]:
-        msg = f"appending file {file} to {args.output}"
-        log.info(msg)
-
-        for name in lgdos:
-            obj, _ = store.read(name, file)
-            # need to remove nested LGDOs from obj too before appending
-            if isinstance(obj, Table):
-                _inplace_table_filter(name, obj, obj_list)
-
-            store.write(obj, name, args.output, wo_mode="append")
-
-    # 5. reset datatypes of the "group-like" structs
-
-    if lgdo_structs != {}:
-        output_file = store.gimme_file(args.output, mode="a")
-        for struct, struct_dtype in lgdo_structs.items():
-            msg = f"reset datatype of struct {struct} to {struct_dtype}"
-            log.debug(msg)
-
-            output_file[struct].attrs["datatype"] = struct_dtype
-        output_file.close()
+    lh5concat(
+        lh5_files=args.lh5_file,
+        overwrite=args.overwrite,
+        output=args.output,
+        include_list=args.include,
+        exclude_list=args.exclude,
+    )

lgdo/lh5/__init__.py

@@ -18,6 +18,7 @@ __all__ = [
     "DEFAULT_HDF5_SETTINGS",
     "LH5Iterator",
     "LH5Store",
+    "concat",
     "load_dfs",
     "load_nda",
     "ls",

lgdo/lh5/_serializers/read/composite.py

@@ -353,15 +353,13 @@ def _h5_read_table(
             table = Table(col_dict=col_dict, attrs=attrs)
 
         # set (write) loc to end of tree
-        table.loc = n_rows_read
+        table.resize(do_warn=True)
         return table, n_rows_read
 
     # We have read all fields into the object buffer. Run
     # checks: All columns should be the same size. So update
     # table's size as necessary, warn if any mismatches are found
     obj_buf.resize(do_warn=True)
-    # set (write) loc to end of tree
-    obj_buf.loc = obj_buf_start + n_rows_read
 
     # check attributes
     utils.check_obj_buf_attrs(obj_buf.attrs, attrs, fname, oname)

lgdo/lh5/_serializers/read/utils.py

@@ -34,7 +34,7 @@ def build_field_mask(field_mask: Mapping[str, bool] | Collection[str]) -> defaul
             default = not field_mask[next(iter(field_mask.keys()))]
         return defaultdict(lambda: default, field_mask)
     if isinstance(field_mask, (list, tuple, set)):
-        return defaultdict(bool, {field: True for field in field_mask})
+        return defaultdict(bool, dict.fromkeys(field_mask, True))
     if isinstance(field_mask, defaultdict):
         return field_mask
     msg = "bad field_mask type"

lgdo/lh5/_serializers/read/vector_of_vectors.py

@@ -123,7 +123,7 @@ def _h5_read_vector_of_vectors(
                 )
                 msg = (
                     f"cumulative_length non-increasing between entries "
-                    f"{start_row} and {start_row+n_rows_read}"
+                    f"{start_row} and {start_row + n_rows_read}"
                 )
                 raise LH5DecodeError(msg, fname, oname)
 

lgdo/lh5/concat.py

@@ -0,0 +1,219 @@
+from __future__ import annotations
+
+import fnmatch
+import logging
+
+from lgdo.lh5 import LH5Iterator
+
+from .. import Array, Scalar, Struct, Table, VectorOfVectors, lh5
+
+log = logging.getLogger(__name__)
+
+
+def _get_obj_list(
+    lh5_files: list, include_list: list | None = None, exclude_list: list | None = None
+) -> list[str]:
+    """Extract a list of lh5 objects to concatenate.
+
+    Parameters
+    ----------
+    lh5_files
+        list of input files to concatenate.
+    include_list
+        patterns for tables to include.
+    exclude_list
+        patterns for tables to exclude.
+
+    """
+    file0 = lh5_files[0]
+    obj_list_full = set(lh5.ls(file0, recursive=True))
+
+    # let's remove objects with nested LGDOs inside
+    to_remove = set()
+    for name in obj_list_full:
+        if len(fnmatch.filter(obj_list_full, f"{name}/*")) > 1:
+            to_remove.add(name)
+    obj_list_full -= to_remove
+
+    obj_list = set()
+    # now first remove excluded stuff
+    if exclude_list is not None:
+        for exc in exclude_list:
+            obj_list_full -= set(fnmatch.filter(obj_list_full, exc.strip("/")))
+
+    # then make list of included, based on latest list
+    if include_list is not None:
+        for inc in include_list:
+            obj_list |= set(fnmatch.filter(obj_list_full, inc.strip("/")))
+    else:
+        obj_list = obj_list_full
+
+    # sort
+    return sorted(obj_list)
+
+
+def _get_lgdos(file, obj_list):
+    """Get name of LGDO objects."""
+
+    store = lh5.LH5Store()
+    h5f0 = store.gimme_file(file)
+
+    lgdos = []
+    lgdo_structs = {}
+
+    # loop over object list in the first file
+    for name in obj_list:
+        # now loop over groups starting from root
+        current = ""
+        for item in name.split("/"):
+            current = f"{current}/{item}".strip("/")
+
+            if current in lgdos:
+                break
+
+            # not even an LGDO (i.e. a plain HDF5 group)!
+            if "datatype" not in h5f0[current].attrs:
+                continue
+
+            # read as little as possible
+            obj = store.read(current, h5f0, n_rows=1)
+            if isinstance(obj, (Table, Array, VectorOfVectors)):
+                lgdos.append(current)
+
+            elif isinstance(obj, Struct):
+                # structs might be used in a "group-like" fashion (i.e. they might only
+                # contain array-like objects).
+                # note: handle after handling tables, as tables also satisfy this check.
+                lgdo_structs[current] = obj.attrs["datatype"]
+                continue
+
+            elif isinstance(obj, Scalar):
+                msg = f"cannot concat scalar field {current}"
+                log.warning(msg)
+
+            break
+
+    msg = f"first-level, array-like objects: {lgdos}"
+    log.info(msg)
+
+    msg = f"nested structs: {lgdo_structs}"
+    log.info(msg)
+
+    h5f0.close()
+
+    if lgdos == []:
+        msg = "did not find any field to concatenate, exit"
+        raise RuntimeError(msg)
+
+    return lgdos, lgdo_structs
+
+
+def _inplace_table_filter(name, table, obj_list):
+    """filter objects nested in this LGDO"""
+    skm = fnmatch.filter(obj_list, f"{name}/*")
+    kept = {it.removeprefix(name).strip("/").split("/")[0] for it in skm}
+
+    # now remove fields
+    for k in list(table.keys()):
+        if k not in kept:
+            table.remove_column(k)
+
+    msg = f"fields left in table '{name}': {table.keys()}"
+    log.debug(msg)
+
+    # recurse!
+    for k2, v2 in table.items():
+        if not isinstance(v2, Table):
+            continue
+
+        _inplace_table_filter(f"{name}/{k2}", v2, obj_list)
+
+
+def _remove_nested_fields(lgdos: dict, obj_list: list):
+    """Remove (nested) table fields based on obj_list."""
+
+    for key, val in lgdos.items():
+        if not isinstance(val, Table):
+            continue
+
+        _inplace_table_filter(key, val, obj_list)
+
+
+def lh5concat(
+    lh5_files: list,
+    output: str,
+    overwrite: bool = False,
+    *,
+    include_list: list | None = None,
+    exclude_list: list | None = None,
+) -> None:
+    """Concatenate LGDO Arrays, VectorOfVectors and Tables in LH5 files.
+
+    Parameters
+    ----------
+    lh5_files
+        list of input files to concatenate.
+    output
+        path to the output file
+    include_list
+        patterns for tables to include.
+    exclude_list
+        patterns for tables to exclude.
+    """
+
+    if len(lh5_files) < 2:
+        msg = "you must provide at least two input files"
+        raise RuntimeError(msg)
+
+    # determine list of objects by recursively ls'ing first file
+    obj_list = _get_obj_list(
+        lh5_files, include_list=include_list, exclude_list=exclude_list
+    )
+
+    msg = f"objects matching include patterns {include_list} in {lh5_files[0]}: {obj_list}"
+    log.info(msg)
+
+    lgdos, lgdo_structs = _get_lgdos(lh5_files[0], obj_list)
+    first_done = False
+    store = lh5.LH5Store()
+
+    # loop over lgdo objects
+    for lgdo in lgdos:
+        # iterate over the files
+        for lh5_obj in LH5Iterator(lh5_files, lgdo):
+            data = {lgdo: lh5_obj}
+
+            # remove the nested fields
+            _remove_nested_fields(data, obj_list)
+
+            if first_done is False:
+                msg = f"creating output file {output}"
+                log.info(msg)
+
+                store.write(
+                    data[lgdo],
+                    lgdo,
+                    output,
+                    wo_mode="overwrite_file"
+                    if (overwrite and not first_done)
+                    else "write_safe",
+                )
+                first_done = True
+
+            else:
+                msg = f"appending to {output}"
+                log.info(msg)
+
+                if isinstance(data[lgdo], Table):
+                    _inplace_table_filter(lgdo, data[lgdo], obj_list)
+
+                store.write(data[lgdo], lgdo, output, wo_mode="append")
+
+    if lgdo_structs != {}:
+        output_file = store.gimme_file(output, mode="a")
+        for struct, struct_dtype in lgdo_structs.items():
+            msg = f"reset datatype of struct {struct} to {struct_dtype}"
+            log.debug(msg)
+
+            output_file[struct].attrs["datatype"] = struct_dtype
+        output_file.close()