legend-pydataobj 1.11.6__tar.gz → 1.11.7__tar.gz

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.11.6
+Version: 1.11.7
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration
@@ -726,6 +726,7 @@ Requires-Dist: pylegendtestdata; extra == "test"
 Requires-Dist: pytest>=6.0; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: dbetto; extra == "test"
+Dynamic: license-file
 
 # legend-pydataobj
 

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/pyproject.toml

@@ -81,7 +81,7 @@ test = [
 
 [project.scripts]
 lh5ls = "lgdo.cli:lh5ls"
-lh5concat = "lgdo.cli:lh5concat"
+lh5concat = "lgdo.cli:lh5concat_cli"
 
 [tool.setuptools]
 include-package-data = true

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/src/legend_pydataobj.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.11.6
+Version: 1.11.7
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration
@@ -726,6 +726,7 @@ Requires-Dist: pylegendtestdata; extra == "test"
 Requires-Dist: pytest>=6.0; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Requires-Dist: dbetto; extra == "test"
+Dynamic: license-file
 
 # legend-pydataobj
 

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/src/legend_pydataobj.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ src/lgdo/compression/radware.py
 src/lgdo/compression/utils.py
 src/lgdo/compression/varlen.py
 src/lgdo/lh5/__init__.py
+src/lgdo/lh5/concat.py
 src/lgdo/lh5/core.py
 src/lgdo/lh5/datatype.py
 src/lgdo/lh5/exceptions.py
@@ -68,6 +69,7 @@ tests/compression/test_uleb128_zigzag_diff.py
 tests/compression/sigcompress/LDQTA_r117_20200110T105115Z_cal_geds_raw-0.dat
 tests/compression/sigcompress/special-wf-clipped.dat
 tests/lh5/conftest.py
+tests/lh5/test_concat.py
 tests/lh5/test_core.py
 tests/lh5/test_exceptions.py
 tests/lh5/test_lh5_datatype.py

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/src/legend_pydataobj.egg-info/entry_points.txt

@@ -1,3 +1,3 @@
 [console_scripts]
-lh5concat = lgdo.cli:lh5concat
+lh5concat = lgdo.cli:lh5concat_cli
 lh5ls = lgdo.cli:lh5ls

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/src/lgdo/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.11.6'
-__version_tuple__ = version_tuple = (1, 11, 6)
+__version__ = version = '1.11.7'
+__version_tuple__ = version_tuple = (1, 11, 7)

legend_pydataobj-1.11.7/src/lgdo/cli.py

@@ -0,0 +1,183 @@
+"""legend-pydataobj's command line interface utilities."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+
+from . import __version__, lh5
+from . import logging as lgdogging  # eheheh
+from .lh5.concat import lh5concat
+
+log = logging.getLogger(__name__)
+
+
+def lh5ls(args=None):
+    """:func:`.lh5.show` command line interface."""
+    parser = argparse.ArgumentParser(
+        prog="lh5ls", description="Inspect LEGEND HDF5 (LH5) file contents"
+    )
+
+    # global options
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="""Print legend-pydataobj version and exit""",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="""Increase the program verbosity""",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="""Increase the program verbosity to maximum""",
+    )
+
+    parser.add_argument(
+        "lh5_file",
+        help="""Input LH5 file""",
+    )
+    parser.add_argument("lh5_group", nargs="?", help="""LH5 group.""", default="/")
+    parser.add_argument(
+        "--attributes", "-a", action="store_true", help="""Print HDF5 attributes too"""
+    )
+    parser.add_argument(
+        "--depth",
+        "-d",
+        type=int,
+        default=None,
+        help="""Maximum tree depth of groups to print""",
+    )
+    parser.add_argument(
+        "--detail",
+        action="store_true",
+        help="""Print details about datasets""",
+    )
+
+    args = parser.parse_args(args)
+
+    if args.verbose:
+        lgdogging.setup(logging.DEBUG)
+    elif args.debug:
+        lgdogging.setup(logging.DEBUG, logging.root)
+    else:
+        lgdogging.setup()
+
+    if args.version:
+        print(__version__)  # noqa: T201
+        sys.exit()
+
+    lh5.show(
+        args.lh5_file,
+        args.lh5_group,
+        attrs=args.attributes,
+        depth=args.depth,
+        detail=args.detail,
+    )
+
+
+def lh5concat_cli(args=None):
+    """Command line interface for concatenating array-like LGDOs in LH5 files."""
+    parser = argparse.ArgumentParser(
+        prog="lh5concat",
+        description="""
+Concatenate LGDO Arrays, VectorOfVectors and Tables in LH5 files.
+
+Examples
+--------
+
+Concatenate all eligible objects in file{1,2}.lh5 into concat.lh5:
+
+  $ lh5concat -o concat.lh5 file1.lh5 file2.lh5
+
+Include only the /data/table1 Table:
+
+  $ lh5concat -o concat.lh5 -i /data/table1/* file1.lh5 file2.lh5
+
+Exclude the /data/table1/col1 Table column:
+
+  $ lh5concat -o concat.lh5 -e /data/table1/col1 file1.lh5 file2.lh5
+        """,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    # global options
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="""Print legend-pydataobj version and exit""",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="""Increase the program verbosity""",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="""Increase the program verbosity to maximum""",
+    )
+
+    parser.add_argument(
+        "lh5_file",
+        nargs="+",
+        help="""Input LH5 files""",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        help="""Output file""",
+        default="lh5concat-output.lh5",
+    )
+    parser.add_argument(
+        "--overwrite",
+        "-w",
+        action="store_true",
+        help="""Overwrite output file""",
+    )
+    parser.add_argument(
+        "--include",
+        "-i",
+        help="""Regular expression (fnmatch style) for object names that should
+        be concatenated. To include full tables, you need to explicitly include
+        all its columns with e.g. '/path/to/table/*'. The option can be passed
+        multiple times to provide a list of patterns.
+        """,
+        action="append",
+        default=None,
+    )
+    parser.add_argument(
+        "--exclude",
+        "-e",
+        help="""List of object names that should be excluded. Takes priority
+        over --include. See --include help for more details.
+        """,
+        action="append",
+        default=None,
+    )
+
+    args = parser.parse_args(args)
+
+    if args.verbose:
+        lgdogging.setup(logging.INFO, log)
+    elif args.debug:
+        lgdogging.setup(logging.DEBUG, logging.root)
+    else:
+        lgdogging.setup()
+
+    if args.version:
+        print(__version__)  # noqa: T201
+        sys.exit()
+
+    lh5concat(
+        lh5_files=args.lh5_file,
+        overwrite=args.overwrite,
+        output=args.output,
+        include_list=args.include,
+        exclude_list=args.exclude,
+    )

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/src/lgdo/lh5/__init__.py

@@ -18,6 +18,7 @@ __all__ = [
     "DEFAULT_HDF5_SETTINGS",
     "LH5Iterator",
     "LH5Store",
+    "concat",
     "load_dfs",
     "load_nda",
     "ls",

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/src/lgdo/lh5/_serializers/read/utils.py

@@ -34,7 +34,7 @@ def build_field_mask(field_mask: Mapping[str, bool] | Collection[str]) -> defaul
             default = not field_mask[next(iter(field_mask.keys()))]
         return defaultdict(lambda: default, field_mask)
     if isinstance(field_mask, (list, tuple, set)):
-        return defaultdict(bool, {field: True for field in field_mask})
+        return defaultdict(bool, dict.fromkeys(field_mask, True))
     if isinstance(field_mask, defaultdict):
         return field_mask
     msg = "bad field_mask type"

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/src/lgdo/lh5/_serializers/read/vector_of_vectors.py

@@ -123,7 +123,7 @@ def _h5_read_vector_of_vectors(
                 )
                 msg = (
                     f"cumulative_length non-increasing between entries "
-                    f"{start_row} and {start_row+n_rows_read}"
+                    f"{start_row} and {start_row + n_rows_read}"
                 )
                 raise LH5DecodeError(msg, fname, oname)
 

legend_pydataobj-1.11.7/src/lgdo/lh5/concat.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+import fnmatch
+import logging
+
+from lgdo.lh5 import LH5Iterator
+
+from .. import Array, Scalar, Struct, Table, VectorOfVectors, lh5
+
+log = logging.getLogger(__name__)
+
+
+def _get_obj_list(
+    lh5_files: list, include_list: list | None = None, exclude_list: list | None = None
+) -> list[str]:
+    """Extract a list of lh5 objects to concatenate.
+
+    Parameters
+    ----------
+    lh5_files
+        list of input files to concatenate.
+    include_list
+        patterns for tables to include.
+    exclude_list
+        patterns for tables to exclude.
+
+    """
+    file0 = lh5_files[0]
+    obj_list_full = set(lh5.ls(file0, recursive=True))
+
+    # let's remove objects with nested LGDOs inside
+    to_remove = set()
+    for name in obj_list_full:
+        if len(fnmatch.filter(obj_list_full, f"{name}/*")) > 1:
+            to_remove.add(name)
+    obj_list_full -= to_remove
+
+    obj_list = set()
+    # now first remove excluded stuff
+    if exclude_list is not None:
+        for exc in exclude_list:
+            obj_list_full -= set(fnmatch.filter(obj_list_full, exc.strip("/")))
+
+    # then make list of included, based on latest list
+    if include_list is not None:
+        for inc in include_list:
+            obj_list |= set(fnmatch.filter(obj_list_full, inc.strip("/")))
+    else:
+        obj_list = obj_list_full
+
+    # sort
+    return sorted(obj_list)
+
+
+def _get_lgdos(file, obj_list):
+    """Get name of LGDO objects."""
+
+    store = lh5.LH5Store()
+    h5f0 = store.gimme_file(file)
+
+    lgdos = []
+    lgdo_structs = {}
+
+    # loop over object list in the first file
+    for name in obj_list:
+        # now loop over groups starting from root
+        current = ""
+        for item in name.split("/"):
+            current = f"{current}/{item}".strip("/")
+
+            if current in lgdos:
+                break
+
+            # not even an LGDO (i.e. a plain HDF5 group)!
+            if "datatype" not in h5f0[current].attrs:
+                continue
+
+            # read as little as possible
+            obj, _ = store.read(current, h5f0, n_rows=1)
+            if isinstance(obj, (Table, Array, VectorOfVectors)):
+                lgdos.append(current)
+
+            elif isinstance(obj, Struct):
+                # structs might be used in a "group-like" fashion (i.e. they might only
+                # contain array-like objects).
+                # note: handle after handling tables, as tables also satisfy this check.
+                lgdo_structs[current] = obj.attrs["datatype"]
+                continue
+
+            elif isinstance(obj, Scalar):
+                msg = f"cannot concat scalar field {current}"
+                log.warning(msg)
+
+            break
+
+    msg = f"first-level, array-like objects: {lgdos}"
+    log.info(msg)
+
+    msg = f"nested structs: {lgdo_structs}"
+    log.info(msg)
+
+    h5f0.close()
+
+    if lgdos == []:
+        msg = "did not find any field to concatenate, exit"
+        raise RuntimeError(msg)
+
+    return lgdos, lgdo_structs
+
+
+def _inplace_table_filter(name, table, obj_list):
+    """filter objects nested in this LGDO"""
+    skm = fnmatch.filter(obj_list, f"{name}/*")
+    kept = {it.removeprefix(name).strip("/").split("/")[0] for it in skm}
+
+    # now remove fields
+    for k in list(table.keys()):
+        if k not in kept:
+            table.remove_column(k)
+
+    msg = f"fields left in table '{name}': {table.keys()}"
+    log.debug(msg)
+
+    # recurse!
+    for k2, v2 in table.items():
+        if not isinstance(v2, Table):
+            continue
+
+        _inplace_table_filter(f"{name}/{k2}", v2, obj_list)
+
+
+def _remove_nested_fields(lgdos: dict, obj_list: list):
+    """Remove (nested) table fields based on obj_list."""
+
+    for key, val in lgdos.items():
+        if not isinstance(val, Table):
+            continue
+
+        _inplace_table_filter(key, val, obj_list)
+
+
+def _slice(obj, n_rows):
+    ak_obj = obj.view_as("ak")[:n_rows]
+    obj_type = type(obj)
+    return obj_type(ak_obj)
+
+
+def lh5concat(
+    lh5_files: list,
+    output: str,
+    overwrite: bool = False,
+    *,
+    include_list: list | None = None,
+    exclude_list: list | None = None,
+) -> None:
+    """Concatenate LGDO Arrays, VectorOfVectors and Tables in LH5 files.
+
+    Parameters
+    ----------
+    lh5_files
+        list of input files to concatenate.
+    output
+        path to the output file
+    include_list
+        patterns for tables to include.
+    exclude_list
+        patterns for tables to exclude.
+    """
+
+    if len(lh5_files) < 2:
+        msg = "you must provide at least two input files"
+        raise RuntimeError(msg)
+
+    # determine list of objects by recursively ls'ing first file
+    obj_list = _get_obj_list(
+        lh5_files, include_list=include_list, exclude_list=exclude_list
+    )
+
+    msg = f"objects matching include patterns {include_list} in {lh5_files[0]}: {obj_list}"
+    log.info(msg)
+
+    lgdos, lgdo_structs = _get_lgdos(lh5_files[0], obj_list)
+    first_done = False
+    store = lh5.LH5Store()
+
+    # loop over lgdo objects
+    for lgdo in lgdos:
+        # iterate over the files
+        for lh5_obj, _, n_rows in LH5Iterator(lh5_files, lgdo):
+            data = {lgdo: _slice(lh5_obj, n_rows)}
+
+            # remove the nested fields
+            _remove_nested_fields(data, obj_list)
+
+            if first_done is False:
+                msg = f"creating output file {output}"
+                log.info(msg)
+
+                store.write(
+                    data[lgdo],
+                    lgdo,
+                    output,
+                    wo_mode="overwrite_file"
+                    if (overwrite and not first_done)
+                    else "write_safe",
+                )
+                first_done = True
+
+            else:
+                msg = f"appending to {output}"
+                log.info(msg)
+
+                if isinstance(data[lgdo], Table):
+                    _inplace_table_filter(lgdo, data[lgdo], obj_list)
+
+                store.write(data[lgdo], lgdo, output, wo_mode="append")
+
+    if lgdo_structs != {}:
+        output_file = store.gimme_file(output, mode="a")
+        for struct, struct_dtype in lgdo_structs.items():
+            msg = f"reset datatype of struct {struct} to {struct_dtype}"
+            log.debug(msg)
+
+            output_file[struct].attrs["datatype"] = struct_dtype
+        output_file.close()

{legend_pydataobj-1.11.6 → legend_pydataobj-1.11.7}/src/lgdo/types/vectorofvectors.py

@@ -138,7 +138,7 @@ class VectorOfVectors(LGDO):
             # FIXME: have to copy the buffers, otherwise self will not own the
             # data and self.resize() will fail. Is it possible to avoid this?
             flattened_data = np.copy(
-                container.pop(f"node{data.ndim-1}-data", np.empty(0, dtype=dtype))
+                container.pop(f"node{data.ndim - 1}-data", np.empty(0, dtype=dtype))
             )
 
             # if user-provided dtype is different than dtype from Awkward, cast

legend_pydataobj-1.11.6/tests/test_cli.py → legend_pydataobj-1.11.7/tests/lh5/test_concat.py

@@ -1,21 +1,13 @@
 from __future__ import annotations
 
 import numpy as np
+import pytest
 
-from lgdo import cli, lh5, types
+from lgdo import lh5, types
+from lgdo.lh5 import concat
 
 
-def test_lh5ls(lgnd_test_data):
-    cli.lh5ls(
-        [
-            "-a",
-            lgnd_test_data.get_path("lh5/LDQTA_r117_20200110T105115Z_cal_geds_raw.lh5"),
-            "geds/raw",
-        ]
-    )
-
-
-def test_lh5concat(lgnd_test_data, tmptestdir):
+def test_concat(lgnd_test_data, tmptestdir):
     infile1 = lgnd_test_data.get_path(
         "lh5/prod-ref-l200/generated/tier/raw/phy/p03/r001/l200-p03-r001-phy-20230322T160139Z-tier_raw.lh5"
     )
@@ -23,7 +15,7 @@ def test_lh5concat(lgnd_test_data, tmptestdir):
         "lh5/prod-ref-l200/generated/tier/raw/phy/p03/r001/l200-p03-r001-phy-20230322T170202Z-tier_raw.lh5"
     )
     outfile = f"{tmptestdir}/out.lh5"
-    cli.lh5concat(["--output", outfile, "--", infile1, infile2])
+    concat.lh5concat(output=outfile, lh5_files=[infile1, infile2], overwrite=True)
 
     assert lh5.ls(outfile) == [
         "ch1057600",
@@ -88,25 +80,23 @@ def test_lh5concat(lgnd_test_data, tmptestdir):
         assert np.array_equal(tbl.tracelist[i], tbl2.tracelist[i - 10])
         assert np.array_equal(tbl.waveform.values[i], tbl2.waveform.values[i - 10])
 
-    arg_list = [
-        "--verbose",
-        "--overwrite",
-        "--output",
-        outfile,
-        "--include",
-        "ch1057600/raw/waveform/*",
-        "--",
-        infile1,
-        infile2,
-    ]
-
-    cli.lh5concat(arg_list)
+    concat.lh5concat(
+        output=outfile,
+        lh5_files=[infile1, infile2],
+        overwrite=True,
+        include_list=["ch1057600/raw/waveform/*"],
+    )
     assert lh5.ls(outfile) == [
         "ch1057600",
     ]
 
-    arg_list[5] = "ch1057600/raw/waveform/values"
-    cli.lh5concat(arg_list)
+    concat.lh5concat(
+        output=outfile,
+        lh5_files=[infile1, infile2],
+        overwrite=True,
+        include_list=["ch1057600/raw/waveform/values"],
+    )
+
     assert lh5.ls(outfile, "ch1057600/raw/waveform/") == [
         "ch1057600/raw/waveform/values",
     ]
@@ -114,10 +104,12 @@ def test_lh5concat(lgnd_test_data, tmptestdir):
     tbl, _ = store.read("ch1057600/raw", outfile)
     assert isinstance(tbl, types.Table)
 
-    arg_list[4] = "--exclude"
-    arg_list[5] = "ch1057600/raw/waveform/values"
-
-    cli.lh5concat(arg_list)
+    concat.lh5concat(
+        output=outfile,
+        lh5_files=[infile1, infile2],
+        overwrite=True,
+        exclude_list=["ch1057600/raw/waveform/values"],
+    )
     assert lh5.ls(outfile) == [
         "ch1057600",
         "ch1059201",
@@ -162,8 +154,12 @@ def test_lh5concat(lgnd_test_data, tmptestdir):
     store.write(struct2, "stp", infile2, wo_mode="overwrite_file")
 
     outfile = f"{tmptestdir}/concat_test_struct_out.lh5"
-    cli.lh5concat(["--output", outfile, "--", infile1, infile2])
+    concat.lh5concat(output=outfile, lh5_files=[infile1, infile2], overwrite=True)
 
     out_stp = store.read("stp", outfile)[0]
     assert out_stp.attrs["datatype"] == "struct{x}"
     assert np.all(out_stp.x["col"].nda == np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]))
+
+    # check for reasonable failures
+    with pytest.raises(RuntimeError):
+        concat.lh5concat(output=outfile, lh5_files=[infile1], overwrite=True)

legend_pydataobj-1.11.7/tests/test_cli.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from lgdo import cli, lh5
+
+
+def test_lh5ls(lgnd_test_data):
+    cli.lh5ls(
+        [
+            "-a",
+            lgnd_test_data.get_path("lh5/LDQTA_r117_20200110T105115Z_cal_geds_raw.lh5"),
+            "geds/raw",
+        ]
+    )
+
+
+def test_lh5concat(lgnd_test_data, tmptestdir):
+    infile1 = lgnd_test_data.get_path(
+        "lh5/prod-ref-l200/generated/tier/raw/phy/p03/r001/l200-p03-r001-phy-20230322T160139Z-tier_raw.lh5"
+    )
+    infile2 = lgnd_test_data.get_path(
+        "lh5/prod-ref-l200/generated/tier/raw/phy/p03/r001/l200-p03-r001-phy-20230322T170202Z-tier_raw.lh5"
+    )
+    outfile = f"{tmptestdir}/out.lh5"
+
+    cli.lh5concat_cli(
+        ["--output", outfile, "--debug", "--overwrite", "--", infile1, infile2]
+    )
+
+    assert lh5.ls(outfile) == [
+        "ch1057600",
+        "ch1059201",
+        "ch1062405",
+        "ch1084803",
+        "ch1084804",
+        "ch1121600",
+    ]