legend-pydataobj 1.5.1__py3-none-any.whl → 1.6.0__py3-none-any.whl

{legend_pydataobj-1.5.1.dist-info → legend_pydataobj-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: legend_pydataobj
-Version: 1.5.1
+Version: 1.6.0
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

legend_pydataobj-1.6.0.dist-info/RECORD

@@ -0,0 +1,54 @@
+lgdo/__init__.py,sha256=nv9kORuX2FCA6rQLbH959E0fuGMfZvHb0H5uyrLr2WI,3046
+lgdo/_version.py,sha256=Ry70pc5l-IBhT9gahlkNwZPp4g0CzVEWqsat9H-UASY,411
+lgdo/cli.py,sha256=hHc0Cz4ZXEOwo55oIukHAhHUkw09ePr4m3sO7eUbWHA,8018
+lgdo/lgdo_utils.py,sha256=6a2YWEwpyEMXlAyTHZMO01aqxy6SxJzPZkGNWKNWuS0,2567
+lgdo/lh5_store.py,sha256=xHwzbKNueEtFwScxrgfvCo2_bWKS6j7ojrpeF9kQflc,8483
+lgdo/logging.py,sha256=82wIOj7l7xr3WYyeHdpSXbbjzHJsy-uRyKYUYx2vMfQ,1003
+lgdo/units.py,sha256=nbJ0JTNqlhHUXiBXT3k6qhRpSfMk5_9yW7EeC0dhMuQ,151
+lgdo/utils.py,sha256=9t_GYdB8aQhZ4Vz6ujmASzwCgTuP7ZdINtPTVPyIR6E,3661
+lgdo/compression/__init__.py,sha256=gqbdx4NnpCcW-C7kUXV-hVUZFiNlbCwIbs3uzFe4AFE,1127
+lgdo/compression/base.py,sha256=82cQJujfvoAOKBFx761dEcx_xM02TBCBBuBo6i78tuI,838
+lgdo/compression/generic.py,sha256=tF3UhLJbUDcovLxpIzgQRxFSjZ5Fz3uDRy9kI4mFntQ,2515
+lgdo/compression/radware.py,sha256=VbKAvi18h48Fz-ZxMEg64yD1ezaw1NkMZazxurdyMmc,24015
+lgdo/compression/utils.py,sha256=W2RkBrxPpXlat84dnU9Ad7d_tTws0irtGl7O1dNWjnk,1140
+lgdo/compression/varlen.py,sha256=6ZZUItyoOfygDdE0DyoISeFZfqdbH6xl7T0eclfarzg,15127
+lgdo/lh5/__init__.py,sha256=KzWF6HI-6N1NqQUm8LAxMmDbg0rgRY4DAaJ2s7w2tLM,811
+lgdo/lh5/core.py,sha256=ap7K6GrbjdF4n-hvPPVeXOk76OF0S4uBclbP3mNHk04,10780
+lgdo/lh5/datatype.py,sha256=6OfLmoqrfPxZLSmGXIF9xFIwyY6u0yoXT_lr1k_5NcM,1619
+lgdo/lh5/exceptions.py,sha256=QWStQD27Qrm4oYs5Z3UAIoq4y7X-f_Z6QWCBCH0DXwE,1006
+lgdo/lh5/iterator.py,sha256=eqH9a_ZjEhgqJUZbMj36jXK_1Xbx86450DVw7LHNB3Y,12369
+lgdo/lh5/store.py,sha256=sYX1harVGRyP0oq1LGq2qrFhorutkev9MOovwhzEWZ4,6670
+lgdo/lh5/tools.py,sha256=AH0RuaUEJ7Tfzb964KnuVdxCKrGqwNP8XDt2iq4829g,8386
+lgdo/lh5/utils.py,sha256=xSTv0obtsUxO38JpkPKd2FbzENlhuTvYR0aexFU1ZQc,6640
+lgdo/lh5/_serializers/__init__.py,sha256=7zvTmBdp-pqS0ium6cKKjEvcqIND-kBC7319G5wMq5Y,1213
+lgdo/lh5/_serializers/read/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lgdo/lh5/_serializers/read/array.py,sha256=DLguBkiVNXZuUk7LdalixA8uISx_SETVPXWk-26HYmk,933
+lgdo/lh5/_serializers/read/composite.py,sha256=M3RR4SEyFGtb4qsscJOtB-P4HhdMQw0_JDIWKkUOsKA,12066
+lgdo/lh5/_serializers/read/encoded.py,sha256=LTdSqEH5tJOdqcSCrMLWutYiQ4FwBFIPF7FDZ-iallw,3744
+lgdo/lh5/_serializers/read/ndarray.py,sha256=4vdgHwj_yIon1KvSiOkDSHq7CJhYtFcjjSHijsvp1aU,3481
+lgdo/lh5/_serializers/read/scalar.py,sha256=YwvA6kyNUh6H0kh2L7bzfgLkA8Et2dQFjp2nFnRmeGI,755
+lgdo/lh5/_serializers/read/utils.py,sha256=K_HDQ_H-vtbs_gEif1MTtFki4qh6lw-5HE7b-7_s-9Q,417
+lgdo/lh5/_serializers/read/vector_of_vectors.py,sha256=nlaoOuDPDQvZWslGm3YxyRkcFodfSBEXRKwY046VDvM,6318
+lgdo/lh5/_serializers/write/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lgdo/lh5/_serializers/write/array.py,sha256=Gosg8rOCH_2dRMj_oNSWyXuoYXDjy0OK--GCYWswR4U,2803
+lgdo/lh5/_serializers/write/composite.py,sha256=f3b4YeOoUr8y1wA7zsKEFT5mIwX8SD0MYQ40unMRyQc,8460
+lgdo/lh5/_serializers/write/scalar.py,sha256=gkcF2WVBR3aQYl0EynbVUocx4y3r8tvPfQYQJjkPvP4,643
+lgdo/lh5/_serializers/write/vector_of_vectors.py,sha256=mZuC7NIb-IkmJ9wgn37TTvFTLLAFR71iivrY4yiSJZM,2912
+lgdo/types/__init__.py,sha256=_cHZEpsh7OGsA243WBK2a2UmCp3NSA0MjXYf95y728k,771
+lgdo/types/array.py,sha256=sUxh1CNCaefrnybt5qdjmmMpVQa_RqFxUv1tJ_pyBbc,6537
+lgdo/types/arrayofequalsizedarrays.py,sha256=DOGJiTmc1QCdm7vLbE6uIRXoMPtt8uuCfmwQawgWf5s,4949
+lgdo/types/encoded.py,sha256=JW4U5ow7KLMzhKnmhdnxbC3SZJAs4bOEDZWKG4KY1uU,15293
+lgdo/types/fixedsizearray.py,sha256=7Fj4QS9ubaeEf2tM3HwjSs6AuG8hKSYaT6Hy7Y_VHdQ,1525
+lgdo/types/lgdo.py,sha256=UnJDi1emQYVgH_H29Vipfs4LelPopxG5pgZUu1eKOlw,2761
+lgdo/types/scalar.py,sha256=c5Es2vyDqyWTPV6mujzfIzMpC1jNWkEIcvYyWQUxH3Q,1933
+lgdo/types/struct.py,sha256=rLtyPthut2wjwiOqwntPrYjjt3uRQbTLWWjiewfKm20,3979
+lgdo/types/table.py,sha256=YL_71PalmGnh4YvaDlK0oY4aeypYRAETwJEkRHVOUu8,16887
+lgdo/types/vectorofvectors.py,sha256=9pKtPfvh0vQ-nmg68LUM8cMttoxTl0BG3QH5awQtSxo,24503
+lgdo/types/vovutils.py,sha256=7BWPP0BSj-92ifbCIUBcfqxG5-TS8uxujTyJJuDFI04,10302
+lgdo/types/waveformtable.py,sha256=f2tS4f1OEoYaTM5ldCX9zmw8iSISCT3t3wS1SrPdu_o,9901
+legend_pydataobj-1.6.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+legend_pydataobj-1.6.0.dist-info/METADATA,sha256=WuFjFye6W_q3YdhE5tHCQlcKNaOCml-TK7ItWAM_f34,44353
+legend_pydataobj-1.6.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+legend_pydataobj-1.6.0.dist-info/entry_points.txt,sha256=Uu5MTlppBZxB4QGlLv-oX8FqACWjAZDNii__TBDJwLQ,72
+legend_pydataobj-1.6.0.dist-info/top_level.txt,sha256=KyR-EUloqiXcQ62IWnzBmtInDtvsHl4q2ZJAZgTcLXE,5
+legend_pydataobj-1.6.0.dist-info/RECORD,,

{legend_pydataobj-1.5.1.dist-info → legend_pydataobj-1.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{legend_pydataobj-1.5.1.dist-info → legend_pydataobj-1.6.0.dist-info}/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+lh5concat = lgdo.cli:lh5concat
 lh5ls = lgdo.cli:lh5ls

lgdo/__init__.py

@@ -15,10 +15,12 @@ basic data object classes are:
   :attr:`nda` attribute.
 * :class:`.ArrayOfEqualSizedArrays`: multi-dimensional :class:`numpy.ndarray`.
   Access data via the :attr:`nda` attribute.
-* :class:`.VectorOfVectors`: a variable length array of variable length arrays.
-  Implemented as a pair of :class:`.Array`: :attr:`flattened_data` holding the
-  raw data, and :attr:`cumulative_length` whose ith element is the sum of the
-  lengths of the vectors with ``index <= i``
+* :class:`.VectorOfVectors`: an n-dimensional variable length array of variable
+  length arrays.  Implemented as a pair of datasets: :attr:`flattened_data`
+  holding the raw data (:class:`.Array` or :class:`.VectorOfVectors`, if the
+  vector dimension is greater than 2), and :attr:`cumulative_length` (always an
+  :class:`.Array`) whose i-th element is the sum of the lengths of the vectors
+  with ``index <= i``
 * :class:`.VectorOfEncodedVectors`: an array of variable length *encoded*
   arrays. Implemented as a :class:`.VectorOfVectors` :attr:`encoded_data`
   holding the encoded vectors and an :class:`.Array` :attr:`decoded_size`
@@ -37,6 +39,7 @@ is done via the class :class:`.lh5_store.LH5Store`. LH5 files can also be
 browsed easily in python like any `HDF5 <https://www.hdfgroup.org>`_ file using
 `h5py <https://www.h5py.org>`_.
 """
+
 from __future__ import annotations
 
 from ._version import version as __version__

lgdo/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.5.1'
-__version_tuple__ = version_tuple = (1, 5, 1)
+__version__ = version = '1.6.0'
+__version_tuple__ = version_tuple = (1, 6, 0)

lgdo/cli.py

@@ -1,16 +1,19 @@
 """legend-pydataobj's command line interface utilities."""
+
 from __future__ import annotations
 
 import argparse
+import fnmatch
 import logging
 import sys
 
-import lgdo
-import lgdo.logging
-from lgdo.lh5 import show
+from . import Array, Table, VectorOfVectors, __version__, lh5
+from . import logging as lgdogging  # eheheh
+
+log = logging.getLogger(__name__)
 
 
-def lh5ls():
+def lh5ls(args=None):
     """:func:`.lh5.show` command line interface."""
     parser = argparse.ArgumentParser(
         prog="lh5ls", description="Inspect LEGEND HDF5 (LH5) file contents"
@@ -18,7 +21,9 @@ def lh5ls():
 
     # global options
     parser.add_argument(
-        "--version", action="store_true", help="""Print lgdo version and exit"""
+        "--version",
+        action="store_true",
+        help="""Print legend-pydataobj version and exit""",
     )
     parser.add_argument(
         "--verbose",
@@ -34,7 +39,7 @@ def lh5ls():
 
     parser.add_argument(
         "lh5_file",
-        help="""Input LH5 file.""",
+        help="""Input LH5 file""",
     )
     parser.add_argument("lh5_group", nargs="?", help="""LH5 group.""", default="/")
     parser.add_argument(
@@ -48,17 +53,237 @@ def lh5ls():
         help="""Maximum tree depth of groups to print""",
     )
 
-    args = parser.parse_args()
+    args = parser.parse_args(args)
 
     if args.verbose:
-        lgdo.logging.setup(logging.DEBUG)
+        lgdogging.setup(logging.DEBUG)
     elif args.debug:
-        lgdo.logging.setup(logging.DEBUG, logging.root)
+        lgdogging.setup(logging.DEBUG, logging.root)
     else:
-        lgdo.logging.setup()
+        lgdogging.setup()
 
     if args.version:
-        print(lgdo.__version__)  # noqa: T201
+        print(__version__)  # noqa: T201
         sys.exit()
 
-    show(args.lh5_file, args.lh5_group, attrs=args.attributes, depth=args.depth)
+    lh5.show(args.lh5_file, args.lh5_group, attrs=args.attributes, depth=args.depth)
+
+
+def lh5concat(args=None):
+    """Command line interface for concatenating array-like LGDOs in LH5 files."""
+    parser = argparse.ArgumentParser(
+        prog="lh5concat",
+        description="""
+Concatenate LGDO Arrays, VectorOfVectors and Tables in LH5 files.
+
+Examples
+--------
+
+Concatenate all eligible objects in file{1,2}.lh5 into concat.lh5:
+
+  $ lh5concat -o concat.lh5 file1.lh5 file2.lh5
+
+Include only the /data/table1 Table:
+
+  $ lh5concat -o concat.lh5 -i /data/table1/* file1.lh5 file2.lh5
+
+Exclude the /data/table1/col1 Table column:
+
+  $ lh5concat -o concat.lh5 -e /data/table1/col1 file1.lh5 file2.lh5
+        """,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    # global options
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="""Print legend-pydataobj version and exit""",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="""Increase the program verbosity""",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="""Increase the program verbosity to maximum""",
+    )
+
+    parser.add_argument(
+        "lh5_file",
+        nargs="+",
+        help="""Input LH5 files""",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        help="""Output file""",
+        default="lh5concat-output.lh5",
+    )
+    parser.add_argument(
+        "--overwrite",
+        "-w",
+        action="store_true",
+        help="""Overwrite output file""",
+    )
+    parser.add_argument(
+        "--include",
+        "-i",
+        help="""Regular expression (fnmatch style) for object names that should
+        be concatenated. To include full tables, you need to explicitly include
+        all its columns with e.g. '/path/to/table/*'. The option can be passed
+        multiple times to provide a list of patterns.
+        """,
+        action="append",
+        default=None,
+    )
+    parser.add_argument(
+        "--exclude",
+        "-e",
+        help="""List of object names that should be excluded. Takes priority
+        over --include. See --include help for more details.
+        """,
+        action="append",
+        default=None,
+    )
+
+    args = parser.parse_args(args)
+
+    if args.verbose:
+        lgdogging.setup(logging.INFO, log)
+    elif args.debug:
+        lgdogging.setup(logging.DEBUG, logging.root)
+    else:
+        lgdogging.setup()
+
+    if args.version:
+        print(__version__)  # noqa: T201
+        sys.exit()
+
+    if len(args.lh5_file) < 2:
+        msg = "you must provide at least two input files"
+        raise RuntimeError(msg)
+
+    # determine list of objects by recursively ls'ing first file
+    file0 = args.lh5_file[0]
+    obj_list_full = set(lh5.ls(file0, recursive=True))
+
+    # let's remove objects with nested LGDOs inside
+    to_remove = set()
+    for name in obj_list_full:
+        if len(fnmatch.filter(obj_list_full, f"{name}/*")) > 1:
+            to_remove.add(name)
+    obj_list_full -= to_remove
+
+    obj_list = set()
+    # now first remove excluded stuff
+    if args.exclude is not None:
+        for exc in args.exclude:
+            obj_list_full -= set(fnmatch.filter(obj_list_full, exc.strip("/")))
+
+    # then make list of included, based on latest list
+    if args.include is not None:
+        for inc in args.include:
+            obj_list |= set(fnmatch.filter(obj_list_full, inc.strip("/")))
+    else:
+        obj_list = obj_list_full
+
+    # sort
+    obj_list = sorted(obj_list)
+
+    msg = f"objects matching include patterns {args.include} in {file0}: {obj_list}"
+    log.debug(msg)
+
+    # 1. read first valid lgdo from left to right
+    store = lh5.LH5Store()
+    h5f0 = store.gimme_file(file0)
+    lgdos = {}
+    # loop over object list in the first file
+    for name in obj_list:
+        # now loop over groups starting from root
+        current = ""
+        for item in name.split("/"):
+            current = f"{current}/{item}".strip("/")
+
+            if current in lgdos:
+                break
+
+            # not even an LGDO!
+            if "datatype" not in h5f0[current].attrs:
+                continue
+
+            # read as little as possible
+            obj, _ = store.read(current, h5f0, n_rows=1)
+            if isinstance(obj, (Table, Array, VectorOfVectors)):
+                # read all!
+                obj, _ = store.read(current, h5f0)
+                lgdos[current] = obj
+
+            break
+
+    msg = f"first-level, array-like objects: {lgdos.keys()}"
+    log.debug(msg)
+
+    h5f0.close()
+
+    # 2. remove (nested) table fields based on obj_list
+
+    def _inplace_table_filter(name, table, obj_list):
+        # filter objects nested in this LGDO
+        skm = fnmatch.filter(obj_list, f"{name}/*")
+        kept = {it.removeprefix(name).strip("/").split("/")[0] for it in skm}
+
+        # now remove fields
+        for k in list(table.keys()):
+            if k not in kept:
+                table.remove_column(k)
+
+        msg = f"fields left in table '{name}': {table.keys()}"
+        log.debug(msg)
+
+        # recurse!
+        for k2, v2 in table.items():
+            if not isinstance(v2, Table):
+                continue
+
+            _inplace_table_filter(f"{name}/{k2}", v2, obj_list)
+
+    for key, val in lgdos.items():
+        if not isinstance(val, Table):
+            continue
+
+        _inplace_table_filter(key, val, obj_list)
+
+    # 3. write to output file
+    msg = f"creating output file {args.output}"
+    log.info(msg)
+
+    first_done = False
+    for name, obj in lgdos.items():
+        store.write(
+            obj,
+            name,
+            args.output,
+            wo_mode="overwrite_file"
+            if (args.overwrite and not first_done)
+            else "write_safe",
+        )
+
+        first_done = True
+
+    # 4. loop over rest of files/names and write-append
+
+    for file in args.lh5_file[1:]:
+        msg = f"appending file {file} to {args.output}"
+        log.info(msg)
+
+        for name in lgdos:
+            obj, _ = store.read(name, file)
+            # need to remove nested LGDOs from obj too before appending
+            if isinstance(obj, Table):
+                _inplace_table_filter(name, obj, obj_list)
+
+            store.write(obj, name, args.output, wo_mode="append")

lgdo/compression/__init__.py

@@ -21,6 +21,7 @@ interface for encoding/decoding :class:`~.lgdo.LGDO`\ s.
 >>> enc_wft = compression.encode(wftable, RadwareSigcompress(codec_shift=-23768)
 >>> compression.decode(enc_wft) # == wftbl
 """
+
 from __future__ import annotations
 
 from .base import WaveformCodec

lgdo/lh5/__init__.py

@@ -4,18 +4,26 @@ is done via the class :class:`.store.LH5Store`. LH5 files can also be
 browsed easily in python like any `HDF5 <https://www.hdfgroup.org>`_ file using
 `h5py <https://www.h5py.org>`_.
 """
+
 from __future__ import annotations
 
+from ._serializers.write.array import DEFAULT_HDF5_SETTINGS
+from .core import read, read_as, write
 from .iterator import LH5Iterator
 from .store import LH5Store
-from .tools import load_dfs, load_nda, ls, read_as, show
+from .tools import load_dfs, load_nda, ls, show
+from .utils import read_n_rows
 
 __all__ = [
+    "DEFAULT_HDF5_SETTINGS",
     "LH5Iterator",
     "LH5Store",
     "load_dfs",
     "load_nda",
+    "read",
+    "write",
     "read_as",
     "ls",
+    "read_n_rows",
     "show",
 ]

lgdo/lh5/_serializers/__init__.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from .read.array import (
+    _h5_read_array,
+    _h5_read_array_of_equalsized_arrays,
+    _h5_read_fixedsize_array,
+    _h5_read_ndarray,
+)
+from .read.composite import (
+    _h5_read_lgdo,
+    _h5_read_struct,
+    _h5_read_table,
+)
+from .read.encoded import (
+    _h5_read_array_of_encoded_equalsized_arrays,
+    _h5_read_vector_of_encoded_vectors,
+)
+from .read.scalar import _h5_read_scalar
+from .read.vector_of_vectors import _h5_read_vector_of_vectors
+from .write.array import _h5_write_array
+from .write.composite import _h5_write_lgdo, _h5_write_struct
+from .write.scalar import _h5_write_scalar
+from .write.vector_of_vectors import _h5_write_vector_of_vectors
+
+__all__ = [
+    "_h5_read_lgdo",
+    "_h5_read_vector_of_vectors",
+    "_h5_read_ndarray",
+    "_h5_read_array",
+    "_h5_read_encoded_array",
+    "_h5_read_fixedsize_array",
+    "_h5_read_array_of_equalsized_arrays",
+    "_h5_read_struct",
+    "_h5_read_table",
+    "_h5_read_scalar",
+    "_h5_read_array_of_encoded_equalsized_arrays",
+    "_h5_read_vector_of_encoded_vectors",
+    "_h5_write_scalar",
+    "_h5_write_array",
+    "_h5_write_vector_of_vectors",
+    "_h5_write_struct",
+    "_h5_write_lgdo",
+]

lgdo/lh5/_serializers/read/array.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import logging
+
+from ....types import Array, ArrayOfEqualSizedArrays, FixedSizeArray
+from . import utils
+from .ndarray import _h5_read_ndarray
+
+log = logging.getLogger(__name__)
+
+
+def _h5_read_array_generic(type_, name, h5f, **kwargs):
+    nda, attrs, n_rows_to_read = _h5_read_ndarray(name, h5f, **kwargs)
+
+    obj_buf = kwargs["obj_buf"]
+
+    if obj_buf is None:
+        return type_(nda=nda, attrs=attrs), n_rows_to_read
+
+    utils.check_obj_buf_attrs(obj_buf.attrs, attrs, h5f, name)
+
+    return obj_buf, n_rows_to_read
+
+
+def _h5_read_array(name, h5f, **kwargs):
+    return _h5_read_array_generic(Array, name, h5f, **kwargs)
+
+
+def _h5_read_fixedsize_array(name, h5f, **kwargs):
+    return _h5_read_array_generic(FixedSizeArray, name, h5f, **kwargs)
+
+
+def _h5_read_array_of_equalsized_arrays(name, h5f, **kwargs):
+    return _h5_read_array_generic(ArrayOfEqualSizedArrays, name, h5f, **kwargs)