legend-pydataobj 1.5.0a5__tar.gz → 1.6.0__tar.gz

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: legend_pydataobj
-Version: 1.5.0a5
+Version: 1.6.0
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/pyproject.toml

@@ -79,6 +79,7 @@ test = [
 
 [project.scripts]
 lh5ls = "lgdo.cli:lh5ls"
+lh5concat = "lgdo.cli:lh5concat"
 
 [tool.setuptools]
 include-package-data = true
@@ -144,7 +145,6 @@ isort.required-imports = ["from __future__ import annotations"]
 "tests/**" = ["T20"]
 "noxfile.py" = ["T20"]
 
-
 [tool.pylint]
 py-version = "3.8"
 ignore-paths = [".*/_version.py"]
@@ -157,3 +157,14 @@ messages_control.disable = [
   "missing-module-docstring",
   "wrong-import-position",
 ]
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
+xfail_strict = true
+filterwarnings = ["error", 'ignore:\nPyarrow:DeprecationWarning']
+log_cli_level = "info"
+testpaths = "tests"
+
+[tool.codespell]
+ignore-words-list = "nd, unparseable, compiletime, crate"

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/src/legend_pydataobj.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: legend_pydataobj
-Version: 1.5.0a5
+Version: 1.6.0
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/src/legend_pydataobj.egg-info/SOURCES.txt

@@ -23,10 +23,27 @@ src/lgdo/compression/radware.py
 src/lgdo/compression/utils.py
 src/lgdo/compression/varlen.py
 src/lgdo/lh5/__init__.py
+src/lgdo/lh5/core.py
+src/lgdo/lh5/datatype.py
+src/lgdo/lh5/exceptions.py
 src/lgdo/lh5/iterator.py
 src/lgdo/lh5/store.py
 src/lgdo/lh5/tools.py
 src/lgdo/lh5/utils.py
+src/lgdo/lh5/_serializers/__init__.py
+src/lgdo/lh5/_serializers/read/__init__.py
+src/lgdo/lh5/_serializers/read/array.py
+src/lgdo/lh5/_serializers/read/composite.py
+src/lgdo/lh5/_serializers/read/encoded.py
+src/lgdo/lh5/_serializers/read/ndarray.py
+src/lgdo/lh5/_serializers/read/scalar.py
+src/lgdo/lh5/_serializers/read/utils.py
+src/lgdo/lh5/_serializers/read/vector_of_vectors.py
+src/lgdo/lh5/_serializers/write/__init__.py
+src/lgdo/lh5/_serializers/write/array.py
+src/lgdo/lh5/_serializers/write/composite.py
+src/lgdo/lh5/_serializers/write/scalar.py
+src/lgdo/lh5/_serializers/write/vector_of_vectors.py
 src/lgdo/types/__init__.py
 src/lgdo/types/array.py
 src/lgdo/types/arrayofequalsizedarrays.py
@@ -37,6 +54,7 @@ src/lgdo/types/scalar.py
 src/lgdo/types/struct.py
 src/lgdo/types/table.py
 src/lgdo/types/vectorofvectors.py
+src/lgdo/types/vovutils.py
 src/lgdo/types/waveformtable.py
 tests/conftest.py
 tests/test_cli.py
@@ -48,9 +66,14 @@ tests/compression/test_str2wfcodec.py
 tests/compression/test_uleb128_zigzag_diff.py
 tests/compression/sigcompress/LDQTA_r117_20200110T105115Z_cal_geds_raw-0.dat
 tests/compression/sigcompress/special-wf-clipped.dat
+tests/lh5/conftest.py
+tests/lh5/test_core.py
+tests/lh5/test_lh5_datatype.py
 tests/lh5/test_lh5_iterator.py
 tests/lh5/test_lh5_store.py
+tests/lh5/test_lh5_tools.py
 tests/lh5/test_lh5_utils.py
+tests/lh5/test_lh5_write.py
 tests/types/test_array.py
 tests/types/test_arrayofequalsizedarrays.py
 tests/types/test_encoded.py
@@ -61,4 +84,5 @@ tests/types/test_struct.py
 tests/types/test_table.py
 tests/types/test_table_eval.py
 tests/types/test_vectorofvectors.py
+tests/types/test_vovutils.py
 tests/types/test_waveformtable.py

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/src/legend_pydataobj.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+lh5concat = lgdo.cli:lh5concat
 lh5ls = lgdo.cli:lh5ls

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/src/lgdo/__init__.py

@@ -15,10 +15,12 @@ basic data object classes are:
   :attr:`nda` attribute.
 * :class:`.ArrayOfEqualSizedArrays`: multi-dimensional :class:`numpy.ndarray`.
   Access data via the :attr:`nda` attribute.
-* :class:`.VectorOfVectors`: a variable length array of variable length arrays.
-  Implemented as a pair of :class:`.Array`: :attr:`flattened_data` holding the
-  raw data, and :attr:`cumulative_length` whose ith element is the sum of the
-  lengths of the vectors with ``index <= i``
+* :class:`.VectorOfVectors`: an n-dimensional variable length array of variable
+  length arrays.  Implemented as a pair of datasets: :attr:`flattened_data`
+  holding the raw data (:class:`.Array` or :class:`.VectorOfVectors`, if the
+  vector dimension is greater than 2), and :attr:`cumulative_length` (always an
+  :class:`.Array`) whose i-th element is the sum of the lengths of the vectors
+  with ``index <= i``
 * :class:`.VectorOfEncodedVectors`: an array of variable length *encoded*
   arrays. Implemented as a :class:`.VectorOfVectors` :attr:`encoded_data`
   holding the encoded vectors and an :class:`.Array` :attr:`decoded_size`
@@ -37,6 +39,7 @@ is done via the class :class:`.lh5_store.LH5Store`. LH5 files can also be
 browsed easily in python like any `HDF5 <https://www.hdfgroup.org>`_ file using
 `h5py <https://www.h5py.org>`_.
 """
+
 from __future__ import annotations
 
 from ._version import version as __version__

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/src/lgdo/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.5.0a5'
-__version_tuple__ = version_tuple = (1, 5, 0)
+__version__ = version = '1.6.0'
+__version_tuple__ = version_tuple = (1, 6, 0)

legend_pydataobj-1.6.0/src/lgdo/cli.py

@@ -0,0 +1,289 @@
+"""legend-pydataobj's command line interface utilities."""
+
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import logging
+import sys
+
+from . import Array, Table, VectorOfVectors, __version__, lh5
+from . import logging as lgdogging  # eheheh
+
+log = logging.getLogger(__name__)
+
+
+def lh5ls(args=None):
+    """:func:`.lh5.show` command line interface."""
+    parser = argparse.ArgumentParser(
+        prog="lh5ls", description="Inspect LEGEND HDF5 (LH5) file contents"
+    )
+
+    # global options
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="""Print legend-pydataobj version and exit""",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="""Increase the program verbosity""",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="""Increase the program verbosity to maximum""",
+    )
+
+    parser.add_argument(
+        "lh5_file",
+        help="""Input LH5 file""",
+    )
+    parser.add_argument("lh5_group", nargs="?", help="""LH5 group.""", default="/")
+    parser.add_argument(
+        "--attributes", "-a", action="store_true", help="""Print HDF5 attributes too"""
+    )
+    parser.add_argument(
+        "--depth",
+        "-d",
+        type=int,
+        default=None,
+        help="""Maximum tree depth of groups to print""",
+    )
+
+    args = parser.parse_args(args)
+
+    if args.verbose:
+        lgdogging.setup(logging.DEBUG)
+    elif args.debug:
+        lgdogging.setup(logging.DEBUG, logging.root)
+    else:
+        lgdogging.setup()
+
+    if args.version:
+        print(__version__)  # noqa: T201
+        sys.exit()
+
+    lh5.show(args.lh5_file, args.lh5_group, attrs=args.attributes, depth=args.depth)
+
+
+def lh5concat(args=None):
+    """Command line interface for concatenating array-like LGDOs in LH5 files."""
+    parser = argparse.ArgumentParser(
+        prog="lh5concat",
+        description="""
+Concatenate LGDO Arrays, VectorOfVectors and Tables in LH5 files.
+
+Examples
+--------
+
+Concatenate all eligible objects in file{1,2}.lh5 into concat.lh5:
+
+  $ lh5concat -o concat.lh5 file1.lh5 file2.lh5
+
+Include only the /data/table1 Table:
+
+  $ lh5concat -o concat.lh5 -i /data/table1/* file1.lh5 file2.lh5
+
+Exclude the /data/table1/col1 Table column:
+
+  $ lh5concat -o concat.lh5 -e /data/table1/col1 file1.lh5 file2.lh5
+        """,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+
+    # global options
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="""Print legend-pydataobj version and exit""",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="""Increase the program verbosity""",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="""Increase the program verbosity to maximum""",
+    )
+
+    parser.add_argument(
+        "lh5_file",
+        nargs="+",
+        help="""Input LH5 files""",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        help="""Output file""",
+        default="lh5concat-output.lh5",
+    )
+    parser.add_argument(
+        "--overwrite",
+        "-w",
+        action="store_true",
+        help="""Overwrite output file""",
+    )
+    parser.add_argument(
+        "--include",
+        "-i",
+        help="""Regular expression (fnmatch style) for object names that should
+        be concatenated. To include full tables, you need to explicitly include
+        all its columns with e.g. '/path/to/table/*'. The option can be passed
+        multiple times to provide a list of patterns.
+        """,
+        action="append",
+        default=None,
+    )
+    parser.add_argument(
+        "--exclude",
+        "-e",
+        help="""List of object names that should be excluded. Takes priority
+        over --include. See --include help for more details.
+        """,
+        action="append",
+        default=None,
+    )
+
+    args = parser.parse_args(args)
+
+    if args.verbose:
+        lgdogging.setup(logging.INFO, log)
+    elif args.debug:
+        lgdogging.setup(logging.DEBUG, logging.root)
+    else:
+        lgdogging.setup()
+
+    if args.version:
+        print(__version__)  # noqa: T201
+        sys.exit()
+
+    if len(args.lh5_file) < 2:
+        msg = "you must provide at least two input files"
+        raise RuntimeError(msg)
+
+    # determine list of objects by recursively ls'ing first file
+    file0 = args.lh5_file[0]
+    obj_list_full = set(lh5.ls(file0, recursive=True))
+
+    # let's remove objects with nested LGDOs inside
+    to_remove = set()
+    for name in obj_list_full:
+        if len(fnmatch.filter(obj_list_full, f"{name}/*")) > 1:
+            to_remove.add(name)
+    obj_list_full -= to_remove
+
+    obj_list = set()
+    # now first remove excluded stuff
+    if args.exclude is not None:
+        for exc in args.exclude:
+            obj_list_full -= set(fnmatch.filter(obj_list_full, exc.strip("/")))
+
+    # then make list of included, based on latest list
+    if args.include is not None:
+        for inc in args.include:
+            obj_list |= set(fnmatch.filter(obj_list_full, inc.strip("/")))
+    else:
+        obj_list = obj_list_full
+
+    # sort
+    obj_list = sorted(obj_list)
+
+    msg = f"objects matching include patterns {args.include} in {file0}: {obj_list}"
+    log.debug(msg)
+
+    # 1. read first valid lgdo from left to right
+    store = lh5.LH5Store()
+    h5f0 = store.gimme_file(file0)
+    lgdos = {}
+    # loop over object list in the first file
+    for name in obj_list:
+        # now loop over groups starting from root
+        current = ""
+        for item in name.split("/"):
+            current = f"{current}/{item}".strip("/")
+
+            if current in lgdos:
+                break
+
+            # not even an LGDO!
+            if "datatype" not in h5f0[current].attrs:
+                continue
+
+            # read as little as possible
+            obj, _ = store.read(current, h5f0, n_rows=1)
+            if isinstance(obj, (Table, Array, VectorOfVectors)):
+                # read all!
+                obj, _ = store.read(current, h5f0)
+                lgdos[current] = obj
+
+            break
+
+    msg = f"first-level, array-like objects: {lgdos.keys()}"
+    log.debug(msg)
+
+    h5f0.close()
+
+    # 2. remove (nested) table fields based on obj_list
+
+    def _inplace_table_filter(name, table, obj_list):
+        # filter objects nested in this LGDO
+        skm = fnmatch.filter(obj_list, f"{name}/*")
+        kept = {it.removeprefix(name).strip("/").split("/")[0] for it in skm}
+
+        # now remove fields
+        for k in list(table.keys()):
+            if k not in kept:
+                table.remove_column(k)
+
+        msg = f"fields left in table '{name}': {table.keys()}"
+        log.debug(msg)
+
+        # recurse!
+        for k2, v2 in table.items():
+            if not isinstance(v2, Table):
+                continue
+
+            _inplace_table_filter(f"{name}/{k2}", v2, obj_list)
+
+    for key, val in lgdos.items():
+        if not isinstance(val, Table):
+            continue
+
+        _inplace_table_filter(key, val, obj_list)
+
+    # 3. write to output file
+    msg = f"creating output file {args.output}"
+    log.info(msg)
+
+    first_done = False
+    for name, obj in lgdos.items():
+        store.write(
+            obj,
+            name,
+            args.output,
+            wo_mode="overwrite_file"
+            if (args.overwrite and not first_done)
+            else "write_safe",
+        )
+
+        first_done = True
+
+    # 4. loop over rest of files/names and write-append
+
+    for file in args.lh5_file[1:]:
+        msg = f"appending file {file} to {args.output}"
+        log.info(msg)
+
+        for name in lgdos:
+            obj, _ = store.read(name, file)
+            # need to remove nested LGDOs from obj too before appending
+            if isinstance(obj, Table):
+                _inplace_table_filter(name, obj, obj_list)
+
+            store.write(obj, name, args.output, wo_mode="append")

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/src/lgdo/compression/__init__.py

@@ -21,6 +21,7 @@ interface for encoding/decoding :class:`~.lgdo.LGDO`\ s.
 >>> enc_wft = compression.encode(wftable, RadwareSigcompress(codec_shift=-23768)
 >>> compression.decode(enc_wft) # == wftbl
 """
+
 from __future__ import annotations
 
 from .base import WaveformCodec

{legend_pydataobj-1.5.0a5 → legend_pydataobj-1.6.0}/src/lgdo/lh5/__init__.py

@@ -4,18 +4,26 @@ is done via the class :class:`.store.LH5Store`. LH5 files can also be
 browsed easily in python like any `HDF5 <https://www.hdfgroup.org>`_ file using
 `h5py <https://www.h5py.org>`_.
 """
+
 from __future__ import annotations
 
+from ._serializers.write.array import DEFAULT_HDF5_SETTINGS
+from .core import read, read_as, write
 from .iterator import LH5Iterator
 from .store import LH5Store
-from .tools import load_dfs, load_nda, ls, read_as, show
+from .tools import load_dfs, load_nda, ls, show
+from .utils import read_n_rows
 
 __all__ = [
+    "DEFAULT_HDF5_SETTINGS",
     "LH5Iterator",
     "LH5Store",
     "load_dfs",
     "load_nda",
+    "read",
+    "write",
     "read_as",
     "ls",
+    "read_n_rows",
     "show",
 ]

legend_pydataobj-1.6.0/src/lgdo/lh5/_serializers/__init__.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from .read.array import (
+    _h5_read_array,
+    _h5_read_array_of_equalsized_arrays,
+    _h5_read_fixedsize_array,
+    _h5_read_ndarray,
+)
+from .read.composite import (
+    _h5_read_lgdo,
+    _h5_read_struct,
+    _h5_read_table,
+)
+from .read.encoded import (
+    _h5_read_array_of_encoded_equalsized_arrays,
+    _h5_read_vector_of_encoded_vectors,
+)
+from .read.scalar import _h5_read_scalar
+from .read.vector_of_vectors import _h5_read_vector_of_vectors
+from .write.array import _h5_write_array
+from .write.composite import _h5_write_lgdo, _h5_write_struct
+from .write.scalar import _h5_write_scalar
+from .write.vector_of_vectors import _h5_write_vector_of_vectors
+
+__all__ = [
+    "_h5_read_lgdo",
+    "_h5_read_vector_of_vectors",
+    "_h5_read_ndarray",
+    "_h5_read_array",
+    "_h5_read_encoded_array",
+    "_h5_read_fixedsize_array",
+    "_h5_read_array_of_equalsized_arrays",
+    "_h5_read_struct",
+    "_h5_read_table",
+    "_h5_read_scalar",
+    "_h5_read_array_of_encoded_equalsized_arrays",
+    "_h5_read_vector_of_encoded_vectors",
+    "_h5_write_scalar",
+    "_h5_write_array",
+    "_h5_write_vector_of_vectors",
+    "_h5_write_struct",
+    "_h5_write_lgdo",
+]

legend_pydataobj-1.6.0/src/lgdo/lh5/_serializers/read/array.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import logging
+
+from ....types import Array, ArrayOfEqualSizedArrays, FixedSizeArray
+from . import utils
+from .ndarray import _h5_read_ndarray
+
+log = logging.getLogger(__name__)
+
+
+def _h5_read_array_generic(type_, name, h5f, **kwargs):
+    nda, attrs, n_rows_to_read = _h5_read_ndarray(name, h5f, **kwargs)
+
+    obj_buf = kwargs["obj_buf"]
+
+    if obj_buf is None:
+        return type_(nda=nda, attrs=attrs), n_rows_to_read
+
+    utils.check_obj_buf_attrs(obj_buf.attrs, attrs, h5f, name)
+
+    return obj_buf, n_rows_to_read
+
+
+def _h5_read_array(name, h5f, **kwargs):
+    return _h5_read_array_generic(Array, name, h5f, **kwargs)
+
+
+def _h5_read_fixedsize_array(name, h5f, **kwargs):
+    return _h5_read_array_generic(FixedSizeArray, name, h5f, **kwargs)
+
+
+def _h5_read_array_of_equalsized_arrays(name, h5f, **kwargs):
+    return _h5_read_array_generic(ArrayOfEqualSizedArrays, name, h5f, **kwargs)