legend-pydataobj 1.12.0a1__tar.gz → 1.12.0a3__tar.gz

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.12.0a1
+Version: 1.12.0a3
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/pyproject.toml

@@ -122,7 +122,7 @@ extend-select = [
   "PIE",      # flake8-pie
   "PL",       # pylint
   "PT",       # flake8-pytest-style
-  #"PTH",      # flake8-use-pathlib
+  "PTH",      # flake8-use-pathlib
   "RET",      # flake8-return
   "RUF",      # Ruff-specific
   "SIM",      # flake8-simplify
@@ -167,7 +167,7 @@ minversion = "6.0"
 addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
 xfail_strict = true
 filterwarnings = ["error", 'ignore:\nPyarrow:DeprecationWarning']
-log_cli_level = "info"
+log_cli_level = "INFO"
 testpaths = "tests"
 
 [tool.codespell]

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/legend_pydataobj.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: legend_pydataobj
-Version: 1.12.0a1
+Version: 1.12.0a3
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/legend_pydataobj.egg-info/SOURCES.txt

@@ -12,7 +12,6 @@ src/lgdo/__init__.py
 src/lgdo/_version.py
 src/lgdo/cli.py
 src/lgdo/lgdo_utils.py
-src/lgdo/lh5_store.py
 src/lgdo/logging.py
 src/lgdo/units.py
 src/lgdo/utils.py

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/__init__.py

@@ -45,7 +45,7 @@ browsed easily in python like any `HDF5 <https://www.hdfgroup.org>`_ file using
 from __future__ import annotations
 
 from ._version import version as __version__
-from .lh5_store import LH5Iterator, LH5Store, load_dfs, load_nda, ls, show
+from .lh5 import LH5Iterator, ls, read, read_as, read_n_rows, show, write
 from .types import (
     LGDO,
     Array,
@@ -69,7 +69,6 @@ __all__ = [
     "FixedSizeArray",
     "Histogram",
     "LH5Iterator",
-    "LH5Store",
     "Scalar",
     "Struct",
     "Table",
@@ -77,8 +76,10 @@ __all__ = [
     "VectorOfVectors",
     "WaveformTable",
     "__version__",
-    "load_dfs",
-    "load_nda",
     "ls",
+    "read",
+    "read_as",
+    "read_n_rows",
     "show",
+    "write",
 ]

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.12.0a1'
-__version_tuple__ = version_tuple = (1, 12, 0)
+__version__ = version = '1.12.0a3'
+__version_tuple__ = version_tuple = (1, 12, 0, 'a3')

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/lh5/__init__.py

@@ -11,7 +11,7 @@ from ._serializers.write.array import DEFAULT_HDF5_SETTINGS
 from .core import read, read_as, write
 from .iterator import LH5Iterator
 from .store import LH5Store
-from .tools import load_dfs, load_nda, ls, show
+from .tools import ls, show
 from .utils import read_n_rows
 
 __all__ = [
@@ -19,8 +19,6 @@ __all__ = [
     "LH5Iterator",
     "LH5Store",
     "concat",
-    "load_dfs",
-    "load_nda",
     "ls",
     "read",
     "read_as",

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/lh5/_serializers/write/composite.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import logging
-import os
 from inspect import signature
+from pathlib import Path
 
 import h5py
 
@@ -53,7 +53,7 @@ def _h5_write_lgdo(
     # change any object in the file. So we use file:append for
     # write_object:overwrite.
     if not isinstance(lh5_file, h5py.File):
-        mode = "w" if wo_mode == "of" or not os.path.exists(lh5_file) else "a"
+        mode = "w" if wo_mode == "of" or not Path(lh5_file).exists() else "a"
         lh5_file = h5py.File(lh5_file, mode=mode, **file_kwargs)
 
     log.debug(

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/lh5/core.py

@@ -113,7 +113,11 @@ def read(
         lh5_obj = lh5_file[name]
     elif isinstance(lh5_file, str):
         lh5_file = h5py.File(lh5_file, mode="r", locking=locking)
-        lh5_obj = lh5_file[name]
+        try:
+            lh5_obj = lh5_file[name]
+        except KeyError as ke:
+            err = f"Object {name} not found in file {lh5_file.filename}"
+            raise KeyError(err) from ke
     else:
         if obj_buf is not None:
             obj_buf.resize(obj_buf_start)

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/lh5/store.py

@@ -6,11 +6,11 @@ HDF5 files.
 from __future__ import annotations
 
 import logging
-import os
 import sys
 from collections import OrderedDict
 from collections.abc import Mapping, Sequence
 from inspect import signature
+from pathlib import Path
 from typing import Any
 
 import h5py
@@ -92,16 +92,16 @@ class LH5Store:
             return self.files[lh5_file]
 
         if self.base_path != "":
-            full_path = os.path.join(self.base_path, lh5_file)
+            full_path = Path(self.base_path) / lh5_file
         else:
-            full_path = lh5_file
+            full_path = Path(lh5_file)
 
-        file_exists = os.path.exists(full_path)
+        file_exists = full_path.exists()
         if mode != "r":
-            directory = os.path.dirname(full_path)
-            if directory != "" and not os.path.exists(directory):
+            directory = full_path.parent
+            if directory != "" and not full_path.parent.exists():
                 log.debug(f"making path {directory}")
-                os.makedirs(directory)
+                directory.mkdir(parents=True, exist_ok=True)
 
         if mode == "r" and not file_exists:
             msg = f"file {full_path} not found"

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/lh5/tools.py

@@ -1,16 +1,10 @@
 from __future__ import annotations
 
 import fnmatch
-import glob
 import logging
-import os
 from copy import copy
-from warnings import warn
 
 import h5py
-import numpy as np
-import pandas as pd
-from numpy.typing import NDArray
 
 from . import utils
 from .store import LH5Store
@@ -223,108 +217,3 @@ def show(
             break
 
         key = k_new
-
-
-def load_nda(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[NDArray | list | tuple] | None = None,
-) -> dict[str, NDArray]:
-    r"""Build a dictionary of :class:`numpy.ndarray`\ s from LH5 data.
-
-    Given a list of files, a list of LH5 table parameters, and an optional
-    group path, return a NumPy array with all values for each parameter.
-
-    Parameters
-    ----------
-    f_list
-        A list of files. Can contain wildcards.
-    par_list
-        A list of parameters to read from each file.
-    lh5_group
-        group path within which to find the specified parameters.
-    idx_list
-        for fancy-indexed reads. Must be one index array for each file in
-        `f_list`.
-
-    Returns
-    -------
-    par_data
-        A dictionary of the parameter data keyed by the elements of `par_list`.
-        Each entry contains the data for the specified parameter concatenated
-        over all files in `f_list`.
-    """
-    warn(
-        "load_nda() is deprecated. "
-        "Please replace it with LH5Store.read(...).view_as('np'), "
-        "or just read_as(..., 'np'). "
-        "load_nda() will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-
-    if isinstance(f_list, str):
-        f_list = [f_list]
-        if idx_list is not None:
-            idx_list = [idx_list]
-    if idx_list is not None and len(f_list) != len(idx_list):
-        msg = f"f_list length ({len(f_list)}) != idx_list length ({len(idx_list)})!"
-        raise ValueError(msg)
-
-    # Expand wildcards
-    f_list = [f for f_wc in f_list for f in sorted(glob.glob(os.path.expandvars(f_wc)))]
-
-    sto = LH5Store()
-    par_data = {par: [] for par in par_list}
-    for ii, ff in enumerate(f_list):
-        f = sto.gimme_file(ff, "r")
-        for par in par_list:
-            if f"{lh5_group}/{par}" not in f:
-                msg = f"'{lh5_group}/{par}' not in file {ff}"
-                raise RuntimeError(msg)
-
-            if idx_list is None:
-                data, _ = sto.read(f"{lh5_group}/{par}", f)
-            else:
-                data, _ = sto.read(f"{lh5_group}/{par}", f, idx=idx_list[ii])
-            if not data:
-                continue
-            par_data[par].append(data.nda)
-    return {par: np.concatenate(par_data[par]) for par in par_list}
-
-
-def load_dfs(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[NDArray | list | tuple] | None = None,
-) -> pd.DataFrame:
-    """Build a :class:`pandas.DataFrame` from LH5 data.
-
-    Given a list of files (can use wildcards), a list of LH5 columns, and
-    optionally the group path, return a :class:`pandas.DataFrame` with all
-    values for each parameter.
-
-    See Also
-    --------
-    :func:`load_nda`
-
-    Returns
-    -------
-    dataframe
-        contains columns for each parameter in `par_list`, and rows containing
-        all data for the associated parameters concatenated over all files in
-        `f_list`.
-    """
-    warn(
-        "load_dfs() is deprecated. "
-        "Please replace it with LH5Store.read(...).view_as('pd'), "
-        "or just read_as(..., 'pd'). "
-        "load_dfs() will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return pd.DataFrame(
-        load_nda(f_list, par_list, lh5_group=lh5_group, idx_list=idx_list)
-    )

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/lh5/utils.py

@@ -7,6 +7,7 @@ import logging
 import os
 import string
 from collections.abc import Mapping, Sequence
+from pathlib import Path
 from typing import Any
 
 import h5py
@@ -183,14 +184,15 @@ def expand_path(
         Unique absolute path, or list of all absolute paths
     """
     if base_path is not None and base_path != "":
-        base_path = os.path.expanduser(os.path.expandvars(base_path))
-        path = os.path.join(base_path, path)
+        base_path = Path(os.path.expandvars(base_path)).expanduser()
+        path = base_path / path
 
     # first expand variables
     _path = expand_vars(path, substitute)
 
     # then expand wildcards
-    paths = sorted(glob.glob(os.path.expanduser(_path)))
+    # pathlib glob works differently so use glob for now
+    paths = sorted(glob.glob(str(Path(_path).expanduser())))  # noqa: PTH207
 
     if base_path is not None and base_path != "":
         paths = [os.path.relpath(p, base_path) for p in paths]

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/types/array.py

@@ -6,7 +6,7 @@ corresponding utilities.
 from __future__ import annotations
 
 import logging
-from collections.abc import Iterator
+from collections.abc import Collection, Iterator
 from typing import Any
 
 import awkward as ak
@@ -126,19 +126,27 @@ class Array(LGDOCollection):
         "Set capacity to be minimum needed to support Array size"
         self.reserve_capacity(np.prod(self.shape))
 
-    def resize(self, new_size: int, trim=False) -> None:
+    def resize(self, new_size: int | Collection[int], trim=False) -> None:
         """Set size of Array in rows. Only change capacity if it must be
         increased to accommodate new rows; in this case double capacity.
-        If trim is True, capacity will be set to match size."""
+        If trim is True, capacity will be set to match size. If new_size
+        is an int, do not change size of inner dimensions.
 
-        self._size = new_size
+        If new_size is a collection, internal memory will be re-allocated, so
+        this should be done only rarely!"""
 
-        if trim and new_size != self.get_capacity:
-            self.reserve_capacity(new_size)
+        if isinstance(new_size, Collection):
+            self._size = new_size[0]
+            self._nda.resize(new_size)
+        else:
+            self._size = new_size
+
+            if trim and new_size != self.get_capacity:
+                self.reserve_capacity(new_size)
 
-        # If capacity is not big enough, set to next power of 2 big enough
-        if new_size > self.get_capacity():
-            self.reserve_capacity(int(2 ** (np.ceil(np.log2(new_size)))))
+            # If capacity is not big enough, set to next power of 2 big enough
+            if new_size > self.get_capacity():
+                self.reserve_capacity(int(2 ** (np.ceil(np.log2(new_size)))))
 
     def append(self, value: np.ndarray) -> None:
         "Append value to end of array (with copy)"

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/types/vectorofvectors.py

@@ -130,20 +130,48 @@ class VectorOfVectors(LGDOCollection):
 
             # ak.to_buffer helps in de-serialization
             # NOTE: ak.to_packed() needed?
-            form, length, container = ak.to_buffers(ak.to_packed(data))
-
-            # NOTE: node#-data is not even in the dict if the awkward array is empty
-            # NOTE: if the data arg was a numpy array, to_buffers() preserves
-            # the original dtype
-            # FIXME: have to copy the buffers, otherwise self will not own the
-            # data and self.resize() will fail. Is it possible to avoid this?
-            flattened_data = np.copy(
-                container.pop(f"node{data.ndim - 1}-data", np.empty(0, dtype=dtype))
-            )
+            form, _, container = ak.to_buffers(ak.to_packed(data))
+
+            # check if bytestring
+            curr = form
+            for _ in range(data.ndim - 1):
+                curr = curr.content
+            if (
+                "__array__" in curr.parameters
+                and curr.parameters["__array__"] == "bytestring"
+            ):
+                diffs = np.diff(container[f"node{data.ndim - 1}-offsets"])
+                if (diffs != diffs[0]).all():
+                    err_msg = "Non uniform string lengths not supported"
+                    raise NotImplementedError(err_msg)
+                flattened_data = np.asarray(
+                    ak.enforce_type(
+                        ak.unflatten(
+                            container.pop(
+                                f"node{data.ndim}-data", np.empty(0, dtype=dtype)
+                            ),
+                            diffs[0],
+                        ),
+                        "bytes",
+                    )
+                )
 
-            # if user-provided dtype is different than dtype from Awkward, cast
-            # NOTE: makes a copy only if needed
-            flattened_data = np.asarray(flattened_data, dtype=dtype)
+                # if user-provided dtype is different than dtype from Awkward, cast
+                # NOTE: makes a copy only if needed
+                flattened_data = np.asarray(flattened_data, dtype=dtype)
+            else:
+                # NOTE: node#-data is not even in the dict if the awkward array is empty
+                # NOTE: if the data arg was a numpy array, to_buffers() preserves
+                # the original dtype
+                # FIXME: have to copy the buffers, otherwise self will not own the
+                # data and self.resize() will fail. Is it possible to avoid this?
+                flattened_data = np.copy(
+                    container.pop(f"node{data.ndim - 1}-data", np.empty(0, dtype=dtype))
+                )
+
+                # if user-provided dtype is different than dtype from Awkward, cast
+                # NOTE: makes a copy only if needed
+                flattened_data = np.asarray(flattened_data, dtype=dtype)
 
             # start from innermost VoV and build nested structure
             for i in range(data.ndim - 2, -1, -1):
@@ -476,7 +504,10 @@ class VectorOfVectors(LGDOCollection):
             else:
                 nan_val = np.nan
             vovutils._nb_fill(
-                vec, lens, nan_val, self.flattened_data.nda[start : cum_lens[-1]]
+                vec,
+                lens,
+                np.array([nan_val]).astype(self.flattened_data.nda.dtype),
+                self.flattened_data.nda[start : cum_lens[-1]],
             )
 
             # add new vector(s) length to cumulative_length
@@ -627,11 +658,25 @@ class VectorOfVectors(LGDOCollection):
             offsets[1:] = self.cumulative_length.nda
             offsets[0] = 0
 
-            content = (
-                ak.contents.NumpyArray(self.flattened_data.nda)
-                if self.ndim == 2
-                else self.flattened_data.view_as(library, with_units=with_units).layout
-            )
+            if self.ndim != 2:
+                content = self.flattened_data.view_as(
+                    library, with_units=with_units
+                ).layout
+            # need to handle strings separately
+            elif np.issubdtype(self.flattened_data.nda.dtype, np.bytes_):
+                byte_arrays = []
+                for s in self.flattened_data.nda:
+                    # Convert each string to array of bytes
+                    byte_array = np.frombuffer(s, dtype=np.uint8)
+                    byte_arrays.append(byte_array)
+                max_len = max(len(b) for b in byte_arrays)
+                raw_arrays = ak.contents.NumpyArray(np.concatenate(byte_arrays))
+                array_of_chars = ak.contents.RegularArray(
+                    raw_arrays, max_len, parameters={"__array__": "bytes"}
+                )
+                content = ak.enforce_type(array_of_chars, "bytes", highlevel=False)
+            else:
+                content = ak.contents.NumpyArray(self.flattened_data.nda)
 
             layout = ak.contents.ListOffsetArray(
                 offsets=ak.index.Index(offsets),

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/types/vovutils.py

@@ -131,7 +131,7 @@ def _nb_fill(
     for i, ll in enumerate(len_in):
         stop = start + ll
         if ll > max_len:
-            flattened_array_out[start : start + max_len] = aoa_in[i, :]
+            flattened_array_out[start : start + max_len] = aoa_in[i, :max_len]
             flattened_array_out[start + max_len : stop] = nan_val
         else:
             flattened_array_out[start:stop] = aoa_in[i, :ll]

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/src/lgdo/types/waveformtable.py

@@ -112,12 +112,10 @@ class WaveformTable(Table):
         if not isinstance(t0, Array):
             shape = (size,)
             t0_dtype = t0.dtype if hasattr(t0, "dtype") else np.float32
-            nda = (
-                t0 if isinstance(t0, np.ndarray) else np.full(shape, t0, dtype=t0_dtype)
-            )
-            if nda.shape != shape:
-                nda.resize(shape, refcheck=True)
-            t0 = Array(nda=nda)
+            if isinstance(t0, np.ndarray):
+                t0 = Array(nda=t0, shape=shape, dtype=t0_dtype)
+            else:
+                t0 = Array(fill_val=t0, shape=shape, dtype=t0_dtype)
 
         if t0_units is not None:
             t0.attrs["units"] = f"{t0_units}"
@@ -125,12 +123,11 @@ class WaveformTable(Table):
         if not isinstance(dt, Array):
             shape = (size,)
             dt_dtype = dt.dtype if hasattr(dt, "dtype") else np.float32
-            nda = (
-                dt if isinstance(dt, np.ndarray) else np.full(shape, dt, dtype=dt_dtype)
-            )
-            if nda.shape != shape:
-                nda.resize(shape, refcheck=True)
-            dt = Array(nda=nda)
+            if isinstance(dt, np.ndarray):
+                dt = Array(nda=dt, shape=shape, dtype=dt_dtype)
+            else:
+                dt = Array(fill_val=dt, shape=shape, dtype=dt_dtype)
+
         if dt_units is not None:
             dt.attrs["units"] = f"{dt_units}"
 
@@ -174,14 +171,15 @@ class WaveformTable(Table):
                         if hasattr(values, "dtype")
                         else np.dtype(np.float64)
                     )
-                nda = (
-                    values
-                    if isinstance(values, np.ndarray)
-                    else np.zeros(shape, dtype=dtype)
-                )
-                if nda.shape != shape:
-                    nda.resize(shape, refcheck=True)
-                values = ArrayOfEqualSizedArrays(dims=(1, 1), nda=nda)
+                if isinstance(values, np.ndarray):
+                    values = ArrayOfEqualSizedArrays(
+                        dims=(1, 1), nda=values, shape=shape, dtype=dtype
+                    )
+                else:
+                    values = ArrayOfEqualSizedArrays(
+                        dims=(1, 1), fill_val=0, shape=shape, dtype=dtype
+                    )
+
         if values_units is not None:
             values.attrs["units"] = f"{values_units}"
 
@@ -215,7 +213,7 @@ class WaveformTable(Table):
             return
         shape = self.values.nda.shape
         shape = (shape[0], wf_len)
-        self.values.nda.resize(shape, refcheck=True)
+        self.values.resize(shape)
 
     def resize_wf_len(self, new_len: int) -> None:
         """Alias for `wf_len.setter`, for when we want to make it clear in

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/tests/compression/test_radware_sigcompress.py

@@ -22,7 +22,7 @@ def read_sigcompress_c_output(filename: str):
     enc_wf_c = np.empty(0, dtype=np.uint16)
     nsig_c = None
     shift = None
-    with open(filename) as f:
+    with Path(filename).open() as f:
         nsig_c = int(f.readline())  # first number in the file
         shift = int(f.readline())  # second number in the file
         for line in f.readlines():  # then the waveform
@@ -35,7 +35,7 @@ def read_sigcompress_c_output_multi(filename: str):
     enc_wf_c = []
     nsig_c = np.empty(0, dtype="uint32")
     shift = np.empty(0, dtype="int32")
-    with open(filename) as f:
+    with Path(filename).open() as f:
         for line in f:
             parts = line.split()
             nsig_c = np.append(nsig_c, np.uint32(parts[0]))

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/tests/conftest.py

@@ -1,20 +1,20 @@
 from __future__ import annotations
 
-import os
 import shutil
 import uuid
 from getpass import getuser
+from pathlib import Path
 from tempfile import gettempdir
 
 import pytest
 from legendtestdata import LegendTestData
 
-_tmptestdir = os.path.join(gettempdir(), f"lgdo-tests-{getuser()}-{uuid.uuid4()!s}")
+_tmptestdir = Path(gettempdir()) / f"lgdo-tests-{getuser()}-{uuid.uuid4()!s}"
 
 
 @pytest.fixture(scope="session")
 def tmptestdir():
-    os.mkdir(_tmptestdir)
+    Path(_tmptestdir).mkdir(parents=True, exist_ok=True)
     return _tmptestdir
 
 

{legend_pydataobj-1.12.0a1 → legend_pydataobj-1.12.0a3}/tests/lh5/test_lh5_utils.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from pathlib import Path
 
 import pytest
 
@@ -36,7 +37,7 @@ def test_expand_path(lgnd_test_data):
             "lh5/prod-ref-l200/generated/tier/dsp/cal/p03/r001/l200-p03-r001-cal-20230318T012228Z-tier_dsp.lh5"
         ),
     ]
-    base_dir = os.path.dirname(files[0])
+    base_dir = Path(files[0]).parent
 
     assert utils.expand_path(f"{base_dir}/*20230318T012144Z*") == files[0]