legend-pydataobj 1.11.13__py3-none-any.whl → 1.12.0__py3-none-any.whl

lgdo/lh5/store.py

@@ -5,21 +5,20 @@ HDF5 files.
 
 from __future__ import annotations
 
-import bisect
 import logging
-import os
 import sys
 from collections import OrderedDict
 from collections.abc import Mapping, Sequence
 from inspect import signature
+from pathlib import Path
 from typing import Any
 
 import h5py
-import numpy as np
 from numpy.typing import ArrayLike
 
 from .. import types
 from . import _serializers, utils
+from .core import read
 
 log = logging.getLogger(__name__)
 
@@ -93,16 +92,16 @@ class LH5Store:
             return self.files[lh5_file]
 
         if self.base_path != "":
-            full_path = os.path.join(self.base_path, lh5_file)
+            full_path = Path(self.base_path) / lh5_file
         else:
-            full_path = lh5_file
+            full_path = Path(lh5_file)
 
-        file_exists = os.path.exists(full_path)
+        file_exists = full_path.exists()
         if mode != "r":
-            directory = os.path.dirname(full_path)
-            if directory != "" and not os.path.exists(directory):
+            directory = full_path.parent
+            if directory != "" and not full_path.parent.exists():
                 log.debug(f"making path {directory}")
-                os.makedirs(directory)
+                directory.mkdir(parents=True, exist_ok=True)
 
         if mode == "r" and not file_exists:
             msg = f"file {full_path} not found"
@@ -155,7 +154,7 @@ class LH5Store:
         """Returns an LH5 object appropriate for use as a pre-allocated buffer
         in a read loop. Sets size to `size` if object has a size.
         """
-        obj, n_rows = self.read(name, lh5_file, n_rows=0, field_mask=field_mask)
+        obj = self.read(name, lh5_file, n_rows=0, field_mask=field_mask)
         if hasattr(obj, "resize") and size is not None:
             obj.resize(new_size=size)
         return obj
@@ -182,72 +181,20 @@ class LH5Store:
         """
         # grab files from store
         if isinstance(lh5_file, (str, h5py.File)):
-            lh5_obj = self.gimme_file(lh5_file, "r", **file_kwargs)[name]
+            h5f = self.gimme_file(lh5_file, "r", **file_kwargs)
         else:
-            lh5_files = list(lh5_file)
-            n_rows_read = 0
-
-            for i, h5f in enumerate(lh5_files):
-                if (
-                    isinstance(idx, (list, tuple))
-                    and len(idx) > 0
-                    and not np.isscalar(idx[0])
-                ):
-                    # a list of lists: must be one per file
-                    idx_i = idx[i]
-                elif idx is not None:
-                    # make idx a proper tuple if it's not one already
-                    if not (isinstance(idx, tuple) and len(idx) == 1):
-                        idx = (idx,)
-                    # idx is a long continuous array
-                    n_rows_i = utils.read_n_rows(name, h5f)
-                    # find the length of the subset of idx that contains indices
-                    # that are less than n_rows_i
-                    n_rows_to_read_i = bisect.bisect_left(idx[0], n_rows_i)
-                    # now split idx into idx_i and the remainder
-                    idx_i = np.array(idx[0])[:n_rows_to_read_i]
-                    idx = np.array(idx[0])[n_rows_to_read_i:] - n_rows_i
-                else:
-                    idx_i = None
-                n_rows_i = n_rows - n_rows_read
-
-                obj_buf, n_rows_read_i = self.read(
-                    name,
-                    h5f,
-                    start_row,
-                    n_rows_i,
-                    idx_i,
-                    use_h5idx,
-                    field_mask,
-                    obj_buf,
-                    obj_buf_start,
-                    decompress,
-                )
-
-                n_rows_read += n_rows_read_i
-                if n_rows_read >= n_rows or obj_buf is None:
-                    return obj_buf, n_rows_read
-                start_row = 0
-                obj_buf_start += n_rows_read_i
-            return obj_buf, n_rows_read
-
-        if isinstance(idx, (list, tuple)) and len(idx) > 0 and not np.isscalar(idx[0]):
-            idx = idx[0]
-        if isinstance(idx, np.ndarray) and idx.dtype == np.dtype("?"):
-            idx = np.where(idx)[0]
-
-        return _serializers._h5_read_lgdo(
-            lh5_obj.id,
-            lh5_obj.file.filename,
-            lh5_obj.name,
-            start_row=start_row,
-            n_rows=n_rows,
-            idx=idx,
-            use_h5idx=use_h5idx,
-            field_mask=field_mask,
-            obj_buf=obj_buf,
-            obj_buf_start=obj_buf_start,
-            decompress=decompress,
+            h5f = [self.gimme_file(f, "r", **file_kwargs) for f in lh5_file]
+        return read(
+            name,
+            h5f,
+            start_row,
+            n_rows,
+            idx,
+            use_h5idx,
+            field_mask,
+            obj_buf,
+            obj_buf_start,
+            decompress,
         )
 
     def write(

lgdo/lh5/tools.py

@@ -1,16 +1,10 @@
 from __future__ import annotations
 
 import fnmatch
-import glob
 import logging
-import os
 from copy import copy
-from warnings import warn
 
 import h5py
-import numpy as np
-import pandas as pd
-from numpy.typing import NDArray
 
 from . import utils
 from .store import LH5Store
@@ -223,108 +217,3 @@ def show(
             break
 
         key = k_new
-
-
-def load_nda(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[NDArray | list | tuple] | None = None,
-) -> dict[str, NDArray]:
-    r"""Build a dictionary of :class:`numpy.ndarray`\ s from LH5 data.
-
-    Given a list of files, a list of LH5 table parameters, and an optional
-    group path, return a NumPy array with all values for each parameter.
-
-    Parameters
-    ----------
-    f_list
-        A list of files. Can contain wildcards.
-    par_list
-        A list of parameters to read from each file.
-    lh5_group
-        group path within which to find the specified parameters.
-    idx_list
-        for fancy-indexed reads. Must be one index array for each file in
-        `f_list`.
-
-    Returns
-    -------
-    par_data
-        A dictionary of the parameter data keyed by the elements of `par_list`.
-        Each entry contains the data for the specified parameter concatenated
-        over all files in `f_list`.
-    """
-    warn(
-        "load_nda() is deprecated. "
-        "Please replace it with LH5Store.read(...).view_as('np'), "
-        "or just read_as(..., 'np'). "
-        "load_nda() will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-
-    if isinstance(f_list, str):
-        f_list = [f_list]
-        if idx_list is not None:
-            idx_list = [idx_list]
-    if idx_list is not None and len(f_list) != len(idx_list):
-        msg = f"f_list length ({len(f_list)}) != idx_list length ({len(idx_list)})!"
-        raise ValueError(msg)
-
-    # Expand wildcards
-    f_list = [f for f_wc in f_list for f in sorted(glob.glob(os.path.expandvars(f_wc)))]
-
-    sto = LH5Store()
-    par_data = {par: [] for par in par_list}
-    for ii, ff in enumerate(f_list):
-        f = sto.gimme_file(ff, "r")
-        for par in par_list:
-            if f"{lh5_group}/{par}" not in f:
-                msg = f"'{lh5_group}/{par}' not in file {ff}"
-                raise RuntimeError(msg)
-
-            if idx_list is None:
-                data, _ = sto.read(f"{lh5_group}/{par}", f)
-            else:
-                data, _ = sto.read(f"{lh5_group}/{par}", f, idx=idx_list[ii])
-            if not data:
-                continue
-            par_data[par].append(data.nda)
-    return {par: np.concatenate(par_data[par]) for par in par_list}
-
-
-def load_dfs(
-    f_list: str | list[str],
-    par_list: list[str],
-    lh5_group: str = "",
-    idx_list: list[NDArray | list | tuple] | None = None,
-) -> pd.DataFrame:
-    """Build a :class:`pandas.DataFrame` from LH5 data.
-
-    Given a list of files (can use wildcards), a list of LH5 columns, and
-    optionally the group path, return a :class:`pandas.DataFrame` with all
-    values for each parameter.
-
-    See Also
-    --------
-    :func:`load_nda`
-
-    Returns
-    -------
-    dataframe
-        contains columns for each parameter in `par_list`, and rows containing
-        all data for the associated parameters concatenated over all files in
-        `f_list`.
-    """
-    warn(
-        "load_dfs() is deprecated. "
-        "Please replace it with LH5Store.read(...).view_as('pd'), "
-        "or just read_as(..., 'pd'). "
-        "load_dfs() will be removed in a future release.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    return pd.DataFrame(
-        load_nda(f_list, par_list, lh5_group=lh5_group, idx_list=idx_list)
-    )

lgdo/lh5/utils.py

@@ -7,6 +7,7 @@ import logging
 import os
 import string
 from collections.abc import Mapping, Sequence
+from pathlib import Path
 from typing import Any
 
 import h5py
@@ -153,7 +154,7 @@ def expand_vars(expr: str, substitute: dict[str, str] | None = None) -> str:
 
     # use provided mapping
     # then expand env variables
-    return os.path.expandvars(string.Template(expr).safe_substitute(substitute))
+    return os.path.expandvars(string.Template(str(expr)).safe_substitute(substitute))
 
 
 def expand_path(
@@ -183,14 +184,15 @@ def expand_path(
         Unique absolute path, or list of all absolute paths
     """
     if base_path is not None and base_path != "":
-        base_path = os.path.expanduser(os.path.expandvars(base_path))
-        path = os.path.join(base_path, path)
+        base_path = Path(os.path.expandvars(base_path)).expanduser()
+        path = base_path / path
 
     # first expand variables
     _path = expand_vars(path, substitute)
 
     # then expand wildcards
-    paths = sorted(glob.glob(os.path.expanduser(_path)))
+    # pathlib glob works differently so use glob for now
+    paths = sorted(glob.glob(str(Path(_path).expanduser())))  # noqa: PTH207
 
     if base_path is not None and base_path != "":
         paths = [os.path.relpath(p, base_path) for p in paths]

lgdo/types/array.py

@@ -6,7 +6,7 @@ corresponding utilities.
 from __future__ import annotations
 
 import logging
-from collections.abc import Iterator
+from collections.abc import Collection, Iterator
 from typing import Any
 
 import awkward as ak
@@ -17,12 +17,12 @@ import pint_pandas  # noqa: F401
 
 from .. import utils
 from ..units import default_units_registry as u
-from .lgdo import LGDO
+from .lgdo import LGDOCollection
 
 log = logging.getLogger(__name__)
 
 
-class Array(LGDO):
+class Array(LGDOCollection):
     r"""Holds an :class:`numpy.ndarray` and attributes.
 
     :class:`Array` (and the other various array types) holds an `nda` instead
@@ -78,11 +78,7 @@ class Array(LGDO):
         elif isinstance(nda, Array):
             nda = nda.nda
 
-        elif not isinstance(nda, np.ndarray):
-            nda = np.array(nda)
-
         self.nda = nda
-        self.dtype = self.nda.dtype
 
         super().__init__(attrs)
 
@@ -96,18 +92,91 @@ class Array(LGDO):
         return dt + "<" + nd + ">{" + et + "}"
 
     def __len__(self) -> int:
-        return len(self.nda)
-
-    def resize(self, new_size: int) -> None:
-        new_shape = (new_size,) + self.nda.shape[1:]
-        return self.nda.resize(new_shape, refcheck=True)
+        return self._size
+
+    @property
+    def nda(self):
+        return self._nda[: self._size, ...] if self._nda.shape != () else self._nda
+
+    @nda.setter
+    def nda(self, value):
+        self._nda = value if isinstance(value, np.ndarray) else np.array(value)
+        self._size = len(self._nda) if self._nda.shape != () else 0
+
+    @property
+    def dtype(self):
+        return self._nda.dtype
+
+    @property
+    def shape(self):
+        return (len(self),) + self._nda.shape[1:]
+
+    def reserve_capacity(self, capacity: int) -> None:
+        "Set size (number of rows) of internal memory buffer"
+        if capacity < len(self):
+            msg = "Cannot reduce capacity below Array length"
+            raise ValueError(msg)
+        self._nda.resize((capacity,) + self._nda.shape[1:], refcheck=False)
+
+    def get_capacity(self) -> int:
+        "Get capacity (i.e. max size before memory must be re-allocated)"
+        return len(self._nda)
+
+    def trim_capacity(self) -> None:
+        "Set capacity to be minimum needed to support Array size"
+        self.reserve_capacity(np.prod(self.shape))
+
+    def resize(self, new_size: int | Collection[int], trim=False) -> None:
+        """Set size of Array in rows. Only change capacity if it must be
+        increased to accommodate new rows; in this case double capacity.
+        If trim is True, capacity will be set to match size. If new_size
+        is an int, do not change size of inner dimensions.
+
+        If new_size is a collection, internal memory will be re-allocated, so
+        this should be done only rarely!"""
+
+        if isinstance(new_size, Collection):
+            self._size = new_size[0]
+            self._nda.resize(new_size)
+        else:
+            self._size = new_size
+
+            if trim and new_size != self.get_capacity:
+                self.reserve_capacity(new_size)
+
+            # If capacity is not big enough, set to next power of 2 big enough
+            if new_size > self.get_capacity():
+                self.reserve_capacity(int(2 ** (np.ceil(np.log2(new_size)))))
 
     def append(self, value: np.ndarray) -> None:
-        self.resize(len(self) + 1)
-        self.nda[-1] = value
+        "Append value to end of array (with copy)"
+        self.insert(len(self), value)
 
     def insert(self, i: int, value: int | float) -> None:
-        self.nda = np.insert(self.nda, i, value)
+        "Insert value into row i (with copy)"
+        if i > len(self):
+            msg = f"index {i} is out of bounds for array with size {len(self)}"
+            raise IndexError(msg)
+
+        value = np.array(value)
+        if value.shape == self.shape[1:]:
+            self.resize(len(self) + 1)
+            self[i + 1 :] = self[i:-1]
+            self[i] = value
+        elif value.shape[1:] == self.shape[1:]:
+            self.resize(len(self) + len(value))
+            self[i + len(value) :] = self[i : -len(value)]
+            self[i : i + len(value)] = value
+        else:
+            msg = f"Could not insert value with shape {value.shape} into Array with shape {self.shape}"
+            raise ValueError(msg)
+
+    def replace(self, i: int, value: int | float) -> None:
+        "Replace value at row i"
+        if i >= len(self):
+            msg = f"index {i} is out of bounds for array with size {len(self)}"
+            raise IndexError(msg)
+        self[i] = value
 
     def __getitem__(self, key):
         return self.nda[key]

lgdo/types/encoded.py

@@ -11,12 +11,12 @@ from numpy.typing import NDArray
 
 from .. import utils
 from .array import Array
-from .lgdo import LGDO
+from .lgdo import LGDOCollection
 from .scalar import Scalar
 from .vectorofvectors import VectorOfVectors
 
 
-class VectorOfEncodedVectors(LGDO):
+class VectorOfEncodedVectors(LGDOCollection):
     """An array of variable-length encoded arrays.
 
     Used to represent an encoded :class:`.VectorOfVectors`. In addition to an
@@ -92,6 +92,17 @@ class VectorOfEncodedVectors(LGDO):
 
         return False
 
+    def reserve_capacity(self, *capacity: int) -> None:
+        self.encoded_data.reserve_capacity(*capacity)
+        self.decoded_size.reserve_capacity(capacity[0])
+
+    def get_capacity(self) -> tuple:
+        return (self.decoded_size.get_capacity, *self.encoded_data.get_capacity())
+
+    def trim_capacity(self) -> None:
+        self.encoded_data.trim_capacity()
+        self.decoded_size.trim_capacity()
+
     def resize(self, new_size: int) -> None:
         """Resize vector along the first axis.
 
@@ -102,21 +113,6 @@ class VectorOfEncodedVectors(LGDO):
         self.encoded_data.resize(new_size)
         self.decoded_size.resize(new_size)
 
-    def append(self, value: tuple[NDArray, int]) -> None:
-        """Append a 1D encoded vector at the end.
-
-        Parameters
-        ----------
-        value
-            a tuple holding the encoded array and its decoded size.
-
-        See Also
-        --------
-        .VectorOfVectors.append
-        """
-        self.encoded_data.append(value[0])
-        self.decoded_size.append(value[1])
-
     def insert(self, i: int, value: tuple[NDArray, int]) -> None:
         """Insert an encoded vector at index `i`.
 
@@ -282,7 +278,7 @@ class VectorOfEncodedVectors(LGDO):
         raise ValueError(msg)
 
 
-class ArrayOfEncodedEqualSizedArrays(LGDO):
+class ArrayOfEncodedEqualSizedArrays(LGDOCollection):
     """An array of encoded arrays with equal decoded size.
 
     Used to represent an encoded :class:`.ArrayOfEqualSizedArrays`. In addition
@@ -349,14 +345,23 @@ class ArrayOfEncodedEqualSizedArrays(LGDO):
 
         return False
 
-    def resize(self, new_size: int) -> None:
+    def reserve_capacity(self, *capacity: int) -> None:
+        self.encoded_data.reserve_capacity(capacity)
+
+    def get_capacity(self) -> tuple:
+        return self.encoded_data.get_capacity()
+
+    def trim_capacity(self) -> None:
+        self.encoded_data.trim_capacity()
+
+    def resize(self, new_size: int, trim: bool = False) -> None:
         """Resize array along the first axis.
 
         See Also
         --------
         .VectorOfVectors.resize
         """
-        self.encoded_data.resize(new_size)
+        self.encoded_data.resize(new_size, trim)
 
     def append(self, value: NDArray) -> None:
         """Append a 1D encoded array at the end.

lgdo/types/histogram.py

@@ -424,7 +424,7 @@ class Histogram(Struct):
             dict.__setitem__(self, name, obj)
         else:
             msg = "histogram fields cannot be mutated "
-            raise TypeError(msg)
+            raise AttributeError(msg)
 
     def __getattr__(self, name: str) -> None:
         # do not allow for new attributes on this

lgdo/types/lgdo.py

@@ -92,3 +92,53 @@ class LGDO(ABC):
 
     def __repr__(self) -> str:
         return self.__class__.__name__ + f"(attrs={self.attrs!r})"
+
+
+class LGDOCollection(LGDO):
+    """Abstract base class representing a LEGEND Collection Object (LGDO).
+    This defines the interface for classes used as table columns.
+    """
+
+    @abstractmethod
+    def __init__(self, attrs: dict[str, Any] | None = None) -> None:
+        super().__init__(attrs)
+
+    @abstractmethod
+    def __len__(self) -> int:
+        """Provides ``__len__`` for this array-like class."""
+
+    @abstractmethod
+    def reserve_capacity(self, capacity: int) -> None:
+        """Reserve capacity (in rows) for later use. Internal memory buffers
+        will have enough entries to store this many rows.
+        """
+
+    @abstractmethod
+    def get_capacity(self) -> int:
+        "get reserved capacity of internal memory buffers in rows"
+
+    @abstractmethod
+    def trim_capacity(self) -> None:
+        """set capacity to only what is required to store current contents
+        of LGDOCollection
+        """
+
+    @abstractmethod
+    def resize(self, new_size: int, trim: bool = False) -> None:
+        """Return this LGDO's datatype attribute string."""
+
+    def append(self, val) -> None:
+        "append val to end of LGDOCollection"
+        self.insert(len(self), val)
+
+    @abstractmethod
+    def insert(self, i: int, val) -> None:
+        "insert val into LGDOCollection at position i"
+
+    @abstractmethod
+    def replace(self, i: int, val) -> None:
+        "replace item at position i with val in LGDOCollection"
+
+    def clear(self, trim: bool = False) -> None:
+        "set size of LGDOCollection to zero"
+        self.resize(0, trim=trim)