legend-pydataobj 1.11.8__py3-none-any.whl → 1.11.10__py3-none-any.whl

lgdo/lh5/store.py

@@ -5,20 +5,21 @@ HDF5 files.
 
 from __future__ import annotations
 
+import bisect
 import logging
+import os
 import sys
 from collections import OrderedDict
 from collections.abc import Mapping, Sequence
 from inspect import signature
-from pathlib import Path
 from typing import Any
 
 import h5py
+import numpy as np
 from numpy.typing import ArrayLike
 
 from .. import types
 from . import _serializers, utils
-from .core import read
 
 log = logging.getLogger(__name__)
 
@@ -92,16 +93,16 @@ class LH5Store:
             return self.files[lh5_file]
 
         if self.base_path != "":
-            full_path = Path(self.base_path) / lh5_file
+            full_path = os.path.join(self.base_path, lh5_file)
         else:
-            full_path = Path(lh5_file)
+            full_path = lh5_file
 
-        file_exists = full_path.exists()
+        file_exists = os.path.exists(full_path)
         if mode != "r":
-            directory = full_path.parent
-            if directory != "" and not full_path.parent.exists():
+            directory = os.path.dirname(full_path)
+            if directory != "" and not os.path.exists(directory):
                 log.debug(f"making path {directory}")
-                directory.mkdir(parents=True, exist_ok=True)
+                os.makedirs(directory)
 
         if mode == "r" and not file_exists:
             msg = f"file {full_path} not found"
@@ -154,7 +155,7 @@ class LH5Store:
         """Returns an LH5 object appropriate for use as a pre-allocated buffer
         in a read loop. Sets size to `size` if object has a size.
         """
-        obj = self.read(name, lh5_file, n_rows=0, field_mask=field_mask)
+        obj, n_rows = self.read(name, lh5_file, n_rows=0, field_mask=field_mask)
         if hasattr(obj, "resize") and size is not None:
             obj.resize(new_size=size)
         return obj
@@ -181,20 +182,72 @@ class LH5Store:
         """
         # grab files from store
         if isinstance(lh5_file, (str, h5py.File)):
-            h5f = self.gimme_file(lh5_file, "r", **file_kwargs)
+            lh5_obj = self.gimme_file(lh5_file, "r", **file_kwargs)[name]
         else:
-            h5f = [self.gimme_file(f, "r", **file_kwargs) for f in lh5_file]
-        return read(
-            name,
-            h5f,
-            start_row,
-            n_rows,
-            idx,
-            use_h5idx,
-            field_mask,
-            obj_buf,
-            obj_buf_start,
-            decompress,
+            lh5_files = list(lh5_file)
+            n_rows_read = 0
+
+            for i, h5f in enumerate(lh5_files):
+                if (
+                    isinstance(idx, (list, tuple))
+                    and len(idx) > 0
+                    and not np.isscalar(idx[0])
+                ):
+                    # a list of lists: must be one per file
+                    idx_i = idx[i]
+                elif idx is not None:
+                    # make idx a proper tuple if it's not one already
+                    if not (isinstance(idx, tuple) and len(idx) == 1):
+                        idx = (idx,)
+                    # idx is a long continuous array
+                    n_rows_i = utils.read_n_rows(name, h5f)
+                    # find the length of the subset of idx that contains indices
+                    # that are less than n_rows_i
+                    n_rows_to_read_i = bisect.bisect_left(idx[0], n_rows_i)
+                    # now split idx into idx_i and the remainder
+                    idx_i = np.array(idx[0])[:n_rows_to_read_i]
+                    idx = np.array(idx[0])[n_rows_to_read_i:] - n_rows_i
+                else:
+                    idx_i = None
+                n_rows_i = n_rows - n_rows_read
+
+                obj_buf, n_rows_read_i = self.read(
+                    name,
+                    h5f,
+                    start_row,
+                    n_rows_i,
+                    idx_i,
+                    use_h5idx,
+                    field_mask,
+                    obj_buf,
+                    obj_buf_start,
+                    decompress,
+                )
+
+                n_rows_read += n_rows_read_i
+                if n_rows_read >= n_rows or obj_buf is None:
+                    return obj_buf, n_rows_read
+                start_row = 0
+                obj_buf_start += n_rows_read_i
+            return obj_buf, n_rows_read
+
+        if isinstance(idx, (list, tuple)) and len(idx) > 0 and not np.isscalar(idx[0]):
+            idx = idx[0]
+        if isinstance(idx, np.ndarray) and idx.dtype == np.dtype("?"):
+            idx = np.where(idx)[0]
+
+        return _serializers._h5_read_lgdo(
+            lh5_obj.id,
+            lh5_obj.file.filename,
+            lh5_obj.name,
+            start_row=start_row,
+            n_rows=n_rows,
+            idx=idx,
+            use_h5idx=use_h5idx,
+            field_mask=field_mask,
+            obj_buf=obj_buf,
+            obj_buf_start=obj_buf_start,
+            decompress=decompress,
         )
 
     def write(

lgdo/lh5/tools.py

@@ -1,10 +1,16 @@
 from __future__ import annotations
 
 import fnmatch
+import glob
 import logging
+import os
 from copy import copy
+from warnings import warn
 
 import h5py
+import numpy as np
+import pandas as pd
+from numpy.typing import NDArray
 
 from . import utils
 from .store import LH5Store
@@ -217,3 +223,108 @@ def show(
             break
 
         key = k_new
+
+
+def load_nda(
+    f_list: str | list[str],
+    par_list: list[str],
+    lh5_group: str = "",
+    idx_list: list[NDArray | list | tuple] | None = None,
+) -> dict[str, NDArray]:
+    r"""Build a dictionary of :class:`numpy.ndarray`\ s from LH5 data.
+
+    Given a list of files, a list of LH5 table parameters, and an optional
+    group path, return a NumPy array with all values for each parameter.
+
+    Parameters
+    ----------
+    f_list
+        A list of files. Can contain wildcards.
+    par_list
+        A list of parameters to read from each file.
+    lh5_group
+        group path within which to find the specified parameters.
+    idx_list
+        for fancy-indexed reads. Must be one index array for each file in
+        `f_list`.
+
+    Returns
+    -------
+    par_data
+        A dictionary of the parameter data keyed by the elements of `par_list`.
+        Each entry contains the data for the specified parameter concatenated
+        over all files in `f_list`.
+    """
+    warn(
+        "load_nda() is deprecated. "
+        "Please replace it with LH5Store.read(...).view_as('np'), "
+        "or just read_as(..., 'np'). "
+        "load_nda() will be removed in a future release.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    if isinstance(f_list, str):
+        f_list = [f_list]
+        if idx_list is not None:
+            idx_list = [idx_list]
+    if idx_list is not None and len(f_list) != len(idx_list):
+        msg = f"f_list length ({len(f_list)}) != idx_list length ({len(idx_list)})!"
+        raise ValueError(msg)
+
+    # Expand wildcards
+    f_list = [f for f_wc in f_list for f in sorted(glob.glob(os.path.expandvars(f_wc)))]
+
+    sto = LH5Store()
+    par_data = {par: [] for par in par_list}
+    for ii, ff in enumerate(f_list):
+        f = sto.gimme_file(ff, "r")
+        for par in par_list:
+            if f"{lh5_group}/{par}" not in f:
+                msg = f"'{lh5_group}/{par}' not in file {ff}"
+                raise RuntimeError(msg)
+
+            if idx_list is None:
+                data, _ = sto.read(f"{lh5_group}/{par}", f)
+            else:
+                data, _ = sto.read(f"{lh5_group}/{par}", f, idx=idx_list[ii])
+            if not data:
+                continue
+            par_data[par].append(data.nda)
+    return {par: np.concatenate(par_data[par]) for par in par_list}
+
+
+def load_dfs(
+    f_list: str | list[str],
+    par_list: list[str],
+    lh5_group: str = "",
+    idx_list: list[NDArray | list | tuple] | None = None,
+) -> pd.DataFrame:
+    """Build a :class:`pandas.DataFrame` from LH5 data.
+
+    Given a list of files (can use wildcards), a list of LH5 columns, and
+    optionally the group path, return a :class:`pandas.DataFrame` with all
+    values for each parameter.
+
+    See Also
+    --------
+    :func:`load_nda`
+
+    Returns
+    -------
+    dataframe
+        contains columns for each parameter in `par_list`, and rows containing
+        all data for the associated parameters concatenated over all files in
+        `f_list`.
+    """
+    warn(
+        "load_dfs() is deprecated. "
+        "Please replace it with LH5Store.read(...).view_as('pd'), "
+        "or just read_as(..., 'pd'). "
+        "load_dfs() will be removed in a future release.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return pd.DataFrame(
+        load_nda(f_list, par_list, lh5_group=lh5_group, idx_list=idx_list)
+    )

lgdo/lh5/utils.py

@@ -7,7 +7,6 @@ import logging
 import os
 import string
 from collections.abc import Mapping, Sequence
-from pathlib import Path
 from typing import Any
 
 import h5py
@@ -154,7 +153,7 @@ def expand_vars(expr: str, substitute: dict[str, str] | None = None) -> str:
 
     # use provided mapping
     # then expand env variables
-    return os.path.expandvars(string.Template(str(expr)).safe_substitute(substitute))
+    return os.path.expandvars(string.Template(expr).safe_substitute(substitute))
 
 
 def expand_path(
@@ -184,15 +183,14 @@ def expand_path(
         Unique absolute path, or list of all absolute paths
     """
     if base_path is not None and base_path != "":
-        base_path = Path(os.path.expandvars(base_path)).expanduser()
-        path = base_path / path
+        base_path = os.path.expanduser(os.path.expandvars(base_path))
+        path = os.path.join(base_path, path)
 
     # first expand variables
     _path = expand_vars(path, substitute)
 
     # then expand wildcards
-    # pathlib glob works differently so use glob for now
-    paths = sorted(glob.glob(str(Path(_path).expanduser())))  # noqa: PTH207
+    paths = sorted(glob.glob(os.path.expanduser(_path)))
 
     if base_path is not None and base_path != "":
         paths = [os.path.relpath(p, base_path) for p in paths]

lgdo/lh5_store.py

@@ -0,0 +1,284 @@
+"""
+.. warning::
+    This subpackage is deprecated, use :mod:`lgdo.lh5`.
+"""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Iterator
+from typing import Union
+from warnings import warn
+
+import h5py
+import numpy as np
+import pandas as pd
+
+from . import lh5
+from .types import (
+    Array,
+    ArrayOfEncodedEqualSizedArrays,  # noqa: F401
+    ArrayOfEqualSizedArrays,  # noqa: F401
+    FixedSizeArray,  # noqa: F401
+    Histogram,  # noqa: F401
+    Scalar,
+    Struct,
+    Table,  # noqa: F401
+    VectorOfEncodedVectors,  # noqa: F401
+    VectorOfVectors,
+    WaveformTable,  # noqa: F401
+)
+
+LGDO = Union[Array, Scalar, Struct, VectorOfVectors]
+
+
+class LH5Iterator(lh5.LH5Iterator):
+    """
+    .. warning::
+        This class is deprecated, use :class:`lgdo.lh5.iterator.LH5Iterator`.
+
+    """
+
+    def __init__(
+        self,
+        lh5_files: str | list[str],
+        groups: str | list[str],
+        base_path: str = "",
+        entry_list: list[int] | list[list[int]] | None = None,
+        entry_mask: list[bool] | list[list[bool]] | None = None,
+        field_mask: dict[str, bool] | list[str] | tuple[str] | None = None,
+        buffer_len: int = 3200,
+        friend: Iterator | None = None,
+    ) -> None:
+        warn(
+            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator."
+            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
+            "lgdo.lh5_store will be removed in a future release.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(
+            lh5_files,
+            groups,
+            base_path,
+            entry_list,
+            entry_mask,
+            field_mask,
+            buffer_len,
+            friend,
+        )
+
+    def write_object(
+        self,
+        obj: LGDO,
+        name: str,
+        lh5_file: str | h5py.File,
+        group: str | h5py.Group = "/",
+        start_row: int = 0,
+        n_rows: int | None = None,
+        wo_mode: str = "append",
+        write_start: int = 0,
+        **h5py_kwargs,
+    ) -> None:
+        """
+        .. warning::
+            This method is deprecated, use :meth:`lgdo.lh5.iterator.LH5Iterator.write`.
+
+        """
+        warn(
+            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator. "
+            "The object you are calling this function from uses the old LH5Iterator class."
+            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
+            "lgdo.lh5_store will be removed in a future release.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        self.write(
+            obj,
+            name,
+            lh5_file,
+            group,
+            start_row,
+            n_rows,
+            wo_mode,
+            write_start,
+            h5py_kwargs,
+        )
+
+    def read_object(
+        self,
+        name: str,
+        lh5_file: str | h5py.File | list[str | h5py.File],
+        start_row: int = 0,
+        n_rows: int = sys.maxsize,
+        idx: np.ndarray | list | tuple | list[np.ndarray | list | tuple] = None,
+        field_mask: dict[str, bool] | list[str] | tuple[str] | None = None,
+        obj_buf: LGDO = None,
+        obj_buf_start: int = 0,
+        decompress: bool = True,
+    ) -> tuple[LGDO, int]:
+        """
+        .. warning::
+            This method is deprecated, use :meth:`lgdo.lh5.iterator.LH5Iterator.read`.
+
+        """
+        warn(
+            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Iterator. "
+            "The object you are calling this function from uses the old LH5Iterator class."
+            "Please replace 'from lgdo.lh5_store import LH5Iterator' with 'from lgdo.lh5 import LH5Iterator'."
+            "lgdo.lh5_store will be removed in a future release.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.read(
+            name,
+            lh5_file,
+            start_row,
+            n_rows,
+            idx,
+            field_mask,
+            obj_buf,
+            obj_buf_start,
+            decompress,
+        )
+
+
+class LH5Store(lh5.LH5Store):
+    """
+    .. warning::
+        This class is deprecated, use :class:`lgdo.lh5.iterator.LH5Store`.
+
+    """
+
+    def __init__(self, base_path: str = "", keep_open: bool = False):
+        warn(
+            "lgdo.lh5_store has moved to a subfolder lgdo.lh5 containing LH5Store. "
+            "Please replace 'from lgdo.lh5_store import LH5Store' with 'from lgdo.lh5 import LH5Store'."
+            "lgdo.lh5_store will be removed in a future release.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(base_path, keep_open)
+
+    def read_object(
+        self,
+        name: str,
+        lh5_file: str | h5py.File | list[str | h5py.File],
+        **kwargs,
+    ) -> tuple[LGDO, int]:
+        """
+        .. warning::
+            This method is deprecated, use :meth:`lgdo.lh5.store.LH5Store.read`.
+
+        """
+        warn(
+            "LH5Store.read_object() has been renamed to LH5Store.read(), "
+            "Please update your code."
+            "LH5Store.read_object() will be removed in a future release.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return super().read(self, name, lh5_file, **kwargs)
+
+    def write_object(
+        self,
+        obj: LGDO,
+        name: str,
+        lh5_file: str | h5py.File,
+        **kwargs,
+    ) -> tuple[LGDO, int]:
+        """
+        .. warning::
+            This method is deprecated, use :meth:`lgdo.lh5.store.LH5Store.write`.
+
+        """
+        warn(
+            "LH5Store.write_object() has been renamed to LH5Store.write(), "
+            "Please update your code."
+            "LH5Store.write_object() will be removed in a future release.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return super().read(self, obj, name, lh5_file, **kwargs)
+
+
+def load_dfs(
+    f_list: str | list[str],
+    par_list: list[str],
+    lh5_group: str = "",
+    idx_list: list[np.ndarray | list | tuple] | None = None,
+) -> pd.DataFrame:
+    """
+    .. warning::
+        This function is deprecated, use :meth:`lgdo.types.lgdo.LGDO.view_as` to
+        view LGDO data as a Pandas data structure.
+
+    """
+    warn(
+        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
+        "Please replace 'from lgdo.lh5_store import load_dfs' with 'from lgdo.lh5 import load_dfs'. "
+        "lgdo.lh5_store will be removed in a future release.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return lh5.load_dfs(f_list, par_list, lh5_group, idx_list)
+
+
+def load_nda(
+    f_list: str | list[str],
+    par_list: list[str],
+    lh5_group: str = "",
+    idx_list: list[np.ndarray | list | tuple] | None = None,
+) -> dict[str, np.ndarray]:
+    """
+    .. warning::
+        This function is deprecated, use :meth:`lgdo.types.lgdo.LGDO.view_as` to
+        view LGDO data as a NumPy data structure.
+
+    """
+    warn(
+        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
+        "Please replace 'from lgdo.lh5_store import load_nda' with 'from lgdo.lh5 import load_nda'. "
+        "lgdo.lh5_store will be removed in a future release.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return lh5.load_nda(f_list, par_list, lh5_group, idx_list)
+
+
+def ls(lh5_file: str | h5py.Group, lh5_group: str = "") -> list[str]:
+    """
+    .. warning::
+        This function is deprecated, import :func:`lgdo.lh5.tools.ls`.
+
+    """
+    warn(
+        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
+        "Please replace 'from lgdo.lh5_store import ls' with 'from lgdo.lh5 import ls'. "
+        "lgdo.lh5_store will be removed in a future release.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return lh5.ls(lh5_file, lh5_group)
+
+
+def show(
+    lh5_file: str | h5py.Group,
+    lh5_group: str = "/",
+    attrs: bool = False,
+    indent: str = "",
+    header: bool = True,
+) -> None:
+    """
+    .. warning::
+        This function is deprecated, import :func:`lgdo.lh5.tools.show`.
+
+    """
+    warn(
+        "lgdo.lh5_store has moved to a subfolder lgdo.lh5. "
+        "Please replace 'from lgdo.lh5_store import show' with 'from lgdo.lh5 import show'. "
+        "lgdo.lh5_store will be removed in a future release.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    lh5.show(lh5_file, lh5_group, attrs, indent, header)