legend-pydataobj 1.9.0__tar.gz → 1.10.0__tar.gz

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: legend_pydataobj
-Version: 1.9.0
+Version: 1.10.0
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration
@@ -700,7 +700,7 @@ License-File: LICENSE
 Requires-Dist: awkward>=2
 Requires-Dist: awkward-pandas
 Requires-Dist: colorlog
-Requires-Dist: h5py>=3.2
+Requires-Dist: h5py>=3.10
 Requires-Dist: hdf5plugin
 Requires-Dist: hist
 Requires-Dist: numba!=0.53.*,!=0.54.*

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.0}/pyproject.toml

@@ -34,7 +34,7 @@ dependencies = [
     "awkward>=2",
     "awkward-pandas",
     "colorlog",
-    "h5py>=3.2",
+    "h5py>=3.10",
     "hdf5plugin",
     "hist",
     "numba!=0.53.*,!=0.54.*",

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.0}/src/legend_pydataobj.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: legend_pydataobj
-Version: 1.9.0
+Version: 1.10.0
 Summary: LEGEND Python Data Objects
 Author: The LEGEND Collaboration
 Maintainer: The LEGEND Collaboration
@@ -700,7 +700,7 @@ License-File: LICENSE
 Requires-Dist: awkward>=2
 Requires-Dist: awkward-pandas
 Requires-Dist: colorlog
-Requires-Dist: h5py>=3.2
+Requires-Dist: h5py>=3.10
 Requires-Dist: hdf5plugin
 Requires-Dist: hist
 Requires-Dist: numba!=0.53.*,!=0.54.*

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.0}/src/legend_pydataobj.egg-info/requires.txt

@@ -1,7 +1,7 @@
 awkward>=2
 awkward-pandas
 colorlog
-h5py>=3.2
+h5py>=3.10
 hdf5plugin
 hist
 numba!=0.53.*,!=0.54.*

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.0}/src/lgdo/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.9.0'
-__version_tuple__ = version_tuple = (1, 9, 0)
+__version__ = version = '1.10.0'
+__version_tuple__ = version_tuple = (1, 10, 0)

legend_pydataobj-1.10.0/src/lgdo/lh5/_serializers/read/array.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import logging
+
+from ....types import Array, ArrayOfEqualSizedArrays, FixedSizeArray
+from . import utils
+from .ndarray import _h5_read_ndarray
+
+log = logging.getLogger(__name__)
+
+
+def _h5_read_array_generic(type_, h5d, fname, oname, **kwargs):
+    nda, attrs, n_rows_to_read = _h5_read_ndarray(h5d, fname, oname, **kwargs)
+
+    obj_buf = kwargs["obj_buf"]
+
+    if obj_buf is None:
+        return type_(nda=nda, attrs=attrs), n_rows_to_read
+
+    utils.check_obj_buf_attrs(obj_buf.attrs, attrs, fname, oname)
+
+    return obj_buf, n_rows_to_read
+
+
+def _h5_read_array(h5d, fname, oname, **kwargs):
+    return _h5_read_array_generic(Array, h5d, fname, oname, **kwargs)
+
+
+def _h5_read_fixedsize_array(h5d, fname, oname, **kwargs):
+    return _h5_read_array_generic(FixedSizeArray, h5d, fname, oname, **kwargs)
+
+
+def _h5_read_array_of_equalsized_arrays(h5d, fname, oname, **kwargs):
+    return _h5_read_array_generic(ArrayOfEqualSizedArrays, h5d, fname, oname, **kwargs)

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.0}/src/lgdo/lh5/_serializers/read/composite.py

@@ -23,7 +23,6 @@ from ....types import (
 )
 from ... import datatype as dtypeutils
 from ...exceptions import LH5DecodeError
-from ...utils import read_n_rows
 from . import utils
 from .array import (
     _h5_read_array,
@@ -42,6 +41,8 @@ log = logging.getLogger(__name__)
 
 def _h5_read_lgdo(
     h5o,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -51,69 +52,23 @@ def _h5_read_lgdo(
     obj_buf_start=0,
     decompress=True,
 ):
-    # Handle list-of-files recursively
-    if not isinstance(h5o, (h5py.Group, h5py.Dataset)):
-        lh5_objs = list(h5o)
-        n_rows_read = 0
-
-        for i, _h5o in enumerate(lh5_objs):
-            if isinstance(idx, list) and len(idx) > 0 and not np.isscalar(idx[0]):
-                # a list of lists: must be one per file
-                idx_i = idx[i]
-            elif idx is not None:
-                # make idx a proper tuple if it's not one already
-                if not (isinstance(idx, tuple) and len(idx) == 1):
-                    idx = (idx,)
-                # idx is a long continuous array
-                n_rows_i = read_n_rows(_h5o)
-                # find the length of the subset of idx that contains indices
-                # that are less than n_rows_i
-                n_rows_to_read_i = bisect.bisect_left(idx[0], n_rows_i)
-                # now split idx into idx_i and the remainder
-                idx_i = (idx[0][:n_rows_to_read_i],)
-                idx = (idx[0][n_rows_to_read_i:] - n_rows_i,)
-            else:
-                idx_i = None
-            n_rows_i = n_rows - n_rows_read
-
-            obj_buf, n_rows_read_i = _h5_read_lgdo(
-                _h5o,
-                start_row=start_row,
-                n_rows=n_rows_i,
-                idx=idx_i,
-                use_h5idx=use_h5idx,
-                field_mask=field_mask,
-                obj_buf=obj_buf,
-                obj_buf_start=obj_buf_start,
-                decompress=decompress,
-            )
-
-            n_rows_read += n_rows_read_i
-            if n_rows_read >= n_rows or obj_buf is None:
-                return obj_buf, n_rows_read
-            start_row = 0
-            obj_buf_start += n_rows_read_i
-
-        return obj_buf, n_rows_read
-
     log.debug(
-        f"reading {h5o.file.filename}:{h5o.name}[{start_row}:{n_rows}], decompress = {decompress}, "
+        f"reading {fname}:{oname}[{start_row}:{n_rows}], decompress = {decompress}, "
         + (f" with field mask {field_mask}" if field_mask else "")
     )
 
-    # make idx a proper tuple if it's not one already
-    if not (isinstance(idx, tuple) and len(idx) == 1) and idx is not None:
-        idx = (idx,)
-
+    attrs = utils.read_attrs(h5o, fname, oname)
     try:
-        lgdotype = dtypeutils.datatype(h5o.attrs["datatype"])
+        lgdotype = dtypeutils.datatype(attrs["datatype"])
     except KeyError as e:
         msg = "dataset not in file or missing 'datatype' attribute"
-        raise LH5DecodeError(msg, h5o) from e
+        raise LH5DecodeError(msg, fname, oname) from e
 
     if lgdotype is Scalar:
         return _h5_read_scalar(
             h5o,
+            fname,
+            oname,
             obj_buf=obj_buf,
         )
 
@@ -125,7 +80,7 @@ def _h5_read_lgdo(
         if len(field_mask) > 0:
             default = not field_mask[next(iter(field_mask.keys()))]
         field_mask = defaultdict(lambda: default, field_mask)
-    elif isinstance(field_mask, (list, tuple)):
+    elif isinstance(field_mask, (list, tuple, set)):
         field_mask = defaultdict(bool, {field: True for field in field_mask})
     elif not isinstance(field_mask, defaultdict):
         msg = "bad field_mask type"
@@ -134,6 +89,8 @@ def _h5_read_lgdo(
     if lgdotype is Struct:
         return _h5_read_struct(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -145,20 +102,22 @@ def _h5_read_lgdo(
     # Below here is all array-like types. So trim idx if needed
     if idx is not None:
         # check if idx is just an ordered list of the integers if so can ignore
-        if (idx[0] == np.arange(0, len(idx[0]), 1)).all():
-            if n_rows > len(idx[0]):
-                n_rows = len(idx[0])
+        if (idx == np.arange(0, len(idx), 1)).all():
+            if n_rows > len(idx):
+                n_rows = len(idx)
             idx = None
         else:
             # chop off indices < start_row
-            i_first_valid = bisect.bisect_left(idx[0], start_row)
-            idxa = idx[0][i_first_valid:]
+            i_first_valid = bisect.bisect_left(idx, start_row)
+            idxa = idx[i_first_valid:]
             # don't readout more than n_rows indices
-            idx = (idxa[:n_rows],)  # works even if n_rows > len(idxa)
+            idx = idxa[:n_rows]  # works even if n_rows > len(idxa)
 
     if lgdotype is Table:
         return _h5_read_table(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -172,6 +131,8 @@ def _h5_read_lgdo(
     if lgdotype is Histogram:
         return _h5_read_histogram(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -185,6 +146,8 @@ def _h5_read_lgdo(
     if lgdotype is ArrayOfEncodedEqualSizedArrays:
         return _h5_read_array_of_encoded_equalsized_arrays(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -197,6 +160,8 @@ def _h5_read_lgdo(
     if lgdotype is VectorOfEncodedVectors:
         return _h5_read_vector_of_encoded_vectors(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -209,6 +174,8 @@ def _h5_read_lgdo(
     if lgdotype is VectorOfVectors:
         return _h5_read_vector_of_vectors(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -220,6 +187,8 @@ def _h5_read_lgdo(
     if lgdotype is FixedSizeArray:
         return _h5_read_fixedsize_array(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -231,6 +200,8 @@ def _h5_read_lgdo(
     if lgdotype is ArrayOfEqualSizedArrays:
         return _h5_read_array_of_equalsized_arrays(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -242,6 +213,8 @@ def _h5_read_lgdo(
     if lgdotype is Array:
         return _h5_read_array(
             h5o,
+            fname,
+            oname,
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -251,11 +224,13 @@ def _h5_read_lgdo(
         )
 
     msg = f"no rule to decode {lgdotype.__name__} from LH5"
-    raise LH5DecodeError(msg, h5o)
+    raise LH5DecodeError(msg, fname, oname)
 
 
 def _h5_read_struct(
     h5g,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -268,7 +243,7 @@ def _h5_read_struct(
     # table... Maybe should emit a warning? Or allow them to be
     # dicts keyed by field name?
 
-    attrs = dict(h5g.attrs)
+    attrs = utils.read_attrs(h5g, fname, oname)
 
     # determine fields to be read out
     all_fields = dtypeutils.get_struct_fields(attrs["datatype"])
@@ -286,20 +261,26 @@ def _h5_read_struct(
     for field in selected_fields:
         # support for integer keys
         field_key = int(field) if attrs.get("int_keys") else str(field)
+        h5o = h5py.h5o.open(h5g, field.encode("utf-8"))
         obj_dict[field_key], _ = _h5_read_lgdo(
-            h5g[field],
+            h5o,
+            fname,
+            f"{oname}/{field}",
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
             use_h5idx=use_h5idx,
             decompress=decompress,
         )
+        h5o.close()
 
     return Struct(obj_dict=obj_dict, attrs=attrs), 1
 
 
 def _h5_read_table(
     h5g,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -311,9 +292,9 @@ def _h5_read_table(
 ):
     if obj_buf is not None and not isinstance(obj_buf, Table):
         msg = "provided object buffer is not a Table"
-        raise LH5DecodeError(msg, h5g)
+        raise LH5DecodeError(msg, fname, oname)
 
-    attrs = dict(h5g.attrs)
+    attrs = utils.read_attrs(h5g, fname, oname)
 
     # determine fields to be read out
     all_fields = dtypeutils.get_struct_fields(attrs["datatype"])
@@ -334,12 +315,15 @@ def _h5_read_table(
         if obj_buf is not None:
             if not isinstance(obj_buf, Table) or field not in obj_buf:
                 msg = "provided object buffer is not a Table or columns are missing"
-                raise LH5DecodeError(msg, h5g)
+                raise LH5DecodeError(msg, fname, oname)
 
             fld_buf = obj_buf[field]
 
+        h5o = h5py.h5o.open(h5g, field.encode("utf-8"))
         col_dict[field], n_rows_read = _h5_read_lgdo(
-            h5g[field],
+            h5o,
+            fname,
+            f"{oname}/{field}",
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -348,6 +332,7 @@ def _h5_read_table(
             obj_buf_start=obj_buf_start,
             decompress=decompress,
         )
+        h5o.close()
 
         if obj_buf is not None and obj_buf_start + n_rows_read > len(obj_buf):
             obj_buf.resize(obj_buf_start + n_rows_read)
@@ -359,12 +344,12 @@ def _h5_read_table(
         n_rows_read = rows_read[0]
     else:
         n_rows_read = 0
-        log.warning(f"Table '{h5g.name}' has no fields specified by {field_mask=}")
+        log.warning(f"Table '{oname}' has no fields specified by {field_mask=}")
 
     for n in rows_read[1:]:
         if n != n_rows_read:
             log.warning(
-                f"Table '{h5g.name}' got strange n_rows_read = {n}, "
+                f"Table '{oname}' got strange n_rows_read = {n}, "
                 "{n_rows_read} was expected ({rows_read})"
             )
 
@@ -396,13 +381,15 @@ def _h5_read_table(
     obj_buf.loc = obj_buf_start + n_rows_read
 
     # check attributes
-    utils.check_obj_buf_attrs(obj_buf.attrs, attrs, h5g)
+    utils.check_obj_buf_attrs(obj_buf.attrs, attrs, fname, oname)
 
     return obj_buf, n_rows_read
 
 
 def _h5_read_histogram(
     h5g,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -414,17 +401,20 @@ def _h5_read_histogram(
 ):
     if obj_buf is not None or obj_buf_start != 0:
         msg = "reading a histogram into an existing object buffer is not supported"
-        raise LH5DecodeError(msg, h5g)
+        raise LH5DecodeError(msg, fname, oname)
 
     struct, n_rows_read = _h5_read_struct(
         h5g,
-        start_row,
-        n_rows,
-        idx,
-        use_h5idx,
-        field_mask,
-        decompress,
+        fname,
+        oname,
+        start_row=start_row,
+        n_rows=n_rows,
+        idx=idx,
+        use_h5idx=use_h5idx,
+        field_mask=field_mask,
+        decompress=decompress,
     )
+
     binning = []
     for _, a in struct.binning.items():
         be = a.binedges
@@ -434,7 +424,7 @@ def _h5_read_histogram(
             b = (be, None, None, None, a.closedleft.value)
         else:
             msg = "unexpected binning of histogram"
-            raise LH5DecodeError(msg, h5g)
+            raise LH5DecodeError(msg, fname, oname)
         ax = Histogram.Axis(*b)
         # copy attrs to "clone" the "whole" struct.
         ax.attrs = a.getattrs(datatype=True)

{legend_pydataobj-1.9.0 → legend_pydataobj-1.10.0}/src/lgdo/lh5/_serializers/read/encoded.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 import logging
 import sys
 
+import h5py
+
 from .... import compression as compress
 from ....types import (
     ArrayOfEncodedEqualSizedArrays,
@@ -13,6 +15,7 @@ from .array import (
     _h5_read_array,
 )
 from .scalar import _h5_read_scalar
+from .utils import read_attrs
 from .vector_of_vectors import _h5_read_vector_of_vectors
 
 log = logging.getLogger(__name__)
@@ -20,21 +23,29 @@ log = logging.getLogger(__name__)
 
 def _h5_read_array_of_encoded_equalsized_arrays(
     h5g,
+    fname,
+    oname,
     **kwargs,
 ):
-    return _h5_read_encoded_array(ArrayOfEncodedEqualSizedArrays, h5g, **kwargs)
+    return _h5_read_encoded_array(
+        ArrayOfEncodedEqualSizedArrays, h5g, fname, oname, **kwargs
+    )
 
 
 def _h5_read_vector_of_encoded_vectors(
     h5g,
+    fname,
+    oname,
     **kwargs,
 ):
-    return _h5_read_encoded_array(VectorOfEncodedVectors, h5g, **kwargs)
+    return _h5_read_encoded_array(VectorOfEncodedVectors, h5g, fname, oname, **kwargs)
 
 
 def _h5_read_encoded_array(
     lgdotype,
     h5g,
+    fname,
+    oname,
     start_row=0,
     n_rows=sys.maxsize,
     idx=None,
@@ -45,11 +56,11 @@ def _h5_read_encoded_array(
 ):
     if lgdotype not in (ArrayOfEncodedEqualSizedArrays, VectorOfEncodedVectors):
         msg = f"unsupported read of encoded type {lgdotype.__name__}"
-        raise LH5DecodeError(msg, h5g)
+        raise LH5DecodeError(msg, fname, oname)
 
     if not decompress and obj_buf is not None and not isinstance(obj_buf, lgdotype):
         msg = f"object buffer is not a {lgdotype.__name__}"
-        raise LH5DecodeError(msg, h5g)
+        raise LH5DecodeError(msg, fname, oname)
 
     # read out decoded_size, either a Scalar or an Array
     decoded_size_buf = encoded_data_buf = None
@@ -58,8 +69,11 @@ def _h5_read_encoded_array(
         encoded_data_buf = obj_buf.encoded_data
 
     if lgdotype is VectorOfEncodedVectors:
+        h5o = h5py.h5o.open(h5g, b"decoded_size")
         decoded_size, _ = _h5_read_array(
-            h5g["decoded_size"],
+            h5o,
+            fname,
+            f"{oname}/decoded_size",
             start_row=start_row,
             n_rows=n_rows,
             idx=idx,
@@ -67,16 +81,23 @@ def _h5_read_encoded_array(
             obj_buf=None if decompress else decoded_size_buf,
             obj_buf_start=0 if decompress else obj_buf_start,
         )
-
+        h5o.close()
     else:
+        h5o = h5py.h5o.open(h5g, b"decoded_size")
         decoded_size, _ = _h5_read_scalar(
-            h5g["decoded_size"],
+            h5o,
+            fname,
+            f"{oname}/decoded_size",
             obj_buf=None if decompress else decoded_size_buf,
         )
+        h5o.close()
 
     # read out encoded_data, a VectorOfVectors
+    h5o = h5py.h5o.open(h5g, b"encoded_data")
     encoded_data, n_rows_read = _h5_read_vector_of_vectors(
-        h5g["encoded_data"],
+        h5o,
+        fname,
+        f"{oname}/encoded_data",
         start_row=start_row,
         n_rows=n_rows,
         idx=idx,
@@ -84,6 +105,7 @@ def _h5_read_encoded_array(
         obj_buf=None if decompress else encoded_data_buf,
         obj_buf_start=0 if decompress else obj_buf_start,
     )
+    h5o.close()
 
     # return the still encoded data in the buffer object, if there
     if obj_buf is not None and not decompress:
@@ -93,7 +115,7 @@ def _h5_read_encoded_array(
     rawdata = lgdotype(
         encoded_data=encoded_data,
         decoded_size=decoded_size,
-        attrs=dict(h5g.attrs),
+        attrs=read_attrs(h5g, fname, oname),
     )
 
     # already return if no decompression is requested

legend_pydataobj-1.10.0/src/lgdo/lh5/_serializers/read/ndarray.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import logging
+import sys
+from bisect import bisect_left
+
+import h5py
+import numpy as np
+
+from ....types import Array
+from ... import datatype
+from ...exceptions import LH5DecodeError
+from .utils import read_attrs
+
+log = logging.getLogger(__name__)
+
+
+def _h5_read_ndarray(
+    h5d,
+    fname,
+    oname,
+    start_row=0,
+    n_rows=sys.maxsize,
+    idx=None,
+    use_h5idx=False,
+    obj_buf=None,
+    obj_buf_start=0,
+):
+    if obj_buf is not None and not isinstance(obj_buf, Array):
+        msg = "object buffer is not an Array"
+        raise LH5DecodeError(msg, fname, oname)
+
+    # compute the number of rows to read
+    # we culled idx above for start_row and n_rows, now we have to apply
+    # the constraint of the length of the dataset
+    try:
+        fspace = h5d.get_space()
+        ds_n_rows = fspace.shape[0]
+    except AttributeError as e:
+        msg = "does not seem to be an HDF5 dataset"
+        raise LH5DecodeError(msg, fname, oname) from e
+
+    if idx is not None:
+        if len(idx) > 0 and idx[-1] >= ds_n_rows:
+            log.warning("idx indexed past the end of the array in the file. Culling...")
+            n_rows_to_read = bisect_left(idx[0], ds_n_rows)
+            idx = (idx[:n_rows_to_read],)
+            if len(idx) == 0:
+                log.warning("idx empty after culling.")
+        n_rows_to_read = len(idx)
+    else:
+        n_rows_to_read = ds_n_rows - start_row
+    if n_rows_to_read > n_rows:
+        n_rows_to_read = n_rows
+
+    if idx is None:
+        fspace.select_hyperslab(
+            (start_row,) + (0,) * (h5d.rank - 1),
+            (1,) * h5d.rank,
+            None,
+            (n_rows_to_read,) + fspace.shape[1:],
+        )
+    elif use_h5idx:
+        # Note that h5s will automatically merge adjacent elements into a range
+        fspace.select_none()
+        for i in idx:
+            fspace.select_hyperslab(
+                (i,) + (0,) * (h5d.rank - 1),
+                (1,) * h5d.rank,
+                None,
+                (1,) + fspace.shape[1:],
+                h5py.h5s.SELECT_OR,
+            )
+
+    # Now read the array
+    if obj_buf is not None and n_rows_to_read > 0:
+        buf_size = obj_buf_start + n_rows_to_read
+        if len(obj_buf) < buf_size:
+            obj_buf.resize(buf_size)
+        dest_sel = np.s_[obj_buf_start:buf_size]
+
+        if idx is None or use_h5idx:
+            mspace = h5py.h5s.create_simple(obj_buf.nda.shape)
+            mspace.select_hyperslab(
+                (obj_buf_start,) + (0,) * (h5d.rank - 1),
+                (1,) * h5d.rank,
+                None,
+                (n_rows_to_read,) + fspace.shape[1:],
+            )
+            h5d.read(mspace, fspace, obj_buf.nda)
+        else:
+            tmp = np.empty(fspace.shape, h5d.dtype)
+            h5d.read(fspace, fspace, tmp)
+            obj_buf.nda[dest_sel, ...] = tmp[idx, ...]
+        nda = obj_buf.nda
+    elif n_rows == 0:
+        tmp_shape = (0,) + h5d.shape[1:]
+        nda = np.empty(tmp_shape, h5d.dtype)
+    else:
+        mspace = h5py.h5s.create_simple((n_rows_to_read,) + fspace.shape[1:])
+        nda = np.empty(mspace.shape, h5d.dtype)
+        if idx is None or use_h5idx:
+            h5d.read(mspace, fspace, nda)
+        else:
+            tmp = np.empty(fspace.shape, h5d.dtype)
+            h5d.read(fspace, fspace, tmp)
+            nda[:, ...] = tmp[idx, ...]
+
+    # Finally, set attributes and return objects
+    attrs = read_attrs(h5d, fname, oname)
+
+    # special handling for bools
+    # (c and Julia store as uint8 so cast to bool)
+    if datatype.get_nested_datatype_string(attrs["datatype"]) == "bool":
+        nda = nda.astype(np.bool_)
+
+    return (nda, attrs, n_rows_to_read)