uproot 5.3.7__tar.gz → 5.3.8__tar.gz

{uproot-5.3.7 → uproot-5.3.8}/.github/workflows/deploy.yml

@@ -21,6 +21,10 @@ jobs:
     needs: [build_dist]
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && github.event.action == 'published'
+    permissions:
+      id-token: write
+      attestations: write
+      contents: read
 
     steps:
       - uses: actions/download-artifact@v4
@@ -31,6 +35,11 @@ jobs:
       - name: List distributions to be deployed
         run: ls -lha dist/
 
+      - name: Generate artifact attestation for sdist and wheel
+        uses: actions/attest-build-provenance@173725a1209d09b31f9d30a3890cf2757ebbff0d # v1.1.2
+        with:
+          subject-path: dist/uproot-*
+
       - uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.pypi_password }}

{uproot-5.3.7 → uproot-5.3.8}/.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
       - id: black
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.3
+    rev: v0.4.7
     hooks:
       - id: ruff
         args: [--fix, --show-fixes]

{uproot-5.3.7 → uproot-5.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: uproot
-Version: 5.3.7
+Version: 5.3.8
 Summary: ROOT I/O in pure Python and NumPy.
 Project-URL: Download, https://github.com/scikit-hep/uproot5/releases
 Project-URL: Homepage, https://github.com/scikit-hep/uproot5

{uproot-5.3.7 → uproot-5.3.8}/src/uproot/_util.py

@@ -13,7 +13,6 @@ import itertools
 import numbers
 import os
 import re
-import sys
 import warnings
 from collections.abc import Iterable
 from pathlib import Path
@@ -28,8 +27,6 @@ import uproot.source.chunk
 import uproot.source.fsspec
 import uproot.source.object
 
-macos = sys.platform == "darwin"
-
 
 def tobytes(array):
     """

{uproot-5.3.7 → uproot-5.3.8}/src/uproot/models/RNTuple.py

@@ -480,9 +480,13 @@ in file {self.file.file_path}"""
         pagelist = linklist[ncol]
         dtype_byte = self.column_records[ncol].type
         dtype_str = uproot.const.rntuple_col_num_to_dtype_dict[dtype_byte]
-        dtype = numpy.dtype("bool") if dtype_str == "bit" else numpy.dtype(dtype_str)
-
         total_len = numpy.sum([desc.num_elements for desc in pagelist], dtype=int)
+        if dtype_str == "switch":
+            dtype = numpy.dtype([("index", "int64"), ("tag", "int32")])
+        elif dtype_str == "bit":
+            dtype = numpy.dtype("bool")
+        else:
+            dtype = numpy.dtype(dtype_str)
         res = numpy.empty(total_len, dtype)
         split = 14 <= dtype_byte <= 21 or 26 <= dtype_byte <= 28
         zigzag = 26 <= dtype_byte <= 28
@@ -568,8 +572,8 @@ in file {self.file.file_path}"""
 
 # Supporting function and classes
 def _split_switch_bits(content):
-    kindex = numpy.bitwise_and(content, numpy.int64(0x00000000000FFFFF))
-    tags = (content >> 44).astype("int8") - 1
+    tags = content["tag"].astype(numpy.dtype("int8")) - 1
+    kindex = content["index"]
     return kindex, tags
 
 

uproot-5.3.8/src/uproot/source/coalesce.py

@@ -0,0 +1,132 @@
+"""Read coalescing algorithms
+
+Inspired in part by https://github.com/cms-sw/cmssw/blob/master/IOPool/TFileAdaptor/src/ReadRepacker.h
+"""
+
+from __future__ import annotations
+
+import queue
+from concurrent.futures import Future
+from dataclasses import dataclass
+from typing import Callable
+
+import uproot.source.chunk
+
+
+@dataclass
+class CoalesceConfig:
+    max_range_gap: int = 32 * 1024
+    max_request_ranges: int = 1024
+    max_request_bytes: int = 10 * 1024 * 1024
+    min_first_request_bytes: int = 32 * 1024
+
+
+DEFAULT_CONFIG = CoalesceConfig()
+
+
+class SliceFuture:
+    def __init__(self, parent: Future, s: slice | int):
+        self._parent = parent
+        self._s = s
+
+    def add_done_callback(self, callback, *, context=None):
+        self._parent.add_done_callback(callback)
+
+    def result(self, timeout=None):
+        return self._parent.result(timeout=timeout)[self._s]
+
+
+@dataclass
+class RangeRequest:
+    start: int
+    stop: int
+    future: Future | None
+
+
+@dataclass
+class Cluster:
+    ranges: list[RangeRequest]
+
+    @property
+    def start(self):
+        # since these are built from sorted ranges, this is the min start
+        return self.ranges[0].start
+
+    @property
+    def stop(self):
+        return max(range.stop for range in self.ranges)
+
+    def __len__(self):
+        return self.stop - self.start
+
+    def set_future(self, future: Future):
+        for range in self.ranges:
+            local_start = range.start - self.start
+            local_stop = range.stop - self.start
+            range.future = SliceFuture(future, slice(local_start, local_stop))
+
+
+@dataclass
+class CoalescedRequest:
+    clusters: list[Cluster]
+
+    def ranges(self):
+        return [(cluster.start, cluster.stop) for cluster in self.clusters]
+
+    def set_future(self, future: Future):
+        for i, cluster in enumerate(self.clusters):
+            cluster.set_future(SliceFuture(future, i))
+
+
+def _merge_adjacent(ranges: list[RangeRequest], config: CoalesceConfig):
+    sorted_ranges = sorted(ranges, key=lambda r: r.start)
+    cluster = Cluster([])
+    for current_range in sorted_ranges:
+        if cluster.ranges and current_range.start - cluster.stop > config.max_range_gap:
+            yield cluster
+            cluster = Cluster([])
+        cluster.ranges.append(current_range)
+    if cluster.ranges:
+        yield cluster
+
+
+def _coalesce(ranges: list[RangeRequest], config: CoalesceConfig):
+    clusters: list[Cluster] = []
+    request_bytes: int = 0
+    first_request = True
+    for cluster in _merge_adjacent(ranges, config):
+        if clusters and (
+            len(clusters) + 1 >= config.max_request_ranges
+            or request_bytes + len(cluster) >= config.max_request_bytes
+            or (first_request and request_bytes >= config.min_first_request_bytes)
+        ):
+            yield CoalescedRequest(clusters)
+            clusters = []
+            request_bytes = 0
+            first_request = False
+        clusters.append(cluster)
+        request_bytes += len(cluster)
+    if clusters:
+        yield CoalescedRequest(clusters)
+
+
+def coalesce_requests(
+    ranges: list[tuple[int, int]],
+    submit_fn: Callable[[list[tuple[int, int]]], Future],
+    source: uproot.source.chunk.Source,
+    notifications: queue.Queue,
+    config: CoalesceConfig | None = None,
+):
+    if config is None:
+        config = DEFAULT_CONFIG
+    all_requests = [RangeRequest(start, stop, None) for start, stop in ranges]
+    for merged_request in _coalesce(all_requests, config):
+        future = submit_fn(merged_request.ranges())
+        merged_request.set_future(future)
+
+    def chunkify(req: RangeRequest):
+        chunk = uproot.source.chunk.Chunk(source, req.start, req.stop, req.future)
+        req.future.add_done_callback(uproot.source.chunk.notifier(chunk, notifications))
+        return chunk
+
+    return list(map(chunkify, all_requests))

{uproot-5.3.7 → uproot-5.3.8}/src/uproot/source/fsspec.py

@@ -12,26 +12,14 @@ import fsspec.asyn
 import uproot
 import uproot.source.chunk
 import uproot.source.futures
-
-
-class PartFuture:
-    """For splitting the result of fs._cat_ranges into its components"""
-
-    def __init__(self, parent_future: concurrent.futures.Future, part_index: int):
-        self._parent = parent_future
-        self._part_index = part_index
-
-    def add_done_callback(self, callback, *, context=None):
-        self._parent.add_done_callback(callback)
-
-    def result(self, timeout=None):
-        return self._parent.result(timeout=timeout)[self._part_index]
+from uproot.source.coalesce import CoalesceConfig, coalesce_requests
 
 
 class FSSpecSource(uproot.source.chunk.Source):
     """
     Args:
         file_path (str): A URL for the file to open.
+        coalesce_config (struct, optional): Configuration options for read coalescing
         **kwargs (dict): any extra arguments to be forwarded to the particular
             FileSystem instance constructor. This might include S3 access keys,
             or HTTP headers, etc.
@@ -40,8 +28,11 @@ class FSSpecSource(uproot.source.chunk.Source):
     to get many chunks in one request.
     """
 
-    def __init__(self, file_path: str, **options):
+    def __init__(
+        self, file_path: str, coalesce_config: CoalesceConfig | None = None, **options
+    ):
         super().__init__()
+        self._coalesce_config = coalesce_config
         self._fs, self._file_path = fsspec.core.url_to_fs(
             file_path, **self.extract_fsspec_options(options)
         )
@@ -50,7 +41,6 @@ class FSSpecSource(uproot.source.chunk.Source):
         self._async_impl = self._fs.async_impl
 
         self._file = None
-        self._fh = None
 
         self._open()
 
@@ -75,25 +65,20 @@ class FSSpecSource(uproot.source.chunk.Source):
         return f"<{type(self).__name__} {path} at 0x{id(self):012x}>"
 
     def __getstate__(self):
-        self._fh = None
         state = dict(self.__dict__)
         state.pop("_executor")
         state.pop("_file")
-        state.pop("_fh")
         return state
 
     def __setstate__(self, state):
         self.__dict__ = state
         self._file = None
-        self._fh = None
         self._open()
 
     def __enter__(self):
-        self._fh = self._file.__enter__()
         return self
 
     def __exit__(self, exception_type, exception_value, traceback):
-        self._fh = None
         self._file.__exit__(exception_type, exception_value, traceback)
         self._executor.shutdown()
 
@@ -110,20 +95,16 @@ class FSSpecSource(uproot.source.chunk.Source):
         self._num_requests += 1
         self._num_requested_chunks += 1
         self._num_requested_bytes += stop - start
-        if self._fh:
-            self._fh.seek(start)
-            data = self._fh.read(stop - start)
-        else:
-            data = self._fs.cat_file(self._file_path, start, stop)
+        data = self._fs.cat_file(self._file_path, start=start, end=stop)
         future = uproot.source.futures.TrivialFuture(data)
         return uproot.source.chunk.Chunk(self, start, stop, future)
 
     def chunks(
-        self, ranges: list[(int, int)], notifications: queue.Queue
+        self, ranges: list[tuple[int, int]], notifications: queue.Queue
     ) -> list[uproot.source.chunk.Chunk]:
         """
         Args:
-            ranges (list of (int, int) 2-tuples): Intervals to fetch
+            ranges (list of tuple[int, int] 2-tuples): Intervals to fetch
                 as (start, stop) pairs in a single request, if possible.
             notifications (``queue.Queue``): Indicator of completed
                 chunks. After each gets filled, it is ``put`` on the
@@ -171,29 +152,23 @@ class FSSpecSource(uproot.source.chunk.Source):
             # TODO: when python 3.8 is dropped, use `asyncio.to_thread` instead (also remove the try/except block above)
             return await to_thread(blocking_func, *args, **kwargs)
 
-        paths = [self._file_path] * len(ranges)
-        starts = [start for start, _ in ranges]
-        ends = [stop for _, stop in ranges]
-        # _cat_ranges is async while cat_ranges is not.
-        coroutine = (
-            self._fs._cat_ranges(paths=paths, starts=starts, ends=ends)
-            if self._async_impl
-            else async_wrapper_thread(
-                self._fs.cat_ranges, paths=paths, starts=starts, ends=ends
+        def submit(request_ranges: list[tuple[int, int]]):
+            paths = [self._file_path] * len(request_ranges)
+            starts = [start for start, _ in request_ranges]
+            ends = [stop for _, stop in request_ranges]
+            # _cat_ranges is async while cat_ranges is not.
+            coroutine = (
+                self._fs._cat_ranges(paths=paths, starts=starts, ends=ends)
+                if self._async_impl
+                else async_wrapper_thread(
+                    self._fs.cat_ranges, paths=paths, starts=starts, ends=ends
+                )
             )
-        )
-
-        future = self._executor.submit(coroutine)
+            return self._executor.submit(coroutine)
 
-        chunks = []
-        for index, (start, stop) in enumerate(ranges):
-            chunk_future = PartFuture(future, index)
-            chunk = uproot.source.chunk.Chunk(self, start, stop, chunk_future)
-            chunk_future.add_done_callback(
-                uproot.source.chunk.notifier(chunk, notifications)
-            )
-            chunks.append(chunk)
-        return chunks
+        return coalesce_requests(
+            ranges, submit, self, notifications, config=self._coalesce_config
+        )
 
     @property
     def async_impl(self) -> bool:

{uproot-5.3.7 → uproot-5.3.8}/src/uproot/streamers.py

@@ -999,14 +999,14 @@ class Model_TStreamerBasicType(Model_TStreamerElement):
                 else:
                     # AwkwardForth testing E: test_0637's 01,02,05,08,09,11,12,13,15,16,29,35,39,45,46,47,49,50,56
                     read_members.append("        if forth_obj is not None:")
-                    for i in range(len(formats[0])):
+                    for i in range(len(formats[-1])):
                         read_members.extend(
                             [
                                 "           key = key_number ; key_number += 1",
                                 '           form_key = f"node{key}-data"',
-                                f'           nested_forth_stash = af.Node(f"node{{key}}", field_name={fields[0][i]!r}, form_details={{ "class": "NumpyArray", "primitive": "{af.struct_to_dtype_name[formats[0][i]]}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}})',
-                                f'           nested_forth_stash.header_code.append(f"output {{form_key}} {af.struct_to_dtype_name[formats[0][i]]}\\n")',
-                                f'           nested_forth_stash.pre_code.append(f"stream !{formats[0][i]}-> {{form_key}}\\n")',
+                                f'           nested_forth_stash = af.Node(f"node{{key}}", field_name={fields[-1][i]!r}, form_details={{ "class": "NumpyArray", "primitive": "{af.struct_to_dtype_name[formats[-1][i]]}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}})',
+                                f'           nested_forth_stash.header_code.append(f"output {{form_key}} {af.struct_to_dtype_name[formats[-1][i]]}\\n")',
+                                f'           nested_forth_stash.pre_code.append(f"stream !{formats[-1][i]}-> {{form_key}}\\n")',
                                 "           forth_obj.add_node(nested_forth_stash)",
                             ]
                         )

{uproot-5.3.7 → uproot-5.3.8}/src/uproot/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '5.3.7'
-__version_tuple__ = version_tuple = (5, 3, 7)
+__version__ = version = '5.3.8'
+__version_tuple__ = version_tuple = (5, 3, 8)

{uproot-5.3.7 → uproot-5.3.8}/tests/test_0008_start_interpretation.py

@@ -49,8 +49,7 @@ def test_file_header(use_isal):
 
 @pytest.mark.parametrize("use_deflate", [False, True])
 def test_file_header(use_deflate):
-    # FIXME: https://github.com/dcwatson/deflate/issues/42
-    if use_deflate and not uproot._util.macos:
+    if use_deflate:
         pytest.importorskip("deflate")
     uproot.ZLIB.use_deflate = use_deflate
     filename = skhep_testdata.data_path("uproot-Zmumu.root")

{uproot-5.3.7 → uproot-5.3.8}/tests/test_0416_writing_compressed_data.py

@@ -34,8 +34,7 @@ def test_ZLIB(use_isal):
 
 @pytest.mark.parametrize("use_deflate", [False, True])
 def test_deflate(use_deflate):
-    # FIXME: https://github.com/dcwatson/deflate/issues/42
-    if use_deflate and not uproot._util.macos:
+    if use_deflate:
         pytest.importorskip("deflate")
         uproot.ZLIB.library = "deflate"
     else:
@@ -134,8 +133,7 @@ def test_histogram_ZLIB(tmp_path, use_isal):
 
 @pytest.mark.parametrize("use_deflate", [False, True])
 def test_histogram_deflate(tmp_path, use_deflate):
-    # FIXME: https://github.com/dcwatson/deflate/issues/42
-    if use_deflate and not uproot._util.macos:
+    if use_deflate:
         pytest.importorskip("deflate")
         uproot.ZLIB.library = "deflate"
     else:
@@ -593,8 +591,7 @@ def test_multicompression_5(tmp_path, use_isal):
 
 @pytest.mark.parametrize("use_deflate", [False, True])
 def test_multicompression_1_deflate(tmp_path, use_deflate):
-    # FIXME: https://github.com/dcwatson/deflate/issues/42
-    if use_deflate and not uproot._util.macos:
+    if use_deflate:
         pytest.importorskip("deflate")
         uproot.ZLIB.library = "deflate"
     else:
@@ -628,8 +625,7 @@ def test_multicompression_1_deflate(tmp_path, use_deflate):
 
 @pytest.mark.parametrize("use_deflate", [False, True])
 def test_multicompression_2_deflate(tmp_path, use_deflate):
-    # FIXME: https://github.com/dcwatson/deflate/issues/42
-    if use_deflate and not uproot._util.macos:
+    if use_deflate:
         pytest.importorskip("deflate")
         uproot.ZLIB.library = "deflate"
     else:
@@ -662,8 +658,7 @@ def test_multicompression_2_deflate(tmp_path, use_deflate):
 
 @pytest.mark.parametrize("use_deflate", [False, True])
 def test_multicompression_3_deflate(tmp_path, use_deflate):
-    # FIXME: https://github.com/dcwatson/deflate/issues/42
-    if use_deflate and not uproot._util.macos:
+    if use_deflate:
         pytest.importorskip("deflate")
         uproot.ZLIB.library = "deflate"
     else:
@@ -697,8 +692,7 @@ def test_multicompression_3_deflate(tmp_path, use_deflate):
 
 @pytest.mark.parametrize("use_deflate", [False, True])
 def test_multicompression_4_deflate(tmp_path, use_deflate):
-    # FIXME: https://github.com/dcwatson/deflate/issues/42
-    if use_deflate and not uproot._util.macos:
+    if use_deflate:
         pytest.importorskip("deflate")
         uproot.ZLIB.library = "deflate"
     else:
@@ -730,8 +724,7 @@ def test_multicompression_4_deflate(tmp_path, use_deflate):
 
 @pytest.mark.parametrize("use_deflate", [False, True])
 def test_multicompression_5_deflate(tmp_path, use_deflate):
-    # FIXME: https://github.com/dcwatson/deflate/issues/42
-    if use_deflate and not uproot._util.macos:
+    if use_deflate:
         pytest.importorskip("deflate")
         uproot.ZLIB.library = "deflate"
     else:

{uproot-5.3.7 → uproot-5.3.8}/tests/test_0662_rntuple_stl_containers.py

@@ -13,9 +13,6 @@ import uproot
 ak = pytest.importorskip("awkward")
 
 
-@pytest.mark.skip(
-    reason="FIXME: skipping test_ntuple_stl_containers.root until #928 is fixed"
-)
 def test_rntuple_stl_containers():
     filename = skhep_testdata.data_path("test_ntuple_stl_containers.root")
     with uproot.open(filename) as f:

{uproot-5.3.7 → uproot-5.3.8}/tests/test_0692_fsspec_reading.py

@@ -97,10 +97,6 @@ def test_open_fsspec_local():
 )
 def test_open_fsspec_s3(handler):
     pytest.importorskip("s3fs")
-    if sys.version_info < (3, 11):
-        pytest.skip(
-            "https://github.com/scikit-hep/uproot5/pull/1012",
-        )
 
     with uproot.open(
         "s3://pivarski-princeton/pythia_ppZee_run17emb.picoDst.root:PicoDst",
@@ -461,10 +457,6 @@ def test_fsspec_globbing_xrootd_no_files(handler):
 )
 def test_fsspec_globbing_s3(handler):
     pytest.importorskip("s3fs")
-    if sys.version_info < (3, 11):
-        pytest.skip(
-            "https://github.com/scikit-hep/uproot5/pull/1012",
-        )
 
     iterator = uproot.iterate(
         {"s3://pivarski-princeton/pythia_ppZee_run17emb.*.root": "PicoDst"},

{uproot-5.3.7 → uproot-5.3.8}/tests/test_1191_rntuple_fixes.py

@@ -1,5 +1,6 @@
 # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE
 
+import pytest
 import skhep_testdata
 
 import uproot
@@ -57,6 +58,9 @@ def test_empty_page_list():
         assert data[0] == 0
 
 
+@pytest.mark.skip(
+    reason="The file takes too long to download (about 5 seconds). Need to find a smaller test file."
+)
 def test_multiple_page_lists():
     url = "http://root.cern/files/tutorials/ntpl004_dimuon_v1rc2.root"
     with uproot.open(f"simplecache::{url}") as f:

uproot-5.3.8/tests/test_1198_coalesce.py

@@ -0,0 +1,39 @@
+import pytest
+from uproot.source.coalesce import CoalesceConfig, RangeRequest, _coalesce, Future
+
+
+@pytest.mark.parametrize(
+    "config",
+    [
+        CoalesceConfig(),
+        CoalesceConfig(max_range_gap=2, max_request_ranges=1),
+    ],
+    ids=["default", "tiny"],
+)
+@pytest.mark.parametrize(
+    "ranges",
+    [
+        [(1, 3), (4, 6), (10, 20)],
+        [(1, 3), (10, 20), (4, 6), (9, 10)],
+        [(1, 3), (10, 20), (6, 15)],
+        [(1, 3), (10, 20), (6, 25)],
+    ],
+    ids=["sorted", "jumbled", "overlapped", "nested"],
+)
+def test_coalesce(ranges, config):
+    data = b"abcdefghijklmnopqurstuvwxyz"
+
+    all_requests = [RangeRequest(start, stop, None) for start, stop in ranges]
+    nreq = 0
+    for merged_request in _coalesce(all_requests, config):
+        future = Future()
+        future.set_result([data[start:stop] for start, stop in merged_request.ranges()])
+        merged_request.set_future(future)
+        nreq += 1
+
+    if config.max_range_gap == 2:
+        assert nreq > 1
+
+    for req in all_requests:
+        assert req.future
+        assert req.future.result() == data[req.start : req.stop]

uproot-5.3.8/tests/test_1221_AwkwardForth_bug.py

@@ -0,0 +1,22 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE
+
+import pytest
+import uproot
+import skhep_testdata
+
+
+def test():
+    with uproot.open(skhep_testdata.data_path("uproot-issue-1221.root")) as file:
+        record = file["TrkAna"]["trkana"]["demtsh"].array()[-1, -1, -1]
+        assert record.tolist() == {
+            "wdot": -0.6311486959457397,
+            "dhit": True,
+            "poca": {
+                "fCoordinates": {
+                    "fX": -526.5504760742188,
+                    "fY": -195.0541534423828,
+                    "fZ": 1338.90771484375,
+                }
+            },
+            "dactive": True,
+        }