fsspec-data 0.2.0__tar.gz → 0.2.3__tar.gz

{fsspec_data-0.2.0 → fsspec_data-0.2.3}/Cargo.lock

@@ -233,7 +233,7 @@ version = "54.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9"
 dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
 ]
 
 [[package]]
@@ -296,9 +296,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.13.0"
+version = "2.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
+checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
 
 [[package]]
 name = "brotli"
@@ -341,9 +341,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "cc"
-version = "1.2.67"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
+checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -464,7 +464,7 @@ dependencies = [
 
 [[package]]
 name = "fsspec_data"
-version = "0.2.0"
+version = "0.2.3"
 dependencies = [
  "arrow",
  "bytes",
@@ -474,7 +474,7 @@ dependencies = [
 
 [[package]]
 name = "fsspec_data_py"
-version = "0.2.0"
+version = "0.2.3"
 dependencies = [
  "arrow",
  "fsspec_data",
@@ -483,21 +483,21 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
 
 [[package]]
 name = "futures-task"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
 
 [[package]]
 name = "futures-util"
-version = "0.3.32"
+version = "0.3.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -732,7 +732,7 @@ version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
 dependencies = [
- "twox-hash 2.1.2",
+ "twox-hash 2.1.3",
 ]
 
 [[package]]
@@ -901,15 +901,15 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
 
 [[package]]
 name = "portable-atomic"
-version = "1.13.1"
+version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.106"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
 dependencies = [
  "unicode-ident",
 ]
@@ -961,7 +961,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -974,14 +974,14 @@ dependencies = [
  "proc-macro2",
  "pyo3-build-config",
  "quote",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.46"
+version = "1.0.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
 dependencies = [
  "proc-macro2",
 ]
@@ -1000,9 +1000,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
 
 [[package]]
 name = "regex"
-version = "1.13.0"
+version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
+checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1012,9 +1012,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.15"
+version = "0.4.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db"
+checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1062,38 +1062,38 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
 
 [[package]]
 name = "serde"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
 dependencies = [
  "serde_core",
 ]
 
 [[package]]
 name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.2",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
 dependencies = [
  "itoa",
  "memchr",
@@ -1128,9 +1128,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "snap"
-version = "1.1.1"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
+checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886"
 
 [[package]]
 name = "static_assertions"
@@ -1140,9 +1140,20 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
 [[package]]
 name = "syn"
-version = "2.0.118"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1157,22 +1168,22 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "thiserror"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
+checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.18"
+version = "2.0.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
+checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 3.0.2",
 ]
 
 [[package]]
@@ -1207,9 +1218,9 @@ dependencies = [
 
 [[package]]
 name = "twox-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9"
 
 [[package]]
 name = "unicode-ident"
@@ -1276,7 +1287,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.119",
  "wasm-bindgen-shared",
 ]
 
@@ -1310,7 +1321,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1321,7 +1332,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]
@@ -1371,7 +1382,7 @@ checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.119",
 ]
 
 [[package]]

{fsspec_data-0.2.0 → fsspec_data-0.2.3}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fsspec_data_py"
-version = "0.2.0"
+version = "0.2.3"
 edition = "2021"
 publish = false
 
@@ -12,7 +12,7 @@ crate-type = ["cdylib"]
 [dependencies]
 fsspec_data = { path = "./rust", version = "*" }
 arrow = { version = "54.3.1", features = ["pyarrow"] }
-pyo3 = { version = "0.23", features = ["abi3-py311", "extension-module"] }
+pyo3 = { version = "0.23", features = ["abi3-py310", "extension-module"] }
 
 [profile.release]
 panic = "abort"

{fsspec_data-0.2.0 → fsspec_data-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec-data
-Version: 0.2.0
+Version: 0.2.3
 Summary: Arrow interchange planning for the fsspec data ecosystem
 Project-URL: Repository, https://github.com/1kbgz/fsspec-data
 Project-URL: Homepage, https://github.com/1kbgz/fsspec-data
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Programming Language :: Rust
-Requires-Python: >=3.11
+Requires-Python: >=3.10
 Requires-Dist: fsspec
 Requires-Dist: pyarrow
 Provides-Extra: develop

{fsspec_data-0.2.0 → fsspec_data-0.2.3}/fsspec_data/__init__.py

@@ -15,7 +15,7 @@ from .interchange import (
     plan_schema,
 )
 
-__version__ = "0.2.0"
+__version__ = "0.2.3"
 
 __all__ = [
     "Codec",

{fsspec_data-0.2.0 → fsspec_data-0.2.3}/fsspec_data/filesystem.py

@@ -70,11 +70,14 @@ class DataFileSystem(ChainedFileSystem):
             raise ValueError("fsspec-data is a read-only filesystem")
 
         path = self._strip_protocol(path)
-        output = self._convert(path)
         file = tempfile.SpooledTemporaryFile(max_size=self.spool_max_size, mode="w+b")
-        file.write(output)
-        file.seek(0)
-        self._sizes[path] = len(output)
+        try:
+            self._convert(path, file)
+            self._sizes[path] = file.tell()
+            file.seek(0)
+        except Exception:
+            file.close()
+            raise
         return file
 
     def info(self, path: str, **kwargs: Any) -> dict[str, Any]:
@@ -94,31 +97,31 @@ class DataFileSystem(ChainedFileSystem):
         entry = self.info(path, **kwargs)
         return [entry] if detail else [entry["name"]]
 
-    def _convert(self, path: str) -> bytes:
+    def _convert(self, path: str, output) -> None:
         provided_format = self.provided_format or _format_from_path(self.fo)
         requested_format = self.requested_format or _format_from_path(path)
-        source = self.fs.cat_file(self.fo)
-        decoded = DEFAULT_REGISTRY.get(provided_format).decode_batches(
-            source,
-            schema=self.provided_schema,
-            batch_size=self.batch_size,
-            row_limit=self.row_limit,
-            byte_limit=self.byte_limit,
-        )
-        if self.provided_schema is not None and not decoded.schema.equals(self.provided_schema, check_metadata=False):
-            raise ValueError("provided schema does not match the source schema")
-
-        provided_schema = self.provided_schema or decoded.schema
-        requested_schema = self.requested_schema or provided_schema
-        plan = InterchangeRequest(
-            provided_format,
-            requested_format,
-            provided_schema,
-            requested_schema,
-            self.schema_policy,
-        ).plan()
-        batches = tuple(plan.apply_batch(batch) for batch in decoded.batches)
-        return DEFAULT_REGISTRY.get(requested_format).encode_batches(batches, schema=requested_schema)
+        with self.fs.open(self.fo, "rb") as source:
+            decoded = DEFAULT_REGISTRY.get(provided_format).iter_batches(
+                source,
+                schema=self.provided_schema,
+                batch_size=self.batch_size,
+                row_limit=self.row_limit,
+                byte_limit=self.byte_limit,
+            )
+            if self.provided_schema is not None and not decoded.schema.equals(self.provided_schema, check_metadata=False):
+                raise ValueError("provided schema does not match the source schema")
+
+            provided_schema = self.provided_schema or decoded.schema
+            requested_schema = self.requested_schema or provided_schema
+            plan = InterchangeRequest(
+                provided_format,
+                requested_format,
+                provided_schema,
+                requested_schema,
+                self.schema_policy,
+            ).plan()
+            batches = (plan.apply_batch(batch) for batch in decoded)
+            DEFAULT_REGISTRY.get(requested_format).encode_batches_to(batches, output, schema=requested_schema)
 
 
 _SUFFIX_FORMATS = {

{fsspec_data-0.2.0 → fsspec_data-0.2.3}/fsspec_data/interchange.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+from collections.abc import Iterable
 from dataclasses import dataclass, replace
 from enum import Enum
 from importlib import import_module
-from typing import Any
+from itertools import chain
+from typing import Any, BinaryIO
 
 import pyarrow as pa
 import pyarrow.compute as pc
@@ -94,9 +96,34 @@ class Codec:
                 raise TypeError("batches must contain pyarrow.RecordBatch objects")
         return _rust.encode_batches(self.format.value, schema, batches)
 
+    def encode_batches_to(
+        self,
+        batches: Iterable[pa.RecordBatch],
+        output: BinaryIO,
+        *,
+        schema: pa.Schema | None = None,
+    ) -> None:
+        batches = iter(batches)
+        if schema is None:
+            try:
+                first = next(batches)
+            except StopIteration as error:
+                raise ValueError("schema is required when encoding no record batches") from error
+            if not isinstance(first, pa.RecordBatch):
+                raise TypeError("batches must contain pyarrow.RecordBatch objects")
+            schema = first.schema
+            batches = chain((first,), batches)
+        schema = _ensure_pyarrow(schema)
+        writer = _rust.start_codec_writer(self.format.value, schema, output)
+        for batch in batches:
+            if not isinstance(batch, pa.RecordBatch):
+                raise TypeError("batches must contain pyarrow.RecordBatch objects")
+            writer.write_batch(batch)
+        writer.finish()
+
     def decode_batches(
         self,
-        data: bytes | bytearray | memoryview,
+        data: bytes | bytearray | memoryview | BinaryIO,
         *,
         schema: pa.Schema | None = None,
         batch_size: int = 1024,
@@ -113,25 +140,35 @@ class Codec:
 
     def iter_batches(
         self,
-        data: bytes | bytearray | memoryview,
+        data: bytes | bytearray | memoryview | BinaryIO,
         *,
         schema: pa.Schema | None = None,
         batch_size: int = 1024,
         row_limit: int | None = None,
         byte_limit: int | None = None,
     ) -> DecodedBatchStream:
-        if not isinstance(data, (bytes, bytearray, memoryview)):
-            raise TypeError("data must be bytes-like")
         if schema is not None:
             schema = _ensure_pyarrow(schema)
-        decoded_schema, native = _rust.decode_stream(
-            self.format.value,
-            bytes(data),
-            schema,
-            batch_size,
-            row_limit,
-            byte_limit,
-        )
+        if isinstance(data, (bytes, bytearray, memoryview)):
+            decoded_schema, native = _rust.decode_stream(
+                self.format.value,
+                bytes(data),
+                schema,
+                batch_size,
+                row_limit,
+                byte_limit,
+            )
+        elif hasattr(data, "read"):
+            decoded_schema, native = _rust.decode_reader(
+                self.format.value,
+                data,
+                schema,
+                batch_size,
+                row_limit,
+                byte_limit,
+            )
+        else:
+            raise TypeError("data must be bytes-like or a binary file-like object")
         return DecodedBatchStream(decoded_schema, native)
 
 
@@ -204,7 +241,7 @@ class InterchangePlan:
 
     def iter_batches(
         self,
-        data: bytes | bytearray | memoryview,
+        data: bytes | bytearray | memoryview | BinaryIO,
         *,
         batch_size: int = 1024,
         row_limit: int | None = None,
@@ -222,7 +259,7 @@ class InterchangePlan:
 
     def convert(
         self,
-        data: bytes | bytearray | memoryview,
+        data: bytes | bytearray | memoryview | BinaryIO,
         *,
         batch_size: int = 1024,
         row_limit: int | None = None,

{fsspec_data-0.2.0 → fsspec_data-0.2.3}/fsspec_data/tests/test_all.py

@@ -1,3 +1,6 @@
+import io
+from typing import Any
+
 import pyarrow as pa
 import pyarrow.parquet as pq
 import pytest
@@ -280,6 +283,43 @@ def test_text_codec_round_trip_preserves_schema_and_rows(format, prefix):
     assert pa.Table.from_batches(decoded.batches).to_pylist() == pa.Table.from_batches(batches).to_pylist()
 
 
+@pytest.mark.parametrize("format", list(DataFormat))
+def test_codec_encodes_batches_to_binary_file(format):
+    provided, _ = schemas()
+    batches = (
+        pa.RecordBatch.from_arrays([pa.array([1, 2]), pa.array(["ada", None])], schema=provided),
+        pa.RecordBatch.from_arrays([pa.array([3]), pa.array(["margaret"])], schema=provided),
+    )
+    codec = DEFAULT_REGISTRY.get(format)
+    output = io.BytesIO()
+
+    codec.encode_batches_to(iter(batches), output)
+
+    decode_schema = provided if format in {DataFormat.CSV, DataFormat.JSONL} else None
+    decoded = codec.decode_batches(output.getvalue(), schema=decode_schema)
+    assert pa.Table.from_batches(decoded.batches).to_pylist() == pa.Table.from_batches(batches).to_pylist()
+
+
+def test_codec_file_writer_requires_schema_for_empty_stream():
+    codec = DEFAULT_REGISTRY.get(DataFormat.ARROW)
+
+    with pytest.raises(ValueError, match="schema is required"):
+        codec.encode_batches_to((), io.BytesIO())
+
+
+def test_codec_file_writer_rejects_non_batches():
+    provided, _ = schemas()
+    codec = DEFAULT_REGISTRY.get(DataFormat.ARROW)
+    invalid: Any = object()
+
+    with pytest.raises(TypeError, match="pyarrow.RecordBatch"):
+        codec.encode_batches_to(iter([invalid]), io.BytesIO())
+
+    batch = pa.record_batch([[1], ["ada"]], schema=provided)
+    with pytest.raises(TypeError, match="pyarrow.RecordBatch"):
+        codec.encode_batches_to(iter([batch, invalid]), io.BytesIO(), schema=provided)
+
+
 @pytest.mark.parametrize("format", [DataFormat.CSV, DataFormat.JSONL])
 def test_text_codec_decode_requires_schema(format):
     codec = DEFAULT_REGISTRY.get(format)
@@ -302,6 +342,28 @@ def test_lazy_decode_applies_batch_and_row_limits():
     assert pa.Table.from_batches(batches).column("id").to_pylist() == [1, 2, 3, 4]
 
 
+def test_lazy_decode_reads_binary_file_without_eagerly_consuming_it():
+    class CountingReader(io.BytesIO):
+        def __init__(self, data):
+            super().__init__(data)
+            self.bytes_read = 0
+
+        def read(self, size=-1):
+            data = super().read(size)
+            self.bytes_read += len(data)
+            return data
+
+    schema = pa.schema([("id", pa.int64())])
+    batch = pa.record_batch([pa.array(range(1024))], schema=schema)
+    codec = DEFAULT_REGISTRY.get(DataFormat.ARROW)
+    encoded = codec.encode_batches((batch,) * 128)
+    source = CountingReader(encoded)
+    stream = codec.iter_batches(source)
+
+    assert next(stream).num_rows == 1024
+    assert source.bytes_read < len(encoded)
+
+
 def test_lazy_decode_observes_cancellation():
     schema = pa.schema([("id", pa.int64())])
     codec = DEFAULT_REGISTRY.get(DataFormat.ARROW)