fsspec-data 0.2.2__tar.gz → 0.2.3__tar.gz

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/Cargo.lock

@@ -464,7 +464,7 @@ dependencies = [
 
 [[package]]
 name = "fsspec_data"
-version = "0.2.2"
+version = "0.2.3"
 dependencies = [
  "arrow",
  "bytes",
@@ -474,7 +474,7 @@ dependencies = [
 
 [[package]]
 name = "fsspec_data_py"
-version = "0.2.2"
+version = "0.2.3"
 dependencies = [
  "arrow",
  "fsspec_data",

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fsspec_data_py"
-version = "0.2.2"
+version = "0.2.3"
 edition = "2021"
 publish = false
 
@@ -12,7 +12,7 @@ crate-type = ["cdylib"]
 [dependencies]
 fsspec_data = { path = "./rust", version = "*" }
 arrow = { version = "54.3.1", features = ["pyarrow"] }
-pyo3 = { version = "0.23", features = ["abi3-py311", "extension-module"] }
+pyo3 = { version = "0.23", features = ["abi3-py310", "extension-module"] }
 
 [profile.release]
 panic = "abort"

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fsspec-data
-Version: 0.2.2
+Version: 0.2.3
 Summary: Arrow interchange planning for the fsspec data ecosystem
 Project-URL: Repository, https://github.com/1kbgz/fsspec-data
 Project-URL: Homepage, https://github.com/1kbgz/fsspec-data
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Programming Language :: Rust
-Requires-Python: >=3.11
+Requires-Python: >=3.10
 Requires-Dist: fsspec
 Requires-Dist: pyarrow
 Provides-Extra: develop

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/fsspec_data/__init__.py

@@ -15,7 +15,7 @@ from .interchange import (
     plan_schema,
 )
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"
 
 __all__ = [
     "Codec",

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/fsspec_data/filesystem.py

@@ -70,11 +70,14 @@ class DataFileSystem(ChainedFileSystem):
             raise ValueError("fsspec-data is a read-only filesystem")
 
         path = self._strip_protocol(path)
-        output = self._convert(path)
         file = tempfile.SpooledTemporaryFile(max_size=self.spool_max_size, mode="w+b")
-        file.write(output)
-        file.seek(0)
-        self._sizes[path] = len(output)
+        try:
+            self._convert(path, file)
+            self._sizes[path] = file.tell()
+            file.seek(0)
+        except Exception:
+            file.close()
+            raise
         return file
 
     def info(self, path: str, **kwargs: Any) -> dict[str, Any]:
@@ -94,31 +97,31 @@ class DataFileSystem(ChainedFileSystem):
         entry = self.info(path, **kwargs)
         return [entry] if detail else [entry["name"]]
 
-    def _convert(self, path: str) -> bytes:
+    def _convert(self, path: str, output) -> None:
         provided_format = self.provided_format or _format_from_path(self.fo)
         requested_format = self.requested_format or _format_from_path(path)
-        source = self.fs.cat_file(self.fo)
-        decoded = DEFAULT_REGISTRY.get(provided_format).decode_batches(
-            source,
-            schema=self.provided_schema,
-            batch_size=self.batch_size,
-            row_limit=self.row_limit,
-            byte_limit=self.byte_limit,
-        )
-        if self.provided_schema is not None and not decoded.schema.equals(self.provided_schema, check_metadata=False):
-            raise ValueError("provided schema does not match the source schema")
-
-        provided_schema = self.provided_schema or decoded.schema
-        requested_schema = self.requested_schema or provided_schema
-        plan = InterchangeRequest(
-            provided_format,
-            requested_format,
-            provided_schema,
-            requested_schema,
-            self.schema_policy,
-        ).plan()
-        batches = tuple(plan.apply_batch(batch) for batch in decoded.batches)
-        return DEFAULT_REGISTRY.get(requested_format).encode_batches(batches, schema=requested_schema)
+        with self.fs.open(self.fo, "rb") as source:
+            decoded = DEFAULT_REGISTRY.get(provided_format).iter_batches(
+                source,
+                schema=self.provided_schema,
+                batch_size=self.batch_size,
+                row_limit=self.row_limit,
+                byte_limit=self.byte_limit,
+            )
+            if self.provided_schema is not None and not decoded.schema.equals(self.provided_schema, check_metadata=False):
+                raise ValueError("provided schema does not match the source schema")
+
+            provided_schema = self.provided_schema or decoded.schema
+            requested_schema = self.requested_schema or provided_schema
+            plan = InterchangeRequest(
+                provided_format,
+                requested_format,
+                provided_schema,
+                requested_schema,
+                self.schema_policy,
+            ).plan()
+            batches = (plan.apply_batch(batch) for batch in decoded)
+            DEFAULT_REGISTRY.get(requested_format).encode_batches_to(batches, output, schema=requested_schema)
 
 
 _SUFFIX_FORMATS = {

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/fsspec_data/interchange.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+from collections.abc import Iterable
 from dataclasses import dataclass, replace
 from enum import Enum
 from importlib import import_module
-from typing import Any
+from itertools import chain
+from typing import Any, BinaryIO
 
 import pyarrow as pa
 import pyarrow.compute as pc
@@ -94,9 +96,34 @@ class Codec:
                 raise TypeError("batches must contain pyarrow.RecordBatch objects")
         return _rust.encode_batches(self.format.value, schema, batches)
 
+    def encode_batches_to(
+        self,
+        batches: Iterable[pa.RecordBatch],
+        output: BinaryIO,
+        *,
+        schema: pa.Schema | None = None,
+    ) -> None:
+        batches = iter(batches)
+        if schema is None:
+            try:
+                first = next(batches)
+            except StopIteration as error:
+                raise ValueError("schema is required when encoding no record batches") from error
+            if not isinstance(first, pa.RecordBatch):
+                raise TypeError("batches must contain pyarrow.RecordBatch objects")
+            schema = first.schema
+            batches = chain((first,), batches)
+        schema = _ensure_pyarrow(schema)
+        writer = _rust.start_codec_writer(self.format.value, schema, output)
+        for batch in batches:
+            if not isinstance(batch, pa.RecordBatch):
+                raise TypeError("batches must contain pyarrow.RecordBatch objects")
+            writer.write_batch(batch)
+        writer.finish()
+
     def decode_batches(
         self,
-        data: bytes | bytearray | memoryview,
+        data: bytes | bytearray | memoryview | BinaryIO,
         *,
         schema: pa.Schema | None = None,
         batch_size: int = 1024,
@@ -113,25 +140,35 @@ class Codec:
 
     def iter_batches(
         self,
-        data: bytes | bytearray | memoryview,
+        data: bytes | bytearray | memoryview | BinaryIO,
         *,
         schema: pa.Schema | None = None,
         batch_size: int = 1024,
         row_limit: int | None = None,
         byte_limit: int | None = None,
     ) -> DecodedBatchStream:
-        if not isinstance(data, (bytes, bytearray, memoryview)):
-            raise TypeError("data must be bytes-like")
         if schema is not None:
             schema = _ensure_pyarrow(schema)
-        decoded_schema, native = _rust.decode_stream(
-            self.format.value,
-            bytes(data),
-            schema,
-            batch_size,
-            row_limit,
-            byte_limit,
-        )
+        if isinstance(data, (bytes, bytearray, memoryview)):
+            decoded_schema, native = _rust.decode_stream(
+                self.format.value,
+                bytes(data),
+                schema,
+                batch_size,
+                row_limit,
+                byte_limit,
+            )
+        elif hasattr(data, "read"):
+            decoded_schema, native = _rust.decode_reader(
+                self.format.value,
+                data,
+                schema,
+                batch_size,
+                row_limit,
+                byte_limit,
+            )
+        else:
+            raise TypeError("data must be bytes-like or a binary file-like object")
         return DecodedBatchStream(decoded_schema, native)
 
 
@@ -204,7 +241,7 @@ class InterchangePlan:
 
     def iter_batches(
         self,
-        data: bytes | bytearray | memoryview,
+        data: bytes | bytearray | memoryview | BinaryIO,
         *,
         batch_size: int = 1024,
         row_limit: int | None = None,
@@ -222,7 +259,7 @@ class InterchangePlan:
 
     def convert(
         self,
-        data: bytes | bytearray | memoryview,
+        data: bytes | bytearray | memoryview | BinaryIO,
         *,
         batch_size: int = 1024,
         row_limit: int | None = None,

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/fsspec_data/tests/test_all.py

@@ -1,3 +1,6 @@
+import io
+from typing import Any
+
 import pyarrow as pa
 import pyarrow.parquet as pq
 import pytest
@@ -280,6 +283,43 @@ def test_text_codec_round_trip_preserves_schema_and_rows(format, prefix):
     assert pa.Table.from_batches(decoded.batches).to_pylist() == pa.Table.from_batches(batches).to_pylist()
 
 
+@pytest.mark.parametrize("format", list(DataFormat))
+def test_codec_encodes_batches_to_binary_file(format):
+    provided, _ = schemas()
+    batches = (
+        pa.RecordBatch.from_arrays([pa.array([1, 2]), pa.array(["ada", None])], schema=provided),
+        pa.RecordBatch.from_arrays([pa.array([3]), pa.array(["margaret"])], schema=provided),
+    )
+    codec = DEFAULT_REGISTRY.get(format)
+    output = io.BytesIO()
+
+    codec.encode_batches_to(iter(batches), output)
+
+    decode_schema = provided if format in {DataFormat.CSV, DataFormat.JSONL} else None
+    decoded = codec.decode_batches(output.getvalue(), schema=decode_schema)
+    assert pa.Table.from_batches(decoded.batches).to_pylist() == pa.Table.from_batches(batches).to_pylist()
+
+
+def test_codec_file_writer_requires_schema_for_empty_stream():
+    codec = DEFAULT_REGISTRY.get(DataFormat.ARROW)
+
+    with pytest.raises(ValueError, match="schema is required"):
+        codec.encode_batches_to((), io.BytesIO())
+
+
+def test_codec_file_writer_rejects_non_batches():
+    provided, _ = schemas()
+    codec = DEFAULT_REGISTRY.get(DataFormat.ARROW)
+    invalid: Any = object()
+
+    with pytest.raises(TypeError, match="pyarrow.RecordBatch"):
+        codec.encode_batches_to(iter([invalid]), io.BytesIO())
+
+    batch = pa.record_batch([[1], ["ada"]], schema=provided)
+    with pytest.raises(TypeError, match="pyarrow.RecordBatch"):
+        codec.encode_batches_to(iter([batch, invalid]), io.BytesIO(), schema=provided)
+
+
 @pytest.mark.parametrize("format", [DataFormat.CSV, DataFormat.JSONL])
 def test_text_codec_decode_requires_schema(format):
     codec = DEFAULT_REGISTRY.get(format)
@@ -302,6 +342,28 @@ def test_lazy_decode_applies_batch_and_row_limits():
     assert pa.Table.from_batches(batches).column("id").to_pylist() == [1, 2, 3, 4]
 
 
+def test_lazy_decode_reads_binary_file_without_eagerly_consuming_it():
+    class CountingReader(io.BytesIO):
+        def __init__(self, data):
+            super().__init__(data)
+            self.bytes_read = 0
+
+        def read(self, size=-1):
+            data = super().read(size)
+            self.bytes_read += len(data)
+            return data
+
+    schema = pa.schema([("id", pa.int64())])
+    batch = pa.record_batch([pa.array(range(1024))], schema=schema)
+    codec = DEFAULT_REGISTRY.get(DataFormat.ARROW)
+    encoded = codec.encode_batches((batch,) * 128)
+    source = CountingReader(encoded)
+    stream = codec.iter_batches(source)
+
+    assert next(stream).num_rows == 1024
+    assert source.bytes_read < len(encoded)
+
+
 def test_lazy_decode_observes_cancellation():
     schema = pa.schema([("id", pa.int64())])
     codec = DEFAULT_REGISTRY.get(DataFormat.ARROW)

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/fsspec_data/tests/test_filesystem.py

@@ -1,9 +1,11 @@
+import io
+
 import fsspec
 import pyarrow as pa
 import pyarrow.parquet as pq
 import pytest
 
-from fsspec_data import DataFileSystem
+from fsspec_data import Codec, DataFileSystem, DecodedBatchStream
 
 SCHEMA = pa.schema([pa.field("id", pa.int64(), nullable=False), pa.field("name", pa.string())])
 SCHEMA_OPTIONS = {
@@ -34,6 +36,43 @@ def test_chained_filesystem_converts_csv_to_seekable_parquet(memory_fs):
     assert table.to_pylist() == [{"id": 1, "name": "ada"}, {"id": 2, "name": "grace"}]
 
 
+def test_chained_filesystem_reads_source_without_cat_file(memory_fs, monkeypatch):
+    rows = [f"{index},name-{index}\n".encode() for index in range(50_000)]
+    encoded = b"id,name\n" + b"".join(rows)
+    bytes_read = 0
+
+    class CountingReader(io.BytesIO):
+        def read(self, size=-1):
+            nonlocal bytes_read
+            data = super().read(size)
+            bytes_read += len(data)
+            return data
+
+    monkeypatch.setattr(memory_fs, "open", lambda *args, **kwargs: CountingReader(encoded))
+    monkeypatch.setattr(memory_fs, "cat_file", lambda *args, **kwargs: pytest.fail("cat_file should not be called"))
+    fs = DataFileSystem(
+        fo="orders.csv",
+        fs=memory_fs,
+        provided_schema=SCHEMA_OPTIONS,
+        batch_size=1,
+        row_limit=1,
+    )
+
+    assert pq.read_table(fs.open("orders.parquet")).to_pylist() == [{"id": 0, "name": "name-0"}]
+    assert bytes_read < len(encoded)
+
+
+def test_chained_filesystem_streams_batches_into_spooled_output(memory_fs, monkeypatch):
+    monkeypatch.setattr(DecodedBatchStream, "collect", lambda self: pytest.fail("decoded batches should not be collected"))
+    monkeypatch.setattr(Codec, "encode_batches", lambda self, *args, **kwargs: pytest.fail("encoded bytes should not be collected"))
+    fs = DataFileSystem(fo="orders.csv", fs=memory_fs, provided_schema=SCHEMA_OPTIONS)
+
+    assert pq.read_table(fs.open("orders.parquet")).to_pylist() == [
+        {"id": 1, "name": "ada"},
+        {"id": 2, "name": "grace"},
+    ]
+
+
 def test_fsspec_url_builds_chained_filesystem(memory_fs):
     fsspec.register_implementation("fsspec-data", DataFileSystem, clobber=True)
 

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/pyproject.toml

@@ -13,8 +13,8 @@ authors = [
 description = "Arrow interchange planning for the fsspec data ecosystem"
 readme = "README.md"
 license = { text = "Apache-2.0" }
-version = "0.2.2"
-requires-python = ">=3.11"
+version = "0.2.3"
+requires-python = ">=3.10"
 keywords = ["arrow", "data", "fsspec", "interchange"]
 
 classifiers = [
@@ -65,7 +65,7 @@ Repository = "https://github.com/1kbgz/fsspec-data"
 Homepage = "https://github.com/1kbgz/fsspec-data"
 
 [tool.bumpversion]
-current_version = "0.2.2"
+current_version = "0.2.3"
 commit = true
 tag = false
 commit_args = "-s"
@@ -91,7 +91,7 @@ search = 'version = "{current_version}"'
 replace = 'version = "{new_version}"'
 
 [tool.cibuildwheel]
-build = "cp311-*"
+build = "cp310-*"
 test-command = "pytest -vvv {project}/fsspec_data/tests"
 test-extras = "develop"
 

{fsspec_data-0.2.2 → fsspec_data-0.2.3}/rust/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fsspec_data"
-version = "0.2.2"
+version = "0.2.3"
 authors = ["the fsspec-data authors <dev@1kbgz.com>"]
 edition = "2021"
 license = "Apache-2.0"