vgi-python 0.28.0__tar.gz → 0.29.0__tar.gz

{vgi_python-0.28.0 → vgi_python-0.29.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.28.0
+Version: 0.29.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python
@@ -162,7 +162,7 @@ Requires-Dist: httpx2>=2.9.1
 Requires-Dist: platformdirs
 Requires-Dist: pyarrow
 Requires-Dist: typer>=0.9
-Requires-Dist: vgi-rpc>=0.40.0
+Requires-Dist: vgi-rpc>=0.43.0
 Provides-Extra: azure
 Requires-Dist: azure-identity>=1.16.0; extra == 'azure'
 Requires-Dist: pymssql>=2.3.0; extra == 'azure'

{vgi_python-0.28.0 → vgi_python-0.29.0}/docs/aggregate-functions.md

@@ -88,16 +88,63 @@ class SumFunction(AggregateFunction[SumState]):
 
 ## State Class
 
-The state class must be a `dataclass` extending `ArrowSerializableDataclass`. Each field needs an `ArrowType` annotation for serialization:
+States are serialized to bytes and stored in `FunctionStorage` between RPC calls. The
+framework requires exactly two things of a state type — `serialize_to_bytes()` and
+`deserialize_from_bytes()`, the [`StreamStateCodec`][vgi.function.StreamStateCodec]
+protocol — and treats the bytes as opaque.
+
+The usual answer is a `dataclass` extending `ArrowSerializableDataclass`, which writes
+both methods for you. Each field needs an `ArrowType` annotation for serialization:
 
 ```python
+from dataclasses import dataclass
+from typing import Annotated
+
+import pyarrow as pa
+from vgi_rpc import ArrowSerializableDataclass, ArrowType
+
+
 @dataclass(kw_only=True)
 class AvgState(ArrowSerializableDataclass):
     total: Annotated[float, ArrowType(pa.float64())] = 0.0
     count: Annotated[int, ArrowType(pa.int64())] = 0
 ```
 
-States are serialized to bytes and stored in `FunctionStorage` between RPC calls. Use simple scalar types (int, float, str, bytes) for efficient serialization.
+Use simple scalar types (int, float, str, bytes) for efficient serialization.
+
+You are not required to go through Arrow. A state that is a couple of integers pays for
+a schema message, a batch message and an end-of-stream marker on every group, every
+batch — packing it yourself is both smaller and faster, and it lets a Python worker match
+the state encoding of a sibling VGI implementation in another language:
+
+```python
+import struct
+from dataclasses import dataclass
+from typing import ClassVar
+
+
+@dataclass(kw_only=True)
+class SumState:
+    """Same two counters, packed as little-endian int64s."""
+
+    total: int = 0
+    count: int = 0
+
+    _STRUCT: ClassVar[struct.Struct] = struct.Struct("<qq")
+
+    def serialize_to_bytes(self) -> bytes:
+        return self._STRUCT.pack(self.total, self.count)
+
+    @classmethod
+    def deserialize_from_bytes(cls, data: bytes) -> "SumState":
+        total, count = cls._STRUCT.unpack(data)
+        return cls(total=total, count=count)
+```
+
+The codec must round-trip exactly: `T.deserialize_from_bytes(s.serialize_to_bytes())` has
+to equal `s` for every state the aggregate can produce, including `initial_state()`. The
+framework cannot check that, and a lossy codec surfaces as wrong aggregate results rather
+than as an error. A `TState` that has neither method is rejected at class-definition time.
 
 ## Method Reference
 
@@ -286,6 +333,50 @@ worker = Worker(
 
 The framework automatically detects `AggregateFunction` subclasses and registers them with the correct function type in the catalog.
 
+## Invoking From Python
+
+Real users reach aggregates through DuckDB, which drives the all-unary aggregate RPCs from its hash-aggregate operator. Non-DuckDB callers drive the same protocol through `Client`.
+
+`aggregate_function()` is the convenience shape — it keys the groups client-side, pumps every batch through `update`, and finalizes in chunks:
+
+```python test="skip"
+import pyarrow as pa
+
+from vgi.client import Client
+
+batch = pa.RecordBatch.from_pydict({"cat": ["a", "b", "a"], "value": [1, 10, 2]})
+
+with Client("vgi-fixture-worker") as client:
+    result = client.aggregate_function(
+        function_name="vgi_sum",
+        schema_name="main",
+        input=[batch],
+        group_by=["cat"],
+    )
+    # {'cat': ['a', 'b'], 'result': [3, 10]}
+```
+
+Every input column not named in `group_by` is passed to the aggregate as a value column, in batch order — so order the columns to match the declared `Param`s. Omitting `group_by` gives the global-aggregate shape (`SELECT vgi_sum(x) FROM t`), which returns one row even for empty input.
+
+For the raw protocol — caller-allocated group ids, `combine`, and the optional window RPCs — use `aggregate_session()`:
+
+```python test="skip"
+with Client("vgi-fixture-worker") as client:
+    with client.aggregate_session(
+        function_name="vgi_sum",
+        schema_name="main",
+        input_schema=pa.schema([pa.field("value", pa.int64())]),
+    ) as session:
+        session.update(
+            group_ids=[0, 0, 1],
+            batch=pa.RecordBatch.from_pydict({"value": [1, 2, 100]}),
+        )
+        session.combine(source_group_ids=[1], target_group_ids=[0])
+        session.finalize([0, 1])  # -> {'result': [103, 100]}
+```
+
+The session is destroyed on exit, so worker-side group state never outlives the `with` block. Window-capable aggregates add `window_init()` / `window()` / `window_batch()` / `window_destroy()` on the same session; streaming-partitioned aggregates use `client.aggregate_streaming(...)` instead (see below).
+
 ## Streaming-Partitioned Variant
 
 For `OVER (PARTITION BY ... ORDER BY ...)` queries against unbounded inputs (e.g. running aggregates across years of trade history), the standard windowed path materializes each partition in DuckDB memory before the aggregate sees it — fine for bounded data, OOMs at scale.
@@ -339,6 +430,28 @@ class MyRunningAgg(AggregateFunction[MyState]):
 
 Queries that don't satisfy all of these fall back to the standard windowed path automatically. The streaming path is opt-in and additive — it does not replace `update`/`combine`/`finalize`, which still service `GROUP BY` queries normally.
 
+From Python, `Client.aggregate_streaming()` opens the session and closes it on exit. The chunk schema is positional: partition keys first, then order keys, then value columns.
+
+```python test="skip"
+schema = pa.schema(
+    [pa.field("k", pa.string()), pa.field("ts", pa.int64()), pa.field("v", pa.int64())]
+)
+
+with Client("vgi-fixture-worker") as client:
+    with client.aggregate_streaming(
+        function_name="vgi_streaming_sum",
+        schema_name="main",
+        input_schema=schema,
+        partition_key_count=1,
+        order_key_count=1,
+    ) as session:
+        chunk = pa.RecordBatch.from_pydict(
+            {"k": ["a", "b", "a", "b"], "ts": [1, 1, 2, 2], "v": [1, 10, 2, 20]},
+            schema=schema,
+        )
+        session.chunk(chunk)  # -> {'result': [1, 10, 3, 30]}
+```
+
 **When pre-aggregation is the better answer.** For most analytics shapes — "EOD positions per book per day, carrying forward across days" — pre-aggregating the input is the cleanest pattern in plain SQL:
 
 ```sql

vgi_python-0.29.0/examples/cache_worker.py

@@ -0,0 +1,124 @@
+# /// script
+# requires-python = ">=3.13"
+# dependencies = ["vgi-python"]
+# ///
+"""A table function that advertises its result as cacheable, and revalidates it.
+
+Caching is *advertised*, not requested: the worker attaches ``vgi.cache.*``
+metadata to the **first** data batch it emits, and the client (the DuckDB
+extension) decides what to do with it. Nothing is cached unless you say so.
+
+This worker exposes ``rates()``, standing in for a slow upstream — a remote API,
+a rate-limited service — whose answer is worth reusing. It shows the whole
+vocabulary:
+
+- a **freshness lifetime** (``ttl``), so repeat queries inside the window never
+  reach the worker at all;
+- a **validator** (``etag``) plus ``revalidatable=True``, which is what lets the
+  client ask "is this still good?" instead of paying for a recompute;
+- the **304-equivalent** reply — a 0-row batch carrying
+  ``CacheControl(not_modified=True)`` — which tells the client its stored payload
+  is still fresh.
+
+``out`` is typed as vgi-rpc's ``OutputCollector``, which knows nothing about
+caching; the object the framework actually passes is a VGI wrapper that accepts
+``cache_control=``. Cast to :class:`vgi.protocol.VgiOutputCollector` to reach it
+with types intact — every cache-aware function does this.
+
+    ATTACH 'rates' (TYPE vgi, LOCATION 'uv run cache_worker.py');
+    SELECT * FROM rates.rates();   -- second call inside the TTL never lands here
+"""
+
+from dataclasses import dataclass
+from typing import cast
+
+import pyarrow as pa
+
+from vgi import Worker
+from vgi.cache_control import CacheControl
+from vgi.catalog import Catalog, Schema
+from vgi.protocol import VgiOutputCollector
+from vgi.table_function import (
+    OutputCollector,
+    ProcessParams,
+    TableFunctionGenerator,
+    bind_fixed_schema,
+    init_single_worker,
+)
+
+_SCHEMA = pa.schema([("currency", pa.string()), ("rate", pa.float64())])
+
+# Stands in for whatever makes your upstream's answer change. A real worker
+# would derive this from the source — a last-modified header, a version column,
+# a content hash — and it is the ONLY thing revalidation compares.
+_DATA_VERSION = '"rates-v3"'
+
+_ROWS = {"currency": ["EUR", "GBP", "JPY"], "rate": [1.09, 1.27, 0.0067]}
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class RatesArgs:
+    """``rates()`` takes no arguments — the whole table is the result."""
+
+
+@bind_fixed_schema
+@init_single_worker
+class Rates(TableFunctionGenerator[RatesArgs, None]):
+    """Emit the rate table once, advertising it as cacheable for 5 minutes."""
+
+    FIXED_SCHEMA = _SCHEMA
+
+    class Meta:
+        """Function metadata."""
+
+        name = "rates"
+        description = "Exchange rates from a slow upstream, cacheable for 5 minutes"
+
+    @classmethod
+    def process(cls, params: ProcessParams[RatesArgs, None], state: None, out: OutputCollector) -> None:
+        """Answer the scan — or, if the client's copy is still current, say so."""
+        vgi_out = cast(VgiOutputCollector, out)
+
+        # Conditional request: the client holds a stale-but-revalidatable copy
+        # and is asking whether it may keep it. Both validators are None on a
+        # normal call, so this branch is simply skipped.
+        if params.if_none_match == _DATA_VERSION:
+            vgi_out.emit(
+                pa.RecordBatch.from_pylist([], schema=_SCHEMA),  # zero rows: "keep what you have"
+                cache_control=CacheControl(
+                    ttl=300,
+                    etag=_DATA_VERSION,
+                    revalidatable=True,
+                    not_modified=True,
+                ),
+            )
+            out.finish()
+            return
+
+        # Normal path: stream the result and advertise how it may be reused.
+        # The metadata rides on the FIRST data batch; attaching it later has no
+        # effect, because by then the client has decided how to treat the stream.
+        vgi_out.emit(
+            pa.RecordBatch.from_pydict(_ROWS, schema=_SCHEMA),
+            cache_control=CacheControl(
+                ttl=300,  # reusable for 5 minutes without asking
+                etag=_DATA_VERSION,  # ...and after that, cheap to revalidate
+                revalidatable=True,  # gates whether the client ever asks
+                stale_while_revalidate=60,  # serve stale while refreshing behind it
+                stale_if_error=600,  # serve stale rather than fail
+            ),
+        )
+        out.finish()
+
+
+class CacheWorker(Worker):
+    """A worker exposing the ``rates`` catalog."""
+
+    catalog = Catalog(
+        name="rates",
+        schemas=[Schema(name="main", functions=[Rates])],
+    )
+
+
+if __name__ == "__main__":
+    CacheWorker().run()

vgi_python-0.29.0/examples/copy_format_worker.py

@@ -0,0 +1,178 @@
+# /// script
+# requires-python = ">=3.13"
+# dependencies = ["vgi-python"]
+# ///
+"""Custom ``COPY ... FROM`` and ``COPY ... TO`` formats, in one catalog.
+
+A VGI catalog can register its own ``COPY`` formats, so users read and write a
+format DuckDB has never heard of. Both directions reuse machinery that already
+exists, which is why they add so few new rules:
+
+- a :class:`CopyFromFunction` is an ordinary producer-mode **table function**;
+- a :class:`CopyToFunction` is a **buffered** function with no Source phase —
+  ``write`` is the sink (per batch, parallel), ``close`` is the combine (once).
+
+The format here is tab-separated values, one record per line, no header, no
+quoting. Deliberately trivial — the point is the wiring, not the parser.
+
+Note the reader and writer declare **different** format names (``tsvlite`` and
+``tsvlite_out``). A reader and a writer that share one name look fine on the
+Python side — the catalog advertises both, keyed by (direction, format) — but the
+extension registers COPY functions by name alone, so the writer is silently
+dropped and ``COPY ... TO`` fails with "COPY TO is not supported for FORMAT".
+Give each direction its own name.
+
+Two rules the framework enforces, and this file demonstrates:
+
+1. **The reader must emit ``expected_schema`` exactly.** DuckDB inserts no cast
+   between the scan and the INSERT, so the target table's columns are the output
+   schema — that is why ``on_bind`` is ``@final`` on a COPY-FROM function.
+2. **The writer must finish inside ``close``.** There is no finalize phase, and
+   ``write`` and ``close`` may run in different processes, so shards go through
+   ``params.storage`` scoped by ``execution_id`` — never on ``self``.
+
+The SQL ``FORMAT`` name is qualified by the catalog alias you attached as, NOT
+the bare ``COPY_*_FORMAT`` string — the extension namespaces formats per attach,
+so two workers may both call theirs ``tsvlite``:
+
+    ATTACH 'tsv' (TYPE vgi, LOCATION 'uv run copy_format_worker.py');
+    CREATE TABLE people (name VARCHAR, age BIGINT);
+    COPY people FROM 'people.tsv' (FORMAT 'tsv.tsvlite');
+    COPY (SELECT * FROM people) TO 'out.tsv' (FORMAT 'tsv.tsvlite_out', header true);
+"""
+
+from dataclasses import dataclass
+from typing import Annotated
+
+import pyarrow as pa
+
+from vgi import Arg, Worker
+from vgi.catalog import Catalog, Schema
+from vgi.copy_from_function import CopyFromFunction
+from vgi.copy_to_function import CopyToFunction
+from vgi.table_buffering_function import TableBufferingParams
+from vgi.table_function import OutputCollector, ProcessParams
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class ReadOptions:
+    """Options accepted by ``COPY ... FROM ... (FORMAT tsvlite, ...)``.
+
+    These are ordinary ``Arg``-annotated arguments — the source path is NOT one
+    of them; it arrives on the bind. Each ``doc`` becomes the option's
+    description in ``vgi_copy_formats()``.
+    """
+
+    skip_rows: Annotated[int, Arg("skip_rows", doc="Leading lines to discard", default=0)] = 0
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class WriteOptions:
+    """Options accepted by ``COPY ... TO ... (FORMAT 'tsv.tsvlite_out', ...)``."""
+
+    header: Annotated[bool, Arg("header", doc="Write a header line of column names", default=False)] = False
+
+
+class ReadTsvLite(CopyFromFunction[ReadOptions]):
+    """Read a tab-separated file into the COPY target table."""
+
+    COPY_FROM_FORMAT = "tsvlite"
+    COPY_FROM_COMMENT = "Tab-separated, one record per line, no quoting"
+
+    @classmethod
+    def read(
+        cls,
+        *,
+        path: str,
+        options: ReadOptions,
+        expected_schema: pa.Schema,
+        params: ProcessParams[ReadOptions],
+        out: OutputCollector,
+    ) -> None:
+        """Parse ``path`` and emit batches matching ``expected_schema`` exactly."""
+        with open(path, encoding="utf-8") as handle:
+            lines = [line.rstrip("\n") for line in handle if line.strip()]
+        lines = lines[options.skip_rows :]
+
+        # Split into columns positionally, then let Arrow cast each column to the
+        # type the target table declared. Emitting a mismatched type or arity is
+        # rejected by the extension at COPY bind, not silently coerced.
+        columns: list[list[str | None]] = [[] for _ in expected_schema]
+        for line in lines:
+            cells = line.split("\t")
+            for index in range(len(expected_schema)):
+                value = cells[index] if index < len(cells) else ""
+                columns[index].append(value if value != "" else None)
+
+        out.emit(
+            pa.RecordBatch.from_arrays(
+                [
+                    pa.array(col, type=pa.string()).cast(field.type)
+                    for col, field in zip(columns, expected_schema, strict=True)
+                ],
+                schema=expected_schema,
+            )
+        )
+        out.finish()
+
+
+class WriteTsvLite(CopyToFunction[WriteOptions]):
+    """Write query results out as a tab-separated file."""
+
+    COPY_TO_FORMAT = "tsvlite_out"
+    COPY_TO_COMMENT = "Tab-separated, one record per line, no quoting"
+
+    @classmethod
+    def write(
+        cls,
+        *,
+        batch: pa.RecordBatch,
+        options: WriteOptions,
+        file_path: str,
+        params: TableBufferingParams[WriteOptions],
+    ) -> None:
+        """Sink: stash this batch as a shard. Runs per batch, possibly in parallel."""
+        rows = ["\t".join("" if value is None else str(value) for value in row.values()) for row in batch.to_pylist()]
+        # execution_id-scoped, because close() may run in a different process.
+        params.storage.state_append(b"shards", b"", "\n".join(rows).encode("utf-8"))
+
+    @classmethod
+    def close(
+        cls,
+        *,
+        options: WriteOptions,
+        file_path: str,
+        params: TableBufferingParams[WriteOptions],
+    ) -> int:
+        """Combine: read every shard back and write the file, once. Returns rows written."""
+        shards = params.storage.state_log_scan(b"shards", b"")
+        body = [chunk.decode("utf-8") for _key, chunk in shards if chunk]
+
+        written = 0
+        with open(file_path, "w", encoding="utf-8") as handle:
+            if options.header:
+                schema = params.init_call.bind_call.input_schema
+                handle.write("\t".join(schema.names) + "\n")
+            for chunk in body:
+                handle.write(chunk + "\n")
+                written += len(chunk.split("\n"))
+        # Called even for an empty COPY — the file is created either way.
+        return written
+
+
+class TsvWorker(Worker):
+    """A worker whose catalog advertises both COPY directions.
+
+    The declarative ``Catalog`` introspects its function list for
+    ``CopyFromFunction`` / ``CopyToFunction`` subclasses and advertises what it
+    finds, so registering them here is all the wiring there is.
+    """
+
+    catalog = Catalog(
+        name="tsv",
+        schemas=[Schema(name="main", functions=[ReadTsvLite, WriteTsvLite])],
+    )
+
+
+if __name__ == "__main__":
+    TsvWorker().run()

{vgi_python-0.28.0 → vgi_python-0.29.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.28.0"
+version = "0.29.0"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [
@@ -40,7 +40,7 @@ dependencies = [
     "pyarrow",
     "typer>=0.9",
     "platformdirs",
-    "vgi-rpc>=0.40.0",
+    "vgi-rpc>=0.43.0",
     "httpx2>=2.9.1",
 ]
 

{vgi_python-0.28.0 → vgi_python-0.29.0}/tests/catalog/test_serialization.py

@@ -1283,7 +1283,7 @@ class TestScanBranchSerialization:
         Lenient producers (the Rust/Go/Java SDKs, tolerated by the DuckDB C++
         client) omit the IPC stream for an empty nested list, so a strict
         consumer must accept a zero-branch result rather than raising — otherwise
-        the Cedar proxy (the only strict Python consumer in the path) fails
+        Rowfence (the only strict Python consumer in the path) fails
         closed on a legitimate response.
         """
         from vgi.catalog import ScanBranchesResult