pl-row-encode 0.3.1__cp39-abi3-win_amd64.whl

pl_row_encode/__init__.py

@@ -0,0 +1,197 @@
+"""Row-level, type-preserving encode/decode for Polars columns.
+
+`encode(*cols)` packs a set of columns into a single `Binary` column where each value is
+an opaque, self-describing token (the polars-row encoding of the row plus an embedded
+schema header). `decode(...)` / `decode_series(...)` reverse it back into a `Struct`.
+
+The token is self-describing, so the schema does not need to be stored anywhere external
+to round-trip through a vendor that only holds the opaque bytes.
+
+Encode paths:
+    * :func:`encode` -- lazy expr producing a token column.
+    * :func:`encode_series` -- eager, returns the token `Series` directly.
+    * :func:`get_header` -- pull the schema header out of a token / token `Series`.
+
+Decode paths:
+    * :func:`decode` -- fully lazy; you supply the ``schema_header`` bytes (zero peeking).
+    * :func:`decode_peek` -- lazy bulk decode; sniffs the header from one materialized token.
+    * :func:`decode_series` -- fully eager, schema-free.
+"""
+
+from __future__ import annotations
+
+import struct
+from pathlib import Path
+from typing import TYPE_CHECKING, cast
+
+import polars as pl
+from polars.plugins import register_plugin_function
+
+if TYPE_CHECKING:
+    from polars._typing import IntoExpr
+    from typing import Union
+
+    Frame = Union[pl.DataFrame, pl.LazyFrame]
+
+__all__ = [
+    "encode",
+    "encode_series",
+    "get_header",
+    "decode",
+    "decode_peek",
+    "decode_series",
+]
+
+_LIB = Path(__file__).parent
+
+
+def encode(*exprs: IntoExpr) -> pl.Expr:
+    """Encode one or more columns into a single self-describing `Binary` token column.
+
+    >>> import polars as pl
+    >>> df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]})
+    >>> out = df.select(encode("x", "y").alias("tok"))
+    >>> out.schema
+    Schema({'tok': Binary})
+    >>> out.select(decode_peek(out, "tok").alias("row")).to_series().to_list()
+    [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]
+    """
+    if not exprs:
+        msg = "encode() requires at least one column"
+        raise ValueError(msg)
+    return register_plugin_function(
+        plugin_path=_LIB,
+        function_name="row_encode",
+        args=list(exprs),
+        is_elementwise=True,
+    )
+
+
+def encode_series(*series: pl.Series) -> pl.Series:
+    """Eagerly encode one or more `Series` into a single `Binary` token `Series`.
+
+    The eager counterpart to :func:`encode`: instead of a lazy expression it returns the
+    materialized token column directly, so callers holding `Series` (not a frame) can get
+    tokens — and, via :func:`get_header`, the schema header — in one step.
+
+    >>> import polars as pl
+    >>> tok = encode_series(pl.Series("x", [1, 2, 3]))
+    >>> tok.dtype
+    Binary
+    >>> decode_series(tok).to_list()
+    [{'x': 1}, {'x': 2}, {'x': 3}]
+    """
+    if not series:
+        msg = "encode_series() requires at least one series"
+        raise ValueError(msg)
+    return pl.select(encode(*(pl.lit(s) for s in series))).to_series()
+
+
+def get_header(token: bytes | pl.Series) -> bytes:
+    """Lift the `[u32 len][header]` schema prefix out of a token.
+
+    Accepts either a single token (`bytes`) or a whole token `Series`, in which case the
+    header is read from its first non-null value. The returned bytes are exactly what
+    :func:`decode` expects as its ``schema_header`` argument, enabling a fully lazy decode
+    without re-sniffing the data.
+
+    The first four bytes are the little-endian length of the header that follows:
+
+    >>> import polars as pl
+    >>> tok = encode_series(pl.Series("x", [1, 2, 3]))
+    >>> header = get_header(tok)
+    >>> import struct
+    >>> struct.unpack_from("<I", header)[0] == len(header) - 4
+    True
+    >>> get_header(tok[0]) == header
+    True
+    >>> decode_series(tok).to_list() == pl.select(
+    ...     decode(pl.lit(tok), schema_header=header)
+    ... ).to_series().to_list()
+    True
+    """
+    if isinstance(token, pl.Series):
+        first = next((v for v in token if v is not None), None)
+        if first is None:
+            msg = "cannot extract header from an all-null / empty Series"
+            raise ValueError(msg)
+        token = first
+    if len(token) < 4:
+        msg = "token too short to contain a schema header"
+        raise ValueError(msg)
+    (header_len,) = struct.unpack_from("<I", token, 0)
+    return token[: 4 + header_len]
+
+
+# TODO: Should be able to do this without the schema_header
+# TODO: What about regular dataframes?
+def decode(expr: IntoExpr, *, schema_header: bytes) -> pl.Expr:
+    """Decode a `Binary` token column back into a `Struct`.
+
+    `schema_header` is the header prefix of any token in the column (see
+    :func:`decode_series` for the eager path that extracts it for you). It is required so
+    the output `Struct` dtype can be resolved before the data is materialized, which the
+    Polars lazy engine needs.
+
+    >>> import polars as pl
+    >>> df = pl.DataFrame({"x": [1, 2]}).select(encode("x").alias("tok"))
+    >>> header = get_header(df.to_series())
+    >>> df.select(decode("tok", schema_header=header).alias("row")).to_series().to_list()
+    [{'x': 1}, {'x': 2}]
+    """
+    return register_plugin_function(
+        plugin_path=_LIB,
+        function_name="row_decode",
+        args=[expr],
+        is_elementwise=True,
+        kwargs={"schema_header": schema_header},
+    )
+
+
+def decode_peek(frame: Frame, column: str) -> pl.Expr:
+    """Build a lazy `decode` expr for `column`, sniffing the schema header for you.
+
+    Unlike :func:`decode`, the caller does not supply a ``schema_header``. This collects a
+    single token from `frame` to read the embedded header, then returns the normal lazy
+    `decode` expression with that header baked in -- so the bulk decode still runs lazily
+    while only one value is materialized up front.
+
+    `frame` may be a `DataFrame` or `LazyFrame`. A peek is required because the output
+    `Struct` dtype must be known before the lazy engine sees any data (see :func:`decode`).
+
+    >>> import polars as pl
+    >>> df = pl.DataFrame({"x": [1, 2], "y": ["a", "b"]}).select(
+    ...     encode("x", "y").alias("tok")
+    ... )
+    >>> df.select(decode_peek(df, "tok").alias("row")).to_series().to_list()
+    [{'x': 1, 'y': 'a'}, {'x': 2, 'y': 'b'}]
+    """
+    peek = cast(
+        "pl.DataFrame",
+        frame.lazy().select(pl.col(column).drop_nulls().first()).collect(),
+    )
+    first = peek.to_series()[0] if peek.height else None
+    if first is None:
+        msg = (
+            "cannot infer schema: no non-null token in the column; "
+            "use decode(schema_header=...)"
+        )
+        raise ValueError(msg)
+    header = get_header(first)
+    return decode(pl.col(column), schema_header=header)
+
+
+# TODO: Should try and unify this with the regular decoder
+def decode_series(s: pl.Series) -> pl.Series:
+    """Eagerly decode a `Binary` token Series into a `Struct` Series, schema-free.
+
+    The schema header is read directly from the first non-null token, so the caller does
+    not need to supply or retain any schema.
+
+    >>> import polars as pl
+    >>> tok = encode_series(pl.Series("x", [10, 20]))
+    >>> decode_series(tok).to_list()
+    [{'x': 10}, {'x': 20}]
+    """
+    header = get_header(s)
+    return pl.select(decode(pl.lit(s), schema_header=header)).to_series()

pl_row_encode-0.3.1.dist-info/METADATA

@@ -0,0 +1,92 @@
+Metadata-Version: 2.4
+Name: pl-row-encode
+Version: 0.3.1
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: OS Independent
+Requires-Dist: polars>=1.34,<2
+License-File: LICENSE
+Summary: Row-level, type-preserving encode/decode for Polars columns
+Keywords: polars,plugin,encode,decode,serialization,rust
+Author-email: Tyler Riccio <tylerriccio8@gmail.com>
+License-Expression: MIT
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/tylerriccio33/pl-row-encode
+Project-URL: Repository, https://github.com/tylerriccio33/pl-row-encode
+
+# pl-row-encode
+
+A Polars plugin for **row-level, type-preserving encode/decode**.
+
+`encode(*cols)` packs a set of columns into a single `Binary` column where each value is an
+opaque, **self-describing** token: the [`polars-row`](https://docs.rs/polars-row) encoding of
+the row, prefixed with an embedded schema header. `decode_series(...)` reverses it back into a
+`Struct`, recovering the original dtypes **without needing any external schema**.
+
+```
+DataFrame
+  -> encode(*cols)
+  -> opaque bytes
+  -> decode(...)   # (row bytes -> Struct -> original typed columns)
+  -> DataFrame
+```
+
+The type information rides with the token and can be decoded on the spot at some later date.
+
+## Token layout
+
+Each `Binary` value is:
+
+```
+[ u32 header_len (LE) ][ header bytes ][ row bytes ]
+```
+
+`header` is a bincode-serialized `Vec<Field>` (logical schema); `row bytes` is the
+unordered `polars-row` encoding of that single row. Embedding the header per value makes
+every token independently decodable.
+
+## Usage
+
+```python
+import polars as pl
+from pl_row_encode import encode, decode_series
+
+df = pl.DataFrame({"id": [1, 2], "name": ["alice", "bob"]})
+
+tokens = df.select(tok=encode("id", "name"))["tok"]   # dtype: Binary
+# ... hand `tokens` to a vendor, get them back ...
+
+decoded = decode_series(tokens).struct.unnest()        # back to id / name with dtypes
+```
+
+For the lazy engine, the output `Struct` dtype must be known up front, so pass a token's
+header explicitly:
+
+```python
+from pl_row_encode import decode
+header = ...  # the [u32 len][header] prefix of any token
+lf.select(decode("tok", schema_header=header)).collect()
+```
+
+## Development
+
+```bash
+make develop   # build the Rust extension into the venv (uv run maturin develop)
+make test      # build + run pytest
+make lint      # ruff + ty
+```
+
+The first `make develop` compiles the full Polars Rust workspace and takes a few minutes;
+subsequent builds are incremental and fast.
+
+## Notes / limitations
+
+- Built on `polars-row`, the same machinery Polars uses internally for sort/group-by row
+  encoding — lossless for primitive, string, boolean, temporal, and nested types.
+- `decode_series` infers the schema from the first non-null token, so an all-null/empty
+  Series needs the explicit `decode(schema_header=...)` form.
+

pl_row_encode-0.3.1.dist-info/RECORD

@@ -0,0 +1,7 @@
+pl_row_encode/__init__.py,sha256=yH6TtK3su-kwKgqmmnq5adTNVOAMai68NlsnRO-xGi4,7443
+pl_row_encode/_internal.pyd,sha256=ZFSk8yzOpSOBwcstLPbjF3d4rKDTMqHSg69Srpqls5A,27591168
+pl_row_encode-0.3.1.dist-info/METADATA,sha256=Y9Way1gW8ZgHVGE5zkfa1oQkni_pqjb3NpNs7izyajw,3219
+pl_row_encode-0.3.1.dist-info/WHEEL,sha256=dB50znGoqD95tHNeV5UBZvlfLNwgEh0iQcswp5YahQ4,95
+pl_row_encode-0.3.1.dist-info/licenses/LICENSE,sha256=NbFGVa7pEgkuRWafOKEf0lUJ_XBdwRFAW-e5AzPbk9w,1090
+pl_row_encode-0.3.1.dist-info/sboms/pl_row_encode.cyclonedx.json,sha256=tPTc_REDrsDGlsmY217w9wV6c8O3EeCmTvzDmUggR_c,239574
+pl_row_encode-0.3.1.dist-info/RECORD,,

pl_row_encode-0.3.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.13.3)
+Root-Is-Purelib: false
+Tag: cp39-abi3-win_amd64

pl_row_encode-0.3.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tyler Riccio
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.