matview-cache 0.1.0__py3-none-any.whl

matview/codecs/__init__.py

@@ -0,0 +1,159 @@
+"""Codecs encode a Python object to bytes and back, and optionally split it into parts.
+
+Codecs are named, and the name is recorded per recipe, so decoding is a lookup and
+never an inference.
+
+**The byte-canonical requirement.** Because fingerprints key on content digests, a
+codec must produce identical bytes for identical input, on every machine, every time.
+A codec that fails this defeats early cutoff silently: nothing errors, the cascade
+simply never stops.
+
+Concrete rules, from the spec:
+
+- Parquet is disqualified out of the box. It stamps a creation timestamp and writer
+  version into the file, so two encodings of the same table differ. Arrow IPC with
+  compression off, fixed schema field ordering and no custom metadata is the safe default.
+- No codec may embed a timestamp, hostname, PID, or library version.
+- Compression, if ever added, must be at a pinned level with a pinned implementation,
+  or it silently breaks the guarantee across environments.
+
+Serialization is never by Python object graph, only by an explicit format. That rules
+out the obvious shortcut for arbitrary objects, and it is what keeps the cache readable
+from other languages and safe to share.
+"""
+
+from __future__ import annotations
+
+from typing import Final, Protocol, runtime_checkable
+
+from matview.errors import CodecError, CodecUnavailable
+
+__all__ = [
+    "CODEC_NAMES",
+    "BaseCodec",
+    "Codec",
+    "CodecError",
+    "CodecUnavailable",
+    "codec_is_known",
+    "get_codec",
+    "register_codec",
+]
+
+
+@runtime_checkable
+class Codec(Protocol):
+    """Encode a value to bytes and back."""
+
+    name: str
+
+    def encode(self, value: object) -> bytes | dict[str, bytes]:
+        """Return bytes for one unpartitioned artifact, or a dict of `part -> bytes`."""
+        ...
+
+    def decode(self, data: bytes) -> object:
+        """Decode one artifact's bytes."""
+        ...
+
+    def decode_parts(self, parts: dict[str, bytes]) -> object:
+        """Reassemble a partitioned value from every part, in sorted-label order.
+
+        Only partitioning codecs implement this. See DECISIONS.md D11.
+        """
+        ...
+
+
+class BaseCodec:
+    """Shared behaviour. A codec that never partitions inherits the right refusal."""
+
+    name: str = ""
+
+    def encode(self, value: object) -> bytes | dict[str, bytes]:  # pragma: no cover - abstract
+        raise NotImplementedError
+
+    def decode(self, data: bytes) -> object:  # pragma: no cover - abstract
+        raise NotImplementedError
+
+    def decode_parts(self, parts: dict[str, bytes]) -> object:
+        raise CodecError(
+            f"codec {self.name!r} does not partition, so it cannot reassemble "
+            f"{len(parts)} parts. A partitioned recipe needs a partitioning codec."
+        )
+
+
+#: Every codec name this build knows about, whether or not its extra is installed.
+CODEC_NAMES: Final = frozenset(
+    {
+        "json",
+        "arrow",
+        "arrow-pandas",
+        "arrow-series",
+        "arrow-year",
+        "arrow-year-pandas",
+        "npy",
+    }
+)
+
+_BUILTIN_MODULES: Final = {
+    "json": "matview.codecs._json",
+    "arrow": "matview.codecs._arrow",
+    "arrow-series": "matview.codecs._arrow",
+    "arrow-pandas": "matview.codecs._arrow_pandas",
+    "arrow-year": "matview.codecs._arrow",
+    "arrow-year-pandas": "matview.codecs._arrow_pandas",
+    "npy": "matview.codecs._npy",
+}
+
+#: Which optional extra installs each codec. Not derivable from the name: `arrow-pandas`
+#: shares the Arrow encoding but needs pandas rather than polars.
+_EXTRAS: Final = {
+    "json": "json",
+    "arrow": "arrow",
+    "arrow-series": "arrow",
+    "arrow-pandas": "pandas",
+    "arrow-year": "arrow",
+    "arrow-year-pandas": "pandas",
+    "npy": "npy",
+}
+
+_REGISTRY: dict[str, Codec] = {}
+
+
+def register_codec(codec: Codec) -> None:
+    """Register a codec under its own name. Re-registering the same name replaces it."""
+    if not codec.name:
+        raise CodecError("a codec must have a name")
+    _REGISTRY[codec.name] = codec
+
+
+def get_codec(name: str) -> Codec:
+    """Look up a codec, importing its optional extra on first use."""
+    if name in _REGISTRY:
+        return _REGISTRY[name]
+
+    module = _BUILTIN_MODULES.get(name)
+    if module is None:
+        known = ", ".join(sorted(set(_REGISTRY) | CODEC_NAMES))
+        raise CodecError(f"unknown codec {name!r}; known codecs: {known}")
+
+    import importlib
+
+    try:
+        importlib.import_module(module)
+    except ImportError as exc:
+        extra = _EXTRAS.get(name, name)
+        raise CodecUnavailable(
+            f"codec {name!r} needs its optional extra: pip install 'matview-cache[{extra}]' ({exc})"
+        ) from exc
+
+    if name not in _REGISTRY:  # pragma: no cover - a builtin module that failed to register
+        raise CodecError(f"module {module} did not register codec {name!r}")
+    return _REGISTRY[name]
+
+
+def codec_is_known(name: str) -> bool:
+    """Whether a codec name resolves, without importing its extra.
+
+    Used at registration so a typo fails at import time rather than at the first
+    compute, while an uninstalled extra does not stop a recipe being registered.
+    """
+    return name in _REGISTRY or name in _BUILTIN_MODULES

matview/codecs/_arrow.py

@@ -0,0 +1,104 @@
+"""Arrow IPC codecs for polars.
+
+The encoding itself lives in `_arrow_ipc`, which imports only pyarrow, so what is stored
+does not depend on which dataframe library produced it. Parquet is disqualified out of
+the box: it stamps a creation timestamp and writer version into the file, so two
+encodings of the same table differ and every downstream fingerprint changes for no reason.
+
+Two names share one encoding, because the spec is explicit that "the codec is a property
+of the recipe, so decoding is a lookup and never an inference":
+
+- `arrow` decodes to a DataFrame.
+- `arrow-series` decodes to a Series, for a recipe whose output genuinely is one.
+
+Both accept either on encode. Inspecting the table to guess which the caller wanted would
+be exactly the inference the spec rules out. See DECISIONS.md D14.
+
+polars is imported at module level on purpose. This module is itself imported lazily, only
+when a recipe asks for one of these codecs, and that import failing is how `get_codec`
+learns to raise CodecUnavailable with an install hint.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import polars as pl
+
+from matview.codecs import BaseCodec, register_codec
+from matview.codecs._arrow_ipc import canonical_table, encode_table, read_table
+from matview.codecs._arrow_parts import combine_years, slice_by_year
+from matview.errors import CodecError
+
+__all__ = ["ArrowCodec", "ArrowSeriesCodec", "ArrowYearCodec"]
+
+
+def _to_table(value: object) -> Any:
+    if isinstance(value, pl.DataFrame):
+        frame = value
+    elif isinstance(value, pl.Series):
+        # A Series becomes a one-column table under its own name, or "" when unnamed.
+        frame = value.to_frame()
+    else:
+        raise CodecError(
+            f"the arrow codecs encode a polars DataFrame or Series, got "
+            f"{type(value).__module__}.{type(value).__qualname__}"
+        )
+    return canonical_table(frame.to_arrow())
+
+
+class ArrowCodec(BaseCodec):
+    name = "arrow"
+
+    def encode(self, value: object) -> bytes | dict[str, bytes]:
+        return encode_table(_to_table(value))
+
+    def decode(self, data: bytes) -> object:
+        return pl.from_arrow(read_table(data))
+
+
+class ArrowSeriesCodec(ArrowCodec):
+    name = "arrow-series"
+
+    def decode(self, data: bytes) -> object:
+        frame: Any = super().decode(data)
+        if len(frame.columns) != 1:
+            raise CodecError(
+                f"codec 'arrow-series' expects one column, found {len(frame.columns)}: "
+                f"{', '.join(frame.columns)}. A multi-column value wants codec 'arrow'."
+            )
+        return frame.to_series()
+
+
+class ArrowYearCodec(ArrowCodec):
+    """`arrow` bytes, split into one artifact per calendar year.
+
+    Registered as `arrow-year`, partitioning on a column named `date`. For any other
+    column name, register your own instance:
+
+        register_codec(ArrowYearCodec(name="bars-year", column="day"))
+
+    An **empty** frame has no years, so it stores as a single unpartitioned artifact
+    rather than as zero parts. An empty series is a value, not a miss (rule 9), and zero
+    parts would store nothing at all. See DECISIONS.md D43.
+    """
+
+    def __init__(self, name: str = "arrow-year", column: str = "date") -> None:
+        self.name = name
+        self.column = column
+
+    def encode(self, value: object) -> bytes | dict[str, bytes]:
+        table = _to_table(value)
+        if table.num_rows == 0:
+            return encode_table(table)
+        return {
+            label: encode_table(part) for label, part in slice_by_year(table, self.column).items()
+        }
+
+    def decode_parts(self, parts: dict[str, bytes]) -> object:
+        return pl.from_arrow(combine_years({label: read_table(d) for label, d in parts.items()}))
+
+
+register_codec(ArrowCodec())
+register_codec(ArrowSeriesCodec())
+register_codec(ArrowYearCodec())

matview/codecs/_arrow_ipc.py

@@ -0,0 +1,89 @@
+"""The canonical Arrow IPC encoding, shared by every Arrow-backed codec.
+
+Library-agnostic on purpose: it imports pyarrow and nothing else, so a consumer who
+works in pandas does not have to install polars to read its own cache, and the bytes are
+defined here rather than by whichever dataframe library happened to produce the table.
+
+That last point is the whole reason this module exists separately. Two things had to be
+taken out of the producing library's hands:
+
+- **The writer.** polars' own `write_ipc_stream` is not byte-stable across polars
+  releases (see DECISIONS.md D29), so the stream is written through pyarrow under
+  explicit options.
+- **The offset width.** `string`/`large_string`, `binary`/`large_binary` and
+  `list`/`large_list` differ only in whether offsets are 32- or 64-bit. It carries no
+  semantic difference, and each library picks a side: polars produces `large_string`,
+  pandas 2 produces `string`, pandas 3 produces `large_string`. Left alone, the same
+  logical table gets different digests depending on its producer and on a minor version
+  of it. Normalizing to the large variant pins the choice to matview. See DECISIONS.md D39.
+"""
+
+from __future__ import annotations
+
+import io
+from typing import Any
+
+import pyarrow as pa
+
+from matview.errors import CodecError
+
+__all__ = ["canonical_table", "encode_table", "read_table", "widen", "write_options"]
+
+
+def write_options() -> Any:
+    """Explicit and pinned, so no pyarrow default can drift into the bytes."""
+    return pa.ipc.IpcWriteOptions(
+        compression=None,
+        metadata_version=pa.ipc.MetadataVersion.V5,
+        allow_64bit=False,
+        use_threads=False,
+    )
+
+
+def widen(kind: Any) -> Any:
+    """Normalize a type's offset width, recursively.
+
+    Only the 32/64-bit offset pairs are touched, because only they are a pure encoding
+    choice. Notably a dictionary type's index width is *not* normalized: it varies with
+    cardinality, so it is data, not an encoding detail, and forcing one width would make
+    a categorical column larger than it needs to be. A recipe whose output includes a
+    categorical should cast it explicitly if it wants a digest independent of its
+    producer.
+    """
+    if pa.types.is_string(kind):
+        return pa.large_string()
+    if pa.types.is_binary(kind):
+        return pa.large_binary()
+    if pa.types.is_list(kind) or pa.types.is_large_list(kind):
+        return pa.large_list(widen(kind.value_type))
+    if pa.types.is_struct(kind):
+        return pa.struct([pa.field(f.name, widen(f.type), f.nullable) for f in kind])
+    if pa.types.is_map(kind):
+        return pa.map_(widen(kind.key_type), widen(kind.item_type))
+    return kind
+
+
+def canonical_table(table: Any) -> Any:
+    """Put a table into the one form matview stores.
+
+    No schema or field metadata — polars and pandas both record their own there, and it
+    would differ between producers of identical data. Offset widths normalized. One
+    record batch, so that how a table happened to be assembled cannot leak into the bytes.
+    """
+    fields = [pa.field(f.name, widen(f.type), f.nullable) for f in table.schema]
+    return table.cast(pa.schema(fields)).combine_chunks()
+
+
+def encode_table(table: Any) -> bytes:
+    sink = io.BytesIO()
+    with pa.ipc.new_stream(sink, table.schema, options=write_options()) as writer:
+        writer.write_table(table)
+    return sink.getvalue()
+
+
+def read_table(data: bytes) -> Any:
+    try:
+        with pa.ipc.open_stream(io.BytesIO(data)) as reader:
+            return reader.read_all()
+    except Exception as exc:
+        raise CodecError(f"stored bytes are not an Arrow IPC stream: {exc}") from exc

matview/codecs/_arrow_pandas.py

@@ -0,0 +1,76 @@
+"""The `arrow-pandas` codec: the same bytes as `arrow`, decoded to a pandas DataFrame.
+
+A consumer working in pandas should not have to install polars to read its own cache, so
+this module imports pandas and pyarrow and not polars. The encoding is shared, and
+because offset widths are normalized in `_arrow_ipc`, a table stored here is
+byte-identical to the same table stored by the `arrow` codec — verified across pandas 2.2
+and 3.0, which disagree about how they map strings.
+
+The index is deliberately not preserved. A pandas index is presentation rather than data,
+and `preserve_index=True` would both add a column and write pandas metadata into the
+schema, which would make the digest depend on the producing library after all.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+import pyarrow as pa
+
+from matview.codecs import BaseCodec, register_codec
+from matview.codecs._arrow_ipc import canonical_table, encode_table, read_table
+from matview.codecs._arrow_parts import combine_years, slice_by_year
+from matview.errors import CodecError
+
+__all__ = ["ArrowPandasCodec", "ArrowYearPandasCodec"]
+
+
+class ArrowPandasCodec(BaseCodec):
+    name = "arrow-pandas"
+
+    def _to_table(self, value: object) -> Any:
+        if isinstance(value, pd.Series):
+            # A Series becomes a one-column table under its own name, as in the polars codec.
+            value = value.to_frame(name=value.name if value.name is not None else "")
+        if not isinstance(value, pd.DataFrame):
+            raise CodecError(
+                f"the {self.name} codec encodes a pandas DataFrame or Series, got "
+                f"{type(value).__module__}.{type(value).__qualname__}"
+            )
+        return canonical_table(pa.Table.from_pandas(value, preserve_index=False))
+
+    def encode(self, value: object) -> bytes | dict[str, bytes]:
+        return encode_table(self._to_table(value))
+
+    def decode(self, data: bytes) -> object:
+        table: Any = read_table(data)
+        return table.to_pandas()
+
+
+class ArrowYearPandasCodec(ArrowPandasCodec):
+    """`arrow-year` bytes, decoded to a pandas DataFrame.
+
+    Byte-identical to `arrow-year`, like the pair these build on, so a year-partitioned
+    cache is not partitioned by dataframe library either.
+    """
+
+    def __init__(self, name: str = "arrow-year-pandas", column: str = "date") -> None:
+        self.name = name
+        self.column = column
+
+    def encode(self, value: object) -> bytes | dict[str, bytes]:
+        table = self._to_table(value)
+        if table.num_rows == 0:
+            return encode_table(table)
+        return {
+            label: encode_table(part) for label, part in slice_by_year(table, self.column).items()
+        }
+
+    def decode_parts(self, parts: dict[str, bytes]) -> object:
+        table = combine_years({label: read_table(data) for label, data in parts.items()})
+        return table.to_pandas()
+
+
+register_codec(ArrowPandasCodec())
+register_codec(ArrowYearPandasCodec())

matview/codecs/_arrow_parts.py

@@ -0,0 +1,99 @@
+"""Calendar-year slicing for the Arrow codecs. Library-agnostic: pyarrow only.
+
+The financial case wants calendar-year partitions, and the reason is write amplification
+rather than anything about SQLite. With roughly 5,000 symbols and 50 indicators, a
+whole-series encoding is around 40 KB, so a daily append rewrites all of it — on the order
+of 10 GB per refresh against a remote store. Yearly parts rewrite only the current year,
+cutting that to roughly 500 MB. Monthly would cut it further but multiplies row count by
+240 and makes range reads stitch many pieces, which is why this is years and not months.
+
+A closed year is then immutable, and matview needs nothing extra for that to pay off: a
+recipe reading one part records a dep edge on *that part alone*, so a write to the current
+year does not invalidate anything built from a closed one.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from matview.codecs._arrow_ipc import canonical_table
+from matview.errors import CodecError
+
+__all__ = ["combine_years", "slice_by_year", "year_labels"]
+
+#: Zero-padded, so that sorting labels as text agrees with sorting years as numbers.
+#: The core treats `part` as an opaque string and the fingerprint sorts dependencies by
+#: UTF-16 code unit, under which "999" sorts after "1000". Padding removes the question.
+_LABEL = "{:04d}"
+
+
+def _partition_values(table: Any, column: str) -> Any:
+    """Validate the partitioning column and return its values, read in UTC.
+
+    Timezone-aware timestamps are read **in UTC**. This is not a detail: the same instant
+    is 2023 in UTC and 2024 in Asia/Tokyo, so without a fixed rule a row near a year
+    boundary would land in a different part depending on a column's timezone metadata, and
+    its digest would move with it. A `Date` column has no such ambiguity.
+    """
+    if column not in table.schema.names:
+        raise CodecError(
+            f"the year codecs partition on a column named {column!r}, which this value does "
+            f"not have. Columns present: {', '.join(table.schema.names) or '(none)'}. Register "
+            f"your own instance with column=... if it is called something else."
+        )
+
+    values = table[column]
+    kind = values.type
+    if pa.types.is_timestamp(kind):
+        if kind.tz is not None:
+            # Reinterpreting as naive reads the stored instant, which Arrow keeps in UTC.
+            values = values.cast(pa.timestamp(kind.unit))
+    elif not pa.types.is_date(kind):
+        raise CodecError(
+            f"column {column!r} is {kind}, which has no calendar year. The year codecs "
+            f"partition on a date or timestamp column."
+        )
+
+    if values.null_count:
+        raise CodecError(
+            f"column {column!r} has {values.null_count} null value(s), which belong to no "
+            f"year. Drop or fill them before storing: silently bucketing them would lose "
+            f"rows on the next read."
+        )
+    return values
+
+
+def year_labels(table: Any, column: str) -> list[str]:
+    """The part labels this table would produce, without slicing it."""
+    years = pc.year(_partition_values(table, column))
+    return sorted({_LABEL.format(year) for year in years.to_pylist()})
+
+
+def slice_by_year(table: Any, column: str) -> dict[str, Any]:
+    """Split a table into one canonical table per calendar year.
+
+    Rows are sorted by the partitioning column first, so that a frame which is merely
+    reordered encodes to the same bytes rather than churning every fingerprint above it.
+    Arrow's sort is stable, so rows sharing a date keep their input order — which makes
+    this fully canonical for the one-row-per-date case the design has, and no more than
+    that. A recipe with several rows per date and no meaningful order of its own should
+    sort them itself.
+    """
+    # Validated before sorting, not after: sort_by on a column that is absent or of the
+    # wrong type raises a raw Arrow error, and a codec should say which column it wanted
+    # and how to change it.
+    _partition_values(table, column)
+    ordered = table.sort_by([(column, "ascending")])
+    years = pc.year(_partition_values(ordered, column))
+    return {
+        _LABEL.format(year): canonical_table(ordered.filter(pc.equal(years, year)))
+        for year in sorted(set(years.to_pylist()))
+    }
+
+
+def combine_years(parts: dict[str, Any]) -> Any:
+    """Reassemble the parts in label order, which is year order because labels are padded."""
+    return pa.concat_tables([parts[label] for label in sorted(parts)])

matview/codecs/_json.py

@@ -0,0 +1,36 @@
+"""The `json` codec: JSON Canonicalization Scheme in, plain `json.loads` out.
+
+The spec asks this codec to sort keys and pin float formatting. JCS does both, is
+already a dependency for key hashing, and is specified well enough that another
+language produces the same bytes, so it is one canonicalization to get right instead
+of two. See DECISIONS.md D22.
+"""
+
+from __future__ import annotations
+
+import json
+
+from matview.canon import jcs
+from matview.codecs import BaseCodec, register_codec
+from matview.errors import CodecError
+
+__all__ = ["JsonCodec"]
+
+
+class JsonCodec(BaseCodec):
+    name = "json"
+
+    def encode(self, value: object) -> bytes | dict[str, bytes]:
+        try:
+            return jcs(value)
+        except Exception as exc:
+            raise CodecError(f"value is not JSON-canonicalizable: {exc}") from exc
+
+    def decode(self, data: bytes) -> object:
+        try:
+            return json.loads(data.decode("utf-8"))
+        except Exception as exc:
+            raise CodecError(f"stored bytes are not valid JSON: {exc}") from exc
+
+
+register_codec(JsonCodec())

matview/codecs/_npy.py

@@ -0,0 +1,53 @@
+"""The `npy` codec: NumPy's own array format.
+
+The .npy header records dtype, shape and element order and nothing else — no timestamp,
+hostname, PID or library version — so the bytes are stable across machines and releases.
+
+Object arrays are refused rather than serialized. NumPy would fall back to serializing
+the Python object graph for them, which would make the cache Python-only and unsafe to
+share, and would foreclose the whole foreign-recipe design. Every call here passes
+`allow_pickle=False`, on write and on read.
+"""
+
+from __future__ import annotations
+
+import io
+from typing import Any
+
+import numpy as np
+
+from matview.codecs import BaseCodec, register_codec
+from matview.errors import CodecError
+
+# numpy is imported at module level on purpose; see the note in _arrow.py.
+
+__all__ = ["NpyCodec"]
+
+
+class NpyCodec(BaseCodec):
+    name = "npy"
+
+    def encode(self, value: object) -> bytes | dict[str, bytes]:
+        if not isinstance(value, np.ndarray):
+            raise CodecError(
+                f"the npy codec encodes a numpy ndarray, got "
+                f"{type(value).__module__}.{type(value).__qualname__}"
+            )
+        if value.dtype.hasobject:
+            raise CodecError(
+                "the npy codec refuses object arrays: storing one would mean serializing "
+                "a Python object graph, which is neither portable nor safe to share"
+            )
+        buffer = io.BytesIO()
+        np.save(buffer, value, allow_pickle=False)
+        return buffer.getvalue()
+
+    def decode(self, data: bytes) -> object:
+        try:
+            array: Any = np.load(io.BytesIO(data), allow_pickle=False)
+        except Exception as exc:
+            raise CodecError(f"stored bytes are not a valid .npy payload: {exc}") from exc
+        return array
+
+
+register_codec(NpyCodec())