hotdata-materialized 0.1.0__py3-none-any.whl

hotdata_materialized/__init__.py

@@ -0,0 +1,44 @@
+"""hotdata-materialized: offload expensive Django query results to Hotdata.
+
+Public surface: the @materialize decorator and MaterializedFrame handle,
+built on fingerprinting, the remote registry, and the entry store.
+"""
+
+from .conf import Config
+from .client import HotdataClients, get_clients, reset_clients
+from .decorator import MaterializedFrame, materialize
+from .exceptions import (
+    ConfigurationError,
+    FingerprintError,
+    MaterializedError,
+    RegistryError,
+    StoreError,
+)
+from .fingerprint import fingerprint_call, fingerprint_queryset
+from .indexes import BM25, Vector
+from .registry import Registry, RegistryEntry
+from .store import EntryStore
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "materialize",
+    "MaterializedFrame",
+    "BM25",
+    "Vector",
+    "Config",
+    "HotdataClients",
+    "get_clients",
+    "reset_clients",
+    "MaterializedError",
+    "ConfigurationError",
+    "FingerprintError",
+    "RegistryError",
+    "StoreError",
+    "fingerprint_call",
+    "fingerprint_queryset",
+    "Registry",
+    "RegistryEntry",
+    "EntryStore",
+    "__version__",
+]

hotdata_materialized/_arrow.py

@@ -0,0 +1,44 @@
+"""Arrow/parquet encoding helpers shared by the registry and the entry store."""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+from typing import Optional
+
+import pyarrow as pa
+import pyarrow.ipc as pa_ipc
+import pyarrow.parquet as pq
+
+
+def table_to_parquet_bytes(table: pa.Table) -> bytes:
+    buffer = io.BytesIO()
+    pq.write_table(table, buffer)
+    return buffer.getvalue()
+
+
+def table_to_ipc_bytes(table: pa.Table) -> bytes:
+    sink = io.BytesIO()
+    with pa_ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue()
+
+
+def encode_inline_payload(table: pa.Table, threshold_bytes: int) -> Optional[str]:
+    # threshold bounds the payload as stored (post-base64), not the raw IPC
+    payload = base64.b64encode(table_to_ipc_bytes(table)).decode("ascii")
+    if len(payload) > threshold_bytes:
+        return None
+    return payload
+
+
+def decode_inline_payload(payload: str) -> pa.Table:
+    with pa_ipc.open_stream(io.BytesIO(base64.b64decode(payload))) as reader:
+        return reader.read_all()
+
+
+def schema_to_json(table: pa.Table) -> str:
+    return json.dumps(
+        [{"name": field.name, "type": str(field.type)} for field in table.schema]
+    )

hotdata_materialized/_sql.py

@@ -0,0 +1,42 @@
+"""SQL literal and identifier encoding.
+
+POST /v1/queries takes SQL text only — there are no bind parameters — so
+every value written into SQL must go through quote_literal(), and every
+caller-supplied identifier (column names in search) through quote_ident().
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import math
+import re
+
+from .exceptions import RegistryError
+
+_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z")
+
+
+def quote_ident(name: str) -> str:
+    if not _IDENT.match(name):
+        raise ValueError(f"invalid SQL identifier: {name!r}")
+    return name
+
+
+def quote_literal(value) -> str:
+    if value is None:
+        return "NULL"
+    if isinstance(value, bool):
+        return "TRUE" if value else "FALSE"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        if math.isnan(value) or math.isinf(value):
+            raise RegistryError(f"cannot encode non-finite float: {value!r}")
+        return repr(value)
+    if isinstance(value, (_dt.datetime, _dt.date)):
+        return quote_literal(value.isoformat())
+    if isinstance(value, str):
+        if "\x00" in value:
+            raise RegistryError("cannot encode string containing NUL byte")
+        return "'" + value.replace("'", "''") + "'"
+    raise RegistryError(f"cannot encode SQL literal of type {type(value).__name__}")

hotdata_materialized/client.py

@@ -0,0 +1,50 @@
+"""Hotdata SDK client construction.
+
+One ApiClient (and its connection pool) per process, shared by the query,
+databases, and uploads resource APIs.
+"""
+
+from __future__ import annotations
+
+import threading
+from typing import Optional
+
+import hotdata
+from hotdata.arrow import ResultsApi
+from hotdata.query import QueryApi
+from hotdata.uploads import UploadsApi
+
+from .conf import Config
+
+
+class HotdataClients:
+    def __init__(self, config: Config):
+        sdk_config = hotdata.Configuration(
+            host=config.api_url,
+            api_key=config.api_key,
+            workspace_id=config.workspace_id,
+        )
+        self.api_client = hotdata.ApiClient(sdk_config)
+        self.query = QueryApi(self.api_client)
+        self.databases = hotdata.DatabasesApi(self.api_client)
+        self.uploads = UploadsApi(self.api_client)
+        self.results = ResultsApi(self.api_client)
+        self.indexes = hotdata.IndexesApi(self.api_client)
+
+
+_lock = threading.Lock()
+_clients: Optional[HotdataClients] = None
+
+
+def get_clients(config: Optional[Config] = None) -> HotdataClients:
+    global _clients
+    with _lock:
+        if _clients is None:
+            _clients = HotdataClients(config or Config.from_django())
+        return _clients
+
+
+def reset_clients() -> None:
+    global _clients
+    with _lock:
+        _clients = None

hotdata_materialized/conf.py

@@ -0,0 +1,70 @@
+"""Configuration, loaded from the HOTDATA_MATERIALIZED dict in Django settings.
+
+The Config dataclass is plain and injectable so the library (and its tests)
+can run without a configured Django project.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict
+
+from .exceptions import ConfigurationError
+
+DEFAULT_API_URL = "https://api.hotdata.dev"
+
+_REQUIRED_KEYS = ("API_KEY", "WORKSPACE_ID")
+
+
+@dataclass(frozen=True)
+class Config:
+    api_key: str
+    workspace_id: str
+    api_url: str = DEFAULT_API_URL
+    registry_database: str = "materialized_registry"
+    inline_threshold_bytes: int = 2 * 1024 * 1024
+    entry_prefix: str = "mz_"
+    # Server-side entry-database expiry is set to ttl + grace: a safety net
+    # behind the sweep, never the primary eviction path.
+    entry_expiry_grace_seconds: int = 24 * 3600
+    # Write-behind is the default: a miss returns as soon as the caller's data
+    # is in hand and the persist runs on a small background pool.
+    background: bool = True
+    background_max_workers: int = 2
+
+    @classmethod
+    def from_dict(cls, raw: Dict[str, Any]) -> "Config":
+        missing = [key for key in _REQUIRED_KEYS if not raw.get(key)]
+        if missing:
+            raise ConfigurationError(
+                "HOTDATA_MATERIALIZED is missing required keys: " + ", ".join(missing)
+            )
+        return cls(
+            api_key=raw["API_KEY"],
+            workspace_id=raw["WORKSPACE_ID"],
+            api_url=raw.get("API_URL", DEFAULT_API_URL),
+            registry_database=raw.get("REGISTRY_DATABASE", cls.registry_database),
+            inline_threshold_bytes=raw.get(
+                "INLINE_THRESHOLD_BYTES", cls.inline_threshold_bytes
+            ),
+            entry_prefix=raw.get("ENTRY_PREFIX", cls.entry_prefix),
+            entry_expiry_grace_seconds=raw.get(
+                "ENTRY_EXPIRY_GRACE_SECONDS", cls.entry_expiry_grace_seconds
+            ),
+            background=raw.get("BACKGROUND", cls.background),
+            background_max_workers=raw.get(
+                "BACKGROUND_MAX_WORKERS", cls.background_max_workers
+            ),
+        )
+
+    @classmethod
+    def from_django(cls) -> "Config":
+        from django.conf import settings
+
+        raw = getattr(settings, "HOTDATA_MATERIALIZED", None)
+        if not isinstance(raw, dict):
+            raise ConfigurationError(
+                "Define a HOTDATA_MATERIALIZED dict in Django settings "
+                "(required keys: API_KEY, WORKSPACE_ID)."
+            )
+        return cls.from_dict(raw)

hotdata_materialized/decorator.py

@@ -0,0 +1,237 @@
+"""The @materialize decorator and the MaterializedFrame handle.
+
+Wrap an expensive function with @materialize: a hit returns a frame backed by
+Hotdata without calling the function; a miss calls it, returns a frame over
+the computed result immediately, and persists write-behind. The wrapped
+function must return something table-shaped: a pyarrow.Table, a pandas
+DataFrame, or an iterable of dicts (e.g. a Django .values() queryset).
+
+Fail-open: if the hit check or the persist raises a MaterializedError, the
+function runs and its result is served uncached — the cache can degrade to
+"no cache," never to "no page."
+"""
+
+from __future__ import annotations
+
+import functools
+import logging
+import re
+import threading
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
+
+import pyarrow as pa
+
+from .client import get_clients
+from .conf import Config
+from .exceptions import MaterializedError
+from ._sql import quote_ident, quote_literal
+from .fingerprint import fingerprint_call
+from .indexes import BM25, Vector
+from .registry import STATUS_READY, Registry, RegistryEntry
+from .store import DATA_SCHEMA, DATA_TABLE, EntryStore
+
+logger = logging.getLogger(__name__)
+
+_THIS = re.compile(r"\bthis\b")
+# quoted chunks ('...' with '' escapes, "..." with "" escapes) pass through
+# untouched so a literal like WHERE label = 'this' is never rewritten
+_SQL_CHUNKS = re.compile(r"('(?:[^']|'')*'|\"(?:[^\"]|\"\")*\")")
+
+
+def _rewrite_this(sql: str) -> str:
+    return "".join(
+        part if part.startswith(("'", '"')) else _THIS.sub(DATA_TABLE, part)
+        for part in _SQL_CHUNKS.split(sql)
+    )
+
+
+class MaterializedFrame:
+    """Handle over a materialized entry. One interface, two backings: a fresh
+    miss wraps the locally computed table; a hit reads from Hotdata lazily."""
+
+    def __init__(
+        self,
+        store: EntryStore,
+        entry: RegistryEntry,
+        table: Optional[pa.Table] = None,
+        *,
+        cached: bool,
+    ):
+        self._store = store
+        self.entry = entry
+        self._table = table
+        self.cached = cached
+
+    def arrow(self) -> pa.Table:
+        if self._table is None:
+            self._table = self._store.read_table(self.entry)
+        return self._table
+
+    def to_pylist(self) -> List[Dict[str, Any]]:
+        return self.arrow().to_pylist()
+
+    def df(self) -> Any:
+        """The data as a pandas.DataFrame (requires pandas)."""
+        return self.arrow().to_pandas()
+
+    def sql(self, sql: str) -> pa.Table:
+        """Run SQL server-side against this entry's database; the identifier
+        `this` names the cached data (e.g. "SELECT x FROM this LIMIT 5").
+
+        Requires a persisted entry: available on hits, or after the
+        write-behind persist lands (a fresh miss raises StoreError until
+        then; use background=False on the decorator to persist inline)."""
+        return self._store.query_table(self.entry, _rewrite_this(sql))
+
+    def search(self, query: str, *, column: str, limit: int = 10) -> pa.Table:
+        """BM25 keyword search over the entry, best matches first. Requires
+        a BM25 index on the column (declare with @materialize(index=BM25(...));
+        a freshly missed entry may error until the index build finishes)."""
+        table_ref = f"default.{DATA_SCHEMA}.{DATA_TABLE}"
+        return self._store.query_table(
+            self.entry,
+            f"SELECT * FROM bm25_search({quote_literal(table_ref)}, "
+            f"{quote_literal(quote_ident(column))}, {quote_literal(query)}) "
+            f"ORDER BY score DESC LIMIT {int(limit)}",
+        )
+
+    def vector_search(self, query: str, *, column: str, limit: int = 10) -> pa.Table:
+        """Semantic search over the entry, nearest first. Requires a vector
+        index (declare with @materialize(index=Vector(...))). Pass the indexed
+        source column; the query text is embedded server-side."""
+        return self._store.query_table(
+            self.entry,
+            f"SELECT *, vector_distance({quote_ident(column)}, "
+            f"{quote_literal(query)}) AS distance "
+            f"FROM {DATA_TABLE} ORDER BY distance LIMIT {int(limit)}",
+        )
+
+    def __len__(self) -> int:
+        return self.arrow().num_rows
+
+
+_runtime_lock = threading.Lock()
+_runtime: Optional[Tuple[Registry, EntryStore]] = None
+
+
+def get_runtime() -> Tuple[Registry, EntryStore]:
+    """The process-wide (Registry, EntryStore) pair, built from Django
+    settings on first use."""
+    global _runtime
+    with _runtime_lock:
+        if _runtime is None:
+            config = Config.from_django()
+            clients = get_clients(config)
+            registry = Registry(clients, config)
+            _runtime = (registry, EntryStore(clients, config, registry))
+        return _runtime
+
+
+def reset_runtime() -> None:
+    global _runtime
+    with _runtime_lock:
+        _runtime = None
+
+
+def to_arrow(result: Any) -> pa.Table:
+    """Normalize a captured result into a pyarrow.Table."""
+    if isinstance(result, pa.Table):
+        return result
+    if type(result).__module__.split(".")[0] == "pandas":
+        return pa.Table.from_pandas(result)
+    rows = list(result)
+    if rows and not isinstance(rows[0], dict):
+        raise TypeError(
+            "materialize expects a pyarrow.Table, a pandas.DataFrame, or an "
+            "iterable of dicts (e.g. a .values() queryset); got an iterable "
+            f"of {type(rows[0]).__name__}"
+        )
+    return pa.Table.from_pylist(rows)
+
+
+def materialize(
+    key: Optional[str] = None,
+    *,
+    ttl: Optional[int] = 3600,
+    version: int = 0,
+    background: Optional[bool] = None,
+    key_fn: Optional[Callable[..., Any]] = None,
+    index: Union[BM25, Vector, Sequence[Union[BM25, Vector]], None] = None,
+) -> Callable[[Callable[..., Any]], Callable[..., MaterializedFrame]]:
+    """Materialize a function's result into Hotdata.
+
+        @materialize(ttl=3600)
+        def revenue_by_region():
+            return Order.objects.values("region").annotate(revenue=Sum("total"))
+
+    The call returns a MaterializedFrame either way; `.cached` says which path
+    served it. `version=` busts the cache on code changes; `key_fn=` maps
+    non-JSON-serializable arguments to a stable identity. `index=` declares
+    BM25/Vector search indexes built when the entry persists.
+    """
+    declarations: Tuple[Union[BM25, Vector], ...]
+    if index is None:
+        declarations = ()
+    elif isinstance(index, (BM25, Vector)):
+        declarations = (index,)
+    else:
+        declarations = tuple(index)
+    embedded = any(
+        isinstance(d, Vector) and d.provider is not None for d in declarations
+    )
+    if embedded and len(declarations) > 1:
+        # platform constraint: embedding-backed vector indexes cannot coexist
+        # with other indexes on the same table
+        raise ValueError(
+            "an embedding-backed Vector index (provider=...) cannot be "
+            "combined with other indexes on the same entry"
+        )
+
+    def decorate(func: Callable[..., Any]) -> Callable[..., MaterializedFrame]:
+        @functools.wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> MaterializedFrame:
+            registry, store = get_runtime()
+            fingerprint = fingerprint_call(
+                func, args, kwargs, version=version, key_fn=key_fn
+            )
+            entry = None
+            try:
+                entry = registry.lookup(fingerprint)
+            except MaterializedError:
+                logger.warning(
+                    "hit check for %s failed; running uncached",
+                    func.__qualname__,
+                    exc_info=True,
+                )
+            if (
+                entry is not None
+                and entry.status == STATUS_READY
+                and not entry.is_expired(store._now())
+            ):
+                return MaterializedFrame(store, entry, cached=True)
+
+            result = func(*args, **kwargs)
+            table = to_arrow(result)
+            label = key or func.__qualname__
+            try:
+                pending = store.materialize(
+                    fingerprint,
+                    table,
+                    key=label,
+                    ttl=ttl,
+                    version=version,
+                    background=background,
+                    indexes=declarations,
+                )
+            except MaterializedError:
+                logger.warning(
+                    "persist of %s failed; serving the result uncached",
+                    label,
+                    exc_info=True,
+                )
+                pending = RegistryEntry(fingerprint=fingerprint, key=label)
+            return MaterializedFrame(store, pending, table=table, cached=False)
+
+        return wrapper
+
+    return decorate

hotdata_materialized/exceptions.py

@@ -0,0 +1,22 @@
+"""Exception hierarchy. Everything raised by this package subclasses
+MaterializedError so callers can catch broadly."""
+
+
+class MaterializedError(Exception):
+    """Base class for all hotdata-materialized errors."""
+
+
+class ConfigurationError(MaterializedError):
+    """The HOTDATA_MATERIALIZED setting is missing or incomplete."""
+
+
+class FingerprintError(MaterializedError):
+    """A call could not be fingerprinted deterministically."""
+
+
+class RegistryError(MaterializedError):
+    """A registry read or write failed."""
+
+
+class StoreError(MaterializedError):
+    """Materializing or evicting an entry failed."""

hotdata_materialized/fingerprint.py

@@ -0,0 +1,77 @@
+"""Entry fingerprinting.
+
+A fingerprint content-addresses one materialized entry. False misses are
+safe; false hits are not — so canonicalization rejects anything it cannot
+serialize deterministically instead of falling back to repr(), whose output
+is process-dependent for arbitrary objects.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import decimal
+import hashlib
+import json
+import uuid
+
+from .exceptions import FingerprintError
+
+SHORT_LENGTH = 16
+
+
+def _stable_default(value):
+    # Type-tagged so date(2026, 7, 22) and the string "2026-07-22" (etc.)
+    # cannot fingerprint identically — a false hit serves someone else's data.
+    if isinstance(value, _dt.datetime):
+        return f"datetime:{value.isoformat()}"
+    if isinstance(value, _dt.date):
+        return f"date:{value.isoformat()}"
+    if isinstance(value, _dt.time):
+        return f"time:{value.isoformat()}"
+    if isinstance(value, decimal.Decimal):
+        return f"decimal:{value}"
+    if isinstance(value, uuid.UUID):
+        return f"uuid:{value}"
+    if isinstance(value, (set, frozenset)):
+        return sorted(value)
+    if isinstance(value, (bytes, bytearray)):
+        return f"bytes:{value.hex()}"
+    raise FingerprintError(
+        f"cannot fingerprint value of type {type(value).__name__}; "
+        "pass JSON-serializable arguments or supply key_fn= on the decorator"
+    )
+
+
+def canonical_json(obj) -> str:
+    try:
+        return json.dumps(
+            obj, sort_keys=True, separators=(",", ":"), default=_stable_default
+        )
+    except TypeError as exc:
+        raise FingerprintError(str(exc)) from exc
+
+
+def _digest(canonical: str) -> str:
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def fingerprint_call(func, args=(), kwargs=None, *, version=0, key_fn=None) -> str:
+    ref = f"{func.__module__}.{func.__qualname__}"
+    if key_fn is not None:
+        call = key_fn(*args, **(kwargs or {}))
+    else:
+        call = {"args": list(args), "kwargs": kwargs or {}}
+    return _digest(canonical_json({"fn": ref, "version": version, "call": call}))
+
+
+def fingerprint_queryset(queryset, *, version=0) -> str:
+    # (sql, params) from the compiler, not str(queryset.query) — the latter
+    # inlines parameters unreliably and is not stable across backends.
+    sql, params = queryset.query.get_compiler(queryset.db).as_sql()
+    return _digest(
+        canonical_json({"sql": sql, "params": list(params), "version": version})
+    )
+
+
+def short(fingerprint: str) -> str:
+    return fingerprint[:SHORT_LENGTH]

hotdata_materialized/indexes.py

@@ -0,0 +1,33 @@
+"""Search-index declarations for @materialize(index=...).
+
+Indexes are created right after the entry's data loads; builds run as
+background jobs on the Hotdata side, so a freshly missed entry may briefly
+error on search until its index is ready.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class BM25:
+    """Keyword (BM25) index over a text column; query via frame.search()."""
+
+    column: str
+
+
+@dataclass(frozen=True)
+class Vector:
+    """Vector index; query via frame.vector_search().
+
+    With `provider` set (an embedding provider id), the column is treated
+    as text and embedded server-side; vector_search then targets the source
+    column and query text is embedded automatically. An embedding-backed
+    index cannot share a table with other indexes."""
+
+    column: str
+    provider: Optional[str] = None
+    metric: Optional[str] = None  # "l2" | "cosine" | "dot"
+    dimensions: Optional[int] = None