memcove 0.3.4__py3-none-any.whl

memcove/__init__.py

@@ -0,0 +1,3 @@
+"""Memcove — a lakehouse-backed memory service for LLM agents, exposed over MCP."""
+
+__version__ = "0.1.0"

memcove/core/arrow_io.py

@@ -0,0 +1,35 @@
+"""Decode inline payloads into Arrow tables for the ingest write path."""
+
+from __future__ import annotations
+
+import base64
+import io
+
+import pyarrow as pa
+import pyarrow.ipc as ipc
+
+from memcove.core.errors import IngestError
+
+
+def from_json_records(records: list[dict]) -> pa.Table:
+    """Build an Arrow table from a list of JSON row objects."""
+    if not records:
+        raise IngestError("inline json_records payload is empty")
+    try:
+        return pa.Table.from_pylist(records)
+    except Exception as exc:  # noqa: BLE001
+        raise IngestError(f"could not build table from json records: {exc}") from exc
+
+
+def from_arrow_ipc_b64(data_b64: str) -> pa.Table:
+    """Decode a base64 Arrow IPC stream into a table."""
+    try:
+        raw = base64.b64decode(data_b64)
+        with ipc.open_stream(io.BytesIO(raw)) as reader:
+            return reader.read_all()
+    except Exception as exc:  # noqa: BLE001
+        raise IngestError(f"could not decode arrow_ipc payload: {exc}") from exc
+
+
+def estimate_bytes(table: pa.Table) -> int:
+    return table.nbytes

memcove/core/audit.py

@@ -0,0 +1,27 @@
+"""Structured audit log for guarded data operations.
+
+Every accepted read/derive/export emits one JSON line on the ``memcove.audit``
+logger: which tenant ran what (rewritten) SQL and how much it returned. Operators
+route this logger wherever their audit sink lives. Auditing must never break a
+request, so serialization failures are swallowed.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import orjson
+
+_log = logging.getLogger("memcove.audit")
+
+
+def audit(event: str, **fields) -> None:
+    """Emit a single structured audit record."""
+    try:
+        _log.info(
+            orjson.dumps(
+                {"event": event, **fields}, default=str, option=orjson.OPT_SORT_KEYS
+            ).decode()
+        )
+    except Exception:  # noqa: BLE001 - auditing is best-effort, never fatal
+        _log.info("audit event=%s (unserializable fields)", event)

memcove/core/catalog.py

@@ -0,0 +1,183 @@
+"""PyIceberg REST-catalog client — the write / ingest path.
+
+Ingested data (inline, s3 parquet, or uploaded parquet) becomes a PyArrow
+table and is written here. Reads and derivations go through Trino instead
+(see ``trino_client.py``); both speak to the same REST catalog + MinIO.
+"""
+
+from __future__ import annotations
+
+import re
+from functools import lru_cache
+
+import pyarrow as pa
+from pyiceberg.catalog import Catalog, load_catalog
+
+from memcove.core.config import get_settings
+from memcove.core.errors import ObjectExistsError, ObjectNotFoundError, SchemaMismatchError
+
+WriteMode = str  # "create" | "replace" | "append"
+
+
+def _norm_type(t: pa.DataType) -> str:
+    """Canonical type string for schema comparison.
+
+    Iceberg forces several arrow types onto a canonical storage form, and
+    ``Schema.as_arrow()`` reads them back in that form. Comparing the raw
+    ``str(type)`` of an incoming write against the read-back schema would then
+    false-reject a genuinely same-shape replace/append. Fold every conversion
+    Iceberg makes so both sides converge:
+
+      * ``large_string`` / ``large_binary`` / ``large_list`` <- string/binary/list
+        (an arrow encoding detail Iceberg picks on read-back).
+      * ``timestamp[ns|ms|s]`` -> ``timestamp[us]`` (Iceberg timestamps are microsecond).
+      * ``int8`` / ``int16`` -> ``int32`` (Iceberg has no integer narrower than 32-bit).
+
+    Applied to both sides, so a real retype (e.g. ``int64`` -> ``string``) still fails.
+    Regex-based so the folding reaches into nested ``list``/``struct`` types too.
+    """
+    s = str(t).replace("large_", "")
+    s = re.sub(r"timestamp\[(ns|ms|s)", "timestamp[us", s)
+    s = re.sub(r"\bint(8|16)\b", "int32", s)
+    return s
+
+
+def _assert_schema_compatible(
+    existing: pa.Schema, incoming: pa.Schema, *, op: str, label: str
+) -> None:
+    """Reject a replace/append whose schema differs from the existing object.
+
+    Compatibility is deterministic: the set of column names must match and each
+    column's Arrow type must be equal (nullability and large-vs-not encoding are
+    ignored — Iceberg matches by name). A changed shape is rejected rather than
+    silently evolved, so downstream SQL never breaks. To change shape, ``forget()``
+    then ``create()``.
+    """
+    existing_map = {f.name: _norm_type(f.type) for f in existing}
+    incoming_map = {f.name: _norm_type(f.type) for f in incoming}
+    if existing_map == incoming_map:
+        return
+    added = sorted(set(incoming_map) - set(existing_map))
+    removed = sorted(set(existing_map) - set(incoming_map))
+    retyped = sorted(
+        f"{n} ({existing_map[n]} -> {incoming_map[n]})"
+        for n in set(existing_map) & set(incoming_map)
+        if existing_map[n] != incoming_map[n]
+    )
+    parts = []
+    if added:
+        parts.append(f"added={added}")
+    if removed:
+        parts.append(f"removed={removed}")
+    if retyped:
+        parts.append(f"retyped={retyped}")
+    raise SchemaMismatchError(
+        f"{op} of '{label}' is schema-incompatible with the existing object "
+        f"({'; '.join(parts)}); forget() then create() to change an object's shape"
+    )
+
+
+@lru_cache
+def get_catalog() -> Catalog:
+    s = get_settings()
+    return load_catalog(
+        s.iceberg_catalog_name,
+        **{
+            "type": "rest",
+            "uri": s.iceberg_rest_uri,
+            "warehouse": s.iceberg_warehouse,
+            "s3.endpoint": s.s3_endpoint,
+            "s3.access-key-id": s.s3_access_key,
+            "s3.secret-access-key": s.s3_secret_key,
+            "s3.region": s.s3_region,
+            "s3.path-style-access": str(s.s3_path_style).lower(),
+        },
+    )
+
+
+def ensure_namespace(namespace: str) -> None:
+    get_catalog().create_namespace_if_not_exists((namespace,))
+
+
+def table_exists(namespace: str, label: str) -> bool:
+    return get_catalog().table_exists(f"{namespace}.{label}")
+
+
+def list_labels(namespace: str) -> list[str]:
+    try:
+        idents = get_catalog().list_tables((namespace,))
+    except Exception:  # noqa: BLE001 - namespace may not exist yet
+        return []
+    return [ident[-1] for ident in idents]
+
+
+def list_namespaces() -> list[str]:
+    """All tenant namespaces known to the catalog (single-component names)."""
+    return [ns[0] for ns in get_catalog().list_namespaces() if ns]
+
+
+def drop_table(namespace: str, label: str) -> None:
+    cat = get_catalog()
+    ident = f"{namespace}.{label}"
+    if not cat.table_exists(ident):
+        raise ObjectNotFoundError(f"object '{label}' does not exist")
+    cat.drop_table(ident)
+
+
+def write_arrow(
+    namespace: str, label: str, table: pa.Table, mode: WriteMode = "create"
+) -> int:
+    """Create/replace/append an Iceberg table from an Arrow table.
+
+    Returns the number of rows written in this call.
+    """
+    cat = get_catalog()
+    ensure_namespace(namespace)
+    ident = f"{namespace}.{label}"
+    exists = cat.table_exists(ident)
+
+    if mode == "create":
+        if exists:
+            raise ObjectExistsError(
+                f"object '{label}' already exists (use mode=replace or mode=append)"
+            )
+        iceberg_table = cat.create_table(ident, schema=table.schema)
+        iceberg_table.append(table)
+    elif mode == "replace":
+        if exists:
+            # Atomic full-table replace: overwrite() commits delete+append in a
+            # single catalog transaction, so a concurrent reader sees the old rows
+            # or the new rows, never a missing table (the old drop-then-create had a
+            # window where a crash lost the data and a reader saw the table vanish).
+            iceberg_table = cat.load_table(ident)
+            _assert_schema_compatible(
+                iceberg_table.schema().as_arrow(), table.schema, op="replace", label=label
+            )
+            iceberg_table.overwrite(table)
+        else:
+            iceberg_table = cat.create_table(ident, schema=table.schema)
+            iceberg_table.append(table)
+    elif mode == "append":
+        if exists:
+            iceberg_table = cat.load_table(ident)
+            _assert_schema_compatible(
+                iceberg_table.schema().as_arrow(), table.schema, op="append", label=label
+            )
+            iceberg_table.append(table)
+        else:
+            iceberg_table = cat.create_table(ident, schema=table.schema)
+            iceberg_table.append(table)
+    else:
+        raise ValueError(f"unknown write mode: {mode!r}")
+
+    return table.num_rows
+
+
+def load_schema(namespace: str, label: str) -> list[tuple[str, str]]:
+    """Return [(column_name, iceberg_type), ...] for an object."""
+    cat = get_catalog()
+    ident = f"{namespace}.{label}"
+    if not cat.table_exists(ident):
+        raise ObjectNotFoundError(f"object '{label}' does not exist")
+    tbl = cat.load_table(ident)
+    return [(f.name, str(f.field_type)) for f in tbl.schema().fields]

memcove/core/config.py

@@ -0,0 +1,121 @@
+"""Runtime configuration, loaded from environment (see .env.example)."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_prefix="MEMCOVE_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    # MCP server
+    host: str = "0.0.0.0"
+    port: int = 8090
+
+    # Iceberg REST catalog
+    iceberg_rest_uri: str = "http://localhost:8181"
+    iceberg_warehouse: str = "s3://warehouse/"
+    iceberg_catalog_name: str = "memcove"
+
+    # Object store
+    s3_endpoint: str = "http://localhost:9000"
+    s3_region: str = "us-east-1"
+    s3_access_key: str = "minio"
+    s3_secret_key: str = "minio12345"
+    s3_path_style: bool = True
+    warehouse_bucket: str = "warehouse"
+    staging_bucket: str = "memcove-staging"
+    artifacts_bucket: str = "memcove-artifacts"
+
+    # Trino
+    trino_host: str = "localhost"
+    trino_port: int = 8080
+    trino_user: str = "memcove"  # service principal; connect identity when not impersonating
+    trino_catalog: str = "iceberg"
+    trino_http_scheme: str = "http"  # "https" for TLS-fronted Trino in real deployments
+    # When true, each request connects to Trino AS the caller's tenant so the operator's
+    # own Trino access control applies per tenant (defense-in-depth beneath the SQL guard).
+    # Requires the service principal to hold impersonation rights + a configured grant
+    # backend. Off by default so local/dev works with a single identity.
+    trino_impersonation: bool = False
+    # Session properties applied to every data connection (generic passthrough so the
+    # operator sets whatever their Trino version supports — resource caps, etc.), e.g.
+    # {"query_max_run_time":"60s","query_max_scan_physical_bytes":"10GB"}.
+    trino_session_properties: dict[str, str] = {}
+
+    # Arrow Flight streaming data plane (M3)
+    flight_host: str = "0.0.0.0"  # bind address
+    flight_port: int = 8815
+    flight_advertise_uri: str = "grpc://localhost:8815"  # what clients are told to dial
+    # HMAC secret for signing Flight tickets/descriptors so a client cannot forge one
+    # to read/write another tenant. MUST be overridden in any real deployment.
+    flight_ticket_secret: str = "dev-insecure-change-me"
+    flight_ticket_ttl_seconds: int = 300  # signed tickets expire after this many seconds
+
+    # Postgres registry
+    pg_dsn: str = "postgresql://memcove:memcove@localhost:5433/memcove"
+    # Connection pool: the registry opens a fresh connection per op otherwise, and the
+    # reconciler + synchronous read-repair add per-op churn. min_size connections are
+    # kept warm; the pool grows to max_size under load. pool_timeout bounds how long a
+    # caller waits for a free connection before raising (a subclass of OperationalError,
+    # so a saturated/unreachable registry is still swallowed by the guarded-write path).
+    # Kept at 10s (not psycopg_pool's 30s default) so an unreachable registry fails
+    # closer to the old connect-per-call fast-fail; registry ops are milliseconds, so a
+    # >10s wait for a free connection only happens during a real outage, not under load.
+    pg_pool_min_size: int = 1
+    pg_pool_max_size: int = 10
+    pg_pool_timeout: float = 10.0
+
+    # Reconciler / read-repair (write-atomicity self-healing). The reconciler diffs the
+    # Iceberg catalog against the Postgres registry to backfill missing rows and drop
+    # dangling ones. Deletion is fail-safe: an empty/failed namespace listing deletes
+    # nothing, a row must be absent across this many consecutive sweeps before deletion,
+    # and a sweep that would delete more than the cap ratio of a namespace aborts + alerts.
+    reconcile_min_absent_sweeps: int = 2
+    reconcile_deletion_cap_ratio: float = 0.25
+    # The ratio cap only applies once a sweep would delete more than this many rows,
+    # so small namespaces can still clean up a single dangling row (1 of 2 rows is 50%
+    # but is not a mass deletion). The cap exists to stop a wipe, not routine cleanup.
+    reconcile_deletion_cap_min: int = 3
+
+    # Guardrails
+    preview_row_cap: int = 1000
+    inline_bytes_cap: int = 8 * 1024 * 1024
+    export_row_cap: int = 5_000_000
+    presign_ttl_seconds: int = 3600
+
+    # Tenancy. Default: trust a tenant header set by the auth proxy (dev/simple).
+    tenant_header: str = "x-memcove-tenant"
+    default_tenant: str = "default"
+
+    # Provisioning map (optional). When tenant_subject_header is set, the tenant is
+    # resolved by mapping the proxy-provided identity (subject, else a matching group)
+    # through tenant_map -> internal tenant id, instead of trusting a raw tenant value.
+    # This is the seam for "don't feed a raw OIDC sub straight through".
+    # Provisioning mode is fail-closed: when tenant_subject_header is set, an identity
+    # absent from tenant_map is rejected (never falls through to the raw tenant header).
+    tenant_subject_header: str = ""  # e.g. "x-auth-subject"; empty = direct tenant header
+    tenant_group_header: str = ""  # e.g. "x-auth-groups" (comma-separated)
+    tenant_map: dict[str, str] = {}  # subject/group -> internal tenant id
+
+    # Shared read-only reference plane (the gateway): schemas every tenant may SELECT
+    # from but none may write. These are NOT rewritten to the caller's namespace by the
+    # SQL guard; they resolve to themselves. Per-domain schemas contain blast radius.
+    shared_schemas: list[str] = ["ref_market"]
+
+    # Ingest allowlist: agent-supplied `s3_parquet` URIs must start with one of these
+    # prefixes. Empty list = agent s3_parquet ingest is DISABLED (fail closed) to avoid
+    # a confused-deputy read of any bucket the service credential can reach.
+    allowed_s3_ingest_prefixes: list[str] = []
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()

memcove/core/errors.py

@@ -0,0 +1,40 @@
+"""Typed errors surfaced to MCP callers as structured messages."""
+
+from __future__ import annotations
+
+
+class MemcoveError(Exception):
+    """Base class for all Memcove errors."""
+
+
+class SqlGuardError(MemcoveError):
+    """Raised when submitted SQL violates the safety policy."""
+
+
+class ObjectNotFoundError(MemcoveError):
+    """Raised when a referenced label does not exist for the tenant."""
+
+
+class ObjectExistsError(MemcoveError):
+    """Raised when creating an object whose label already exists."""
+
+
+class SchemaMismatchError(MemcoveError):
+    """Raised when incoming data's schema is incompatible with an existing object.
+
+    A ``replace`` or ``append`` must keep the object's shape; changing columns or
+    types is rejected so downstream SQL never breaks silently. To change shape,
+    ``forget()`` the object then ``create()`` it fresh.
+    """
+
+
+class IngestError(MemcoveError):
+    """Raised when an ingest source cannot be read or written."""
+
+
+class TenancyError(MemcoveError):
+    """Raised when a tenant cannot be resolved."""
+
+
+class TicketError(MemcoveError):
+    """Raised when a Flight ticket/descriptor fails signature or expiry checks."""

memcove/core/models.py

@@ -0,0 +1,92 @@
+"""Domain models for the Memcove control plane."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class SourceKind(str, Enum):
+    """How an object's data first entered Memcove."""
+
+    INLINE = "inline"
+    S3_PARQUET = "s3_parquet"
+    UPLOAD = "upload_handle"
+    STREAM = "stream"
+    DERIVED = "derived"
+    # A row the reconciler (or synchronous read-repair) backfilled for an Iceberg
+    # table that had no registry row — e.g. after a crash between the data write and
+    # the registry write. Marks that user metadata (tags/producing_sql/lineage) was
+    # lost and should be re-supplied by re-running the producing operation.
+    RECONCILED = "reconciled"
+
+
+class ColumnSchema(BaseModel):
+    name: str
+    type: str
+
+
+class Lineage(BaseModel):
+    """Provenance of a derived object."""
+
+    parents: list[str] = Field(default_factory=list)  # parent labels
+    producing_sql: str | None = None
+
+
+class MemoryObject(BaseModel):
+    """A labeled, namespaced reference to an Iceberg table.
+
+    Identity is ``<tenant>.<label>`` which maps to the Iceberg table
+    ``<catalog>.<tenant>.<label>``.
+    """
+
+    tenant: str
+    label: str
+    table_ident: str  # fully-qualified iceberg identifier, e.g. memcove.t_acme.events
+    source: SourceKind
+    source_ref: str | None = None  # e.g. the s3 uri or upload handle
+    schema_: list[ColumnSchema] = Field(default_factory=list, alias="schema")
+    row_count: int | None = None
+    size_bytes: int | None = None
+    tags: list[str] = Field(default_factory=list)
+    lineage: Lineage = Field(default_factory=Lineage)
+    created_at: datetime | None = None
+    updated_at: datetime | None = None
+    # True when the data write committed but the registry metadata write failed. The
+    # object is queryable, but tags/producing_sql/lineage may be missing until a
+    # reconciler sweep or a re-run repairs them. Built from values in hand, so the
+    # response does not depend on re-reading the registry that just failed.
+    metadata_pending: bool = False
+
+    model_config = {"populate_by_name": True}
+
+
+class PreviewResult(BaseModel):
+    """A capped tabular result handed back through the control plane."""
+
+    columns: list[str]
+    rows: list[list[Any]]
+    row_count: int  # number of rows in this preview
+    truncated: bool  # True if more rows exist beyond the cap
+
+
+class ArtifactRef(BaseModel):
+    """A materialized export living in object storage."""
+
+    uri: str  # s3:// location
+    presigned_url: str  # time-limited GET URL
+    format: str  # parquet | csv | json
+    row_count: int
+    size_bytes: int
+    expires_in_seconds: int
+
+
+class UploadTicket(BaseModel):
+    """A presigned slot for out-of-band parquet upload."""
+
+    upload_handle: str  # staging key the client passes back to ingest_object
+    presigned_url: str  # presigned PUT URL
+    expires_in_seconds: int

memcove/core/naming.py

@@ -0,0 +1,20 @@
+"""Validation for object labels."""
+
+from __future__ import annotations
+
+import re
+
+from memcove.core.errors import MemcoveError
+
+_LABEL_RE = re.compile(r"^[a-z][a-z0-9_]{0,127}$")
+
+
+def validate_label(label: str) -> str:
+    """Ensure a label is a safe, lowercase Iceberg table name."""
+    candidate = (label or "").strip().lower()
+    if not _LABEL_RE.match(candidate):
+        raise MemcoveError(
+            f"invalid label {label!r}: use lowercase letters, digits and underscores, "
+            "starting with a letter (max 128 chars)"
+        )
+    return candidate