lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/objectstore.py

@@ -0,0 +1,162 @@
+"""Content-addressed objects in Postgres (`objects`), scoped to one Domain (design 6.1)."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any
+
+from pydantic import JsonValue
+from sqlalchemy import Table, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import Session
+
+from lorebase_core.canonical import ObjectType, canonical_bytes, normalize, object_hash
+from lorebase_store.bulk import bulk_write, chunks
+from lorebase_store.models import metadata
+from lorebase_store.settings import WriteSettings
+
+OBJECTS: Table = metadata.tables["objects"]
+
+OBJECT_TYPE_CODES: Mapping[ObjectType, int] = MappingProxyType(
+    {
+        ObjectType.SCHEMA_DOMAIN: 1,
+        ObjectType.SCHEMA_KIND: 2,
+        ObjectType.SCHEMA_RELATION: 3,
+        ObjectType.SCHEMA_TRAIT: 4,
+        ObjectType.SCHEMA_OPERATION: 5,
+        ObjectType.SCHEMA_DERIVATION: 6,
+        ObjectType.SCHEMA_SOURCE: 7,
+        ObjectType.RECORD: 8,
+        ObjectType.LINK: 9,
+    }
+)
+"""`objects.object_type` codes. Stored data depends on them: never renumber."""
+
+OBJECT_TYPES_BY_CODE: Mapping[int, ObjectType] = MappingProxyType(
+    {code: object_type for object_type, code in OBJECT_TYPE_CODES.items()}
+)
+
+
+@dataclass(frozen=True, slots=True)
+class EncodedObject:
+    """An object ready to store: hash, type, normalized body and canonical size in bytes."""
+
+    hash: bytes
+    object_type: ObjectType
+    body: dict[str, Any]
+    size: int
+
+
+def encode_object(object_type: ObjectType | str, content: Mapping[str, Any]) -> EncodedObject:
+    """Normalize `content` and compute its object hash (design 6.1)."""
+    kind = ObjectType(object_type)
+    body = normalize(content)
+    if not isinstance(body, dict):
+        raise TypeError("object content must be a JSON object")
+    return EncodedObject(
+        hash=object_hash(kind, body),
+        object_type=kind,
+        body=body,
+        size=len(canonical_bytes(body)),
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class StoredBody:
+    """Type and body of a stored object."""
+
+    object_type: ObjectType
+    body: dict[str, Any]
+
+
+class PgObjectStore:
+    """Async batched reads and bulk writes of one Domain's objects."""
+
+    def __init__(
+        self, session: AsyncSession, domain_id: str, *, writes: WriteSettings, batch: int = 1000
+    ) -> None:
+        self._session = session
+        self._domain = domain_id
+        self._writes = writes
+        self._batch = batch
+
+    async def get_many(self, hashes: Iterable[bytes]) -> dict[bytes, StoredBody]:
+        """Stored objects among `hashes`; absent hashes are left out."""
+        found: dict[bytes, StoredBody] = {}
+        for chunk in chunks(list(dict.fromkeys(hashes)), self._batch):
+            rows = await self._session.execute(
+                select(OBJECTS.c.hash, OBJECTS.c.object_type, OBJECTS.c.body).where(
+                    OBJECTS.c.domain_id == self._domain, OBJECTS.c.hash.in_(chunk)
+                )
+            )
+            for digest, code, body in rows:
+                found[digest] = StoredBody(OBJECT_TYPES_BY_CODE[code], body)
+        return found
+
+    async def put_many(self, objects: Iterable[EncodedObject]) -> None:
+        """Persist objects; objects already stored are left unchanged."""
+        unique = {item.hash: item for item in objects}
+        rows = [
+            {
+                "domain_id": self._domain,
+                "hash": item.hash,
+                "object_type": OBJECT_TYPE_CODES[item.object_type],
+                "body": item.body,
+                "size": item.size,
+            }
+            for item in unique.values()
+        ]
+        await bulk_write(self._session, OBJECTS, rows, writes=self._writes)
+
+
+class SessionObjectStore:
+    """Synchronous core `ObjectStore` of one Domain over a synchronous `Session`.
+
+    Like `PgNodeStore` it runs inside `AsyncSession.run_sync`: reads query `objects` (and are
+    cached for the life of the store), writes are buffered and handed to
+    `PgObjectStore.put_many` with `take_pending`.
+    """
+
+    def __init__(self, session: Session, domain_id: str, *, batch: int = 1000) -> None:
+        self._session = session
+        self._domain = domain_id
+        self._batch = batch
+        self._known: dict[bytes, dict[str, Any]] = {}
+        self._pending: dict[bytes, EncodedObject] = {}
+
+    def get(self, object_hash: bytes) -> JsonValue | None:
+        pending = self._pending.get(object_hash)
+        if pending is not None:
+            return pending.body
+        if object_hash not in self._known:
+            self.prefetch([object_hash])
+        return self._known.get(object_hash)
+
+    def put(self, object_type: str, payload: JsonValue) -> bytes:
+        if not isinstance(payload, dict):
+            raise TypeError("object content must be a JSON object")
+        encoded = encode_object(object_type, payload)
+        self._pending.setdefault(encoded.hash, encoded)
+        return encoded.hash
+
+    def prefetch(self, hashes: Iterable[bytes]) -> None:
+        """Load the given objects in batched queries; known ones are skipped."""
+        missing = [
+            h for h in dict.fromkeys(hashes) if h not in self._known and h not in self._pending
+        ]
+        for chunk in chunks(missing, self._batch):
+            rows = self._session.execute(
+                select(OBJECTS.c.hash, OBJECTS.c.body).where(
+                    OBJECTS.c.domain_id == self._domain, OBJECTS.c.hash.in_(chunk)
+                )
+            )
+            for digest, body in rows:
+                self._known[digest] = body
+
+    def take_pending(self) -> list[EncodedObject]:
+        """Return and forget the buffered writes."""
+        pending, self._pending = self._pending, {}
+        self._known.update({digest: item.body for digest, item in pending.items()})
+        return list(pending.values())

lorebase_store/outbox_events.py

@@ -0,0 +1,166 @@
+"""Event types of the outbox and the sensitive values their payloads carry (design 10.2, 10.5).
+
+**Types.** A commit writes `revision.committed`; an Operation that declares side effects writes
+one `operation.<name>` event per name, in its commit transaction. Webhooks subscribe to exact
+types or to `operation.*`, every Operation event.
+
+**Sensitive values.** A payload never holds a value that needs a clearance as such: the writer
+wraps it as `{"$sensitive": {"tags": [...], "value": ...}}`, the value of a vault Attribute
+being its vault reference (`{"$vault": token}`), never the personal data. `disclose` renders a
+payload for one subscriber when it is sent: a wrapped value whose tags the subscriber is
+cleared for is shown (a vault reference opened; an erased one shows `erased`), any other is
+replaced by `masked`, and so is a vault reference found outside a wrapper. A wrapper without
+tags is never cleared, and writers wrap with every tag any value they did not produce whose
+JSON holds `$sensitive` or `$vault` keys (`carries_markers`), so a value written by a caller
+cannot pose as a wrapper or a reference. Stored payloads and deliveries therefore keep no vault
+value, and an erasure reaches deliveries not sent yet. Values of sensitive Attributes outside
+the vault are kept as committed, like the Revision objects that hold them.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Collection, Iterable, Iterator, Mapping
+from typing import Any
+
+from lorebase_core.model import VAULT_KEY, Sensitivity, vault_token
+from lorebase_core.model.operation import SIDE_EFFECT_MAX_LENGTH
+from lorebase_store.commit.pipeline import COMMITTED_EVENT
+from lorebase_store.vault import VaultState, VaultValue
+
+OPERATION_EVENT_PREFIX = "operation."
+OPERATION_EVENTS = "operation.*"
+"""Subscribes to the events of every Operation side effect."""
+EVENT_TYPES: tuple[str, ...] = (COMMITTED_EVENT, OPERATION_EVENTS)
+"""Event types a webhook subscribes to when it names none."""
+SENSITIVE_KEY = "$sensitive"
+
+_OPERATION_EVENT = re.compile(r"operation\.[a-z][a-z0-9]*(?:_[a-z0-9]+)*")
+
+
+def operation_event_type(side_effect: str) -> str:
+    """The outbox event type of an Operation side effect."""
+    return f"{OPERATION_EVENT_PREFIX}{side_effect}"
+
+
+def event_type_problem(name: str) -> str | None:
+    """Why `name` is not an event type a webhook may subscribe to; None when it is."""
+    if name in EVENT_TYPES:
+        return None
+    if (
+        _OPERATION_EVENT.fullmatch(name)
+        and len(name) - len(OPERATION_EVENT_PREFIX) <= SIDE_EFFECT_MAX_LENGTH
+    ):
+        return None
+    return (
+        f"unknown event type {name!r}: expected {COMMITTED_EVENT}, {OPERATION_EVENTS} or "
+        "operation.<side effect>"
+    )
+
+
+def subscribed(event_type: str, subscriptions: Collection[str]) -> bool:
+    """Whether a subscription to `subscriptions` receives events of `event_type`."""
+    if event_type in subscriptions:
+        return True
+    return event_type.startswith(OPERATION_EVENT_PREFIX) and OPERATION_EVENTS in subscriptions
+
+
+def sensitive_value(tags: Iterable[Sensitivity], value: Any) -> dict[str, Any]:
+    """`value` wrapped with the tags that clear it (see module doc)."""
+    return {SENSITIVE_KEY: {"tags": sorted(tag.value for tag in tags), "value": value}}
+
+
+def carries_markers(value: Any) -> bool:
+    """Whether `value` holds, at any depth, an object with a `$sensitive` or `$vault` key."""
+    if isinstance(value, dict):
+        return (
+            SENSITIVE_KEY in value
+            or VAULT_KEY in value
+            or any(map(carries_markers, value.values()))
+        )
+    if isinstance(value, list):
+        return any(map(carries_markers, value))
+    return False
+
+
+def _wrapped(node: Any) -> tuple[frozenset[str], Any] | None:
+    if isinstance(node, dict) and len(node) == 1 and SENSITIVE_KEY in node:
+        body = node[SENSITIVE_KEY]
+        if isinstance(body, dict) and isinstance(body.get("tags"), list):
+            return frozenset(str(t) for t in body["tags"]), body.get("value")
+    return None
+
+
+def _cleared(tags: frozenset[str], clearances: Collection[str]) -> bool:
+    return bool(tags) and all(tag in clearances for tag in tags)
+
+
+def cleared_tokens(payload: Any, clearances: Collection[str]) -> set[str]:
+    """Vault tokens that `disclose` opens for a subscriber with `clearances`."""
+    return set(_tokens(payload, clearances, inside=False))
+
+
+def _tokens(node: Any, clearances: Collection[str], *, inside: bool) -> Iterator[str]:
+    wrapped = _wrapped(node)
+    if wrapped is not None:
+        tags, value = wrapped
+        if _cleared(tags, clearances):
+            yield from _tokens(value, clearances, inside=True)
+        return
+    if isinstance(node, dict):
+        token = vault_token(node) if VAULT_KEY in node else None
+        if token is not None:
+            if inside:
+                yield token
+            return
+        for item in node.values():
+            yield from _tokens(item, clearances, inside=inside)
+    elif isinstance(node, list):
+        for item in node:
+            yield from _tokens(item, clearances, inside=inside)
+
+
+def disclose(
+    payload: Any,
+    clearances: Collection[str],
+    opened: Mapping[str, VaultValue],
+    *,
+    masked: str,
+    erased: str,
+) -> Any:
+    """`payload` as a subscriber with `clearances` receives it (see module doc). `opened`
+    holds the vault values of `cleared_tokens`."""
+    return _disclose(payload, frozenset(clearances), opened, masked, erased, inside=False)
+
+
+def _disclose(
+    node: Any,
+    clearances: frozenset[str],
+    opened: Mapping[str, VaultValue],
+    masked: str,
+    erased: str,
+    *,
+    inside: bool,
+) -> Any:
+    wrapped = _wrapped(node)
+    if wrapped is not None:
+        tags, value = wrapped
+        if not _cleared(tags, clearances):
+            return None if value is None else masked
+        return _disclose(value, clearances, opened, masked, erased, inside=True)
+    if isinstance(node, dict):
+        if VAULT_KEY in node:
+            token = vault_token(node)
+            if token is None or not inside:
+                return masked
+            found = opened.get(token)
+            if found is None or found.state is VaultState.UNAVAILABLE:
+                return masked
+            return erased if found.state is VaultState.ERASED else found.value
+        return {
+            key: _disclose(item, clearances, opened, masked, erased, inside=inside)
+            for key, item in node.items()
+        }
+    if isinstance(node, list):
+        return [_disclose(item, clearances, opened, masked, erased, inside=inside) for item in node]
+    return node

lorebase_store/profiling/__init__.py

@@ -0,0 +1,58 @@
+"""Profilers of structured sources for schema inference (``lorebase_core.infer``).
+
+They read CSV and TSV files (``delimited``), Postgres tables (``postgres``), MySQL, MariaDB
+and SQL Server tables (``sql_databases``), rows read as records (``records``: JSON, JSON Lines,
+Parquet and API Sources, whose readers live in Scribe) and uploaded JSON, JSON Lines and
+Parquet files (``record_files``) into ``TableProfile`` objects.
+Limits come from the ``store.profiling`` settings.
+"""
+
+from lorebase_store.profiling.delimited import (
+    CsvProfileError,
+    profile_bytes,
+    profile_file,
+    profile_text,
+    table_name,
+)
+from lorebase_store.profiling.postgres import PostgresProfileError, profile_postgres, type_spec
+from lorebase_store.profiling.record_files import (
+    ParquetUnavailableError,
+    profile_json_bytes,
+    profile_jsonl_bytes,
+    profile_parquet_bytes,
+)
+from lorebase_store.profiling.records import (
+    RecordProfileError,
+    RecordProfiler,
+    RowSampler,
+    profile_records,
+)
+from lorebase_store.profiling.sql_databases import (
+    SqlDialect,
+    SqlDriverError,
+    SqlProfileError,
+    profile_sql_database,
+)
+
+__all__ = [
+    "CsvProfileError",
+    "ParquetUnavailableError",
+    "PostgresProfileError",
+    "RecordProfileError",
+    "RecordProfiler",
+    "RowSampler",
+    "SqlDialect",
+    "SqlDriverError",
+    "SqlProfileError",
+    "profile_bytes",
+    "profile_file",
+    "profile_json_bytes",
+    "profile_jsonl_bytes",
+    "profile_parquet_bytes",
+    "profile_postgres",
+    "profile_records",
+    "profile_sql_database",
+    "profile_text",
+    "table_name",
+    "type_spec",
+]

lorebase_store/profiling/delimited.py

@@ -0,0 +1,109 @@
+"""CSV and TSV profiler: delimiter sniffing, encoding fallback and row sampling (stdlib ``csv``).
+
+The first row is the header. Every row is counted; ``sample_rows`` of them are kept, either the
+first ones (``head``) or a uniform seeded sample (``reservoir``, rows kept in file order). Short
+rows are padded with nulls and extra cells are dropped. All values stay text: inference
+detects their types.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+from collections.abc import Iterable, Iterator
+from itertools import chain
+from pathlib import Path
+from typing import TextIO
+
+from lorebase_core.infer import ColumnProfile, TableProfile
+from lorebase_store.profiling.records import RowSampler
+from lorebase_store.settings import CsvProfilingSettings
+
+TSV_SUFFIXES = (".tsv", ".tab")
+DEFAULT_DELIMITER = ","
+
+
+class CsvProfileError(ValueError):
+    """The file cannot be read as delimited text."""
+
+
+def _delimiter(sample: str, name: str, settings: CsvProfilingSettings) -> str:
+    if name.lower().endswith(TSV_SUFFIXES):
+        return "\t"
+    try:
+        return csv.Sniffer().sniff(sample, delimiters=settings.delimiters).delimiter
+    except csv.Error:
+        return DEFAULT_DELIMITER
+
+
+def _sample(
+    rows: Iterator[list[str]], settings: CsvProfilingSettings
+) -> tuple[list[list[str]], int]:
+    """Kept rows (in file order) and the number of rows read."""
+    sampler: RowSampler[list[str]] = RowSampler(
+        settings.sample_rows, settings.sampling, settings.seed
+    )
+    for row in rows:
+        sampler.add(row)
+    return sampler.kept(), sampler.total
+
+
+def _header(cells: list[str]) -> list[str]:
+    return [cell.strip() or f"column_{index + 1}" for index, cell in enumerate(cells)]
+
+
+def profile_text(stream: TextIO, name: str, settings: CsvProfilingSettings) -> TableProfile:
+    """Profile delimited text; ``name`` is the table name (a file name keeps its suffix rules)."""
+    head = stream.read(settings.sniff_bytes)
+    head += stream.readline()
+    delimiter = _delimiter(head, name, settings)
+    lines: Iterable[str] = chain(io.StringIO(head), stream)
+    reader = csv.reader(lines, delimiter=delimiter)
+    try:
+        first = next(reader, None)
+        if first is None:
+            raise CsvProfileError(f"{name}: the file is empty")
+        header = _header(first)
+        if len(header) > settings.max_columns:
+            raise CsvProfileError(
+                f"{name}: {len(header)} columns, more than the {settings.max_columns} allowed"
+            )
+        rows = (row for row in reader if any(cell.strip() for cell in row))
+        kept, total = _sample(rows, settings)
+    except csv.Error as error:
+        raise CsvProfileError(f"{name}: not valid delimited text ({error})") from None
+    columns = tuple(
+        ColumnProfile(
+            column,
+            [row[index] if index < len(row) else None for row in kept],
+        )
+        for index, column in enumerate(header)
+    )
+    return TableProfile(table_name(name), columns, row_count=total)
+
+
+def table_name(file_name: str) -> str:
+    """Table name of a file: the base name without directories and suffix."""
+    return Path(file_name.replace("\\", "/")).stem or file_name
+
+
+def profile_bytes(data: bytes, name: str, settings: CsvProfilingSettings) -> TableProfile:
+    """Profile an uploaded file, trying ``settings.encodings`` in order."""
+    for encoding in settings.encodings:
+        try:
+            text = data.decode(encoding)
+        except (UnicodeDecodeError, LookupError):
+            continue
+        return profile_text(io.StringIO(text, newline=""), name, settings)
+    raise CsvProfileError(f"{name}: not decodable as {', '.join(settings.encodings)}")
+
+
+def profile_file(path: Path, settings: CsvProfilingSettings) -> TableProfile:
+    """Profile a file on disk without loading it whole, trying ``settings.encodings``."""
+    for encoding in settings.encodings:
+        try:
+            with path.open(encoding=encoding, newline="") as stream:
+                return profile_text(stream, path.name, settings)
+        except (UnicodeDecodeError, LookupError):
+            continue
+    raise CsvProfileError(f"{path.name}: not decodable as {', '.join(settings.encodings)}")