lorebase-core 0.2.3__py3-none-any.whl

lorebase_core/model/schema.py

@@ -0,0 +1,219 @@
+"""The ``Schema`` aggregate: every schema element of one Revision, with lookup helpers.
+
+Elements are keyed by id. Api name lookups return ``None`` for unknown names; when a schema
+is invalid (duplicate api names) they return one of the candidates, and ``validate_schema``
+reports the duplicates.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Mapping
+from functools import cached_property
+from typing import Any, Self
+
+from pydantic import Field, model_validator
+
+from lorebase_core.model.attribute import Attribute
+from lorebase_core.model.base import Model, SchemaElement
+from lorebase_core.model.derivation import Derivation
+from lorebase_core.model.domain import Domain
+from lorebase_core.model.ids import (
+    AttributeId,
+    DerivationId,
+    KindId,
+    OperationId,
+    RelationId,
+    SourceId,
+    TraitId,
+)
+from lorebase_core.model.kind import Kind
+from lorebase_core.model.operation import Operation
+from lorebase_core.model.paths import PathRelation, path_relations
+from lorebase_core.model.record import parse_ref
+from lorebase_core.model.relation import LinkDirection, Relation
+from lorebase_core.model.source import Source
+from lorebase_core.model.trait import Trait
+
+#: Names of the lazily built lookup indexes (cached properties).
+_INDEXES = (
+    "_kinds_by_name",
+    "_traits_by_name",
+    "_relations_by_name",
+    "_operations_by_name",
+    "_derivations_by_name",
+    "_sources_by_name",
+    "_paths",
+    "_paths_by_name",
+)
+
+
+class Schema(Model):
+    domain: Domain
+    kinds: dict[KindId, Kind] = Field(default_factory=dict)
+    relations: dict[RelationId, Relation] = Field(default_factory=dict)
+    traits: dict[TraitId, Trait] = Field(default_factory=dict)
+    operations: dict[OperationId, Operation] = Field(default_factory=dict)
+    derivations: dict[DerivationId, Derivation] = Field(default_factory=dict)
+    sources: dict[SourceId, Source] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def _check_keys(self) -> Self:
+        collections: tuple[tuple[str, dict[str, SchemaElement]], ...] = (
+            ("kinds", dict(self.kinds)),
+            ("relations", dict(self.relations)),
+            ("traits", dict(self.traits)),
+            ("operations", dict(self.operations)),
+            ("derivations", dict(self.derivations)),
+            ("sources", dict(self.sources)),
+        )
+        for name, elements in collections:
+            for key, element in elements.items():
+                if key != element.id:
+                    raise ValueError(f"{name}: key {key} does not match element id {element.id}")
+        return self
+
+    def model_copy(self, *, update: Mapping[str, Any] | None = None, deep: bool = False) -> Self:
+        """Copy the Schema; lookup indexes are rebuilt for the copy."""
+        copied = super().model_copy(update=update, deep=deep)
+        for name in _INDEXES:
+            copied.__dict__.pop(name, None)
+        return copied
+
+    # Iteration -----------------------------------------------------------------
+
+    def elements(self) -> Iterator[SchemaElement]:
+        """Every schema element except the Domain."""
+        yield from self.kinds.values()
+        yield from self.relations.values()
+        yield from self.traits.values()
+        yield from self.operations.values()
+        yield from self.derivations.values()
+        yield from self.sources.values()
+
+    # Api name lookups ------------------------------------------------------------
+
+    @cached_property
+    def _kinds_by_name(self) -> dict[str, Kind]:
+        return {kind.api_name: kind for kind in self.kinds.values()}
+
+    @cached_property
+    def _traits_by_name(self) -> dict[str, Trait]:
+        return {trait.api_name: trait for trait in self.traits.values()}
+
+    @cached_property
+    def _relations_by_name(self) -> dict[str, tuple[Relation, LinkDirection]]:
+        index: dict[str, tuple[Relation, LinkDirection]] = {}
+        for relation in self.relations.values():
+            index[relation.inverse_api_name] = (relation, LinkDirection.INVERSE)
+            index[relation.api_name] = (relation, LinkDirection.FORWARD)
+        return index
+
+    @cached_property
+    def _operations_by_name(self) -> dict[str, Operation]:
+        return {operation.api_name: operation for operation in self.operations.values()}
+
+    @cached_property
+    def _derivations_by_name(self) -> dict[str, Derivation]:
+        return {derivation.api_name: derivation for derivation in self.derivations.values()}
+
+    @cached_property
+    def _sources_by_name(self) -> dict[str, Source]:
+        return {source.api_name: source for source in self.sources.values()}
+
+    @cached_property
+    def _paths(self) -> dict[str, PathRelation]:
+        return path_relations(self.derivations.values(), self.relations)
+
+    @cached_property
+    def _paths_by_name(self) -> dict[str, tuple[PathRelation, LinkDirection]]:
+        index: dict[str, tuple[PathRelation, LinkDirection]] = {}
+        for path in self._paths.values():
+            inverse = path.relation.inverse_api_name
+            if inverse:
+                index[inverse] = (path, LinkDirection.INVERSE)
+            index[path.relation.api_name] = (path, LinkDirection.FORWARD)
+        return index
+
+    def kind_by_api_name(self, api_name: str) -> Kind | None:
+        return self._kinds_by_name.get(api_name)
+
+    def trait_by_api_name(self, api_name: str) -> Trait | None:
+        return self._traits_by_name.get(api_name)
+
+    def relation_by_api_name(self, api_name: str) -> tuple[Relation, LinkDirection] | None:
+        """Resolve a forward or inverse Relation name to the Relation and the direction."""
+        return self._relations_by_name.get(api_name)
+
+    def path_by_api_name(self, api_name: str) -> tuple[PathRelation, LinkDirection] | None:
+        """Resolve the forward or inverse name of a path Derivation (a virtual Relation)."""
+        return self._paths_by_name.get(api_name)
+
+    def path_relations(self) -> dict[str, PathRelation]:
+        """The valid path Derivations as virtual Relations, by Derivation id."""
+        return self._paths
+
+    def link_by_api_name(self, api_name: str) -> tuple[Relation, LinkDirection] | None:
+        """Resolve a Relation name or a path Derivation name (its virtual Relation, whose id
+        is the Derivation id) to the Relation and the direction."""
+        found = self._relations_by_name.get(api_name)
+        if found is not None:
+            return found
+        path = self._paths_by_name.get(api_name)
+        return None if path is None else (path[0].relation, path[1])
+
+    def link_relation(self, relation_id: str) -> Relation | None:
+        """A Relation by id, or the virtual Relation of a valid path Derivation."""
+        relation = self.relations.get(relation_id)
+        if relation is not None:
+            return relation
+        path = self._paths.get(relation_id)
+        return None if path is None else path.relation
+
+    def operation_by_api_name(self, api_name: str) -> Operation | None:
+        return self._operations_by_name.get(api_name)
+
+    def derivation_by_api_name(self, api_name: str) -> Derivation | None:
+        return self._derivations_by_name.get(api_name)
+
+    def source_by_api_name(self, api_name: str) -> Source | None:
+        return self._sources_by_name.get(api_name)
+
+    def attribute_by_api_name(self, kind_id: str, api_name: str) -> Attribute | None:
+        kind = self.kinds.get(kind_id)
+        return kind.attribute_by_api_name(api_name) if kind is not None else None
+
+    # Structural lookups ----------------------------------------------------------
+
+    def attribute(self, kind_id: str, attribute_id: str) -> Attribute | None:
+        kind = self.kinds.get(kind_id)
+        return kind.attributes.get(attribute_id) if kind is not None else None
+
+    def relations_of(self, kind_id: str) -> list[Relation]:
+        """Relations with ``kind_id`` at either end."""
+        return [r for r in self.relations.values() if kind_id in (r.from_kind, r.to_kind)]
+
+    def kinds_implementing(self, trait_id: str) -> list[Kind]:
+        return [k for k in self.kinds.values() if any(t.trait_id == trait_id for t in k.traits)]
+
+    def resolve_ref(self, ref: str) -> tuple[KindId, str]:
+        """Resolve ``Supplier/S-1001`` to ``(kind_id, key)``.
+
+        Raises ``KeyError`` for unknown Kinds.
+        """
+        kind_api_name, key = parse_ref(ref)
+        kind = self.kind_by_api_name(kind_api_name)
+        if kind is None:
+            raise KeyError(f"unknown Kind {kind_api_name!r}")
+        return kind.id, key
+
+    def attribute_owner(self, attribute_id: AttributeId) -> Kind | Relation | Trait | None:
+        """The Kind, Relation or Trait declaring ``attribute_id``."""
+        owners: list[Kind | Relation | Trait] = [
+            *self.kinds.values(),
+            *self.relations.values(),
+            *self.traits.values(),
+        ]
+        for owner in owners:
+            if attribute_id in owner.attributes:
+                return owner
+        return None

lorebase_core/model/source.py

@@ -0,0 +1,204 @@
+"""Source (design 5.9): external backing data mapped into a Kind. Model only."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import ClassVar, Self
+
+from pydantic import Field, JsonValue, model_validator
+
+from lorebase_core.model.base import ElementStatus, Model, SchemaElement
+from lorebase_core.model.ids import AttributeId, KindId, RelationId, SourceId
+
+#: Prefix of secret references; connections are never literals (design 5.9, 13.4).
+SECRET_REFERENCE_PREFIX = "secret://"
+
+
+class SourceType(StrEnum):
+    POSTGRES_TABLE = "postgres_table"
+    POSTGRES_QUERY = "postgres_query"
+    MYSQL_TABLE = "mysql_table"
+    MYSQL_QUERY = "mysql_query"
+    MSSQL_TABLE = "mssql_table"
+    MSSQL_QUERY = "mssql_query"
+    CSV = "csv"
+    JSON = "json"
+    JSONL = "jsonl"
+    PARQUET = "parquet"
+    HTTP_JSON = "http_json"
+
+
+class SyncMode(StrEnum):
+    SNAPSHOT = "snapshot"
+    INCREMENTAL = "incremental"
+
+
+class MergePolicy(StrEnum):
+    AUTO_WHEN_CLEAN = "auto_when_clean"
+    PROPOSAL = "proposal"
+    AUTO_WITH_AUTHORITY = "auto_with_authority"
+
+
+class TransformName(StrEnum):
+    CAST = "cast"
+    TRIM = "trim"
+    LOWER = "lower"
+    UPPER = "upper"
+    PARSE_DATE = "parse_date"
+    CONCAT = "concat"
+    CONSTANT = "constant"
+    MAP = "map"
+    NULL_IF = "null_if"
+
+
+class Transform(Model):
+    name: TransformName
+    args: dict[str, JsonValue] = Field(default_factory=dict)
+
+
+class AttributeMapping(Model):
+    """Maps one column or one expression, through transforms, to an Attribute."""
+
+    column: str | None = None
+    expression: str | None = None
+    transforms: tuple[Transform, ...] = ()
+
+    @model_validator(mode="after")
+    def _check_input(self) -> Self:
+        if (self.column is None) == (self.expression is None):
+            raise ValueError("attribute mapping needs exactly one of column or expression")
+        return self
+
+
+class RelationMapping(Model):
+    """Foreign key columns of the Source row that identify the ``to`` Record of ``relation``."""
+
+    relation: RelationId
+    columns: tuple[str, ...] = Field(min_length=1)
+
+
+class TargetBranchPolicy(Model):
+    branch_template: str = "source/{source}"
+    merge_policy: MergePolicy = MergePolicy.PROPOSAL
+
+
+class PaginationType(StrEnum):
+    NONE = "none"
+    CURSOR = "cursor"
+    PAGE = "page"
+    OFFSET = "offset"
+    LINK = "link"
+
+
+class HttpPagination(Model):
+    """How an ``http_json`` Source asks for the next page.
+
+    ``cursor`` sends the value found at ``cursor_path`` of a response as ``cursor_param`` of the
+    next request; ``page`` counts ``page_param`` up from ``start_page``; ``offset`` moves
+    ``offset_param`` by ``page_size``; ``link`` follows the ``rel="next"`` URL of the ``Link``
+    header. ``size_param`` (with ``page_size``) asks for the page size.
+    """
+
+    type: PaginationType = PaginationType.NONE
+    cursor_path: str | None = Field(default=None, min_length=1)
+    cursor_param: str = Field(default="cursor", min_length=1)
+    page_param: str = Field(default="page", min_length=1)
+    start_page: int = Field(default=1, ge=0)
+    offset_param: str = Field(default="offset", min_length=1)
+    size_param: str | None = Field(default=None, min_length=1)
+    page_size: int | None = Field(default=None, ge=1)
+
+    @model_validator(mode="after")
+    def _check_type(self) -> Self:
+        if self.type is PaginationType.CURSOR and self.cursor_path is None:
+            raise ValueError("cursor pagination needs cursor_path")
+        if self.type is PaginationType.OFFSET and self.page_size is None:
+            raise ValueError("offset pagination needs page_size")
+        if self.size_param is not None and self.page_size is None:
+            raise ValueError("size_param needs page_size")
+        return self
+
+
+#: Request headers an ``http_json`` Source may not set: Scribe owns the target and the framing.
+RESERVED_HEADERS = frozenset(
+    {"host", "content-length", "transfer-encoding", "connection", "te", "upgrade"}
+)
+#: Header names containing one of these carry credentials: their values must be secret
+#: references, so no credential is ever stored in a schema.
+CREDENTIAL_HEADER_HINTS = (
+    "auth",
+    "token",
+    "secret",
+    "key",
+    "password",
+    "cookie",
+    "session",
+    "signature",
+)
+
+
+def credential_header(name: str) -> bool:
+    """Whether the value of header ``name`` must be a secret reference."""
+    lowered = name.lower()
+    return any(hint in lowered for hint in CREDENTIAL_HEADER_HINTS)
+
+
+class HttpOptions(Model):
+    """Requests of an ``http_json`` Source (the URL is the value of its ``connection``)."""
+
+    #: Header values; values of credential headers (``credential_header``) must be secret
+    #: references, resolved by Scribe like the connection.
+    headers: dict[str, str] = Field(default_factory=dict)
+    pagination: HttpPagination = Field(default_factory=HttpPagination)
+    #: Query parameter carrying the watermark of the previous run on incremental reads, so the
+    #: server can filter; rows are filtered by the watermark column either way.
+    since_param: str | None = Field(default=None, min_length=1)
+    #: Pages read at most by one run, below ``scribe.sources.http_json.max_pages``.
+    max_pages: int | None = Field(default=None, ge=1)
+
+    @model_validator(mode="after")
+    def _check_headers(self) -> Self:
+        for name, value in self.headers.items():
+            if not name.strip() or name.lower() in RESERVED_HEADERS:
+                raise ValueError(f"header {name!r} cannot be set by a Source")
+            if credential_header(name) and not value.startswith(SECRET_REFERENCE_PREFIX):
+                raise ValueError(
+                    f"header {name!r} carries credentials: its value must be a secret "
+                    f"reference ({SECRET_REFERENCE_PREFIX}...), never a literal"
+                )
+        return self
+
+
+class Source(SchemaElement):
+    object_type: ClassVar[str] = "schema.source"
+
+    id: SourceId
+    type: SourceType
+    connection: str
+    #: Table of a ``postgres_table``, ``mysql_table`` or ``mssql_table`` Source, optionally
+    #: schema-qualified (``erp.orders``); ``None`` reads the table named by the last segment
+    #: of the api name.
+    table: str | None = None
+    #: Read-only ``SELECT`` of a ``postgres_query``, ``mysql_query`` or ``mssql_query`` Source.
+    query: str | None = None
+    #: ``json`` and ``http_json``: dotted path of the array of records in the document
+    #: (``data.items``); ``None``: the document is the array.
+    records_path: str | None = Field(default=None, min_length=1)
+    #: ``json``, ``jsonl``, ``http_json`` and ``parquet``: nested objects become columns named
+    #: ``parent<separator>child``; ``None`` keeps them as JSON values.
+    flatten_separator: str | None = Field(default=None, min_length=1)
+    #: Requests of an ``http_json`` Source.
+    http: HttpOptions | None = None
+    target_kind: KindId
+    #: Source columns producing the primary key, in primary key order.
+    key_mapping: tuple[str, ...] = ()
+    attribute_mapping: dict[AttributeId, AttributeMapping] = Field(default_factory=dict)
+    relation_mapping: tuple[RelationMapping, ...] = ()
+    scope_filter: str | None = None
+    sync_mode: SyncMode = SyncMode.SNAPSHOT
+    watermark_column: str | None = None
+    tombstone_column: str | None = None
+    #: Cron expression; ``None`` means manual runs only.
+    schedule: str | None = None
+    target_branch_policy: TargetBranchPolicy = Field(default_factory=TargetBranchPolicy)
+    status: ElementStatus = ElementStatus.ACTIVE

lorebase_core/model/trait.py

@@ -0,0 +1,37 @@
+"""Trait (design 5.6): a shared set of Attributes that Kinds implement by mapping."""
+
+from __future__ import annotations
+
+from typing import ClassVar, Self
+
+from pydantic import Field, model_validator
+
+from lorebase_core.model.base import ElementStatus, Model, SchemaElement
+from lorebase_core.model.ids import AttributeId, TraitId
+from lorebase_core.types import TypeSpec
+
+
+class TraitAttribute(Model):
+    id: AttributeId
+    api_name: str
+    display_name: str = ""
+    description: str = ""
+    type: TypeSpec
+    required: bool = False
+
+
+class Trait(SchemaElement):
+    object_type: ClassVar[str] = "schema.trait"
+
+    id: TraitId
+    attributes: dict[AttributeId, TraitAttribute] = Field(default_factory=dict)
+    status: ElementStatus = ElementStatus.ACTIVE
+
+    @model_validator(mode="after")
+    def _check_keys(self) -> Self:
+        for key, attribute in self.attributes.items():
+            if key != attribute.id:
+                raise ValueError(
+                    f"Trait {self.id}: attribute map key {key} does not match id {attribute.id}"
+                )
+        return self

lorebase_core/model/vault.py

@@ -0,0 +1,43 @@
+"""Vault references (design 6.1, 13.5).
+
+The value of an Attribute stored in the vault is replaced, in Record objects, by a reference
+``{"$vault": "<token>"}``. Objects, hashes, diffs and merges see the reference only; the value
+lives encrypted in the store's ``vault`` table, and erasing it leaves every Revision intact
+while reads of the reference answer that the value was erased.
+
+Tokens are opaque here: the store derives them (``lorebase_store.vault``) so that the same
+value of the same Attribute of the same Record gives the same token, which keeps unchanged
+values unchanged across writes, branches and merges.
+"""
+
+from __future__ import annotations
+
+import re
+
+from pydantic import JsonValue
+
+VAULT_KEY = "$vault"
+"""The only member of a vault reference."""
+
+TOKEN_PREFIX = "vlt_"
+TOKEN_PATTERN = re.compile(r"vlt_[0-9a-f]{32}")
+"""Tokens minted by the store: the prefix and 128 bits in lowercase hex."""
+
+
+def vault_reference(token: str) -> dict[str, JsonValue]:
+    """The canonical value that stands for a vaulted value."""
+    return {VAULT_KEY: token}
+
+
+def vault_token(value: object) -> str | None:
+    """The token of a well-formed vault reference; None for any other value."""
+    if isinstance(value, dict) and len(value) == 1:
+        token = value.get(VAULT_KEY)
+        if isinstance(token, str) and TOKEN_PATTERN.fullmatch(token):
+            return token
+    return None
+
+
+def looks_like_reference(value: object) -> bool:
+    """Whether `value` is shaped like a vault reference, well formed or not."""
+    return isinstance(value, dict) and VAULT_KEY in value