lorebase-core 0.2.3__py3-none-any.whl

lorebase_core/model/lineage.py

@@ -0,0 +1,114 @@
+"""Lineage (design 5.11, AC-08): provenance entries attached to changes, not to content.
+
+A provenance entry describes one origin. Each change in a Revision changeset carries a
+``LineageMap`` from the Attribute ids it changed to provenance entry ids. Provenance is never
+part of content hashes.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Annotated, Literal, Self
+
+from pydantic import Field, field_validator, model_validator
+
+from lorebase_core.model.base import Model
+from lorebase_core.model.ids import AttributeId, OperationId, ProvenanceId, SourceId
+from lorebase_core.types import AttributeType, TypeSpec, encode_value
+
+_TIMESTAMP = TypeSpec.of(AttributeType.TIMESTAMP)
+
+
+class OriginType(StrEnum):
+    OPERATION = "operation"
+    SOURCE_SYNC = "source_sync"
+    EXTRACTION = "extraction"
+    MERGE_RESOLUTION = "merge_resolution"
+    SCHEMA_MIGRATION = "schema_migration"
+    IMPORT = "import"
+
+
+class OperationOrigin(Model):
+    type: Literal["operation"] = "operation"
+    invocation_id: str
+    #: Operation api name; built-in Operations (``create_supplier``) have no stored id.
+    operation: str
+    operation_id: OperationId | None = None
+    principal: str
+
+
+class SourceSyncOrigin(Model):
+    type: Literal["source_sync"] = "source_sync"
+    source_id: SourceId
+    run_id: str
+    row_locator: str | None = None
+
+
+class ExtractionOrigin(Model):
+    type: Literal["extraction"] = "extraction"
+    job_id: str
+    document_id: str
+    span_start: int = Field(ge=0)
+    span_end: int = Field(ge=0)
+    model_id: str
+    prompt_version: str
+
+    @model_validator(mode="after")
+    def _check_span(self) -> Self:
+        if self.span_end < self.span_start:
+            raise ValueError("span_end must not precede span_start")
+        return self
+
+
+class MergeResolutionOrigin(Model):
+    type: Literal["merge_resolution"] = "merge_resolution"
+    proposal_id: str
+    conflict_id: str
+    resolver: str
+
+
+class SchemaMigrationOrigin(Model):
+    """A value written by a schema change (cast or default) on ``attribute_id``."""
+
+    type: Literal["schema_migration"] = "schema_migration"
+    attribute_id: AttributeId
+    from_type: TypeSpec | None = None
+    to_type: TypeSpec | None = None
+
+
+class ImportOrigin(Model):
+    type: Literal["import"] = "import"
+    batch_id: str
+
+
+Origin = Annotated[
+    OperationOrigin
+    | SourceSyncOrigin
+    | ExtractionOrigin
+    | MergeResolutionOrigin
+    | SchemaMigrationOrigin
+    | ImportOrigin,
+    Field(discriminator="type"),
+]
+
+
+class ProvenanceEntry(Model):
+    """One Lineage origin: ``{id, origin, actor, at}``; ``at`` is a canonical timestamp."""
+
+    id: ProvenanceId
+    origin: Origin
+    actor: str
+    at: str
+
+    @field_validator("at", mode="before")
+    @classmethod
+    def _canonical_timestamp(cls, value: object) -> object:
+        return encode_value(_TIMESTAMP, value)
+
+    @property
+    def origin_type(self) -> OriginType:
+        return OriginType(self.origin.type)
+
+
+#: Per change: changed Attribute id -> provenance entry id.
+LineageMap = dict[AttributeId, ProvenanceId]

lorebase_core/model/operation.py

@@ -0,0 +1,161 @@
+"""Operation (design 5.7): a named, typed, governed write.
+
+Expressions (preconditions, step values, review policies) are CEL source strings (AC-06).
+They are stored as written; type checking and evaluation belong to the expression module.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Annotated, ClassVar, Literal, Self
+
+from pydantic import Field, model_validator
+
+from lorebase_core.model.base import ElementStatus, Model, SchemaElement, snake_case
+from lorebase_core.model.ids import AttributeId, KindId, OperationId, RelationId
+from lorebase_core.model.kind import BuiltinOperation, Kind
+from lorebase_core.model.relation import Relation
+from lorebase_core.types import TypeSpec
+
+#: A CEL expression, stored as source text.
+Expression = Annotated[str, Field(min_length=1)]
+
+SIDE_EFFECT_MAX_LENGTH = 64
+"""Longest side effect name (static validation); the outbox event type is
+`operation.<name>`."""
+
+
+class OperationParameter(Model):
+    """A parameter. Record reference parameters use a ``ref`` type naming the Kind."""
+
+    name: str
+    type: TypeSpec
+    required: bool = False
+    description: str = ""
+
+
+class Precondition(Model):
+    expression: Expression
+    message: str
+
+
+class CreateRecordStep(Model):
+    step: Literal["create_record"] = "create_record"
+    kind: KindId
+    values: dict[AttributeId, Expression] = Field(default_factory=dict)
+
+
+class UpdateRecordStep(Model):
+    step: Literal["update_record"] = "update_record"
+    kind: KindId
+    target: Expression
+    values: dict[AttributeId, Expression] = Field(default_factory=dict)
+
+
+class DeleteRecordStep(Model):
+    step: Literal["delete_record"] = "delete_record"
+    kind: KindId
+    target: Expression
+
+
+class CreateLinkStep(Model):
+    step: Literal["create_link"] = "create_link"
+    relation: RelationId
+    from_: Expression = Field(alias="from")
+    to: Expression
+    values: dict[AttributeId, Expression] = Field(default_factory=dict)
+
+
+class DeleteLinkStep(Model):
+    step: Literal["delete_link"] = "delete_link"
+    relation: RelationId
+    from_: Expression = Field(alias="from")
+    to: Expression
+
+
+OperationStep = Annotated[
+    CreateRecordStep | UpdateRecordStep | DeleteRecordStep | CreateLinkStep | DeleteLinkStep,
+    Field(discriminator="step"),
+]
+
+
+class ReviewMode(StrEnum):
+    NEVER = "never"
+    ALWAYS = "always"
+    EXPRESSION = "expression"
+
+
+class ReviewPolicy(Model):
+    """``never``, ``always``, or a CEL ``expression`` (true sends the edits to a Proposal)."""
+
+    mode: ReviewMode = ReviewMode.NEVER
+    expression: Expression | None = None
+
+    @model_validator(mode="after")
+    def _check_expression(self) -> Self:
+        if (self.mode is ReviewMode.EXPRESSION) != (self.expression is not None):
+            raise ValueError(
+                "review policy has an expression if and only if its mode is 'expression'"
+            )
+        return self
+
+
+class RateLimit(Model):
+    max_calls: int = Field(ge=1)
+    per_seconds: int = Field(ge=1)
+
+
+class Operation(SchemaElement):
+    object_type: ClassVar[str] = "schema.operation"
+
+    id: OperationId
+    parameters: tuple[OperationParameter, ...] = ()
+    preconditions: tuple[Precondition, ...] = ()
+    steps: tuple[OperationStep, ...] = ()
+    review_policy: ReviewPolicy = Field(default_factory=ReviewPolicy)
+    permissions: tuple[str, ...] = ()
+    side_effects: tuple[str, ...] = ()
+    """Named events written to the outbox in the commit transaction (design 10.5); unique
+    snake_case names, checked by static validation so that stored schemas always load."""
+    rate_limit: RateLimit | None = None
+    status: ElementStatus = ElementStatus.ACTIVE
+
+    @property
+    def uses_expressions(self) -> bool:
+        """True when the Operation carries CEL expressions."""
+        return (
+            bool(self.preconditions or self.steps)
+            or self.review_policy.mode is ReviewMode.EXPRESSION
+        )
+
+
+def builtin_operation_names(kind: Kind) -> list[str]:
+    """Names of the built-in Operations enabled for ``kind``: ``create_{kind}`` and so on."""
+    base = snake_case(kind.api_name)
+    return [f"{operation.value}_{base}" for operation in kind.operations_builtin]
+
+
+def builtin_link_operation_names(relation: Relation) -> list[str]:
+    """``link_{relation}`` and ``unlink_{relation}`` for an explicit Relation."""
+    return [f"link_{relation.api_name}", f"unlink_{relation.api_name}"]
+
+
+__all__ = [
+    "SIDE_EFFECT_MAX_LENGTH",
+    "BuiltinOperation",
+    "CreateLinkStep",
+    "CreateRecordStep",
+    "DeleteLinkStep",
+    "DeleteRecordStep",
+    "Expression",
+    "Operation",
+    "OperationParameter",
+    "OperationStep",
+    "Precondition",
+    "RateLimit",
+    "ReviewMode",
+    "ReviewPolicy",
+    "UpdateRecordStep",
+    "builtin_link_operation_names",
+    "builtin_operation_names",
+]

lorebase_core/model/paths.py

@@ -0,0 +1,141 @@
+"""Path Derivations as virtual Relations (design 5.8).
+
+A `path` Derivation composes Relations, `A -r1-> B -r2-> C`, into a virtual Relation `A -> C`
+named by the Derivation's `api_name` (forward) and `inverse_api_name` (inverse, optional).
+`PathRelation` is what readers and the materializer work with: the steps resolved to
+Relations and a virtual `Relation` (id = the Derivation id, explicit backing, no Attributes)
+that code reading Links by Relation can use unchanged.
+
+Semantics, shared by every evaluation (materialized rows, reads at the head, reads at a past
+Revision): the Links of the path are the distinct `(a, c)` pairs such that `c` is reached from
+`a` through one Link per step. The first step follows every Link of `a`; every later step
+follows, from each intermediate Record, at most `fanout` Links, those with the smallest far
+keys. The bound depends on the intermediate Record only, never on where the walk started, so
+the pairs of one `a` can be recomputed alone. Paths have a fixed length, so a cycle in the data
+ends the walk like any other step (a path may lead back to its start: `(a, a)` is a pair).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from dataclasses import dataclass
+
+from lorebase_core.model.derivation import Derivation, PathDerivation
+from lorebase_core.model.relation import Cardinality, LinkDirection, Relation, RelationBacking
+
+
+@dataclass(frozen=True, slots=True)
+class PathHop:
+    """One step of a path: a Relation followed in a direction."""
+
+    relation: Relation
+    direction: LinkDirection
+
+    @property
+    def near_kind(self) -> str:
+        return self.relation.endpoint(self.direction)
+
+    @property
+    def far_kind(self) -> str:
+        return self.relation.far_endpoint(self.direction)
+
+
+@dataclass(frozen=True, slots=True)
+class PathRelation:
+    """A valid path Derivation with its steps resolved (see the module docstring)."""
+
+    derivation: Derivation
+    relation: Relation
+    steps: tuple[PathHop, ...]
+
+    @property
+    def id(self) -> str:
+        return self.derivation.id
+
+    @property
+    def materialized(self) -> bool:
+        definition = self.derivation.definition
+        assert isinstance(definition, PathDerivation)
+        return definition.materialized
+
+    @property
+    def kinds(self) -> tuple[str, ...]:
+        """Kind ids along the path, from the first `near` Kind to the last `far` Kind."""
+        return (self.steps[0].near_kind, *(step.far_kind for step in self.steps))
+
+    @property
+    def inner_kinds(self) -> tuple[str, ...]:
+        """Kind ids of the intermediate Records."""
+        return self.kinds[1:-1]
+
+
+def path_relation(derivation: Derivation, relations: Mapping[str, Relation]) -> PathRelation | None:
+    """The resolved path of a `path` Derivation; `None` for other variants and for paths
+    naming unknown Relations or not continuing from one step to the next (schema validation
+    reports those)."""
+    definition = derivation.definition
+    if not isinstance(definition, PathDerivation):
+        return None
+    steps: list[PathHop] = []
+    for step in definition.steps:
+        relation = relations.get(step.relation)
+        if relation is None:
+            return None
+        hop = PathHop(relation, step.direction)
+        if steps and steps[-1].far_kind != hop.near_kind:
+            return None
+        steps.append(hop)
+    virtual = Relation.model_construct(
+        id=derivation.id,
+        api_name=derivation.api_name,
+        display_name=derivation.display_name,
+        description=derivation.description,
+        inverse_api_name=definition.inverse_api_name or "",
+        from_kind=steps[0].near_kind,
+        to_kind=steps[-1].far_kind,
+        cardinality=Cardinality.MANY_TO_MANY,
+        attributes={},
+        backing=RelationBacking.EXPLICIT,
+        status=derivation.status,
+    )
+    return PathRelation(derivation, virtual, tuple(steps))
+
+
+def path_relations(
+    derivations: Iterable[Derivation], relations: Mapping[str, Relation]
+) -> dict[str, PathRelation]:
+    """The valid path Derivations by Derivation id."""
+    found: dict[str, PathRelation] = {}
+    for derivation in derivations:
+        path = path_relation(derivation, relations)
+        if path is not None:
+            found[derivation.id] = path
+    return found
+
+
+def bounded(far_keys: Iterable[str], fanout: int | None) -> list[str]:
+    """The far keys one intermediate Record contributes: distinct, smallest first, at most
+    `fanout` (all when `None`)."""
+    ordered = sorted(set(far_keys))
+    return ordered if fanout is None else ordered[:fanout]
+
+
+def walk(
+    starts: Iterable[str],
+    steps: Sequence[Callable[[Sequence[str]], Mapping[str, Sequence[str]]]],
+    fanout: int | None,
+) -> dict[str, set[str]]:
+    """Pairs of a path from `starts`, with each step given as a function from near keys to
+    their far keys (synchronous loaders: tree reads hold every Link in memory)."""
+    reached: dict[str, set[str]] = {start: {start} for start in starts}
+    for index, step in enumerate(steps):
+        frontier = sorted({key for keys in reached.values() for key in keys})
+        found = step(frontier) if frontier else {}
+        nexts = {
+            near: (list(far) if index == 0 else bounded(far, fanout)) for near, far in found.items()
+        }
+        reached = {
+            start: {far for near in keys for far in nexts.get(near, ())}
+            for start, keys in reached.items()
+        }
+    return reached

lorebase_core/model/record.py

@@ -0,0 +1,127 @@
+"""Record and Link (design 5.1, 5.10).
+
+A Record is identified by ``(kind, key)`` (AC-02); a Link by ``(relation, from_key, to_key)``,
+with no parallel Links (AC-03). Values are keyed by Attribute id and hold canonical values;
+null and absent are the same, so ``None`` values are dropped.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import ClassVar, NamedTuple
+from urllib.parse import quote
+
+from pydantic import Field, JsonValue, field_validator
+
+from lorebase_core.model.base import Model
+from lorebase_core.model.ids import AttributeId, KindId, RelationId
+from lorebase_core.model.kind import Kind
+from lorebase_core.types import normalize_key, to_text
+
+#: Separator between Kind api name and key in external references (``Supplier/S-1001``).
+REF_SEPARATOR = "/"
+#: Separator between components of composite keys (components are percent-encoded).
+KEY_COMPONENT_SEPARATOR = ","
+
+
+class RecordIdentity(NamedTuple):
+    kind: str
+    key: str
+
+
+class LinkIdentity(NamedTuple):
+    relation: str
+    from_key: str
+    to_key: str
+
+
+def _drop_absent(values: dict[str, JsonValue]) -> dict[str, JsonValue]:
+    return {attribute: value for attribute, value in values.items() if value is not None}
+
+
+class Record(Model):
+    object_type: ClassVar[str] = "record"
+
+    kind: KindId
+    key: str
+    values: dict[AttributeId, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("key")
+    @classmethod
+    def _normalize_key(cls, key: str) -> str:
+        return normalize_key(key)
+
+    @field_validator("values")
+    @classmethod
+    def _drop_absent_values(cls, values: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        return _drop_absent(values)
+
+    @property
+    def identity(self) -> RecordIdentity:
+        return RecordIdentity(self.kind, self.key)
+
+
+class Link(Model):
+    object_type: ClassVar[str] = "link"
+
+    relation: RelationId
+    from_key: str = Field(alias="from")
+    to_key: str = Field(alias="to")
+    values: dict[AttributeId, JsonValue] = Field(default_factory=dict)
+
+    @field_validator("from_key", "to_key")
+    @classmethod
+    def _normalize_key(cls, key: str) -> str:
+        return normalize_key(key)
+
+    @field_validator("values")
+    @classmethod
+    def _drop_absent_values(cls, values: dict[str, JsonValue]) -> dict[str, JsonValue]:
+        return _drop_absent(values)
+
+    @property
+    def identity(self) -> LinkIdentity:
+        return LinkIdentity(self.relation, self.from_key, self.to_key)
+
+
+def render_key(components: Sequence[str]) -> str:
+    """Render primary key component texts as a canonical key string.
+
+    A single component is the key itself; composite keys percent-encode each component and
+    join them with ``,`` so that rendering is unambiguous.
+    """
+    if not components:
+        raise ValueError("a key needs at least one component")
+    if len(components) == 1:
+        return components[0]
+    return KEY_COMPONENT_SEPARATOR.join(quote(component, safe="") for component in components)
+
+
+def record_key(kind: Kind, values: Mapping[str, JsonValue]) -> str | None:
+    """The key of a Record of ``kind`` from its canonical primary key values.
+
+    Returns ``None`` when the Kind has no natural key or a primary key value is absent.
+    """
+    if not kind.primary_key:
+        return None
+    components: list[str] = []
+    for attribute_id in kind.primary_key:
+        value = values.get(attribute_id)
+        attribute = kind.attributes.get(attribute_id)
+        if value is None or attribute is None:
+            return None
+        components.append(to_text(attribute.type, value))
+    return render_key(components)
+
+
+def format_ref(kind_api_name: str, key: str) -> str:
+    """External reference ``{kind_api_name}/{key}`` (design 5.1)."""
+    return f"{kind_api_name}{REF_SEPARATOR}{key}"
+
+
+def parse_ref(ref: str) -> tuple[str, str]:
+    """Split an external reference into ``(kind_api_name, key)``; keys may contain ``/``."""
+    kind_api_name, separator, key = ref.partition(REF_SEPARATOR)
+    if not separator or not kind_api_name or not key:
+        raise ValueError(f"not a Record reference: {ref!r}")
+    return kind_api_name, key

lorebase_core/model/relation.py

@@ -0,0 +1,83 @@
+"""Relation (design 5.5): a typed, directional connection between two Kinds."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import ClassVar, Self
+
+from pydantic import Field, model_validator
+
+from lorebase_core.model.attribute import Attribute
+from lorebase_core.model.base import ElementStatus, SchemaElement
+from lorebase_core.model.ids import AttributeId, KindId, RelationId
+from lorebase_core.model.kind import check_attribute_keys
+
+
+class Cardinality(StrEnum):
+    """``x_to_y``: how many ``from`` Records (x) and ``to`` Records (y) one Link side allows.
+
+    ``one_to_many``: each ``to`` Record has at most one incoming Link;
+    ``many_to_one``: each ``from`` Record has at most one outgoing Link.
+    """
+
+    ONE_TO_ONE = "one_to_one"
+    ONE_TO_MANY = "one_to_many"
+    MANY_TO_ONE = "many_to_one"
+    MANY_TO_MANY = "many_to_many"
+
+
+class OnDelete(StrEnum):
+    CASCADE = "cascade"
+    RESTRICT = "restrict"
+
+
+class RelationBacking(StrEnum):
+    """``explicit``: Links are stored. ``foreign_key``: Links derive from a ``ref`` Attribute."""
+
+    EXPLICIT = "explicit"
+    FOREIGN_KEY = "foreign_key"
+
+
+class LinkDirection(StrEnum):
+    FORWARD = "forward"
+    INVERSE = "inverse"
+
+
+class Relation(SchemaElement):
+    object_type: ClassVar[str] = "schema.relation"
+
+    id: RelationId
+    inverse_api_name: str
+    from_kind: KindId
+    to_kind: KindId
+    cardinality: Cardinality = Cardinality.MANY_TO_MANY
+    required_from: bool = False
+    attributes: dict[AttributeId, Attribute] = Field(default_factory=dict)
+    on_delete: OnDelete = OnDelete.CASCADE
+    backing: RelationBacking = RelationBacking.EXPLICIT
+    #: For ``foreign_key`` backing: the ``ref`` Attribute of ``from_kind`` holding the ``to`` key.
+    foreign_key_attribute: AttributeId | None = None
+    status: ElementStatus = ElementStatus.ACTIVE
+
+    @model_validator(mode="after")
+    def _check_keys(self) -> Self:
+        check_attribute_keys(self.attributes, f"Relation {self.id}")
+        return self
+
+    @property
+    def single_per_from(self) -> bool:
+        """Each ``from`` Record has at most one Link of this Relation."""
+        return self.cardinality in (Cardinality.ONE_TO_ONE, Cardinality.MANY_TO_ONE)
+
+    @property
+    def single_per_to(self) -> bool:
+        """Each ``to`` Record has at most one Link of this Relation."""
+        return self.cardinality in (Cardinality.ONE_TO_ONE, Cardinality.ONE_TO_MANY)
+
+    def endpoint(self, direction: LinkDirection) -> KindId:
+        """The Kind a traversal in ``direction`` starts from."""
+        return self.from_kind if direction is LinkDirection.FORWARD else self.to_kind
+
+    def far_endpoint(self, direction: LinkDirection) -> KindId:
+        """The Kind a traversal in ``direction`` arrives at."""
+        return self.to_kind if direction is LinkDirection.FORWARD else self.from_kind