lorebase-core 0.2.3__py3-none-any.whl

lorebase_core/model/attribute.py

@@ -0,0 +1,72 @@
+"""Attribute (design 5.4): a typed field on a Kind, or a link Attribute on a Relation."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+from pydantic import Field, JsonValue, field_validator
+
+from lorebase_core.model.base import Model
+from lorebase_core.model.ids import AttributeId, DerivationId, SourceId
+from lorebase_core.types import Constraints, TypeSpec
+
+
+class Sensitivity(StrEnum):
+    PII = "pii"
+    CONFIDENTIAL = "confidential"
+    RESTRICTED = "restricted"
+
+
+class SearchWeight(StrEnum):
+    """Full-text weight of a searchable Attribute (``A`` is the strongest)."""
+
+    A = "A"
+    B = "B"
+    C = "C"
+    D = "D"
+
+
+class Attribute(Model):
+    """A typed field. ``default`` is a value of ``type`` (stored in canonical form).
+
+    ``vault`` stores the values of a ``pii`` Attribute of a Kind in the vault (design 13.5):
+    Record objects hold a vault reference instead of the value (``lorebase_core.model.vault``).
+    Unset (None) keeps values inline; the field is then left out of canonical content, so
+    schemas written before it keep their hashes.
+    """
+
+    id: AttributeId
+    api_name: str
+    display_name: str = ""
+    description: str = ""
+    type: TypeSpec
+    required: bool = False
+    unique: bool = False
+    default: JsonValue = None
+    constraints: Constraints = Field(default_factory=Constraints)
+    sensitivity: tuple[Sensitivity, ...] = ()
+    indexed: bool = False
+    searchable: bool = False
+    search_weight: SearchWeight | None = None
+    derivation_id: DerivationId | None = None
+    authority: SourceId | None = None
+    vault: bool | None = None
+
+    @field_validator("sensitivity")
+    @classmethod
+    def _sort_sensitivity(cls, tags: tuple[Sensitivity, ...]) -> tuple[Sensitivity, ...]:
+        return tuple(sorted(set(tags)))
+
+    @property
+    def is_derived(self) -> bool:
+        return self.derivation_id is not None
+
+    @property
+    def is_sensitive(self) -> bool:
+        """Whether reads need a clearance for this Attribute's values (design 10.2)."""
+        return bool(self.sensitivity)
+
+    @property
+    def in_vault(self) -> bool:
+        """Whether values are stored in the vault, Record objects holding references."""
+        return self.vault is True

lorebase_core/model/base.py

@@ -0,0 +1,61 @@
+"""Shared base classes and naming rules of schema elements."""
+
+from __future__ import annotations
+
+import re
+from enum import StrEnum
+from typing import ClassVar
+
+from pydantic import BaseModel, ConfigDict, JsonValue
+
+
+class Model(BaseModel):
+    """Frozen model base. ``content()`` is the JSON payload used for canonical hashing."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True)
+
+    def content(self) -> dict[str, JsonValue]:
+        """JSON-compatible payload: field aliases, absent (``None``) fields omitted."""
+        return self.model_dump(mode="json", by_alias=True, exclude_none=True)
+
+
+class ElementStatus(StrEnum):
+    DRAFT = "draft"
+    ACTIVE = "active"
+    DEPRECATED = "deprecated"
+
+
+class SchemaElement(Model):
+    """A versioned schema element: stable ``id`` plus renamable ``api_name`` (AC-01)."""
+
+    #: Object type used by the canonical hashing scheme (design 6.1).
+    object_type: ClassVar[str]
+
+    id: str
+    api_name: str
+    display_name: str = ""
+    description: str = ""
+
+
+class NameStyle(StrEnum):
+    """Api name styles (design 5.1)."""
+
+    SNAKE = "snake_case"
+    PASCAL = "PascalCase"
+    DOTTED_SNAKE = "dotted.snake_case"
+
+
+_NAME_PATTERNS: dict[NameStyle, re.Pattern[str]] = {
+    NameStyle.SNAKE: re.compile(r"[a-z][a-z0-9]*(?:_[a-z0-9]+)*"),
+    NameStyle.PASCAL: re.compile(r"[A-Z][A-Za-z0-9]*"),
+    NameStyle.DOTTED_SNAKE: re.compile(r"[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*"),
+}
+
+
+def is_valid_api_name(name: str, style: NameStyle) -> bool:
+    return _NAME_PATTERNS[style].fullmatch(name) is not None
+
+
+def snake_case(name: str) -> str:
+    """``RiskFlag`` -> ``risk_flag``; used to derive built-in Operation names."""
+    return re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", name).lower()

lorebase_core/model/derivation.py

@@ -0,0 +1,95 @@
+"""Derivation (design 5.8): a computed Attribute or Relation. Model only; no evaluation here.
+
+Derived values are not part of Record content and never merge (AC-07).
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Annotated, ClassVar, Literal, Self
+
+from pydantic import Field, model_validator
+
+from lorebase_core.model.base import ElementStatus, Model, SchemaElement
+from lorebase_core.model.ids import AttributeId, DerivationId, KindId, RelationId
+from lorebase_core.model.operation import Expression
+from lorebase_core.model.relation import LinkDirection
+
+
+class AggregateFunction(StrEnum):
+    COUNT = "count"
+    SUM = "sum"
+    MIN = "min"
+    MAX = "max"
+    AVG = "avg"
+
+
+class ExpressionDerivation(Model):
+    """A CEL expression over the Record's own Attributes, computing ``kind.attribute``."""
+
+    variant: Literal["expression"] = "expression"
+    kind: KindId
+    attribute: AttributeId
+    expression: Expression
+
+
+class AggregateDerivation(Model):
+    """An aggregate over the Records reached from ``kind`` through ``relation`` in ``direction``.
+
+    ``relation`` is a Relation or a path Derivation (its virtual Relation).
+    ``source_attribute`` is an Attribute of the Kind at the far end; ``count`` needs none.
+    """
+
+    variant: Literal["aggregate"] = "aggregate"
+    kind: KindId
+    attribute: AttributeId
+    relation: RelationId | DerivationId
+    direction: LinkDirection = LinkDirection.FORWARD
+    function: AggregateFunction
+    source_attribute: AttributeId | None = None
+    filter: Expression | None = None
+
+    @model_validator(mode="after")
+    def _check_source(self) -> Self:
+        if self.function is not AggregateFunction.COUNT and self.source_attribute is None:
+            raise ValueError(f"aggregate {self.function} requires source_attribute")
+        return self
+
+
+class PathStep(Model):
+    relation: RelationId
+    direction: LinkDirection = LinkDirection.FORWARD
+
+
+class PathDerivation(Model):
+    """A composed Relation ``A -r1-> B -r2-> C`` exposed as a virtual Relation ``A -> C``.
+
+    The virtual Relation's name is the Derivation's ``api_name``.
+    """
+
+    variant: Literal["path"] = "path"
+    steps: tuple[PathStep, ...] = Field(min_length=2)
+    inverse_api_name: str | None = None
+    materialized: bool = False
+
+
+DerivationDefinition = Annotated[
+    ExpressionDerivation | AggregateDerivation | PathDerivation,
+    Field(discriminator="variant"),
+]
+
+
+class Derivation(SchemaElement):
+    object_type: ClassVar[str] = "schema.derivation"
+
+    id: DerivationId
+    definition: DerivationDefinition
+    status: ElementStatus = ElementStatus.ACTIVE
+
+    @property
+    def target(self) -> tuple[KindId, AttributeId] | None:
+        """The ``(kind, attribute)`` this Derivation computes; ``None`` for path Derivations."""
+        definition = self.definition
+        if isinstance(definition, ExpressionDerivation | AggregateDerivation):
+            return (definition.kind, definition.attribute)
+        return None

lorebase_core/model/domain.py

@@ -0,0 +1,71 @@
+"""Domain (design 5.2): the isolation boundary for data, permissions and storage."""
+
+from __future__ import annotations
+
+from typing import Annotated, ClassVar
+
+from pydantic import Field, JsonValue, StringConstraints, field_validator
+
+from lorebase_core.model.base import Model
+from lorebase_core.types import AttributeType, TypeSpec, encode_value
+
+DomainSlug = Annotated[str, StringConstraints(pattern=r"^[a-z][a-z0-9_-]*$")]
+
+_TIMESTAMP = TypeSpec.of(AttributeType.TIMESTAMP)
+
+#: Upper bounds of the example questions of a Domain; they are shown as buttons.
+MAX_EXAMPLES = 20
+MAX_EXAMPLE_LABEL = 80
+MAX_EXAMPLE_QUESTION = 500
+
+
+class DomainExample(Model):
+    """A sample question about the Domain, offered by clients as a ready-made prompt."""
+
+    label: Annotated[
+        str, StringConstraints(strip_whitespace=True, min_length=1, max_length=MAX_EXAMPLE_LABEL)
+    ]
+    question: Annotated[
+        str,
+        StringConstraints(strip_whitespace=True, min_length=1, max_length=MAX_EXAMPLE_QUESTION),
+    ]
+
+
+class Domain(Model):
+    object_type: ClassVar[str] = "schema.domain"
+
+    id: DomainSlug
+    display_name: str = ""
+    description: str = ""
+    default_branch: str = "main"
+    settings: dict[str, JsonValue] = Field(default_factory=dict)
+    #: Optional sample questions; absent (not empty) when there are none, so the canonical
+    #: content of Domains written before this field existed is unchanged.
+    examples: tuple[DomainExample, ...] | None = Field(default=None, max_length=MAX_EXAMPLES)
+    #: Canonical RFC 3339 UTC timestamp.
+    created_at: str | None = None
+    created_by: str | None = None
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def _canonical_timestamp(cls, value: object) -> object:
+        return encode_value(_TIMESTAMP, value)
+
+    @field_validator("examples", mode="before")
+    @classmethod
+    def _empty_examples_are_absent(cls, value: object) -> object:
+        if isinstance(value, list | tuple) and len(value) == 0:
+            return None
+        return value
+
+    @field_validator("examples")
+    @classmethod
+    def _unique_labels(
+        cls, value: tuple[DomainExample, ...] | None
+    ) -> tuple[DomainExample, ...] | None:
+        if value is not None:
+            labels = [example.label for example in value]
+            duplicates = sorted({label for label in labels if labels.count(label) > 1})
+            if duplicates:
+                raise ValueError(f"duplicate example labels: {', '.join(duplicates)}")
+        return value

lorebase_core/model/embedding.py

@@ -0,0 +1,193 @@
+"""Embedding templates (design 8.4, 9.3): the text of a Record that Scribe embeds.
+
+A Kind opts into semantic search by setting ``search.embedding_template``, a text with
+``{attribute}`` placeholders naming Attributes of the Kind by api name, plus ``{_kind}`` (the
+Kind's display name, else its api name) and ``{_key}`` (the Record key). ``{{`` and ``}}``
+write literal braces. Example::
+
+    search:
+      embedding_template: "{legal_name} ({country}). {description}"
+
+Rendering is deterministic: absent values render as nothing, booleans as ``true``/``false``,
+lists as their items joined by ``, ``, objects as compact JSON, and runs of whitespace collapse
+to one space. Derived Attributes cannot be named (their values are not part of the Record's
+content hash, which keys the stored vectors), and sensitive Attributes only when
+``search.embedding_sensitive`` is true.
+
+``KindTemplate.fingerprint`` changes whenever the rendered text of some Record could change
+for reasons other than its values (template text, the Attributes the names resolve to, the
+Kind name used by ``{_kind}``, the rendering rules), so a job can tell which Kinds need their
+texts rendered again.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+from pydantic import JsonValue
+
+from lorebase_core.model.kind import Kind
+
+RENDERING_VERSION = 1
+"""Bumped when the rendering rules change, which changes every fingerprint."""
+
+KIND_FIELD = "_kind"
+KEY_FIELD = "_key"
+SPECIAL_FIELDS = frozenset({KIND_FIELD, KEY_FIELD})
+
+_TOKEN = re.compile(r"\{\{|\}\}|\{([^{}]*)\}|[{}]")
+_WHITESPACE = re.compile(r"\s+")
+
+
+class TemplateError(ValueError):
+    """An embedding template that cannot be parsed or names something it cannot use."""
+
+
+@dataclass(frozen=True, slots=True)
+class Placeholder:
+    name: str
+
+
+@dataclass(frozen=True, slots=True)
+class EmbeddingTemplate:
+    """A parsed template: literal text and placeholders, in order."""
+
+    source: str
+    parts: tuple[str | Placeholder, ...]
+
+    @property
+    def fields(self) -> tuple[str, ...]:
+        """Placeholder names in order of first use."""
+        return tuple(dict.fromkeys(p.name for p in self.parts if isinstance(p, Placeholder)))
+
+
+def parse_template(source: str) -> EmbeddingTemplate:
+    """Parse ``source``; `TemplateError` on an unbalanced brace or an empty placeholder."""
+    parts: list[str | Placeholder] = []
+    literal: list[str] = []
+    position = 0
+    for match in _TOKEN.finditer(source):
+        literal.append(source[position : match.start()])
+        position = match.end()
+        token = match.group(0)
+        if token in ("{{", "}}"):
+            literal.append(token[0])
+            continue
+        name = match.group(1)
+        if name is None:
+            raise TemplateError(
+                f"unbalanced {token!r} at offset {match.start()}; write {token * 2}"
+            )
+        name = name.strip()
+        if not name:
+            raise TemplateError(f"empty placeholder at offset {match.start()}")
+        if any(literal):
+            parts.append("".join(literal))
+        literal = []
+        parts.append(Placeholder(name))
+    literal.append(source[position:])
+    if any(literal):
+        parts.append("".join(literal))
+    if not any(isinstance(p, Placeholder) for p in parts):
+        raise TemplateError("the template names no Attribute")
+    return EmbeddingTemplate(source, tuple(parts))
+
+
+@dataclass(frozen=True, slots=True)
+class KindTemplate:
+    """The embedding template of one Kind, resolved against its Attributes."""
+
+    kind_id: str
+    template: EmbeddingTemplate
+    attributes: Mapping[str, str]
+    """Placeholder name -> Attribute id (special fields are not listed)."""
+    kind_label: str
+    fingerprint: str
+
+    def render(self, key: str, values: Mapping[str, JsonValue]) -> str:
+        """The text of the Record ``key`` with ``values`` (keyed by Attribute id)."""
+        out: list[str] = []
+        for part in self.template.parts:
+            if isinstance(part, str):
+                out.append(part)
+            elif part.name == KIND_FIELD:
+                out.append(self.kind_label)
+            elif part.name == KEY_FIELD:
+                out.append(key)
+            else:
+                out.append(_format(values.get(self.attributes[part.name])))
+        return _WHITESPACE.sub(" ", "".join(out)).strip()
+
+
+def template_problems(kind: Kind) -> list[str]:
+    """Why the Kind's template cannot be used; empty when it can (or there is none)."""
+    try:
+        _resolve(kind)
+    except TemplateError as error:
+        return [str(error)]
+    return []
+
+
+def kind_template(kind: Kind) -> KindTemplate | None:
+    """The resolved template of ``kind``; None without one. Raises `TemplateError`."""
+    return _resolve(kind)
+
+
+def _resolve(kind: Kind) -> KindTemplate | None:
+    source = kind.search.embedding_template
+    if source is None or not source.strip():
+        return None
+    template = parse_template(source)
+    allow_sensitive = bool(kind.search.embedding_sensitive)
+    attributes: dict[str, str] = {}
+    for name in template.fields:
+        if name in SPECIAL_FIELDS:
+            continue
+        attribute = kind.attribute_by_api_name(name)
+        if attribute is None:
+            raise TemplateError(f"{{{name}}} is not an Attribute of Kind {kind.api_name!r}")
+        if attribute.is_derived:
+            raise TemplateError(f"{{{name}}} is derived; templates read stored values only")
+        if attribute.in_vault:
+            raise TemplateError(
+                f"{{{name}}} is stored in the vault; templates cannot read vault values"
+            )
+        if attribute.sensitivity and not allow_sensitive:
+            raise TemplateError(
+                f"{{{name}}} is sensitive ({', '.join(attribute.sensitivity)}); set "
+                "search.embedding_sensitive to embed it"
+            )
+        attributes[name] = attribute.id
+    label = kind.display_name or kind.api_name
+    fingerprint = hashlib.sha256(
+        json.dumps(
+            {
+                "v": RENDERING_VERSION,
+                "kind": kind.id,
+                "template": source,
+                "attributes": attributes,
+                "label": label if KIND_FIELD in template.fields else None,
+            },
+            sort_keys=True,
+            separators=(",", ":"),
+        ).encode("utf-8")
+    ).hexdigest()
+    return KindTemplate(kind.id, template, attributes, label, fingerprint)
+
+
+def _format(value: JsonValue) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, str):
+        return value
+    if isinstance(value, int | float):
+        return str(value)
+    if isinstance(value, list):
+        return ", ".join(text for text in (_format(item) for item in value) if text)
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)

lorebase_core/model/ids.py

@@ -0,0 +1,78 @@
+"""Stable ids (design 5.1, AC-01): prefixed ULIDs such as ``knd_01JB8Z...``."""
+
+from __future__ import annotations
+
+import secrets
+import time
+from enum import StrEnum
+from typing import Annotated
+
+from pydantic import StringConstraints
+
+CROCKFORD_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+ULID_LENGTH = 26
+ULID_REGEX = "[0-7][0-9A-HJKMNP-TV-Z]{25}"
+_TIMESTAMP_BITS = 48
+_RANDOM_BYTES = 10
+
+
+class IdPrefix(StrEnum):
+    KIND = "knd"
+    ATTRIBUTE = "att"
+    RELATION = "rel"
+    TRAIT = "trt"
+    OPERATION = "opr"
+    DERIVATION = "drv"
+    SOURCE = "src"
+    PROVENANCE = "prv"
+
+
+KindId = Annotated[str, StringConstraints(pattern=rf"^knd_{ULID_REGEX}$")]
+AttributeId = Annotated[str, StringConstraints(pattern=rf"^att_{ULID_REGEX}$")]
+RelationId = Annotated[str, StringConstraints(pattern=rf"^rel_{ULID_REGEX}$")]
+TraitId = Annotated[str, StringConstraints(pattern=rf"^trt_{ULID_REGEX}$")]
+OperationId = Annotated[str, StringConstraints(pattern=rf"^opr_{ULID_REGEX}$")]
+DerivationId = Annotated[str, StringConstraints(pattern=rf"^drv_{ULID_REGEX}$")]
+SourceId = Annotated[str, StringConstraints(pattern=rf"^src_{ULID_REGEX}$")]
+ProvenanceId = Annotated[str, StringConstraints(pattern=rf"^prv_{ULID_REGEX}$")]
+
+
+def new_ulid(*, timestamp_ms: int | None = None, randomness: bytes | None = None) -> str:
+    """Mint a ULID. Pass ``timestamp_ms`` and ``randomness`` (10 bytes) for deterministic ids."""
+    millis = time.time_ns() // 1_000_000 if timestamp_ms is None else timestamp_ms
+    if not 0 <= millis < 2**_TIMESTAMP_BITS:
+        raise ValueError("ULID timestamp must fit in 48 bits")
+    entropy = secrets.token_bytes(_RANDOM_BYTES) if randomness is None else randomness
+    if len(entropy) != _RANDOM_BYTES:
+        raise ValueError(f"ULID randomness must be {_RANDOM_BYTES} bytes")
+    number = (millis << (8 * _RANDOM_BYTES)) | int.from_bytes(entropy, "big")
+    chars = []
+    for _ in range(ULID_LENGTH):
+        number, digit = divmod(number, 32)
+        chars.append(CROCKFORD_ALPHABET[digit])
+    return "".join(reversed(chars))
+
+
+def new_id(
+    prefix: IdPrefix, *, timestamp_ms: int | None = None, randomness: bytes | None = None
+) -> str:
+    """Mint a prefixed id, for example ``new_id(IdPrefix.KIND)`` -> ``knd_01JB8Z...``."""
+    return f"{prefix.value}_{new_ulid(timestamp_ms=timestamp_ms, randomness=randomness)}"
+
+
+def is_ulid(text: str) -> bool:
+    return (
+        len(text) == ULID_LENGTH
+        and text[0] in CROCKFORD_ALPHABET[:8]
+        and all(ch in CROCKFORD_ALPHABET for ch in text)
+    )
+
+
+def id_prefix(element_id: str) -> IdPrefix | None:
+    """The prefix of a well-formed id, or ``None``."""
+    prefix, _, rest = element_id.partition("_")
+    try:
+        parsed = IdPrefix(prefix)
+    except ValueError:
+        return None
+    return parsed if is_ulid(rest) else None

lorebase_core/model/kind.py

@@ -0,0 +1,85 @@
+"""Kind (design 5.3): a type of thing in a Domain."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import ClassVar, Self
+
+from pydantic import Field, model_validator
+
+from lorebase_core.model.attribute import Attribute
+from lorebase_core.model.base import ElementStatus, Model, SchemaElement
+from lorebase_core.model.ids import AttributeId, KindId, TraitId
+
+
+class BuiltinOperation(StrEnum):
+    """Built-in Operations a Kind can enable (design 5.7)."""
+
+    CREATE = "create"
+    UPDATE = "update"
+    DELETE = "delete"
+
+
+class TraitImplementation(Model):
+    """A Kind implementing a Trait.
+
+    ``attribute_map`` maps Trait Attribute ids to Kind Attribute ids.
+    """
+
+    trait_id: TraitId
+    attribute_map: dict[AttributeId, AttributeId] = Field(default_factory=dict)
+
+
+class KindSearch(Model):
+    """Search settings of a Kind. Full-text fields come from searchable Attributes.
+
+    ``embedding_template`` opts the Kind into semantic search: Scribe embeds the text it
+    renders for each Record (``lorebase_core.model.embedding``). ``embedding_sensitive`` lets
+    the template name sensitive Attributes; unset, they are refused.
+    """
+
+    embedding_template: str | None = None
+    embedding_sensitive: bool | None = None
+
+
+def check_attribute_keys(attributes: dict[str, Attribute], owner: str) -> None:
+    for key, attribute in attributes.items():
+        if key != attribute.id:
+            raise ValueError(
+                f"{owner}: attribute map key {key} does not match attribute id {attribute.id}"
+            )
+
+
+class Kind(SchemaElement):
+    object_type: ClassVar[str] = "schema.kind"
+
+    id: KindId
+    primary_key: tuple[AttributeId, ...] = ()
+    title_attribute: AttributeId | None = None
+    attributes: dict[AttributeId, Attribute] = Field(default_factory=dict)
+    traits: tuple[TraitImplementation, ...] = ()
+    status: ElementStatus = ElementStatus.ACTIVE
+    search: KindSearch = Field(default_factory=KindSearch)
+    icon: str | None = None
+    color: str | None = None
+    operations_builtin: tuple[BuiltinOperation, ...] = (
+        BuiltinOperation.CREATE,
+        BuiltinOperation.UPDATE,
+        BuiltinOperation.DELETE,
+    )
+
+    @model_validator(mode="after")
+    def _check_keys(self) -> Self:
+        check_attribute_keys(self.attributes, f"Kind {self.id}")
+        return self
+
+    @property
+    def has_natural_key(self) -> bool:
+        """True when Records are keyed by primary key values, False when keys are minted ULIDs."""
+        return bool(self.primary_key)
+
+    def attribute_by_api_name(self, api_name: str) -> Attribute | None:
+        for attribute in self.attributes.values():
+            if attribute.api_name == api_name:
+                return attribute
+        return None