lorebase-core 0.2.3__py3-none-any.whl

lorebase_core/expr/values.py

@@ -0,0 +1,169 @@
+"""Runtime values of evaluation and their conversion from and to canonical values.
+
+``int`` is int, ``double`` float, ``decimal`` Decimal, ``timestamp`` an aware datetime in
+UTC, ``date`` date, ``duration`` timedelta, lists are lists and maps dicts. Schema values
+use the classes below. An absent Attribute reads as ``None`` (null).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import date, datetime, timedelta
+from decimal import Decimal
+
+from pydantic import JsonValue
+
+from lorebase_core.expr.errors import ExprError, ExprErrorCode, evaluation_error
+from lorebase_core.model.kind import Kind
+from lorebase_core.model.record import format_ref
+from lorebase_core.types import (
+    AttributeType,
+    EncodingError,
+    GeoPoint,
+    TypeSpec,
+    decode_value,
+    encode_value,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Ref:
+    """A Record reference: Kind id and key."""
+
+    kind: str
+    key: str
+
+
+@dataclass(frozen=True, slots=True)
+class ObjectValue:
+    """A value of an object type (``params``, ``principal``)."""
+
+    name: str
+    fields: Mapping[str, object]
+
+
+@dataclass(eq=False, slots=True)
+class RecordValue:
+    """A loaded Record. Attribute values are decoded on first access."""
+
+    kind: Kind
+    key: str
+    values: Mapping[str, JsonValue]
+    _decoded: dict[str, object] = field(default_factory=dict)
+
+    @property
+    def ref(self) -> Ref:
+        return Ref(self.kind.id, self.key)
+
+    def has(self, name: str) -> bool:
+        attribute = self.kind.attribute_by_api_name(name)
+        return attribute is not None and self.values.get(attribute.id) is not None
+
+    def get(self, name: str) -> object:
+        """Decoded value of the Attribute called ``name`` (``None`` when absent)."""
+        if name in self._decoded:
+            return self._decoded[name]
+        attribute = self.kind.attribute_by_api_name(name)
+        if attribute is None:
+            raise evaluation_error(f"Kind {self.kind.api_name} has no attribute {name!r}")
+        value = to_runtime(attribute.type, self.values.get(attribute.id))
+        self._decoded[name] = value
+        return value
+
+    def __repr__(self) -> str:
+        return f"RecordValue({format_ref(self.kind.api_name, self.key)})"
+
+
+@dataclass(eq=False, slots=True)
+class LinksValue:
+    """``links`` of the subject Record; fields are resolved lazily by the evaluator."""
+
+    kind: Kind
+    key: str
+
+
+def to_runtime(spec: TypeSpec, canonical: object) -> object:
+    """Runtime value of a canonical value of ``spec`` (``None`` stays ``None``)."""
+    if canonical is None:
+        return None
+    try:
+        value = decode_value(spec, canonical)
+    except EncodingError as error:
+        raise evaluation_error(f"stored value is invalid: {error.message}") from None
+    return _wrap(spec, value)
+
+
+def _wrap(spec: TypeSpec, value: object) -> object:
+    if spec.name is AttributeType.REF:
+        assert spec.kind_ref is not None
+        assert isinstance(value, str)
+        return Ref(spec.kind_ref, value)
+    if spec.name is AttributeType.ARRAY and isinstance(value, list):
+        assert spec.element is not None
+        element = spec.element
+        return [_wrap(element, item) for item in value]
+    return value
+
+
+def to_raw(value: object) -> object:
+    """A runtime value as input for ``encode_value`` (Record references become keys)."""
+    if isinstance(value, Ref):
+        return value.key
+    if isinstance(value, RecordValue):
+        return value.key
+    if isinstance(value, list):
+        return [to_raw(item) for item in value]
+    if isinstance(value, dict):
+        return {str(k): to_raw(v) for k, v in value.items()}
+    if isinstance(value, ObjectValue):
+        return {k: to_raw(v) for k, v in value.fields.items()}
+    return value
+
+
+def encode_result(spec: TypeSpec, value: object) -> JsonValue:
+    """Encode an evaluation result for an Attribute of type ``spec``.
+
+    Raises ``ExprError`` with code ``expression.value_invalid`` when the value does not fit.
+    """
+    if isinstance(value, Ref) and spec.name is AttributeType.REF and value.kind != spec.kind_ref:
+        raise ExprError(ExprErrorCode.VALUE_INVALID, "reference to a Record of another Kind")
+    try:
+        return encode_value(spec, to_raw(value))
+    except EncodingError as error:
+        raise ExprError(ExprErrorCode.VALUE_INVALID, error.message) from None
+
+
+def type_name(value: object) -> str:
+    """CEL name of the type of a runtime value, for error messages."""
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "bool"
+    if isinstance(value, int):
+        return "int"
+    if isinstance(value, float):
+        return "double"
+    if isinstance(value, Decimal):
+        return "decimal"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, bytes):
+        return "bytes"
+    if isinstance(value, datetime):
+        return "timestamp"
+    if isinstance(value, date):
+        return "date"
+    if isinstance(value, timedelta):
+        return "duration"
+    if isinstance(value, list):
+        return "list"
+    if isinstance(value, dict):
+        return "map"
+    if isinstance(value, Ref):
+        return "ref"
+    if isinstance(value, RecordValue):
+        return "record"
+    if isinstance(value, GeoPoint):
+        return "geo_point"
+    return "object"

lorebase_core/expr/visibility.py

@@ -0,0 +1,363 @@
+"""What a reader limited to some Kinds sees of a schema (design 13.3, Kind-scoped invisibility).
+
+`visible_schema(schema, kinds)` is the schema as a caller whose grants are narrowed to the
+Kinds `kinds` (api names) sees it: the other Kinds are not there at all, and neither is
+anything that would tell about them. Reads resolve names against this schema, so a name the
+caller may not read fails exactly like a name that does not exist.
+
+Left out, besides the invisible Kinds:
+
+- Relations with an invisible end, and path Derivations with a step left out;
+- Derivations computing an Attribute of an invisible Kind, aggregating over a Relation left
+  out, or reading (an expression's `links.<name>` fields, an aggregate's source Attribute and
+  filter) anything left out, transitively; the derived Attributes they compute go with them;
+- Attributes of type `ref` (or arrays of `ref`) to an invisible Kind;
+- Trait implementations mapping an Attribute left out, and Traits implemented by nothing left
+  in, or declaring a reference to an invisible Kind;
+- Operations whose parameters, steps or expressions name anything left out;
+- Sources writing an invisible Kind or a Relation or Attribute left out;
+- Kinds keyed by a reference to an invisible Kind, and Kinds whose titles, full-text index
+  or embeddings come from an Attribute left out (a derived searchable or title Attribute
+  counting Records of an invisible Kind): their keys, stored titles and search vectors would
+  tell about it, so they are invisible too.
+
+Elements are conservative supersets: an element that may tell about an invisible Kind is left
+out, even when it would not in practice.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Collection
+
+from lorebase_core.expr.ast import Ident, Node, Select, parse
+from lorebase_core.expr.derivations import _names, _nodes, derivation_reads
+from lorebase_core.expr.environment import LINKS, link_fields
+from lorebase_core.expr.errors import ExprError
+from lorebase_core.expr.limits import ExprLimits
+from lorebase_core.model.attribute import Attribute
+from lorebase_core.model.derivation import (
+    AggregateDerivation,
+    Derivation,
+    ExpressionDerivation,
+    PathDerivation,
+)
+from lorebase_core.model.kind import Kind
+from lorebase_core.model.operation import CreateLinkStep, DeleteLinkStep, Operation
+from lorebase_core.model.relation import Relation
+from lorebase_core.model.schema import Schema
+from lorebase_core.model.source import Source
+from lorebase_core.model.trait import Trait
+from lorebase_core.types import TypeSpec
+
+_WORD = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+
+def visible_schema(
+    schema: Schema, kinds: Collection[str], limits: ExprLimits | None = None
+) -> Schema:
+    """`schema` as seen by a reader of the Kinds named `kinds` (api names) only."""
+    limits = limits or ExprLimits()
+    shown = {k.id for k in schema.kinds.values() if k.api_name in kinds}
+    if len(shown) == len(schema.kinds):
+        return schema
+    while True:
+        relations, paths, dropped, derivations = _project(schema, shown, limits)
+        # A Kind whose keys hold keys of a Kind left out, or whose titles, full-text index or
+        # embeddings are computed from something left out, would tell about it through them:
+        # it is left out too.
+        tainted = {
+            kind_id
+            for kind_id in shown
+            if _indexes_hidden(schema.kinds[kind_id], dropped)
+            or _keys_hidden(schema.kinds[kind_id], shown)
+        }
+        if not tainted:
+            break
+        shown -= tainted
+    sources = {
+        s.id for s in schema.sources.values() if _source_visible(s, shown, relations, dropped)
+    }
+    kept_kinds = {
+        kind_id: _kind(kind, dropped, sources)
+        for kind_id, kind in schema.kinds.items()
+        if kind_id in shown
+    }
+    kept_relations = {
+        rid: relation.model_copy(
+            update={"attributes": _attributes(relation.attributes, dropped, sources)}
+        )
+        for rid, relation in schema.relations.items()
+        if rid in relations
+    }
+    # An implementation mapping a Trait Attribute to an Attribute left out goes with it.
+    implementations = {
+        kind_id: tuple(
+            t for t in kind.traits if not any(a in dropped for a in t.attribute_map.values())
+        )
+        for kind_id, kind in kept_kinds.items()
+    }
+    implemented = {t.trait_id for found in implementations.values() for t in found}
+    traits = {
+        tid: trait
+        for tid, trait in schema.traits.items()
+        if (tid in implemented or not schema.kinds_implementing(tid))
+        and not any(_refs_hidden(a.type, shown) for a in trait.attributes.values())
+    }
+    kept_kinds = {
+        kind_id: kind.model_copy(
+            update={"traits": tuple(t for t in implementations[kind_id] if t.trait_id in traits)}
+        )
+        for kind_id, kind in kept_kinds.items()
+    }
+    hidden_words = _hidden_words(schema, shown, relations | paths, dropped)
+    operations = {
+        oid: operation
+        for oid, operation in schema.operations.items()
+        if _operation_visible(operation, shown, relations, dropped, hidden_words)
+    }
+    return schema.model_copy(
+        update={
+            "kinds": kept_kinds,
+            "relations": kept_relations,
+            "traits": traits,
+            "operations": operations,
+            "derivations": {
+                d: schema.derivations[d] for d in schema.derivations if d in derivations
+            },
+            "sources": {s: schema.sources[s] for s in schema.sources if s in sources},
+        }
+    )
+
+
+def _project(
+    schema: Schema, shown: Collection[str], limits: ExprLimits
+) -> tuple[set[str], set[str], set[str], set[str]]:
+    """Relations, path Derivations, left out Attributes and Derivations for the Kinds
+    `shown` (ids)."""
+    relations = {
+        r.id for r in schema.relations.values() if r.from_kind in shown and r.to_kind in shown
+    }
+    paths = {
+        path.id
+        for path in schema.path_relations().values()
+        if all(step.relation.id in relations for step in path.steps)
+    }
+    dropped = _hidden_attributes(schema, shown)
+    derivations = _visible_derivations(schema, shown, relations | paths, dropped, limits)
+    for kind in schema.kinds.values():
+        dropped |= {
+            a.id
+            for a in kind.attributes.values()
+            if a.derivation_id is not None and a.derivation_id not in derivations
+        }
+    return relations, paths, dropped, derivations
+
+
+def _indexes_hidden(kind: Kind, dropped: Collection[str]) -> bool:
+    """Whether a left out Attribute of `kind` feeds its titles, full-text index or
+    embeddings."""
+    template = kind.search.embedding_template
+    words = set(_WORD.findall(template)) if template else set()
+    return any(
+        a.id in dropped and (a.searchable or a.id == kind.title_attribute or a.api_name in words)
+        for a in kind.attributes.values()
+    )
+
+
+def _refs_hidden(spec: TypeSpec, shown: Collection[str]) -> bool:
+    """Whether a type holds keys of a Kind not in `shown`."""
+    if spec.kind_ref is not None and spec.kind_ref not in shown:
+        return True
+    return spec.element is not None and _refs_hidden(spec.element, shown)
+
+
+def _keys_hidden(kind: Kind, shown: Collection[str]) -> bool:
+    """Whether a primary key Attribute of `kind` holds keys of a Kind not in `shown`."""
+    return any(
+        (a := kind.attributes.get(attribute_id)) is not None and _refs_hidden(a.type, shown)
+        for attribute_id in kind.primary_key
+    )
+
+
+def _hidden_attributes(schema: Schema, shown: Collection[str]) -> set[str]:
+    """Stored Attributes holding keys of Kinds not in `shown`."""
+    owners: list[Kind | Relation] = [*schema.kinds.values(), *schema.relations.values()]
+    return {
+        a.id for owner in owners for a in owner.attributes.values() if _refs_hidden(a.type, shown)
+    }
+
+
+def _expression_links(
+    schema: Schema, kind_id: str, source: str, limits: ExprLimits
+) -> set[str] | None:
+    """Ids of the Relations (or path Derivations) an expression over a Record of `kind_id`
+    reads through `links.<name>`; None when it uses `links` otherwise (every one, then)."""
+    try:
+        root = parse(source, limits)
+    except ExprError:
+        return set()  # not evaluated: expression validation reports it
+    names = _names(root)
+    if LINKS in names.idents and _bare_links(root):
+        return None
+    fields = link_fields(schema, kind_id)
+    return {fields[name].relation.id for name in names.links if name in fields}
+
+
+def _bare_links(root: Node) -> bool:
+    """Whether `links` appears other than as `links.<name>`."""
+    selected = sum(
+        1
+        for node in _nodes(root)
+        if isinstance(node, Select)
+        and isinstance(node.operand, Ident)
+        and node.operand.name == LINKS
+    )
+    named = sum(1 for node in _nodes(root) if isinstance(node, Ident) and node.name == LINKS)
+    return named > selected
+
+
+def _derivation_visible(
+    schema: Schema,
+    derivation: Derivation,
+    shown: Collection[str],
+    links: Collection[str],
+    dropped: Collection[str],
+    limits: ExprLimits,
+) -> bool:
+    definition = derivation.definition
+    if isinstance(definition, PathDerivation):
+        return derivation.id in links
+    if definition.kind not in shown:
+        return False
+    if isinstance(definition, AggregateDerivation):
+        if definition.relation not in links:
+            return False
+        relation = schema.link_relation(definition.relation)
+        if relation is None:
+            return False
+        if definition.filter is not None:
+            far = relation.far_endpoint(definition.direction)
+            used = _expression_links(schema, far, definition.filter, limits)
+            if used is None or not used <= set(links):
+                return False
+    elif isinstance(definition, ExpressionDerivation):
+        used = _expression_links(schema, definition.kind, definition.expression, limits)
+        if used is None or not used <= set(links):
+            return False
+    reads = derivation_reads(schema, derivation, limits)
+    return all(kind in shown and attribute not in dropped for kind, attribute in reads)
+
+
+def _visible_derivations(
+    schema: Schema,
+    shown: Collection[str],
+    links: Collection[str],
+    dropped: set[str],
+    limits: ExprLimits,
+) -> set[str]:
+    """Ids of the Derivations left in; `dropped` gains the derived Attributes left out."""
+    visible = set(schema.derivations)
+    changed = True
+    while changed:
+        changed = False
+        for derivation_id in sorted(visible):
+            derivation = schema.derivations[derivation_id]
+            if _derivation_visible(schema, derivation, shown, links, dropped, limits):
+                continue
+            visible.discard(derivation_id)
+            target = derivation.target
+            if target is not None:
+                dropped.add(target[1])
+            changed = True
+    return visible
+
+
+def _attributes(
+    attributes: dict[str, Attribute], dropped: Collection[str], sources: Collection[str]
+) -> dict[str, Attribute]:
+    found: dict[str, Attribute] = {}
+    for attribute_id, attribute in attributes.items():
+        if attribute_id in dropped:
+            continue
+        if attribute.authority is not None and attribute.authority not in sources:
+            attribute = attribute.model_copy(update={"authority": None})
+        found[attribute_id] = attribute
+    return found
+
+
+def _kind(kind: Kind, dropped: Collection[str], sources: Collection[str]) -> Kind:
+    attributes = _attributes(kind.attributes, dropped, sources)
+    return kind.model_copy(update={"attributes": attributes})
+
+
+def _source_visible(
+    source: Source, shown: Collection[str], relations: Collection[str], dropped: Collection[str]
+) -> bool:
+    return (
+        source.target_kind in shown
+        and all(m.relation in relations for m in source.relation_mapping)
+        and not any(a in dropped for a in source.attribute_mapping)
+    )
+
+
+def _hidden_words(
+    schema: Schema, shown: Collection[str], links: Collection[str], dropped: Collection[str]
+) -> frozenset[str]:
+    """Names of what is left out (Kinds, Relations both ways, path Derivations, Attributes)."""
+    words: set[str] = {k.api_name for k in schema.kinds.values() if k.id not in shown}
+    for relation in schema.relations.values():
+        if relation.id not in links:
+            words |= {relation.api_name, relation.inverse_api_name}
+    for path in schema.path_relations().values():
+        if path.id not in links:
+            words |= {path.relation.api_name, path.relation.inverse_api_name}
+    for kind in schema.kinds.values():
+        words |= {a.api_name for a in kind.attributes.values() if a.id in dropped}
+    return frozenset(w for w in words if w)
+
+
+def _operation_visible(
+    operation: Operation,
+    shown: Collection[str],
+    relations: Collection[str],
+    dropped: Collection[str],
+    hidden_words: Collection[str],
+) -> bool:
+    if any(_refs_hidden(p.type, shown) for p in operation.parameters):
+        return False
+    expressions: list[str] = [p.expression for p in operation.preconditions]
+    if operation.review_policy.expression is not None:
+        expressions.append(operation.review_policy.expression)
+    for step in operation.steps:
+        if isinstance(step, CreateLinkStep | DeleteLinkStep):
+            if step.relation not in relations:
+                return False
+            expressions += [step.from_, step.to]
+        else:
+            if step.kind not in shown:
+                return False
+            target = getattr(step, "target", None)
+            if target is not None:
+                expressions.append(target)
+        values: dict[str, str] = getattr(step, "values", {})
+        if any(a in dropped for a in values):
+            return False
+        expressions += list(values.values())
+    return not any(set(_WORD.findall(e)) & set(hidden_words) for e in expressions)
+
+
+def element_ids(schema: Schema) -> set[str]:
+    """Ids of every schema element and of every Attribute of a Kind, Relation or Trait."""
+    ids = {element.id for element in schema.elements()}
+    owners: list[Kind | Relation | Trait] = [
+        *schema.kinds.values(),
+        *schema.relations.values(),
+        *schema.traits.values(),
+    ]
+    for owner in owners:
+        ids |= set(owner.attributes)
+    return ids
+
+
+__all__ = ["element_ids", "visible_schema"]

lorebase_core/infer/__init__.py

@@ -0,0 +1,36 @@
+"""Deterministic schema inference from structured sources (extraction tier 0, design 9.1).
+
+``infer_schema`` turns ``TableProfile`` objects (columns with sampled or complete values, row
+counts and whatever a catalog declares) into a ``SchemaProposal``: a draft ``Schema`` with Kinds,
+typed Attributes with constraints, primary keys and Relations with cardinality, the evidence
+behind each element and the issues a reviewer must settle. No model is called and nothing is
+read or written: profilers (CSV, database catalogs) live outside the core.
+
+Thresholds and naming rules are ``InferenceOptions``; see docs/inference.md.
+"""
+
+from lorebase_core.infer.engine import infer_schema
+from lorebase_core.infer.options import InferenceOptions
+from lorebase_core.infer.profile import ColumnProfile, DeclaredForeignKey, TableProfile
+from lorebase_core.infer.proposal import (
+    Evidence,
+    InferenceIssue,
+    InferenceIssueCode,
+    SchemaProposal,
+    TableOutcome,
+    TableRole,
+)
+
+__all__ = [
+    "ColumnProfile",
+    "DeclaredForeignKey",
+    "Evidence",
+    "InferenceIssue",
+    "InferenceIssueCode",
+    "InferenceOptions",
+    "SchemaProposal",
+    "TableOutcome",
+    "TableProfile",
+    "TableRole",
+    "infer_schema",
+]