lorebase-keeper 0.2.3__py3-none-any.whl

lorebase_keeper/services/schema_suggestions.py

@@ -0,0 +1,418 @@
+"""Review of schema suggestions (design 9.2 step 10).
+
+Extraction jobs aggregate what they saw outside the schema into `schema_suggestions`
+(`lorebase_scribe.extraction.discovery`). Reviewers list them, reject them, or accept a set of
+them: the accepted suggestions become one schema change set (`suggestion_patch`) committed on a
+new branch (`keeper.schema_suggestions.branch_prefix` and the first suggestion's id) and
+proposed into the default branch as a Proposal. Nothing reaches the default branch until that
+Proposal merges; new Kinds and Relations are drafts for a modeler to describe and activate.
+
+Listings compare each suggestion with the default branch's schema of now: one that duplicates
+an element meanwhile is flagged (`duplicate_of`) and cannot be accepted, and one that
+duplicates a sensitive Attribute the caller may not read is listed without examples and
+samples.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from lorebase_core.infer.suggestions import (
+    PROPOSED,
+    AttributeGuess,
+    ExistingMatch,
+    Suggestion,
+    SuggestionError,
+    SuggestionOptions,
+    SuggestionType,
+    existing_match,
+    infer_attribute,
+    sensitive_attributes,
+    suggestion_patch,
+)
+from lorebase_core.model import Kind, Schema
+from lorebase_keeper.auth.principals import Principal
+from lorebase_keeper.context import StoreAccess, require_domain
+from lorebase_keeper.errors import CONFLICT, NOT_FOUND, VALIDATION_FAILED, ProblemError
+from lorebase_keeper.reading.sensitivity import Masking
+from lorebase_keeper.services import branches as branch_service
+from lorebase_keeper.services import proposals as proposal_service
+from lorebase_keeper.services import schema as schema_service
+from lorebase_keeper.services.config import (
+    BranchSettings,
+    ProposalSettings,
+    SchemaSuggestionSettings,
+)
+from lorebase_keeper.services.errors import Finding, violations_problem
+from lorebase_store.commit.schema import load_compiled
+from lorebase_store.repositories import (
+    MaterializedRepository,
+    MergeOutcome,
+    SchemaSuggestionRepository,
+    SuggestionRow,
+)
+from lorebase_store.repositories.suggestions import ACCEPTED, PENDING, REJECTED
+
+DECISIONS = (PENDING, ACCEPTED, REJECTED)
+ISSUES_LISTED = 50
+
+
+@dataclass(frozen=True)
+class SuggestionView:
+    """A stored suggestion as the caller may see it, against the current schema."""
+
+    row: SuggestionRow
+    kind: str | None
+    target: str | None
+    guess: AttributeGuess | None
+    duplicate: ExistingMatch | None
+    withheld: bool
+
+
+@dataclass(frozen=True)
+class SuggestionPage:
+    views: list[SuggestionView]
+    next_position: dict[str, Any] | None
+
+
+@dataclass(frozen=True)
+class DecisionInput:
+    suggestion: str
+    decision: str
+    name: str | None = None
+
+
+@dataclass
+class DecisionOutcome:
+    views: list[SuggestionView]
+    proposal_id: str | None = None
+    proposal_title: str | None = None
+    branch: str | None = None
+    changes: list[schema_service.ElementChange] = field(default_factory=list)
+
+
+def invalid(pointer: str, message: str, **extra: Any) -> ProblemError:
+    return ProblemError(
+        VALIDATION_FAILED,
+        "One or more request members are invalid.",
+        extensions={"errors": [{"pointer": pointer, "message": message, **extra}]},
+    )
+
+
+async def default_schema(store: StoreAccess, domain_id: str) -> tuple[str, Schema]:
+    """The default branch's name and schema."""
+    async with store.sessions() as session, session.begin():
+        domain = await require_domain(session, domain_id)
+        compiled = await MaterializedRepository(
+            session, domain_id, domain.default_branch, writes=store.runtime.settings.writes
+        ).schema()
+    if compiled is None:
+        raise ProblemError(NOT_FOUND, f"Branch {domain.default_branch!r} is not materialized.")
+    return domain.default_branch, load_compiled(compiled.compiled)
+
+
+def _kind_label(schema: Schema, ref: str | None) -> str | None:
+    if ref is None:
+        return None
+    if ref.startswith(PROPOSED):
+        return ref
+    kind = schema.kinds.get(ref)
+    return ref if kind is None else kind.api_name
+
+
+def view_of(
+    row: SuggestionRow,
+    schema: Schema,
+    options: SuggestionOptions,
+    masking: Masking | None,
+) -> SuggestionView:
+    """What a listing shows of `row` against `schema` for a caller with `masking`."""
+    type_ = SuggestionType(row.suggestion_type)
+    duplicate = existing_match(
+        schema,
+        type_,
+        row.name,
+        kind=row.kind_ref,
+        target=row.target_ref,
+        options=options,
+        tags=sensitive_attributes(schema),
+    )
+    withheld = False
+    if duplicate is not None and duplicate.sensitive:
+        owner = schema.attribute_owner(duplicate.id)
+        attribute = owner.attributes.get(duplicate.id) if isinstance(owner, Kind) else None
+        kinds = (owner.id,) if isinstance(owner, Kind) else ()
+        withheld = masking is None or attribute is None or masking.hidden(attribute, kinds)
+    guess = None
+    if type_ is SuggestionType.ATTRIBUTE and row.samples and not withheld:
+        guess = infer_attribute(row.name, row.samples, options)
+    return SuggestionView(
+        row=row,
+        kind=_kind_label(schema, row.kind_ref),
+        target=_kind_label(schema, row.target_ref),
+        guess=guess,
+        duplicate=duplicate,
+        withheld=withheld,
+    )
+
+
+async def page(
+    store: StoreAccess,
+    domain_id: str,
+    masking: Masking | None,
+    *,
+    statuses: Sequence[str],
+    types: Sequence[str],
+    all_support: bool,
+    limit: int,
+    position: dict[str, Any] | None,
+) -> SuggestionPage:
+    """Suggestions by support, the most supported first: pending ones need
+    `schema.suggestions.min_support` (pending Kinds also `kind_min_attributes`), unless
+    `all_support`, and at most `max_suggestions` are listed in all."""
+    options = store.runtime.schema.suggestions
+    _, schema = await default_schema(store, domain_id)
+    listed = 0
+    after = None
+    if position is not None:
+        try:
+            after = (int(position["support"]), str(position["id"]))
+            listed = int(position["listed"])
+        except (KeyError, TypeError, ValueError):
+            raise ProblemError(
+                VALIDATION_FAILED, "The cursor is not a suggestion cursor."
+            ) from None
+    room = options.max_suggestions - listed
+    if room <= 0:
+        return SuggestionPage([], None)
+    take = min(limit, room)
+    async with store.sessions() as session, session.begin():
+        await require_domain(session, domain_id)
+        rows = await SchemaSuggestionRepository(session, domain_id).page(
+            statuses=statuses,
+            types=types,
+            min_support=1 if all_support else options.min_support,
+            kind_min_attributes=0 if all_support else options.kind_min_attributes,
+            limit=take + 1,
+            after=after,
+        )
+    more = len(rows) > take
+    rows = rows[:take]
+    following = None
+    if more and rows and listed + len(rows) < options.max_suggestions:
+        last = rows[-1]
+        following = {"support": last.support, "id": last.id, "listed": listed + len(rows)}
+    return SuggestionPage([view_of(r, schema, options, masking) for r in rows], following)
+
+
+async def merge_variants(store: StoreAccess, domain_id: str) -> MergeOutcome:
+    """Fold the pending Attribute suggestions already stored that are variants of one another
+    (`schema.suggestions.merge_similarity`) into the best supported one of each set."""
+    async with store.sessions() as session, session.begin():
+        await require_domain(session, domain_id)
+        return await SchemaSuggestionRepository(session, domain_id).merge_variants(
+            options=store.runtime.schema.suggestions, now=store.clock()
+        )
+
+
+async def decide(
+    store: StoreAccess,
+    domain_id: str,
+    decisions: Sequence[DecisionInput],
+    principal: Principal,
+    *,
+    settings: SchemaSuggestionSettings,
+    branches: BranchSettings,
+    proposals: ProposalSettings,
+    masking: Masking | None,
+) -> DecisionOutcome:
+    """Reject, reset or accept suggestions. Accepted ones are applied together on a new branch
+    and proposed into the default branch; nothing is decided when that fails."""
+    if len(decisions) > settings.max_decisions:
+        raise invalid("/body/decisions", f"at most {settings.max_decisions} per request")
+    ids = [d.suggestion for d in decisions]
+    if len(set(ids)) != len(ids):
+        raise invalid("/body/decisions", "a suggestion is named twice")
+    options = store.runtime.schema.suggestions
+    default_branch, schema = await default_schema(store, domain_id)
+    async with store.sessions() as session, session.begin():
+        await require_domain(session, domain_id)
+        rows = await SchemaSuggestionRepository(session, domain_id).get_many(ids)
+    for index, decision in enumerate(decisions):
+        row = rows.get(decision.suggestion)
+        if row is None:
+            raise ProblemError(NOT_FOUND, f"Suggestion {decision.suggestion!r} not found.")
+        if row.status == ACCEPTED:
+            raise ProblemError(
+                CONFLICT,
+                f"Suggestion {row.id!r} was accepted into Proposal {row.proposal_id!r}.",
+                extensions={"proposal": row.proposal_id},
+            )
+        if decision.name is not None and not decision.name.strip():
+            raise invalid(f"/body/decisions/{index}/name", "must not be empty")
+    accepted = [d for d in decisions if d.decision == ACCEPTED]
+    outcome = DecisionOutcome(views=[])
+    now = store.clock()
+    if accepted:
+        await _accept(
+            store,
+            domain_id,
+            [(d, rows[d.suggestion]) for d in accepted],
+            schema,
+            default_branch,
+            principal,
+            options,
+            settings,
+            branches,
+            proposals,
+            outcome,
+            masking,
+        )
+    async with store.sessions() as session, session.begin():
+        await require_domain(session, domain_id)
+        repository = SchemaSuggestionRepository(session, domain_id)
+        if accepted:
+            await repository.decide(
+                [d.suggestion for d in accepted],
+                status=ACCEPTED,
+                decided_by=principal.id,
+                decided_at=now,
+                proposal_id=outcome.proposal_id,
+                branch=outcome.branch,
+            )
+        for status in (REJECTED, PENDING):
+            chosen = [d.suggestion for d in decisions if d.decision == status]
+            await repository.decide(
+                chosen,
+                status=status,
+                decided_by=principal.id if status == REJECTED else None,
+                decided_at=now if status == REJECTED else None,
+            )
+        stored = await repository.get_many(ids)
+    outcome.views = [view_of(stored[i], schema, options, masking) for i in ids]
+    return outcome
+
+
+def _patch(
+    rows: Sequence[tuple[DecisionInput, SuggestionRow]],
+    schema: Schema,
+    options: SuggestionOptions,
+) -> dict[str, Any]:
+    suggestions: list[Suggestion] = []
+    for index, (decision, row) in enumerate(rows):
+        type_ = SuggestionType(row.suggestion_type)
+        duplicate = existing_match(
+            schema, type_, row.name, kind=row.kind_ref, target=row.target_ref, options=options
+        )
+        if duplicate is not None:
+            raise invalid(
+                f"/body/decisions/{index}",
+                f"the schema has {duplicate.api_name!r} ({duplicate.section}) already",
+                suggestion=row.id,
+            )
+        suggestions.append(
+            Suggestion(
+                id=row.id,
+                type=type_,
+                name=decision.name.strip() if decision.name else row.name,
+                kind=row.kind_ref,
+                target=row.target_ref,
+                samples=tuple(row.samples),
+                labels=tuple(row.labels),
+            )
+        )
+    try:
+        return suggestion_patch(schema, suggestions, options)
+    except SuggestionError as error:
+        position = next(i for i, (_, row) in enumerate(rows) if row.id == error.suggestion)
+        raise invalid(
+            f"/body/decisions/{position}", str(error), suggestion=error.suggestion
+        ) from None
+
+
+def _issues_problem(issues: Sequence[Finding]) -> ProblemError:
+    return violations_problem(issues, limit=ISSUES_LISTED)
+
+
+async def _accept(
+    store: StoreAccess,
+    domain_id: str,
+    rows: Sequence[tuple[DecisionInput, SuggestionRow]],
+    schema: Schema,
+    default_branch: str,
+    principal: Principal,
+    options: SuggestionOptions,
+    settings: SchemaSuggestionSettings,
+    branches: BranchSettings,
+    proposals: ProposalSettings,
+    outcome: DecisionOutcome,
+    masking: Masking | None,
+) -> None:
+    change = schema_service.ChangeSetInput(patch=_patch(rows, schema, options))
+    try:
+        target = schema_service.target_schema(schema, change)
+    except schema_service.ChangeSetInvalidError as error:
+        raise _issues_problem(error.issues) from None
+    issues = schema_service.static_issues(target, store.runtime.schema.rules())
+    if issues:
+        raise _issues_problem(issues)
+    branch = f"{settings.branch_prefix}{rows[0][1].id}"
+    branch_service.check_ref_name(branch, branches, "/body/decisions")
+    await branch_service.create(
+        store,
+        domain_id,
+        branch,
+        principal,
+        branches,
+        source=default_branch,
+        revision=None,
+        idempotency_key=None,
+    )
+    names = ", ".join(row.name for _, row in rows)
+    try:
+        applied = await schema_service.apply(
+            store,
+            domain_id,
+            branch,
+            change,
+            principal,
+            message=f"Schema suggestions: {names}",
+            expected_head=None,
+            dry_run=False,
+            cascade=False,
+            clearance=None if masking is None else masking.clearance,
+        )
+        created = await proposal_service.create(
+            store,
+            domain_id,
+            proposal_service.NewProposal(
+                source_ref=branch,
+                target_ref=default_branch,
+                title=f"Schema suggestions: {names}"[:200],
+                description="Suggested by document extraction: "
+                + ", ".join(f"{row.suggestion_type} {row.name} ({row.id})" for _, row in rows),
+            ),
+            principal,
+            proposals,
+            idempotency_key=None,
+        )
+    except schema_service.ChangeSetInvalidError as error:
+        await _drop(store, domain_id, branch, branches)
+        raise _issues_problem(error.issues) from None
+    except Exception:
+        await _drop(store, domain_id, branch, branches)
+        raise
+    outcome.branch = branch
+    outcome.changes = applied.changes
+    outcome.proposal_id = created.view.proposal.id
+    outcome.proposal_title = created.view.proposal.title
+
+
+async def _drop(store: StoreAccess, domain_id: str, branch: str, settings: BranchSettings) -> None:
+    """Delete the branch of a change set that could not be proposed (best effort)."""
+    try:
+        await branch_service.delete(store, domain_id, branch, settings)
+    except ProblemError:
+        return

lorebase_keeper/services/source_preview.py

@@ -0,0 +1,127 @@
+"""Preview of the expression mappings of a candidate Source (design 5.9) on sample rows.
+
+The candidate is a schema change set naming the Source: the schema it produces is validated
+like a schema change, then the `expression` of every Attribute mapping of the Source is
+evaluated on each row the caller sends, with the engine Scribe maps rows with
+(`lorebase_core.expr.rows`, within `schema.expression_limits`). The answer holds the value of
+each expression before the mapping's transforms; transforms and the coercion to the Attribute
+type are the mapper's (`lorebase_scribe.sources.mapping`, which Keeper does not import).
+
+The rows are the caller's own input and the answer is made of them: no Record is read, nothing is
+stored and no connection is opened, so reading the schema is the only permission involved (a
+grant narrowed to a Kind is enough when it names the Source's target Kind). Expressions are
+compiled once over every column the rows have; a column a row lacks is null, as in Scribe.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import datetime
+
+from pydantic import JsonValue
+
+from lorebase_core.expr import ExprError, ExprLimits, RowExpressions, row_value
+from lorebase_core.model import Schema, Source
+from lorebase_core.validate import SchemaRules
+from lorebase_keeper.services.derivation_preview import CandidateInvalidError
+from lorebase_keeper.services.errors import Finding
+from lorebase_keeper.services.schema import (
+    ChangeSetInput,
+    ChangeSetInvalidError,
+    static_issues,
+    target_schema,
+)
+
+SOURCE_NOT_FOUND = "source.not_found"
+
+
+@dataclass(frozen=True)
+class ExpressionCell:
+    """What one expression gives for one row: the value, or why there is none."""
+
+    value: JsonValue
+    error_code: str | None = None
+    error_message: str | None = None
+
+
+@dataclass(frozen=True)
+class SourceCandidate:
+    """A validated candidate: the schema the change set asks for and the Source in it."""
+
+    schema: Schema
+    source: Source
+
+
+def candidate_of(
+    current: Schema,
+    change: ChangeSetInput,
+    name: str,
+    rules: SchemaRules,
+    *,
+    authorize: Callable[[Schema, Source | None], None] | None = None,
+) -> SourceCandidate:
+    """The Source `name` of the schema `change` asks for, once the schema is valid.
+
+    `authorize` is called with the schema the change set produces and the Source in it (None
+    when there is none) before the schema is validated, so a caller who may not read the
+    Source's target Kind is refused before any issue is reported. Raises
+    `CandidateInvalidError` when the change set cannot be read, fails static validation or
+    `name` is not a Source of it.
+    """
+    try:
+        schema = target_schema(current, change)
+    except ChangeSetInvalidError as invalid:
+        raise CandidateInvalidError(invalid.issues) from None
+    source = schema.source_by_api_name(name)
+    if authorize is not None:
+        authorize(schema, source)
+    issues = static_issues(schema, rules)
+    if issues:
+        raise CandidateInvalidError(issues)
+    if source is None:
+        raise CandidateInvalidError(
+            [Finding(SOURCE_NOT_FOUND, f"No Source {name!r} in the schema.", ("sources",))]
+        )
+    return SourceCandidate(schema, source)
+
+
+def expression_attributes(candidate: SourceCandidate) -> dict[str, str]:
+    """Api name of each Attribute mapped by an expression, with the expression."""
+    schema, source = candidate.schema, candidate.source
+    kind = schema.kinds[source.target_kind]
+    return {
+        kind.attributes[attribute_id].api_name: mapping.expression
+        for attribute_id, mapping in source.attribute_mapping.items()
+        if mapping.expression is not None and attribute_id in kind.attributes
+    }
+
+
+def evaluate(
+    candidate: SourceCandidate,
+    rows: Sequence[Mapping[str, JsonValue]],
+    *,
+    limits: ExprLimits,
+    now: datetime,
+) -> list[dict[str, ExpressionCell]]:
+    """The value of every expression mapping of the Source for each row, in row order.
+
+    Expressions are compiled once over every column the rows have, as Scribe compiles them
+    over the columns of the Source: a column a row lacks is null."""
+    expressions = expression_attributes(candidate)
+    columns: dict[str, None] = {}
+    for row in rows:
+        columns.update(dict.fromkeys(row))
+    programs = RowExpressions(candidate.schema, columns, limits)
+    cells: list[dict[str, ExpressionCell]] = []
+    for row in rows:
+        found: dict[str, ExpressionCell] = {}
+        for attribute, expression in expressions.items():
+            try:
+                found[attribute] = ExpressionCell(
+                    row_value(programs.evaluate(expression, row, now=now))
+                )
+            except ExprError as error:
+                found[attribute] = ExpressionCell(None, error.code.value, error.describe())
+        cells.append(found)
+    return cells