lorebase-keeper 0.2.3__py3-none-any.whl

lorebase_keeper/kindhealth/models.py

@@ -0,0 +1,179 @@
+"""Response shapes of the health reports. Names are api names; no Attribute value is ever
+printed except the normalized value of a duplicate on an Attribute without sensitivity tags.
+For Attributes whose values the caller may not read, validation issues carry the generic
+`value.invalid` code and duplicate groups are only counted (`DuplicateReport.hidden`)."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from lorebase_core.model.relation import RelationBacking
+from lorebase_keeper.reading.views import Served
+
+
+class IssueGroup(BaseModel):
+    """Records failing the same rule of the current schema."""
+
+    code: str = Field(
+        description="Validation issue code, e.g. value.required or value.pattern; value.invalid "
+        "for an Attribute whose values the caller may not read."
+    )
+    attribute: str | None = Field(default=None, description="Attribute api name, when one.")
+    count: int
+    examples: list[str] = Field(
+        description="Keys of some of the Records; none for an Attribute whose values the "
+        "caller may not read."
+    )
+
+
+class ViolationReport(BaseModel):
+    """Result of re-validating Records with the schema of the branch head."""
+
+    checked: int = Field(description="Records re-validated.")
+    sampled: bool = Field(description="Only a subset of the Kind's Records was re-validated.")
+    invalid_records: int = Field(description="Re-validated Records with at least one issue.")
+    issues: list[IssueGroup]
+    issues_truncated: bool = False
+
+
+class DuplicateGroup(BaseModel):
+    """Records whose value of one Attribute is the same once case and spacing are ignored."""
+
+    attribute: str
+    unique: bool = Field(description="The Attribute is unique: this is a violation, not a hint.")
+    value: str | None = Field(
+        description="The normalized value; null for Attributes with sensitivity tags."
+    )
+    count: int
+    keys: list[str]
+
+
+class HiddenDuplicates(BaseModel):
+    """Duplicates on an Attribute whose values the caller may not read: counted, not listed
+    (which Records share a value would tell that their values are equal)."""
+
+    attribute: str
+    unique: bool = Field(description="The Attribute is unique: this is a violation, not a hint.")
+    groups: int = Field(description="Groups of Records sharing a value.")
+    records: int = Field(description="Records in those groups.")
+
+
+class DuplicateReport(BaseModel):
+    """Normalized exact duplicates on unique and title Attributes."""
+
+    attributes: list[str] = Field(description="Attributes checked.")
+    groups: list[DuplicateGroup]
+    total_groups: int = Field(description="Groups found, listed or not (`hidden` included).")
+    skipped: Literal["too_many_records", "disabled"] | None = None
+    hidden: list[HiddenDuplicates] = Field(
+        default_factory=list,
+        description="Attributes with duplicates whose groups are not listed: the caller is "
+        "not cleared for their values.",
+    )
+
+
+class DerivedReport(BaseModel):
+    """Derived Attributes whose materialized value is missing, so reads cannot serve it.
+
+    Only presence is checked: values are not recomputed and compared with the stored ones
+    (`lorebase_store.materialize.verify` with `derived=True` does that).
+    """
+
+    attributes: list[str]
+    records_missing_value: int
+    stale: bool
+
+
+class LastChange(BaseModel):
+    """The latest Revision on the branch that changed a Record of the Kind."""
+
+    revision: str
+    at: datetime
+
+
+class LinkRef(BaseModel):
+    from_key: str
+    to_key: str
+
+
+class OrphanLinks(BaseModel):
+    """Links, or foreign keys, whose endpoint Record does not exist."""
+
+    count: int
+    missing_from: int = Field(description="Links whose `from` Record is missing.")
+    missing_to: int = Field(description="Links whose `to` Record is missing.")
+    examples: list[LinkRef]
+
+
+class RequiredGaps(BaseModel):
+    """Records of the `from` Kind without the Link a required Relation demands."""
+
+    missing: int
+    examples: list[str] = Field(description="Keys of some of the Records.")
+
+
+class RelationHealth(BaseModel):
+    relation: str = Field(description="Relation api name, forward direction.")
+    from_kind: str
+    to_kind: str
+    backing: RelationBacking
+    links: int
+    orphan_links: OrphanLinks
+    required: bool
+    required_gaps: RequiredGaps | None = Field(
+        default=None, description="Present for Relations with `required_from`."
+    )
+    direction: Literal["outgoing", "incoming", "self"] | None = Field(
+        default=None, description="In a Kind report: the side the Kind is on."
+    )
+
+
+class KindHealth(BaseModel):
+    kind: str
+    records: int
+    last_change: LastChange | None = None
+    violations: ViolationReport
+    duplicates: DuplicateReport
+    derived: DerivedReport | None = Field(
+        default=None, description="Present for Kinds with derived Attributes."
+    )
+
+
+class HealthSummary(BaseModel):
+    ok: bool = Field(
+        description="No violations, orphan Links, required-relation gaps or duplicates of unique "
+        "Attributes found. Duplicate candidates on title Attributes and stale derived values "
+        "are reported but do not make a report not ok."
+    )
+    kinds: int
+    records: int
+    links: int
+    invalid_records: int
+    orphan_links: int
+    required_relation_gaps: int
+    duplicate_groups: int
+    stale_kinds: int = Field(description="Kinds with derived values missing.")
+    sampled_kinds: int = Field(description="Kinds re-validated on a sample only.")
+
+
+class DomainHealthOut(Served):
+    """Health of a Domain at the head of a branch, for the Kinds the caller may read."""
+
+    domain: str
+    branch: str
+    summary: HealthSummary
+    kinds: list[KindHealth]
+    relations: list[RelationHealth]
+
+
+class KindHealthOut(Served):
+    """Health of one Kind, with the Relations it takes part in."""
+
+    domain: str
+    branch: str
+    summary: HealthSummary
+    health: KindHealth
+    relations: list[RelationHealth]

lorebase_keeper/kindhealth/report.py

@@ -0,0 +1,154 @@
+"""Assembly of the health reports from the scans.
+
+A report covers the Kinds the caller may read (`KindScope`) and the Relations whose both ends
+are such Kinds, so nothing about a Kind the caller cannot see is counted or listed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from functools import partial
+
+from sqlalchemy import func, select
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from lorebase_core.model import Attribute, Kind, Relation
+from lorebase_keeper.errors import UNAVAILABLE, ProblemError
+from lorebase_keeper.kindhealth import scan
+from lorebase_keeper.kindhealth.config import HealthSettings
+from lorebase_keeper.kindhealth.models import HealthSummary, KindHealth, RelationHealth
+from lorebase_keeper.reading.points import ReadPoint
+from lorebase_keeper.reading.sensitivity import Masking, generalize_issues
+
+
+@dataclass(frozen=True, slots=True)
+class Findings:
+    """What a scan found: Kind reports, Relation reports and the totals over them."""
+
+    kinds: list[KindHealth]
+    relations: list[RelationHealth]
+    summary: HealthSummary
+
+
+def visible_kinds(point: ReadPoint) -> list[Kind]:
+    """The Kinds the caller may read, by api name."""
+    kinds = [k for k in point.schema.kinds.values() if point.scope.allows(k.api_name)]
+    return sorted(kinds, key=lambda k: k.api_name)
+
+
+def visible_relations(point: ReadPoint, kind: Kind | None = None) -> list[Relation]:
+    """Relations between Kinds the caller may read; only those of `kind` when given."""
+    schema = point.schema
+    found: list[Relation] = []
+    for relation in schema.relations.values():
+        ends = [schema.kinds.get(relation.from_kind), schema.kinds.get(relation.to_kind)]
+        if any(end is None or not point.scope.allows(end.api_name) for end in ends):
+            continue
+        if kind is None or kind.id in (relation.from_kind, relation.to_kind):
+            found.append(relation)
+    return sorted(found, key=lambda r: r.api_name)
+
+
+async def scan_branch(
+    session: AsyncSession,
+    point: ReadPoint,
+    settings: HealthSettings,
+    kinds: Sequence[Kind],
+    relations: Sequence[Relation],
+) -> Findings:
+    """Run every check for `kinds` and `relations` at the head of the point's branch."""
+    where = scan.Branch(point.domain.id, point.layout.branch, point.layout)
+    schema = point.schema
+    masking = point.masking
+    await session.execute(
+        select(func.set_config("statement_timeout", str(settings.statement_timeout_ms), True))
+    )
+    try:
+        counts = await scan.record_counts(session, where)
+        stored_links = await scan.link_counts(session, where)
+        changes = await scan.last_changes(session, where, [k.id for k in kinds])
+        reports: list[KindHealth] = []
+        for kind in kinds:
+            total = counts.get(kind.id, 0)
+            reports.append(
+                KindHealth(
+                    kind=kind.api_name,
+                    records=total,
+                    last_change=changes.get(kind.id),
+                    violations=await scan.validate_records(
+                        session,
+                        where,
+                        schema,
+                        kind,
+                        total,
+                        settings,
+                        lambda issues: generalize_issues(issues, (schema,), masking),
+                        partial(_hidden, masking, kind),
+                        projected=bool(point.hidden),
+                    ),
+                    duplicates=await scan.find_duplicates(
+                        session,
+                        where,
+                        kind,
+                        total,
+                        settings,
+                        masking.sensitive,
+                        partial(_hidden, masking, kind),
+                    ),
+                    derived=await scan.derived_status(session, where, kind, total),
+                )
+            )
+        related = [
+            await scan.relation_health(session, where, schema, relation, stored_links, settings)
+            for relation in relations
+        ]
+    except DBAPIError as error:
+        if scan.is_timeout(error):
+            raise ProblemError(
+                UNAVAILABLE,
+                "The health scan took longer than read.health.statement_timeout_ms allows. "
+                "Ask for a single Kind, or raise the limit.",
+            ) from None
+        raise
+    return Findings(reports, related, summarize(reports, related))
+
+
+def _hidden(masking: Masking, kind: Kind, attribute: Attribute) -> bool:
+    """Whether the caller may not read values of `attribute` of `kind`."""
+    return masking.hidden(attribute, (kind.id,))
+
+
+def summarize(kinds: Sequence[KindHealth], relations: Sequence[RelationHealth]) -> HealthSummary:
+    """Totals over Kind and Relation reports."""
+    invalid = sum(k.violations.invalid_records for k in kinds)
+    orphans = sum(r.orphan_links.count for r in relations)
+    gaps = sum(r.required_gaps.missing for r in relations if r.required_gaps is not None)
+    duplicates = sum(k.duplicates.total_groups for k in kinds)
+    unique = sum(1 for k in kinds for g in k.duplicates.groups if g.unique) + sum(
+        h.groups for k in kinds for h in k.duplicates.hidden if h.unique
+    )
+    return HealthSummary(
+        ok=not (invalid or orphans or gaps or unique),
+        kinds=len(kinds),
+        records=sum(k.records for k in kinds),
+        links=sum(r.links for r in relations),
+        invalid_records=invalid,
+        orphan_links=orphans,
+        required_relation_gaps=gaps,
+        duplicate_groups=duplicates,
+        stale_kinds=sum(1 for k in kinds if k.derived is not None and k.derived.stale),
+        sampled_kinds=sum(1 for k in kinds if k.violations.sampled),
+    )
+
+
+def with_direction(relations: Sequence[RelationHealth], kind: Kind) -> list[RelationHealth]:
+    """The Relation reports of a Kind report, each marked with the side the Kind is on."""
+    out: list[RelationHealth] = []
+    for item in relations:
+        outgoing = item.from_kind == kind.api_name
+        incoming = item.to_kind == kind.api_name
+        side = "self" if outgoing and incoming else "outgoing" if outgoing else "incoming"
+        out.append(item.model_copy(update={"direction": side}))
+    return out