artest-core 0.2.0__py3-none-any.whl

artest/core/__init__.py

@@ -0,0 +1,158 @@
+"""Domain models, run events and plugin protocols for Artest.
+
+Every pipeline stage reads and writes these types, and the TypeScript contract is generated from
+them. This package depends on pydantic only, so plugin authors can depend on it cheaply.
+"""
+
+from artest.core.artifacts import ArtifactRef
+from artest.core.capture import (
+    AxeImpact,
+    AxeReport,
+    AxeViolation,
+    Capture,
+    ConsoleEntry,
+    ConsoleLevel,
+    ElementRef,
+    FieldMessage,
+    FixedLayer,
+    FormProbe,
+    NetworkSummary,
+    PastEdge,
+    ResourceType,
+    ResourceWeight,
+    ValidityReason,
+    Viewport,
+    WebVitals,
+)
+from artest.core.events import (
+    DomainEvent,
+    ExplorerActed,
+    FindingMerged,
+    FindingRejected,
+    FindingReported,
+    FunnelModelBuilt,
+    PageCaptured,
+    StepLeftOut,
+    TrainingProvidersAllowed,
+    WalkEnded,
+    domain_event,
+)
+from artest.core.findings import (
+    Claim,
+    Evidence,
+    Finding,
+    FindingCategory,
+    FindingSource,
+    RejectionReason,
+    Report,
+    Severity,
+    SourceKind,
+)
+from artest.core.funnel import (
+    Action,
+    ActionKind,
+    Cta,
+    ExploreOutcome,
+    Form,
+    FormField,
+    FunnelFacts,
+    FunnelGraph,
+    FunnelStep,
+    MessageKind,
+    PageMessage,
+    PriceKind,
+    PriceMention,
+    StepKind,
+    StepView,
+    Transition,
+    TrustKind,
+    TrustSignal,
+)
+from artest.core.geometry import Box, Size
+from artest.core.ids import CaptureId, ElementId, FindingId, StepId
+from artest.core.links import link_address
+from artest.core.plugins import RULE_API_VERSION, RULES_GROUP, Rule, RuleClaim, RuleContext, StepPage
+from artest.core.product import PRODUCT, Product
+from artest.core.variants import DomPatch
+
+CONTRACT_VERSION = "0.18.0"
+"""Semver of the data contract shared with the web app and run bundles (ADR-0015).
+
+Additive changes bump the minor version. Removing or renaming anything bumps the major version.
+"""
+
+__all__ = [
+    "CONTRACT_VERSION",
+    "PRODUCT",
+    "RULES_GROUP",
+    "RULE_API_VERSION",
+    "Action",
+    "ActionKind",
+    "ArtifactRef",
+    "AxeImpact",
+    "AxeReport",
+    "AxeViolation",
+    "Box",
+    "Capture",
+    "CaptureId",
+    "Claim",
+    "ConsoleEntry",
+    "ConsoleLevel",
+    "Cta",
+    "DomPatch",
+    "DomainEvent",
+    "ElementId",
+    "ElementRef",
+    "Evidence",
+    "ExploreOutcome",
+    "ExplorerActed",
+    "FieldMessage",
+    "Finding",
+    "FindingCategory",
+    "FindingId",
+    "FindingMerged",
+    "FindingRejected",
+    "FindingReported",
+    "FindingSource",
+    "FixedLayer",
+    "Form",
+    "FormField",
+    "FormProbe",
+    "FunnelFacts",
+    "FunnelGraph",
+    "FunnelModelBuilt",
+    "FunnelStep",
+    "MessageKind",
+    "NetworkSummary",
+    "PageCaptured",
+    "PageMessage",
+    "PastEdge",
+    "PriceKind",
+    "PriceMention",
+    "Product",
+    "RejectionReason",
+    "Report",
+    "ResourceType",
+    "ResourceWeight",
+    "Rule",
+    "RuleClaim",
+    "RuleContext",
+    "Severity",
+    "Size",
+    "SourceKind",
+    "StepId",
+    "StepKind",
+    "StepLeftOut",
+    "StepPage",
+    "StepView",
+    "TrainingProvidersAllowed",
+    "Transition",
+    "TrustKind",
+    "TrustSignal",
+    "ValidityReason",
+    "Viewport",
+    "WalkEnded",
+    "WebVitals",
+    "domain_event",
+    "link_address",
+]

artest/core/artifacts.py

@@ -0,0 +1,19 @@
+from typing import Annotated
+
+from pydantic import Field, StringConstraints
+
+from artest.core.base import CoreModel
+
+Sha256 = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{64}$")]
+
+
+class ArtifactRef(CoreModel):
+    """A pointer to immutable content in the artifact store, addressed by its SHA-256.
+
+    Events carry these references instead of inlining screenshots or DOM snapshots, so the journal stays
+    small and the same bytes are stored once however often they're referenced.
+    """
+
+    sha256: Sha256
+    media_type: str = Field(pattern=r"^[a-z]+/[a-z0-9.+-]+$")
+    size: int = Field(ge=0)

artest/core/base.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, ConfigDict
+
+
+class CoreModel(BaseModel):
+    """Base for every contract model.
+
+    Models are immutable, so a value read from the journal can't be changed by accident, and strict
+    about unknown fields, so a typo or an invented field fails loudly instead of being dropped. Fields
+    with defaults are always serialised, so the generated TypeScript marks them as required too.
+    """
+
+    model_config = ConfigDict(
+        frozen=True,
+        extra="forbid",
+        allow_inf_nan=False,
+        json_schema_serialization_defaults_required=True,
+    )

artest/core/capture.py

@@ -0,0 +1,323 @@
+from enum import StrEnum
+from typing import Self
+
+from pydantic import AwareDatetime, Field, model_validator
+
+from artest.core.artifacts import ArtifactRef
+from artest.core.base import CoreModel
+from artest.core.geometry import Box, Size
+from artest.core.ids import CaptureId, ElementId, capture_of
+
+
+class Viewport(StrEnum):
+    DESKTOP = "desktop"
+    MOBILE = "mobile"
+
+
+class PastEdge(StrEnum):
+    """How a shopper gets at what reaches past the screen's right edge."""
+
+    SCROLLS = "scrolls"
+    """A box around it scrolls sideways, with room to."""
+    CLIPPED = "clipped"
+    """Something cuts it off, so no scrolling brings it on the screen."""
+    PAGE = "page"
+    """Nothing clips it, so the page itself goes on sideways."""
+
+
+class ElementRef(CoreModel):
+    """An element of a capture that an agent may cite as evidence.
+
+    The selector and box are measured by the browser. A model only ever refers to the element by ``eid``.
+    """
+
+    eid: ElementId
+    role: str
+    name: str = Field(max_length=300, description="Accessible name")
+    tag: str
+    text: str = Field(default="", max_length=500, description="Visible text, truncated")
+    selector: str
+    box: Box
+    visible: bool
+    covered: bool = Field(
+        default=False,
+        description="As the page stands, nothing of it shows at any scroll position a shopper can reach: "
+        "something solid is drawn over it wherever it is on the screen, it is clipped away where it is, or "
+        "it is see-through, or holds only words a shopper can't read and shows nothing else. A sticky bar it "
+        "scrolls clear of doesn't count. Where the browser can't say, and in captures from before this was "
+        "measured, it reads False",
+    )
+    words_shown: str | None = Field(
+        default=None,
+        max_length=500,
+        description="Of its text, the words a shopper can read, where that isn't all of them: others are "
+        "see-through, tiny, moved off the page or clipped away, though the browser lays them out. Empty if "
+        "none can be read, though it may still show, by a picture or a background. None if all can, and in "
+        "captures from before this was measured. Only what agents are told and may quote goes by it",
+    )
+    overlaid: bool = Field(
+        default=False,
+        description="Something solid is drawn over most of it in the capture's screenshot, such as a popup "
+        "fixed to the screen. Unless it is also covered, a shopper can scroll it clear",
+    )
+    fixed: bool = Field(
+        default=False,
+        description="It keeps its place on the screen however far the page scrolls, and the capture's fixed "
+        "layer draws it. In captures from before this was measured, it reads False",
+    )
+    past_edge: PastEdge | None = Field(
+        default=None,
+        description="For an element reaching past the screen's right edge, how a shopper gets at the rest of "
+        "it. None for any other, and in captures from before this was measured, which don't say",
+    )
+    attributes: dict[str, str] = Field(
+        default_factory=dict[str, str],
+        description="The attributes rules care about, such as type, autocomplete, required and href",
+    )
+    form: ElementId | None = Field(default=None, description="The form this control belongs to, if any")
+
+    @property
+    def capture_id(self) -> str:
+        return capture_of(self.eid)
+
+
+class WebVitals(CoreModel):
+    """Lab Web Vitals of the page a capture shows, measured by the web-vitals library in the browser.
+
+    They come from one visit on the explorer's machine and connection, with nothing cached, so they
+    show how a page behaves, not what shoppers experience. A page reached by client-side routing is a
+    soft navigation: its paints count from the click that routed to it, it made no request of its own,
+    and its layout shifts start afresh. A metric the page gave no reading for is None.
+    """
+
+    navigation_url: str = Field(
+        pattern=r"^https?://",
+        description="Where the measured navigation went; another page than the capture's when the page "
+        "changed its address without a navigation the browser recognised",
+    )
+    soft_navigation: bool = Field(description="Whether the page was reached by client-side routing")
+    ttfb_ms: float | None = Field(ge=0, description="Time to first byte; None after a soft navigation")
+    fcp_ms: float | None = Field(ge=0, description="First Contentful Paint")
+    lcp_ms: float | None = Field(ge=0, description="Largest Contentful Paint")
+    cls: float | None = Field(ge=0, description="Cumulative Layout Shift, unitless")
+    inp_lab_ms: float | None = Field(
+        ge=0,
+        description="Lab INP: the slowest response, from input to the next paint, to the explorer's own "
+        "clicks on this page since the previous capture. Field INP comes from real shoppers' "
+        "interactions and can differ a lot. 16 for a click answered within 16 ms, the shortest time the "
+        "browser reports. None when there was no such click. A click that loads another document can't be "
+        "timed: its page is gone before the next paint",
+    )
+
+
+class ResourceType(StrEnum):
+    DOCUMENT = "document"
+    SCRIPT = "script"
+    STYLESHEET = "stylesheet"
+    IMAGE = "image"
+    FONT = "font"
+    MEDIA = "media"
+    FETCH = "fetch"
+    OTHER = "other"
+
+
+class ResourceWeight(CoreModel):
+    requests: int = Field(ge=0)
+    transfer_bytes: int = Field(ge=0, description="Response headers and bodies as received, compressed")
+
+
+class NetworkSummary(CoreModel):
+    """What the page loaded over the network since the previous capture, from its own document on.
+
+    The explorer's browser keeps no HTTP cache, so a page loads everything it needs as on a first
+    visit. A request still running at the capture is counted with the next capture of the same document.
+    """
+
+    requests: int = Field(ge=0)
+    transfer_bytes: int = Field(ge=0, description="Response headers and bodies as received, compressed")
+    failed: int = Field(
+        ge=0,
+        description="Requests that got an HTTP error status, or no response for a reason other than "
+        "the page cancelling them",
+    )
+    by_type: dict[ResourceType, ResourceWeight] = Field(default_factory=dict[ResourceType, ResourceWeight])
+
+    @model_validator(mode="after")
+    def _totals_add_up(self) -> Self:
+        weights = self.by_type.values()
+        if (self.requests, self.transfer_bytes) != (
+            sum(weight.requests for weight in weights),
+            sum(weight.transfer_bytes for weight in weights),
+        ):
+            msg = "the network totals must add up to the weights by type"
+            raise ValueError(msg)
+        return self
+
+
+class ConsoleLevel(StrEnum):
+    ERROR = "error"
+    WARNING = "warning"
+
+
+class ConsoleEntry(CoreModel):
+    """A console error or warning, or an exception nothing on the page caught."""
+
+    level: ConsoleLevel
+    text: str = Field(max_length=500, description="The message, truncated")
+    uncaught: bool = Field(default=False, description="Whether it is an exception nothing caught")
+    url: str | None = Field(default=None, description="The script or resource it came from, if known")
+
+
+class AxeImpact(StrEnum):
+    MINOR = "minor"
+    MODERATE = "moderate"
+    SERIOUS = "serious"
+    CRITICAL = "critical"
+
+
+class AxeViolation(CoreModel):
+    rule: str = Field(description="The axe-core rule ID, such as color-contrast")
+    impact: AxeImpact | None
+    description: str = Field(description="What the rule asks for, in axe-core's words")
+    help_url: str
+    tags: tuple[str, ...] = Field(description="axe-core's tags, such as wcag2aa and wcag143")
+    nodes: int = Field(ge=1, description="How many elements on the page fail the rule")
+    elements: tuple[ElementId, ...] = Field(
+        default=(),
+        description="The failing elements that are in the capture's element list, which leaves out "
+        "containers such as a list's",
+    )
+
+
+class AxeReport(CoreModel):
+    """What axe-core found wrong on the page, as the capture saw it. Frames on the page aren't checked."""
+
+    version: str = Field(description="The axe-core version that checked the page")
+    violations: tuple[AxeViolation, ...] = ()
+
+
+class ValidityReason(StrEnum):
+    """Why the browser's own validation flagged a field, after the browser's ValidityState."""
+
+    VALUE_MISSING = "value-missing"
+    CUSTOM_ERROR = "custom-error"
+    TYPE_MISMATCH = "type-mismatch"
+    PATTERN_MISMATCH = "pattern-mismatch"
+    TOO_SHORT = "too-short"
+    TOO_LONG = "too-long"
+    RANGE_UNDERFLOW = "range-underflow"
+    RANGE_OVERFLOW = "range-overflow"
+    STEP_MISMATCH = "step-mismatch"
+    BAD_INPUT = "bad-input"
+
+
+class FieldMessage(CoreModel):
+    field: ElementId
+    reason: ValidityReason
+    message: str = Field(
+        max_length=300, description="What the browser showed: its own words, or the page's if it set some"
+    )
+
+
+class FormProbe(CoreModel):
+    """The explorer submitted a checkout form empty just before this capture (ADR-0011).
+
+    The messages a page writes into itself are among the capture's elements. Those of the browser's own
+    validation are not: the browser shows them in a bubble outside the page, one at a time from the
+    first, and the screenshot leaves it out, so they are kept here. Only fields the browser flagged at
+    the submit have one: a form that turns that validation off, or a script that stops the submit
+    first, leaves none.
+    """
+
+    submitted_on: CaptureId = Field(description="The capture the form was submitted from")
+    form: ElementId | None = Field(description="The form in this capture; None once the page has none")
+    browser_messages: tuple[FieldMessage, ...] = Field(
+        default=(), description="The browser's own validation messages, by field in this capture"
+    )
+
+
+class FixedLayer(CoreModel):
+    """What it takes to draw the elements fixed to the screen where each screen would show them (ADR-0012).
+
+    A full-page screenshot draws them once, where the capture's screen had them, so a tile further down the
+    page wouldn't show them where a shopper there sees them. Boxes are in the page's terms, as elements' are.
+    """
+
+    page: ArtifactRef = Field(description="The full-page screenshot with those elements left out")
+    layer: ArtifactRef = Field(description="One screen with only those elements on it, see-through elsewhere")
+    screen: Box = Field(description="The part of the page the capture's screen showed")
+    solid: tuple[Box, ...] = Field(default=(), description="What is solid among those elements")
+
+
+class Capture(CoreModel):
+    """One page state in one viewport, as the explorer saw it.
+
+    The heavy parts (screenshot, DOM, accessibility tree) live in the artifact store; the capture keeps
+    references to them plus the element list agents and rules work from, and what the browser measured.
+    Captures recorded before Artest measured something hold None for it.
+    """
+
+    id: CaptureId
+    url: str = Field(pattern=r"^https?://")
+    title: str
+    viewport: Viewport
+    viewport_size: Size
+    page_size: Size
+    captured_at: AwareDatetime
+    screenshot: ArtifactRef
+    dom: ArtifactRef
+    aria: ArtifactRef
+    elements: tuple[ElementRef, ...] = ()
+    vitals: WebVitals | None = None
+    network: NetworkSummary | None = None
+    console: tuple[ConsoleEntry, ...] | None = Field(
+        default=None,
+        description="Errors and warnings since the previous capture, from this page's document on",
+    )
+    axe: AxeReport | None = None
+    probe: FormProbe | None = Field(default=None, description="The empty submit that led to this capture")
+    settled: bool = Field(
+        default=True,
+        description="Whether the page had stopped asking its server for things when it was captured. False "
+        "when it was taken as it stood, as a request stayed open, such as a chat widget's long poll. "
+        "Captures from before this was measured read True",
+    )
+    fixed_layer: FixedLayer | None = Field(
+        default=None,
+        description="What draws the elements fixed to the screen on each screen. None on a page with none "
+        "that can be drawn alone, and in captures from before this was measured",
+    )
+
+    @property
+    def images(self) -> tuple[ArtifactRef, ...]:
+        """Every picture of the page in the artifact store, which is what a vision agent may be shown."""
+        layer = self.fixed_layer
+        return (self.screenshot,) if layer is None else (self.screenshot, layer.page, layer.layer)
+
+    @model_validator(mode="after")
+    def _elements_belong_to_this_capture(self) -> Self:
+        seen: set[str] = set()
+        for element in self.elements:
+            if element.capture_id != self.id:
+                msg = f"element {element.eid} does not belong to capture {self.id}"
+                raise ValueError(msg)
+            if element.eid in seen:
+                msg = f"duplicate element id {element.eid}"
+                raise ValueError(msg)
+            seen.add(element.eid)
+        for eid in self._cited():
+            if eid not in seen:
+                msg = f"{eid} is cited but is not an element of capture {self.id}"
+                raise ValueError(msg)
+        return self
+
+    def _cited(self) -> list[str]:
+        """The elements that what the browser measured points at."""
+        cited = [eid for violation in self.axe.violations for eid in violation.elements] if self.axe else []
+        if self.probe is not None:
+            cited += [message.field for message in self.probe.browser_messages]
+            cited += [self.probe.form] if self.probe.form is not None else []
+        return cited
+
+    def element(self, eid: str) -> ElementRef | None:
+        return next((element for element in self.elements if element.eid == eid), None)

artest/core/events.py

@@ -0,0 +1,125 @@
+"""Domain events: what the pipeline tells the journal, the live UI and the exported report.
+
+Control events (a step started, a run paused) belong to the runtime, which knows nothing about CRO. The
+engine combines both into the full run event union.
+"""
+
+from typing import Annotated, Literal
+
+from pydantic import Field, TypeAdapter
+
+from artest.core.artifacts import ArtifactRef
+from artest.core.base import CoreModel
+from artest.core.capture import Viewport
+from artest.core.findings import Finding, FindingCategory, FindingSource, RejectionReason
+from artest.core.funnel import Action, ExploreOutcome, FunnelFacts
+from artest.core.geometry import Box
+from artest.core.ids import CaptureId, FindingId, StepId
+
+
+class PageCaptured(CoreModel):
+    type: Literal["page_captured"] = "page_captured"
+    capture_id: CaptureId
+    url: str
+    title: str
+    viewport: Viewport
+    screenshot: ArtifactRef
+    capture: ArtifactRef = Field(description="The full Capture as JSON in the artifact store")
+
+
+class ExplorerActed(CoreModel):
+    type: Literal["explorer_acted"] = "explorer_acted"
+    viewport: Viewport
+    capture_id: CaptureId = Field(description="The page the action was taken on")
+    action: Action
+    target: Box | None = Field(default=None, description="Where the element was, for the UI's agent cursor")
+
+
+class WalkEnded(CoreModel):
+    """How one viewport's walk ended, journaled before anything reviews what it reached, so that whoever runs
+    the audit and whoever reads its report know what the review never saw."""
+
+    type: Literal["walk_ended"] = "walk_ended"
+    viewport: Viewport
+    outcome: ExploreOutcome
+    reason: str = Field(description="Why it ended there, in a sentence of Artest's own")
+    url: str | None = Field(default=None, description="The last page it captured, if it captured any")
+    stopped_at: str | None = Field(
+        default=None,
+        description="The words of the button or field it stopped at, if it did: the page's, not Artest's",
+    )
+
+
+class FunnelModelBuilt(CoreModel):
+    type: Literal["funnel_model_built"] = "funnel_model_built"
+    graph: ArtifactRef
+    step_count: int = Field(ge=1)
+    facts: FunnelFacts
+
+
+class FindingReported(CoreModel):
+    type: Literal["finding_reported"] = "finding_reported"
+    finding: Finding
+
+
+class FindingMerged(CoreModel):
+    """A finding about a problem already found, merged into the earlier finding instead of reported apart.
+
+    It carries the earlier finding as it is now: the same ID, with the new source's report, evidence
+    and source added, and a higher confidence when a rule and an agent agree (ADR-0012).
+    """
+
+    type: Literal["finding_merged"] = "finding_merged"
+    finding: Finding
+    merged: FindingId = Field(
+        description="The ID the evidence gate gave the finding that merged in, which no finding keeps"
+    )
+
+
+class FindingRejected(CoreModel):
+    """A claimed finding the evidence gate dropped, kept on record so the UI can show what and why."""
+
+    type: Literal["finding_rejected"] = "finding_rejected"
+    source: FindingSource
+    step_id: str = Field(description="The step the claim was about, which may not exist")
+    viewport: Viewport
+    category: FindingCategory
+    title: str
+    reason: RejectionReason
+    detail: str = Field(description="What failed, such as which element was not in the capture")
+
+
+class TrainingProvidersAllowed(CoreModel):
+    """The user let this site's pages go to providers that may train on prompts, or whose terms Artest doesn't
+    know (ADR-0006). Kept on record so the run and its report say so."""
+
+    type: Literal["training_providers_allowed"] = "training_providers_allowed"
+    url: str = Field(description="The site whose pages may go to them")
+
+
+class StepLeftOut(CoreModel):
+    """A step an agent was to review but left out, kept on record so the UI can show what wasn't reviewed
+    and why."""
+
+    type: Literal["step_left_out"] = "step_left_out"
+    source: FindingSource = Field(description="The agent that left the step out")
+    step_id: StepId
+    viewport: Viewport
+    reason: str = Field(description="Why, in a sentence")
+
+
+DomainEvent = Annotated[
+    PageCaptured
+    | ExplorerActed
+    | WalkEnded
+    | FunnelModelBuilt
+    | FindingReported
+    | FindingMerged
+    | FindingRejected
+    | StepLeftOut
+    | TrainingProvidersAllowed,
+    Field(discriminator="type"),
+]
+
+domain_event: TypeAdapter[DomainEvent] = TypeAdapter(DomainEvent)
+"""Validates and serialises any domain event, dispatching on its ``type``."""