artest-engine 0.2.0__py3-none-any.whl

artest/engine/__init__.py

@@ -0,0 +1,5 @@
+"""The Artest pipeline: stages, agents, prompts, rules and report export.
+
+The only package that knows the whole flow from exploring a funnel to learning from results. Its
+stages talk to each other only through artest.core models.
+"""

artest/engine/agents/__init__.py

@@ -0,0 +1,74 @@
+"""Agents: small prompts with one output schema each, whose claims all pass the evidence gate.
+
+An agent names no model. It declares what it needs from one, and the run's routing profile gives it the
+model (ADR-0008).
+"""
+
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Protocol
+
+from artest.core import Capture, Finding, FindingRejected, FunnelGraph, StepLeftOut, Viewport
+from artest.engine.agents import checkout_ux, copy_persuasion, mobile_layout, trust_pricing
+from artest.engine.agents.review import FailedReviews, Screenshots
+from artest.llm import CallObserver, Gateway, ModelSpec, Needs
+
+
+class ReviewFunnel(Protocol):
+    async def __call__(
+        self,
+        gateway: Gateway,
+        model: ModelSpec,
+        graph: FunnelGraph,
+        captures: Mapping[str, Capture],
+        *,
+        ids: Iterator[str],
+        viewport: Viewport = Viewport.DESKTOP,
+        observer: CallObserver | None = None,
+        screenshots: Screenshots | None = None,
+        failed: FailedReviews | None = None,
+    ) -> Sequence[Finding | FindingRejected | StepLeftOut]:
+        """Every claim the agent made, through the evidence gate, in order, after the steps it left out."""
+        ...
+
+
+@dataclass(frozen=True)
+class Agent:
+    title: str
+    """What people call it, as the benchmark table does."""
+    name: str
+    """What its model calls, findings and routing name it."""
+    review: ReviewFunnel
+    needs: Needs
+    """What the model that does its work must do. The router picks that model from the run's profile."""
+    looks_at: Callable[[FunnelGraph], frozenset[str]] | None = None
+    """For an agent that looks at screenshots, the captures of a funnel whose screenshots it reads. A
+    recording keeps them, since its replay draws the agent's images from them again. One that only reads
+    text has None."""
+
+    def __post_init__(self) -> None:
+        if self.needs.vision != (self.looks_at is not None):
+            msg = f"the {self.title} agent needs vision exactly when it says which screenshots it looks at"
+            raise ValueError(msg)
+
+
+AGENTS = (
+    Agent("checkout UX", checkout_ux.SOURCE.name, checkout_ux.review_checkout, checkout_ux.NEEDS),
+    Agent(
+        "copy & persuasion", copy_persuasion.SOURCE.name, copy_persuasion.review_copy, copy_persuasion.NEEDS
+    ),
+    Agent("trust & pricing", trust_pricing.SOURCE.name, trust_pricing.review_trust, trust_pricing.NEEDS),
+    Agent(
+        "mobile layout",
+        mobile_layout.SOURCE.name,
+        mobile_layout.review_mobile,
+        mobile_layout.NEEDS,
+        mobile_layout.looks_at,
+    ),
+)
+"""Artest's agents, which review a funnel after the rules, one after another in this order, which finding
+IDs follow, except that one the routing profile has no model for doesn't run. Each looks at different steps
+or for different problems, and leaves what the rules measure to them. An agent may still see a rule's
+problem from its own side: the mobile agent a main call to action too small to tap, or content past the
+screen's side, and the trust agent a cart "Total" that leaves out the shipping. Findings of an agent and a
+rule about one element merge (ADR-0012)."""

artest/engine/agents/checkout_ux.py

@@ -0,0 +1,270 @@
+"""The checkout UX agent: what in each checkout step makes shoppers give up (DESIGN.md §8).
+
+It reads one checkout step at a time and judges what no rule can: whether the order needs each field the
+step asks a shopper to fill in, whether labels, buttons and error messages say what to do, whether the way
+on as a guest is easy to find, and whether something draws shoppers away or the checkout adds a fee. It
+checks each step for a short list of faults, writing its check of each before any finding, and a fault the
+step gives nothing to judge, such as error wording on a step that shows no error, isn't asked about. Last, it
+is asked about anything else on the step that would make a shopper give up, in any of its categories, since
+no list names every problem a real checkout has. Whether checkout works without an account, how many fields
+it asks for, generic error messages, when shipping costs show, links out of the checkout and an open coupon
+field are the rules' (ADR-0025), and whether prices are clear and add up is the trust and pricing agent's,
+so its prompt leaves them out. The page's messages, like its elements, reach the model only in a page block.
+"""
+
+from collections.abc import Callable, Iterator, Mapping
+from typing import NamedTuple
+
+from pydantic import Field
+
+from artest.core import (
+    Capture,
+    Claim,
+    Finding,
+    FindingCategory,
+    FindingRejected,
+    FindingSource,
+    FunnelGraph,
+    FunnelStep,
+    MessageKind,
+    SourceKind,
+    StepKind,
+    StepLeftOut,
+    StepView,
+    Viewport,
+)
+from artest.engine.agents.prompting import SITE_NOTICE, UNTRUSTED_PAGE, message_block, page_block
+from artest.engine.agents.review import (
+    CHECKS_FIRST,
+    CheckedReview,
+    FailedReviews,
+    Screenshots,
+    fault_lines,
+    review_steps,
+)
+from artest.engine.funnel.classify import is_field
+from artest.llm import CallObserver, Gateway, LlmRequest, Message, ModelSpec, Needs
+
+SOURCE = FindingSource(kind=SourceKind.AGENT, name="checkout-ux")
+NEEDS = Needs(reasoning="strong")
+"""Its faults call for judgement about the whole step, such as whether the order needs a field the step
+asks for, or whether the way on as a guest is easy to find next to signing in: work for a model that
+reasons well (DESIGN.md §8)."""
+MAX_FINDINGS = 5
+
+type Asked = Callable[[FunnelGraph, StepView, Capture], bool]
+
+
+def _always(_graph: FunnelGraph, _view: StepView, _capture: Capture) -> bool:
+    return True
+
+
+def _has_fields(_graph: FunnelGraph, _view: StepView, capture: Capture) -> bool:
+    """Whether the page has a field a shopper fills in or picks from, by its tag: a checkout built without
+    a form element has fields too, and so does one that marks required fields only with an asterisk."""
+    return any(element.visible and is_field(element) for element in capture.elements)
+
+
+def _shows_errors(_graph: FunnelGraph, view: StepView, _capture: Capture) -> bool:
+    """Whether the page shows an error, or was captured after the explorer submitted its form empty: a page
+    may write its errors as plain text, which the funnel model finds only by what that submit brought up,
+    and may miss even then."""
+    return view.submitted_empty or any(message.kind is MessageKind.ERROR for message in view.messages)
+
+
+def _offers_guest_checkout(graph: FunnelGraph, _view: StepView, _capture: Capture) -> bool:
+    return graph.facts.guest_checkout is True
+
+
+class Fault(NamedTuple):
+    text: str
+    category: FindingCategory | None
+    """The category of a finding about it; None for the last fault, which may be in any of the agent's."""
+    asked: Asked = _always
+    """Whether the step gives the fault something to judge: one about error wording needs an error."""
+
+
+# Checkout that works only with an account is the forced-account rule's, so this is asked only where
+# code found a way on as a guest.
+GUEST_OPTION = Fault(
+    "the way to check out as a guest is harder to find than signing in or creating an account, such as a "
+    "small link below a big sign-in form",
+    FindingCategory.ACCOUNT_FRICTION,
+    _offers_guest_checkout,
+)
+# How many fields the checkout asks for, optional ones included, is the form-length rule's: which of
+# them the order needs isn't.
+NEEDLESS_FIELD = Fault(
+    "a field the shopper must fill in that the order doesn't need, such as a date of birth, where the "
+    "page doesn't say why it asks",
+    FindingCategory.FORM_FRICTION,
+    _has_fields,
+)
+# No rule judges whether a label is clear, and the copy agent looks for placeholder text on the landing
+# and product pages only, so the checkout's own fields and options are this agent's.
+UNCLEAR_LABEL = Fault(
+    "a field or option whose label leaves a shopper unsure what to enter or choose, such as placeholder "
+    "text like Lorem ipsum, or no word on the format a field wants",
+    FindingCategory.COPY_CLARITY,
+    _has_fields,
+)
+# The error-quality rule reports the generic messages its word lists know, such as "Invalid input" or
+# "This field is required", and a form whose only messages are the browser's generic ones. Any other
+# wording that leaves the shopper guessing is beyond those lists. When both report one message, the two
+# findings merge (ADR-0012).
+ERROR_WITHOUT_FIX = Fault(
+    "an error message that doesn't say which field is wrong or what to enter instead, such as "
+    '"Postcode is invalid" or "Please complete all required fields" (one such as "Enter a postcode, '
+    'for example 10115" is fine)',
+    FindingCategory.INPUT_ERRORS,
+    _shows_errors,
+)
+# The checkout-distraction rule counts the links that lead out of the checkout, reporting three or
+# more, and reports an open coupon field. What draws the eye without being a way out, such as products
+# offered to add, it can't count.
+DISTRACTION = Fault(
+    "a promotion, offer or pop-up, other than links, a coupon field and a cookie or consent notice, that "
+    "draws attention away from finishing the order, such as a sale banner or products offered to add",
+    FindingCategory.CHECKOUT_DISTRACTION,
+)
+VAGUE_BUTTON = Fault(
+    "the button that goes on to the next step or places the order, labelled so vaguely that a shopper "
+    'can\'t tell what it does, such as "Submit" or "OK"',
+    FindingCategory.COPY_CLARITY,
+)
+# When shipping costs first show is the shipping-reveal rule's, and whether the payment and review
+# steps' prices are clear and add up is the trust and pricing agent's: a fee the checkout adds, however
+# clearly, is neither's. An unclear fee both report becomes one finding when both cite it. Tax is left
+# out: a sales tax worked out from the address is expected, and a tax line a price includes adds nothing.
+ADDED_FEE = Fault(
+    "a fee or surcharge the checkout adds on top of the items, shipping and tax, such as a service, "
+    "handling or payment fee",
+    FindingCategory.PRICING_TRANSPARENCY,
+)
+_FORM = (NEEDLESS_FIELD, UNCLEAR_LABEL, ERROR_WITHOUT_FIX, DISTRACTION, VAGUE_BUTTON)
+FAULTS = {
+    StepKind.CHECKOUT_ACCOUNT: (GUEST_OPTION, *_FORM),
+    StepKind.CHECKOUT_ADDRESS: _FORM,
+    StepKind.CHECKOUT_SHIPPING: (ADDED_FEE, *_FORM),
+    StepKind.CHECKOUT_PAYMENT: (ADDED_FEE, *_FORM),
+    StepKind.CHECKOUT_REVIEW: (ADDED_FEE, *_FORM),
+}
+"""The faults each checkout step is checked for, in the order the agent checks them, before the last."""
+CATEGORIES = frozenset(fault.category for faults in FAULTS.values() for fault in faults if fault.category)
+# Asked on every step, after the listed faults: anything else that would make a shopper give up, in any of
+# the agent's categories. The gate then takes all of them on every step.
+ANYTHING_ELSE = Fault(
+    "any other problem on this step that would make a shopper give up, which the faults above don't cover",
+    None,
+)
+
+
+def open_faults(graph: FunnelGraph, step: FunnelStep, view: StepView, capture: Capture) -> tuple[Fault, ...]:
+    """The faults to ask about on ``step``: those its page gives something to judge, then the last."""
+    return (*(fault for fault in FAULTS[step.kind] if fault.asked(graph, view, capture)), ANYTHING_ELSE)
+
+
+SYSTEM = f"""You check one step of an online shop's checkout for faults that make shoppers give up. The \
+message lists the faults to check on that step. If the step has none of them, return an empty list. \
+Report only what would make a shopper give up, not wording you would merely polish. Code already checks \
+whether checkout works without an account, how many fields it asks for, error messages that say only \
+"Invalid input" or "This field is required", whether shipping costs are shown before the total, links \
+that lead out of the checkout and a coupon field, and other checks cover prices, trust signals, tap sizes \
+and accessibility. {SITE_NOTICE}
+
+{CHECKS_FIRST}
+
+Rules:
+- Each finding cites 1 to 3 element IDs from a page list, copied exactly, such as p4:e12.
+- If you give a quote, copy it word for word from a page list.
+- Give each finding the category named after its fault.
+- Report at most {MAX_FINDINGS} findings, the most important first.
+- {UNTRUSTED_PAGE}"""
+
+
+class CheckoutReview(CheckedReview):
+    """What the agent answers: its check of each fault, then a claim for each it found on the step."""
+
+    findings: list[Claim] = Field(default_factory=list[Claim], max_length=MAX_FINDINGS)
+
+
+def review_request(
+    model: ModelSpec, graph: FunnelGraph, step: FunnelStep, view: StepView, capture: Capture
+) -> LlmRequest:
+    # The step's URL is the site's own text, so it stays out of the lines code writes.
+    any_category = f"whichever of {', '.join(sorted(CATEGORIES))} fits"
+    faults = [
+        f"{fault.text} (category: {fault.category or any_category})"
+        for fault in open_faults(graph, step, view, capture)
+    ]
+    user = "\n".join(
+        [
+            f"Checkout step {step.id}: {step.kind}, {capture.viewport} view",
+            "",
+            "The page's elements, one per line: ID, role, name and details.",
+            page_block(capture),
+            *_messages(view, capture),
+            "",
+            "Check this step for these faults:",
+            *fault_lines(tuple(faults)),
+        ]
+    )
+    return LlmRequest(
+        model=model,
+        messages=(Message.system(SYSTEM), Message.user(user)),
+        max_tokens=1500,
+        purpose=SOURCE.name,
+    )
+
+
+def _messages(view: StepView, capture: Capture) -> list[str]:
+    """The step's messages, under a line that says when the page showed them.
+
+    A page without messages gets no list: saying "none" read as something the explorer saw on the page,
+    and the model quoted it as such. If the explorer submitted its form empty, a line says so, since the
+    agent is then asked about error wording that the page may have written as plain text.
+    """
+    if not view.messages:
+        if not view.submitted_empty:
+            return []
+        return [
+            "",
+            "The explorer submitted the step's form empty just before this view was captured, so any error "
+            "message the page shows is among its elements.",
+        ]
+    after = " after the explorer submitted its form empty" if view.submitted_empty else ""
+    return [
+        "",
+        f"Messages on the page{after}, one per line: ID, kind, text and the field it is about.",
+        message_block(view.messages, capture),
+    ]
+
+
+async def review_checkout(
+    gateway: Gateway,
+    model: ModelSpec,
+    graph: FunnelGraph,
+    captures: Mapping[str, Capture],
+    *,
+    ids: Iterator[str],
+    viewport: Viewport = Viewport.DESKTOP,
+    observer: CallObserver | None = None,
+    screenshots: Screenshots | None = None,
+    failed: FailedReviews | None = None,
+) -> list[Finding | FindingRejected | StepLeftOut]:
+    """Review every checkout step in ``viewport``, and pass each claim through the evidence gate."""
+    return await review_steps(
+        gateway,
+        graph,
+        captures,
+        reviews=lambda step: step.kind in FAULTS,
+        ask=lambda step, view, capture: review_request(model, graph, step, view, capture),
+        answer=CheckoutReview,
+        claims=lambda review: review.findings,
+        source=SOURCE,
+        allowed=CATEGORIES,
+        ids=ids,
+        viewport=viewport,
+        observer=observer,
+        failed=failed,
+    )

artest/engine/agents/copy_persuasion.py

@@ -0,0 +1,138 @@
+"""The copy and persuasion agent: words on the pages that sell that leave shoppers unsure (DESIGN.md §8).
+
+It reads the landing and product pages, where a shopper decides whether the shop and the product are
+worth it, and judges what no rule can: whether the words say what is sold and what a button does, and
+whether placeholder text was left behind. It checks each page for a short list of faults, writing its check
+of each before any finding, and last for anything else in the words that would leave a shopper unsure. It
+reads each element's text whole, since its subject is the words. Forms, errors, prices, trust signals and
+tap sizes belong to the rules and the other agents, so its prompt leaves them out.
+"""
+
+from collections.abc import Iterator, Mapping
+
+from pydantic import Field
+
+from artest.core import (
+    Capture,
+    Claim,
+    Finding,
+    FindingCategory,
+    FindingRejected,
+    FindingSource,
+    FunnelGraph,
+    FunnelStep,
+    SourceKind,
+    StepKind,
+    StepLeftOut,
+    StepView,
+    Viewport,
+)
+from artest.engine.agents.prompting import SITE_NOTICE, SKIPPED_ROLES, UNTRUSTED_PAGE, page_block
+from artest.engine.agents.review import (
+    CHECKS_FIRST,
+    CheckedReview,
+    FailedReviews,
+    Screenshots,
+    fault_lines,
+    review_steps,
+)
+from artest.llm import CallObserver, Gateway, LlmRequest, Message, ModelSpec, Needs
+
+SOURCE = FindingSource(kind=SourceKind.AGENT, name="copy-persuasion")
+NEEDS = Needs()
+"""It reads a page's words for a short, fixed list of faults, which is basic work for a model that reads."""
+CATEGORIES = frozenset({FindingCategory.COPY_CLARITY})
+MAX_FINDINGS = 3
+SKIPPED = SKIPPED_ROLES | {"form"}
+"""A form is named after all the text inside it, which the lines of its fields and labels list again."""
+
+_PLACEHOLDER = "placeholder or leftover text, such as Lorem ipsum"
+_VAGUE_LABEL = (
+    "a button or link labelled so vaguely that a shopper can't tell what it does, such as "
+    '"Click here" or "Submit"'
+)
+FAULTS = {
+    StepKind.LANDING: (_PLACEHOLDER, _VAGUE_LABEL, "a main heading that doesn't say what the shop sells"),
+    StepKind.PRODUCT: (_PLACEHOLDER, _VAGUE_LABEL, "text that doesn't say what the product is"),
+}
+"""The faults each kind of page is checked for, in the order the agent checks them, before the last."""
+ANYTHING_ELSE = (
+    "any other problem in the page's words that would leave a shopper unsure what is sold or what to do "
+    "next, which the faults above don't cover"
+)
+"""Asked on every page, after the listed faults."""
+
+SYSTEM = f"""You check the copy on one page of an online shop for faults that make shoppers leave. The \
+message lists the faults to check on that page. If the page has none of them, return an empty list. \
+Report only what would make a shopper leave, not copy you would merely polish. Other checks cover forms, \
+errors, prices, shipping, reviews and other trust signals, tap sizes and accessibility. {SITE_NOTICE}
+
+{CHECKS_FIRST}
+
+Rules:
+- Each finding cites 1 to 3 element IDs from the page list, copied exactly, such as p2:e12.
+- If you give a quote, copy it word for word from the page list.
+- Use only the category {FindingCategory.COPY_CLARITY}.
+- Report at most {MAX_FINDINGS} findings, the most important first.
+- {UNTRUSTED_PAGE}"""
+
+
+class CopyReview(CheckedReview):
+    """What the agent answers: its check of each fault, then a claim for each it found on the page."""
+
+    findings: list[Claim] = Field(default_factory=list[Claim], max_length=MAX_FINDINGS)
+
+
+def review_request(model: ModelSpec, step: FunnelStep, capture: Capture) -> LlmRequest:
+    # The step's URL is the site's own text, so it stays out of the lines code writes.
+    user = "\n".join(
+        [
+            f"Step {step.id}: {step.kind} page, {capture.viewport} view",
+            "",
+            "The page's elements, one per line: ID, role, name and details.",
+            page_block(capture, skip=SKIPPED),
+            "",
+            "Check this page for these faults:",
+            *fault_lines((*FAULTS[step.kind], ANYTHING_ELSE)),
+        ]
+    )
+    return LlmRequest(
+        model=model,
+        messages=(Message.system(SYSTEM), Message.user(user)),
+        max_tokens=1500,
+        purpose=SOURCE.name,
+    )
+
+
+async def review_copy(
+    gateway: Gateway,
+    model: ModelSpec,
+    graph: FunnelGraph,
+    captures: Mapping[str, Capture],
+    *,
+    ids: Iterator[str],
+    viewport: Viewport = Viewport.DESKTOP,
+    observer: CallObserver | None = None,
+    screenshots: Screenshots | None = None,
+    failed: FailedReviews | None = None,
+) -> list[Finding | FindingRejected | StepLeftOut]:
+    """Review the landing and product pages in ``viewport``, and pass each claim through the gate."""
+
+    def ask(step: FunnelStep, _view: StepView, capture: Capture) -> LlmRequest:
+        return review_request(model, step, capture)
+
+    return await review_steps(
+        gateway,
+        graph,
+        captures,
+        reviews=lambda step: step.kind in FAULTS,
+        ask=ask,
+        answer=CopyReview,
+        claims=lambda review: review.findings,
+        source=SOURCE,
+        allowed=CATEGORIES,
+        ids=ids,
+        viewport=viewport,
+        observer=observer,
+        failed=failed,
+    )