prettyplay 0.0.2__py3-none-any.whl

prettyplay/llm/provider.py

@@ -0,0 +1,279 @@
+"""The unified LLM port of the library and the provider factory."""
+
+from ..config import Config
+from .models import ComplianceFinding, FailureClassification, GroupFailureClassification, ScenarioStep
+
+
+class LLMProvider:
+    """The single LLM port: step code generation, failure classification, the group diagnosis and the verdict."""
+
+    def generate_step_code(  # noqa: PLR0913, PLR0917 — the signature is fixed by the port contract
+        self,
+        prompt: str,
+        user_instructions: str,
+        instruction: str,
+        step_type: str,
+        previous_steps: list[ScenarioStep],
+        group_prompt: str | None,
+        inputs: dict[str, str],
+        declarations: list[str],
+        snapshot: str,
+        page_url: str | None,
+        screenshot: bytes | None,
+        cheat_sheet: str,
+        attempt_history: list[str],
+        recommendation: str | None,
+        guidance: str | None,
+    ) -> str:
+        """Generate step code of the fixed form working through the standard Playwright sync API.
+
+        Args:
+            prompt: the system prompt text supplied by the calling engine;
+                applied verbatim as the system message.
+            user_instructions: the project's code style instructions supplied
+                by the calling engine from the generation_prompt setting;
+                empty — the request carries no instructions block, non-empty —
+                rendered verbatim as a separate USER INSTRUCTIONS block of the
+                user content, identically in both implementations.
+            instruction: the prepared instruction of the step — the rendered
+                plain-text sentence with actual values embedded; rendered as
+                the STEP block; the raw template sentence never reaches the
+                request.
+            step_type: the type of the step; rendered as a STEP TYPE line
+                immediately before the STEP line, identically in both
+                implementations; takes no part in step addressing.
+            previous_steps: the typed scenario records of the previous steps
+                of the test, in execution order — each the prepared
+                instruction recorded at the step's execution plus its
+                permanent group membership; rendered by the provider
+                implementations as the PREVIOUS STEPS block with the group
+                entries marked, identically in both.
+            group_prompt: the group prompt of the current step's group;
+                None — an ordinary step, no GROUP PROMPT block; non-empty —
+                rendered by the provider implementations verbatim as a
+                separate GROUP PROMPT block immediately before the PREVIOUS
+                STEPS block, identically in both; takes no part in step
+                addressing.
+            inputs: the call-local input bindings of the step; non-empty —
+                rendered by the provider implementations as the INPUTS block
+                immediately after the STEP line, one ``name = value`` line
+                per binding, identically in both; empty — no block.
+            declarations: the declared result names of the step; non-empty —
+                rendered by the provider implementations as the RESULTS
+                block immediately after the INPUTS block stating the result
+                contract — the code returns a dictionary of exactly the
+                declared names to non-blank strings observed on the page;
+                empty — no block, the success-without-result code form.
+            snapshot: the accessibility snapshot of the current page.
+            page_url: the current URL of the page; non-empty — rendered by
+                the provider implementations as its own PAGE URL line
+                immediately after the PAGE SNAPSHOT block of the user
+                content, identically in both; None — no line; supplied by
+                the engine and steering generation paths.
+            screenshot: an optional PNG image of the page; passed only when
+                the project enables screenshots.
+            cheat_sheet: the compact standard Playwright sync API reference
+                supplied by the calling engine — rendered as the CHEAT SHEET
+                block after the scenario inputs of the user content,
+                identically in both implementations; guidance, not an
+                allowlist — everything standard stays allowed.
+            attempt_history: the rendered per-step attempt records — every
+                record a complete multi-line verbatim record composed by the
+                calling engine, the original cached code anchored as record 0
+                when it exists, the last record the code being fixed;
+                non-empty — rendered as a separate HISTORY block after the
+                USER INSTRUCTIONS block, every record verbatim, no
+                collapsing, no size limits; empty — no block; the list takes
+                the place of the former existing_code, error and
+                guidance_history inputs.
+            recommendation: the diagnosis of the classification that preceded
+                the regeneration; non-empty — rendered as a separate
+                RECOMMENDATION block after the HISTORY block, None — no
+                block.
+            guidance: the engineer guidance message of the interactive
+                steering; non-empty — rendered as a separate USER GUIDANCE
+                block, None — no block.
+
+        Returns:
+            The generated step code of the fixed form, working through the
+            standard Playwright sync API — imports from playwright.sync_api
+            and the Python standard library only, global at the top level of
+            the code block.
+
+        Raises:
+            NotImplementedError: the port itself carries no implementation.
+        """
+        raise NotImplementedError("LLMProvider is a port; use create_provider() to select an implementation")
+
+    def classify_step_failure(  # noqa: PLR0913, PLR0917 — the signature is fixed by the port contract
+        self,
+        prompt: str,
+        user_instructions: str,
+        step_text: str,
+        code: str,
+        error: str,
+        snapshot: str,
+        screenshot: bytes | None,
+    ) -> FailureClassification:
+        """Classify a failed cached step — the former classify_failure, renamed nominally.
+
+        Args:
+            prompt: the system prompt text supplied by the calling engine;
+                applied verbatim as the system message.
+            user_instructions: the project's classification guidance supplied
+                by the calling engine from the classification_prompt setting;
+                empty — the request carries no instructions block, non-empty —
+                rendered verbatim as a separate USER INSTRUCTIONS block placed
+                last of the user content, identically in both implementations.
+            step_text: the sentence of the failed step — the prepared
+                instruction, never the raw template sentence; a non-template
+                step passes its sentence unchanged.
+            code: the existing step code that failed.
+            error: the human-readable description of the failure.
+            snapshot: the accessibility snapshot of the current page.
+            screenshot: an optional PNG image of the page; passed only when
+                the project enables screenshots.
+
+        Returns:
+            The classification verdict.
+
+        Raises:
+            NotImplementedError: the port itself carries no implementation.
+        """
+        raise NotImplementedError("LLMProvider is a port; use create_provider() to select an implementation")
+
+    def classify_group_failure(  # noqa: PLR0913, PLR0917 — the signature is fixed by the port contract
+        self,
+        prompt: str,
+        user_instructions: str,
+        group_prompt: str,
+        group_steps: list[str],
+        previous_steps: list[ScenarioStep],
+        step_text: str,
+        attempt_history: list[str],
+        snapshot: str,
+        screenshot: bytes | None,
+    ) -> GroupFailureClassification:
+        """Diagnose a failed group step with the whole interaction in view.
+
+        Args:
+            prompt: the group diagnosis system prompt supplied by the calling
+                engine; applied verbatim as the system message.
+            user_instructions: the project's classification guidance supplied
+                by the calling engine from the classification_prompt setting;
+                the diagnosis request carries them exactly as a classification
+                request does — empty, no instructions block; non-empty,
+                rendered verbatim as a separate USER INSTRUCTIONS block placed
+                last of the user content, identically in both implementations.
+            group_prompt: the group prompt of the diagnosed group, verbatim.
+            group_steps: the composed verbatim traces of the group's steps in
+                execution order — each the prepared instruction, the outcome
+                and the URL before -> after transition, supplied by the
+                calling engine.
+            previous_steps: the typed scenario records of the previous steps
+                of the test, in execution order — each the prepared
+                instruction recorded at the step's execution plus its
+                permanent group membership; rendered by the provider
+                implementations as the PREVIOUS STEPS block immediately before
+                the GROUP STEPS block with the group entries marked,
+                identically in both; empty — no block.
+            step_text: the prepared instruction of the failed step — never
+                the raw template sentence; a non-template step passes its
+                sentence unchanged.
+            attempt_history: the rendered verbatim records of the failed
+                step's attempt history; non-empty — rendered as a separate
+                HISTORY block, every record verbatim, no collapsing, no size
+                limits; empty — no block.
+            snapshot: the accessibility snapshot of the current page.
+            screenshot: an optional PNG image of the page; passed only when
+                the project enables screenshots.
+
+        Returns:
+            The diagnosis verdict — parsed strictly through
+            :func:`~prettyplay.llm.parse_group_failure_classification`; a
+            degraded answer is the conservative incurable, never a granted
+            regeneration.
+
+        Raises:
+            NotImplementedError: the port itself carries no implementation.
+        """
+        raise NotImplementedError("LLMProvider is a port; use create_provider() to select an implementation")
+
+    def check_instruction_compliance(  # noqa: PLR0913, PLR0917 — the signature is fixed by the port contract
+        self,
+        prompt: str,
+        user_instructions: str,
+        instruction: str,
+        step_type: str,
+        inputs: dict[str, str],
+        declarations: list[str],
+        code: str,
+        attempt_history: list[str],
+    ) -> list[ComplianceFinding]:
+        """Check the successfully executed candidate code in two dimensions.
+
+        Args:
+            prompt: the gate system prompt text supplied by the calling
+                engine; applied verbatim as the system message.
+            user_instructions: the project's generation instructions
+                supplied by the calling engine from the generation_prompt
+                setting; the calling engine guarantees non-empty — the gate
+                never runs on empty instructions.
+            instruction: the prepared plain-text instruction of the generated
+                step — rendered as the STEP block; the raw template sentence
+                never reaches the request.
+            step_type: the type of the step; rendered as a STEP TYPE line
+                immediately before the STEP line, identically in both
+                implementations.
+            inputs: the call-local input bindings of the step; non-empty —
+                rendered by the provider implementations as the INPUTS block
+                immediately after the STEP block, one ``name = value`` line
+                per binding, identically in both; empty — no block.
+            declarations: the reached capture names of the step; non-empty —
+                rendered by the provider implementations as the RESULTS block
+                immediately after the INPUTS block stating the exact-key,
+                observed non-blank string return contract, identically in
+                both; empty — no block, the success-without-result form.
+            code: the successfully executed candidate code.
+            attempt_history: the rendered per-step attempt records — the
+                ground truth of what was already tried; non-empty — rendered
+                as a separate ATTEMPT HISTORY block, every record verbatim,
+                no collapsing, no size limits; empty — no block.
+
+        Returns:
+            The parsed findings; an empty list means compliant — one verdict
+            request per executed candidate; the block order INSTRUCTIONS,
+            STEP, the optional INPUTS and RESULTS sections, ATTEMPT HISTORY,
+            CODE.
+
+        Raises:
+            NotImplementedError: the port itself carries no implementation.
+        """
+        raise NotImplementedError("LLMProvider is a port; use create_provider() to select an implementation")
+
+
+def create_provider(config: Config) -> LLMProvider:
+    """Select and construct the LLM provider from configuration — the implementations are interchangeable.
+
+    Args:
+        config: project settings; the provider setting selects the SDK
+            implementation.
+
+    Returns:
+        The selected provider implementation.
+
+    Raises:
+        ValueError: the provider setting names no supported provider.
+    """
+    # deferred: the implementations subclass the port defined in this module,
+    # so a top-level import here would be circular
+    from .anthropic_provider import AnthropicProvider  # noqa: PLC0415
+    from .openai_provider import OpenAIProvider  # noqa: PLC0415
+
+    if config.provider == "openai":
+        return OpenAIProvider(config)
+
+    if config.provider == "anthropic":
+        return AnthropicProvider(config)
+
+    raise ValueError(f"unsupported provider {config.provider!r}: expected one of openai, anthropic")

prettyplay/reporting/.usages/hooks.md

@@ -0,0 +1,61 @@
+# Step and healing hooks
+
+Domain: event callbacks of prettyplay. Audience: integrators building custom reporting, metrics or CI reactions on top of step execution.
+
+StepHooks is a thin callback contract. The library calls the matching method synchronously while a step executes. The base implementation of every method is a no-op — override only the events you need. Hook implementations are registered on the main library object at the start of a test.
+
+## Events
+
+| Method | When | Payload |
+|---|---|---|
+| on_step_started | a step started executing | step_text, step_type (action or assertion) |
+| on_step_passed | the step finished successfully | step_text, step_type |
+| on_step_failed | the step failed | step_text, step_type, error — the **full rendered failure message** (see below) |
+| on_step_verdict | the terminal failure carried a verdict (fires after on_step_failed) | step_text, category (rot, product_defect, fixable, incurable), explanation, recommendation |
+| on_step_finished | the step ended — always the last step event, regardless of outcome | step_text, step_type, outcome (passed or failed) |
+| on_group_started | a group block started (once per entered group, zero-step groups included) | group_prompt |
+| on_group_passed | the group block completed without an exception — a recovered group reports passed; a zero-step group reports passed | group_prompt |
+| on_group_failed | the group block exited through an exception — a failed step, a refused recovery or an author exception alike; the exception still propagates | group_prompt |
+| on_group_finished | the group block ended — always the last group event, regardless of outcome, after on_group_passed/on_group_failed | group_prompt |
+| on_generation_started | a generation attempt started | step_text, attempt (1-based, one per LLM request) |
+| on_healing_started | healing of a failed cached step started | step_text, category (rot, product_defect, fixable, incurable — a group recovery row reports the diagnosis label recoverable) |
+| on_healed | the step healed, cache updated | step_text, explanation (why it failed, what changed — interactive healings report here too) |
+| on_cache_saved | step code written to the cache | step_text, filename |
+| on_cache_skipped | cache write skipped | step_text, reason (e.g. read-only cache) |
+
+## The error payload of on_step_failed
+
+`error` carries the full structured render of the terminal failure — the same multi-line text the raised exception carries and the log record writes: the first line with the class name of the terminal failure and the authored reason, the `---` separated step/error section, the conditional received/cause/Call log details section and the unpadded verdict block. Display it verbatim in reports; do not parse it — structured data arrives through on_step_verdict fields. In strict mode on_generation_started and on_healing_started never fire: classification is the only LLM call.
+
+## Attempt semantics
+
+Hook events fire per step or per LLM attempt — never per execution retry. Settle re-executions of the same step code (transient failure absorption) emit no hook events: they are visible only as settle_retry log records. on_step_finished closes every step exactly once, passed or failed; on_group_finished closes every entered group exactly once the same way — the group verdict is the block outcome, never the traces: a step healed by the group recovery leaves its verbatim failed trace record behind while the group reports passed.
+
+## Example
+
+```python
+from prettyplay.reporting import StepHooks
+
+
+class FailureMonitor(StepHooks):
+    def on_step_failed(self, step_text: str, step_type: str, error: str) -> None:
+        print(f"failed [{step_type}]: {step_text}\n{error}")  # the full render, verbatim
+
+    def on_step_verdict(self, step_text: str, category: str, explanation: str, recommendation: str) -> None:
+        print(f"verdict [{category}]: {step_text} — {explanation} → {recommendation}")
+
+    def on_step_finished(self, step_text: str, step_type: str, outcome: str) -> None:
+        print(f"finished [{outcome}]: {step_text}")
+
+    def on_healed(self, step_text: str, explanation: str) -> None:
+        print(f"healed: {step_text} — {explanation}")
+```
+
+## Rules
+
+- Handlers run synchronously inside step execution: keep them fast
+- A raising handler is logged as a warning and skipped — the test run never fails because of a hook
+- on_step_verdict fires only when the verdict exists: LLM unavailability skips the verdict quietly (WARNING in the log)
+- The verdict payload is built from the verdict object of the failure — never parsed from the rendered `error` text
+- Payload values are plain strings; the attempt counter of on_generation_started is an int
+- Step texts land in logs and hooks: never put secrets or personal data into a step sentence

prettyplay/reporting/__init__.py

@@ -0,0 +1,6 @@
+"""Facade of the prettyplay.reporting cell: the visibility of step execution."""
+
+from .hooks import StepHooks
+from .reporter import StepReporter
+
+__all__ = ["StepHooks", "StepReporter"]

prettyplay/reporting/hooks.py

@@ -0,0 +1,47 @@
+"""Thin callback contract for integrator-side reactions to step execution events."""
+
+
+class StepHooks:
+    """Callback contract of the fourteen prettyplay events; every base method is a no-op."""
+
+    def on_step_started(self, step_text: str, step_type: str) -> None:
+        """A step started executing; ``step_type`` is action or assertion."""
+
+    def on_step_passed(self, step_text: str, step_type: str) -> None:
+        """The step finished successfully."""
+
+    def on_step_failed(self, step_text: str, step_type: str, error: str) -> None:
+        """The step failed; ``error`` is the rendered failure message, same as the exception text."""
+
+    def on_step_verdict(self, step_text: str, category: str, explanation: str, recommendation: str) -> None:
+        """The terminal failure of the step carried a verdict; fires after on_step_failed."""
+
+    def on_step_finished(self, step_text: str, step_type: str, outcome: str) -> None:
+        """The step ended — the closing event of every step, fired exactly once regardless of outcome."""
+
+    def on_group_started(self, group_prompt: str) -> None:
+        """A group block started; ``group_prompt`` is the group prompt, verbatim."""
+
+    def on_group_passed(self, group_prompt: str) -> None:
+        """The group block completed without an exception — a recovered group reports passed here."""
+
+    def on_group_failed(self, group_prompt: str) -> None:
+        """The group block exited through an exception — a failed step, a refused recovery or an author error."""
+
+    def on_group_finished(self, group_prompt: str) -> None:
+        """The group block ended — the closing event of every group, fired exactly once regardless of outcome."""
+
+    def on_generation_started(self, step_text: str, attempt: int) -> None:
+        """A code generation attempt started; ``attempt`` is the 1-based attempt number."""
+
+    def on_healing_started(self, step_text: str, category: str) -> None:
+        """Healing of a failed cached step started; ``category`` is rot, product_defect, fixable or incurable."""
+
+    def on_healed(self, step_text: str, explanation: str) -> None:
+        """The step was healed and the cache updated; ``explanation`` says why and what changed."""
+
+    def on_cache_saved(self, step_text: str, filename: str) -> None:
+        """Step code was written to the cache file ``filename``."""
+
+    def on_cache_skipped(self, step_text: str, reason: str) -> None:
+        """The cache write was skipped; ``reason`` names the cause, e.g. a read-only cache."""

prettyplay/reporting/reporter.py

@@ -0,0 +1,70 @@
+"""The single visibility point of prettyplay: the ``prettyplay`` logger plus hook fan-out."""
+
+import logging
+
+from .hooks import StepHooks
+
+#: Events written at WARNING; every other event is step/generation/healing lifecycle INFO.
+_WARNING_EVENTS = frozenset({"on_cache_skipped"})
+
+#: Attributes Logger.makeRecord forbids overwriting: collision keys get a ``ctx_`` prefix
+#: in the log record only — hooks always receive the original payload kwargs.
+_LOG_RECORD_RESERVED = frozenset(
+    {
+        "name",
+        "msg",
+        "message",
+        "args",
+        "levelname",
+        "levelno",
+        "pathname",
+        "filename",
+        "module",
+        "exc_info",
+        "funcName",
+        "lineno",
+        "created",
+        "msecs",
+        "relativeCreated",
+        "thread",
+        "threadName",
+        "process",
+        "processName",
+        "stack_info",
+        "asctime",
+        "taskName",
+    }
+)
+
+
+class StepReporter:
+    """Dispatches every step, generation, healing and cache event to the logger and hooks."""
+
+    def __init__(self, hooks: list[StepHooks]) -> None:
+        """Keep the hooks list by reference in the public ``hooks`` attribute.
+
+        Args:
+            hooks: callback implementations registered by the integrator; may be empty.
+                ``add_hooks`` of the main object appends into this same list.
+        """
+        self.hooks = hooks
+        self._logger = logging.getLogger("prettyplay")
+
+    def emit(self, event: str, payload: dict[str, str | int]) -> None:
+        """Dispatch one event to the logger and to every hook, in registration order.
+
+        Args:
+            event: the event name — equals a ``StepHooks`` method name exactly.
+            payload: the payload fields of the event; values are strings, the
+                attempt counter is an int.
+        """
+        level = logging.WARNING if event in _WARNING_EVENTS else logging.INFO
+        extra = {f"ctx_{key}" if key in _LOG_RECORD_RESERVED else key: value for key, value in payload.items()}
+
+        self._logger.log(level, event, extra=extra)
+
+        for hook in self.hooks:
+            try:
+                getattr(hook, event)(**payload)
+            except Exception:
+                self._logger.warning("hook call failed", extra={"event": event, "hook": type(hook).__name__})

prettyplay/runtime.py

@@ -0,0 +1,81 @@
+"""Per-test composition root of the library: the objects one test owns."""
+
+from __future__ import annotations
+
+import atexit
+
+from .cache import RunBudgets
+from .config import Config
+from .driver import DriverSession, PageFacade
+from .llm import LLMProvider, create_provider
+
+
+class PrettyplayRuntime:
+    """The composition root: the per-test objects one test composes — nothing shared between tests.
+
+    Attributes:
+        _config: validated project settings of the test.
+        _budgets: the per-test attempt registry owned by this runtime.
+        _driver: the browser driver of the test; ``None`` until first access.
+        _provider: the LLM provider of the test; ``None`` until first access.
+    """
+
+    def __init__(self, config: Config) -> None:
+        """Compose the eager per-test objects from the config.
+
+        Args:
+            config: validated project settings of the test.
+        """
+        self._config = config
+        self._budgets = RunBudgets(config.generation_attempts, config.healing_attempts)
+        self._driver: DriverSession | None = None
+        self._provider: LLMProvider | None = None
+        atexit.register(self.close)
+
+    @property
+    def config(self) -> Config:
+        """The validated settings of the test."""
+        return self._config
+
+    @property
+    def budgets(self) -> RunBudgets:
+        """The per-test attempt registry owned by this one runtime."""
+        return self._budgets
+
+    @property
+    def driver(self) -> DriverSession:
+        """The browser driver of the test, constructed lazily exactly once.
+
+        Returns:
+            The browser driver — a construction after a ``close`` re-arms
+            the ``atexit`` hook the close dropped.
+        """
+        if self._driver is None:
+            self._driver = DriverSession(self._config)
+            atexit.register(self.close)
+
+        return self._driver
+
+    @property
+    def provider(self) -> LLMProvider:
+        """The LLM provider of the test, constructed lazily exactly once."""
+        if self._provider is None:
+            self._provider = create_provider(self._config)
+
+        return self._provider
+
+    def open_page(self) -> PageFacade:
+        """Open a fresh isolated page of the test driver.
+
+        Returns:
+            The facade of the new page of a fresh isolated context.
+        """
+        return self.driver.open_context()
+
+    def close(self) -> None:
+        """Stop the browser driver of the test; idempotent; drops the ``atexit`` hook."""
+        atexit.unregister(self.close)
+
+        if self._driver is not None:
+            self._driver.close()
+            self._driver = None