aetherius 0.2.0__py3-none-any.whl

aetherius/recorder/_vector_js.py

@@ -0,0 +1,174 @@
+"""In-page capture for the Vector recorder: observe API calls, pick a request and its JSON fields.
+
+Symmetric with the DOM recorder, but for Act I. It patches ``fetch``/``XMLHttpRequest`` (and treats a
+JSON *document* — navigating straight to an API URL — as a request too), lists the JSON responses in a
+floating panel, and lets the user turn one into an ``http.request`` + JSONPath extraction. The panel
+shows a **structured summary** of the response (a top-level array → "extract N records" with
+auto-detected scalar fields; an object → a chip per key) rather than an infinite tree: simpler, and it
+matches real API shapes (a list of records, or an object of fields/arrays).
+
+Everything funnels through ``window.__aeVector.emit(request, extract)``, which the panel calls and
+integration tests drive directly. The Python side (:mod:`.vector_backend`) turns it into a Blueprint.
+"""
+
+from __future__ import annotations
+
+# Self-invoking, guarded. Emits through the shared __aetherius_capture binding with the http_request
+# kind. Auth-bearing headers are handled Python-side (turned into secrets), never stored literally.
+VECTOR_JS = r"""
+(() => {
+  if (window.__aeVector) return;
+  const OVERLAY_ID = '__ae-recorder-overlay';
+  const ACCENT = '#c9a94a';
+  const requests = [];
+  const send = (p) => { try { window.__aetherius_capture(JSON.stringify(p)); } catch (e) {} };
+
+  const headersToObj = (h) => {
+    const o = {};
+    try {
+      if (h && typeof h.forEach === 'function') h.forEach((v, k) => { o[k] = v; });
+      else if (h && typeof h === 'object') Object.assign(o, h);
+    } catch (e) {}
+    return o;
+  };
+  const bodyToStr = (b) => (b == null ? null : (typeof b === 'string' ? b : null));
+
+  let refresh = () => {};
+  const record = (r) => {
+    if ((r.contentType || '').includes('json')) { requests.push(r); refresh(); }
+  };
+
+  const origFetch = window.fetch;
+  if (origFetch) {
+    window.fetch = function (...args) {
+      const arg0 = args[0], init = args[1] || {};
+      const url = (typeof arg0 === 'string') ? arg0 : (arg0 && arg0.url) || '';
+      const method = (init.method || (arg0 && arg0.method) || 'GET').toUpperCase();
+      const headers = headersToObj(init.headers);
+      const body = bodyToStr(init.body);
+      return origFetch.apply(this, args).then((resp) => {
+        const ct = resp.headers.get('content-type') || '';
+        resp.clone().text()
+          .then((text) => record({ method, url, headers, body, status: resp.status, contentType: ct, responseText: text }))
+          .catch(() => {});
+        return resp;
+      });
+    };
+  }
+
+  const xOpen = XMLHttpRequest.prototype.open, xSend = XMLHttpRequest.prototype.send;
+  XMLHttpRequest.prototype.open = function (m, u) {
+    this.__ae = { method: (m || 'GET').toUpperCase(), url: u };
+    return xOpen.apply(this, arguments);
+  };
+  XMLHttpRequest.prototype.send = function (b) {
+    const xhr = this, info = xhr.__ae || {};
+    xhr.addEventListener('load', () => {
+      const ct = xhr.getResponseHeader('content-type') || '';
+      record({ method: info.method, url: info.url, headers: {}, body: bodyToStr(b),
+               status: xhr.status, contentType: ct, responseText: xhr.responseText });
+    });
+    return xSend.apply(this, arguments);
+  };
+
+  window.__aeVector = {
+    requests,
+    emit(request, extract) { send({ kind: 'http_request', request, extract }); },
+  };
+
+  // ── UI ──────────────────────────────────────────────────────────────────────
+  const el = (tag, css, text) => {
+    const n = document.createElement(tag);
+    if (css) n.style.cssText = css;
+    if (text != null) n.textContent = text;
+    return n;
+  };
+  const chip = (label, onclick) => {
+    const b = el('button', 'margin:2px;padding:3px 7px;font:12px system-ui;cursor:pointer;' +
+      'border-radius:5px;border:1px solid ' + ACCENT + '55;background:#1c1830;color:#e7e2f3;', label);
+    b.addEventListener('click', (e) => { e.preventDefault(); e.stopPropagation(); onclick(); });
+    return b;
+  };
+
+  const scalarFields = (obj) => {  // { key: '$.key' } for each scalar field of a sample record
+    const out = {};
+    if (obj && typeof obj === 'object' && !Array.isArray(obj)) {
+      for (const k of Object.keys(obj)) {
+        const v = obj[k];
+        if (v == null || typeof v !== 'object') out[k] = '$.' + k;
+      }
+    }
+    return out;
+  };
+
+  let host, root, statusEl, listEl, detailEl;
+  const setStatus = (t) => { if (statusEl) statusEl.textContent = t; };
+
+  const pickRecords = (request, path, sample, name) => {
+    window.__aeVector.emit(request, { name, path, fields: scalarFields(sample) });
+    setStatus('Added records "' + name + '". Pick more, or Finish.');
+  };
+  const pickValue = (request, path, name) => {
+    window.__aeVector.emit(request, { name, path });
+    setStatus('Added "' + name + '". Pick more, or Finish.');
+  };
+
+  const showDetail = (request) => {
+    detailEl.innerHTML = '';
+    let data;
+    try { data = JSON.parse(request.responseText); } catch (e) { detailEl.textContent = 'Not JSON.'; return; }
+    detailEl.appendChild(el('div', 'color:' + ACCENT + ';margin:4px 0;', request.method + ' ' + request.url));
+    if (Array.isArray(data)) {
+      detailEl.appendChild(chip('Extract ' + data.length + ' records', () =>
+        pickRecords(request, '$[*]', data[0], 'records')));
+    } else if (data && typeof data === 'object') {
+      for (const k of Object.keys(data)) {
+        const v = data[k];
+        if (Array.isArray(v)) {
+          detailEl.appendChild(chip(k + ' (' + v.length + ' records)', () =>
+            pickRecords(request, '$.' + k + '[*]', v[0], k)));
+        } else if (v == null || typeof v !== 'object') {
+          detailEl.appendChild(chip(k, () => pickValue(request, '$.' + k, k)));
+        }
+      }
+    } else {
+      detailEl.appendChild(chip('Extract value', () => pickValue(request, '$', 'value')));
+    }
+  };
+
+  const install = () => {
+    if (!document.documentElement || document.getElementById(OVERLAY_ID)) return;
+    host = el('div'); host.id = OVERLAY_ID; host.style.cssText = 'all:initial;position:fixed;z-index:2147483647;';
+    document.documentElement.appendChild(host);
+    root = host.attachShadow({ mode: 'open' });
+    const bar = el('div', 'position:fixed;top:10px;right:10px;width:320px;max-height:80vh;overflow:auto;' +
+      'padding:8px 10px;font:12px system-ui;background:#171327ee;color:#e7e2f3;' +
+      'border:1px solid ' + ACCENT + '66;border-radius:8px;box-shadow:0 6px 24px #000a;');
+    bar.appendChild(el('div', 'font-weight:600;color:' + ACCENT + ';', 'Aetherius recorder — API'));
+    statusEl = el('div', 'color:#b9b2d0;margin:4px 0;', 'Browse to trigger API calls, then pick one.');
+    bar.appendChild(statusEl);
+    bar.appendChild(el('div', 'color:#8f88a8;margin-top:4px;', 'Requests'));
+    listEl = el('div'); bar.appendChild(listEl);
+    detailEl = el('div', 'margin-top:6px;border-top:1px solid #ffffff22;padding-top:4px;'); bar.appendChild(detailEl);
+    bar.appendChild(chip('Finish', () => send({ kind: 'finish' })));
+    root.appendChild(bar);
+
+    refresh = () => {
+      listEl.innerHTML = '';
+      requests.slice(-12).forEach((r) => {
+        const label = r.method + ' ' + (r.url.length > 34 ? '…' + r.url.slice(-33) : r.url);
+        listEl.appendChild(chip(label, () => showDetail(r)));
+      });
+    };
+    refresh();
+    if ((document.contentType || '').includes('json')) {
+      record({ method: 'GET', url: location.href, headers: {}, body: null,
+               status: 200, contentType: 'application/json',
+               responseText: document.body ? document.body.textContent : '' });
+    }
+  };
+
+  if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', install);
+  else install();
+})();
+"""

aetherius/recorder/base.py

@@ -0,0 +1,92 @@
+"""Recorder backends behind a common interface, selected by Act — a mirror of the driver pattern.
+
+A recorder captures a demonstration in a live browser and turns it into a Blueprint. *How* it captures
+and *what* it emits depends on the Act: Continuum records DOM interactions, Vector records network
+calls. Both plug into the same generic browser shell (:mod:`.session`) through this interface, and the
+registry maps ``act -> backend`` exactly as ``ACT_CAPABILITIES`` maps ``act -> capabilities``.
+
+Acts without a recorder yet (Oracle, Phantom) are declared here as pending holes: asking for one
+raises a clear :class:`RecorderError` instead of failing obscurely, so the CLI/Console can say so
+honestly — the same "honest milestone" pattern used elsewhere (PendingScreen, DependencyError).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Callable, Protocol
+
+from ..core.errors import RecorderError
+
+if TYPE_CHECKING:
+    from .session import RecordingSession
+
+# Acts that are real Aetherius Acts but whose recorder is a later milestone (documented in
+# docs/recorder.md). Kept distinct from a plainly unknown act so the message can be specific.
+_PENDING_RECORDERS: dict[str, str] = {
+    "oracle": "visual annotation (bounding boxes on screenshots) feeding training/",
+    "phantom": "goal-based demonstration",
+}
+
+_BACKENDS: dict[str, Callable[..., "RecorderBackend"]] = {}
+
+
+@dataclass
+class RecordingResult:
+    """What a backend produces from a session: the pieces of the Blueprint (minus the envelope)."""
+
+    steps: list[dict[str, Any]] = field(default_factory=list)
+    secrets: list[str] = field(default_factory=list)
+    inputs: dict[str, Any] = field(default_factory=dict)
+    outputs: dict[str, Any] = field(default_factory=dict)
+
+
+class RecorderBackend(Protocol):
+    """One Act's way of recording. The generic session drives it through these three hooks."""
+
+    act: str
+
+    def init_scripts(self) -> list[str]:
+        """Scripts to inject (``add_init_script``) into every document before the page runs."""
+        ...
+
+    def attach(self, session: "RecordingSession") -> None:
+        """Register bindings and page hooks on *session* (called once the context exists)."""
+        ...
+
+    def result(self) -> RecordingResult:
+        """Transform everything captured during the session into a RecordingResult."""
+        ...
+
+
+def register_backend(act: str, factory: Callable[..., "RecorderBackend"]) -> None:
+    """Register a backend factory for *act* (called at backend-module import).
+
+    The factory accepts arbitrary keyword options and picks the ones its Act understands, so
+    ``get_recorder`` can forward Act-specific knobs (e.g. ``credentials_as_secrets``) uniformly.
+    """
+    _BACKENDS[act] = factory
+
+
+def _ensure_loaded() -> None:
+    """Import the backend modules so their registrations run (idempotent, dependency-light)."""
+    from . import continuum_backend, vector_backend  # noqa: F401  (import for side effects)
+
+
+def get_recorder(act: str, **options: Any) -> "RecorderBackend":
+    """Return a fresh backend for *act* (forwarding *options*), or raise a clear RecorderError."""
+    _ensure_loaded()
+    factory = _BACKENDS.get(act)
+    if factory is not None:
+        return factory(**options)
+    if act in _PENDING_RECORDERS:
+        raise RecorderError(
+            f"The recorder for act {act!r} is a pending milestone "
+            f"({_PENDING_RECORDERS[act]}); see docs/recorder.md."
+        )
+    raise RecorderError(f"Unknown act {act!r} (no recorder backend).")
+
+
+def recorder_acts() -> set[str]:
+    """Acts that have a working recorder today (single source of truth, like IMPLEMENTED_ACTS)."""
+    _ensure_loaded()
+    return set(_BACKENDS)

aetherius/recorder/blueprint_recorder.py

@@ -0,0 +1,72 @@
+"""Blueprint recorder: launch a visible browser, capture a demonstration, emit a clean Blueprint.
+
+Act-agnostic orchestrator: it picks the recorder backend for the requested Act (:mod:`.base`), drives
+a generic :class:`~.session.RecordingSession`, assembles the backend's result into a Blueprint, and
+re-validates the produced file through the canonical loader/validator — so the recorder can only ever
+hand back a schema-valid, runnable Blueprint (or fail loudly). What is captured and how it maps to
+steps is the backend's job (Continuum: DOM; Vector: network).
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from pathlib import Path
+
+from ..builder.factory import assemble_blueprint, slugify_name
+from ..core.blueprint.loader import load_blueprint
+from ..core.blueprint.validator import validate_for_act
+from ..core.errors import BlueprintValidationError
+from .base import get_recorder
+from .session import NotifyCallback, RecordingSession
+
+# assemble_blueprint is re-exported from builder.factory: Blueprint construction is the builder's
+# job, and the recorder is one of its callers. Kept importable here for existing callers and tests.
+__all__ = ["assemble_blueprint", "record_blueprint"]
+
+
+def record_blueprint(
+    name: str,
+    start_url: str,
+    *,
+    act: str = "continuum",
+    out_dir: Path | str | None = None,
+    on_event: NotifyCallback | None = None,
+    stop_event: threading.Event | None = None,
+    credentials_as_secrets: bool = True,
+) -> Path:
+    """Record a demonstration for *act* in a live browser and write a validated Blueprint file.
+
+    Returns the path of the written ``.blueprint.json`` (under *out_dir* or ``./blueprints/``).
+    Raises :class:`~aetherius.core.errors.RecorderError` if *act* has no recorder backend, or
+    :class:`BlueprintValidationError` if the assembled Blueprint fails canonical validation.
+    """
+    backend = get_recorder(act, credentials_as_secrets=credentials_as_secrets)
+    session = RecordingSession(start_url, backend, on_event=on_event, stop_event=stop_event)
+    result = session.record()
+
+    description = (
+        f"Genere par le recorder Aetherius (act {act}) a partir d'une demo sur {start_url}."
+    )
+    blueprint = assemble_blueprint(
+        name,
+        result.steps,
+        result.secrets,
+        act=act,
+        inputs=result.inputs,
+        outputs=result.outputs,
+        description=description,
+    )
+
+    target_dir = Path(out_dir) if out_dir is not None else Path.cwd() / "blueprints"
+    target_dir.mkdir(parents=True, exist_ok=True)
+    path = target_dir / f"{slugify_name(name)}.blueprint.json"
+    path.write_text(json.dumps(blueprint, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+    try:
+        validate_for_act(load_blueprint(path))
+    except Exception as exc:  # a produced-invalid Blueprint is our bug: surface it, do not hide it
+        raise BlueprintValidationError(
+            f"The recorder produced an invalid Blueprint: {exc}"
+        ) from exc
+    return path

aetherius/recorder/capture.py

@@ -0,0 +1,59 @@
+"""DOM capture data types shared by the Continuum backend and its tests.
+
+The live-browser lifecycle now lives in the generic :mod:`.session`; this module holds the typed
+representation of a captured DOM event and the parser that builds an element descriptor from the JSON
+the in-page scripts report (:mod:`._capture_js`, :mod:`._overlay_js`).
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from .selector_synth import Candidate, ElementDescriptor
+
+# Overlay pick kinds, carrying an extraction/meta ``config`` rather than a plain action. ``finish`` is
+# a control message handled by the backend (end the session), never turned into a recorded event.
+_OVERLAY_KINDS = frozenset({"extract", "extract_records", "wait_for", "parameterize"})
+
+
+@dataclass(frozen=True, slots=True)
+class RecordedEvent:
+    """One captured user action, before it is transformed into a Blueprint step.
+
+    ``descriptor`` is the target element (the repeating container for records) or ``None`` for a
+    navigation. A password field never carries its value: ``redacted`` is set instead. ``config``
+    carries an overlay pick's details (name / as / attr / list / fields / input name).
+    """
+
+    kind: str  # navigate | click | fill | select | press | extract | extract_records | wait_for | parameterize
+    descriptor: ElementDescriptor | None = None
+    value: str | None = None
+    key: str | None = None
+    url: str | None = None
+    redacted: bool = False
+    config: dict[str, Any] | None = None
+    ts: float = field(default_factory=time.monotonic)
+
+
+def _descriptor_from_raw(raw: dict[str, Any]) -> ElementDescriptor:
+    """Build an :class:`ElementDescriptor` from the JSON an in-page script sent."""
+    candidates = tuple(
+        Candidate(
+            strategy=str(c["strategy"]),
+            selector=str(c["selector"]),
+            selector_type=str(c.get("selector_type", "css")),
+            unique=bool(c["unique"]),
+        )
+        for c in raw.get("candidates", [])
+    )
+    return ElementDescriptor(
+        tag=str(raw.get("tag", "")),
+        css_path=str(raw.get("css_path", "")),
+        candidates=candidates,
+        text=raw.get("text"),
+        name=raw.get("name"),
+        field_type=raw.get("field_type"),
+        autocomplete=raw.get("autocomplete"),
+    )

aetherius/recorder/continuum_backend.py

@@ -0,0 +1,119 @@
+"""Continuum recorder backend: capture DOM interactions and picks into a browser Blueprint.
+
+Injects the shared selector primitives, the action-capture script and the pick overlay; collects the
+DOM events they report (plus top-level navigations); and transforms them into ``continuum`` steps via
+:mod:`._transform`. This is the Act II half of the Act-agnostic recorder (:mod:`.base`).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from ._capture_js import RECORDER_JS
+from ._overlay_js import OVERLAY_JS
+from ._selector_js import SELECTOR_JS
+from ._transform import events_to_steps
+from .base import RecordingResult, register_backend
+from .capture import _OVERLAY_KINDS, RecordedEvent, _descriptor_from_raw
+from .selector_synth import synthesize
+from .session import RecordingSession
+
+
+def describe_event(event: RecordedEvent) -> str:
+    """Sober one-line description of a captured DOM action, for the Console event log / CLI."""
+    if event.kind == "navigate" or (event.kind == "click" and event.url):
+        return f"navigate  {event.url or ''}"
+    selector = synthesize(event.descriptor).selector if event.descriptor else "?"
+    config = event.config or {}
+    if event.kind == "extract":
+        return f"extract {config.get('name', '?')}  {selector}"
+    if event.kind == "extract_records":
+        fields = ", ".join(f.get("name", "?") for f in config.get("fields", []))
+        return f"extract table {config.get('name', '?')} ({fields})  {selector}"
+    if event.kind == "wait_for":
+        return f"wait_for  {selector}"
+    if event.kind == "parameterize":
+        return f"input {config.get('name', '?')}  {selector}"
+    if event.kind == "press":
+        return f"press {event.key or 'Enter'}  {selector}"
+    return f"{event.kind}  {selector}"
+
+
+class ContinuumRecorder:
+    """Records DOM interactions and overlay picks into a ``continuum`` Blueprint."""
+
+    act = "continuum"
+
+    def __init__(self, *, credentials_as_secrets: bool = True) -> None:
+        self._credentials_as_secrets = credentials_as_secrets
+        self._events: list[RecordedEvent] = []
+        self._last_nav_url: str | None = None
+        self._session: RecordingSession | None = None
+
+    def init_scripts(self) -> list[str]:
+        return [SELECTOR_JS, RECORDER_JS, OVERLAY_JS]
+
+    def attach(self, session: RecordingSession) -> None:
+        self._session = session
+        session.expose("__aetherius_capture", self._on_binding)
+        session.on_new_page(self._wire_page)
+
+    def result(self) -> RecordingResult:
+        steps, secrets, inputs, outputs = events_to_steps(
+            self._events, credentials_as_secrets=self._credentials_as_secrets
+        )
+        return RecordingResult(steps=steps, secrets=secrets, inputs=inputs, outputs=outputs)
+
+    # ── capture ────────────────────────────────────────────────────────────────
+    def _wire_page(self, page: Any) -> None:
+        page.on("framenavigated", self._on_framenavigated)
+
+    def _on_framenavigated(self, frame: Any) -> None:
+        if frame.parent_frame is not None:
+            return  # only the main frame is a user-visible navigation
+        url = frame.url
+        if not url or url == "about:blank" or url == self._last_nav_url:
+            return
+        self._last_nav_url = url
+        self._emit(RecordedEvent(kind="navigate", url=url))
+
+    def _on_binding(self, _source: dict[str, Any], payload: str) -> None:
+        try:
+            data = json.loads(payload)
+        except (TypeError, ValueError):
+            return
+        kind = str(data.get("kind", ""))
+        if kind == "finish":
+            if self._session is not None:
+                self._session.finish()  # overlay Finish: end without closing the window
+            return
+        raw = data.get("descriptor")
+        descriptor = _descriptor_from_raw(raw) if raw else None
+        if kind in _OVERLAY_KINDS:
+            config = {k: v for k, v in data.items() if k not in ("kind", "descriptor")}
+            self._emit(RecordedEvent(kind=kind, descriptor=descriptor, config=config))
+            return
+        self._emit(
+            RecordedEvent(
+                kind=kind,
+                descriptor=descriptor,
+                value=data.get("value"),
+                key=data.get("key"),
+                url=data.get("href"),  # set for a link click: replayed as a navigate, not a click
+                redacted=bool(data.get("redacted", False)),
+            )
+        )
+
+    def _emit(self, event: RecordedEvent) -> None:
+        self._events.append(event)
+        if self._session is not None:
+            self._session.notify(describe_event(event))
+
+
+register_backend(
+    "continuum",
+    lambda **o: ContinuumRecorder(
+        credentials_as_secrets=bool(o.get("credentials_as_secrets", True))
+    ),
+)