frontend-perception-engine 0.1.0__py3-none-any.whl

navigation/perception/exploration.py

@@ -0,0 +1,99 @@
+"""Phase 4: CRG-guided exploration with live verification."""
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any
+
+from navigation.browser_use.hints import GraphHintResolver
+from navigation.codeGraph.interface import ICodeGraph
+
+from .verification import SuccessCriteria, read_current_url, verify
+
+
+@dataclass(slots=True)
+class ExplorationStep:
+    action: str
+    detail: str
+    url: str = ""
+
+    def to_dict(self) -> dict:
+        return {"action": self.action, "detail": self.detail, "url": self.url}
+
+
+@dataclass(slots=True)
+class ExplorationResult:
+    goal: str
+    ok: bool
+    final_url: str
+    steps: list[ExplorationStep] = field(default_factory=list)
+    hint_summary: str = ""
+    error: str | None = None
+
+    def to_dict(self) -> dict:
+        return {
+            "goal": self.goal,
+            "ok": self.ok,
+            "final_url": self.final_url,
+            "hint_summary": self.hint_summary,
+            "steps": [s.to_dict() for s in self.steps],
+            "error": self.error,
+        }
+
+
+async def explore_with_hints(
+    session: Any,
+    code_graph: ICodeGraph | None,
+    goal: str,
+    *,
+    base_url: str,
+    success: SuccessCriteria,
+    candidate_paths: list[str] | None = None,
+) -> ExplorationResult:
+    """Use CRG hints to pick a path, navigate, verify — no blind agent loop."""
+    steps: list[ExplorationStep] = []
+    resolver = GraphHintResolver(code_graph)
+    hint = resolver.resolve(goal)
+    steps.append(ExplorationStep("crg.resolve", hint.summary, await read_current_url(session)))
+
+    paths = list(candidate_paths or [])
+    for kw in resolver.extract_keywords(goal):
+        if kw not in paths:
+            paths.append(f"/{kw}")
+
+    # CRG-related file paths often mention edge-lab
+    for fp in hint.related_files:
+        if "edge" in fp.lower():
+            paths.insert(0, "/edge-lab")
+
+    if "/edge-lab" not in paths:
+        paths.append("/edge-lab")
+
+    b = base_url.rstrip("/")
+    ok = False
+    final_url = ""
+    for path in paths:
+        target = path if path.startswith("http") else f"{b}/{path.lstrip('/')}"
+        try:
+            await session.navigate_to(target)
+            await asyncio.sleep(0.35)
+            final_url = await read_current_url(session)
+            steps.append(ExplorationStep("navigate", target, final_url))
+            result = await verify(session, success)
+            if result.ok:
+                ok = True
+                break
+        except Exception as exc:
+            steps.append(ExplorationStep("navigate.error", str(exc), await read_current_url(session)))
+
+    if not ok:
+        final_url = await read_current_url(session)
+
+    return ExplorationResult(
+        goal=goal,
+        ok=ok,
+        final_url=final_url,
+        steps=steps,
+        hint_summary=hint.summary,
+        error=None if ok else "no candidate path verified",
+    )

navigation/perception/feature_flags.py

@@ -0,0 +1,83 @@
+"""Phase 4: detect feature-gated UI vs hard failures."""
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any
+
+from .verification import read_page_text, read_current_url
+
+
+@dataclass(slots=True)
+class FeatureFlagProbe:
+    route: str
+    flag_name: str
+    available_without_flag: bool
+    available_with_flag: bool
+    in_code_graph: bool = True
+    status: str = "unknown"  # feature_gated | enabled | missing
+
+    def to_dict(self) -> dict:
+        return {
+            "route": self.route,
+            "flag_name": self.flag_name,
+            "available_without_flag": self.available_without_flag,
+            "available_with_flag": self.available_with_flag,
+            "in_code_graph": self.in_code_graph,
+            "status": self.status,
+        }
+
+
+@dataclass(slots=True)
+class FeatureFlagResult:
+    ok: bool
+    probes: list[FeatureFlagProbe] = field(default_factory=list)
+    error: str | None = None
+
+    def to_dict(self) -> dict:
+        return {"ok": self.ok, "probes": [p.to_dict() for p in self.probes], "error": self.error}
+
+
+async def probe_feature_flag(
+    session: Any,
+    base_url: str,
+    route: str,
+    *,
+    flag_query: str,
+    feature_text: str,
+) -> FeatureFlagResult:
+    b = base_url.rstrip("/")
+    path = route if route.startswith("/") else f"/{route}"
+    probes: list[FeatureFlagProbe] = []
+
+    try:
+        await session.navigate_to(f"{b}{path}")
+        await asyncio.sleep(0.3)
+        text_off = (await read_page_text(session, include_dom_text=True)).lower()
+        off_has_feature = feature_text.lower() in text_off
+
+        await session.navigate_to(f"{b}{path}?{flag_query}")
+        await asyncio.sleep(0.3)
+        text_on = (await read_page_text(session, include_dom_text=True)).lower()
+        on_has_feature = feature_text.lower() in text_on
+
+        status = "feature_gated"
+        if on_has_feature and not off_has_feature:
+            status = "feature_gated"
+        elif on_has_feature and off_has_feature:
+            status = "enabled"
+        else:
+            status = "missing"
+
+        probe = FeatureFlagProbe(
+            route=path,
+            flag_name=flag_query.split("=")[0],
+            available_without_flag=off_has_feature,
+            available_with_flag=on_has_feature,
+            status=status,
+        )
+        probes.append(probe)
+        ok = status == "feature_gated"
+        return FeatureFlagResult(ok=ok, probes=probes)
+    except Exception as exc:
+        return FeatureFlagResult(ok=False, probes=probes, error=str(exc))

navigation/perception/file_upload.py

@@ -0,0 +1,84 @@
+"""Phase 4: programmatic file upload via CDP."""
+from __future__ import annotations
+
+import asyncio
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .verification import evaluate_js, read_page_text
+
+
+@dataclass(slots=True)
+class FileUploadResult:
+    ok: bool
+    filename: str
+    displayed: bool
+    error: str | None = None
+
+    def to_dict(self) -> dict:
+        return {
+            "ok": self.ok,
+            "filename": self.filename,
+            "displayed": self.displayed,
+            "error": self.error,
+        }
+
+
+async def upload_test_file(
+    session: Any,
+    *,
+    input_selector: str = '[data-testid="file-input"]',
+    content: str = "edge-lab upload test",
+    suffix: str = ".txt",
+) -> FileUploadResult:
+    tmp_path: Path | None = None
+    try:
+        with tempfile.NamedTemporaryFile("w", suffix=suffix, delete=False, encoding="utf-8") as f:
+            f.write(content)
+            tmp_path = Path(f.name)
+
+        filename = tmp_path.name
+        cdp = await session.get_or_create_cdp_session(target_id=None, focus=True)
+        doc = await cdp.cdp_client.send.DOM.getDocument(session_id=cdp.session_id)
+        root_id = doc["root"]["nodeId"]
+        sel = await cdp.cdp_client.send.DOM.querySelector(
+            params={"nodeId": root_id, "selector": input_selector},
+            session_id=cdp.session_id,
+        )
+        node_id = sel.get("nodeId")
+        if not node_id:
+            return FileUploadResult(False, filename, False, "file input not found")
+
+        await cdp.cdp_client.send.DOM.setFileInputFiles(
+            params={"files": [str(tmp_path.resolve())], "nodeId": node_id},
+            session_id=cdp.session_id,
+        )
+
+        await evaluate_js(
+            session,
+            f"""
+            (() => {{
+              const input = document.querySelector({input_selector!r});
+              if (!input) return false;
+              input.dispatchEvent(new Event('change', {{ bubbles: true }}));
+              input.dispatchEvent(new Event('input', {{ bubbles: true }}));
+              return true;
+            }})()
+            """,
+        )
+        await asyncio.sleep(0.3)
+
+        page = await read_page_text(session, include_dom_text=True)
+        base_name = Path(filename).name
+        displayed = base_name in page or "uploaded:" in page.lower()
+        return FileUploadResult(displayed, base_name, displayed)
+    except Exception as exc:
+        return FileUploadResult(False, tmp_path.name if tmp_path else "", False, str(exc))
+    finally:
+        if tmp_path and tmp_path.exists():
+            try:
+                tmp_path.unlink()
+            except Exception:
+                pass

navigation/perception/flow_graph.py

@@ -0,0 +1,121 @@
+"""Phase 3: flow graph — verified checkpoints, not just routes."""
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+
+from .verification import SuccessCriteria
+
+
+@dataclass(slots=True)
+class Checkpoint:
+    name: str
+    success: SuccessCriteria
+    instruction: str | None = None
+    navigate_url: str | None = None
+    ensure_url: str | None = None
+    prerequisite_state: str | None = None
+    max_steps: int = 8
+    max_attempts: int = 3
+
+    @property
+    def is_deterministic(self) -> bool:
+        return self.instruction is None and self.navigate_url is not None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "mode": "deterministic" if self.is_deterministic else "agent",
+            "navigate_url": self.navigate_url,
+            "instruction": self.instruction,
+            "ensure_url": self.ensure_url,
+            "prerequisite_state": self.prerequisite_state,
+            "success": {
+                "url_contains": self.success.url_contains,
+                "text_contains": self.success.text_contains,
+            },
+        }
+
+
+@dataclass(slots=True)
+class FlowGraph:
+    name: str
+    description: str
+    checkpoints: list[Checkpoint] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "description": self.description,
+            "checkpoints": [c.to_dict() for c in self.checkpoints],
+        }
+
+
+def validation_form_flow() -> FlowGraph:
+    return FlowGraph(
+        name="validation-form",
+        description="Probe validation rules then submit valid data",
+        checkpoints=[
+            Checkpoint(
+                name="open-form",
+                navigate_url="/forms/validation",
+                success=SuccessCriteria(
+                    url_contains=["/forms/validation"],
+                    text_contains=["Validated form"],
+                ),
+            ),
+            Checkpoint(
+                name="submit-empty",
+                instruction="Click Validate & submit without filling fields",
+                ensure_url="/forms/validation",
+                success=SuccessCriteria(
+                    url_contains=["/forms/validation"],
+                    text_contains=["Invalid email address"],
+                ),
+                max_steps=4,
+            ),
+            Checkpoint(
+                name="submit-valid",
+                instruction=(
+                    "Fill Email test@example.com, Phone 1234567890, Age 25, "
+                    "check terms, click Validate & submit"
+                ),
+                ensure_url="/forms/validation",
+                success=SuccessCriteria(
+                    url_contains=["/forms/validation"],
+                    text_contains=["Form is valid"],
+                ),
+                max_steps=8,
+            ),
+        ],
+    )
+
+
+def shop_order_flow() -> FlowGraph:
+    return FlowGraph(
+        name="shop-order",
+        description="Buy Pulse Watch through checkout",
+        checkpoints=[
+            Checkpoint(
+                name="open-product",
+                navigate_url="/shop/product/p3",
+                success=SuccessCriteria(url_contains=["/shop/product/p3"]),
+            ),
+            Checkpoint(
+                name="checkout-to-confirmation",
+                instruction=(
+                    "Buy now, checkout, fill shipping and payment, place order"
+                ),
+                prerequisite_state="cart_with_items",
+                success=SuccessCriteria(url_contains=["/shop/checkout/confirmation"]),
+                max_steps=35,
+            ),
+        ],
+    )
+
+
+FLOWS: dict[str, Callable[[], FlowGraph]] = {
+    "validation-form": validation_form_flow,
+    "shop-order": shop_order_flow,
+}

navigation/perception/form_probe.py

@@ -0,0 +1,148 @@
+"""Phase 1: probe forms — invalid submit first, extract rules, then valid submit."""
+from __future__ import annotations
+
+import asyncio
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from .scripted_actions import click_button_text
+from .verification import SuccessCriteria, evaluate_js, read_page_text, verify
+
+
+@dataclass(slots=True)
+class ValidationRule:
+    field: str
+    message: str
+    source: str = "live_probe"
+
+    def to_dict(self) -> dict:
+        return {"field": self.field, "message": self.message, "source": self.source}
+
+
+@dataclass(slots=True)
+class FormProbeResult:
+    form_url: str
+    ok: bool
+    rules: list[ValidationRule] = field(default_factory=list)
+    invalid_verified: bool = False
+    valid_verified: bool = False
+    error: str | None = None
+
+    def to_dict(self) -> dict:
+        return {
+            "form_url": self.form_url,
+            "ok": self.ok,
+            "rules": [r.to_dict() for r in self.rules],
+            "invalid_verified": self.invalid_verified,
+            "valid_verified": self.valid_verified,
+            "error": self.error,
+        }
+
+
+async def _extract_error_messages(session: Any) -> list[ValidationRule]:
+    raw = await evaluate_js(
+        session,
+        """
+        Array.from(document.querySelectorAll('.error-text'))
+          .map(el => el.textContent.trim())
+          .filter(Boolean)
+        """,
+    )
+    rules: list[ValidationRule] = []
+    if not raw:
+        return rules
+    messages = raw if isinstance(raw, list) else [raw]
+    field_hints = {
+        "email": "email",
+        "phone": "phone",
+        "age": "age",
+        "18": "age",
+        "terms": "terms",
+        "digit": "phone",
+    }
+    for msg in messages:
+        msg_str = str(msg)
+        field = "unknown"
+        lower = msg_str.lower()
+        for hint, name in field_hints.items():
+            if hint in lower:
+                field = name
+                break
+        rules.append(ValidationRule(field=field, message=msg_str))
+    return rules
+
+
+async def _fill_validation_form(session: Any) -> bool:
+    return bool(
+        await evaluate_js(
+            session,
+            """
+            (() => {
+              const form = document.querySelector('form.card');
+              if (!form) return false;
+              const inputs = Array.from(form.querySelectorAll('input'));
+              if (inputs.length < 4) return false;
+              const [email, phone, age, terms] = inputs;
+              const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set;
+              if (!setter) return false;
+              for (const [el, val] of [[email, 'test@example.com'], [phone, '1234567890'], [age, '25']]) {
+                setter.call(el, val);
+                el.dispatchEvent(new Event('input', { bubbles: true }));
+                el.dispatchEvent(new Event('change', { bubbles: true }));
+              }
+              if (!terms.checked) terms.click();
+              return true;
+            })()
+            """,
+        )
+    )
+
+
+EXPECTED_RULES = (
+    "Invalid email address",
+    "10 digits",
+    "18 or older",
+    "accept the terms",
+)
+
+
+async def probe_validation_form(session: Any, base_url: str) -> FormProbeResult:
+    path = "/forms/validation"
+    url = f"{base_url.rstrip('/')}{path}"
+    try:
+        await session.navigate_to(url)
+        await asyncio.sleep(0.3)
+
+        await click_button_text(session, "Validate & submit")
+        await asyncio.sleep(0.3)
+
+        invalid = await verify(
+            session,
+            SuccessCriteria(url_contains=["/forms/validation"], text_contains=["Invalid email address"]),
+        )
+        rules = await _extract_error_messages(session)
+
+        await _fill_validation_form(session)
+        await click_button_text(session, "Validate & submit")
+        await asyncio.sleep(0.3)
+
+        valid = await verify(
+            session,
+            SuccessCriteria(url_contains=["/forms/validation"], text_contains=["Form is valid"]),
+        )
+
+        haystack = (await read_page_text(session, include_dom_text=True)).lower()
+        expected_hits = sum(1 for e in EXPECTED_RULES if e.lower() in haystack or any(e.lower() in r.message.lower() for r in rules))
+
+        ok = invalid.ok and valid.ok and len(rules) >= 1
+        return FormProbeResult(
+            form_url=url,
+            ok=ok,
+            rules=rules,
+            invalid_verified=invalid.ok,
+            valid_verified=valid.ok,
+            error=None if ok else f"rules={len(rules)} expected_hits={expected_hits}",
+        )
+    except Exception as exc:
+        return FormProbeResult(form_url=url, ok=False, error=str(exc))

navigation/perception/iframe_context.py

@@ -0,0 +1,76 @@
+"""Phase 4: iframe-aware interaction (same-origin / srcdoc frames)."""
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import Any
+
+from .verification import evaluate_js, read_page_text
+
+
+@dataclass(slots=True)
+class IframeProbeResult:
+    ok: bool
+    frame_count: int
+    clicked_in_frame: bool
+    frame_text: str = ""
+    error: str | None = None
+
+    def to_dict(self) -> dict:
+        return {
+            "ok": self.ok,
+            "frame_count": self.frame_count,
+            "clicked_in_frame": self.clicked_in_frame,
+            "frame_text": self.frame_text[:500],
+            "error": self.error,
+        }
+
+
+async def probe_iframe_interaction(
+    session: Any,
+    *,
+    iframe_selector: str = '[data-testid="edge-iframe"]',
+    button_id: str = "frame-btn",
+    expected_text: str = "clicked",
+) -> IframeProbeResult:
+    try:
+        frame_count = await evaluate_js(
+            session,
+            "document.querySelectorAll('iframe').length",
+        )
+        frame_count = int(frame_count or 0)
+
+        clicked = await evaluate_js(
+            session,
+            f"""
+            (() => {{
+              const frame = document.querySelector({iframe_selector!r});
+              if (!frame || !frame.contentDocument) return false;
+              const btn = frame.contentDocument.getElementById({button_id!r});
+              if (!btn) return false;
+              btn.click();
+              const out = frame.contentDocument.getElementById('out');
+              return out && out.textContent === {expected_text!r};
+            }})()
+            """,
+        )
+
+        frame_text = await evaluate_js(
+            session,
+            f"""
+            (() => {{
+              const frame = document.querySelector({iframe_selector!r});
+              return frame?.contentDocument?.body?.innerText || '';
+            }})()
+            """,
+        )
+
+        ok = bool(clicked) and frame_count >= 1
+        return IframeProbeResult(
+            ok=ok,
+            frame_count=frame_count,
+            clicked_in_frame=bool(clicked),
+            frame_text=str(frame_text or ""),
+        )
+    except Exception as exc:
+        return IframeProbeResult(ok=False, frame_count=0, clicked_in_frame=False, error=str(exc))