webpilot-engine 2.0.0__py3-none-any.whl

supervisor/shell.py

@@ -0,0 +1,194 @@
+"""Interactive REPL Shell Mode (Thin Client).
+
+Provides an interactive console prompt for step-by-step browser automation,
+communicating directly with the Master Supervisor (Layer 1).
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from typing import List
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+from services.inspection_formatter_service import InspectionFormatterService
+from supervisor.client import SupervisorClient
+from supervisor.contracts import SupervisorActionRequest, SupervisorActionResponse
+
+
+def run_interactive_shell(host: str = "127.0.0.1", port: int = 9333):
+    """Runs the interactive REPL session."""
+    client = SupervisorClient(base_url=f"http://{host}:{port}")
+    try:
+        client.ensure_running()
+    except Exception as exc:
+        print(f"[!] Failed to connect to Master Supervisor: {exc}")
+        return
+
+    print("\n" + "═" * 70)
+    print(" 🚀 WEBPILOT — INTERACTIVE AUTONOMOUS REPL SHELL")
+    print("═" * 70)
+    print(" Commands:")
+    print("  • open <url>             : Navigate to target URL")
+    print("  • fill <k>=<v> [<k>=<v>] : Populate one or more fields")
+    print("  • press <button_text>    : Click button with reactive mutation tracking")
+    print("  • upload <key>=<path>    : Upload document/CV")
+    print("  • inspect                : Inspect current page form schema")
+    print("  • tabs                   : List all open tabs")
+    print("  • tab <index>            : Switch to specific tab index (0-based)")
+    print("  • cookies <path>         : Save refreshed session cookies")
+    print("  • screenshot <path>      : Capture visual screenshot")
+    print("  • status                 : View runtime PIDs and tier health")
+    print("  • restart                : Cascading kill of Layer 2 & 3 with instant restart")
+    print("  • exit / quit            : Exit shell")
+    print("═" * 70 + "\n")
+
+    current_url = ""
+
+    while True:
+        try:
+            line = input("webpilot> ").strip()
+        except (KeyboardInterrupt, EOFError):
+            print("\nExiting interactive shell.")
+            break
+
+        if not line:
+            continue
+
+        parts = line.split(maxsplit=1)
+        cmd = parts[0].lower()
+        arg = parts[1] if len(parts) > 1 else ""
+
+        if cmd in ("exit", "quit", "q"):
+            print("Exiting interactive shell. Background supervisor remains active.")
+            break
+
+        elif cmd == "status":
+            st = client.get_status()
+            print("\n[📊 Supervisor & Worker Telemetry]:")
+            for k, v in st.items():
+                print(f" • {k:<25}: {v}")
+            print()
+
+        elif cmd == "restart":
+            print("[*] Initiating cascading kill of Layer 2 & 3...")
+            res = client.restart_worker()
+            print(f"[✓] {res.message}\n")
+
+        elif cmd in ("open", "goto"):
+            if not arg:
+                print("[!] Error: URL required, e.g. open https://example.com")
+                continue
+            current_url = arg.strip()
+            print(f"[*] Navigating to: {current_url}...")
+            req = SupervisorActionRequest(action="apply", url=current_url)
+            res = client.execute(req)
+            _print_apply_response(res)
+
+        elif cmd == "fill":
+            if not arg:
+                print("[!] Error: field mapping required, e.g. fill username=user@mail.com")
+                continue
+            fill_args = [a.strip() for a in arg.split(" ") if "=" in a]
+            if not fill_args:
+                fill_args = [arg.strip()]
+            req = SupervisorActionRequest(action="apply", fill_arguments=fill_args)
+            res = client.execute(req)
+            _print_apply_response(res)
+
+        elif cmd == "press":
+            if not arg:
+                print("[!] Error: button label required, e.g. press 'Sign In'")
+                continue
+            btn = arg.strip().strip("'\"")
+            print(f"[*] Pressing button: '{btn}'...")
+            req = SupervisorActionRequest(action="apply", press_buttons=[btn])
+            res = client.execute(req)
+            _print_apply_response(res)
+
+        elif cmd == "upload":
+            if not arg:
+                print("[!] Error: upload path required, e.g. upload resume=C:/cv.pdf")
+                continue
+            req = SupervisorActionRequest(action="apply", upload_files=[arg.strip()])
+            res = client.execute(req)
+            _print_apply_response(res)
+
+        elif cmd == "inspect":
+            print("[*] Inspecting active page schema...")
+            req = SupervisorActionRequest(action="inspect")
+            res = client.execute(req)
+            if res.schema:
+                print("\n" + InspectionFormatterService.format_summary(res.schema))
+            else:
+                print(f"[!] Inspection failed: {res.message}")
+
+        elif cmd == "tabs":
+            req = SupervisorActionRequest(action="list_tabs")
+            res = client.execute(req)
+            print("\n" + "═" * 70)
+            print(" 📑 OPEN BROWSER TABS")
+            print("═" * 70)
+            if not res.tabs:
+                print(" [!] No open tabs detected.")
+            else:
+                for t in res.tabs:
+                    act = " [*ACTIVE*]" if t.get("is_active") else ""
+                    print(f" • Tab #{t['index']}{act}: '{t['title']}'\n   URL: {t['url']}")
+            print("═" * 70 + "\n")
+
+        elif cmd == "tab":
+            if not arg or not arg.isdigit():
+                print("[!] Error: Tab index required, e.g. tab 0")
+                continue
+            idx = int(arg)
+            req = SupervisorActionRequest(action="switch_tab", tab_index=idx)
+            res = client.execute(req)
+            print(f"[✓] {res.message}\n")
+
+        elif cmd == "cookies":
+            save_path = arg.strip() or "session_cookies.json"
+            req = SupervisorActionRequest(action="apply", cookies_path=save_path)
+            res = client.execute(req)
+            print(f"[✓] Cookies requested for save: {save_path}\n")
+
+        elif cmd == "screenshot":
+            save_path = arg.strip() or "screenshot.png"
+            req = SupervisorActionRequest(action="apply", screenshot_path=save_path)
+            res = client.execute(req)
+            print(f"[✓] Screenshot saved: {save_path}\n")
+
+        else:
+            print(f"[!] Unknown command '{cmd}'. Type 'open', 'fill', 'press', 'inspect', 'tabs', 'restart', or 'exit'.")
+
+
+def _print_apply_response(res: SupervisorActionResponse):
+    """Formats output lines and reactive events from worker."""
+    if not res.success:
+        print(f"[!] Action failed: {res.message}")
+        return
+
+    print(" 📄 ACTIVE PAGE TELEMETRY")
+    print("─" * 70)
+    print(f" • Tab #{res.active_tab_index} (Total: {res.total_tabs}) | Title: '{res.active_title}'")
+    print(f" • URL: {res.active_url}")
+    print("═" * 70)
+
+    for line in res.output_lines:
+        print(f" {line}")
+
+    if res.reactive_event:
+        rev = res.reactive_event
+        if rev.get("redirected"):
+            print(f"\n[🔀 Redirect / Navigation]: {rev.get('new_url')}")
+        if rev.get("new_window"):
+            print(f"\n[🌐 New Window Opened]: {rev.get('new_url')}")
+        if rev.get("modal_opened"):
+            print(f"\n[🪟 Modal Appeared]: {rev.get('modal_text')}")
+        if res.schema:
+            print(InspectionFormatterService.format_summary(res.schema))
+
+    print()

supervisor/worker_process.py

@@ -0,0 +1,400 @@
+"""Layer 2: Operational Worker Process.
+
+Runs as a supervised subprocess of Master Supervisor (Layer 1).
+Holds Playwright, Chromium browser, contexts, pages, and active session state.
+Communicates via stdin/stdout JSON-RPC.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+from typing import Any, Dict, List, Optional
+
+# Ensure project root is on sys.path
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+from adapters.browser_adapter import BrowserAdapter
+from adapters.upload_adapter import UploadAdapter
+from common.cookies import save_cookies
+from builder import FormPayloadBuilder
+from config.settings import get_settings, load_device_profile
+from core.models import DeviceProfile
+from orchestration.context import SessionContext
+from orchestration.session_coordinator import SessionCoordinator
+from services.auth_navigation_service import AuthNavigationService
+from services.field_interaction_service import FieldInteractionService
+from services.inspection_formatter_service import InspectionFormatterService
+from services.inspection_service import InspectionService
+from services.reactive_interaction_service import ReactiveInteractionService
+from supervisor.contracts import SupervisorActionRequest, SupervisorActionResponse
+from validators.form_validator import FormValidator
+
+
+class OperationalWorker:
+    """Manages active Playwright runtime and executes form/browser automation tasks."""
+
+    def __init__(self, headed: bool = False):
+        self.settings = get_settings()
+        self.headed = headed
+        self.coordinator: Optional[SessionCoordinator] = None
+
+    def _ensure_session(self, req: SupervisorActionRequest) -> SessionCoordinator:
+        """Initializes or returns active SessionCoordinator."""
+        if (
+            self.coordinator is None
+            or self.coordinator.page is None
+            or self.coordinator.page.is_closed()
+        ):
+            effective_cookies = req.cookies_path or self.settings.cookies_path
+            session_ctx = SessionContext(
+                settings=self.settings,
+                device_profile=load_device_profile(),
+                headless=not req.headed if req.headed else not self.headed,
+                cookies_path=effective_cookies,
+                keep_alive=False,
+                close_session=False,
+                timeout_minutes=req.timeout_minutes,
+                tab_index=req.tab_index,
+            )
+            self.coordinator = SessionCoordinator(session_ctx)
+            self.coordinator.start()
+        return self.coordinator
+
+    def handle_request(self, req: SupervisorActionRequest) -> SupervisorActionResponse:
+        """Dispatches action to internal services and returns structured response."""
+        action = req.action.lower().strip()
+        lines: List[str] = []
+
+        try:
+            if action in ("ping", "status"):
+                return self._handle_status(req)
+
+            elif action == "list_tabs":
+                return self._handle_list_tabs(req)
+
+            elif action == "switch_tab":
+                return self._handle_switch_tab(req)
+
+            elif action == "inspect":
+                return self._handle_inspect(req)
+
+            elif action == "apply":
+                return self._handle_apply(req)
+
+            elif action in ("stop", "shutdown"):
+                self.shutdown()
+                return SupervisorActionResponse(
+                    success=True,
+                    action=action,
+                    message="Operational Worker shut down cleanly.",
+                )
+
+            else:
+                return SupervisorActionResponse(
+                    success=False,
+                    action=action,
+                    message=f"Unknown action: '{action}'",
+                )
+
+        except Exception as exc:
+            return SupervisorActionResponse(
+                success=False,
+                action=action,
+                message=f"Error executing {action}: {exc}",
+                output_lines=[f"[!] Worker exception: {exc}"],
+            )
+
+    def _handle_status(self, req: SupervisorActionRequest) -> SupervisorActionResponse:
+        coord = self.coordinator
+        if not coord or not coord.page:
+            return SupervisorActionResponse(
+                success=True,
+                action="status",
+                message="Worker is idle (no active browser yet).",
+                worker_pid=os.getpid(),
+                total_tabs=0,
+            )
+
+        adapter = coord.adapter
+        return SupervisorActionResponse(
+            success=True,
+            action="status",
+            message="Worker active with running browser.",
+            browser_pid=adapter.browser_pid,
+            worker_pid=os.getpid(),
+            active_tab_index=adapter.get_active_tab_index(),
+            active_url=coord.page.url if coord.page else "",
+            active_title=coord.page.title() if coord.page else "",
+            total_tabs=len(coord.context.pages) if coord.context else 0,
+            tabs=adapter.list_open_tabs(),
+        )
+
+    def _handle_list_tabs(self, req: SupervisorActionRequest) -> SupervisorActionResponse:
+        coord = self._ensure_session(req)
+        adapter = coord.adapter
+        tabs = adapter.list_open_tabs()
+        return SupervisorActionResponse(
+            success=True,
+            action="list_tabs",
+            message=f"Retrieved {len(tabs)} tabs.",
+            browser_pid=adapter.browser_pid,
+            worker_pid=os.getpid(),
+            total_tabs=len(tabs),
+            tabs=tabs,
+            active_tab_index=adapter.get_active_tab_index(),
+            active_url=coord.page.url if coord.page else "",
+        )
+
+    def _handle_switch_tab(self, req: SupervisorActionRequest) -> SupervisorActionResponse:
+        coord = self._ensure_session(req)
+        if req.tab_index is None:
+            return SupervisorActionResponse(
+                success=False,
+                action="switch_tab",
+                message="tab_index is required for switch_tab action.",
+            )
+        page = coord.switch_to_tab(req.tab_index)
+        return SupervisorActionResponse(
+            success=True,
+            action="switch_tab",
+            message=f"Switched to Tab #{req.tab_index}: '{page.title()}'",
+            active_tab_index=req.tab_index,
+            active_url=page.url,
+            active_title=page.title(),
+            total_tabs=len(coord.context.pages) if coord.context else 0,
+        )
+
+    def _handle_inspect(self, req: SupervisorActionRequest) -> SupervisorActionResponse:
+        coord = self._ensure_session(req)
+        page = coord.page
+        assert page is not None
+
+        if req.url:
+            coord.adapter.navigate(req.url)
+            page = coord.page
+
+        # Landing portal transition if needed
+        auth_svc = AuthNavigationService(page, coord.context)
+        if req.url and ("jobs." in page.url or page.locator("button.dropdown-toggle, .dialogApplyBtn").count() > 0):
+            auth_svc.bypass_job_landing_portal()
+            page = coord.switch_to_latest_page()
+
+        inspection_svc = InspectionService(page)
+        schema = inspection_svc.inspect(unpack_options=req.unpack_options)
+
+        if req.screenshot_path:
+            coord.adapter.capture_screenshot(req.screenshot_path, full_page=True)
+
+        return SupervisorActionResponse(
+            success=True,
+            action="inspect",
+            message=f"Inspected page: '{page.title()}'",
+            browser_pid=coord.adapter.browser_pid,
+            worker_pid=os.getpid(),
+            active_tab_index=coord.adapter.get_active_tab_index(),
+            active_url=page.url,
+            active_title=page.title(),
+            total_tabs=len(coord.context.pages) if coord.context else 0,
+            schema=schema,
+        )
+
+    def _handle_apply(self, req: SupervisorActionRequest) -> SupervisorActionResponse:
+        coord = self._ensure_session(req)
+        page = coord.page
+        ctx = coord.context
+        assert page is not None and ctx is not None
+
+        lines: List[str] = []
+
+        if req.tab_index is not None:
+            page = coord.switch_to_tab(req.tab_index)
+
+        if req.url:
+            lines.append(f"[*] Navigating to target URL: {req.url}")
+            coord.adapter.navigate(req.url)
+            page = coord.page
+        else:
+            if page.url and page.url != "about:blank":
+                lines.append(f"[*] Working on active page in existing session: {page.url}")
+            else:
+                return SupervisorActionResponse(
+                    success=False,
+                    action="apply",
+                    message="Target URL (--url) is required when starting a new session without an active page.",
+                )
+
+        auth_svc = AuthNavigationService(page, ctx)
+
+        # 1. Landing portal bypass
+        if req.url and ("jobs." in page.url or page.locator("button.dropdown-toggle, .dialogApplyBtn").count() > 0):
+            lines.append("[*] Bypassing landing page to active application...")
+            auth_svc.bypass_job_landing_portal()
+            page = coord.switch_to_latest_page()
+
+        # 2. Wait for form readiness & expand accordions
+        auth_svc.wait_for_form_ready(self.settings.form_ready_timeout_ms)
+        lines.append("[*] Ensuring all form sections are expanded...")
+        try:
+            auth_svc.expand_all_sections(self.settings.expand_section_wait_ms)
+        except Exception:
+            pass
+
+        # 3. Assemble fields to set
+        fields_to_set = []
+        if req.data_path and os.path.isfile(req.data_path):
+            fields_to_set.extend(FormPayloadBuilder.load_from_file(req.data_path))
+        if req.fill_arguments:
+            fields_to_set.extend(FormPayloadBuilder.parse_cli_fill_arguments(req.fill_arguments))
+
+        field_svc = FieldInteractionService(page)
+        confirmed = []
+        failed = []
+
+        if fields_to_set:
+            lines.append(f"[*] Populating {len(fields_to_set)} field(s)...")
+            for key, val in fields_to_set:
+                success = field_svc.set_field(key, val)
+                if success:
+                    confirmed.append((key, val))
+                    lines.append(f"  [✓] Set '{key}' -> '{val}'")
+                else:
+                    failed.append((key, val))
+                    lines.append(f"  [~] Unmatched/Failed '{key}' -> '{val}'")
+            lines.append(f"[*] Fields summary: {len(confirmed)} set | {len(failed)} skipped.")
+
+        # 4. Handle uploads
+        if req.upload_files:
+            upload_adapter = UploadAdapter(page)
+            for up in req.upload_files:
+                if "=" in up:
+                    kw, path = up.split("=", 1)
+                    kw, path = kw.strip(), path.strip()
+                else:
+                    kw, path = "", up.strip()
+                if os.path.isfile(path):
+                    lines.append(f"[*] Uploading '{path}' (Keyword: '{kw}')...")
+                    upload_adapter.upload_document(path, kw)
+
+        # 5. Handle buttons with reactive observation
+        inspection_svc = InspectionService(page)
+        validator = FormValidator(page)
+        reactive_svc = ReactiveInteractionService(
+            page=page,
+            context=ctx,
+            inspection_service=inspection_svc,
+            form_validator=validator,
+            on_active_page_change=lambda p: coord.switch_to_latest_page(),
+        )
+
+        last_reactive_event: Optional[Dict[str, Any]] = None
+        if req.press_buttons:
+            for btn in req.press_buttons:
+                lines.append(f"[*] Pressing button: '{btn}'...")
+                res = reactive_svc.click_button(btn)
+                if not res.get("success"):
+                    lines.append(f"  [!] Failed to find or click button: '{btn}'")
+                    continue
+                lines.append(f"  [✓] Successfully clicked '{btn}'.")
+                page = coord.switch_to_latest_page()
+                last_reactive_event = res
+
+                if res.get("new_window"):
+                    lines.append(f"\n[🌐 New Window / Tab Opened]: {res['new_url']}")
+                elif res.get("redirected"):
+                    lines.append(f"\n[🔀 Redirect / Navigation Detected]: {res['new_url']}")
+                elif res.get("modal_opened"):
+                    lines.append(f"\n[🪟 New Modal / Dialog Appeared]:\n{res['modal_text']}")
+                elif res.get("new_inputs_count", 0) > 0:
+                    lines.append(f"\n[📋 Dynamic Form Expansion (+{res['new_inputs_count']} new inputs)]")
+
+        # 6. Capture screenshot if requested
+        if req.screenshot_path:
+            coord.adapter.capture_screenshot(req.screenshot_path, full_page=True)
+            lines.append(f"[✓] Form state captured: {req.screenshot_path}")
+
+        # 7. Save refreshed cookies if requested
+        if req.cookies_path and ctx:
+            try:
+                cookies = ctx.cookies()
+                save_cookies(req.cookies_path, cookies)
+                lines.append(f"[✓] Persisted {len(cookies)} cookies to: {req.cookies_path}")
+            except Exception as exc:
+                lines.append(f"[!] Warning saving cookies: {exc}")
+
+        # 8. Validation checks
+        validation_errors = validator.get_validation_errors()
+        if validation_errors:
+            lines.append(f"[!] Validation Notices: {validation_errors}")
+        else:
+            lines.append("[✓] Zero validation errors detected!")
+
+        schema = None
+        if req.auto_inspect:
+            schema = inspection_svc.inspect(unpack_options=False)
+
+        return SupervisorActionResponse(
+            success=True,
+            action="apply",
+            message="Form action executed successfully.",
+            browser_pid=coord.adapter.browser_pid,
+            worker_pid=os.getpid(),
+            active_tab_index=coord.adapter.get_active_tab_index(),
+            active_url=page.url,
+            active_title=page.title(),
+            total_tabs=len(ctx.pages) if ctx else 0,
+            tabs=coord.adapter.list_open_tabs(),
+            confirmed_fields=confirmed,
+            failed_fields=failed,
+            validation_errors=validation_errors,
+            schema=schema,
+            reactive_event=last_reactive_event,
+            output_lines=lines,
+        )
+
+    def shutdown(self) -> None:
+        """Terminates Playwright coordinator and all child browser tabs."""
+        if self.coordinator:
+            try:
+                self.coordinator.close()
+            except Exception:
+                pass
+            self.coordinator = None
+
+
+def run_worker_loop():
+    """Reads JSON requests from stdin, executes, writes JSON response to stdout."""
+    rpc_out = sys.stdout
+    # Redirect all print/logging to stderr to keep JSON-RPC channel strictly pure
+    sys.stdout = sys.stderr
+
+    worker = OperationalWorker()
+    for raw_line in sys.stdin:
+        line = raw_line.strip()
+        if not line:
+            continue
+        try:
+            data = json.loads(line)
+            req = SupervisorActionRequest.from_dict(data)
+            res = worker.handle_request(req)
+            out = json.dumps(res.to_dict(), ensure_ascii=False)
+            rpc_out.write(out + "\n")
+            rpc_out.flush()
+            if req.action in ("stop", "shutdown"):
+                break
+        except Exception as exc:
+            err_res = SupervisorActionResponse(
+                success=False,
+                action="unknown",
+                message=f"Fatal worker parse error: {exc}",
+            )
+            rpc_out.write(json.dumps(err_res.to_dict(), ensure_ascii=False) + "\n")
+            rpc_out.flush()
+
+
+if __name__ == "__main__":
+    run_worker_loop()

validators/__init__.py

@@ -0,0 +1,9 @@
+"""Validators package for universal web applier."""
+
+from validators.payload_validator import PayloadValidator
+from validators.form_validator import FormValidator
+
+__all__ = [
+    "PayloadValidator",
+    "FormValidator",
+]

validators/form_validator.py

@@ -0,0 +1,28 @@
+"""Form validation and error banner inspection."""
+
+from typing import List
+from playwright.sync_api import Page
+from constants.selectors import SELECTOR_VALIDATION_ERRORS
+
+class FormValidator:
+    """Inspects the active DOM for validation notices, errors, and alerts."""
+
+    def __init__(self, page: Page):
+        self.page = page
+
+    def get_validation_errors(self) -> List[str]:
+        """Collects error alerts, validation messages, or red warning texts currently shown."""
+        if self.page is None:
+            return []
+
+        errors = self.page.evaluate("""(selector) => {
+            const errorNodes = Array.from(document.querySelectorAll(selector));
+            const texts = errorNodes.map(e => e.innerText.trim()).filter(t => t.length > 0);
+
+            // Also inspect lines starting with error glyphs
+            const bodyLines = document.body.innerText.split('\\n').map(l => l.trim());
+            const errLines = bodyLines.filter(l => l.includes('is required') || l.includes('Please correct the errors'));
+            return Array.from(new Set([...texts, ...errLines]));
+        }""", SELECTOR_VALIDATION_ERRORS)
+
+        return list(errors)

validators/payload_validator.py

@@ -0,0 +1,25 @@
+"""Validation logic for form payloads and CLI inputs."""
+
+from typing import Any, Dict, List, Tuple, Union
+from core.exceptions import ValidationError
+
+class PayloadValidator:
+    """Validates candidate payloads and CLI key-value arguments."""
+
+    @staticmethod
+    def validate_key_value_pair(key: Any, val: Any) -> Tuple[str, Any]:
+        """Validates that a field key is a non-empty string."""
+        clean_key = str(key).strip()
+        if not clean_key:
+            raise ValidationError("Field key cannot be empty or whitespace")
+        return clean_key, val
+
+    @staticmethod
+    def validate_payload_list(items: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:
+        """Ensures all items in payload are valid (key, value) tuples."""
+        validated = []
+        for item in items:
+            if not isinstance(item, (tuple, list)) or len(item) != 2:
+                raise ValidationError(f"Invalid payload item structure: {item}")
+            validated.append(PayloadValidator.validate_key_value_pair(item[0], item[1]))
+        return validated