webpilot-engine 2.0.0__py3-none-any.whl

supervisor/client.py

@@ -0,0 +1,171 @@
+"""Lightweight HTTP Client for communicating with the Master Supervisor (Layer 1).
+
+Includes Zero-Touch Auto-Spawn capability: automatically boots the background
+supervisor host detached via WMI if not already running.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Dict, Optional
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+from supervisor.contracts import SupervisorActionRequest, SupervisorActionResponse
+
+DEFAULT_SUPERVISOR_URL: str = "http://127.0.0.1:9333"
+
+
+class SupervisorClient:
+    """Thin client for interacting with the Master Supervisor daemon."""
+
+    def __init__(self, base_url: str = DEFAULT_SUPERVISOR_URL):
+        self.base_url = base_url.rstrip("/")
+
+    def is_running(self) -> bool:
+        """Pings the supervisor health endpoint."""
+        try:
+            req = urllib.request.Request(f"{self.base_url}/ping", headers={"User-Agent": "WebPilotClient/2.0"})
+            with urllib.request.urlopen(req, timeout=0.8) as resp:
+                return resp.status == 200
+        except Exception:
+            return False
+
+    def ensure_running(self) -> None:
+        """Checks if supervisor is alive; if not, launches it detached via WMI."""
+        if self.is_running():
+            return
+
+        print("[*] Master Supervisor (Layer 1) not running. Auto-spawning background host...")
+        script_path = os.path.join(PROJECT_ROOT, "supervisor", "master_daemon.py")
+        py_exe = sys.executable
+
+        # Launch detached via WMI outside any parent terminal Job Object
+        if sys.platform == "win32":
+            if getattr(sys, "frozen", False):
+                py_exe = sys.executable
+                cwd = os.path.dirname(py_exe)
+                cmd_line = f'`"{py_exe}`" --run-daemon'
+            else:
+                script_path = os.path.join(PROJECT_ROOT, "supervisor", "master_daemon.py")
+                py_exe = sys.executable
+                cwd = PROJECT_ROOT
+                cmd_line = f'`"{py_exe}`" `"{script_path}`"'
+
+            ps_script = f"""
+$cwd = '{cwd}'
+$cmd = '{cmd_line}'
+$proc = ([wmiclass]'Win32_Process').Create($cmd, $cwd, $null)
+if ($proc.ReturnValue -eq 0) {{
+    Write-Output $proc.ProcessId
+}} else {{
+    exit 1
+}}
+"""
+            encoded = base64.b64encode(ps_script.encode("utf-16le")).decode("ascii")
+            subprocess.run(
+                ["powershell", "-NoProfile", "-NonInteractive", "-EncodedCommand", encoded],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=5,
+            )
+        else:
+            subprocess.Popen(
+                [sys.executable, script_path],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                start_new_session=True,
+                cwd=PROJECT_ROOT,
+            )
+
+        # Wait up to 4 seconds for supervisor to become active
+        start = time.time()
+        while time.time() - start < 4.0:
+            if self.is_running():
+                print("[✓] Master Supervisor is ready and active.")
+                return
+            time.sleep(0.2)
+
+        raise RuntimeError("Failed to auto-spawn Master Supervisor on http://127.0.0.1:9333.")
+
+    def execute(self, action_req: SupervisorActionRequest) -> SupervisorActionResponse:
+        """Sends an action request to the supervisor and returns structured response."""
+        self.ensure_running()
+        url = f"{self.base_url}/execute"
+        body = json.dumps(action_req.to_dict()).encode("utf-8")
+        req = urllib.request.Request(
+            url,
+            data=body,
+            headers={"Content-Type": "application/json", "User-Agent": "WebPilotClient/2.0"},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=90) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+                return SupervisorActionResponse.from_dict(data)
+        except Exception as exc:
+            return SupervisorActionResponse(
+                success=False,
+                action=action_req.action,
+                message=f"Network error communicating with Master Supervisor: {exc}",
+            )
+
+    def restart_worker(self) -> SupervisorActionResponse:
+        """Triggers cascading kill of Layer 2 Worker and Chromium, then respawns fresh."""
+        self.ensure_running()
+        req = urllib.request.Request(f"{self.base_url}/restart", data=b"{}", method="POST")
+        try:
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+                return SupervisorActionResponse.from_dict(data)
+        except Exception as exc:
+            return SupervisorActionResponse(
+                success=False,
+                action="restart",
+                message=f"Failed to restart worker: {exc}",
+            )
+
+    def stop_service(self) -> SupervisorActionResponse:
+        """Completely terminates Master Supervisor and all worker tiers."""
+        if not self.is_running():
+            return SupervisorActionResponse(
+                success=True,
+                action="stop",
+                message="Master Supervisor is already stopped.",
+            )
+
+        req = urllib.request.Request(f"{self.base_url}/stop", data=b"{}", method="POST")
+        try:
+            with urllib.request.urlopen(req, timeout=10) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+                return SupervisorActionResponse.from_dict(data)
+        except Exception as exc:
+            return SupervisorActionResponse(
+                success=False,
+                action="stop",
+                message=f"Error stopping service: {exc}",
+            )
+
+    def get_status(self) -> Dict[str, Any]:
+        """Retrieves runtime health and PID telemetry."""
+        if not self.is_running():
+            return {"status": "stopped", "supervisor_alive": False}
+
+        try:
+            req = urllib.request.Request(f"{self.base_url}/status")
+            with urllib.request.urlopen(req, timeout=2) as resp:
+                data = json.loads(resp.read().decode("utf-8"))
+                data["status"] = "running"
+                data["supervisor_alive"] = True
+                return data
+        except Exception:
+            return {"status": "unreachable", "supervisor_alive": False}

supervisor/contracts.py

@@ -0,0 +1,62 @@
+"""Data contracts for Master Supervisor, Worker Process, and CLI Client."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class SupervisorActionRequest:
+    """Action payload sent from CLI or Shell to the Master Supervisor."""
+    action: str  # 'apply', 'inspect', 'list_tabs', 'switch_tab', 'restart_worker', 'status', 'stop'
+    url: Optional[str] = None
+    data_path: Optional[str] = None
+    fill_arguments: List[str] = field(default_factory=list)
+    press_buttons: List[str] = field(default_factory=list)
+    upload_files: List[str] = field(default_factory=list)
+    cookies_path: Optional[str] = None
+    submit: bool = False
+    screenshot_path: Optional[str] = None
+    tab_index: Optional[int] = None
+    unpack_options: bool = False
+    headed: bool = False
+    auto_inspect: bool = True
+    timeout_minutes: int = 30
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> SupervisorActionRequest:
+        return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
+
+
+@dataclass
+class SupervisorActionResponse:
+    """Standardized response returned by Master Supervisor to CLI or Shell."""
+    success: bool
+    action: str
+    message: str = ""
+    browser_pid: Optional[int] = None
+    worker_pid: Optional[int] = None
+    active_tab_index: Optional[int] = None
+    active_url: Optional[str] = None
+    active_title: Optional[str] = None
+    total_tabs: int = 0
+    tabs: List[Dict[str, Any]] = field(default_factory=list)
+    telemetry: Dict[str, Any] = field(default_factory=dict)
+    schema: Optional[Dict[str, Any]] = None
+    confirmed_fields: List[Any] = field(default_factory=list)
+    failed_fields: List[Any] = field(default_factory=list)
+    validation_errors: List[str] = field(default_factory=list)
+    reactive_event: Optional[Dict[str, Any]] = None
+    output_lines: List[str] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> SupervisorActionResponse:
+        return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})

supervisor/master_daemon.py

@@ -0,0 +1,303 @@
+"""Layer 1: Master Supervisor Daemon Host.
+
+Runs as a lightweight, persistent background daemon (Zero-Admin, Standard User).
+Supervises Layer 2 (Operational Worker) and enforces cascading teardown.
+Exposes a minimal HTTP endpoint on 127.0.0.1:9333 for CLI and Shell clients.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any, Dict, Optional
+
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+from supervisor.contracts import SupervisorActionRequest, SupervisorActionResponse
+
+DEFAULT_SUPERVISOR_HOST: str = "127.0.0.1"
+DEFAULT_SUPERVISOR_PORT: int = 9333
+DEFAULT_INACTIVITY_TIMEOUT_SECONDS: int = 1800  # 30 minutes
+LOG_FILE: str = os.path.join(PROJECT_ROOT, "master_daemon.log")
+
+
+class LogWriter:
+    """Safe file-backed stream writer for headless daemon background logging."""
+    def __init__(self, filepath: str):
+        self.filepath = filepath
+
+    def write(self, s: str) -> None:
+        if s:
+            try:
+                with open(self.filepath, "a", encoding="utf-8") as f:
+                    f.write(s)
+            except Exception:
+                pass
+
+    def flush(self) -> None:
+        pass
+
+
+def log_daemon(msg: str) -> None:
+    """Appends timestamped event to master_daemon.log."""
+    try:
+        ts = time.strftime("%Y-%m-%d %H:%M:%S")
+        with open(LOG_FILE, "a", encoding="utf-8") as f:
+            f.write(f"[{ts}] {msg}\n")
+    except Exception:
+        pass
+
+
+class MasterSupervisor:
+    """Layer 1 Master Supervisor: controls Worker process lifecycle and inactivity watchdog."""
+
+    def __init__(
+        self,
+        host: str = DEFAULT_SUPERVISOR_HOST,
+        port: int = DEFAULT_SUPERVISOR_PORT,
+        inactivity_timeout_seconds: int = DEFAULT_INACTIVITY_TIMEOUT_SECONDS,
+    ):
+        self.host = host
+        self.port = port
+        self.inactivity_timeout_seconds = inactivity_timeout_seconds
+        self.worker_process: Optional[subprocess.Popen] = None
+        self.last_active_time: float = time.time()
+        self.lock = threading.Lock()
+        self._is_running = True
+
+        # Launch watchdog thread
+        self.watchdog_thread = threading.Thread(target=self._watchdog_loop, daemon=True)
+        self.watchdog_thread.start()
+
+    def _ensure_worker(self) -> subprocess.Popen:
+        """Spawns Layer 2 Worker subprocess if not currently running."""
+        with self.lock:
+            if self.worker_process is None or self.worker_process.poll() is not None:
+                if getattr(sys, "frozen", False):
+                    worker_cmd = [sys.executable, "--run-worker"]
+                    worker_cwd = os.path.dirname(sys.executable)
+                else:
+                    worker_cmd = [sys.executable, "-m", "supervisor.worker_process"]
+                    worker_cwd = PROJECT_ROOT
+
+                self.worker_process = subprocess.Popen(
+                    worker_cmd,
+                    stdin=subprocess.PIPE,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                    bufsize=1,
+                    cwd=worker_cwd,
+                )
+            return self.worker_process
+
+    def execute_action(self, req: SupervisorActionRequest) -> SupervisorActionResponse:
+        """Proxies action request to Layer 2 Worker and returns structured response."""
+        self.last_active_time = time.time()
+
+        if req.action == "restart_worker":
+            return self.restart_worker()
+
+        if req.action in ("stop", "shutdown"):
+            self.terminate_worker()
+            self._is_running = False
+            return SupervisorActionResponse(
+                success=True,
+                action="stop",
+                message="Master Supervisor and all worker tiers shut down.",
+            )
+
+        worker = self._ensure_worker()
+        assert worker.stdin is not None and worker.stdout is not None
+
+        req_line = json.dumps(req.to_dict()) + "\n"
+        try:
+            worker.stdin.write(req_line)
+            worker.stdin.flush()
+
+            data = None
+            while True:
+                line = worker.stdout.readline()
+                if not line:
+                    break
+                line_str = line.strip()
+                if not line_str:
+                    continue
+                try:
+                    data = json.loads(line_str)
+                    break
+                except Exception:
+                    log_daemon(f"Skipping non-JSON worker stdout: {line_str}")
+                    continue
+
+            if not data:
+                log_daemon("Worker returned no valid JSON response.")
+                self.terminate_worker()
+                return SupervisorActionResponse(
+                    success=False,
+                    action=req.action,
+                    message="Operational Worker failed to return valid JSON response.",
+                )
+
+            return SupervisorActionResponse.from_dict(data)
+
+        except Exception as exc:
+            return SupervisorActionResponse(
+                success=False,
+                action=req.action,
+                message=f"Supervisor dispatch error: {exc}",
+            )
+
+    def restart_worker(self) -> SupervisorActionResponse:
+        """Cascading kill of Layer 2 Worker and all child Chromium processes, then respawns fresh."""
+        with self.lock:
+            old_pid = self.worker_process.pid if self.worker_process else None
+            self.terminate_worker()
+            # Spawn fresh Worker
+            new_worker = subprocess.Popen(
+                [sys.executable, "-m", "supervisor.worker_process"],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                bufsize=1,
+                cwd=PROJECT_ROOT,
+            )
+            self.worker_process = new_worker
+            self.last_active_time = time.time()
+
+        return SupervisorActionResponse(
+            success=True,
+            action="restart_worker",
+            message=f"Cascading kill completed for PID {old_pid}. Fresh Worker spawned (PID: {new_worker.pid}).",
+            worker_pid=new_worker.pid,
+        )
+
+    def terminate_worker(self) -> None:
+        """Forcefully kills the Layer 2 Worker process tree."""
+        proc = self.worker_process
+        self.worker_process = None
+        if proc and proc.poll() is None:
+            try:
+                if sys.platform == "win32":
+                    subprocess.run(
+                        ["taskkill", "/F", "/T", "/PID", str(proc.pid)],
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL,
+                    )
+                else:
+                    proc.kill()
+            except Exception:
+                pass
+
+    def get_status(self) -> Dict[str, Any]:
+        """Returns telemetry for Supervisor and Worker."""
+        worker_alive = self.worker_process is not None and self.worker_process.poll() is None
+        worker_pid = self.worker_process.pid if worker_alive else None
+        return {
+            "supervisor_pid": os.getpid(),
+            "supervisor_port": self.port,
+            "worker_alive": worker_alive,
+            "worker_pid": worker_pid,
+            "idle_seconds": round(time.time() - self.last_active_time, 1),
+            "inactivity_timeout_seconds": self.inactivity_timeout_seconds,
+        }
+
+    def _watchdog_loop(self) -> None:
+        """Periodically checks inactivity. Kills idle Worker to release 100% of browser RAM."""
+        while self._is_running:
+            time.sleep(15)
+            if self.worker_process and self.worker_process.poll() is None:
+                idle = time.time() - self.last_active_time
+                if idle > self.inactivity_timeout_seconds:
+                    print(f"[*] Inactivity watchdog triggered ({idle:.0f}s idle). Terminating Worker to release RAM.")
+                    self.terminate_worker()
+
+
+def create_supervisor_handler(supervisor: MasterSupervisor):
+    """Factory creating an HTTP Request Handler bound to the supervisor instance."""
+
+    class SupervisorHTTPHandler(BaseHTTPRequestHandler):
+        def log_message(self, format, *args):
+            # Suppress default stdout HTTP access logs
+            return
+
+        def _send_json(self, status_code: int, data: Dict[str, Any]):
+            body = json.dumps(data, ensure_ascii=False).encode("utf-8")
+            self.send_response(status_code)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+        def do_GET(self):
+            if self.path == "/ping":
+                self._send_json(200, {"status": "ok", "supervisor_pid": os.getpid()})
+            elif self.path == "/status":
+                status = supervisor.get_status()
+                self._send_json(200, status)
+            else:
+                self._send_json(404, {"error": "Not Found"})
+
+        def do_POST(self):
+            content_length = int(self.headers.get("Content-Length", 0))
+            raw_body = self.rfile.read(content_length).decode("utf-8") if content_length > 0 else "{}"
+
+            try:
+                data = json.loads(raw_body)
+            except Exception:
+                data = {}
+
+            if self.path == "/execute":
+                req = SupervisorActionRequest.from_dict(data)
+                res = supervisor.execute_action(req)
+                self._send_json(200, res.to_dict())
+            elif self.path == "/restart":
+                res = supervisor.restart_worker()
+                self._send_json(200, res.to_dict())
+            elif self.path == "/stop":
+                req = SupervisorActionRequest(action="stop")
+                res = supervisor.execute_action(req)
+                self._send_json(200, res.to_dict())
+                # Schedule daemon shutdown in background thread
+                threading.Thread(target=self.server.shutdown, daemon=True).start()
+            else:
+                self._send_json(404, {"error": "Not Found"})
+
+    return SupervisorHTTPHandler
+
+
+def run_master_daemon(port: int = DEFAULT_SUPERVISOR_PORT):
+    """Starts the Master Supervisor HTTP service."""
+    sys.stdout = LogWriter(LOG_FILE)
+    sys.stderr = LogWriter(LOG_FILE)
+    log_daemon(f"run_master_daemon starting on port {port} (PID: {os.getpid()})")
+    try:
+        supervisor = MasterSupervisor(port=port)
+        handler_class = create_supervisor_handler(supervisor)
+        server = HTTPServer((DEFAULT_SUPERVISOR_HOST, port), handler_class)
+        log_daemon(f"Master Supervisor listening on http://{DEFAULT_SUPERVISOR_HOST}:{port}")
+        print(f"[*] Master Supervisor (Layer 1) started on http://{DEFAULT_SUPERVISOR_HOST}:{port} (PID: {os.getpid()})")
+        server.serve_forever()
+    except Exception as exc:
+        log_daemon(f"Master Supervisor fatal exception: {exc}")
+        raise
+    finally:
+        try:
+            supervisor.terminate_worker()
+        except Exception:
+            pass
+        log_daemon("Master Supervisor stopped.")
+        print("[*] Master Supervisor stopped.")
+
+
+if __name__ == "__main__":
+    port = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_SUPERVISOR_PORT
+    run_master_daemon(port=port)