webpilot-engine 2.0.4__tar.gz → 2.0.5__tar.gz

{webpilot_engine-2.0.4 → webpilot_engine-2.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: webpilot-engine
-Version: 2.0.4
+Version: 2.0.5
 Summary: WebPilot: Enterprise Autonomous Browser Engine, Universal Form Inspector & AI Agent Web Runtime
 Author: Abdulkarim Salih, WebPilot Core Team
 License: MIT

{webpilot_engine-2.0.4 → webpilot_engine-2.0.5}/adapters/browser_adapter.py

@@ -253,15 +253,10 @@ class BrowserAdapter(BrowserAdapterProtocol):
                 args=["--disable-blink-features=AutomationControlled"],
             )
             try:
-                if sys.platform == "win32":
-                    ps_c = f'(Get-CimInstance Win32_Process | Where-Object {{ $_.ParentProcessId -eq {os.getpid()} }}).ProcessId | Select-Object -First 1'
-                    out = subprocess.check_output(["powershell", "-NoProfile", "-Command", ps_c], text=True).strip()
-                    if out.isdigit():
-                        self.browser_pid = int(out)
-                else:
-                    out = subprocess.check_output(["pgrep", "-P", str(os.getpid())], text=True).strip().split()
-                    if out and out[0].isdigit():
-                        self.browser_pid = int(out[0])
+                from common.process_manager import get_process_manager
+                children = get_process_manager().get_child_pids(os.getpid())
+                if children:
+                    self.browser_pid = children[0]
             except Exception:
                 pass
 

{webpilot_engine-2.0.4 → webpilot_engine-2.0.5}/adapters/live_session_manager.py

@@ -124,16 +124,8 @@ class LiveSessionManager:
         pid = state.get("pid")
         if pid:
             try:
-                # Terminate on Windows
-                if sys.platform == "win32":
-                    subprocess.run(
-                        ["taskkill", "/F", "/T", "/PID", str(pid)],
-                        stdout=subprocess.DEVNULL,
-                        stderr=subprocess.DEVNULL,
-                    )
-                else:
-                    os.kill(pid, 9)
-                return True
+                from common.process_manager import get_process_manager
+                return get_process_manager().kill_process_tree(pid, force=True)
             except Exception:
                 pass
         return False
@@ -267,62 +259,23 @@ class LiveSessionManager:
     @classmethod
     def _launch_wmi_process(cls, cmd_str: str, cwd: Optional[str] = None) -> Optional[int]:
         """Launches a detached process via Windows WMI outside any parent Job Object."""
+        from common.process_manager import get_process_manager
         working_dir = cwd or PROJECT_ROOT
-        ps_script = f"""
-$startup = ([wmiclass]"Win32_ProcessStartup").CreateInstance()
-$startup.ShowWindow = 0
-$proc = ([wmiclass]"Win32_Process").Create(@'
-{cmd_str}
-'@, $null, $startup)
-if ($proc.ReturnValue -eq 0) {{
-    Write-Output $proc.ProcessId
-}} else {{
-    exit 1
-}}
-"""
-        try:
-            encoded = base64.b64encode(ps_script.encode("utf-16le")).decode("ascii")
-            res = subprocess.run(
-                ["powershell", "-NoProfile", "-NonInteractive", "-EncodedCommand", encoded],
-                capture_output=True,
-                text=True,
-                timeout=10,
-            )
-            out = res.stdout.strip()
-            if res.returncode == 0 and out.isdigit():
-                return int(out)
-            else:
-                print(f"[!] WMI launch non-zero/non-digit: code={res.returncode}, out='{out}', err='{res.stderr.strip()}'")
-        except Exception as exc:
-            print(f"[!] WMI launch exception: {exc}")
-        return None
+        return get_process_manager().launch_detached(cmd_str, cwd=working_dir)
 
     @classmethod
     def _spawn_watchdog_supervisor(cls, browser_pid: int, port: int, timeout_seconds: int) -> None:
         """Launches a lightweight detached background watchdog to terminate idle browser."""
-        watchdog_cmd = f'"{sys.executable}" -m adapters.live_session_manager {browser_pid} {port} {timeout_seconds}'
-
-        if sys.platform == "win32":
-            wmi_pid = cls._launch_wmi_process(watchdog_cmd)
-            if wmi_pid is not None:
-                return
-
-        popen_kwargs: Dict[str, Any] = {
-            "stdout": subprocess.DEVNULL,
-            "stderr": subprocess.DEVNULL,
-        }
-        if sys.platform == "win32":
-            popen_kwargs["creationflags"] = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
-        else:
-            popen_kwargs["start_new_session"] = True
-
-        try:
-            subprocess.Popen(
-                [sys.executable, "-m", "adapters.live_session_manager", str(browser_pid), str(port), str(timeout_seconds)],
-                **popen_kwargs,
-            )
-        except Exception:
-            pass
+        from common.process_manager import get_process_manager
+        watchdog_cmd = [
+            sys.executable,
+            "-m",
+            "adapters.live_session_manager",
+            str(browser_pid),
+            str(port),
+            str(timeout_seconds),
+        ]
+        get_process_manager().launch_detached(watchdog_cmd, cwd=PROJECT_ROOT)
 
 
 if __name__ == "__main__":

{webpilot_engine-2.0.4 → webpilot_engine-2.0.5}/builder.py

@@ -34,7 +34,17 @@ class FormPayloadBuilder:
 
     @staticmethod
     def load_from_file(filepath: str) -> List[Tuple[str, Any]]:
-        """Load field key-value pairs from an external JSON file."""
+        """Load field key-value pairs from an external JSON file or stdin ('-' or '@stdin')."""
+        import sys
+        from common.secrets import resolve_secret_value
+
+        if filepath in ("-", "@stdin"):
+            try:
+                raw_data = json.load(sys.stdin)
+            except Exception as exc:
+                raise ValidationError(f"Invalid JSON read from standard input: {exc}") from exc
+            return FormPayloadBuilder.normalize_payload(raw_data)
+
         if not os.path.isfile(filepath):
             raise ConfigurationError(f"Form payload data file not found: {filepath}")
 
@@ -48,18 +58,21 @@ class FormPayloadBuilder:
 
     @staticmethod
     def normalize_payload(raw_data: Union[Dict[str, Any], List[Any]]) -> List[Tuple[str, Any]]:
-        """Normalizes dict or list structures into a clean list of (field, value) pairs."""
+        """Normalizes dict or list structures into a clean list of (field, value) pairs and resolves secrets."""
+        from common.secrets import resolve_secret_value
+
         fields: List[Tuple[str, Any]] = []
 
         if isinstance(raw_data, dict):
             for k, v in raw_data.items():
-                fields.append((str(k), v))
+                fields.append((str(k), resolve_secret_value(v, prompt_label=str(k))))
         elif isinstance(raw_data, list):
             for item in raw_data:
                 if isinstance(item, (list, tuple)) and len(item) == 2:
-                    fields.append((str(item[0]), item[1]))
+                    fields.append((str(item[0]), resolve_secret_value(item[1], prompt_label=str(item[0]))))
                 elif isinstance(item, dict) and "key" in item and "value" in item:
-                    fields.append((str(item["key"]), item["value"]))
+                    k = str(item["key"])
+                    fields.append((k, resolve_secret_value(item["value"], prompt_label=k)))
                 else:
                     raise ValidationError(f"Unrecognized payload entry structure: {item}")
         else:
@@ -68,16 +81,8 @@ class FormPayloadBuilder:
         return fields
 
     @staticmethod
-    def parse_cli_fill_arguments(fill_args: Optional[List[str]]) -> List[Tuple[str, str]]:
-        """Parses CLI --fill 'key=value' arguments into key-value pairs."""
-        if not fill_args:
-            return []
-
-        parsed: List[Tuple[str, str]] = []
-        for item in fill_args:
-            if "=" in item:
-                k, v = item.split("=", 1)
-                parsed.append((k.strip(), v.strip()))
-            else:
-                raise ValidationError(f"Invalid fill argument format: '{item}' (Expected 'key=value')")
-        return parsed
+    def parse_cli_fill_arguments(fill_args: Optional[List[str]]) -> List[Tuple[str, Any]]:
+        """Parses CLI --fill arguments into key-value pairs and resolves secrets (@env, @stdin, @file)."""
+        from common.secrets import parse_and_resolve_fill_args
+        return parse_and_resolve_fill_args(fill_args)
+

{webpilot_engine-2.0.4 → webpilot_engine-2.0.5}/cli.py

@@ -116,11 +116,29 @@ def apply_cmd(args: argparse.Namespace) -> None:
     """Executes the universal form filling and submission workflow."""
     auto_inspect = not getattr(args, "no_auto_inspect", False)
 
+    from common.secrets import parse_and_resolve_fill_args
+    from builder import FormPayloadBuilder
+
+    raw_fill_args = args.fill or []
+    resolved_pairs = parse_and_resolve_fill_args(raw_fill_args)
+    effective_fill_args = [f"{k}={v}" for k, v in resolved_pairs]
+
+    effective_data_path = args.data
+    if args.data in ("-", "@stdin"):
+        try:
+            stdin_json = json.load(sys.stdin)
+            stdin_pairs = FormPayloadBuilder.normalize_payload(stdin_json)
+            effective_fill_args.extend([f"{k}={v}" for k, v in stdin_pairs])
+            effective_data_path = None
+        except Exception as exc:
+            print(f"[!] Error reading JSON payload from stdin: {exc}", file=sys.stderr)
+            sys.exit(1)
+
     if getattr(args, "standalone", False):
         req = ApplyRequestContext(
             url=args.url,
-            data_path=args.data,
-            fill_arguments=args.fill or [],
+            data_path=effective_data_path,
+            fill_arguments=effective_fill_args,
             press_buttons=args.press or [],
             upload_files=args.upload or [],
             cookies_path=args.cookies,
@@ -165,8 +183,8 @@ def apply_cmd(args: argparse.Namespace) -> None:
     req = SupervisorActionRequest(
         action="apply",
         url=args.url,
-        data_path=args.data,
-        fill_arguments=args.fill or [],
+        data_path=effective_data_path,
+        fill_arguments=effective_fill_args,
         press_buttons=args.press or [],
         upload_files=args.upload or [],
         cookies_path=args.cookies,

webpilot_engine-2.0.5/common/process_manager.py

@@ -0,0 +1,259 @@
+"""Cross-Platform Process Management and Isolation Layer.
+
+Isolates OS-specific process lifecycle mechanisms (WMI, CIM, taskkill on Windows;
+killpg, pgrep, signals on POSIX/Linux/macOS) behind a clean, unified ProcessManager interface.
+"""
+
+from __future__ import annotations
+
+import base64
+import os
+import signal
+import subprocess
+import sys
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Union
+
+
+class BaseProcessManager(ABC):
+    """Abstract interface for system process management and detached lifecycle execution."""
+
+    @abstractmethod
+    def launch_detached(
+        self,
+        cmd: Union[str, List[str]],
+        cwd: Optional[str] = None,
+        env: Optional[Dict[str, str]] = None,
+    ) -> Optional[int]:
+        """Launches a detached process decoupled from the caller's terminal and process group."""
+        pass
+
+    @abstractmethod
+    def kill_process_tree(self, pid: int, force: bool = True) -> bool:
+        """Kills the target process and all descendant child processes (cascading teardown)."""
+        pass
+
+    @abstractmethod
+    def get_child_pids(self, parent_pid: int) -> List[int]:
+        """Returns direct child process IDs for the given parent PID."""
+        pass
+
+    @abstractmethod
+    def is_process_running(self, pid: int) -> bool:
+        """Checks if a process with the specified PID is currently active."""
+        pass
+
+
+class WindowsProcessManager(BaseProcessManager):
+    """Windows-specific implementation using PowerShell WMI/CIM and taskkill."""
+
+    def launch_detached(
+        self,
+        cmd: Union[str, List[str]],
+        cwd: Optional[str] = None,
+        env: Optional[Dict[str, str]] = None,
+    ) -> Optional[int]:
+        target_cwd = cwd or os.getcwd()
+        if isinstance(cmd, list):
+            cmd_str = " ".join(f'"{arg}"' if " " in arg and not arg.startswith('"') else arg for arg in cmd)
+        else:
+            cmd_str = cmd
+
+        # Primary approach: Win32_Process.Create via WMI to break out of terminal Job Objects
+        ps_script = f"""
+$cwd = '{target_cwd}'
+$cmd = @'
+{cmd_str}
+'@
+$startup = ([wmiclass]'Win32_ProcessStartup').CreateInstance()
+$startup.ShowWindow = 0
+$proc = ([wmiclass]'Win32_Process').Create($cmd, $cwd, $startup)
+if ($proc.ReturnValue -eq 0) {{
+    Write-Output $proc.ProcessId
+}} else {{
+    exit 1
+}}
+"""
+        try:
+            encoded = base64.b64encode(ps_script.encode("utf-16le")).decode("ascii")
+            res = subprocess.run(
+                ["powershell", "-NoProfile", "-NonInteractive", "-EncodedCommand", encoded],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            out = res.stdout.strip()
+            if res.returncode == 0 and out.isdigit():
+                return int(out)
+        except Exception:
+            pass
+
+        # Fallback approach: subprocess.Popen with DETACHED_PROCESS
+        try:
+            cmd_args = cmd if isinstance(cmd, list) else cmd_str
+            creation_flags = (
+                getattr(subprocess, "DETACHED_PROCESS", 0x00000008)
+                | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200)
+            )
+            proc = subprocess.Popen(
+                cmd_args,
+                cwd=target_cwd,
+                env=env,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                stdin=subprocess.DEVNULL,
+                creationflags=creation_flags,
+            )
+            return proc.pid
+        except Exception:
+            return None
+
+    def kill_process_tree(self, pid: int, force: bool = True) -> bool:
+        if not pid or pid <= 0:
+            return False
+        args = ["taskkill"]
+        if force:
+            args.append("/F")
+        args.extend(["/T", "/PID", str(pid)])
+        try:
+            res = subprocess.run(
+                args,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=10,
+            )
+            return res.returncode == 0
+        except Exception:
+            return False
+
+    def get_child_pids(self, parent_pid: int) -> List[int]:
+        if not parent_pid or parent_pid <= 0:
+            return []
+        try:
+            ps_c = f"(Get-CimInstance Win32_Process | Where-Object {{ $_.ParentProcessId -eq {parent_pid} }}).ProcessId"
+            out = subprocess.check_output(
+                ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps_c],
+                text=True,
+                stderr=subprocess.DEVNULL,
+                timeout=8,
+            ).strip()
+            pids = []
+            for line in out.splitlines():
+                line = line.strip()
+                if line.isdigit():
+                    pids.append(int(line))
+            return pids
+        except Exception:
+            return []
+
+    def is_process_running(self, pid: int) -> bool:
+        if not pid or pid <= 0:
+            return False
+        try:
+            ps_c = f"(Get-Process -Id {pid} -ErrorAction SilentlyContinue) -ne $null"
+            out = subprocess.check_output(
+                ["powershell", "-NoProfile", "-NonInteractive", "-Command", ps_c],
+                text=True,
+                stderr=subprocess.DEVNULL,
+                timeout=5,
+            ).strip()
+            return out.lower() == "true"
+        except Exception:
+            return False
+
+
+class PosixProcessManager(BaseProcessManager):
+    """POSIX/Linux/macOS implementation using setsid, signal.SIGKILL, and pgrep."""
+
+    def launch_detached(
+        self,
+        cmd: Union[str, List[str]],
+        cwd: Optional[str] = None,
+        env: Optional[Dict[str, str]] = None,
+    ) -> Optional[int]:
+        target_cwd = cwd or os.getcwd()
+        try:
+            cmd_args = cmd if isinstance(cmd, list) else cmd.split()
+            proc = subprocess.Popen(
+                cmd_args,
+                cwd=target_cwd,
+                env=env,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                stdin=subprocess.DEVNULL,
+                start_new_session=True,  # setsid decoupling
+            )
+            return proc.pid
+        except Exception:
+            return None
+
+    def kill_process_tree(self, pid: int, force: bool = True) -> bool:
+        if not pid or pid <= 0:
+            return False
+        sig = signal.SIGKILL if force else signal.SIGTERM
+        killed = False
+        # Strategy 1: Process group kill if leader
+        try:
+            pgid = os.getpgid(pid)
+            os.killpg(pgid, sig)
+            killed = True
+        except Exception:
+            pass
+
+        # Strategy 2: pkill children then kill target
+        try:
+            subprocess.run(["pkill", "-P", str(pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            os.kill(pid, sig)
+            killed = True
+        except Exception:
+            pass
+
+        return killed
+
+    def get_child_pids(self, parent_pid: int) -> List[int]:
+        if not parent_pid or parent_pid <= 0:
+            return []
+        try:
+            out = subprocess.check_output(
+                ["pgrep", "-P", str(parent_pid)],
+                text=True,
+                stderr=subprocess.DEVNULL,
+                timeout=5,
+            ).strip()
+            pids = []
+            for line in out.splitlines():
+                line = line.strip()
+                if line.isdigit():
+                    pids.append(int(line))
+            return pids
+        except Exception:
+            return []
+
+    def is_process_running(self, pid: int) -> bool:
+        if not pid or pid <= 0:
+            return False
+        try:
+            os.kill(pid, 0)
+            return True
+        except OSError:
+            return False
+
+
+_GLOBAL_PROCESS_MANAGER: Optional[BaseProcessManager] = None
+
+
+def get_process_manager() -> BaseProcessManager:
+    """Returns the platform-appropriate ProcessManager singleton."""
+    global _GLOBAL_PROCESS_MANAGER
+    if _GLOBAL_PROCESS_MANAGER is None:
+        if sys.platform == "win32":
+            _GLOBAL_PROCESS_MANAGER = WindowsProcessManager()
+        else:
+            _GLOBAL_PROCESS_MANAGER = PosixProcessManager()
+    return _GLOBAL_PROCESS_MANAGER
+
+
+def set_process_manager(manager: Optional[BaseProcessManager]) -> None:
+    """Allows injecting custom or mock ProcessManager implementations for testing."""
+    global _GLOBAL_PROCESS_MANAGER
+    _GLOBAL_PROCESS_MANAGER = manager

webpilot_engine-2.0.5/common/secrets.py

@@ -0,0 +1,120 @@
+"""Secrets resolution and credential hygiene utilities.
+
+Enables secure injection of sensitive form values without leaving plain-text
+passwords in terminal history, process lists, or shell logfiles.
+Supports:
+  - @env:VARIABLE_NAME (reads secret from specified environment variable)
+  - @stdin (reads securely from stdin/pipe without shell echo)
+  - @file:FILE_PATH (reads secret from a local protected file)
+"""
+
+from __future__ import annotations
+
+import getpass
+import os
+import sys
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from core.exceptions import ConfigurationError, ValidationError
+
+
+def resolve_secret_value(value: Any, prompt_label: Optional[str] = None) -> Any:
+    """Resolves secret references (@env:VAR, @stdin, @file:PATH) to actual string values."""
+    if not isinstance(value, str):
+        return value
+
+    trimmed = value.strip()
+
+    # 1. Environment Variable Reference (@env:VAR_NAME)
+    if trimmed.startswith("@env:"):
+        var_name = trimmed[5:].strip()
+        if not var_name:
+            raise ValidationError("Empty environment variable name specified in '@env:' reference.")
+        env_val = os.environ.get(var_name)
+        if env_val is None:
+            raise ConfigurationError(
+                f"Referenced environment variable '{var_name}' is not set. "
+                f"Please define '{var_name}' before invoking WebPilot."
+            )
+        return env_val
+
+    # 2. Standard Input Reference (@stdin or pure '-')
+    if trimmed in ("@stdin", "-"):
+        if sys.stdin.isatty():
+            # Interactive terminal: prompt securely without terminal echo
+            label = prompt_label or "secret value"
+            try:
+                entered = getpass.getpass(f"Enter {label}: ")
+                return entered
+            except Exception as exc:
+                raise ValidationError(f"Failed to read secret from stdin: {exc}") from exc
+        else:
+            # Piped or redirected input: read single line from pipe
+            line = sys.stdin.readline()
+            return line.rstrip("\r\n")
+
+    # 3. Secure File Reference (@file:PATH)
+    if trimmed.startswith("@file:"):
+        file_path = trimmed[6:].strip()
+        if not os.path.isfile(file_path):
+            raise ConfigurationError(f"Referenced secret file not found: {file_path}")
+        try:
+            with open(file_path, "r", encoding="utf-8") as f:
+                return f.read().rstrip("\r\n")
+        except Exception as exc:
+            raise ConfigurationError(f"Failed to read secret file '{file_path}': {exc}") from exc
+
+    return value
+
+
+def resolve_payload_secrets(data: Any) -> Any:
+    """Recursively resolves secret references in arbitrary dictionary or list structures."""
+    if isinstance(data, dict):
+        return {k: resolve_payload_secrets(v) for k, v in data.items()}
+    elif isinstance(data, list):
+        return [resolve_payload_secrets(item) for item in data]
+    elif isinstance(data, tuple) and len(data) == 2:
+        return (data[0], resolve_secret_value(data[1], prompt_label=str(data[0])))
+    elif isinstance(data, str):
+        return resolve_secret_value(data)
+    return data
+
+
+def parse_and_resolve_fill_args(fill_args: Optional[List[str]]) -> List[Tuple[str, Any]]:
+    """Parses list of CLI --fill arguments and resolves any @env, @stdin, or @file references.
+
+    Also handles batch input from stdin if an argument is literally '-' or '@stdin'.
+    """
+    if not fill_args:
+        return []
+
+    results: List[Tuple[str, Any]] = []
+
+    for item in fill_args:
+        trimmed = item.strip()
+
+        # Check for batch stdin ingestion
+        if trimmed in ("-", "@stdin"):
+            if not sys.stdin.isatty():
+                # Read all lines from pipe: each line expected to be key=value
+                for line in sys.stdin:
+                    line_str = line.strip()
+                    if line_str and "=" in line_str and not line_str.startswith("#"):
+                        k, v = line_str.split("=", 1)
+                        resolved_v = resolve_secret_value(v.strip(), prompt_label=k.strip())
+                        results.append((k.strip(), resolved_v))
+            continue
+
+        if "=" not in trimmed:
+            raise ValidationError(
+                f"Invalid --fill argument format: '{item}'. "
+                "Expected 'key=value', e.g. --fill 'username=john' or --fill 'password=@env:MY_PASS'"
+            )
+
+        k, v = trimmed.split("=", 1)
+        key = k.strip()
+        raw_val = v.strip()
+        resolved_val = resolve_secret_value(raw_val, prompt_label=key)
+        results.append((key, resolved_val))
+
+    return results

{webpilot_engine-2.0.4 → webpilot_engine-2.0.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "webpilot-engine"
-version = "2.0.4"
+version = "2.0.5"
 description = "WebPilot: Enterprise Autonomous Browser Engine, Universal Form Inspector & AI Agent Web Runtime"
 readme = "README.md"
 authors = [