forgexa-cli 1.43.2__tar.gz → 1.43.3__tar.gz

{forgexa_cli-1.43.2 → forgexa_cli-1.43.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.43.2
+Version: 1.43.3
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License-Expression: MIT

{forgexa_cli-1.43.2 → forgexa_cli-1.43.3}/forgexa_cli/__init__.py

@@ -1,2 +1,2 @@
 """forgexa-cli — Forgexa command-line client."""
-__version__ = "1.43.2"
+__version__ = "1.43.3"

{forgexa_cli-1.43.2 → forgexa_cli-1.43.3}/forgexa_cli/agent_core.py

@@ -1766,6 +1766,8 @@ def copilot_base_env(env: dict[str, str], *, copilot_home: str | Path) -> dict[s
     env = dict(env)
     env["TERM"] = "dumb"  # suppress TTY-detection that suspends the process
     env["COPILOT_HOME"] = str(copilot_home)
+    env.setdefault("TOKIO_WORKER_THREADS", "2")
+    env["FORGEXA_AGENT_PROCESS"] = "1"
     return env
 
 
@@ -1808,6 +1810,113 @@ def is_copilot_model_discovery_auth_error(*messages: str | None) -> bool:
     )
 
 
+def is_runtime_resource_exhaustion_error(*messages: str | None) -> bool:
+    """Return whether the OS refused to create an agent process or thread."""
+    detail = " ".join(str(message or "").lower() for message in messages)
+    return any(
+        pattern in detail
+        for pattern in (
+            "can't spawn worker thread",
+            "failed to load ca certificates off thread",
+            "resource temporarily unavailable (os error 11)",
+            "pthread_create failed",
+        )
+    )
+
+
+def collect_runtime_resource_metrics(
+    *,
+    proc_root: str | Path = "/proc",
+    cgroup_root: str | Path = "/sys/fs/cgroup",
+) -> dict[str, Any]:
+    """Collect Linux process/thread headroom without optional dependencies."""
+    metrics: dict[str, Any] = {}
+    if not sys.platform.startswith("linux"):
+        return metrics
+
+    proc_path = Path(proc_root)
+    cgroup_path = Path(cgroup_root)
+    constraints: list[tuple[str, int, int]] = []
+
+    try:
+        for line in (proc_path / "self" / "cgroup").read_text().splitlines():
+            hierarchy, controllers, relative_path = line.split(":", 2)
+            relative = relative_path.lstrip("/")
+            if hierarchy == "0" and not controllers:
+                pid_dir = cgroup_path / relative
+            elif "pids" in controllers.split(","):
+                pid_dir = cgroup_path / "pids" / relative
+            else:
+                continue
+            current_text = (pid_dir / "pids.current").read_text().strip()
+            maximum_text = (pid_dir / "pids.max").read_text().strip()
+            if maximum_text != "max":
+                constraints.append(("cgroup", int(current_text), int(maximum_text)))
+            break
+    except (OSError, ValueError):
+        pass
+
+    try:
+        import resource
+
+        soft_limit, _ = resource.getrlimit(resource.RLIMIT_NPROC)
+        if soft_limit != resource.RLIM_INFINITY:
+            current_uid = os.getuid()
+            user_threads = 0
+            for status_path in proc_path.glob("[0-9]*/status"):
+                try:
+                    real_uid: int | None = None
+                    threads = 1
+                    for line in status_path.read_text().splitlines():
+                        if line.startswith("Uid:"):
+                            real_uid = int(line.split()[1])
+                        elif line.startswith("Threads:"):
+                            threads = int(line.split()[1])
+                    if real_uid == current_uid:
+                        user_threads += threads
+                except (OSError, ValueError, IndexError):
+                    continue
+            constraints.append(("rlimit_nproc", user_threads, int(soft_limit)))
+    except (ImportError, OSError, ValueError):
+        pass
+
+    if constraints:
+        source, current, limit = min(
+            constraints,
+            key=lambda item: item[2] - item[1],
+        )
+        metrics.update({
+            "process_limit_source": source,
+            "process_current": current,
+            "process_limit": limit,
+            "process_remaining": max(0, limit - current),
+        })
+
+    try:
+        for line in (proc_path / "meminfo").read_text().splitlines():
+            if line.startswith("MemAvailable:"):
+                metrics["memory_available_mb"] = int(line.split()[1]) // 1024
+                break
+    except (OSError, ValueError, IndexError):
+        pass
+    return metrics
+
+
+def runtime_resource_exhaustion_reason(metrics: dict[str, Any]) -> str | None:
+    """Return a blocking reason when an agent lacks safe process headroom."""
+    process_remaining = metrics.get("process_remaining")
+    if isinstance(process_remaining, int) and process_remaining < 32:
+        return (
+            f"Only {process_remaining} process/thread slots remain "
+            f"({metrics.get('process_current')}/{metrics.get('process_limit')} used via "
+            f"{metrics.get('process_limit_source')})"
+        )
+    memory_available_mb = metrics.get("memory_available_mb")
+    if isinstance(memory_available_mb, int) and memory_available_mb < 256:
+        return f"Only {memory_available_mb} MiB memory is available"
+    return None
+
+
 def copilot_auth_state_signature(copilot_home: str | Path) -> str | None:
     """Return a content signature for Copilot's persisted authentication state."""
     config_file = Path(copilot_home) / "config.json"

{forgexa_cli-1.43.2 → forgexa_cli-1.43.3}/forgexa_cli/daemon.py

@@ -910,7 +910,7 @@ except (ImportError, ModuleNotFoundError):
 # DAEMON_VERSION is the protocol/logic version of the daemon code.
 # Kept in sync with pyproject.toml version via bump-version.sh.
 # CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
-DAEMON_VERSION = "1.43.2"
+DAEMON_VERSION = "1.43.3"
 
 
 def _detect_client_type() -> str:
@@ -4783,6 +4783,10 @@ class ProcessManager:
         "websocket receive failed",
         "misdirected request",
         "could not retrieve the list of available models",
+        "can't spawn worker thread",
+        "failed to load ca certificates off thread",
+        "resource temporarily unavailable (os error 11)",
+        "pthread_create failed",
         "failed to initialize mcp client",
         # "api error" removed: too broad — matches agent-generated code/output
         # discussing API errors. Real API transport errors are covered by the
@@ -5077,6 +5081,38 @@ class ProcessManager:
         on_chunk: Any = None,
     ) -> TaskResult:
         """Execute an agent CLI and collect results."""
+        resource_metrics = agent_core.collect_runtime_resource_metrics()
+        resource_reason = agent_core.runtime_resource_exhaustion_reason(
+            resource_metrics,
+        )
+        if resource_reason:
+            check = {
+                "id": "process_headroom",
+                "status": "fail",
+                "blocking": True,
+                "failure_class": "resource_exhausted",
+                "detail": resource_reason,
+                "remediation": (
+                    "Stop orphaned agent processes or raise the user/cgroup PID limit, "
+                    "then retry"
+                ),
+            }
+            logger.error(
+                "Refusing to launch agent '%s' for task %s: %s",
+                agent.agent_id,
+                task.task_id,
+                resource_reason,
+            )
+            return TaskResult(
+                status="failed",
+                exit_code=-1,
+                stdout="",
+                stderr="",
+                error=f"Runtime resource preflight failed: {resource_reason}",
+                failure_code="runtime_preflight_failed",
+                preflight={"scope": "runtime", "checks": [check]},
+            )
+
         # Centralized, agent-agnostic version gate: AgentDiscovery.discover()
         # already computed min_version_ok/version_warning against
         # AGENT_REGISTRY[agent_id]["min_version"] (see _check_agent_min_version).
@@ -6399,6 +6435,14 @@ class ProcessManager:
                     stdout=stdout[-settings.AGENT_MAX_OUTPUT_SIZE:],
                     stderr=stderr[-10000:],
                     error=failure_error,
+                    failure_code=(
+                        "runtime_resource_exhausted"
+                        if agent_core.is_runtime_resource_exhaustion_error(
+                            failure_error,
+                            stderr,
+                        )
+                        else ""
+                    ),
                     metrics=metrics,
                 )
         except asyncio.CancelledError:
@@ -6748,26 +6792,11 @@ class ProcessManager:
     async def cancel(self, task_id: str):
         proc = self.active_processes.pop(task_id, None)
         if proc:
+            _kill_proc(proc)
             try:
-                if sys.platform != "win32":
-                    import signal as _signal
-                    try:
-                        os.killpg(os.getpgid(proc.pid), _signal.SIGKILL)
-                    except (ProcessLookupError, PermissionError, OSError):
-                        pass
-                else:
-                    import subprocess as _subprocess
-                    _subprocess.run(
-                        ["taskkill", "/F", "/T", "/PID", str(proc.pid)],
-                        capture_output=True,
-                    )
+                await asyncio.wait_for(proc.wait(), timeout=5)
             except Exception:
-                pass
-            finally:
-                try:
-                    proc.kill()
-                except Exception:
-                    pass
+                logger.warning("Timed out reaping agent process for task %s", task_id)
 
 
 # ── Progress Reporter ──
@@ -6967,8 +6996,48 @@ class RuntimePreflightMonitor:
             await self._check_git_executable(),
             await self._check_workspace_root(),
             await self._check_workspace_disk(),
+            await self._check_process_headroom(),
         ]
 
+    @staticmethod
+    async def _check_process_headroom() -> dict:
+        metrics = agent_core.collect_runtime_resource_metrics()
+        reason = agent_core.runtime_resource_exhaustion_reason(metrics)
+        if reason:
+            return {
+                "id": "process_headroom",
+                "status": "fail",
+                "blocking": True,
+                "failure_class": "resource_exhausted",
+                "detail": reason,
+                "remediation": (
+                    "Stop orphaned agent processes or raise the user/cgroup PID limit, "
+                    "then restart the daemon"
+                ),
+            }
+        if metrics:
+            detail = (
+                f"{metrics['process_remaining']} process/thread slots remain"
+                if "process_remaining" in metrics
+                else f"{metrics.get('memory_available_mb', 'unknown')} MiB memory available"
+            )
+            return {
+                "id": "process_headroom",
+                "status": "pass",
+                "blocking": True,
+                "failure_class": "",
+                "detail": detail,
+                "remediation": "",
+            }
+        return {
+            "id": "process_headroom",
+            "status": "unknown",
+            "blocking": False,
+            "failure_class": "",
+            "detail": "Process/thread headroom is unavailable on this platform",
+            "remediation": "",
+        }
+
     @staticmethod
     async def _run_command_timeout(cmd: list[str], timeout: float = 5) -> str | None:
         """Run a command with a deadline. Returns stdout on success, None on timeout/error.
@@ -7273,15 +7342,17 @@ class HeartbeatService:
 
     def _collect_system_metrics(self) -> dict:
         """Basic system metrics."""
+        metrics = agent_core.collect_runtime_resource_metrics()
         try:
             import psutil
-            return {
+            metrics.update({
                 "cpu_percent": psutil.cpu_percent(),
                 "memory_percent": psutil.virtual_memory().percent,
                 "disk_percent": psutil.disk_usage("/").percent,
-            }
+            })
         except ImportError:
-            return {}
+            pass
+        return metrics
 
 
 # ── Log Uploader ──
@@ -8619,6 +8690,12 @@ class RuntimeDaemon:
         """Main entry point."""
         # Prevent multiple daemon instances on the same machine
         self._acquire_lock()
+        cleaned_processes = self._cleanup_stale_agent_processes()
+        if cleaned_processes:
+            logger.warning(
+                "Cleaned up %d orphaned Forgexa agent process group(s)",
+                cleaned_processes,
+            )
 
         # Mint a local-dev token if no token is available yet
         if not self.api_token and not self._has_stored_credentials():
@@ -9328,7 +9405,15 @@ class RuntimeDaemon:
             # Guard: if the agent already produced file changes in the workspace, it DID
             # meaningful work — don't trigger fallback even if it crashed after completing.
             # Let the recovery logic (step 4.1) handle non-zero exit with committed work.
-            agent_failure_kind = ProcessManager.agent_failure_kind(result)
+            runtime_resource_failure = result.failure_code in {
+                "runtime_preflight_failed",
+                "runtime_resource_exhausted",
+            }
+            agent_failure_kind = (
+                None
+                if runtime_resource_failure
+                else ProcessManager.agent_failure_kind(result)
+            )
             is_agent_fallback_eligible = agent_failure_kind is not None
             is_silent_idle_timeout = self.process_manager.is_silent_idle_timeout(result)
             fallback_reason = (
@@ -11465,7 +11550,13 @@ class RuntimeDaemon:
             # try the next available agent before giving up. An explicit
             # override is deliberately a single-agent attempt.
             fallback_scope = "override" if agent_override else "auto"
-            allow_fallback = not bool(agent_override)
+            allow_fallback = (
+                not bool(agent_override)
+                and result.failure_code not in {
+                    "runtime_preflight_failed",
+                    "runtime_resource_exhausted",
+                }
+            )
             generation_attempts: list[dict] = []
             agent_failure_kind = ProcessManager.agent_failure_kind(result)
             agent_failure_label = (
@@ -14081,15 +14172,78 @@ class RuntimeDaemon:
                 except Exception as e:
                     logger.debug("Could not clean temp dir %s: %s", d, e)
 
+    @staticmethod
+    def _cleanup_stale_agent_processes(
+        *,
+        proc_root: str | Path = "/proc",
+        temp_root: str | Path | None = None,
+    ) -> int:
+        """Kill orphaned agent groups identifiable as Forgexa-owned on Linux."""
+        if not sys.platform.startswith("linux"):
+            return 0
+        import signal as _signal
+        import tempfile
+
+        root = Path(proc_root)
+        temp = Path(temp_root or tempfile.gettempdir()).resolve()
+        current_pid = os.getpid()
+        current_group = os.getpgrp()
+        cleaned = 0
+        try:
+            process_dirs = sorted(
+                (path for path in root.iterdir() if path.name.isdigit()),
+                key=lambda path: int(path.name),
+            )
+        except OSError:
+            return 0
+
+        for process_dir in process_dirs:
+            pid = int(process_dir.name)
+            if pid == current_pid:
+                continue
+            try:
+                entries = (process_dir / "environ").read_bytes().split(b"\0")
+                environment = {}
+                for entry in entries:
+                    if b"=" not in entry:
+                        continue
+                    key, value = entry.split(b"=", 1)
+                    environment[key.decode(errors="ignore")] = value.decode(errors="ignore")
+                marked = environment.get("FORGEXA_AGENT_PROCESS") == "1"
+                isolated_home = environment.get("COPILOT_HOME", "")
+                legacy_copilot = False
+                if isolated_home:
+                    isolated_path = Path(isolated_home).resolve()
+                    legacy_copilot = (
+                        isolated_path.parent == temp
+                        and isolated_path.name.startswith("copilot-")
+                    )
+                if not (marked or legacy_copilot):
+                    continue
+                process_group = os.getpgid(pid)
+                if process_group == current_group:
+                    continue
+                os.killpg(process_group, _signal.SIGKILL)
+                cleaned += 1
+            except (OSError, ValueError):
+                continue
+        return cleaned
+
     async def _shutdown_gracefully(self):
         """Graceful shutdown."""
         logger.info("Shutting down daemon...")
         self._shutdown = True
 
-        # Cancel active tasks
-        for task_id, task in self.active_tasks.items():
+        # Cancel task coroutines, then terminate and reap every tracked process.
+        tasks = list(self.active_tasks.values())
+        for task in tasks:
             task.cancel()
-            await self.process_manager.cancel(task_id)
+        for process_id in list(self.process_manager.active_processes):
+            await self.process_manager.cancel(process_id)
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+        self.active_tasks.clear()
+        self._task_connections.clear()
 
         # Stop preflight monitor
         await self.preflight_monitor.stop()

{forgexa_cli-1.43.2 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.43.2
+Version: 1.43.3
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License-Expression: MIT

{forgexa_cli-1.43.2 → forgexa_cli-1.43.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "forgexa-cli"
-version = "1.43.2"
+version = "1.43.3"
 description = "Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform"
 requires-python = ">=3.9"
 license = "MIT"