forgexa-cli 1.8.7__tar.gz → 1.8.10__tar.gz

{forgexa_cli-1.8.7 → forgexa_cli-1.8.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.8.7
+Version: 1.8.10
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License: MIT

{forgexa_cli-1.8.7 → forgexa_cli-1.8.10}/forgexa_cli/__init__.py

@@ -1,2 +1,2 @@
 """forgexa-cli — Forgexa command-line client."""
-__version__ = "1.8.7"
+__version__ = "1.8.10"

{forgexa_cli-1.8.7 → forgexa_cli-1.8.10}/forgexa_cli/daemon.py

@@ -352,7 +352,11 @@ except (ImportError, ModuleNotFoundError):
 
         @property
         def AGENT_TIMEOUT(self) -> int:
-            return int(os.environ.get("AGENT_TIMEOUT", "3600"))
+            return int(os.environ.get("AGENT_TIMEOUT", "14400"))  # 4-hour absolute ceiling
+
+        @property
+        def AGENT_IDLE_TIMEOUT(self) -> int:
+            return int(os.environ.get("AGENT_IDLE_TIMEOUT", "600"))  # 10-min idle (stdout+fs) = hung agent
 
         @property
         def GIT_CLONE_TIMEOUT(self) -> int:
@@ -392,7 +396,7 @@ except (ImportError, ModuleNotFoundError):
 # DAEMON_VERSION is the protocol/logic version of the daemon code.
 # Kept in sync with pyproject.toml version via bump-version.sh.
 # CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
-DAEMON_VERSION = "1.8.7"
+DAEMON_VERSION = "1.8.10"
 
 
 def _detect_client_type() -> str:
@@ -633,6 +637,11 @@ class TaskResult:
     lines_added: int = 0
     lines_removed: int = 0
     error: str = ""
+    # failure_code is forwarded to the server to drive retry policy.
+    # Key values:
+    #   "all_agents_rate_limited" — daemon tried every installed agent, all
+    #       hit rate/quota limits.  Server must NOT retry on the same runtime.
+    failure_code: str = ""
     artifacts: list[dict] = field(default_factory=list)
     observations: list[dict] = field(default_factory=list)
     metrics: dict = field(default_factory=dict)
@@ -1753,15 +1762,34 @@ class WorkspaceManager:
                     key_path_safe = key_path.replace("\\", "/") if sys.platform == "win32" else key_path
                     # RC1 (Windows): os.chmod(S_IRUSR) does not set proper NTFS ACLs.
                     # Windows OpenSSH rejects keys that aren't exclusively owner-readable
-                    # ("UNPROTECTED PRIVATE KEY FILE").  StrictModes=no bypasses this.
+                    # ("UNPROTECTED PRIVATE KEY FILE").  Fix with icacls to set the ACLs
+                    # correctly.  NOTE: StrictModes is an sshd_config option (server side);
+                    # passing -o StrictModes=no to the SSH client is invalid and causes:
+                    #   "command-line: line 0: Bad configuration option: strictmodes"
                     # RC2 (Windows): /dev/null doesn't exist on Windows native OpenSSH
                     # (C:\Windows\System32\OpenSSH\ssh.exe). Use NUL instead.
                     if sys.platform == "win32":
                         _known_hosts_null = "NUL"
-                        _strict_modes_opt = " -o StrictModes=no"
+                        try:
+                            import subprocess as _subp
+                            _username = (
+                                os.environ.get("USERNAME")
+                                or os.environ.get("USER")
+                                or ""
+                            )
+                            if _username:
+                                _subp.run(
+                                    [
+                                        "icacls", key_path,
+                                        "/inheritance:r",
+                                        "/grant:r", f"{_username}:(R)",
+                                    ],
+                                    capture_output=True, check=False, timeout=10,
+                                )
+                        except Exception:
+                            pass
                     else:
                         _known_hosts_null = "/dev/null"
-                        _strict_modes_opt = ""
                     env = {
                         **os.environ,
                         "GIT_SSH_COMMAND": (
@@ -1777,7 +1805,6 @@ class WorkspaceManager:
                             f" -o ConnectTimeout=30"
                             f" -o ServerAliveInterval=30"
                             f" -o ServerAliveCountMax=3"
-                            f"{_strict_modes_opt}"
                         ),
                     }
                 except Exception:
@@ -1797,6 +1824,13 @@ class WorkspaceManager:
                 if git_prefix_args:
                     env = {**(env or os.environ), "GIT_TERMINAL_PROMPT": "0"}
 
+        # Always enable long-path support.  On Windows this removes git's own
+        # 260-char path limit (Windows also needs HKLM LongPathsEnabled=1 or
+        # the Win10 1607+ Group Policy, but at a minimum we ensure git won't
+        # reject long paths on platforms where it is already enabled).
+        # On Linux/macOS this is a no-op.
+        longpath_args = ["-c", "core.longpaths=true"]
+
         # start_new_session=True puts git in its own process group.
         # On timeout we send SIGKILL to the entire group, which includes
         # any ssh/gpg/credential-helper children that git forked — preventing
@@ -1804,7 +1838,7 @@ class WorkspaceManager:
         # Windows note: start_new_session creates a new console process group;
         # we use taskkill /T there instead of killpg.
         proc = await asyncio.create_subprocess_exec(
-            "git", *git_prefix_args, *args,
+            "git", *longpath_args, *git_prefix_args, *args,
             stdout=asyncio.subprocess.PIPE,
             stderr=asyncio.subprocess.PIPE,
             cwd=str(cwd) if cwd else None,
@@ -1865,6 +1899,92 @@ class WorkspaceManager:
 # ── Process Manager ──
 
 
+def _kill_proc(proc: asyncio.subprocess.Process) -> None:
+    """Kill a subprocess and its entire process group.
+
+    A plain ``proc.kill()`` only terminates the direct child; grandchildren
+    (npm, yarn, ssh, git, etc.) spawned by the agent stay alive, keep pipes
+    open, and exhaust system resources.  ``os.killpg`` sends SIGKILL to the
+    whole process group, reliably cleaning up all descendants.
+    """
+    try:
+        if sys.platform != "win32":
+            import signal as _signal
+            try:
+                os.killpg(os.getpgid(proc.pid), _signal.SIGKILL)
+            except (ProcessLookupError, PermissionError, OSError):
+                pass
+        else:
+            import subprocess as _subprocess
+            _subprocess.run(
+                ["taskkill", "/F", "/T", "/PID", str(proc.pid)],
+                capture_output=True,
+            )
+    except Exception:
+        pass
+    finally:
+        try:
+            proc.kill()
+        except Exception:
+            pass
+
+
+class _IdleTimeoutError(asyncio.TimeoutError):
+    """Raised when an agent process produces no stdout for longer than AGENT_IDLE_TIMEOUT.
+
+    Subclasses asyncio.TimeoutError so existing ``except asyncio.TimeoutError``
+    handlers catch it, but callers can distinguish it from an absolute wall-clock
+    timeout via ``isinstance(exc, _IdleTimeoutError)`` or ``exc.idle_seconds``.
+    """
+
+    def __init__(self, idle_seconds: float) -> None:
+        super().__init__(f"idle:{idle_seconds:.0f}s")
+        self.idle_seconds = idle_seconds
+
+
+def _workspace_has_recent_activity(
+    workspace_path: "Path", since_monotonic: float, max_depth: int = 4
+) -> bool:
+    """Return True if any file under workspace_path was modified after since_monotonic.
+
+    Converts the monotonic timestamp to a wall-clock value for mtime comparison.
+    Scans the directory tree (up to max_depth levels) with os.scandir, skips
+    .git, and returns on the first matching file for speed.
+
+    This is the key secondary signal used by _stream_process to distinguish
+    "agent is silent but legitimately working" (e.g. running npm install,
+    compiling TypeScript, executing test suites) from "agent is truly hung".
+    All silent-but-busy operations (package installs, compilation, test runs,
+    git operations) write files to disk, so a positive result here means we
+    must NOT kill the process even if stdout has been idle for a long time.
+    """
+    since_wall = time.time() - (time.monotonic() - since_monotonic)
+
+    def _scan(path: "Path", depth: int) -> bool:
+        if depth > max_depth:
+            return False
+        try:
+            for entry in os.scandir(str(path)):
+                if entry.name == ".git":
+                    continue  # skip version-control metadata
+                try:
+                    if entry.stat(follow_symlinks=False).st_mtime > since_wall:
+                        return True
+                    if entry.is_dir(follow_symlinks=False) and depth < max_depth:
+                        if _scan(Path(entry.path), depth + 1):
+                            return True
+                except (OSError, PermissionError):
+                    pass
+        except (OSError, PermissionError):
+            pass
+        return False
+
+    try:
+        return _scan(workspace_path, 0)
+    except Exception:
+        return False  # never let a filesystem check crash the agent run
+
+
 class ProcessManager:
     """Manages Agent CLI subprocess lifecycle."""
 
@@ -2312,6 +2432,7 @@ class ProcessManager:
         timeout: int,
         task_id: str,
         on_chunk: Any,
+        workspace_path: "Path | None" = None,
     ) -> tuple[str, str, int]:
         """Stream stdout line-by-line from a subprocess, flushing to on_chunk.
 
@@ -2323,7 +2444,22 @@ class ProcessManager:
           deadlock when the process fills the stderr buffer.
         - on_chunk(lines) is called with each decoded line so the caller can
           forward to the progress reporter without waiting for completion.
+        - Idle timeout: if the agent produces no stdout for AGENT_IDLE_TIMEOUT
+          seconds the code checks for filesystem activity in workspace_path
+          before deciding to kill.  If files were recently modified the agent
+          is doing silent work (npm install, compilation, test runs, etc.) and
+          the idle timer is reset.  Only when BOTH stdout AND the filesystem
+          are idle does the process get killed.  This eliminates false-positive
+          kills at the idle boundary.
+        - Absolute timeout (``timeout`` param): hard ceiling for zombie-process
+          prevention.  Always kills at this boundary (no extension), but logs
+          filesystem activity status for post-mortem observability.
         """
+        idle_timeout: int = getattr(settings, "AGENT_IDLE_TIMEOUT", 600)
+        _start_time = time.monotonic()
+        # Mutable state shared between nested coroutines (list avoids nonlocal)
+        _last_activity_at: list[float] = [time.monotonic()]
+
         # Write prompt and close stdin so the agent knows input is done.
         if stdin_input and proc.stdin:
             try:
@@ -2345,13 +2481,76 @@ class ProcessManager:
             if not proc.stdout:
                 return
             while True:
+                # ── Timeout checks ────────────────────────────────────────────
+                now = time.monotonic()
+                elapsed_abs = now - _start_time
+                if elapsed_abs >= timeout:
+                    # Absolute ceiling reached — hard zombie-process prevention.
+                    # Log filesystem status for observability but always kill;
+                    # never extend the absolute boundary.
+                    _fs_active = workspace_path and _workspace_has_recent_activity(
+                        workspace_path, _last_activity_at[0]
+                    )
+                    logger.warning(
+                        "Task %s absolute timeout %.0fs reached — killing "
+                        "(workspace filesystem %s)",
+                        task_id, elapsed_abs,
+                        "was active" if _fs_active else "was idle",
+                    )
+                    _kill_proc(proc)
+                    raise asyncio.TimeoutError(
+                        f"Timed out after {timeout}s (absolute limit)"
+                    )
+
+                idle_elapsed = now - _last_activity_at[0]
+                if idle_elapsed >= idle_timeout:
+                    # Before killing, check if the agent is doing silent
+                    # filesystem work (npm install, compilation, test runs, git
+                    # operations, etc.).  These produce no stdout but DO modify
+                    # files — killing at the idle boundary would be a false
+                    # positive.  Only kill when BOTH signals agree: no stdout
+                    # AND no filesystem activity.
+                    if workspace_path and _workspace_has_recent_activity(
+                        workspace_path, _last_activity_at[0]
+                    ):
+                        # Files modified since last stdout → agent is working
+                        # silently.  Reset idle timer and continue.
+                        _last_activity_at[0] = time.monotonic()
+                        logger.info(
+                            "Task %s: stdout idle %.0fs but workspace files "
+                            "modified — resetting idle timer (agent working "
+                            "silently)",
+                            task_id, idle_elapsed,
+                        )
+                    else:
+                        # No stdout AND no filesystem activity → truly hung.
+                        logger.warning(
+                            "Task %s agent idle %.0fs — no stdout, no "
+                            "filesystem activity; killing hung process",
+                            task_id, idle_elapsed,
+                        )
+                        _kill_proc(proc)
+                        raise _IdleTimeoutError(idle_elapsed)
+
+                # Check interval capped at 30s so the loop stays responsive
+                # even when both timeouts are far away.
+                check_interval = min(
+                    idle_timeout - idle_elapsed + 0.5,   # until idle fires
+                    timeout - elapsed_abs + 0.5,          # until absolute fires
+                    30.0,
+                )
+
+                # ── Read one line with a bounded wait ─────────────────────────
                 try:
-                    line_bytes = await proc.stdout.readline()
+                    line_bytes = await asyncio.wait_for(
+                        proc.stdout.readline(), timeout=check_interval
+                    )
+                except asyncio.TimeoutError:
+                    # readline timed out within check_interval — no new output
+                    # yet.  Loop back to re-evaluate idle/absolute conditions.
+                    continue
                 except (ValueError, asyncio.LimitOverrunError, Exception) as exc:
-                    # Line exceeded stream buffer limit (LimitOverrunError
-                    # converted to ValueError by readline(), but catch broadly
-                    # to handle edge cases in different Python versions).
-                    # Fall back to reading remaining data in bulk.
+                    # Line exceeded stream buffer limit — drain remaining bulk.
                     logger.warning(
                         "Stream read error for task %s (%s: %s), draining remaining output",
                         task_id, type(exc).__name__, exc,
@@ -2370,8 +2569,12 @@ class ProcessManager:
                                     except Exception:
                                         pass
                     break
+
                 if not line_bytes:
-                    break
+                    break  # EOF — process exited normally
+
+                # ── New output received — reset idle timer ────────────────────
+                _last_activity_at[0] = time.monotonic()
                 line = line_bytes.decode(errors="replace").rstrip("\n")
                 stdout_lines.append(line)
                 if on_chunk:
@@ -2381,35 +2584,17 @@ class ProcessManager:
                         pass  # never let on_chunk crash the agent run
 
         try:
+            # Outer wait_for uses timeout+idle_timeout as generous safety net.
+            # In practice _read_stdout handles both idle and absolute killing
+            # before this fires.
             await asyncio.wait_for(
                 asyncio.gather(_read_stdout(), _read_stderr()),
-                timeout=timeout,
+                timeout=timeout + idle_timeout + 60,
             )
-        except asyncio.TimeoutError:
-            # Kill the entire process group so that child processes (npm, yarn,
-            # ssh, git, etc.) spawned by the agent are also terminated.  A plain
-            # proc.kill() only kills the direct subprocess; any grandchildren
-            # become orphaned, keep pipes open, and can exhaust system resources.
-            try:
-                if sys.platform != "win32":
-                    import signal as _signal
-                    try:
-                        os.killpg(os.getpgid(proc.pid), _signal.SIGKILL)
-                    except (ProcessLookupError, PermissionError, OSError):
-                        pass
-                else:
-                    import subprocess as _subprocess
-                    _subprocess.run(
-                        ["taskkill", "/F", "/T", "/PID", str(proc.pid)],
-                        capture_output=True,
-                    )
-            except Exception:
-                pass
-            finally:
-                try:
-                    proc.kill()
-                except Exception:
-                    pass
+        except asyncio.TimeoutError as _exc:
+            # This branch fires if the outer safety net triggers (extremely
+            # rare) or if _IdleTimeoutError propagates from _read_stdout.
+            _kill_proc(proc)
             # Drain any remaining output after kill
             try:
                 remaining, _ = await asyncio.wait_for(proc.communicate(), timeout=5)
@@ -2419,7 +2604,7 @@ class ProcessManager:
                             stdout_lines.append(line)
             except Exception:
                 pass
-            raise  # re-raise so callers can set result.error
+            raise  # re-raise (_IdleTimeoutError preserves subclass type)
 
         await proc.wait()
         stdout = "\n".join(stdout_lines)
@@ -2480,7 +2665,8 @@ class ProcessManager:
             )
             self.active_processes[task_id] = proc
             stdout, stderr, returncode = await self._stream_process(
-                proc, prompt.encode(), timeout, task_id, on_chunk
+                proc, prompt.encode(), timeout, task_id, on_chunk,
+                workspace_path=cwd,
             )
 
             # Parse Claude JSON output for metrics
@@ -2521,13 +2707,13 @@ class ProcessManager:
                     error=f"Claude exited with code {returncode}: {stderr[-500:]}",
                     metrics=metrics,
                 )
-        except asyncio.TimeoutError:
-            if task_id in self.active_processes:
-                self.active_processes[task_id].kill()
-            return TaskResult(
-                status="failed", exit_code=-1, stdout="", stderr="",
-                error=f"Timed out after {timeout}s",
-            )
+        except asyncio.TimeoutError as exc:
+            _kill_proc(self.active_processes.pop(task_id, None) or proc)
+            _err = (
+                f"Agent idle for {exc.idle_seconds:.0f}s without output — process terminated. "
+                "Task may require more context decomposition or a different agent."
+            ) if isinstance(exc, _IdleTimeoutError) else f"Timed out after {timeout}s (absolute limit)"
+            return TaskResult(status="failed", exit_code=-1, stdout="", stderr="", error=_err)
         except Exception as exc:
             logger.exception("Claude stream error for task %s", task_id)
             if task_id in self.active_processes:
@@ -2697,7 +2883,8 @@ class ProcessManager:
             )
             self.active_processes[task_id] = proc
             stdout, stderr, returncode = await self._stream_process(
-                proc, None, timeout, task_id, on_chunk
+                proc, None, timeout, task_id, on_chunk,
+                workspace_path=cwd,
             )
 
             # Parse copilot JSONL output for metrics
@@ -2725,13 +2912,13 @@ class ProcessManager:
                     error=f"Copilot exited with code {effective_rc}: {stderr[-500:]}",
                     metrics=metrics,
                 )
-        except asyncio.TimeoutError:
-            if task_id in self.active_processes:
-                self.active_processes[task_id].kill()
-            return TaskResult(
-                status="failed", exit_code=-1, stdout="", stderr="",
-                error=f"Timed out after {timeout}s",
-            )
+        except asyncio.TimeoutError as exc:
+            _kill_proc(self.active_processes.pop(task_id, None) or proc)
+            _err = (
+                f"Agent idle for {exc.idle_seconds:.0f}s without output — process terminated. "
+                "Task may require more context decomposition or a different agent."
+            ) if isinstance(exc, _IdleTimeoutError) else f"Timed out after {timeout}s (absolute limit)"
+            return TaskResult(status="failed", exit_code=-1, stdout="", stderr="", error=_err)
         except Exception as exc:
             logger.exception("Copilot stream error for task %s", task_id)
             if task_id in self.active_processes:
@@ -2772,7 +2959,8 @@ class ProcessManager:
             self.active_processes[task_id] = proc
             stdin_bytes = stdin_input.encode() if stdin_input else None
             stdout, stderr, returncode = await self._stream_process(
-                proc, stdin_bytes, timeout, task_id, on_chunk
+                proc, stdin_bytes, timeout, task_id, on_chunk,
+                workspace_path=cwd,
             )
             status = "success" if returncode == 0 else "failed"
             return TaskResult(
@@ -2782,33 +2970,13 @@ class ProcessManager:
                 stderr=stderr[-10000:],
                 error="" if status == "success" else f"Exited with code {returncode}",
             )
-        except asyncio.TimeoutError:
-            proc = self.active_processes.pop(task_id, None)
-            if proc:
-                try:
-                    if sys.platform != "win32":
-                        import signal as _signal
-                        try:
-                            os.killpg(os.getpgid(proc.pid), _signal.SIGKILL)
-                        except (ProcessLookupError, PermissionError, OSError):
-                            pass
-                    else:
-                        import subprocess as _subprocess
-                        _subprocess.run(
-                            ["taskkill", "/F", "/T", "/PID", str(proc.pid)],
-                            capture_output=True,
-                        )
-                except Exception:
-                    pass
-                finally:
-                    try:
-                        proc.kill()
-                    except Exception:
-                        pass
-            return TaskResult(
-                status="failed", exit_code=-1, stdout="", stderr="",
-                error=f"Timed out after {timeout}s",
-            )
+        except asyncio.TimeoutError as exc:
+            _kill_proc(self.active_processes.pop(task_id, None) or proc)
+            _err = (
+                f"Agent idle for {exc.idle_seconds:.0f}s without output — process terminated. "
+                "Task may require more context decomposition or a different agent."
+            ) if isinstance(exc, _IdleTimeoutError) else f"Timed out after {timeout}s (absolute limit)"
+            return TaskResult(status="failed", exit_code=-1, stdout="", stderr="", error=_err)
         except Exception as exc:
             logger.exception("CLI stream error for task %s", task_id)
             if task_id in self.active_processes:
@@ -3351,6 +3519,7 @@ class ProgressReporter:
             "stdout_tail": result.stdout[-20000:] if result.stdout else "",
             "stderr_tail": result.stderr[-5000:] if result.stderr else "",
             "error": result.error,
+            "failure_code": result.failure_code,
             "files_changed": result.files_changed,
             "lines_added": result.lines_added,
             "lines_removed": result.lines_removed,
@@ -4250,6 +4419,110 @@ class RuntimeDaemon:
             )
             logger.info("Workspace ready: %s", workspace_path)
 
+            # 2.1 Workspace health check: detect broken checkout (Windows filename-
+            # too-long or other git checkout failure that leaves the working tree
+            # empty while the git index still tracks all source files).
+            # If this is not caught the agent will run `git add -A` and commit a
+            # catastrophic mass-deletion (e.g. SI-434: 47,566 files deleted).
+            try:
+                _index_count_out = await self._git(
+                    "ls-files", "--cached", "--", ".", cwd=workspace_path,
+                    timeout=30,
+                )
+                _index_count = len([l for l in _index_count_out.splitlines() if l.strip()])
+                if _index_count > 500:
+                    # Count physical files (exclude .git/)
+                    _phys_count = sum(1 for _ in workspace_path.rglob("*")
+                                      if _.is_file() and ".git" not in _.parts)
+                    _ratio = _phys_count / _index_count
+                    if _ratio < 0.20:
+                        # Less than 20 % of tracked files exist on disk — almost
+                        # certainly a failed git checkout (e.g. Windows path-length
+                        # limit).  Abort rather than letting the agent commit a
+                        # mass-deletion.
+                        _longpath_hint = (
+                            " Enable Windows long-path support: run "
+                            "`git config --global core.longpaths true` and enable "
+                            "LongPathsEnabled in Windows Group Policy / Registry "
+                            "(HKLM\\SYSTEM\\CurrentControlSet\\Control\\FileSystem\\LongPathsEnabled=1)."
+                            if sys.platform == "win32" else ""
+                        )
+                        raise RuntimeError(
+                            f"Workspace health check failed: only {_phys_count}/{_index_count} "
+                            f"tracked files exist on disk ({_ratio:.0%}).  "
+                            f"The git checkout likely failed due to filename-length limitations."
+                            f"{_longpath_hint}"
+                        )
+                    elif _ratio < 0.80:
+                        logger.warning(
+                            "Workspace health check warning: only %d/%d tracked files "
+                            "exist on disk (%.0f%%) for task %s — checkout may be incomplete.",
+                            _phys_count, _index_count, _ratio * 100, task.task_id,
+                        )
+            except RuntimeError:
+                raise
+            except Exception as _health_exc:
+                logger.warning("Workspace health check error (non-fatal): %s", _health_exc)
+
+
+            # agent run starts from a completely clean slate.  This covers:
+            #   • Type change: removes old-type files (e.g. PRD.md/SDD.md) so they
+            #     don't coexist with the new type's files (e.g. diagnosis.md).
+            #   • Same-type fresh re-analysis: removes extra files the agent may
+            #     have written that fall outside the expected type profile.
+            # Using a whole-directory wipe is more reliable than the old
+            # cleanup_stale_docs approach (which only deleted known-profile files).
+            if task.node_type == "analysis" and (
+                task.input_data.get("wipe_analysis_dir")
+                # Backwards-compat: older server versions send cleanup_stale_docs
+                or task.input_data.get("cleanup_stale_docs")
+                # Also wipe if analysis_mode is explicitly "fresh" (belt+suspenders)
+                or task.input_data.get("analysis_mode") == "fresh"
+            ):
+                output_dir_raw = (
+                    task.input_data.get("analysis_output_dir")
+                    or task.input_data.get("output_dir")
+                    or ""
+                )
+                output_dir_norm = str(output_dir_raw).replace("\\", "/").lstrip("./").rstrip("/")
+                if output_dir_norm:
+                    dir_to_wipe = workspace_path / output_dir_norm
+                    if dir_to_wipe.is_dir():
+                        existing_files = [f for f in dir_to_wipe.iterdir() if f.is_file()]
+                        if existing_files:
+                            try:
+                                # Stage all deletions with git rm
+                                await self._git(
+                                    "rm", "-r", "--cached", "--ignore-unmatch",
+                                    output_dir_norm,
+                                    cwd=workspace_path,
+                                )
+                                # Remove physical files
+                                shutil.rmtree(str(dir_to_wipe), ignore_errors=True)
+                                # Commit the wipe so the branch diff is clean
+                                await self._git(
+                                    "-c", "user.name=Forgexa Agent",
+                                    "-c", "user.email=agent@forgexa.net",
+                                    "commit", "-m",
+                                    f"cleanup: wipe analysis docs in {output_dir_norm} before fresh re-analysis",
+                                    cwd=workspace_path,
+                                )
+                                logger.info(
+                                    "Wiped %d analysis doc(s) from %s for task %s (fresh analysis)",
+                                    len(existing_files), output_dir_norm, task.task_id,
+                                )
+                            except Exception:
+                                logger.warning(
+                                    "Could not wipe analysis dir %s for task %s "
+                                    "(proceeding anyway — agent will overwrite)",
+                                    output_dir_norm, task.task_id, exc_info=True,
+                                )
+                        else:
+                            logger.debug(
+                                "Analysis dir %s is already empty for task %s",
+                                output_dir_norm, task.task_id,
+                            )
+
             # 3. Run agent with real-time output streaming + periodic progress heartbeat
             await reporter.report_progress(task.task_id, 10, "running_agent")
 
@@ -4419,6 +4692,10 @@ class RuntimeDaemon:
                         f"Original error: {result.error}"
                     )
                     result.status = "failed"
+                    # Signal to the server that ALL installed agents were tried and
+                    # all are rate/quota limited.  The server must NOT re-enqueue on
+                    # the same runtime — that would hit the same quota wall.
+                    result.failure_code = "all_agents_rate_limited"
 
             # 4. Collect git info BEFORE commit (shows uncommitted changes)
             pre_commit_git = await self.process_manager._collect_git_info(workspace_path)
@@ -4447,26 +4724,49 @@ class RuntimeDaemon:
                         f"(node_type={task.node_type}, agent={agent.agent_id})"
                     )
 
-            # 4.1 Recovery: agent exited non-zero but already committed code
-            # (e.g. OpenCode EBADF crash on exit after successful work)
-            if result.status == "failed" and result.exit_code not in (None, -1):
-                committed_git = await self.process_manager._collect_git_info_vs_parent(workspace_path)
-                has_committed_changes = bool(committed_git.get("files_changed"))
-                has_no_uncommitted = not pre_commit_git.get("files_changed")
-                has_tokens = (
-                    int(result.metrics.get("token_input", 0) or 0)
-                    + int(result.metrics.get("token_output", 0) or 0)
-                ) > 0
-                has_meaningful_output = self.process_manager.has_meaningful_agent_output(result)
-                if has_committed_changes and has_no_uncommitted and (has_tokens or has_meaningful_output):
-                    logger.warning(
-                        "Task %s agent exited with code %s but has committed changes — "
-                        "recovering as success (agent likely crashed during cleanup)",
-                        task.task_id, result.exit_code,
+            # 4.1 Recovery: agent exited non-zero but already committed code.
+            # Covers two scenarios:
+            #   A. Process crash (e.g. OpenCode EBADF on exit after successful work):
+            #      exit_code is a real non-negative/non-(-1) value.
+            #   B. Timeout with committed work: agent finished its task and committed
+            #      before the idle/absolute timeout fired.  exit_code=-1 but the commits
+            #      are real — don't discard them.
+            if result.status == "failed":
+                _error_lower = (result.error or "").lower()
+                is_timeout_failure = (
+                    "idle for" in _error_lower
+                    or "timed out" in _error_lower
+                    or "absolute limit" in _error_lower
+                )
+                can_attempt_recovery = (
+                    is_timeout_failure               # timeout: also allow exit_code=-1
+                    or result.exit_code not in (None, -1)  # crash: original guard
+                )
+                if can_attempt_recovery:
+                    committed_git = await self.process_manager._collect_git_info_vs_parent(workspace_path)
+                    has_committed_changes = bool(committed_git.get("files_changed"))
+                    has_no_uncommitted = not pre_commit_git.get("files_changed")
+                    has_tokens = (
+                        int(result.metrics.get("token_input", 0) or 0)
+                        + int(result.metrics.get("token_output", 0) or 0)
+                    ) > 0
+                    has_meaningful_output = self.process_manager.has_meaningful_agent_output(result)
+                    # Timeout recovery requires stronger evidence: committed work + tokens.
+                    # Crash recovery (original): committed + (tokens OR meaningful output).
+                    sufficient_evidence = (
+                        (has_committed_changes and has_no_uncommitted and has_tokens and has_meaningful_output)
+                        if is_timeout_failure
+                        else (has_committed_changes and has_no_uncommitted and (has_tokens or has_meaningful_output))
                     )
-                    result.status = "success"
-                    result.error = ""
-                    result.metrics["recovered_from_exit_code"] = result.exit_code
+                    if sufficient_evidence:
+                        _reason = "timed out but already committed changes" if is_timeout_failure else f"exited with code {result.exit_code}"
+                        logger.warning(
+                            "Task %s agent %s — recovering as success",
+                            task.task_id, _reason,
+                        )
+                        result.status = "success"
+                        result.error = ""
+                        result.metrics["recovered_from_exit_code"] = result.exit_code
 
             # 4.5 Layer 2: Validation gate — check outputs before committing
             if result.status == "success":
@@ -5081,7 +5381,14 @@ class RuntimeDaemon:
                 ],
             )
 
-            # Build a targeted fix prompt with output directory context
+            # Save the original prompt BEFORE building the retry variant so we
+            # can include it in fix_prompt.  Without this the agent receives only
+            # "fix validation errors" with zero task context and responds with
+            # "I don't have a specific task to execute yet." (root cause confirmed
+            # via Copilot JSONL output for SI-434/SI-446).
+            original_prompt = task.input_prompt
+
+            # Build a targeted fix prompt: original task + validation issues.
             _input = task.input_data or {}
             _fix_doc_dir = (
                 _input.get("output_dir")
@@ -5089,8 +5396,11 @@ class RuntimeDaemon:
                 or ""
             )
             fix_prompt = (
-                "The previous execution produced output with validation errors.\n"
-                "Please fix ALL of the following issues:\n\n"
+                f"{original_prompt}\n\n"
+                "---\n\n"
+                "**IMPORTANT – Validation Retry:** The previous execution attempt "
+                "did not produce all required output. Please complete the task above "
+                "and ensure ALL of the following issues are resolved:\n\n"
                 f"{issues_text}\n\n"
             )
             if _fix_doc_dir:
@@ -5105,7 +5415,6 @@ class RuntimeDaemon:
             )
 
             # Override task prompt temporarily
-            original_prompt = task.input_prompt
             task.input_prompt = fix_prompt
 
             try:

{forgexa_cli-1.8.7 → forgexa_cli-1.8.10}/forgexa_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.8.7
+Version: 1.8.10
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License: MIT

{forgexa_cli-1.8.7 → forgexa_cli-1.8.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "forgexa-cli"
-version = "1.8.7"
+version = "1.8.10"
 description = "Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform"
 requires-python = ">=3.9"
 license = { text = "MIT" }