forgexa-cli 1.12.2__tar.gz → 1.13.2__tar.gz

{forgexa_cli-1.12.2 → forgexa_cli-1.13.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.12.2
+Version: 1.13.2
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License: MIT
@@ -25,6 +25,7 @@ Classifier: Topic :: Software Development :: Build Tools
 Classifier: Topic :: Software Development :: Quality Assurance
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.24
 Provides-Extra: daemon
 Requires-Dist: httpx>=0.24; extra == "daemon"
 
@@ -32,7 +33,7 @@ Requires-Dist: httpx>=0.24; extra == "daemon"
 
 Command-line client and agent runtime for the [Forgexa](https://forgexa.net) platform.
 
-Lightweight — communicates with the Forgexa server via REST API using only Python stdlib.
+Communicates with the Forgexa server via REST API.
 Includes a built-in **daemon** that discovers local AI agents (Claude Code, Codex, Gemini CLI, OpenCode, Kimi Code, etc.) and executes tasks on behalf of the server.
 
 ## Installation
@@ -44,6 +45,10 @@ pip install forgexa-cli
 # Or with pipx (isolated environment)
 pipx install forgexa-cli
 
+# Upgrade later
+forgexa upgrade
+forgexa upgrade --target-version 1.12.3
+
 # Verify installation
 forgexa version
 ```
@@ -104,6 +109,27 @@ forgexa board --project <project-id>
 | `forgexa daemon stop` | Stop local daemon |
 | `forgexa runtimes list` | List your runtimes |
 | `forgexa version` | Show CLI version |
+| `forgexa upgrade` | Upgrade to the latest CLI release |
+| `forgexa upgrade --target-version <version>` | Upgrade or roll forward to a specific release |
+
+## Self Upgrade
+
+```bash
+# Upgrade to the latest published release
+forgexa upgrade
+
+# Upgrade to a specific published release
+forgexa upgrade --target-version 1.12.3
+```
+
+The command detects how `forgexa-cli` was installed:
+
+- `pipx` install: uses `pipx upgrade forgexa-cli`, or `pipx install --force forgexa-cli==<version>` for a pinned version
+- `pip` install: uses `python -m pip install --upgrade forgexa-cli`
+
+For safety, `forgexa upgrade` stops any running local daemon before upgrading. Restart it afterward with `forgexa daemon start`.
+
+`forgexa upgrade` is intended for standard PyPI / pipx installs. Editable installs, local path installs, and VCS installs are rejected with a manual upgrade hint.
 
 ## Configuration
 

{forgexa_cli-1.12.2 → forgexa_cli-1.13.2}/README.md

@@ -2,7 +2,7 @@
 
 Command-line client and agent runtime for the [Forgexa](https://forgexa.net) platform.
 
-Lightweight — communicates with the Forgexa server via REST API using only Python stdlib.
+Communicates with the Forgexa server via REST API.
 Includes a built-in **daemon** that discovers local AI agents (Claude Code, Codex, Gemini CLI, OpenCode, Kimi Code, etc.) and executes tasks on behalf of the server.
 
 ## Installation
@@ -14,6 +14,10 @@ pip install forgexa-cli
 # Or with pipx (isolated environment)
 pipx install forgexa-cli
 
+# Upgrade later
+forgexa upgrade
+forgexa upgrade --target-version 1.12.3
+
 # Verify installation
 forgexa version
 ```
@@ -74,6 +78,27 @@ forgexa board --project <project-id>
 | `forgexa daemon stop` | Stop local daemon |
 | `forgexa runtimes list` | List your runtimes |
 | `forgexa version` | Show CLI version |
+| `forgexa upgrade` | Upgrade to the latest CLI release |
+| `forgexa upgrade --target-version <version>` | Upgrade or roll forward to a specific release |
+
+## Self Upgrade
+
+```bash
+# Upgrade to the latest published release
+forgexa upgrade
+
+# Upgrade to a specific published release
+forgexa upgrade --target-version 1.12.3
+```
+
+The command detects how `forgexa-cli` was installed:
+
+- `pipx` install: uses `pipx upgrade forgexa-cli`, or `pipx install --force forgexa-cli==<version>` for a pinned version
+- `pip` install: uses `python -m pip install --upgrade forgexa-cli`
+
+For safety, `forgexa upgrade` stops any running local daemon before upgrading. Restart it afterward with `forgexa daemon start`.
+
+`forgexa upgrade` is intended for standard PyPI / pipx installs. Editable installs, local path installs, and VCS installs are rejected with a manual upgrade hint.
 
 ## Configuration
 

{forgexa_cli-1.12.2 → forgexa_cli-1.13.2}/forgexa_cli/__init__.py

@@ -1,2 +1,2 @@
 """forgexa-cli — Forgexa command-line client."""
-__version__ = "1.12.2"
+__version__ = "1.13.2"

{forgexa_cli-1.12.2 → forgexa_cli-1.13.2}/forgexa_cli/daemon.py

@@ -104,6 +104,70 @@ def _save_cli_tokens(access_token: str, refresh_token: str | None = None) -> Non
     token_path.chmod(0o600)
 
 
+def _looks_like_pipx_environment(python_path: str | None = None) -> bool:
+    candidate = (python_path or sys.executable or "").replace("\\", "/").lower()
+    return "/pipx/venvs/" in candidate
+
+
+def _httpx_install_commands(python: str, deps_dir: str) -> list[tuple[str, list[str]]]:
+    # Use one isolated install path across all OSes so the daemon avoids
+    # mutating system or user site-packages.
+    return [
+        (
+            "pip install --target (isolated deps)",
+            [
+                python,
+                "-m",
+                "pip",
+                "install",
+                "--target",
+                deps_dir,
+                "--quiet",
+                "--upgrade",
+                "httpx>=0.24",
+                "httpcore",
+                "certifi",
+            ],
+        )
+    ]
+
+
+def _httpx_recovery_hints(os_name: str, python_path: str) -> list[str]:
+    if _looks_like_pipx_environment(python_path):
+        return [
+            "pipx upgrade forgexa-cli          # preferred repair for pipx installs",
+            "pipx inject forgexa-cli httpx httpcore certifi",
+            "pipx reinstall forgexa-cli",
+        ]
+
+    if os_name == "Linux":
+        return [
+            "python3 -m pip install --upgrade forgexa-cli",
+            "pip3 install --upgrade forgexa-cli",
+            "pipx install forgexa-cli          # recommended clean install",
+            "pip3 install --user httpx httpcore certifi",
+        ]
+    if os_name == "Darwin":
+        return [
+            "python3 -m pip install --upgrade forgexa-cli",
+            "pip3 install --upgrade forgexa-cli",
+            "pipx install forgexa-cli          # recommended clean install",
+            "pip3 install httpx httpcore certifi",
+        ]
+    if os_name == "Windows":
+        return [
+            "py -m pip install --upgrade forgexa-cli",
+            "pip install --upgrade forgexa-cli",
+            "pipx install forgexa-cli          # recommended clean install",
+            "py -m pip install httpx httpcore certifi",
+        ]
+    return [
+        "python -m pip install --upgrade forgexa-cli",
+        "pipx install forgexa-cli",
+        "python -m pip install httpx httpcore certifi",
+    ]
+
+
 def _try_install_httpx(deps_dir: str) -> tuple[bool, str]:
     """Try to install httpx to a user-writable directory.
 
@@ -113,37 +177,29 @@ def _try_install_httpx(deps_dir: str) -> tuple[bool, str]:
     - macOS .app bundles (sandboxed Python)
     - Windows portable installs
     - Docker containers with read-only system dirs
+    - pipx isolated venvs (bootstrap pip via ensurepip first)
 
     Returns (success, error_detail).
     """
     os.makedirs(deps_dir, exist_ok=True)
     python = sys.executable or "python3"
 
-    # Try pip --target first (most universally compatible).
-    # Falls back to --user, then --break-system-packages as last resort.
-    # We explicitly list httpcore and certifi alongside httpx because pip
-    # --target may skip transitive deps it finds in system site-packages,
-    # even though they won't be importable from the isolated deps directory.
-    # certifi must be included so that its cacert.pem bundle is copied into
-    # the deps dir; without it httpx raises FileNotFoundError when building
-    # the default SSL context (observed on Python 3.14 standalone / Windows).
-    strategies: list[tuple[str, list[str]]] = [
-        (
-            "pip install --target (isolated deps)",
-            [python, "-m", "pip", "install", "--target", deps_dir,
-             "--quiet", "--upgrade", "httpx>=0.24", "httpcore", "certifi"],
-        ),
-        (
-            "pip install --user",
-            [python, "-m", "pip", "install", "--user", "--quiet",
-             "httpx>=0.24", "httpcore", "certifi"],
-        ),
-        (
-            "pip install --break-system-packages",
-            [python, "-m", "pip", "install", "--quiet",
-             "--break-system-packages", "httpx>=0.24", "httpcore", "certifi"],
-        ),
-    ]
+    # Strategy 0: Bootstrap pip via ensurepip (stdlib).
+    # pipx venvs don't include pip by design, so `python -m pip` fails with
+    # "No module named pip". ensurepip.bootstrap() installs pip into the
+    # current environment without any external tools.
+    try:
+        import ensurepip as _ensurepip
+        _ensurepip.bootstrap(upgrade=True)
+    except Exception:
+        pass  # non-fatal; pip strategies below will detect availability
+
+    # Use a private deps directory on every OS. This avoids distro-specific
+    # package manager behavior, user-site visibility issues inside virtualenvs,
+    # and system-package mutations. We explicitly install httpcore and certifi
+    # because pip may otherwise reuse system copies that are not importable from
+    # the isolated target directory.
+    strategies = _httpx_install_commands(python, deps_dir)
 
     last_error = ""
     for label, cmd in strategies:
@@ -177,30 +233,7 @@ def _die_missing_httpx(detail: str) -> None:
     """Print a clear, actionable error and exit when httpx cannot be loaded."""
     os_name = platform.system()
     python_path = sys.executable or "(unknown)"
-
-    if os_name == "Linux":
-        hints = [
-            "pip3 install --user httpx",
-            "sudo apt install python3-httpx          # Debian/Ubuntu",
-            "sudo dnf install python3-httpx          # Fedora/RHEL",
-            "pip3 install forgexa-cli[daemon]",
-        ]
-    elif os_name == "Darwin":
-        hints = [
-            "pip3 install httpx",
-            "brew install python3 && pip3 install httpx",
-            "pip3 install forgexa-cli[daemon]",
-        ]
-    elif os_name == "Windows":
-        hints = [
-            "pip install httpx",
-            "pip install forgexa-cli[daemon]",
-        ]
-    else:
-        hints = [
-            "pip3 install httpx",
-            "pip3 install forgexa-cli[daemon]",
-        ]
+    hints = _httpx_recovery_hints(os_name, python_path)
 
     hint_lines = "\n".join(f"    {h}" for h in hints)
     msg = (
@@ -216,7 +249,7 @@ def _die_missing_httpx(detail: str) -> None:
         f"  Platform:  {os_name} ({platform.machine()})\n"
         f"  Detail:    {detail}\n"
         "\n"
-        "  Please install it manually with one of these commands:\n"
+        "  Please repair the installation with one of these commands:\n"
         "\n"
         f"{hint_lines}\n"
         "\n"
@@ -392,7 +425,7 @@ except (ImportError, ModuleNotFoundError):
 
         @property
         def AGENT_IDLE_TIMEOUT(self) -> int:
-            return int(os.environ.get("AGENT_IDLE_TIMEOUT", "600"))  # 10-min idle (stdout+fs) = hung agent
+            return int(os.environ.get("AGENT_IDLE_TIMEOUT", "1200"))  # 20-min idle (stdout+fs) = hung agent — matches backend config.py default
 
         @property
         def GIT_CLONE_TIMEOUT(self) -> int:
@@ -420,6 +453,22 @@ except (ImportError, ModuleNotFoundError):
                 return int(env_clone)
             return 1800
 
+        @property
+        def GIT_PUSH_TIMEOUT(self) -> int:
+            """Timeout (seconds) for git push operations.
+
+            Standalone runtimes use the same precedence model as fetch: allow an
+            explicit push timeout, otherwise inherit the clone timeout override,
+            then fall back to the 30-minute safe default used by the backend.
+            """
+            env_push = os.environ.get("GIT_PUSH_TIMEOUT")
+            if env_push:
+                return int(env_push)
+            env_clone = os.environ.get("GIT_CLONE_TIMEOUT")
+            if env_clone:
+                return int(env_clone)
+            return 1800
+
         @property
         def GIT_CLONE_FILTER(self) -> str:
             """Optional partial-clone filter, e.g. 'blob:none' for blobless clones.
@@ -474,7 +523,7 @@ except (ImportError, ModuleNotFoundError):
 # DAEMON_VERSION is the protocol/logic version of the daemon code.
 # Kept in sync with pyproject.toml version via bump-version.sh.
 # CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
-DAEMON_VERSION = "1.12.2"
+DAEMON_VERSION = "1.13.2"
 
 
 def _detect_client_type() -> str:
@@ -1315,6 +1364,70 @@ class WorkspaceManager:
             args.extend(["--branch", branch])
         return tuple(args)
 
+    @staticmethod
+    def _is_retryable_git_transport_error(exc: Exception | str) -> bool:
+        err = str(exc).lower()
+        retry_markers = (
+            "timeout",
+            "timed out",
+            "server not responding",
+            "unexpected disconnect",
+            "early eof",
+            "invalid index-pack output",
+            "fetch-pack",
+            "remote hung up unexpectedly",
+            "connection reset",
+            "connection closed",
+            "connection refused",
+            "network is unreachable",
+        )
+        return any(marker in err for marker in retry_markers)
+
+    async def _clone_repo(
+        self,
+        repo_url: str,
+        dest: Path,
+        *,
+        branch: str | None = None,
+        project_key: str = "default",
+        max_attempts: int = 3,
+    ) -> None:
+        import shutil
+
+        last_exc: RuntimeError | None = None
+        for attempt in range(1, max_attempts + 1):
+            try:
+                await self._git(
+                    *self._clone_args(branch=branch),
+                    repo_url,
+                    str(dest),
+                    timeout=settings.GIT_CLONE_TIMEOUT,
+                    project_key=project_key,
+                )
+                return
+            except RuntimeError as exc:
+                last_exc = exc
+                if attempt >= max_attempts or not self._is_retryable_git_transport_error(exc):
+                    # Clean up any partial directory git may have created before giving up.
+                    # If not cleaned up now, _is_healthy_worktree will detect it as broken
+                    # on the next call and remove it, but cleaning here gives a faster path.
+                    shutil.rmtree(dest, ignore_errors=True)
+                    raise
+
+                logger.warning(
+                    "Clone attempt %d/%d failed for %s -> %s: %s — retrying",
+                    attempt,
+                    max_attempts,
+                    repo_url,
+                    dest,
+                    exc,
+                )
+                shutil.rmtree(dest, ignore_errors=True)
+                await asyncio.sleep(attempt * 5)
+
+        if last_exc is not None:
+            raise last_exc
+
     async def prepare_workspace(self, project: dict, task: TaskInfo) -> Path:
         """Create or reuse a workspace for the given task.
 
@@ -2059,9 +2172,11 @@ class WorkspaceManager:
 
         # Ensure _main repo is present and up-to-date
         if not main_repo.exists():
-            await self._git(
-                *self._clone_args(branch=default_branch),
-                repo_url, str(main_repo), timeout=settings.GIT_CLONE_TIMEOUT, project_key=project_key,
+            await self._clone_repo(
+                repo_url,
+                main_repo,
+                branch=default_branch,
+                project_key=project_key,
             )
         else:
             # Before fetching, verify _main has a valid HEAD and at least one ref.
@@ -2089,9 +2204,11 @@ class WorkspaceManager:
                 )
                 await self._safe_rmtree_main(main_repo)
                 try:
-                    await self._git(
-                        *self._clone_args(branch=default_branch),
-                        repo_url, str(main_repo), timeout=settings.GIT_CLONE_TIMEOUT, project_key=project_key,
+                    await self._clone_repo(
+                        repo_url,
+                        main_repo,
+                        branch=default_branch,
+                        project_key=project_key,
                     )
                 except Exception:
                     shutil.rmtree(main_repo, ignore_errors=True)
@@ -2121,9 +2238,10 @@ class WorkspaceManager:
                         )
                         await self._safe_rmtree_main(main_repo)
                         try:
-                            await self._git(
-                                *self._clone_args(branch=default_branch),
-                                repo_url, str(main_repo), timeout=settings.GIT_CLONE_TIMEOUT,
+                            await self._clone_repo(
+                                repo_url,
+                                main_repo,
+                                branch=default_branch,
                                 project_key=project_key,
                             )
                         except Exception:
@@ -2233,9 +2351,10 @@ class WorkspaceManager:
                         )
                     except Exception:
                         ws_path.mkdir(parents=True, exist_ok=True)
-                        await self._git(
-                            *self._clone_args(),
-                            repo_url, str(ws_path), timeout=settings.GIT_CLONE_TIMEOUT, project_key=project_key,
+                        await self._clone_repo(
+                            repo_url,
+                            ws_path,
+                            project_key=project_key,
                         )
                         # Ensure we're on the correct branch after clone
                         try:
@@ -2258,9 +2377,10 @@ class WorkspaceManager:
                 except Exception:
                     # Fallback to simple clone
                     ws_path.mkdir(parents=True, exist_ok=True)
-                    await self._git(
-                        *self._clone_args(),
-                        repo_url, str(ws_path), timeout=settings.GIT_CLONE_TIMEOUT, project_key=project_key,
+                    await self._clone_repo(
+                        repo_url,
+                        ws_path,
+                        project_key=project_key,
                     )
                     # Ensure we're on the correct branch after clone
                     try:
@@ -2672,8 +2792,11 @@ class ProcessManager:
                 msg_data = data.get("data") or {}
                 if isinstance(msg_data.get("content"), str) and msg_data["content"].strip():
                     has_result = True
+            elif ev_type == "assistant.message_start":
+                has_assistant_events = True
             elif ev_type == "assistant.message_delta":
                 has_meaningful_content = True
+                has_assistant_events = True
             # ── Generic / Claude "result" event ────────────────────────────
             elif ev_type == "result":
                 # Copilot result format: {"type":"result","exitCode":0,"usage":{...}}
@@ -2796,6 +2919,19 @@ class ProcessManager:
             or signals["has_meaningful_content"]
         )
 
+    @staticmethod
+    def _copilot_completed_without_result(stdout: str) -> bool:
+        """Detect Copilot runs that completed a turn but omitted the final result event."""
+        signals = ProcessManager._extract_output_signals(stdout)
+        return bool(
+            signals["has_turn_completed"]
+            and signals["has_assistant_events"]
+            and signals["has_meaningful_content"]
+            and not signals["has_result"]
+            and not signals["has_turn_failed"]
+            and not signals["error_messages"]
+        )
+
     @staticmethod
     def is_rate_limited(result: "TaskResult") -> bool:
         """Check if an agent failure warrants trying a different agent.
@@ -3810,6 +3946,7 @@ class ProcessManager:
             # Copilot always exits 0 on normal completion; check result.exitCode
             # from the JSONL "result" event for a true success signal.
             copilot_exit = self._extract_copilot_exit_code(stdout)
+            completed_without_result = self._copilot_completed_without_result(stdout)
             effective_rc = copilot_exit if copilot_exit is not None else returncode
 
             if effective_rc == 0 and returncode == 0:
@@ -3820,6 +3957,21 @@ class ProcessManager:
                     stderr=stderr[-10000:],
                     metrics=metrics,
                 )
+            elif copilot_exit is None and returncode != 0 and completed_without_result:
+                logger.warning(
+                    "Copilot exited with return code %s for task %s despite a completed assistant turn and no result event; recovering to validation path",
+                    returncode,
+                    task_id,
+                )
+                metrics["recovered_missing_result_event"] = True
+                metrics["recovered_process_returncode"] = returncode
+                return TaskResult(
+                    status="success",
+                    exit_code=0,
+                    stdout=stdout[-settings.AGENT_MAX_OUTPUT_SIZE:],
+                    stderr=stderr[-10000:],
+                    metrics=metrics,
+                )
             else:
                 return TaskResult(
                     status="failed",
@@ -5097,6 +5249,8 @@ class RuntimeDaemon:
         self.active_tasks: dict[str, asyncio.Task] = {}
         # Track which ServerConnection owns each task (for reporting)
         self._task_connections: dict[str, ServerConnection] = {}
+        self._project_execution_locks: dict[str, asyncio.Lock] = {}
+        self._project_execution_locks_mu: asyncio.Lock = asyncio.Lock()
         self._shutdown = False
 
         self.connections: list[ServerConnection] = []
@@ -5105,6 +5259,76 @@ class RuntimeDaemon:
         self.process_manager = ProcessManager()
         self._lock_file = None  # File lock to prevent multiple daemon instances
 
+    @staticmethod
+    def _project_execution_key(project: dict | None, task_hint: str) -> str:
+        project_info = project or {}
+        for candidate in (
+            project_info.get("project_key"),
+            project_info.get("id"),
+            project_info.get("repo_url"),
+            project_info.get("name"),
+            task_hint,
+        ):
+            if candidate:
+                return str(candidate)
+        return task_hint or "default"
+
+    async def _get_project_execution_lock(self, project_key: str) -> asyncio.Lock:
+        async with self._project_execution_locks_mu:
+            if project_key not in self._project_execution_locks:
+                self._project_execution_locks[project_key] = asyncio.Lock()
+            return self._project_execution_locks[project_key]
+
+    @staticmethod
+    def _resolve_task_output_dir(task: TaskInfo) -> str:
+        input_data = task.input_data or {}
+        context = input_data.get("context") or {}
+
+        if task.node_type == "analysis":
+            raw_dir = (
+                input_data.get("analysis_output_dir")
+                or context.get("analysis_output_dir")
+                or input_data.get("output_dir")
+                or context.get("output_dir")
+                or ""
+            )
+        elif task.node_type in {"design", "coding", "testing", "review", "delivery"}:
+            raw_dir = (
+                input_data.get("output_dir")
+                or context.get("output_dir")
+                or ""
+            )
+        else:
+            raw_dir = ""
+
+        return str(raw_dir).replace("\\", "/").lstrip("./").rstrip("/")
+
+    def _ensure_task_output_dir(self, workspace_path: Path, task: TaskInfo) -> None:
+        output_dir = self._resolve_task_output_dir(task)
+        if not output_dir:
+            return
+
+        target_dir = workspace_path / output_dir
+        try:
+            if not target_dir.resolve().is_relative_to(workspace_path.resolve()):
+                logger.warning(
+                    "Task %s output dir %r resolves outside workspace %s — skipping mkdir",
+                    task.task_id,
+                    output_dir,
+                    workspace_path,
+                )
+                return
+        except Exception:
+            logger.warning(
+                "Task %s output dir %r could not be resolved safely under %s",
+                task.task_id,
+                output_dir,
+                workspace_path,
+            )
+            return
+
+        target_dir.mkdir(parents=True, exist_ok=True)
+
     @staticmethod
     async def _mint_local_dev_token() -> str:
         """Mint a JWT for local development when no explicit token is configured.
@@ -5468,6 +5692,22 @@ class RuntimeDaemon:
     async def _execute_task(self, task: TaskInfo, conn: ServerConnection):
         """Execute a single task, reporting to the originating server connection."""
         reporter = conn.reporter
+        # Use workspace-level granularity (requirement_workflow_id or graph_id) so that
+        # tasks for DIFFERENT requirements can run in parallel on the same daemon, while
+        # tasks for the SAME requirement (e.g. analysis + retry) are still serialized to
+        # prevent non-fast-forward push conflicts.  Prefixed with project_key to avoid
+        # cross-project UUID collisions.
+        _proj_key = (task.project or {}).get("project_key") or str((task.project or {}).get("id") or "default")
+        _ws_key = task.requirement_workflow_id or str(task.graph_id or task.task_id)
+        project_lock_key = f"{_proj_key}:{_ws_key}"
+        project_lock = await self._get_project_execution_lock(project_lock_key)
+        if project_lock.locked():
+            logger.info(
+                "Task %s waiting for project execution lock %s",
+                task.task_id,
+                project_lock_key,
+            )
+        await project_lock.acquire()
         try:
             # 1. Find the right agent
             agent = self._select_agent(task.agent_type, task.fallback_chain)
@@ -5499,7 +5739,13 @@ class RuntimeDaemon:
             workspace_path = await self.workspace_manager.prepare_workspace(
                 task.project, task
             )
+            if not workspace_path.exists():
+                raise RuntimeError(
+                    f"Workspace directory does not exist after prepare_workspace: {workspace_path}. "
+                    "This indicates a git clone or worktree setup failure (check SSH/network)."
+                )
             logger.info("Workspace ready: %s", workspace_path)
+            self._ensure_task_output_dir(workspace_path, task)
 
             # 2.1 Workspace health check: detect broken checkout (Windows filename-
             # too-long or other git checkout failure that leaves the working tree
@@ -6073,6 +6319,8 @@ class RuntimeDaemon:
                 status="failed", exit_code=-1, stdout="", stderr="",
                 error=str(e),
             ))
+        finally:
+            project_lock.release()
 
     def _select_fallback_agent(
         self, current_agent_id: str, fallback_chain: list[str] | None, tried: set[str]
@@ -6359,6 +6607,18 @@ class RuntimeDaemon:
         user_prompt = aj.get("user_prompt", "")
 
         reporter_url = f"{conn.server_url.rstrip('/')}/api/v1/runtimes/{conn.runtime_id}/ai-jobs/{job_id}"
+        # Use workspace-level key (project:requirement or project:job_id) so AI jobs
+        # for different requirements run in parallel while same-requirement jobs serialize.
+        _aj_proj_key = project_info.get("project_key") or str(project_info.get("id") or "default")
+        project_lock_key = f"{_aj_proj_key}:{requirement_key or job_id}"
+        project_lock = await self._get_project_execution_lock(project_lock_key)
+        if project_lock.locked():
+            logger.info(
+                "AIJob %s waiting for project execution lock %s",
+                job_id,
+                project_lock_key,
+            )
+        await project_lock.acquire()
 
         try:
             # Report progress: starting
@@ -6568,6 +6828,8 @@ class RuntimeDaemon:
                 )
             except Exception:
                 pass
+        finally:
+            project_lock.release()
 
     async def _validate_and_retry(
         self,