forgexa-cli 1.58.1__tar.gz → 1.58.3__tar.gz

{forgexa_cli-1.58.1 → forgexa_cli-1.58.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.58.1
+Version: 1.58.3
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License-Expression: MIT

{forgexa_cli-1.58.1 → forgexa_cli-1.58.3}/forgexa_cli/__init__.py

@@ -1,2 +1,2 @@
 """forgexa-cli — Forgexa command-line client."""
-__version__ = "1.58.1"
+__version__ = "1.58.3"

{forgexa_cli-1.58.1 → forgexa_cli-1.58.3}/forgexa_cli/daemon.py

@@ -1058,6 +1058,32 @@ except (ImportError, ModuleNotFoundError):
         def GIT_CLONE_TIMEOUT(self) -> int:
             return int(os.environ.get("GIT_CLONE_TIMEOUT", "1800"))
 
+        @property
+        def GIT_WORKTREE_TIMEOUT(self) -> int:
+            """Timeout (seconds) for `git worktree add` operations.
+
+            `worktree add` materialises a full checkout of the target branch
+            into the new worktree.  On large repos (e.g. HP SI) or busy disks
+            this easily exceeds the 120 s default for ordinary git commands
+            (SI-1181: a fresh-start worktree add timed out after 120 s and
+            failed the node even though the checkout was still progressing).
+
+            Precedence:
+              1. GIT_WORKTREE_TIMEOUT env var (explicit override)
+              2. GIT_CLONE_TIMEOUT env var (backwards compat — an operator
+                 who raised the clone timeout for a huge repo wants local
+                 checkouts to get comparable headroom)
+              3. 600 s default (10 min — local disk I/O only, no network,
+                 so shorter than the clone/fetch/push ceiling)
+            """
+            env_worktree = os.environ.get("GIT_WORKTREE_TIMEOUT")
+            if env_worktree:
+                return int(env_worktree)
+            env_clone = os.environ.get("GIT_CLONE_TIMEOUT")
+            if env_clone:
+                return int(env_clone)
+            return 600
+
         @property
         def GIT_FETCH_TIMEOUT(self) -> int:
             """Timeout (seconds) for incremental git fetch operations.
@@ -1096,6 +1122,26 @@ except (ImportError, ModuleNotFoundError):
                 return int(env_clone)
             return 1800
 
+        @property
+        def GIT_INDEX_LOCK_WAIT(self) -> int:
+            """Seconds to wait for a concurrently-held git index.lock (SI-1509).
+
+            A live git process (huge add/status on a slow disk) legitimately
+            holds the lock for minutes; failing the node sync after seconds of
+            retries just burns a requeue cycle on the same contention.
+            """
+            return int(os.environ.get("GIT_INDEX_LOCK_WAIT", "300"))
+
+        @property
+        def GIT_STALE_LOCK_AGE(self) -> int:
+            """Age (seconds) after which an index.lock is orphaned (SI-1509).
+
+            A lock older than this with the sync still failing is left behind
+            by a SIGKILL'd git process (daemon timeout kills the whole process
+            group) and is removed, as git's own error message recommends.
+            """
+            return int(os.environ.get("GIT_STALE_LOCK_AGE", "180"))
+
         @property
         def GIT_CLONE_FILTER(self) -> str:
             """Optional partial-clone filter, e.g. 'blob:none' for blobless clones.
@@ -1150,7 +1196,7 @@ except (ImportError, ModuleNotFoundError):
 # DAEMON_VERSION is the protocol/logic version of the daemon code.
 # Kept in sync with pyproject.toml version via bump-version.sh.
 # CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
-DAEMON_VERSION = "1.58.1"
+DAEMON_VERSION = "1.58.3"
 
 
 def _detect_client_type() -> str:
@@ -3419,7 +3465,10 @@ class WorkspaceManager:
                 else:
                     try:
                         # Ensure we're on the correct branch
-                        await self._git("checkout", branch_name, cwd=ws_path)
+                        await self._git_index_op(
+                            "checkout", branch_name, cwd=ws_path,
+                            context=f"final sync checkout {branch_name} in {ws_path}",
+                        )
                     except RuntimeError as exc:
                         sync_error = f"checkout branch failed: {exc}"
                     else:
@@ -3427,9 +3476,11 @@ class WorkspaceManager:
                         # branch has diverged (force-pushed by prior phase),
                         # reset --hard is used below.
                         try:
-                            await self._git(
+                            await self._git_index_op(
                                 "pull", "--ff-only", "origin", branch_name,
                                 cwd=ws_path, project_key=project_key,
+                                timeout=settings.GIT_FETCH_TIMEOUT,
+                                context=f"final sync pull --ff-only {branch_name} in {ws_path}",
                             )
                             sync_success = True
                         except RuntimeError as exc:
@@ -3439,9 +3490,10 @@ class WorkspaceManager:
                     # ff-only failed (diverged or remote not yet created) — try
                     # reset --hard to force-sync to whatever the remote has.
                     try:
-                        await self._git(
+                        await self._git_index_op(
                             "reset", "--hard", f"origin/{branch_name}",
                             cwd=ws_path,
+                            context=f"final sync reset to origin/{branch_name} in {ws_path}",
                         )
                         sync_success = True
                     except RuntimeError as exc:
@@ -3826,6 +3878,157 @@ class WorkspaceManager:
             backup_tag,
         )
 
+    # ── git index.lock contention recovery (SI-1509) ─────────────────────
+    # A git command that writes the index (checkout, reset, pull, add,
+    # commit, status-refresh) refuses to run while the worktree's
+    # index.lock exists.  Two very different situations produce that file:
+    #   1. A LIVE concurrent git process (e.g. a huge `git add`/`git status`
+    #      on a slow disk) — must be waited out; removing the lock mid-op
+    #      corrupts the index.
+    #   2. A STALE lock orphaned by a SIGKILL'd process (the daemon kills
+    #      agent process trees on timeout) — never disappears on its own
+    #      and must be removed; git's own error message says exactly this.
+    # Before SI-1509 neither case was handled: sync failed after ~6 s of
+    # blind retries with a misleading "fetch timed out or credentials
+    # missing/invalid" message, and the node was requeued to repeat the
+    # same failure (the real checkout error text was dropped).
+
+    _INDEX_LOCK_RE = re.compile(r"'([^']*index\.lock)'")
+
+    @classmethod
+    def _extract_index_lock_path(cls, err_text: str) -> Path | None:
+        """Parse the index.lock path git reports when it refuses to proceed.
+
+        Git prints the lock path in single quotes in every locale, e.g.
+          fatal: Unable to create '/w/…/_main/.git/worktrees/<id>/index.lock': File exists.
+          fatal: 无法创建 '/w/…/index.lock'：文件已存在。
+        """
+        m = cls._INDEX_LOCK_RE.search(err_text)
+        if not m:
+            return None
+        raw = m.group(1)
+        # On Windows git may print an MSYS-style path (/c/…); leave those
+        # alone — only recover locks whose path we can act on directly.
+        p = Path(raw)
+        return p if p.is_absolute() else None
+
+    @staticmethod
+    def _linked_worktree_index_lock_path(
+        ws_path: Path, main_repo: Path | None,
+    ) -> Path | None:
+        """Return the index.lock path of a structurally-valid linked worktree.
+
+        Unlike `_is_healthy_worktree` this isolates the *lock* condition:
+        if this returns a path, the worktree is fine apart from being
+        locked, so lock recovery alone can restore it without the
+        destructive remove-and-recreate path.
+        """
+        git_file = ws_path / ".git"
+        if not git_file.exists() or git_file.is_dir():
+            return None
+        try:
+            content = git_file.read_text().strip()
+        except OSError:
+            return None
+        if not content.startswith("gitdir: "):
+            return None
+        gitdir = Path(content.split("gitdir: ", 1)[1].strip())
+        if main_repo is not None:
+            try:
+                gitdir.resolve().relative_to((main_repo / ".git" / "worktrees").resolve())
+            except (ValueError, OSError):
+                return None
+        lock = gitdir / "index.lock"
+        return lock if lock.exists() else None
+
+    async def _wait_or_remove_index_lock(
+        self, lock_path: Path, *, max_wait: float, stale_age: float,
+        context: str, poll_interval: float = 5.0,
+    ) -> bool:
+        """Wait out a live index.lock holder; remove the lock once stale.
+
+        Policy:
+          • Poll every 5 s; the holder finishing removes the lock itself.
+          • If the lock's mtime age exceeds ``stale_age`` seconds, the
+            holder is almost certainly gone (SIGKILL leaves exactly this
+            state) — unlink it as git's error message recommends.
+          • Give up after ``max_wait`` seconds and return False so callers
+            can fall back to their existing failure handling.
+        """
+        deadline = time.monotonic() + max_wait
+        last_age = -1.0
+        while True:
+            if not lock_path.exists():
+                if last_age >= 0:
+                    logger.info(
+                        "%s: index.lock %s disappeared — holder finished; retrying",
+                        context, lock_path,
+                    )
+                return True
+            try:
+                age = time.time() - lock_path.stat().st_mtime
+            except FileNotFoundError:
+                continue  # lock vanished between exists() and stat()
+            except OSError:
+                age = 0.0  # unreadable mtime (permissions) — treat as fresh
+            if age > stale_age:
+                try:
+                    lock_path.unlink()
+                except FileNotFoundError:
+                    return True
+                except OSError as rm_exc:
+                    logger.warning(
+                        "%s: could not remove stale index.lock %s (%s) — retrying",
+                        context, lock_path, rm_exc,
+                    )
+                else:
+                    logger.warning(
+                        "%s: removed stale index.lock %s (age %.0fs > %.0fs; "
+                        "left behind by an interrupted git process)",
+                        context, lock_path, age, stale_age,
+                    )
+                    return True
+            if time.monotonic() >= deadline:
+                logger.warning(
+                    "%s: index.lock %s still held after %.0fs — giving up",
+                    context, lock_path, max_wait,
+                )
+                return False
+            if last_age < 0:
+                logger.info(
+                    "%s: waiting up to %.0fs for concurrent git process holding %s",
+                    context, max_wait, lock_path,
+                )
+            last_age = age
+            await asyncio.sleep(poll_interval)
+
+    async def _git_index_op(
+        self, *args: str, cwd: Path | str, context: str,
+        project_key: str | None = None, timeout: int = 120,
+    ) -> str:
+        """Run a git command that writes the worktree index, with lock recovery.
+
+        On index.lock contention (live or stale) this waits out or clears
+        the lock and retries once instead of failing the whole node sync —
+        see the SI-1509 note above the helpers.
+        """
+        try:
+            return await self._git(*args, cwd=cwd, timeout=timeout, project_key=project_key)
+        except RuntimeError as exc:
+            lock_path = self._extract_index_lock_path(str(exc))
+            if lock_path is None:
+                raise
+            recovered = await self._wait_or_remove_index_lock(
+                lock_path,
+                max_wait=settings.GIT_INDEX_LOCK_WAIT,
+                stale_age=settings.GIT_STALE_LOCK_AGE,
+                context=context,
+            )
+            if not recovered:
+                raise
+            logger.info("%s: index.lock cleared — retrying git %s", context, args[0])
+            return await self._git(*args, cwd=cwd, timeout=timeout, project_key=project_key)
+
     async def _is_healthy_worktree(
         self,
         ws_path: Path,
@@ -4426,11 +4629,35 @@ class WorkspaceManager:
             # valid main repo and has no stale Git index lock. Worktrees break
             # when directories are moved or an interrupted Git process leaves
             # its index locked.
-            if not await self._is_healthy_worktree(
+            _worktree_healthy = await self._is_healthy_worktree(
                 ws_path,
                 main_repo=main_repo,
                 require_unlocked=True,
-            ):
+            )
+            if not _worktree_healthy:
+                # SI-1509: when the only problem is an index.lock (live
+                # concurrent git process, or a stale one left by a SIGKILL'd
+                # process), recovering just the lock keeps the worktree and
+                # any uncommitted state intact — no need to nuke and re-clone.
+                _lock_path = self._linked_worktree_index_lock_path(ws_path, main_repo)
+                if _lock_path is not None:
+                    logger.warning(
+                        "Worktree %s is locked by %s — attempting lock recovery before "
+                        "falling back to worktree recreation",
+                        ws_path, _lock_path,
+                    )
+                    if await self._wait_or_remove_index_lock(
+                        _lock_path,
+                        max_wait=settings.GIT_INDEX_LOCK_WAIT,
+                        stale_age=settings.GIT_STALE_LOCK_AGE,
+                        context=f"worktree health check for {ws_path}",
+                    ):
+                        _worktree_healthy = await self._is_healthy_worktree(
+                            ws_path,
+                            main_repo=main_repo,
+                            require_unlocked=True,
+                        )
+            if not _worktree_healthy:
                 logger.warning("Broken worktree detected at %s — removing and recreating", ws_path)
                 await self._remove_broken_worktree(main_repo, ws_path, workspace_key)
             else:
@@ -4660,15 +4887,25 @@ class WorkspaceManager:
                     sync_success = False
                     for _sync_attempt in range(3):
                         try:
-                            await self._git("checkout", branch_name, cwd=ws_path)
+                            await self._git_index_op(
+                                "checkout", branch_name, cwd=ws_path,
+                                context=f"sync checkout {branch_name} in {ws_path}",
+                            )
                         except RuntimeError:
                             # Branch might not exist locally yet — create tracking branch
                             try:
-                                await self._git(
+                                await self._git_index_op(
                                     "checkout", "-B", branch_name, f"origin/{branch_name}",
                                     cwd=ws_path,
+                                    context=f"sync checkout -B {branch_name} in {ws_path}",
                                 )
                             except RuntimeError as exc:
+                                # SI-1509: record the REAL checkout error (lock
+                                # contention, bad ref, …) — without this the
+                                # raised message fell back to a misleading
+                                # "fetch timed out or credentials missing/invalid"
+                                # even though every fetch had succeeded.
+                                _last_sync_err = str(exc)[:300]
                                 if _sync_attempt < 2:
                                     logger.info(
                                         "Branch %s not available yet (attempt %d/3), re-fetching...",
@@ -4693,13 +4930,15 @@ class WorkspaceManager:
                                     logger.warning("Failed to checkout %s after retries: %s", branch_name, exc)
                         # Reset working tree to match remote branch (fast-forward)
                         try:
-                            await self._git(
+                            await self._git_index_op(
                                 "reset", "--hard", f"origin/{branch_name}",
                                 cwd=ws_path,
+                                context=f"sync reset to origin/{branch_name} in {ws_path}",
                             )
                             sync_success = True
                             break
                         except RuntimeError as exc:
+                            _last_sync_err = str(exc)[:300]
                             if _sync_attempt < 2:
                                 logger.info(
                                     "Could not reset to origin/%s (attempt %d/3), re-fetching...",
@@ -4761,7 +5000,9 @@ class WorkspaceManager:
                                 logger.warning("Could not remove stale worktree %s: %s", ws_path, _rm_exc)
                             _err_detail = (
                                 f"Git error: {_last_sync_err}" if _last_sync_err
-                                else "fetch timed out or credentials missing/invalid"
+                                else "checkout/reset failed repeatedly with no captured "
+                                     "git error (possible stale index.lock or interrupted "
+                                     "git process on the daemon host)"
                             )
                             raise RuntimeError(
                                 f"Failed to sync branch '{branch_name}' from remote after 3 attempts "
@@ -5221,7 +5462,8 @@ class WorkspaceManager:
                         ) from recovery_exc
                 self._safe_rmtree(ws_path)
                 raise RuntimeError(
-                    f"Failed to create clean fresh worktree for branch '{branch_name}'. "
+                    f"Failed to create clean fresh worktree for branch '{branch_name}': "
+                    f"{self._summarize_git_error(exc)}. "
                     "Refusing to reuse a stale local branch."
                 ) from exc
         else:
@@ -5257,13 +5499,18 @@ class WorkspaceManager:
         """Optional cleanup after task. For now, keep workspaces for debugging."""
         pass
 
-    async def _git(self, *args: str, cwd: Path | str | None = None, timeout: int = 120,
+    async def _git(self, *args: str, cwd: Path | str | None = None, timeout: int | None = None,
                    project_key: str | None = None) -> str:
         """Run a git command safely using exec (no shell injection).
 
         If a project SSH key is registered, remote-touching commands
         (clone, fetch, pull, push) will use GIT_SSH_COMMAND.
 
+        Timeout resolution (SI-1181): when ``timeout`` is None the default
+        is 120 s for ordinary git commands, but `worktree add` — which
+        materialises a full checkout of the branch — gets the dedicated
+        GIT_WORKTREE_TIMEOUT (default 600 s) instead.
+
         Process-group isolation: git is started in its own session so that
         SIGKILL on timeout propagates to all of git's children (especially
         the ssh subprocess that git forks for remote operations).  Without
@@ -5276,6 +5523,15 @@ class WorkspaceManager:
         accepts but never sends the git protocol banner) is detected within
         ~90 s rather than waiting forever.
         """
+        if timeout is None:
+            # `git worktree add` is a full checkout of the target branch;
+            # on large repos (e.g. HP SI) or busy disks it legitimately
+            # exceeds the 120 s default for ordinary git commands.
+            timeout = (
+                settings.GIT_WORKTREE_TIMEOUT
+                if args[:2] == ("worktree", "add")
+                else 120
+            )
         env = None
         git_prefix_args: list[str] = []
         # Inject SSH key for commands that touch the remote

{forgexa_cli-1.58.1 → forgexa_cli-1.58.3}/forgexa_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.58.1
+Version: 1.58.3
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License-Expression: MIT

{forgexa_cli-1.58.1 → forgexa_cli-1.58.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "forgexa-cli"
-version = "1.58.1"
+version = "1.58.3"
 description = "Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform"
 requires-python = ">=3.9"
 license = "MIT"