forgexa-cli 1.42.1__tar.gz → 1.43.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/PKG-INFO +1 -1
  2. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/__init__.py +1 -1
  3. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/agent_core.py +243 -1
  4. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/daemon.py +241 -30
  5. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/PKG-INFO +1 -1
  6. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/pyproject.toml +1 -1
  7. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/README.md +0 -0
  8. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/_build_config.py +0 -0
  9. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/_local_bind.py +0 -0
  10. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/autoupgrade.py +0 -0
  11. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/main.py +0 -0
  12. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/py.typed +0 -0
  13. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/SOURCES.txt +0 -0
  14. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/dependency_links.txt +0 -0
  15. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/entry_points.txt +0 -0
  16. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/requires.txt +0 -0
  17. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/top_level.txt +0 -0
  18. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/setup.cfg +0 -0
  19. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_auth_and_runtime_commands.py +0 -0
  20. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_autoupgrade.py +0 -0
  21. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_check_command.py +0 -0
  22. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_expiry_warnings_and_revoke.py +0 -0
  23. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_local_bind_commands.py +0 -0
  24. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_runtime_credentials.py +0 -0
  25. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_session_credentials.py +0 -0
  26. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_silent_install.py +0 -0
  27. {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_upgrade_observability.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: forgexa-cli
3
- Version: 1.42.1
3
+ Version: 1.43.3
4
4
  Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
5
5
  Author-email: Jason Sun <dev.winds@gmail.com>
6
6
  License-Expression: MIT
@@ -1,2 +1,2 @@
1
1
  """forgexa-cli — Forgexa command-line client."""
2
- __version__ = "1.42.1"
2
+ __version__ = "1.43.3"
@@ -1766,9 +1766,157 @@ def copilot_base_env(env: dict[str, str], *, copilot_home: str | Path) -> dict[s
1766
1766
  env = dict(env)
1767
1767
  env["TERM"] = "dumb" # suppress TTY-detection that suspends the process
1768
1768
  env["COPILOT_HOME"] = str(copilot_home)
1769
+ env.setdefault("TOKIO_WORKER_THREADS", "2")
1770
+ env["FORGEXA_AGENT_PROCESS"] = "1"
1769
1771
  return env
1770
1772
 
1771
1773
 
1774
+ COPILOT_TOKEN_ENV_VARS = ("COPILOT_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN")
1775
+
1776
+
1777
+ def copilot_interactive_home() -> Path:
1778
+ """Return the login home selected for interactive Copilot CLI use."""
1779
+ configured_home = os.environ.get("COPILOT_HOME", "").strip()
1780
+ if configured_home:
1781
+ return Path(configured_home).expanduser()
1782
+ return Path.home() / ".copilot"
1783
+
1784
+
1785
+ def copilot_recovery_home(destination_home: str | Path) -> Path | None:
1786
+ """Return a populated login home distinct from the failed destination."""
1787
+ destination = Path(destination_home).resolve()
1788
+ candidates = (copilot_interactive_home(), Path.home() / ".copilot")
1789
+ seen: set[Path] = set()
1790
+ for candidate in candidates:
1791
+ resolved = candidate.resolve()
1792
+ if resolved in seen or resolved == destination:
1793
+ continue
1794
+ seen.add(resolved)
1795
+ if (resolved / "config.json").is_file():
1796
+ return resolved
1797
+ return None
1798
+
1799
+
1800
+ def is_copilot_model_discovery_auth_error(*messages: str | None) -> bool:
1801
+ """Return whether Copilot failed before work while resolving model access."""
1802
+ detail = " ".join(str(message or "").lower() for message in messages)
1803
+ return (
1804
+ "421" in detail
1805
+ and "misdirected request" in detail
1806
+ and (
1807
+ "retrieve the list of available models" in detail
1808
+ or "failed to load models" in detail
1809
+ )
1810
+ )
1811
+
1812
+
1813
+ def is_runtime_resource_exhaustion_error(*messages: str | None) -> bool:
1814
+ """Return whether the OS refused to create an agent process or thread."""
1815
+ detail = " ".join(str(message or "").lower() for message in messages)
1816
+ return any(
1817
+ pattern in detail
1818
+ for pattern in (
1819
+ "can't spawn worker thread",
1820
+ "failed to load ca certificates off thread",
1821
+ "resource temporarily unavailable (os error 11)",
1822
+ "pthread_create failed",
1823
+ )
1824
+ )
1825
+
1826
+
1827
+ def collect_runtime_resource_metrics(
1828
+ *,
1829
+ proc_root: str | Path = "/proc",
1830
+ cgroup_root: str | Path = "/sys/fs/cgroup",
1831
+ ) -> dict[str, Any]:
1832
+ """Collect Linux process/thread headroom without optional dependencies."""
1833
+ metrics: dict[str, Any] = {}
1834
+ if not sys.platform.startswith("linux"):
1835
+ return metrics
1836
+
1837
+ proc_path = Path(proc_root)
1838
+ cgroup_path = Path(cgroup_root)
1839
+ constraints: list[tuple[str, int, int]] = []
1840
+
1841
+ try:
1842
+ for line in (proc_path / "self" / "cgroup").read_text().splitlines():
1843
+ hierarchy, controllers, relative_path = line.split(":", 2)
1844
+ relative = relative_path.lstrip("/")
1845
+ if hierarchy == "0" and not controllers:
1846
+ pid_dir = cgroup_path / relative
1847
+ elif "pids" in controllers.split(","):
1848
+ pid_dir = cgroup_path / "pids" / relative
1849
+ else:
1850
+ continue
1851
+ current_text = (pid_dir / "pids.current").read_text().strip()
1852
+ maximum_text = (pid_dir / "pids.max").read_text().strip()
1853
+ if maximum_text != "max":
1854
+ constraints.append(("cgroup", int(current_text), int(maximum_text)))
1855
+ break
1856
+ except (OSError, ValueError):
1857
+ pass
1858
+
1859
+ try:
1860
+ import resource
1861
+
1862
+ soft_limit, _ = resource.getrlimit(resource.RLIMIT_NPROC)
1863
+ if soft_limit != resource.RLIM_INFINITY:
1864
+ current_uid = os.getuid()
1865
+ user_threads = 0
1866
+ for status_path in proc_path.glob("[0-9]*/status"):
1867
+ try:
1868
+ real_uid: int | None = None
1869
+ threads = 1
1870
+ for line in status_path.read_text().splitlines():
1871
+ if line.startswith("Uid:"):
1872
+ real_uid = int(line.split()[1])
1873
+ elif line.startswith("Threads:"):
1874
+ threads = int(line.split()[1])
1875
+ if real_uid == current_uid:
1876
+ user_threads += threads
1877
+ except (OSError, ValueError, IndexError):
1878
+ continue
1879
+ constraints.append(("rlimit_nproc", user_threads, int(soft_limit)))
1880
+ except (ImportError, OSError, ValueError):
1881
+ pass
1882
+
1883
+ if constraints:
1884
+ source, current, limit = min(
1885
+ constraints,
1886
+ key=lambda item: item[2] - item[1],
1887
+ )
1888
+ metrics.update({
1889
+ "process_limit_source": source,
1890
+ "process_current": current,
1891
+ "process_limit": limit,
1892
+ "process_remaining": max(0, limit - current),
1893
+ })
1894
+
1895
+ try:
1896
+ for line in (proc_path / "meminfo").read_text().splitlines():
1897
+ if line.startswith("MemAvailable:"):
1898
+ metrics["memory_available_mb"] = int(line.split()[1]) // 1024
1899
+ break
1900
+ except (OSError, ValueError, IndexError):
1901
+ pass
1902
+ return metrics
1903
+
1904
+
1905
+ def runtime_resource_exhaustion_reason(metrics: dict[str, Any]) -> str | None:
1906
+ """Return a blocking reason when an agent lacks safe process headroom."""
1907
+ process_remaining = metrics.get("process_remaining")
1908
+ if isinstance(process_remaining, int) and process_remaining < 32:
1909
+ return (
1910
+ f"Only {process_remaining} process/thread slots remain "
1911
+ f"({metrics.get('process_current')}/{metrics.get('process_limit')} used via "
1912
+ f"{metrics.get('process_limit_source')})"
1913
+ )
1914
+ memory_available_mb = metrics.get("memory_available_mb")
1915
+ if isinstance(memory_available_mb, int) and memory_available_mb < 256:
1916
+ return f"Only {memory_available_mb} MiB memory is available"
1917
+ return None
1918
+
1919
+
1772
1920
  def copilot_auth_state_signature(copilot_home: str | Path) -> str | None:
1773
1921
  """Return a content signature for Copilot's persisted authentication state."""
1774
1922
  config_file = Path(copilot_home) / "config.json"
@@ -1786,6 +1934,65 @@ def _copilot_auth_lock_path(copilot_home: str | Path) -> Path:
1786
1934
  return home.with_name(f".{home.name}-auth-state.lock")
1787
1935
 
1788
1936
 
1937
+ def replace_copilot_auth_state(
1938
+ source_home: str | Path,
1939
+ destination_home: str | Path,
1940
+ expected_destination_signature: str | None = None,
1941
+ ) -> bool:
1942
+ """Replace an isolated Copilot auth file from a known login home."""
1943
+ source = Path(source_home) / "config.json"
1944
+ destination_dir = Path(destination_home)
1945
+ destination = destination_dir / source.name
1946
+ if not source.is_file():
1947
+ return False
1948
+ lock_file = None
1949
+ temp_name: str | None = None
1950
+ try:
1951
+ source_signature = copilot_auth_state_signature(source_home)
1952
+ if source_signature is None:
1953
+ return False
1954
+ destination_dir.mkdir(parents=True, exist_ok=True)
1955
+ lock_file = acquire_file_lock(
1956
+ _copilot_auth_lock_path(destination_dir),
1957
+ blocking=True,
1958
+ )
1959
+ if lock_file is None:
1960
+ logger.warning("Could not lock Copilot authentication state for restore")
1961
+ return False
1962
+ destination_signature = copilot_auth_state_signature(destination_home)
1963
+ if (
1964
+ expected_destination_signature is not None
1965
+ and destination_signature != expected_destination_signature
1966
+ ):
1967
+ return False
1968
+ if destination_signature == source_signature:
1969
+ return False
1970
+ temp_fd, temp_name = tempfile.mkstemp(prefix=".config.json.", dir=destination_dir)
1971
+ os.close(temp_fd)
1972
+ shutil.copy2(source, temp_name)
1973
+ os.chmod(temp_name, 0o600)
1974
+ for attempt in range(3):
1975
+ try:
1976
+ os.replace(temp_name, destination)
1977
+ temp_name = None
1978
+ break
1979
+ except PermissionError:
1980
+ if sys.platform != "win32" or attempt == 2:
1981
+ raise
1982
+ time.sleep(0.05)
1983
+ return True
1984
+ except OSError:
1985
+ logger.warning("Could not restore interactive Copilot credentials", exc_info=True)
1986
+ return False
1987
+ finally:
1988
+ if temp_name is not None:
1989
+ try:
1990
+ Path(temp_name).unlink(missing_ok=True)
1991
+ except OSError:
1992
+ logger.debug("Could not remove temporary Copilot config", exc_info=True)
1993
+ release_file_lock(lock_file)
1994
+
1995
+
1789
1996
  def persist_copilot_auth_state(
1790
1997
  run_home: str | Path,
1791
1998
  persistent_home: str | Path,
@@ -1821,6 +2028,15 @@ def persist_copilot_auth_state(
1821
2028
  destination_signature = copilot_auth_state_signature(destination_dir)
1822
2029
  if destination_signature == source_signature:
1823
2030
  return
2031
+ if (
2032
+ baseline_signature is not None
2033
+ and destination_signature != baseline_signature
2034
+ ):
2035
+ logger.info(
2036
+ "Skipping stale Copilot authentication update because another "
2037
+ "run refreshed the persistent state first"
2038
+ )
2039
+ return
1824
2040
  if baseline_signature is None and destination.is_file():
1825
2041
  if destination.stat().st_mtime_ns >= source.stat().st_mtime_ns:
1826
2042
  return
@@ -1867,7 +2083,7 @@ def prepare_copilot_home() -> str:
1867
2083
  authentication state written by isolated runs is retained even when it
1868
2084
  has no counterpart in the interactive home.
1869
2085
  """
1870
- source_root = Path.home() / ".copilot"
2086
+ source_root = copilot_interactive_home()
1871
2087
  target_root = Path.home() / ".forgexa" / "copilot-home"
1872
2088
  lock_file = None
1873
2089
  try:
@@ -1879,6 +2095,8 @@ def prepare_copilot_home() -> str:
1879
2095
  if lock_file is None:
1880
2096
  logger.warning("Could not lock Copilot home for synchronization")
1881
2097
  return str(target_root)
2098
+ if source_root.resolve() == target_root.resolve():
2099
+ return str(target_root)
1882
2100
  if not source_root.exists():
1883
2101
  return str(target_root)
1884
2102
 
@@ -1920,6 +2138,20 @@ def prepare_copilot_home() -> str:
1920
2138
  migration_marker.touch(exist_ok=True)
1921
2139
  os.chmod(migration_marker, 0o600)
1922
2140
 
2141
+ source_auth_signature = copilot_auth_state_signature(source_root)
2142
+ source_signature_marker = target_root.parent / ".copilot-login-source.sha256"
2143
+ recorded_source_signature: str | None = None
2144
+ if source_signature_marker.is_file():
2145
+ try:
2146
+ recorded_source_signature = source_signature_marker.read_text().strip()
2147
+ except OSError:
2148
+ logger.debug("Could not read Copilot login source signature", exc_info=True)
2149
+ source_auth_changed = bool(
2150
+ source_auth_signature
2151
+ and recorded_source_signature is not None
2152
+ and source_auth_signature != recorded_source_signature
2153
+ )
2154
+ source_config_synced = False
1923
2155
  for child in source_root.iterdir():
1924
2156
  if child.name in {"logs", "session-state"}:
1925
2157
  continue
@@ -1937,11 +2169,15 @@ def prepare_copilot_home() -> str:
1937
2169
  elif child.is_file():
1938
2170
  if (
1939
2171
  child.name == "config.json"
2172
+ and not source_auth_changed
1940
2173
  and dest.is_file()
1941
2174
  and dest.stat().st_mtime_ns >= child.stat().st_mtime_ns
1942
2175
  ):
2176
+ source_config_synced = True
1943
2177
  continue
1944
2178
  shutil.copy2(child, dest)
2179
+ if child.name == "config.json":
2180
+ source_config_synced = True
1945
2181
  except OSError as exc:
1946
2182
  logger.debug(
1947
2183
  "Copilot home mirror skipped %s -> %s: %s",
@@ -1949,6 +2185,12 @@ def prepare_copilot_home() -> str:
1949
2185
  dest,
1950
2186
  exc,
1951
2187
  )
2188
+ if source_config_synced and source_auth_signature:
2189
+ try:
2190
+ source_signature_marker.write_text(source_auth_signature)
2191
+ os.chmod(source_signature_marker, 0o600)
2192
+ except OSError:
2193
+ logger.debug("Could not record Copilot login source signature", exc_info=True)
1952
2194
  except OSError:
1953
2195
  logger.warning("Could not prepare writable Copilot home", exc_info=True)
1954
2196
  finally:
@@ -910,7 +910,7 @@ except (ImportError, ModuleNotFoundError):
910
910
  # DAEMON_VERSION is the protocol/logic version of the daemon code.
911
911
  # Kept in sync with pyproject.toml version via bump-version.sh.
912
912
  # CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
913
- DAEMON_VERSION = "1.42.1"
913
+ DAEMON_VERSION = "1.43.3"
914
914
 
915
915
 
916
916
  def _detect_client_type() -> str:
@@ -4781,7 +4781,12 @@ class ProcessManager:
4781
4781
  "error during sign-in",
4782
4782
  "websockettransporterror",
4783
4783
  "websocket receive failed",
4784
- "421 misdirected request",
4784
+ "misdirected request",
4785
+ "could not retrieve the list of available models",
4786
+ "can't spawn worker thread",
4787
+ "failed to load ca certificates off thread",
4788
+ "resource temporarily unavailable (os error 11)",
4789
+ "pthread_create failed",
4785
4790
  "failed to initialize mcp client",
4786
4791
  # "api error" removed: too broad — matches agent-generated code/output
4787
4792
  # discussing API errors. Real API transport errors are covered by the
@@ -5076,6 +5081,38 @@ class ProcessManager:
5076
5081
  on_chunk: Any = None,
5077
5082
  ) -> TaskResult:
5078
5083
  """Execute an agent CLI and collect results."""
5084
+ resource_metrics = agent_core.collect_runtime_resource_metrics()
5085
+ resource_reason = agent_core.runtime_resource_exhaustion_reason(
5086
+ resource_metrics,
5087
+ )
5088
+ if resource_reason:
5089
+ check = {
5090
+ "id": "process_headroom",
5091
+ "status": "fail",
5092
+ "blocking": True,
5093
+ "failure_class": "resource_exhausted",
5094
+ "detail": resource_reason,
5095
+ "remediation": (
5096
+ "Stop orphaned agent processes or raise the user/cgroup PID limit, "
5097
+ "then retry"
5098
+ ),
5099
+ }
5100
+ logger.error(
5101
+ "Refusing to launch agent '%s' for task %s: %s",
5102
+ agent.agent_id,
5103
+ task.task_id,
5104
+ resource_reason,
5105
+ )
5106
+ return TaskResult(
5107
+ status="failed",
5108
+ exit_code=-1,
5109
+ stdout="",
5110
+ stderr="",
5111
+ error=f"Runtime resource preflight failed: {resource_reason}",
5112
+ failure_code="runtime_preflight_failed",
5113
+ preflight={"scope": "runtime", "checks": [check]},
5114
+ )
5115
+
5079
5116
  # Centralized, agent-agnostic version gate: AgentDiscovery.discover()
5080
5117
  # already computed min_version_ok/version_warning against
5081
5118
  # AGENT_REGISTRY[agent_id]["min_version"] (see _check_agent_min_version).
@@ -5996,7 +6033,9 @@ class ProcessManager:
5996
6033
  *,
5997
6034
  disable_builtin_mcps: bool = False,
5998
6035
  _retry_without_effort: bool = False,
6036
+ _retry_without_token_env: bool = False,
5999
6037
  _isolated_copilot_home: str | None = None,
6038
+ _auth_state: dict[str, bool] | None = None,
6000
6039
  ) -> TaskResult:
6001
6040
  """Run GitHub Copilot CLI in non-interactive JSON-streaming mode.
6002
6041
 
@@ -6032,6 +6071,8 @@ class ProcessManager:
6032
6071
  # session-state persists from the prior attempt all todos are already
6033
6072
  # "done" and the agent exits immediately without producing any output.
6034
6073
  _is_outer_call = _isolated_copilot_home is None
6074
+ if _auth_state is None:
6075
+ _auth_state = {"persist": True}
6035
6076
  _master_copilot_home: str | None = None
6036
6077
  _copilot_auth_baseline: str | None = None
6037
6078
  if _is_outer_call:
@@ -6059,10 +6100,13 @@ class ProcessManager:
6059
6100
  os.environ.copy(),
6060
6101
  copilot_home=_isolated_copilot_home,
6061
6102
  )
6103
+ if _retry_without_token_env:
6104
+ for key in agent_core.COPILOT_TOKEN_ENV_VARS:
6105
+ env.pop(key, None)
6062
6106
  auth_env_var = next(
6063
6107
  (
6064
6108
  key
6065
- for key in ("COPILOT_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN")
6109
+ for key in agent_core.COPILOT_TOKEN_ENV_VARS
6066
6110
  if env.get(key)
6067
6111
  ),
6068
6112
  None,
@@ -6230,11 +6274,58 @@ class ProcessManager:
6230
6274
  on_chunk,
6231
6275
  disable_builtin_mcps=True,
6232
6276
  _retry_without_effort=_retry_without_effort,
6277
+ _retry_without_token_env=_retry_without_token_env,
6233
6278
  _isolated_copilot_home=_isolated_copilot_home,
6279
+ _auth_state=_auth_state,
6234
6280
  )
6235
6281
  retry_result.metrics["copilot_builtin_mcp_fallback"] = True
6236
6282
  return retry_result
6237
6283
 
6284
+ model_discovery_error = agent_core.is_copilot_model_discovery_auth_error(
6285
+ *structured_errors,
6286
+ stderr,
6287
+ )
6288
+ if model_discovery_error and not self._copilot_called_any_tools(stdout):
6289
+ isolated_config = Path(_isolated_copilot_home) / "config.json"
6290
+ recovery_home = agent_core.copilot_recovery_home(
6291
+ _isolated_copilot_home,
6292
+ )
6293
+ can_retry_native_auth = bool(
6294
+ isolated_config.is_file()
6295
+ or recovery_home is not None
6296
+ )
6297
+ if not _retry_without_token_env and can_retry_native_auth:
6298
+ restored_interactive_auth = bool(
6299
+ recovery_home
6300
+ and await asyncio.to_thread(
6301
+ agent_core.replace_copilot_auth_state,
6302
+ recovery_home,
6303
+ _isolated_copilot_home,
6304
+ )
6305
+ )
6306
+ logger.warning(
6307
+ "Copilot model discovery failed with HTTP 421 before work for "
6308
+ "task %s; retrying once with token environment variables removed%s",
6309
+ task_id,
6310
+ " and interactive login restored" if restored_interactive_auth else "",
6311
+ )
6312
+ retry_result = await self._run_copilot(
6313
+ agent,
6314
+ prompt,
6315
+ cwd,
6316
+ timeout,
6317
+ task_id,
6318
+ on_chunk,
6319
+ disable_builtin_mcps=disable_builtin_mcps,
6320
+ _retry_without_effort=_retry_without_effort,
6321
+ _retry_without_token_env=True,
6322
+ _isolated_copilot_home=_isolated_copilot_home,
6323
+ _auth_state=_auth_state,
6324
+ )
6325
+ retry_result.metrics["copilot_model_discovery_recovery"] = True
6326
+ return retry_result
6327
+ _auth_state["persist"] = False
6328
+
6238
6329
  if copilot_exit == 0 and returncode == 0:
6239
6330
  return TaskResult(
6240
6331
  status="success",
@@ -6290,7 +6381,9 @@ class ProcessManager:
6290
6381
  agent, prompt, cwd, timeout, task_id, on_chunk,
6291
6382
  disable_builtin_mcps=disable_builtin_mcps,
6292
6383
  _retry_without_effort=True,
6384
+ _retry_without_token_env=_retry_without_token_env,
6293
6385
  _isolated_copilot_home=_isolated_copilot_home,
6386
+ _auth_state=_auth_state,
6294
6387
  )
6295
6388
  # Exhausted retries — emit an actionable diagnostic hint
6296
6389
  if effective_rc == 1 and _no_tools and not copilot_specific_error:
@@ -6342,6 +6435,14 @@ class ProcessManager:
6342
6435
  stdout=stdout[-settings.AGENT_MAX_OUTPUT_SIZE:],
6343
6436
  stderr=stderr[-10000:],
6344
6437
  error=failure_error,
6438
+ failure_code=(
6439
+ "runtime_resource_exhausted"
6440
+ if agent_core.is_runtime_resource_exhaustion_error(
6441
+ failure_error,
6442
+ stderr,
6443
+ )
6444
+ else ""
6445
+ ),
6345
6446
  metrics=metrics,
6346
6447
  )
6347
6448
  except asyncio.CancelledError:
@@ -6390,7 +6491,7 @@ class ProcessManager:
6390
6491
  finally:
6391
6492
  self.active_processes.pop(task_id, None)
6392
6493
  if _is_outer_call and _isolated_copilot_home:
6393
- if _master_copilot_home:
6494
+ if _master_copilot_home and _auth_state["persist"]:
6394
6495
  await asyncio.to_thread(
6395
6496
  agent_core.persist_copilot_auth_state,
6396
6497
  _isolated_copilot_home,
@@ -6691,26 +6792,11 @@ class ProcessManager:
6691
6792
  async def cancel(self, task_id: str):
6692
6793
  proc = self.active_processes.pop(task_id, None)
6693
6794
  if proc:
6795
+ _kill_proc(proc)
6694
6796
  try:
6695
- if sys.platform != "win32":
6696
- import signal as _signal
6697
- try:
6698
- os.killpg(os.getpgid(proc.pid), _signal.SIGKILL)
6699
- except (ProcessLookupError, PermissionError, OSError):
6700
- pass
6701
- else:
6702
- import subprocess as _subprocess
6703
- _subprocess.run(
6704
- ["taskkill", "/F", "/T", "/PID", str(proc.pid)],
6705
- capture_output=True,
6706
- )
6797
+ await asyncio.wait_for(proc.wait(), timeout=5)
6707
6798
  except Exception:
6708
- pass
6709
- finally:
6710
- try:
6711
- proc.kill()
6712
- except Exception:
6713
- pass
6799
+ logger.warning("Timed out reaping agent process for task %s", task_id)
6714
6800
 
6715
6801
 
6716
6802
  # ── Progress Reporter ──
@@ -6910,8 +6996,48 @@ class RuntimePreflightMonitor:
6910
6996
  await self._check_git_executable(),
6911
6997
  await self._check_workspace_root(),
6912
6998
  await self._check_workspace_disk(),
6999
+ await self._check_process_headroom(),
6913
7000
  ]
6914
7001
 
7002
+ @staticmethod
7003
+ async def _check_process_headroom() -> dict:
7004
+ metrics = agent_core.collect_runtime_resource_metrics()
7005
+ reason = agent_core.runtime_resource_exhaustion_reason(metrics)
7006
+ if reason:
7007
+ return {
7008
+ "id": "process_headroom",
7009
+ "status": "fail",
7010
+ "blocking": True,
7011
+ "failure_class": "resource_exhausted",
7012
+ "detail": reason,
7013
+ "remediation": (
7014
+ "Stop orphaned agent processes or raise the user/cgroup PID limit, "
7015
+ "then restart the daemon"
7016
+ ),
7017
+ }
7018
+ if metrics:
7019
+ detail = (
7020
+ f"{metrics['process_remaining']} process/thread slots remain"
7021
+ if "process_remaining" in metrics
7022
+ else f"{metrics.get('memory_available_mb', 'unknown')} MiB memory available"
7023
+ )
7024
+ return {
7025
+ "id": "process_headroom",
7026
+ "status": "pass",
7027
+ "blocking": True,
7028
+ "failure_class": "",
7029
+ "detail": detail,
7030
+ "remediation": "",
7031
+ }
7032
+ return {
7033
+ "id": "process_headroom",
7034
+ "status": "unknown",
7035
+ "blocking": False,
7036
+ "failure_class": "",
7037
+ "detail": "Process/thread headroom is unavailable on this platform",
7038
+ "remediation": "",
7039
+ }
7040
+
6915
7041
  @staticmethod
6916
7042
  async def _run_command_timeout(cmd: list[str], timeout: float = 5) -> str | None:
6917
7043
  """Run a command with a deadline. Returns stdout on success, None on timeout/error.
@@ -7216,15 +7342,17 @@ class HeartbeatService:
7216
7342
 
7217
7343
  def _collect_system_metrics(self) -> dict:
7218
7344
  """Basic system metrics."""
7345
+ metrics = agent_core.collect_runtime_resource_metrics()
7219
7346
  try:
7220
7347
  import psutil
7221
- return {
7348
+ metrics.update({
7222
7349
  "cpu_percent": psutil.cpu_percent(),
7223
7350
  "memory_percent": psutil.virtual_memory().percent,
7224
7351
  "disk_percent": psutil.disk_usage("/").percent,
7225
- }
7352
+ })
7226
7353
  except ImportError:
7227
- return {}
7354
+ pass
7355
+ return metrics
7228
7356
 
7229
7357
 
7230
7358
  # ── Log Uploader ──
@@ -8562,6 +8690,12 @@ class RuntimeDaemon:
8562
8690
  """Main entry point."""
8563
8691
  # Prevent multiple daemon instances on the same machine
8564
8692
  self._acquire_lock()
8693
+ cleaned_processes = self._cleanup_stale_agent_processes()
8694
+ if cleaned_processes:
8695
+ logger.warning(
8696
+ "Cleaned up %d orphaned Forgexa agent process group(s)",
8697
+ cleaned_processes,
8698
+ )
8565
8699
 
8566
8700
  # Mint a local-dev token if no token is available yet
8567
8701
  if not self.api_token and not self._has_stored_credentials():
@@ -9271,7 +9405,15 @@ class RuntimeDaemon:
9271
9405
  # Guard: if the agent already produced file changes in the workspace, it DID
9272
9406
  # meaningful work — don't trigger fallback even if it crashed after completing.
9273
9407
  # Let the recovery logic (step 4.1) handle non-zero exit with committed work.
9274
- agent_failure_kind = ProcessManager.agent_failure_kind(result)
9408
+ runtime_resource_failure = result.failure_code in {
9409
+ "runtime_preflight_failed",
9410
+ "runtime_resource_exhausted",
9411
+ }
9412
+ agent_failure_kind = (
9413
+ None
9414
+ if runtime_resource_failure
9415
+ else ProcessManager.agent_failure_kind(result)
9416
+ )
9275
9417
  is_agent_fallback_eligible = agent_failure_kind is not None
9276
9418
  is_silent_idle_timeout = self.process_manager.is_silent_idle_timeout(result)
9277
9419
  fallback_reason = (
@@ -11408,7 +11550,13 @@ class RuntimeDaemon:
11408
11550
  # try the next available agent before giving up. An explicit
11409
11551
  # override is deliberately a single-agent attempt.
11410
11552
  fallback_scope = "override" if agent_override else "auto"
11411
- allow_fallback = not bool(agent_override)
11553
+ allow_fallback = (
11554
+ not bool(agent_override)
11555
+ and result.failure_code not in {
11556
+ "runtime_preflight_failed",
11557
+ "runtime_resource_exhausted",
11558
+ }
11559
+ )
11412
11560
  generation_attempts: list[dict] = []
11413
11561
  agent_failure_kind = ProcessManager.agent_failure_kind(result)
11414
11562
  agent_failure_label = (
@@ -14024,15 +14172,78 @@ class RuntimeDaemon:
14024
14172
  except Exception as e:
14025
14173
  logger.debug("Could not clean temp dir %s: %s", d, e)
14026
14174
 
14175
+ @staticmethod
14176
+ def _cleanup_stale_agent_processes(
14177
+ *,
14178
+ proc_root: str | Path = "/proc",
14179
+ temp_root: str | Path | None = None,
14180
+ ) -> int:
14181
+ """Kill orphaned agent groups identifiable as Forgexa-owned on Linux."""
14182
+ if not sys.platform.startswith("linux"):
14183
+ return 0
14184
+ import signal as _signal
14185
+ import tempfile
14186
+
14187
+ root = Path(proc_root)
14188
+ temp = Path(temp_root or tempfile.gettempdir()).resolve()
14189
+ current_pid = os.getpid()
14190
+ current_group = os.getpgrp()
14191
+ cleaned = 0
14192
+ try:
14193
+ process_dirs = sorted(
14194
+ (path for path in root.iterdir() if path.name.isdigit()),
14195
+ key=lambda path: int(path.name),
14196
+ )
14197
+ except OSError:
14198
+ return 0
14199
+
14200
+ for process_dir in process_dirs:
14201
+ pid = int(process_dir.name)
14202
+ if pid == current_pid:
14203
+ continue
14204
+ try:
14205
+ entries = (process_dir / "environ").read_bytes().split(b"\0")
14206
+ environment = {}
14207
+ for entry in entries:
14208
+ if b"=" not in entry:
14209
+ continue
14210
+ key, value = entry.split(b"=", 1)
14211
+ environment[key.decode(errors="ignore")] = value.decode(errors="ignore")
14212
+ marked = environment.get("FORGEXA_AGENT_PROCESS") == "1"
14213
+ isolated_home = environment.get("COPILOT_HOME", "")
14214
+ legacy_copilot = False
14215
+ if isolated_home:
14216
+ isolated_path = Path(isolated_home).resolve()
14217
+ legacy_copilot = (
14218
+ isolated_path.parent == temp
14219
+ and isolated_path.name.startswith("copilot-")
14220
+ )
14221
+ if not (marked or legacy_copilot):
14222
+ continue
14223
+ process_group = os.getpgid(pid)
14224
+ if process_group == current_group:
14225
+ continue
14226
+ os.killpg(process_group, _signal.SIGKILL)
14227
+ cleaned += 1
14228
+ except (OSError, ValueError):
14229
+ continue
14230
+ return cleaned
14231
+
14027
14232
  async def _shutdown_gracefully(self):
14028
14233
  """Graceful shutdown."""
14029
14234
  logger.info("Shutting down daemon...")
14030
14235
  self._shutdown = True
14031
14236
 
14032
- # Cancel active tasks
14033
- for task_id, task in self.active_tasks.items():
14237
+ # Cancel task coroutines, then terminate and reap every tracked process.
14238
+ tasks = list(self.active_tasks.values())
14239
+ for task in tasks:
14034
14240
  task.cancel()
14035
- await self.process_manager.cancel(task_id)
14241
+ for process_id in list(self.process_manager.active_processes):
14242
+ await self.process_manager.cancel(process_id)
14243
+ if tasks:
14244
+ await asyncio.gather(*tasks, return_exceptions=True)
14245
+ self.active_tasks.clear()
14246
+ self._task_connections.clear()
14036
14247
 
14037
14248
  # Stop preflight monitor
14038
14249
  await self.preflight_monitor.stop()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: forgexa-cli
3
- Version: 1.42.1
3
+ Version: 1.43.3
4
4
  Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
5
5
  Author-email: Jason Sun <dev.winds@gmail.com>
6
6
  License-Expression: MIT
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "forgexa-cli"
3
- version = "1.42.1"
3
+ version = "1.43.3"
4
4
  description = "Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform"
5
5
  requires-python = ">=3.9"
6
6
  license = "MIT"
File without changes
File without changes