forgexa-cli 1.42.1__tar.gz → 1.43.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/PKG-INFO +1 -1
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/__init__.py +1 -1
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/agent_core.py +243 -1
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/daemon.py +241 -30
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/PKG-INFO +1 -1
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/pyproject.toml +1 -1
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/README.md +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/_build_config.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/_local_bind.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/autoupgrade.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/main.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli/py.typed +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/SOURCES.txt +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/dependency_links.txt +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/entry_points.txt +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/requires.txt +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/forgexa_cli.egg-info/top_level.txt +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/setup.cfg +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_auth_and_runtime_commands.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_autoupgrade.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_check_command.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_expiry_warnings_and_revoke.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_local_bind_commands.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_runtime_credentials.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_session_credentials.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_silent_install.py +0 -0
- {forgexa_cli-1.42.1 → forgexa_cli-1.43.3}/tests/test_upgrade_observability.py +0 -0
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""forgexa-cli — Forgexa command-line client."""
|
|
2
|
-
__version__ = "1.
|
|
2
|
+
__version__ = "1.43.3"
|
|
@@ -1766,9 +1766,157 @@ def copilot_base_env(env: dict[str, str], *, copilot_home: str | Path) -> dict[s
|
|
|
1766
1766
|
env = dict(env)
|
|
1767
1767
|
env["TERM"] = "dumb" # suppress TTY-detection that suspends the process
|
|
1768
1768
|
env["COPILOT_HOME"] = str(copilot_home)
|
|
1769
|
+
env.setdefault("TOKIO_WORKER_THREADS", "2")
|
|
1770
|
+
env["FORGEXA_AGENT_PROCESS"] = "1"
|
|
1769
1771
|
return env
|
|
1770
1772
|
|
|
1771
1773
|
|
|
1774
|
+
COPILOT_TOKEN_ENV_VARS = ("COPILOT_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN")
|
|
1775
|
+
|
|
1776
|
+
|
|
1777
|
+
def copilot_interactive_home() -> Path:
|
|
1778
|
+
"""Return the login home selected for interactive Copilot CLI use."""
|
|
1779
|
+
configured_home = os.environ.get("COPILOT_HOME", "").strip()
|
|
1780
|
+
if configured_home:
|
|
1781
|
+
return Path(configured_home).expanduser()
|
|
1782
|
+
return Path.home() / ".copilot"
|
|
1783
|
+
|
|
1784
|
+
|
|
1785
|
+
def copilot_recovery_home(destination_home: str | Path) -> Path | None:
|
|
1786
|
+
"""Return a populated login home distinct from the failed destination."""
|
|
1787
|
+
destination = Path(destination_home).resolve()
|
|
1788
|
+
candidates = (copilot_interactive_home(), Path.home() / ".copilot")
|
|
1789
|
+
seen: set[Path] = set()
|
|
1790
|
+
for candidate in candidates:
|
|
1791
|
+
resolved = candidate.resolve()
|
|
1792
|
+
if resolved in seen or resolved == destination:
|
|
1793
|
+
continue
|
|
1794
|
+
seen.add(resolved)
|
|
1795
|
+
if (resolved / "config.json").is_file():
|
|
1796
|
+
return resolved
|
|
1797
|
+
return None
|
|
1798
|
+
|
|
1799
|
+
|
|
1800
|
+
def is_copilot_model_discovery_auth_error(*messages: str | None) -> bool:
|
|
1801
|
+
"""Return whether Copilot failed before work while resolving model access."""
|
|
1802
|
+
detail = " ".join(str(message or "").lower() for message in messages)
|
|
1803
|
+
return (
|
|
1804
|
+
"421" in detail
|
|
1805
|
+
and "misdirected request" in detail
|
|
1806
|
+
and (
|
|
1807
|
+
"retrieve the list of available models" in detail
|
|
1808
|
+
or "failed to load models" in detail
|
|
1809
|
+
)
|
|
1810
|
+
)
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
def is_runtime_resource_exhaustion_error(*messages: str | None) -> bool:
|
|
1814
|
+
"""Return whether the OS refused to create an agent process or thread."""
|
|
1815
|
+
detail = " ".join(str(message or "").lower() for message in messages)
|
|
1816
|
+
return any(
|
|
1817
|
+
pattern in detail
|
|
1818
|
+
for pattern in (
|
|
1819
|
+
"can't spawn worker thread",
|
|
1820
|
+
"failed to load ca certificates off thread",
|
|
1821
|
+
"resource temporarily unavailable (os error 11)",
|
|
1822
|
+
"pthread_create failed",
|
|
1823
|
+
)
|
|
1824
|
+
)
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
def collect_runtime_resource_metrics(
|
|
1828
|
+
*,
|
|
1829
|
+
proc_root: str | Path = "/proc",
|
|
1830
|
+
cgroup_root: str | Path = "/sys/fs/cgroup",
|
|
1831
|
+
) -> dict[str, Any]:
|
|
1832
|
+
"""Collect Linux process/thread headroom without optional dependencies."""
|
|
1833
|
+
metrics: dict[str, Any] = {}
|
|
1834
|
+
if not sys.platform.startswith("linux"):
|
|
1835
|
+
return metrics
|
|
1836
|
+
|
|
1837
|
+
proc_path = Path(proc_root)
|
|
1838
|
+
cgroup_path = Path(cgroup_root)
|
|
1839
|
+
constraints: list[tuple[str, int, int]] = []
|
|
1840
|
+
|
|
1841
|
+
try:
|
|
1842
|
+
for line in (proc_path / "self" / "cgroup").read_text().splitlines():
|
|
1843
|
+
hierarchy, controllers, relative_path = line.split(":", 2)
|
|
1844
|
+
relative = relative_path.lstrip("/")
|
|
1845
|
+
if hierarchy == "0" and not controllers:
|
|
1846
|
+
pid_dir = cgroup_path / relative
|
|
1847
|
+
elif "pids" in controllers.split(","):
|
|
1848
|
+
pid_dir = cgroup_path / "pids" / relative
|
|
1849
|
+
else:
|
|
1850
|
+
continue
|
|
1851
|
+
current_text = (pid_dir / "pids.current").read_text().strip()
|
|
1852
|
+
maximum_text = (pid_dir / "pids.max").read_text().strip()
|
|
1853
|
+
if maximum_text != "max":
|
|
1854
|
+
constraints.append(("cgroup", int(current_text), int(maximum_text)))
|
|
1855
|
+
break
|
|
1856
|
+
except (OSError, ValueError):
|
|
1857
|
+
pass
|
|
1858
|
+
|
|
1859
|
+
try:
|
|
1860
|
+
import resource
|
|
1861
|
+
|
|
1862
|
+
soft_limit, _ = resource.getrlimit(resource.RLIMIT_NPROC)
|
|
1863
|
+
if soft_limit != resource.RLIM_INFINITY:
|
|
1864
|
+
current_uid = os.getuid()
|
|
1865
|
+
user_threads = 0
|
|
1866
|
+
for status_path in proc_path.glob("[0-9]*/status"):
|
|
1867
|
+
try:
|
|
1868
|
+
real_uid: int | None = None
|
|
1869
|
+
threads = 1
|
|
1870
|
+
for line in status_path.read_text().splitlines():
|
|
1871
|
+
if line.startswith("Uid:"):
|
|
1872
|
+
real_uid = int(line.split()[1])
|
|
1873
|
+
elif line.startswith("Threads:"):
|
|
1874
|
+
threads = int(line.split()[1])
|
|
1875
|
+
if real_uid == current_uid:
|
|
1876
|
+
user_threads += threads
|
|
1877
|
+
except (OSError, ValueError, IndexError):
|
|
1878
|
+
continue
|
|
1879
|
+
constraints.append(("rlimit_nproc", user_threads, int(soft_limit)))
|
|
1880
|
+
except (ImportError, OSError, ValueError):
|
|
1881
|
+
pass
|
|
1882
|
+
|
|
1883
|
+
if constraints:
|
|
1884
|
+
source, current, limit = min(
|
|
1885
|
+
constraints,
|
|
1886
|
+
key=lambda item: item[2] - item[1],
|
|
1887
|
+
)
|
|
1888
|
+
metrics.update({
|
|
1889
|
+
"process_limit_source": source,
|
|
1890
|
+
"process_current": current,
|
|
1891
|
+
"process_limit": limit,
|
|
1892
|
+
"process_remaining": max(0, limit - current),
|
|
1893
|
+
})
|
|
1894
|
+
|
|
1895
|
+
try:
|
|
1896
|
+
for line in (proc_path / "meminfo").read_text().splitlines():
|
|
1897
|
+
if line.startswith("MemAvailable:"):
|
|
1898
|
+
metrics["memory_available_mb"] = int(line.split()[1]) // 1024
|
|
1899
|
+
break
|
|
1900
|
+
except (OSError, ValueError, IndexError):
|
|
1901
|
+
pass
|
|
1902
|
+
return metrics
|
|
1903
|
+
|
|
1904
|
+
|
|
1905
|
+
def runtime_resource_exhaustion_reason(metrics: dict[str, Any]) -> str | None:
|
|
1906
|
+
"""Return a blocking reason when an agent lacks safe process headroom."""
|
|
1907
|
+
process_remaining = metrics.get("process_remaining")
|
|
1908
|
+
if isinstance(process_remaining, int) and process_remaining < 32:
|
|
1909
|
+
return (
|
|
1910
|
+
f"Only {process_remaining} process/thread slots remain "
|
|
1911
|
+
f"({metrics.get('process_current')}/{metrics.get('process_limit')} used via "
|
|
1912
|
+
f"{metrics.get('process_limit_source')})"
|
|
1913
|
+
)
|
|
1914
|
+
memory_available_mb = metrics.get("memory_available_mb")
|
|
1915
|
+
if isinstance(memory_available_mb, int) and memory_available_mb < 256:
|
|
1916
|
+
return f"Only {memory_available_mb} MiB memory is available"
|
|
1917
|
+
return None
|
|
1918
|
+
|
|
1919
|
+
|
|
1772
1920
|
def copilot_auth_state_signature(copilot_home: str | Path) -> str | None:
|
|
1773
1921
|
"""Return a content signature for Copilot's persisted authentication state."""
|
|
1774
1922
|
config_file = Path(copilot_home) / "config.json"
|
|
@@ -1786,6 +1934,65 @@ def _copilot_auth_lock_path(copilot_home: str | Path) -> Path:
|
|
|
1786
1934
|
return home.with_name(f".{home.name}-auth-state.lock")
|
|
1787
1935
|
|
|
1788
1936
|
|
|
1937
|
+
def replace_copilot_auth_state(
|
|
1938
|
+
source_home: str | Path,
|
|
1939
|
+
destination_home: str | Path,
|
|
1940
|
+
expected_destination_signature: str | None = None,
|
|
1941
|
+
) -> bool:
|
|
1942
|
+
"""Replace an isolated Copilot auth file from a known login home."""
|
|
1943
|
+
source = Path(source_home) / "config.json"
|
|
1944
|
+
destination_dir = Path(destination_home)
|
|
1945
|
+
destination = destination_dir / source.name
|
|
1946
|
+
if not source.is_file():
|
|
1947
|
+
return False
|
|
1948
|
+
lock_file = None
|
|
1949
|
+
temp_name: str | None = None
|
|
1950
|
+
try:
|
|
1951
|
+
source_signature = copilot_auth_state_signature(source_home)
|
|
1952
|
+
if source_signature is None:
|
|
1953
|
+
return False
|
|
1954
|
+
destination_dir.mkdir(parents=True, exist_ok=True)
|
|
1955
|
+
lock_file = acquire_file_lock(
|
|
1956
|
+
_copilot_auth_lock_path(destination_dir),
|
|
1957
|
+
blocking=True,
|
|
1958
|
+
)
|
|
1959
|
+
if lock_file is None:
|
|
1960
|
+
logger.warning("Could not lock Copilot authentication state for restore")
|
|
1961
|
+
return False
|
|
1962
|
+
destination_signature = copilot_auth_state_signature(destination_home)
|
|
1963
|
+
if (
|
|
1964
|
+
expected_destination_signature is not None
|
|
1965
|
+
and destination_signature != expected_destination_signature
|
|
1966
|
+
):
|
|
1967
|
+
return False
|
|
1968
|
+
if destination_signature == source_signature:
|
|
1969
|
+
return False
|
|
1970
|
+
temp_fd, temp_name = tempfile.mkstemp(prefix=".config.json.", dir=destination_dir)
|
|
1971
|
+
os.close(temp_fd)
|
|
1972
|
+
shutil.copy2(source, temp_name)
|
|
1973
|
+
os.chmod(temp_name, 0o600)
|
|
1974
|
+
for attempt in range(3):
|
|
1975
|
+
try:
|
|
1976
|
+
os.replace(temp_name, destination)
|
|
1977
|
+
temp_name = None
|
|
1978
|
+
break
|
|
1979
|
+
except PermissionError:
|
|
1980
|
+
if sys.platform != "win32" or attempt == 2:
|
|
1981
|
+
raise
|
|
1982
|
+
time.sleep(0.05)
|
|
1983
|
+
return True
|
|
1984
|
+
except OSError:
|
|
1985
|
+
logger.warning("Could not restore interactive Copilot credentials", exc_info=True)
|
|
1986
|
+
return False
|
|
1987
|
+
finally:
|
|
1988
|
+
if temp_name is not None:
|
|
1989
|
+
try:
|
|
1990
|
+
Path(temp_name).unlink(missing_ok=True)
|
|
1991
|
+
except OSError:
|
|
1992
|
+
logger.debug("Could not remove temporary Copilot config", exc_info=True)
|
|
1993
|
+
release_file_lock(lock_file)
|
|
1994
|
+
|
|
1995
|
+
|
|
1789
1996
|
def persist_copilot_auth_state(
|
|
1790
1997
|
run_home: str | Path,
|
|
1791
1998
|
persistent_home: str | Path,
|
|
@@ -1821,6 +2028,15 @@ def persist_copilot_auth_state(
|
|
|
1821
2028
|
destination_signature = copilot_auth_state_signature(destination_dir)
|
|
1822
2029
|
if destination_signature == source_signature:
|
|
1823
2030
|
return
|
|
2031
|
+
if (
|
|
2032
|
+
baseline_signature is not None
|
|
2033
|
+
and destination_signature != baseline_signature
|
|
2034
|
+
):
|
|
2035
|
+
logger.info(
|
|
2036
|
+
"Skipping stale Copilot authentication update because another "
|
|
2037
|
+
"run refreshed the persistent state first"
|
|
2038
|
+
)
|
|
2039
|
+
return
|
|
1824
2040
|
if baseline_signature is None and destination.is_file():
|
|
1825
2041
|
if destination.stat().st_mtime_ns >= source.stat().st_mtime_ns:
|
|
1826
2042
|
return
|
|
@@ -1867,7 +2083,7 @@ def prepare_copilot_home() -> str:
|
|
|
1867
2083
|
authentication state written by isolated runs is retained even when it
|
|
1868
2084
|
has no counterpart in the interactive home.
|
|
1869
2085
|
"""
|
|
1870
|
-
source_root =
|
|
2086
|
+
source_root = copilot_interactive_home()
|
|
1871
2087
|
target_root = Path.home() / ".forgexa" / "copilot-home"
|
|
1872
2088
|
lock_file = None
|
|
1873
2089
|
try:
|
|
@@ -1879,6 +2095,8 @@ def prepare_copilot_home() -> str:
|
|
|
1879
2095
|
if lock_file is None:
|
|
1880
2096
|
logger.warning("Could not lock Copilot home for synchronization")
|
|
1881
2097
|
return str(target_root)
|
|
2098
|
+
if source_root.resolve() == target_root.resolve():
|
|
2099
|
+
return str(target_root)
|
|
1882
2100
|
if not source_root.exists():
|
|
1883
2101
|
return str(target_root)
|
|
1884
2102
|
|
|
@@ -1920,6 +2138,20 @@ def prepare_copilot_home() -> str:
|
|
|
1920
2138
|
migration_marker.touch(exist_ok=True)
|
|
1921
2139
|
os.chmod(migration_marker, 0o600)
|
|
1922
2140
|
|
|
2141
|
+
source_auth_signature = copilot_auth_state_signature(source_root)
|
|
2142
|
+
source_signature_marker = target_root.parent / ".copilot-login-source.sha256"
|
|
2143
|
+
recorded_source_signature: str | None = None
|
|
2144
|
+
if source_signature_marker.is_file():
|
|
2145
|
+
try:
|
|
2146
|
+
recorded_source_signature = source_signature_marker.read_text().strip()
|
|
2147
|
+
except OSError:
|
|
2148
|
+
logger.debug("Could not read Copilot login source signature", exc_info=True)
|
|
2149
|
+
source_auth_changed = bool(
|
|
2150
|
+
source_auth_signature
|
|
2151
|
+
and recorded_source_signature is not None
|
|
2152
|
+
and source_auth_signature != recorded_source_signature
|
|
2153
|
+
)
|
|
2154
|
+
source_config_synced = False
|
|
1923
2155
|
for child in source_root.iterdir():
|
|
1924
2156
|
if child.name in {"logs", "session-state"}:
|
|
1925
2157
|
continue
|
|
@@ -1937,11 +2169,15 @@ def prepare_copilot_home() -> str:
|
|
|
1937
2169
|
elif child.is_file():
|
|
1938
2170
|
if (
|
|
1939
2171
|
child.name == "config.json"
|
|
2172
|
+
and not source_auth_changed
|
|
1940
2173
|
and dest.is_file()
|
|
1941
2174
|
and dest.stat().st_mtime_ns >= child.stat().st_mtime_ns
|
|
1942
2175
|
):
|
|
2176
|
+
source_config_synced = True
|
|
1943
2177
|
continue
|
|
1944
2178
|
shutil.copy2(child, dest)
|
|
2179
|
+
if child.name == "config.json":
|
|
2180
|
+
source_config_synced = True
|
|
1945
2181
|
except OSError as exc:
|
|
1946
2182
|
logger.debug(
|
|
1947
2183
|
"Copilot home mirror skipped %s -> %s: %s",
|
|
@@ -1949,6 +2185,12 @@ def prepare_copilot_home() -> str:
|
|
|
1949
2185
|
dest,
|
|
1950
2186
|
exc,
|
|
1951
2187
|
)
|
|
2188
|
+
if source_config_synced and source_auth_signature:
|
|
2189
|
+
try:
|
|
2190
|
+
source_signature_marker.write_text(source_auth_signature)
|
|
2191
|
+
os.chmod(source_signature_marker, 0o600)
|
|
2192
|
+
except OSError:
|
|
2193
|
+
logger.debug("Could not record Copilot login source signature", exc_info=True)
|
|
1952
2194
|
except OSError:
|
|
1953
2195
|
logger.warning("Could not prepare writable Copilot home", exc_info=True)
|
|
1954
2196
|
finally:
|
|
@@ -910,7 +910,7 @@ except (ImportError, ModuleNotFoundError):
|
|
|
910
910
|
# DAEMON_VERSION is the protocol/logic version of the daemon code.
|
|
911
911
|
# Kept in sync with pyproject.toml version via bump-version.sh.
|
|
912
912
|
# CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
|
|
913
|
-
DAEMON_VERSION = "1.
|
|
913
|
+
DAEMON_VERSION = "1.43.3"
|
|
914
914
|
|
|
915
915
|
|
|
916
916
|
def _detect_client_type() -> str:
|
|
@@ -4781,7 +4781,12 @@ class ProcessManager:
|
|
|
4781
4781
|
"error during sign-in",
|
|
4782
4782
|
"websockettransporterror",
|
|
4783
4783
|
"websocket receive failed",
|
|
4784
|
-
"
|
|
4784
|
+
"misdirected request",
|
|
4785
|
+
"could not retrieve the list of available models",
|
|
4786
|
+
"can't spawn worker thread",
|
|
4787
|
+
"failed to load ca certificates off thread",
|
|
4788
|
+
"resource temporarily unavailable (os error 11)",
|
|
4789
|
+
"pthread_create failed",
|
|
4785
4790
|
"failed to initialize mcp client",
|
|
4786
4791
|
# "api error" removed: too broad — matches agent-generated code/output
|
|
4787
4792
|
# discussing API errors. Real API transport errors are covered by the
|
|
@@ -5076,6 +5081,38 @@ class ProcessManager:
|
|
|
5076
5081
|
on_chunk: Any = None,
|
|
5077
5082
|
) -> TaskResult:
|
|
5078
5083
|
"""Execute an agent CLI and collect results."""
|
|
5084
|
+
resource_metrics = agent_core.collect_runtime_resource_metrics()
|
|
5085
|
+
resource_reason = agent_core.runtime_resource_exhaustion_reason(
|
|
5086
|
+
resource_metrics,
|
|
5087
|
+
)
|
|
5088
|
+
if resource_reason:
|
|
5089
|
+
check = {
|
|
5090
|
+
"id": "process_headroom",
|
|
5091
|
+
"status": "fail",
|
|
5092
|
+
"blocking": True,
|
|
5093
|
+
"failure_class": "resource_exhausted",
|
|
5094
|
+
"detail": resource_reason,
|
|
5095
|
+
"remediation": (
|
|
5096
|
+
"Stop orphaned agent processes or raise the user/cgroup PID limit, "
|
|
5097
|
+
"then retry"
|
|
5098
|
+
),
|
|
5099
|
+
}
|
|
5100
|
+
logger.error(
|
|
5101
|
+
"Refusing to launch agent '%s' for task %s: %s",
|
|
5102
|
+
agent.agent_id,
|
|
5103
|
+
task.task_id,
|
|
5104
|
+
resource_reason,
|
|
5105
|
+
)
|
|
5106
|
+
return TaskResult(
|
|
5107
|
+
status="failed",
|
|
5108
|
+
exit_code=-1,
|
|
5109
|
+
stdout="",
|
|
5110
|
+
stderr="",
|
|
5111
|
+
error=f"Runtime resource preflight failed: {resource_reason}",
|
|
5112
|
+
failure_code="runtime_preflight_failed",
|
|
5113
|
+
preflight={"scope": "runtime", "checks": [check]},
|
|
5114
|
+
)
|
|
5115
|
+
|
|
5079
5116
|
# Centralized, agent-agnostic version gate: AgentDiscovery.discover()
|
|
5080
5117
|
# already computed min_version_ok/version_warning against
|
|
5081
5118
|
# AGENT_REGISTRY[agent_id]["min_version"] (see _check_agent_min_version).
|
|
@@ -5996,7 +6033,9 @@ class ProcessManager:
|
|
|
5996
6033
|
*,
|
|
5997
6034
|
disable_builtin_mcps: bool = False,
|
|
5998
6035
|
_retry_without_effort: bool = False,
|
|
6036
|
+
_retry_without_token_env: bool = False,
|
|
5999
6037
|
_isolated_copilot_home: str | None = None,
|
|
6038
|
+
_auth_state: dict[str, bool] | None = None,
|
|
6000
6039
|
) -> TaskResult:
|
|
6001
6040
|
"""Run GitHub Copilot CLI in non-interactive JSON-streaming mode.
|
|
6002
6041
|
|
|
@@ -6032,6 +6071,8 @@ class ProcessManager:
|
|
|
6032
6071
|
# session-state persists from the prior attempt all todos are already
|
|
6033
6072
|
# "done" and the agent exits immediately without producing any output.
|
|
6034
6073
|
_is_outer_call = _isolated_copilot_home is None
|
|
6074
|
+
if _auth_state is None:
|
|
6075
|
+
_auth_state = {"persist": True}
|
|
6035
6076
|
_master_copilot_home: str | None = None
|
|
6036
6077
|
_copilot_auth_baseline: str | None = None
|
|
6037
6078
|
if _is_outer_call:
|
|
@@ -6059,10 +6100,13 @@ class ProcessManager:
|
|
|
6059
6100
|
os.environ.copy(),
|
|
6060
6101
|
copilot_home=_isolated_copilot_home,
|
|
6061
6102
|
)
|
|
6103
|
+
if _retry_without_token_env:
|
|
6104
|
+
for key in agent_core.COPILOT_TOKEN_ENV_VARS:
|
|
6105
|
+
env.pop(key, None)
|
|
6062
6106
|
auth_env_var = next(
|
|
6063
6107
|
(
|
|
6064
6108
|
key
|
|
6065
|
-
for key in
|
|
6109
|
+
for key in agent_core.COPILOT_TOKEN_ENV_VARS
|
|
6066
6110
|
if env.get(key)
|
|
6067
6111
|
),
|
|
6068
6112
|
None,
|
|
@@ -6230,11 +6274,58 @@ class ProcessManager:
|
|
|
6230
6274
|
on_chunk,
|
|
6231
6275
|
disable_builtin_mcps=True,
|
|
6232
6276
|
_retry_without_effort=_retry_without_effort,
|
|
6277
|
+
_retry_without_token_env=_retry_without_token_env,
|
|
6233
6278
|
_isolated_copilot_home=_isolated_copilot_home,
|
|
6279
|
+
_auth_state=_auth_state,
|
|
6234
6280
|
)
|
|
6235
6281
|
retry_result.metrics["copilot_builtin_mcp_fallback"] = True
|
|
6236
6282
|
return retry_result
|
|
6237
6283
|
|
|
6284
|
+
model_discovery_error = agent_core.is_copilot_model_discovery_auth_error(
|
|
6285
|
+
*structured_errors,
|
|
6286
|
+
stderr,
|
|
6287
|
+
)
|
|
6288
|
+
if model_discovery_error and not self._copilot_called_any_tools(stdout):
|
|
6289
|
+
isolated_config = Path(_isolated_copilot_home) / "config.json"
|
|
6290
|
+
recovery_home = agent_core.copilot_recovery_home(
|
|
6291
|
+
_isolated_copilot_home,
|
|
6292
|
+
)
|
|
6293
|
+
can_retry_native_auth = bool(
|
|
6294
|
+
isolated_config.is_file()
|
|
6295
|
+
or recovery_home is not None
|
|
6296
|
+
)
|
|
6297
|
+
if not _retry_without_token_env and can_retry_native_auth:
|
|
6298
|
+
restored_interactive_auth = bool(
|
|
6299
|
+
recovery_home
|
|
6300
|
+
and await asyncio.to_thread(
|
|
6301
|
+
agent_core.replace_copilot_auth_state,
|
|
6302
|
+
recovery_home,
|
|
6303
|
+
_isolated_copilot_home,
|
|
6304
|
+
)
|
|
6305
|
+
)
|
|
6306
|
+
logger.warning(
|
|
6307
|
+
"Copilot model discovery failed with HTTP 421 before work for "
|
|
6308
|
+
"task %s; retrying once with token environment variables removed%s",
|
|
6309
|
+
task_id,
|
|
6310
|
+
" and interactive login restored" if restored_interactive_auth else "",
|
|
6311
|
+
)
|
|
6312
|
+
retry_result = await self._run_copilot(
|
|
6313
|
+
agent,
|
|
6314
|
+
prompt,
|
|
6315
|
+
cwd,
|
|
6316
|
+
timeout,
|
|
6317
|
+
task_id,
|
|
6318
|
+
on_chunk,
|
|
6319
|
+
disable_builtin_mcps=disable_builtin_mcps,
|
|
6320
|
+
_retry_without_effort=_retry_without_effort,
|
|
6321
|
+
_retry_without_token_env=True,
|
|
6322
|
+
_isolated_copilot_home=_isolated_copilot_home,
|
|
6323
|
+
_auth_state=_auth_state,
|
|
6324
|
+
)
|
|
6325
|
+
retry_result.metrics["copilot_model_discovery_recovery"] = True
|
|
6326
|
+
return retry_result
|
|
6327
|
+
_auth_state["persist"] = False
|
|
6328
|
+
|
|
6238
6329
|
if copilot_exit == 0 and returncode == 0:
|
|
6239
6330
|
return TaskResult(
|
|
6240
6331
|
status="success",
|
|
@@ -6290,7 +6381,9 @@ class ProcessManager:
|
|
|
6290
6381
|
agent, prompt, cwd, timeout, task_id, on_chunk,
|
|
6291
6382
|
disable_builtin_mcps=disable_builtin_mcps,
|
|
6292
6383
|
_retry_without_effort=True,
|
|
6384
|
+
_retry_without_token_env=_retry_without_token_env,
|
|
6293
6385
|
_isolated_copilot_home=_isolated_copilot_home,
|
|
6386
|
+
_auth_state=_auth_state,
|
|
6294
6387
|
)
|
|
6295
6388
|
# Exhausted retries — emit an actionable diagnostic hint
|
|
6296
6389
|
if effective_rc == 1 and _no_tools and not copilot_specific_error:
|
|
@@ -6342,6 +6435,14 @@ class ProcessManager:
|
|
|
6342
6435
|
stdout=stdout[-settings.AGENT_MAX_OUTPUT_SIZE:],
|
|
6343
6436
|
stderr=stderr[-10000:],
|
|
6344
6437
|
error=failure_error,
|
|
6438
|
+
failure_code=(
|
|
6439
|
+
"runtime_resource_exhausted"
|
|
6440
|
+
if agent_core.is_runtime_resource_exhaustion_error(
|
|
6441
|
+
failure_error,
|
|
6442
|
+
stderr,
|
|
6443
|
+
)
|
|
6444
|
+
else ""
|
|
6445
|
+
),
|
|
6345
6446
|
metrics=metrics,
|
|
6346
6447
|
)
|
|
6347
6448
|
except asyncio.CancelledError:
|
|
@@ -6390,7 +6491,7 @@ class ProcessManager:
|
|
|
6390
6491
|
finally:
|
|
6391
6492
|
self.active_processes.pop(task_id, None)
|
|
6392
6493
|
if _is_outer_call and _isolated_copilot_home:
|
|
6393
|
-
if _master_copilot_home:
|
|
6494
|
+
if _master_copilot_home and _auth_state["persist"]:
|
|
6394
6495
|
await asyncio.to_thread(
|
|
6395
6496
|
agent_core.persist_copilot_auth_state,
|
|
6396
6497
|
_isolated_copilot_home,
|
|
@@ -6691,26 +6792,11 @@ class ProcessManager:
|
|
|
6691
6792
|
async def cancel(self, task_id: str):
|
|
6692
6793
|
proc = self.active_processes.pop(task_id, None)
|
|
6693
6794
|
if proc:
|
|
6795
|
+
_kill_proc(proc)
|
|
6694
6796
|
try:
|
|
6695
|
-
|
|
6696
|
-
import signal as _signal
|
|
6697
|
-
try:
|
|
6698
|
-
os.killpg(os.getpgid(proc.pid), _signal.SIGKILL)
|
|
6699
|
-
except (ProcessLookupError, PermissionError, OSError):
|
|
6700
|
-
pass
|
|
6701
|
-
else:
|
|
6702
|
-
import subprocess as _subprocess
|
|
6703
|
-
_subprocess.run(
|
|
6704
|
-
["taskkill", "/F", "/T", "/PID", str(proc.pid)],
|
|
6705
|
-
capture_output=True,
|
|
6706
|
-
)
|
|
6797
|
+
await asyncio.wait_for(proc.wait(), timeout=5)
|
|
6707
6798
|
except Exception:
|
|
6708
|
-
|
|
6709
|
-
finally:
|
|
6710
|
-
try:
|
|
6711
|
-
proc.kill()
|
|
6712
|
-
except Exception:
|
|
6713
|
-
pass
|
|
6799
|
+
logger.warning("Timed out reaping agent process for task %s", task_id)
|
|
6714
6800
|
|
|
6715
6801
|
|
|
6716
6802
|
# ── Progress Reporter ──
|
|
@@ -6910,8 +6996,48 @@ class RuntimePreflightMonitor:
|
|
|
6910
6996
|
await self._check_git_executable(),
|
|
6911
6997
|
await self._check_workspace_root(),
|
|
6912
6998
|
await self._check_workspace_disk(),
|
|
6999
|
+
await self._check_process_headroom(),
|
|
6913
7000
|
]
|
|
6914
7001
|
|
|
7002
|
+
@staticmethod
|
|
7003
|
+
async def _check_process_headroom() -> dict:
|
|
7004
|
+
metrics = agent_core.collect_runtime_resource_metrics()
|
|
7005
|
+
reason = agent_core.runtime_resource_exhaustion_reason(metrics)
|
|
7006
|
+
if reason:
|
|
7007
|
+
return {
|
|
7008
|
+
"id": "process_headroom",
|
|
7009
|
+
"status": "fail",
|
|
7010
|
+
"blocking": True,
|
|
7011
|
+
"failure_class": "resource_exhausted",
|
|
7012
|
+
"detail": reason,
|
|
7013
|
+
"remediation": (
|
|
7014
|
+
"Stop orphaned agent processes or raise the user/cgroup PID limit, "
|
|
7015
|
+
"then restart the daemon"
|
|
7016
|
+
),
|
|
7017
|
+
}
|
|
7018
|
+
if metrics:
|
|
7019
|
+
detail = (
|
|
7020
|
+
f"{metrics['process_remaining']} process/thread slots remain"
|
|
7021
|
+
if "process_remaining" in metrics
|
|
7022
|
+
else f"{metrics.get('memory_available_mb', 'unknown')} MiB memory available"
|
|
7023
|
+
)
|
|
7024
|
+
return {
|
|
7025
|
+
"id": "process_headroom",
|
|
7026
|
+
"status": "pass",
|
|
7027
|
+
"blocking": True,
|
|
7028
|
+
"failure_class": "",
|
|
7029
|
+
"detail": detail,
|
|
7030
|
+
"remediation": "",
|
|
7031
|
+
}
|
|
7032
|
+
return {
|
|
7033
|
+
"id": "process_headroom",
|
|
7034
|
+
"status": "unknown",
|
|
7035
|
+
"blocking": False,
|
|
7036
|
+
"failure_class": "",
|
|
7037
|
+
"detail": "Process/thread headroom is unavailable on this platform",
|
|
7038
|
+
"remediation": "",
|
|
7039
|
+
}
|
|
7040
|
+
|
|
6915
7041
|
@staticmethod
|
|
6916
7042
|
async def _run_command_timeout(cmd: list[str], timeout: float = 5) -> str | None:
|
|
6917
7043
|
"""Run a command with a deadline. Returns stdout on success, None on timeout/error.
|
|
@@ -7216,15 +7342,17 @@ class HeartbeatService:
|
|
|
7216
7342
|
|
|
7217
7343
|
def _collect_system_metrics(self) -> dict:
|
|
7218
7344
|
"""Basic system metrics."""
|
|
7345
|
+
metrics = agent_core.collect_runtime_resource_metrics()
|
|
7219
7346
|
try:
|
|
7220
7347
|
import psutil
|
|
7221
|
-
|
|
7348
|
+
metrics.update({
|
|
7222
7349
|
"cpu_percent": psutil.cpu_percent(),
|
|
7223
7350
|
"memory_percent": psutil.virtual_memory().percent,
|
|
7224
7351
|
"disk_percent": psutil.disk_usage("/").percent,
|
|
7225
|
-
}
|
|
7352
|
+
})
|
|
7226
7353
|
except ImportError:
|
|
7227
|
-
|
|
7354
|
+
pass
|
|
7355
|
+
return metrics
|
|
7228
7356
|
|
|
7229
7357
|
|
|
7230
7358
|
# ── Log Uploader ──
|
|
@@ -8562,6 +8690,12 @@ class RuntimeDaemon:
|
|
|
8562
8690
|
"""Main entry point."""
|
|
8563
8691
|
# Prevent multiple daemon instances on the same machine
|
|
8564
8692
|
self._acquire_lock()
|
|
8693
|
+
cleaned_processes = self._cleanup_stale_agent_processes()
|
|
8694
|
+
if cleaned_processes:
|
|
8695
|
+
logger.warning(
|
|
8696
|
+
"Cleaned up %d orphaned Forgexa agent process group(s)",
|
|
8697
|
+
cleaned_processes,
|
|
8698
|
+
)
|
|
8565
8699
|
|
|
8566
8700
|
# Mint a local-dev token if no token is available yet
|
|
8567
8701
|
if not self.api_token and not self._has_stored_credentials():
|
|
@@ -9271,7 +9405,15 @@ class RuntimeDaemon:
|
|
|
9271
9405
|
# Guard: if the agent already produced file changes in the workspace, it DID
|
|
9272
9406
|
# meaningful work — don't trigger fallback even if it crashed after completing.
|
|
9273
9407
|
# Let the recovery logic (step 4.1) handle non-zero exit with committed work.
|
|
9274
|
-
|
|
9408
|
+
runtime_resource_failure = result.failure_code in {
|
|
9409
|
+
"runtime_preflight_failed",
|
|
9410
|
+
"runtime_resource_exhausted",
|
|
9411
|
+
}
|
|
9412
|
+
agent_failure_kind = (
|
|
9413
|
+
None
|
|
9414
|
+
if runtime_resource_failure
|
|
9415
|
+
else ProcessManager.agent_failure_kind(result)
|
|
9416
|
+
)
|
|
9275
9417
|
is_agent_fallback_eligible = agent_failure_kind is not None
|
|
9276
9418
|
is_silent_idle_timeout = self.process_manager.is_silent_idle_timeout(result)
|
|
9277
9419
|
fallback_reason = (
|
|
@@ -11408,7 +11550,13 @@ class RuntimeDaemon:
|
|
|
11408
11550
|
# try the next available agent before giving up. An explicit
|
|
11409
11551
|
# override is deliberately a single-agent attempt.
|
|
11410
11552
|
fallback_scope = "override" if agent_override else "auto"
|
|
11411
|
-
allow_fallback =
|
|
11553
|
+
allow_fallback = (
|
|
11554
|
+
not bool(agent_override)
|
|
11555
|
+
and result.failure_code not in {
|
|
11556
|
+
"runtime_preflight_failed",
|
|
11557
|
+
"runtime_resource_exhausted",
|
|
11558
|
+
}
|
|
11559
|
+
)
|
|
11412
11560
|
generation_attempts: list[dict] = []
|
|
11413
11561
|
agent_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
11414
11562
|
agent_failure_label = (
|
|
@@ -14024,15 +14172,78 @@ class RuntimeDaemon:
|
|
|
14024
14172
|
except Exception as e:
|
|
14025
14173
|
logger.debug("Could not clean temp dir %s: %s", d, e)
|
|
14026
14174
|
|
|
14175
|
+
@staticmethod
|
|
14176
|
+
def _cleanup_stale_agent_processes(
|
|
14177
|
+
*,
|
|
14178
|
+
proc_root: str | Path = "/proc",
|
|
14179
|
+
temp_root: str | Path | None = None,
|
|
14180
|
+
) -> int:
|
|
14181
|
+
"""Kill orphaned agent groups identifiable as Forgexa-owned on Linux."""
|
|
14182
|
+
if not sys.platform.startswith("linux"):
|
|
14183
|
+
return 0
|
|
14184
|
+
import signal as _signal
|
|
14185
|
+
import tempfile
|
|
14186
|
+
|
|
14187
|
+
root = Path(proc_root)
|
|
14188
|
+
temp = Path(temp_root or tempfile.gettempdir()).resolve()
|
|
14189
|
+
current_pid = os.getpid()
|
|
14190
|
+
current_group = os.getpgrp()
|
|
14191
|
+
cleaned = 0
|
|
14192
|
+
try:
|
|
14193
|
+
process_dirs = sorted(
|
|
14194
|
+
(path for path in root.iterdir() if path.name.isdigit()),
|
|
14195
|
+
key=lambda path: int(path.name),
|
|
14196
|
+
)
|
|
14197
|
+
except OSError:
|
|
14198
|
+
return 0
|
|
14199
|
+
|
|
14200
|
+
for process_dir in process_dirs:
|
|
14201
|
+
pid = int(process_dir.name)
|
|
14202
|
+
if pid == current_pid:
|
|
14203
|
+
continue
|
|
14204
|
+
try:
|
|
14205
|
+
entries = (process_dir / "environ").read_bytes().split(b"\0")
|
|
14206
|
+
environment = {}
|
|
14207
|
+
for entry in entries:
|
|
14208
|
+
if b"=" not in entry:
|
|
14209
|
+
continue
|
|
14210
|
+
key, value = entry.split(b"=", 1)
|
|
14211
|
+
environment[key.decode(errors="ignore")] = value.decode(errors="ignore")
|
|
14212
|
+
marked = environment.get("FORGEXA_AGENT_PROCESS") == "1"
|
|
14213
|
+
isolated_home = environment.get("COPILOT_HOME", "")
|
|
14214
|
+
legacy_copilot = False
|
|
14215
|
+
if isolated_home:
|
|
14216
|
+
isolated_path = Path(isolated_home).resolve()
|
|
14217
|
+
legacy_copilot = (
|
|
14218
|
+
isolated_path.parent == temp
|
|
14219
|
+
and isolated_path.name.startswith("copilot-")
|
|
14220
|
+
)
|
|
14221
|
+
if not (marked or legacy_copilot):
|
|
14222
|
+
continue
|
|
14223
|
+
process_group = os.getpgid(pid)
|
|
14224
|
+
if process_group == current_group:
|
|
14225
|
+
continue
|
|
14226
|
+
os.killpg(process_group, _signal.SIGKILL)
|
|
14227
|
+
cleaned += 1
|
|
14228
|
+
except (OSError, ValueError):
|
|
14229
|
+
continue
|
|
14230
|
+
return cleaned
|
|
14231
|
+
|
|
14027
14232
|
async def _shutdown_gracefully(self):
|
|
14028
14233
|
"""Graceful shutdown."""
|
|
14029
14234
|
logger.info("Shutting down daemon...")
|
|
14030
14235
|
self._shutdown = True
|
|
14031
14236
|
|
|
14032
|
-
# Cancel
|
|
14033
|
-
|
|
14237
|
+
# Cancel task coroutines, then terminate and reap every tracked process.
|
|
14238
|
+
tasks = list(self.active_tasks.values())
|
|
14239
|
+
for task in tasks:
|
|
14034
14240
|
task.cancel()
|
|
14035
|
-
|
|
14241
|
+
for process_id in list(self.process_manager.active_processes):
|
|
14242
|
+
await self.process_manager.cancel(process_id)
|
|
14243
|
+
if tasks:
|
|
14244
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
14245
|
+
self.active_tasks.clear()
|
|
14246
|
+
self._task_connections.clear()
|
|
14036
14247
|
|
|
14037
14248
|
# Stop preflight monitor
|
|
14038
14249
|
await self.preflight_monitor.stop()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|