forgexa-cli 1.41.4__tar.gz → 1.42.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/PKG-INFO +1 -1
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli/__init__.py +1 -1
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli/daemon.py +246 -73
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli.egg-info/PKG-INFO +1 -1
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/pyproject.toml +1 -1
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/README.md +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli/_build_config.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli/_local_bind.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli/agent_core.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli/autoupgrade.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli/main.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli/py.typed +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli.egg-info/SOURCES.txt +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli.egg-info/dependency_links.txt +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli.egg-info/entry_points.txt +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli.egg-info/requires.txt +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/forgexa_cli.egg-info/top_level.txt +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/setup.cfg +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_auth_and_runtime_commands.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_autoupgrade.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_check_command.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_expiry_warnings_and_revoke.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_local_bind_commands.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_runtime_credentials.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_session_credentials.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_silent_install.py +0 -0
- {forgexa_cli-1.41.4 → forgexa_cli-1.42.0}/tests/test_upgrade_observability.py +0 -0
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""forgexa-cli — Forgexa command-line client."""
|
|
2
|
-
__version__ = "1.
|
|
2
|
+
__version__ = "1.42.0"
|
|
@@ -910,7 +910,7 @@ except (ImportError, ModuleNotFoundError):
|
|
|
910
910
|
# DAEMON_VERSION is the protocol/logic version of the daemon code.
|
|
911
911
|
# Kept in sync with pyproject.toml version via bump-version.sh.
|
|
912
912
|
# CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
|
|
913
|
-
DAEMON_VERSION = "1.
|
|
913
|
+
DAEMON_VERSION = "1.42.0"
|
|
914
914
|
|
|
915
915
|
|
|
916
916
|
def _detect_client_type() -> str:
|
|
@@ -1234,8 +1234,9 @@ class TaskResult:
|
|
|
1234
1234
|
error: str = ""
|
|
1235
1235
|
# failure_code is forwarded to the server to drive retry policy.
|
|
1236
1236
|
# Key values:
|
|
1237
|
-
# "all_agents_rate_limited" — daemon tried
|
|
1238
|
-
#
|
|
1237
|
+
# "all_agents_rate_limited" / "all_agents_unavailable" — daemon tried
|
|
1238
|
+
# every installed agent and none can currently run. Server must NOT
|
|
1239
|
+
# retry on the same runtime.
|
|
1239
1240
|
# "validation_retry_exhausted" — required-output validation still failed
|
|
1240
1241
|
# after the allowed repair agents. Server must not auto-retry it.
|
|
1241
1242
|
failure_code: str = ""
|
|
@@ -4728,6 +4729,7 @@ class ProcessManager:
|
|
|
4728
4729
|
"rate limit",
|
|
4729
4730
|
"rate_limit",
|
|
4730
4731
|
"quota exceeded",
|
|
4732
|
+
"quota exhausted",
|
|
4731
4733
|
"monthly quota", # Copilot: "You have exceeded your monthly quota"
|
|
4732
4734
|
"too many requests",
|
|
4733
4735
|
"overloaded",
|
|
@@ -4749,6 +4751,8 @@ class ProcessManager:
|
|
|
4749
4751
|
"name or service not known",
|
|
4750
4752
|
"no such host",
|
|
4751
4753
|
"network is unreachable",
|
|
4754
|
+
"oauth/access_token",
|
|
4755
|
+
"error during sign-in",
|
|
4752
4756
|
"websockettransporterror",
|
|
4753
4757
|
"websocket receive failed",
|
|
4754
4758
|
"421 misdirected request",
|
|
@@ -4881,25 +4885,43 @@ class ProcessManager:
|
|
|
4881
4885
|
"""Implementation lives in agent_core._copilot_called_any_tools."""
|
|
4882
4886
|
return _copilot_called_any_tools(stdout)
|
|
4883
4887
|
|
|
4888
|
+
@staticmethod
|
|
4889
|
+
def agent_failure_kind(result: "TaskResult") -> str | None:
|
|
4890
|
+
"""Classify a fallback-eligible failure without conflating its cause."""
|
|
4891
|
+
if result.failure_code in {"all_agents_rate_limited", "all_agents_unavailable"}:
|
|
4892
|
+
return None
|
|
4893
|
+
error_text = ProcessManager._failure_pattern_channels(result).lower()
|
|
4894
|
+
if any(p in error_text for p in ProcessManager.RATE_LIMIT_PATTERNS):
|
|
4895
|
+
return "rate_limit"
|
|
4896
|
+
if any(p in error_text for p in ProcessManager.AGENT_UNAVAILABLE_PATTERNS):
|
|
4897
|
+
return "agent_unavailable"
|
|
4898
|
+
return None
|
|
4899
|
+
|
|
4884
4900
|
@staticmethod
|
|
4885
4901
|
def is_rate_limited(result: "TaskResult") -> bool:
|
|
4886
|
-
"""
|
|
4902
|
+
"""Return whether an agent actually hit a rate or usage limit."""
|
|
4903
|
+
return ProcessManager.agent_failure_kind(result) == "rate_limit"
|
|
4887
4904
|
|
|
4888
|
-
|
|
4889
|
-
|
|
4905
|
+
@staticmethod
|
|
4906
|
+
def is_agent_fallback_eligible(result: "TaskResult") -> bool:
|
|
4907
|
+
"""Return whether another agent may recover this agent-side failure."""
|
|
4908
|
+
return ProcessManager.agent_failure_kind(result) is not None
|
|
4890
4909
|
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
|
|
4897
|
-
|
|
4898
|
-
|
|
4899
|
-
|
|
4900
|
-
|
|
4901
|
-
|
|
4902
|
-
|
|
4910
|
+
@staticmethod
|
|
4911
|
+
def all_agents_failure_code(failure_kinds: list[str]) -> str:
|
|
4912
|
+
"""Return the terminal code for exhausted fallback-eligible agents."""
|
|
4913
|
+
if failure_kinds and all(kind == "rate_limit" for kind in failure_kinds):
|
|
4914
|
+
return "all_agents_rate_limited"
|
|
4915
|
+
return "all_agents_unavailable"
|
|
4916
|
+
|
|
4917
|
+
@staticmethod
|
|
4918
|
+
def all_agents_failure_label(failure_kinds: list[str]) -> str:
|
|
4919
|
+
"""Return an accurate human-readable summary for exhausted agents."""
|
|
4920
|
+
if failure_kinds and all(kind == "rate_limit" for kind in failure_kinds):
|
|
4921
|
+
return "rate-limited"
|
|
4922
|
+
if failure_kinds and all(kind == "agent_unavailable" for kind in failure_kinds):
|
|
4923
|
+
return "unavailable"
|
|
4924
|
+
return "unable to run"
|
|
4903
4925
|
|
|
4904
4926
|
@staticmethod
|
|
4905
4927
|
def is_silent_idle_timeout(result: "TaskResult") -> bool:
|
|
@@ -5654,6 +5676,22 @@ class ProcessManager:
|
|
|
5654
5676
|
result.metrics.update(parsed_metrics)
|
|
5655
5677
|
return result
|
|
5656
5678
|
|
|
5679
|
+
@staticmethod
|
|
5680
|
+
def _opencode_suggested_model(output: str) -> str | None:
|
|
5681
|
+
"""Extract OpenCode's first replacement model from a missing-model error."""
|
|
5682
|
+
match = re.search(
|
|
5683
|
+
r"model not found\s*:\s*(.+?)(?:[.?!]\s+|\s+)did you mean:\s*"
|
|
5684
|
+
r"([A-Za-z0-9][A-Za-z0-9._:/-]*)",
|
|
5685
|
+
output or "",
|
|
5686
|
+
re.IGNORECASE,
|
|
5687
|
+
)
|
|
5688
|
+
if not match:
|
|
5689
|
+
return None
|
|
5690
|
+
missing_model, suggested_model = match.groups()
|
|
5691
|
+
if missing_model.strip().lower() == suggested_model.lower():
|
|
5692
|
+
return None
|
|
5693
|
+
return suggested_model
|
|
5694
|
+
|
|
5657
5695
|
async def _run_opencode(
|
|
5658
5696
|
self, agent: DiscoveredAgent, prompt: str, cwd: Path, timeout: int, task_id: str,
|
|
5659
5697
|
on_chunk: Any = None,
|
|
@@ -5678,7 +5716,7 @@ class ProcessManager:
|
|
|
5678
5716
|
No need to re-check it here.
|
|
5679
5717
|
"""
|
|
5680
5718
|
# Apply model override if configured (e.g. FACTORY_OPENCODE_MODEL=copilot/gpt-4.1)
|
|
5681
|
-
model_override = os.environ.get("FACTORY_OPENCODE_MODEL")
|
|
5719
|
+
model_override = os.environ.get("FACTORY_OPENCODE_MODEL", "").strip() or None
|
|
5682
5720
|
# Isolate each opencode run in its own data directory to prevent
|
|
5683
5721
|
# concurrent processes from racing on the shared SQLite WAL file.
|
|
5684
5722
|
tmp_data_root = tempfile.mkdtemp(prefix=f"opencode-{task_id[:8]}-")
|
|
@@ -5702,23 +5740,43 @@ class ProcessManager:
|
|
|
5702
5740
|
output = "\n".join(
|
|
5703
5741
|
part for part in (result.stdout, result.stderr, result.error) if part
|
|
5704
5742
|
).lower()
|
|
5705
|
-
|
|
5743
|
+
suggested_model = self._opencode_suggested_model(output)
|
|
5744
|
+
retry_model: str | None = None
|
|
5745
|
+
recovery_metric = ""
|
|
5746
|
+
if (
|
|
5747
|
+
result.status == "failed"
|
|
5748
|
+
and model_override
|
|
5749
|
+
and "model not found" in output
|
|
5750
|
+
and model_override.lower() in output
|
|
5751
|
+
):
|
|
5752
|
+
recovery_metric = "opencode_model_override_fallback"
|
|
5706
5753
|
logger.warning(
|
|
5707
5754
|
"OpenCode model override %r is unavailable for task %s; retrying without it",
|
|
5708
5755
|
model_override,
|
|
5709
5756
|
task_id,
|
|
5710
5757
|
)
|
|
5758
|
+
elif result.status == "failed" and suggested_model:
|
|
5759
|
+
retry_model = suggested_model
|
|
5760
|
+
recovery_metric = "opencode_default_model_fallback"
|
|
5761
|
+
logger.warning(
|
|
5762
|
+
"OpenCode configured model is unavailable for task %s; retrying with %r",
|
|
5763
|
+
task_id,
|
|
5764
|
+
suggested_model,
|
|
5765
|
+
)
|
|
5766
|
+
|
|
5767
|
+
if recovery_metric:
|
|
5711
5768
|
retry_env = dict(env)
|
|
5712
5769
|
retry_env.pop("FACTORY_OPENCODE_MODEL", None)
|
|
5713
5770
|
cmd = agent_core.build_opencode_command(
|
|
5714
5771
|
agent.command,
|
|
5715
5772
|
cwd=cwd,
|
|
5773
|
+
model=retry_model,
|
|
5716
5774
|
prompt_file=prompt_file,
|
|
5717
5775
|
)
|
|
5718
5776
|
result = await self._run_cli(
|
|
5719
5777
|
cmd, cwd, timeout, task_id, on_chunk=on_chunk, env=retry_env,
|
|
5720
5778
|
)
|
|
5721
|
-
result.metrics[
|
|
5779
|
+
result.metrics[recovery_metric] = True
|
|
5722
5780
|
finally:
|
|
5723
5781
|
shutil.rmtree(tmp_data_root, ignore_errors=True)
|
|
5724
5782
|
|
|
@@ -5882,6 +5940,27 @@ class ProcessManager:
|
|
|
5882
5940
|
"""Implementation lives in agent_core.prepare_copilot_home."""
|
|
5883
5941
|
return agent_core.prepare_copilot_home()
|
|
5884
5942
|
|
|
5943
|
+
@staticmethod
|
|
5944
|
+
def _copilot_builtin_mcp_failure(stdout: str) -> str | None:
|
|
5945
|
+
"""Return a startup failure from Copilot's built-in GitHub MCP server."""
|
|
5946
|
+
for line in (stdout or "").splitlines():
|
|
5947
|
+
try:
|
|
5948
|
+
event = json.loads(line)
|
|
5949
|
+
except (TypeError, ValueError):
|
|
5950
|
+
continue
|
|
5951
|
+
if not isinstance(event, dict):
|
|
5952
|
+
continue
|
|
5953
|
+
if event.get("type") != "session.mcp_server_status_changed":
|
|
5954
|
+
continue
|
|
5955
|
+
data = event.get("data") or {}
|
|
5956
|
+
if not isinstance(data, dict):
|
|
5957
|
+
continue
|
|
5958
|
+
if data.get("serverName") != "github-mcp-server" or data.get("status") != "failed":
|
|
5959
|
+
continue
|
|
5960
|
+
message = data.get("error") or data.get("message") or ""
|
|
5961
|
+
return str(message).strip() or "GitHub MCP server failed to initialize"
|
|
5962
|
+
return None
|
|
5963
|
+
|
|
5885
5964
|
async def _run_copilot(
|
|
5886
5965
|
self, agent: DiscoveredAgent, prompt: str, cwd: Path, timeout: int, task_id: str,
|
|
5887
5966
|
on_chunk: Any = None,
|
|
@@ -6097,6 +6176,36 @@ class ProcessManager:
|
|
|
6097
6176
|
break
|
|
6098
6177
|
effective_rc = copilot_exit if copilot_exit is not None else returncode
|
|
6099
6178
|
|
|
6179
|
+
builtin_mcp_error = (
|
|
6180
|
+
self._copilot_builtin_mcp_failure(stdout)
|
|
6181
|
+
if not disable_builtin_mcps
|
|
6182
|
+
else None
|
|
6183
|
+
)
|
|
6184
|
+
if (
|
|
6185
|
+
builtin_mcp_error
|
|
6186
|
+
and not signals["has_meaningful_content"]
|
|
6187
|
+
and not self._copilot_called_any_tools(stdout)
|
|
6188
|
+
):
|
|
6189
|
+
logger.warning(
|
|
6190
|
+
"Copilot built-in GitHub MCP failed before work for task %s; "
|
|
6191
|
+
"retrying with built-in MCPs disabled: %s",
|
|
6192
|
+
task_id,
|
|
6193
|
+
builtin_mcp_error[:300],
|
|
6194
|
+
)
|
|
6195
|
+
retry_result = await self._run_copilot(
|
|
6196
|
+
agent,
|
|
6197
|
+
prompt,
|
|
6198
|
+
cwd,
|
|
6199
|
+
timeout,
|
|
6200
|
+
task_id,
|
|
6201
|
+
on_chunk,
|
|
6202
|
+
disable_builtin_mcps=True,
|
|
6203
|
+
_retry_without_effort=_retry_without_effort,
|
|
6204
|
+
_isolated_copilot_home=_isolated_copilot_home,
|
|
6205
|
+
)
|
|
6206
|
+
retry_result.metrics["copilot_builtin_mcp_fallback"] = True
|
|
6207
|
+
return retry_result
|
|
6208
|
+
|
|
6100
6209
|
if copilot_exit == 0 and returncode == 0:
|
|
6101
6210
|
return TaskResult(
|
|
6102
6211
|
status="success",
|
|
@@ -9111,15 +9220,24 @@ class RuntimeDaemon:
|
|
|
9111
9220
|
# Guard: if the agent already produced file changes in the workspace, it DID
|
|
9112
9221
|
# meaningful work — don't trigger fallback even if it crashed after completing.
|
|
9113
9222
|
# Let the recovery logic (step 4.1) handle non-zero exit with committed work.
|
|
9114
|
-
|
|
9223
|
+
agent_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
9224
|
+
is_agent_fallback_eligible = agent_failure_kind is not None
|
|
9115
9225
|
is_silent_idle_timeout = self.process_manager.is_silent_idle_timeout(result)
|
|
9116
9226
|
fallback_reason = (
|
|
9117
|
-
|
|
9118
|
-
if
|
|
9227
|
+
agent_failure_kind
|
|
9228
|
+
if is_agent_fallback_eligible
|
|
9119
9229
|
else "agent_idle_timeout"
|
|
9120
9230
|
if is_silent_idle_timeout
|
|
9121
9231
|
else ""
|
|
9122
9232
|
)
|
|
9233
|
+
fallback_label = (
|
|
9234
|
+
"rate-limited"
|
|
9235
|
+
if agent_failure_kind == "rate_limit"
|
|
9236
|
+
else "unavailable"
|
|
9237
|
+
if agent_failure_kind == "agent_unavailable"
|
|
9238
|
+
else "timed out"
|
|
9239
|
+
)
|
|
9240
|
+
fallback_failure_kinds = [agent_failure_kind or fallback_reason] if fallback_reason else []
|
|
9123
9241
|
_skip_fallback = False
|
|
9124
9242
|
if fallback_reason:
|
|
9125
9243
|
_pre_fallback_git = await self.process_manager._collect_git_info(workspace_path)
|
|
@@ -9160,7 +9278,7 @@ class RuntimeDaemon:
|
|
|
9160
9278
|
await reporter.report_progress(
|
|
9161
9279
|
task.task_id, 10,
|
|
9162
9280
|
f"agent_fallback: retrying with {agent.agent_id}",
|
|
9163
|
-
output_lines=[f"[daemon] Agent {
|
|
9281
|
+
output_lines=[f"[daemon] Agent {fallback_label}, switching to {agent.agent_id}"],
|
|
9164
9282
|
)
|
|
9165
9283
|
|
|
9166
9284
|
# Re-run with fallback agent
|
|
@@ -9210,27 +9328,39 @@ class RuntimeDaemon:
|
|
|
9210
9328
|
# A silent hang gets exactly one alternate-agent attempt.
|
|
9211
9329
|
# Unlike quota failures, do not serially spend the full idle
|
|
9212
9330
|
# timeout on every installed CLI when the host is offline.
|
|
9213
|
-
|
|
9331
|
+
fallback_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
9332
|
+
if fallback_failure_kind:
|
|
9333
|
+
fallback_failure_kinds.append(fallback_failure_kind)
|
|
9334
|
+
fallback_label = (
|
|
9335
|
+
"rate-limited"
|
|
9336
|
+
if fallback_failure_kind == "rate_limit"
|
|
9337
|
+
else "unavailable"
|
|
9338
|
+
)
|
|
9339
|
+
if not ProcessManager.is_agent_fallback_eligible(result):
|
|
9214
9340
|
break # Success or non-retriable failure
|
|
9215
9341
|
|
|
9216
9342
|
logger.warning(
|
|
9217
|
-
"Fallback agent '%s' also
|
|
9218
|
-
agent.agent_id,
|
|
9343
|
+
"Fallback agent '%s' also %s for task %s",
|
|
9344
|
+
agent.agent_id,
|
|
9345
|
+
"rate-limited" if fallback_failure_kind == "rate_limit" else "unavailable",
|
|
9346
|
+
task.task_id,
|
|
9219
9347
|
)
|
|
9220
9348
|
fallback_agent = self._select_fallback_agent(
|
|
9221
9349
|
agent.agent_id, task.fallback_chain, tried_agents
|
|
9222
9350
|
)
|
|
9223
9351
|
|
|
9224
|
-
if
|
|
9352
|
+
if ProcessManager.is_agent_fallback_eligible(result):
|
|
9353
|
+
failure_label = ProcessManager.all_agents_failure_label(
|
|
9354
|
+
fallback_failure_kinds,
|
|
9355
|
+
)
|
|
9225
9356
|
result.error = (
|
|
9226
|
-
f"All agents
|
|
9357
|
+
f"All agents {failure_label} (tried: {', '.join(tried_agents)}). "
|
|
9227
9358
|
f"Original error: {result.error}"
|
|
9228
9359
|
)
|
|
9229
9360
|
result.status = "failed"
|
|
9230
|
-
|
|
9231
|
-
|
|
9232
|
-
|
|
9233
|
-
result.failure_code = "all_agents_rate_limited"
|
|
9361
|
+
result.failure_code = ProcessManager.all_agents_failure_code(
|
|
9362
|
+
fallback_failure_kinds,
|
|
9363
|
+
)
|
|
9234
9364
|
|
|
9235
9365
|
# 4. Collect git info BEFORE commit (shows uncommitted changes)
|
|
9236
9366
|
pre_commit_git = await self.process_manager._collect_git_info(workspace_path)
|
|
@@ -9360,11 +9490,20 @@ class RuntimeDaemon:
|
|
|
9360
9490
|
# block (step 3) only checked the *initial* run; if that succeeded but
|
|
9361
9491
|
# the agent became rate-limited during a validation retry, we need a
|
|
9362
9492
|
# second fallback pass here so the task is attempted on a fresh agent.
|
|
9363
|
-
|
|
9493
|
+
post_validation_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
9494
|
+
post_validation_failure_kinds = (
|
|
9495
|
+
[post_validation_failure_kind] if post_validation_failure_kind else []
|
|
9496
|
+
)
|
|
9497
|
+
post_validation_failure_label = (
|
|
9498
|
+
"rate-limited"
|
|
9499
|
+
if post_validation_failure_kind == "rate_limit"
|
|
9500
|
+
else "unavailable"
|
|
9501
|
+
)
|
|
9502
|
+
if result.status == "failed" and post_validation_failure_kind and not _skip_fallback:
|
|
9364
9503
|
logger.warning(
|
|
9365
|
-
"Agent '%s'
|
|
9504
|
+
"Agent '%s' failed during validation retry for task %s (%s) — "
|
|
9366
9505
|
"attempting post-validation agent fallback",
|
|
9367
|
-
agent.agent_id, task.task_id,
|
|
9506
|
+
agent.agent_id, task.task_id, post_validation_failure_kind,
|
|
9368
9507
|
)
|
|
9369
9508
|
_pv_fallback = self._select_fallback_agent(
|
|
9370
9509
|
agent.agent_id, task.fallback_chain, tried_agents
|
|
@@ -9380,14 +9519,22 @@ class RuntimeDaemon:
|
|
|
9380
9519
|
task.task_id, 10,
|
|
9381
9520
|
f"agent_fallback: retrying with {agent.agent_id}",
|
|
9382
9521
|
output_lines=[
|
|
9383
|
-
f"[daemon] Agent
|
|
9522
|
+
f"[daemon] Agent {post_validation_failure_label} during validation, "
|
|
9384
9523
|
f"switching to {agent.agent_id}",
|
|
9385
9524
|
],
|
|
9386
9525
|
)
|
|
9387
9526
|
result = await self.process_manager.run_agent(
|
|
9388
9527
|
agent, task, workspace_path, on_chunk=on_output_chunk,
|
|
9389
9528
|
)
|
|
9390
|
-
|
|
9529
|
+
fallback_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
9530
|
+
if fallback_failure_kind:
|
|
9531
|
+
post_validation_failure_kinds.append(fallback_failure_kind)
|
|
9532
|
+
post_validation_failure_label = (
|
|
9533
|
+
"rate-limited"
|
|
9534
|
+
if fallback_failure_kind == "rate_limit"
|
|
9535
|
+
else "unavailable"
|
|
9536
|
+
)
|
|
9537
|
+
if not ProcessManager.is_agent_fallback_eligible(result):
|
|
9391
9538
|
# Fallback agent ran successfully (or hit a non-rate-limit
|
|
9392
9539
|
# failure) — re-run the validation gate and update git state.
|
|
9393
9540
|
if result.status == "success":
|
|
@@ -9413,13 +9560,18 @@ class RuntimeDaemon:
|
|
|
9413
9560
|
)
|
|
9414
9561
|
# If every agent we tried is still rate-limited, signal the server
|
|
9415
9562
|
# NOT to re-enqueue — it would hit the same quota wall immediately.
|
|
9416
|
-
if
|
|
9563
|
+
if ProcessManager.is_agent_fallback_eligible(result):
|
|
9564
|
+
failure_label = ProcessManager.all_agents_failure_label(
|
|
9565
|
+
post_validation_failure_kinds,
|
|
9566
|
+
)
|
|
9417
9567
|
result.error = (
|
|
9418
|
-
f"All agents
|
|
9568
|
+
f"All agents {failure_label} (tried: {', '.join(tried_agents)}). "
|
|
9419
9569
|
f"Original error: {result.error}"
|
|
9420
9570
|
)
|
|
9421
9571
|
result.status = "failed"
|
|
9422
|
-
result.failure_code =
|
|
9572
|
+
result.failure_code = ProcessManager.all_agents_failure_code(
|
|
9573
|
+
post_validation_failure_kinds,
|
|
9574
|
+
)
|
|
9423
9575
|
|
|
9424
9576
|
# 4.55 Analysis/design/fix nodes must update their deliverables in THIS run.
|
|
9425
9577
|
# Existing files from a prior iteration are not sufficient evidence.
|
|
@@ -10742,19 +10894,19 @@ class RuntimeDaemon:
|
|
|
10742
10894
|
cr_result = await self.process_manager.run_agent(
|
|
10743
10895
|
cr_agent, _cr_task, workspace_path,
|
|
10744
10896
|
)
|
|
10745
|
-
#
|
|
10746
|
-
while
|
|
10897
|
+
# Try an alternate agent after a recoverable agent-side failure.
|
|
10898
|
+
while ProcessManager.is_agent_fallback_eligible(cr_result):
|
|
10747
10899
|
_cr_fallback = self._select_fallback_agent(
|
|
10748
10900
|
cr_agent.agent_id, fallback_chain, _cr_tried,
|
|
10749
10901
|
)
|
|
10750
10902
|
if not _cr_fallback:
|
|
10751
10903
|
logger.warning(
|
|
10752
|
-
"deployment_merge: all agents rate-limited for job %s",
|
|
10904
|
+
"deployment_merge: all agents unavailable/rate-limited for job %s",
|
|
10753
10905
|
job_id,
|
|
10754
10906
|
)
|
|
10755
10907
|
break
|
|
10756
10908
|
logger.warning(
|
|
10757
|
-
"deployment_merge: agent '%s' rate-limited, falling back to '%s'",
|
|
10909
|
+
"deployment_merge: agent '%s' unavailable/rate-limited, falling back to '%s'",
|
|
10758
10910
|
cr_agent.agent_id, _cr_fallback.agent_id,
|
|
10759
10911
|
)
|
|
10760
10912
|
cr_agent = _cr_fallback
|
|
@@ -10763,7 +10915,7 @@ class RuntimeDaemon:
|
|
|
10763
10915
|
cr_result = await self.process_manager.run_agent(
|
|
10764
10916
|
cr_agent, _cr_task, workspace_path,
|
|
10765
10917
|
)
|
|
10766
|
-
if not
|
|
10918
|
+
if not ProcessManager.is_agent_fallback_eligible(cr_result):
|
|
10767
10919
|
# Remember the last agent that actually ran to completion.
|
|
10768
10920
|
resolved_agent_id = cr_agent.agent_id
|
|
10769
10921
|
except Exception as exc:
|
|
@@ -11175,12 +11327,14 @@ class RuntimeDaemon:
|
|
|
11175
11327
|
agent_id: str,
|
|
11176
11328
|
task_result: TaskResult,
|
|
11177
11329
|
*,
|
|
11178
|
-
|
|
11330
|
+
agent_failure_kind: str | None,
|
|
11179
11331
|
) -> dict:
|
|
11180
11332
|
if task_result.status == "success":
|
|
11181
11333
|
outcome = "success"
|
|
11182
|
-
elif
|
|
11334
|
+
elif agent_failure_kind == "rate_limit":
|
|
11183
11335
|
outcome = "rate_limited"
|
|
11336
|
+
elif agent_failure_kind == "agent_unavailable":
|
|
11337
|
+
outcome = "agent_unavailable"
|
|
11184
11338
|
elif task_result.failure_code == "agent_idle_timeout":
|
|
11185
11339
|
outcome = "timeout"
|
|
11186
11340
|
elif isinstance(task_result.exit_code, int) and task_result.exit_code != 0:
|
|
@@ -11196,37 +11350,48 @@ class RuntimeDaemon:
|
|
|
11196
11350
|
entry["detail"] = f"rc={task_result.exit_code}"
|
|
11197
11351
|
return entry
|
|
11198
11352
|
|
|
11199
|
-
# 3.5 Agent fallback: if the chosen agent
|
|
11353
|
+
# 3.5 Agent fallback: if the chosen agent has a recoverable failure,
|
|
11200
11354
|
# try the next available agent before giving up. An explicit
|
|
11201
11355
|
# override is deliberately a single-agent attempt.
|
|
11202
11356
|
fallback_scope = "override" if agent_override else "auto"
|
|
11203
11357
|
allow_fallback = not bool(agent_override)
|
|
11204
11358
|
generation_attempts: list[dict] = []
|
|
11205
|
-
|
|
11359
|
+
agent_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
11360
|
+
agent_failure_label = (
|
|
11361
|
+
"rate-limited"
|
|
11362
|
+
if agent_failure_kind == "rate_limit"
|
|
11363
|
+
else "unavailable"
|
|
11364
|
+
)
|
|
11206
11365
|
generation_attempts.append(_generation_attempt(
|
|
11207
11366
|
agent.agent_id,
|
|
11208
11367
|
result,
|
|
11209
|
-
|
|
11368
|
+
agent_failure_kind=agent_failure_kind,
|
|
11210
11369
|
))
|
|
11370
|
+
generation_failure_kinds = [agent_failure_kind] if agent_failure_kind else []
|
|
11211
11371
|
_aj_tried: set[str] = {agent.agent_id}
|
|
11212
|
-
if
|
|
11213
|
-
result.failure_code =
|
|
11214
|
-
|
|
11372
|
+
if agent_failure_kind and not allow_fallback and not result.failure_code:
|
|
11373
|
+
result.failure_code = (
|
|
11374
|
+
"agent_rate_limited"
|
|
11375
|
+
if agent_failure_kind == "rate_limit"
|
|
11376
|
+
else "agent_unavailable"
|
|
11377
|
+
)
|
|
11378
|
+
while agent_failure_kind and allow_fallback:
|
|
11215
11379
|
_aj_fallback = self._select_fallback_agent(agent.agent_id, [], _aj_tried)
|
|
11216
11380
|
if not _aj_fallback:
|
|
11217
|
-
|
|
11218
|
-
|
|
11381
|
+
result.failure_code = ProcessManager.all_agents_failure_code(
|
|
11382
|
+
generation_failure_kinds,
|
|
11383
|
+
)
|
|
11219
11384
|
break
|
|
11220
11385
|
logger.warning(
|
|
11221
|
-
"AIJob %s: agent '%s'
|
|
11222
|
-
job_id, agent.agent_id, _aj_fallback.agent_id,
|
|
11386
|
+
"AIJob %s: agent '%s' failed (%s), falling back to '%s'",
|
|
11387
|
+
job_id, agent.agent_id, agent_failure_kind, _aj_fallback.agent_id,
|
|
11223
11388
|
)
|
|
11224
11389
|
await conn.client.post(
|
|
11225
11390
|
f"{reporter_url}/progress",
|
|
11226
11391
|
json={
|
|
11227
11392
|
"current_step": f"agent_fallback: retrying with {_aj_fallback.agent_id}",
|
|
11228
11393
|
"output_lines": [
|
|
11229
|
-
f"[daemon] Agent
|
|
11394
|
+
f"[daemon] Agent {agent_failure_label}, switching to {_aj_fallback.agent_id}",
|
|
11230
11395
|
],
|
|
11231
11396
|
"progress_pct": 15,
|
|
11232
11397
|
},
|
|
@@ -11265,11 +11430,18 @@ class RuntimeDaemon:
|
|
|
11265
11430
|
except asyncio.CancelledError:
|
|
11266
11431
|
pass
|
|
11267
11432
|
await _flush_output_to_server()
|
|
11268
|
-
|
|
11433
|
+
agent_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
11434
|
+
if agent_failure_kind:
|
|
11435
|
+
generation_failure_kinds.append(agent_failure_kind)
|
|
11436
|
+
agent_failure_label = (
|
|
11437
|
+
"rate-limited"
|
|
11438
|
+
if agent_failure_kind == "rate_limit"
|
|
11439
|
+
else "unavailable"
|
|
11440
|
+
)
|
|
11269
11441
|
generation_attempts.append(_generation_attempt(
|
|
11270
11442
|
agent.agent_id,
|
|
11271
11443
|
result,
|
|
11272
|
-
|
|
11444
|
+
agent_failure_kind=agent_failure_kind,
|
|
11273
11445
|
))
|
|
11274
11446
|
|
|
11275
11447
|
generation_attempt_count = len(generation_attempts)
|
|
@@ -11313,7 +11485,7 @@ class RuntimeDaemon:
|
|
|
11313
11485
|
max_content=max_content,
|
|
11314
11486
|
)
|
|
11315
11487
|
|
|
11316
|
-
# Preserve
|
|
11488
|
+
# Preserve all-agent terminal failures so the server does NOT re-enqueue.
|
|
11317
11489
|
_failure_code = result.failure_code if result.failure_code else (
|
|
11318
11490
|
"agent_error" if result.status != "success" else ""
|
|
11319
11491
|
)
|
|
@@ -11577,17 +11749,18 @@ class RuntimeDaemon:
|
|
|
11577
11749
|
finally:
|
|
11578
11750
|
task.input_prompt = original_prompt
|
|
11579
11751
|
|
|
11580
|
-
# If the agent
|
|
11752
|
+
# If the agent had a fallback-eligible failure during this validation retry,
|
|
11581
11753
|
# bail out immediately so the outer execution loop can trigger agent
|
|
11582
11754
|
# fallback. Continuing to retry with the same rate-limited agent is
|
|
11583
11755
|
# pointless; it will hit the same wall every time.
|
|
11584
|
-
# Returning early also preserves the
|
|
11585
|
-
#
|
|
11586
|
-
if ProcessManager.
|
|
11756
|
+
# Returning early also preserves the failure in result.error so the
|
|
11757
|
+
# outer execution loop can select an alternate agent.
|
|
11758
|
+
if ProcessManager.is_agent_fallback_eligible(result):
|
|
11587
11759
|
logger.warning(
|
|
11588
|
-
"Agent '%s'
|
|
11760
|
+
"Agent '%s' failed during validation retry for task %s (%s) "
|
|
11589
11761
|
"(attempt %d/%d) — aborting validation retries for agent fallback",
|
|
11590
|
-
agent.agent_id, task.task_id,
|
|
11762
|
+
agent.agent_id, task.task_id,
|
|
11763
|
+
ProcessManager.agent_failure_kind(result), attempt + 1, max_retries,
|
|
11591
11764
|
)
|
|
11592
11765
|
return result
|
|
11593
11766
|
|
|
@@ -13400,13 +13573,13 @@ class RuntimeDaemon:
|
|
|
13400
13573
|
workspace_path,
|
|
13401
13574
|
)
|
|
13402
13575
|
|
|
13403
|
-
# Agent fallback:
|
|
13576
|
+
# Agent fallback: try another CLI after a recoverable agent-side failure.
|
|
13404
13577
|
_cr_tried: set[str] = {agent.agent_id}
|
|
13405
|
-
while
|
|
13578
|
+
while ProcessManager.is_agent_fallback_eligible(resolve_result):
|
|
13406
13579
|
_cr_fallback = self._select_fallback_agent(agent.agent_id, task.fallback_chain, _cr_tried)
|
|
13407
13580
|
if not _cr_fallback:
|
|
13408
13581
|
logger.warning(
|
|
13409
|
-
"All agents rate-limited for conflict resolution of task %s — aborting merge",
|
|
13582
|
+
"All agents unavailable/rate-limited for conflict resolution of task %s — aborting merge",
|
|
13410
13583
|
task.task_id,
|
|
13411
13584
|
)
|
|
13412
13585
|
try:
|
|
@@ -13415,7 +13588,7 @@ class RuntimeDaemon:
|
|
|
13415
13588
|
pass
|
|
13416
13589
|
return
|
|
13417
13590
|
logger.warning(
|
|
13418
|
-
"Conflict resolution: agent '%s' rate-limited for task %s, "
|
|
13591
|
+
"Conflict resolution: agent '%s' unavailable/rate-limited for task %s, "
|
|
13419
13592
|
"falling back to '%s'",
|
|
13420
13593
|
agent.agent_id, task.task_id, _cr_fallback.agent_id,
|
|
13421
13594
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|