forgexa-cli 1.41.3__tar.gz → 1.41.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/PKG-INFO +1 -1
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli/__init__.py +1 -1
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli/daemon.py +273 -78
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli.egg-info/PKG-INFO +1 -1
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/pyproject.toml +1 -1
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/README.md +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli/_build_config.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli/_local_bind.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli/agent_core.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli/autoupgrade.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli/main.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli/py.typed +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli.egg-info/SOURCES.txt +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli.egg-info/dependency_links.txt +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli.egg-info/entry_points.txt +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli.egg-info/requires.txt +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/forgexa_cli.egg-info/top_level.txt +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/setup.cfg +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_auth_and_runtime_commands.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_autoupgrade.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_check_command.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_expiry_warnings_and_revoke.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_local_bind_commands.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_runtime_credentials.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_session_credentials.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_silent_install.py +0 -0
- {forgexa_cli-1.41.3 → forgexa_cli-1.41.5}/tests/test_upgrade_observability.py +0 -0
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""forgexa-cli — Forgexa command-line client."""
|
|
2
|
-
__version__ = "1.41.
|
|
2
|
+
__version__ = "1.41.5"
|
|
@@ -910,7 +910,7 @@ except (ImportError, ModuleNotFoundError):
|
|
|
910
910
|
# DAEMON_VERSION is the protocol/logic version of the daemon code.
|
|
911
911
|
# Kept in sync with pyproject.toml version via bump-version.sh.
|
|
912
912
|
# CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
|
|
913
|
-
DAEMON_VERSION = "1.41.
|
|
913
|
+
DAEMON_VERSION = "1.41.5"
|
|
914
914
|
|
|
915
915
|
|
|
916
916
|
def _detect_client_type() -> str:
|
|
@@ -1234,8 +1234,9 @@ class TaskResult:
|
|
|
1234
1234
|
error: str = ""
|
|
1235
1235
|
# failure_code is forwarded to the server to drive retry policy.
|
|
1236
1236
|
# Key values:
|
|
1237
|
-
# "all_agents_rate_limited" — daemon tried
|
|
1238
|
-
#
|
|
1237
|
+
# "all_agents_rate_limited" / "all_agents_unavailable" — daemon tried
|
|
1238
|
+
# every installed agent and none can currently run. Server must NOT
|
|
1239
|
+
# retry on the same runtime.
|
|
1239
1240
|
# "validation_retry_exhausted" — required-output validation still failed
|
|
1240
1241
|
# after the allowed repair agents. Server must not auto-retry it.
|
|
1241
1242
|
failure_code: str = ""
|
|
@@ -1557,7 +1558,12 @@ def _normalize_ai_job_output(
|
|
|
1557
1558
|
if not raw_output:
|
|
1558
1559
|
return "", None
|
|
1559
1560
|
|
|
1560
|
-
if task_type not in {
|
|
1561
|
+
if task_type not in {
|
|
1562
|
+
"qa_scenario_generate_prompt",
|
|
1563
|
+
"qa_scenario_preview",
|
|
1564
|
+
"qa_script_preview",
|
|
1565
|
+
"test_script_generate",
|
|
1566
|
+
}:
|
|
1561
1567
|
return raw_output, None
|
|
1562
1568
|
|
|
1563
1569
|
extracted_output = _daemon_extract_agent_output(agent_id, raw_output)
|
|
@@ -4723,6 +4729,7 @@ class ProcessManager:
|
|
|
4723
4729
|
"rate limit",
|
|
4724
4730
|
"rate_limit",
|
|
4725
4731
|
"quota exceeded",
|
|
4732
|
+
"quota exhausted",
|
|
4726
4733
|
"monthly quota", # Copilot: "You have exceeded your monthly quota"
|
|
4727
4734
|
"too many requests",
|
|
4728
4735
|
"overloaded",
|
|
@@ -4744,6 +4751,8 @@ class ProcessManager:
|
|
|
4744
4751
|
"name or service not known",
|
|
4745
4752
|
"no such host",
|
|
4746
4753
|
"network is unreachable",
|
|
4754
|
+
"oauth/access_token",
|
|
4755
|
+
"error during sign-in",
|
|
4747
4756
|
"websockettransporterror",
|
|
4748
4757
|
"websocket receive failed",
|
|
4749
4758
|
"421 misdirected request",
|
|
@@ -4876,25 +4885,43 @@ class ProcessManager:
|
|
|
4876
4885
|
"""Implementation lives in agent_core._copilot_called_any_tools."""
|
|
4877
4886
|
return _copilot_called_any_tools(stdout)
|
|
4878
4887
|
|
|
4888
|
+
@staticmethod
|
|
4889
|
+
def agent_failure_kind(result: "TaskResult") -> str | None:
|
|
4890
|
+
"""Classify a fallback-eligible failure without conflating its cause."""
|
|
4891
|
+
if result.failure_code in {"all_agents_rate_limited", "all_agents_unavailable"}:
|
|
4892
|
+
return None
|
|
4893
|
+
error_text = ProcessManager._failure_pattern_channels(result).lower()
|
|
4894
|
+
if any(p in error_text for p in ProcessManager.RATE_LIMIT_PATTERNS):
|
|
4895
|
+
return "rate_limit"
|
|
4896
|
+
if any(p in error_text for p in ProcessManager.AGENT_UNAVAILABLE_PATTERNS):
|
|
4897
|
+
return "agent_unavailable"
|
|
4898
|
+
return None
|
|
4899
|
+
|
|
4879
4900
|
@staticmethod
|
|
4880
4901
|
def is_rate_limited(result: "TaskResult") -> bool:
|
|
4881
|
-
"""
|
|
4902
|
+
"""Return whether an agent actually hit a rate or usage limit."""
|
|
4903
|
+
return ProcessManager.agent_failure_kind(result) == "rate_limit"
|
|
4882
4904
|
|
|
4883
|
-
|
|
4884
|
-
|
|
4905
|
+
@staticmethod
|
|
4906
|
+
def is_agent_fallback_eligible(result: "TaskResult") -> bool:
|
|
4907
|
+
"""Return whether another agent may recover this agent-side failure."""
|
|
4908
|
+
return ProcessManager.agent_failure_kind(result) is not None
|
|
4885
4909
|
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
|
|
4897
|
-
|
|
4910
|
+
@staticmethod
|
|
4911
|
+
def all_agents_failure_code(failure_kinds: list[str]) -> str:
|
|
4912
|
+
"""Return the terminal code for exhausted fallback-eligible agents."""
|
|
4913
|
+
if failure_kinds and all(kind == "rate_limit" for kind in failure_kinds):
|
|
4914
|
+
return "all_agents_rate_limited"
|
|
4915
|
+
return "all_agents_unavailable"
|
|
4916
|
+
|
|
4917
|
+
@staticmethod
|
|
4918
|
+
def all_agents_failure_label(failure_kinds: list[str]) -> str:
|
|
4919
|
+
"""Return an accurate human-readable summary for exhausted agents."""
|
|
4920
|
+
if failure_kinds and all(kind == "rate_limit" for kind in failure_kinds):
|
|
4921
|
+
return "rate-limited"
|
|
4922
|
+
if failure_kinds and all(kind == "agent_unavailable" for kind in failure_kinds):
|
|
4923
|
+
return "unavailable"
|
|
4924
|
+
return "unable to run"
|
|
4898
4925
|
|
|
4899
4926
|
@staticmethod
|
|
4900
4927
|
def is_silent_idle_timeout(result: "TaskResult") -> bool:
|
|
@@ -5062,7 +5089,11 @@ class ProcessManager:
|
|
|
5062
5089
|
timeout,
|
|
5063
5090
|
task.task_id,
|
|
5064
5091
|
on_chunk,
|
|
5065
|
-
disable_builtin_mcps=task.input_data.get("task_type")
|
|
5092
|
+
disable_builtin_mcps=task.input_data.get("task_type") in {
|
|
5093
|
+
"test_script_generate",
|
|
5094
|
+
"qa_scenario_preview",
|
|
5095
|
+
"qa_script_preview",
|
|
5096
|
+
},
|
|
5066
5097
|
)
|
|
5067
5098
|
else:
|
|
5068
5099
|
result = await self._run_generic(agent, prompt, workspace_path, timeout, task.task_id, on_chunk)
|
|
@@ -5645,6 +5676,22 @@ class ProcessManager:
|
|
|
5645
5676
|
result.metrics.update(parsed_metrics)
|
|
5646
5677
|
return result
|
|
5647
5678
|
|
|
5679
|
+
@staticmethod
|
|
5680
|
+
def _opencode_suggested_model(output: str) -> str | None:
|
|
5681
|
+
"""Extract OpenCode's first replacement model from a missing-model error."""
|
|
5682
|
+
match = re.search(
|
|
5683
|
+
r"model not found\s*:\s*(.+?)(?:[.?!]\s+|\s+)did you mean:\s*"
|
|
5684
|
+
r"([A-Za-z0-9][A-Za-z0-9._:/-]*)",
|
|
5685
|
+
output or "",
|
|
5686
|
+
re.IGNORECASE,
|
|
5687
|
+
)
|
|
5688
|
+
if not match:
|
|
5689
|
+
return None
|
|
5690
|
+
missing_model, suggested_model = match.groups()
|
|
5691
|
+
if missing_model.strip().lower() == suggested_model.lower():
|
|
5692
|
+
return None
|
|
5693
|
+
return suggested_model
|
|
5694
|
+
|
|
5648
5695
|
async def _run_opencode(
|
|
5649
5696
|
self, agent: DiscoveredAgent, prompt: str, cwd: Path, timeout: int, task_id: str,
|
|
5650
5697
|
on_chunk: Any = None,
|
|
@@ -5669,7 +5716,7 @@ class ProcessManager:
|
|
|
5669
5716
|
No need to re-check it here.
|
|
5670
5717
|
"""
|
|
5671
5718
|
# Apply model override if configured (e.g. FACTORY_OPENCODE_MODEL=copilot/gpt-4.1)
|
|
5672
|
-
model_override = os.environ.get("FACTORY_OPENCODE_MODEL")
|
|
5719
|
+
model_override = os.environ.get("FACTORY_OPENCODE_MODEL", "").strip() or None
|
|
5673
5720
|
# Isolate each opencode run in its own data directory to prevent
|
|
5674
5721
|
# concurrent processes from racing on the shared SQLite WAL file.
|
|
5675
5722
|
tmp_data_root = tempfile.mkdtemp(prefix=f"opencode-{task_id[:8]}-")
|
|
@@ -5693,23 +5740,43 @@ class ProcessManager:
|
|
|
5693
5740
|
output = "\n".join(
|
|
5694
5741
|
part for part in (result.stdout, result.stderr, result.error) if part
|
|
5695
5742
|
).lower()
|
|
5696
|
-
|
|
5743
|
+
suggested_model = self._opencode_suggested_model(output)
|
|
5744
|
+
retry_model: str | None = None
|
|
5745
|
+
recovery_metric = ""
|
|
5746
|
+
if (
|
|
5747
|
+
result.status == "failed"
|
|
5748
|
+
and model_override
|
|
5749
|
+
and "model not found" in output
|
|
5750
|
+
and model_override.lower() in output
|
|
5751
|
+
):
|
|
5752
|
+
recovery_metric = "opencode_model_override_fallback"
|
|
5697
5753
|
logger.warning(
|
|
5698
5754
|
"OpenCode model override %r is unavailable for task %s; retrying without it",
|
|
5699
5755
|
model_override,
|
|
5700
5756
|
task_id,
|
|
5701
5757
|
)
|
|
5758
|
+
elif result.status == "failed" and suggested_model:
|
|
5759
|
+
retry_model = suggested_model
|
|
5760
|
+
recovery_metric = "opencode_default_model_fallback"
|
|
5761
|
+
logger.warning(
|
|
5762
|
+
"OpenCode configured model is unavailable for task %s; retrying with %r",
|
|
5763
|
+
task_id,
|
|
5764
|
+
suggested_model,
|
|
5765
|
+
)
|
|
5766
|
+
|
|
5767
|
+
if recovery_metric:
|
|
5702
5768
|
retry_env = dict(env)
|
|
5703
5769
|
retry_env.pop("FACTORY_OPENCODE_MODEL", None)
|
|
5704
5770
|
cmd = agent_core.build_opencode_command(
|
|
5705
5771
|
agent.command,
|
|
5706
5772
|
cwd=cwd,
|
|
5773
|
+
model=retry_model,
|
|
5707
5774
|
prompt_file=prompt_file,
|
|
5708
5775
|
)
|
|
5709
5776
|
result = await self._run_cli(
|
|
5710
5777
|
cmd, cwd, timeout, task_id, on_chunk=on_chunk, env=retry_env,
|
|
5711
5778
|
)
|
|
5712
|
-
result.metrics[
|
|
5779
|
+
result.metrics[recovery_metric] = True
|
|
5713
5780
|
finally:
|
|
5714
5781
|
shutil.rmtree(tmp_data_root, ignore_errors=True)
|
|
5715
5782
|
|
|
@@ -5873,6 +5940,27 @@ class ProcessManager:
|
|
|
5873
5940
|
"""Implementation lives in agent_core.prepare_copilot_home."""
|
|
5874
5941
|
return agent_core.prepare_copilot_home()
|
|
5875
5942
|
|
|
5943
|
+
@staticmethod
|
|
5944
|
+
def _copilot_builtin_mcp_failure(stdout: str) -> str | None:
|
|
5945
|
+
"""Return a startup failure from Copilot's built-in GitHub MCP server."""
|
|
5946
|
+
for line in (stdout or "").splitlines():
|
|
5947
|
+
try:
|
|
5948
|
+
event = json.loads(line)
|
|
5949
|
+
except (TypeError, ValueError):
|
|
5950
|
+
continue
|
|
5951
|
+
if not isinstance(event, dict):
|
|
5952
|
+
continue
|
|
5953
|
+
if event.get("type") != "session.mcp_server_status_changed":
|
|
5954
|
+
continue
|
|
5955
|
+
data = event.get("data") or {}
|
|
5956
|
+
if not isinstance(data, dict):
|
|
5957
|
+
continue
|
|
5958
|
+
if data.get("serverName") != "github-mcp-server" or data.get("status") != "failed":
|
|
5959
|
+
continue
|
|
5960
|
+
message = data.get("error") or data.get("message") or ""
|
|
5961
|
+
return str(message).strip() or "GitHub MCP server failed to initialize"
|
|
5962
|
+
return None
|
|
5963
|
+
|
|
5876
5964
|
async def _run_copilot(
|
|
5877
5965
|
self, agent: DiscoveredAgent, prompt: str, cwd: Path, timeout: int, task_id: str,
|
|
5878
5966
|
on_chunk: Any = None,
|
|
@@ -6088,6 +6176,36 @@ class ProcessManager:
|
|
|
6088
6176
|
break
|
|
6089
6177
|
effective_rc = copilot_exit if copilot_exit is not None else returncode
|
|
6090
6178
|
|
|
6179
|
+
builtin_mcp_error = (
|
|
6180
|
+
self._copilot_builtin_mcp_failure(stdout)
|
|
6181
|
+
if not disable_builtin_mcps
|
|
6182
|
+
else None
|
|
6183
|
+
)
|
|
6184
|
+
if (
|
|
6185
|
+
builtin_mcp_error
|
|
6186
|
+
and not signals["has_meaningful_content"]
|
|
6187
|
+
and not self._copilot_called_any_tools(stdout)
|
|
6188
|
+
):
|
|
6189
|
+
logger.warning(
|
|
6190
|
+
"Copilot built-in GitHub MCP failed before work for task %s; "
|
|
6191
|
+
"retrying with built-in MCPs disabled: %s",
|
|
6192
|
+
task_id,
|
|
6193
|
+
builtin_mcp_error[:300],
|
|
6194
|
+
)
|
|
6195
|
+
retry_result = await self._run_copilot(
|
|
6196
|
+
agent,
|
|
6197
|
+
prompt,
|
|
6198
|
+
cwd,
|
|
6199
|
+
timeout,
|
|
6200
|
+
task_id,
|
|
6201
|
+
on_chunk,
|
|
6202
|
+
disable_builtin_mcps=True,
|
|
6203
|
+
_retry_without_effort=_retry_without_effort,
|
|
6204
|
+
_isolated_copilot_home=_isolated_copilot_home,
|
|
6205
|
+
)
|
|
6206
|
+
retry_result.metrics["copilot_builtin_mcp_fallback"] = True
|
|
6207
|
+
return retry_result
|
|
6208
|
+
|
|
6091
6209
|
if copilot_exit == 0 and returncode == 0:
|
|
6092
6210
|
return TaskResult(
|
|
6093
6211
|
status="success",
|
|
@@ -9102,15 +9220,24 @@ class RuntimeDaemon:
|
|
|
9102
9220
|
# Guard: if the agent already produced file changes in the workspace, it DID
|
|
9103
9221
|
# meaningful work — don't trigger fallback even if it crashed after completing.
|
|
9104
9222
|
# Let the recovery logic (step 4.1) handle non-zero exit with committed work.
|
|
9105
|
-
|
|
9223
|
+
agent_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
9224
|
+
is_agent_fallback_eligible = agent_failure_kind is not None
|
|
9106
9225
|
is_silent_idle_timeout = self.process_manager.is_silent_idle_timeout(result)
|
|
9107
9226
|
fallback_reason = (
|
|
9108
|
-
|
|
9109
|
-
if
|
|
9227
|
+
agent_failure_kind
|
|
9228
|
+
if is_agent_fallback_eligible
|
|
9110
9229
|
else "agent_idle_timeout"
|
|
9111
9230
|
if is_silent_idle_timeout
|
|
9112
9231
|
else ""
|
|
9113
9232
|
)
|
|
9233
|
+
fallback_label = (
|
|
9234
|
+
"rate-limited"
|
|
9235
|
+
if agent_failure_kind == "rate_limit"
|
|
9236
|
+
else "unavailable"
|
|
9237
|
+
if agent_failure_kind == "agent_unavailable"
|
|
9238
|
+
else "timed out"
|
|
9239
|
+
)
|
|
9240
|
+
fallback_failure_kinds = [agent_failure_kind or fallback_reason] if fallback_reason else []
|
|
9114
9241
|
_skip_fallback = False
|
|
9115
9242
|
if fallback_reason:
|
|
9116
9243
|
_pre_fallback_git = await self.process_manager._collect_git_info(workspace_path)
|
|
@@ -9151,7 +9278,7 @@ class RuntimeDaemon:
|
|
|
9151
9278
|
await reporter.report_progress(
|
|
9152
9279
|
task.task_id, 10,
|
|
9153
9280
|
f"agent_fallback: retrying with {agent.agent_id}",
|
|
9154
|
-
output_lines=[f"[daemon] Agent {
|
|
9281
|
+
output_lines=[f"[daemon] Agent {fallback_label}, switching to {agent.agent_id}"],
|
|
9155
9282
|
)
|
|
9156
9283
|
|
|
9157
9284
|
# Re-run with fallback agent
|
|
@@ -9201,27 +9328,39 @@ class RuntimeDaemon:
|
|
|
9201
9328
|
# A silent hang gets exactly one alternate-agent attempt.
|
|
9202
9329
|
# Unlike quota failures, do not serially spend the full idle
|
|
9203
9330
|
# timeout on every installed CLI when the host is offline.
|
|
9204
|
-
|
|
9331
|
+
fallback_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
9332
|
+
if fallback_failure_kind:
|
|
9333
|
+
fallback_failure_kinds.append(fallback_failure_kind)
|
|
9334
|
+
fallback_label = (
|
|
9335
|
+
"rate-limited"
|
|
9336
|
+
if fallback_failure_kind == "rate_limit"
|
|
9337
|
+
else "unavailable"
|
|
9338
|
+
)
|
|
9339
|
+
if not ProcessManager.is_agent_fallback_eligible(result):
|
|
9205
9340
|
break # Success or non-retriable failure
|
|
9206
9341
|
|
|
9207
9342
|
logger.warning(
|
|
9208
|
-
"Fallback agent '%s' also
|
|
9209
|
-
agent.agent_id,
|
|
9343
|
+
"Fallback agent '%s' also %s for task %s",
|
|
9344
|
+
agent.agent_id,
|
|
9345
|
+
"rate-limited" if fallback_failure_kind == "rate_limit" else "unavailable",
|
|
9346
|
+
task.task_id,
|
|
9210
9347
|
)
|
|
9211
9348
|
fallback_agent = self._select_fallback_agent(
|
|
9212
9349
|
agent.agent_id, task.fallback_chain, tried_agents
|
|
9213
9350
|
)
|
|
9214
9351
|
|
|
9215
|
-
if
|
|
9352
|
+
if ProcessManager.is_agent_fallback_eligible(result):
|
|
9353
|
+
failure_label = ProcessManager.all_agents_failure_label(
|
|
9354
|
+
fallback_failure_kinds,
|
|
9355
|
+
)
|
|
9216
9356
|
result.error = (
|
|
9217
|
-
f"All agents
|
|
9357
|
+
f"All agents {failure_label} (tried: {', '.join(tried_agents)}). "
|
|
9218
9358
|
f"Original error: {result.error}"
|
|
9219
9359
|
)
|
|
9220
9360
|
result.status = "failed"
|
|
9221
|
-
|
|
9222
|
-
|
|
9223
|
-
|
|
9224
|
-
result.failure_code = "all_agents_rate_limited"
|
|
9361
|
+
result.failure_code = ProcessManager.all_agents_failure_code(
|
|
9362
|
+
fallback_failure_kinds,
|
|
9363
|
+
)
|
|
9225
9364
|
|
|
9226
9365
|
# 4. Collect git info BEFORE commit (shows uncommitted changes)
|
|
9227
9366
|
pre_commit_git = await self.process_manager._collect_git_info(workspace_path)
|
|
@@ -9351,11 +9490,20 @@ class RuntimeDaemon:
|
|
|
9351
9490
|
# block (step 3) only checked the *initial* run; if that succeeded but
|
|
9352
9491
|
# the agent became rate-limited during a validation retry, we need a
|
|
9353
9492
|
# second fallback pass here so the task is attempted on a fresh agent.
|
|
9354
|
-
|
|
9493
|
+
post_validation_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
9494
|
+
post_validation_failure_kinds = (
|
|
9495
|
+
[post_validation_failure_kind] if post_validation_failure_kind else []
|
|
9496
|
+
)
|
|
9497
|
+
post_validation_failure_label = (
|
|
9498
|
+
"rate-limited"
|
|
9499
|
+
if post_validation_failure_kind == "rate_limit"
|
|
9500
|
+
else "unavailable"
|
|
9501
|
+
)
|
|
9502
|
+
if result.status == "failed" and post_validation_failure_kind and not _skip_fallback:
|
|
9355
9503
|
logger.warning(
|
|
9356
|
-
"Agent '%s'
|
|
9504
|
+
"Agent '%s' failed during validation retry for task %s (%s) — "
|
|
9357
9505
|
"attempting post-validation agent fallback",
|
|
9358
|
-
agent.agent_id, task.task_id,
|
|
9506
|
+
agent.agent_id, task.task_id, post_validation_failure_kind,
|
|
9359
9507
|
)
|
|
9360
9508
|
_pv_fallback = self._select_fallback_agent(
|
|
9361
9509
|
agent.agent_id, task.fallback_chain, tried_agents
|
|
@@ -9371,14 +9519,22 @@ class RuntimeDaemon:
|
|
|
9371
9519
|
task.task_id, 10,
|
|
9372
9520
|
f"agent_fallback: retrying with {agent.agent_id}",
|
|
9373
9521
|
output_lines=[
|
|
9374
|
-
f"[daemon] Agent
|
|
9522
|
+
f"[daemon] Agent {post_validation_failure_label} during validation, "
|
|
9375
9523
|
f"switching to {agent.agent_id}",
|
|
9376
9524
|
],
|
|
9377
9525
|
)
|
|
9378
9526
|
result = await self.process_manager.run_agent(
|
|
9379
9527
|
agent, task, workspace_path, on_chunk=on_output_chunk,
|
|
9380
9528
|
)
|
|
9381
|
-
|
|
9529
|
+
fallback_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
9530
|
+
if fallback_failure_kind:
|
|
9531
|
+
post_validation_failure_kinds.append(fallback_failure_kind)
|
|
9532
|
+
post_validation_failure_label = (
|
|
9533
|
+
"rate-limited"
|
|
9534
|
+
if fallback_failure_kind == "rate_limit"
|
|
9535
|
+
else "unavailable"
|
|
9536
|
+
)
|
|
9537
|
+
if not ProcessManager.is_agent_fallback_eligible(result):
|
|
9382
9538
|
# Fallback agent ran successfully (or hit a non-rate-limit
|
|
9383
9539
|
# failure) — re-run the validation gate and update git state.
|
|
9384
9540
|
if result.status == "success":
|
|
@@ -9404,13 +9560,18 @@ class RuntimeDaemon:
|
|
|
9404
9560
|
)
|
|
9405
9561
|
# If every agent we tried is still rate-limited, signal the server
|
|
9406
9562
|
# NOT to re-enqueue — it would hit the same quota wall immediately.
|
|
9407
|
-
if
|
|
9563
|
+
if ProcessManager.is_agent_fallback_eligible(result):
|
|
9564
|
+
failure_label = ProcessManager.all_agents_failure_label(
|
|
9565
|
+
post_validation_failure_kinds,
|
|
9566
|
+
)
|
|
9408
9567
|
result.error = (
|
|
9409
|
-
f"All agents
|
|
9568
|
+
f"All agents {failure_label} (tried: {', '.join(tried_agents)}). "
|
|
9410
9569
|
f"Original error: {result.error}"
|
|
9411
9570
|
)
|
|
9412
9571
|
result.status = "failed"
|
|
9413
|
-
result.failure_code =
|
|
9572
|
+
result.failure_code = ProcessManager.all_agents_failure_code(
|
|
9573
|
+
post_validation_failure_kinds,
|
|
9574
|
+
)
|
|
9414
9575
|
|
|
9415
9576
|
# 4.55 Analysis/design/fix nodes must update their deliverables in THIS run.
|
|
9416
9577
|
# Existing files from a prior iteration are not sufficient evidence.
|
|
@@ -10733,19 +10894,19 @@ class RuntimeDaemon:
|
|
|
10733
10894
|
cr_result = await self.process_manager.run_agent(
|
|
10734
10895
|
cr_agent, _cr_task, workspace_path,
|
|
10735
10896
|
)
|
|
10736
|
-
#
|
|
10737
|
-
while
|
|
10897
|
+
# Try an alternate agent after a recoverable agent-side failure.
|
|
10898
|
+
while ProcessManager.is_agent_fallback_eligible(cr_result):
|
|
10738
10899
|
_cr_fallback = self._select_fallback_agent(
|
|
10739
10900
|
cr_agent.agent_id, fallback_chain, _cr_tried,
|
|
10740
10901
|
)
|
|
10741
10902
|
if not _cr_fallback:
|
|
10742
10903
|
logger.warning(
|
|
10743
|
-
"deployment_merge: all agents rate-limited for job %s",
|
|
10904
|
+
"deployment_merge: all agents unavailable/rate-limited for job %s",
|
|
10744
10905
|
job_id,
|
|
10745
10906
|
)
|
|
10746
10907
|
break
|
|
10747
10908
|
logger.warning(
|
|
10748
|
-
"deployment_merge: agent '%s' rate-limited, falling back to '%s'",
|
|
10909
|
+
"deployment_merge: agent '%s' unavailable/rate-limited, falling back to '%s'",
|
|
10749
10910
|
cr_agent.agent_id, _cr_fallback.agent_id,
|
|
10750
10911
|
)
|
|
10751
10912
|
cr_agent = _cr_fallback
|
|
@@ -10754,7 +10915,7 @@ class RuntimeDaemon:
|
|
|
10754
10915
|
cr_result = await self.process_manager.run_agent(
|
|
10755
10916
|
cr_agent, _cr_task, workspace_path,
|
|
10756
10917
|
)
|
|
10757
|
-
if not
|
|
10918
|
+
if not ProcessManager.is_agent_fallback_eligible(cr_result):
|
|
10758
10919
|
# Remember the last agent that actually ran to completion.
|
|
10759
10920
|
resolved_agent_id = cr_agent.agent_id
|
|
10760
10921
|
except Exception as exc:
|
|
@@ -11024,6 +11185,15 @@ class RuntimeDaemon:
|
|
|
11024
11185
|
|
|
11025
11186
|
# 2. Prepare workspace (using project info + requirement branch)
|
|
11026
11187
|
full_prompt = f"{system_prompt}\n\n{user_prompt}" if system_prompt else user_prompt
|
|
11188
|
+
task_timeout = settings.AGENT_TIMEOUT
|
|
11189
|
+
if task_type in {"qa_scenario_preview", "qa_script_preview"}:
|
|
11190
|
+
try:
|
|
11191
|
+
task_timeout = min(
|
|
11192
|
+
max(int(input_ctx.get("timeout_seconds", settings.AGENT_TIMEOUT)), 30),
|
|
11193
|
+
settings.AGENT_TIMEOUT,
|
|
11194
|
+
)
|
|
11195
|
+
except (TypeError, ValueError):
|
|
11196
|
+
pass
|
|
11027
11197
|
fake_task = TaskInfo(
|
|
11028
11198
|
task_id=job_id,
|
|
11029
11199
|
# Reuse a stable workspace key when the server provides one so
|
|
@@ -11034,7 +11204,7 @@ class RuntimeDaemon:
|
|
|
11034
11204
|
agent_type=agent_type,
|
|
11035
11205
|
input_prompt=full_prompt,
|
|
11036
11206
|
input_data={"task_type": task_type},
|
|
11037
|
-
timeout_seconds=
|
|
11207
|
+
timeout_seconds=task_timeout,
|
|
11038
11208
|
max_retries=0,
|
|
11039
11209
|
retry_count=0,
|
|
11040
11210
|
project=project_info,
|
|
@@ -11157,12 +11327,14 @@ class RuntimeDaemon:
|
|
|
11157
11327
|
agent_id: str,
|
|
11158
11328
|
task_result: TaskResult,
|
|
11159
11329
|
*,
|
|
11160
|
-
|
|
11330
|
+
agent_failure_kind: str | None,
|
|
11161
11331
|
) -> dict:
|
|
11162
11332
|
if task_result.status == "success":
|
|
11163
11333
|
outcome = "success"
|
|
11164
|
-
elif
|
|
11334
|
+
elif agent_failure_kind == "rate_limit":
|
|
11165
11335
|
outcome = "rate_limited"
|
|
11336
|
+
elif agent_failure_kind == "agent_unavailable":
|
|
11337
|
+
outcome = "agent_unavailable"
|
|
11166
11338
|
elif task_result.failure_code == "agent_idle_timeout":
|
|
11167
11339
|
outcome = "timeout"
|
|
11168
11340
|
elif isinstance(task_result.exit_code, int) and task_result.exit_code != 0:
|
|
@@ -11178,37 +11350,48 @@ class RuntimeDaemon:
|
|
|
11178
11350
|
entry["detail"] = f"rc={task_result.exit_code}"
|
|
11179
11351
|
return entry
|
|
11180
11352
|
|
|
11181
|
-
# 3.5 Agent fallback: if the chosen agent
|
|
11353
|
+
# 3.5 Agent fallback: if the chosen agent has a recoverable failure,
|
|
11182
11354
|
# try the next available agent before giving up. An explicit
|
|
11183
11355
|
# override is deliberately a single-agent attempt.
|
|
11184
11356
|
fallback_scope = "override" if agent_override else "auto"
|
|
11185
11357
|
allow_fallback = not bool(agent_override)
|
|
11186
11358
|
generation_attempts: list[dict] = []
|
|
11187
|
-
|
|
11359
|
+
agent_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
11360
|
+
agent_failure_label = (
|
|
11361
|
+
"rate-limited"
|
|
11362
|
+
if agent_failure_kind == "rate_limit"
|
|
11363
|
+
else "unavailable"
|
|
11364
|
+
)
|
|
11188
11365
|
generation_attempts.append(_generation_attempt(
|
|
11189
11366
|
agent.agent_id,
|
|
11190
11367
|
result,
|
|
11191
|
-
|
|
11368
|
+
agent_failure_kind=agent_failure_kind,
|
|
11192
11369
|
))
|
|
11370
|
+
generation_failure_kinds = [agent_failure_kind] if agent_failure_kind else []
|
|
11193
11371
|
_aj_tried: set[str] = {agent.agent_id}
|
|
11194
|
-
if
|
|
11195
|
-
result.failure_code =
|
|
11196
|
-
|
|
11372
|
+
if agent_failure_kind and not allow_fallback and not result.failure_code:
|
|
11373
|
+
result.failure_code = (
|
|
11374
|
+
"agent_rate_limited"
|
|
11375
|
+
if agent_failure_kind == "rate_limit"
|
|
11376
|
+
else "agent_unavailable"
|
|
11377
|
+
)
|
|
11378
|
+
while agent_failure_kind and allow_fallback:
|
|
11197
11379
|
_aj_fallback = self._select_fallback_agent(agent.agent_id, [], _aj_tried)
|
|
11198
11380
|
if not _aj_fallback:
|
|
11199
|
-
|
|
11200
|
-
|
|
11381
|
+
result.failure_code = ProcessManager.all_agents_failure_code(
|
|
11382
|
+
generation_failure_kinds,
|
|
11383
|
+
)
|
|
11201
11384
|
break
|
|
11202
11385
|
logger.warning(
|
|
11203
|
-
"AIJob %s: agent '%s'
|
|
11204
|
-
job_id, agent.agent_id, _aj_fallback.agent_id,
|
|
11386
|
+
"AIJob %s: agent '%s' failed (%s), falling back to '%s'",
|
|
11387
|
+
job_id, agent.agent_id, agent_failure_kind, _aj_fallback.agent_id,
|
|
11205
11388
|
)
|
|
11206
11389
|
await conn.client.post(
|
|
11207
11390
|
f"{reporter_url}/progress",
|
|
11208
11391
|
json={
|
|
11209
11392
|
"current_step": f"agent_fallback: retrying with {_aj_fallback.agent_id}",
|
|
11210
11393
|
"output_lines": [
|
|
11211
|
-
f"[daemon] Agent
|
|
11394
|
+
f"[daemon] Agent {agent_failure_label}, switching to {_aj_fallback.agent_id}",
|
|
11212
11395
|
],
|
|
11213
11396
|
"progress_pct": 15,
|
|
11214
11397
|
},
|
|
@@ -11247,11 +11430,18 @@ class RuntimeDaemon:
|
|
|
11247
11430
|
except asyncio.CancelledError:
|
|
11248
11431
|
pass
|
|
11249
11432
|
await _flush_output_to_server()
|
|
11250
|
-
|
|
11433
|
+
agent_failure_kind = ProcessManager.agent_failure_kind(result)
|
|
11434
|
+
if agent_failure_kind:
|
|
11435
|
+
generation_failure_kinds.append(agent_failure_kind)
|
|
11436
|
+
agent_failure_label = (
|
|
11437
|
+
"rate-limited"
|
|
11438
|
+
if agent_failure_kind == "rate_limit"
|
|
11439
|
+
else "unavailable"
|
|
11440
|
+
)
|
|
11251
11441
|
generation_attempts.append(_generation_attempt(
|
|
11252
11442
|
agent.agent_id,
|
|
11253
11443
|
result,
|
|
11254
|
-
|
|
11444
|
+
agent_failure_kind=agent_failure_kind,
|
|
11255
11445
|
))
|
|
11256
11446
|
|
|
11257
11447
|
generation_attempt_count = len(generation_attempts)
|
|
@@ -11271,7 +11461,11 @@ class RuntimeDaemon:
|
|
|
11271
11461
|
|
|
11272
11462
|
# 5. Report completion
|
|
11273
11463
|
# For deliverables: allow up to 200K chars (full document); others: last 20K
|
|
11274
|
-
max_content =
|
|
11464
|
+
max_content = (
|
|
11465
|
+
200000 if task_type == "deliverable_generate"
|
|
11466
|
+
else 50000 if task_type in {"qa_scenario_preview", "qa_script_preview"}
|
|
11467
|
+
else 20000
|
|
11468
|
+
)
|
|
11275
11469
|
normalized_output, scenario_items = _normalize_ai_job_output(
|
|
11276
11470
|
task_type=task_type,
|
|
11277
11471
|
agent_id=agent.agent_id,
|
|
@@ -11283,7 +11477,7 @@ class RuntimeDaemon:
|
|
|
11283
11477
|
scenario_ids=input_ctx.get("scenario_ids", []),
|
|
11284
11478
|
workspace_path=workspace_path,
|
|
11285
11479
|
)
|
|
11286
|
-
if task_type
|
|
11480
|
+
if task_type in {"qa_scenario_generate_prompt", "qa_scenario_preview", "qa_script_preview"}:
|
|
11287
11481
|
output_content, scenario_items = _prepare_ai_job_output(
|
|
11288
11482
|
task_type=task_type,
|
|
11289
11483
|
agent_id=agent.agent_id,
|
|
@@ -11291,7 +11485,7 @@ class RuntimeDaemon:
|
|
|
11291
11485
|
max_content=max_content,
|
|
11292
11486
|
)
|
|
11293
11487
|
|
|
11294
|
-
# Preserve
|
|
11488
|
+
# Preserve all-agent terminal failures so the server does NOT re-enqueue.
|
|
11295
11489
|
_failure_code = result.failure_code if result.failure_code else (
|
|
11296
11490
|
"agent_error" if result.status != "success" else ""
|
|
11297
11491
|
)
|
|
@@ -11555,17 +11749,18 @@ class RuntimeDaemon:
|
|
|
11555
11749
|
finally:
|
|
11556
11750
|
task.input_prompt = original_prompt
|
|
11557
11751
|
|
|
11558
|
-
# If the agent
|
|
11752
|
+
# If the agent had a fallback-eligible failure during this validation retry,
|
|
11559
11753
|
# bail out immediately so the outer execution loop can trigger agent
|
|
11560
11754
|
# fallback. Continuing to retry with the same rate-limited agent is
|
|
11561
11755
|
# pointless; it will hit the same wall every time.
|
|
11562
|
-
# Returning early also preserves the
|
|
11563
|
-
#
|
|
11564
|
-
if ProcessManager.
|
|
11756
|
+
# Returning early also preserves the failure in result.error so the
|
|
11757
|
+
# outer execution loop can select an alternate agent.
|
|
11758
|
+
if ProcessManager.is_agent_fallback_eligible(result):
|
|
11565
11759
|
logger.warning(
|
|
11566
|
-
"Agent '%s'
|
|
11760
|
+
"Agent '%s' failed during validation retry for task %s (%s) "
|
|
11567
11761
|
"(attempt %d/%d) — aborting validation retries for agent fallback",
|
|
11568
|
-
agent.agent_id, task.task_id,
|
|
11762
|
+
agent.agent_id, task.task_id,
|
|
11763
|
+
ProcessManager.agent_failure_kind(result), attempt + 1, max_retries,
|
|
11569
11764
|
)
|
|
11570
11765
|
return result
|
|
11571
11766
|
|
|
@@ -13378,13 +13573,13 @@ class RuntimeDaemon:
|
|
|
13378
13573
|
workspace_path,
|
|
13379
13574
|
)
|
|
13380
13575
|
|
|
13381
|
-
# Agent fallback:
|
|
13576
|
+
# Agent fallback: try another CLI after a recoverable agent-side failure.
|
|
13382
13577
|
_cr_tried: set[str] = {agent.agent_id}
|
|
13383
|
-
while
|
|
13578
|
+
while ProcessManager.is_agent_fallback_eligible(resolve_result):
|
|
13384
13579
|
_cr_fallback = self._select_fallback_agent(agent.agent_id, task.fallback_chain, _cr_tried)
|
|
13385
13580
|
if not _cr_fallback:
|
|
13386
13581
|
logger.warning(
|
|
13387
|
-
"All agents rate-limited for conflict resolution of task %s — aborting merge",
|
|
13582
|
+
"All agents unavailable/rate-limited for conflict resolution of task %s — aborting merge",
|
|
13388
13583
|
task.task_id,
|
|
13389
13584
|
)
|
|
13390
13585
|
try:
|
|
@@ -13393,7 +13588,7 @@ class RuntimeDaemon:
|
|
|
13393
13588
|
pass
|
|
13394
13589
|
return
|
|
13395
13590
|
logger.warning(
|
|
13396
|
-
"Conflict resolution: agent '%s' rate-limited for task %s, "
|
|
13591
|
+
"Conflict resolution: agent '%s' unavailable/rate-limited for task %s, "
|
|
13397
13592
|
"falling back to '%s'",
|
|
13398
13593
|
agent.agent_id, task.task_id, _cr_fallback.agent_id,
|
|
13399
13594
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|