forgexa-cli 1.8.10__tar.gz → 1.9.1__tar.gz

{forgexa_cli-1.8.10 → forgexa_cli-1.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.8.10
+Version: 1.9.1
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License: MIT

{forgexa_cli-1.8.10 → forgexa_cli-1.9.1}/forgexa_cli/__init__.py

@@ -1,2 +1,2 @@
 """forgexa-cli — Forgexa command-line client."""
-__version__ = "1.8.10"
+__version__ = "1.9.1"

{forgexa_cli-1.8.10 → forgexa_cli-1.9.1}/forgexa_cli/daemon.py

@@ -396,7 +396,7 @@ except (ImportError, ModuleNotFoundError):
 # DAEMON_VERSION is the protocol/logic version of the daemon code.
 # Kept in sync with pyproject.toml version via bump-version.sh.
 # CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
-DAEMON_VERSION = "1.8.10"
+DAEMON_VERSION = "1.9.1"
 
 
 def _detect_client_type() -> str:
@@ -2150,10 +2150,54 @@ class ProcessManager:
             "has_turn_failed": has_turn_failed,
             "has_result": has_result,
             "has_meaningful_content": has_meaningful_content,
+            "has_assistant_events": has_assistant_events,
             "error_messages": error_messages,
             "json_line_count": json_line_count,
         }
 
+    @staticmethod
+    def _should_scan_short_success_stdout(stdout: str, signals: dict[str, Any]) -> bool:
+        """Return True when success-shaped stdout is short enough to be an error blob.
+
+        Real agent work output can legitimately mention quota/rate-limit terms, so
+        we never scan arbitrary stdout for success cases. The safe exception is a
+        tiny stdout payload with no assistant/result/turn-complete signals; in that
+        shape the CLI usually failed before making a real model call and printed a
+        plain-text transport/quota error like "API Error: 429 ...".
+        """
+        stdout_stripped = (stdout or "").strip()
+        if not stdout_stripped or len(stdout_stripped) >= 500:
+            return False
+        return not (
+            signals.get("has_result")
+            or signals.get("has_turn_completed")
+            or signals.get("has_assistant_events")
+        )
+
+    @staticmethod
+    def _failure_pattern_channels(result: "TaskResult") -> str:
+        """Build the text window safe to scan for quota/backend failure patterns."""
+        stdout = result.stdout or ""
+        stderr = result.stderr or ""
+        error = result.error or ""
+
+        if result.status != "success":
+            if result.exit_code == 0:
+                return "\n".join(part for part in (stderr, error) if part)
+            return "\n".join(part for part in (stderr, error, stdout[-3000:]) if part)
+
+        error_channels = "\n".join(part for part in (stderr, error) if part)
+        has_token_usage = (
+            int(result.metrics.get("token_input", 0) or 0)
+            + int(result.metrics.get("token_output", 0) or 0)
+        ) > 0
+        signals = ProcessManager._extract_output_signals(
+            "\n".join(part for part in (stdout, stderr) if part)
+        )
+        if not has_token_usage and ProcessManager._should_scan_short_success_stdout(stdout, signals):
+            error_channels = "\n".join(filter(None, [error_channels, stdout.strip()]))
+        return error_channels
+
     @staticmethod
     def has_meaningful_agent_output(result: "TaskResult") -> bool:
         """Return True when the agent emitted real user-meaningful output."""
@@ -2172,29 +2216,12 @@ class ProcessManager:
         Returns True for rate/quota limits AND API unavailability errors,
         since a different agent (using a different API backend) may succeed.
 
-        IMPORTANT: Only checks stderr and error message.  When exit code is
-        non-zero, also checks the tail of stdout (last 3000 chars) since the
-        error is likely at the end.  When exit code is 0 (agent reported
-        success but _detect_agent_output_failure set status to failed), do
-        NOT scan stdout — it contains the agent's work output (configs, code)
-        which naturally has terms like "rate_limit", "API_RATE_LIMIT_PER_MINUTE"
-        that trigger false positives.
+        For true success cases we still avoid scanning arbitrary stdout.
+        The one safe exception is a tiny stdout payload with no assistant/result
+        signals, which strongly indicates a pre-call CLI failure printed as
+        plain text (for example "API Error: 429 ...").
         """
-        if result.status == "success":
-            return False
-        # When exit code is 0, _detect_agent_output_failure already checked
-        # stderr+error for rate-limit patterns.  Don't re-scan stdout here.
-        if result.exit_code == 0:
-            error_text = (
-                (result.stderr or "")
-                + "\n" + (result.error or "")
-            ).lower()
-        else:
-            error_text = (
-                (result.stderr or "")
-                + "\n" + (result.error or "")
-                + "\n" + (result.stdout or "")[-3000:]
-            ).lower()
+        error_text = ProcessManager._failure_pattern_channels(result).lower()
         return (
             any(p in error_text for p in ProcessManager.RATE_LIMIT_PATTERNS)
             or any(p in error_text for p in ProcessManager.AGENT_UNAVAILABLE_PATTERNS)
@@ -2214,16 +2241,13 @@ class ProcessManager:
         if result.status != "success":
             return None
 
-        # For exit-code-0 (success) cases, only scan stderr and the error field
-        # for rate-limit / unavailability patterns.  Stdout contains the agent's
-        # actual task output (code, configs, analysis docs) which may legitimately
-        # contain substrings like "rate_limit", "429", "quota", etc. — e.g. writing
-        # a config file with API_RATE_LIMIT_PER_MINUTE=1000 would previously trigger
-        # a false "quota exhaustion" failure even though the agent succeeded.
-        # stdout[-N:] is only safe to scan when the agent already failed (exit != 0),
-        # which is handled by is_rate_limited() called at the orchestrator level.
-        error_only_channels = (result.stderr or "") + "\n" + (result.error or "")
-        pattern_failure = ProcessManager._has_failure_pattern(error_only_channels)
+        # For exit-code-0 (success) cases, avoid scanning arbitrary stdout for
+        # quota keywords. The only safe stdout exception is a tiny payload with
+        # no success signals, which typically means the CLI failed before making
+        # a real model call and printed a plain-text error like "API Error: 429".
+        pattern_failure = ProcessManager._has_failure_pattern(
+            ProcessManager._failure_pattern_channels(result)
+        )
         if pattern_failure:
             return pattern_failure
 
@@ -2239,8 +2263,13 @@ class ProcessManager:
         has_turn_failed = signals["has_turn_failed"]
         has_result = signals["has_result"]
         has_meaningful_content = signals["has_meaningful_content"]
+        has_assistant_events = signals["has_assistant_events"]
         error_messages = signals["error_messages"]
         json_line_count = signals["json_line_count"]
+        has_token_usage = (
+            int(result.metrics.get("token_input", 0) or 0)
+            + int(result.metrics.get("token_output", 0) or 0)
+        ) > 0
 
         stderr_lower = stderr.lower()
         if (
@@ -2272,14 +2301,27 @@ class ProcessManager:
                 and not has_meaningful_content and json_line_count > 0):
             return f"Agent encountered errors without producing output: {error_messages[0]}"
 
+        # Plain-text pre-call failures (not JSONL) can still exit 0 on some
+        # agent CLIs. When stdout is tiny and lacks any structural success
+        # signals, treat explicit error markers as agent failure so fallback can
+        # run locally instead of relying on server-side re-enqueue.
+        stdout_stripped = stdout.strip()
+        stdout_lower = stdout_stripped.lower()
+        if (
+            not has_token_usage
+            and ProcessManager._should_scan_short_success_stdout(stdout, signals)
+            and any(marker in stdout_lower for marker in ("api error", "exception", "forbidden", "unauthorized"))
+        ):
+            return stdout_stripped.splitlines()[-1][:300]
+
         # ── Claude: JSON output mode but no result object and no content ──
         if agent_id == "claude" and json_line_count > 0:
-            if not has_result and not has_meaningful_content:
+            if not has_result and not has_meaningful_content and not has_assistant_events:
                 return "Claude produced no result output"
 
         # ── Copilot: JSONL mode but no turn completion and no content ──
         if agent_id == "copilot" and json_line_count > 0:
-            if not has_result and not has_meaningful_content:
+            if not has_result and not has_meaningful_content and not has_assistant_events:
                 return "Copilot produced no result output (check GitHub authentication: run 'gh auth login')"
 
         return None
@@ -2347,8 +2389,9 @@ class ProcessManager:
         return normalized
 
     def _required_deliverable_paths(self, task: TaskInfo) -> set[str]:
-        # For analysis nodes, deliverables live in analysis_output_dir (docs/requirements/...)
-        # For other nodes, use output_dir (docs/implements/...)
+        # For analysis nodes, deliverables live in analysis_output_dir (docs/requirements/<key>/analysis)
+        # For delivery nodes, deliverables live in output_dir (docs/requirements/<key>/delivery)
+        # For other nodes, use output_dir (docs/requirements/<key>/implement)
         if task.node_type == "analysis":
             output_dir = str(
                 (task.input_data or {}).get("analysis_output_dir", "")
@@ -2366,6 +2409,9 @@ class ProcessManager:
             required_files = _get_analysis_outputs_for_type(req_type)
         elif task.node_type == "design":
             required_files = ["design.md"]
+        elif task.node_type == "delivery":
+            # Required docs come from node input_data (set by delivery_doc_service)
+            required_files = (task.input_data or {}).get("required_docs") or ["release-note.md"]
         else:
             return set()
 
@@ -4781,6 +4827,72 @@ class RuntimeDaemon:
                 except Exception:
                     logger.exception("Validation gate error for task %s (proceeding anyway)", task.task_id)
 
+            # 4.6 Post-validation rate-limit fallback.
+            # _validate_and_retry returns early (preserving the rate-limit error)
+            # when the agent hits a quota wall mid-retry.  The initial-run fallback
+            # block (step 3) only checked the *initial* run; if that succeeded but
+            # the agent became rate-limited during a validation retry, we need a
+            # second fallback pass here so the task is attempted on a fresh agent.
+            if result.status == "failed" and self.process_manager.is_rate_limited(result) and not _skip_fallback:
+                logger.warning(
+                    "Agent '%s' rate-limited during validation retry for task %s — "
+                    "attempting post-validation agent fallback",
+                    agent.agent_id, task.task_id,
+                )
+                _pv_fallback = self._select_fallback_agent(
+                    agent.agent_id, task.fallback_chain, tried_agents
+                )
+                while _pv_fallback:
+                    logger.info(
+                        "Post-validation fallback: '%s' → '%s' for task %s",
+                        agent.agent_id, _pv_fallback.agent_id, task.task_id,
+                    )
+                    agent = _pv_fallback
+                    tried_agents.add(agent.agent_id)
+                    await reporter.report_progress(
+                        task.task_id, 10,
+                        f"agent_fallback: retrying with {agent.agent_id}",
+                        output_lines=[
+                            f"[daemon] Agent rate-limited during validation, "
+                            f"switching to {agent.agent_id}",
+                        ],
+                    )
+                    result = await self.process_manager.run_agent(
+                        agent, task, workspace_path, on_chunk=on_output_chunk,
+                    )
+                    if not self.process_manager.is_rate_limited(result):
+                        # Fallback agent ran successfully (or hit a non-rate-limit
+                        # failure) — re-run the validation gate and update git state.
+                        if result.status == "success":
+                            try:
+                                result = await self._validate_and_retry(
+                                    agent, task, workspace_path, result,
+                                    reporter, on_output_chunk, max_retries=2,
+                                )
+                                pre_commit_git = await self.process_manager._collect_git_info(workspace_path)
+                            except Exception:
+                                logger.exception(
+                                    "Post-validation gate error for task %s (proceeding anyway)",
+                                    task.task_id,
+                                )
+                        break
+                    logger.warning(
+                        "Post-validation fallback agent '%s' also rate-limited for task %s",
+                        agent.agent_id, task.task_id,
+                    )
+                    _pv_fallback = self._select_fallback_agent(
+                        agent.agent_id, task.fallback_chain, tried_agents
+                    )
+                # If every agent we tried is still rate-limited, signal the server
+                # NOT to re-enqueue — it would hit the same quota wall immediately.
+                if self.process_manager.is_rate_limited(result):
+                    result.error = (
+                        f"All agents unavailable/rate-limited (tried: {', '.join(tried_agents)}). "
+                        f"Original error: {result.error}"
+                    )
+                    result.status = "failed"
+                    result.failure_code = "all_agents_rate_limited"
+
             # 4.55 Analysis/design nodes must update their deliverables in THIS run.
             # Existing files from a prior iteration are not sufficient evidence.
             if result.status == "success" and task.node_type in ("analysis", "design"):
@@ -5262,6 +5374,38 @@ class RuntimeDaemon:
             # Flush any remaining buffered lines after agent finishes
             await _flush_output_to_server()
 
+            # 3.5 Agent fallback: if the chosen agent hit a rate/quota limit,
+            # try the next available agent before giving up.
+            _aj_tried: set[str] = {agent.agent_id}
+            while self.process_manager.is_rate_limited(result):
+                _aj_fallback = self._select_fallback_agent(agent.agent_id, [], _aj_tried)
+                if not _aj_fallback:
+                    # All agents exhausted — signal server not to re-enqueue.
+                    result.failure_code = "all_agents_rate_limited"
+                    break
+                logger.warning(
+                    "AIJob %s: agent '%s' rate-limited, falling back to '%s'",
+                    job_id, agent.agent_id, _aj_fallback.agent_id,
+                )
+                await conn.client.post(
+                    f"{reporter_url}/progress",
+                    json={
+                        "current_step": f"agent_fallback: retrying with {_aj_fallback.agent_id}",
+                        "output_lines": [
+                            f"[daemon] Agent rate-limited, switching to {_aj_fallback.agent_id}",
+                        ],
+                        "progress_pct": 15,
+                    },
+                    timeout=5,
+                )
+                agent = _aj_fallback
+                _aj_tried.add(agent.agent_id)
+                fake_task.agent_type = agent.agent_id
+                result = await self.process_manager.run_agent(
+                    agent, fake_task, workspace_path, on_chunk=on_chunk,
+                )
+                await _flush_output_to_server()
+
             # 4. Auto-commit if successful
             input_ctx = aj.get("input_context", {})
             git_info = {}
@@ -5308,6 +5452,10 @@ class RuntimeDaemon:
                             except Exception:
                                 pass
 
+            # Preserve all_agents_rate_limited so the server does NOT re-enqueue.
+            _failure_code = result.failure_code if result.failure_code else (
+                "agent_error" if result.status != "success" else ""
+            )
             complete_payload = {
                 "status": "success" if result.status == "success" else "failed",
                 "output_content": output_content,
@@ -5321,7 +5469,7 @@ class RuntimeDaemon:
                 "resolved_agent": agent.agent_id,
                 "git_info": git_info,
                 "error": result.error if result.status != "success" else "",
-                "failure_code": "agent_error" if result.status != "success" else "",
+                "failure_code": _failure_code,
             }
 
             await conn.client.post(
@@ -5424,6 +5572,20 @@ class RuntimeDaemon:
             finally:
                 task.input_prompt = original_prompt
 
+            # If the agent hit a rate/quota limit during this validation retry,
+            # bail out immediately so the outer execution loop can trigger agent
+            # fallback.  Continuing to retry with the same rate-limited agent is
+            # pointless; it will hit the same wall every time.
+            # Returning early also preserves the rate-limit error in result.error
+            # so that is_rate_limited() can detect it in the caller.
+            if ProcessManager.is_rate_limited(result):
+                logger.warning(
+                    "Agent '%s' rate-limited during validation retry for task %s "
+                    "(attempt %d/%d) — aborting validation retries for agent fallback",
+                    agent.agent_id, task.task_id, attempt + 1, max_retries,
+                )
+                return result
+
         # Final check after all retries
         remaining = self._validate_outputs(workspace_path, task, result)
         if remaining:
@@ -5897,7 +6059,7 @@ class RuntimeDaemon:
         for f in files[:30]:
             path = f["path"].lower()
             fname = path.rsplit("/", 1)[-1]
-            if "docs/requirements" in path:
+            if "docs/requirements" in path or "docs/workitems" in path:
                 buckets["Analysis deliverables"].append(f)
             elif (
                 "_test" in fname or fname.startswith("test_")
@@ -6107,24 +6269,49 @@ class RuntimeDaemon:
         # 5. Run the agent with the conflict resolution prompt
         logger.info("Invoking %s to resolve %d conflict(s)...", agent.agent_id, len(conflicted_files))
         try:
+            _cr_task = TaskInfo(
+                task_id=f"{task.task_id}-conflict-resolve",
+                graph_id=task.graph_id,
+                node_type="conflict_resolution",
+                agent_type=agent.agent_id,
+                input_prompt=resolve_prompt,
+                input_data={},
+                timeout_seconds=min(task.timeout_seconds, 300),  # cap at 5 min
+                max_retries=0,
+                retry_count=0,
+                project=task.project,
+                work_item=task.work_item,
+            )
             resolve_result = await self.process_manager.run_agent(
                 agent,
-                TaskInfo(
-                    task_id=f"{task.task_id}-conflict-resolve",
-                    graph_id=task.graph_id,
-                    node_type="conflict_resolution",
-                    agent_type=agent.agent_id,
-                    input_prompt=resolve_prompt,
-                    input_data={},
-                    timeout_seconds=min(task.timeout_seconds, 300),  # cap at 5 min
-                    max_retries=0,
-                    retry_count=0,
-                    project=task.project,
-                    work_item=task.work_item,
-                ),
+                _cr_task,
                 workspace_path,
             )
 
+            # Agent fallback: if the primary agent is rate-limited, try others.
+            _cr_tried: set[str] = {agent.agent_id}
+            while self.process_manager.is_rate_limited(resolve_result):
+                _cr_fallback = self._select_fallback_agent(agent.agent_id, task.fallback_chain, _cr_tried)
+                if not _cr_fallback:
+                    logger.warning(
+                        "All agents rate-limited for conflict resolution of task %s — aborting merge",
+                        task.task_id,
+                    )
+                    try:
+                        await git("merge", "--abort", cwd=workspace_path)
+                    except RuntimeError:
+                        pass
+                    return
+                logger.warning(
+                    "Conflict resolution: agent '%s' rate-limited for task %s, "
+                    "falling back to '%s'",
+                    agent.agent_id, task.task_id, _cr_fallback.agent_id,
+                )
+                agent = _cr_fallback
+                _cr_tried.add(agent.agent_id)
+                _cr_task.agent_type = agent.agent_id
+                resolve_result = await self.process_manager.run_agent(agent, _cr_task, workspace_path)
+
             # 6. Check if conflicts are resolved
             proc = await asyncio.create_subprocess_exec(
                 "git", "diff", "--name-only", "--diff-filter=U",

{forgexa_cli-1.8.10 → forgexa_cli-1.9.1}/forgexa_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.8.10
+Version: 1.9.1
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License: MIT

{forgexa_cli-1.8.10 → forgexa_cli-1.9.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "forgexa-cli"
-version = "1.8.10"
+version = "1.9.1"
 description = "Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform"
 requires-python = ">=3.9"
 license = { text = "MIT" }