forgexa-cli 1.19.0__tar.gz → 1.20.0__tar.gz

{forgexa_cli-1.19.0 → forgexa_cli-1.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.19.0
+Version: 1.20.0
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License-Expression: MIT

{forgexa_cli-1.19.0 → forgexa_cli-1.20.0}/forgexa_cli/__init__.py

@@ -1,2 +1,2 @@
 """forgexa-cli — Forgexa command-line client."""
-__version__ = "1.19.0"
+__version__ = "1.20.0"

{forgexa_cli-1.19.0 → forgexa_cli-1.20.0}/forgexa_cli/daemon.py

@@ -555,7 +555,7 @@ except (ImportError, ModuleNotFoundError):
 # DAEMON_VERSION is the protocol/logic version of the daemon code.
 # Kept in sync with pyproject.toml version via bump-version.sh.
 # CLIENT_TYPE identifies which packaging/distribution this daemon runs in.
-DAEMON_VERSION = "1.19.0"
+DAEMON_VERSION = "1.20.0"
 
 
 def _detect_client_type() -> str:
@@ -5495,6 +5495,13 @@ class ProcessManager:
                     pass
             if data.get("duration_ms") and "duration_seconds" not in metrics:
                 metrics["duration_seconds"] = data["duration_ms"] / 1000.0
+            # Turn usage (Claude CLI result event) — baseline telemetry for
+            # node-type turn budgeting (see test-script quality audit M0).
+            if data.get("num_turns") and "num_turns" not in metrics:
+                try:
+                    metrics["num_turns"] = int(data["num_turns"])
+                except (TypeError, ValueError):
+                    pass
 
         # ── 3. Prefer "result" type line — most authoritative ──
         for data in reversed(parsed):
@@ -6697,6 +6704,146 @@ class ServerConnection:
 # ── Main Daemon ──
 
 
+def _categorize_validation_issue(issue: str) -> str:
+    """Map a free-text validation issue to a stable category for telemetry.
+
+    Baseline data for the test-script quality initiative (M0): lets us rank
+    validation failure modes without parsing free text. Order matters —
+    specific patterns are checked before generic ones.
+    """
+    text = issue.lower()
+    if "script_content" in text:
+        return "empty_script_content"
+    if "script_path" in text and "does not exist" in text:
+        return "missing_script_file"
+    if "not valid json" in text:
+        return "invalid_json"
+    if "e2e" in text and "test case" in text:
+        return "missing_e2e_case"
+    if "coverage" in text:
+        return "coverage_gap"
+    if "scope" in text and "violation" in text:
+        return "scope_violation"
+    if "missing" in text or "not found" in text:
+        return "missing_file"
+    if "is empty" in text:
+        return "empty_file"
+    return "other"
+
+
+def _extract_testing_artifacts(workspace_path: Path, output_dir: str) -> dict | None:
+    """Summarize test-cases.json artifact counts for telemetry (best effort).
+
+    Returns None when the file is missing/unparseable; never raises.
+    """
+    try:
+        tc_path = workspace_path / output_dir / "test-cases.json"
+        if not output_dir or not tc_path.exists():
+            return None
+        data = json.loads(tc_path.read_text(encoding="utf-8"))
+        cases = data.get("test_cases") or []
+        by_type: dict[str, int] = {}
+        scripted = 0
+        e2e_scripted = 0
+        for case in cases:
+            if not isinstance(case, dict):
+                continue
+            case_type = (case.get("test_type") or "unknown").lower()
+            by_type[case_type] = by_type.get(case_type, 0) + 1
+            if (case.get("script_content") or "").strip():
+                scripted += 1
+                if case_type == "e2e":
+                    e2e_scripted += 1
+        return {
+            "total_cases": len(cases),
+            "by_type": by_type,
+            "scripted_cases": scripted,
+            "e2e_scripted": e2e_scripted,
+        }
+    except Exception:
+        logger.debug("testing artifact telemetry extraction failed", exc_info=True)
+        return None
+
+
+def _validate_test_evidence(
+    base: Path, workspace_path: Path, e2e_case_ids: list[str], max_anchor_checks: int = 50
+) -> list[str]:
+    """Validate the M1 evidence bundle (test-evidence.json) for e2e test cases.
+
+    Deterministic, zero-environment checks: file exists and parses, locators/
+    routes are present, every e2e case has a case_evidence entry, and each
+    evidence anchor is a literal string that actually appears in its declared
+    source_file (rejects invented selectors/routes). Items explicitly marked
+    runtime_evidence_required are exempt from anchor checks by design.
+    """
+    issues: list[str] = []
+    ev_path = base / "test-evidence.json"
+    if not ev_path.exists():
+        return [
+            "test-evidence.json not found. It is MANDATORY when test-intent.json "
+            "has e2e intents: record every route/selector used by E2E scripts with "
+            "source_file + anchor (a literal string copied from that file), plus "
+            "case_evidence mapping each e2e test case to its route/locator refs."
+        ]
+    try:
+        ev = json.loads(ev_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, UnicodeDecodeError) as e:
+        return [f"test-evidence.json is not valid JSON: {e}"]
+
+    locators = ev.get("locators") or []
+    routes = ev.get("routes") or []
+    if not locators and not routes:
+        issues.append(
+            "test-evidence.json has no 'locators' or 'routes' entries — E2E scripts "
+            "must ground their selectors/routes in source-file evidence."
+        )
+
+    case_evidence = ev.get("case_evidence") or []
+    covered_ids = {
+        str(entry.get("test_case_id"))
+        for entry in case_evidence
+        if isinstance(entry, dict) and entry.get("test_case_id")
+    }
+    missing_cases = [cid for cid in e2e_case_ids if cid not in covered_ids]
+    if missing_cases:
+        issues.append(
+            f"test-evidence.json missing case_evidence for e2e test cases: "
+            f"{missing_cases[:10]}. Each e2e test case needs a case_evidence entry "
+            f"with route_refs/locator_refs."
+        )
+
+    checked = 0
+    for item in list(locators) + list(routes):
+        if checked >= max_anchor_checks:
+            break
+        if not isinstance(item, dict) or item.get("runtime_evidence_required"):
+            continue
+        checked += 1
+        label = item.get("selector") or item.get("path") or item.get("id") or "?"
+        src = (item.get("source_file") or "").strip()
+        anchor = (item.get("anchor") or "").strip()
+        if not src or not anchor:
+            issues.append(
+                f"evidence item '{label}' missing 'source_file' or 'anchor' — "
+                f"anchor must be a literal string copied from source_file."
+            )
+            continue
+        fpath = workspace_path / src
+        if not fpath.exists():
+            issues.append(f"evidence source_file not found: '{src}' (for '{label}')")
+            continue
+        try:
+            content = fpath.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            continue
+        if anchor not in content:
+            issues.append(
+                f"evidence anchor not found in '{src}': '{anchor[:80]}' (for '{label}'). "
+                f"Copy the exact literal from the source file — do not paraphrase."
+            )
+    return issues
+
+
 class RuntimeDaemon:
     """Main Daemon orchestrator — runs on the host machine.
 
@@ -8457,6 +8604,7 @@ class RuntimeDaemon:
                     # Not gated by _requires_structured_artifacts so that bugfix with e2e
                     # intents gets the same E2E chain validation as feature/improvement.
                     _e2e_intent_ids: set[str] = set()
+                    _e2e_case_ids: list[str] = []
                     _analysis_dir = (
                         _input.get("analysis_output_dir")
                         or (_input.get("context") or {}).get("analysis_output_dir")
@@ -8493,6 +8641,13 @@ class RuntimeDaemon:
                         try:
                             tc_data = _json.loads(tc_path.read_text(encoding="utf-8"))
                             cases = tc_data.get("test_cases", [])
+                            _e2e_case_ids = [
+                                str(_tc.get("id"))
+                                for _tc in cases
+                                if isinstance(_tc, dict)
+                                and (_tc.get("test_type") or "").lower() == "e2e"
+                                and _tc.get("id")
+                            ]
                             if not cases:
                                 issues.append("test-cases.json exists but contains no test cases")
                             else:
@@ -8580,6 +8735,15 @@ class RuntimeDaemon:
                     elif _requires_structured_artifacts or _requires_e2e_artifacts:
                         issues.append(f"coverage-matrix.json not found in {doc_dir or 'workspace root'}")
 
+                    # --- test-evidence.json validation (M1 evidence bundle) ---
+                    # Required whenever e2e intents exist: every route/selector
+                    # used by E2E scripts must be grounded in source-file evidence
+                    # (anchor literals are verified against the declared files).
+                    if _requires_e2e_artifacts:
+                        issues.extend(
+                            _validate_test_evidence(base, workspace_path, _e2e_case_ids)
+                        )
+
                     # --- test-report.md validation ---
                     report_path = base / "test-report.md"
                     if not report_path.exists():
@@ -8897,6 +9061,9 @@ class RuntimeDaemon:
                 )
             issues = self._validate_outputs(workspace_path, task, result)
             if not issues:
+                self._record_testing_telemetry(
+                    workspace_path, task, result, retries_used=attempt
+                )
                 return result  # All validations passed
 
             issues_text = "\n".join(f"- {iss}" for iss in issues)
@@ -9031,8 +9198,38 @@ class RuntimeDaemon:
                     len(remaining), max_retries, task.task_id,
                 )
             result.metrics["validation_issues"] = remaining
+        self._record_testing_telemetry(workspace_path, task, result, retries_used=max_retries)
         return result
 
+    def _record_testing_telemetry(
+        self, workspace_path: Path, task: TaskInfo, result: TaskResult, retries_used: int
+    ) -> None:
+        """Attach validation/testing telemetry to result.metrics (best effort).
+
+        M0 baseline for the test-script quality initiative: validation retry
+        usage, stable failure categories, and testing artifact counts. These
+        travel with the completion payload (metrics are kept in full by the
+        backend) so baselines can be built without parsing free text.
+        Never raises.
+        """
+        try:
+            result.metrics["validation_retries_used"] = retries_used
+            issues = result.metrics.get("validation_issues") or []
+            if issues:
+                result.metrics["validation_issue_categories"] = sorted(
+                    {_categorize_validation_issue(str(issue)) for issue in issues}
+                )
+            if task.node_type != "testing":
+                return
+            input_data = task.input_data or {}
+            context = input_data.get("context") or {}
+            output_dir = input_data.get("output_dir") or context.get("output_dir") or ""
+            artifacts = _extract_testing_artifacts(workspace_path, output_dir)
+            if artifacts:
+                result.metrics["testing_artifacts"] = artifacts
+        except Exception:
+            logger.debug("testing telemetry recording failed", exc_info=True)
+
     async def _collect_analysis_artifacts(
         self, workspace_path: Path, task: TaskInfo, result: TaskResult
     ) -> None:

{forgexa_cli-1.19.0 → forgexa_cli-1.20.0}/forgexa_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forgexa-cli
-Version: 1.19.0
+Version: 1.20.0
 Summary: Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform
 Author-email: Jason Sun <dev.winds@gmail.com>
 License-Expression: MIT

{forgexa_cli-1.19.0 → forgexa_cli-1.20.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "forgexa-cli"
-version = "1.19.0"
+version = "1.20.0"
 description = "Forgexa CLI — command-line client and AI agent runtime for the Forgexa platform"
 requires-python = ">=3.9"
 license = "MIT"