agentcompass 1.0.0__py3-none-any.whl

agentcompass/analyzers/basic/exception_analyzer.py

@@ -0,0 +1,167 @@
+import logging
+import os
+import re
+
+from agentcompass.runtime import ANALYZERS, AnalysisResult, AnalyzerCategory, BaseAnalyzer, RunResult
+
+logger = logging.getLogger(__name__)
+
+_FRAME_RE = re.compile(r'^\s*File "([^"]+)", line \d+, in (\S+)', re.MULTILINE)
+
+_EXC_LINE_RE = re.compile(r'^([A-Za-z_][\w.]*): (.*)$', re.MULTILINE)
+
+_CHAIN_SEP_RE = re.compile(r'(?:The above exception was the direct cause of the following exception'
+                           r'|During handling of the above exception, another exception occurred):')
+
+
+def _normalize_message(msg: str) -> str:
+    if not msg:
+        return ""
+    s = msg
+    s = re.sub(r'0x[0-9a-fA-F]+', '<hex>', s)
+    s = re.sub(r'\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b', '<uuid>', s)
+    s = re.sub(r'\b\d{1,3}(?:\.\d{1,3}){3}\b', '<ip>', s)
+    s = re.sub(r'(?:/[\w.\-]+)+', '<path>', s)
+    # s = re.sub(r'\b\d+(?:\.\d+)?\b', '<n>', s)
+    s = re.sub(r'"[^"]*"', '<s>', s)
+    s = re.sub(r"'[^']*'", '<s>', s)
+    return s.strip()
+
+
+def _traceback_fingerprint(error: str) -> dict:
+    """Parse a Python traceback into a stable aggregation fingerprint.
+    """
+    out = {
+        "error_type": "",
+        "error_location": "",
+        "error_message_normalized": "",
+        "error_fingerprint": "",
+        "error_cause_type": "",
+        "error_cause_location": "",
+        "error_cause": "",
+    }
+    if not error or not error.strip():
+        return out
+
+    exc_lines = _EXC_LINE_RE.findall(error)
+    frames = [(m.start(), m.group(1), m.group(2)) for m in _FRAME_RE.finditer(error)]
+    sep = _CHAIN_SEP_RE.search(error)
+
+    if not exc_lines:
+        out["error_type"] = error
+        out["error_fingerprint"] = _normalize_message(error.strip()) or "<error>"
+        return out
+
+    final_type, final_msg = exc_lines[-1]
+    chained = bool(sep) and len(exc_lines) >= 2
+    cause_type, cause_msg = exc_lines[0] if chained else ("", "")
+
+    final_loc = ""
+    cause_loc = ""
+    if frames:
+        _, ff_path, ff_func = frames[-1]
+        final_loc = f"{os.path.basename(ff_path)}:{ff_func}"
+        if chained:
+            before = [f for f in frames if f[0] < sep.start()]
+            if before:
+                _, cf_path, cf_func = before[-1]
+                cause_loc = f"{os.path.basename(cf_path)}:{cf_func}"
+
+    final_msg_n = _normalize_message(final_msg)
+    out["error_type"] = f"{final_type}: {final_msg_n}"
+    out["error_location"] = final_loc
+
+    if chained:
+        cause_msg_n = _normalize_message(cause_msg)
+        out["error_cause_type"] = cause_type
+        out["error_cause_location"] = cause_loc
+        out["error_cause"] = f"{cause_type}: {cause_msg}"
+        out["error_message_normalized"] = (f"{final_msg_n} (caused by: {cause_type}: {cause_msg_n})")
+        out["error_fingerprint"] = (f"{final_type} @ {final_loc or '<unknown>'} "
+                                    f"(caused by: {cause_type} @ {cause_loc or '<unknown>'})")
+    else:
+        out["error_message_normalized"] = final_msg_n
+        out["error_fingerprint"] = f"{final_type} @ {final_loc or '<unknown>'}"
+    return out
+
+
+@ANALYZERS.register()
+class ExceptionAnalyzer(BaseAnalyzer):
+    id = "ExceptionAnalyzer"
+    description = "Detect execution issues and preserved diagnostic evidence, and parse tracebacks into stable, aggregable fingerprints (error type, location, cause)."
+    category = AnalyzerCategory.ENV_FRAMEWORK_ERROR
+    data_requirements = ["$.issues"]
+    datasets = []
+    conf = {"only_incorrect": False, "threshold": 0.0}
+    distribution_fields = {
+        "error_type": "value_counts",
+        "error_fingerprint": "value_counts",
+        "task_status": "value_counts",
+    }
+
+    async def analysis(self, task, prepared, result: RunResult, req, plan) -> AnalysisResult:
+        is_badcase = False
+        details = {}
+
+        if result is None:
+            return AnalysisResult(task_id=task.task_id, is_badcase=None, error="RunResult is None")
+
+        has_error_status = any(issue.severity in {"fatal", "error"} for issue in result.issues)
+        diagnostics = result.artifacts.get("execution_diagnostics") or "\n".join(issue.message
+                                                                                 for issue in result.issues)
+        has_error_field = bool(diagnostics)
+
+        is_badcase = has_error_status or has_error_field
+
+        if has_error_status:
+            details["task_status"] = f"{result.status.value}"
+        if has_error_field:
+            details["error_snippet"] = diagnostics[:500]
+            details.update(_traceback_fingerprint(diagnostics))
+
+        return AnalysisResult(
+            task_id=task.task_id,
+            is_badcase=is_badcase,
+            score=1 if is_badcase else 0,
+            details=details,
+        )
+
+
+@ANALYZERS.register()
+class TerminalBench2ExceptionAnalyzer(ExceptionAnalyzer):
+    """TerminalBench2-specific exception analysis.
+
+    Extends the generic ExceptionAnalyzer with environment error detection
+    from eval_raw_data.test_output.
+    """
+    id = "TerminalBench2ExceptionAnalyzer"
+    description = "TerminalBench2 variant of ExceptionAnalyzer that also detects environment-level errors from eval test output (terminal_bench_2 only)."
+    datasets = ["terminal_bench_2", "terminal_bench_2_verified", "terminal_bench_2_1"]
+    base_analyzer = "ExceptionAnalyzer"
+    priority = 1
+
+    async def analysis(self, task, prepared, result: RunResult, req, plan) -> AnalysisResult:
+        base_result = await super().analysis(task, prepared, result, req, plan)
+
+        extra = result.extra or {}
+        eval_raw = extra.get("eval_raw_data") or {}
+        test_output = eval_raw.get("testcase_output")
+        if test_output and self._detect_env_error(test_output):
+            base_result.is_badcase = True
+            base_result.details["eval_environment_error"] = True
+            base_result.details["eval_environment_error_snippet"] = str(test_output)[:500]
+            if base_result.score is None or base_result.score < 1:
+                base_result.score = 1
+
+        return base_result
+
+    @staticmethod
+    def _detect_env_error(test_output: str) -> bool:
+        """Detect environment-level errors from test output."""
+        signals = [
+            "connection refused", "connection timed out", "name or service not known", "no such file or directory",
+            "permission denied", "address already in use", "environment error", "command not found",
+            "E: Could not get lock /var/lib/dpkg/lock-frontend", "No such file or directory"
+        ]
+        lower = str(test_output).lower()
+        return any(s in lower for s in signals)

agentcompass/analyzers/basic/json_error_analyzer.py

@@ -0,0 +1,177 @@
+import json
+import logging
+from typing import List
+
+from agentcompass.runtime import ANALYZERS, AnalysisResult, AnalyzerCategory, BaseAnalyzer, RunResult
+
+logger = logging.getLogger(__name__)
+
+
+def _extract_json_content(response: str) -> tuple[str, List[str]]:
+    """Extract JSON content from response, handling extra text."""
+    warnings = []
+
+    # Try to find JSON object boundaries
+    json_start = -1
+    json_end = -1
+    brace_count = 0
+    in_string = False
+    escape_next = False
+
+    for i, char in enumerate(response):
+        if escape_next:
+            escape_next = False
+            continue
+
+        if char == "\\":
+            escape_next = True
+            continue
+
+        if char == '"' and not escape_next:
+            in_string = not in_string
+            continue
+
+        if not in_string:
+            if char == "{":
+                if brace_count == 0:
+                    json_start = i
+                brace_count += 1
+            elif char == "}":
+                brace_count -= 1
+                if brace_count == 0 and json_start != -1:
+                    json_end = i + 1
+                    break
+
+    if json_start == -1 or json_end == -1:
+        return "", ["No valid JSON object found"]
+
+    # Check for extra text
+    before_text = response[:json_start].strip()
+    after_text = response[json_end:].strip()
+
+    if before_text:
+        warnings.append("Extra text detected before JSON object")
+    if after_text:
+        warnings.append("Extra text detected after JSON object")
+
+    return response[json_start:json_end], warnings
+
+
+def _check_json(text: str) -> str:
+    """Classify a text string's JSON status.
+
+    Returns:
+        "clean_json"           – text is valid JSON as-is
+        "json_with_extra_text" – valid JSON exists but is surrounded by other text
+        "no_json"              – no valid JSON found
+    """
+    if not text or not text.strip():
+        return "no_json"
+
+    stripped = text.strip()
+
+    json_content, extra_text_warnings = _extract_json_content(stripped)
+    if not json_content:
+        return "no_json"
+
+    try:
+        _ = json.loads(json_content)
+    except (json.JSONDecodeError, ValueError):
+        return "no_json"
+
+    if extra_text_warnings:
+        return "json_with_extra_text"
+
+    return "clean_json"
+
+
+@ANALYZERS.register()
+class JSONErrorAnalyzer(BaseAnalyzer):
+    """Detect JSON formatting issues in assistant content (terminal_bench_2 only)."""
+
+    id = "JSONErrorAnalyzer"
+    description = "Detect JSON formatting issues in assistant content — missing JSON or JSON wrapped in extra text (terminal_bench_2 only)."
+    category = AnalyzerCategory.ERROR
+    datasets = ["terminal_bench_2", "terminal_bench_2_verified", "terminal_bench_2_1"]
+    data_requirements = ["$.trajectory.steps[*].assistant_content"]
+    conf = {"only_incorrect": False, "threshold": 0.0}
+    distribution_fields = {
+        "json_error_types": "value_counts",
+        "no_json_steps": "numeric_stats",
+        "json_with_extra_text_steps": "numeric_stats",
+    }
+
+    async def analysis(self, task, prepared, result: RunResult, req, plan) -> AnalysisResult:
+        if result is None or result.trajectory is None:
+            return AnalysisResult(
+                task_id=task.task_id,
+                is_badcase=None,
+                error="no trajectory available",
+            )
+
+        steps = result.trajectory.steps
+        if not steps:
+            return AnalysisResult(
+                task_id=task.task_id,
+                is_badcase=None,
+                error="empty trajectory",
+            )
+
+        total_llm_steps = 0
+        no_json_count = 0
+        extra_text_count = 0
+        error_types: list[str] = []
+        error_steps_detail: list[dict] = []
+
+        for step in steps:
+            ac = step.assistant_content
+            if ac is None:
+                continue
+            total_llm_steps += 1
+
+            content = getattr(ac, "content", "") or ""
+            if isinstance(content, list):
+                content = " ".join(item.get("text", "") if isinstance(item, dict) else str(item) for item in content)
+
+            status = _check_json(content)
+
+            if status == "no_json":
+                no_json_count += 1
+                error_types.append("no_json")
+                error_steps_detail.append({
+                    "step_id": step.step_id,
+                    "error_type": "no_json",
+                    "content_snippet": content[:300],
+                })
+            elif status == "json_with_extra_text":
+                extra_text_count += 1
+                error_types.append("json_with_extra_text")
+                error_steps_detail.append({
+                    "step_id": step.step_id,
+                    "error_type": "json_with_extra_text",
+                    "content_snippet": content[:300],
+                })
+
+        # Task-level score: average across steps (no_json=1, json_with_extra_text=0.5, clean=0)
+        if total_llm_steps > 0:
+            score = round((no_json_count * 1.0 + extra_text_count * 0.5) / total_llm_steps, 4)
+        else:
+            score = 0.0
+
+        is_badcase = score > 0
+
+        details: dict = {
+            "total_llm_steps": total_llm_steps,
+            "no_json_steps": no_json_count,
+            "json_with_extra_text_steps": extra_text_count,
+            "json_error_types": error_types,
+        }
+        if error_steps_detail:
+            details["error_steps_detail"] = error_steps_detail
+
+        return AnalysisResult(
+            task_id=task.task_id,
+            is_badcase=is_badcase,
+            score=score,
+            details=details,
+        )

agentcompass/analyzers/basic/keyword_analyzer.py

@@ -0,0 +1,156 @@
+import logging
+
+from agentcompass.runtime import ANALYZERS, AnalysisResult, AnalyzerCategory, BaseAnalyzer, RunResult
+
+logger = logging.getLogger(__name__)
+
+
+def _find_keywords(text: str, keywords: list[str]) -> list[str]:
+    """Return the subset of *keywords* found in *text* (case-insensitive)."""
+    if not text or not keywords:
+        return []
+    lower = text.lower()
+    return [kw for kw in keywords if kw.lower() in lower]
+
+
+class KeywordAnalyzerBase(BaseAnalyzer):
+    """Base class for keyword detection.
+
+    Subclasses set ``check_fields`` to specify which ``assistant_content``
+    fields to inspect.  Keywords come from ``conf["keywords"]``.
+
+        keywords      – list of keyword strings to search for (required)
+        only_incorrect – only analyse incorrect samples
+        threshold      – overall badcase ratio threshold
+    """
+
+    description = "Base class for forbidden/sensitive keyword detection across assistant_content fields; subclasses choose which fields to inspect and which keywords to search for."
+    check_fields: list[str] = ["content", "reasoning_content"]
+
+    def _find_in_field(self, text: str) -> list[str]:
+        keywords = self.conf.get("keywords") or []
+        return _find_keywords(text, keywords)
+
+    async def analysis(self, task, prepared, result: RunResult, req, plan) -> AnalysisResult:
+        if result is None or result.trajectory is None:
+            return AnalysisResult(
+                task_id=task.task_id,
+                is_badcase=None,
+                error="no trajectory available",
+            )
+
+        steps = result.trajectory.steps
+        if not steps:
+            return AnalysisResult(
+                task_id=task.task_id,
+                is_badcase=None,
+                error="empty trajectory",
+            )
+
+        keywords = self.conf.get("keywords") or []
+        if not keywords:
+            return AnalysisResult(
+                task_id=task.task_id,
+                is_badcase=None,
+                error="no keywords configured",
+            )
+
+        total_llm_steps = 0
+        matched_steps = 0
+        all_matched_keywords: list[str] = []
+        matched_steps_detail: list[dict] = []
+
+        for step in steps:
+            ac = step.assistant_content
+            if ac is None:
+                continue
+            total_llm_steps += 1
+
+            step_matched = False
+            step_fields: dict = {}
+
+            for field_name in self.check_fields:
+                text = getattr(ac, field_name, "") or ""
+                if isinstance(text, list):
+                    text = " ".join(item.get("text", "") if isinstance(item, dict) else str(item) for item in text)
+                found = self._find_in_field(text)
+                if found:
+                    step_matched = True
+                    all_matched_keywords.extend(found)
+                    step_fields[field_name] = found
+
+            if step_matched:
+                matched_steps += 1
+                matched_steps_detail.append({
+                    "step_id": step.step_id,
+                    "matched_fields": step_fields,
+                })
+
+        is_badcase = matched_steps > 0
+
+        details: dict = {
+            "total_llm_steps": total_llm_steps,
+            "matched_steps": matched_steps,
+            "matched_keywords": all_matched_keywords,
+        }
+        if matched_steps_detail:
+            details["matched_steps_detail"] = matched_steps_detail
+
+        return AnalysisResult(
+            task_id=task.task_id,
+            is_badcase=is_badcase,
+            score=1 if is_badcase else 0,
+            details=details,
+        )
+
+
+@ANALYZERS.register()
+class ContentKeywordAnalyzer(KeywordAnalyzerBase):
+    """Detect forbidden/sensitive keywords in assistant content."""
+
+    id = "ContentKeywordAnalyzer"
+    description = "Detect forbidden/sensitive raw tokens (e.g. leaked special tokens like <think>, <tool_call>) in assistant content."
+    category = AnalyzerCategory.BEHAVIOR
+    datasets = []
+    data_requirements = ["$.trajectory.steps[*].assistant_content"]
+    check_fields = ["content"]
+    conf: dict = {
+        "threshold":
+        0.0,
+        "only_incorrect":
+        False,
+        "keywords": [
+            "<think>", "</think>", "<tool>", "</tool>", "<tool_call>", "</tool_call>", "<|end|>", "<|start|>",
+            "<|channe|>", "<|message|>"
+        ],
+    }
+    distribution_fields = {
+        "matched_steps": "numeric_stats",
+        "matched_keywords": "value_counts",
+    }
+
+
+@ANALYZERS.register()
+class ReasoningKeywordAnalyzer(KeywordAnalyzerBase):
+    """Detect forbidden/sensitive keywords in assistant reasoning_content."""
+
+    id = "ReasoningKeywordAnalyzer"
+    description = "Detect forbidden/sensitive raw tokens (e.g. leaked special tokens like <think>, <tool_call>) in assistant reasoning_content."
+    category = AnalyzerCategory.BEHAVIOR
+    datasets = []
+    data_requirements = ["$.trajectory.steps[*].assistant_content"]
+    check_fields = ["reasoning_content"]
+    conf: dict = {
+        "threshold":
+        0.0,
+        "only_incorrect":
+        False,
+        "keywords": [
+            "<think>", "</think>", "<tool>", "</tool>", "<tool_call>", "</tool_call>", "<|end|>", "<|start|>",
+            "<|channe|>", "<|message|>"
+        ],
+    }
+    distribution_fields = {
+        "matched_steps": "numeric_stats",
+        "matched_keywords": "value_counts",
+    }