zeno-cli 0.3.4__py3-none-any.whl

zeno_session_intel/signals.py

@@ -0,0 +1,319 @@
+"""Agent-behavioral signal extraction from a parsed transcript.
+
+Ported from agentsview (MIT, see THIRD_PARTY_LICENSES.md): tool-health
+(``internal/signals/toolhealth.go``), context pressure/compaction
+(``internal/signals/context.go``), outcome classification
+(``internal/signals/outcome.go``), and the health score (``internal/signals/score.go``,
+ported verbatim - the penalty constants are the contract).
+
+These score *agent* behavior and are stored on ``transcript_sessions``. They are
+ORTHOGONAL to zeno's human-cognition ``attention_score``; the two complement, never
+overlap. Pure computation over a ``ParsedSession``; stdlib-only, never raises.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+
+from .model import ParsedSession
+from .taxonomy import normalize_tool_category
+
+# tool-failure content heuristics (toolhealth.go IsFailure)
+_GOROUTINE = re.compile(r"goroutine \d+")
+_EXIT_CODE = re.compile(r"exit (status|code) [1-9]\d*")
+_JS_AT = re.compile(r"^\s+at ", re.MULTILINE)
+_BASH_FAIL_SUBSTR = ("command not found", "permission denied", "traceback (most recent call last)")
+_GIVE_UP = (
+    "i'm unable to",
+    "i am unable to",
+    "i can't proceed",
+    "i cannot proceed",
+    "i don't have access",
+)
+_EDIT_PATH_KEYS = ("file_path", "path", "target_file", "filePath")
+
+
+@dataclass
+class SessionSignals:
+    message_count: int = 0
+    user_message_count: int = 0
+    total_output_tokens: int = 0
+    peak_context_tokens: int = 0
+    is_automated: bool = False
+    tool_failure_signal_count: int = 0
+    tool_retry_count: int = 0
+    edit_churn_count: int = 0
+    consecutive_failure_max: int = 0
+    final_failure_streak: int = 0
+    compaction_count: int = 0
+    mid_task_compaction_count: int = 0
+    compaction_boundaries: list[int] = field(default_factory=list)  # message ordinals
+    context_pressure_max: float | None = None
+    has_tool_calls: bool = False
+    has_context_data: bool = False
+    ended_with_role: str = ""
+    outcome: str = "unknown"
+    outcome_confidence: str = "low"
+    health_score: int | None = None
+    health_grade: str = ""
+    penalties: dict[str, int] = field(default_factory=dict)
+
+
+def _is_failure(category: str, result_content: str, is_error: bool) -> bool:
+    """A tool result is a failure if flagged is_error, or its content matches the
+    category-specific heuristics (Bash error shapes; Edit/Write 'FAILED')."""
+    if is_error:
+        return True
+    text = result_content or ""
+    low = text.lower()
+    if category in ("Bash",):
+        if any(s in low for s in _BASH_FAIL_SUBSTR):
+            return True
+        if _GOROUTINE.search(text) or _EXIT_CODE.search(low):
+            return True
+        if len(_JS_AT.findall(text)) >= 3:
+            return True
+    if category in ("Edit", "Write") and "FAILED" in text:
+        return True
+    return False
+
+
+def _tool_failure_sequence(session: ParsedSession) -> list[bool]:
+    """Per-tool-result failure flags in message order. A result is matched to the
+    tool_use it answers (by category, via the preceding assistant turn's tool_uses)."""
+    # map tool_use_id -> category for results that carry an id; else fall back to
+    # the most recent assistant tool_use category.
+    id_to_cat: dict[str, str] = {}
+    last_cat = "Other"
+    seq: list[bool] = []
+    for m in session.messages:
+        for tu in m.tool_uses:
+            cat = normalize_tool_category(tu.name)
+            last_cat = cat
+            if tu.id:
+                id_to_cat[tu.id] = cat
+        for tr in m.tool_results:
+            cat = id_to_cat.get(tr.tool_use_id, last_cat)
+            seq.append(_is_failure(cat, tr.content, tr.is_error))
+    return seq
+
+
+def _retry_count(session: ParsedSession) -> int:
+    """Runs of 3+ identical (tool_name, input_json) -> sum(runLen - 1)."""
+    calls = [(tu.name, tu.input_json) for m in session.messages for tu in m.tool_uses]
+    total = 0
+    i = 0
+    n = len(calls)
+    while i < n:
+        j = i + 1
+        while j < n and calls[j] == calls[i]:
+            j += 1
+        run = j - i
+        if run >= 3:
+            total += run - 1
+        i = j
+    return total
+
+
+def _edit_churn_count(session: ParsedSession) -> int:
+    """Excess Edit/Write touches: a touch counts as churn when the same file already
+    had >=2 touches within the trailing 10-ordinal window."""
+    touches: list[tuple[int, str]] = []  # (ordinal, file)
+    for m in session.messages:
+        for tu in m.tool_uses:
+            if normalize_tool_category(tu.name) in ("Edit", "Write"):
+                path = _extract_path(tu.input_json)
+                if path:
+                    touches.append((m.ordinal, path))
+    churn = 0
+    for idx, (ordinal, path) in enumerate(touches):
+        prior = [1 for o, p in touches[:idx] if p == path and (ordinal - o) <= 10]
+        if len(prior) >= 2:
+            churn += 1
+    return churn
+
+
+def _extract_path(input_json: str) -> str:
+    if not input_json:
+        return ""
+    try:
+        obj = json.loads(input_json)
+    except Exception:
+        return ""
+    if isinstance(obj, dict):
+        for k in _EDIT_PATH_KEYS:
+            v = obj.get(k)
+            if isinstance(v, str) and v:
+                return v
+    return ""
+
+
+def _compactions(session: ParsedSession) -> tuple[int, list[int]]:
+    """Count >30% context-token drops between consecutive turns that carry context.
+    Returns (count, list of boundary message-indexes)."""
+    count = 0
+    boundaries: list[int] = []
+    prev: int | None = None
+    for idx, m in enumerate(session.messages):
+        ctx = m.context_tokens
+        if ctx <= 0:
+            continue
+        if prev is not None and prev * 0.7 > ctx:
+            count += 1
+            boundaries.append(idx)
+        prev = ctx
+    return count, boundaries
+
+
+def _mid_task_compactions(session: ParsedSession, boundaries: list[int]) -> int:
+    """A compaction is mid-task when the agent resumes similar work across it: >=2 tool
+    names shared between the 5 messages after the boundary and the 10 before it."""
+    msgs = session.messages
+    mid = 0
+    for b in boundaries:
+        before = {
+            normalize_tool_category(tu.name) for m in msgs[max(0, b - 10) : b] for tu in m.tool_uses
+        }
+        after = {normalize_tool_category(tu.name) for m in msgs[b : b + 5] for tu in m.tool_uses}
+        if len(before & after) >= 2:
+            mid += 1
+    return mid
+
+
+def _classify_outcome(
+    *,
+    is_automated: bool,
+    message_count: int,
+    ended_with_role: str,
+    final_failure_streak: int,
+    last_text: str,
+) -> tuple[str, str]:
+    """Port of outcome.go: heuristic on conversation shape, not content quality."""
+    if is_automated:
+        return "unknown", "low"
+    if message_count == 2 and ended_with_role == "assistant":
+        return "completed", "medium"
+    if message_count < 3:
+        return "unknown", "low"
+    if ended_with_role == "user":
+        return "abandoned", ("high" if message_count >= 10 else "medium")
+    if final_failure_streak >= 3:
+        return "errored", "medium"
+    if ended_with_role == "assistant":
+        gave_up = any(p in last_text.lower() for p in _GIVE_UP)
+        return "completed", ("low" if gave_up else "medium")
+    return "unknown", "low"
+
+
+def _cap(raw: int, cap: int) -> int:
+    return cap if raw > cap else raw
+
+
+def compute_health_score(s: SessionSignals) -> tuple[int | None, str, dict[str, int]]:
+    """Verbatim port of agentsview signals/score.go ComputeHealthScore.
+
+    Start 100, floor 0. Unscored (None) iff outcome==unknown AND confidence==low AND no
+    tool/context data. Grade: >=90 A, >=75 B, >=60 C, >=40 D, else F.
+    """
+    basis = ["outcome"]
+    if s.has_tool_calls:
+        basis.append("tool_health")
+    if s.has_context_data:
+        basis.append("context_pressure")
+    can_score = (s.outcome != "unknown" or s.outcome_confidence != "low") or len(basis) > 1
+    if not can_score:
+        return None, "", {}
+
+    pen: dict[str, int] = {}
+    if s.outcome == "errored":
+        pen["outcome_errored"] = 30
+    elif s.outcome == "abandoned":
+        pen["outcome_abandoned"] = 15
+    if s.has_tool_calls:
+        if (p := _cap(s.tool_failure_signal_count * 3, 30)) > 0:
+            pen["tool_failure_signals"] = p
+        if (p := _cap(s.tool_retry_count * 5, 25)) > 0:
+            pen["tool_retries"] = p
+        if (p := _cap(s.edit_churn_count * 4, 20)) > 0:
+            pen["edit_churn"] = p
+        if s.consecutive_failure_max >= 3:
+            pen["consecutive_failures"] = 10
+    if s.has_context_data:
+        if s.compaction_count >= 2 and (p := _cap((s.compaction_count - 1) * 5, 15)) > 0:
+            pen["compactions"] = p
+        if s.mid_task_compaction_count > 0 and (p := _cap(s.mid_task_compaction_count * 8, 18)) > 0:
+            pen["mid_task_compactions"] = p
+        if s.context_pressure_max is not None and s.context_pressure_max > 0.9:
+            pen["context_pressure_high"] = 10
+
+    score = max(0, 100 - sum(pen.values()))
+    grade = (
+        "A"
+        if score >= 90
+        else "B" if score >= 75 else "C" if score >= 60 else "D" if score >= 40 else "F"
+    )
+    return score, grade, pen
+
+
+def extract_signals(session: ParsedSession, *, context_window: int | None = None) -> SessionSignals:
+    """Compute all behavioral signals + the health score for one parsed session."""
+    msgs = session.messages
+    s = SessionSignals()
+    s.message_count = len(msgs)
+    s.user_message_count = sum(1 for m in msgs if m.role == "user")
+    s.total_output_tokens = sum(int(m.output_tokens or 0) for m in msgs)
+    s.peak_context_tokens = max((m.context_tokens for m in msgs), default=0)
+    s.has_tool_calls = any(m.tool_uses for m in msgs)
+    s.has_context_data = any(m.context_tokens > 0 for m in msgs)
+    s.ended_with_role = msgs[-1].role if msgs else ""
+
+    # automation heuristic: a one-shot autonomous run (>=6 turns, <=1 human turn).
+    s.is_automated = s.message_count >= 6 and s.user_message_count <= 1 and s.has_tool_calls
+
+    failures = _tool_failure_sequence(session)
+    s.tool_failure_signal_count = sum(1 for f in failures if f)
+    s.consecutive_failure_max = _max_run(failures)
+    s.final_failure_streak = _trailing_run(failures)
+    s.tool_retry_count = _retry_count(session)
+    s.edit_churn_count = _edit_churn_count(session)
+
+    s.compaction_count, boundaries = _compactions(session)
+    s.mid_task_compaction_count = _mid_task_compactions(session, boundaries)
+    s.compaction_boundaries = [msgs[i].ordinal for i in boundaries if 0 <= i < len(msgs)]
+    if context_window and s.peak_context_tokens > 0:
+        s.context_pressure_max = round(s.peak_context_tokens / float(context_window), 4)
+
+    last_text = ""
+    for m in reversed(msgs):
+        if m.role == "assistant" and m.content:
+            last_text = m.content
+            break
+    s.outcome, s.outcome_confidence = _classify_outcome(
+        is_automated=s.is_automated,
+        message_count=s.message_count,
+        ended_with_role=s.ended_with_role,
+        final_failure_streak=s.final_failure_streak,
+        last_text=last_text,
+    )
+    s.health_score, s.health_grade, s.penalties = compute_health_score(s)
+    return s
+
+
+def _max_run(flags: list[bool]) -> int:
+    best = run = 0
+    for f in flags:
+        run = run + 1 if f else 0
+        best = max(best, run)
+    return best
+
+
+def _trailing_run(flags: list[bool]) -> int:
+    run = 0
+    for f in reversed(flags):
+        if f:
+            run += 1
+        else:
+            break
+    return run

zeno_session_intel/taxonomy.py

@@ -0,0 +1,71 @@
+"""Tool-name -> category normalization for tool-mix analytics.
+
+Ported from agentsview ``internal/parser/taxonomy.go`` (MIT, see THIRD_PARTY_LICENSES.md):
+a flat name->bucket map plus a ``"subagent" in name -> Task`` fallback, collapsing the
+long tail of agent tool names into 9 stable buckets. Categorize at ingest so the
+dashboard tool-mix is a plain ``GROUP BY category``.
+"""
+
+from __future__ import annotations
+
+CATEGORIES = ("Read", "Edit", "Write", "Bash", "Grep", "Glob", "Task", "Tool", "Other")
+
+# explicit name -> category (lowercased keys). Covers Claude Code + Codex + Cursor names.
+_MAP: dict[str, str] = {
+    # read
+    "read": "Read",
+    "read_file": "Read",
+    "notebookread": "Read",
+    "view": "Read",
+    # edit
+    "edit": "Edit",
+    "multiedit": "Edit",
+    "str_replace_based_edit_tool": "Edit",
+    "search_replace": "Edit",
+    "apply_patch": "Edit",
+    "notebookedit": "Edit",
+    # write
+    "write": "Write",
+    "create_file": "Write",
+    "write_file": "Write",
+    # bash / shell
+    "bash": "Bash",
+    "bashoutput": "Bash",
+    "killshell": "Bash",
+    "run_terminal_cmd": "Bash",
+    "shell": "Bash",
+    "local_shell": "Bash",
+    # grep / search
+    "grep": "Grep",
+    "codebase_search": "Grep",
+    "search": "Grep",
+    # glob
+    "glob": "Glob",
+    "list_dir": "Glob",
+    "file_search": "Glob",
+    # task / subagent
+    "task": "Task",
+    "agent": "Task",
+    # generic tool buckets (web, mcp, todo, etc. that are not core file/shell ops)
+    "webfetch": "Tool",
+    "websearch": "Tool",
+    "web_search": "Tool",
+    "fetch": "Tool",
+    "todowrite": "Tool",
+    "update_plan": "Tool",
+}
+
+
+def normalize_tool_category(name: str | None) -> str:
+    """Map a raw tool name to one of the 9 buckets. Unknown -> Other; any name
+    containing 'subagent' -> Task (mirrors agentsview's fallback)."""
+    if not name:
+        return "Other"
+    key = name.strip().lower()
+    if key in _MAP:
+        return _MAP[key]
+    if "subagent" in key:
+        return "Task"
+    if key.startswith("mcp__") or key.startswith("mcp_"):
+        return "Tool"
+    return "Other"