traceval 0.1.1__py3-none-any.whl

traceval/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.1"

traceval/analyze/__init__.py

@@ -0,0 +1,75 @@
+from pathlib import Path
+from typing import Any
+
+from traceval.analyze.cluster import JaccardClusterer
+from traceval.analyze.coverage import compute_coverage
+from traceval.analyze.outcomes import label_trace, load_user_rules
+from traceval.analyze.report import render_report
+from traceval.store import TraceStore
+
+
+def run_analysis(
+    db_path: Path,
+    rules_path: Path | None = None,
+    evals_dir: Path | None = None,
+    output_dir: Path | None = None,
+) -> dict[str, Any]:
+    store = TraceStore(db_path)
+    try:
+        # 1. Load user rules if any
+        user_rules = None
+        if rules_path:
+            user_rules = load_user_rules(rules_path)
+
+        # 2. Label all traces
+        traces = list(store.list_traces())
+        for trace in traces:
+            outcome = label_trace(trace, user_rules=user_rules)
+            trace.outcome = outcome
+            store.save_trace(trace)
+
+        # 3. Fetch fresh labeled traces
+        labeled_traces = list(store.list_traces())
+
+        # 4. Cluster traces
+        clusterer = JaccardClusterer()
+        clusters = clusterer.cluster(labeled_traces)
+
+        # 5. Compute coverage of clusters against existing evals
+        coverage = compute_coverage(clusters, evals_dir, labeled_traces)
+
+        # 6. Render reports if output_dir is provided
+        summary = {
+            "total_traces": len(labeled_traces),
+            "outcomes": {},
+            "clusters": [],
+        }
+
+        # Populate summary outcomes counts
+        outcome_counts: dict[str, int] = {}
+        for t in labeled_traces:
+            lbl = t.outcome.label if t.outcome else "unknown"
+            outcome_counts[lbl] = outcome_counts.get(lbl, 0) + 1
+        summary["outcomes"] = outcome_counts
+
+        summary["clusters"] = [
+            {
+                "id": c.id,
+                "name": c.name,
+                "tool_signature": c.tool_signature,
+                "top_terms": c.top_terms,
+                "trace_count": len(c.trace_ids),
+                "trace_ids": c.trace_ids,
+                "coverage_count": coverage.get(c.id, 0),
+            }
+            for c in clusters
+        ]
+
+        if output_dir:
+            output_dir.mkdir(parents=True, exist_ok=True)
+            html_path = output_dir / "report.html"
+            render_report(summary, coverage, html_path)
+
+        return summary
+    finally:
+        store.close()

traceval/analyze/cluster.py

@@ -0,0 +1,226 @@
+import hashlib
+import math
+import re
+from collections import Counter
+from typing import Protocol
+
+from pydantic import BaseModel
+
+from traceval.model import Trace
+
+
+class Cluster(BaseModel):
+    id: str
+    name: str
+    trace_ids: list[str]
+    tool_signature: str
+    top_terms: list[str]
+
+
+class Clusterer(Protocol):
+    def cluster(self, traces: list[Trace]) -> list[Cluster]: ...
+
+
+# Standard English stopwords
+STOPWORDS: set[str] = {
+    "a",
+    "an",
+    "the",
+    "and",
+    "or",
+    "but",
+    "is",
+    "are",
+    "was",
+    "were",
+    "to",
+    "for",
+    "in",
+    "of",
+    "on",
+    "at",
+    "by",
+    "with",
+    "this",
+    "that",
+    "these",
+    "those",
+    "i",
+    "you",
+    "he",
+    "she",
+    "it",
+    "we",
+    "they",
+    "my",
+    "your",
+    "his",
+    "her",
+    "its",
+    "our",
+    "their",
+    "me",
+    "him",
+    "us",
+    "them",
+    "what",
+    "where",
+    "how",
+    "why",
+    "please",
+    "can",
+    "get",
+    "show",
+    "find",
+}
+
+
+def tokenize(text: str) -> list[str]:
+    # Lowercase and strip non-alphanumeric
+    text = text.lower()
+    words = re.findall(r"\b[a-z0-9_]+\b", text)
+    return [w for w in words if w not in STOPWORDS]
+
+
+def get_ngrams(tokens: list[str], max_n: int = 3) -> set[tuple[str, ...]]:
+    ngrams: set[tuple[str, ...]] = set()
+    n_tokens = len(tokens)
+    for n in range(1, min(max_n + 1, n_tokens + 1)):
+        for i in range(n_tokens - n + 1):
+            ngrams.add(tuple(tokens[i : i + n]))
+    return ngrams
+
+
+def get_tool_signature(trace: Trace) -> str:
+    tool_names = []
+    for step in trace.steps:
+        if step.kind == "tool" and step.tool:
+            tool_names.append(step.tool.name)
+    return ">".join(tool_names) if tool_names else ""
+
+
+class JaccardClusterer:
+    def __init__(self, jaccard_threshold: float = 0.35) -> None:
+        self.jaccard_threshold = jaccard_threshold
+
+    def cluster(self, traces: list[Trace]) -> list[Cluster]:
+        if not traces:
+            return []
+
+        # 1. Group traces by exact tool signature
+        traces_by_sig: dict[str, list[Trace]] = {}
+        for trace in traces:
+            sig = get_tool_signature(trace)
+            traces_by_sig.setdefault(sig, []).append(trace)
+
+        raw_clusters: list[tuple[str, list[Trace]]] = []
+
+        # 2. Within each signature group, apply greedy Jaccard
+        # clustering on task_input ngrams
+        for sig, sig_traces in traces_by_sig.items():
+            # Keep trace list sorted by trace_id to ensure determinism
+            sorted_traces = sorted(sig_traces, key=lambda t: t.trace_id)
+
+            sig_clusters: list[list[Trace]] = []
+            cluster_ngrams: list[set[tuple[str, ...]]] = []
+
+            for trace in sorted_traces:
+                tokens = tokenize(trace.task_input)
+                ngrams = get_ngrams(tokens)
+
+                matched_idx = -1
+                for idx, seed_ngrams in enumerate(cluster_ngrams):
+                    if not ngrams or not seed_ngrams:
+                        jaccard = 0.0
+                    else:
+                        intersection = len(ngrams.intersection(seed_ngrams))
+                        union = len(ngrams.union(seed_ngrams))
+                        jaccard = intersection / union if union > 0 else 0.0
+
+                    if jaccard >= self.jaccard_threshold:
+                        matched_idx = idx
+                        break
+
+                if matched_idx != -1:
+                    sig_clusters[matched_idx].append(trace)
+                else:
+                    sig_clusters.append([trace])
+                    cluster_ngrams.append(ngrams)
+
+            for group in sig_clusters:
+                raw_clusters.append((sig, group))
+
+        # 3. Calculate TF-IDF of terms per cluster to name them
+        # Term counts per cluster
+        cluster_term_docs: list[Counter[str]] = []
+        doc_frequencies: Counter[str] = Counter()
+        all_terms: set[str] = set()
+
+        for _sig, group in raw_clusters:
+            terms = []
+            for t in group:
+                terms.extend(tokenize(t.task_input))
+            counter = Counter(terms)
+            cluster_term_docs.append(counter)
+            for term in counter:
+                doc_frequencies[term] += 1
+                all_terms.add(term)
+
+        num_clusters = len(raw_clusters)
+        idfs: dict[str, float] = {}
+        for term in all_terms:
+            # Standard smooth IDF formula
+            idfs[term] = math.log((num_clusters + 1) / (doc_frequencies[term] + 1)) + 1
+
+        clusters: list[Cluster] = []
+        for c_idx, (sig, group) in enumerate(raw_clusters):
+            # Compute TF-IDF for each term in this cluster
+            term_scores = {}
+            counter = cluster_term_docs[c_idx]
+            for term, count in counter.items():
+                tf = count  # raw term frequency in the cluster docs
+                term_scores[term] = tf * idfs[term]
+
+            # Sort terms by TF-IDF score
+            sorted_terms = sorted(
+                term_scores.keys(), key=lambda term: term_scores[term], reverse=True
+            )
+            top_terms = sorted_terms[:3]
+
+            # Reconstruct signature string for title (replace '>' with ' -> ')
+            formatted_sig = sig.replace(">", " -> ") if sig else ""
+
+            # Outcome suffix (optional, if we want to differentiate failure clusters)
+            outcomes = [t.outcome.label if t.outcome else "unknown" for t in group]
+            outcome_counter = Counter(outcomes)
+            dominant_outcome = outcome_counter.most_common(1)[0][0]
+
+            # Build name
+            name_parts = []
+            if top_terms:
+                name_parts.append(" ".join(top_terms))
+            if formatted_sig:
+                name_parts.append(formatted_sig)
+            if dominant_outcome != "success":
+                name_parts.append(f"({dominant_outcome})")
+
+            name = " -> ".join(name_parts) if name_parts else "unclassified"
+
+            # Stable content hash ID based on sorted list of trace_ids in the cluster
+            sorted_ids = sorted(t.trace_id for t in group)
+            joined_ids = ",".join(sorted_ids)
+            content_hash = hashlib.sha256(joined_ids.encode("utf-8")).hexdigest()[:8]
+            cluster_id = f"c_{content_hash}"
+
+            clusters.append(
+                Cluster(
+                    id=cluster_id,
+                    name=name,
+                    trace_ids=sorted_ids,
+                    tool_signature=sig,
+                    top_terms=top_terms,
+                )
+            )
+
+        # Sort clusters by id for deterministic outputs
+        return sorted(clusters, key=lambda c: c.id)

traceval/analyze/coverage.py

@@ -0,0 +1,84 @@
+from pathlib import Path
+
+import yaml
+
+from traceval.analyze.cluster import Cluster, get_ngrams, tokenize
+from traceval.model import Trace
+
+
+def compute_coverage(
+    clusters: list[Cluster],
+    evals_dir: Path | None,
+    traces: list[Trace],
+) -> dict[str, int]:
+    coverage: dict[str, int] = {c.id: 0 for c in clusters}
+    if not evals_dir or not evals_dir.exists():
+        return coverage
+
+    # 1. Parse all eval files (*.yaml) in evals_dir recursively
+    eval_cases = []
+    for p in evals_dir.rglob("*.yaml"):
+        # Skip potential metadata yaml like traceval.yaml
+        if p.name == "traceval.yaml":
+            continue
+        try:
+            with open(p, encoding="utf-8") as f:
+                data = yaml.safe_load(f)
+                if isinstance(data, dict) and "input" in data:
+                    eval_cases.append(data)
+        except Exception:
+            pass
+
+    # 2. For each eval case, map to a cluster
+    traces_by_id = {t.trace_id: t for t in traces}
+    cluster_by_id = {c.id: c for c in clusters}
+
+    for case in eval_cases:
+        mapped_cluster_id = None
+
+        # Scenario A: case has a explicit 'cluster' field
+        case_cluster = case.get("cluster")
+        if case_cluster and case_cluster in cluster_by_id:
+            mapped_cluster_id = case_cluster
+
+        # Scenario B: look up the source trace and find which cluster contains it
+        elif "source_trace_id" in case:
+            tid = case["source_trace_id"]
+            for c in clusters:
+                if tid in c.trace_ids:
+                    mapped_cluster_id = c.id
+                    break
+
+        # Scenario C: fallback to input similarity match
+        if not mapped_cluster_id:
+            case_input = case.get("input", "")
+            case_ngrams = get_ngrams(tokenize(case_input))
+            best_sim = -1.0
+            best_cid = None
+
+            for c in clusters:
+                # Compare similarity with each trace in the cluster
+                for tid in c.trace_ids:
+                    t = traces_by_id.get(tid)
+                    if not t:
+                        continue
+                    t_ngrams = get_ngrams(tokenize(t.task_input))
+                    if not case_ngrams or not t_ngrams:
+                        sim = 0.0
+                    else:
+                        union_len = len(case_ngrams.union(t_ngrams))
+                        sim = (
+                            len(case_ngrams.intersection(t_ngrams)) / union_len
+                            if union_len > 0
+                            else 0.0
+                        )
+                    if sim > best_sim:
+                        best_sim = sim
+                        best_cid = c.id
+            if best_sim >= 0.35:
+                mapped_cluster_id = best_cid
+
+        if mapped_cluster_id:
+            coverage[mapped_cluster_id] += 1
+
+    return coverage

traceval/analyze/outcomes.py

@@ -0,0 +1,254 @@
+import importlib.util
+import logging
+import re
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from traceval.model import Outcome, Trace
+
+
+class Rule:
+    def __init__(
+        self,
+        rule_id: str,
+        description: str,
+        fn: Callable[[Trace], Outcome | None],
+    ) -> None:
+        self.rule_id = rule_id
+        self.description = description
+        self.fn = fn
+
+
+def R_TOOL_ERROR(trace: Trace) -> Outcome | None:
+    for step in trace.steps:
+        if step.kind == "tool" and step.tool and step.tool.error:
+            return Outcome(
+                label="tool_error",
+                reason=f"Tool '{step.tool.name}' failed with error: {step.tool.error}",
+                labeled_by="rule",
+                rule_id="R_TOOL_ERROR",
+            )
+    return None
+
+
+def R_LLM_ERROR(trace: Trace) -> Outcome | None:
+    for step in trace.steps:
+        if step.kind == "llm" and step.llm and step.llm.error:
+            return Outcome(
+                label="bad_output",
+                reason=(
+                    f"LLM call in step {step.index} failed with error: {step.llm.error}"
+                ),
+                labeled_by="rule",
+                rule_id="R_LLM_ERROR",
+            )
+    return None
+
+
+def R_LOOP(trace: Trace, loop_step_threshold: int = 25) -> Outcome | None:
+    # 1. Check total steps
+    if len(trace.steps) > loop_step_threshold:
+        return Outcome(
+            label="loop",
+            reason=(
+                f"Total steps ({len(trace.steps)}) exceeded loop step "
+                f"threshold ({loop_step_threshold})"
+            ),
+            labeled_by="rule",
+            rule_id="R_LOOP",
+        )
+
+    # 2. Check identical consecutive tool calls (>=3 times)
+    consec_count = 0
+    last_name = None
+    last_args = None
+
+    for step in trace.steps:
+        if step.kind == "tool" and step.tool:
+            curr_name = step.tool.name
+            curr_args = step.tool.arguments_json
+            if curr_name == last_name and curr_args == last_args:
+                consec_count += 1
+                if consec_count >= 3:
+                    return Outcome(
+                        label="loop",
+                        reason=(
+                            f"Tool '{curr_name}' called with identical "
+                            f"arguments {consec_count} times consecutively"
+                        ),
+                        labeled_by="rule",
+                        rule_id="R_LOOP",
+                    )
+            else:
+                consec_count = 1
+                last_name = curr_name
+                last_args = curr_args
+        else:
+            # reset consecutive count if another step kind occurs
+            consec_count = 0
+            last_name = None
+            last_args = None
+
+    return None
+
+
+def R_TIMEOUT(trace: Trace, timeout_s: float = 300.0) -> Outcome | None:
+    if not trace.ended_at:
+        return Outcome(
+            label="timeout",
+            reason="Trace has not completed (ended_at is missing)",
+            labeled_by="rule",
+            rule_id="R_TIMEOUT",
+        )
+
+    duration = (trace.ended_at - trace.started_at).total_seconds()
+    if duration > timeout_s:
+        return Outcome(
+            label="timeout",
+            reason=f"Trace duration ({duration:.1f}s) exceeded limit of {timeout_s}s",
+            labeled_by="rule",
+            rule_id="R_TIMEOUT",
+        )
+    return None
+
+
+DEFAULT_VALIDATION_REGEXES = [
+    re.compile(r"validation\s+error", re.IGNORECASE),
+    re.compile(r"field\s+required", re.IGNORECASE),
+    re.compile(r"value\s+is\s+not\s+a\s+valid", re.IGNORECASE),
+    re.compile(r"json-schema", re.IGNORECASE),
+    re.compile(r"json\s+schema\s+error", re.IGNORECASE),
+]
+
+
+def R_VALIDATION(trace: Trace) -> Outcome | None:
+    if not trace.final_output:
+        return None
+    for pattern in DEFAULT_VALIDATION_REGEXES:
+        if pattern.search(trace.final_output):
+            return Outcome(
+                label="validation_error",
+                reason=(
+                    f"Final output matched validation signature pattern: "
+                    f"'{pattern.pattern}'"
+                ),
+                labeled_by="rule",
+                rule_id="R_VALIDATION",
+            )
+    return None
+
+
+def R_EMPTY_OUTPUT(trace: Trace) -> Outcome | None:
+    if trace.task_input and not trace.final_output:
+        return Outcome(
+            label="bad_output",
+            reason="Task input is non-empty, but final output is empty or missing",
+            labeled_by="rule",
+            rule_id="R_EMPTY_OUTPUT",
+        )
+    return None
+
+
+def R_DEFAULT_SUCCESS(trace: Trace) -> Outcome | None:
+    if trace.final_output:
+        return Outcome(
+            label="success",
+            reason="Trace completed with non-empty final output",
+            labeled_by="rule",
+            rule_id="R_DEFAULT_SUCCESS",
+        )
+    return None
+
+
+def R_UNKNOWN(trace: Trace) -> Outcome | None:
+    return Outcome(
+        label="unknown",
+        reason="No rules matched this trace",
+        labeled_by="rule",
+        rule_id="R_UNKNOWN",
+    )
+
+
+BUILTIN_RULES = [
+    R_TOOL_ERROR,
+    R_LLM_ERROR,
+    R_LOOP,
+    R_TIMEOUT,
+    R_VALIDATION,
+    R_EMPTY_OUTPUT,
+    R_DEFAULT_SUCCESS,
+    R_UNKNOWN,
+]
+
+
+def load_user_rules(rules_path: str | Path) -> list[Any]:
+    path = Path(rules_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Custom rules file not found: {rules_path}")
+
+    spec = importlib.util.spec_from_file_location("custom_rules", path)
+    if spec is None or spec.loader is None:
+        raise ValueError(f"Could not load custom rules from {rules_path}")
+
+    mod = importlib.util.module_from_spec(spec)
+    sys.modules["custom_rules"] = mod
+    spec.loader.exec_module(mod)
+
+    rules = getattr(mod, "RULES", [])
+    return rules
+
+
+def label_trace(
+    trace: Trace,
+    user_rules: list[Any] | None = None,
+    loop_step_threshold: int = 25,
+    timeout_s: float = 300.0,
+) -> Outcome:
+    # 1. Run user rules first
+    if user_rules:
+        for rule in user_rules:
+            # support both Rule objects and simple callables
+            fn = getattr(rule, "fn", rule)
+            rule_id = getattr(rule, "rule_id", getattr(fn, "__name__", "user_rule"))
+            try:
+                res = fn(trace)
+                if res is not None:
+                    # ensure correct metadata is set for user rule label
+                    if isinstance(res, Outcome):
+                        return Outcome(
+                            label=res.label,
+                            reason=res.reason,
+                            labeled_by="user_rule",
+                            rule_id=res.rule_id or rule_id,
+                        )
+                    return Outcome(
+                        label=res,
+                        reason=f"Matched custom rule '{rule_id}'",
+                        labeled_by="user_rule",
+                        rule_id=rule_id,
+                    )
+            except Exception as e:
+                logger = logging.getLogger("traceval.analyze")
+                logger.error("Error executing user rule '%s': %s", rule_id, str(e))
+
+    # 2. Run built-in rules
+    # loop and timeout require extra arguments
+    for rule_fn in BUILTIN_RULES:
+        if rule_fn is R_LOOP:
+            res = R_LOOP(trace, loop_step_threshold=loop_step_threshold)
+        elif rule_fn is R_TIMEOUT:
+            res = R_TIMEOUT(trace, timeout_s=timeout_s)
+        else:
+            res = rule_fn(trace)
+
+        if res is not None:
+            return res
+
+    return Outcome(
+        label="unknown",
+        reason="Default fallback",
+        labeled_by="rule",
+        rule_id="R_UNKNOWN",
+    )