agent-ablation 0.3.0__py3-none-any.whl

agent_ablation/__init__.py

@@ -0,0 +1,76 @@
+"""
+agent-ablation: Leave-one-out ablation testing, backward elimination, and ROI evaluation for multi-agent systems.
+"""
+
+from agent_ablation.adapters.autogen import from_autogen_messages
+from agent_ablation.adapters.crewai import from_crewai_tasks
+from agent_ablation.adapters.langgraph import from_langgraph_messages, from_records
+from agent_ablation.adapters.vercel import from_ai_sdk_steps
+from agent_ablation.core import (
+    aggregate_batch_results,
+    batch_ablation,
+    batch_ablation_async,
+    majority_vote,
+    run_ablation,
+    run_ablation_async,
+    run_backward_elimination,
+    run_backward_elimination_async,
+    run_pairwise_ablation,
+)
+from agent_ablation.models import (
+    AblationResult,
+    AgentRoiMetrics,
+    BackwardEliminationResult,
+    BackwardEliminationStep,
+    BatchAblationSummary,
+    Finding,
+    GroundTruthSummary,
+    PairwiseAblationItem,
+    PairwiseAblationResult,
+    PerAgentAblation,
+    PerAgentStats,
+    PruningRecommendation,
+    RoiSummary,
+)
+from agent_ablation.reporters.formatters import (
+    format_ascii_table,
+    format_markdown_report,
+)
+
+__version__ = "0.3.0"
+
+__all__ = [
+    # Models
+    "Finding",
+    "PerAgentAblation",
+    "AblationResult",
+    "AgentRoiMetrics",
+    "PruningRecommendation",
+    "RoiSummary",
+    "GroundTruthSummary",
+    "PerAgentStats",
+    "BatchAblationSummary",
+    "BackwardEliminationStep",
+    "BackwardEliminationResult",
+    "PairwiseAblationItem",
+    "PairwiseAblationResult",
+    # Core Functions
+    "run_ablation",
+    "run_ablation_async",
+    "batch_ablation",
+    "batch_ablation_async",
+    "aggregate_batch_results",
+    "run_backward_elimination",
+    "run_backward_elimination_async",
+    "run_pairwise_ablation",
+    "majority_vote",
+    # Adapters
+    "from_langgraph_messages",
+    "from_records",
+    "from_crewai_tasks",
+    "from_autogen_messages",
+    "from_ai_sdk_steps",
+    # Reporters
+    "format_markdown_report",
+    "format_ascii_table",
+]

agent_ablation/adapters/__init__.py

@@ -0,0 +1,16 @@
+"""
+Framework adapters for agent-ablation.
+"""
+
+from agent_ablation.adapters.autogen import from_autogen_messages
+from agent_ablation.adapters.crewai import from_crewai_tasks
+from agent_ablation.adapters.langgraph import from_langgraph_messages, from_records
+from agent_ablation.adapters.vercel import from_ai_sdk_steps
+
+__all__ = [
+    "from_langgraph_messages",
+    "from_records",
+    "from_crewai_tasks",
+    "from_autogen_messages",
+    "from_ai_sdk_steps",
+]

agent_ablation/adapters/autogen.py

@@ -0,0 +1,73 @@
+"""
+Adapter for AutoGen multi-agent chat message histories.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+from agent_ablation.models import Finding
+
+
+def _get_field(obj: Any, key: str, default: Any = None) -> Any:
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    return getattr(obj, key, default)
+
+
+def from_autogen_messages(
+    messages: Sequence[Any],
+    score_of: Callable[[Any], float],
+    confidence_of: Optional[Callable[[Any], Optional[float]]] = None,
+    tokens_of: Optional[Callable[[Any], Optional[int]]] = None,
+    cost_of: Optional[Callable[[Any], Optional[float]]] = None,
+) -> List[Finding]:
+    """
+    Converts AutoGen conversation history into `Finding` objects.
+    """
+    findings: List[Finding] = []
+
+    for message in messages:
+        if message is None:
+            continue
+
+        name = _get_field(message, "name")
+        role = _get_field(message, "role")
+
+        agent_id: Optional[str] = None
+        if isinstance(name, str) and name.strip():
+            agent_id = name.strip()
+        elif isinstance(role, str) and role not in ("user", "system") and role.strip():
+            agent_id = role.strip()
+
+        if not agent_id:
+            continue
+
+        score = float(score_of(message))
+        confidence = float(confidence_of(message)) if confidence_of and confidence_of(message) is not None else None
+        tokens = int(tokens_of(message)) if tokens_of and tokens_of(message) is not None else None
+        cost = float(cost_of(message)) if cost_of and cost_of(message) is not None else None
+
+        content = _get_field(message, "content")
+        context = _get_field(message, "context")
+
+        metadata: Dict[str, Any] = {}
+        if isinstance(content, dict):
+            metadata.update(content)
+        elif content is not None:
+            metadata["raw"] = content
+
+        if isinstance(context, dict):
+            metadata["context"] = context
+
+        finding = Finding(
+            agent_id=agent_id,
+            score=score,
+            confidence=confidence,
+            tokens=tokens,
+            cost=cost,
+            metadata=metadata if metadata else None,
+        )
+        findings.append(finding)
+
+    return findings

agent_ablation/adapters/crewai.py

@@ -0,0 +1,111 @@
+"""
+Adapter for CrewAI task outputs and agent payloads.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+from agent_ablation.models import Finding
+
+
+def _get_field(obj: Any, key: str, default: Any = None) -> Any:
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    return getattr(obj, key, default)
+
+
+def from_crewai_tasks(
+    outputs: Sequence[Any],
+    score_of: Optional[Callable[[Any], float]] = None,
+    agent_id_of: Optional[Callable[[Any], str]] = None,
+    confidence_of: Optional[Callable[[Any], Optional[float]]] = None,
+    cost_of: Optional[Callable[[Any], Optional[float]]] = None,
+    tokens_of: Optional[Callable[[Any], Optional[int]]] = None,
+) -> List[Finding]:
+    """
+    Converts CrewAI task outputs or agent payloads into `Finding` objects.
+    """
+    findings: List[Finding] = []
+
+    for output in outputs:
+        if output is None:
+            continue
+
+        agent_id: Optional[str] = None
+        if agent_id_of is not None:
+            agent_id = agent_id_of(output)
+        else:
+            agent_val = _get_field(output, "agent")
+            if isinstance(agent_val, str):
+                agent_id = agent_val
+            elif agent_val is not None:
+                agent_id = _get_field(agent_val, "role") or _get_field(agent_val, "name")
+
+        if not agent_id or not isinstance(agent_id, str) or not agent_id.strip():
+            continue
+
+        agent_id = agent_id.strip()
+
+        score: float = 0.0
+        json_dict = _get_field(output, "json_dict")
+        raw = _get_field(output, "raw")
+
+        if score_of is not None:
+            score = float(score_of(output))
+        else:
+            raw_score = _get_field(output, "score")
+            if raw_score is not None:
+                score = float(raw_score)
+            elif isinstance(json_dict, dict) and "score" in json_dict:
+                score = float(json_dict["score"])
+
+        confidence: Optional[float] = None
+        if confidence_of is not None:
+            conf_val = confidence_of(output)
+            if conf_val is not None:
+                confidence = float(conf_val)
+        elif isinstance(json_dict, dict) and "confidence" in json_dict:
+            confidence = float(json_dict["confidence"])
+
+        cost: Optional[float] = None
+        if cost_of is not None:
+            cost_val = cost_of(output)
+            if cost_val is not None:
+                cost = float(cost_val)
+        else:
+            cost_val = _get_field(output, "cost")
+            if cost_val is not None:
+                cost = float(cost_val)
+
+        tokens: Optional[int] = None
+        if tokens_of is not None:
+            tokens_val = tokens_of(output)
+            if tokens_val is not None:
+                tokens = int(tokens_val)
+        else:
+            tokens_val = _get_field(output, "tokens")
+            if tokens_val is not None:
+                tokens = int(tokens_val)
+
+        latency_ms_val = _get_field(output, "latency_ms") or _get_field(output, "latencyMs")
+        latency_ms = float(latency_ms_val) if latency_ms_val is not None else None
+
+        metadata: Dict[str, Any] = {}
+        if isinstance(json_dict, dict):
+            metadata.update(json_dict)
+        if raw:
+            metadata["raw"] = raw
+
+        finding = Finding(
+            agent_id=agent_id,
+            score=score,
+            confidence=confidence,
+            cost=cost,
+            tokens=tokens,
+            latency_ms=latency_ms,
+            metadata=metadata if metadata else None,
+        )
+        findings.append(finding)
+
+    return findings

agent_ablation/adapters/langgraph.py

@@ -0,0 +1,109 @@
+"""
+Adapter for LangGraph / LangChain agent message histories and generic records.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+
+from agent_ablation.models import Finding
+
+
+def _get_field(obj: Any, key: str, default: Any = None) -> Any:
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    return getattr(obj, key, default)
+
+
+def from_langgraph_messages(
+    messages: Sequence[Any],
+    score_of: Callable[[Any], float],
+    confidence_of: Optional[Callable[[Any], Optional[float]]] = None,
+) -> List[Finding]:
+    """
+    Converts a sequence of LangGraph-style agent messages into `Finding` objects for ablation.
+
+    Rules:
+    - Entries without a valid string `name` are ignored.
+    - `score_of(message)` is invoked for each named message to extract its score.
+    - `confidence_of(message)` is invoked if provided.
+    - Object/dict `content` is stored directly as `metadata`.
+    - String or other primitive `content` is wrapped inside `{"raw": content}` as `metadata`.
+
+    :param messages: Sequence of message-like objects or dicts (e.g. `state.messages`).
+    :param score_of: Required mapping function to extract the numerical score from each message.
+    :param confidence_of: Optional mapping function to extract a numerical confidence.
+    :returns: List of `Finding` objects ready for `run_ablation`.
+    """
+    findings: List[Finding] = []
+
+    for message in messages:
+        if message is None:
+            continue
+
+        name = _get_field(message, "name")
+        if not isinstance(name, str) or not name.strip():
+            continue
+
+        agent_id = name.strip()
+        score = float(score_of(message))
+        confidence = float(confidence_of(message)) if confidence_of and confidence_of(message) is not None else None
+
+        content = _get_field(message, "content")
+        metadata: Optional[Dict[str, Any]] = None
+
+        if isinstance(content, dict):
+            metadata = dict(content)
+        elif isinstance(content, str):
+            metadata = {"raw": content}
+        elif content is not None:
+            metadata = {"raw": content}
+
+        finding = Finding(
+            agent_id=agent_id,
+            score=score,
+            confidence=confidence,
+            metadata=metadata,
+        )
+        findings.append(finding)
+
+    return findings
+
+
+def from_records(
+    records: Sequence[Any],
+    agent_id: Callable[[Any, int], str],
+    score_of: Callable[[Any, int], float],
+    confidence_of: Optional[Callable[[Any, int], Optional[float]]] = None,
+) -> List[Finding]:
+    """
+    Converts an arbitrary sequence of records into `Finding` objects using user-supplied mapping functions.
+    Preserves the original record in `metadata`.
+
+    :param records: Sequence of arbitrary records or dicts.
+    :param agent_id: Function returning the agent ID string for each record `(record, index) -> str`.
+    :param score_of: Function returning the score for each record `(record, index) -> float`.
+    :param confidence_of: Optional function returning confidence `(record, index) -> Optional[float]`.
+    :returns: List of `Finding` objects ready for `run_ablation`.
+    """
+    findings: List[Finding] = []
+
+    for index, record in enumerate(records):
+        aid = agent_id(record, index)
+        score = float(score_of(record, index))
+        confidence = float(confidence_of(record, index)) if confidence_of and confidence_of(record, index) is not None else None
+
+        if isinstance(record, dict):
+            metadata: Dict[str, Any] = dict(record)
+        else:
+            metadata = {"raw": record}
+
+        finding = Finding(
+            agent_id=aid,
+            score=score,
+            confidence=confidence,
+            metadata=metadata,
+        )
+        findings.append(finding)
+
+    return findings

agent_ablation/adapters/vercel.py

@@ -0,0 +1,96 @@
+"""
+Adapter for Vercel AI SDK step and tool execution traces.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+from agent_ablation.models import Finding
+
+
+def _get_field(obj: Any, key: str, default: Any = None) -> Any:
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    return getattr(obj, key, default)
+
+
+def from_ai_sdk_steps(
+    steps: Sequence[Any],
+    score_of: Callable[[Any], float],
+    agent_id_of: Optional[Callable[[Any], str]] = None,
+    confidence_of: Optional[Callable[[Any], Optional[float]]] = None,
+    cost_of: Optional[Callable[[Any], Optional[float]]] = None,
+) -> List[Finding]:
+    """
+    Converts Vercel AI SDK tool call steps or agent trace steps into `Finding` objects.
+    """
+    findings: List[Finding] = []
+
+    for step in steps:
+        if step is None:
+            continue
+
+        agent_id: Optional[str] = None
+        if agent_id_of is not None:
+            agent_id = agent_id_of(step)
+        else:
+            agent_id = (
+                _get_field(step, "toolName")
+                or _get_field(step, "tool_name")
+                or _get_field(step, "stepType")
+                or _get_field(step, "step_type")
+            )
+
+        if not agent_id or not isinstance(agent_id, str) or not agent_id.strip():
+            continue
+
+        agent_id = agent_id.strip()
+        score = float(score_of(step))
+        confidence = float(confidence_of(step)) if confidence_of and confidence_of(step) is not None else None
+        cost = float(cost_of(step)) if cost_of and cost_of(step) is not None else None
+
+        usage = _get_field(step, "usage")
+        tokens: Optional[int] = None
+        if isinstance(usage, dict):
+            if "totalTokens" in usage:
+                tokens = int(usage["totalTokens"])
+            elif "total_tokens" in usage:
+                tokens = int(usage["total_tokens"])
+            else:
+                prompt_t = usage.get("promptTokens", usage.get("prompt_tokens", 0))
+                comp_t = usage.get("completionTokens", usage.get("completion_tokens", 0))
+                total = (prompt_t or 0) + (comp_t or 0)
+                tokens = int(total) if total > 0 else None
+        elif usage is not None:
+            total_t = getattr(usage, "total_tokens", getattr(usage, "totalTokens", None))
+            if total_t is not None:
+                tokens = int(total_t)
+
+        latency_ms_val = _get_field(step, "latencyMs") or _get_field(step, "latency_ms")
+        latency_ms = float(latency_ms_val) if latency_ms_val is not None else None
+
+        result_val = _get_field(step, "result")
+        args_val = _get_field(step, "args")
+
+        metadata: Dict[str, Any] = {}
+        if isinstance(result_val, dict):
+            metadata.update(result_val)
+        elif result_val is not None:
+            metadata["result"] = result_val
+
+        if args_val is not None:
+            metadata["args"] = args_val
+
+        finding = Finding(
+            agent_id=agent_id,
+            score=score,
+            confidence=confidence,
+            tokens=tokens,
+            cost=cost,
+            latency_ms=latency_ms,
+            metadata=metadata if metadata else None,
+        )
+        findings.append(finding)
+
+    return findings