entropy-agent-eval 0.1.0__py3-none-any.whl

entropy_agent_eval/__init__.py

@@ -0,0 +1,13 @@
+from entropy_agent_eval.evaluator import EntropicAgentScore, EntropyEvaluator, EvaluationReport
+from entropy_agent_eval.models import AgentEvent, AgentRun, InformationState
+
+__all__ = [
+    "AgentEvent",
+    "AgentRun",
+    "EntropicAgentScore",
+    "EntropyEvaluator",
+    "EvaluationReport",
+    "InformationState",
+]
+
+__version__ = "0.1.0"

entropy_agent_eval/adapters/__init__.py

@@ -0,0 +1,3 @@
+from entropy_agent_eval.adapters.generic import EventRecorder, normalize_events
+
+__all__ = ["EventRecorder", "normalize_events"]

entropy_agent_eval/adapters/generic.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from typing import Any, Iterable, List, Mapping, Optional
+
+from entropy_agent_eval.models import AgentEvent, AgentRun, InformationState
+
+
+def normalize_events(events: Iterable[Mapping[str, Any]]) -> List[AgentEvent]:
+    """Normalize dictionaries from arbitrary agent runtimes into AgentEvent objects."""
+
+    return [AgentEvent.from_mapping(event) for event in events]
+
+
+class EventRecorder:
+    """Small framework-agnostic recorder for custom agent loops."""
+
+    def __init__(self, task_id: str) -> None:
+        self.task_id = task_id
+        self.events: list[AgentEvent] = []
+
+    def record(self, kind: str, name: str, **metadata: Any) -> None:
+        timestamp = metadata.pop("timestamp", None)
+        self.events.append(AgentEvent(kind=kind, name=name, timestamp=timestamp, metadata=metadata))
+
+    def action(self, name: str, **metadata: Any) -> None:
+        self.record("action", name, **metadata)
+
+    def tool(self, name: str, **metadata: Any) -> None:
+        self.record("tool", name, **metadata)
+
+    def llm(self, name: str = "llm", **metadata: Any) -> None:
+        self.record("llm", name, **metadata)
+
+    def to_run(
+        self,
+        *,
+        success: Optional[bool] = None,
+        cost: float = 0.0,
+        before_state: Optional[InformationState] = None,
+        after_state: Optional[InformationState] = None,
+        outcome: Optional[str] = None,
+        **metadata: Any,
+    ) -> AgentRun:
+        return AgentRun(
+            task_id=self.task_id,
+            events=list(self.events),
+            success=success,
+            cost=cost,
+            before_state=before_state,
+            after_state=after_state,
+            outcome=outcome,
+            metadata=metadata,
+        )

entropy_agent_eval/adapters/google_adk.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Any, Iterable, Mapping
+
+from entropy_agent_eval.models import AgentEvent, AgentRun
+
+
+def runs_from_adk_events(
+    task_id: str,
+    events: Iterable[Mapping[str, Any]],
+    *,
+    success: bool | None = None,
+    cost: float = 0.0,
+    outcome: str | None = None,
+) -> AgentRun:
+    normalized = []
+    for event in events:
+        kind = str(event.get("kind") or event.get("event_type") or event.get("type") or "event")
+        name = str(
+            event.get("name")
+            or event.get("tool_name")
+            or event.get("agent_name")
+            or event.get("model")
+            or kind
+        )
+        timestamp = event.get("timestamp") or event.get("time")
+        metadata = {
+            key: value
+            for key, value in event.items()
+            if key not in {"kind", "event_type", "type", "name", "tool_name", "agent_name", "model", "timestamp", "time"}
+        }
+        normalized.append(AgentEvent(kind=kind, name=name, timestamp=timestamp, metadata=metadata))
+    return AgentRun(task_id=task_id, events=normalized, success=success, cost=cost, outcome=outcome)

entropy_agent_eval/adapters/langchain.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from entropy_agent_eval.adapters.generic import EventRecorder
+from entropy_agent_eval.models import AgentRun
+
+
+try:
+    from langchain_core.callbacks import BaseCallbackHandler
+except Exception:
+    BaseCallbackHandler = object  # type: ignore[misc,assignment]
+
+
+class EntropyCallbackHandler(BaseCallbackHandler):  # type: ignore[misc]
+    def __init__(self, task_id: str) -> None:
+        self.recorder = EventRecorder(task_id)
+
+    def on_tool_start(self, serialized: dict[str, Any], input_str: str, **kwargs: Any) -> None:
+        name = serialized.get("name") or serialized.get("id") or "tool"
+        self.recorder.tool(str(name), input=input_str, **_compact(kwargs))
+
+    def on_llm_start(self, serialized: dict[str, Any], prompts: list[str], **kwargs: Any) -> None:
+        name = serialized.get("name") or serialized.get("id") or "llm"
+        self.recorder.llm(str(name), prompt_count=len(prompts), **_compact(kwargs))
+
+    def on_chain_start(self, serialized: dict[str, Any], inputs: dict[str, Any], **kwargs: Any) -> None:
+        name = serialized.get("name") or serialized.get("id") or "chain"
+        self.recorder.action(str(name), input_keys=sorted(inputs.keys()), **_compact(kwargs))
+
+    def to_run(
+        self,
+        *,
+        success: Optional[bool] = None,
+        cost: float = 0.0,
+        outcome: Optional[str] = None,
+        **metadata: Any,
+    ) -> AgentRun:
+        return self.recorder.to_run(success=success, cost=cost, outcome=outcome, **metadata)
+
+
+def _compact(kwargs: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in kwargs.items() if key in {"run_id", "parent_run_id", "tags"}}

entropy_agent_eval/benchmarks/__init__.py

@@ -0,0 +1,5 @@
+from entropy_agent_eval.benchmarks.base import AgentCallable, BenchmarkTask, run_benchmark
+from entropy_agent_eval.benchmarks.coding_tasks import CODING_TASKS
+from entropy_agent_eval.benchmarks.qa_tasks import QA_TASKS
+
+__all__ = ["AgentCallable", "BenchmarkTask", "CODING_TASKS", "QA_TASKS", "run_benchmark"]

entropy_agent_eval/benchmarks/base.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any, Iterable, List, Mapping, Protocol
+
+from entropy_agent_eval.models import AgentRun
+
+
+@dataclass(frozen=True)
+class BenchmarkTask:
+    id: str
+    prompt: str
+    expected: str | None = None
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+
+class AgentCallable(Protocol):
+    def __call__(self, task: BenchmarkTask) -> AgentRun | Mapping[str, Any]:
+        """Execute a task and return an AgentRun or AgentRun-compatible mapping."""
+
+
+def run_benchmark(tasks: Iterable[BenchmarkTask], agent: AgentCallable) -> List[AgentRun]:
+    """Run tasks through any callable agent and normalize outputs."""
+
+    runs: list[AgentRun] = []
+    for task in tasks:
+        started = time.perf_counter()
+        result = agent(task)
+        elapsed_ms = (time.perf_counter() - started) * 1000
+        run = result if isinstance(result, AgentRun) else AgentRun.from_mapping(result)
+        run.task_id = run.task_id if run.task_id != "unknown" else task.id
+        run.latency_ms = run.latency_ms if run.latency_ms is not None else elapsed_ms
+        run.metadata.setdefault("benchmark_prompt", task.prompt)
+        if task.expected is not None:
+            run.metadata.setdefault("expected", task.expected)
+        runs.append(run)
+    return runs

entropy_agent_eval/benchmarks/coding_tasks.py

@@ -0,0 +1,17 @@
+from entropy_agent_eval.benchmarks.base import BenchmarkTask
+
+
+CODING_TASKS = [
+    BenchmarkTask(
+        id="code-sort",
+        prompt="Write a Python function that returns a sorted copy of a list.",
+        expected="Function should not mutate the input and should return ascending order.",
+        metadata={"difficulty": "easy"},
+    ),
+    BenchmarkTask(
+        id="code-dedupe-stable",
+        prompt="Write a Python function that removes duplicates while preserving order.",
+        expected="Function should preserve the first occurrence of each item.",
+        metadata={"difficulty": "medium"},
+    ),
+]

entropy_agent_eval/benchmarks/qa_tasks.py

@@ -0,0 +1,16 @@
+from entropy_agent_eval.benchmarks.base import BenchmarkTask
+
+
+QA_TASKS = [
+    BenchmarkTask(
+        id="qa-capital-france",
+        prompt="What is the capital of France?",
+        expected="Paris",
+        metadata={"difficulty": "easy"},
+    ),
+    BenchmarkTask(
+        id="qa-entropy-definition",
+        prompt="In one sentence, define Shannon entropy.",
+        metadata={"difficulty": "medium"},
+    ),
+]

entropy_agent_eval/cli.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import argparse
+import json
+from typing import Sequence
+
+from entropy_agent_eval.evaluator import EntropyEvaluator
+from entropy_agent_eval.io import load_runs
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="eea", description="Evaluate agent logs with entropy metrics.")
+    parser.add_argument("path", help="JSON or JSONL run log path")
+    parser.add_argument("--per-run", action="store_true", help="Emit per-run metrics instead of corpus metrics")
+    parser.add_argument("--indent", type=int, default=2, help="JSON indentation")
+    args = parser.parse_args(argv)
+
+    runs = load_runs(args.path)
+    evaluator = EntropyEvaluator()
+    payload = (
+        [dict(evaluator.evaluate_run(run)) for run in runs]
+        if args.per_run
+        else evaluator.evaluate(runs).as_dict()
+    )
+    print(json.dumps(payload, indent=args.indent, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

entropy_agent_eval/evaluator.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from statistics import mean
+from typing import Dict, Iterable, Mapping, Optional
+
+from entropy_agent_eval.metrics.core import (
+    entropy_reduction,
+    exploration_efficiency,
+    normalized_entropy,
+    shannon_entropy,
+)
+from entropy_agent_eval.metrics.robustness import robustness_summary
+from entropy_agent_eval.metrics.temporal import entropy_curve, rolling_entropy_curve
+from entropy_agent_eval.models import AgentRun
+
+
+@dataclass(frozen=True)
+class EntropicAgentScore:
+    """Weighted composite score.
+
+    Defaults reward success, information gain, and exploration efficiency while
+    penalizing monetary or token-normalized cost.
+    """
+
+    success_weight: float = 1.0
+    information_gain_weight: float = 1.0
+    exploration_efficiency_weight: float = 1.0
+    cost_weight: float = 1.0
+
+    def compute(
+        self,
+        success_rate: float,
+        information_gain: float,
+        exploration_efficiency_value: float,
+        cost: float,
+    ) -> float:
+        return (
+            self.success_weight * success_rate
+            + self.information_gain_weight * information_gain
+            + self.exploration_efficiency_weight * exploration_efficiency_value
+            - self.cost_weight * cost
+        )
+
+
+@dataclass
+class EvaluationReport:
+    runs: int
+    action_entropy: float
+    action_entropy_normalized: float
+    tool_entropy: float
+    tool_entropy_normalized: float
+    trajectory_entropy: float
+    trajectory_entropy_normalized: float
+    success_rate: Optional[float]
+    information_gain: float
+    exploration_efficiency: Optional[float]
+    mean_cost: float
+    entropic_agent_score: Optional[float]
+    robustness: Dict[str, object] = field(default_factory=dict)
+
+    def as_dict(self) -> Dict[str, object]:
+        return {
+            "runs": self.runs,
+            "action_entropy": self.action_entropy,
+            "action_entropy_normalized": self.action_entropy_normalized,
+            "tool_entropy": self.tool_entropy,
+            "tool_entropy_normalized": self.tool_entropy_normalized,
+            "trajectory_entropy": self.trajectory_entropy,
+            "trajectory_entropy_normalized": self.trajectory_entropy_normalized,
+            "success_rate": self.success_rate,
+            "information_gain": self.information_gain,
+            "exploration_efficiency": self.exploration_efficiency,
+            "mean_cost": self.mean_cost,
+            "entropic_agent_score": self.entropic_agent_score,
+            "robustness": self.robustness,
+        }
+
+
+class EntropyEvaluator:
+    """Compute entropy metrics for one run or a corpus of runs."""
+
+    def __init__(self, score: Optional[EntropicAgentScore] = None) -> None:
+        self.score = score or EntropicAgentScore()
+
+    @staticmethod
+    def action_entropy(actions: Iterable[str]) -> float:
+        return shannon_entropy(list(actions))
+
+    @staticmethod
+    def tool_entropy(tools: Iterable[str]) -> float:
+        return shannon_entropy(list(tools))
+
+    @staticmethod
+    def trajectory_entropy(trajectories: Iterable[Iterable[str] | str]) -> float:
+        normalized = [
+            tuple(trajectory) if not isinstance(trajectory, str) else trajectory
+            for trajectory in trajectories
+        ]
+        return shannon_entropy(normalized)
+
+    @staticmethod
+    def information_gain(before: Iterable[float], after: Iterable[float]) -> float:
+        return entropy_reduction(list(before), list(after))
+
+    @staticmethod
+    def entropy_curve(symbols: Iterable[str]) -> list[float]:
+        return entropy_curve(symbols)
+
+    @staticmethod
+    def rolling_entropy_curve(symbols: Iterable[str], window_size: int) -> list[float]:
+        return rolling_entropy_curve(symbols, window_size)
+
+    def evaluate_run(self, run: AgentRun) -> Mapping[str, object]:
+        actions = run.actions
+        tools = run.tools
+        gain = (
+            entropy_reduction(run.before_state.values(), run.after_state.values())
+            if run.before_state and run.after_state
+            else 0.0
+        )
+        return {
+            "task_id": run.task_id,
+            "action_entropy": shannon_entropy(actions),
+            "action_entropy_normalized": normalized_entropy(actions),
+            "tool_entropy": shannon_entropy(tools),
+            "tool_entropy_normalized": normalized_entropy(tools),
+            "trajectory": list(run.trajectory_tuple()),
+            "trajectory_length": len(run.events),
+            "entropy_curve": entropy_curve(actions),
+            "information_gain": gain,
+            "success": run.success,
+            "cost": run.cost,
+        }
+
+    def evaluate(self, runs: Iterable[AgentRun]) -> EvaluationReport:
+        materialized = list(runs)
+        actions = [action for run in materialized for action in run.actions]
+        tools = [tool for run in materialized for tool in run.tools]
+        trajectories = [run.trajectory_tuple() for run in materialized]
+        successes = [run.success for run in materialized if run.success is not None]
+        success_rate = mean([1.0 if success else 0.0 for success in successes]) if successes else None
+        gains = [
+            entropy_reduction(run.before_state.values(), run.after_state.values())
+            for run in materialized
+            if run.before_state and run.after_state
+        ]
+        information_gain = mean(gains) if gains else 0.0
+        mean_cost = mean([run.cost for run in materialized]) if materialized else 0.0
+        action_h = shannon_entropy(actions)
+        efficiency = (
+            exploration_efficiency(success_rate, action_h) if success_rate is not None else None
+        )
+        score = (
+            self.score.compute(success_rate, information_gain, efficiency, mean_cost)
+            if success_rate is not None and efficiency is not None
+            else None
+        )
+
+        return EvaluationReport(
+            runs=len(materialized),
+            action_entropy=action_h,
+            action_entropy_normalized=normalized_entropy(actions),
+            tool_entropy=shannon_entropy(tools),
+            tool_entropy_normalized=normalized_entropy(tools),
+            trajectory_entropy=shannon_entropy(trajectories),
+            trajectory_entropy_normalized=normalized_entropy(trajectories),
+            success_rate=success_rate,
+            information_gain=information_gain,
+            exploration_efficiency=efficiency,
+            mean_cost=mean_cost,
+            entropic_agent_score=score,
+            robustness=robustness_summary(materialized),
+        )

entropy_agent_eval/experiments/__init__.py

@@ -0,0 +1,21 @@
+from entropy_agent_eval.experiments.runner import ExperimentAgent, ExperimentConfig, ExperimentResult
+from entropy_agent_eval.experiments.learning_roadmap import load_learning_roadmap_tasks
+from entropy_agent_eval.experiments.reference_agents import (
+    DirectAgent,
+    PlannerAgent,
+    ReferenceAgent,
+    SearchCodeAgent,
+    SearchOnlyAgent,
+)
+
+__all__ = [
+    "DirectAgent",
+    "ExperimentAgent",
+    "ExperimentConfig",
+    "ExperimentResult",
+    "PlannerAgent",
+    "ReferenceAgent",
+    "SearchCodeAgent",
+    "SearchOnlyAgent",
+    "load_learning_roadmap_tasks",
+]

entropy_agent_eval/experiments/google_adk_roadmap.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import asyncio
+
+from entropy_agent_eval.adapters import EventRecorder
+from entropy_agent_eval.benchmarks import BenchmarkTask
+from entropy_agent_eval.experiments.learning_roadmap import (
+    ROADMAP_SYSTEM_PROMPT,
+    build_roadmap_context,
+    build_roadmap_prompt,
+    grade_roadmap_response,
+    roadmap_information_states,
+)
+from entropy_agent_eval.models import AgentRun
+
+
+class GoogleADKLearningRoadmapAgent:
+    name = "google-adk-learning-roadmap"
+
+    def __init__(self, model: str = "gemini-2.5-flash") -> None:
+        try:
+            from google.adk.agents import LlmAgent
+            from google.adk.runners import Runner
+            from google.adk.sessions import InMemorySessionService
+            from google.genai import types
+        except ImportError as exc:
+            raise RuntimeError(
+                "Install Google ADK support with: pip install 'entropy-agent-eval[google-adk]'"
+            ) from exc
+        self.model_name = model
+        self.agent_class = LlmAgent
+        self.runner_class = Runner
+        self.session_service_class = InMemorySessionService
+        self.types = types
+
+    def run(self, task: BenchmarkTask, repetition: int) -> AgentRun:
+        return asyncio.run(self._run_async(task, repetition))
+
+    async def _run_async(self, task: BenchmarkTask, repetition: int) -> AgentRun:
+        recorder = EventRecorder(task_id=task.id)
+        context = self._prepare_context(task, recorder)
+        prompt = build_roadmap_prompt(task, context)
+        recorder.llm(self.model_name, provider="google-adk")
+        response_text = await self._call_adk(task, prompt, repetition)
+        expected_terms = list(task.metadata.get("expected_terms", []))
+        success = grade_roadmap_response(response_text, expected_terms)
+        before, after = roadmap_information_states(success, expected_terms, response_text)
+        return recorder.to_run(
+            success=success,
+            cost=0.0,
+            before_state=before,
+            after_state=after,
+            outcome="usable_roadmap" if success else "incomplete_roadmap",
+            agent_name=self.name,
+            provider="google-adk",
+            model=self.model_name,
+            repetition=repetition,
+            response=response_text,
+        )
+
+    def _prepare_context(self, task: BenchmarkTask, recorder: EventRecorder) -> dict[str, object]:
+        recorder.tool("assess_learner_profile")
+        recorder.tool("select_learning_modules")
+        recorder.tool("build_weekly_schedule")
+        recorder.tool("design_assessment_checkpoints")
+        return build_roadmap_context(task)
+
+    async def _call_adk(self, task: BenchmarkTask, prompt: str, repetition: int) -> str:
+        app_name = "learning_roadmap_experiment"
+        user_id = "eea_user"
+        session_id = f"{task.id}_{repetition}"
+        agent = self.agent_class(
+            name="learning_roadmap_agent",
+            model=self.model_name,
+            instruction=ROADMAP_SYSTEM_PROMPT,
+            description="Creates structured learning roadmaps.",
+        )
+        session_service = self.session_service_class()
+        await session_service.create_session(
+            app_name=app_name,
+            user_id=user_id,
+            session_id=session_id,
+        )
+        runner = self.runner_class(
+            agent=agent,
+            app_name=app_name,
+            session_service=session_service,
+        )
+        content = self.types.Content(role="user", parts=[self.types.Part(text=prompt)])
+        final_response = ""
+        async for event in runner.run_async(
+            user_id=user_id,
+            session_id=session_id,
+            new_message=content,
+        ):
+            if event.is_final_response() and event.content and event.content.parts:
+                final_response = event.content.parts[0].text or ""
+                break
+        return final_response

entropy_agent_eval/experiments/langchain_roadmap.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from entropy_agent_eval.adapters import EventRecorder
+from entropy_agent_eval.benchmarks import BenchmarkTask
+from entropy_agent_eval.experiments.learning_roadmap import (
+    ROADMAP_SYSTEM_PROMPT,
+    build_roadmap_context,
+    build_roadmap_prompt,
+    grade_roadmap_response,
+    roadmap_information_states,
+)
+from entropy_agent_eval.models import AgentRun
+
+
+class LangChainLearningRoadmapAgent:
+    name = "langchain-learning-roadmap"
+
+    def __init__(self, model: str = "gpt-4o-mini", temperature: float = 0.0) -> None:
+        try:
+            from langchain_core.messages import HumanMessage, SystemMessage
+            from langchain_openai import ChatOpenAI
+        except ImportError as exc:
+            raise RuntimeError(
+                "Install LangChain support with: pip install 'entropy-agent-eval[langchain]'"
+            ) from exc
+        self.model_name = model
+        self.llm = ChatOpenAI(model=model, temperature=temperature)
+        self.human_message = HumanMessage
+        self.system_message = SystemMessage
+
+    def run(self, task: BenchmarkTask, repetition: int) -> AgentRun:
+        recorder = EventRecorder(task_id=task.id)
+        context = self._prepare_context(task, recorder)
+        prompt = build_roadmap_prompt(task, context)
+        recorder.llm(self.model_name, provider="langchain")
+        response = self.llm.invoke(
+            [
+                self.system_message(content=ROADMAP_SYSTEM_PROMPT),
+                self.human_message(content=prompt),
+            ]
+        )
+        text = str(getattr(response, "content", response))
+        expected_terms = list(task.metadata.get("expected_terms", []))
+        success = grade_roadmap_response(text, expected_terms)
+        before, after = roadmap_information_states(success, expected_terms, text)
+        usage = getattr(response, "usage_metadata", None) or {}
+        cost = _cost_from_usage(usage)
+        return recorder.to_run(
+            success=success,
+            cost=cost,
+            before_state=before,
+            after_state=after,
+            outcome="usable_roadmap" if success else "incomplete_roadmap",
+            agent_name=self.name,
+            provider="langchain",
+            model=self.model_name,
+            repetition=repetition,
+            response=text,
+            usage=usage,
+        )
+
+    def _prepare_context(self, task: BenchmarkTask, recorder: EventRecorder) -> dict[str, object]:
+        recorder.tool("assess_learner_profile")
+        recorder.tool("select_learning_modules")
+        recorder.tool("build_weekly_schedule")
+        recorder.tool("design_assessment_checkpoints")
+        return build_roadmap_context(task)
+
+
+def _cost_from_usage(usage: dict[str, object]) -> float:
+    total_tokens = usage.get("total_tokens") or usage.get("total_token_count") or 0
+    try:
+        return round(float(total_tokens) * 0.000001, 6)
+    except (TypeError, ValueError):
+        return 0.0