tei-loop 0.1.0__py3-none-any.whl

examples/example_crewai.py

@@ -0,0 +1,35 @@
+"""
+Example: Using TEI with a CrewAI crew.
+
+TEI wraps the crew's kickoff as a callable.
+"""
+import asyncio
+from tei_loop import TEILoop
+
+
+# Assuming you have a CrewAI crew:
+# from my_crew import my_crew
+#
+# def run_crew(query: str) -> str:
+#     result = my_crew.kickoff(inputs={"query": query})
+#     return str(result)
+#
+# async def main():
+#     loop = TEILoop(agent=run_crew, verbose=True)
+#     result = await loop.run("Research the latest AI trends")
+#     print(result.summary())
+
+
+def mock_crewai_agent(query: str) -> str:
+    """Placeholder — replace with your actual CrewAI crew."""
+    return f"CrewAI would research: {query}"
+
+
+async def main():
+    loop = TEILoop(agent=mock_crewai_agent, verbose=True)
+    result = await loop.evaluate_only("Research competitor pricing strategies")
+    print(result.summary())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/example_custom_python.py

@@ -0,0 +1,33 @@
+"""
+Example: Using TEI with a custom Python agent.
+
+This is the simplest case — your agent is a plain Python function.
+"""
+import asyncio
+from tei_loop import TEILoop
+
+
+def my_summarizer(text: str) -> str:
+    """Your existing agent — any function that takes input and returns output."""
+    from openai import OpenAI
+    client = OpenAI()
+    response = client.chat.completions.create(
+        model="gpt-5.2-mini",
+        messages=[
+            {"role": "system", "content": "Summarize the following text concisely."},
+            {"role": "user", "content": text},
+        ],
+    )
+    return response.choices[0].message.content
+
+
+async def main():
+    loop = TEILoop(agent=my_summarizer, verbose=True)
+
+    result = await loop.run("The quick brown fox jumped over the lazy dog. "
+                            "It was a sunny day in the forest.")
+    print(result.summary())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/example_langgraph.py

@@ -0,0 +1,40 @@
+"""
+Example: Using TEI with a LangGraph agent.
+
+TEI wraps the compiled graph as a callable — no changes to your graph needed.
+"""
+import asyncio
+from tei_loop import TEILoop
+
+
+# Assuming you have a LangGraph compiled graph:
+# from my_app import compiled_graph
+#
+# async def main():
+#     loop = TEILoop(
+#         agent=lambda q: compiled_graph.invoke({"messages": [("user", q)]}),
+#         verbose=True,
+#     )
+#     result = await loop.run("What restaurants are near me?")
+#     print(result.summary())
+
+# Or using the LangGraph adapter for deeper tracing:
+# from tei_loop.adapters.langgraph import LangGraphAdapter
+#
+# adapter = LangGraphAdapter(compiled_graph)
+# loop = TEILoop(agent=adapter.run, verbose=True)
+
+
+def mock_langgraph_agent(query: str) -> str:
+    """Placeholder — replace with your actual LangGraph agent."""
+    return f"LangGraph would process: {query}"
+
+
+async def main():
+    loop = TEILoop(agent=mock_langgraph_agent, verbose=True)
+    result = await loop.evaluate_only("Find Italian restaurants in SF for 4 people tonight")
+    print(result.summary())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tei_loop/__init__.py

@@ -0,0 +1,46 @@
+"""
+TEI Loop: Target -> Evaluate -> Improve
+
+A self-improving loop for agentic systems.
+
+Usage:
+    from tei_loop import TEILoop
+
+    loop = TEILoop(agent=my_agent_function)
+    result = await loop.run("user query")
+    print(result.summary())
+"""
+
+from .models import (
+    Dimension,
+    TEIConfig,
+    TEIResult,
+    EvalResult,
+    Trace,
+    TraceStep,
+    Failure,
+    Fix,
+    RunMode,
+)
+from .loop import TEILoop
+from .tracer import tei_trace
+from .evaluator import TEIEvaluator
+from .improver import TEIImprover
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "TEILoop",
+    "TEIConfig",
+    "TEIResult",
+    "EvalResult",
+    "Trace",
+    "TraceStep",
+    "Dimension",
+    "Failure",
+    "Fix",
+    "RunMode",
+    "TEIEvaluator",
+    "TEIImprover",
+    "tei_trace",
+]

tei_loop/adapters/__init__.py

@@ -0,0 +1,5 @@
+from .generic import GenericAdapter
+from .langgraph import LangGraphAdapter
+from .crewai import CrewAIAdapter
+
+__all__ = ["GenericAdapter", "LangGraphAdapter", "CrewAIAdapter"]

tei_loop/adapters/crewai.py

@@ -0,0 +1,237 @@
+"""
+CrewAI Adapter — deep task-level tracing.
+
+Hooks into CrewAI's Crew execution to capture each task as a separate
+TraceStep, giving TEI per-task visibility for diagnosis.
+
+Works with CrewAI >=0.60. If CrewAI is not installed, the adapter
+still works via generic kickoff wrapping.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import time
+from typing import Any, Callable, Optional
+
+from ..tracer import TEITracer, _safe_serialize
+from ..models import Trace, TraceStep
+
+
+class CrewAIAdapter:
+    """Wraps a CrewAI Crew for TEI with per-task tracing."""
+
+    def __init__(self, crew: Any, name: str = "crewai_agent"):
+        self.crew = crew
+        self.name = name
+        self._tracer = TEITracer()
+
+    async def run(self, query: Any, context: Optional[dict[str, Any]] = None) -> Trace:
+        self._tracer.start()
+        overall_start = time.time()
+
+        inputs = {"query": str(query)}
+        if context:
+            inputs.update(context)
+
+        output = None
+        error_msg = None
+
+        try:
+            self._trace_crew_config()
+
+            if self._has_task_callbacks():
+                output = await self._run_with_callbacks(inputs)
+            elif hasattr(self.crew, "kickoff_async"):
+                output = await self._run_async(inputs)
+            else:
+                output = await self._run_sync(inputs)
+        except Exception as e:
+            error_msg = f"{type(e).__name__}: {e}"
+
+        overall_ms = (time.time() - overall_start) * 1000
+        self._tracer.add_step(
+            name="crewai_total",
+            step_type="orchestrator",
+            input_data=query,
+            output_data=output,
+            duration_ms=overall_ms,
+            error=error_msg,
+            metadata={
+                "adapter": "crewai",
+                "task_count": self._get_task_count(),
+                "agent_count": self._get_agent_count(),
+            },
+        )
+
+        return self._tracer.finish(agent_input=query, agent_output=output)
+
+    def _trace_crew_config(self) -> None:
+        """Capture crew configuration as a trace step for diagnostic context."""
+        agents_info = []
+        if hasattr(self.crew, "agents"):
+            for ag in self.crew.agents:
+                agents_info.append({
+                    "role": getattr(ag, "role", "unknown"),
+                    "goal": str(getattr(ag, "goal", ""))[:200],
+                    "backstory": str(getattr(ag, "backstory", ""))[:200],
+                    "tools": [
+                        getattr(t, "name", str(t))
+                        for t in getattr(ag, "tools", [])
+                    ],
+                })
+
+        tasks_info = []
+        if hasattr(self.crew, "tasks"):
+            for task in self.crew.tasks:
+                tasks_info.append({
+                    "description": str(getattr(task, "description", ""))[:200],
+                    "expected_output": str(getattr(task, "expected_output", ""))[:200],
+                    "agent_role": getattr(
+                        getattr(task, "agent", None), "role", "unassigned"
+                    ),
+                })
+
+        self._tracer.add_step(
+            name="crew_config",
+            step_type="crewai_config",
+            metadata={
+                "agents": agents_info,
+                "tasks": tasks_info,
+                "process": str(getattr(self.crew, "process", "sequential")),
+            },
+        )
+
+    def _has_task_callbacks(self) -> bool:
+        """Check if crew tasks support callback injection."""
+        if not hasattr(self.crew, "tasks"):
+            return False
+        tasks = self.crew.tasks
+        if not tasks:
+            return False
+        return hasattr(tasks[0], "callback")
+
+    async def _run_with_callbacks(self, inputs: dict[str, Any]) -> Any:
+        """Inject per-task callbacks to capture task-level output."""
+        task_results: list[dict[str, Any]] = []
+
+        original_callbacks = []
+        for i, task in enumerate(self.crew.tasks):
+            original_callbacks.append(getattr(task, "callback", None))
+
+            task_name = getattr(
+                getattr(task, "agent", None), "role", f"task_{i}"
+            )
+            task_desc = str(getattr(task, "description", ""))[:200]
+
+            def make_callback(idx: int, name: str, desc: str):
+                task_start = time.time()
+
+                def cb(output: Any) -> None:
+                    task_ms = (time.time() - task_start) * 1000
+                    raw_output = getattr(output, "raw", str(output))
+
+                    self._tracer.add_step(
+                        name=name,
+                        step_type="crewai_task",
+                        input_data=desc,
+                        output_data=raw_output,
+                        duration_ms=task_ms,
+                        metadata={
+                            "task_index": idx,
+                            "agent_role": name,
+                        },
+                    )
+                    task_results.append({
+                        "task_index": idx,
+                        "agent": name,
+                        "output_preview": str(raw_output)[:300],
+                    })
+
+                return cb
+
+            task.callback = make_callback(i, task_name, task_desc)
+
+        try:
+            if hasattr(self.crew, "kickoff_async"):
+                result = await self.crew.kickoff_async(inputs=inputs)
+            else:
+                result = await asyncio.to_thread(
+                    self.crew.kickoff, inputs=inputs
+                )
+        finally:
+            for i, task in enumerate(self.crew.tasks):
+                task.callback = original_callbacks[i]
+
+        return self._extract_output(result)
+
+    async def _run_async(self, inputs: dict[str, Any]) -> Any:
+        """Fallback: async kickoff without per-task callbacks."""
+        task_start = time.time()
+        result = await self.crew.kickoff_async(inputs=inputs)
+        task_ms = (time.time() - task_start) * 1000
+
+        self._tracer.add_step(
+            name="kickoff_async",
+            step_type="crewai_kickoff",
+            input_data=inputs,
+            output_data=self._extract_output(result),
+            duration_ms=task_ms,
+        )
+
+        self._trace_task_outputs(result)
+        return self._extract_output(result)
+
+    async def _run_sync(self, inputs: dict[str, Any]) -> Any:
+        """Fallback: sync kickoff wrapped in thread."""
+        task_start = time.time()
+        result = await asyncio.to_thread(self.crew.kickoff, inputs=inputs)
+        task_ms = (time.time() - task_start) * 1000
+
+        self._tracer.add_step(
+            name="kickoff",
+            step_type="crewai_kickoff",
+            input_data=inputs,
+            output_data=self._extract_output(result),
+            duration_ms=task_ms,
+        )
+
+        self._trace_task_outputs(result)
+        return self._extract_output(result)
+
+    def _trace_task_outputs(self, crew_output: Any) -> None:
+        """Extract per-task outputs from CrewOutput (post-hoc tracing)."""
+        tasks_output = getattr(crew_output, "tasks_output", None)
+        if not tasks_output:
+            return
+
+        for i, task_out in enumerate(tasks_output):
+            agent_name = getattr(task_out, "agent", f"task_{i}")
+            raw = getattr(task_out, "raw", str(task_out))
+
+            self._tracer.add_step(
+                name=str(agent_name),
+                step_type="crewai_task_output",
+                output_data=raw,
+                metadata={
+                    "task_index": i,
+                    "description": str(
+                        getattr(task_out, "description", "")
+                    )[:200],
+                },
+            )
+
+    def _extract_output(self, result: Any) -> Any:
+        if hasattr(result, "raw"):
+            return result.raw
+        return str(result)
+
+    def _get_task_count(self) -> int:
+        return len(getattr(self.crew, "tasks", []))
+
+    def _get_agent_count(self) -> int:
+        return len(getattr(self.crew, "agents", []))
+
+    def __repr__(self) -> str:
+        return f"CrewAIAdapter(name={self.name!r})"

tei_loop/adapters/generic.py

@@ -0,0 +1,41 @@
+"""
+Generic Python Adapter.
+
+Wraps any Python callable (sync or async) as a TEI-compatible agent.
+This is the primary adapter and covers 100% of agents since every
+framework (LangGraph, CrewAI, custom) produces a Python callable.
+"""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any, Callable, Optional
+
+from ..tracer import TEITracer, run_and_trace
+from ..models import Trace
+
+
+class GenericAdapter:
+    """Wraps any Python callable as a TEI agent."""
+
+    def __init__(
+        self,
+        agent_fn: Callable,
+        name: str = "agent",
+        description: str = "",
+    ):
+        self.agent_fn = agent_fn
+        self.name = name or getattr(agent_fn, "__name__", "agent")
+        self.description = description
+        self._tracer = TEITracer()
+
+    async def run(self, query: Any, context: Optional[dict[str, Any]] = None) -> Trace:
+        return await run_and_trace(
+            self.agent_fn,
+            query,
+            tracer=self._tracer,
+            context=context,
+        )
+
+    def __repr__(self) -> str:
+        return f"GenericAdapter(name={self.name!r})"

tei_loop/adapters/langgraph.py

@@ -0,0 +1,159 @@
+"""
+LangGraph Adapter — deep node-level tracing.
+
+Hooks into LangGraph's compiled graph to capture each node execution
+as a separate TraceStep, giving TEI per-node visibility for diagnosis.
+
+Works with LangGraph >=0.2 compiled StateGraphs.
+If LangGraph is not installed, falls back to generic black-box tracing.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import time
+from typing import Any, Callable, Optional
+
+from ..tracer import TEITracer, _safe_serialize
+from ..models import Trace, TraceStep
+
+
+class LangGraphAdapter:
+    """Wraps a LangGraph compiled graph for TEI with per-node tracing."""
+
+    def __init__(self, graph: Any, name: str = "langgraph_agent"):
+        self.graph = graph
+        self.name = name
+        self._tracer = TEITracer()
+
+    async def run(self, query: Any, context: Optional[dict[str, Any]] = None) -> Trace:
+        self._tracer.start()
+        overall_start = time.time()
+
+        input_payload = self._build_input(query)
+        output = None
+        error_msg = None
+
+        try:
+            if self._has_stream_events():
+                output = await self._run_with_stream(input_payload)
+            elif hasattr(self.graph, "ainvoke"):
+                output = await self._run_async(input_payload)
+            else:
+                output = await self._run_sync(input_payload)
+        except Exception as e:
+            error_msg = f"{type(e).__name__}: {e}"
+
+        overall_ms = (time.time() - overall_start) * 1000
+        self._tracer.add_step(
+            name="langgraph_total",
+            step_type="orchestrator",
+            input_data=query,
+            output_data=output,
+            duration_ms=overall_ms,
+            error=error_msg,
+            metadata={"adapter": "langgraph", "node_count": len(self._tracer._steps)},
+        )
+
+        return self._tracer.finish(agent_input=query, agent_output=output)
+
+    def _build_input(self, query: Any) -> dict[str, Any]:
+        if isinstance(query, dict):
+            return query
+        return {"messages": [("user", str(query))]}
+
+    def _has_stream_events(self) -> bool:
+        return hasattr(self.graph, "astream_events")
+
+    async def _run_with_stream(self, input_payload: dict[str, Any]) -> Any:
+        """Use LangGraph's astream_events to capture per-node execution."""
+        final_output = None
+        current_node: Optional[str] = None
+        node_start: float = 0.0
+        node_input: Any = None
+
+        async for event in self.graph.astream_events(input_payload, version="v2"):
+            kind = event.get("event", "")
+            name = event.get("name", "")
+            data = event.get("data", {})
+
+            if kind == "on_chain_start" and name != self.name:
+                current_node = name
+                node_start = time.time()
+                node_input = data.get("input")
+
+            elif kind == "on_chain_end" and name == current_node:
+                node_output = data.get("output")
+                node_ms = (time.time() - node_start) * 1000
+
+                self._tracer.add_step(
+                    name=current_node,
+                    step_type="langgraph_node",
+                    input_data=node_input,
+                    output_data=node_output,
+                    duration_ms=node_ms,
+                    metadata={"event_kind": kind},
+                )
+                final_output = node_output
+                current_node = None
+
+            elif kind == "on_chain_error" and name == current_node:
+                node_ms = (time.time() - node_start) * 1000
+                self._tracer.add_step(
+                    name=current_node,
+                    step_type="langgraph_node",
+                    input_data=node_input,
+                    duration_ms=node_ms,
+                    error=str(data.get("error", "unknown")),
+                )
+                current_node = None
+
+            elif kind == "on_tool_start":
+                self._tracer.add_step(
+                    name=name,
+                    step_type="tool_call",
+                    input_data=data.get("input"),
+                    metadata={"tool_name": name},
+                )
+
+            elif kind == "on_tool_end":
+                if self._tracer._steps and self._tracer._steps[-1].name == name:
+                    self._tracer._steps[-1].output_data = _safe_serialize(
+                        data.get("output")
+                    )
+
+        return final_output
+
+    async def _run_async(self, input_payload: dict[str, Any]) -> Any:
+        """Fallback: ainvoke without streaming (less granular)."""
+        node_start = time.time()
+        result = await self.graph.ainvoke(input_payload)
+        node_ms = (time.time() - node_start) * 1000
+
+        self._tracer.add_step(
+            name="ainvoke",
+            step_type="langgraph_invoke",
+            input_data=input_payload,
+            output_data=result,
+            duration_ms=node_ms,
+        )
+        return result
+
+    async def _run_sync(self, input_payload: dict[str, Any]) -> Any:
+        """Fallback: sync invoke wrapped in thread."""
+        node_start = time.time()
+        result = await asyncio.to_thread(self.graph.invoke, input_payload)
+        node_ms = (time.time() - node_start) * 1000
+
+        self._tracer.add_step(
+            name="invoke",
+            step_type="langgraph_invoke",
+            input_data=input_payload,
+            output_data=result,
+            duration_ms=node_ms,
+        )
+        return result
+
+    def __repr__(self) -> str:
+        return f"LangGraphAdapter(name={self.name!r})"