tei-loop 0.1.0__tar.gz

tei_loop-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Orkhan Javadli
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tei_loop-0.1.0/PKG-INFO

@@ -0,0 +1,185 @@
+Metadata-Version: 2.4
+Name: tei-loop
+Version: 0.1.0
+Summary: Target, Evaluate, Improve: A self-improving loop for agentic systems
+Author-email: Orkhan Javadli <ojavadli@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/ojavadli/tei-loop
+Project-URL: Repository, https://github.com/ojavadli/tei-loop
+Keywords: agents,evaluation,improvement,llm,agentic-systems,self-improving
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2.0
+Provides-Extra: openai
+Requires-Dist: openai>=1.40; extra == "openai"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.34; extra == "anthropic"
+Provides-Extra: google
+Requires-Dist: google-generativeai>=0.8; extra == "google"
+Provides-Extra: all
+Requires-Dist: openai>=1.40; extra == "all"
+Requires-Dist: anthropic>=0.34; extra == "all"
+Requires-Dist: google-generativeai>=0.8; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Dynamic: license-file
+
+# TEI Loop
+
+**Target, Evaluate, Improve** — a self-improving loop for agentic systems.
+
+TEI wraps any Python agent as a black box, evaluates its output across 4 dimensions using assertion-based LLM judges, and automatically applies targeted fixes when failures are detected.
+
+## Install
+
+```bash
+pip install tei-loop
+
+# With your preferred LLM provider:
+pip install 'tei-loop[openai]'    # OpenAI
+pip install 'tei-loop[anthropic]' # Anthropic
+pip install 'tei-loop[google]'    # Google Gemini
+pip install 'tei-loop[all]'       # All providers
+```
+
+## Quick Start
+
+```python
+import asyncio
+from tei_loop import TEILoop
+
+# Your existing agent — any function that takes input and returns output
+def my_agent(query: str) -> str:
+    # ... your agent logic ...
+    return result
+
+async def main():
+    loop = TEILoop(agent=my_agent)
+
+    # Evaluate only (baseline measurement)
+    result = await loop.evaluate_only("your test query")
+    print(result.summary())
+
+    # Full TEI loop (evaluate + improve + retry)
+    result = await loop.run("your test query")
+    print(result.summary())
+
+    # Before/after comparison
+    comparison = await loop.compare("your test query")
+    print(f"Baseline: {comparison['baseline'].baseline_score:.2f}")
+    print(f"With TEI: {comparison['improved'].final_score:.2f}")
+
+asyncio.run(main())
+```
+
+## CLI
+
+```bash
+# Evaluate your agent (baseline)
+tei evaluate my_agent.py --query "test input" --verbose
+
+# Run full improvement loop
+tei improve my_agent.py --query "test input" --max-retries 3
+
+# Before/after comparison
+tei compare my_agent.py --query "test input"
+
+# Generate config file
+tei init
+```
+
+TEI CLI looks for a function named `agent`, `run`, or `main` in your Python file.
+
+## How It Works
+
+### 1. Target
+Define what success looks like. TEI evaluates across 4 dimensions:
+
+| Dimension | What it checks |
+|---|---|
+| **Target Alignment** | Did the agent pursue the correct objective? |
+| **Reasoning Soundness** | Was the reasoning logical and non-contradictory? |
+| **Execution Accuracy** | Were the right tools called with correct parameters? |
+| **Output Integrity** | Is the output complete, accurate, and consistent? |
+
+### 2. Evaluate
+TEI runs 4 LLM judges in parallel (~3-5 seconds). Each judge produces verifiable assertions, not subjective scores. Every claim is backed by evidence from the agent's output.
+
+### 3. Improve
+When a dimension fails, TEI applies the targeted fix strategy:
+
+| Failure | Fix Strategy |
+|---|---|
+| Target drift | Re-anchor to original objective |
+| Flawed reasoning | Regenerate plan with failure context |
+| Execution errors | Correct tool calls and parameters |
+| Output issues | Repair factual errors and fill gaps |
+
+The loop retries automatically. Each cycle is sharper because the last was diagnosed.
+
+## Two Modes
+
+**Runtime mode** (default): Per-query, 1-3 retries, fixes individual failures in seconds.
+
+**Development mode**: Across many queries, proposes permanent prompt improvements.
+
+```python
+# Development mode
+dev_results = await loop.develop(
+    queries=["query1", "query2", "query3", ...],
+    max_iterations=50,
+)
+print(f"Avg improvement: {dev_results['avg_improvement']:+.2f}")
+```
+
+## Configuration
+
+TEI auto-detects your LLM provider from environment variables (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`). No new accounts or API keys needed.
+
+```python
+loop = TEILoop(
+    agent=my_agent,
+    eval_llm="gpt-5.2",          # Smartest for evaluation
+    improve_llm="gpt-5.2-mini",  # Cost-effective for fixes
+    max_retries=3,
+    verbose=True,
+)
+```
+
+Or via `.tei.yaml`:
+
+```bash
+tei init  # generates config file
+```
+
+## Cost
+
+| Scenario | Cost |
+|---|---|
+| Agent passes all dimensions | ~$0.005 |
+| One improvement cycle | ~$0.025 |
+| Full 3-retry loop | ~$0.07 |
+
+TEI shows the cost estimate before running.
+
+## Works With
+
+TEI wraps any Python callable. No framework lock-in:
+
+- **LangGraph** agents
+- **CrewAI** crews
+- **Custom Python** functions
+- **FastAPI** endpoints
+- **Any callable** that takes input and returns output
+
+## License
+
+MIT

tei_loop-0.1.0/README.md

@@ -0,0 +1,151 @@
+# TEI Loop
+
+**Target, Evaluate, Improve** — a self-improving loop for agentic systems.
+
+TEI wraps any Python agent as a black box, evaluates its output across 4 dimensions using assertion-based LLM judges, and automatically applies targeted fixes when failures are detected.
+
+## Install
+
+```bash
+pip install tei-loop
+
+# With your preferred LLM provider:
+pip install 'tei-loop[openai]'    # OpenAI
+pip install 'tei-loop[anthropic]' # Anthropic
+pip install 'tei-loop[google]'    # Google Gemini
+pip install 'tei-loop[all]'       # All providers
+```
+
+## Quick Start
+
+```python
+import asyncio
+from tei_loop import TEILoop
+
+# Your existing agent — any function that takes input and returns output
+def my_agent(query: str) -> str:
+    # ... your agent logic ...
+    return result
+
+async def main():
+    loop = TEILoop(agent=my_agent)
+
+    # Evaluate only (baseline measurement)
+    result = await loop.evaluate_only("your test query")
+    print(result.summary())
+
+    # Full TEI loop (evaluate + improve + retry)
+    result = await loop.run("your test query")
+    print(result.summary())
+
+    # Before/after comparison
+    comparison = await loop.compare("your test query")
+    print(f"Baseline: {comparison['baseline'].baseline_score:.2f}")
+    print(f"With TEI: {comparison['improved'].final_score:.2f}")
+
+asyncio.run(main())
+```
+
+## CLI
+
+```bash
+# Evaluate your agent (baseline)
+tei evaluate my_agent.py --query "test input" --verbose
+
+# Run full improvement loop
+tei improve my_agent.py --query "test input" --max-retries 3
+
+# Before/after comparison
+tei compare my_agent.py --query "test input"
+
+# Generate config file
+tei init
+```
+
+TEI CLI looks for a function named `agent`, `run`, or `main` in your Python file.
+
+## How It Works
+
+### 1. Target
+Define what success looks like. TEI evaluates across 4 dimensions:
+
+| Dimension | What it checks |
+|---|---|
+| **Target Alignment** | Did the agent pursue the correct objective? |
+| **Reasoning Soundness** | Was the reasoning logical and non-contradictory? |
+| **Execution Accuracy** | Were the right tools called with correct parameters? |
+| **Output Integrity** | Is the output complete, accurate, and consistent? |
+
+### 2. Evaluate
+TEI runs 4 LLM judges in parallel (~3-5 seconds). Each judge produces verifiable assertions, not subjective scores. Every claim is backed by evidence from the agent's output.
+
+### 3. Improve
+When a dimension fails, TEI applies the targeted fix strategy:
+
+| Failure | Fix Strategy |
+|---|---|
+| Target drift | Re-anchor to original objective |
+| Flawed reasoning | Regenerate plan with failure context |
+| Execution errors | Correct tool calls and parameters |
+| Output issues | Repair factual errors and fill gaps |
+
+The loop retries automatically. Each cycle is sharper because the last was diagnosed.
+
+## Two Modes
+
+**Runtime mode** (default): Per-query, 1-3 retries, fixes individual failures in seconds.
+
+**Development mode**: Across many queries, proposes permanent prompt improvements.
+
+```python
+# Development mode
+dev_results = await loop.develop(
+    queries=["query1", "query2", "query3", ...],
+    max_iterations=50,
+)
+print(f"Avg improvement: {dev_results['avg_improvement']:+.2f}")
+```
+
+## Configuration
+
+TEI auto-detects your LLM provider from environment variables (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`). No new accounts or API keys needed.
+
+```python
+loop = TEILoop(
+    agent=my_agent,
+    eval_llm="gpt-5.2",          # Smartest for evaluation
+    improve_llm="gpt-5.2-mini",  # Cost-effective for fixes
+    max_retries=3,
+    verbose=True,
+)
+```
+
+Or via `.tei.yaml`:
+
+```bash
+tei init  # generates config file
+```
+
+## Cost
+
+| Scenario | Cost |
+|---|---|
+| Agent passes all dimensions | ~$0.005 |
+| One improvement cycle | ~$0.025 |
+| Full 3-retry loop | ~$0.07 |
+
+TEI shows the cost estimate before running.
+
+## Works With
+
+TEI wraps any Python callable. No framework lock-in:
+
+- **LangGraph** agents
+- **CrewAI** crews
+- **Custom Python** functions
+- **FastAPI** endpoints
+- **Any callable** that takes input and returns output
+
+## License
+
+MIT

tei_loop-0.1.0/examples/example_crewai.py

@@ -0,0 +1,35 @@
+"""
+Example: Using TEI with a CrewAI crew.
+
+TEI wraps the crew's kickoff as a callable.
+"""
+import asyncio
+from tei_loop import TEILoop
+
+
+# Assuming you have a CrewAI crew:
+# from my_crew import my_crew
+#
+# def run_crew(query: str) -> str:
+#     result = my_crew.kickoff(inputs={"query": query})
+#     return str(result)
+#
+# async def main():
+#     loop = TEILoop(agent=run_crew, verbose=True)
+#     result = await loop.run("Research the latest AI trends")
+#     print(result.summary())
+
+
+def mock_crewai_agent(query: str) -> str:
+    """Placeholder — replace with your actual CrewAI crew."""
+    return f"CrewAI would research: {query}"
+
+
+async def main():
+    loop = TEILoop(agent=mock_crewai_agent, verbose=True)
+    result = await loop.evaluate_only("Research competitor pricing strategies")
+    print(result.summary())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tei_loop-0.1.0/examples/example_custom_python.py

@@ -0,0 +1,33 @@
+"""
+Example: Using TEI with a custom Python agent.
+
+This is the simplest case — your agent is a plain Python function.
+"""
+import asyncio
+from tei_loop import TEILoop
+
+
+def my_summarizer(text: str) -> str:
+    """Your existing agent — any function that takes input and returns output."""
+    from openai import OpenAI
+    client = OpenAI()
+    response = client.chat.completions.create(
+        model="gpt-5.2-mini",
+        messages=[
+            {"role": "system", "content": "Summarize the following text concisely."},
+            {"role": "user", "content": text},
+        ],
+    )
+    return response.choices[0].message.content
+
+
+async def main():
+    loop = TEILoop(agent=my_summarizer, verbose=True)
+
+    result = await loop.run("The quick brown fox jumped over the lazy dog. "
+                            "It was a sunny day in the forest.")
+    print(result.summary())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tei_loop-0.1.0/examples/example_langgraph.py

@@ -0,0 +1,40 @@
+"""
+Example: Using TEI with a LangGraph agent.
+
+TEI wraps the compiled graph as a callable — no changes to your graph needed.
+"""
+import asyncio
+from tei_loop import TEILoop
+
+
+# Assuming you have a LangGraph compiled graph:
+# from my_app import compiled_graph
+#
+# async def main():
+#     loop = TEILoop(
+#         agent=lambda q: compiled_graph.invoke({"messages": [("user", q)]}),
+#         verbose=True,
+#     )
+#     result = await loop.run("What restaurants are near me?")
+#     print(result.summary())
+
+# Or using the LangGraph adapter for deeper tracing:
+# from tei_loop.adapters.langgraph import LangGraphAdapter
+#
+# adapter = LangGraphAdapter(compiled_graph)
+# loop = TEILoop(agent=adapter.run, verbose=True)
+
+
+def mock_langgraph_agent(query: str) -> str:
+    """Placeholder — replace with your actual LangGraph agent."""
+    return f"LangGraph would process: {query}"
+
+
+async def main():
+    loop = TEILoop(agent=mock_langgraph_agent, verbose=True)
+    result = await loop.evaluate_only("Find Italian restaurants in SF for 4 people tonight")
+    print(result.summary())
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tei_loop-0.1.0/pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tei-loop"
+version = "0.1.0"
+description = "Target, Evaluate, Improve: A self-improving loop for agentic systems"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.9"
+authors = [
+    {name = "Orkhan Javadli", email = "ojavadli@gmail.com"},
+]
+keywords = [
+    "agents",
+    "evaluation",
+    "improvement",
+    "llm",
+    "agentic-systems",
+    "self-improving",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "pydantic>=2.0",
+]
+
+[project.optional-dependencies]
+openai = ["openai>=1.40"]
+anthropic = ["anthropic>=0.34"]
+google = ["google-generativeai>=0.8"]
+all = [
+    "openai>=1.40",
+    "anthropic>=0.34",
+    "google-generativeai>=0.8",
+]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "ruff>=0.5",
+]
+
+[project.scripts]
+tei = "tei_loop.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/ojavadli/tei-loop"
+Repository = "https://github.com/ojavadli/tei-loop"
+
+[tool.setuptools.packages.find]
+where = ["."]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tei_loop/tests"]

tei_loop-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tei_loop-0.1.0/tei_loop/__init__.py

@@ -0,0 +1,46 @@
+"""
+TEI Loop: Target -> Evaluate -> Improve
+
+A self-improving loop for agentic systems.
+
+Usage:
+    from tei_loop import TEILoop
+
+    loop = TEILoop(agent=my_agent_function)
+    result = await loop.run("user query")
+    print(result.summary())
+"""
+
+from .models import (
+    Dimension,
+    TEIConfig,
+    TEIResult,
+    EvalResult,
+    Trace,
+    TraceStep,
+    Failure,
+    Fix,
+    RunMode,
+)
+from .loop import TEILoop
+from .tracer import tei_trace
+from .evaluator import TEIEvaluator
+from .improver import TEIImprover
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "TEILoop",
+    "TEIConfig",
+    "TEIResult",
+    "EvalResult",
+    "Trace",
+    "TraceStep",
+    "Dimension",
+    "Failure",
+    "Fix",
+    "RunMode",
+    "TEIEvaluator",
+    "TEIImprover",
+    "tei_trace",
+]

tei_loop-0.1.0/tei_loop/adapters/__init__.py

@@ -0,0 +1,5 @@
+from .generic import GenericAdapter
+from .langgraph import LangGraphAdapter
+from .crewai import CrewAIAdapter
+
+__all__ = ["GenericAdapter", "LangGraphAdapter", "CrewAIAdapter"]