entropy-ai 0.1.0__tar.gz

entropy_ai-0.1.0/PKG-INFO

@@ -0,0 +1,34 @@
+Metadata-Version: 2.4
+Name: entropy-ai
+Version: 0.1.0
+Summary: Behavioral evaluation framework for non-deterministic AI agents
+Author-email: Satyam Singh <satyamsingh7734@gmail.com>
+Requires-Python: >=3.9
+Requires-Dist: pandas
+Requires-Dist: matplotlib
+Requires-Dist: jinja2
+Requires-Dist: rich
+Requires-Dist: pyyaml
+Provides-Extra: pdf
+Requires-Dist: reportlab; extra == "pdf"
+Provides-Extra: jupyter
+Requires-Dist: nbformat; extra == "jupyter"
+Provides-Extra: hf
+Requires-Dist: datasets; extra == "hf"
+Provides-Extra: langchain
+Requires-Dist: langchain; extra == "langchain"
+Provides-Extra: langgraph
+Requires-Dist: langgraph; extra == "langgraph"
+Provides-Extra: openai
+Requires-Dist: openai; extra == "openai"
+Requires-Dist: openai-agents; extra == "openai"
+Provides-Extra: crewai
+Requires-Dist: crewai; extra == "crewai"
+Provides-Extra: pydanticai
+Requires-Dist: pydantic-ai; extra == "pydanticai"
+Provides-Extra: autogen
+Requires-Dist: autogen; extra == "autogen"
+Provides-Extra: google-adk
+Requires-Dist: google-adk; extra == "google-adk"
+Provides-Extra: mcp
+Requires-Dist: mcp; extra == "mcp"

entropy_ai-0.1.0/README.md

@@ -0,0 +1,125 @@
+# EntroPy
+
+Behavioral evaluation framework for non-deterministic AI agents.
+
+Measures what other frameworks miss: **uncertainty, drift, stochastic
+behavior, and emergent failures** — via Monte Carlo evaluation over repeated
+runs and entropy-based metrics. Framework-agnostic, local-first, zero vendor
+lock-in.
+
+## Install
+
+```bash
+pip install -e .                       # core (pandas, matplotlib, jinja2, rich, pyyaml)
+pip install "entropy-ai[langchain]"    # optional: a framework adapter
+pip install "entropy-ai[pdf,jupyter]"  # optional: PDF / Jupyter reports
+```
+
+## Quick start
+
+```python
+from entropy import Suite, Dataset, Case
+
+def my_agent(inp): ...
+    # return an output, an AgentRun, or a dict with events
+
+dataset = Dataset([Case(input="q1", expected="correct answer")])
+results = Suite(seed=42).run(my_agent, dataset, trials=100)
+print(results)   # success_rate, behavioral_entropy, drift_score, ...
+```
+
+The canonical output (spec §9 / §21) includes: `success_rate`, `reliability`,
+`variance`, `robustness`, `entropy`, `confidence_interval`, `behavioral_entropy`,
+`behavioral_variance`, `trajectory_entropy`, `trajectory_diversity`, `tool_reliability`,
+`drift_score`, `goal_stability`, `recovery_score`, `emergent_behavior`, and more.
+56 metrics total; `entropy.default_metrics()` lists the canonical set.
+
+## Tracing
+
+```python
+from entropy import trace, event, instrument
+with trace() as t:
+    event("action", "tool:search")
+# or auto-wrap any agent:
+wrapped = instrument(my_agent)          # returns an AgentRun per call
+```
+
+## Custom metrics (plugin system)
+
+```python
+from entropy import metric
+@metric("my_metric")
+class MyMetric:
+    def compute(self, batch): ...       # batch has outputs/events/successes/costs/...
+```
+
+## Simulation, Chaos, Observability
+
+```python
+from entropy import AdversarialSimulator, ChaosRunner, Watcher
+adv = AdversarialSimulator(seed=0)
+for v in adv.iterate("book a flight"):
+    print(v)                            # typo / injection / distraction / leak variants
+
+ChaosRunner(seed=0).run(agent, dataset, ["api_fail", "timeout"])   # fault injection
+Watcher(agent, dataset, trials=20).watch(rounds=10)                # live drift + anomalies
+```
+
+## Framework adapters (lazy)
+
+```python
+from entropy import from_langchain, from_langgraph, from_openai, from_custom
+runnable = from_langchain(my_langchain_agent)   # maps framework runs -> AgentRun
+```
+
+All 9 adapters (LangChain, LangGraph, OpenAI Agents, CrewAI, PydanticAI,
+AutoGen, Google ADK, MCP, custom) import their framework **lazily**, so
+`import entropy` works with none of them installed.
+
+## CLI
+
+```bash
+entropy init                      # scaffold a project
+entropy run    --agent m:a --dataset d.json --trials 100
+entropy test   --agent m:a --dataset d.json --baseline base.json [--update]
+entropy benchmark --agents m:a m:b --dataset d.json
+entropy chaos   --agent m:a --dataset d.json --faults api_fail,timeout
+entropy simulate --kind adversarial --text "book a flight"
+entropy ui      --agent m:a --dataset d.json --serve
+entropy watch   --agent m:a --dataset d.json --rounds 10
+entropy report  --results r.json --format html
+entropy dataset --path d.json
+entropy doctor                      # check optional deps
+```
+
+## Reports
+
+```python
+from entropy import export_report
+export_report(results, "html", "report.html")      # also: json, markdown, pdf, jupyter
+```
+
+## Layout
+
+`core/` (models, tracing, exporters, suite) · `metrics/` (registry + all
+categories) · `datasets/` · `simulation/` · `chaos/` · `regression/` ·
+`benchmarks/` · `observability/` · `reports/` · `dashboard/` · `integrations/`
+(adapters) · `cli/` · `plugins/`.
+
+Tests: `python -m pytest tests/`.
+
+## Documentation
+
+The static documentation site lives in [`docs/`](docs/) — plain
+HTML/CSS/JS with no build step, deployable to GitHub Pages.
+
+- Source: `docs/index.html`, `docs/guide/*.html`, `docs/api.html`,
+  `docs/examples.html`, plus `docs/assets/` (CSS, JS, logo, search index).
+- Local preview: serve the folder, e.g. `python -m http.server -d docs 8000`
+  then open `http://localhost:8000`.
+- Deploy: push to `main`; the
+  [`.github/workflows/pages.yml`](.github/workflows/pages.yml) workflow
+  uploads `docs/` and publishes it via GitHub Pages. Alternatively, set
+  **Settings → Pages → Build and deployment → Source** to *Deploy from a
+  branch* and choose the `main` branch `/docs` folder.
+

entropy_ai-0.1.0/entropy/__init__.py

@@ -0,0 +1,52 @@
+"""EntroPy — behavioral evaluation for non-deterministic AI agents.
+
+Measure uncertainty, drift, stochastic behavior, and emergent failures via
+Monte Carlo evaluation and entropy-based metrics.
+"""
+from __future__ import annotations
+
+from .core import (
+    AgentRun, Event, Trace, Node, Edge,
+    ActionEvent, ReasoningEvent, ToolCallEvent, ObservationEvent,
+    MemoryReadEvent, MemoryWriteEvent, ErrorEvent, StateTransitionEvent,
+    instrument, trace, event, current_trace,
+    Suite, assert_stable,
+)
+from .metrics import metric, Metric, Batch, default_metrics, layers
+from .datasets import (
+    Dataset, Case, GoldenCase, Scenario, BehaviorCase, FailureCase, AdversarialCase,
+)
+from .simulation import (
+    UserSimulator, ScriptedUserSimulator, RandomUserSimulator, LLMUserSimulator,
+    Environment, StatefulEnv, GridWorld, AdversarialSimulator,
+)
+from .chaos import ChaosRunner, inject as chaos_inject, available as chaos_available
+from .observability import Watcher, render_metric_heatmap
+from .dashboard import build_dashboard, serve as serve_dashboard
+from .reports import export as export_report
+from .integrations import (
+    adapter, find_adapter, list_adapters,
+    from_langchain, from_langgraph, from_openai, from_crewai,
+    from_pydanticai, from_autogen, from_google_adk, from_mcp, from_custom,
+)
+from .plugins import exporter, discover
+
+__all__ = [
+    "Suite", "Dataset", "Case", "GoldenCase", "Scenario", "BehaviorCase",
+    "FailureCase", "AdversarialCase",
+    "UserSimulator", "ScriptedUserSimulator", "RandomUserSimulator", "LLMUserSimulator",
+    "Environment", "StatefulEnv", "GridWorld", "AdversarialSimulator",
+    "ChaosRunner", "chaos_inject", "chaos_available",
+    "Watcher", "render_metric_heatmap", "build_dashboard", "serve_dashboard",
+    "export_report",
+    "AgentRun", "Event", "Trace", "Node", "Edge",
+    "ActionEvent", "ReasoningEvent", "ToolCallEvent", "ObservationEvent",
+    "MemoryReadEvent", "MemoryWriteEvent", "ErrorEvent", "StateTransitionEvent",
+    "instrument", "trace", "event", "current_trace",
+    "metric", "Batch", "default_metrics", "layers",
+    "adapter", "find_adapter", "list_adapters",
+    "from_langchain", "from_langgraph", "from_openai", "from_crewai",
+    "from_pydanticai", "from_autogen", "from_google_adk", "from_mcp", "from_custom",
+    "exporter", "discover",
+    "assert_stable",
+]

entropy_ai-0.1.0/entropy/benchmarks/__init__.py

@@ -0,0 +1,10 @@
+"""Benchmarks package."""
+from __future__ import annotations
+
+from .bench import (
+    load_ref, load_dataset, benchmark_variants, benchmark_refs, to_markdown,
+)
+
+__all__ = [
+    "load_ref", "load_dataset", "benchmark_variants", "benchmark_refs", "to_markdown",
+]

entropy_ai-0.1.0/entropy/benchmarks/bench.py

@@ -0,0 +1,59 @@
+"""Benchmarking (spec §14): compare models/prompts/frameworks/agents/versions."""
+from __future__ import annotations
+
+import importlib
+from typing import Any, Callable, Dict, List, Tuple
+
+from ..core.suite import Suite
+from ..datasets.loaders import Dataset
+
+
+def load_ref(ref: str):
+    """Load ``module:attr`` (agent/function) or ``module:attr`` callable."""
+    mod_name, attr = ref.split(":")
+    mod = importlib.import_module(mod_name)
+    return getattr(mod, attr)
+
+
+def load_dataset(path: str) -> Dataset:
+    p = path.lower()
+    if p.endswith(".jsonl"):
+        return Dataset.from_jsonl(path)
+    if p.endswith((".yaml", ".yml")):
+        return Dataset.from_yaml(path)
+    if p.endswith(".csv"):
+        return Dataset.from_csv(path)
+    return Dataset.from_json(path)
+
+
+def benchmark_variants(
+    variants: List[Tuple[str, Callable]], dataset, trials: int = 100, seed: int = 0,
+) -> List[Tuple[str, dict]]:
+    """Run the suite for each (name, agent) variant and return rows."""
+    rows = []
+    for name, agent in variants:
+        res = Suite(seed=seed).run(agent, dataset, trials=trials)
+        rows.append((name, res))
+    return rows
+
+
+def benchmark_refs(
+    refs: List[str], dataset_path: str, trials: int = 100, seed: int = 0,
+) -> List[Tuple[str, dict]]:
+    ds = load_dataset(dataset_path)
+    variants = [(r, load_ref(r)) for r in refs]
+    return benchmark_variants(variants, ds, trials=trials, seed=seed)
+
+
+def to_markdown(rows: List[Tuple[str, dict]], metrics: List[str] | None = None) -> str:
+    if not rows:
+        return "(no results)"
+    if metrics is None:
+        metrics = [k for k, v in rows[0][1].items() if isinstance(v, (int, float))]
+    header = ["variant"] + metrics
+    lines = ["| " + " | ".join(header) + " |",
+             "| " + " | ".join("---" for _ in header) + " |"]
+    for name, res in rows:
+        cells = [name] + [f"{res.get(m, 0):.4f}" for m in metrics]
+        lines.append("| " + " | ".join(cells) + " |")
+    return "\n".join(lines)

entropy_ai-0.1.0/entropy/chaos/__init__.py

@@ -0,0 +1,7 @@
+"""Chaos package (spec §10)."""
+from __future__ import annotations
+
+from .faults import FAULTS, inject, available
+from .runner import ChaosRunner
+
+__all__ = ["FAULTS", "inject", "available", "ChaosRunner"]

entropy_ai-0.1.0/entropy/chaos/faults.py

@@ -0,0 +1,101 @@
+"""Fault injection (spec §10): 8 fault types as agent wrappers."""
+from __future__ import annotations
+
+import random
+from typing import Callable, Dict, List
+
+from ..core.models import AgentRun, ErrorEvent, ToolCallEvent, MemoryWriteEvent
+
+
+def _wrap(agent: Callable, fault_fn, p: float, rng: random.Random) -> Callable:
+    def wrapped(inp):
+        if rng.random() < p:
+            return fault_fn(agent, inp)
+        return agent(inp)
+    wrapped.__name__ = getattr(fault_fn, "__name__", "fault")
+    return wrapped
+
+
+# --- individual faults ----------------------------------------------------
+def api_fail(agent, inp):
+    raise RuntimeError("CHAOS: simulated API failure")
+
+
+def timeout(agent, inp):
+    raise TimeoutError("CHAOS: simulated timeout")
+
+
+def rate_limit(agent, inp):
+    raise RuntimeError("CHAOS: 429 rate limited")
+
+
+def network_delay(agent, inp):
+    raise ConnectionError("CHAOS: simulated network delay/reset")
+
+
+def tool_crash(agent, inp):
+    res = agent(inp)
+    if isinstance(res, AgentRun):
+        events = [ToolCallEvent("__crashed__") if e.type == "tool" else e
+                  for e in res.events]
+        return AgentRun(run_id=res.run_id, input=inp, output=res.output,
+                        events=events, cost=res.cost,
+                        metadata={**res.metadata, "error": "tool_crash"})
+    return AgentRun(run_id="", input=inp, output=res,
+                    events=[ToolCallEvent("__crashed__")], metadata={"error": "tool_crash"})
+
+
+def memory_corrupt(agent, inp):
+    res = agent(inp)
+    if isinstance(res, AgentRun):
+        events = [MemoryWriteEvent(e.name, {**e.data, "polluting": True})
+                  if e.type == "memory_write" else e for e in res.events]
+        return AgentRun(run_id=res.run_id, input=inp, output=res.output,
+                        events=events, cost=res.cost, metadata=res.metadata)
+    return res
+
+
+def token_limit(agent, inp, limit: int = 16):
+    res = agent(inp)
+    out = res.output if isinstance(res, AgentRun) else res
+    text = out if isinstance(out, str) else str(out)
+    if len(text) > limit:
+        truncated = text[:limit]
+        if isinstance(res, AgentRun):
+            return AgentRun(run_id=res.run_id, input=inp, output=truncated,
+                            events=res.events, cost=res.cost,
+                            metadata={**res.metadata, "error": "token_limit"})
+        return AgentRun(run_id="", input=inp, output=truncated, metadata={"error": "token_limit"})
+    return res
+
+
+def malformed_output(agent, inp):
+    res = agent(inp)
+    if isinstance(res, AgentRun):
+        return AgentRun(run_id=res.run_id, input=inp, output="[[MALFORMED]]",
+                        events=res.events, cost=res.cost,
+                        metadata={**res.metadata, "error": "malformed"})
+    return AgentRun(run_id="", input=inp, output="[[MALFORMED]]", metadata={"error": "malformed"})
+
+
+FAULTS: Dict[str, Callable] = {
+    "api_fail": api_fail,
+    "timeout": timeout,
+    "rate_limit": rate_limit,
+    "network_delay": network_delay,
+    "tool_crash": tool_crash,
+    "memory_corrupt": memory_corrupt,
+    "token_limit": token_limit,
+    "malformed_output": malformed_output,
+}
+
+
+def inject(agent: Callable, fault: str, p: float = 1.0, seed: int = 0) -> Callable:
+    """Wrap ``agent`` so fault ``fault`` fires with probability ``p``."""
+    if fault not in FAULTS:
+        raise ValueError(f"unknown fault {fault!r}; have {sorted(FAULTS)}")
+    return _wrap(agent, FAULTS[fault], p, random.Random(seed))
+
+
+def available() -> List[str]:
+    return list(FAULTS)

entropy_ai-0.1.0/entropy/chaos/runner.py

@@ -0,0 +1,40 @@
+"""Chaos runner (spec §10): run the suite under each fault and compare."""
+from __future__ import annotations
+
+import random
+from typing import Callable, Iterable, List
+
+from ..core.suite import Suite
+from .faults import FAULTS, inject
+
+
+class ChaosRunner:
+    """Run an agent under baseline and each requested fault."""
+
+    def __init__(self, seed: int = 0):
+        self.seed = seed
+
+    def run(self, agent: Callable, dataset, faults: List[str] | str,
+            trials: int = 50) -> dict:
+        if isinstance(faults, str):
+            faults = [faults]
+        unknown = [f for f in faults if f not in FAULTS]
+        if unknown:
+            raise ValueError(f"unknown faults: {unknown}")
+        rng = random.Random(self.seed)
+        baseline = Suite(seed=self.seed).run(agent, dataset, trials=trials)
+        out = {"baseline": baseline, "faults": {}}
+        for f in faults:
+            wrapped = inject(agent, f, p=1.0, seed=self.seed)
+            out["faults"][f] = Suite(seed=self.seed).run(wrapped, dataset, trials=trials)
+        return out
+
+    def summary(self, result: dict) -> str:
+        lines = [f"{'fault':16} {'success':>9} {'fail':>9} {'drift':>9}"]
+        b = result["baseline"]
+        lines.append(f"{'(baseline)':16} {b['success_rate']:9.4f} "
+                     f"{1 - b['success_rate']:9.4f} {b['drift_score']:9.4f}")
+        for f, r in result["faults"].items():
+            lines.append(f"{f:16} {r['success_rate']:9.4f} "
+                         f"{1 - r['success_rate']:9.4f} {r['drift_score']:9.4f}")
+        return "\n".join(lines)

entropy_ai-0.1.0/entropy/cli/__init__.py

@@ -0,0 +1,6 @@
+"""Entropy CLI package."""
+from __future__ import annotations
+
+from .main import build_parser, main
+
+__all__ = ["build_parser", "main"]

entropy_ai-0.1.0/entropy/cli/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .main import main
+
+if __name__ == "__main__":
+    sys.exit(main())