agobs 1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentlens/__init__.py +48 -0
- agentlens/_diag.py +32 -0
- agentlens/adapters/__init__.py +16 -0
- agentlens/adapters/anthropic_agents.py +52 -0
- agentlens/adapters/autogen.py +61 -0
- agentlens/adapters/common.py +46 -0
- agentlens/adapters/crewai.py +66 -0
- agentlens/adapters/fastmcp.py +56 -0
- agentlens/adapters/langchain.py +183 -0
- agentlens/adapters/langgraph.py +17 -0
- agentlens/adapters/llamaindex.py +73 -0
- agentlens/adapters/mcp.py +47 -0
- agentlens/adapters/openai_agents.py +125 -0
- agentlens/adapters/pydanticai.py +92 -0
- agentlens/adapters/strands.py +92 -0
- agentlens/cli.py +57 -0
- agentlens/client.py +109 -0
- agentlens/events.py +124 -0
- agentlens/instrument/__init__.py +41 -0
- agentlens/instrument/_patch.py +119 -0
- agentlens/instrument/anthropic.py +32 -0
- agentlens/instrument/gemini.py +31 -0
- agentlens/instrument/groq.py +32 -0
- agentlens/instrument/litellm.py +33 -0
- agentlens/instrument/openai.py +32 -0
- agentlens/monitor.py +43 -0
- agentlens/server/__init__.py +0 -0
- agentlens/server/api.py +115 -0
- agentlens/server/app.py +68 -0
- agentlens/server/db.py +471 -0
- agentlens/server/docs.py +435 -0
- agentlens/server/llm.py +167 -0
- agentlens/server/pricing.py +100 -0
- agentlens/server/static/app.js +268 -0
- agentlens/server/static/fonts.css +46 -0
- agentlens/server/static/style.css +245 -0
- agentlens/server/templates/agents.html +1 -0
- agentlens/server/templates/base.html +38 -0
- agentlens/server/templates/cost.html +1 -0
- agentlens/server/templates/failures.html +1 -0
- agentlens/server/templates/memory.html +1 -0
- agentlens/server/templates/overview.html +1 -0
- agentlens/server/templates/run_detail.html +1 -0
- agentlens/server/templates/runs.html +1 -0
- agentlens/server/templates/tools.html +1 -0
- agentlens/storage.py +59 -0
- agentlens/tracing.py +178 -0
- agobs-1.0.dist-info/METADATA +164 -0
- agobs-1.0.dist-info/RECORD +52 -0
- agobs-1.0.dist-info/WHEEL +5 -0
- agobs-1.0.dist-info/entry_points.txt +2 -0
- agobs-1.0.dist-info/top_level.txt +1 -0
agentlens/__init__.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""AgentLens — local-first diagnostics & observability for AI agents.
|
|
2
|
+
|
|
3
|
+
Answers one question: *"Why did my agent behave this way?"*
|
|
4
|
+
|
|
5
|
+
Zero-config quickstart::
|
|
6
|
+
|
|
7
|
+
from agentlens import monitor
|
|
8
|
+
monitor.start() # auto-patches installed LLM SDKs
|
|
9
|
+
# ... run your agent ...
|
|
10
|
+
|
|
11
|
+
Manual instrumentation (framework-agnostic)::
|
|
12
|
+
|
|
13
|
+
import agentlens
|
|
14
|
+
agentlens.init(project="research-bot")
|
|
15
|
+
with agentlens.trace_agent("planner"):
|
|
16
|
+
agentlens.log_tool("web_search", args={"q": "..."}, result=[...], retry_count=1)
|
|
17
|
+
agentlens.log_memory("read", "user_prefs", hit=True)
|
|
18
|
+
agentlens.log_llm(model="gpt-4o", input_tokens=1200, output_tokens=400)
|
|
19
|
+
|
|
20
|
+
Then ``agentlens ui`` to explore the dashboard (default http://127.0.0.1:7180).
|
|
21
|
+
"""
|
|
22
|
+
from . import monitor
|
|
23
|
+
from .client import flush, get_client, init
|
|
24
|
+
from .events import Span, SpanKind
|
|
25
|
+
from .tracing import (
|
|
26
|
+
current_session_id,
|
|
27
|
+
current_trace_id,
|
|
28
|
+
log_decision,
|
|
29
|
+
log_error,
|
|
30
|
+
log_llm,
|
|
31
|
+
log_memory,
|
|
32
|
+
log_retrieval,
|
|
33
|
+
log_tool,
|
|
34
|
+
trace,
|
|
35
|
+
trace_agent,
|
|
36
|
+
trace_session,
|
|
37
|
+
trace_workflow,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
__version__ = "0.1.0"
|
|
41
|
+
|
|
42
|
+
__all__ = [
|
|
43
|
+
"init", "flush", "get_client", "monitor",
|
|
44
|
+
"trace", "trace_session", "trace_workflow", "trace_agent",
|
|
45
|
+
"current_trace_id", "current_session_id",
|
|
46
|
+
"log_llm", "log_tool", "log_memory", "log_decision", "log_retrieval", "log_error",
|
|
47
|
+
"Span", "SpanKind", "__version__",
|
|
48
|
+
]
|
agentlens/_diag.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Diagnostics for the framework adapters and auto-patchers.
|
|
2
|
+
|
|
3
|
+
Adapters hook into framework internals (callback signatures, method names) that
|
|
4
|
+
move between versions. When they drift the failure is silent — a span just stops
|
|
5
|
+
being captured. These helpers turn that silence into a visible
|
|
6
|
+
``AgentLensWarning`` so version drift is noticed instead of producing empty
|
|
7
|
+
dashboards.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import warnings
|
|
12
|
+
from typing import Iterable
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class AgentLensWarning(UserWarning):
|
|
16
|
+
"""Emitted when an adapter/patcher can't hook something it expected to."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def warn(message: str) -> None:
|
|
20
|
+
warnings.warn(f"[agentlens] {message}", AgentLensWarning, stacklevel=3)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def require_methods(obj: object, methods: Iterable[str], what: str) -> None:
|
|
24
|
+
"""Warn if ``obj`` is missing every one of ``methods`` (so the wrapper would
|
|
25
|
+
silently capture nothing). ``methods`` is treated as "at least one must
|
|
26
|
+
exist"."""
|
|
27
|
+
present = [m for m in methods if callable(getattr(obj, m, None))]
|
|
28
|
+
if not present:
|
|
29
|
+
warn(
|
|
30
|
+
f"{what}: {type(obj).__name__} has none of {list(methods)} — "
|
|
31
|
+
f"that will not be captured (framework version drift?)"
|
|
32
|
+
)
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Framework adapters.
|
|
2
|
+
|
|
3
|
+
Auto-patching (``monitor.start()``) covers the raw LLM SDKs. Agent frameworks
|
|
4
|
+
expose their own callback/hook systems that carry far richer structure (agent
|
|
5
|
+
boundaries, tool calls, decisions), so each framework gets a dedicated adapter
|
|
6
|
+
here. Every adapter is import-time safe: importing this package never requires
|
|
7
|
+
any framework to be installed.
|
|
8
|
+
|
|
9
|
+
Common, framework-agnostic helpers live in ``common`` (``wrap_tool``,
|
|
10
|
+
``traced_tool``) and work everywhere.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from .common import traced_tool, wrap_tool
|
|
15
|
+
|
|
16
|
+
__all__ = ["traced_tool", "wrap_tool"]
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Anthropic Agent SDK (``claude-agent-sdk``) adapter.
|
|
2
|
+
|
|
3
|
+
The Claude Agent SDK runs tools you define and streams messages. The most
|
|
4
|
+
reliable, version-stable instrumentation point is the tools themselves: wrap
|
|
5
|
+
each tool callable so its invocation logs a tool span (args, result, errors).
|
|
6
|
+
Raw Claude API calls the SDK makes are additionally captured by
|
|
7
|
+
``monitor.start()`` auto-patching the underlying ``anthropic`` client.
|
|
8
|
+
|
|
9
|
+
Example::
|
|
10
|
+
|
|
11
|
+
from agentlens.adapters.anthropic_agents import traced_tool
|
|
12
|
+
|
|
13
|
+
@traced_tool("get_weather")
|
|
14
|
+
async def get_weather(args): ...
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import functools
|
|
19
|
+
import time
|
|
20
|
+
from typing import Any, Callable, Optional
|
|
21
|
+
|
|
22
|
+
from .common import traced_tool, wrap_tool
|
|
23
|
+
|
|
24
|
+
instrument_tool = wrap_tool
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def traced_async_tool(name: Optional[str] = None):
|
|
28
|
+
"""Async variant of :func:`traced_tool` for ``async def`` tool handlers."""
|
|
29
|
+
def deco(fn: Callable) -> Callable:
|
|
30
|
+
tool_name = name or getattr(fn, "__name__", "tool")
|
|
31
|
+
|
|
32
|
+
@functools.wraps(fn)
|
|
33
|
+
async def wrapper(*args, **kwargs):
|
|
34
|
+
from ..tracing import log_tool
|
|
35
|
+
|
|
36
|
+
t0 = time.time()
|
|
37
|
+
try:
|
|
38
|
+
result = await fn(*args, **kwargs)
|
|
39
|
+
except Exception as e:
|
|
40
|
+
log_tool(tool_name, args={"args": args, "kwargs": kwargs}, error=e,
|
|
41
|
+
duration_ms=(time.time() - t0) * 1000.0)
|
|
42
|
+
raise
|
|
43
|
+
log_tool(tool_name, args={"args": args, "kwargs": kwargs}, result=result,
|
|
44
|
+
duration_ms=(time.time() - t0) * 1000.0)
|
|
45
|
+
return result
|
|
46
|
+
|
|
47
|
+
return wrapper
|
|
48
|
+
|
|
49
|
+
return deco
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
__all__ = ["traced_tool", "traced_async_tool", "wrap_tool", "instrument_tool"]
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""AutoGen (AG2 / pyautogen) adapter.
|
|
2
|
+
|
|
3
|
+
AutoGen conversable agents support registered reply/hook functions. This adapter
|
|
4
|
+
instruments an agent so each generated reply logs an agent span, and provides a
|
|
5
|
+
tool wrapper for function-calling tools.
|
|
6
|
+
|
|
7
|
+
Example::
|
|
8
|
+
|
|
9
|
+
from agentlens.adapters.autogen import instrument_agent
|
|
10
|
+
instrument_agent(assistant)
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from .. import client as _client
|
|
17
|
+
from ..events import Span, SpanKind, jsonable
|
|
18
|
+
from .common import wrap_tool
|
|
19
|
+
|
|
20
|
+
instrument_tool = wrap_tool
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def instrument_agent(agent: Any) -> Any:
|
|
24
|
+
"""Hook an AutoGen agent so its replies log agent spans. Uses
|
|
25
|
+
``register_hook('process_message_before_send')`` when available, else wraps
|
|
26
|
+
``generate_reply``."""
|
|
27
|
+
name = getattr(agent, "name", None) or type(agent).__name__
|
|
28
|
+
|
|
29
|
+
def _log(content: Any) -> None:
|
|
30
|
+
try:
|
|
31
|
+
ev = Span(project=_client.get_project(), kind=SpanKind.AGENT.value, name=name,
|
|
32
|
+
attributes={"agent_name": name, "output": jsonable(content)})
|
|
33
|
+
_client.get_client().log_event(ev.finish().model_dump())
|
|
34
|
+
except Exception:
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
if hasattr(agent, "register_hook"):
|
|
38
|
+
def hook(sender=None, message=None, recipient=None, silent=None, **kw):
|
|
39
|
+
_log(message)
|
|
40
|
+
return message
|
|
41
|
+
try:
|
|
42
|
+
agent.register_hook("process_message_before_send", hook)
|
|
43
|
+
return agent
|
|
44
|
+
except Exception:
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
# fallback: wrap generate_reply
|
|
48
|
+
original = getattr(agent, "generate_reply", None)
|
|
49
|
+
if callable(original):
|
|
50
|
+
import functools
|
|
51
|
+
|
|
52
|
+
@functools.wraps(original)
|
|
53
|
+
def wrapper(*a, **k):
|
|
54
|
+
reply = original(*a, **k)
|
|
55
|
+
_log(reply)
|
|
56
|
+
return reply
|
|
57
|
+
try:
|
|
58
|
+
agent.generate_reply = wrapper
|
|
59
|
+
except Exception:
|
|
60
|
+
pass
|
|
61
|
+
return agent
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Framework-agnostic instrumentation helpers.
|
|
2
|
+
|
|
3
|
+
These work with any agent framework (or none): wrap a plain callable as a tool
|
|
4
|
+
so each invocation logs a ``tool`` span — including its arguments, result,
|
|
5
|
+
duration, and any exception (which automatically feeds the Failure Explorer).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import functools
|
|
10
|
+
import time
|
|
11
|
+
from typing import Any, Callable, Optional
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def wrap_tool(fn: Callable, name: Optional[str] = None) -> Callable:
|
|
15
|
+
"""Return a wrapped callable that logs a ``tool`` span per call. Re-raises
|
|
16
|
+
the original exception after recording it, so behavior is unchanged."""
|
|
17
|
+
tool_name = name or getattr(fn, "__name__", None) or "tool"
|
|
18
|
+
|
|
19
|
+
@functools.wraps(fn)
|
|
20
|
+
def wrapper(*args, **kwargs):
|
|
21
|
+
from ..tracing import log_tool
|
|
22
|
+
|
|
23
|
+
t0 = time.time()
|
|
24
|
+
try:
|
|
25
|
+
result = fn(*args, **kwargs)
|
|
26
|
+
except Exception as e:
|
|
27
|
+
log_tool(tool_name, args={"args": args, "kwargs": kwargs}, error=e,
|
|
28
|
+
duration_ms=(time.time() - t0) * 1000.0)
|
|
29
|
+
raise
|
|
30
|
+
log_tool(tool_name, args={"args": args, "kwargs": kwargs}, result=result,
|
|
31
|
+
duration_ms=(time.time() - t0) * 1000.0)
|
|
32
|
+
return result
|
|
33
|
+
|
|
34
|
+
return wrapper
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def traced_tool(name: Optional[str] = None):
|
|
38
|
+
"""Decorator form of :func:`wrap_tool`.
|
|
39
|
+
|
|
40
|
+
@traced_tool()
|
|
41
|
+
def web_search(q): ...
|
|
42
|
+
"""
|
|
43
|
+
def deco(fn: Callable) -> Callable:
|
|
44
|
+
return wrap_tool(fn, name)
|
|
45
|
+
|
|
46
|
+
return deco
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""CrewAI adapter.
|
|
2
|
+
|
|
3
|
+
CrewAI agents/crews accept ``step_callback`` and ``task_callback`` hooks, and
|
|
4
|
+
their tools are plain callables. This adapter provides:
|
|
5
|
+
|
|
6
|
+
* ``step_callback`` / ``task_callback`` — pass to ``Agent(...)`` / ``Crew(...)``
|
|
7
|
+
to log agent-step and task spans,
|
|
8
|
+
* ``instrument_tools(tools)`` — wrap a list of tools so each invocation logs a
|
|
9
|
+
tool span (uses the framework-agnostic ``wrap_tool``).
|
|
10
|
+
|
|
11
|
+
Example::
|
|
12
|
+
|
|
13
|
+
from agentlens.adapters.crewai import step_callback, instrument_tools
|
|
14
|
+
agent = Agent(..., tools=instrument_tools(tools), step_callback=step_callback)
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from typing import Any, List
|
|
19
|
+
|
|
20
|
+
from ..events import SpanKind, jsonable
|
|
21
|
+
from ..tracing import _log
|
|
22
|
+
from .common import wrap_tool
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def step_callback(step: Any) -> None:
|
|
26
|
+
"""Log a CrewAI agent step as an agent span (best-effort field extraction).
|
|
27
|
+
|
|
28
|
+
Uses the contextvar-aware logger so the span nests under whatever trace is
|
|
29
|
+
active — wrap the crew kickoff in ``agentlens.trace_workflow(...)`` to keep a
|
|
30
|
+
crew's steps in one run instead of scattering them into top-level rows."""
|
|
31
|
+
try:
|
|
32
|
+
_log(SpanKind.AGENT.value, "crewai.step",
|
|
33
|
+
{"agent_name": getattr(step, "agent", None) and str(getattr(step, "agent")),
|
|
34
|
+
"output": jsonable(getattr(step, "output", None) or step)})
|
|
35
|
+
except Exception:
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def task_callback(task_output: Any) -> None:
|
|
40
|
+
"""Log a CrewAI task result as a workflow span (nested under the active trace)."""
|
|
41
|
+
try:
|
|
42
|
+
_log(SpanKind.WORKFLOW.value, "crewai.task",
|
|
43
|
+
{"output": jsonable(getattr(task_output, "raw", None) or task_output)})
|
|
44
|
+
except Exception:
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def instrument_tools(tools: List[Any]) -> List[Any]:
|
|
49
|
+
"""Wrap each tool's underlying callable so calls log tool spans. Works for
|
|
50
|
+
CrewAI ``BaseTool`` instances (wraps ``_run``/``run``) and plain functions."""
|
|
51
|
+
out = []
|
|
52
|
+
for t in tools or []:
|
|
53
|
+
for attr in ("_run", "run", "func"):
|
|
54
|
+
fn = getattr(t, attr, None)
|
|
55
|
+
if callable(fn):
|
|
56
|
+
name = getattr(t, "name", None) or getattr(fn, "__name__", "tool")
|
|
57
|
+
try:
|
|
58
|
+
setattr(t, attr, wrap_tool(fn, name))
|
|
59
|
+
except Exception:
|
|
60
|
+
pass
|
|
61
|
+
break
|
|
62
|
+
else:
|
|
63
|
+
if callable(t):
|
|
64
|
+
t = wrap_tool(t)
|
|
65
|
+
out.append(t)
|
|
66
|
+
return out
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""FastMCP adapter.
|
|
2
|
+
|
|
3
|
+
FastMCP keeps registered tools in a tool manager; this adapter wraps each tool's
|
|
4
|
+
underlying function so calls log tool spans. Call ``instrument_server(mcp)``
|
|
5
|
+
after your tools are registered.
|
|
6
|
+
|
|
7
|
+
Example::
|
|
8
|
+
|
|
9
|
+
from fastmcp import FastMCP
|
|
10
|
+
from agentlens.adapters.fastmcp import instrument_server
|
|
11
|
+
|
|
12
|
+
mcp = FastMCP("demo")
|
|
13
|
+
|
|
14
|
+
@mcp.tool()
|
|
15
|
+
def add(a: int, b: int) -> int: return a + b
|
|
16
|
+
|
|
17
|
+
instrument_server(mcp)
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from .anthropic_agents import traced_async_tool
|
|
24
|
+
from .common import traced_tool, wrap_tool
|
|
25
|
+
|
|
26
|
+
instrument_tool = wrap_tool
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def instrument_server(mcp: Any) -> Any:
|
|
30
|
+
"""Wrap each registered FastMCP tool's function with a tool-span logger."""
|
|
31
|
+
manager = getattr(mcp, "_tool_manager", None) or getattr(mcp, "tool_manager", None)
|
|
32
|
+
tools = None
|
|
33
|
+
if manager is not None:
|
|
34
|
+
tools = getattr(manager, "_tools", None) or getattr(manager, "tools", None)
|
|
35
|
+
tools = tools if tools is not None else getattr(mcp, "_tools", None)
|
|
36
|
+
|
|
37
|
+
items = []
|
|
38
|
+
if isinstance(tools, dict):
|
|
39
|
+
items = list(tools.items())
|
|
40
|
+
elif isinstance(tools, (list, tuple)):
|
|
41
|
+
items = [(getattr(t, "name", i), t) for i, t in enumerate(tools)]
|
|
42
|
+
|
|
43
|
+
for key, tool in items:
|
|
44
|
+
for attr in ("fn", "func", "handler", "callback"):
|
|
45
|
+
fn = getattr(tool, attr, None)
|
|
46
|
+
if callable(fn) and not getattr(fn, "__agentlens_patched__", False):
|
|
47
|
+
name = getattr(tool, "name", None) or str(key)
|
|
48
|
+
try:
|
|
49
|
+
setattr(tool, attr, wrap_tool(fn, name))
|
|
50
|
+
except Exception:
|
|
51
|
+
pass
|
|
52
|
+
break
|
|
53
|
+
return mcp
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
__all__ = ["traced_tool", "traced_async_tool", "wrap_tool", "instrument_tool", "instrument_server"]
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""LangChain adapter: a callback handler that converts LangChain run events
|
|
2
|
+
into AgentLens spans.
|
|
3
|
+
|
|
4
|
+
Usage::
|
|
5
|
+
|
|
6
|
+
from agentlens.adapters.langchain import AgentLensCallbackHandler
|
|
7
|
+
chain.invoke(question, config={"callbacks": [AgentLensCallbackHandler()]})
|
|
8
|
+
|
|
9
|
+
LangGraph runs through the same LangChain callback system, so this handler works
|
|
10
|
+
for LangGraph graphs too (see ``adapters/langgraph.py``).
|
|
11
|
+
|
|
12
|
+
The pure mapping helpers below (dict in -> span dict out) are testable without
|
|
13
|
+
LangChain installed.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import time
|
|
18
|
+
from typing import Any, Dict, List, Optional
|
|
19
|
+
|
|
20
|
+
from .. import client as _client
|
|
21
|
+
from ..events import Span, SpanKind, estimate_tokens, jsonable, normalize_result
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
from langchain_core.callbacks import BaseCallbackHandler # type: ignore
|
|
25
|
+
except ImportError: # pragma: no cover
|
|
26
|
+
BaseCallbackHandler = object
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
# --------------------------------------------------------------- pure mapping
|
|
30
|
+
|
|
31
|
+
def llm_span(prompts: List[str], response_text: str, model: Optional[str],
|
|
32
|
+
trace_id: str, parent_span_id: Optional[str], project: str,
|
|
33
|
+
start_time: float, token_usage: Optional[Dict[str, Any]] = None,
|
|
34
|
+
status: str = "ok") -> Dict[str, Any]:
|
|
35
|
+
usage = token_usage or {}
|
|
36
|
+
ev = Span(
|
|
37
|
+
trace_id=trace_id, parent_span_id=parent_span_id, project=project,
|
|
38
|
+
kind=SpanKind.LLM.value, name=model or "langchain.llm", start_time=start_time,
|
|
39
|
+
attributes={"model": model, "provider": "langchain", "prompt": "\n\n".join(prompts),
|
|
40
|
+
"response": response_text, "input_tokens": usage.get("prompt_tokens"),
|
|
41
|
+
"output_tokens": usage.get("completion_tokens")},
|
|
42
|
+
)
|
|
43
|
+
return ev.finish(status).model_dump()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def tool_span(tool_name: str, args: Any, result: Any, trace_id: str,
|
|
47
|
+
parent_span_id: Optional[str], project: str, start_time: float,
|
|
48
|
+
error: Any = None) -> Dict[str, Any]:
|
|
49
|
+
j_args, j_result = jsonable(args), jsonable(result)
|
|
50
|
+
attrs: Dict[str, Any] = {"tool_name": tool_name, "input": j_args, "output": j_result,
|
|
51
|
+
"args": j_args, "result": j_result}
|
|
52
|
+
status = "ok"
|
|
53
|
+
if error is not None:
|
|
54
|
+
status = "error"
|
|
55
|
+
attrs["error"] = repr(error)
|
|
56
|
+
attrs["exception"] = type(error).__name__ if isinstance(error, BaseException) else None
|
|
57
|
+
ev = Span(trace_id=trace_id, parent_span_id=parent_span_id, project=project,
|
|
58
|
+
kind=SpanKind.TOOL.value, name=tool_name, start_time=start_time, attributes=attrs)
|
|
59
|
+
return ev.finish(status).model_dump()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# ------------------------------------------------------------------- handler
|
|
63
|
+
|
|
64
|
+
class AgentLensCallbackHandler(BaseCallbackHandler):
|
|
65
|
+
"""Maps on_chain_* to the run/agent boundary, on_tool_* to tool spans,
|
|
66
|
+
on_retriever_* to retrieval spans and on_llm_* to llm spans."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, project: Optional[str] = None):
|
|
69
|
+
if BaseCallbackHandler is object:
|
|
70
|
+
raise ImportError("LangChain is not installed. Run: pip install agentlens[langchain]")
|
|
71
|
+
self.project = project or _client.get_project()
|
|
72
|
+
self._trace_id: Optional[str] = None
|
|
73
|
+
self._root_run: Optional[str] = None
|
|
74
|
+
self._starts: Dict[str, float] = {}
|
|
75
|
+
self._names: Dict[str, str] = {}
|
|
76
|
+
self._trace_start: Optional[float] = None
|
|
77
|
+
self._name: Optional[str] = None
|
|
78
|
+
|
|
79
|
+
def _ensure_trace(self) -> str:
|
|
80
|
+
if self._trace_id is None:
|
|
81
|
+
self._trace_id = Span().trace_id
|
|
82
|
+
return self._trace_id
|
|
83
|
+
|
|
84
|
+
def _emit(self, ev: Dict[str, Any]) -> None:
|
|
85
|
+
_client.get_client().log_event(ev)
|
|
86
|
+
|
|
87
|
+
# -- chain = run/agent boundary --------------------------------------
|
|
88
|
+
def on_chain_start(self, serialized, inputs, *, run_id, parent_run_id=None, **kw):
|
|
89
|
+
if parent_run_id is None and self._root_run is None:
|
|
90
|
+
self._root_run = str(run_id)
|
|
91
|
+
self._trace_id = Span().trace_id
|
|
92
|
+
self._trace_start = time.time()
|
|
93
|
+
self._name = (serialized or {}).get("name") if isinstance(serialized, dict) else None
|
|
94
|
+
|
|
95
|
+
def on_chain_end(self, outputs, *, run_id, parent_run_id=None, **kw):
|
|
96
|
+
if str(run_id) == self._root_run:
|
|
97
|
+
ev = Span(trace_id=self._trace_id, project=self.project, kind=SpanKind.WORKFLOW.value,
|
|
98
|
+
name=self._name or "langchain.chain", start_time=self._trace_start or time.time(),
|
|
99
|
+
attributes={"name": self._name})
|
|
100
|
+
self._emit(ev.finish().model_dump())
|
|
101
|
+
self._root_run = None
|
|
102
|
+
|
|
103
|
+
def on_chain_error(self, error, *, run_id, parent_run_id=None, **kw):
|
|
104
|
+
if str(run_id) == self._root_run:
|
|
105
|
+
ev = Span(trace_id=self._trace_id, project=self.project, kind=SpanKind.WORKFLOW.value,
|
|
106
|
+
name=self._name or "langchain.chain", start_time=self._trace_start or time.time(),
|
|
107
|
+
attributes={"error": repr(error), "exception": type(error).__name__})
|
|
108
|
+
self._emit(ev.finish("error").model_dump())
|
|
109
|
+
self._root_run = None
|
|
110
|
+
|
|
111
|
+
# -- tools -----------------------------------------------------------
|
|
112
|
+
def on_tool_start(self, serialized, input_str, *, run_id, parent_run_id=None, **kw):
|
|
113
|
+
rid = str(run_id)
|
|
114
|
+
self._starts[rid] = time.time()
|
|
115
|
+
self._names[rid] = (serialized or {}).get("name", "tool") if isinstance(serialized, dict) else "tool"
|
|
116
|
+
self._names[rid + ":in"] = input_str
|
|
117
|
+
|
|
118
|
+
def on_tool_end(self, output, *, run_id, parent_run_id=None, **kw):
|
|
119
|
+
rid = str(run_id)
|
|
120
|
+
self._emit(tool_span(self._names.pop(rid, "tool"), self._names.pop(rid + ":in", None),
|
|
121
|
+
output, self._ensure_trace(), None, self.project,
|
|
122
|
+
self._starts.pop(rid, time.time())))
|
|
123
|
+
|
|
124
|
+
def on_tool_error(self, error, *, run_id, parent_run_id=None, **kw):
|
|
125
|
+
rid = str(run_id)
|
|
126
|
+
self._emit(tool_span(self._names.pop(rid, "tool"), self._names.pop(rid + ":in", None),
|
|
127
|
+
None, self._ensure_trace(), None, self.project,
|
|
128
|
+
self._starts.pop(rid, time.time()), error=error))
|
|
129
|
+
|
|
130
|
+
# -- retriever -------------------------------------------------------
|
|
131
|
+
def on_retriever_start(self, serialized, query, *, run_id, parent_run_id=None, **kw):
|
|
132
|
+
rid = str(run_id)
|
|
133
|
+
self._starts[rid] = time.time()
|
|
134
|
+
self._names[rid] = query
|
|
135
|
+
|
|
136
|
+
def on_retriever_end(self, documents, *, run_id, parent_run_id=None, **kw):
|
|
137
|
+
rid = str(run_id)
|
|
138
|
+
results = [normalize_result(d) for d in (documents or [])]
|
|
139
|
+
ev = Span(trace_id=self._ensure_trace(), project=self.project, kind=SpanKind.RETRIEVAL.value,
|
|
140
|
+
name="langchain.retriever", start_time=self._starts.pop(rid, time.time()),
|
|
141
|
+
attributes={"query": self._names.pop(rid, ""), "results": results, "top_k": len(results)})
|
|
142
|
+
self._emit(ev.finish().model_dump())
|
|
143
|
+
|
|
144
|
+
# -- llm -------------------------------------------------------------
|
|
145
|
+
def on_llm_start(self, serialized, prompts, *, run_id, parent_run_id=None, **kw):
|
|
146
|
+
rid = str(run_id)
|
|
147
|
+
self._starts[rid] = time.time()
|
|
148
|
+
self._names[rid] = "\n\n".join(prompts)
|
|
149
|
+
|
|
150
|
+
def on_chat_model_start(self, serialized, messages, *, run_id, parent_run_id=None, **kw):
|
|
151
|
+
rid = str(run_id)
|
|
152
|
+
self._starts[rid] = time.time()
|
|
153
|
+
flat = []
|
|
154
|
+
for batch in messages:
|
|
155
|
+
for m in batch:
|
|
156
|
+
flat.append(f"{getattr(m, 'type', 'msg')}: {getattr(m, 'content', m)}")
|
|
157
|
+
self._names[rid] = "\n".join(flat)
|
|
158
|
+
|
|
159
|
+
def on_llm_end(self, response, *, run_id, parent_run_id=None, **kw):
|
|
160
|
+
rid = str(run_id)
|
|
161
|
+
text, model, usage = "", None, None
|
|
162
|
+
try:
|
|
163
|
+
out = getattr(response, "llm_output", None) or {}
|
|
164
|
+
model = out.get("model_name") or out.get("model")
|
|
165
|
+
usage = out.get("token_usage") or out.get("usage")
|
|
166
|
+
gen = response.generations[0][0]
|
|
167
|
+
msg = getattr(gen, "message", None)
|
|
168
|
+
text = getattr(gen, "text", "") or getattr(msg, "content", "")
|
|
169
|
+
meta = getattr(msg, "response_metadata", None) or {}
|
|
170
|
+
model = model or meta.get("model_name") or meta.get("model")
|
|
171
|
+
usage = usage or meta.get("token_usage")
|
|
172
|
+
um = getattr(msg, "usage_metadata", None)
|
|
173
|
+
if not usage and um:
|
|
174
|
+
usage = {"prompt_tokens": um.get("input_tokens"), "completion_tokens": um.get("output_tokens")}
|
|
175
|
+
except (AttributeError, IndexError):
|
|
176
|
+
pass
|
|
177
|
+
self._emit(llm_span([self._names.pop(rid, "")], text, model, self._ensure_trace(),
|
|
178
|
+
None, self.project, self._starts.pop(rid, time.time()), usage))
|
|
179
|
+
|
|
180
|
+
def on_llm_error(self, error, *, run_id, parent_run_id=None, **kw):
|
|
181
|
+
rid = str(run_id)
|
|
182
|
+
self._emit(llm_span([self._names.pop(rid, "")], repr(error), None, self._ensure_trace(),
|
|
183
|
+
None, self.project, self._starts.pop(rid, time.time()), status="error"))
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""LangGraph adapter.
|
|
2
|
+
|
|
3
|
+
LangGraph executes nodes through the LangChain callback system, so the
|
|
4
|
+
``AgentLensCallbackHandler`` from the LangChain adapter captures graph node
|
|
5
|
+
LLM/tool/retrieval activity directly. Pass it in the run config::
|
|
6
|
+
|
|
7
|
+
from agentlens.adapters.langgraph import AgentLensCallbackHandler
|
|
8
|
+
graph.invoke(state, config={"callbacks": [AgentLensCallbackHandler()]})
|
|
9
|
+
|
|
10
|
+
Each graph node fires chain start/end events; tool nodes fire tool events; the
|
|
11
|
+
parent/child span tree reconstructs the workflow path in the Run detail view.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from .langchain import AgentLensCallbackHandler
|
|
16
|
+
|
|
17
|
+
__all__ = ["AgentLensCallbackHandler"]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""LlamaIndex adapter: a callback handler mapping LlamaIndex events to spans.
|
|
2
|
+
|
|
3
|
+
Usage::
|
|
4
|
+
|
|
5
|
+
from llama_index.core import Settings
|
|
6
|
+
from llama_index.core.callbacks import CallbackManager
|
|
7
|
+
from agentlens.adapters.llamaindex import AgentLensLlamaHandler
|
|
8
|
+
|
|
9
|
+
Settings.callback_manager = CallbackManager([AgentLensLlamaHandler()])
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import time
|
|
14
|
+
from typing import Any, Dict, Optional
|
|
15
|
+
|
|
16
|
+
from .. import client as _client
|
|
17
|
+
from ..events import Span, SpanKind, jsonable, normalize_result
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
from llama_index.core.callbacks.base_handler import BaseCallbackHandler # type: ignore
|
|
21
|
+
from llama_index.core.callbacks.schema import CBEventType # type: ignore
|
|
22
|
+
_OK = True
|
|
23
|
+
except ImportError: # pragma: no cover
|
|
24
|
+
BaseCallbackHandler = object
|
|
25
|
+
CBEventType = None
|
|
26
|
+
_OK = False
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class AgentLensLlamaHandler(BaseCallbackHandler):
|
|
30
|
+
"""Maps LLM / FUNCTION_CALL / AGENT_STEP / RETRIEVE events to AgentLens spans."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, project: Optional[str] = None):
|
|
33
|
+
if not _OK:
|
|
34
|
+
raise ImportError("LlamaIndex is not installed. Run: pip install agentlens[llamaindex]")
|
|
35
|
+
super().__init__(event_starts_to_ignore=[], event_ends_to_ignore=[])
|
|
36
|
+
self.project = project or _client.get_project()
|
|
37
|
+
self._trace_id = Span().trace_id
|
|
38
|
+
self._starts: Dict[str, float] = {}
|
|
39
|
+
self._payloads: Dict[str, Any] = {}
|
|
40
|
+
|
|
41
|
+
def start_trace(self, trace_id: Optional[str] = None) -> None:
|
|
42
|
+
self._trace_id = Span().trace_id
|
|
43
|
+
|
|
44
|
+
def end_trace(self, trace_id: Optional[str] = None, trace_map=None) -> None:
|
|
45
|
+
pass
|
|
46
|
+
|
|
47
|
+
def on_event_start(self, event_type, payload=None, event_id="", parent_id="", **kw) -> str:
|
|
48
|
+
self._starts[event_id] = time.time()
|
|
49
|
+
self._payloads[event_id] = payload or {}
|
|
50
|
+
return event_id
|
|
51
|
+
|
|
52
|
+
def on_event_end(self, event_type, payload=None, event_id="", **kw) -> None:
|
|
53
|
+
t0 = self._starts.pop(event_id, time.time())
|
|
54
|
+
payload = payload or self._payloads.pop(event_id, {})
|
|
55
|
+
name = str(getattr(event_type, "value", event_type))
|
|
56
|
+
kind, attrs = self._map(event_type, payload)
|
|
57
|
+
ev = Span(trace_id=self._trace_id, project=self.project, kind=kind,
|
|
58
|
+
name=name, start_time=t0, attributes=attrs)
|
|
59
|
+
_client.get_client().log_event(ev.finish().model_dump())
|
|
60
|
+
|
|
61
|
+
def _map(self, event_type, payload) -> tuple:
|
|
62
|
+
et = str(getattr(event_type, "value", event_type)).lower()
|
|
63
|
+
if "llm" in et:
|
|
64
|
+
return SpanKind.LLM.value, {"model": jsonable(payload.get("serialized", {})),
|
|
65
|
+
"response": jsonable(payload.get("response"))}
|
|
66
|
+
if "retrieve" in et:
|
|
67
|
+
nodes = payload.get("nodes") or []
|
|
68
|
+
return SpanKind.RETRIEVAL.value, {"query": jsonable(payload.get("query_str")),
|
|
69
|
+
"results": [normalize_result(n) for n in nodes]}
|
|
70
|
+
if "function" in et or "tool" in et or "agent" in et:
|
|
71
|
+
jp = jsonable(payload)
|
|
72
|
+
return SpanKind.TOOL.value, {"tool_name": et, "input": jp, "args": jp}
|
|
73
|
+
return SpanKind.OTHER.value, {"payload": jsonable(payload)}
|