brenducate-sdk 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brenducate_sdk-0.1.0/PKG-INFO +104 -0
- brenducate_sdk-0.1.0/README.md +81 -0
- brenducate_sdk-0.1.0/agentpulse/__init__.py +23 -0
- brenducate_sdk-0.1.0/agentpulse/circuit_breaker.py +178 -0
- brenducate_sdk-0.1.0/agentpulse/client.py +82 -0
- brenducate_sdk-0.1.0/agentpulse/integrations/__init__.py +13 -0
- brenducate_sdk-0.1.0/agentpulse/integrations/langchain.py +185 -0
- brenducate_sdk-0.1.0/agentpulse/integrations/openai.py +139 -0
- brenducate_sdk-0.1.0/agentpulse/monitor.py +420 -0
- brenducate_sdk-0.1.0/brenducate/__init__.py +23 -0
- brenducate_sdk-0.1.0/brenducate/circuit_breaker.py +178 -0
- brenducate_sdk-0.1.0/brenducate/client.py +82 -0
- brenducate_sdk-0.1.0/brenducate/integrations/__init__.py +13 -0
- brenducate_sdk-0.1.0/brenducate/integrations/langchain.py +185 -0
- brenducate_sdk-0.1.0/brenducate/integrations/openai.py +139 -0
- brenducate_sdk-0.1.0/brenducate/monitor.py +420 -0
- brenducate_sdk-0.1.0/brenducate_sdk.egg-info/PKG-INFO +104 -0
- brenducate_sdk-0.1.0/brenducate_sdk.egg-info/SOURCES.txt +22 -0
- brenducate_sdk-0.1.0/brenducate_sdk.egg-info/dependency_links.txt +1 -0
- brenducate_sdk-0.1.0/brenducate_sdk.egg-info/requires.txt +2 -0
- brenducate_sdk-0.1.0/brenducate_sdk.egg-info/top_level.txt +2 -0
- brenducate_sdk-0.1.0/pyproject.toml +29 -0
- brenducate_sdk-0.1.0/setup.cfg +4 -0
- brenducate_sdk-0.1.0/setup.py +32 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: brenducate-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Production observability, circuit-breaker guardrails, and prompt drift monitoring for autonomous LLM agents.
|
|
5
|
+
Home-page: https://brenducate.com
|
|
6
|
+
Author: Brenducate
|
|
7
|
+
Author-email: founder@brenducate.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Project-URL: Homepage, https://brenducate.com
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
Requires-Dist: httpx>=0.27.0
|
|
18
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
19
|
+
Dynamic: author
|
|
20
|
+
Dynamic: author-email
|
|
21
|
+
Dynamic: home-page
|
|
22
|
+
Dynamic: requires-python
|
|
23
|
+
|
|
24
|
+
# AgentPulse Python SDK (`agentpulse-sdk`)
|
|
25
|
+
|
|
26
|
+
Production observability, prompt drift sentinel, and sub-millisecond circuit breakers for autonomous LLM agents (LangChain, AutoGen, CrewAI, LlamaIndex, or raw OpenAI/Anthropic).
|
|
27
|
+
|
|
28
|
+
## 🚀 Quickstart
|
|
29
|
+
|
|
30
|
+
### 1. Install
|
|
31
|
+
```bash
|
|
32
|
+
pip install agentpulse-sdk
|
|
33
|
+
# or install locally from source:
|
|
34
|
+
pip install -e sdk/
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### 2. Wrap your Agent Function
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from agentpulse import AgentPulse, CircuitBreaker
|
|
41
|
+
|
|
42
|
+
# Initialize sentinel
|
|
43
|
+
pulse = AgentPulse(
|
|
44
|
+
api_key="ap_live_your_api_key_here",
|
|
45
|
+
base_url="http://localhost:8000", # Use https://api.agentpulse.io in production
|
|
46
|
+
environment="production"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# Protect your agent with a circuit breaker
|
|
50
|
+
@pulse.monitor(
|
|
51
|
+
agent_id="customer-support-copilot",
|
|
52
|
+
model="claude-3-7-sonnet",
|
|
53
|
+
circuit_breaker=CircuitBreaker(
|
|
54
|
+
max_hops=6, # Abort if agent loops > 6 hops
|
|
55
|
+
max_cost_usd=0.50, # Kill if run exceeds $0.50
|
|
56
|
+
kill_on_infinite_loop=True, # Kill repetitive duplicate tool calls
|
|
57
|
+
max_duplicate_calls=2
|
|
58
|
+
)
|
|
59
|
+
)
|
|
60
|
+
def run_my_agent(user_query: str):
|
|
61
|
+
# Record reasoning steps or tool calls inside your loop
|
|
62
|
+
pulse.record_step(
|
|
63
|
+
step_type="tool",
|
|
64
|
+
title="CRM Search",
|
|
65
|
+
content="Searching customer ticket history",
|
|
66
|
+
tool_call_str='crm.search(query="invoice 9812")',
|
|
67
|
+
input_tokens=1500,
|
|
68
|
+
output_tokens=300
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
# If the agent attempts a runaway recursive loop, CircuitBreakerException
|
|
72
|
+
# is raised in <1ms, killing the run and logging the incident to your dashboard!
|
|
73
|
+
return "Resolved query."
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### 3. Context Manager Syntax
|
|
77
|
+
|
|
78
|
+
You can also use AgentPulse as an explicit context manager:
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
with pulse.trace("sql-analyst", model="gpt-4o") as t:
|
|
82
|
+
t.record_step(
|
|
83
|
+
step_type="tool",
|
|
84
|
+
title="Execute SQL",
|
|
85
|
+
tool_call_str="SELECT * FROM users",
|
|
86
|
+
input_tokens=800,
|
|
87
|
+
output_tokens=200
|
|
88
|
+
)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## 🛡️ Circuit Breakers & Guardrails
|
|
92
|
+
|
|
93
|
+
| Guardrail | Parameter | Default | Behavior |
|
|
94
|
+
| :--- | :--- | :--- | :--- |
|
|
95
|
+
| **Max Hops** | `max_hops` | `8` | Aborts run if total reasoning steps / tool calls exceeds limit. |
|
|
96
|
+
| **Cost Ceiling** | `max_cost_usd` | `$1.00` | Calculates token spend against live model pricing; aborts if exceeded. |
|
|
97
|
+
| **Infinite Recursion** | `kill_on_infinite_loop` | `True` | Intercepts repetitive duplicate queries with zero parameter change. |
|
|
98
|
+
| **Duplicate Threshold** | `max_duplicate_calls` | `2` | Number of allowable consecutive duplicate calls before termination. |
|
|
99
|
+
|
|
100
|
+
## 📦 Features
|
|
101
|
+
- **Zero latency impact**: Telemetry payloads are delivered asynchronously using non-blocking HTTP workers.
|
|
102
|
+
- **In-process guardrail checks**: All circuit-breaker decisions execute in memory in `<1ms` without waiting for network responses.
|
|
103
|
+
- **Auto cost calculation**: Built-in pricing matrix for Claude 3.7/3.5, GPT-4o, Gemini 2.5, DeepSeek R1, and Llama 3.3.
|
|
104
|
+
- **Framework agnostic**: Works natively with LangChain, AutoGen, CrewAI, LiteLLM, or plain Python functions.
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# AgentPulse Python SDK (`agentpulse-sdk`)
|
|
2
|
+
|
|
3
|
+
Production observability, prompt drift sentinel, and sub-millisecond circuit breakers for autonomous LLM agents (LangChain, AutoGen, CrewAI, LlamaIndex, or raw OpenAI/Anthropic).
|
|
4
|
+
|
|
5
|
+
## 🚀 Quickstart
|
|
6
|
+
|
|
7
|
+
### 1. Install
|
|
8
|
+
```bash
|
|
9
|
+
pip install agentpulse-sdk
|
|
10
|
+
# or install locally from source:
|
|
11
|
+
pip install -e sdk/
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
### 2. Wrap your Agent Function
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from agentpulse import AgentPulse, CircuitBreaker
|
|
18
|
+
|
|
19
|
+
# Initialize sentinel
|
|
20
|
+
pulse = AgentPulse(
|
|
21
|
+
api_key="ap_live_your_api_key_here",
|
|
22
|
+
base_url="http://localhost:8000", # Use https://api.agentpulse.io in production
|
|
23
|
+
environment="production"
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# Protect your agent with a circuit breaker
|
|
27
|
+
@pulse.monitor(
|
|
28
|
+
agent_id="customer-support-copilot",
|
|
29
|
+
model="claude-3-7-sonnet",
|
|
30
|
+
circuit_breaker=CircuitBreaker(
|
|
31
|
+
max_hops=6, # Abort if agent loops > 6 hops
|
|
32
|
+
max_cost_usd=0.50, # Kill if run exceeds $0.50
|
|
33
|
+
kill_on_infinite_loop=True, # Kill repetitive duplicate tool calls
|
|
34
|
+
max_duplicate_calls=2
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
def run_my_agent(user_query: str):
|
|
38
|
+
# Record reasoning steps or tool calls inside your loop
|
|
39
|
+
pulse.record_step(
|
|
40
|
+
step_type="tool",
|
|
41
|
+
title="CRM Search",
|
|
42
|
+
content="Searching customer ticket history",
|
|
43
|
+
tool_call_str='crm.search(query="invoice 9812")',
|
|
44
|
+
input_tokens=1500,
|
|
45
|
+
output_tokens=300
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# If the agent attempts a runaway recursive loop, CircuitBreakerException
|
|
49
|
+
# is raised in <1ms, killing the run and logging the incident to your dashboard!
|
|
50
|
+
return "Resolved query."
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### 3. Context Manager Syntax
|
|
54
|
+
|
|
55
|
+
You can also use AgentPulse as an explicit context manager:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
with pulse.trace("sql-analyst", model="gpt-4o") as t:
|
|
59
|
+
t.record_step(
|
|
60
|
+
step_type="tool",
|
|
61
|
+
title="Execute SQL",
|
|
62
|
+
tool_call_str="SELECT * FROM users",
|
|
63
|
+
input_tokens=800,
|
|
64
|
+
output_tokens=200
|
|
65
|
+
)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## 🛡️ Circuit Breakers & Guardrails
|
|
69
|
+
|
|
70
|
+
| Guardrail | Parameter | Default | Behavior |
|
|
71
|
+
| :--- | :--- | :--- | :--- |
|
|
72
|
+
| **Max Hops** | `max_hops` | `8` | Aborts run if total reasoning steps / tool calls exceeds limit. |
|
|
73
|
+
| **Cost Ceiling** | `max_cost_usd` | `$1.00` | Calculates token spend against live model pricing; aborts if exceeded. |
|
|
74
|
+
| **Infinite Recursion** | `kill_on_infinite_loop` | `True` | Intercepts repetitive duplicate queries with zero parameter change. |
|
|
75
|
+
| **Duplicate Threshold** | `max_duplicate_calls` | `2` | Number of allowable consecutive duplicate calls before termination. |
|
|
76
|
+
|
|
77
|
+
## 📦 Features
|
|
78
|
+
- **Zero latency impact**: Telemetry payloads are delivered asynchronously using non-blocking HTTP workers.
|
|
79
|
+
- **In-process guardrail checks**: All circuit-breaker decisions execute in memory in `<1ms` without waiting for network responses.
|
|
80
|
+
- **Auto cost calculation**: Built-in pricing matrix for Claude 3.7/3.5, GPT-4o, Gemini 2.5, DeepSeek R1, and Llama 3.3.
|
|
81
|
+
- **Framework agnostic**: Works natively with LangChain, AutoGen, CrewAI, LiteLLM, or plain Python functions.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""
|
|
2
|
+
agentpulse — Production LLM Agent Observability SDK
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
from agentpulse import AgentPulse, CircuitBreaker, ExecutionState
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .monitor import AgentPulse
|
|
9
|
+
from .circuit_breaker import CircuitBreaker, CircuitBreakerException, ExecutionState
|
|
10
|
+
from .client import AgentPulseClient
|
|
11
|
+
from .integrations import wrap_openai, AgentPulseCallbackHandler
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.0"
|
|
14
|
+
__all__ = [
|
|
15
|
+
"AgentPulse",
|
|
16
|
+
"CircuitBreaker",
|
|
17
|
+
"CircuitBreakerException",
|
|
18
|
+
"ExecutionState",
|
|
19
|
+
"AgentPulseClient",
|
|
20
|
+
"wrap_openai",
|
|
21
|
+
"AgentPulseCallbackHandler",
|
|
22
|
+
]
|
|
23
|
+
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""
|
|
2
|
+
AgentPulse SDK — Circuit Breaker Engine
|
|
3
|
+
|
|
4
|
+
Tracks per-execution state, steps, token consumption, and raises
|
|
5
|
+
CircuitBreakerException when any guardrail threshold is crossed.
|
|
6
|
+
Runs entirely in-process with zero network overhead (<1ms kill decision).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Optional, List, Dict, Any
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# Standard model pricing per 1,000,000 tokens (USD)
|
|
14
|
+
MODEL_PRICING: Dict[str, Dict[str, float]] = {
|
|
15
|
+
"claude-3-7-sonnet": {"input": 3.00, "output": 15.00},
|
|
16
|
+
"claude-3-5-sonnet": {"input": 3.00, "output": 15.00},
|
|
17
|
+
"claude-3-5-haiku": {"input": 0.80, "output": 4.00},
|
|
18
|
+
"gpt-4o": {"input": 2.50, "output": 10.00},
|
|
19
|
+
"gpt-4o-mini": {"input": 0.15, "output": 0.60},
|
|
20
|
+
"o1": {"input": 15.00, "output": 60.00},
|
|
21
|
+
"o3-mini": {"input": 1.10, "output": 4.40},
|
|
22
|
+
"gemini-2.5-pro": {"input": 1.25, "output": 5.00},
|
|
23
|
+
"gemini-2.5-flash": {"input": 0.10, "output": 0.40},
|
|
24
|
+
"llama-3.3-70b": {"input": 0.60, "output": 1.80},
|
|
25
|
+
"deepseek-r1": {"input": 0.55, "output": 2.19},
|
|
26
|
+
"default": {"input": 2.00, "output": 8.00},
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def calculate_token_cost(model_name: str, input_tokens: int, output_tokens: int) -> float:
|
|
31
|
+
"""Calculate USD cost based on standard model token pricing."""
|
|
32
|
+
name_clean = model_name.lower().strip()
|
|
33
|
+
pricing = MODEL_PRICING.get("default")
|
|
34
|
+
for key, p in MODEL_PRICING.items():
|
|
35
|
+
if key in name_clean:
|
|
36
|
+
pricing = p
|
|
37
|
+
break
|
|
38
|
+
|
|
39
|
+
in_cost = (input_tokens / 1_000_000.0) * pricing["input"]
|
|
40
|
+
out_cost = (output_tokens / 1_000_000.0) * pricing["output"]
|
|
41
|
+
return round(in_cost + out_cost, 6)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class CircuitBreakerException(Exception):
|
|
45
|
+
"""Raised when a guardrail threshold is crossed and execution is aborted."""
|
|
46
|
+
def __init__(self, reason: str, hops: int, cost_usd: float, tokens: int):
|
|
47
|
+
self.reason = reason
|
|
48
|
+
self.hops = hops
|
|
49
|
+
self.cost_usd = cost_usd
|
|
50
|
+
self.tokens = tokens
|
|
51
|
+
super().__init__(f"[AgentPulse] Circuit breaker tripped: {reason}")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class CircuitBreaker:
|
|
56
|
+
"""
|
|
57
|
+
Configurable guardrail policy for a monitored agent.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
max_hops: Kill if the agent makes more than this many
|
|
61
|
+
sequential tool calls / reasoning steps.
|
|
62
|
+
max_cost_usd: Kill if the estimated token cost for a single
|
|
63
|
+
run exceeds this dollar amount.
|
|
64
|
+
kill_on_infinite_loop: Kill if the exact same tool call string is
|
|
65
|
+
repeated consecutively more than max_duplicate_calls times.
|
|
66
|
+
max_duplicate_calls: Duplicate call threshold (default: 2).
|
|
67
|
+
"""
|
|
68
|
+
max_hops: int = 8
|
|
69
|
+
max_cost_usd: float = 1.00
|
|
70
|
+
kill_on_infinite_loop: bool = True
|
|
71
|
+
max_duplicate_calls: int = 2
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class ExecutionState:
|
|
76
|
+
"""Mutable run-time state tracked per agent execution."""
|
|
77
|
+
hops: int = 0
|
|
78
|
+
input_tokens: int = 0
|
|
79
|
+
output_tokens: int = 0
|
|
80
|
+
cost_usd: float = 0.0
|
|
81
|
+
steps: List[Dict[str, Any]] = field(default_factory=list)
|
|
82
|
+
_tool_call_history: List[str] = field(default_factory=list)
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def total_tokens(self) -> int:
|
|
86
|
+
return self.input_tokens + self.output_tokens
|
|
87
|
+
|
|
88
|
+
def add_step(
|
|
89
|
+
self,
|
|
90
|
+
step_type: str,
|
|
91
|
+
title: str,
|
|
92
|
+
content: str,
|
|
93
|
+
tool_call: Optional[str] = None
|
|
94
|
+
) -> None:
|
|
95
|
+
"""Add a step to the execution graph."""
|
|
96
|
+
self.steps.append({
|
|
97
|
+
"step_index": len(self.steps) + 1,
|
|
98
|
+
"step_type": step_type,
|
|
99
|
+
"title": title,
|
|
100
|
+
"content": content,
|
|
101
|
+
"tool_call": tool_call,
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
def record_hop(
|
|
105
|
+
self,
|
|
106
|
+
breaker: CircuitBreaker,
|
|
107
|
+
input_tokens: int = 0,
|
|
108
|
+
output_tokens: int = 0,
|
|
109
|
+
cost_usd: Optional[float] = None,
|
|
110
|
+
tool_call_str: Optional[str] = None,
|
|
111
|
+
step_type: str = "tool",
|
|
112
|
+
title: Optional[str] = None,
|
|
113
|
+
content: Optional[str] = None,
|
|
114
|
+
model_for_cost: Optional[str] = None,
|
|
115
|
+
) -> None:
|
|
116
|
+
"""
|
|
117
|
+
Record a single reasoning step or tool call.
|
|
118
|
+
Raises CircuitBreakerException immediately if any guardrail is violated.
|
|
119
|
+
"""
|
|
120
|
+
self.hops += 1
|
|
121
|
+
self.input_tokens += input_tokens
|
|
122
|
+
self.output_tokens += output_tokens
|
|
123
|
+
|
|
124
|
+
# Auto-calculate cost if not provided explicitly
|
|
125
|
+
if cost_usd is not None:
|
|
126
|
+
hop_cost = cost_usd
|
|
127
|
+
elif model_for_cost:
|
|
128
|
+
hop_cost = calculate_token_cost(model_for_cost, input_tokens, output_tokens)
|
|
129
|
+
else:
|
|
130
|
+
hop_cost = 0.0
|
|
131
|
+
|
|
132
|
+
self.cost_usd += hop_cost
|
|
133
|
+
|
|
134
|
+
# Add to step history
|
|
135
|
+
step_title = title or (f"Tool Call: {tool_call_str[:40]}..." if tool_call_str else f"Step {self.hops}")
|
|
136
|
+
step_content = content or (tool_call_str or f"Completed hop {self.hops}")
|
|
137
|
+
self.add_step(
|
|
138
|
+
step_type=step_type,
|
|
139
|
+
title=step_title,
|
|
140
|
+
content=step_content,
|
|
141
|
+
tool_call=tool_call_str,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
# ── Guardrail 1: Maximum hop count ──────────────────────────────────
|
|
145
|
+
if self.hops > breaker.max_hops:
|
|
146
|
+
raise CircuitBreakerException(
|
|
147
|
+
reason=f"Maximum hop limit exceeded ({self.hops} > {breaker.max_hops})",
|
|
148
|
+
hops=self.hops,
|
|
149
|
+
cost_usd=self.cost_usd,
|
|
150
|
+
tokens=self.total_tokens,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
# ── Guardrail 2: Cost ceiling ────────────────────────────────────────
|
|
154
|
+
if self.cost_usd > breaker.max_cost_usd:
|
|
155
|
+
raise CircuitBreakerException(
|
|
156
|
+
reason=f"Single-run cost ceiling exceeded (${self.cost_usd:.4f} > ${breaker.max_cost_usd:.2f})",
|
|
157
|
+
hops=self.hops,
|
|
158
|
+
cost_usd=self.cost_usd,
|
|
159
|
+
tokens=self.total_tokens,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
# ── Guardrail 3: Infinite loop / repetitive tool call ────────────────
|
|
163
|
+
if breaker.kill_on_infinite_loop and tool_call_str:
|
|
164
|
+
self._tool_call_history.append(tool_call_str)
|
|
165
|
+
tail = self._tool_call_history[-breaker.max_duplicate_calls - 1:]
|
|
166
|
+
if (
|
|
167
|
+
len(tail) > breaker.max_duplicate_calls
|
|
168
|
+
and len(set(tail)) == 1
|
|
169
|
+
):
|
|
170
|
+
raise CircuitBreakerException(
|
|
171
|
+
reason=(
|
|
172
|
+
f"Repetitive tool call detected: same call repeated "
|
|
173
|
+
f"{breaker.max_duplicate_calls + 1}x consecutively: '{tool_call_str[:80]}'"
|
|
174
|
+
),
|
|
175
|
+
hops=self.hops,
|
|
176
|
+
cost_usd=self.cost_usd,
|
|
177
|
+
tokens=self.total_tokens,
|
|
178
|
+
)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""
|
|
2
|
+
AgentPulse SDK — Async HTTP Client
|
|
3
|
+
|
|
4
|
+
Sends telemetry payloads to the AgentPulse Ops ingestion API.
|
|
5
|
+
Uses a fire-and-forget async pattern so it never adds latency to
|
|
6
|
+
the agent's main execution path.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
import asyncio
|
|
11
|
+
import time
|
|
12
|
+
from typing import List, Optional, Dict, Any
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class AgentPulseClient:
|
|
16
|
+
"""
|
|
17
|
+
Thin async HTTP client for the AgentPulse Ops API.
|
|
18
|
+
Sends traces asynchronously — if the network is down, it fails
|
|
19
|
+
silently so it never disrupts your agent's execution.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, api_key: str, base_url: str):
|
|
23
|
+
self.api_key = api_key
|
|
24
|
+
self.base_url = base_url.rstrip("/")
|
|
25
|
+
self._headers = {
|
|
26
|
+
"Authorization": f"Bearer {api_key}",
|
|
27
|
+
"Content-Type": "application/json",
|
|
28
|
+
"User-Agent": "agentpulse-sdk/0.1.0",
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
def send_trace(self, payload: Dict[str, Any]) -> None:
|
|
32
|
+
"""
|
|
33
|
+
Fire-and-forget trace submission.
|
|
34
|
+
Runs in a background thread so it never blocks the caller.
|
|
35
|
+
"""
|
|
36
|
+
try:
|
|
37
|
+
# Try to get the running event loop (e.g. inside async agent frameworks)
|
|
38
|
+
loop = asyncio.get_event_loop()
|
|
39
|
+
if loop.is_running():
|
|
40
|
+
asyncio.ensure_future(self._post_trace_async(payload))
|
|
41
|
+
else:
|
|
42
|
+
loop.run_until_complete(self._post_trace_async(payload))
|
|
43
|
+
except RuntimeError:
|
|
44
|
+
# No event loop at all — run synchronously as a fallback
|
|
45
|
+
self._post_trace_sync(payload)
|
|
46
|
+
|
|
47
|
+
async def _post_trace_async(self, payload: Dict[str, Any]) -> None:
|
|
48
|
+
try:
|
|
49
|
+
async with httpx.AsyncClient(timeout=5.0) as client:
|
|
50
|
+
await client.post(
|
|
51
|
+
f"{self.base_url}/v1/telemetry/traces",
|
|
52
|
+
json=payload,
|
|
53
|
+
headers=self._headers,
|
|
54
|
+
)
|
|
55
|
+
except Exception:
|
|
56
|
+
pass # Never raise — telemetry is best-effort
|
|
57
|
+
|
|
58
|
+
def _post_trace_sync(self, payload: Dict[str, Any]) -> None:
|
|
59
|
+
try:
|
|
60
|
+
with httpx.Client(timeout=5.0) as client:
|
|
61
|
+
client.post(
|
|
62
|
+
f"{self.base_url}/v1/telemetry/traces",
|
|
63
|
+
json=payload,
|
|
64
|
+
headers=self._headers,
|
|
65
|
+
)
|
|
66
|
+
except Exception:
|
|
67
|
+
pass # Never raise — telemetry is best-effort
|
|
68
|
+
|
|
69
|
+
def get_dashboard_summary(self) -> Optional[Dict]:
|
|
70
|
+
"""Fetch live KPI summary for the current team (synchronous)."""
|
|
71
|
+
try:
|
|
72
|
+
with httpx.Client(timeout=5.0) as client:
|
|
73
|
+
r = client.get(
|
|
74
|
+
f"{self.base_url}/v1/dashboard/summary",
|
|
75
|
+
headers=self._headers,
|
|
76
|
+
)
|
|
77
|
+
r.raise_for_status()
|
|
78
|
+
return r.json()
|
|
79
|
+
except Exception as e:
|
|
80
|
+
print(f"[AgentPulse] Could not fetch summary: {e}")
|
|
81
|
+
return None
|
|
82
|
+
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""
|
|
2
|
+
AgentPulse SDK — Framework Integrations
|
|
3
|
+
Auto-instrumentation for OpenAI, LangChain, LangGraph, and CrewAI.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .openai import wrap_openai
|
|
7
|
+
from .langchain import AgentPulseCallbackHandler
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"wrap_openai",
|
|
11
|
+
"AgentPulseCallbackHandler",
|
|
12
|
+
]
|
|
13
|
+
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""
|
|
2
|
+
AgentPulse SDK — LangChain & LangGraph Integration Callback Handler
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
from agentpulse import AgentPulse, CircuitBreaker
|
|
6
|
+
from agentpulse.integrations.langchain import AgentPulseCallbackHandler
|
|
7
|
+
from langchain.agents import create_openai_tools_agent, AgentExecutor
|
|
8
|
+
|
|
9
|
+
pulse = AgentPulse(api_key="ap_live_...")
|
|
10
|
+
handler = AgentPulseCallbackHandler(
|
|
11
|
+
pulse=pulse,
|
|
12
|
+
agent_id="langchain-agent",
|
|
13
|
+
circuit_breaker=CircuitBreaker(max_hops=6, max_cost_usd=0.50)
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
executor = AgentExecutor(agent=agent, tools=tools, callbacks=[handler])
|
|
17
|
+
result = executor.invoke({"input": "Analyze customer support ticket"})
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import time
|
|
21
|
+
from typing import Any, Dict, List, Optional
|
|
22
|
+
from ..circuit_breaker import CircuitBreaker, CircuitBreakerException, ExecutionState, calculate_token_cost
|
|
23
|
+
from ..monitor import AgentPulse
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
from langchain_core.callbacks.base import BaseCallbackHandler
|
|
27
|
+
except ImportError:
|
|
28
|
+
# Graceful fallback if langchain_core is not installed
|
|
29
|
+
class BaseCallbackHandler:
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class AgentPulseCallbackHandler(BaseCallbackHandler):
|
|
34
|
+
"""
|
|
35
|
+
LangChain & LangGraph Callback Handler.
|
|
36
|
+
Automatically captures LLM generations, tool calls, token usage, latency,
|
|
37
|
+
and enforces AgentPulse Circuit Breakers on LangChain workflows.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
pulse: Optional[AgentPulse] = None,
|
|
43
|
+
agent_id: str = "langchain-agent",
|
|
44
|
+
circuit_breaker: Optional[CircuitBreaker] = None,
|
|
45
|
+
):
|
|
46
|
+
if pulse is None:
|
|
47
|
+
import os
|
|
48
|
+
api_key = os.getenv("AGENTPULSE_API_KEY", "ap_live_demo_key_00001")
|
|
49
|
+
base_url = os.getenv("AGENTPULSE_BASE_URL", "http://localhost:8000")
|
|
50
|
+
pulse = AgentPulse(api_key=api_key, base_url=base_url)
|
|
51
|
+
|
|
52
|
+
self.pulse = pulse
|
|
53
|
+
self.agent_id = agent_id
|
|
54
|
+
self.breaker = circuit_breaker or CircuitBreaker()
|
|
55
|
+
self.state = ExecutionState()
|
|
56
|
+
self.start_time = 0.0
|
|
57
|
+
self.initial_prompt = ""
|
|
58
|
+
self.model_name = "claude-3-7-sonnet"
|
|
59
|
+
self._active_runs: Dict[str, float] = {}
|
|
60
|
+
|
|
61
|
+
def on_chain_start(
|
|
62
|
+
self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any
|
|
63
|
+
) -> None:
|
|
64
|
+
"""Capture top-level agent execution start."""
|
|
65
|
+
if not self.initial_prompt:
|
|
66
|
+
self.start_time = time.monotonic()
|
|
67
|
+
# Capture user goal from inputs
|
|
68
|
+
for key in ("input", "query", "question", "prompt", "task"):
|
|
69
|
+
if key in inputs:
|
|
70
|
+
self.initial_prompt = str(inputs[key])[:500]
|
|
71
|
+
break
|
|
72
|
+
if not self.initial_prompt and inputs:
|
|
73
|
+
self.initial_prompt = str(inputs)[:500]
|
|
74
|
+
|
|
75
|
+
self.state.add_step(
|
|
76
|
+
step_type="thought",
|
|
77
|
+
title="LangChain Chain Started",
|
|
78
|
+
content=f"Workflow initiated with objective: {self.initial_prompt[:120]}...",
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def on_tool_start(
|
|
82
|
+
self, serialized: Dict[str, Any], input_str: str, **kwargs: Any
|
|
83
|
+
) -> None:
|
|
84
|
+
"""Capture tool invocation and enforce circuit breaker limits."""
|
|
85
|
+
tool_name = serialized.get("name", "tool") if isinstance(serialized, dict) else "tool"
|
|
86
|
+
tool_call_str = f"{tool_name}({input_str[:120]})"
|
|
87
|
+
|
|
88
|
+
# Enforce hop & loop limits in real time!
|
|
89
|
+
self.state.record_hop(
|
|
90
|
+
breaker=self.breaker,
|
|
91
|
+
input_tokens=0,
|
|
92
|
+
output_tokens=0,
|
|
93
|
+
cost_usd=0.0,
|
|
94
|
+
tool_call_str=tool_call_str,
|
|
95
|
+
step_type="tool",
|
|
96
|
+
title=f"Tool Call: {tool_name}",
|
|
97
|
+
content=f"Arguments: {input_str[:250]}",
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
def on_tool_end(self, output: str, **kwargs: Any) -> None:
|
|
101
|
+
"""Record tool output."""
|
|
102
|
+
if self.state.steps:
|
|
103
|
+
last_step = self.state.steps[-1]
|
|
104
|
+
if last_step.get("step_type") == "tool":
|
|
105
|
+
last_step["content"] += f"\nOutput: {str(output)[:250]}"
|
|
106
|
+
|
|
107
|
+
def on_llm_start(
|
|
108
|
+
self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
|
|
109
|
+
) -> None:
|
|
110
|
+
"""Capture LLM call start."""
|
|
111
|
+
if serialized and "name" in serialized:
|
|
112
|
+
self.model_name = serialized["name"]
|
|
113
|
+
|
|
114
|
+
def on_llm_end(self, response: Any, **kwargs: Any) -> None:
|
|
115
|
+
"""Extract LLM usage statistics (tokens & cost)."""
|
|
116
|
+
input_tokens = 0
|
|
117
|
+
output_tokens = 0
|
|
118
|
+
llm_output = getattr(response, "llm_output", None) or {}
|
|
119
|
+
if isinstance(llm_output, dict):
|
|
120
|
+
token_usage = llm_output.get("token_usage", {})
|
|
121
|
+
if token_usage:
|
|
122
|
+
input_tokens = token_usage.get("prompt_tokens", 0) or 0
|
|
123
|
+
output_tokens = token_usage.get("completion_tokens", 0) or 0
|
|
124
|
+
model_used = llm_output.get("model_name", self.model_name)
|
|
125
|
+
self.model_name = model_used
|
|
126
|
+
|
|
127
|
+
cost = calculate_token_cost(self.model_name, input_tokens, output_tokens)
|
|
128
|
+
self.state.input_tokens += input_tokens
|
|
129
|
+
self.state.output_tokens += output_tokens
|
|
130
|
+
self.state.cost_usd += cost
|
|
131
|
+
|
|
132
|
+
# Check cost ceiling guardrail
|
|
133
|
+
if self.state.cost_usd > self.breaker.max_cost_usd:
|
|
134
|
+
raise CircuitBreakerException(
|
|
135
|
+
reason=f"Single-run cost ceiling exceeded (${self.state.cost_usd:.4f} > ${self.breaker.max_cost_usd:.2f})",
|
|
136
|
+
hops=self.state.hops,
|
|
137
|
+
cost_usd=self.state.cost_usd,
|
|
138
|
+
tokens=self.state.total_tokens,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
|
|
142
|
+
"""Finalize trace and ship to AgentPulse edge ingestion API."""
|
|
143
|
+
latency_ms = int((time.monotonic() - self.start_time) * 1000)
|
|
144
|
+
output_str = str(outputs.get("output", outputs))[:300]
|
|
145
|
+
|
|
146
|
+
self.state.add_step(
|
|
147
|
+
step_type="final",
|
|
148
|
+
title="LangChain Response Completed",
|
|
149
|
+
content=output_str,
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
self.pulse._ship_trace(
|
|
153
|
+
agent_id=self.agent_id,
|
|
154
|
+
model=self.model_name,
|
|
155
|
+
prompt=self.initial_prompt or "LangChain execution",
|
|
156
|
+
state=self.state,
|
|
157
|
+
status="COMPLETED",
|
|
158
|
+
latency_ms=latency_ms,
|
|
159
|
+
guardrail_reason=None,
|
|
160
|
+
circuit_tripped=False,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def on_chain_error(self, error: BaseException, **kwargs: Any) -> None:
|
|
164
|
+
"""Capture chain error or circuit breaker interception."""
|
|
165
|
+
latency_ms = int((time.monotonic() - self.start_time) * 1000)
|
|
166
|
+
is_breaker = isinstance(error, CircuitBreakerException)
|
|
167
|
+
status = "CIRCUIT_KILLED" if is_breaker else "ERROR"
|
|
168
|
+
|
|
169
|
+
self.state.add_step(
|
|
170
|
+
step_type="guardrail",
|
|
171
|
+
title="Circuit Breaker Tripped" if is_breaker else "Execution Error",
|
|
172
|
+
content=str(error),
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
self.pulse._ship_trace(
|
|
176
|
+
agent_id=self.agent_id,
|
|
177
|
+
model=self.model_name,
|
|
178
|
+
prompt=self.initial_prompt or "LangChain execution",
|
|
179
|
+
state=self.state,
|
|
180
|
+
status=status,
|
|
181
|
+
latency_ms=latency_ms,
|
|
182
|
+
guardrail_reason=error.reason if is_breaker else str(error),
|
|
183
|
+
circuit_tripped=is_breaker,
|
|
184
|
+
)
|
|
185
|
+
|