agentbill-py-langchain 4.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentbill_py_langchain-4.0.1/LICENSE +21 -0
- agentbill_py_langchain-4.0.1/PKG-INFO +208 -0
- agentbill_py_langchain-4.0.1/README.md +168 -0
- agentbill_py_langchain-4.0.1/agentbill_langchain/__init__.py +9 -0
- agentbill_py_langchain-4.0.1/agentbill_langchain/callback.py +332 -0
- agentbill_py_langchain-4.0.1/agentbill_py_langchain.egg-info/PKG-INFO +208 -0
- agentbill_py_langchain-4.0.1/agentbill_py_langchain.egg-info/SOURCES.txt +11 -0
- agentbill_py_langchain-4.0.1/agentbill_py_langchain.egg-info/dependency_links.txt +1 -0
- agentbill_py_langchain-4.0.1/agentbill_py_langchain.egg-info/requires.txt +8 -0
- agentbill_py_langchain-4.0.1/agentbill_py_langchain.egg-info/top_level.txt +1 -0
- agentbill_py_langchain-4.0.1/setup.cfg +4 -0
- agentbill_py_langchain-4.0.1/setup.py +38 -0
- agentbill_py_langchain-4.0.1/tests/test_callback.py +121 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 AgentBill
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agentbill-py-langchain
|
|
3
|
+
Version: 4.0.1
|
|
4
|
+
Summary: LangChain callback handler for automatic usage tracking and billing with AgentBill
|
|
5
|
+
Home-page: https://github.com/Agent-Bill/langchain
|
|
6
|
+
Author: AgentBill
|
|
7
|
+
Author-email: dominic@agentbill.io
|
|
8
|
+
Keywords: langchain agentbill ai agent billing usage-tracking openai anthropic llm observability callbacks
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Requires-Python: >=3.8
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: langchain>=0.1.0
|
|
23
|
+
Requires-Dist: requests>=2.28.0
|
|
24
|
+
Provides-Extra: openai
|
|
25
|
+
Requires-Dist: langchain-openai>=0.0.1; extra == "openai"
|
|
26
|
+
Provides-Extra: anthropic
|
|
27
|
+
Requires-Dist: langchain-anthropic>=0.0.1; extra == "anthropic"
|
|
28
|
+
Dynamic: author
|
|
29
|
+
Dynamic: author-email
|
|
30
|
+
Dynamic: classifier
|
|
31
|
+
Dynamic: description
|
|
32
|
+
Dynamic: description-content-type
|
|
33
|
+
Dynamic: home-page
|
|
34
|
+
Dynamic: keywords
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
Dynamic: provides-extra
|
|
37
|
+
Dynamic: requires-dist
|
|
38
|
+
Dynamic: requires-python
|
|
39
|
+
Dynamic: summary
|
|
40
|
+
|
|
41
|
+
# AgentBill LangChain Integration
|
|
42
|
+
|
|
43
|
+
Automatic usage tracking and billing for LangChain applications.
|
|
44
|
+
|
|
45
|
+
[](https://pypi.org/project/agentbill-langchain/)
|
|
46
|
+
[](https://opensource.org/licenses/MIT)
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
Install via pip:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install agentbill-langchain
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
With OpenAI support:
|
|
57
|
+
```bash
|
|
58
|
+
pip install agentbill-langchain[openai]
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
With Anthropic support:
|
|
62
|
+
```bash
|
|
63
|
+
pip install agentbill-langchain[anthropic]
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Quick Start
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from agentbill_langchain import AgentBillCallback
|
|
70
|
+
from langchain_openai import ChatOpenAI
|
|
71
|
+
from langchain.chains import LLMChain
|
|
72
|
+
from langchain.prompts import PromptTemplate
|
|
73
|
+
|
|
74
|
+
# 1. Initialize AgentBill callback
|
|
75
|
+
callback = AgentBillCallback(
|
|
76
|
+
api_key="agb_your_api_key_here", # Get from AgentBill dashboard
|
|
77
|
+
base_url="https://bgwyprqxtdreuutzpbgw.supabase.co",
|
|
78
|
+
customer_id="customer-123",
|
|
79
|
+
debug=True
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# 2. Create LangChain chain with callback
|
|
83
|
+
llm = ChatOpenAI(model="gpt-4o-mini")
|
|
84
|
+
prompt = PromptTemplate.from_template("Tell me a joke about {topic}")
|
|
85
|
+
chain = LLMChain(llm=llm, prompt=prompt)
|
|
86
|
+
|
|
87
|
+
# 3. Run - everything is auto-tracked!
|
|
88
|
+
result = chain.invoke(
|
|
89
|
+
{"topic": "programming"},
|
|
90
|
+
config={"callbacks": [callback]}
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
print(result["text"])
|
|
94
|
+
|
|
95
|
+
# ✅ Automatically captured:
|
|
96
|
+
# - Prompt text (hashed for privacy)
|
|
97
|
+
# - Model name (gpt-4o-mini)
|
|
98
|
+
# - Provider (openai)
|
|
99
|
+
# - Token usage (prompt + completion)
|
|
100
|
+
# - Latency (ms)
|
|
101
|
+
# - Costs (calculated automatically)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Features
|
|
105
|
+
|
|
106
|
+
- ✅ **Zero-config instrumentation** - Just add the callback
|
|
107
|
+
- ✅ **Automatic token tracking** - Captures all LLM calls
|
|
108
|
+
- ✅ **Multi-provider support** - OpenAI, Anthropic, any LangChain LLM
|
|
109
|
+
- ✅ **Chain tracking** - Tracks entire chain executions
|
|
110
|
+
- ✅ **Cost calculation** - Auto-calculates costs per model
|
|
111
|
+
- ✅ **Prompt profitability** - Compare costs vs revenue
|
|
112
|
+
- ✅ **OpenTelemetry compatible** - Standard observability
|
|
113
|
+
|
|
114
|
+
## Advanced Usage
|
|
115
|
+
|
|
116
|
+
### Track Custom Revenue
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
# Track revenue for profitability analysis
|
|
120
|
+
callback.track_revenue(
|
|
121
|
+
event_name="chat_completion",
|
|
122
|
+
revenue=0.50, # What you charged the customer
|
|
123
|
+
metadata={"subscription_tier": "pro"}
|
|
124
|
+
)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Use with Agents
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from langchain.agents import initialize_agent, load_tools
|
|
131
|
+
|
|
132
|
+
tools = load_tools(["serpapi", "llm-math"], llm=llm)
|
|
133
|
+
agent = initialize_agent(
|
|
134
|
+
tools,
|
|
135
|
+
llm,
|
|
136
|
+
agent="zero-shot-react-description",
|
|
137
|
+
callbacks=[callback] # Add callback here
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# All agent steps auto-tracked!
|
|
141
|
+
response = agent.run("What is 25% of 300?")
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### Use with Sequential Chains
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from langchain.chains import SimpleSequentialChain
|
|
148
|
+
|
|
149
|
+
# All chain steps tracked automatically
|
|
150
|
+
overall_chain = SimpleSequentialChain(
|
|
151
|
+
chains=[chain1, chain2, chain3],
|
|
152
|
+
callbacks=[callback]
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
result = overall_chain.run(input_text)
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## Configuration
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
callback = AgentBillCallback(
|
|
162
|
+
api_key="agb_...", # Required - get from dashboard
|
|
163
|
+
base_url="https://...", # Required - your AgentBill instance
|
|
164
|
+
customer_id="customer-123", # Optional - for multi-tenant apps
|
|
165
|
+
account_id="account-456", # Optional - for account-level tracking
|
|
166
|
+
debug=True, # Optional - enable debug logging
|
|
167
|
+
batch_size=10, # Optional - batch signals before sending
|
|
168
|
+
flush_interval=5.0 # Optional - flush interval in seconds
|
|
169
|
+
)
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## How It Works
|
|
173
|
+
|
|
174
|
+
The callback hooks into LangChain's lifecycle:
|
|
175
|
+
|
|
176
|
+
1. **on_llm_start** - Captures prompt, model, provider
|
|
177
|
+
2. **on_llm_end** - Captures tokens, latency, response
|
|
178
|
+
3. **on_llm_error** - Captures errors and retries
|
|
179
|
+
4. **on_chain_start** - Tracks chain execution start
|
|
180
|
+
5. **on_chain_end** - Tracks chain completion
|
|
181
|
+
|
|
182
|
+
All data is sent to AgentBill via the `record-signals` API endpoint with proper authentication.
|
|
183
|
+
|
|
184
|
+
## Supported Models
|
|
185
|
+
|
|
186
|
+
Auto-cost calculation for:
|
|
187
|
+
- OpenAI: GPT-4, GPT-4o, GPT-3.5-turbo, etc.
|
|
188
|
+
- Anthropic: Claude 3.5 Sonnet, Claude 3 Opus, etc.
|
|
189
|
+
- Any LangChain-compatible LLM
|
|
190
|
+
|
|
191
|
+
## Troubleshooting
|
|
192
|
+
|
|
193
|
+
### Not seeing data in dashboard?
|
|
194
|
+
|
|
195
|
+
1. Check API key is correct
|
|
196
|
+
2. Enable `debug=True` to see logs
|
|
197
|
+
3. Verify `base_url` matches your instance
|
|
198
|
+
4. Check network connectivity to AgentBill
|
|
199
|
+
|
|
200
|
+
### Token counts are zero?
|
|
201
|
+
|
|
202
|
+
- Some LLMs don't return token usage
|
|
203
|
+
- Callback will estimate based on response length
|
|
204
|
+
- OpenAI/Anthropic provide accurate counts
|
|
205
|
+
|
|
206
|
+
## License
|
|
207
|
+
|
|
208
|
+
MIT
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# AgentBill LangChain Integration
|
|
2
|
+
|
|
3
|
+
Automatic usage tracking and billing for LangChain applications.
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/agentbill-langchain/)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
Install via pip:
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install agentbill-langchain
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
With OpenAI support:
|
|
17
|
+
```bash
|
|
18
|
+
pip install agentbill-langchain[openai]
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
With Anthropic support:
|
|
22
|
+
```bash
|
|
23
|
+
pip install agentbill-langchain[anthropic]
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Quick Start
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from agentbill_langchain import AgentBillCallback
|
|
30
|
+
from langchain_openai import ChatOpenAI
|
|
31
|
+
from langchain.chains import LLMChain
|
|
32
|
+
from langchain.prompts import PromptTemplate
|
|
33
|
+
|
|
34
|
+
# 1. Initialize AgentBill callback
|
|
35
|
+
callback = AgentBillCallback(
|
|
36
|
+
api_key="agb_your_api_key_here", # Get from AgentBill dashboard
|
|
37
|
+
base_url="https://bgwyprqxtdreuutzpbgw.supabase.co",
|
|
38
|
+
customer_id="customer-123",
|
|
39
|
+
debug=True
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
# 2. Create LangChain chain with callback
|
|
43
|
+
llm = ChatOpenAI(model="gpt-4o-mini")
|
|
44
|
+
prompt = PromptTemplate.from_template("Tell me a joke about {topic}")
|
|
45
|
+
chain = LLMChain(llm=llm, prompt=prompt)
|
|
46
|
+
|
|
47
|
+
# 3. Run - everything is auto-tracked!
|
|
48
|
+
result = chain.invoke(
|
|
49
|
+
{"topic": "programming"},
|
|
50
|
+
config={"callbacks": [callback]}
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
print(result["text"])
|
|
54
|
+
|
|
55
|
+
# ✅ Automatically captured:
|
|
56
|
+
# - Prompt text (hashed for privacy)
|
|
57
|
+
# - Model name (gpt-4o-mini)
|
|
58
|
+
# - Provider (openai)
|
|
59
|
+
# - Token usage (prompt + completion)
|
|
60
|
+
# - Latency (ms)
|
|
61
|
+
# - Costs (calculated automatically)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Features
|
|
65
|
+
|
|
66
|
+
- ✅ **Zero-config instrumentation** - Just add the callback
|
|
67
|
+
- ✅ **Automatic token tracking** - Captures all LLM calls
|
|
68
|
+
- ✅ **Multi-provider support** - OpenAI, Anthropic, any LangChain LLM
|
|
69
|
+
- ✅ **Chain tracking** - Tracks entire chain executions
|
|
70
|
+
- ✅ **Cost calculation** - Auto-calculates costs per model
|
|
71
|
+
- ✅ **Prompt profitability** - Compare costs vs revenue
|
|
72
|
+
- ✅ **OpenTelemetry compatible** - Standard observability
|
|
73
|
+
|
|
74
|
+
## Advanced Usage
|
|
75
|
+
|
|
76
|
+
### Track Custom Revenue
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
# Track revenue for profitability analysis
|
|
80
|
+
callback.track_revenue(
|
|
81
|
+
event_name="chat_completion",
|
|
82
|
+
revenue=0.50, # What you charged the customer
|
|
83
|
+
metadata={"subscription_tier": "pro"}
|
|
84
|
+
)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### Use with Agents
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from langchain.agents import initialize_agent, load_tools
|
|
91
|
+
|
|
92
|
+
tools = load_tools(["serpapi", "llm-math"], llm=llm)
|
|
93
|
+
agent = initialize_agent(
|
|
94
|
+
tools,
|
|
95
|
+
llm,
|
|
96
|
+
agent="zero-shot-react-description",
|
|
97
|
+
callbacks=[callback] # Add callback here
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
# All agent steps auto-tracked!
|
|
101
|
+
response = agent.run("What is 25% of 300?")
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Use with Sequential Chains
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from langchain.chains import SimpleSequentialChain
|
|
108
|
+
|
|
109
|
+
# All chain steps tracked automatically
|
|
110
|
+
overall_chain = SimpleSequentialChain(
|
|
111
|
+
chains=[chain1, chain2, chain3],
|
|
112
|
+
callbacks=[callback]
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
result = overall_chain.run(input_text)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Configuration
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
callback = AgentBillCallback(
|
|
122
|
+
api_key="agb_...", # Required - get from dashboard
|
|
123
|
+
base_url="https://...", # Required - your AgentBill instance
|
|
124
|
+
customer_id="customer-123", # Optional - for multi-tenant apps
|
|
125
|
+
account_id="account-456", # Optional - for account-level tracking
|
|
126
|
+
debug=True, # Optional - enable debug logging
|
|
127
|
+
batch_size=10, # Optional - batch signals before sending
|
|
128
|
+
flush_interval=5.0 # Optional - flush interval in seconds
|
|
129
|
+
)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
## How It Works
|
|
133
|
+
|
|
134
|
+
The callback hooks into LangChain's lifecycle:
|
|
135
|
+
|
|
136
|
+
1. **on_llm_start** - Captures prompt, model, provider
|
|
137
|
+
2. **on_llm_end** - Captures tokens, latency, response
|
|
138
|
+
3. **on_llm_error** - Captures errors and retries
|
|
139
|
+
4. **on_chain_start** - Tracks chain execution start
|
|
140
|
+
5. **on_chain_end** - Tracks chain completion
|
|
141
|
+
|
|
142
|
+
All data is sent to AgentBill via the `record-signals` API endpoint with proper authentication.
|
|
143
|
+
|
|
144
|
+
## Supported Models
|
|
145
|
+
|
|
146
|
+
Auto-cost calculation for:
|
|
147
|
+
- OpenAI: GPT-4, GPT-4o, GPT-3.5-turbo, etc.
|
|
148
|
+
- Anthropic: Claude 3.5 Sonnet, Claude 3 Opus, etc.
|
|
149
|
+
- Any LangChain-compatible LLM
|
|
150
|
+
|
|
151
|
+
## Troubleshooting
|
|
152
|
+
|
|
153
|
+
### Not seeing data in dashboard?
|
|
154
|
+
|
|
155
|
+
1. Check API key is correct
|
|
156
|
+
2. Enable `debug=True` to see logs
|
|
157
|
+
3. Verify `base_url` matches your instance
|
|
158
|
+
4. Check network connectivity to AgentBill
|
|
159
|
+
|
|
160
|
+
### Token counts are zero?
|
|
161
|
+
|
|
162
|
+
- Some LLMs don't return token usage
|
|
163
|
+
- Callback will estimate based on response length
|
|
164
|
+
- OpenAI/Anthropic provide accurate counts
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
MIT
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"""AgentBill LangChain Callback Handler"""
|
|
2
|
+
|
|
3
|
+
import time
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
from typing import Any, Dict, List, Optional
|
|
7
|
+
from uuid import UUID
|
|
8
|
+
|
|
9
|
+
try:
|
|
10
|
+
from langchain.callbacks.base import BaseCallbackHandler
|
|
11
|
+
from langchain.schema import LLMResult
|
|
12
|
+
except ImportError:
|
|
13
|
+
raise ImportError(
|
|
14
|
+
"langchain is not installed. Install with: pip install langchain"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
import requests
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class AgentBillCallback(BaseCallbackHandler):
|
|
21
|
+
"""LangChain callback handler that sends usage data to AgentBill.
|
|
22
|
+
|
|
23
|
+
Example:
|
|
24
|
+
callback = AgentBillCallback(
|
|
25
|
+
api_key="agb_your_key",
|
|
26
|
+
base_url="https://your-instance.supabase.co",
|
|
27
|
+
customer_id="customer-123"
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
llm = ChatOpenAI(callbacks=[callback])
|
|
31
|
+
result = llm.invoke("Hello!")
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
api_key: str,
|
|
37
|
+
base_url: str,
|
|
38
|
+
customer_id: Optional[str] = None,
|
|
39
|
+
account_id: Optional[str] = None,
|
|
40
|
+
debug: bool = False,
|
|
41
|
+
batch_size: int = 10,
|
|
42
|
+
flush_interval: float = 5.0
|
|
43
|
+
):
|
|
44
|
+
"""Initialize AgentBill callback.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
api_key: AgentBill API key (get from dashboard)
|
|
48
|
+
base_url: AgentBill base URL (e.g., https://xxx.supabase.co)
|
|
49
|
+
customer_id: Optional customer ID for tracking
|
|
50
|
+
account_id: Optional account ID for tracking
|
|
51
|
+
debug: Enable debug logging
|
|
52
|
+
batch_size: Number of signals to batch before sending
|
|
53
|
+
flush_interval: Seconds between automatic flushes
|
|
54
|
+
"""
|
|
55
|
+
super().__init__()
|
|
56
|
+
self.api_key = api_key
|
|
57
|
+
self.base_url = base_url.rstrip('/')
|
|
58
|
+
self.customer_id = customer_id
|
|
59
|
+
self.account_id = account_id
|
|
60
|
+
self.debug = debug
|
|
61
|
+
self.batch_size = batch_size
|
|
62
|
+
self.flush_interval = flush_interval
|
|
63
|
+
|
|
64
|
+
# Track active LLM calls
|
|
65
|
+
self._active_runs: Dict[str, Dict[str, Any]] = {}
|
|
66
|
+
|
|
67
|
+
# Batch queue
|
|
68
|
+
self._signal_queue: List[Dict[str, Any]] = []
|
|
69
|
+
self._last_flush = time.time()
|
|
70
|
+
|
|
71
|
+
if self.debug:
|
|
72
|
+
print(f"[AgentBill] Initialized with base_url={self.base_url}")
|
|
73
|
+
|
|
74
|
+
def _hash_prompt(self, text: str) -> str:
|
|
75
|
+
"""Hash prompt for privacy."""
|
|
76
|
+
return hashlib.sha256(text.encode()).hexdigest()
|
|
77
|
+
|
|
78
|
+
def _extract_provider(self, serialized: Dict[str, Any]) -> str:
|
|
79
|
+
"""Extract provider from LLM serialization."""
|
|
80
|
+
# Check id field
|
|
81
|
+
for id_item in serialized.get("id", []):
|
|
82
|
+
if "openai" in id_item.lower():
|
|
83
|
+
return "openai"
|
|
84
|
+
if "anthropic" in id_item.lower():
|
|
85
|
+
return "anthropic"
|
|
86
|
+
if "cohere" in id_item.lower():
|
|
87
|
+
return "cohere"
|
|
88
|
+
if "bedrock" in id_item.lower():
|
|
89
|
+
return "bedrock"
|
|
90
|
+
|
|
91
|
+
# Check kwargs
|
|
92
|
+
kwargs = serialized.get("kwargs", {})
|
|
93
|
+
if "openai" in str(kwargs).lower():
|
|
94
|
+
return "openai"
|
|
95
|
+
if "anthropic" in str(kwargs).lower():
|
|
96
|
+
return "anthropic"
|
|
97
|
+
|
|
98
|
+
return "unknown"
|
|
99
|
+
|
|
100
|
+
def _extract_model(self, serialized: Dict[str, Any]) -> str:
|
|
101
|
+
"""Extract model name from LLM serialization."""
|
|
102
|
+
# Try model_name in kwargs
|
|
103
|
+
kwargs = serialized.get("kwargs", {})
|
|
104
|
+
if "model_name" in kwargs:
|
|
105
|
+
return kwargs["model_name"]
|
|
106
|
+
if "model" in kwargs:
|
|
107
|
+
return kwargs["model"]
|
|
108
|
+
|
|
109
|
+
# Try id field
|
|
110
|
+
for id_item in serialized.get("id", []):
|
|
111
|
+
if "gpt" in id_item.lower():
|
|
112
|
+
return id_item
|
|
113
|
+
if "claude" in id_item.lower():
|
|
114
|
+
return id_item
|
|
115
|
+
|
|
116
|
+
return "unknown"
|
|
117
|
+
|
|
118
|
+
def _send_signal(self, signal: Dict[str, Any]) -> None:
|
|
119
|
+
"""Send signal to AgentBill."""
|
|
120
|
+
try:
|
|
121
|
+
url = f"{self.base_url}/functions/v1/record-signals"
|
|
122
|
+
headers = {
|
|
123
|
+
"Content-Type": "application/json",
|
|
124
|
+
"X-API-Key": self.api_key
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if self.debug:
|
|
128
|
+
print(f"[AgentBill] Sending signal: {signal.get('event_name')}")
|
|
129
|
+
|
|
130
|
+
response = requests.post(url, json=[signal], headers=headers, timeout=10)
|
|
131
|
+
|
|
132
|
+
if response.status_code != 200:
|
|
133
|
+
print(f"[AgentBill] Error sending signal: {response.status_code} {response.text}")
|
|
134
|
+
elif self.debug:
|
|
135
|
+
print(f"[AgentBill] Signal sent successfully")
|
|
136
|
+
|
|
137
|
+
except Exception as e:
|
|
138
|
+
if self.debug:
|
|
139
|
+
print(f"[AgentBill] Error sending signal: {e}")
|
|
140
|
+
|
|
141
|
+
def _queue_signal(self, signal: Dict[str, Any]) -> None:
|
|
142
|
+
"""Add signal to queue and flush if needed."""
|
|
143
|
+
self._signal_queue.append(signal)
|
|
144
|
+
|
|
145
|
+
# Flush if batch size reached or interval exceeded
|
|
146
|
+
now = time.time()
|
|
147
|
+
should_flush = (
|
|
148
|
+
len(self._signal_queue) >= self.batch_size or
|
|
149
|
+
(now - self._last_flush) >= self.flush_interval
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
if should_flush:
|
|
153
|
+
self.flush()
|
|
154
|
+
|
|
155
|
+
def flush(self) -> None:
|
|
156
|
+
"""Flush queued signals to AgentBill."""
|
|
157
|
+
if not self._signal_queue:
|
|
158
|
+
return
|
|
159
|
+
|
|
160
|
+
try:
|
|
161
|
+
url = f"{self.base_url}/functions/v1/record-signals"
|
|
162
|
+
headers = {
|
|
163
|
+
"Content-Type": "application/json",
|
|
164
|
+
"X-API-Key": self.api_key
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if self.debug:
|
|
168
|
+
print(f"[AgentBill] Flushing {len(self._signal_queue)} signals")
|
|
169
|
+
|
|
170
|
+
response = requests.post(
|
|
171
|
+
url,
|
|
172
|
+
json=self._signal_queue,
|
|
173
|
+
headers=headers,
|
|
174
|
+
timeout=10
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
if response.status_code == 200:
|
|
178
|
+
self._signal_queue.clear()
|
|
179
|
+
self._last_flush = time.time()
|
|
180
|
+
if self.debug:
|
|
181
|
+
print(f"[AgentBill] Flush successful")
|
|
182
|
+
else:
|
|
183
|
+
print(f"[AgentBill] Flush error: {response.status_code} {response.text}")
|
|
184
|
+
|
|
185
|
+
except Exception as e:
|
|
186
|
+
if self.debug:
|
|
187
|
+
print(f"[AgentBill] Flush error: {e}")
|
|
188
|
+
|
|
189
|
+
def on_llm_start(
|
|
190
|
+
self,
|
|
191
|
+
serialized: Dict[str, Any],
|
|
192
|
+
prompts: List[str],
|
|
193
|
+
**kwargs: Any
|
|
194
|
+
) -> None:
|
|
195
|
+
"""Called when LLM starts."""
|
|
196
|
+
run_id = kwargs.get("run_id") or str(UUID(int=0))
|
|
197
|
+
|
|
198
|
+
# Extract metadata
|
|
199
|
+
provider = self._extract_provider(serialized)
|
|
200
|
+
model = self._extract_model(serialized)
|
|
201
|
+
|
|
202
|
+
# Store run info
|
|
203
|
+
self._active_runs[str(run_id)] = {
|
|
204
|
+
"start_time": time.time(),
|
|
205
|
+
"prompts": prompts,
|
|
206
|
+
"provider": provider,
|
|
207
|
+
"model": model,
|
|
208
|
+
"prompt_hash": self._hash_prompt(prompts[0]) if prompts else None,
|
|
209
|
+
"prompt_sample": prompts[0][:200] if prompts else None,
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if self.debug:
|
|
213
|
+
print(f"[AgentBill] LLM started: {model} ({provider})")
|
|
214
|
+
|
|
215
|
+
def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
|
|
216
|
+
"""Called when LLM ends."""
|
|
217
|
+
run_id = str(kwargs.get("run_id", ""))
|
|
218
|
+
run_info = self._active_runs.pop(run_id, None)
|
|
219
|
+
|
|
220
|
+
if not run_info:
|
|
221
|
+
return
|
|
222
|
+
|
|
223
|
+
# Calculate latency
|
|
224
|
+
latency_ms = int((time.time() - run_info["start_time"]) * 1000)
|
|
225
|
+
|
|
226
|
+
# Extract token usage
|
|
227
|
+
llm_output = response.llm_output or {}
|
|
228
|
+
token_usage = llm_output.get("token_usage", {})
|
|
229
|
+
|
|
230
|
+
prompt_tokens = token_usage.get("prompt_tokens", 0)
|
|
231
|
+
completion_tokens = token_usage.get("completion_tokens", 0)
|
|
232
|
+
total_tokens = token_usage.get("total_tokens", prompt_tokens + completion_tokens)
|
|
233
|
+
|
|
234
|
+
# Build signal
|
|
235
|
+
signal = {
|
|
236
|
+
"event_name": "langchain_llm_call",
|
|
237
|
+
"model": run_info["model"],
|
|
238
|
+
"provider": run_info["provider"],
|
|
239
|
+
"prompt_hash": run_info["prompt_hash"],
|
|
240
|
+
"prompt_sample": run_info["prompt_sample"],
|
|
241
|
+
"metrics": {
|
|
242
|
+
"prompt_tokens": prompt_tokens,
|
|
243
|
+
"completion_tokens": completion_tokens,
|
|
244
|
+
"total_tokens": total_tokens,
|
|
245
|
+
},
|
|
246
|
+
"latency_ms": latency_ms,
|
|
247
|
+
"data_source": "langchain",
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
# Add customer/account if provided with UUID detection
|
|
251
|
+
if self.customer_id:
|
|
252
|
+
# Check if UUID format - send as customer_id, else customer_external_id
|
|
253
|
+
import re
|
|
254
|
+
uuid_regex = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.IGNORECASE)
|
|
255
|
+
is_uuid = bool(uuid_regex.match(self.customer_id))
|
|
256
|
+
if is_uuid:
|
|
257
|
+
signal["customer_id"] = self.customer_id
|
|
258
|
+
else:
|
|
259
|
+
signal["customer_external_id"] = self.customer_id
|
|
260
|
+
if self.account_id:
|
|
261
|
+
signal["account_id"] = self.account_id
|
|
262
|
+
|
|
263
|
+
# Queue signal
|
|
264
|
+
self._queue_signal(signal)
|
|
265
|
+
|
|
266
|
+
if self.debug:
|
|
267
|
+
print(f"[AgentBill] LLM ended: {total_tokens} tokens, {latency_ms}ms")
|
|
268
|
+
|
|
269
|
+
def on_llm_error(self, error: Exception, **kwargs: Any) -> None:
|
|
270
|
+
"""Called when LLM errors."""
|
|
271
|
+
run_id = str(kwargs.get("run_id", ""))
|
|
272
|
+
self._active_runs.pop(run_id, None)
|
|
273
|
+
|
|
274
|
+
if self.debug:
|
|
275
|
+
print(f"[AgentBill] LLM error: {error}")
|
|
276
|
+
|
|
277
|
+
def on_chain_start(
|
|
278
|
+
self,
|
|
279
|
+
serialized: Dict[str, Any],
|
|
280
|
+
inputs: Dict[str, Any],
|
|
281
|
+
**kwargs: Any
|
|
282
|
+
) -> None:
|
|
283
|
+
"""Called when chain starts."""
|
|
284
|
+
if self.debug:
|
|
285
|
+
chain_name = serialized.get("id", ["unknown"])[-1]
|
|
286
|
+
print(f"[AgentBill] Chain started: {chain_name}")
|
|
287
|
+
|
|
288
|
+
def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
|
|
289
|
+
"""Called when chain ends."""
|
|
290
|
+
if self.debug:
|
|
291
|
+
print(f"[AgentBill] Chain ended")
|
|
292
|
+
|
|
293
|
+
def track_revenue(
|
|
294
|
+
self,
|
|
295
|
+
event_name: str,
|
|
296
|
+
revenue: float,
|
|
297
|
+
metadata: Optional[Dict[str, Any]] = None
|
|
298
|
+
) -> None:
|
|
299
|
+
"""Track revenue for profitability analysis.
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
event_name: Event name (e.g., "chat_completion")
|
|
303
|
+
revenue: Revenue amount (what you charged)
|
|
304
|
+
metadata: Additional metadata
|
|
305
|
+
"""
|
|
306
|
+
signal = {
|
|
307
|
+
"event_name": event_name,
|
|
308
|
+
"conversion_value": revenue,
|
|
309
|
+
"revenue_source": "langchain",
|
|
310
|
+
"data": metadata or {},
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
if self.customer_id:
|
|
314
|
+
# Check if UUID format - send as customer_id, else customer_external_id
|
|
315
|
+
import re
|
|
316
|
+
uuid_regex = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.IGNORECASE)
|
|
317
|
+
is_uuid = bool(uuid_regex.match(self.customer_id))
|
|
318
|
+
if is_uuid:
|
|
319
|
+
signal["customer_id"] = self.customer_id
|
|
320
|
+
else:
|
|
321
|
+
signal["customer_external_id"] = self.customer_id
|
|
322
|
+
if self.account_id:
|
|
323
|
+
signal["account_id"] = self.account_id
|
|
324
|
+
|
|
325
|
+
self._queue_signal(signal)
|
|
326
|
+
|
|
327
|
+
if self.debug:
|
|
328
|
+
print(f"[AgentBill] Revenue tracked: ${revenue}")
|
|
329
|
+
|
|
330
|
+
def __del__(self):
|
|
331
|
+
"""Flush on cleanup."""
|
|
332
|
+
self.flush()
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: agentbill-py-langchain
|
|
3
|
+
Version: 4.0.1
|
|
4
|
+
Summary: LangChain callback handler for automatic usage tracking and billing with AgentBill
|
|
5
|
+
Home-page: https://github.com/Agent-Bill/langchain
|
|
6
|
+
Author: AgentBill
|
|
7
|
+
Author-email: dominic@agentbill.io
|
|
8
|
+
Keywords: langchain agentbill ai agent billing usage-tracking openai anthropic llm observability callbacks
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Requires-Python: >=3.8
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: langchain>=0.1.0
|
|
23
|
+
Requires-Dist: requests>=2.28.0
|
|
24
|
+
Provides-Extra: openai
|
|
25
|
+
Requires-Dist: langchain-openai>=0.0.1; extra == "openai"
|
|
26
|
+
Provides-Extra: anthropic
|
|
27
|
+
Requires-Dist: langchain-anthropic>=0.0.1; extra == "anthropic"
|
|
28
|
+
Dynamic: author
|
|
29
|
+
Dynamic: author-email
|
|
30
|
+
Dynamic: classifier
|
|
31
|
+
Dynamic: description
|
|
32
|
+
Dynamic: description-content-type
|
|
33
|
+
Dynamic: home-page
|
|
34
|
+
Dynamic: keywords
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
Dynamic: provides-extra
|
|
37
|
+
Dynamic: requires-dist
|
|
38
|
+
Dynamic: requires-python
|
|
39
|
+
Dynamic: summary
|
|
40
|
+
|
|
41
|
+
# AgentBill LangChain Integration
|
|
42
|
+
|
|
43
|
+
Automatic usage tracking and billing for LangChain applications.
|
|
44
|
+
|
|
45
|
+
[](https://pypi.org/project/agentbill-langchain/)
|
|
46
|
+
[](https://opensource.org/licenses/MIT)
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
Install via pip:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install agentbill-langchain
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
With OpenAI support:
|
|
57
|
+
```bash
|
|
58
|
+
pip install agentbill-langchain[openai]
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
With Anthropic support:
|
|
62
|
+
```bash
|
|
63
|
+
pip install agentbill-langchain[anthropic]
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Quick Start
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from agentbill_langchain import AgentBillCallback
|
|
70
|
+
from langchain_openai import ChatOpenAI
|
|
71
|
+
from langchain.chains import LLMChain
|
|
72
|
+
from langchain.prompts import PromptTemplate
|
|
73
|
+
|
|
74
|
+
# 1. Initialize AgentBill callback
|
|
75
|
+
callback = AgentBillCallback(
|
|
76
|
+
api_key="agb_your_api_key_here", # Get from AgentBill dashboard
|
|
77
|
+
base_url="https://bgwyprqxtdreuutzpbgw.supabase.co",
|
|
78
|
+
customer_id="customer-123",
|
|
79
|
+
debug=True
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# 2. Create LangChain chain with callback
|
|
83
|
+
llm = ChatOpenAI(model="gpt-4o-mini")
|
|
84
|
+
prompt = PromptTemplate.from_template("Tell me a joke about {topic}")
|
|
85
|
+
chain = LLMChain(llm=llm, prompt=prompt)
|
|
86
|
+
|
|
87
|
+
# 3. Run - everything is auto-tracked!
|
|
88
|
+
result = chain.invoke(
|
|
89
|
+
{"topic": "programming"},
|
|
90
|
+
config={"callbacks": [callback]}
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
print(result["text"])
|
|
94
|
+
|
|
95
|
+
# ✅ Automatically captured:
|
|
96
|
+
# - Prompt text (hashed for privacy)
|
|
97
|
+
# - Model name (gpt-4o-mini)
|
|
98
|
+
# - Provider (openai)
|
|
99
|
+
# - Token usage (prompt + completion)
|
|
100
|
+
# - Latency (ms)
|
|
101
|
+
# - Costs (calculated automatically)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Features
|
|
105
|
+
|
|
106
|
+
- ✅ **Zero-config instrumentation** - Just add the callback
|
|
107
|
+
- ✅ **Automatic token tracking** - Captures all LLM calls
|
|
108
|
+
- ✅ **Multi-provider support** - OpenAI, Anthropic, any LangChain LLM
|
|
109
|
+
- ✅ **Chain tracking** - Tracks entire chain executions
|
|
110
|
+
- ✅ **Cost calculation** - Auto-calculates costs per model
|
|
111
|
+
- ✅ **Prompt profitability** - Compare costs vs revenue
|
|
112
|
+
- ✅ **OpenTelemetry compatible** - Standard observability
|
|
113
|
+
|
|
114
|
+
## Advanced Usage
|
|
115
|
+
|
|
116
|
+
### Track Custom Revenue
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
# Track revenue for profitability analysis
|
|
120
|
+
callback.track_revenue(
|
|
121
|
+
event_name="chat_completion",
|
|
122
|
+
revenue=0.50, # What you charged the customer
|
|
123
|
+
metadata={"subscription_tier": "pro"}
|
|
124
|
+
)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Use with Agents
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from langchain.agents import initialize_agent, load_tools
|
|
131
|
+
|
|
132
|
+
tools = load_tools(["serpapi", "llm-math"], llm=llm)
|
|
133
|
+
agent = initialize_agent(
|
|
134
|
+
tools,
|
|
135
|
+
llm,
|
|
136
|
+
agent="zero-shot-react-description",
|
|
137
|
+
callbacks=[callback] # Add callback here
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
# All agent steps auto-tracked!
|
|
141
|
+
response = agent.run("What is 25% of 300?")
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### Use with Sequential Chains
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from langchain.chains import SimpleSequentialChain
|
|
148
|
+
|
|
149
|
+
# All chain steps tracked automatically
|
|
150
|
+
overall_chain = SimpleSequentialChain(
|
|
151
|
+
chains=[chain1, chain2, chain3],
|
|
152
|
+
callbacks=[callback]
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
result = overall_chain.run(input_text)
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## Configuration
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
callback = AgentBillCallback(
|
|
162
|
+
api_key="agb_...", # Required - get from dashboard
|
|
163
|
+
base_url="https://...", # Required - your AgentBill instance
|
|
164
|
+
customer_id="customer-123", # Optional - for multi-tenant apps
|
|
165
|
+
account_id="account-456", # Optional - for account-level tracking
|
|
166
|
+
debug=True, # Optional - enable debug logging
|
|
167
|
+
batch_size=10, # Optional - batch signals before sending
|
|
168
|
+
flush_interval=5.0 # Optional - flush interval in seconds
|
|
169
|
+
)
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## How It Works
|
|
173
|
+
|
|
174
|
+
The callback hooks into LangChain's lifecycle:
|
|
175
|
+
|
|
176
|
+
1. **on_llm_start** - Captures prompt, model, provider
|
|
177
|
+
2. **on_llm_end** - Captures tokens, latency, response
|
|
178
|
+
3. **on_llm_error** - Captures errors and retries
|
|
179
|
+
4. **on_chain_start** - Tracks chain execution start
|
|
180
|
+
5. **on_chain_end** - Tracks chain completion
|
|
181
|
+
|
|
182
|
+
All data is sent to AgentBill via the `record-signals` API endpoint with proper authentication.
|
|
183
|
+
|
|
184
|
+
## Supported Models
|
|
185
|
+
|
|
186
|
+
Auto-cost calculation for:
|
|
187
|
+
- OpenAI: GPT-4, GPT-4o, GPT-3.5-turbo, etc.
|
|
188
|
+
- Anthropic: Claude 3.5 Sonnet, Claude 3 Opus, etc.
|
|
189
|
+
- Any LangChain-compatible LLM
|
|
190
|
+
|
|
191
|
+
## Troubleshooting
|
|
192
|
+
|
|
193
|
+
### Not seeing data in dashboard?
|
|
194
|
+
|
|
195
|
+
1. Check API key is correct
|
|
196
|
+
2. Enable `debug=True` to see logs
|
|
197
|
+
3. Verify `base_url` matches your instance
|
|
198
|
+
4. Check network connectivity to AgentBill
|
|
199
|
+
|
|
200
|
+
### Token counts are zero?
|
|
201
|
+
|
|
202
|
+
- Some LLMs don't return token usage
|
|
203
|
+
- Callback will estimate based on response length
|
|
204
|
+
- OpenAI/Anthropic provide accurate counts
|
|
205
|
+
|
|
206
|
+
## License
|
|
207
|
+
|
|
208
|
+
MIT
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
setup.py
|
|
4
|
+
agentbill_langchain/__init__.py
|
|
5
|
+
agentbill_langchain/callback.py
|
|
6
|
+
agentbill_py_langchain.egg-info/PKG-INFO
|
|
7
|
+
agentbill_py_langchain.egg-info/SOURCES.txt
|
|
8
|
+
agentbill_py_langchain.egg-info/dependency_links.txt
|
|
9
|
+
agentbill_py_langchain.egg-info/requires.txt
|
|
10
|
+
agentbill_py_langchain.egg-info/top_level.txt
|
|
11
|
+
tests/test_callback.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
agentbill_langchain
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
|
|
3
|
+
with open("README.md", "r", encoding="utf-8") as fh:
|
|
4
|
+
long_description = fh.read()
|
|
5
|
+
|
|
6
|
+
setup(
|
|
7
|
+
name="agentbill-py-langchain",
|
|
8
|
+
version="4.0.1",
|
|
9
|
+
author="AgentBill",
|
|
10
|
+
author_email="dominic@agentbill.io",
|
|
11
|
+
description="LangChain callback handler for automatic usage tracking and billing with AgentBill",
|
|
12
|
+
long_description=long_description,
|
|
13
|
+
long_description_content_type="text/markdown",
|
|
14
|
+
url="https://github.com/Agent-Bill/langchain",
|
|
15
|
+
packages=find_packages(),
|
|
16
|
+
classifiers=[
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
20
|
+
"License :: OSI Approved :: MIT License",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.8",
|
|
23
|
+
"Programming Language :: Python :: 3.9",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
],
|
|
28
|
+
python_requires=">=3.8",
|
|
29
|
+
install_requires=[
|
|
30
|
+
"langchain>=0.1.0",
|
|
31
|
+
"requests>=2.28.0",
|
|
32
|
+
],
|
|
33
|
+
extras_require={
|
|
34
|
+
"openai": ["langchain-openai>=0.0.1"],
|
|
35
|
+
"anthropic": ["langchain-anthropic>=0.0.1"],
|
|
36
|
+
},
|
|
37
|
+
keywords="langchain agentbill ai agent billing usage-tracking openai anthropic llm observability callbacks",
|
|
38
|
+
)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Tests for AgentBill LangChain callback handler."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from unittest.mock import Mock, patch
|
|
5
|
+
from agentbill_langchain import AgentBillCallback
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@pytest.fixture
|
|
9
|
+
def callback():
|
|
10
|
+
"""Create a callback instance for testing."""
|
|
11
|
+
return AgentBillCallback(
|
|
12
|
+
api_key="test-api-key",
|
|
13
|
+
base_url="https://test.agentbill.com",
|
|
14
|
+
customer_id="test-customer",
|
|
15
|
+
debug=False
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_callback_initialization(callback):
|
|
20
|
+
"""Test callback is initialized correctly."""
|
|
21
|
+
assert callback is not None
|
|
22
|
+
assert callback.api_key == "test-api-key"
|
|
23
|
+
assert callback.base_url == "https://test.agentbill.com"
|
|
24
|
+
assert callback.customer_id == "test-customer"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_callback_on_llm_start(callback):
|
|
28
|
+
"""Test callback handles LLM start event."""
|
|
29
|
+
serialized = {"name": "OpenAI"}
|
|
30
|
+
prompts = ["Test prompt"]
|
|
31
|
+
|
|
32
|
+
# Should not raise exception
|
|
33
|
+
callback.on_llm_start(serialized, prompts)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_callback_on_llm_end(callback):
|
|
37
|
+
"""Test callback handles LLM end event."""
|
|
38
|
+
response = Mock()
|
|
39
|
+
response.llm_output = {
|
|
40
|
+
"token_usage": {
|
|
41
|
+
"prompt_tokens": 10,
|
|
42
|
+
"completion_tokens": 20,
|
|
43
|
+
"total_tokens": 30
|
|
44
|
+
},
|
|
45
|
+
"model_name": "gpt-4"
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
# Should not raise exception
|
|
49
|
+
callback.on_llm_end(response)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_callback_on_llm_error(callback):
|
|
53
|
+
"""Test callback handles LLM error event."""
|
|
54
|
+
error = Exception("Test error")
|
|
55
|
+
|
|
56
|
+
# Should not raise exception
|
|
57
|
+
callback.on_llm_error(error)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_callback_on_chain_start(callback):
|
|
61
|
+
"""Test callback handles chain start event."""
|
|
62
|
+
serialized = {"name": "LLMChain"}
|
|
63
|
+
inputs = {"input": "test"}
|
|
64
|
+
|
|
65
|
+
# Should not raise exception
|
|
66
|
+
callback.on_chain_start(serialized, inputs)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_callback_on_chain_end(callback):
|
|
70
|
+
"""Test callback handles chain end event."""
|
|
71
|
+
outputs = {"output": "test"}
|
|
72
|
+
|
|
73
|
+
# Should not raise exception
|
|
74
|
+
callback.on_chain_end(outputs)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_callback_on_tool_start(callback):
|
|
78
|
+
"""Test callback handles tool start event."""
|
|
79
|
+
serialized = {"name": "Calculator"}
|
|
80
|
+
input_str = "2+2"
|
|
81
|
+
|
|
82
|
+
# Should not raise exception
|
|
83
|
+
callback.on_tool_start(serialized, input_str)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_callback_on_tool_end(callback):
|
|
87
|
+
"""Test callback handles tool end event."""
|
|
88
|
+
output = "4"
|
|
89
|
+
|
|
90
|
+
# Should not raise exception
|
|
91
|
+
callback.on_tool_end(output)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@patch('requests.post')
|
|
95
|
+
def test_callback_flush(mock_post, callback):
|
|
96
|
+
"""Test callback flushes data correctly."""
|
|
97
|
+
mock_post.return_value.status_code = 200
|
|
98
|
+
|
|
99
|
+
# Simulate some activity
|
|
100
|
+
callback.on_llm_start({"name": "OpenAI"}, ["test"])
|
|
101
|
+
|
|
102
|
+
response = Mock()
|
|
103
|
+
response.llm_output = {
|
|
104
|
+
"token_usage": {
|
|
105
|
+
"prompt_tokens": 10,
|
|
106
|
+
"completion_tokens": 20,
|
|
107
|
+
"total_tokens": 30
|
|
108
|
+
},
|
|
109
|
+
"model_name": "gpt-4"
|
|
110
|
+
}
|
|
111
|
+
callback.on_llm_end(response)
|
|
112
|
+
|
|
113
|
+
# Should not raise exception
|
|
114
|
+
callback.flush()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_callback_context_manager(callback):
|
|
118
|
+
"""Test callback works as context manager."""
|
|
119
|
+
with callback as cb:
|
|
120
|
+
assert cb is callback
|
|
121
|
+
# Should clean up properly
|