llmtrace-observability 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llmtrace_observability-0.1.1/PKG-INFO +42 -0
- llmtrace_observability-0.1.1/README.md +24 -0
- llmtrace_observability-0.1.1/llmtrace/__init__.py +23 -0
- llmtrace_observability-0.1.1/llmtrace/client.py +358 -0
- llmtrace_observability-0.1.1/llmtrace/demo.py +61 -0
- llmtrace_observability-0.1.1/llmtrace/integrations/__init__.py +1 -0
- llmtrace_observability-0.1.1/llmtrace/integrations/anthropic.py +40 -0
- llmtrace_observability-0.1.1/llmtrace/integrations/openai.py +63 -0
- llmtrace_observability-0.1.1/llmtrace_observability.egg-info/PKG-INFO +42 -0
- llmtrace_observability-0.1.1/llmtrace_observability.egg-info/SOURCES.txt +14 -0
- llmtrace_observability-0.1.1/llmtrace_observability.egg-info/dependency_links.txt +1 -0
- llmtrace_observability-0.1.1/llmtrace_observability.egg-info/entry_points.txt +2 -0
- llmtrace_observability-0.1.1/llmtrace_observability.egg-info/requires.txt +1 -0
- llmtrace_observability-0.1.1/llmtrace_observability.egg-info/top_level.txt +1 -0
- llmtrace_observability-0.1.1/pyproject.toml +31 -0
- llmtrace_observability-0.1.1/setup.cfg +4 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: llmtrace-observability
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: LLMTrace SDK - one-line LLM observability for OpenAI / Anthropic and plain functions
|
|
5
|
+
Author: LLMTrace Contributors
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/zfan980816/llmtrace
|
|
8
|
+
Keywords: llm,observability,tracing,openai,anthropic,langsmith-alternative
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
14
|
+
Classifier: Topic :: System :: Monitoring
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
Requires-Dist: httpx>=0.27
|
|
18
|
+
|
|
19
|
+
# llmtrace-sdk
|
|
20
|
+
|
|
21
|
+
One-line LLM observability for LLMTrace - a self-hosted LangSmith alternative.
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install llmtrace-sdk
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Quick start
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from llmtrace import wrap_openai
|
|
31
|
+
|
|
32
|
+
client = wrap_openai(openai.OpenAI()) # done. every call is traced
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Also ships with:
|
|
36
|
+
- `wrap_anthropic` - Anthropic client (streaming supported)
|
|
37
|
+
- `@trace` / `@span` decorators - sync & async plain functions
|
|
38
|
+
- `record_llm_call` - manual LLM call recording
|
|
39
|
+
- batch upload + disk buffer - retries after network failures
|
|
40
|
+
- PII masking - phone / email / ID numbers
|
|
41
|
+
|
|
42
|
+
See the server docs for setup.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# llmtrace-sdk
|
|
2
|
+
|
|
3
|
+
One-line LLM observability for LLMTrace - a self-hosted LangSmith alternative.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install llmtrace-sdk
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Quick start
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
from llmtrace import wrap_openai
|
|
13
|
+
|
|
14
|
+
client = wrap_openai(openai.OpenAI()) # done. every call is traced
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Also ships with:
|
|
18
|
+
- `wrap_anthropic` - Anthropic client (streaming supported)
|
|
19
|
+
- `@trace` / `@span` decorators - sync & async plain functions
|
|
20
|
+
- `record_llm_call` - manual LLM call recording
|
|
21
|
+
- batch upload + disk buffer - retries after network failures
|
|
22
|
+
- PII masking - phone / email / ID numbers
|
|
23
|
+
|
|
24
|
+
See the server docs for setup.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from .client import configure, record_llm_call, span, trace
|
|
2
|
+
|
|
3
|
+
__all__ = [
|
|
4
|
+
"configure",
|
|
5
|
+
"record_llm_call",
|
|
6
|
+
"span",
|
|
7
|
+
"trace",
|
|
8
|
+
"wrap_openai",
|
|
9
|
+
"wrap_anthropic",
|
|
10
|
+
]
|
|
11
|
+
__version__ = "0.1.0"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def wrap_openai(client):
|
|
15
|
+
"""Auto-instrument an OpenAI SDK v1 client (chat.completions.create)."""
|
|
16
|
+
from .integrations.openai import wrap_openai as _wrap
|
|
17
|
+
return _wrap(client)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def wrap_anthropic(client):
|
|
21
|
+
"""Auto-instrument an Anthropic SDK client (messages.create, non-streaming)."""
|
|
22
|
+
from .integrations.anthropic import wrap_anthropic as _wrap
|
|
23
|
+
return _wrap(client)
|
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
"""llmtrace: minimal SDK for reporting LLM traces to an LLMTrace server."""
|
|
2
|
+
|
|
3
|
+
import atexit
|
|
4
|
+
import inspect
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import queue
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
import tempfile
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
import urllib.request
|
|
14
|
+
import uuid
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
from contextvars import ContextVar
|
|
17
|
+
from datetime import datetime, timedelta, timezone
|
|
18
|
+
from functools import wraps
|
|
19
|
+
|
|
20
|
+
_config = {"api_url": None, "api_key": None, "project_id": None, "buffer_path": None}
|
|
21
|
+
_queue = queue.Queue()
|
|
22
|
+
_sender_started = False
|
|
23
|
+
_max_retries = 3
|
|
24
|
+
_buffer_lock = threading.Lock()
|
|
25
|
+
|
|
26
|
+
_current_trace = ContextVar("llmtrace_current_trace", default=None)
|
|
27
|
+
|
|
28
|
+
MASK_PATTERNS = {
|
|
29
|
+
"phone": (re.compile(r"(?<!\d)1[3-9]\d{9}(?!\d)"), "[phone]"),
|
|
30
|
+
"email": (re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"), "[email]"),
|
|
31
|
+
"id_card": (re.compile(r"(?<!\d)\d{17}[\dXx](?!\d)"), "[id]"),
|
|
32
|
+
}
|
|
33
|
+
_mask_patterns = []
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _now_iso():
|
|
37
|
+
return datetime.now(timezone.utc).isoformat()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _new_state(session_id=None):
|
|
41
|
+
return {"trace_id": str(uuid.uuid4()), "session_id": session_id, "spans": [], "stack": []}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _safe_repr(value):
|
|
45
|
+
try:
|
|
46
|
+
return repr(value)
|
|
47
|
+
except Exception:
|
|
48
|
+
return "<unrepresentable>"
|
|
49
|
+
|
|
50
|
+
def _jsonable(value):
|
|
51
|
+
"""Return value as-is when JSON-serializable, else a readable repr."""
|
|
52
|
+
try:
|
|
53
|
+
json.dumps(value)
|
|
54
|
+
return value
|
|
55
|
+
except Exception:
|
|
56
|
+
return _safe_repr(value)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _capture_input(args, kwargs):
|
|
60
|
+
"""Capture function input the way LangSmith would: a single positional
|
|
61
|
+
argument is stored raw; multiple args / kwargs become structured dicts."""
|
|
62
|
+
if kwargs:
|
|
63
|
+
cap = {k: _jsonable(v) for k, v in kwargs.items()}
|
|
64
|
+
if args:
|
|
65
|
+
cap["args"] = [_jsonable(a) for a in args]
|
|
66
|
+
return cap
|
|
67
|
+
if len(args) == 1:
|
|
68
|
+
return _jsonable(args[0])
|
|
69
|
+
if args:
|
|
70
|
+
return [_jsonable(a) for a in args]
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def configure(api_url=None, api_key=None, project_id=None, masking=None, buffer=True, buffer_path=None):
|
|
75
|
+
"""Configure the SDK. Env vars: LLMTRACE_URL / LLMTRACE_API_KEY / LLMTRACE_PROJECT_ID.
|
|
76
|
+
|
|
77
|
+
masking: True enables all built-in PII rules, or a list like ["phone", "email"].
|
|
78
|
+
buffer: failed payloads are persisted to disk and retried on next startup.
|
|
79
|
+
buffer_path: override the buffer file location.
|
|
80
|
+
"""
|
|
81
|
+
global _sender_started, _mask_patterns
|
|
82
|
+
_config["api_url"] = api_url or os.getenv("LLMTRACE_URL", "http://localhost:8000")
|
|
83
|
+
_config["api_key"] = api_key or os.getenv("LLMTRACE_API_KEY", "")
|
|
84
|
+
_config["project_id"] = project_id or os.getenv("LLMTRACE_PROJECT_ID", "")
|
|
85
|
+
_config["buffer_path"] = (
|
|
86
|
+
None if buffer is False
|
|
87
|
+
else buffer_path or os.getenv("LLMTRACE_BUFFER_PATH")
|
|
88
|
+
or os.path.join(tempfile.gettempdir(), "llmtrace-buffer.jsonl")
|
|
89
|
+
)
|
|
90
|
+
if masking is True:
|
|
91
|
+
keys = list(MASK_PATTERNS)
|
|
92
|
+
elif isinstance(masking, (list, tuple)):
|
|
93
|
+
keys = masking
|
|
94
|
+
else:
|
|
95
|
+
keys = []
|
|
96
|
+
_mask_patterns = [MASK_PATTERNS[k] for k in keys if k in MASK_PATTERNS]
|
|
97
|
+
if not _sender_started:
|
|
98
|
+
threading.Thread(target=_sender_loop, daemon=True).start()
|
|
99
|
+
_sender_started = True
|
|
100
|
+
atexit.register(_drain, 2.0)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _request(path, body):
|
|
104
|
+
url = _config["api_url"].rstrip("/") + path
|
|
105
|
+
headers = {"Content-Type": "application/json", "X-API-Key": _config["api_key"] or ""}
|
|
106
|
+
if _config["project_id"]:
|
|
107
|
+
headers["X-Project-Id"] = str(_config["project_id"])
|
|
108
|
+
req = urllib.request.Request(url, data=body, headers=headers, method="POST")
|
|
109
|
+
with urllib.request.urlopen(req, timeout=10) as resp:
|
|
110
|
+
resp.read()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _buffer_save(batch):
|
|
114
|
+
"""Persist failed payloads to disk (JSON lines, 5MB cap) for later retry."""
|
|
115
|
+
path = _config["buffer_path"]
|
|
116
|
+
if not path:
|
|
117
|
+
return
|
|
118
|
+
with _buffer_lock, open(path, "a", encoding="utf-8") as f:
|
|
119
|
+
for p in batch:
|
|
120
|
+
f.write(json.dumps(p, default=str) + "\n")
|
|
121
|
+
try:
|
|
122
|
+
if os.path.getsize(path) > 5 * 1024 * 1024:
|
|
123
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
124
|
+
tail = f.readlines()[-2000:]
|
|
125
|
+
with open(path, "w", encoding="utf-8") as f:
|
|
126
|
+
f.writelines(tail)
|
|
127
|
+
except OSError:
|
|
128
|
+
pass
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _buffer_load():
|
|
132
|
+
"""Read and clear the disk buffer. Returns payloads from previous runs."""
|
|
133
|
+
path = _config["buffer_path"]
|
|
134
|
+
if not path or not os.path.exists(path):
|
|
135
|
+
return []
|
|
136
|
+
payloads = []
|
|
137
|
+
with open(path, "r", encoding="utf-8") as f:
|
|
138
|
+
for line in f:
|
|
139
|
+
line = line.strip()
|
|
140
|
+
if not line:
|
|
141
|
+
continue
|
|
142
|
+
try:
|
|
143
|
+
payloads.append(json.loads(line))
|
|
144
|
+
except Exception:
|
|
145
|
+
pass
|
|
146
|
+
try:
|
|
147
|
+
os.remove(path)
|
|
148
|
+
except OSError:
|
|
149
|
+
pass
|
|
150
|
+
return payloads
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _post_batch(batch):
|
|
154
|
+
body = json.dumps({"traces": batch}, default=str).encode("utf-8")
|
|
155
|
+
_request("/api/v1/traces/ingest-batch", body)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _send_with_retry(batch):
|
|
159
|
+
for attempt in range(_max_retries):
|
|
160
|
+
try:
|
|
161
|
+
_post_batch(batch)
|
|
162
|
+
return True
|
|
163
|
+
except Exception:
|
|
164
|
+
time.sleep(0.5 * (attempt + 1))
|
|
165
|
+
_buffer_save(batch)
|
|
166
|
+
return False
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _sender_loop():
|
|
170
|
+
buffered = _buffer_load()
|
|
171
|
+
if buffered:
|
|
172
|
+
_send_with_retry(buffered)
|
|
173
|
+
while True:
|
|
174
|
+
batch = []
|
|
175
|
+
try:
|
|
176
|
+
batch.append(_queue.get(timeout=2.0))
|
|
177
|
+
except queue.Empty:
|
|
178
|
+
continue
|
|
179
|
+
deadline = time.time() + 1.0
|
|
180
|
+
while len(batch) < 50:
|
|
181
|
+
remaining = deadline - time.time()
|
|
182
|
+
if remaining <= 0:
|
|
183
|
+
break
|
|
184
|
+
try:
|
|
185
|
+
batch.append(_queue.get(timeout=remaining))
|
|
186
|
+
except queue.Empty:
|
|
187
|
+
break
|
|
188
|
+
_send_with_retry(batch)
|
|
189
|
+
_queue.task_done() # 配合 _queue.join():发送完成(或落盘缓冲)后解除阻塞
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _drain(timeout=2.0):
|
|
193
|
+
batch = []
|
|
194
|
+
deadline = time.time() + timeout
|
|
195
|
+
while time.time() < deadline:
|
|
196
|
+
try:
|
|
197
|
+
batch.append(_queue.get_nowait())
|
|
198
|
+
except queue.Empty:
|
|
199
|
+
break
|
|
200
|
+
if batch:
|
|
201
|
+
_send_with_retry(batch)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _apply_masks(obj):
|
|
205
|
+
if not _mask_patterns:
|
|
206
|
+
return obj
|
|
207
|
+
if isinstance(obj, str):
|
|
208
|
+
for pattern, replacement in _mask_patterns:
|
|
209
|
+
obj = pattern.sub(replacement, obj)
|
|
210
|
+
return obj
|
|
211
|
+
if isinstance(obj, dict):
|
|
212
|
+
return {k: _apply_masks(v) for k, v in obj.items()}
|
|
213
|
+
if isinstance(obj, list):
|
|
214
|
+
return [_apply_masks(v) for v in obj]
|
|
215
|
+
return obj
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _submit(state):
|
|
219
|
+
roots = [s for s in state["spans"] if s["parent_span_id"] is None]
|
|
220
|
+
root = roots[0] if roots else None
|
|
221
|
+
payload = {
|
|
222
|
+
"name": (root or {}).get("name", "trace"),
|
|
223
|
+
"session_id": state.get("session_id"),
|
|
224
|
+
"status": (root or {}).get("status", "ok"),
|
|
225
|
+
"error": (root or {}).get("error"),
|
|
226
|
+
"input": (root or {}).get("input"),
|
|
227
|
+
"output": (root or {}).get("output"),
|
|
228
|
+
"metadata": {"trace_id": state["trace_id"]},
|
|
229
|
+
"start_time": (root or {}).get("start_time") or _now_iso(),
|
|
230
|
+
"end_time": (root or {}).get("end_time") or _now_iso(),
|
|
231
|
+
"spans": [
|
|
232
|
+
{
|
|
233
|
+
"id": s["id"],
|
|
234
|
+
"parent_span_id": s["parent_span_id"],
|
|
235
|
+
"name": s["name"],
|
|
236
|
+
"span_type": s["span_type"],
|
|
237
|
+
"model": s.get("model"),
|
|
238
|
+
"status": s["status"],
|
|
239
|
+
"error": s.get("error"),
|
|
240
|
+
"input": s.get("input"),
|
|
241
|
+
"output": s.get("output"),
|
|
242
|
+
"metadata": s.get("metadata"),
|
|
243
|
+
"prompt_tokens": s.get("prompt_tokens", 0),
|
|
244
|
+
"completion_tokens": s.get("completion_tokens", 0),
|
|
245
|
+
"total_tokens": s.get("total_tokens", 0),
|
|
246
|
+
"start_time": s["start_time"],
|
|
247
|
+
"end_time": s["end_time"],
|
|
248
|
+
}
|
|
249
|
+
for s in state["spans"]
|
|
250
|
+
],
|
|
251
|
+
}
|
|
252
|
+
_queue.put(_apply_masks(payload))
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
@contextmanager
|
|
256
|
+
def span(name, span_type="chain", *, input=None, model=None, metadata=None, session_id=None):
|
|
257
|
+
"""Record a span. Creates a new trace automatically at the outermost level."""
|
|
258
|
+
state = _current_trace.get()
|
|
259
|
+
is_root = state is None
|
|
260
|
+
if state is None:
|
|
261
|
+
state = _new_state(session_id=session_id)
|
|
262
|
+
|
|
263
|
+
rec = {
|
|
264
|
+
"id": str(uuid.uuid4()),
|
|
265
|
+
"parent_span_id": state["stack"][-1] if state["stack"] else None,
|
|
266
|
+
"name": name,
|
|
267
|
+
"span_type": span_type,
|
|
268
|
+
"model": model,
|
|
269
|
+
"status": "ok",
|
|
270
|
+
"error": None,
|
|
271
|
+
"input": input,
|
|
272
|
+
"output": None,
|
|
273
|
+
"metadata": metadata,
|
|
274
|
+
"prompt_tokens": 0,
|
|
275
|
+
"completion_tokens": 0,
|
|
276
|
+
"total_tokens": 0,
|
|
277
|
+
}
|
|
278
|
+
state["spans"].append(rec)
|
|
279
|
+
state["stack"].append(rec["id"])
|
|
280
|
+
token = _current_trace.set(state)
|
|
281
|
+
t0 = time.perf_counter()
|
|
282
|
+
start_iso = _now_iso()
|
|
283
|
+
try:
|
|
284
|
+
yield rec
|
|
285
|
+
except BaseException as exc:
|
|
286
|
+
rec["status"] = "error"
|
|
287
|
+
rec["error"] = type(exc).__name__ + ": " + str(exc)
|
|
288
|
+
raise
|
|
289
|
+
finally:
|
|
290
|
+
rec["start_time"] = start_iso
|
|
291
|
+
rec["end_time"] = _now_iso()
|
|
292
|
+
rec["latency_ms"] = round((time.perf_counter() - t0) * 1000)
|
|
293
|
+
state["stack"].pop()
|
|
294
|
+
_current_trace.reset(token)
|
|
295
|
+
if is_root and not state["stack"]:
|
|
296
|
+
if not _config["api_url"]:
|
|
297
|
+
configure()
|
|
298
|
+
_submit(state)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def trace(name=None, *, span_type="chain", session_id=None, metadata=None):
|
|
302
|
+
"""Decorator version of span() for sync AND async functions."""
|
|
303
|
+
def deco(fn):
|
|
304
|
+
if inspect.iscoroutinefunction(fn):
|
|
305
|
+
@wraps(fn)
|
|
306
|
+
async def async_wrapper(*args, **kwargs):
|
|
307
|
+
with span(name or fn.__name__, span_type=span_type,
|
|
308
|
+
input=_capture_input(args, kwargs),
|
|
309
|
+
metadata=metadata, session_id=session_id) as rec:
|
|
310
|
+
result = await fn(*args, **kwargs)
|
|
311
|
+
rec["output"] = result
|
|
312
|
+
return result
|
|
313
|
+
return async_wrapper
|
|
314
|
+
|
|
315
|
+
@wraps(fn)
|
|
316
|
+
def wrapper(*args, **kwargs):
|
|
317
|
+
with span(name or fn.__name__, span_type=span_type,
|
|
318
|
+
input=_capture_input(args, kwargs),
|
|
319
|
+
metadata=metadata, session_id=session_id) as rec:
|
|
320
|
+
result = fn(*args, **kwargs)
|
|
321
|
+
rec["output"] = result
|
|
322
|
+
return result
|
|
323
|
+
return wrapper
|
|
324
|
+
return deco
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def record_llm_call(model, input=None, output=None, prompt_tokens=0,
|
|
328
|
+
completion_tokens=0, latency_ms=None, metadata=None, error=None):
|
|
329
|
+
"""Record a single LLM call as a span (type=llm) inside the current trace."""
|
|
330
|
+
state = _current_trace.get()
|
|
331
|
+
is_root = state is None
|
|
332
|
+
if state is None:
|
|
333
|
+
state = _new_state()
|
|
334
|
+
start_dt = datetime.now(timezone.utc)
|
|
335
|
+
end_dt = start_dt + timedelta(milliseconds=latency_ms or 0)
|
|
336
|
+
rec = {
|
|
337
|
+
"id": str(uuid.uuid4()),
|
|
338
|
+
"parent_span_id": state["stack"][-1] if state["stack"] else None,
|
|
339
|
+
"name": model or "llm-call",
|
|
340
|
+
"span_type": "llm",
|
|
341
|
+
"model": model,
|
|
342
|
+
"status": "error" if error else "ok",
|
|
343
|
+
"error": error,
|
|
344
|
+
"input": input,
|
|
345
|
+
"output": output,
|
|
346
|
+
"metadata": metadata,
|
|
347
|
+
"prompt_tokens": prompt_tokens,
|
|
348
|
+
"completion_tokens": completion_tokens,
|
|
349
|
+
"total_tokens": prompt_tokens + completion_tokens,
|
|
350
|
+
"start_time": start_dt.isoformat(),
|
|
351
|
+
"end_time": end_dt.isoformat(),
|
|
352
|
+
"latency_ms": latency_ms or 0,
|
|
353
|
+
}
|
|
354
|
+
state["spans"].append(rec)
|
|
355
|
+
if is_root:
|
|
356
|
+
if not _config["api_url"]:
|
|
357
|
+
configure()
|
|
358
|
+
_submit(state)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""llmtrace demo - send a sample trace so you can see the UI working."""
|
|
2
|
+
import argparse
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
import uuid
|
|
7
|
+
|
|
8
|
+
from .client import configure, span
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def main():
|
|
12
|
+
parser = argparse.ArgumentParser(prog="llmtrace", description="LLMTrace SDK CLI")
|
|
13
|
+
sub = parser.add_subparsers(dest="command")
|
|
14
|
+
demo = sub.add_parser("demo", help="send a sample trace so you can see the dashboard with data")
|
|
15
|
+
demo.add_argument("--url", default=os.environ.get("LLMTRACE_URL", "http://localhost:8000"))
|
|
16
|
+
demo.add_argument("--key", default=os.environ.get("LLMTRACE_API_KEY", ""))
|
|
17
|
+
demo.add_argument("--project", default=os.environ.get("LLMTRACE_PROJECT_ID", "default"))
|
|
18
|
+
args = parser.parse_args()
|
|
19
|
+
|
|
20
|
+
if args.command != "demo":
|
|
21
|
+
parser.print_help()
|
|
22
|
+
sys.exit(1)
|
|
23
|
+
|
|
24
|
+
configure(api_url=args.url, api_key=args.key, project_id=args.project)
|
|
25
|
+
sid = "demo-" + uuid.uuid4().hex[:8]
|
|
26
|
+
|
|
27
|
+
with span(name="demo.conversation", span_type="chain", session_id=sid,
|
|
28
|
+
input={"query": "what is llmtrace?"}) as root:
|
|
29
|
+
with span(name="retrieve_context", span_type="retrieval",
|
|
30
|
+
input={"query": "what is llmtrace?"}) as sp:
|
|
31
|
+
time.sleep(0.06)
|
|
32
|
+
sp["output"] = {"documents": [
|
|
33
|
+
"LLMTrace is a self-hosted LLM observability platform, Apache-2.0.",
|
|
34
|
+
"Single Docker container, SQLite for trials, Postgres for production.",
|
|
35
|
+
]}
|
|
36
|
+
sp["metadata"] = {"source": "llmtrace-demo"}
|
|
37
|
+
with span(name="chat.completion", span_type="llm", model="gpt-4o-mini",
|
|
38
|
+
input={"messages": [
|
|
39
|
+
{"role": "system", "content": "You are a helpful assistant."},
|
|
40
|
+
{"role": "user", "content": "what is llmtrace?"},
|
|
41
|
+
]}) as sp:
|
|
42
|
+
time.sleep(0.12)
|
|
43
|
+
sp["output"] = {"role": "assistant",
|
|
44
|
+
"content": "LLMTrace is a self-hosted LLM observability platform, similar to LangSmith but keeps data on your own server."}
|
|
45
|
+
sp["prompt_tokens"] = 18
|
|
46
|
+
sp["completion_tokens"] = 24
|
|
47
|
+
sp["total_tokens"] = 42
|
|
48
|
+
sp["metadata"] = {"cost_usd": 0.000042, "finish_reason": "stop"}
|
|
49
|
+
|
|
50
|
+
import urllib.request
|
|
51
|
+
try:
|
|
52
|
+
urllib.request.urlopen(args.url.rstrip("/") + "/api/health", timeout=3)
|
|
53
|
+
except Exception:
|
|
54
|
+
print(f"warning: could not reach {args.url} - trace is queued and will retry")
|
|
55
|
+
print(f"trace sent: {args.url}")
|
|
56
|
+
print(f"session id: {sid}")
|
|
57
|
+
print(f"open {args.url}/sessions and look for {sid}")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
if __name__ == "__main__":
|
|
61
|
+
main()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Auto-instrumentation for the Anthropic python SDK (non-streaming).
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
import anthropic
|
|
5
|
+
import llmtrace
|
|
6
|
+
|
|
7
|
+
client = llmtrace.wrap_anthropic(anthropic.Anthropic())
|
|
8
|
+
msg = client.messages.create(model="claude-3-5-haiku", messages=[...])
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from ..client import span
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def wrap_anthropic(client):
|
|
15
|
+
"""Patch client.messages.create so every call is traced."""
|
|
16
|
+
original = client.messages.create
|
|
17
|
+
|
|
18
|
+
def traced_create(*args, **kwargs):
|
|
19
|
+
if kwargs.get("stream"):
|
|
20
|
+
# Streaming (message stream manager) is not instrumented yet.
|
|
21
|
+
return original(*args, **kwargs)
|
|
22
|
+
model = kwargs.get("model")
|
|
23
|
+
messages = kwargs.get("messages")
|
|
24
|
+
with span(model or "anthropic-messages", span_type="llm", input=messages, model=model) as rec:
|
|
25
|
+
response = original(*args, **kwargs)
|
|
26
|
+
texts = [
|
|
27
|
+
block.text
|
|
28
|
+
for block in getattr(response, "content", [])
|
|
29
|
+
if hasattr(block, "text")
|
|
30
|
+
]
|
|
31
|
+
rec["output"] = "".join(texts) or None
|
|
32
|
+
usage = getattr(response, "usage", None)
|
|
33
|
+
if usage is not None:
|
|
34
|
+
rec["prompt_tokens"] = getattr(usage, "input_tokens", 0) or 0
|
|
35
|
+
rec["completion_tokens"] = getattr(usage, "output_tokens", 0) or 0
|
|
36
|
+
rec["total_tokens"] = rec["prompt_tokens"] + rec["completion_tokens"]
|
|
37
|
+
return response
|
|
38
|
+
|
|
39
|
+
client.messages.create = traced_create
|
|
40
|
+
return client
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Auto-instrumentation for the OpenAI python SDK (v1.x client).
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
from openai import OpenAI
|
|
5
|
+
import llmtrace
|
|
6
|
+
|
|
7
|
+
client = llmtrace.wrap_openai(OpenAI())
|
|
8
|
+
resp = client.chat.completions.create(model="gpt-4o-mini", messages=[...])
|
|
9
|
+
# -> an llm span with model / input / output / token usage / latency is recorded
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from ..client import span
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _record_response(rec, response):
|
|
16
|
+
choices = getattr(response, "choices", None)
|
|
17
|
+
if choices:
|
|
18
|
+
msg = getattr(choices[0], "message", None)
|
|
19
|
+
rec["output"] = getattr(msg, "content", None) or getattr(choices[0], "text", None)
|
|
20
|
+
usage = getattr(response, "usage", None)
|
|
21
|
+
if usage is not None:
|
|
22
|
+
rec["prompt_tokens"] = getattr(usage, "prompt_tokens", 0) or 0
|
|
23
|
+
rec["completion_tokens"] = getattr(usage, "completion_tokens", 0) or 0
|
|
24
|
+
rec["total_tokens"] = rec["prompt_tokens"] + rec["completion_tokens"]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def wrap_openai(client):
|
|
28
|
+
"""Patch client.chat.completions.create so every call is traced."""
|
|
29
|
+
original = client.chat.completions.create
|
|
30
|
+
|
|
31
|
+
def traced_create(*args, **kwargs):
|
|
32
|
+
model = kwargs.get("model")
|
|
33
|
+
messages = kwargs.get("messages")
|
|
34
|
+
if kwargs.get("stream"):
|
|
35
|
+
return _traced_stream(original, args, kwargs, model, messages)
|
|
36
|
+
with span(model or "openai-chat", span_type="llm", input=messages, model=model) as rec:
|
|
37
|
+
response = original(*args, **kwargs)
|
|
38
|
+
_record_response(rec, response)
|
|
39
|
+
return response
|
|
40
|
+
|
|
41
|
+
client.chat.completions.create = traced_create
|
|
42
|
+
return client
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _traced_stream(original, args, kwargs, model, messages):
|
|
46
|
+
"""Generator wrapper: keeps the span open for the whole stream iteration."""
|
|
47
|
+
with span(model or "openai-chat", span_type="llm", input=messages, model=model) as rec:
|
|
48
|
+
pieces = []
|
|
49
|
+
for chunk in original(*args, **kwargs):
|
|
50
|
+
try:
|
|
51
|
+
delta = chunk.choices[0].delta
|
|
52
|
+
text = getattr(delta, "content", None)
|
|
53
|
+
except (AttributeError, IndexError, TypeError):
|
|
54
|
+
text = None
|
|
55
|
+
if text:
|
|
56
|
+
pieces.append(text)
|
|
57
|
+
usage = getattr(chunk, "usage", None)
|
|
58
|
+
if usage is not None:
|
|
59
|
+
rec["prompt_tokens"] = getattr(usage, "prompt_tokens", 0) or 0
|
|
60
|
+
rec["completion_tokens"] = getattr(usage, "completion_tokens", 0) or 0
|
|
61
|
+
rec["total_tokens"] = rec["prompt_tokens"] + rec["completion_tokens"]
|
|
62
|
+
yield chunk
|
|
63
|
+
rec["output"] = "".join(pieces) or None
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: llmtrace-observability
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: LLMTrace SDK - one-line LLM observability for OpenAI / Anthropic and plain functions
|
|
5
|
+
Author: LLMTrace Contributors
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/zfan980816/llmtrace
|
|
8
|
+
Keywords: llm,observability,tracing,openai,anthropic,langsmith-alternative
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
14
|
+
Classifier: Topic :: System :: Monitoring
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
Requires-Dist: httpx>=0.27
|
|
18
|
+
|
|
19
|
+
# llmtrace-sdk
|
|
20
|
+
|
|
21
|
+
One-line LLM observability for LLMTrace - a self-hosted LangSmith alternative.
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install llmtrace-sdk
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Quick start
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from llmtrace import wrap_openai
|
|
31
|
+
|
|
32
|
+
client = wrap_openai(openai.OpenAI()) # done. every call is traced
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Also ships with:
|
|
36
|
+
- `wrap_anthropic` - Anthropic client (streaming supported)
|
|
37
|
+
- `@trace` / `@span` decorators - sync & async plain functions
|
|
38
|
+
- `record_llm_call` - manual LLM call recording
|
|
39
|
+
- batch upload + disk buffer - retries after network failures
|
|
40
|
+
- PII masking - phone / email / ID numbers
|
|
41
|
+
|
|
42
|
+
See the server docs for setup.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
llmtrace/__init__.py
|
|
4
|
+
llmtrace/client.py
|
|
5
|
+
llmtrace/demo.py
|
|
6
|
+
llmtrace/integrations/__init__.py
|
|
7
|
+
llmtrace/integrations/anthropic.py
|
|
8
|
+
llmtrace/integrations/openai.py
|
|
9
|
+
llmtrace_observability.egg-info/PKG-INFO
|
|
10
|
+
llmtrace_observability.egg-info/SOURCES.txt
|
|
11
|
+
llmtrace_observability.egg-info/dependency_links.txt
|
|
12
|
+
llmtrace_observability.egg-info/entry_points.txt
|
|
13
|
+
llmtrace_observability.egg-info/requires.txt
|
|
14
|
+
llmtrace_observability.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
httpx>=0.27
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
llmtrace
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "llmtrace-observability"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "LLMTrace SDK - one-line LLM observability for OpenAI / Anthropic and plain functions"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "Apache-2.0" }
|
|
12
|
+
authors = [{ name = "LLMTrace Contributors" }]
|
|
13
|
+
keywords = ["llm", "observability", "tracing", "openai", "anthropic", "langsmith-alternative"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 4 - Beta",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: Apache Software License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Software Development :: Debuggers",
|
|
20
|
+
"Topic :: System :: Monitoring",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["httpx>=0.27"]
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
llmtrace = "llmtrace.demo:main"
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/zfan980816/llmtrace"
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.packages.find]
|
|
31
|
+
include = ["llmtrace*"]
|