freesolo 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- freesolo/__init__.py +21 -0
- freesolo/client.py +385 -0
- freesolo/decorators.py +62 -0
- freesolo/providers.py +185 -0
- freesolo/sanitize.py +123 -0
- freesolo-0.1.0.dist-info/METADATA +100 -0
- freesolo-0.1.0.dist-info/RECORD +8 -0
- freesolo-0.1.0.dist-info/WHEEL +4 -0
freesolo/__init__.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from .client import Tracer, configure, get_current_trace_id, get_tracer, start_trace
|
|
2
|
+
from .decorators import trace
|
|
3
|
+
from .providers import (
|
|
4
|
+
instrument_anthropic,
|
|
5
|
+
instrument_client,
|
|
6
|
+
instrument_google,
|
|
7
|
+
instrument_openai,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"Tracer",
|
|
12
|
+
"configure",
|
|
13
|
+
"start_trace",
|
|
14
|
+
"get_tracer",
|
|
15
|
+
"get_current_trace_id",
|
|
16
|
+
"trace",
|
|
17
|
+
"instrument_openai",
|
|
18
|
+
"instrument_anthropic",
|
|
19
|
+
"instrument_google",
|
|
20
|
+
"instrument_client",
|
|
21
|
+
]
|
freesolo/client.py
ADDED
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import contextvars
|
|
5
|
+
import functools
|
|
6
|
+
import os
|
|
7
|
+
import uuid
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from datetime import UTC, datetime
|
|
10
|
+
from typing import Any, Callable
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
from .sanitize import sanitize_value
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _utc_now() -> datetime:
|
|
18
|
+
return datetime.now(UTC)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _env(name: str) -> str | None:
|
|
22
|
+
value = os.getenv(name)
|
|
23
|
+
return value or None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class TracerConfig:
|
|
28
|
+
base_url: str | None = None
|
|
29
|
+
api_key: str | None = None
|
|
30
|
+
timeout_seconds: float = 5.0
|
|
31
|
+
enabled: bool = True
|
|
32
|
+
endpoint_path: str = "/api/traces/ingest"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class SpanRecord:
|
|
37
|
+
started_at: datetime
|
|
38
|
+
input_payload: Any = None
|
|
39
|
+
output_payload: Any = None
|
|
40
|
+
provider: str | None = None
|
|
41
|
+
model: str | None = None
|
|
42
|
+
input_tokens: int | None = None
|
|
43
|
+
output_tokens: int | None = None
|
|
44
|
+
duration_ms: int | None = None
|
|
45
|
+
|
|
46
|
+
def as_payload(self) -> dict[str, Any]:
|
|
47
|
+
return {
|
|
48
|
+
"provider": self.provider,
|
|
49
|
+
"model": self.model,
|
|
50
|
+
"duration_ms": self.duration_ms,
|
|
51
|
+
"input_tokens": self.input_tokens,
|
|
52
|
+
"output_tokens": self.output_tokens,
|
|
53
|
+
"input_payload": self.input_payload,
|
|
54
|
+
"output_payload": self.output_payload,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class TraceRecord:
|
|
60
|
+
trace_id: str
|
|
61
|
+
metadata: dict[str, Any] | None = None
|
|
62
|
+
spans: list[SpanRecord] = field(default_factory=list)
|
|
63
|
+
managed: bool = False
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class ActiveSpan:
|
|
68
|
+
trace: TraceRecord
|
|
69
|
+
is_root: bool
|
|
70
|
+
span: SpanRecord
|
|
71
|
+
trace_token: contextvars.Token[TraceRecord | None] | None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
_current_trace: contextvars.ContextVar[TraceRecord | None] = contextvars.ContextVar(
|
|
75
|
+
"freesolo_current_trace",
|
|
76
|
+
default=None,
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class Tracer:
|
|
81
|
+
def __init__(self) -> None:
|
|
82
|
+
self._config = TracerConfig(
|
|
83
|
+
base_url=((_env("FREESOLO_BASE_URL") or "").rstrip("/") or None),
|
|
84
|
+
api_key=_env("FREESOLO_API_KEY"),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def config(self) -> TracerConfig:
|
|
89
|
+
return self._config
|
|
90
|
+
|
|
91
|
+
def configure(
|
|
92
|
+
self,
|
|
93
|
+
*,
|
|
94
|
+
base_url: str | None = None,
|
|
95
|
+
api_key: str | None = None,
|
|
96
|
+
) -> None:
|
|
97
|
+
if base_url is not None:
|
|
98
|
+
self._config.base_url = base_url.rstrip("/")
|
|
99
|
+
if api_key is not None:
|
|
100
|
+
self._config.api_key = api_key
|
|
101
|
+
|
|
102
|
+
def start_trace(
|
|
103
|
+
self,
|
|
104
|
+
trace_id: str | None = None,
|
|
105
|
+
*,
|
|
106
|
+
metadata: dict[str, Any] | None = None,
|
|
107
|
+
) -> _TraceContext:
|
|
108
|
+
return _TraceContext(
|
|
109
|
+
tracer=self,
|
|
110
|
+
trace=TraceRecord(
|
|
111
|
+
trace_id=trace_id or str(uuid.uuid4()),
|
|
112
|
+
metadata=metadata,
|
|
113
|
+
managed=True,
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def _start_span(
|
|
118
|
+
self,
|
|
119
|
+
*,
|
|
120
|
+
input_payload: Any,
|
|
121
|
+
provider: str | None = None,
|
|
122
|
+
model: str | None = None,
|
|
123
|
+
) -> ActiveSpan:
|
|
124
|
+
trace = _current_trace.get()
|
|
125
|
+
is_root = trace is None
|
|
126
|
+
trace_token: contextvars.Token[TraceRecord | None] | None = None
|
|
127
|
+
|
|
128
|
+
if trace is None:
|
|
129
|
+
trace = TraceRecord(trace_id=str(uuid.uuid4()))
|
|
130
|
+
trace_token = _current_trace.set(trace)
|
|
131
|
+
|
|
132
|
+
span = SpanRecord(
|
|
133
|
+
started_at=_utc_now(),
|
|
134
|
+
input_payload=input_payload,
|
|
135
|
+
provider=provider,
|
|
136
|
+
model=model,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
return ActiveSpan(
|
|
140
|
+
trace=trace,
|
|
141
|
+
is_root=is_root,
|
|
142
|
+
span=span,
|
|
143
|
+
trace_token=trace_token,
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def _finish_span(
|
|
147
|
+
self,
|
|
148
|
+
active_span: ActiveSpan,
|
|
149
|
+
*,
|
|
150
|
+
output_payload: Any = None,
|
|
151
|
+
usage: dict[str, int | None] | None = None,
|
|
152
|
+
) -> None:
|
|
153
|
+
ended_at = _utc_now()
|
|
154
|
+
active_span.span.duration_ms = int(
|
|
155
|
+
(ended_at - active_span.span.started_at).total_seconds() * 1000
|
|
156
|
+
)
|
|
157
|
+
active_span.span.output_payload = output_payload
|
|
158
|
+
|
|
159
|
+
if usage:
|
|
160
|
+
active_span.span.input_tokens = usage.get("input_tokens")
|
|
161
|
+
active_span.span.output_tokens = usage.get("output_tokens")
|
|
162
|
+
|
|
163
|
+
active_span.trace.spans.append(active_span.span)
|
|
164
|
+
|
|
165
|
+
if not active_span.is_root or active_span.trace.managed:
|
|
166
|
+
return
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
self._emit_trace(active_span.trace)
|
|
170
|
+
finally:
|
|
171
|
+
if active_span.trace_token is not None:
|
|
172
|
+
_current_trace.reset(active_span.trace_token)
|
|
173
|
+
|
|
174
|
+
async def _finish_span_async(
|
|
175
|
+
self,
|
|
176
|
+
active_span: ActiveSpan,
|
|
177
|
+
*,
|
|
178
|
+
output_payload: Any = None,
|
|
179
|
+
usage: dict[str, int | None] | None = None,
|
|
180
|
+
) -> None:
|
|
181
|
+
ended_at = _utc_now()
|
|
182
|
+
active_span.span.duration_ms = int(
|
|
183
|
+
(ended_at - active_span.span.started_at).total_seconds() * 1000
|
|
184
|
+
)
|
|
185
|
+
active_span.span.output_payload = output_payload
|
|
186
|
+
|
|
187
|
+
if usage:
|
|
188
|
+
active_span.span.input_tokens = usage.get("input_tokens")
|
|
189
|
+
active_span.span.output_tokens = usage.get("output_tokens")
|
|
190
|
+
|
|
191
|
+
active_span.trace.spans.append(active_span.span)
|
|
192
|
+
|
|
193
|
+
if not active_span.is_root or active_span.trace.managed:
|
|
194
|
+
return
|
|
195
|
+
|
|
196
|
+
try:
|
|
197
|
+
await self._emit_trace_async(active_span.trace)
|
|
198
|
+
finally:
|
|
199
|
+
if active_span.trace_token is not None:
|
|
200
|
+
_current_trace.reset(active_span.trace_token)
|
|
201
|
+
|
|
202
|
+
def create_payload(self, trace: TraceRecord) -> dict[str, Any]:
|
|
203
|
+
return {
|
|
204
|
+
"metadata": trace.metadata,
|
|
205
|
+
"spans": [span.as_payload() for span in trace.spans],
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
def _emit_trace(self, trace: TraceRecord) -> None:
|
|
209
|
+
if (
|
|
210
|
+
not self._config.enabled
|
|
211
|
+
or not self._config.base_url
|
|
212
|
+
or not self._config.api_key
|
|
213
|
+
):
|
|
214
|
+
return
|
|
215
|
+
|
|
216
|
+
payload = self.create_payload(trace)
|
|
217
|
+
endpoint = f"{self._config.base_url}{self._config.endpoint_path}"
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
with httpx.Client(timeout=self._config.timeout_seconds) as client:
|
|
221
|
+
client.post(
|
|
222
|
+
endpoint,
|
|
223
|
+
headers={
|
|
224
|
+
"Authorization": f"Bearer {self._config.api_key}",
|
|
225
|
+
"Content-Type": "application/json",
|
|
226
|
+
},
|
|
227
|
+
json=payload,
|
|
228
|
+
).raise_for_status()
|
|
229
|
+
except Exception:
|
|
230
|
+
return
|
|
231
|
+
|
|
232
|
+
async def _emit_trace_async(self, trace: TraceRecord) -> None:
|
|
233
|
+
if (
|
|
234
|
+
not self._config.enabled
|
|
235
|
+
or not self._config.base_url
|
|
236
|
+
or not self._config.api_key
|
|
237
|
+
):
|
|
238
|
+
return
|
|
239
|
+
|
|
240
|
+
payload = self.create_payload(trace)
|
|
241
|
+
endpoint = f"{self._config.base_url}{self._config.endpoint_path}"
|
|
242
|
+
|
|
243
|
+
try:
|
|
244
|
+
async with httpx.AsyncClient(
|
|
245
|
+
timeout=self._config.timeout_seconds
|
|
246
|
+
) as client:
|
|
247
|
+
response = await client.post(
|
|
248
|
+
endpoint,
|
|
249
|
+
headers={
|
|
250
|
+
"Authorization": f"Bearer {self._config.api_key}",
|
|
251
|
+
"Content-Type": "application/json",
|
|
252
|
+
},
|
|
253
|
+
json=payload,
|
|
254
|
+
)
|
|
255
|
+
response.raise_for_status()
|
|
256
|
+
except Exception:
|
|
257
|
+
return
|
|
258
|
+
|
|
259
|
+
def wrap_callable(
|
|
260
|
+
self,
|
|
261
|
+
func: Callable[..., Any],
|
|
262
|
+
*,
|
|
263
|
+
input_builder: Callable[[tuple[Any, ...], dict[str, Any]], Any],
|
|
264
|
+
output_builder: Callable[[Any], Any],
|
|
265
|
+
attribute_builder: Callable[[tuple[Any, ...], dict[str, Any]], dict[str, Any] | None]
|
|
266
|
+
| None = None,
|
|
267
|
+
usage_builder: Callable[[Any], dict[str, int | None] | None] | None = None,
|
|
268
|
+
) -> Callable[..., Any]:
|
|
269
|
+
if asyncio.iscoroutinefunction(func):
|
|
270
|
+
|
|
271
|
+
@functools.wraps(func)
|
|
272
|
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
273
|
+
attributes = attribute_builder(args, kwargs) if attribute_builder else {}
|
|
274
|
+
active_span = self._start_span(
|
|
275
|
+
input_payload=input_builder(args, kwargs),
|
|
276
|
+
provider=attributes.get("provider"),
|
|
277
|
+
model=attributes.get("model"),
|
|
278
|
+
)
|
|
279
|
+
try:
|
|
280
|
+
result = await func(*args, **kwargs)
|
|
281
|
+
except BaseException:
|
|
282
|
+
await self._finish_span_async(active_span)
|
|
283
|
+
raise
|
|
284
|
+
|
|
285
|
+
usage = usage_builder(result) if usage_builder else None
|
|
286
|
+
await self._finish_span_async(
|
|
287
|
+
active_span,
|
|
288
|
+
output_payload=output_builder(result),
|
|
289
|
+
usage=usage,
|
|
290
|
+
)
|
|
291
|
+
return result
|
|
292
|
+
|
|
293
|
+
return async_wrapper
|
|
294
|
+
|
|
295
|
+
@functools.wraps(func)
|
|
296
|
+
def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
297
|
+
attributes = attribute_builder(args, kwargs) if attribute_builder else {}
|
|
298
|
+
active_span = self._start_span(
|
|
299
|
+
input_payload=input_builder(args, kwargs),
|
|
300
|
+
provider=attributes.get("provider"),
|
|
301
|
+
model=attributes.get("model"),
|
|
302
|
+
)
|
|
303
|
+
try:
|
|
304
|
+
result = func(*args, **kwargs)
|
|
305
|
+
except BaseException:
|
|
306
|
+
self._finish_span(active_span)
|
|
307
|
+
raise
|
|
308
|
+
|
|
309
|
+
usage = usage_builder(result) if usage_builder else None
|
|
310
|
+
self._finish_span(
|
|
311
|
+
active_span,
|
|
312
|
+
output_payload=output_builder(result),
|
|
313
|
+
usage=usage,
|
|
314
|
+
)
|
|
315
|
+
return result
|
|
316
|
+
|
|
317
|
+
return sync_wrapper
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
@dataclass
|
|
321
|
+
class _TraceContext:
|
|
322
|
+
tracer: Tracer
|
|
323
|
+
trace: TraceRecord
|
|
324
|
+
trace_token: contextvars.Token[TraceRecord | None] | None = None
|
|
325
|
+
|
|
326
|
+
def __enter__(self) -> str:
|
|
327
|
+
self.trace_token = _current_trace.set(self.trace)
|
|
328
|
+
return self.trace.trace_id
|
|
329
|
+
|
|
330
|
+
def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
|
|
331
|
+
try:
|
|
332
|
+
if self.trace.spans:
|
|
333
|
+
self.tracer._emit_trace(self.trace)
|
|
334
|
+
finally:
|
|
335
|
+
if self.trace_token is not None:
|
|
336
|
+
_current_trace.reset(self.trace_token)
|
|
337
|
+
|
|
338
|
+
async def __aenter__(self) -> str:
|
|
339
|
+
return self.__enter__()
|
|
340
|
+
|
|
341
|
+
async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
|
|
342
|
+
try:
|
|
343
|
+
if self.trace.spans:
|
|
344
|
+
await self.tracer._emit_trace_async(self.trace)
|
|
345
|
+
finally:
|
|
346
|
+
if self.trace_token is not None:
|
|
347
|
+
_current_trace.reset(self.trace_token)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
_default_tracer = Tracer()
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def get_tracer() -> Tracer:
|
|
354
|
+
return _default_tracer
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def configure(*, base_url: str | None = None, api_key: str | None = None) -> None:
|
|
358
|
+
_default_tracer.configure(base_url=base_url, api_key=api_key)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def start_trace(
|
|
362
|
+
trace_id: str | None = None,
|
|
363
|
+
*,
|
|
364
|
+
metadata: dict[str, Any] | None = None,
|
|
365
|
+
) -> _TraceContext:
|
|
366
|
+
return _default_tracer.start_trace(trace_id=trace_id, metadata=metadata)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def get_current_trace_id() -> str | None:
|
|
370
|
+
trace = _current_trace.get()
|
|
371
|
+
if trace is None:
|
|
372
|
+
return None
|
|
373
|
+
|
|
374
|
+
return trace.trace_id
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def sanitize_call_args(args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
378
|
+
return {
|
|
379
|
+
"args": sanitize_value(args),
|
|
380
|
+
"kwargs": sanitize_value(kwargs),
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def sanitize_result(result: Any) -> Any:
|
|
385
|
+
return sanitize_value(result)
|
freesolo/decorators.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import functools
|
|
5
|
+
from typing import Any, Callable
|
|
6
|
+
|
|
7
|
+
from .client import get_tracer, sanitize_call_args, sanitize_result
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def trace(
|
|
11
|
+
*,
|
|
12
|
+
capture_input: bool = True,
|
|
13
|
+
capture_output: bool = True,
|
|
14
|
+
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
|
15
|
+
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
16
|
+
tracer = get_tracer()
|
|
17
|
+
|
|
18
|
+
if asyncio.iscoroutinefunction(func):
|
|
19
|
+
|
|
20
|
+
@functools.wraps(func)
|
|
21
|
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
22
|
+
active_span = tracer._start_span( # noqa: SLF001
|
|
23
|
+
input_payload=sanitize_call_args(args, kwargs)
|
|
24
|
+
if capture_input
|
|
25
|
+
else None,
|
|
26
|
+
)
|
|
27
|
+
try:
|
|
28
|
+
result = await func(*args, **kwargs)
|
|
29
|
+
except BaseException:
|
|
30
|
+
await tracer._finish_span_async(active_span) # noqa: SLF001
|
|
31
|
+
raise
|
|
32
|
+
|
|
33
|
+
await tracer._finish_span_async( # noqa: SLF001
|
|
34
|
+
active_span,
|
|
35
|
+
output_payload=sanitize_result(result) if capture_output else None,
|
|
36
|
+
)
|
|
37
|
+
return result
|
|
38
|
+
|
|
39
|
+
return async_wrapper
|
|
40
|
+
|
|
41
|
+
@functools.wraps(func)
|
|
42
|
+
def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
43
|
+
active_span = tracer._start_span( # noqa: SLF001
|
|
44
|
+
input_payload=sanitize_call_args(args, kwargs)
|
|
45
|
+
if capture_input
|
|
46
|
+
else None,
|
|
47
|
+
)
|
|
48
|
+
try:
|
|
49
|
+
result = func(*args, **kwargs)
|
|
50
|
+
except BaseException:
|
|
51
|
+
tracer._finish_span(active_span) # noqa: SLF001
|
|
52
|
+
raise
|
|
53
|
+
|
|
54
|
+
tracer._finish_span( # noqa: SLF001
|
|
55
|
+
active_span,
|
|
56
|
+
output_payload=sanitize_result(result) if capture_output else None,
|
|
57
|
+
)
|
|
58
|
+
return result
|
|
59
|
+
|
|
60
|
+
return sync_wrapper
|
|
61
|
+
|
|
62
|
+
return decorator
|
freesolo/providers.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from .client import get_tracer, sanitize_call_args, sanitize_result
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _extract_usage(result: Any) -> dict[str, int | None] | None:
|
|
9
|
+
if hasattr(result, "usage"):
|
|
10
|
+
usage = getattr(result, "usage")
|
|
11
|
+
elif isinstance(result, dict):
|
|
12
|
+
usage = result.get("usage")
|
|
13
|
+
else:
|
|
14
|
+
usage = None
|
|
15
|
+
|
|
16
|
+
if usage is None:
|
|
17
|
+
return None
|
|
18
|
+
|
|
19
|
+
input_tokens = getattr(usage, "input_tokens", None)
|
|
20
|
+
output_tokens = getattr(usage, "output_tokens", None)
|
|
21
|
+
|
|
22
|
+
if isinstance(usage, dict):
|
|
23
|
+
input_tokens = usage.get("input_tokens", usage.get("prompt_tokens"))
|
|
24
|
+
output_tokens = usage.get("output_tokens", usage.get("completion_tokens"))
|
|
25
|
+
|
|
26
|
+
if hasattr(usage, "prompt_tokens"):
|
|
27
|
+
input_tokens = input_tokens or getattr(usage, "prompt_tokens", None)
|
|
28
|
+
if hasattr(usage, "completion_tokens"):
|
|
29
|
+
output_tokens = output_tokens or getattr(usage, "completion_tokens", None)
|
|
30
|
+
|
|
31
|
+
return {
|
|
32
|
+
"input_tokens": input_tokens,
|
|
33
|
+
"output_tokens": output_tokens,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _extract_model(args: tuple[Any, ...], kwargs: dict[str, Any]) -> str | None:
|
|
38
|
+
if isinstance(kwargs.get("model"), str):
|
|
39
|
+
return kwargs["model"]
|
|
40
|
+
|
|
41
|
+
if args and isinstance(args[0], str):
|
|
42
|
+
return args[0]
|
|
43
|
+
|
|
44
|
+
return None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _wrap_method(
|
|
48
|
+
owner: Any,
|
|
49
|
+
method_name: str,
|
|
50
|
+
*,
|
|
51
|
+
provider: str,
|
|
52
|
+
) -> None:
|
|
53
|
+
if owner is None or not hasattr(owner, method_name):
|
|
54
|
+
return
|
|
55
|
+
|
|
56
|
+
original = getattr(owner, method_name)
|
|
57
|
+
tracer = get_tracer()
|
|
58
|
+
|
|
59
|
+
wrapped = tracer.wrap_callable(
|
|
60
|
+
original,
|
|
61
|
+
input_builder=sanitize_call_args,
|
|
62
|
+
output_builder=sanitize_result,
|
|
63
|
+
attribute_builder=lambda args, kwargs: {
|
|
64
|
+
"provider": provider,
|
|
65
|
+
"model": _extract_model(args, kwargs),
|
|
66
|
+
},
|
|
67
|
+
usage_builder=_extract_usage,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
setattr(owner, method_name, wrapped)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def instrument_openai(client: Any) -> Any:
|
|
74
|
+
chat = getattr(client, "chat", None)
|
|
75
|
+
completions = getattr(chat, "completions", None)
|
|
76
|
+
legacy_completions = getattr(client, "completions", None)
|
|
77
|
+
responses = getattr(client, "responses", None)
|
|
78
|
+
embeddings = getattr(client, "embeddings", None)
|
|
79
|
+
images = getattr(client, "images", None)
|
|
80
|
+
audio = getattr(client, "audio", None)
|
|
81
|
+
transcriptions = getattr(audio, "transcriptions", None)
|
|
82
|
+
translations = getattr(audio, "translations", None)
|
|
83
|
+
speech = getattr(audio, "speech", None)
|
|
84
|
+
|
|
85
|
+
_wrap_method(
|
|
86
|
+
completions,
|
|
87
|
+
"create",
|
|
88
|
+
provider="openai",
|
|
89
|
+
)
|
|
90
|
+
_wrap_method(
|
|
91
|
+
legacy_completions,
|
|
92
|
+
"create",
|
|
93
|
+
provider="openai",
|
|
94
|
+
)
|
|
95
|
+
_wrap_method(
|
|
96
|
+
responses,
|
|
97
|
+
"create",
|
|
98
|
+
provider="openai",
|
|
99
|
+
)
|
|
100
|
+
_wrap_method(
|
|
101
|
+
embeddings,
|
|
102
|
+
"create",
|
|
103
|
+
provider="openai",
|
|
104
|
+
)
|
|
105
|
+
_wrap_method(
|
|
106
|
+
images,
|
|
107
|
+
"generate",
|
|
108
|
+
provider="openai",
|
|
109
|
+
)
|
|
110
|
+
_wrap_method(
|
|
111
|
+
images,
|
|
112
|
+
"edit",
|
|
113
|
+
provider="openai",
|
|
114
|
+
)
|
|
115
|
+
_wrap_method(
|
|
116
|
+
images,
|
|
117
|
+
"create_variation",
|
|
118
|
+
provider="openai",
|
|
119
|
+
)
|
|
120
|
+
_wrap_method(
|
|
121
|
+
transcriptions,
|
|
122
|
+
"create",
|
|
123
|
+
provider="openai",
|
|
124
|
+
)
|
|
125
|
+
_wrap_method(
|
|
126
|
+
translations,
|
|
127
|
+
"create",
|
|
128
|
+
provider="openai",
|
|
129
|
+
)
|
|
130
|
+
_wrap_method(
|
|
131
|
+
speech,
|
|
132
|
+
"create",
|
|
133
|
+
provider="openai",
|
|
134
|
+
)
|
|
135
|
+
return client
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def instrument_anthropic(client: Any) -> Any:
|
|
139
|
+
messages = getattr(client, "messages", None)
|
|
140
|
+
beta = getattr(client, "beta", None)
|
|
141
|
+
beta_messages = getattr(beta, "messages", None)
|
|
142
|
+
|
|
143
|
+
_wrap_method(
|
|
144
|
+
messages,
|
|
145
|
+
"create",
|
|
146
|
+
provider="anthropic",
|
|
147
|
+
)
|
|
148
|
+
_wrap_method(
|
|
149
|
+
beta_messages,
|
|
150
|
+
"create",
|
|
151
|
+
provider="anthropic",
|
|
152
|
+
)
|
|
153
|
+
return client
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def instrument_google(client: Any) -> Any:
|
|
157
|
+
models = getattr(client, "models", None)
|
|
158
|
+
|
|
159
|
+
_wrap_method(
|
|
160
|
+
models,
|
|
161
|
+
"generate_content",
|
|
162
|
+
provider="google",
|
|
163
|
+
)
|
|
164
|
+
_wrap_method(
|
|
165
|
+
models,
|
|
166
|
+
"generate_images",
|
|
167
|
+
provider="google",
|
|
168
|
+
)
|
|
169
|
+
_wrap_method(
|
|
170
|
+
models,
|
|
171
|
+
"embed_content",
|
|
172
|
+
provider="google",
|
|
173
|
+
)
|
|
174
|
+
return client
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def instrument_client(client: Any) -> Any:
|
|
178
|
+
module_name = client.__class__.__module__.lower()
|
|
179
|
+
if "openai" in module_name:
|
|
180
|
+
return instrument_openai(client)
|
|
181
|
+
if "anthropic" in module_name:
|
|
182
|
+
return instrument_anthropic(client)
|
|
183
|
+
if "google" in module_name or "genai" in module_name:
|
|
184
|
+
return instrument_google(client)
|
|
185
|
+
return client
|
freesolo/sanitize.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import dataclasses
|
|
5
|
+
import hashlib
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Mapping, Sequence
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
DATA_URL_RE = re.compile(r"^data:([^;,]+)?(;base64)?,(.*)$", re.DOTALL)
|
|
11
|
+
MAX_STRING_LENGTH = 4000
|
|
12
|
+
MAX_COLLECTION_ITEMS = 50
|
|
13
|
+
MAX_DEPTH = 8
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _truncate_text(value: str) -> str:
|
|
17
|
+
if len(value) <= MAX_STRING_LENGTH:
|
|
18
|
+
return value
|
|
19
|
+
omitted = len(value) - MAX_STRING_LENGTH
|
|
20
|
+
return f"{value[:MAX_STRING_LENGTH]}... <truncated {omitted} chars>"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _estimate_base64_bytes(payload: str) -> int:
|
|
24
|
+
padding = payload.count("=")
|
|
25
|
+
return max(0, (len(payload) * 3) // 4 - padding)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _summarize_data_url(value: str) -> dict[str, Any]:
|
|
29
|
+
match = DATA_URL_RE.match(value)
|
|
30
|
+
if not match:
|
|
31
|
+
return {"type": "string", "value": _truncate_text(value)}
|
|
32
|
+
|
|
33
|
+
media_type = match.group(1) or "application/octet-stream"
|
|
34
|
+
is_base64 = bool(match.group(2))
|
|
35
|
+
data = match.group(3)
|
|
36
|
+
|
|
37
|
+
if is_base64:
|
|
38
|
+
approx_bytes = _estimate_base64_bytes(data)
|
|
39
|
+
digest = hashlib.sha256(data[:8192].encode("utf-8")).hexdigest()
|
|
40
|
+
return {
|
|
41
|
+
"type": "data_url",
|
|
42
|
+
"media_type": media_type,
|
|
43
|
+
"encoding": "base64",
|
|
44
|
+
"approx_bytes": approx_bytes,
|
|
45
|
+
"sha256_prefix": digest,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return {
|
|
49
|
+
"type": "data_url",
|
|
50
|
+
"media_type": media_type,
|
|
51
|
+
"encoding": "urlencoded",
|
|
52
|
+
"approx_chars": len(data),
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _summarize_bytes(value: bytes | bytearray | memoryview) -> dict[str, Any]:
|
|
57
|
+
raw = bytes(value)
|
|
58
|
+
return {
|
|
59
|
+
"type": "bytes",
|
|
60
|
+
"length": len(raw),
|
|
61
|
+
"sha256_prefix": hashlib.sha256(raw[:8192]).hexdigest(),
|
|
62
|
+
"base64_preview": base64.b64encode(raw[:24]).decode("ascii"),
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def sanitize_value(value: Any, *, _depth: int = 0) -> Any:
|
|
67
|
+
if _depth >= MAX_DEPTH:
|
|
68
|
+
return {"type": "max_depth_reached", "repr": repr(value)[:500]}
|
|
69
|
+
|
|
70
|
+
if value is None or isinstance(value, (bool, int, float)):
|
|
71
|
+
return value
|
|
72
|
+
|
|
73
|
+
if isinstance(value, str):
|
|
74
|
+
if value.startswith("data:"):
|
|
75
|
+
return _summarize_data_url(value)
|
|
76
|
+
return _truncate_text(value)
|
|
77
|
+
|
|
78
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
79
|
+
return _summarize_bytes(value)
|
|
80
|
+
|
|
81
|
+
if dataclasses.is_dataclass(value):
|
|
82
|
+
return sanitize_value(dataclasses.asdict(value), _depth=_depth + 1)
|
|
83
|
+
|
|
84
|
+
if hasattr(value, "model_dump") and callable(value.model_dump):
|
|
85
|
+
try:
|
|
86
|
+
return sanitize_value(value.model_dump(), _depth=_depth + 1)
|
|
87
|
+
except Exception:
|
|
88
|
+
return {"type": "model_dump_error", "repr": repr(value)[:500]}
|
|
89
|
+
|
|
90
|
+
if hasattr(value, "dict") and callable(value.dict):
|
|
91
|
+
try:
|
|
92
|
+
return sanitize_value(value.dict(), _depth=_depth + 1)
|
|
93
|
+
except Exception:
|
|
94
|
+
return {"type": "dict_error", "repr": repr(value)[:500]}
|
|
95
|
+
|
|
96
|
+
if isinstance(value, Mapping):
|
|
97
|
+
items = list(value.items())[:MAX_COLLECTION_ITEMS]
|
|
98
|
+
output: dict[str, Any] = {}
|
|
99
|
+
for key, item_value in items:
|
|
100
|
+
output[str(key)] = sanitize_value(item_value, _depth=_depth + 1)
|
|
101
|
+
if len(value) > MAX_COLLECTION_ITEMS:
|
|
102
|
+
output["__truncated_items__"] = len(value) - MAX_COLLECTION_ITEMS
|
|
103
|
+
return output
|
|
104
|
+
|
|
105
|
+
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
|
106
|
+
items = list(value)[:MAX_COLLECTION_ITEMS]
|
|
107
|
+
output = [sanitize_value(item, _depth=_depth + 1) for item in items]
|
|
108
|
+
if len(value) > MAX_COLLECTION_ITEMS:
|
|
109
|
+
output.append(
|
|
110
|
+
{
|
|
111
|
+
"type": "truncated_items",
|
|
112
|
+
"count": len(value) - MAX_COLLECTION_ITEMS,
|
|
113
|
+
}
|
|
114
|
+
)
|
|
115
|
+
return output
|
|
116
|
+
|
|
117
|
+
if hasattr(value, "__dict__"):
|
|
118
|
+
try:
|
|
119
|
+
return sanitize_value(vars(value), _depth=_depth + 1)
|
|
120
|
+
except Exception:
|
|
121
|
+
return {"type": "object", "repr": repr(value)[:500]}
|
|
122
|
+
|
|
123
|
+
return {"type": "repr", "value": repr(value)[:500]}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: freesolo
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Decorator-based LLM tracing package with OpenAI and Anthropic client instrumentation.
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: httpx>=0.27.0
|
|
7
|
+
Provides-Extra: dev
|
|
8
|
+
Requires-Dist: ruff>=0.11.0; extra == 'dev'
|
|
9
|
+
Provides-Extra: examples
|
|
10
|
+
Requires-Dist: anthropic>=0.40.0; extra == 'examples'
|
|
11
|
+
Requires-Dist: openai>=1.0.0; extra == 'examples'
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# freesolo
|
|
15
|
+
|
|
16
|
+
`freesolo` is a Python tracing package for the Reinforcement Labs app.
|
|
17
|
+
|
|
18
|
+
It is designed for the lowest-friction integration possible:
|
|
19
|
+
|
|
20
|
+
1. Install the package
|
|
21
|
+
2. Configure one endpoint + API key
|
|
22
|
+
3. Start a trace once
|
|
23
|
+
4. Add `@trace()` to your functions
|
|
24
|
+
5. Optionally wrap your OpenAI, Anthropic, or Google client for automatic provider spans
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
cd tracing
|
|
30
|
+
uv sync
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
To run the bundled example apps:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
cd tracing
|
|
37
|
+
uv sync --extra examples
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Environment
|
|
41
|
+
|
|
42
|
+
The package reads these environment variables by default:
|
|
43
|
+
|
|
44
|
+
- `FREESOLO_BASE_URL`
|
|
45
|
+
- `FREESOLO_API_KEY`
|
|
46
|
+
|
|
47
|
+
The frontend app must also have:
|
|
48
|
+
|
|
49
|
+
- `FREESOLO_API_KEY`
|
|
50
|
+
- `SUPABASE_SECRET_KEY`
|
|
51
|
+
|
|
52
|
+
The canonical database schema lives in `frontend/supabase/migrations/`.
|
|
53
|
+
There is no separate tracing-side migration folder because both the ingest API
|
|
54
|
+
and the Supabase project are owned by the frontend app in this repo.
|
|
55
|
+
|
|
56
|
+
Runnable examples live in [`examples/`](./examples/README.md). They will
|
|
57
|
+
autoload env vars from `frontend/.env`, `tracing/.env`, or `tracing/examples/.env`.
|
|
58
|
+
|
|
59
|
+
## Quickstart
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from openai import OpenAI
|
|
63
|
+
from freesolo import configure, instrument_openai, start_trace, trace
|
|
64
|
+
|
|
65
|
+
configure(
|
|
66
|
+
base_url="https://your-app.com",
|
|
67
|
+
api_key="freesolo_api_key",
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
client = instrument_openai(OpenAI())
|
|
71
|
+
|
|
72
|
+
@trace()
|
|
73
|
+
def run_agent(question: str) -> str:
|
|
74
|
+
result = client.chat.completions.create(
|
|
75
|
+
model="gpt-4.1",
|
|
76
|
+
messages=[
|
|
77
|
+
{"role": "user", "content": question},
|
|
78
|
+
],
|
|
79
|
+
)
|
|
80
|
+
return result.choices[0].message.content or ""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
with start_trace("support-agent-run"):
|
|
84
|
+
print(run_agent("How do I reset my password?"))
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## What Gets Stored
|
|
88
|
+
|
|
89
|
+
- Trace metadata if you explicitly pass it to `start_trace(..., metadata=...)`
|
|
90
|
+
- Spans from decorators
|
|
91
|
+
- OpenAI / Anthropic / Google request + response payloads
|
|
92
|
+
- Token usage when available
|
|
93
|
+
- Image inputs summarized safely instead of storing full base64 blobs
|
|
94
|
+
|
|
95
|
+
## Notes
|
|
96
|
+
|
|
97
|
+
- If you do nothing except add `@trace()`, you still get useful spans.
|
|
98
|
+
- Use `start_trace(trace_id=...)` when you want several decorated functions to land in one trace.
|
|
99
|
+
- If you also wrap the LLM client, you get dedicated provider/model spans.
|
|
100
|
+
- Delivery is best-effort by default. Trace ingestion failures do not break your app.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
freesolo/__init__.py,sha256=aGF-ixYiozElpTctZcn1Wz_m_2aB2nl6Yx-NSQKz8FM,464
|
|
2
|
+
freesolo/client.py,sha256=hsBrFTUKVZESjSoblmshnUnyeavwgWp-oZwaxIWWYx0,11284
|
|
3
|
+
freesolo/decorators.py,sha256=gI658oCLMvlsaGByUtl1VxIYnoqWwcDdc0BvnkM-n9E,2004
|
|
4
|
+
freesolo/providers.py,sha256=u8UstFMBZj7fLKZfEDanlqts0fe33GiHO0iqOIMWqMU,4534
|
|
5
|
+
freesolo/sanitize.py,sha256=tiHRuWne5E3_69taD59Ml00KV7oSEHeLHhlXODx10ug,4007
|
|
6
|
+
freesolo-0.1.0.dist-info/METADATA,sha256=SBfTt2ECaz6kOOxN1Xn8mzXE4jTvbypBfAjNkIW0n50,2709
|
|
7
|
+
freesolo-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
8
|
+
freesolo-0.1.0.dist-info/RECORD,,
|