synth-ai 0.2.2.dev0__py3-none-any.whl → 0.2.4.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synth_ai/cli/__init__.py +66 -0
- synth_ai/cli/balance.py +205 -0
- synth_ai/cli/calc.py +70 -0
- synth_ai/cli/demo.py +74 -0
- synth_ai/{cli.py → cli/legacy_root_backup.py} +60 -15
- synth_ai/cli/man.py +103 -0
- synth_ai/cli/recent.py +126 -0
- synth_ai/cli/root.py +184 -0
- synth_ai/cli/status.py +126 -0
- synth_ai/cli/traces.py +136 -0
- synth_ai/cli/watch.py +508 -0
- synth_ai/config/base_url.py +53 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/analyze_semantic_words_markdown.py +252 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/filter_traces_sft_duckdb_v2_backup.py +413 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/filter_traces_sft_turso.py +760 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/kick_off_ft_synth.py +34 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/test_crafter_react_agent_lm_synth.py +1740 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/test_crafter_react_agent_lm_synth_v2_backup.py +1318 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/filter_traces_sft_duckdb_v2_backup.py +386 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/filter_traces_sft_turso.py +580 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/run_rollouts_for_models_and_compare_v2_backup.py +1352 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/run_rollouts_for_models_and_compare_v3.py +4 -4
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/test_crafter_react_agent_openai_v2_backup.py +2551 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_trace_evaluation.py +1 -1
- synth_ai/environments/examples/crafter_classic/agent_demos/example_v3_usage.py +1 -1
- synth_ai/environments/examples/crafter_classic/agent_demos/old/traces/session_crafter_episode_16_15227b68-2906-416f-acc4-d6a9b4fa5828_20250725_001154.json +1363 -1
- synth_ai/environments/examples/crafter_classic/agent_demos/test_crafter_react_agent.py +3 -3
- synth_ai/environments/examples/crafter_classic/environment.py +1 -1
- synth_ai/environments/examples/crafter_custom/environment.py +1 -1
- synth_ai/environments/examples/enron/dataset/corbt___enron_emails_sample_questions/default/0.0.0/293c9fe8170037e01cc9cf5834e0cd5ef6f1a6bb/dataset_info.json +1 -0
- synth_ai/environments/examples/nethack/helpers/achievements.json +64 -0
- synth_ai/environments/examples/red/units/test_exploration_strategy.py +1 -1
- synth_ai/environments/examples/red/units/test_menu_bug_reproduction.py +5 -5
- synth_ai/environments/examples/red/units/test_movement_debug.py +2 -2
- synth_ai/environments/examples/red/units/test_retry_movement.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/available_envs.json +122 -0
- synth_ai/environments/examples/sokoban/verified_puzzles.json +54987 -0
- synth_ai/environments/service/core_routes.py +1 -1
- synth_ai/experimental/synth_oss.py +446 -0
- synth_ai/learning/core.py +21 -0
- synth_ai/learning/gateway.py +4 -0
- synth_ai/learning/prompts/gepa.py +0 -0
- synth_ai/learning/prompts/mipro.py +8 -0
- synth_ai/lm/__init__.py +3 -0
- synth_ai/lm/core/main.py +4 -0
- synth_ai/lm/core/main_v3.py +238 -122
- synth_ai/lm/core/vendor_clients.py +4 -0
- synth_ai/lm/provider_support/openai.py +11 -2
- synth_ai/lm/vendors/base.py +7 -0
- synth_ai/lm/vendors/openai_standard.py +339 -4
- synth_ai/lm/vendors/openai_standard_responses.py +243 -0
- synth_ai/lm/vendors/synth_client.py +155 -5
- synth_ai/lm/warmup.py +54 -17
- synth_ai/tracing/__init__.py +18 -0
- synth_ai/tracing_v1/__init__.py +29 -14
- synth_ai/tracing_v3/__init__.py +2 -2
- synth_ai/tracing_v3/abstractions.py +62 -17
- synth_ai/tracing_v3/config.py +13 -7
- synth_ai/tracing_v3/db_config.py +6 -6
- synth_ai/tracing_v3/hooks.py +1 -1
- synth_ai/tracing_v3/llm_call_record_helpers.py +350 -0
- synth_ai/tracing_v3/lm_call_record_abstractions.py +257 -0
- synth_ai/tracing_v3/session_tracer.py +5 -5
- synth_ai/tracing_v3/tests/test_concurrent_operations.py +1 -1
- synth_ai/tracing_v3/tests/test_llm_call_records.py +672 -0
- synth_ai/tracing_v3/tests/test_session_tracer.py +43 -9
- synth_ai/tracing_v3/tests/test_turso_manager.py +1 -1
- synth_ai/tracing_v3/turso/manager.py +18 -11
- synth_ai/tracing_v3/turso/models.py +1 -0
- synth_ai/tui/__main__.py +13 -0
- synth_ai/tui/dashboard.py +329 -0
- synth_ai/v0/tracing/__init__.py +0 -0
- synth_ai/{tracing → v0/tracing}/base_client.py +3 -3
- synth_ai/{tracing → v0/tracing}/client_manager.py +1 -1
- synth_ai/{tracing → v0/tracing}/context.py +1 -1
- synth_ai/{tracing → v0/tracing}/decorators.py +11 -11
- synth_ai/v0/tracing/events/__init__.py +0 -0
- synth_ai/{tracing → v0/tracing}/events/manage.py +4 -4
- synth_ai/{tracing → v0/tracing}/events/scope.py +6 -6
- synth_ai/{tracing → v0/tracing}/events/store.py +3 -3
- synth_ai/{tracing → v0/tracing}/immediate_client.py +6 -6
- synth_ai/{tracing → v0/tracing}/log_client_base.py +2 -2
- synth_ai/{tracing → v0/tracing}/retry_queue.py +3 -3
- synth_ai/{tracing → v0/tracing}/trackers.py +2 -2
- synth_ai/{tracing → v0/tracing}/upload.py +4 -4
- synth_ai/v0/tracing_v1/__init__.py +16 -0
- synth_ai/{tracing_v1 → v0/tracing_v1}/base_client.py +3 -3
- synth_ai/{tracing_v1 → v0/tracing_v1}/client_manager.py +1 -1
- synth_ai/{tracing_v1 → v0/tracing_v1}/context.py +1 -1
- synth_ai/{tracing_v1 → v0/tracing_v1}/decorators.py +11 -11
- synth_ai/v0/tracing_v1/events/__init__.py +0 -0
- synth_ai/{tracing_v1 → v0/tracing_v1}/events/manage.py +4 -4
- synth_ai/{tracing_v1 → v0/tracing_v1}/events/scope.py +6 -6
- synth_ai/{tracing_v1 → v0/tracing_v1}/events/store.py +3 -3
- synth_ai/{tracing_v1 → v0/tracing_v1}/immediate_client.py +6 -6
- synth_ai/{tracing_v1 → v0/tracing_v1}/log_client_base.py +2 -2
- synth_ai/{tracing_v1 → v0/tracing_v1}/retry_queue.py +3 -3
- synth_ai/{tracing_v1 → v0/tracing_v1}/trackers.py +2 -2
- synth_ai/{tracing_v1 → v0/tracing_v1}/upload.py +4 -4
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.4.dev2.dist-info}/METADATA +100 -5
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.4.dev2.dist-info}/RECORD +115 -75
- /synth_ai/{tracing/events/__init__.py → compound/cais.py} +0 -0
- /synth_ai/{tracing_v1/events/__init__.py → environments/examples/crafter_classic/debug_translation.py} +0 -0
- /synth_ai/{tracing → v0/tracing}/abstractions.py +0 -0
- /synth_ai/{tracing → v0/tracing}/config.py +0 -0
- /synth_ai/{tracing → v0/tracing}/local.py +0 -0
- /synth_ai/{tracing → v0/tracing}/utils.py +0 -0
- /synth_ai/{tracing_v1 → v0/tracing_v1}/abstractions.py +0 -0
- /synth_ai/{tracing_v1 → v0/tracing_v1}/config.py +0 -0
- /synth_ai/{tracing_v1 → v0/tracing_v1}/local.py +0 -0
- /synth_ai/{tracing_v1 → v0/tracing_v1}/utils.py +0 -0
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.4.dev2.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.4.dev2.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.4.dev2.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.4.dev2.dist-info}/top_level.txt +0 -0
@@ -47,6 +47,73 @@ class AsyncSynthClient:
|
|
47
47
|
},
|
48
48
|
)
|
49
49
|
|
50
|
+
async def responses_create(
|
51
|
+
self,
|
52
|
+
model: str,
|
53
|
+
messages: List[Dict[str, Any]],
|
54
|
+
previous_response_id: Optional[str] = None,
|
55
|
+
tools: Optional[List[Dict[str, Any]]] = None,
|
56
|
+
tool_choice: Optional[Union[str, Dict[str, Any]]] = "auto",
|
57
|
+
**kwargs,
|
58
|
+
) -> Dict[str, Any]:
|
59
|
+
"""
|
60
|
+
Create response using Synth Responses API.
|
61
|
+
|
62
|
+
Args:
|
63
|
+
model: Model identifier
|
64
|
+
messages: List of message dicts with 'role' and 'content'
|
65
|
+
previous_response_id: Optional ID of previous response for thread management
|
66
|
+
tools: List of available tools
|
67
|
+
tool_choice: How to choose tools
|
68
|
+
**kwargs: Additional parameters
|
69
|
+
|
70
|
+
Returns:
|
71
|
+
Responses API-compatible response dict
|
72
|
+
"""
|
73
|
+
await self._ensure_client()
|
74
|
+
|
75
|
+
# Build payload for Responses API
|
76
|
+
payload = {
|
77
|
+
"model": model,
|
78
|
+
"messages": messages,
|
79
|
+
}
|
80
|
+
|
81
|
+
# Add optional parameters
|
82
|
+
if previous_response_id is not None:
|
83
|
+
payload["previous_response_id"] = previous_response_id
|
84
|
+
if tools is not None:
|
85
|
+
payload["tools"] = tools
|
86
|
+
payload["tool_choice"] = tool_choice
|
87
|
+
|
88
|
+
# Add any additional kwargs
|
89
|
+
payload.update(kwargs)
|
90
|
+
|
91
|
+
# Retry logic
|
92
|
+
for attempt in range(self.config.max_retries):
|
93
|
+
try:
|
94
|
+
url = f"{self.config.get_base_url_without_v1()}/v1/responses"
|
95
|
+
response = await self._client.post(url, json=payload)
|
96
|
+
|
97
|
+
if response.status_code == 200:
|
98
|
+
return response.json()
|
99
|
+
|
100
|
+
# Handle rate limits with exponential backoff
|
101
|
+
if response.status_code == 429:
|
102
|
+
wait_time = 2**attempt
|
103
|
+
await asyncio.sleep(wait_time)
|
104
|
+
continue
|
105
|
+
|
106
|
+
# Other errors
|
107
|
+
response.raise_for_status()
|
108
|
+
|
109
|
+
except Exception as e:
|
110
|
+
if attempt == self.config.max_retries - 1:
|
111
|
+
logger.error(f"Failed after {self.config.max_retries} attempts: {e}")
|
112
|
+
raise
|
113
|
+
await asyncio.sleep(2**attempt)
|
114
|
+
|
115
|
+
raise Exception(f"Failed to create response after {self.config.max_retries} attempts")
|
116
|
+
|
50
117
|
async def chat_completions_create(
|
51
118
|
self,
|
52
119
|
model: str,
|
@@ -118,12 +185,32 @@ class AsyncSynthClient:
|
|
118
185
|
# Retry logic
|
119
186
|
for attempt in range(self.config.max_retries):
|
120
187
|
try:
|
121
|
-
|
122
|
-
|
123
|
-
)
|
124
|
-
|
188
|
+
url = f"{self.config.get_base_url_without_v1()}/v1/chat/completions"
|
189
|
+
print(f"🔍 SYNTH DEBUG: Making request to URL: {url}")
|
190
|
+
print(f"🔍 SYNTH DEBUG: Payload keys: {list(payload.keys())}")
|
191
|
+
if 'tools' in payload:
|
192
|
+
print(f"🔍 SYNTH DEBUG: Tools in payload: {len(payload['tools'])} tools")
|
193
|
+
print(f"🔍 SYNTH DEBUG: First tool: {json.dumps(payload['tools'][0], indent=2)}")
|
194
|
+
|
195
|
+
response = await self._client.post(url, json=payload)
|
196
|
+
|
197
|
+
print(f"🔍 SYNTH DEBUG: Response status: {response.status_code}")
|
198
|
+
|
125
199
|
if response.status_code == 200:
|
126
|
-
|
200
|
+
result = response.json()
|
201
|
+
print(f"🔍 SYNTH DEBUG: Response keys: {list(result.keys())}")
|
202
|
+
if 'choices' in result and result['choices']:
|
203
|
+
choice = result['choices'][0]
|
204
|
+
print(f"🔍 SYNTH DEBUG: Choice keys: {list(choice.keys())}")
|
205
|
+
if 'message' in choice:
|
206
|
+
message = choice['message']
|
207
|
+
print(f"🔍 SYNTH DEBUG: Message keys: {list(message.keys())}")
|
208
|
+
if 'tool_calls' in message:
|
209
|
+
print(f"🔍 SYNTH DEBUG: Tool calls: {message['tool_calls']}")
|
210
|
+
else:
|
211
|
+
print(f"🔍 SYNTH DEBUG: No tool_calls in message")
|
212
|
+
print(f"🔍 SYNTH DEBUG: Message content: {message.get('content', 'N/A')[:200]}...")
|
213
|
+
return result
|
127
214
|
|
128
215
|
# Handle rate limits with exponential backoff
|
129
216
|
if response.status_code == 429:
|
@@ -191,6 +278,69 @@ class SyncSynthClient:
|
|
191
278
|
},
|
192
279
|
)
|
193
280
|
|
281
|
+
def responses_create(
|
282
|
+
self,
|
283
|
+
model: str,
|
284
|
+
messages: List[Dict[str, Any]],
|
285
|
+
previous_response_id: Optional[str] = None,
|
286
|
+
tools: Optional[List[Dict[str, Any]]] = None,
|
287
|
+
tool_choice: Optional[Union[str, Dict[str, Any]]] = "auto",
|
288
|
+
**kwargs,
|
289
|
+
) -> Dict[str, Any]:
|
290
|
+
"""
|
291
|
+
Create response using Synth Responses API (sync version).
|
292
|
+
|
293
|
+
See AsyncSynthClient.responses_create for full parameter documentation.
|
294
|
+
"""
|
295
|
+
self._ensure_client()
|
296
|
+
|
297
|
+
# Build payload for Responses API
|
298
|
+
payload = {
|
299
|
+
"model": model,
|
300
|
+
"messages": messages,
|
301
|
+
}
|
302
|
+
|
303
|
+
# Add optional parameters
|
304
|
+
if previous_response_id is not None:
|
305
|
+
payload["previous_response_id"] = previous_response_id
|
306
|
+
if tools is not None:
|
307
|
+
payload["tools"] = tools
|
308
|
+
payload["tool_choice"] = tool_choice
|
309
|
+
|
310
|
+
# Add any additional kwargs
|
311
|
+
payload.update(kwargs)
|
312
|
+
|
313
|
+
# Retry logic
|
314
|
+
for attempt in range(self.config.max_retries):
|
315
|
+
try:
|
316
|
+
response = self._client.post(
|
317
|
+
f"{self.config.get_base_url_without_v1()}/v1/responses", json=payload
|
318
|
+
)
|
319
|
+
|
320
|
+
if response.status_code == 200:
|
321
|
+
return response.json()
|
322
|
+
|
323
|
+
# Handle rate limits
|
324
|
+
if response.status_code == 429:
|
325
|
+
wait_time = 2**attempt
|
326
|
+
logger.warning(f"Rate limited, waiting {wait_time}s...")
|
327
|
+
import time
|
328
|
+
time.sleep(wait_time)
|
329
|
+
continue
|
330
|
+
|
331
|
+
# Other errors
|
332
|
+
error_msg = f"API error {response.status_code}: {response.text}"
|
333
|
+
logger.error(error_msg)
|
334
|
+
raise Exception(error_msg)
|
335
|
+
|
336
|
+
except httpx.TimeoutException:
|
337
|
+
if attempt < self.config.max_retries - 1:
|
338
|
+
logger.warning(f"Timeout on attempt {attempt + 1}, retrying...")
|
339
|
+
continue
|
340
|
+
raise
|
341
|
+
|
342
|
+
raise Exception(f"Failed after {self.config.max_retries} attempts")
|
343
|
+
|
194
344
|
def chat_completions_create(
|
195
345
|
self, model: str, messages: List[Dict[str, Any]], **kwargs
|
196
346
|
) -> Dict[str, Any]:
|
synth_ai/lm/warmup.py
CHANGED
@@ -6,6 +6,8 @@ Handles model preloading and warmup polling.
|
|
6
6
|
import httpx
|
7
7
|
import asyncio
|
8
8
|
import logging
|
9
|
+
import sys
|
10
|
+
import time
|
9
11
|
from typing import Optional, Dict, Any
|
10
12
|
from datetime import datetime, timedelta
|
11
13
|
from .config import SynthConfig
|
@@ -46,9 +48,10 @@ _warmup_status = WarmupStatus()
|
|
46
48
|
async def warmup_synth_model(
|
47
49
|
model_name: str,
|
48
50
|
config: Optional[SynthConfig] = None,
|
49
|
-
max_attempts: int =
|
51
|
+
max_attempts: Optional[int] = None,
|
50
52
|
force: bool = False,
|
51
53
|
verbose: bool = True,
|
54
|
+
gpu_preference: Optional[str] = None,
|
52
55
|
) -> bool:
|
53
56
|
"""
|
54
57
|
Warm up a model on the Synth backend using fire-and-forget approach.
|
@@ -73,6 +76,8 @@ async def warmup_synth_model(
|
|
73
76
|
|
74
77
|
async with httpx.AsyncClient() as client:
|
75
78
|
headers = {"Authorization": f"Bearer {config.api_key}"}
|
79
|
+
if gpu_preference:
|
80
|
+
headers["X-GPU-Preference"] = gpu_preference
|
76
81
|
|
77
82
|
# Step 1: Start warmup (fire and forget)
|
78
83
|
try:
|
@@ -84,8 +89,11 @@ async def warmup_synth_model(
|
|
84
89
|
|
85
90
|
if response.status_code == 200:
|
86
91
|
response_data = response.json()
|
87
|
-
if response_data.get("status") in ["warming", "already_warming"
|
92
|
+
if response_data.get("status") in ["warming", "already_warming"]:
|
88
93
|
pass
|
94
|
+
elif response_data.get("status") == "already_warmed":
|
95
|
+
_warmup_status.mark_warm(model_name)
|
96
|
+
return True
|
89
97
|
else:
|
90
98
|
logger.warning(f"Unexpected warmup response: {response_data}")
|
91
99
|
else:
|
@@ -98,8 +106,13 @@ async def warmup_synth_model(
|
|
98
106
|
logger.warning(f"Warmup start failed: {e}")
|
99
107
|
return False
|
100
108
|
|
101
|
-
# Step 2: Poll status until ready
|
102
|
-
|
109
|
+
# Step 2: Poll status until ready (indefinite by default)
|
110
|
+
spinner = "|/-\\"
|
111
|
+
spin_idx = 0
|
112
|
+
start_time = time.time()
|
113
|
+
attempt = 0
|
114
|
+
while True:
|
115
|
+
attempt += 1
|
103
116
|
try:
|
104
117
|
response = await client.get(
|
105
118
|
f"{config.get_base_url_without_v1()}/warmup/status/{model_name}",
|
@@ -113,34 +126,58 @@ async def warmup_synth_model(
|
|
113
126
|
|
114
127
|
if status == "warmed":
|
115
128
|
_warmup_status.mark_warm(model_name)
|
129
|
+
# Final spinner line as success
|
130
|
+
elapsed = int(time.time() - start_time)
|
131
|
+
sys.stdout.write(f"\r✅ Warmed {model_name} in {elapsed}s \n")
|
132
|
+
sys.stdout.flush()
|
116
133
|
return True
|
117
134
|
elif status == "failed":
|
118
135
|
error = status_data.get("error", "Unknown error")
|
119
136
|
logger.error(f"❌ Warmup failed for {model_name}: {error}")
|
137
|
+
sys.stdout.write(f"\r❌ Warmup failed: {error} \n")
|
138
|
+
sys.stdout.flush()
|
120
139
|
return False
|
121
|
-
elif status == "warming":
|
122
|
-
# Still warming up, continue polling
|
123
|
-
pass
|
124
|
-
elif status == "not_started":
|
125
|
-
# Warmup hasn't started yet, continue polling
|
126
|
-
pass
|
127
140
|
else:
|
128
|
-
|
141
|
+
# Treat unknown statuses (e.g., "cold") as still warming
|
142
|
+
elapsed = int(time.time() - start_time)
|
143
|
+
wheel = spinner[spin_idx % len(spinner)]
|
144
|
+
spin_idx += 1
|
145
|
+
label = status or "pending"
|
146
|
+
sys.stdout.write(
|
147
|
+
f"\r⏳ Warming {model_name} [{wheel}] status={label} elapsed={elapsed}s"
|
148
|
+
)
|
149
|
+
sys.stdout.flush()
|
129
150
|
|
130
151
|
# Short sleep between status checks
|
131
152
|
await asyncio.sleep(2.0)
|
132
153
|
|
133
154
|
except httpx.TimeoutException:
|
134
|
-
|
135
|
-
|
155
|
+
# Continue polling; update spinner line
|
156
|
+
elapsed = int(time.time() - start_time)
|
157
|
+
wheel = spinner[spin_idx % len(spinner)]
|
158
|
+
spin_idx += 1
|
159
|
+
sys.stdout.write(
|
160
|
+
f"\r⏳ Warming {model_name} [{wheel}] status=timeout elapsed={elapsed}s"
|
161
|
+
)
|
162
|
+
sys.stdout.flush()
|
136
163
|
await asyncio.sleep(1.0)
|
137
164
|
except Exception as e:
|
138
|
-
|
139
|
-
|
165
|
+
# Continue polling; update spinner line with error label
|
166
|
+
elapsed = int(time.time() - start_time)
|
167
|
+
wheel = spinner[spin_idx % len(spinner)]
|
168
|
+
spin_idx += 1
|
169
|
+
sys.stdout.write(
|
170
|
+
f"\r⏳ Warming {model_name} [{wheel}] status=error elapsed={elapsed}s"
|
171
|
+
)
|
172
|
+
sys.stdout.flush()
|
140
173
|
await asyncio.sleep(1.0)
|
141
174
|
|
142
|
-
|
143
|
-
|
175
|
+
# Optional max_attempts for callers who want a cap
|
176
|
+
if max_attempts is not None and attempt >= max_attempts:
|
177
|
+
logger.error(f"Failed to warm up {model_name} after {max_attempts} status checks")
|
178
|
+
sys.stdout.write("\n")
|
179
|
+
sys.stdout.flush()
|
180
|
+
return False
|
144
181
|
|
145
182
|
|
146
183
|
def get_warmup_status() -> WarmupStatus:
|
synth_ai/tracing/__init__.py
CHANGED
@@ -0,0 +1,18 @@
|
|
1
|
+
import sys as _sys
|
2
|
+
import importlib as _importlib
|
3
|
+
|
4
|
+
_pkg = _importlib.import_module('synth_ai.v0.tracing')
|
5
|
+
_sys.modules[__name__] = _pkg
|
6
|
+
|
7
|
+
_SUBMODULES = [
|
8
|
+
'abstractions', 'base_client', 'client_manager', 'config', 'context',
|
9
|
+
'decorators', 'immediate_client', 'local', 'log_client_base', 'retry_queue',
|
10
|
+
'trackers', 'upload', 'utils'
|
11
|
+
]
|
12
|
+
for _m in _SUBMODULES:
|
13
|
+
_sys.modules[f'{__name__}.{_m}'] = _importlib.import_module(f'synth_ai.v0.tracing.{_m}')
|
14
|
+
|
15
|
+
_events_pkg = _importlib.import_module('synth_ai.v0.tracing.events')
|
16
|
+
_sys.modules[f'{__name__}.events'] = _events_pkg
|
17
|
+
for _m in ['manage', 'scope', 'store']:
|
18
|
+
_sys.modules[f'{__name__}.events.{_m}'] = _importlib.import_module(f'synth_ai.v0.tracing.events.{_m}')
|
synth_ai/tracing_v1/__init__.py
CHANGED
@@ -1,16 +1,31 @@
|
|
1
|
-
import
|
1
|
+
import sys as _sys
|
2
|
+
import importlib as _importlib
|
2
3
|
|
3
|
-
#
|
4
|
-
|
5
|
-
|
6
|
-
"Please use synth_ai.tracing_v2 instead. "
|
7
|
-
"Backend upload functionality is no longer supported in v1.",
|
8
|
-
DeprecationWarning,
|
9
|
-
stacklevel=2,
|
10
|
-
)
|
4
|
+
# Forward top-level package
|
5
|
+
_pkg = _importlib.import_module('synth_ai.v0.tracing_v1')
|
6
|
+
_sys.modules[__name__] = _pkg
|
11
7
|
|
12
|
-
#
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
8
|
+
# Explicitly forward submodules so `synth_ai.tracing_v1.X` works
|
9
|
+
_SUBMODULES = [
|
10
|
+
'abstractions',
|
11
|
+
'base_client',
|
12
|
+
'client_manager',
|
13
|
+
'config',
|
14
|
+
'context',
|
15
|
+
'decorators',
|
16
|
+
'immediate_client',
|
17
|
+
'local',
|
18
|
+
'log_client_base',
|
19
|
+
'retry_queue',
|
20
|
+
'trackers',
|
21
|
+
'upload',
|
22
|
+
'utils',
|
23
|
+
]
|
24
|
+
for _m in _SUBMODULES:
|
25
|
+
_sys.modules[f'{__name__}.{_m}'] = _importlib.import_module(f'synth_ai.v0.tracing_v1.{_m}')
|
26
|
+
|
27
|
+
# Forward events package and its submodules
|
28
|
+
_events_pkg = _importlib.import_module('synth_ai.v0.tracing_v1.events')
|
29
|
+
_sys.modules[f'{__name__}.events'] = _events_pkg
|
30
|
+
for _m in ['manage', 'scope', 'store']:
|
31
|
+
_sys.modules[f'{__name__}.events.{_m}'] = _importlib.import_module(f'synth_ai.v0.tracing_v1.events.{_m}')
|
synth_ai/tracing_v3/__init__.py
CHANGED
@@ -77,7 +77,7 @@ from .abstractions import (
|
|
77
77
|
BaseEvent,
|
78
78
|
RuntimeEvent,
|
79
79
|
EnvironmentEvent,
|
80
|
-
|
80
|
+
SessionEventMarkovBlanketMessage,
|
81
81
|
TimeRecord,
|
82
82
|
)
|
83
83
|
from .config import TursoConfig
|
@@ -89,7 +89,7 @@ __all__ = [
|
|
89
89
|
"BaseEvent",
|
90
90
|
"RuntimeEvent",
|
91
91
|
"EnvironmentEvent",
|
92
|
-
"
|
92
|
+
"SessionEventMarkovBlanketMessage",
|
93
93
|
"TimeRecord",
|
94
94
|
"TursoConfig",
|
95
95
|
]
|
@@ -18,13 +18,27 @@ Session Structure:
|
|
18
18
|
- SessionTrace: Top-level container for a complete session
|
19
19
|
- SessionTimeStep: Logical steps within a session (e.g., conversation turns)
|
20
20
|
- Events: Individual events that occurred during the timestep
|
21
|
-
- Messages:
|
21
|
+
- Messages: Information passed between subsystems (user, agent, runtime, environments)
|
22
|
+
|
23
|
+
Concepts:
|
24
|
+
---------
|
25
|
+
- Events capture something that happened inside a subsystem. They may or may not be externally
|
26
|
+
visible. Examples include an LLM API call (LMCAISEvent), a tool selection (RuntimeEvent), or
|
27
|
+
a tool execution outcome (EnvironmentEvent).
|
28
|
+
|
29
|
+
- Messages represent information transmitted between subsystems within the session.
|
30
|
+
Messages are used to record communications like: a user sending input to the agent,
|
31
|
+
the agent/runtime sending a tool invocation to an environment, the environment sending a
|
32
|
+
tool result back, and the agent sending a reply to the user. Do not confuse these with
|
33
|
+
provider-specific LLM API "messages" (prompt formatting) — those belong inside an LMCAISEvent
|
34
|
+
as part of its input/output content, not as SessionEventMessages.
|
22
35
|
"""
|
23
36
|
|
24
37
|
from __future__ import annotations
|
25
38
|
from dataclasses import dataclass, field, asdict
|
26
39
|
from datetime import datetime
|
27
40
|
from typing import Any, Dict, List, Optional
|
41
|
+
from .lm_call_record_abstractions import LLMCallRecord
|
28
42
|
|
29
43
|
|
30
44
|
@dataclass
|
@@ -46,18 +60,39 @@ class TimeRecord:
|
|
46
60
|
|
47
61
|
|
48
62
|
@dataclass
|
49
|
-
class
|
50
|
-
"""Message
|
63
|
+
class SessionEventMarkovBlanketMessage:
|
64
|
+
"""Message crossing Markov blanket boundaries between systems in a session.
|
65
|
+
|
66
|
+
IMPORTANT: This represents information transfer BETWEEN distinct systems/subsystems,
|
67
|
+
where each system is conceptualized as having a Markov blanket that separates its
|
68
|
+
internal states from the external environment. These messages cross those boundaries.
|
69
|
+
|
70
|
+
This is NOT for chat messages within an LLM conversation (those belong in LLMCallRecord).
|
71
|
+
Instead, this captures inter-system communication such as:
|
72
|
+
- Human -> Agent system (user providing instructions)
|
73
|
+
- Agent -> Runtime (agent deciding on an action)
|
74
|
+
- Runtime -> Environment (executing a tool/action)
|
75
|
+
- Environment -> Runtime (returning results)
|
76
|
+
- Runtime -> Agent (passing back results)
|
77
|
+
- Agent -> Human (final response)
|
51
78
|
|
52
|
-
|
53
|
-
|
79
|
+
Each system maintains its own internal state and processing, but can only influence
|
80
|
+
other systems through these explicit boundary-crossing messages. This follows the
|
81
|
+
Free Energy Principle where systems minimize surprise by maintaining boundaries.
|
54
82
|
|
55
83
|
Attributes:
|
56
|
-
content: The actual message content (text, JSON, etc.)
|
57
|
-
message_type: Type
|
58
|
-
time_record: Timing information for the
|
59
|
-
metadata:
|
60
|
-
|
84
|
+
content: The actual message content crossing the boundary (text, JSON, etc.)
|
85
|
+
message_type: Type of boundary crossing (e.g., 'observation', 'action', 'result')
|
86
|
+
time_record: Timing information for the boundary crossing
|
87
|
+
metadata: Boundary crossing metadata. Recommended keys:
|
88
|
+
- 'step_id': Timestep identifier
|
89
|
+
- 'from_system_instance_id': UUID of the sending system
|
90
|
+
- 'to_system_instance_id': UUID of the receiving system
|
91
|
+
- 'from_system_role': Role of sender (e.g., 'human', 'agent', 'runtime', 'environment')
|
92
|
+
- 'to_system_role': Role of receiver
|
93
|
+
- 'boundary_type': Type of Markov blanket boundary being crossed
|
94
|
+
- 'call_id': Correlate request/response pairs across boundaries
|
95
|
+
- 'causal_influence': Direction of causal flow
|
61
96
|
"""
|
62
97
|
|
63
98
|
content: str
|
@@ -70,8 +105,9 @@ class SessionEventMessage:
|
|
70
105
|
class BaseEvent:
|
71
106
|
"""Base class for all event types.
|
72
107
|
|
73
|
-
This is the foundation for all events in the tracing system. Every event
|
74
|
-
|
108
|
+
This is the foundation for all events in the tracing system. Every event must
|
109
|
+
have a system identifier and timing information. Events are intra-system facts
|
110
|
+
(they occur within a subsystem) and are not necessarily direct communications.
|
75
111
|
|
76
112
|
Attributes:
|
77
113
|
system_instance_id: Identifier for the system/component that generated
|
@@ -95,8 +131,10 @@ class BaseEvent:
|
|
95
131
|
class RuntimeEvent(BaseEvent):
|
96
132
|
"""Event from runtime system.
|
97
133
|
|
98
|
-
Captures events from the AI system's runtime, typically representing
|
99
|
-
|
134
|
+
Captures events from the AI system's runtime, typically representing decisions
|
135
|
+
or actions taken by the system (e.g., selecting a tool with arguments).
|
136
|
+
Use paired SessionEventMessages to record the communication of this choice to
|
137
|
+
the environment.
|
100
138
|
|
101
139
|
Attributes:
|
102
140
|
actions: List of action identifiers or indices. The interpretation
|
@@ -111,7 +149,9 @@ class RuntimeEvent(BaseEvent):
|
|
111
149
|
class EnvironmentEvent(BaseEvent):
|
112
150
|
"""Event from environment.
|
113
151
|
|
114
|
-
Captures feedback from the environment in response to system actions.
|
152
|
+
Captures feedback from the environment in response to system actions (e.g.,
|
153
|
+
command output, exit codes, observations). Use a paired SessionEventMessage
|
154
|
+
to record the environment-to-agent communication of the result.
|
115
155
|
Follows the Gymnasium/OpenAI Gym convention for compatibility.
|
116
156
|
|
117
157
|
Attributes:
|
@@ -135,6 +175,8 @@ class LMCAISEvent(BaseEvent):
|
|
135
175
|
|
136
176
|
CAIS (Claude AI System) events capture detailed information about LLM calls,
|
137
177
|
including performance metrics, cost tracking, and distributed tracing support.
|
178
|
+
Treat provider-specific prompt/completion structures as part of this event's
|
179
|
+
data. Do not emit them as SessionEventMessages.
|
138
180
|
|
139
181
|
Attributes:
|
140
182
|
model_name: The specific model used (e.g., 'gpt-4', 'claude-3-opus')
|
@@ -148,6 +190,8 @@ class LMCAISEvent(BaseEvent):
|
|
148
190
|
trace_id: OpenTelemetry compatible trace identifier
|
149
191
|
system_state_before: State snapshot before the LLM call
|
150
192
|
system_state_after: State snapshot after the LLM call
|
193
|
+
call_records: List of normalized LLM call records capturing request/response
|
194
|
+
details (messages, tool calls/results, usage, params, etc.).
|
151
195
|
"""
|
152
196
|
|
153
197
|
model_name: str = ""
|
@@ -161,6 +205,7 @@ class LMCAISEvent(BaseEvent):
|
|
161
205
|
trace_id: Optional[str] = None
|
162
206
|
system_state_before: Optional[Dict[str, Any]] = None
|
163
207
|
system_state_after: Optional[Dict[str, Any]] = None
|
208
|
+
call_records: List[LLMCallRecord] = field(default_factory=list)
|
164
209
|
|
165
210
|
|
166
211
|
@dataclass
|
@@ -188,7 +233,7 @@ class SessionTimeStep:
|
|
188
233
|
timestamp: datetime = field(default_factory=datetime.utcnow)
|
189
234
|
turn_number: Optional[int] = None
|
190
235
|
events: List[BaseEvent] = field(default_factory=list)
|
191
|
-
|
236
|
+
markov_blanket_messages: List[SessionEventMarkovBlanketMessage] = field(default_factory=list)
|
192
237
|
step_metadata: Dict[str, Any] = field(default_factory=dict)
|
193
238
|
completed_at: Optional[datetime] = None
|
194
239
|
|
@@ -222,7 +267,7 @@ class SessionTrace:
|
|
222
267
|
created_at: datetime = field(default_factory=datetime.utcnow)
|
223
268
|
session_time_steps: List[SessionTimeStep] = field(default_factory=list)
|
224
269
|
event_history: List[BaseEvent] = field(default_factory=list)
|
225
|
-
|
270
|
+
markov_blanket_message_history: List[SessionEventMarkovBlanketMessage] = field(default_factory=list)
|
226
271
|
metadata: Dict[str, Any] = field(default_factory=dict)
|
227
272
|
session_metadata: Optional[List[Dict[str, Any]]] = None
|
228
273
|
|
synth_ai/tracing_v3/config.py
CHANGED
@@ -9,15 +9,21 @@ class TursoConfig:
|
|
9
9
|
"""Configuration for Turso/sqld connection."""
|
10
10
|
|
11
11
|
# Default values matching serve.sh
|
12
|
-
DEFAULT_DB_FILE = "synth_ai.db"
|
12
|
+
DEFAULT_DB_FILE = "traces/v3/synth_ai.db"
|
13
13
|
DEFAULT_HTTP_PORT = 8080
|
14
14
|
|
15
15
|
# Local embedded database for async SQLAlchemy
|
16
|
-
#
|
17
|
-
|
18
|
-
"
|
19
|
-
|
20
|
-
|
16
|
+
# Resolve to the actual SQLite file used by sqld if the base path is a directory
|
17
|
+
def _resolve_sqlite_db_url() -> str: # type: ignore[no-redef]
|
18
|
+
base_path = os.path.abspath(os.getenv("SQLD_DB_PATH", "traces/v3/synth_ai.db"))
|
19
|
+
# If sqld is managing this DB, the real SQLite file lives under dbs/default/data
|
20
|
+
candidate = os.path.join(base_path, "dbs", "default", "data")
|
21
|
+
if os.path.isdir(base_path) and os.path.exists(candidate):
|
22
|
+
return f"sqlite+aiosqlite:///{candidate}"
|
23
|
+
return f"sqlite+aiosqlite:///{base_path}"
|
24
|
+
|
25
|
+
# Use env override if provided; otherwise resolve based on SQLD layout
|
26
|
+
db_url: str = os.getenv("TURSO_LOCAL_DB_URL", _resolve_sqlite_db_url())
|
21
27
|
|
22
28
|
# Remote database sync configuration
|
23
29
|
sync_url: str = os.getenv("TURSO_DATABASE_URL", "")
|
@@ -40,7 +46,7 @@ class TursoConfig:
|
|
40
46
|
|
41
47
|
# Daemon settings (for local sqld) - match serve.sh defaults
|
42
48
|
sqld_binary: str = os.getenv("SQLD_BINARY", "sqld")
|
43
|
-
sqld_db_path: str = os.getenv("SQLD_DB_PATH", "synth_ai.db")
|
49
|
+
sqld_db_path: str = os.getenv("SQLD_DB_PATH", "traces/v3/synth_ai.db")
|
44
50
|
sqld_http_port: int = int(os.getenv("SQLD_HTTP_PORT", "8080"))
|
45
51
|
sqld_idle_shutdown: int = int(os.getenv("SQLD_IDLE_SHUTDOWN", "0")) # 0 = no idle shutdown
|
46
52
|
|
synth_ai/tracing_v3/db_config.py
CHANGED
@@ -18,7 +18,7 @@ class DatabaseConfig:
|
|
18
18
|
"""Centralized database configuration management."""
|
19
19
|
|
20
20
|
# Default values from serve.sh
|
21
|
-
DEFAULT_DB_FILE = "synth_ai.db"
|
21
|
+
DEFAULT_DB_FILE = "traces/v3/synth_ai.db"
|
22
22
|
DEFAULT_HTTP_PORT = 8080
|
23
23
|
|
24
24
|
def __init__(
|
@@ -58,15 +58,15 @@ class DatabaseConfig:
|
|
58
58
|
|
59
59
|
if os.path.exists(sqld_data_path):
|
60
60
|
# sqld is managing the database
|
61
|
-
logger.
|
61
|
+
logger.debug(f"✅ Using sqld-managed database at: {sqld_data_path}")
|
62
62
|
actual_db_path = sqld_data_path
|
63
63
|
else:
|
64
64
|
# Direct SQLite file
|
65
65
|
if not os.path.exists(abs_path):
|
66
|
-
logger.
|
67
|
-
logger.
|
66
|
+
logger.debug(f"⚠️ Database file not found at: {abs_path}")
|
67
|
+
logger.debug("🔧 Make sure to run './serve.sh' to start the turso/sqld service")
|
68
68
|
else:
|
69
|
-
logger.
|
69
|
+
logger.debug(f"📁 Using direct SQLite file at: {abs_path}")
|
70
70
|
actual_db_path = abs_path
|
71
71
|
|
72
72
|
# SQLite URLs need 3 slashes for absolute paths
|
@@ -147,7 +147,7 @@ def get_default_db_config() -> DatabaseConfig:
|
|
147
147
|
# sqld is already running, don't start a new one
|
148
148
|
sqld_running = True
|
149
149
|
use_sqld = False
|
150
|
-
logger.
|
150
|
+
logger.debug(f"✅ Detected sqld already running on port {sqld_port}")
|
151
151
|
except Exception as e:
|
152
152
|
logger.debug(f"Could not check for sqld process: {e}")
|
153
153
|
|
synth_ai/tracing_v3/hooks.py
CHANGED
@@ -37,7 +37,7 @@ from dataclasses import dataclass
|
|
37
37
|
import asyncio
|
38
38
|
import inspect
|
39
39
|
|
40
|
-
from .abstractions import SessionTrace, SessionTimeStep, BaseEvent,
|
40
|
+
from .abstractions import SessionTrace, SessionTimeStep, BaseEvent, SessionEventMarkovBlanketMessage
|
41
41
|
|
42
42
|
|
43
43
|
@dataclass
|