synth-ai 0.2.4.dev5__py3-none-any.whl โ 0.2.4.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synth_ai/__init__.py +18 -9
- synth_ai/cli/__init__.py +10 -5
- synth_ai/cli/balance.py +22 -17
- synth_ai/cli/calc.py +2 -3
- synth_ai/cli/demo.py +3 -5
- synth_ai/cli/legacy_root_backup.py +58 -32
- synth_ai/cli/man.py +22 -19
- synth_ai/cli/recent.py +9 -8
- synth_ai/cli/root.py +58 -13
- synth_ai/cli/status.py +13 -6
- synth_ai/cli/traces.py +45 -21
- synth_ai/cli/watch.py +40 -37
- synth_ai/config/base_url.py +1 -3
- synth_ai/core/experiment.py +1 -2
- synth_ai/environments/__init__.py +2 -6
- synth_ai/environments/environment/artifacts/base.py +3 -1
- synth_ai/environments/environment/db/sqlite.py +1 -1
- synth_ai/environments/environment/registry.py +19 -20
- synth_ai/environments/environment/resources/sqlite.py +2 -3
- synth_ai/environments/environment/rewards/core.py +3 -2
- synth_ai/environments/environment/tools/__init__.py +6 -4
- synth_ai/environments/examples/crafter_classic/__init__.py +1 -1
- synth_ai/environments/examples/crafter_classic/engine.py +21 -17
- synth_ai/environments/examples/crafter_classic/engine_deterministic_patch.py +1 -0
- synth_ai/environments/examples/crafter_classic/engine_helpers/action_map.py +2 -1
- synth_ai/environments/examples/crafter_classic/engine_helpers/serialization.py +2 -1
- synth_ai/environments/examples/crafter_classic/engine_serialization_patch_v3.py +3 -2
- synth_ai/environments/examples/crafter_classic/environment.py +16 -15
- synth_ai/environments/examples/crafter_classic/taskset.py +2 -2
- synth_ai/environments/examples/crafter_classic/trace_hooks_v3.py +2 -3
- synth_ai/environments/examples/crafter_classic/world_config_patch_simple.py +2 -1
- synth_ai/environments/examples/crafter_custom/crafter/__init__.py +2 -2
- synth_ai/environments/examples/crafter_custom/crafter/config.py +2 -2
- synth_ai/environments/examples/crafter_custom/crafter/env.py +1 -5
- synth_ai/environments/examples/crafter_custom/crafter/objects.py +1 -2
- synth_ai/environments/examples/crafter_custom/crafter/worldgen.py +1 -2
- synth_ai/environments/examples/crafter_custom/dataset_builder.py +5 -5
- synth_ai/environments/examples/crafter_custom/environment.py +13 -13
- synth_ai/environments/examples/crafter_custom/run_dataset.py +5 -5
- synth_ai/environments/examples/enron/art_helpers/email_search_tools.py +2 -2
- synth_ai/environments/examples/enron/art_helpers/local_email_db.py +5 -4
- synth_ai/environments/examples/enron/art_helpers/types_enron.py +2 -1
- synth_ai/environments/examples/enron/engine.py +18 -14
- synth_ai/environments/examples/enron/environment.py +12 -11
- synth_ai/environments/examples/enron/taskset.py +7 -7
- synth_ai/environments/examples/minigrid/__init__.py +6 -6
- synth_ai/environments/examples/minigrid/engine.py +6 -6
- synth_ai/environments/examples/minigrid/environment.py +6 -6
- synth_ai/environments/examples/minigrid/puzzle_loader.py +3 -2
- synth_ai/environments/examples/minigrid/taskset.py +13 -13
- synth_ai/environments/examples/nethack/achievements.py +1 -1
- synth_ai/environments/examples/nethack/engine.py +8 -7
- synth_ai/environments/examples/nethack/environment.py +10 -9
- synth_ai/environments/examples/nethack/helpers/__init__.py +8 -9
- synth_ai/environments/examples/nethack/helpers/action_mapping.py +1 -1
- synth_ai/environments/examples/nethack/helpers/nle_wrapper.py +2 -1
- synth_ai/environments/examples/nethack/helpers/observation_utils.py +1 -1
- synth_ai/environments/examples/nethack/helpers/recording_wrapper.py +3 -4
- synth_ai/environments/examples/nethack/helpers/trajectory_recorder.py +6 -5
- synth_ai/environments/examples/nethack/helpers/visualization/replay_viewer.py +5 -5
- synth_ai/environments/examples/nethack/helpers/visualization/visualizer.py +7 -6
- synth_ai/environments/examples/nethack/taskset.py +5 -5
- synth_ai/environments/examples/red/engine.py +9 -8
- synth_ai/environments/examples/red/engine_helpers/reward_components.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/__init__.py +7 -7
- synth_ai/environments/examples/red/engine_helpers/reward_library/adaptive_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/battle_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/composite_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/economy_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/efficiency_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/exploration_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/novelty_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/pallet_town_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/pokemon_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/social_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/story_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/screen_analysis.py +3 -2
- synth_ai/environments/examples/red/engine_helpers/state_extraction.py +2 -1
- synth_ai/environments/examples/red/environment.py +18 -15
- synth_ai/environments/examples/red/taskset.py +5 -3
- synth_ai/environments/examples/sokoban/engine.py +16 -13
- synth_ai/environments/examples/sokoban/engine_helpers/room_utils.py +3 -2
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/__init__.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/__init__.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/boxoban_env.py +7 -5
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/render_utils.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/room_utils.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env.py +5 -4
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_fixed_targets.py +3 -2
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_pull.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_two_player.py +5 -4
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_variations.py +1 -1
- synth_ai/environments/examples/sokoban/environment.py +15 -14
- synth_ai/environments/examples/sokoban/generate_verified_puzzles.py +5 -3
- synth_ai/environments/examples/sokoban/puzzle_loader.py +3 -2
- synth_ai/environments/examples/sokoban/taskset.py +13 -10
- synth_ai/environments/examples/tictactoe/engine.py +6 -6
- synth_ai/environments/examples/tictactoe/environment.py +8 -7
- synth_ai/environments/examples/tictactoe/taskset.py +6 -5
- synth_ai/environments/examples/verilog/engine.py +4 -3
- synth_ai/environments/examples/verilog/environment.py +11 -10
- synth_ai/environments/examples/verilog/taskset.py +14 -12
- synth_ai/environments/examples/wordle/__init__.py +29 -0
- synth_ai/environments/examples/wordle/engine.py +398 -0
- synth_ai/environments/examples/wordle/environment.py +159 -0
- synth_ai/environments/examples/wordle/helpers/generate_instances_wordfreq.py +75 -0
- synth_ai/environments/examples/wordle/taskset.py +230 -0
- synth_ai/environments/reproducibility/core.py +1 -1
- synth_ai/environments/reproducibility/tree.py +21 -21
- synth_ai/environments/service/app.py +11 -2
- synth_ai/environments/service/core_routes.py +137 -105
- synth_ai/environments/service/external_registry.py +1 -2
- synth_ai/environments/service/registry.py +1 -1
- synth_ai/environments/stateful/core.py +1 -2
- synth_ai/environments/stateful/engine.py +1 -1
- synth_ai/environments/tasks/api.py +4 -4
- synth_ai/environments/tasks/core.py +14 -12
- synth_ai/environments/tasks/filters.py +6 -4
- synth_ai/environments/tasks/utils.py +13 -11
- synth_ai/evals/base.py +2 -3
- synth_ai/experimental/synth_oss.py +4 -4
- synth_ai/learning/gateway.py +1 -3
- synth_ai/learning/prompts/banking77_injection_eval.py +168 -0
- synth_ai/learning/prompts/hello_world_in_context_injection_ex.py +213 -0
- synth_ai/learning/prompts/mipro.py +282 -1
- synth_ai/learning/prompts/random_search.py +246 -0
- synth_ai/learning/prompts/run_mipro_banking77.py +172 -0
- synth_ai/learning/prompts/run_random_search_banking77.py +324 -0
- synth_ai/lm/__init__.py +5 -5
- synth_ai/lm/caching/ephemeral.py +9 -9
- synth_ai/lm/caching/handler.py +20 -20
- synth_ai/lm/caching/persistent.py +10 -10
- synth_ai/lm/config.py +3 -3
- synth_ai/lm/constants.py +7 -7
- synth_ai/lm/core/all.py +17 -3
- synth_ai/lm/core/exceptions.py +0 -2
- synth_ai/lm/core/main.py +26 -41
- synth_ai/lm/core/main_v3.py +20 -10
- synth_ai/lm/core/vendor_clients.py +18 -17
- synth_ai/lm/injection.py +80 -0
- synth_ai/lm/overrides.py +206 -0
- synth_ai/lm/provider_support/__init__.py +1 -1
- synth_ai/lm/provider_support/anthropic.py +51 -24
- synth_ai/lm/provider_support/openai.py +51 -22
- synth_ai/lm/structured_outputs/handler.py +34 -32
- synth_ai/lm/structured_outputs/inject.py +24 -27
- synth_ai/lm/structured_outputs/rehabilitate.py +19 -15
- synth_ai/lm/tools/base.py +17 -16
- synth_ai/lm/unified_interface.py +17 -18
- synth_ai/lm/vendors/base.py +20 -18
- synth_ai/lm/vendors/core/anthropic_api.py +50 -25
- synth_ai/lm/vendors/core/gemini_api.py +31 -36
- synth_ai/lm/vendors/core/mistral_api.py +19 -19
- synth_ai/lm/vendors/core/openai_api.py +11 -10
- synth_ai/lm/vendors/openai_standard.py +144 -88
- synth_ai/lm/vendors/openai_standard_responses.py +74 -61
- synth_ai/lm/vendors/retries.py +9 -1
- synth_ai/lm/vendors/supported/custom_endpoint.py +26 -26
- synth_ai/lm/vendors/supported/deepseek.py +10 -10
- synth_ai/lm/vendors/supported/grok.py +8 -8
- synth_ai/lm/vendors/supported/ollama.py +2 -1
- synth_ai/lm/vendors/supported/openrouter.py +11 -9
- synth_ai/lm/vendors/synth_client.py +69 -63
- synth_ai/lm/warmup.py +8 -7
- synth_ai/tracing/__init__.py +22 -10
- synth_ai/tracing_v1/__init__.py +22 -20
- synth_ai/tracing_v3/__init__.py +7 -7
- synth_ai/tracing_v3/abstractions.py +56 -52
- synth_ai/tracing_v3/config.py +4 -2
- synth_ai/tracing_v3/db_config.py +6 -8
- synth_ai/tracing_v3/decorators.py +29 -30
- synth_ai/tracing_v3/examples/basic_usage.py +12 -12
- synth_ai/tracing_v3/hooks.py +21 -21
- synth_ai/tracing_v3/llm_call_record_helpers.py +85 -98
- synth_ai/tracing_v3/lm_call_record_abstractions.py +2 -4
- synth_ai/tracing_v3/migration_helper.py +3 -5
- synth_ai/tracing_v3/replica_sync.py +30 -32
- synth_ai/tracing_v3/session_tracer.py +35 -29
- synth_ai/tracing_v3/storage/__init__.py +1 -1
- synth_ai/tracing_v3/storage/base.py +8 -7
- synth_ai/tracing_v3/storage/config.py +4 -4
- synth_ai/tracing_v3/storage/factory.py +4 -4
- synth_ai/tracing_v3/storage/utils.py +9 -9
- synth_ai/tracing_v3/turso/__init__.py +3 -3
- synth_ai/tracing_v3/turso/daemon.py +9 -9
- synth_ai/tracing_v3/turso/manager.py +60 -48
- synth_ai/tracing_v3/turso/models.py +24 -19
- synth_ai/tracing_v3/utils.py +5 -5
- synth_ai/tui/__main__.py +1 -1
- synth_ai/tui/cli/query_experiments.py +2 -3
- synth_ai/tui/cli/query_experiments_v3.py +2 -3
- synth_ai/tui/dashboard.py +97 -86
- synth_ai/v0/tracing/abstractions.py +28 -28
- synth_ai/v0/tracing/base_client.py +9 -9
- synth_ai/v0/tracing/client_manager.py +7 -7
- synth_ai/v0/tracing/config.py +7 -7
- synth_ai/v0/tracing/context.py +6 -6
- synth_ai/v0/tracing/decorators.py +6 -5
- synth_ai/v0/tracing/events/manage.py +1 -1
- synth_ai/v0/tracing/events/store.py +5 -4
- synth_ai/v0/tracing/immediate_client.py +4 -5
- synth_ai/v0/tracing/local.py +3 -3
- synth_ai/v0/tracing/log_client_base.py +4 -5
- synth_ai/v0/tracing/retry_queue.py +5 -6
- synth_ai/v0/tracing/trackers.py +25 -25
- synth_ai/v0/tracing/upload.py +6 -0
- synth_ai/v0/tracing_v1/__init__.py +1 -1
- synth_ai/v0/tracing_v1/abstractions.py +28 -28
- synth_ai/v0/tracing_v1/base_client.py +9 -9
- synth_ai/v0/tracing_v1/client_manager.py +7 -7
- synth_ai/v0/tracing_v1/config.py +7 -7
- synth_ai/v0/tracing_v1/context.py +6 -6
- synth_ai/v0/tracing_v1/decorators.py +7 -6
- synth_ai/v0/tracing_v1/events/manage.py +1 -1
- synth_ai/v0/tracing_v1/events/store.py +5 -4
- synth_ai/v0/tracing_v1/immediate_client.py +4 -5
- synth_ai/v0/tracing_v1/local.py +3 -3
- synth_ai/v0/tracing_v1/log_client_base.py +4 -5
- synth_ai/v0/tracing_v1/retry_queue.py +5 -6
- synth_ai/v0/tracing_v1/trackers.py +25 -25
- synth_ai/v0/tracing_v1/upload.py +25 -24
- synth_ai/zyk/__init__.py +1 -0
- {synth_ai-0.2.4.dev5.dist-info โ synth_ai-0.2.4.dev7.dist-info}/METADATA +2 -11
- synth_ai-0.2.4.dev7.dist-info/RECORD +299 -0
- synth_ai-0.2.4.dev5.dist-info/RECORD +0 -287
- {synth_ai-0.2.4.dev5.dist-info โ synth_ai-0.2.4.dev7.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.4.dev5.dist-info โ synth_ai-0.2.4.dev7.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.4.dev5.dist-info โ synth_ai-0.2.4.dev7.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.4.dev5.dist-info โ synth_ai-0.2.4.dev7.dist-info}/top_level.txt +0 -0
@@ -1,28 +1,25 @@
|
|
1
1
|
"""SQLAlchemy declarative models for tracing v3."""
|
2
2
|
|
3
|
-
|
4
|
-
|
3
|
+
import json
|
4
|
+
|
5
5
|
from sqlalchemy import (
|
6
|
+
Boolean,
|
7
|
+
CheckConstraint,
|
6
8
|
Column,
|
7
|
-
Integer,
|
8
|
-
String,
|
9
9
|
DateTime,
|
10
|
-
|
10
|
+
Float,
|
11
11
|
ForeignKey,
|
12
12
|
Index,
|
13
|
+
Integer,
|
14
|
+
String,
|
13
15
|
Text,
|
14
|
-
Float,
|
15
|
-
Boolean,
|
16
|
-
UniqueConstraint,
|
17
|
-
CheckConstraint,
|
18
16
|
TypeDecorator,
|
17
|
+
UniqueConstraint,
|
19
18
|
)
|
20
19
|
from sqlalchemy.ext.declarative import declarative_base
|
21
20
|
from sqlalchemy.orm import relationship
|
22
21
|
from sqlalchemy.sql import func
|
23
22
|
from sqlalchemy.types import UserDefinedType
|
24
|
-
import json
|
25
|
-
|
26
23
|
|
27
24
|
Base = declarative_base()
|
28
25
|
|
@@ -81,10 +78,11 @@ class JSONText(TypeDecorator):
|
|
81
78
|
|
82
79
|
class SessionTrace(Base):
|
83
80
|
"""Database model for session traces.
|
84
|
-
|
81
|
+
|
85
82
|
Stores high-level information about tracing sessions including
|
86
83
|
metadata, statistics, and relationships to timesteps and events.
|
87
84
|
"""
|
85
|
+
|
88
86
|
__tablename__ = "session_traces"
|
89
87
|
|
90
88
|
session_id = Column(String, primary_key=True)
|
@@ -114,10 +112,11 @@ class SessionTrace(Base):
|
|
114
112
|
|
115
113
|
class SessionTimestep(Base):
|
116
114
|
"""Database model for session timesteps.
|
117
|
-
|
115
|
+
|
118
116
|
Represents individual steps within a tracing session, with timing
|
119
117
|
information and relationships to events and messages.
|
120
118
|
"""
|
119
|
+
|
121
120
|
__tablename__ = "session_timesteps"
|
122
121
|
|
123
122
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
@@ -145,11 +144,12 @@ class SessionTimestep(Base):
|
|
145
144
|
|
146
145
|
class Event(Base):
|
147
146
|
"""Database model for events.
|
148
|
-
|
147
|
+
|
149
148
|
Stores all types of events (LM CAIS, environment, runtime) with
|
150
149
|
type-specific fields and common metadata. Supports vector embeddings
|
151
150
|
for similarity search.
|
152
151
|
"""
|
152
|
+
|
153
153
|
__tablename__ = "events"
|
154
154
|
|
155
155
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
@@ -209,10 +209,11 @@ class Event(Base):
|
|
209
209
|
|
210
210
|
class Message(Base):
|
211
211
|
"""Database model for messages.
|
212
|
-
|
212
|
+
|
213
213
|
Stores conversational messages between users, assistants, and systems
|
214
214
|
with support for embeddings and rich metadata.
|
215
215
|
"""
|
216
|
+
|
216
217
|
__tablename__ = "messages"
|
217
218
|
|
218
219
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
@@ -247,10 +248,11 @@ class Message(Base):
|
|
247
248
|
|
248
249
|
class Experiment(Base):
|
249
250
|
"""Database model for experiments.
|
250
|
-
|
251
|
+
|
251
252
|
Groups related sessions and systems for experimental evaluation
|
252
253
|
and comparison. Supports rich configuration and metadata.
|
253
254
|
"""
|
255
|
+
|
254
256
|
__tablename__ = "experiments"
|
255
257
|
|
256
258
|
experiment_id = Column(String, primary_key=True)
|
@@ -277,10 +279,11 @@ class Experiment(Base):
|
|
277
279
|
|
278
280
|
class System(Base):
|
279
281
|
"""Database model for systems.
|
280
|
-
|
282
|
+
|
281
283
|
Represents agents, environments, or runtime systems that participate
|
282
284
|
in tracing sessions. Supports versioning and type classification.
|
283
285
|
"""
|
286
|
+
|
284
287
|
__tablename__ = "systems"
|
285
288
|
|
286
289
|
system_id = Column(String, primary_key=True)
|
@@ -302,10 +305,11 @@ class System(Base):
|
|
302
305
|
|
303
306
|
class SystemVersion(Base):
|
304
307
|
"""Database model for system versions.
|
305
|
-
|
308
|
+
|
306
309
|
Tracks different versions of systems with commit hashes,
|
307
310
|
configuration changes, and relationships to experiments.
|
308
311
|
"""
|
312
|
+
|
309
313
|
__tablename__ = "system_versions"
|
310
314
|
|
311
315
|
version_id = Column(String, primary_key=True)
|
@@ -329,10 +333,11 @@ class SystemVersion(Base):
|
|
329
333
|
|
330
334
|
class ExperimentalSystem(Base):
|
331
335
|
"""Database model for experiment-system relationships.
|
332
|
-
|
336
|
+
|
333
337
|
Junction table linking experiments with specific system versions,
|
334
338
|
allowing tracking of which systems participated in which experiments.
|
335
339
|
"""
|
340
|
+
|
336
341
|
__tablename__ = "experimental_systems"
|
337
342
|
|
338
343
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
synth_ai/tracing_v3/utils.py
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
"""Utility functions for tracing v3."""
|
2
2
|
|
3
|
-
import json
|
4
|
-
from datetime import datetime
|
5
|
-
from typing import Any, Dict, Optional
|
6
3
|
import hashlib
|
4
|
+
import json
|
7
5
|
import uuid
|
6
|
+
from datetime import datetime
|
7
|
+
from typing import Any
|
8
8
|
|
9
9
|
|
10
10
|
def iso_now() -> str:
|
@@ -36,7 +36,7 @@ def generate_experiment_id(name: str) -> str:
|
|
36
36
|
return f"exp_{hash_obj.hexdigest()[:12]}"
|
37
37
|
|
38
38
|
|
39
|
-
def detect_provider(model_name:
|
39
|
+
def detect_provider(model_name: str | None) -> str:
|
40
40
|
"""Detect LLM provider from model name."""
|
41
41
|
if not model_name:
|
42
42
|
return "unknown"
|
@@ -59,7 +59,7 @@ def detect_provider(model_name: Optional[str]) -> str:
|
|
59
59
|
return "unknown"
|
60
60
|
|
61
61
|
|
62
|
-
def calculate_cost(model_name: str, input_tokens: int, output_tokens: int) ->
|
62
|
+
def calculate_cost(model_name: str, input_tokens: int, output_tokens: int) -> float | None:
|
63
63
|
"""Calculate cost in USD based on model and token counts."""
|
64
64
|
# This is a simplified version - in production you'd want a proper pricing table
|
65
65
|
pricing = {
|
synth_ai/tui/__main__.py
CHANGED
@@ -5,9 +5,8 @@ Query experiments and sessions from Turso/sqld using v3 tracing.
|
|
5
5
|
|
6
6
|
import argparse
|
7
7
|
import asyncio
|
8
|
-
|
8
|
+
|
9
9
|
from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
|
10
|
-
import pandas as pd
|
11
10
|
|
12
11
|
|
13
12
|
async def list_experiments(db_url: str):
|
@@ -119,7 +118,7 @@ async def show_experiment_details(db_url: str, experiment_id: str):
|
|
119
118
|
await db.close()
|
120
119
|
|
121
120
|
|
122
|
-
async def show_model_usage(db_url: str, model_name:
|
121
|
+
async def show_model_usage(db_url: str, model_name: str | None = None):
|
123
122
|
"""Show model usage statistics."""
|
124
123
|
db = AsyncSQLTraceManager(db_url)
|
125
124
|
await db.initialize()
|
@@ -5,9 +5,8 @@ Query experiments and sessions from Turso/sqld using v3 tracing.
|
|
5
5
|
|
6
6
|
import argparse
|
7
7
|
import asyncio
|
8
|
-
|
8
|
+
|
9
9
|
from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
|
10
|
-
import pandas as pd
|
11
10
|
|
12
11
|
|
13
12
|
async def list_experiments(db_url: str):
|
@@ -119,7 +118,7 @@ async def show_experiment_details(db_url: str, experiment_id: str):
|
|
119
118
|
await db.close()
|
120
119
|
|
121
120
|
|
122
|
-
async def show_model_usage(db_url: str, model_name:
|
121
|
+
async def show_model_usage(db_url: str, model_name: str | None = None):
|
123
122
|
"""Show model usage statistics."""
|
124
123
|
db = AsyncSQLTraceManager(db_url)
|
125
124
|
await db.initialize()
|
synth_ai/tui/dashboard.py
CHANGED
@@ -5,68 +5,77 @@ Interactive TUI Dashboard for Synth AI experiments.
|
|
5
5
|
Launch with: python -m synth_ai.tui.dashboard
|
6
6
|
"""
|
7
7
|
|
8
|
-
import asyncio
|
9
8
|
import logging
|
10
9
|
from datetime import datetime
|
11
|
-
from typing import List, Optional, Dict, Any
|
12
10
|
from urllib.parse import urlparse
|
13
11
|
|
12
|
+
from textual import on
|
14
13
|
from textual.app import App, ComposeResult
|
15
|
-
from textual.containers import Container, Horizontal, Vertical
|
16
|
-
from textual.widgets import (
|
17
|
-
Header, Footer, DataTable, Static, Input, Button,
|
18
|
-
TabbedContent, TabPane, Label, ProgressBar
|
19
|
-
)
|
20
|
-
from textual.reactive import reactive
|
21
14
|
from textual.binding import Binding
|
22
|
-
from textual import
|
15
|
+
from textual.containers import Container
|
16
|
+
from textual.reactive import reactive
|
23
17
|
from textual.timer import Timer
|
18
|
+
from textual.widgets import (
|
19
|
+
DataTable,
|
20
|
+
Footer,
|
21
|
+
Header,
|
22
|
+
Static,
|
23
|
+
)
|
24
24
|
|
25
25
|
from ..tracing_v3.turso.manager import AsyncSQLTraceManager
|
26
26
|
|
27
|
+
|
27
28
|
class ExperimentRow:
|
28
29
|
"""Data structure for experiment display."""
|
29
|
-
|
30
|
-
|
31
|
-
|
30
|
+
|
31
|
+
def __init__(
|
32
|
+
self,
|
33
|
+
exp_id: str,
|
34
|
+
name: str,
|
35
|
+
description: str,
|
36
|
+
created_at: datetime,
|
37
|
+
sessions: int,
|
38
|
+
events: int,
|
39
|
+
messages: int,
|
40
|
+
cost: float,
|
41
|
+
tokens: int,
|
42
|
+
):
|
32
43
|
self.exp_id = exp_id
|
33
44
|
self.name = name or "Unnamed"
|
34
45
|
self.description = description or ""
|
35
46
|
self.created_at = created_at
|
36
47
|
self.sessions = sessions
|
37
|
-
self.events = events
|
48
|
+
self.events = events
|
38
49
|
self.messages = messages
|
39
50
|
self.cost = cost
|
40
51
|
self.tokens = tokens
|
41
|
-
|
42
|
-
def to_row(self) ->
|
52
|
+
|
53
|
+
def to_row(self) -> list[str]:
|
43
54
|
"""Convert to table row format."""
|
44
55
|
return [
|
45
56
|
self.exp_id[:8], # Shortened ID
|
46
|
-
self.name[:20],
|
57
|
+
self.name[:20], # Truncated name
|
47
58
|
str(self.sessions),
|
48
59
|
str(self.events),
|
49
60
|
str(self.messages),
|
50
61
|
f"${self.cost:.4f}",
|
51
62
|
f"{self.tokens:,}",
|
52
|
-
self.created_at.strftime("%H:%M")
|
63
|
+
self.created_at.strftime("%H:%M"),
|
53
64
|
]
|
54
65
|
|
66
|
+
|
55
67
|
class ExperimentTable(DataTable):
|
56
68
|
"""Custom DataTable for experiments with refresh capability."""
|
57
|
-
|
69
|
+
|
58
70
|
def __init__(self, **kwargs):
|
59
71
|
super().__init__(**kwargs)
|
60
|
-
self.experiments:
|
61
|
-
self.selected_exp_id:
|
62
|
-
|
72
|
+
self.experiments: list[ExperimentRow] = []
|
73
|
+
self.selected_exp_id: str | None = None
|
74
|
+
|
63
75
|
def setup_table(self):
|
64
76
|
"""Initialize table columns."""
|
65
|
-
self.add_columns(
|
66
|
-
|
67
|
-
"Messages", "Cost", "Tokens", "Time"
|
68
|
-
)
|
69
|
-
|
77
|
+
self.add_columns("ID", "Name", "Sessions", "Events", "Messages", "Cost", "Tokens", "Time")
|
78
|
+
|
70
79
|
async def refresh_data(self, db_manager: AsyncSQLTraceManager):
|
71
80
|
"""Refresh experiment data from database."""
|
72
81
|
try:
|
@@ -89,50 +98,51 @@ class ExperimentTable(DataTable):
|
|
89
98
|
GROUP BY e.experiment_id, e.name, e.description, e.created_at
|
90
99
|
ORDER BY e.created_at DESC
|
91
100
|
""")
|
92
|
-
|
101
|
+
|
93
102
|
self.experiments.clear()
|
94
103
|
self.clear()
|
95
|
-
|
104
|
+
|
96
105
|
if not df.empty:
|
97
106
|
for _, row in df.iterrows():
|
98
107
|
exp_row = ExperimentRow(
|
99
|
-
exp_id=row[
|
100
|
-
name=row[
|
101
|
-
description=row[
|
102
|
-
created_at=row[
|
103
|
-
sessions=int(row[
|
104
|
-
events=int(row[
|
105
|
-
messages=int(row[
|
106
|
-
cost=float(row[
|
107
|
-
tokens=int(row[
|
108
|
+
exp_id=row["experiment_id"],
|
109
|
+
name=row["name"],
|
110
|
+
description=row["description"],
|
111
|
+
created_at=row["created_at"],
|
112
|
+
sessions=int(row["num_sessions"] or 0),
|
113
|
+
events=int(row["num_events"] or 0),
|
114
|
+
messages=int(row["num_messages"] or 0),
|
115
|
+
cost=float(row["total_cost"] or 0.0),
|
116
|
+
tokens=int(row["total_tokens"] or 0),
|
108
117
|
)
|
109
118
|
self.experiments.append(exp_row)
|
110
119
|
self.add_row(*exp_row.to_row(), key=exp_row.exp_id)
|
111
|
-
|
120
|
+
|
112
121
|
except Exception as e:
|
113
122
|
logging.error(f"Failed to refresh experiments: {e}")
|
114
|
-
|
115
|
-
def get_selected_experiment(self) ->
|
123
|
+
|
124
|
+
def get_selected_experiment(self) -> ExperimentRow | None:
|
116
125
|
"""Get currently selected experiment."""
|
117
126
|
if self.cursor_row >= 0 and self.cursor_row < len(self.experiments):
|
118
127
|
return self.experiments[self.cursor_row]
|
119
128
|
return None
|
120
129
|
|
130
|
+
|
121
131
|
class ExperimentDetail(Static):
|
122
132
|
"""Detailed view of selected experiment."""
|
123
|
-
|
133
|
+
|
124
134
|
def __init__(self, **kwargs):
|
125
135
|
super().__init__(**kwargs)
|
126
|
-
self.current_experiment:
|
127
|
-
|
128
|
-
def update_experiment(self, experiment:
|
136
|
+
self.current_experiment: ExperimentRow | None = None
|
137
|
+
|
138
|
+
def update_experiment(self, experiment: ExperimentRow | None):
|
129
139
|
"""Update the displayed experiment details."""
|
130
140
|
self.current_experiment = experiment
|
131
141
|
if experiment:
|
132
142
|
details = f"""
|
133
143
|
๐ฌ **{experiment.name}**
|
134
144
|
ID: {experiment.exp_id}
|
135
|
-
Description: {experiment.description or
|
145
|
+
Description: {experiment.description or "No description"}
|
136
146
|
|
137
147
|
๐ **Statistics**
|
138
148
|
Sessions: {experiment.sessions}
|
@@ -141,36 +151,38 @@ Messages: {experiment.messages}
|
|
141
151
|
Cost: ${experiment.cost:.4f}
|
142
152
|
Tokens: {experiment.tokens:,}
|
143
153
|
|
144
|
-
๐ **Created**: {experiment.created_at.strftime(
|
154
|
+
๐ **Created**: {experiment.created_at.strftime("%Y-%m-%d %H:%M:%S")}
|
145
155
|
""".strip()
|
146
156
|
else:
|
147
157
|
details = "Select an experiment to view details"
|
148
|
-
|
158
|
+
|
149
159
|
self.update(details)
|
150
160
|
|
161
|
+
|
151
162
|
class DatabaseStatus(Static):
|
152
163
|
"""Display database connection status."""
|
153
|
-
|
164
|
+
|
154
165
|
connection_status = reactive("๐ด Disconnected")
|
155
|
-
|
166
|
+
|
156
167
|
def __init__(self, **kwargs):
|
157
168
|
super().__init__(**kwargs)
|
158
|
-
|
169
|
+
|
159
170
|
def render(self) -> str:
|
160
171
|
return f"Database: {self.connection_status}"
|
161
|
-
|
172
|
+
|
162
173
|
def set_connected(self, url: str):
|
163
174
|
parsed = urlparse(url)
|
164
175
|
host_info = f"{parsed.hostname}:{parsed.port}" if parsed.port else str(parsed.hostname)
|
165
176
|
self.connection_status = f"๐ข Connected ({host_info})"
|
166
|
-
|
177
|
+
|
167
178
|
def set_disconnected(self, error: str = ""):
|
168
179
|
error_text = f" - {error}" if error else ""
|
169
180
|
self.connection_status = f"๐ด Disconnected{error_text}"
|
170
181
|
|
182
|
+
|
171
183
|
class SynthDashboard(App):
|
172
184
|
"""Main Synth AI TUI Dashboard application."""
|
173
|
-
|
185
|
+
|
174
186
|
CSS = """
|
175
187
|
Screen {
|
176
188
|
layout: grid;
|
@@ -211,73 +223,73 @@ class SynthDashboard(App):
|
|
211
223
|
padding: 0 1;
|
212
224
|
}
|
213
225
|
"""
|
214
|
-
|
226
|
+
|
215
227
|
BINDINGS = [
|
216
228
|
Binding("q", "quit", "Quit"),
|
217
229
|
Binding("r", "refresh", "Refresh"),
|
218
230
|
Binding("d", "toggle_debug", "Debug"),
|
219
231
|
("ctrl+c", "quit", "Quit"),
|
220
232
|
]
|
221
|
-
|
233
|
+
|
222
234
|
def __init__(self, db_url: str = "sqlite+aiosqlite:///./synth_ai.db/dbs/default/data"):
|
223
235
|
super().__init__()
|
224
236
|
self.db_url = db_url
|
225
|
-
self.db_manager:
|
226
|
-
self.refresh_timer:
|
227
|
-
|
237
|
+
self.db_manager: AsyncSQLTraceManager | None = None
|
238
|
+
self.refresh_timer: Timer | None = None
|
239
|
+
|
228
240
|
def compose(self) -> ComposeResult:
|
229
241
|
"""Create the UI layout."""
|
230
242
|
yield Header(show_clock=True)
|
231
|
-
|
243
|
+
|
232
244
|
with Container(id="experiments-table"):
|
233
245
|
yield Static("๐งช Experiments", classes="section-title")
|
234
246
|
yield ExperimentTable(id="experiments")
|
235
|
-
|
247
|
+
|
236
248
|
with Container(id="experiment-detail"):
|
237
|
-
yield Static("๐ Details", classes="section-title")
|
249
|
+
yield Static("๐ Details", classes="section-title")
|
238
250
|
yield ExperimentDetail(id="detail")
|
239
|
-
|
251
|
+
|
240
252
|
with Container(id="status-bar"):
|
241
253
|
yield DatabaseStatus(id="db-status")
|
242
254
|
yield Footer()
|
243
|
-
|
255
|
+
|
244
256
|
async def on_mount(self) -> None:
|
245
257
|
"""Initialize the app when mounted."""
|
246
258
|
# Setup database connection
|
247
259
|
try:
|
248
260
|
self.db_manager = AsyncSQLTraceManager(self.db_url)
|
249
261
|
await self.db_manager.initialize()
|
250
|
-
|
262
|
+
|
251
263
|
db_status = self.query_one("#db-status", DatabaseStatus)
|
252
264
|
db_status.set_connected(self.db_url)
|
253
|
-
|
265
|
+
|
254
266
|
except Exception as e:
|
255
267
|
logging.error(f"Failed to connect to database: {e}")
|
256
268
|
db_status = self.query_one("#db-status", DatabaseStatus)
|
257
269
|
db_status.set_disconnected(str(e))
|
258
|
-
|
270
|
+
|
259
271
|
# Setup experiment table
|
260
272
|
exp_table = self.query_one("#experiments", ExperimentTable)
|
261
273
|
exp_table.setup_table()
|
262
|
-
|
274
|
+
|
263
275
|
# Initial data load
|
264
276
|
await self.action_refresh()
|
265
|
-
|
277
|
+
|
266
278
|
# Start auto-refresh timer (every 5 seconds)
|
267
279
|
self.refresh_timer = self.set_interval(5.0, self._auto_refresh)
|
268
|
-
|
280
|
+
|
269
281
|
async def _auto_refresh(self) -> None:
|
270
282
|
"""Auto-refresh data periodically."""
|
271
283
|
if self.db_manager:
|
272
284
|
exp_table = self.query_one("#experiments", ExperimentTable)
|
273
285
|
await exp_table.refresh_data(self.db_manager)
|
274
|
-
|
286
|
+
|
275
287
|
async def action_refresh(self) -> None:
|
276
288
|
"""Manual refresh action."""
|
277
289
|
if self.db_manager:
|
278
290
|
exp_table = self.query_one("#experiments", ExperimentTable)
|
279
291
|
await exp_table.refresh_data(self.db_manager)
|
280
|
-
|
292
|
+
|
281
293
|
async def action_quit(self) -> None:
|
282
294
|
"""Quit the application."""
|
283
295
|
if self.refresh_timer:
|
@@ -285,45 +297,44 @@ class SynthDashboard(App):
|
|
285
297
|
if self.db_manager:
|
286
298
|
await self.db_manager.close()
|
287
299
|
self.exit()
|
288
|
-
|
300
|
+
|
289
301
|
def action_toggle_debug(self) -> None:
|
290
302
|
"""Toggle debug mode."""
|
291
303
|
# Could add debug panel or logging level toggle
|
292
304
|
pass
|
293
|
-
|
305
|
+
|
294
306
|
@on(DataTable.RowHighlighted, "#experiments")
|
295
307
|
def on_experiment_selected(self, event: DataTable.RowHighlighted) -> None:
|
296
308
|
"""Handle experiment selection."""
|
297
309
|
exp_table = self.query_one("#experiments", ExperimentTable)
|
298
310
|
selected_exp = exp_table.get_selected_experiment()
|
299
|
-
|
311
|
+
|
300
312
|
detail_panel = self.query_one("#detail", ExperimentDetail)
|
301
313
|
detail_panel.update_experiment(selected_exp)
|
302
314
|
|
315
|
+
|
303
316
|
def main():
|
304
317
|
"""Main entry point for the dashboard."""
|
305
318
|
import argparse
|
306
|
-
|
319
|
+
|
307
320
|
parser = argparse.ArgumentParser(description="Synth AI Interactive Dashboard")
|
308
321
|
parser.add_argument(
|
309
|
-
"-u",
|
322
|
+
"-u",
|
323
|
+
"--url",
|
310
324
|
default="sqlite+libsql://http://127.0.0.1:8080",
|
311
|
-
help="Database URL (default: sqlite+libsql://http://127.0.0.1:8080)"
|
325
|
+
help="Database URL (default: sqlite+libsql://http://127.0.0.1:8080)",
|
312
326
|
)
|
313
|
-
parser.add_argument(
|
314
|
-
|
315
|
-
action="store_true",
|
316
|
-
help="Enable debug logging"
|
317
|
-
)
|
318
|
-
|
327
|
+
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
|
328
|
+
|
319
329
|
args = parser.parse_args()
|
320
|
-
|
330
|
+
|
321
331
|
if args.debug:
|
322
332
|
logging.basicConfig(level=logging.DEBUG)
|
323
|
-
|
333
|
+
|
324
334
|
# Run the dashboard
|
325
335
|
app = SynthDashboard(db_url=args.url)
|
326
336
|
app.run()
|
327
337
|
|
338
|
+
|
328
339
|
if __name__ == "__main__":
|
329
|
-
main()
|
340
|
+
main()
|