synth-ai 0.2.4.dev6__py3-none-any.whl → 0.2.4.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synth_ai/__init__.py +18 -9
- synth_ai/cli/__init__.py +10 -5
- synth_ai/cli/balance.py +22 -17
- synth_ai/cli/calc.py +2 -3
- synth_ai/cli/demo.py +3 -5
- synth_ai/cli/legacy_root_backup.py +58 -32
- synth_ai/cli/man.py +22 -19
- synth_ai/cli/recent.py +9 -8
- synth_ai/cli/root.py +58 -13
- synth_ai/cli/status.py +13 -6
- synth_ai/cli/traces.py +45 -21
- synth_ai/cli/watch.py +40 -37
- synth_ai/config/base_url.py +1 -3
- synth_ai/core/experiment.py +1 -2
- synth_ai/environments/__init__.py +2 -6
- synth_ai/environments/environment/artifacts/base.py +3 -1
- synth_ai/environments/environment/db/sqlite.py +1 -1
- synth_ai/environments/environment/registry.py +19 -20
- synth_ai/environments/environment/resources/sqlite.py +2 -3
- synth_ai/environments/environment/rewards/core.py +3 -2
- synth_ai/environments/environment/tools/__init__.py +6 -4
- synth_ai/environments/examples/crafter_classic/__init__.py +1 -1
- synth_ai/environments/examples/crafter_classic/engine.py +13 -13
- synth_ai/environments/examples/crafter_classic/engine_deterministic_patch.py +1 -0
- synth_ai/environments/examples/crafter_classic/engine_helpers/action_map.py +2 -1
- synth_ai/environments/examples/crafter_classic/engine_helpers/serialization.py +2 -1
- synth_ai/environments/examples/crafter_classic/engine_serialization_patch_v3.py +3 -2
- synth_ai/environments/examples/crafter_classic/environment.py +16 -15
- synth_ai/environments/examples/crafter_classic/taskset.py +2 -2
- synth_ai/environments/examples/crafter_classic/trace_hooks_v3.py +2 -3
- synth_ai/environments/examples/crafter_classic/world_config_patch_simple.py +2 -1
- synth_ai/environments/examples/crafter_custom/crafter/__init__.py +2 -2
- synth_ai/environments/examples/crafter_custom/crafter/config.py +2 -2
- synth_ai/environments/examples/crafter_custom/crafter/env.py +1 -5
- synth_ai/environments/examples/crafter_custom/crafter/objects.py +1 -2
- synth_ai/environments/examples/crafter_custom/crafter/worldgen.py +1 -2
- synth_ai/environments/examples/crafter_custom/dataset_builder.py +5 -5
- synth_ai/environments/examples/crafter_custom/environment.py +13 -13
- synth_ai/environments/examples/crafter_custom/run_dataset.py +5 -5
- synth_ai/environments/examples/enron/art_helpers/email_search_tools.py +2 -2
- synth_ai/environments/examples/enron/art_helpers/local_email_db.py +5 -4
- synth_ai/environments/examples/enron/art_helpers/types_enron.py +2 -1
- synth_ai/environments/examples/enron/engine.py +18 -14
- synth_ai/environments/examples/enron/environment.py +12 -11
- synth_ai/environments/examples/enron/taskset.py +7 -7
- synth_ai/environments/examples/minigrid/__init__.py +6 -6
- synth_ai/environments/examples/minigrid/engine.py +6 -6
- synth_ai/environments/examples/minigrid/environment.py +6 -6
- synth_ai/environments/examples/minigrid/puzzle_loader.py +3 -2
- synth_ai/environments/examples/minigrid/taskset.py +13 -13
- synth_ai/environments/examples/nethack/achievements.py +1 -1
- synth_ai/environments/examples/nethack/engine.py +8 -7
- synth_ai/environments/examples/nethack/environment.py +10 -9
- synth_ai/environments/examples/nethack/helpers/__init__.py +8 -9
- synth_ai/environments/examples/nethack/helpers/action_mapping.py +1 -1
- synth_ai/environments/examples/nethack/helpers/nle_wrapper.py +2 -1
- synth_ai/environments/examples/nethack/helpers/observation_utils.py +1 -1
- synth_ai/environments/examples/nethack/helpers/recording_wrapper.py +3 -4
- synth_ai/environments/examples/nethack/helpers/trajectory_recorder.py +6 -5
- synth_ai/environments/examples/nethack/helpers/visualization/replay_viewer.py +5 -5
- synth_ai/environments/examples/nethack/helpers/visualization/visualizer.py +7 -6
- synth_ai/environments/examples/nethack/taskset.py +5 -5
- synth_ai/environments/examples/red/engine.py +9 -8
- synth_ai/environments/examples/red/engine_helpers/reward_components.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/__init__.py +7 -7
- synth_ai/environments/examples/red/engine_helpers/reward_library/adaptive_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/battle_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/composite_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/economy_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/efficiency_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/exploration_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/novelty_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/pallet_town_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/pokemon_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/social_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/story_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/screen_analysis.py +3 -2
- synth_ai/environments/examples/red/engine_helpers/state_extraction.py +2 -1
- synth_ai/environments/examples/red/environment.py +18 -15
- synth_ai/environments/examples/red/taskset.py +5 -3
- synth_ai/environments/examples/sokoban/engine.py +16 -13
- synth_ai/environments/examples/sokoban/engine_helpers/room_utils.py +3 -2
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/__init__.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/__init__.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/boxoban_env.py +7 -5
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/render_utils.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/room_utils.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env.py +5 -4
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_fixed_targets.py +3 -2
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_pull.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_two_player.py +5 -4
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_variations.py +1 -1
- synth_ai/environments/examples/sokoban/environment.py +15 -14
- synth_ai/environments/examples/sokoban/generate_verified_puzzles.py +5 -3
- synth_ai/environments/examples/sokoban/puzzle_loader.py +3 -2
- synth_ai/environments/examples/sokoban/taskset.py +13 -10
- synth_ai/environments/examples/tictactoe/engine.py +6 -6
- synth_ai/environments/examples/tictactoe/environment.py +8 -7
- synth_ai/environments/examples/tictactoe/taskset.py +6 -5
- synth_ai/environments/examples/verilog/engine.py +4 -3
- synth_ai/environments/examples/verilog/environment.py +11 -10
- synth_ai/environments/examples/verilog/taskset.py +14 -12
- synth_ai/environments/examples/wordle/__init__.py +5 -5
- synth_ai/environments/examples/wordle/engine.py +32 -25
- synth_ai/environments/examples/wordle/environment.py +21 -16
- synth_ai/environments/examples/wordle/helpers/generate_instances_wordfreq.py +6 -6
- synth_ai/environments/examples/wordle/taskset.py +20 -12
- synth_ai/environments/reproducibility/core.py +1 -1
- synth_ai/environments/reproducibility/tree.py +21 -21
- synth_ai/environments/service/app.py +3 -2
- synth_ai/environments/service/core_routes.py +104 -110
- synth_ai/environments/service/external_registry.py +1 -2
- synth_ai/environments/service/registry.py +1 -1
- synth_ai/environments/stateful/core.py +1 -2
- synth_ai/environments/stateful/engine.py +1 -1
- synth_ai/environments/tasks/api.py +4 -4
- synth_ai/environments/tasks/core.py +14 -12
- synth_ai/environments/tasks/filters.py +6 -4
- synth_ai/environments/tasks/utils.py +13 -11
- synth_ai/evals/base.py +2 -3
- synth_ai/experimental/synth_oss.py +4 -4
- synth_ai/learning/gateway.py +1 -3
- synth_ai/learning/prompts/banking77_injection_eval.py +15 -10
- synth_ai/learning/prompts/hello_world_in_context_injection_ex.py +26 -14
- synth_ai/learning/prompts/mipro.py +61 -52
- synth_ai/learning/prompts/random_search.py +42 -43
- synth_ai/learning/prompts/run_mipro_banking77.py +32 -20
- synth_ai/learning/prompts/run_random_search_banking77.py +71 -52
- synth_ai/lm/__init__.py +5 -5
- synth_ai/lm/caching/ephemeral.py +9 -9
- synth_ai/lm/caching/handler.py +20 -20
- synth_ai/lm/caching/persistent.py +10 -10
- synth_ai/lm/config.py +3 -3
- synth_ai/lm/constants.py +7 -7
- synth_ai/lm/core/all.py +17 -3
- synth_ai/lm/core/exceptions.py +0 -2
- synth_ai/lm/core/main.py +26 -41
- synth_ai/lm/core/main_v3.py +20 -10
- synth_ai/lm/core/vendor_clients.py +18 -17
- synth_ai/lm/injection.py +7 -8
- synth_ai/lm/overrides.py +21 -19
- synth_ai/lm/provider_support/__init__.py +1 -1
- synth_ai/lm/provider_support/anthropic.py +15 -15
- synth_ai/lm/provider_support/openai.py +23 -21
- synth_ai/lm/structured_outputs/handler.py +34 -32
- synth_ai/lm/structured_outputs/inject.py +24 -27
- synth_ai/lm/structured_outputs/rehabilitate.py +19 -15
- synth_ai/lm/tools/base.py +17 -16
- synth_ai/lm/unified_interface.py +17 -18
- synth_ai/lm/vendors/base.py +20 -18
- synth_ai/lm/vendors/core/anthropic_api.py +36 -27
- synth_ai/lm/vendors/core/gemini_api.py +31 -36
- synth_ai/lm/vendors/core/mistral_api.py +19 -19
- synth_ai/lm/vendors/core/openai_api.py +11 -10
- synth_ai/lm/vendors/openai_standard.py +113 -87
- synth_ai/lm/vendors/openai_standard_responses.py +74 -61
- synth_ai/lm/vendors/retries.py +9 -1
- synth_ai/lm/vendors/supported/custom_endpoint.py +26 -26
- synth_ai/lm/vendors/supported/deepseek.py +10 -10
- synth_ai/lm/vendors/supported/grok.py +8 -8
- synth_ai/lm/vendors/supported/ollama.py +2 -1
- synth_ai/lm/vendors/supported/openrouter.py +11 -9
- synth_ai/lm/vendors/synth_client.py +69 -63
- synth_ai/lm/warmup.py +8 -7
- synth_ai/tracing/__init__.py +22 -10
- synth_ai/tracing_v1/__init__.py +22 -20
- synth_ai/tracing_v3/__init__.py +7 -7
- synth_ai/tracing_v3/abstractions.py +56 -52
- synth_ai/tracing_v3/config.py +4 -2
- synth_ai/tracing_v3/db_config.py +6 -8
- synth_ai/tracing_v3/decorators.py +29 -30
- synth_ai/tracing_v3/examples/basic_usage.py +12 -12
- synth_ai/tracing_v3/hooks.py +21 -21
- synth_ai/tracing_v3/llm_call_record_helpers.py +85 -98
- synth_ai/tracing_v3/lm_call_record_abstractions.py +2 -4
- synth_ai/tracing_v3/migration_helper.py +3 -5
- synth_ai/tracing_v3/replica_sync.py +30 -32
- synth_ai/tracing_v3/session_tracer.py +35 -29
- synth_ai/tracing_v3/storage/__init__.py +1 -1
- synth_ai/tracing_v3/storage/base.py +8 -7
- synth_ai/tracing_v3/storage/config.py +4 -4
- synth_ai/tracing_v3/storage/factory.py +4 -4
- synth_ai/tracing_v3/storage/utils.py +9 -9
- synth_ai/tracing_v3/turso/__init__.py +3 -3
- synth_ai/tracing_v3/turso/daemon.py +9 -9
- synth_ai/tracing_v3/turso/manager.py +60 -48
- synth_ai/tracing_v3/turso/models.py +24 -19
- synth_ai/tracing_v3/utils.py +5 -5
- synth_ai/tui/__main__.py +1 -1
- synth_ai/tui/cli/query_experiments.py +2 -3
- synth_ai/tui/cli/query_experiments_v3.py +2 -3
- synth_ai/tui/dashboard.py +97 -86
- synth_ai/v0/tracing/abstractions.py +28 -28
- synth_ai/v0/tracing/base_client.py +9 -9
- synth_ai/v0/tracing/client_manager.py +7 -7
- synth_ai/v0/tracing/config.py +7 -7
- synth_ai/v0/tracing/context.py +6 -6
- synth_ai/v0/tracing/decorators.py +6 -5
- synth_ai/v0/tracing/events/manage.py +1 -1
- synth_ai/v0/tracing/events/store.py +5 -4
- synth_ai/v0/tracing/immediate_client.py +4 -5
- synth_ai/v0/tracing/local.py +3 -3
- synth_ai/v0/tracing/log_client_base.py +4 -5
- synth_ai/v0/tracing/retry_queue.py +5 -6
- synth_ai/v0/tracing/trackers.py +25 -25
- synth_ai/v0/tracing/upload.py +6 -0
- synth_ai/v0/tracing_v1/__init__.py +1 -1
- synth_ai/v0/tracing_v1/abstractions.py +28 -28
- synth_ai/v0/tracing_v1/base_client.py +9 -9
- synth_ai/v0/tracing_v1/client_manager.py +7 -7
- synth_ai/v0/tracing_v1/config.py +7 -7
- synth_ai/v0/tracing_v1/context.py +6 -6
- synth_ai/v0/tracing_v1/decorators.py +7 -6
- synth_ai/v0/tracing_v1/events/manage.py +1 -1
- synth_ai/v0/tracing_v1/events/store.py +5 -4
- synth_ai/v0/tracing_v1/immediate_client.py +4 -5
- synth_ai/v0/tracing_v1/local.py +3 -3
- synth_ai/v0/tracing_v1/log_client_base.py +4 -5
- synth_ai/v0/tracing_v1/retry_queue.py +5 -6
- synth_ai/v0/tracing_v1/trackers.py +25 -25
- synth_ai/v0/tracing_v1/upload.py +25 -24
- synth_ai/zyk/__init__.py +1 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/METADATA +1 -11
- synth_ai-0.2.4.dev7.dist-info/RECORD +299 -0
- synth_ai-0.2.4.dev6.dist-info/RECORD +0 -299
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/top_level.txt +0 -0
synth_ai/cli/root.py
CHANGED
@@ -12,12 +12,11 @@ import signal
|
|
12
12
|
import subprocess
|
13
13
|
import sys
|
14
14
|
import time
|
15
|
-
from typing import Optional
|
16
15
|
|
17
16
|
import click
|
18
17
|
|
19
18
|
|
20
|
-
def find_sqld_binary() ->
|
19
|
+
def find_sqld_binary() -> str | None:
|
21
20
|
sqld_path = shutil.which("sqld")
|
22
21
|
if sqld_path:
|
23
22
|
return sqld_path
|
@@ -78,9 +77,25 @@ def cli():
|
|
78
77
|
@click.option("--env-port", default=8901, type=int, help="Port for environment service")
|
79
78
|
@click.option("--no-sqld", is_flag=True, help="Skip starting sqld daemon")
|
80
79
|
@click.option("--no-env", is_flag=True, help="Skip starting environment service")
|
81
|
-
@click.option(
|
82
|
-
|
83
|
-
|
80
|
+
@click.option(
|
81
|
+
"--reload/--no-reload",
|
82
|
+
default=False,
|
83
|
+
help="Enable auto-reload (default: off). Or set SYNTH_RELOAD=1",
|
84
|
+
)
|
85
|
+
@click.option(
|
86
|
+
"--force/--no-force",
|
87
|
+
default=True,
|
88
|
+
help="Kill any process already bound to --env-port without prompting",
|
89
|
+
)
|
90
|
+
def serve(
|
91
|
+
db_file: str,
|
92
|
+
sqld_port: int,
|
93
|
+
env_port: int,
|
94
|
+
no_sqld: bool,
|
95
|
+
no_env: bool,
|
96
|
+
reload: bool,
|
97
|
+
force: bool,
|
98
|
+
):
|
84
99
|
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
85
100
|
processes = []
|
86
101
|
|
@@ -100,11 +115,25 @@ def serve(db_file: str, sqld_port: int, env_port: int, no_sqld: bool, no_env: bo
|
|
100
115
|
|
101
116
|
if not no_sqld:
|
102
117
|
try:
|
103
|
-
result = subprocess.run(
|
118
|
+
result = subprocess.run(
|
119
|
+
["pgrep", "-f", f"sqld.*--http-listen-addr.*:{sqld_port}"],
|
120
|
+
capture_output=True,
|
121
|
+
text=True,
|
122
|
+
)
|
104
123
|
if result.returncode != 0:
|
105
124
|
sqld_bin = find_sqld_binary() or install_sqld()
|
106
125
|
click.echo(f"🗄️ Starting sqld (local only) on port {sqld_port}")
|
107
|
-
proc = subprocess.Popen(
|
126
|
+
proc = subprocess.Popen(
|
127
|
+
[
|
128
|
+
sqld_bin,
|
129
|
+
"--db-path",
|
130
|
+
db_file,
|
131
|
+
"--http-listen-addr",
|
132
|
+
f"127.0.0.1:{sqld_port}",
|
133
|
+
],
|
134
|
+
stdout=open("sqld.log", "w"), # noqa: SIM115
|
135
|
+
stderr=subprocess.STDOUT,
|
136
|
+
)
|
108
137
|
processes.append(proc)
|
109
138
|
time.sleep(2)
|
110
139
|
except FileNotFoundError:
|
@@ -118,6 +147,7 @@ def serve(db_file: str, sqld_port: int, env_port: int, no_sqld: bool, no_env: bo
|
|
118
147
|
# Ensure port is free
|
119
148
|
try:
|
120
149
|
import socket
|
150
|
+
|
121
151
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
122
152
|
in_use = s.connect_ex(("127.0.0.1", env_port)) == 0
|
123
153
|
except Exception:
|
@@ -125,7 +155,9 @@ def serve(db_file: str, sqld_port: int, env_port: int, no_sqld: bool, no_env: bo
|
|
125
155
|
if in_use:
|
126
156
|
pids: list[str] = []
|
127
157
|
try:
|
128
|
-
out = subprocess.run(
|
158
|
+
out = subprocess.run(
|
159
|
+
["lsof", "-ti", f":{env_port}"], capture_output=True, text=True
|
160
|
+
)
|
129
161
|
if out.returncode == 0 and out.stdout.strip():
|
130
162
|
pids = [p for p in out.stdout.strip().splitlines() if p]
|
131
163
|
except FileNotFoundError:
|
@@ -136,7 +168,9 @@ def serve(db_file: str, sqld_port: int, env_port: int, no_sqld: bool, no_env: bo
|
|
136
168
|
time.sleep(0.5)
|
137
169
|
else:
|
138
170
|
suffix = f" PIDs: {', '.join(pids)}" if pids else ""
|
139
|
-
if click.confirm(
|
171
|
+
if click.confirm(
|
172
|
+
f"⚠️ Port {env_port} is in use.{suffix} Kill and continue?", default=True
|
173
|
+
):
|
140
174
|
if pids:
|
141
175
|
subprocess.run(["kill", "-9", *pids], check=False)
|
142
176
|
time.sleep(0.5)
|
@@ -158,12 +192,24 @@ def serve(db_file: str, sqld_port: int, env_port: int, no_sqld: bool, no_env: bo
|
|
158
192
|
click.echo(" - Check sqld.log if database issues occur")
|
159
193
|
click.echo(" - Use Ctrl+C to stop all services")
|
160
194
|
reload_enabled = reload or (os.getenv("SYNTH_RELOAD", "0") == "1")
|
161
|
-
click.echo(
|
195
|
+
click.echo(
|
196
|
+
" - Auto-reload ENABLED (code changes restart service)"
|
197
|
+
if reload_enabled
|
198
|
+
else " - Auto-reload DISABLED (stable in-memory sessions)"
|
199
|
+
)
|
162
200
|
click.echo("")
|
163
201
|
|
164
202
|
uvicorn_cmd = [
|
165
|
-
sys.executable,
|
166
|
-
"
|
203
|
+
sys.executable,
|
204
|
+
"-m",
|
205
|
+
"uvicorn",
|
206
|
+
"synth_ai.environments.service.app:app",
|
207
|
+
"--host",
|
208
|
+
"0.0.0.0",
|
209
|
+
"--port",
|
210
|
+
str(env_port),
|
211
|
+
"--log-level",
|
212
|
+
"info",
|
167
213
|
]
|
168
214
|
if reload_enabled:
|
169
215
|
uvicorn_cmd.append("--reload")
|
@@ -181,4 +227,3 @@ def serve(db_file: str, sqld_port: int, env_port: int, no_sqld: bool, no_env: bo
|
|
181
227
|
pass
|
182
228
|
else:
|
183
229
|
click.echo("No services to start.")
|
184
|
-
|
synth_ai/cli/status.py
CHANGED
@@ -4,14 +4,13 @@ CLI: status of agent runs/versions and environment service.
|
|
4
4
|
"""
|
5
5
|
|
6
6
|
import asyncio
|
7
|
-
from typing import Optional
|
8
7
|
|
9
8
|
import click
|
9
|
+
import requests
|
10
|
+
from rich import box
|
10
11
|
from rich.console import Console
|
11
12
|
from rich.panel import Panel
|
12
13
|
from rich.table import Table
|
13
|
-
from rich import box
|
14
|
-
import requests
|
15
14
|
|
16
15
|
|
17
16
|
async def _db_stats(db_url: str) -> dict:
|
@@ -52,7 +51,7 @@ async def _db_stats(db_url: str) -> dict:
|
|
52
51
|
"""
|
53
52
|
)
|
54
53
|
if not versions.empty:
|
55
|
-
out["version_count"] = int(versions.iloc[0]["version_count"])
|
54
|
+
out["version_count"] = int(versions.iloc[0]["version_count"])
|
56
55
|
else:
|
57
56
|
out["version_count"] = 0
|
58
57
|
return out
|
@@ -108,7 +107,11 @@ def register(cli):
|
|
108
107
|
lines.append("")
|
109
108
|
lines.append(f"Env Service: {health_text} [dim]{service_url}[/dim]")
|
110
109
|
if envs_list:
|
111
|
-
lines.append(
|
110
|
+
lines.append(
|
111
|
+
"Environments: "
|
112
|
+
+ ", ".join(sorted(envs_list)[:10])
|
113
|
+
+ (" ..." if len(envs_list) > 10 else "")
|
114
|
+
)
|
112
115
|
|
113
116
|
panel_main = Panel("\n".join(lines), title="Synth AI Status", border_style="cyan")
|
114
117
|
console.print(panel_main)
|
@@ -116,7 +119,11 @@ def register(cli):
|
|
116
119
|
# Systems table
|
117
120
|
sys_df = stats.get("systems")
|
118
121
|
if sys_df is not None and not sys_df.empty:
|
119
|
-
tbl = Table(
|
122
|
+
tbl = Table(
|
123
|
+
title=f"Systems (versions: {stats.get('version_count', 0)})",
|
124
|
+
box=box.SIMPLE,
|
125
|
+
header_style="bold",
|
126
|
+
)
|
120
127
|
tbl.add_column("Type")
|
121
128
|
tbl.add_column("Count", justify="right")
|
122
129
|
for _, r in sys_df.iterrows():
|
synth_ai/cli/traces.py
CHANGED
@@ -3,14 +3,13 @@
|
|
3
3
|
CLI: basic info about traces (runs).
|
4
4
|
"""
|
5
5
|
|
6
|
-
import os
|
7
6
|
import asyncio
|
8
|
-
|
7
|
+
import os
|
9
8
|
|
10
9
|
import click
|
10
|
+
from rich import box
|
11
11
|
from rich.console import Console
|
12
12
|
from rich.table import Table
|
13
|
-
from rich import box
|
14
13
|
|
15
14
|
|
16
15
|
def register(cli):
|
@@ -35,7 +34,7 @@ def register(cli):
|
|
35
34
|
console.print(f"[red]No DB root found:[/red] {root}")
|
36
35
|
return
|
37
36
|
|
38
|
-
entries:
|
37
|
+
entries: list[tuple[str, str]] = []
|
39
38
|
for name in sorted(os.listdir(root)):
|
40
39
|
path = os.path.join(root, name)
|
41
40
|
data_path = os.path.join(path, "data")
|
@@ -51,38 +50,53 @@ def register(cli):
|
|
51
50
|
for dp, _, files in os.walk(dir_path):
|
52
51
|
for fn in files:
|
53
52
|
fp = os.path.join(dp, fn)
|
54
|
-
|
53
|
+
import contextlib
|
54
|
+
with contextlib.suppress(OSError):
|
55
55
|
total += os.path.getsize(fp)
|
56
|
-
except OSError:
|
57
|
-
pass
|
58
56
|
return total
|
59
57
|
|
60
|
-
async def db_counts(db_dir: str) ->
|
58
|
+
async def db_counts(db_dir: str) -> tuple[int, dict[str, int], int, str | None, int]:
|
61
59
|
data_file = os.path.join(db_dir, "data")
|
62
60
|
mgr = AsyncSQLTraceManager(f"sqlite+aiosqlite:///{data_file}")
|
63
61
|
await mgr.initialize()
|
64
62
|
try:
|
65
63
|
traces_df = await mgr.query_traces("SELECT COUNT(*) AS c FROM session_traces")
|
66
|
-
traces_count =
|
64
|
+
traces_count = (
|
65
|
+
int(traces_df.iloc[0]["c"])
|
66
|
+
if traces_df is not None and not traces_df.empty
|
67
|
+
else 0
|
68
|
+
)
|
67
69
|
try:
|
68
70
|
systems_df = await mgr.query_traces(
|
69
71
|
"SELECT system_type, COUNT(*) AS c FROM systems GROUP BY system_type"
|
70
72
|
)
|
71
|
-
system_counts =
|
72
|
-
|
73
|
-
|
74
|
-
|
73
|
+
system_counts = (
|
74
|
+
{
|
75
|
+
str(r["system_type"] or "-"): int(r["c"] or 0)
|
76
|
+
for _, r in (systems_df or []).iterrows()
|
77
|
+
}
|
78
|
+
if systems_df is not None and not systems_df.empty
|
79
|
+
else {}
|
80
|
+
)
|
75
81
|
except Exception:
|
76
82
|
system_counts = {}
|
77
83
|
try:
|
78
84
|
exps_df = await mgr.query_traces("SELECT COUNT(*) AS c FROM experiments")
|
79
|
-
exps_count =
|
85
|
+
exps_count = (
|
86
|
+
int(exps_df.iloc[0]["c"])
|
87
|
+
if exps_df is not None and not exps_df.empty
|
88
|
+
else 0
|
89
|
+
)
|
80
90
|
except Exception:
|
81
91
|
exps_count = 0
|
82
92
|
try:
|
83
|
-
last_df = await mgr.query_traces(
|
93
|
+
last_df = await mgr.query_traces(
|
94
|
+
"SELECT MAX(created_at) AS last_created_at FROM session_traces"
|
95
|
+
)
|
84
96
|
last_created = (
|
85
|
-
str(last_df.iloc[0]["last_created_at"])
|
97
|
+
str(last_df.iloc[0]["last_created_at"])
|
98
|
+
if last_df is not None and not last_df.empty
|
99
|
+
else None
|
86
100
|
)
|
87
101
|
except Exception:
|
88
102
|
last_created = None
|
@@ -102,13 +116,21 @@ def register(cli):
|
|
102
116
|
# DB summary table
|
103
117
|
summary = Table(title="Trace Databases", box=box.SIMPLE, header_style="bold")
|
104
118
|
for col in ["DB", "Traces", "Experiments", "Last Activity", "Size (GB)"]:
|
105
|
-
summary.add_column(
|
119
|
+
summary.add_column(
|
120
|
+
col, justify="right" if col in {"Traces", "Experiments"} else "left"
|
121
|
+
)
|
106
122
|
|
107
|
-
aggregate_systems:
|
123
|
+
aggregate_systems: dict[str, int] = {}
|
108
124
|
total_bytes = 0
|
109
|
-
for name, (
|
125
|
+
for name, (
|
126
|
+
traces_count,
|
127
|
+
system_counts,
|
128
|
+
experiments_count,
|
129
|
+
last_created_at,
|
130
|
+
size_bytes,
|
131
|
+
) in results:
|
110
132
|
total_bytes += int(size_bytes or 0)
|
111
|
-
gb =
|
133
|
+
gb = int(size_bytes or 0) / (1024**3)
|
112
134
|
summary.add_row(
|
113
135
|
name,
|
114
136
|
f"{traces_count:,}",
|
@@ -129,7 +151,9 @@ def register(cli):
|
|
129
151
|
st = Table(title="Per-System (all DBs)", box=box.SIMPLE, header_style="bold")
|
130
152
|
st.add_column("System")
|
131
153
|
st.add_column("Count", justify="right")
|
132
|
-
for sys_name, count in sorted(
|
154
|
+
for sys_name, count in sorted(
|
155
|
+
aggregate_systems.items(), key=lambda x: (-x[1], x[0])
|
156
|
+
):
|
133
157
|
st.add_row(sys_name or "-", f"{int(count):,}")
|
134
158
|
console.print(st)
|
135
159
|
|
synth_ai/cli/watch.py
CHANGED
@@ -7,26 +7,24 @@ Placed beside scope.txt for discoverability.
|
|
7
7
|
|
8
8
|
import asyncio
|
9
9
|
from datetime import datetime
|
10
|
-
from typing import Any
|
10
|
+
from typing import Any
|
11
11
|
|
12
12
|
import click
|
13
|
+
from rich import box
|
14
|
+
from rich.align import Align
|
13
15
|
from rich.console import Console, Group
|
14
|
-
from rich.table import Table
|
15
16
|
from rich.panel import Panel
|
16
|
-
from rich.
|
17
|
-
from rich.live import Live
|
18
|
-
from rich import box
|
19
|
-
import sys
|
17
|
+
from rich.table import Table
|
20
18
|
|
21
19
|
|
22
20
|
class _State:
|
23
21
|
def __init__(self):
|
24
22
|
self.view: str = "experiments" # experiments | experiment | usage | traces | recent
|
25
|
-
self.view_arg:
|
23
|
+
self.view_arg: str | None = None
|
26
24
|
self.limit: int = 20
|
27
25
|
self.hours: float = 24.0
|
28
26
|
self.last_msg: str = "Type 'help' for commands. 'q' to quit."
|
29
|
-
self.error:
|
27
|
+
self.error: str | None = None
|
30
28
|
# UI state for a visible, blinking cursor in the input field
|
31
29
|
self.cursor_on: bool = True
|
32
30
|
|
@@ -80,7 +78,7 @@ async def _fetch_experiments(db_url: str):
|
|
80
78
|
await db.close()
|
81
79
|
|
82
80
|
|
83
|
-
def _experiments_table(df, limit:
|
81
|
+
def _experiments_table(df, limit: int | None = None) -> Table:
|
84
82
|
table = Table(
|
85
83
|
title="Synth AI Experiments",
|
86
84
|
title_style="bold cyan",
|
@@ -90,15 +88,15 @@ def _experiments_table(df, limit: Optional[int] = None) -> Table:
|
|
90
88
|
pad_edge=False,
|
91
89
|
)
|
92
90
|
for col in ["ID", "Name", "Sessions", "Events", "Msgs", "Cost", "Tokens", "Created"]:
|
93
|
-
table.add_column(
|
91
|
+
table.add_column(
|
92
|
+
col, justify="right" if col in {"Sessions", "Events", "Msgs", "Tokens"} else "left"
|
93
|
+
)
|
94
94
|
|
95
95
|
if df is not None and not df.empty:
|
96
96
|
rows = df.itertuples(index=False)
|
97
|
-
count =
|
98
|
-
|
99
|
-
if limit is not None and count >= limit:
|
97
|
+
for count, row in enumerate(rows, start=1):
|
98
|
+
if limit is not None and count > limit:
|
100
99
|
break
|
101
|
-
count += 1
|
102
100
|
table.add_row(
|
103
101
|
_short_id(getattr(row, "experiment_id", "")),
|
104
102
|
str(getattr(row, "name", "Unnamed"))[:28],
|
@@ -115,7 +113,7 @@ def _experiments_table(df, limit: Optional[int] = None) -> Table:
|
|
115
113
|
return table
|
116
114
|
|
117
115
|
|
118
|
-
async def _experiment_detail(db_url: str, experiment_id: str) ->
|
116
|
+
async def _experiment_detail(db_url: str, experiment_id: str) -> dict[str, Any]:
|
119
117
|
from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
|
120
118
|
|
121
119
|
db = AsyncSQLTraceManager(db_url)
|
@@ -158,22 +156,24 @@ async def _experiment_detail(db_url: str, experiment_id: str) -> Dict[str, Any]:
|
|
158
156
|
await db.close()
|
159
157
|
|
160
158
|
|
161
|
-
def _render_experiment_panel(detail:
|
159
|
+
def _render_experiment_panel(detail: dict[str, Any]) -> Panel:
|
162
160
|
if detail.get("not_found"):
|
163
161
|
return Panel("No experiment found for given ID", title="Experiment", border_style="red")
|
164
162
|
|
165
163
|
exp = detail["experiment"]
|
166
164
|
stats = detail.get("stats")
|
167
|
-
lines:
|
165
|
+
lines: list[str] = []
|
168
166
|
lines.append(f"[bold]🧪 {exp['name']}[/bold] ([dim]{exp['experiment_id']}[/dim])")
|
169
167
|
if exp.get("description"):
|
170
|
-
lines.append(exp["description"])
|
168
|
+
lines.append(exp["description"])
|
171
169
|
lines.append("")
|
172
170
|
if stats is not None:
|
173
|
-
lines.append(
|
174
|
-
|
175
|
-
|
176
|
-
|
171
|
+
lines.append(
|
172
|
+
f"[bold]Stats[/bold] Events: {_format_int(stats['total_events'])} "
|
173
|
+
f"Messages: {_format_int(stats['total_messages'])} "
|
174
|
+
f"Cost: {_format_currency(float(stats['total_cost'] or 0.0))} "
|
175
|
+
f"Tokens: {_format_int(stats['total_tokens'])}"
|
176
|
+
)
|
177
177
|
lines.append(f"Created: {exp['created_at']}")
|
178
178
|
lines.append("")
|
179
179
|
sessions = detail.get("sessions", [])
|
@@ -215,7 +215,8 @@ def register(cli):
|
|
215
215
|
console.print(table)
|
216
216
|
console.print(
|
217
217
|
"\n[dim]Tip:[/dim] use [cyan]synth-ai experiment <id>[/cyan] for details, "
|
218
|
-
"[cyan]synth-ai usage[/cyan] for model usage.",
|
218
|
+
"[cyan]synth-ai usage[/cyan] for model usage.",
|
219
|
+
sep="",
|
219
220
|
)
|
220
221
|
|
221
222
|
asyncio.run(_run())
|
@@ -247,12 +248,13 @@ def register(cli):
|
|
247
248
|
help="Database URL",
|
248
249
|
)
|
249
250
|
@click.option("--model", "model_name", default=None, help="Filter by model name")
|
250
|
-
def usage(db_url: str, model_name:
|
251
|
+
def usage(db_url: str, model_name: str | None):
|
251
252
|
"""Show model usage statistics (tokens, cost)."""
|
252
253
|
console = Console()
|
253
254
|
|
254
255
|
async def _run():
|
255
256
|
from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
|
257
|
+
|
256
258
|
db = AsyncSQLTraceManager(db_url)
|
257
259
|
await db.initialize()
|
258
260
|
try:
|
@@ -379,7 +381,7 @@ async def _handle_command(cmd: str, state: _State):
|
|
379
381
|
state.last_msg = f"Unknown command: {cmd}"
|
380
382
|
|
381
383
|
|
382
|
-
def _parse_hours(args:
|
384
|
+
def _parse_hours(args: list[str]) -> float | None:
|
383
385
|
if not args:
|
384
386
|
return None
|
385
387
|
# Accept formats: "6", "--6", "-6", "6h"
|
@@ -393,8 +395,9 @@ def _parse_hours(args: List[str]) -> Optional[float]:
|
|
393
395
|
return None
|
394
396
|
|
395
397
|
|
396
|
-
async def _usage_table(db_url: str, model_name:
|
398
|
+
async def _usage_table(db_url: str, model_name: str | None):
|
397
399
|
from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
|
400
|
+
|
398
401
|
db = AsyncSQLTraceManager(db_url)
|
399
402
|
await db.initialize()
|
400
403
|
try:
|
@@ -416,6 +419,7 @@ async def _usage_table(db_url: str, model_name: Optional[str]):
|
|
416
419
|
|
417
420
|
async def _traces_table(db_url: str, limit: int):
|
418
421
|
from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
|
422
|
+
|
419
423
|
db = AsyncSQLTraceManager(db_url)
|
420
424
|
await db.initialize()
|
421
425
|
try:
|
@@ -425,16 +429,14 @@ async def _traces_table(db_url: str, limit: int):
|
|
425
429
|
|
426
430
|
table = Table(title="Recent Sessions", box=box.SIMPLE, header_style="bold")
|
427
431
|
for col in ["Session", "Experiment", "Events", "Msgs", "Timesteps", "Cost", "Created"]:
|
428
|
-
table.add_column(col, justify="right" if col in {"Events","Msgs","Timesteps"} else "left")
|
432
|
+
table.add_column(col, justify="right" if col in {"Events", "Msgs", "Timesteps"} else "left")
|
429
433
|
|
430
434
|
if df is None or df.empty:
|
431
435
|
table.add_row("-", "No sessions found", "-", "-", "-", "-", "-")
|
432
436
|
else:
|
433
|
-
count =
|
434
|
-
|
435
|
-
if count >= limit:
|
437
|
+
for count, (_, r) in enumerate(df.iterrows(), start=1):
|
438
|
+
if count > limit:
|
436
439
|
break
|
437
|
-
count += 1
|
438
440
|
table.add_row(
|
439
441
|
str(r.get("session_id", ""))[:10],
|
440
442
|
str(r.get("experiment_name", ""))[:24],
|
@@ -450,6 +452,7 @@ async def _traces_table(db_url: str, limit: int):
|
|
450
452
|
async def _recent_table(db_url: str, hours: float, limit: int):
|
451
453
|
# Inline the recent query to avoid cross-module coupling
|
452
454
|
from datetime import timedelta
|
455
|
+
|
453
456
|
from synth_ai.tracing_v3.turso.manager import AsyncSQLTraceManager
|
454
457
|
|
455
458
|
start_time = datetime.now() - timedelta(hours=hours)
|
@@ -483,18 +486,18 @@ async def _recent_table(db_url: str, hours: float, limit: int):
|
|
483
486
|
|
484
487
|
table = Table(title=f"Experiments in last {hours:g}h", header_style="bold", box=box.SIMPLE)
|
485
488
|
for col in ["Experiment", "Runs", "First", "Last", "Events", "Msgs", "Cost", "Tokens"]:
|
486
|
-
table.add_column(
|
489
|
+
table.add_column(
|
490
|
+
col, justify="right" if col in {"Runs", "Events", "Msgs", "Tokens"} else "left"
|
491
|
+
)
|
487
492
|
|
488
493
|
if df is None or df.empty:
|
489
494
|
table.add_row("-", "0", "-", "-", "-", "-", "-", "-")
|
490
495
|
else:
|
491
|
-
count =
|
492
|
-
|
493
|
-
if count >= limit:
|
496
|
+
for count, (_, r) in enumerate(df.iterrows(), start=1):
|
497
|
+
if count > limit:
|
494
498
|
break
|
495
|
-
count += 1
|
496
499
|
name = r.get("name") or "Unnamed"
|
497
|
-
exp_disp = f"{name[:28]} [dim]({(str(r.get('experiment_id',''))[:8])})[/dim]"
|
500
|
+
exp_disp = f"{name[:28]} [dim]({(str(r.get('experiment_id', ''))[:8])})[/dim]"
|
498
501
|
table.add_row(
|
499
502
|
exp_disp,
|
500
503
|
f"{int(r.get('runs', 0)):,}",
|
synth_ai/config/base_url.py
CHANGED
@@ -14,7 +14,6 @@ Normalization: ensure the returned URL ends with "/api".
|
|
14
14
|
import os
|
15
15
|
from typing import Literal
|
16
16
|
|
17
|
-
|
18
17
|
PROD_BASE_URL_DEFAULT = "https://agent-learning.onrender.com"
|
19
18
|
|
20
19
|
|
@@ -28,7 +27,7 @@ def _normalize_base(url: str) -> str:
|
|
28
27
|
return url
|
29
28
|
|
30
29
|
|
31
|
-
def get_learning_v2_base_url(mode: Literal["dev","prod"] = "prod") -> str:
|
30
|
+
def get_learning_v2_base_url(mode: Literal["dev", "prod"] = "prod") -> str:
|
32
31
|
if mode == "prod":
|
33
32
|
prod = os.getenv("SYNTH_PROD_BASE_URL") or PROD_BASE_URL_DEFAULT
|
34
33
|
return _normalize_base(prod)
|
@@ -50,4 +49,3 @@ def get_learning_v2_base_url(mode: Literal["dev","prod"] = "prod") -> str:
|
|
50
49
|
return _normalize_base(dev)
|
51
50
|
|
52
51
|
raise Exception()
|
53
|
-
|
synth_ai/core/experiment.py
CHANGED
@@ -3,11 +3,7 @@
|
|
3
3
|
__version__ = "0.1.5"
|
4
4
|
|
5
5
|
# Import key modules for easier access
|
6
|
-
from . import environment
|
7
|
-
from . import service
|
8
|
-
from . import stateful
|
9
|
-
from . import tasks
|
10
|
-
from . import examples
|
6
|
+
from . import environment, examples, service, stateful, tasks
|
11
7
|
|
12
8
|
__all__ = [
|
13
9
|
"environment",
|
@@ -32,4 +28,4 @@ if "src" not in sys.modules:
|
|
32
28
|
|
33
29
|
# Expose this package as src.synth_env
|
34
30
|
sys.modules["src.synth_env"] = sys.modules[__name__]
|
35
|
-
|
31
|
+
sys.modules["src"].synth_env = sys.modules[__name__]
|