synth-ai 0.2.10__py3-none-any.whl → 0.2.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synth-ai might be problematic. Click here for more details.
- examples/multi_step/task_app_config_notes.md +488 -0
- examples/warming_up_to_rl/configs/eval_stepwise_complex.toml +33 -0
- examples/warming_up_to_rl/configs/eval_stepwise_consistent.toml +26 -0
- examples/warming_up_to_rl/configs/eval_stepwise_per_achievement.toml +36 -0
- examples/warming_up_to_rl/configs/eval_stepwise_simple.toml +30 -0
- examples/warming_up_to_rl/run_eval.py +142 -25
- examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py +146 -2
- synth_ai/api/train/builders.py +25 -14
- synth_ai/api/train/cli.py +29 -6
- synth_ai/api/train/env_resolver.py +18 -19
- synth_ai/api/train/supported_algos.py +8 -5
- synth_ai/api/train/utils.py +6 -1
- synth_ai/cli/__init__.py +4 -2
- synth_ai/cli/_storage.py +19 -0
- synth_ai/cli/balance.py +14 -2
- synth_ai/cli/calc.py +37 -22
- synth_ai/cli/legacy_root_backup.py +12 -14
- synth_ai/cli/recent.py +12 -7
- synth_ai/cli/status.py +4 -3
- synth_ai/cli/task_apps.py +143 -137
- synth_ai/cli/traces.py +4 -3
- synth_ai/cli/watch.py +3 -2
- synth_ai/jobs/client.py +15 -3
- synth_ai/task/server.py +14 -7
- synth_ai/tracing_v3/decorators.py +51 -26
- synth_ai/tracing_v3/examples/basic_usage.py +12 -7
- synth_ai/tracing_v3/llm_call_record_helpers.py +107 -53
- synth_ai/tracing_v3/replica_sync.py +8 -4
- synth_ai/tracing_v3/storage/utils.py +11 -9
- synth_ai/tracing_v3/turso/__init__.py +12 -0
- synth_ai/tracing_v3/turso/daemon.py +2 -1
- synth_ai/tracing_v3/turso/native_manager.py +28 -15
- {synth_ai-0.2.10.dist-info → synth_ai-0.2.12.dist-info}/METADATA +4 -2
- {synth_ai-0.2.10.dist-info → synth_ai-0.2.12.dist-info}/RECORD +38 -31
- {synth_ai-0.2.10.dist-info → synth_ai-0.2.12.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.10.dist-info → synth_ai-0.2.12.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.10.dist-info → synth_ai-0.2.12.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.10.dist-info → synth_ai-0.2.12.dist-info}/top_level.txt +0 -0
synth_ai/cli/task_apps.py
CHANGED
|
@@ -28,10 +28,61 @@ except Exception: # pragma: no cover - fallback
|
|
|
28
28
|
import uuid
|
|
29
29
|
|
|
30
30
|
import click
|
|
31
|
+
from click.exceptions import Abort
|
|
31
32
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
# ---------------------------------------------------------------------------
|
|
34
|
+
# Dynamic imports to avoid hard dependencies during type checking.
|
|
35
|
+
# ---------------------------------------------------------------------------
|
|
36
|
+
ModalDeploymentConfigType = TaskAppConfigType = TaskAppEntryType = Any
|
|
37
|
+
|
|
38
|
+
try: # Resolve base URL defaults lazily
|
|
39
|
+
_config_module = importlib.import_module("synth_ai.config.base_url")
|
|
40
|
+
PROD_BASE_URL_DEFAULT = cast(str, _config_module.PROD_BASE_URL_DEFAULT)
|
|
41
|
+
except Exception: # pragma: no cover - fallback
|
|
42
|
+
PROD_BASE_URL_DEFAULT = "https://agent-learning.onrender.com"
|
|
43
|
+
|
|
44
|
+
try:
|
|
45
|
+
_task_apps_module = importlib.import_module("synth_ai.task.apps")
|
|
46
|
+
ModalDeploymentConfig = cast(
|
|
47
|
+
type[ModalDeploymentConfigType], _task_apps_module.ModalDeploymentConfig
|
|
48
|
+
)
|
|
49
|
+
TaskAppConfig = cast(type[TaskAppConfigType], _task_apps_module.TaskAppConfig)
|
|
50
|
+
TaskAppEntry = cast(type[TaskAppEntryType], _task_apps_module.TaskAppEntry)
|
|
51
|
+
registry = _task_apps_module.registry
|
|
52
|
+
except Exception as exc: # pragma: no cover - critical dependency
|
|
53
|
+
raise RuntimeError("Unable to load task app registry") from exc
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
_task_server_module = importlib.import_module("synth_ai.task.server")
|
|
57
|
+
create_task_app = _task_server_module.create_task_app
|
|
58
|
+
run_task_app = _task_server_module.run_task_app
|
|
59
|
+
except Exception as exc: # pragma: no cover - critical dependency
|
|
60
|
+
raise RuntimeError("Unable to load task app server utilities") from exc
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _load_demo_directory() -> Path | None:
|
|
64
|
+
"""Return the demo task apps directory if available."""
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
module = importlib.import_module("synth_ai.demos.demo_task_apps.core")
|
|
68
|
+
loader = module.load_demo_dir
|
|
69
|
+
demo_dir = loader()
|
|
70
|
+
if isinstance(demo_dir, (str, Path)):
|
|
71
|
+
demo_path = Path(demo_dir)
|
|
72
|
+
if demo_path.exists():
|
|
73
|
+
return demo_path.resolve()
|
|
74
|
+
except Exception:
|
|
75
|
+
return None
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _maybe_import(name: str) -> Any:
|
|
80
|
+
"""Safely import a module by name and return it, or None on failure."""
|
|
81
|
+
|
|
82
|
+
try:
|
|
83
|
+
return importlib.import_module(name)
|
|
84
|
+
except Exception:
|
|
85
|
+
return None
|
|
35
86
|
|
|
36
87
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
37
88
|
|
|
@@ -62,12 +113,12 @@ class AppChoice:
|
|
|
62
113
|
source: str
|
|
63
114
|
description: str | None = None
|
|
64
115
|
aliases: tuple[str, ...] = ()
|
|
65
|
-
entry:
|
|
66
|
-
entry_loader: Callable[[],
|
|
116
|
+
entry: TaskAppEntryType | None = None
|
|
117
|
+
entry_loader: Callable[[], TaskAppEntryType] | None = None
|
|
67
118
|
modal_script: Path | None = None
|
|
68
119
|
lineno: int | None = None
|
|
69
120
|
|
|
70
|
-
def ensure_entry(self) ->
|
|
121
|
+
def ensure_entry(self) -> TaskAppEntryType:
|
|
71
122
|
if self.entry is not None:
|
|
72
123
|
return self.entry
|
|
73
124
|
if self.entry_loader is None:
|
|
@@ -152,17 +203,9 @@ def _candidate_search_roots() -> list[Path]:
|
|
|
152
203
|
"""Only search for task apps in the current working directory and subdirectories."""
|
|
153
204
|
roots: list[Path] = []
|
|
154
205
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
demo_dir = load_demo_dir()
|
|
160
|
-
if demo_dir:
|
|
161
|
-
demo_path = Path(demo_dir)
|
|
162
|
-
if demo_path.exists() and demo_path.is_dir():
|
|
163
|
-
roots.append(demo_path.resolve())
|
|
164
|
-
except Exception:
|
|
165
|
-
pass
|
|
206
|
+
demo_path = _load_demo_directory()
|
|
207
|
+
if demo_path is not None and demo_path.is_dir():
|
|
208
|
+
roots.append(demo_path)
|
|
166
209
|
|
|
167
210
|
# Allow explicit search paths via environment variable
|
|
168
211
|
env_paths = os.environ.get("SYNTH_TASK_APP_SEARCH_PATH")
|
|
@@ -359,7 +402,7 @@ def _collect_task_app_choices() -> list[AppChoice]:
|
|
|
359
402
|
|
|
360
403
|
choices: list[AppChoice] = []
|
|
361
404
|
with contextlib.suppress(Exception):
|
|
362
|
-
|
|
405
|
+
_maybe_import("synth_ai.demos.demo_task_apps")
|
|
363
406
|
# Only use discovered task apps, not registered ones (since we moved them to examples)
|
|
364
407
|
choices.extend(_collect_scanned_task_configs())
|
|
365
408
|
choices.extend(_collect_modal_scripts())
|
|
@@ -515,16 +558,9 @@ def _app_choice_sort_key(choice: AppChoice) -> tuple[int, int, int, int, int, st
|
|
|
515
558
|
|
|
516
559
|
# Further prioritize apps in the demo directory if one is set
|
|
517
560
|
demo_rank = 1
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
demo_dir = load_demo_dir()
|
|
522
|
-
if demo_dir:
|
|
523
|
-
demo_path = Path(demo_dir).resolve()
|
|
524
|
-
if choice.path.is_relative_to(demo_path):
|
|
525
|
-
demo_rank = 0
|
|
526
|
-
except Exception:
|
|
527
|
-
pass
|
|
561
|
+
demo_dir = _load_demo_directory()
|
|
562
|
+
if demo_dir and choice.path.is_relative_to(demo_dir):
|
|
563
|
+
demo_rank = 0
|
|
528
564
|
|
|
529
565
|
modal_rank = 1 if choice.modal_script else 0
|
|
530
566
|
|
|
@@ -598,7 +634,7 @@ def _has_modal_support_in_file(path: Path) -> bool:
|
|
|
598
634
|
return False
|
|
599
635
|
|
|
600
636
|
|
|
601
|
-
def _extract_modal_config_from_file(path: Path) ->
|
|
637
|
+
def _extract_modal_config_from_file(path: Path) -> ModalDeploymentConfigType | None:
|
|
602
638
|
"""Extract ModalDeploymentConfig from a file by parsing the AST."""
|
|
603
639
|
try:
|
|
604
640
|
source = path.read_text(encoding="utf-8")
|
|
@@ -629,7 +665,7 @@ def _extract_modal_config_from_file(path: Path) -> ModalDeploymentConfig | None:
|
|
|
629
665
|
return None
|
|
630
666
|
|
|
631
667
|
|
|
632
|
-
def _build_modal_config_from_ast(modal_call: ast.Call) ->
|
|
668
|
+
def _build_modal_config_from_ast(modal_call: ast.Call) -> ModalDeploymentConfigType | None:
|
|
633
669
|
"""Build a ModalDeploymentConfig from an AST Call node."""
|
|
634
670
|
try:
|
|
635
671
|
# Extract keyword arguments
|
|
@@ -637,44 +673,41 @@ def _build_modal_config_from_ast(modal_call: ast.Call) -> ModalDeploymentConfig
|
|
|
637
673
|
for kw in modal_call.keywords:
|
|
638
674
|
if kw.arg and isinstance(kw.value, ast.Constant):
|
|
639
675
|
kwargs[kw.arg] = kw.value.value
|
|
640
|
-
elif kw.arg == "pip_packages" and isinstance(kw.value, ast.List
|
|
676
|
+
elif kw.arg == "pip_packages" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
641
677
|
# Handle pip_packages list/tuple
|
|
642
|
-
packages = []
|
|
678
|
+
packages: list[str] = []
|
|
643
679
|
for elt in kw.value.elts:
|
|
644
680
|
if isinstance(elt, ast.Constant):
|
|
645
681
|
packages.append(elt.value)
|
|
646
682
|
kwargs[kw.arg] = tuple(packages)
|
|
647
|
-
elif kw.arg == "extra_local_dirs" and isinstance(kw.value, ast.List
|
|
683
|
+
elif kw.arg == "extra_local_dirs" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
648
684
|
# Handle extra_local_dirs list/tuple of tuples
|
|
649
685
|
dirs = []
|
|
650
686
|
for elt in kw.value.elts:
|
|
651
|
-
if isinstance(elt, ast.List
|
|
687
|
+
if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
|
|
652
688
|
src = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
|
|
653
689
|
dst = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
|
|
654
690
|
if src and dst:
|
|
655
691
|
dirs.append((src, dst))
|
|
656
692
|
kwargs[kw.arg] = tuple(dirs)
|
|
657
|
-
elif kw.arg == "secret_names" and isinstance(kw.value, ast.List
|
|
693
|
+
elif kw.arg == "secret_names" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
658
694
|
# Handle secret_names list/tuple
|
|
659
695
|
secrets = []
|
|
660
696
|
for elt in kw.value.elts:
|
|
661
697
|
if isinstance(elt, ast.Constant):
|
|
662
698
|
secrets.append(elt.value)
|
|
663
699
|
kwargs[kw.arg] = tuple(secrets)
|
|
664
|
-
elif kw.arg == "volume_mounts" and isinstance(kw.value, ast.List
|
|
700
|
+
elif kw.arg == "volume_mounts" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
665
701
|
# Handle volume_mounts list/tuple of tuples
|
|
666
702
|
mounts = []
|
|
667
703
|
for elt in kw.value.elts:
|
|
668
|
-
if isinstance(elt, ast.List
|
|
704
|
+
if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
|
|
669
705
|
name = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
|
|
670
706
|
mount = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
|
|
671
707
|
if name and mount:
|
|
672
708
|
mounts.append((name, mount))
|
|
673
709
|
kwargs[kw.arg] = tuple(mounts)
|
|
674
710
|
|
|
675
|
-
# Create ModalDeploymentConfig with extracted arguments
|
|
676
|
-
from synth_ai.task.apps import ModalDeploymentConfig
|
|
677
|
-
|
|
678
711
|
return ModalDeploymentConfig(**kwargs)
|
|
679
712
|
except Exception:
|
|
680
713
|
return None
|
|
@@ -713,7 +746,7 @@ def _prompt_user_for_choice(choices: list[AppChoice]) -> AppChoice:
|
|
|
713
746
|
click.echo(_format_choice(choice, idx))
|
|
714
747
|
try:
|
|
715
748
|
response = click.prompt("Enter choice", default="1", type=str).strip() or "1"
|
|
716
|
-
except (
|
|
749
|
+
except (Abort, EOFError, KeyboardInterrupt) as exc:
|
|
717
750
|
raise click.ClickException("Task app selection cancelled by user") from exc
|
|
718
751
|
if not response.isdigit():
|
|
719
752
|
raise click.ClickException("Selection must be a number")
|
|
@@ -798,7 +831,7 @@ def _import_task_app_module(
|
|
|
798
831
|
|
|
799
832
|
def _load_entry_from_path(
|
|
800
833
|
path: Path, app_id: str, module_search_roots: Sequence[Path] | None = None
|
|
801
|
-
) ->
|
|
834
|
+
) -> TaskAppEntryType:
|
|
802
835
|
resolved = path.resolve()
|
|
803
836
|
search_roots: list[Path] = []
|
|
804
837
|
seen_roots: set[Path] = set()
|
|
@@ -849,8 +882,8 @@ def _load_entry_from_path(
|
|
|
849
882
|
detail = last_error or exc
|
|
850
883
|
raise click.ClickException(f"Failed to import {resolved}: {detail}") from detail
|
|
851
884
|
|
|
852
|
-
config_obj:
|
|
853
|
-
factory_callable: Callable[[],
|
|
885
|
+
config_obj: TaskAppConfigType | None = None
|
|
886
|
+
factory_callable: Callable[[], TaskAppConfigType] | None = None
|
|
854
887
|
|
|
855
888
|
for attr_name in dir(module):
|
|
856
889
|
try:
|
|
@@ -860,7 +893,7 @@ def _load_entry_from_path(
|
|
|
860
893
|
if isinstance(attr, TaskAppConfig) and attr.app_id == app_id:
|
|
861
894
|
config_obj = attr
|
|
862
895
|
|
|
863
|
-
def _return_config(cfg:
|
|
896
|
+
def _return_config(cfg: TaskAppConfigType = attr) -> TaskAppConfigType:
|
|
864
897
|
return cfg
|
|
865
898
|
|
|
866
899
|
factory_callable = _return_config
|
|
@@ -900,8 +933,8 @@ def _load_entry_from_path(
|
|
|
900
933
|
bound_func: Callable[[], TaskAppConfig] = cast(Callable[[], TaskAppConfig], attr) # type: ignore[assignment]
|
|
901
934
|
|
|
902
935
|
def _factory_noargs(
|
|
903
|
-
func: Callable[[],
|
|
904
|
-
) ->
|
|
936
|
+
func: Callable[[], TaskAppConfigType] = bound_func,
|
|
937
|
+
) -> TaskAppConfigType:
|
|
905
938
|
return func()
|
|
906
939
|
|
|
907
940
|
factory_callable = _factory_noargs
|
|
@@ -919,7 +952,7 @@ def _load_entry_from_path(
|
|
|
919
952
|
f"Could not locate TaskAppConfig for '{app_id}' in {resolved}."
|
|
920
953
|
) from exc
|
|
921
954
|
|
|
922
|
-
modal_cfg:
|
|
955
|
+
modal_cfg: ModalDeploymentConfigType | None = None
|
|
923
956
|
for attr_name in dir(module):
|
|
924
957
|
try:
|
|
925
958
|
attr = getattr(module, attr_name)
|
|
@@ -1008,7 +1041,6 @@ def _modal_command_prefix(modal_cli: str) -> list[str]:
|
|
|
1008
1041
|
def _build_modal_app_wrapper(original_script: Path) -> tuple[Path, Path]:
|
|
1009
1042
|
source_dir = original_script.parent.resolve()
|
|
1010
1043
|
repo_root = REPO_ROOT
|
|
1011
|
-
synth_src = (repo_root / "synth_ai").resolve()
|
|
1012
1044
|
temp_root = Path(tempfile.mkdtemp(prefix="synth_modal_app_"))
|
|
1013
1045
|
|
|
1014
1046
|
wrapper_source = textwrap.dedent(
|
|
@@ -1103,7 +1135,7 @@ def _run_modal_script(
|
|
|
1103
1135
|
try:
|
|
1104
1136
|
_preflight_env_key(env_paths_list, crash_on_failure=True)
|
|
1105
1137
|
except Exception as _pf_err:
|
|
1106
|
-
raise click.ClickException(str(_pf_err))
|
|
1138
|
+
raise click.ClickException(str(_pf_err)) from _pf_err
|
|
1107
1139
|
|
|
1108
1140
|
proc_env = os.environ.copy()
|
|
1109
1141
|
pythonpath_entries: list[str] = []
|
|
@@ -1172,10 +1204,8 @@ def _run_modal_script(
|
|
|
1172
1204
|
finally:
|
|
1173
1205
|
if wrapper_info is not None:
|
|
1174
1206
|
wrapper_path, temp_root = wrapper_info
|
|
1175
|
-
|
|
1207
|
+
with contextlib.suppress(Exception):
|
|
1176
1208
|
wrapper_path.unlink(missing_ok=True)
|
|
1177
|
-
except Exception:
|
|
1178
|
-
pass
|
|
1179
1209
|
shutil.rmtree(temp_root, ignore_errors=True)
|
|
1180
1210
|
|
|
1181
1211
|
|
|
@@ -1201,10 +1231,12 @@ def _preflight_env_key(env_paths: Sequence[Path] | None = None, *, crash_on_fail
|
|
|
1201
1231
|
|
|
1202
1232
|
minted = False
|
|
1203
1233
|
if not env_api_key:
|
|
1234
|
+
secrets_module = _maybe_import("synth_ai.learning.rl.secrets")
|
|
1204
1235
|
try:
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1236
|
+
if secrets_module is None:
|
|
1237
|
+
raise RuntimeError("secrets module unavailable")
|
|
1238
|
+
mint_env_key = secrets_module.mint_environment_api_key
|
|
1239
|
+
env_api_key = mint_env_key()
|
|
1208
1240
|
os.environ["ENVIRONMENT_API_KEY"] = env_api_key
|
|
1209
1241
|
os.environ.setdefault("DEV_ENVIRONMENT_API_KEY", env_api_key)
|
|
1210
1242
|
minted = True
|
|
@@ -1347,8 +1379,8 @@ def _preflight_env_key(env_paths: Sequence[Path] | None = None, *, crash_on_fail
|
|
|
1347
1379
|
|
|
1348
1380
|
|
|
1349
1381
|
def _run_modal_with_entry(
|
|
1350
|
-
entry:
|
|
1351
|
-
modal_cfg:
|
|
1382
|
+
entry: TaskAppEntryType,
|
|
1383
|
+
modal_cfg: ModalDeploymentConfigType,
|
|
1352
1384
|
modal_cli: str,
|
|
1353
1385
|
modal_name: str | None,
|
|
1354
1386
|
env_paths: list[Path],
|
|
@@ -1504,7 +1536,7 @@ def _interactive_fill_env(env_path: Path) -> Path | None:
|
|
|
1504
1536
|
value = click.prompt(
|
|
1505
1537
|
label, default=default, show_default=bool(default) or not required
|
|
1506
1538
|
).strip()
|
|
1507
|
-
except (
|
|
1539
|
+
except (Abort, EOFError, KeyboardInterrupt):
|
|
1508
1540
|
click.echo("Aborted env creation.")
|
|
1509
1541
|
return None
|
|
1510
1542
|
if value or not required:
|
|
@@ -1545,7 +1577,7 @@ def _ensure_env_values(env_paths: list[Path], fallback_dir: Path) -> None:
|
|
|
1545
1577
|
|
|
1546
1578
|
|
|
1547
1579
|
def _deploy_entry(
|
|
1548
|
-
entry:
|
|
1580
|
+
entry: TaskAppEntryType,
|
|
1549
1581
|
modal_name: str | None,
|
|
1550
1582
|
dry_run: bool,
|
|
1551
1583
|
modal_cli: str,
|
|
@@ -1573,7 +1605,7 @@ def _deploy_entry(
|
|
|
1573
1605
|
|
|
1574
1606
|
|
|
1575
1607
|
def _modal_serve_entry(
|
|
1576
|
-
entry:
|
|
1608
|
+
entry: TaskAppEntryType,
|
|
1577
1609
|
modal_name: str | None,
|
|
1578
1610
|
modal_cli: str,
|
|
1579
1611
|
env_file: Sequence[str],
|
|
@@ -1673,20 +1705,15 @@ def serve_command(
|
|
|
1673
1705
|
trace_dir: str | None,
|
|
1674
1706
|
trace_db: str | None,
|
|
1675
1707
|
) -> None:
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
demo_dir = load_demo_dir()
|
|
1680
|
-
if demo_dir:
|
|
1681
|
-
demo_path = Path(demo_dir)
|
|
1682
|
-
if not demo_path.is_dir():
|
|
1708
|
+
demo_dir_path = _load_demo_directory()
|
|
1709
|
+
if demo_dir_path:
|
|
1710
|
+
if not demo_dir_path.is_dir():
|
|
1683
1711
|
raise click.ClickException(
|
|
1684
|
-
f"Demo directory not found: {
|
|
1712
|
+
f"Demo directory not found: {demo_dir_path}\nRun 'synth-ai setup' to create a demo."
|
|
1685
1713
|
)
|
|
1686
|
-
os.chdir(
|
|
1687
|
-
click.echo(f"Using demo directory: {
|
|
1688
|
-
|
|
1689
|
-
os.environ["SYNTH_DEMO_DIR"] = str(demo_path.resolve())
|
|
1714
|
+
os.chdir(demo_dir_path)
|
|
1715
|
+
click.echo(f"Using demo directory: {demo_dir_path}\n")
|
|
1716
|
+
os.environ["SYNTH_DEMO_DIR"] = str(demo_dir_path.resolve())
|
|
1690
1717
|
|
|
1691
1718
|
# Prompt for port if not provided
|
|
1692
1719
|
if port is None:
|
|
@@ -1752,9 +1779,10 @@ def info_command(base_url: str | None, api_key: str | None, seeds: tuple[int, ..
|
|
|
1752
1779
|
base = (base_url or _os.getenv("TASK_APP_BASE_URL") or "http://127.0.0.1:8001").rstrip("/")
|
|
1753
1780
|
|
|
1754
1781
|
# Resolve API key, permitting dev fallbacks
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1782
|
+
auth_module = _maybe_import("synth_ai.task.auth")
|
|
1783
|
+
if auth_module is not None:
|
|
1784
|
+
_norm_key = getattr(auth_module, "normalize_environment_api_key", lambda: _os.getenv("ENVIRONMENT_API_KEY"))
|
|
1785
|
+
else:
|
|
1758
1786
|
_norm_key = lambda: _os.getenv("ENVIRONMENT_API_KEY") # noqa: E731
|
|
1759
1787
|
key = (api_key or _norm_key() or "").strip()
|
|
1760
1788
|
if not key:
|
|
@@ -1831,20 +1859,15 @@ def serve_task_group(
|
|
|
1831
1859
|
trace_dir: str | None,
|
|
1832
1860
|
trace_db: str | None,
|
|
1833
1861
|
) -> None:
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
demo_dir = load_demo_dir()
|
|
1838
|
-
if demo_dir:
|
|
1839
|
-
demo_path = Path(demo_dir)
|
|
1840
|
-
if not demo_path.is_dir():
|
|
1862
|
+
demo_dir_path = _load_demo_directory()
|
|
1863
|
+
if demo_dir_path:
|
|
1864
|
+
if not demo_dir_path.is_dir():
|
|
1841
1865
|
raise click.ClickException(
|
|
1842
|
-
f"Demo directory not found: {
|
|
1866
|
+
f"Demo directory not found: {demo_dir_path}\nRun 'synth-ai setup' to create a demo."
|
|
1843
1867
|
)
|
|
1844
|
-
os.chdir(
|
|
1845
|
-
click.echo(f"Using demo directory: {
|
|
1846
|
-
|
|
1847
|
-
os.environ["SYNTH_DEMO_DIR"] = str(demo_path.resolve())
|
|
1868
|
+
os.chdir(demo_dir_path)
|
|
1869
|
+
click.echo(f"Using demo directory: {demo_dir_path}\n")
|
|
1870
|
+
os.environ["SYNTH_DEMO_DIR"] = str(demo_dir_path.resolve())
|
|
1848
1871
|
|
|
1849
1872
|
# Prompt for port if not provided
|
|
1850
1873
|
if port is None:
|
|
@@ -1881,7 +1904,7 @@ def serve_task_group(
|
|
|
1881
1904
|
)
|
|
1882
1905
|
|
|
1883
1906
|
|
|
1884
|
-
def _determine_env_files(entry:
|
|
1907
|
+
def _determine_env_files(entry: TaskAppEntryType, user_env_files: Sequence[str]) -> list[Path]:
|
|
1885
1908
|
resolved: list[Path] = []
|
|
1886
1909
|
for candidate in user_env_files:
|
|
1887
1910
|
p = Path(candidate).expanduser()
|
|
@@ -2068,17 +2091,10 @@ def _validate_required_env_keys() -> None:
|
|
|
2068
2091
|
def _print_demo_next_steps_if_applicable() -> None:
|
|
2069
2092
|
"""Print next steps if currently in a demo directory."""
|
|
2070
2093
|
try:
|
|
2071
|
-
from synth_ai.demos.demo_task_apps.core import load_demo_dir
|
|
2072
|
-
|
|
2073
2094
|
cwd = Path.cwd().resolve()
|
|
2074
|
-
demo_dir =
|
|
2095
|
+
demo_dir = _load_demo_directory()
|
|
2075
2096
|
|
|
2076
|
-
|
|
2077
|
-
if (
|
|
2078
|
-
demo_dir
|
|
2079
|
-
and Path(demo_dir).resolve() == cwd
|
|
2080
|
-
and (cwd / "run_local_rollout_traced.py").exists()
|
|
2081
|
-
):
|
|
2097
|
+
if demo_dir and demo_dir == cwd and (cwd / "run_local_rollout_traced.py").exists():
|
|
2082
2098
|
click.echo("\n" + "=" * 60)
|
|
2083
2099
|
click.echo("Next step: Collect traced rollouts")
|
|
2084
2100
|
click.echo("=" * 60)
|
|
@@ -2088,12 +2104,11 @@ def _print_demo_next_steps_if_applicable() -> None:
|
|
|
2088
2104
|
click.echo("\nRun this 5-10 times to collect diverse traces.")
|
|
2089
2105
|
click.echo("=" * 60 + "\n")
|
|
2090
2106
|
except Exception:
|
|
2091
|
-
# Silently fail - this is just a helpful printout
|
|
2092
2107
|
pass
|
|
2093
2108
|
|
|
2094
2109
|
|
|
2095
2110
|
def _serve_entry(
|
|
2096
|
-
entry:
|
|
2111
|
+
entry: TaskAppEntryType,
|
|
2097
2112
|
host: str,
|
|
2098
2113
|
port: int,
|
|
2099
2114
|
env_file: Sequence[str],
|
|
@@ -2134,13 +2149,13 @@ def _serve_entry(
|
|
|
2134
2149
|
os.environ["SQLD_DB_PATH"] = str(db_path)
|
|
2135
2150
|
os.environ["TURSO_LOCAL_DB_URL"] = db_url
|
|
2136
2151
|
click.echo(f"Tracing DB path set to {db_path}")
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2152
|
+
tracing_config_module = _maybe_import("synth_ai.tracing_v3.config")
|
|
2153
|
+
if tracing_config_module is not None:
|
|
2154
|
+
trace_config = tracing_config_module.CONFIG
|
|
2155
|
+
new_db_url = os.getenv("TURSO_LOCAL_DB_URL") or trace_config.db_url
|
|
2156
|
+
trace_config.db_url = new_db_url
|
|
2157
|
+
if new_db_url:
|
|
2158
|
+
click.echo(f"Tracing DB URL resolved to {new_db_url}")
|
|
2144
2159
|
elif os.getenv("TASKAPP_TRACING_ENABLED"):
|
|
2145
2160
|
click.echo("Tracing enabled via environment variables")
|
|
2146
2161
|
|
|
@@ -2183,18 +2198,14 @@ def deploy_app(
|
|
|
2183
2198
|
) -> None:
|
|
2184
2199
|
"""Deploy a task app to Modal."""
|
|
2185
2200
|
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
demo_dir = load_demo_dir()
|
|
2190
|
-
if demo_dir:
|
|
2191
|
-
demo_path = Path(demo_dir)
|
|
2192
|
-
if not demo_path.is_dir():
|
|
2201
|
+
demo_dir_path = _load_demo_directory()
|
|
2202
|
+
if demo_dir_path:
|
|
2203
|
+
if not demo_dir_path.is_dir():
|
|
2193
2204
|
raise click.ClickException(
|
|
2194
|
-
f"Demo directory not found: {
|
|
2205
|
+
f"Demo directory not found: {demo_dir_path}\nRun 'synth-ai demo' to create a demo."
|
|
2195
2206
|
)
|
|
2196
|
-
os.chdir(
|
|
2197
|
-
click.echo(f"Using demo directory: {
|
|
2207
|
+
os.chdir(demo_dir_path)
|
|
2208
|
+
click.echo(f"Using demo directory: {demo_dir_path}\n")
|
|
2198
2209
|
|
|
2199
2210
|
choice = _select_app_choice(app_id, purpose="deploy")
|
|
2200
2211
|
|
|
@@ -2241,8 +2252,8 @@ def modal_serve_app(
|
|
|
2241
2252
|
|
|
2242
2253
|
|
|
2243
2254
|
def _write_modal_entrypoint(
|
|
2244
|
-
entry:
|
|
2245
|
-
modal_cfg:
|
|
2255
|
+
entry: TaskAppEntryType,
|
|
2256
|
+
modal_cfg: ModalDeploymentConfigType,
|
|
2246
2257
|
override_name: str | None,
|
|
2247
2258
|
*,
|
|
2248
2259
|
dotenv_paths: Sequence[str] | None = None,
|
|
@@ -2291,30 +2302,25 @@ def _write_modal_entrypoint(
|
|
|
2291
2302
|
pip_packages = list(modal_cfg.pip_packages)
|
|
2292
2303
|
# Ensure synth-ai (matching host version if available) is installed in the container
|
|
2293
2304
|
synth_pkg = "synth-ai"
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
host_ver = getattr(_host_synth, "__version__", None)
|
|
2305
|
+
host_synth = _maybe_import("synth_ai")
|
|
2306
|
+
if host_synth is not None:
|
|
2307
|
+
host_ver = getattr(host_synth, "__version__", None)
|
|
2298
2308
|
if host_ver:
|
|
2299
2309
|
synth_pkg = f"synth-ai=={host_ver}"
|
|
2300
|
-
except Exception:
|
|
2301
|
-
pass
|
|
2302
2310
|
if not any(str(p).startswith("synth-ai") for p in pip_packages):
|
|
2303
2311
|
pip_packages.insert(0, synth_pkg)
|
|
2304
2312
|
|
|
2305
2313
|
local_dirs = [(str(Path(src)), dst) for src, dst in modal_cfg.extra_local_dirs]
|
|
2306
2314
|
# Also mount the host synth_ai source if available to ensure latest code is used
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
except Exception:
|
|
2317
|
-
pass
|
|
2315
|
+
if host_synth is not None:
|
|
2316
|
+
try:
|
|
2317
|
+
host_synth_dir = Path(host_synth.__file__).resolve().parent
|
|
2318
|
+
sy_dst = "/opt/synth_ai_repo/synth_ai"
|
|
2319
|
+
candidate = (str(host_synth_dir), sy_dst)
|
|
2320
|
+
if candidate not in local_dirs:
|
|
2321
|
+
local_dirs.insert(0, candidate)
|
|
2322
|
+
except Exception:
|
|
2323
|
+
pass
|
|
2318
2324
|
# Ensure the discovered app directory is mounted, regardless of modal_cfg
|
|
2319
2325
|
if original_path:
|
|
2320
2326
|
discovered_dir = str(Path(original_path).resolve().parent)
|
|
@@ -2544,7 +2550,7 @@ def eval_command(
|
|
|
2544
2550
|
elif isinstance(ef, list):
|
|
2545
2551
|
env_file = tuple(str(x) for x in ef) # type: ignore[assignment]
|
|
2546
2552
|
|
|
2547
|
-
entry:
|
|
2553
|
+
entry: TaskAppEntryType | None = None
|
|
2548
2554
|
if task_app_url is None:
|
|
2549
2555
|
choice = _select_app_choice(app_id, purpose="eval")
|
|
2550
2556
|
entry = choice.ensure_entry()
|
synth_ai/cli/traces.py
CHANGED
|
@@ -11,6 +11,8 @@ from rich import box
|
|
|
11
11
|
from rich.console import Console
|
|
12
12
|
from rich.table import Table
|
|
13
13
|
|
|
14
|
+
from ._storage import load_storage
|
|
15
|
+
|
|
14
16
|
|
|
15
17
|
def register(cli):
|
|
16
18
|
@cli.command()
|
|
@@ -26,8 +28,6 @@ def register(cli):
|
|
|
26
28
|
console = Console()
|
|
27
29
|
|
|
28
30
|
async def _run():
|
|
29
|
-
from synth_ai.tracing_v3.storage.factory import StorageConfig, create_storage
|
|
30
|
-
|
|
31
31
|
# Discover DBs under ./synth_ai.db/dbs (or override via env)
|
|
32
32
|
root = os.getenv("SYNTH_TRACES_ROOT", "./synth_ai.db/dbs")
|
|
33
33
|
if not os.path.isdir(root):
|
|
@@ -58,7 +58,8 @@ def register(cli):
|
|
|
58
58
|
|
|
59
59
|
async def db_counts(db_dir: str) -> tuple[int, dict[str, int], int, str | None, int]:
|
|
60
60
|
data_file = os.path.join(db_dir, "data")
|
|
61
|
-
|
|
61
|
+
create_storage, storage_config = load_storage()
|
|
62
|
+
mgr = create_storage(storage_config(connection_string=f"sqlite+aiosqlite:///{data_file}"))
|
|
62
63
|
await mgr.initialize()
|
|
63
64
|
try:
|
|
64
65
|
traces_df = await mgr.query_traces("SELECT COUNT(*) AS c FROM session_traces")
|
synth_ai/cli/watch.py
CHANGED
|
@@ -16,11 +16,12 @@ from rich.console import Console, Group
|
|
|
16
16
|
from rich.panel import Panel
|
|
17
17
|
from rich.table import Table
|
|
18
18
|
|
|
19
|
-
from
|
|
19
|
+
from ._storage import load_storage
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
def _open_db(db_url: str):
|
|
23
|
-
|
|
23
|
+
create_storage, storage_config = load_storage()
|
|
24
|
+
return create_storage(storage_config(connection_string=db_url))
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
class _State:
|
synth_ai/jobs/client.py
CHANGED
|
@@ -1,10 +1,22 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import importlib
|
|
3
4
|
from typing import Any
|
|
4
5
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
try:
|
|
7
|
+
normalize_model_identifier = importlib.import_module("synth_ai.api.models.supported").normalize_model_identifier
|
|
8
|
+
except Exception as exc: # pragma: no cover - critical dependency
|
|
9
|
+
raise RuntimeError("Unable to load supported model utilities") from exc
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
AsyncHttpClient = importlib.import_module("synth_ai.http").AsyncHttpClient
|
|
13
|
+
except Exception as exc: # pragma: no cover - critical dependency
|
|
14
|
+
raise RuntimeError("Unable to load HTTP client") from exc
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
prepare_sft_job_payload = importlib.import_module("synth_ai.learning.sft.config").prepare_sft_job_payload
|
|
18
|
+
except Exception as exc: # pragma: no cover - critical dependency
|
|
19
|
+
raise RuntimeError("Unable to load SFT configuration helpers") from exc
|
|
8
20
|
|
|
9
21
|
|
|
10
22
|
class FilesApi:
|