synth-ai 0.2.9.dev3__py3-none-any.whl → 0.2.9.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synth-ai might be problematic. Click here for more details.
- examples/analyze_semantic_words.sh +17 -0
- examples/common_old/backend.py +21 -0
- examples/crafter_debug_render.py +180 -0
- examples/evals_old/README.md +98 -0
- examples/evals_old/__init__.py +6 -0
- examples/evals_old/compare_models.py +1037 -0
- examples/evals_old/example_log.md +145 -0
- examples/evals_old/run_demo.sh +126 -0
- examples/evals_old/trace_analysis.py +270 -0
- examples/finetuning_old/_backup_synth_qwen/config.toml +29 -0
- examples/finetuning_old/_backup_synth_qwen/example_log.md +324 -0
- examples/finetuning_old/_backup_synth_qwen/filter_traces.py +60 -0
- examples/finetuning_old/_backup_synth_qwen/filter_traces_achievements.py +239 -0
- examples/finetuning_old/_backup_synth_qwen/purge_v3_traces.py +109 -0
- examples/finetuning_old/_backup_synth_qwen/react_agent_lm.py +1924 -0
- examples/finetuning_old/_backup_synth_qwen/readme.md +49 -0
- examples/finetuning_old/_backup_synth_qwen/run_crafter_qwen4b.py +114 -0
- examples/finetuning_old/_backup_synth_qwen/run_demo.sh +195 -0
- examples/finetuning_old/_backup_synth_qwen/sft_kickoff.py +118 -0
- examples/finetuning_old/synth_qwen_v1/README.md +68 -0
- examples/finetuning_old/synth_qwen_v1/filter_traces.py +60 -0
- examples/finetuning_old/synth_qwen_v1/filter_traces_achievements.py +239 -0
- examples/finetuning_old/synth_qwen_v1/finetune.py +46 -0
- examples/finetuning_old/synth_qwen_v1/hello_ft_model.py +71 -0
- examples/finetuning_old/synth_qwen_v1/infer.py +37 -0
- examples/finetuning_old/synth_qwen_v1/poll.py +44 -0
- examples/finetuning_old/synth_qwen_v1/prepare_data.py +35 -0
- examples/finetuning_old/synth_qwen_v1/purge_v3_traces.py +109 -0
- examples/finetuning_old/synth_qwen_v1/react_agent_lm.py +1932 -0
- examples/finetuning_old/synth_qwen_v1/run_crafter_sft_job.py +207 -0
- examples/finetuning_old/synth_qwen_v1/run_ft_job.py +232 -0
- examples/finetuning_old/synth_qwen_v1/upload_data.py +34 -0
- examples/finetuning_old/synth_qwen_v1/util.py +147 -0
- examples/rl/README.md +169 -0
- examples/rl/configs/eval_base_qwen.toml +15 -0
- examples/rl/configs/eval_rl_qwen.toml +11 -0
- examples/rl/configs/rl_from_base_qwen.toml +35 -0
- examples/rl/configs/rl_from_base_qwen17.toml +74 -0
- examples/rl/configs/rl_from_ft_qwen.toml +35 -0
- examples/rl/download_dataset.py +64 -0
- examples/rl/run_eval.py +435 -0
- examples/rl/run_rl_and_save.py +94 -0
- examples/rl/task_app/README.md +22 -0
- {synth_ai/task/apps → examples/rl/task_app}/math_single_step.py +8 -8
- examples/rl/task_app/math_task_app.py +107 -0
- examples/rl_old/task_app.py +962 -0
- examples/run_crafter_demo.sh +10 -0
- examples/warming_up_to_rl/analyze_trace_db.py +420 -0
- examples/warming_up_to_rl/configs/crafter_fft.toml +48 -0
- examples/warming_up_to_rl/configs/crafter_fft_4b.toml +54 -0
- examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml +20 -0
- examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml +13 -0
- examples/warming_up_to_rl/configs/eval_modal_qwen4b.toml +23 -0
- examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml +73 -0
- examples/warming_up_to_rl/configs/rl_from_ft.toml +56 -0
- examples/warming_up_to_rl/export_trace_sft.py +541 -0
- examples/warming_up_to_rl/groq_test.py +88 -0
- examples/warming_up_to_rl/manage_secrets.py +127 -0
- examples/warming_up_to_rl/old/event_rewards.md +234 -0
- examples/warming_up_to_rl/old/notes.md +73 -0
- examples/warming_up_to_rl/readme.md +172 -0
- examples/warming_up_to_rl/run_eval.py +434 -0
- examples/warming_up_to_rl/run_fft_and_save.py +309 -0
- examples/warming_up_to_rl/run_local_rollout.py +188 -0
- examples/warming_up_to_rl/run_local_rollout_modal.py +160 -0
- examples/warming_up_to_rl/run_local_rollout_parallel.py +342 -0
- examples/warming_up_to_rl/run_local_rollout_traced.py +372 -0
- examples/warming_up_to_rl/run_rl_and_save.py +101 -0
- examples/warming_up_to_rl/run_rollout_remote.py +129 -0
- examples/warming_up_to_rl/task_app/README.md +38 -0
- {synth_ai/task/apps → examples/warming_up_to_rl/task_app}/grpo_crafter.py +7 -7
- examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py +165 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/README.md +173 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/__init__.py +5 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/branching.py +145 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/environment_routes.py +1271 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/__init__.py +1 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/__init__.py +6 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/app.py +1 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/environment.py +429 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/policy.py +442 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/react_agent.py +96 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/shared.py +302 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/tools.py +47 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/hosted_app.py +202 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/__init__.py +5 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/openai_client.py +512 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/main.py +102 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py +985 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/registry.py +197 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py +1749 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/__init__.py +5 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/storage/volume.py +217 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/test_agents.py +160 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/test_service.py +146 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/test_stepwise_rewards.py +58 -0
- examples/warming_up_to_rl/task_app/synth_envs_hosted/utils.py +61 -0
- synth_ai/api/train/config_finder.py +18 -18
- synth_ai/api/train/env_resolver.py +28 -1
- synth_ai/cli/task_apps.py +264 -55
- synth_ai/task/apps/__init__.py +54 -13
- {synth_ai-0.2.9.dev3.dist-info → synth_ai-0.2.9.dev4.dist-info}/METADATA +1 -1
- {synth_ai-0.2.9.dev3.dist-info → synth_ai-0.2.9.dev4.dist-info}/RECORD +107 -12
- {synth_ai-0.2.9.dev3.dist-info → synth_ai-0.2.9.dev4.dist-info}/top_level.txt +1 -0
- {synth_ai-0.2.9.dev3.dist-info → synth_ai-0.2.9.dev4.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.9.dev3.dist-info → synth_ai-0.2.9.dev4.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.9.dev3.dist-info → synth_ai-0.2.9.dev4.dist-info}/licenses/LICENSE +0 -0
|
@@ -18,17 +18,15 @@ class ConfigCandidate:
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
def _iter_candidate_paths() -> Iterable[Path]:
|
|
21
|
-
# Prefer explicit config directories first
|
|
22
|
-
preferred = [
|
|
23
|
-
REPO_ROOT / "configs",
|
|
24
|
-
REPO_ROOT / "examples",
|
|
25
|
-
REPO_ROOT / "training",
|
|
26
|
-
]
|
|
27
21
|
seen: set[Path] = set()
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
22
|
+
|
|
23
|
+
# Prioritize current working directory first
|
|
24
|
+
try:
|
|
25
|
+
cwd = Path.cwd().resolve()
|
|
26
|
+
except Exception:
|
|
27
|
+
cwd = None
|
|
28
|
+
if cwd and cwd.exists():
|
|
29
|
+
for path in cwd.rglob("*.toml"):
|
|
32
30
|
if any(part in _SKIP_DIRS for part in path.parts):
|
|
33
31
|
continue
|
|
34
32
|
resolved = path.resolve()
|
|
@@ -37,14 +35,16 @@ def _iter_candidate_paths() -> Iterable[Path]:
|
|
|
37
35
|
seen.add(resolved)
|
|
38
36
|
yield resolved
|
|
39
37
|
|
|
40
|
-
#
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
38
|
+
# Then look in explicit config directories
|
|
39
|
+
preferred = [
|
|
40
|
+
REPO_ROOT / "configs",
|
|
41
|
+
REPO_ROOT / "examples",
|
|
42
|
+
REPO_ROOT / "training",
|
|
43
|
+
]
|
|
44
|
+
for base in preferred:
|
|
45
|
+
if not base.exists():
|
|
46
|
+
continue
|
|
47
|
+
for path in base.rglob("*.toml"):
|
|
48
48
|
if any(part in _SKIP_DIRS for part in path.parts):
|
|
49
49
|
continue
|
|
50
50
|
resolved = path.resolve()
|
|
@@ -55,17 +55,43 @@ class EnvResolver:
|
|
|
55
55
|
|
|
56
56
|
def _collect_default_candidates(config_path: Path | None) -> list[Path]:
|
|
57
57
|
candidates: list[Path] = []
|
|
58
|
+
cwd = Path.cwd()
|
|
59
|
+
|
|
60
|
+
# Prioritize CWD env files
|
|
61
|
+
cwd_env = cwd / ".env"
|
|
62
|
+
if cwd_env.exists():
|
|
63
|
+
candidates.append(cwd_env.resolve())
|
|
64
|
+
|
|
65
|
+
# Search for additional .env files in CWD subdirectories
|
|
66
|
+
for sub in cwd.glob("**/.env"):
|
|
67
|
+
try:
|
|
68
|
+
resolved = sub.resolve()
|
|
69
|
+
except Exception:
|
|
70
|
+
continue
|
|
71
|
+
if resolved in candidates:
|
|
72
|
+
continue
|
|
73
|
+
# avoid nested venv caches
|
|
74
|
+
if any(part in {".venv", "node_modules", "__pycache__"} for part in resolved.parts):
|
|
75
|
+
continue
|
|
76
|
+
if len(candidates) >= 20:
|
|
77
|
+
break
|
|
78
|
+
candidates.append(resolved)
|
|
79
|
+
|
|
80
|
+
# Then config path env file
|
|
58
81
|
if config_path:
|
|
59
82
|
cfg_env = config_path.parent / ".env"
|
|
60
83
|
if cfg_env.exists():
|
|
61
84
|
candidates.append(cfg_env.resolve())
|
|
85
|
+
|
|
86
|
+
# Then repo env files
|
|
62
87
|
repo_env = REPO_ROOT / ".env"
|
|
63
88
|
if repo_env.exists():
|
|
64
89
|
candidates.append(repo_env.resolve())
|
|
65
90
|
examples_env = REPO_ROOT / "examples" / ".env"
|
|
66
91
|
if examples_env.exists():
|
|
67
92
|
candidates.append(examples_env.resolve())
|
|
68
|
-
|
|
93
|
+
|
|
94
|
+
# Search shallow depth for additional .env files in examples
|
|
69
95
|
for sub in (REPO_ROOT / "examples").glob("**/.env"):
|
|
70
96
|
try:
|
|
71
97
|
resolved = sub.resolve()
|
|
@@ -79,6 +105,7 @@ def _collect_default_candidates(config_path: Path | None) -> list[Path]:
|
|
|
79
105
|
if len(candidates) >= 20:
|
|
80
106
|
break
|
|
81
107
|
candidates.append(resolved)
|
|
108
|
+
|
|
82
109
|
deduped: list[Path] = []
|
|
83
110
|
for path in candidates:
|
|
84
111
|
if path not in deduped:
|
synth_ai/cli/task_apps.py
CHANGED
|
@@ -37,8 +37,6 @@ DEFAULT_IGNORE_DIRS = {
|
|
|
37
37
|
|
|
38
38
|
DEFAULT_SEARCH_RELATIVE = (
|
|
39
39
|
Path("."),
|
|
40
|
-
Path("examples"),
|
|
41
|
-
Path("synth_ai"),
|
|
42
40
|
)
|
|
43
41
|
|
|
44
42
|
|
|
@@ -70,32 +68,21 @@ def _should_ignore_path(path: Path) -> bool:
|
|
|
70
68
|
|
|
71
69
|
|
|
72
70
|
def _candidate_search_roots() -> list[Path]:
|
|
71
|
+
"""Only search for task apps in the current working directory and subdirectories."""
|
|
73
72
|
roots: list[Path] = []
|
|
73
|
+
|
|
74
|
+
# Allow explicit search paths via environment variable
|
|
74
75
|
env_paths = os.environ.get("SYNTH_TASK_APP_SEARCH_PATH")
|
|
75
76
|
if env_paths:
|
|
76
77
|
for chunk in env_paths.split(os.pathsep):
|
|
77
78
|
if chunk:
|
|
78
79
|
roots.append(Path(chunk).expanduser())
|
|
79
80
|
|
|
81
|
+
# Always include current working directory
|
|
80
82
|
cwd = Path.cwd().resolve()
|
|
81
83
|
roots.append(cwd)
|
|
82
84
|
|
|
83
|
-
|
|
84
|
-
try:
|
|
85
|
-
candidate = (cwd / rel).resolve()
|
|
86
|
-
except Exception:
|
|
87
|
-
continue
|
|
88
|
-
roots.append(candidate)
|
|
89
|
-
if REPO_ROOT not in (None, candidate):
|
|
90
|
-
try:
|
|
91
|
-
repo_candidate = (REPO_ROOT / rel).resolve()
|
|
92
|
-
except Exception:
|
|
93
|
-
repo_candidate = None
|
|
94
|
-
if repo_candidate:
|
|
95
|
-
roots.append(repo_candidate)
|
|
96
|
-
|
|
97
|
-
roots.append(REPO_ROOT)
|
|
98
|
-
|
|
85
|
+
# Remove duplicates while preserving order
|
|
99
86
|
seen: set[Path] = set()
|
|
100
87
|
ordered: list[Path] = []
|
|
101
88
|
for root in roots:
|
|
@@ -119,6 +106,10 @@ class _TaskAppConfigVisitor(ast.NodeVisitor):
|
|
|
119
106
|
app_id = _extract_app_id(node)
|
|
120
107
|
if app_id:
|
|
121
108
|
self.matches.append((app_id, getattr(node, "lineno", 0)))
|
|
109
|
+
elif _is_register_task_app_call(node):
|
|
110
|
+
app_id = _extract_register_app_id(node)
|
|
111
|
+
if app_id:
|
|
112
|
+
self.matches.append((app_id, getattr(node, "lineno", 0)))
|
|
122
113
|
self.generic_visit(node)
|
|
123
114
|
|
|
124
115
|
|
|
@@ -142,6 +133,27 @@ def _extract_app_id(node: ast.Call) -> str | None:
|
|
|
142
133
|
return None
|
|
143
134
|
|
|
144
135
|
|
|
136
|
+
def _is_register_task_app_call(node: ast.Call) -> bool:
|
|
137
|
+
func = node.func
|
|
138
|
+
if isinstance(func, ast.Name) and func.id == "register_task_app":
|
|
139
|
+
return True
|
|
140
|
+
if isinstance(func, ast.Attribute) and func.attr == "register_task_app":
|
|
141
|
+
return True
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _extract_register_app_id(node: ast.Call) -> str | None:
|
|
146
|
+
# Look for entry=TaskAppEntry(app_id="...", ...)
|
|
147
|
+
for kw in node.keywords:
|
|
148
|
+
if kw.arg == "entry" and isinstance(kw.value, ast.Call):
|
|
149
|
+
entry_call = kw.value
|
|
150
|
+
if isinstance(entry_call.func, ast.Name) and entry_call.func.id == "TaskAppEntry":
|
|
151
|
+
for entry_kw in entry_call.keywords:
|
|
152
|
+
if entry_kw.arg == "app_id" and isinstance(entry_kw.value, ast.Constant) and isinstance(entry_kw.value.value, str):
|
|
153
|
+
return entry_kw.value.value
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
145
157
|
class _ModalAppVisitor(ast.NodeVisitor):
|
|
146
158
|
def __init__(self) -> None:
|
|
147
159
|
self.app_aliases: set[str] = set()
|
|
@@ -186,12 +198,14 @@ def _extract_modal_app_name(node: ast.Call) -> str | None:
|
|
|
186
198
|
return None
|
|
187
199
|
|
|
188
200
|
|
|
189
|
-
@functools.lru_cache(maxsize=1)
|
|
190
201
|
def _collect_task_app_choices() -> list[AppChoice]:
|
|
202
|
+
# Clear registry to avoid duplicate registration errors
|
|
203
|
+
registry.clear()
|
|
204
|
+
|
|
191
205
|
choices: list[AppChoice] = []
|
|
192
206
|
with contextlib.suppress(Exception):
|
|
193
207
|
import synth_ai.demos.demo_task_apps # noqa: F401
|
|
194
|
-
|
|
208
|
+
# Only use discovered task apps, not registered ones (since we moved them to examples)
|
|
195
209
|
choices.extend(_collect_scanned_task_configs())
|
|
196
210
|
choices.extend(_collect_modal_scripts())
|
|
197
211
|
|
|
@@ -333,10 +347,112 @@ def _choice_has_modal_support(choice: AppChoice) -> bool:
|
|
|
333
347
|
try:
|
|
334
348
|
entry = choice.ensure_entry()
|
|
335
349
|
except click.ClickException:
|
|
336
|
-
|
|
350
|
+
# If we can't load the entry, try to detect Modal support via AST parsing
|
|
351
|
+
return _has_modal_support_in_file(choice.path)
|
|
337
352
|
return entry.modal is not None
|
|
338
353
|
|
|
339
354
|
|
|
355
|
+
def _has_modal_support_in_file(path: Path) -> bool:
|
|
356
|
+
"""Detect if a file has Modal deployment support by parsing the AST."""
|
|
357
|
+
try:
|
|
358
|
+
source = path.read_text(encoding="utf-8")
|
|
359
|
+
tree = ast.parse(source, filename=str(path))
|
|
360
|
+
|
|
361
|
+
# Look for ModalDeploymentConfig in register_task_app calls
|
|
362
|
+
for node in ast.walk(tree):
|
|
363
|
+
if isinstance(node, ast.Call):
|
|
364
|
+
if _is_register_task_app_call(node):
|
|
365
|
+
# Check if the entry has modal=ModalDeploymentConfig(...)
|
|
366
|
+
for kw in node.keywords:
|
|
367
|
+
if kw.arg == "entry" and isinstance(kw.value, ast.Call):
|
|
368
|
+
entry_call = kw.value
|
|
369
|
+
if isinstance(entry_call.func, ast.Name) and entry_call.func.id == "TaskAppEntry":
|
|
370
|
+
for entry_kw in entry_call.keywords:
|
|
371
|
+
if entry_kw.arg == "modal" and isinstance(entry_kw.value, ast.Call):
|
|
372
|
+
modal_call = entry_kw.value
|
|
373
|
+
if isinstance(modal_call.func, ast.Name) and modal_call.func.id == "ModalDeploymentConfig":
|
|
374
|
+
return True
|
|
375
|
+
except Exception:
|
|
376
|
+
pass
|
|
377
|
+
return False
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _extract_modal_config_from_file(path: Path) -> ModalDeploymentConfig | None:
|
|
381
|
+
"""Extract ModalDeploymentConfig from a file by parsing the AST."""
|
|
382
|
+
try:
|
|
383
|
+
source = path.read_text(encoding="utf-8")
|
|
384
|
+
tree = ast.parse(source, filename=str(path))
|
|
385
|
+
|
|
386
|
+
# Look for ModalDeploymentConfig in register_task_app calls
|
|
387
|
+
for node in ast.walk(tree):
|
|
388
|
+
if isinstance(node, ast.Call):
|
|
389
|
+
if _is_register_task_app_call(node):
|
|
390
|
+
# Check if the entry has modal=ModalDeploymentConfig(...)
|
|
391
|
+
for kw in node.keywords:
|
|
392
|
+
if kw.arg == "entry" and isinstance(kw.value, ast.Call):
|
|
393
|
+
entry_call = kw.value
|
|
394
|
+
if isinstance(entry_call.func, ast.Name) and entry_call.func.id == "TaskAppEntry":
|
|
395
|
+
for entry_kw in entry_call.keywords:
|
|
396
|
+
if entry_kw.arg == "modal" and isinstance(entry_kw.value, ast.Call):
|
|
397
|
+
modal_call = entry_kw.value
|
|
398
|
+
if isinstance(modal_call.func, ast.Name) and modal_call.func.id == "ModalDeploymentConfig":
|
|
399
|
+
# Extract the arguments to ModalDeploymentConfig
|
|
400
|
+
return _build_modal_config_from_ast(modal_call)
|
|
401
|
+
except Exception:
|
|
402
|
+
pass
|
|
403
|
+
return None
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _build_modal_config_from_ast(modal_call: ast.Call) -> ModalDeploymentConfig | None:
|
|
407
|
+
"""Build a ModalDeploymentConfig from an AST Call node."""
|
|
408
|
+
try:
|
|
409
|
+
# Extract keyword arguments
|
|
410
|
+
kwargs = {}
|
|
411
|
+
for kw in modal_call.keywords:
|
|
412
|
+
if kw.arg and isinstance(kw.value, ast.Constant):
|
|
413
|
+
kwargs[kw.arg] = kw.value.value
|
|
414
|
+
elif kw.arg == "pip_packages" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
415
|
+
# Handle pip_packages list/tuple
|
|
416
|
+
packages = []
|
|
417
|
+
for elt in kw.value.elts:
|
|
418
|
+
if isinstance(elt, ast.Constant):
|
|
419
|
+
packages.append(elt.value)
|
|
420
|
+
kwargs[kw.arg] = tuple(packages)
|
|
421
|
+
elif kw.arg == "extra_local_dirs" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
422
|
+
# Handle extra_local_dirs list/tuple of tuples
|
|
423
|
+
dirs = []
|
|
424
|
+
for elt in kw.value.elts:
|
|
425
|
+
if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
|
|
426
|
+
src = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
|
|
427
|
+
dst = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
|
|
428
|
+
if src and dst:
|
|
429
|
+
dirs.append((src, dst))
|
|
430
|
+
kwargs[kw.arg] = tuple(dirs)
|
|
431
|
+
elif kw.arg == "secret_names" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
432
|
+
# Handle secret_names list/tuple
|
|
433
|
+
secrets = []
|
|
434
|
+
for elt in kw.value.elts:
|
|
435
|
+
if isinstance(elt, ast.Constant):
|
|
436
|
+
secrets.append(elt.value)
|
|
437
|
+
kwargs[kw.arg] = tuple(secrets)
|
|
438
|
+
elif kw.arg == "volume_mounts" and isinstance(kw.value, (ast.List, ast.Tuple)):
|
|
439
|
+
# Handle volume_mounts list/tuple of tuples
|
|
440
|
+
mounts = []
|
|
441
|
+
for elt in kw.value.elts:
|
|
442
|
+
if isinstance(elt, (ast.List, ast.Tuple)) and len(elt.elts) == 2:
|
|
443
|
+
name = elt.elts[0].value if isinstance(elt.elts[0], ast.Constant) else None
|
|
444
|
+
mount = elt.elts[1].value if isinstance(elt.elts[1], ast.Constant) else None
|
|
445
|
+
if name and mount:
|
|
446
|
+
mounts.append((name, mount))
|
|
447
|
+
kwargs[kw.arg] = tuple(mounts)
|
|
448
|
+
|
|
449
|
+
# Create ModalDeploymentConfig with extracted arguments
|
|
450
|
+
from synth_ai.task.apps import ModalDeploymentConfig
|
|
451
|
+
return ModalDeploymentConfig(**kwargs)
|
|
452
|
+
except Exception:
|
|
453
|
+
return None
|
|
454
|
+
|
|
455
|
+
|
|
340
456
|
def _choice_has_local_support(choice: AppChoice) -> bool:
|
|
341
457
|
if choice.modal_script:
|
|
342
458
|
return False
|
|
@@ -418,6 +534,10 @@ def _load_entry_from_path(path: Path, app_id: str) -> TaskAppEntry:
|
|
|
418
534
|
raise click.ClickException(f"Unable to load Python module from {resolved}")
|
|
419
535
|
module = importlib.util.module_from_spec(spec)
|
|
420
536
|
sys.modules[module_name] = module
|
|
537
|
+
|
|
538
|
+
# Clear registry before importing to avoid duplicate registration errors
|
|
539
|
+
registry.clear()
|
|
540
|
+
|
|
421
541
|
try:
|
|
422
542
|
spec.loader.exec_module(module)
|
|
423
543
|
except Exception as exc:
|
|
@@ -468,10 +588,16 @@ def _load_entry_from_path(path: Path, app_id: str) -> TaskAppEntry:
|
|
|
468
588
|
config_obj = result
|
|
469
589
|
break
|
|
470
590
|
|
|
591
|
+
# If no TaskAppConfig found directly, check if it was registered via register_task_app
|
|
471
592
|
if factory_callable is None or config_obj is None:
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
593
|
+
try:
|
|
594
|
+
# Check if the app was registered in the registry
|
|
595
|
+
entry = registry.get(app_id)
|
|
596
|
+
return entry
|
|
597
|
+
except KeyError:
|
|
598
|
+
raise click.ClickException(
|
|
599
|
+
f"Could not locate TaskAppConfig for '{app_id}' in {resolved}."
|
|
600
|
+
)
|
|
475
601
|
|
|
476
602
|
modal_cfg: ModalDeploymentConfig | None = None
|
|
477
603
|
for attr_name in dir(module):
|
|
@@ -482,6 +608,10 @@ def _load_entry_from_path(path: Path, app_id: str) -> TaskAppEntry:
|
|
|
482
608
|
if isinstance(attr, ModalDeploymentConfig):
|
|
483
609
|
modal_cfg = attr
|
|
484
610
|
break
|
|
611
|
+
|
|
612
|
+
# If no ModalDeploymentConfig found, try to detect it via AST parsing
|
|
613
|
+
if modal_cfg is None:
|
|
614
|
+
modal_cfg = _extract_modal_config_from_file(resolved)
|
|
485
615
|
|
|
486
616
|
description = inspect.getdoc(module) or f"Discovered task app in {resolved.name}"
|
|
487
617
|
env_files: Iterable[str] = getattr(module, "ENV_FILES", ()) # type: ignore[arg-type]
|
|
@@ -507,20 +637,35 @@ def _resolve_env_paths_for_script(script_path: Path, explicit: Sequence[str]) ->
|
|
|
507
637
|
resolved.append(p)
|
|
508
638
|
return resolved
|
|
509
639
|
|
|
640
|
+
# Always prompt for env file selection instead of auto-loading defaults
|
|
510
641
|
script_dir = script_path.parent.resolve()
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
642
|
+
cwd = Path.cwd()
|
|
643
|
+
|
|
644
|
+
# Look for env files in current working directory first, then repo root
|
|
645
|
+
env_candidates = []
|
|
646
|
+
|
|
647
|
+
# Add CWD env files first (prioritized)
|
|
648
|
+
cwd_env_files = sorted(cwd.glob('**/*.env'))
|
|
649
|
+
env_candidates.extend(cwd_env_files)
|
|
650
|
+
|
|
651
|
+
# Add repo root env files
|
|
652
|
+
repo_env_files = sorted(REPO_ROOT.glob('**/*.env'))
|
|
653
|
+
# Avoid duplicates
|
|
654
|
+
for repo_file in repo_env_files:
|
|
655
|
+
if repo_file not in env_candidates:
|
|
656
|
+
env_candidates.append(repo_file)
|
|
657
|
+
|
|
658
|
+
if not env_candidates:
|
|
659
|
+
created = _interactive_create_env(script_dir)
|
|
660
|
+
if created is None:
|
|
661
|
+
raise click.ClickException("Env file required (--env-file) for this task app")
|
|
662
|
+
return [created]
|
|
663
|
+
|
|
664
|
+
click.echo('Select env file to load:')
|
|
665
|
+
for idx, path in enumerate(env_candidates, start=1):
|
|
666
|
+
click.echo(f" {idx}) {path}")
|
|
667
|
+
choice = click.prompt('Enter choice', type=click.IntRange(1, len(env_candidates)))
|
|
668
|
+
return [env_candidates[choice - 1]]
|
|
524
669
|
|
|
525
670
|
|
|
526
671
|
def _run_modal_script(
|
|
@@ -554,7 +699,7 @@ def _run_modal_script(
|
|
|
554
699
|
raise click.ClickException(f"modal {command} failed with exit code {exc.returncode}") from exc
|
|
555
700
|
|
|
556
701
|
|
|
557
|
-
def _preflight_env_key() -> None:
|
|
702
|
+
def _preflight_env_key(crash_on_failure: bool = False) -> None:
|
|
558
703
|
try:
|
|
559
704
|
raw_backend = os.environ.get("BACKEND_BASE_URL") or os.environ.get("SYNTH_BASE_URL") or "http://localhost:8000/api"
|
|
560
705
|
backend_base = raw_backend.rstrip('/')
|
|
@@ -591,13 +736,24 @@ def _preflight_env_key() -> None:
|
|
|
591
736
|
click.echo("[preflight] verifying env key presence…")
|
|
592
737
|
ver = c.get(f"{backend_base.rstrip('/')}/v1/env-keys/verify")
|
|
593
738
|
if ver.status_code == 200 and (ver.json() or {}).get("present"):
|
|
594
|
-
|
|
739
|
+
# Show first and last 5 chars of the API key for verification
|
|
740
|
+
key_preview = f"{env_api_key[:5]}...{env_api_key[-5:]}" if len(env_api_key) > 10 else env_api_key
|
|
741
|
+
click.echo(f"✅ ENVIRONMENT_API_KEY upserted and verified in backend ({key_preview})")
|
|
595
742
|
else:
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
743
|
+
error_msg = "ENVIRONMENT_API_KEY verification failed"
|
|
744
|
+
if crash_on_failure:
|
|
745
|
+
raise click.ClickException(f"[CRITICAL] {error_msg}")
|
|
746
|
+
click.echo(f"[WARN] {error_msg}; proceeding anyway")
|
|
747
|
+
except Exception as e:
|
|
748
|
+
error_msg = f"Failed to encrypt/upload ENVIRONMENT_API_KEY: {e}"
|
|
749
|
+
if crash_on_failure:
|
|
750
|
+
raise click.ClickException(f"[CRITICAL] {error_msg}")
|
|
751
|
+
click.echo(f"[WARN] {error_msg}; proceeding anyway")
|
|
752
|
+
except Exception as e:
|
|
753
|
+
error_msg = f"Backend preflight for ENVIRONMENT_API_KEY failed: {e}"
|
|
754
|
+
if crash_on_failure:
|
|
755
|
+
raise click.ClickException(f"[CRITICAL] {error_msg}")
|
|
756
|
+
click.echo(f"[WARN] {error_msg}; proceeding anyway")
|
|
601
757
|
|
|
602
758
|
|
|
603
759
|
def _run_modal_with_entry(
|
|
@@ -609,6 +765,7 @@ def _run_modal_with_entry(
|
|
|
609
765
|
command: str,
|
|
610
766
|
*,
|
|
611
767
|
dry_run: bool = False,
|
|
768
|
+
original_path: Path | None = None,
|
|
612
769
|
) -> None:
|
|
613
770
|
modal_path = shutil.which(modal_cli)
|
|
614
771
|
if modal_path is None:
|
|
@@ -620,13 +777,14 @@ def _run_modal_with_entry(
|
|
|
620
777
|
fallback_dir = env_paths_list[0].parent if env_paths_list else Path.cwd()
|
|
621
778
|
_ensure_env_values(env_paths_list, fallback_dir)
|
|
622
779
|
_load_env_values(env_paths_list)
|
|
623
|
-
_preflight_env_key()
|
|
780
|
+
_preflight_env_key(crash_on_failure=True)
|
|
624
781
|
|
|
625
782
|
script_path = _write_modal_entrypoint(
|
|
626
783
|
entry,
|
|
627
784
|
modal_cfg,
|
|
628
785
|
modal_name,
|
|
629
786
|
dotenv_paths=dotenv_paths,
|
|
787
|
+
original_path=original_path,
|
|
630
788
|
)
|
|
631
789
|
cmd = [modal_path, command, str(script_path)]
|
|
632
790
|
|
|
@@ -742,6 +900,7 @@ def _deploy_entry(
|
|
|
742
900
|
dry_run: bool,
|
|
743
901
|
modal_cli: str,
|
|
744
902
|
env_file: Sequence[str],
|
|
903
|
+
original_path: Path | None = None,
|
|
745
904
|
) -> None:
|
|
746
905
|
modal_cfg = entry.modal
|
|
747
906
|
if modal_cfg is None:
|
|
@@ -749,7 +908,7 @@ def _deploy_entry(
|
|
|
749
908
|
|
|
750
909
|
env_paths = _determine_env_files(entry, env_file)
|
|
751
910
|
click.echo('Using env file(s): ' + ', '.join(str(p) for p in env_paths))
|
|
752
|
-
_run_modal_with_entry(entry, modal_cfg, modal_cli, modal_name, env_paths, command="deploy", dry_run=dry_run)
|
|
911
|
+
_run_modal_with_entry(entry, modal_cfg, modal_cli, modal_name, env_paths, command="deploy", dry_run=dry_run, original_path=original_path)
|
|
753
912
|
|
|
754
913
|
|
|
755
914
|
def _modal_serve_entry(
|
|
@@ -757,6 +916,7 @@ def _modal_serve_entry(
|
|
|
757
916
|
modal_name: str | None,
|
|
758
917
|
modal_cli: str,
|
|
759
918
|
env_file: Sequence[str],
|
|
919
|
+
original_path: Path | None = None,
|
|
760
920
|
) -> None:
|
|
761
921
|
modal_cfg = entry.modal
|
|
762
922
|
if modal_cfg is None:
|
|
@@ -764,7 +924,7 @@ def _modal_serve_entry(
|
|
|
764
924
|
|
|
765
925
|
env_paths = _determine_env_files(entry, env_file)
|
|
766
926
|
click.echo('Using env file(s): ' + ', '.join(str(p) for p in env_paths))
|
|
767
|
-
_run_modal_with_entry(entry, modal_cfg, modal_cli, modal_name, env_paths, command="serve")
|
|
927
|
+
_run_modal_with_entry(entry, modal_cfg, modal_cli, modal_name, env_paths, command="serve", original_path=original_path)
|
|
768
928
|
|
|
769
929
|
@click.group(
|
|
770
930
|
name='task-app',
|
|
@@ -862,11 +1022,22 @@ def _determine_env_files(entry: TaskAppEntry, user_env_files: Sequence[str]) ->
|
|
|
862
1022
|
if resolved:
|
|
863
1023
|
return resolved
|
|
864
1024
|
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
1025
|
+
# Always prompt for env file selection instead of auto-loading defaults
|
|
1026
|
+
# Look for env files in current working directory first, then repo root
|
|
1027
|
+
cwd = Path.cwd()
|
|
1028
|
+
env_candidates = []
|
|
1029
|
+
|
|
1030
|
+
# Add CWD env files first (prioritized)
|
|
1031
|
+
cwd_env_files = sorted(cwd.glob('**/*.env'))
|
|
1032
|
+
env_candidates.extend(cwd_env_files)
|
|
1033
|
+
|
|
1034
|
+
# Add repo root env files
|
|
1035
|
+
repo_env_files = sorted(REPO_ROOT.glob('**/*.env'))
|
|
1036
|
+
# Avoid duplicates
|
|
1037
|
+
for repo_file in repo_env_files:
|
|
1038
|
+
if repo_file not in env_candidates:
|
|
1039
|
+
env_candidates.append(repo_file)
|
|
1040
|
+
|
|
870
1041
|
if not env_candidates:
|
|
871
1042
|
raise click.ClickException('No env file found. Pass --env-file explicitly.')
|
|
872
1043
|
|
|
@@ -995,7 +1166,7 @@ def deploy_app(app_id: str | None, modal_name: str | None, dry_run: bool, modal_
|
|
|
995
1166
|
return
|
|
996
1167
|
|
|
997
1168
|
entry = choice.ensure_entry()
|
|
998
|
-
_deploy_entry(entry, modal_name, dry_run, modal_cli, env_file)
|
|
1169
|
+
_deploy_entry(entry, modal_name, dry_run, modal_cli, env_file, original_path=choice.path)
|
|
999
1170
|
|
|
1000
1171
|
@task_app_group.command('modal-serve')
|
|
1001
1172
|
@click.argument('app_id', type=str, required=False)
|
|
@@ -1012,7 +1183,7 @@ def modal_serve_app(app_id: str | None, modal_cli: str, modal_name: str | None,
|
|
|
1012
1183
|
return
|
|
1013
1184
|
|
|
1014
1185
|
entry = choice.ensure_entry()
|
|
1015
|
-
_modal_serve_entry(entry, modal_name, modal_cli, env_file)
|
|
1186
|
+
_modal_serve_entry(entry, modal_name, modal_cli, env_file, original_path=choice.path)
|
|
1016
1187
|
|
|
1017
1188
|
|
|
1018
1189
|
def _write_modal_entrypoint(
|
|
@@ -1021,10 +1192,35 @@ def _write_modal_entrypoint(
|
|
|
1021
1192
|
override_name: str | None,
|
|
1022
1193
|
*,
|
|
1023
1194
|
dotenv_paths: Sequence[str] | None = None,
|
|
1195
|
+
original_path: Path | None = None,
|
|
1024
1196
|
) -> Path:
|
|
1025
1197
|
modal_name = override_name or modal_cfg.app_name
|
|
1026
1198
|
|
|
1199
|
+
# For dynamically discovered apps, import the module by its package path
|
|
1200
|
+
# Compute the module name relative to the mounted repo root (/opt/synth_ai_repo)
|
|
1201
|
+
remote_file_str: str | None = None
|
|
1202
|
+
if original_path:
|
|
1203
|
+
try:
|
|
1204
|
+
# Build lookup of local->remote mounts
|
|
1205
|
+
mount_map: list[tuple[Path, Path]] = [
|
|
1206
|
+
(Path(local).resolve(), Path(remote)) for (local, remote) in modal_cfg.extra_local_dirs
|
|
1207
|
+
]
|
|
1208
|
+
orig = Path(original_path).resolve()
|
|
1209
|
+
for local_src, remote_dst in mount_map:
|
|
1210
|
+
with contextlib.suppress(Exception):
|
|
1211
|
+
if orig.is_relative_to(local_src): # py311+
|
|
1212
|
+
remote_file_str = str((remote_dst / orig.relative_to(local_src)).resolve())
|
|
1213
|
+
break
|
|
1214
|
+
try:
|
|
1215
|
+
rel = orig.relative_to(local_src)
|
|
1216
|
+
remote_file_str = str((remote_dst / rel).resolve())
|
|
1217
|
+
break
|
|
1218
|
+
except Exception:
|
|
1219
|
+
pass
|
|
1220
|
+
except Exception:
|
|
1221
|
+
remote_file_str = None
|
|
1027
1222
|
module_name = entry.config_factory.__module__
|
|
1223
|
+
|
|
1028
1224
|
dotenv_paths = [str(Path(path)) for path in (dotenv_paths or [])]
|
|
1029
1225
|
|
|
1030
1226
|
pip_packages = list(modal_cfg.pip_packages)
|
|
@@ -1036,6 +1232,7 @@ def _write_modal_entrypoint(
|
|
|
1036
1232
|
script = f"""from __future__ import annotations
|
|
1037
1233
|
|
|
1038
1234
|
import importlib
|
|
1235
|
+
import importlib.util
|
|
1039
1236
|
import sys
|
|
1040
1237
|
sys.path.insert(0, '/opt/synth_ai_repo')
|
|
1041
1238
|
|
|
@@ -1047,6 +1244,7 @@ from synth_ai.task.server import create_task_app
|
|
|
1047
1244
|
ENTRY_ID = {entry.app_id!r}
|
|
1048
1245
|
MODAL_APP_NAME = {modal_name!r}
|
|
1049
1246
|
MODULE_NAME = {module_name!r}
|
|
1247
|
+
MODULE_FILE = {remote_file_str!r}
|
|
1050
1248
|
DOTENV_PATHS = {dotenv_paths!r}
|
|
1051
1249
|
|
|
1052
1250
|
image = Image.debian_slim(python_version={modal_cfg.python_version!r})
|
|
@@ -1070,8 +1268,19 @@ volume_map = {{}}
|
|
|
1070
1268
|
for vol_name, mount_path in volume_mounts:
|
|
1071
1269
|
volume_map[mount_path] = Volume.from_name(vol_name, create_if_missing=True)
|
|
1072
1270
|
|
|
1073
|
-
|
|
1271
|
+
# Import the module to trigger registration
|
|
1272
|
+
if MODULE_FILE:
|
|
1273
|
+
spec = importlib.util.spec_from_file_location(MODULE_NAME or 'task_app_module', MODULE_FILE)
|
|
1274
|
+
if spec and spec.loader:
|
|
1275
|
+
mod = importlib.util.module_from_spec(spec)
|
|
1276
|
+
sys.modules[MODULE_NAME or 'task_app_module'] = mod
|
|
1277
|
+
spec.loader.exec_module(mod)
|
|
1278
|
+
else:
|
|
1279
|
+
raise RuntimeError("Failed to import task app from file: " + str(MODULE_FILE))
|
|
1280
|
+
else:
|
|
1281
|
+
importlib.import_module(MODULE_NAME)
|
|
1074
1282
|
|
|
1283
|
+
# Get the entry from registry (now that it's registered)
|
|
1075
1284
|
entry = registry.get(ENTRY_ID)
|
|
1076
1285
|
modal_cfg = entry.modal
|
|
1077
1286
|
if modal_cfg is None:
|