synth-ai 0.2.9.dev7__py3-none-any.whl → 0.2.9.dev9__py3-none-any.whl

examples/vlm/run_crafter_vlm_benchmark.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python3
+"""
+Benchmark Crafter performance across prompt modalities (text-only, image-only, both).
+
+For each mode we:
+  * Run 20 seeded episodes (configurable) with GPT-4o mini via OpenAI Chat Completions.
+  * Execute the returned tool calls in the local Crafter environment.
+  * Record achievements/steps and save every rendered frame under `examples/vlm/temp/`.
+
+Concurrency is capped by an asyncio semaphore (default parallelism = 10).
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import json
+import os
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Any
+from uuid import uuid4
+
+from examples.warming_up_to_rl.task_app.synth_envs_hosted.envs.crafter.environment import (
+    CrafterEnvironmentWrapper,
+)
+from examples.warming_up_to_rl.task_app.synth_envs_hosted.envs.crafter.policy import CrafterPolicy
+from openai import AsyncOpenAI
+from synth_ai.environments.examples.crafter_classic.environment import CrafterClassicEnvironment
+from synth_ai.environments.examples.crafter_classic.taskset import (
+    CrafterTaskInstance,
+    CrafterTaskInstanceMetadata,
+)
+from synth_ai.environments.tasks.core import Impetus, Intent
+
+OUTPUT_ROOT = Path("examples/vlm/temp")
+
+
+class Mode(str, Enum):
+    TEXT = "text"
+    IMAGE = "image"
+    BOTH = "both"
+
+
+@dataclass
+class EpisodeResult:
+    mode: Mode
+    seed: int
+    steps_taken: int
+    achievements: set[str]
+    total_reward: float
+    tool_calls: int
+
+
+def _ensure_openai_client(api_key: str | None) -> AsyncOpenAI:
+    if not api_key:
+        raise RuntimeError(
+            "OPENAI_API_KEY must be set to run the VLM benchmark (export the key or add to your .env)."
+        )
+    return AsyncOpenAI(api_key=api_key)
+
+
+def _build_task_instance(seed: int) -> CrafterTaskInstance:
+    impetus = Impetus(instructions="Explore, survive, and unlock achievements.")
+    intent = Intent(rubric={"goal": "Unlock achievements"}, gold_trajectories=None, gold_state_diff={})
+    metadata = CrafterTaskInstanceMetadata(
+        difficulty="custom",
+        seed=seed,
+        num_trees_radius=0,
+        num_cows_radius=0,
+        num_hostiles_radius=0,
+    )
+    instance = CrafterTaskInstance(
+        id=uuid4(),
+        impetus=impetus,
+        intent=intent,
+        metadata=metadata,
+        is_reproducible=True,
+        initial_engine_snapshot=None,
+    )
+    # Engine expects these config keys
+    instance.config = {"seed": seed, "length": 256, "area": [64, 64]}
+    return instance
+
+
+def _save_observation_frame(observation_packet: dict[str, Any], dest_path: Path) -> None:
+    obs = observation_packet.get("observation")
+    if not isinstance(obs, dict):
+        return
+    image_b64 = obs.get("observation_image_base64")
+    if not isinstance(image_b64, str) or not image_b64:
+        return
+    try:
+        dest_path.parent.mkdir(parents=True, exist_ok=True)
+        dest_path.write_bytes(base64.b64decode(image_b64))
+    except Exception:
+        pass  # best effort
+
+
+def _strip_image_fields(observation_packet: dict[str, Any]) -> dict[str, Any]:
+    stripped = json.loads(json.dumps(observation_packet))
+    obs = stripped.get("observation")
+    if isinstance(obs, dict):
+        for key in list(obs.keys()):
+            if key.startswith("observation_image"):
+                obs.pop(key, None)
+    return stripped
+
+
+def _make_image_only_request(request: dict[str, Any]) -> dict[str, Any]:
+    cloned = json.loads(json.dumps(request))
+    for message in cloned.get("messages", []):
+        if message.get("role") != "user":
+            continue
+        content = message.get("content")
+        if isinstance(content, list):
+            image_parts = [
+                item
+                for item in content
+                if isinstance(item, dict) and item.get("type") in {"image_url", "image"}
+            ]
+            message["content"] = image_parts or content
+        elif isinstance(content, str):
+            # No structured parts available; leave as empty string
+            message["content"] = ""
+    return cloned
+
+
+async def _run_episode(
+    *,
+    mode: Mode,
+    seed: int,
+    client: AsyncOpenAI,
+    model: str,
+    max_steps: int,
+    temperature: float,
+    semaphore: asyncio.Semaphore,
+) -> EpisodeResult:
+    async with semaphore:
+        task_instance = _build_task_instance(seed)
+        env = CrafterClassicEnvironment(task_instance)
+        wrapper = CrafterEnvironmentWrapper(env, seed=seed)
+
+        policy = CrafterPolicy(inference_url="openai://chat-completions", model=model)
+        await policy.initialize({"use_tools": True, "model": model})
+
+        observation_packet = await wrapper.initialize()
+        achievements: set[str] = set()
+        total_reward = 0.0
+        steps_taken = 0
+        tool_calls_total = 0
+
+        frames_dir = OUTPUT_ROOT / f"{mode.value}_frames" / f"seed_{seed:04d}"
+        _save_observation_frame(observation_packet, frames_dir / "step_000.png")
+
+        for step_idx in range(max_steps):
+            obs_dict = observation_packet.get("observation")
+            if not isinstance(obs_dict, dict):
+                break
+
+            observation_for_policy: dict[str, Any]
+            metadata_payload: dict[str, Any] = {}
+
+            if mode == Mode.TEXT:
+                observation_for_policy = _strip_image_fields(observation_packet)
+            else:
+                observation_for_policy = json.loads(json.dumps(observation_packet))
+                metadata_payload["raw_observation"] = observation_packet
+
+            obs_text = policy._format_observation_for_llm(observation_for_policy)  # noqa: SLF001
+            _, meta = await policy.step(
+                observation_text=obs_text,
+                metadata=metadata_payload,
+            )
+            inference_request = json.loads(json.dumps(meta["inference_request"]))
+
+            if mode == Mode.IMAGE:
+                inference_request = _make_image_only_request(inference_request)
+
+            inference_request.update(
+                {
+                    "model": model,
+                    "temperature": temperature,
+                    "max_tokens": inference_request.get("max_tokens", 512),
+                }
+            )
+            inference_request.pop("stop_after_tool_calls", None)
+            inference_request.pop("thinking_mode", None)
+            inference_request.pop("thinking_budget", None)
+
+            response = await client.chat.completions.create(**inference_request)
+            response_dict = response.model_dump()
+
+            assistant_tool_calls = CrafterPolicy.parse_response_to_tool_calls(
+                response_dict,
+                use_tools=policy.use_tools,
+            )
+            if not assistant_tool_calls:
+                break
+
+            tool_calls_total += len(assistant_tool_calls)
+            assistant_message = response_dict["choices"][0].get("message") or {}
+            assistant_text = assistant_message.get("content")
+
+            env_response = await wrapper.step(assistant_tool_calls)
+            if not isinstance(env_response, dict):
+                raise RuntimeError(f"Unexpected environment response type: {type(env_response)!r}")
+
+            policy._append_assistant_turn(  # noqa: SLF001
+                assistant_text,
+                assistant_tool_calls,
+                env_response,
+            )
+
+            steps_taken += 1
+            obs = env_response.get("observation")
+            if isinstance(obs, dict):
+                ach = obs.get("achievements_status")
+                if isinstance(ach, dict):
+                    for name, unlocked in ach.items():
+                        if unlocked:
+                            achievements.add(str(name))
+                reward = obs.get("reward_last_step")
+                if isinstance(reward, (int, float)):
+                    total_reward += float(reward)
+
+            _save_observation_frame(env_response, frames_dir / f"step_{step_idx + 1:03d}.png")
+
+            if env_response.get("done"):
+                break
+            observation_packet = env_response
+
+        await wrapper.terminate()
+        return EpisodeResult(
+            mode=mode,
+            seed=seed,
+            steps_taken=steps_taken,
+            achievements=achievements,
+            total_reward=total_reward,
+            tool_calls=tool_calls_total,
+        )
+
+
+def _summarise(results: list[EpisodeResult]) -> dict[str, Any]:
+    grouped: dict[Mode, list[EpisodeResult]] = defaultdict(list)
+    for result in results:
+        grouped[result.mode].append(result)
+
+    summary: dict[str, Any] = {}
+    for mode, mode_results in grouped.items():
+        if not mode_results:
+            continue
+        mean_steps = sum(r.steps_taken for r in mode_results) / len(mode_results)
+        mean_achievements = sum(len(r.achievements) for r in mode_results) / len(mode_results)
+        achievement_counts = Counter()
+        for res in mode_results:
+            achievement_counts.update(res.achievements)
+        summary[mode.value] = {
+            "episodes": len(mode_results),
+            "mean_steps": round(mean_steps, 2),
+            "mean_achievements": round(mean_achievements, 2),
+            "total_tool_calls": sum(r.tool_calls for r in mode_results),
+            "achievements": {name: count for name, count in sorted(achievement_counts.items())},
+        }
+    return summary
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", default="gpt-4o-mini-2024-07-18", help="OpenAI model id to benchmark")
+    parser.add_argument("--seeds", type=int, default=20, help="Number of seeds per mode")
+    parser.add_argument("--steps", type=int, default=10, help="Max steps per episode")
+    parser.add_argument("--temperature", type=float, default=0.6, help="Sampling temperature")
+    parser.add_argument("--concurrency", type=int, default=10, help="Max concurrent OpenAI calls")
+    args = parser.parse_args()
+
+    api_key = os.getenv("OPENAI_API_KEY")
+    client = _ensure_openai_client(api_key)
+    semaphore = asyncio.Semaphore(max(1, args.concurrency))
+
+    OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
+
+    tasks: list[asyncio.Task[EpisodeResult]] = []
+    for mode in (Mode.TEXT, Mode.IMAGE, Mode.BOTH):
+        for seed in range(args.seeds):
+            task = asyncio.create_task(
+                _run_episode(
+                    mode=mode,
+                    seed=seed,
+                    client=client,
+                    model=args.model,
+                    max_steps=args.steps,
+                    temperature=args.temperature,
+                    semaphore=semaphore,
+                )
+            )
+            tasks.append(task)
+
+    results = await asyncio.gather(*tasks)
+    summary = _summarise(results)
+
+    summary_path = OUTPUT_ROOT / "vlm_benchmark_summary.json"
+    summary_path.write_text(json.dumps(summary, indent=2), encoding="utf-8")
+
+    print("\nBenchmark Summary")
+    print("-----------------")
+    print(json.dumps(summary, indent=2))
+    print(f"\nFrames stored under: {OUTPUT_ROOT}/<mode>_frames/seed_xxxx/")
+    print(f"Summary saved to: {summary_path}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/warming_up_to_rl/analyze_trace_db.py

@@ -9,7 +9,7 @@ import sqlite3
 import sys
 from collections import Counter, defaultdict
 from pathlib import Path
-from typing import Any, Dict, List, Set, Tuple
+from typing import Any
 
 Row = sqlite3.Row
 
@@ -56,7 +56,7 @@ def fetch_model_usage(conn: sqlite3.Connection) -> list[dict[str, Any]]:
 def _parse_json(value: Any) -> Any:
     if value is None:
         return None
-    if isinstance(value, (dict, list)):
+    if isinstance(value, dict | list):
         return value
     try:
         return json.loads(value)
@@ -64,7 +64,7 @@ def _parse_json(value: Any) -> Any:
         return None
 
 
-AchievementMap = dict[Tuple[str, int], dict[str, list[str]]]
+AchievementMap = dict[tuple[str, int], dict[str, list[str]]]
 
 
 def fetch_achievement_data(
@@ -162,7 +162,7 @@ def fetch_achievement_data(
         achievement_name_counts.update(achievement_set)
 
     achievement_size_counts: Counter = Counter()
-    for session_id, count in unique_counts_per_session.items():
+    for _session_id, count in unique_counts_per_session.items():
         achievement_size_counts[count] += 1
 
     return (
@@ -295,7 +295,7 @@ def format_reward_summary(outcome: dict[str, Any], breakdown: list[dict[str, Any
 
 
 def compute_model_achievement_stats(
-    conn: sqlite3.Connection, session_unique_sets: dict[str, Set[str]]
+    conn: sqlite3.Connection, session_unique_sets: dict[str, set[str]]
 ) -> dict[str, dict[str, Any]]:
     """Aggregate unique-achievement stats per model."""
 

examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml

@@ -42,9 +42,13 @@ base = "Qwen/Qwen3-4B"
 label = "crafter-rl-from-base"
 
 [rollout]
+env_name = "crafter"
 max_turns = 10
 episodes_per_batch = 64
-policy_name = "crafter"
+policy_name = "crafter-react"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
 
 [evaluation]
 # Run baseline evaluation over the first 100 seeds every 20 training iterations
@@ -55,6 +59,12 @@ seeds = [
 ]
 
 [training]
+num_epochs = 1
+iterations_per_epoch = 10
+batch_size = 16
+group_size = 4
+gradient_accumulation_steps = 1
+learning_rate = 5e-5
 log_interval = 1
 weight_sync_interval = 1
 # Additional RL hyperparameters can go here

examples/warming_up_to_rl/export_trace_sft.py

@@ -8,8 +8,9 @@ import json
 import sqlite3
 import sys
 from collections import Counter, defaultdict
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Set, Tuple
+from typing import Any
 
 Row = sqlite3.Row
 
@@ -23,7 +24,7 @@ def connect(db_path: Path) -> sqlite3.Connection:
 def _parse_json(value: Any) -> Any:
     if value is None:
         return None
-    if isinstance(value, (dict, list)):
+    if isinstance(value, dict | list):
         return value
     try:
         return json.loads(value)
@@ -31,7 +32,7 @@ def _parse_json(value: Any) -> Any:
         return None
 
 
-AchievementMap = dict[Tuple[str, int], dict[str, list[str]]]
+AchievementMap = dict[tuple[str, int], dict[str, list[str]]]
 
 
 def fetch_achievement_data(
@@ -116,7 +117,7 @@ def fetch_achievement_data(
         achievement_name_counts.update(achievement_set)
 
     achievement_size_counts: Counter = Counter()
-    for session_id, count in unique_counts_per_session.items():
+    for _session_id, count in unique_counts_per_session.items():
         achievement_size_counts[count] += 1
 
     return (
@@ -203,25 +204,71 @@ def parse_event_filters(specs: list[str] | None) -> list[tuple[str, float]]:
         if min_val_str:
             try:
                 min_val = float(min_val_str)
-            except ValueError:
+            except ValueError as e:
                 print(f"Invalid event reward specification '{spec}'", file=sys.stderr)
-                raise SystemExit(1)
+                raise SystemExit(1) from e
         filters.append((reward_type, min_val))
     return filters
 
 
-def _collect_text(parts: Iterable[dict[str, Any]] | None) -> str:
-    texts: list[str] = []
+def _collect_content(
+    parts: Iterable[dict[str, Any]] | None,
+) -> tuple[Any, bool]:
+    """Normalise multimodal content parts into OpenAI-style segments."""
+
     if not parts:
-        return ""
+        return "", False
+
+    segments: list[dict[str, Any]] = []
+    has_image = False
+
     for part in parts:
         if not isinstance(part, dict):
             continue
-        if part.get("type") == "text":
+        ptype = part.get("type")
+        if ptype == "text":
             text = part.get("text")
-            if isinstance(text, str) and text:
-                texts.append(text)
-    return "\n".join(texts)
+            if isinstance(text, str):
+                segments.append({"type": "text", "text": text})
+        elif ptype == "image":
+            uri = part.get("uri")
+            mime_type = part.get("mime_type") or "image/png"
+            data_url = None
+            if isinstance(uri, str) and uri.startswith("data:"):
+                data_url = uri
+            else:
+                source = part.get("data") or part.get("source")
+                if isinstance(source, dict):
+                    base64_data = source.get("data")
+                    media_type = source.get("media_type") or mime_type
+                    if isinstance(base64_data, str) and base64_data:
+                        data_url = f"data:{media_type};base64,{base64_data}"
+            if data_url:
+                has_image = True
+                segments.append({"type": "image_url", "image_url": {"url": data_url}})
+        elif ptype == "image_url":
+            image_url = part.get("image_url", {})
+            if isinstance(image_url, dict):
+                url = image_url.get("url")
+                if isinstance(url, str) and url:
+                    has_image = True
+                    segments.append({"type": "image_url", "image_url": {"url": url}})
+
+    if not segments:
+        return "", False
+    if not has_image and len(segments) == 1 and segments[0]["type"] == "text":
+        return segments[0]["text"], False
+    return segments, has_image
+
+
+def _normalise_output_content(content: Any) -> tuple[Any, bool]:
+    if isinstance(content, list):
+        return _collect_content(content)
+    if isinstance(content, str):
+        return content, False
+    if content is None:
+        return "", False
+    return str(content), False
 
 
 def _normalise_tool_calls(tool_calls: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
@@ -251,7 +298,7 @@ def _normalise_tool_calls(tool_calls: list[dict[str, Any]] | None) -> list[dict[
                     except Exception:
                         args = raw
 
-        if isinstance(args, (dict, list)):
+        if isinstance(args, dict | list):
             args_str = json.dumps(args, ensure_ascii=False)
         elif isinstance(args, str):
             args_str = args
@@ -279,7 +326,7 @@ def _normalise_tool_calls(tool_calls: list[dict[str, Any]] | None) -> list[dict[
 def build_sft_dataset(
     conn: sqlite3.Connection,
     achievements_map: AchievementMap,
-    sessions_filter: Set[str],
+    sessions_filter: set[str],
     *,
     allowed_models: set[str] | None = None,
     limit: int | None = None,
@@ -329,14 +376,18 @@ def build_sft_dataset(
 
         for record in call_records:
             messages: list[dict[str, Any]] = []
+            input_has_image = False
             for message in record.get("input_messages", []):
                 role = message.get("role", "unknown")
-                content = _collect_text(message.get("parts"))
-                if not content:
+                content, has_image = _collect_content(message.get("parts"))
+                if (content == "" or content is None) and not has_image:
                     continue
+                if has_image and role == "user":
+                    input_has_image = True
                 messages.append({"role": role, "content": content})
 
-            assistant_content = ""
+            assistant_content_value: Any = ""
+            assistant_has_image = False
             assistant_tool_calls: list[dict[str, Any]] = []
 
             output_text = record.get("output_text")
@@ -351,7 +402,9 @@ def build_sft_dataset(
                 choices = parsed_response.get("choices") or []
                 if choices:
                     message = choices[0].get("message") or {}
-                    assistant_content = message.get("content") or ""
+                    assistant_content_value, assistant_has_image = _normalise_output_content(
+                        message.get("content")
+                    )
                     assistant_tool_calls = _normalise_tool_calls(message.get("tool_calls"))
 
             if not assistant_tool_calls:
@@ -359,12 +412,13 @@ def build_sft_dataset(
 
             assistant_message: dict[str, Any] = {
                 "role": "assistant",
-                "content": assistant_content or "",
+                "content": assistant_content_value,
             }
             if assistant_tool_calls:
                 assistant_message["tool_calls"] = assistant_tool_calls
 
-            if assistant_message.get("content") == "" and not assistant_message.get("tool_calls"):
+            content_empty = assistant_message.get("content") in ("", None)
+            if content_empty and not assistant_message.get("tool_calls"):
                 continue
 
             messages.append(assistant_message)
@@ -385,6 +439,9 @@ def build_sft_dataset(
                     "turned_true": achievements.get("all", []),
                     "cumulative_unique": cumulative_unique[session_id],
                 },
+                "user_has_image": input_has_image,
+                "assistant_has_image": assistant_has_image,
+                "has_image": input_has_image or assistant_has_image,
             }
 
             dataset.append({"messages": messages, "metadata": metadata})

examples/warming_up_to_rl/groq_test.py

@@ -1,7 +1,7 @@
-from __future__ import annotations
-
 """Quick smoke test that drives a rollout through the Groq proxy-backed Crafter Task App."""
 
+from __future__ import annotations
+
 import argparse
 import asyncio
 import os
@@ -29,8 +29,8 @@ def _build_policy_payload(seed: int, model: str) -> dict[str, Any]:
             {
                 "role": "user",
                 "content": (
-                    "Environment seed {seed}. Plan initial survival/crafting steps and then call interact with concrete actions."
-                ).format(seed=seed),
+                    f"Environment seed {seed}. Plan initial survival/crafting steps and then call interact with concrete actions."
+                ),
             },
         ],
     }

examples/warming_up_to_rl/manage_secrets.py

@@ -8,11 +8,10 @@ import subprocess
 import sys
 import tempfile
 from pathlib import Path
-from typing import Dict, Tuple
 
 
-def load_env_file(path: Path) -> Dict[str, str]:
-    env: Dict[str, str] = {}
+def load_env_file(path: Path) -> dict[str, str]:
+    env: dict[str, str] = {}
     if not path.exists():
         raise FileNotFoundError(f".env not found at {path}")
     for line in path.read_text(encoding="utf-8").splitlines():
@@ -24,7 +23,7 @@ def load_env_file(path: Path) -> Dict[str, str]:
     return env
 
 
-def write_temp_env(kv: Dict[str, str]) -> Path:
+def write_temp_env(kv: dict[str, str]) -> Path:
     fd, p = tempfile.mkstemp(prefix="modal_secret_", suffix=".env")
     path = Path(p)
     with os.fdopen(fd, "w", encoding="utf-8") as fh:
@@ -33,14 +32,14 @@ def write_temp_env(kv: Dict[str, str]) -> Path:
     return path
 
 
-def run(cmd: str) -> Tuple[int, str]:
+def run(cmd: str) -> tuple[int, str]:
     proc = subprocess.run(
         cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
     )
     return proc.returncode, proc.stdout
 
 
-def ensure_secret(secret_name: str, kv: Dict[str, str]) -> None:
+def ensure_secret(secret_name: str, kv: dict[str, str]) -> None:
     if not kv:
         print(f"[skip] {secret_name}: no values provided")
         return
@@ -48,10 +47,10 @@ def ensure_secret(secret_name: str, kv: Dict[str, str]) -> None:
     kv_args = " ".join([f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in kv.items()])
 
     # Try plain modal first; fallback to uv run modal
-    def _create() -> Tuple[int, str]:
+    def _create() -> tuple[int, str]:
         return run(f"modal secret create {shlex.quote(secret_name)} {kv_args}")
 
-    def _delete() -> Tuple[int, str]:
+    def _delete() -> tuple[int, str]:
         return run(f"printf 'y\n' | modal secret delete {shlex.quote(secret_name)}")
 
     rc, out = _create()
@@ -86,15 +85,6 @@ def main() -> None:
     env = load_env_file(Path(args.env_path))
 
     # Secrets used by the task app
-    env_secret = {
-        k: v
-        for k, v in {
-            "ENVIRONMENT_API_KEY": env.get("ENVIRONMENT_API_KEY", ""),
-            "dev_environment_api_key": env.get("ENVIRONMENT_API_KEY", ""),
-        }.items()
-        if v
-    }
-
     groq_secret = {
         k: v
         for k, v in {
@@ -118,7 +108,12 @@ def main() -> None:
         {"SYNTH_API_KEY": env.get("SYNTH_API_KEY", "")} if env.get("SYNTH_API_KEY") else {}
     )
 
-    ensure_secret("crafter-environment-sdk", env_secret)
+    env_key = env.get("ENVIRONMENT_API_KEY", "")
+    if env_key:
+        print(
+            "Skipping Modal secret 'crafter-environment-sdk'; the task app now expects "
+            "ENVIRONMENT_API_KEY via --env-file so the CLI-minted value stays in sync."
+        )
     ensure_secret("groq-api-key", groq_secret)
     ensure_secret("openai-api-key", openai_secret)
     if synth_secret: