synth-ai 0.2.2.dev0__py3-none-any.whl → 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synth_ai/cli/__init__.py +66 -0
- synth_ai/cli/balance.py +205 -0
- synth_ai/cli/calc.py +70 -0
- synth_ai/cli/demo.py +74 -0
- synth_ai/{cli.py → cli/legacy_root_backup.py} +60 -15
- synth_ai/cli/man.py +103 -0
- synth_ai/cli/recent.py +126 -0
- synth_ai/cli/root.py +184 -0
- synth_ai/cli/status.py +126 -0
- synth_ai/cli/traces.py +136 -0
- synth_ai/cli/watch.py +508 -0
- synth_ai/config/base_url.py +53 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/analyze_semantic_words_markdown.py +252 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/filter_traces_sft_duckdb_v2_backup.py +413 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/filter_traces_sft_turso.py +646 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/kick_off_ft_synth.py +34 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/test_crafter_react_agent_lm_synth.py +1740 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/test_crafter_react_agent_lm_synth_v2_backup.py +1318 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/filter_traces_sft_duckdb_v2_backup.py +386 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/filter_traces_sft_turso.py +580 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/run_rollouts_for_models_and_compare_v2_backup.py +1352 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_openai_ft/test_crafter_react_agent_openai_v2_backup.py +2551 -0
- synth_ai/environments/examples/crafter_classic/agent_demos/crafter_trace_evaluation.py +1 -1
- synth_ai/environments/examples/crafter_classic/agent_demos/old/traces/session_crafter_episode_16_15227b68-2906-416f-acc4-d6a9b4fa5828_20250725_001154.json +1363 -1
- synth_ai/environments/examples/crafter_classic/agent_demos/test_crafter_react_agent.py +3 -3
- synth_ai/environments/examples/enron/dataset/corbt___enron_emails_sample_questions/default/0.0.0/293c9fe8170037e01cc9cf5834e0cd5ef6f1a6bb/dataset_info.json +1 -0
- synth_ai/environments/examples/nethack/helpers/achievements.json +64 -0
- synth_ai/environments/examples/red/units/test_exploration_strategy.py +1 -1
- synth_ai/environments/examples/red/units/test_menu_bug_reproduction.py +5 -5
- synth_ai/environments/examples/red/units/test_movement_debug.py +2 -2
- synth_ai/environments/examples/red/units/test_retry_movement.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/available_envs.json +122 -0
- synth_ai/environments/examples/sokoban/verified_puzzles.json +54987 -0
- synth_ai/experimental/synth_oss.py +446 -0
- synth_ai/learning/core.py +21 -0
- synth_ai/learning/gateway.py +4 -0
- synth_ai/learning/prompts/mipro.py +0 -0
- synth_ai/lm/__init__.py +3 -0
- synth_ai/lm/core/main.py +4 -0
- synth_ai/lm/core/main_v3.py +68 -13
- synth_ai/lm/core/vendor_clients.py +4 -0
- synth_ai/lm/provider_support/openai.py +11 -2
- synth_ai/lm/vendors/base.py +7 -0
- synth_ai/lm/vendors/openai_standard.py +339 -4
- synth_ai/lm/vendors/openai_standard_responses.py +243 -0
- synth_ai/lm/vendors/synth_client.py +155 -5
- synth_ai/lm/warmup.py +54 -17
- synth_ai/tracing/__init__.py +18 -0
- synth_ai/tracing_v1/__init__.py +29 -14
- synth_ai/tracing_v3/config.py +13 -7
- synth_ai/tracing_v3/db_config.py +6 -6
- synth_ai/tracing_v3/turso/manager.py +8 -8
- synth_ai/tui/__main__.py +13 -0
- synth_ai/tui/dashboard.py +329 -0
- synth_ai/v0/tracing/__init__.py +0 -0
- synth_ai/{tracing → v0/tracing}/base_client.py +3 -3
- synth_ai/{tracing → v0/tracing}/client_manager.py +1 -1
- synth_ai/{tracing → v0/tracing}/context.py +1 -1
- synth_ai/{tracing → v0/tracing}/decorators.py +11 -11
- synth_ai/v0/tracing/events/__init__.py +0 -0
- synth_ai/{tracing → v0/tracing}/events/manage.py +4 -4
- synth_ai/{tracing → v0/tracing}/events/scope.py +6 -6
- synth_ai/{tracing → v0/tracing}/events/store.py +3 -3
- synth_ai/{tracing → v0/tracing}/immediate_client.py +6 -6
- synth_ai/{tracing → v0/tracing}/log_client_base.py +2 -2
- synth_ai/{tracing → v0/tracing}/retry_queue.py +3 -3
- synth_ai/{tracing → v0/tracing}/trackers.py +2 -2
- synth_ai/{tracing → v0/tracing}/upload.py +4 -4
- synth_ai/v0/tracing_v1/__init__.py +16 -0
- synth_ai/{tracing_v1 → v0/tracing_v1}/base_client.py +3 -3
- synth_ai/{tracing_v1 → v0/tracing_v1}/client_manager.py +1 -1
- synth_ai/{tracing_v1 → v0/tracing_v1}/context.py +1 -1
- synth_ai/{tracing_v1 → v0/tracing_v1}/decorators.py +11 -11
- synth_ai/v0/tracing_v1/events/__init__.py +0 -0
- synth_ai/{tracing_v1 → v0/tracing_v1}/events/manage.py +4 -4
- synth_ai/{tracing_v1 → v0/tracing_v1}/events/scope.py +6 -6
- synth_ai/{tracing_v1 → v0/tracing_v1}/events/store.py +3 -3
- synth_ai/{tracing_v1 → v0/tracing_v1}/immediate_client.py +6 -6
- synth_ai/{tracing_v1 → v0/tracing_v1}/log_client_base.py +2 -2
- synth_ai/{tracing_v1 → v0/tracing_v1}/retry_queue.py +3 -3
- synth_ai/{tracing_v1 → v0/tracing_v1}/trackers.py +2 -2
- synth_ai/{tracing_v1 → v0/tracing_v1}/upload.py +4 -4
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.3.dist-info}/METADATA +98 -4
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.3.dist-info}/RECORD +98 -62
- /synth_ai/{tracing/events/__init__.py → environments/examples/crafter_classic/debug_translation.py} +0 -0
- /synth_ai/{tracing_v1/events/__init__.py → learning/prompts/gepa.py} +0 -0
- /synth_ai/{tracing → v0/tracing}/abstractions.py +0 -0
- /synth_ai/{tracing → v0/tracing}/config.py +0 -0
- /synth_ai/{tracing → v0/tracing}/local.py +0 -0
- /synth_ai/{tracing → v0/tracing}/utils.py +0 -0
- /synth_ai/{tracing_v1 → v0/tracing_v1}/abstractions.py +0 -0
- /synth_ai/{tracing_v1 → v0/tracing_v1}/config.py +0 -0
- /synth_ai/{tracing_v1 → v0/tracing_v1}/local.py +0 -0
- /synth_ai/{tracing_v1 → v0/tracing_v1}/utils.py +0 -0
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.3.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.3.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.3.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.2.dev0.dist-info → synth_ai-0.2.3.dist-info}/top_level.txt +0 -0
@@ -12,8 +12,8 @@ from pydantic import BaseModel, ConfigDict, field_validator
|
|
12
12
|
from requests.adapters import HTTPAdapter
|
13
13
|
from urllib3.poolmanager import PoolManager
|
14
14
|
|
15
|
-
from
|
16
|
-
from
|
15
|
+
from .abstractions import Dataset, SystemTrace
|
16
|
+
from .events.store import event_store
|
17
17
|
|
18
18
|
load_dotenv()
|
19
19
|
|
@@ -370,8 +370,8 @@ def upload_helper(
|
|
370
370
|
raise ValueError("SYNTH_API_KEY environment variable not set")
|
371
371
|
base_url = os.getenv("SYNTH_ENDPOINT_OVERRIDE", "https://agent-learning.onrender.com")
|
372
372
|
|
373
|
-
|
374
|
-
|
373
|
+
from .decorators import _local, active_events_var
|
374
|
+
from .trackers import synth_tracker_async, synth_tracker_sync
|
375
375
|
|
376
376
|
# First close any tracker events
|
377
377
|
if hasattr(synth_tracker_async, "active_events"):
|
@@ -0,0 +1,16 @@
|
|
1
|
+
import warnings
|
2
|
+
|
3
|
+
# Issue deprecation warning when this module is imported
|
4
|
+
warnings.warn(
|
5
|
+
"synth_ai.tracing (now tracing_v1) is deprecated. "
|
6
|
+
"Please use synth_ai.tracing_v2 instead. "
|
7
|
+
"Backend upload functionality is no longer supported in v1.",
|
8
|
+
DeprecationWarning,
|
9
|
+
stacklevel=2,
|
10
|
+
)
|
11
|
+
|
12
|
+
# Re-export the main components with deprecation warnings
|
13
|
+
from .config import *
|
14
|
+
from .decorators import *
|
15
|
+
from .trackers import *
|
16
|
+
from .abstractions import *
|
@@ -4,9 +4,9 @@ from abc import ABC, abstractmethod
|
|
4
4
|
from dataclasses import dataclass
|
5
5
|
from typing import Any, Dict, Optional
|
6
6
|
|
7
|
-
from
|
8
|
-
from
|
9
|
-
from
|
7
|
+
from .abstractions import Event
|
8
|
+
from .config import TracingConfig
|
9
|
+
from .events.store import event_store
|
10
10
|
|
11
11
|
logger = logging.getLogger(__name__)
|
12
12
|
|
@@ -8,9 +8,9 @@ from functools import wraps
|
|
8
8
|
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, ParamSpec, TypeVar, Union
|
9
9
|
|
10
10
|
if TYPE_CHECKING:
|
11
|
-
from
|
11
|
+
from .trackers import SynthTrackerAsync, SynthTrackerSync
|
12
12
|
|
13
|
-
from
|
13
|
+
from .abstractions import (
|
14
14
|
AgentComputeStep,
|
15
15
|
ArbitraryInputs,
|
16
16
|
ArbitraryOutputs,
|
@@ -19,28 +19,28 @@ from synth_ai.tracing_v1.abstractions import (
|
|
19
19
|
MessageInputs,
|
20
20
|
MessageOutputs,
|
21
21
|
)
|
22
|
-
from
|
23
|
-
from
|
22
|
+
from .config import EventManagement, LoggingMode, Origin, TracingConfig
|
23
|
+
from .context import (
|
24
24
|
get_current_context,
|
25
25
|
trace_context,
|
26
26
|
)
|
27
|
-
from
|
28
|
-
from
|
29
|
-
from
|
27
|
+
from .events.manage import set_current_event
|
28
|
+
from .events.store import event_store
|
29
|
+
from .immediate_client import (
|
30
30
|
AsyncImmediateLogClient,
|
31
31
|
ImmediateLogClient,
|
32
32
|
)
|
33
|
-
from
|
33
|
+
from .local import (
|
34
34
|
_local,
|
35
35
|
active_events_var,
|
36
36
|
logger,
|
37
37
|
)
|
38
|
-
from
|
39
|
-
from
|
38
|
+
from .retry_queue import initialize_retry_queue, retry_queue
|
39
|
+
from .trackers import (
|
40
40
|
synth_tracker_async,
|
41
41
|
synth_tracker_sync,
|
42
42
|
)
|
43
|
-
from
|
43
|
+
from .utils import get_system_id
|
44
44
|
|
45
45
|
logger = logging.getLogger(__name__)
|
46
46
|
|
File without changes
|
@@ -1,9 +1,9 @@
|
|
1
1
|
import time
|
2
2
|
from typing import Literal, Optional
|
3
3
|
|
4
|
-
from
|
5
|
-
from
|
6
|
-
from
|
4
|
+
from ..abstractions import Event
|
5
|
+
from ..events.store import event_store
|
6
|
+
from ..local import _local, logger
|
7
7
|
|
8
8
|
|
9
9
|
def get_current_event(event_type: str) -> "Event":
|
@@ -75,7 +75,7 @@ def set_current_event(event: Optional["Event"], decorator_type: Literal["sync",
|
|
75
75
|
_local.active_events[unique_key] = event # Unique key for async
|
76
76
|
|
77
77
|
else:
|
78
|
-
from
|
78
|
+
from ..local import (
|
79
79
|
active_events_var,
|
80
80
|
system_id_var,
|
81
81
|
system_instance_id_var,
|
@@ -1,20 +1,20 @@
|
|
1
1
|
import time
|
2
2
|
from contextlib import contextmanager
|
3
3
|
|
4
|
-
from
|
5
|
-
from
|
6
|
-
from
|
4
|
+
from ..abstractions import Event
|
5
|
+
from ..config import LoggingMode
|
6
|
+
from ..decorators import (
|
7
7
|
_local,
|
8
8
|
clear_current_event,
|
9
9
|
get_tracing_config,
|
10
10
|
set_current_event,
|
11
11
|
)
|
12
|
-
from
|
13
|
-
from
|
12
|
+
from ..events.store import event_store
|
13
|
+
from ..immediate_client import (
|
14
14
|
AsyncImmediateLogClient,
|
15
15
|
ImmediateLogClient,
|
16
16
|
)
|
17
|
-
from
|
17
|
+
from ..local import (
|
18
18
|
system_id_var,
|
19
19
|
system_instance_id_var,
|
20
20
|
system_name_var,
|
@@ -4,8 +4,8 @@ import time
|
|
4
4
|
from threading import RLock # Change this import
|
5
5
|
from typing import Any, Dict, List
|
6
6
|
|
7
|
-
from
|
8
|
-
from
|
7
|
+
from ..abstractions import Event, EventPartitionElement, SystemTrace
|
8
|
+
from ..local import ( # Import context variables
|
9
9
|
_local,
|
10
10
|
active_events_var,
|
11
11
|
)
|
@@ -36,7 +36,7 @@ class EventStore:
|
|
36
36
|
# Get system_instance_metadata from context if available
|
37
37
|
system_instance_metadata = {}
|
38
38
|
try:
|
39
|
-
from
|
39
|
+
from ..context import get_current_context
|
40
40
|
|
41
41
|
context = get_current_context()
|
42
42
|
if (
|
@@ -5,10 +5,10 @@ from typing import Dict
|
|
5
5
|
|
6
6
|
import httpx
|
7
7
|
|
8
|
-
from
|
9
|
-
from
|
10
|
-
from
|
11
|
-
from
|
8
|
+
from .abstractions import Event
|
9
|
+
from .client_manager import ClientManager
|
10
|
+
from .config import TracingConfig
|
11
|
+
from .log_client_base import BaseAsyncLogClient, BaseLogClient
|
12
12
|
|
13
13
|
logger = logging.getLogger(__name__)
|
14
14
|
|
@@ -22,7 +22,7 @@ class ImmediateLogClient(BaseLogClient):
|
|
22
22
|
|
23
23
|
def send_event(self, event: Event, system_info: Dict[str, str]) -> bool:
|
24
24
|
"""Send a single event with retries and fallback"""
|
25
|
-
from
|
25
|
+
from .retry_queue import (
|
26
26
|
retry_queue, # Import here to avoid circular import
|
27
27
|
)
|
28
28
|
|
@@ -73,7 +73,7 @@ class AsyncImmediateLogClient(BaseAsyncLogClient):
|
|
73
73
|
|
74
74
|
async def send_event(self, event: Event, system_info: Dict[str, str]) -> bool:
|
75
75
|
"""Send a single event with retries and fallback (async version)"""
|
76
|
-
from
|
76
|
+
from .retry_queue import retry_queue
|
77
77
|
|
78
78
|
if not self.config.api_key:
|
79
79
|
logger.error("No API key provided")
|
@@ -5,8 +5,8 @@ from collections import deque
|
|
5
5
|
from dataclasses import dataclass
|
6
6
|
from typing import Dict, List, Optional, Tuple
|
7
7
|
|
8
|
-
from
|
9
|
-
from
|
8
|
+
from .abstractions import Event
|
9
|
+
from .config import TracingConfig
|
10
10
|
|
11
11
|
logger = logging.getLogger(__name__)
|
12
12
|
|
@@ -88,7 +88,7 @@ class RetryQueue:
|
|
88
88
|
failure_count = 0
|
89
89
|
|
90
90
|
try:
|
91
|
-
from
|
91
|
+
from .immediate_client import (
|
92
92
|
ImmediateLogClient, # Import here to avoid circular import
|
93
93
|
)
|
94
94
|
|
@@ -4,8 +4,8 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union
|
|
4
4
|
|
5
5
|
from pydantic import BaseModel
|
6
6
|
|
7
|
-
from
|
8
|
-
from
|
7
|
+
from .config import VALID_TYPES, Message, ModelParams
|
8
|
+
from .local import _local
|
9
9
|
|
10
10
|
# Existing SynthTrackerSync and SynthTrackerAsync classes...
|
11
11
|
|
@@ -13,8 +13,8 @@ from pydantic import BaseModel, ConfigDict, field_validator
|
|
13
13
|
from requests.adapters import HTTPAdapter
|
14
14
|
from urllib3.poolmanager import PoolManager
|
15
15
|
|
16
|
-
from
|
17
|
-
from
|
16
|
+
from .abstractions import Dataset, SystemTrace
|
17
|
+
from .events.store import event_store
|
18
18
|
|
19
19
|
load_dotenv()
|
20
20
|
|
@@ -390,8 +390,8 @@ def upload_helper(
|
|
390
390
|
raise ValueError("SYNTH_API_KEY environment variable not set")
|
391
391
|
base_url = os.getenv("SYNTH_ENDPOINT_OVERRIDE", "https://agent-learning.onrender.com")
|
392
392
|
|
393
|
-
from
|
394
|
-
from
|
393
|
+
from .decorators import _local, active_events_var
|
394
|
+
from .trackers import synth_tracker_async, synth_tracker_sync
|
395
395
|
|
396
396
|
# First close any tracker events
|
397
397
|
if hasattr(synth_tracker_async, "active_events"):
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: synth-ai
|
3
|
-
Version: 0.2.
|
3
|
+
Version: 0.2.3
|
4
4
|
Summary: Software for aiding the best and multiplying the will - Core AI functionality and tracing
|
5
5
|
Author-email: Synth AI <josh@usesynth.ai>
|
6
6
|
License-Expression: MIT
|
@@ -18,7 +18,7 @@ Requires-Dist: tqdm>=4.66.4
|
|
18
18
|
Requires-Dist: jsonschema>=4.23.0
|
19
19
|
Requires-Dist: backoff>=2.0.0
|
20
20
|
Requires-Dist: typing_extensions>=4.0.0
|
21
|
-
Requires-Dist: openai
|
21
|
+
Requires-Dist: openai>=1.99.0
|
22
22
|
Requires-Dist: anthropic>=0.42.0
|
23
23
|
Requires-Dist: langfuse<3.0.0,>=2.53.9
|
24
24
|
Requires-Dist: opentelemetry-api>=1.27.0
|
@@ -45,6 +45,10 @@ Requires-Dist: google-api-core>=2.25.1
|
|
45
45
|
Requires-Dist: google-generativeai>=0.8.5
|
46
46
|
Requires-Dist: crafter>=1.8.3
|
47
47
|
Requires-Dist: click>=8.1.0
|
48
|
+
Requires-Dist: textual>=1.1.0
|
49
|
+
Requires-Dist: openai-harmony>=0.0.1
|
50
|
+
Requires-Dist: asyncpg>=0.30.0
|
51
|
+
Requires-Dist: aiohttp>=3.8.0
|
48
52
|
Provides-Extra: dev
|
49
53
|
Requires-Dist: build>=1.2.2.post1; extra == "dev"
|
50
54
|
Requires-Dist: twine>=4.0.0; extra == "dev"
|
@@ -103,5 +107,95 @@ pip install synth-ai[all]
|
|
103
107
|
```
|
104
108
|
|
105
109
|
### Spinning Up
|
106
|
-
|
107
|
-
|
110
|
+
|
111
|
+
Start the Synth AI service daemon (includes sqld database + environment service):
|
112
|
+
|
113
|
+
```bash
|
114
|
+
# Start both database daemon (port 8080) and environment service (port 8901)
|
115
|
+
uvx synth-ai serve
|
116
|
+
```
|
117
|
+
|
118
|
+
#### Service Command Options
|
119
|
+
|
120
|
+
```bash
|
121
|
+
uvx synth-ai serve [OPTIONS]
|
122
|
+
```
|
123
|
+
|
124
|
+
**Available Options:**
|
125
|
+
- `--db-file` - Database file path (default: "synth_ai.db")
|
126
|
+
- `--sqld-port` - Port for sqld HTTP interface (default: 8080)
|
127
|
+
- `--env-port` - Port for environment service (default: 8901)
|
128
|
+
- `--no-sqld` - Skip starting sqld database daemon
|
129
|
+
- `--no-env` - Skip starting environment service
|
130
|
+
|
131
|
+
**Examples:**
|
132
|
+
```bash
|
133
|
+
# Start with custom ports
|
134
|
+
uvx synth-ai serve --sqld-port 8081 --env-port 8902
|
135
|
+
|
136
|
+
# Start only the environment service
|
137
|
+
uvx synth-ai serve --no-sqld
|
138
|
+
|
139
|
+
# Start only the database service
|
140
|
+
uvx synth-ai serve --no-env
|
141
|
+
```
|
142
|
+
|
143
|
+
#### What the Serve Command Provides
|
144
|
+
|
145
|
+
**sqld Database Service (port 8080)**
|
146
|
+
- Local SQLite-compatible database server with HTTP API
|
147
|
+
- Automatically downloads and installs sqld binary if needed
|
148
|
+
- Provides persistent storage for agent interactions and traces
|
149
|
+
|
150
|
+
**Environment Service (port 8901)**
|
151
|
+
- FastAPI service for managing AI environments and tasks
|
152
|
+
- Built-in environments: Crafter, Sokoban, MiniGrid, TicTacToe, Verilog, NetHack, Enron
|
153
|
+
- RESTful API for environment initialization, stepping, and termination
|
154
|
+
- Dynamic environment registry for custom environments
|
155
|
+
|
156
|
+
In another terminal, run your first example:
|
157
|
+
|
158
|
+
```bash
|
159
|
+
# Run a Crafter agent demo with Gemini
|
160
|
+
./examples/run_crafter_demo.sh
|
161
|
+
```
|
162
|
+
|
163
|
+
This will:
|
164
|
+
- Start the sqld database daemon with HTTP API on port 8080
|
165
|
+
- Launch the environment service API on port 8901
|
166
|
+
- Run a reactive agent in the Crafter environment using Gemini 1.5 Flash
|
167
|
+
|
168
|
+
#### Demos (Eval + Finetuning)
|
169
|
+
|
170
|
+
You can run interactive demos from the repo without remembering exact commands:
|
171
|
+
|
172
|
+
```bash
|
173
|
+
# Lists all available demos under examples/, then prompts you to choose
|
174
|
+
uvx synth-ai demo
|
175
|
+
```
|
176
|
+
|
177
|
+
Today this includes:
|
178
|
+
- Eval demo: `examples/evals/run_demo.sh`
|
179
|
+
- Prompts for models, episodes, etc.
|
180
|
+
- Runs Crafter rollouts with v3 tracing, then analyzes and filters traces
|
181
|
+
- Writes a JSONL like `ft_data/evals_filtered.jsonl` for downstream use
|
182
|
+
- Finetuning demo: `examples/finetuning/synth_qwen/run_demo.sh`
|
183
|
+
- Guides you through: rollouts → filter v3 traces → prepare SFT JSONL
|
184
|
+
- Pair with `uvpm examples.finetuning.synth_qwen.sft_kickoff` to start an SFT job when ready
|
185
|
+
|
186
|
+
Notes:
|
187
|
+
- Ensure the service is running (`uvx synth-ai serve`) so v3 traces are recorded locally.
|
188
|
+
- Set API configuration for finetuning:
|
189
|
+
- `export LEARNING_V2_BASE_URL="http://localhost:8000/api"` (or your proxy)
|
190
|
+
- `export SYNTH_API_KEY="sk_live_..."`
|
191
|
+
- v3 trace data is stored under `traces/v3/synth_ai.db/` by default. Inspect with `uvx synth-ai traces`.
|
192
|
+
|
193
|
+
### One-Command Demos
|
194
|
+
|
195
|
+
Quickly browse and launch interactive demos under `examples/`:
|
196
|
+
|
197
|
+
```bash
|
198
|
+
uvx synth-ai demo
|
199
|
+
```
|
200
|
+
|
201
|
+
This lists all `run_demo.sh` scripts found in the repo (e.g., eval comparisons, finetuning flows) and lets you pick one to run.
|