synth-ai 0.2.4.dev6__py3-none-any.whl → 0.2.4.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synth_ai/__init__.py +18 -9
- synth_ai/cli/__init__.py +10 -5
- synth_ai/cli/balance.py +22 -17
- synth_ai/cli/calc.py +2 -3
- synth_ai/cli/demo.py +3 -5
- synth_ai/cli/legacy_root_backup.py +58 -32
- synth_ai/cli/man.py +22 -19
- synth_ai/cli/recent.py +9 -8
- synth_ai/cli/root.py +58 -13
- synth_ai/cli/status.py +13 -6
- synth_ai/cli/traces.py +45 -21
- synth_ai/cli/watch.py +40 -37
- synth_ai/config/base_url.py +1 -3
- synth_ai/core/experiment.py +1 -2
- synth_ai/environments/__init__.py +2 -6
- synth_ai/environments/environment/artifacts/base.py +3 -1
- synth_ai/environments/environment/db/sqlite.py +1 -1
- synth_ai/environments/environment/registry.py +19 -20
- synth_ai/environments/environment/resources/sqlite.py +2 -3
- synth_ai/environments/environment/rewards/core.py +3 -2
- synth_ai/environments/environment/tools/__init__.py +6 -4
- synth_ai/environments/examples/crafter_classic/__init__.py +1 -1
- synth_ai/environments/examples/crafter_classic/engine.py +13 -13
- synth_ai/environments/examples/crafter_classic/engine_deterministic_patch.py +1 -0
- synth_ai/environments/examples/crafter_classic/engine_helpers/action_map.py +2 -1
- synth_ai/environments/examples/crafter_classic/engine_helpers/serialization.py +2 -1
- synth_ai/environments/examples/crafter_classic/engine_serialization_patch_v3.py +3 -2
- synth_ai/environments/examples/crafter_classic/environment.py +16 -15
- synth_ai/environments/examples/crafter_classic/taskset.py +2 -2
- synth_ai/environments/examples/crafter_classic/trace_hooks_v3.py +2 -3
- synth_ai/environments/examples/crafter_classic/world_config_patch_simple.py +2 -1
- synth_ai/environments/examples/crafter_custom/crafter/__init__.py +2 -2
- synth_ai/environments/examples/crafter_custom/crafter/config.py +2 -2
- synth_ai/environments/examples/crafter_custom/crafter/env.py +1 -5
- synth_ai/environments/examples/crafter_custom/crafter/objects.py +1 -2
- synth_ai/environments/examples/crafter_custom/crafter/worldgen.py +1 -2
- synth_ai/environments/examples/crafter_custom/dataset_builder.py +5 -5
- synth_ai/environments/examples/crafter_custom/environment.py +13 -13
- synth_ai/environments/examples/crafter_custom/run_dataset.py +5 -5
- synth_ai/environments/examples/enron/art_helpers/email_search_tools.py +2 -2
- synth_ai/environments/examples/enron/art_helpers/local_email_db.py +5 -4
- synth_ai/environments/examples/enron/art_helpers/types_enron.py +2 -1
- synth_ai/environments/examples/enron/engine.py +18 -14
- synth_ai/environments/examples/enron/environment.py +12 -11
- synth_ai/environments/examples/enron/taskset.py +7 -7
- synth_ai/environments/examples/minigrid/__init__.py +6 -6
- synth_ai/environments/examples/minigrid/engine.py +6 -6
- synth_ai/environments/examples/minigrid/environment.py +6 -6
- synth_ai/environments/examples/minigrid/puzzle_loader.py +3 -2
- synth_ai/environments/examples/minigrid/taskset.py +13 -13
- synth_ai/environments/examples/nethack/achievements.py +1 -1
- synth_ai/environments/examples/nethack/engine.py +8 -7
- synth_ai/environments/examples/nethack/environment.py +10 -9
- synth_ai/environments/examples/nethack/helpers/__init__.py +8 -9
- synth_ai/environments/examples/nethack/helpers/action_mapping.py +1 -1
- synth_ai/environments/examples/nethack/helpers/nle_wrapper.py +2 -1
- synth_ai/environments/examples/nethack/helpers/observation_utils.py +1 -1
- synth_ai/environments/examples/nethack/helpers/recording_wrapper.py +3 -4
- synth_ai/environments/examples/nethack/helpers/trajectory_recorder.py +6 -5
- synth_ai/environments/examples/nethack/helpers/visualization/replay_viewer.py +5 -5
- synth_ai/environments/examples/nethack/helpers/visualization/visualizer.py +7 -6
- synth_ai/environments/examples/nethack/taskset.py +5 -5
- synth_ai/environments/examples/red/engine.py +9 -8
- synth_ai/environments/examples/red/engine_helpers/reward_components.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/__init__.py +7 -7
- synth_ai/environments/examples/red/engine_helpers/reward_library/adaptive_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/battle_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/composite_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/economy_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/efficiency_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/exploration_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/novelty_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/pallet_town_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/pokemon_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/social_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/story_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/screen_analysis.py +3 -2
- synth_ai/environments/examples/red/engine_helpers/state_extraction.py +2 -1
- synth_ai/environments/examples/red/environment.py +18 -15
- synth_ai/environments/examples/red/taskset.py +5 -3
- synth_ai/environments/examples/sokoban/engine.py +16 -13
- synth_ai/environments/examples/sokoban/engine_helpers/room_utils.py +3 -2
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/__init__.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/__init__.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/boxoban_env.py +7 -5
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/render_utils.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/room_utils.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env.py +5 -4
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_fixed_targets.py +3 -2
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_pull.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_two_player.py +5 -4
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_variations.py +1 -1
- synth_ai/environments/examples/sokoban/environment.py +15 -14
- synth_ai/environments/examples/sokoban/generate_verified_puzzles.py +5 -3
- synth_ai/environments/examples/sokoban/puzzle_loader.py +3 -2
- synth_ai/environments/examples/sokoban/taskset.py +13 -10
- synth_ai/environments/examples/tictactoe/engine.py +6 -6
- synth_ai/environments/examples/tictactoe/environment.py +8 -7
- synth_ai/environments/examples/tictactoe/taskset.py +6 -5
- synth_ai/environments/examples/verilog/engine.py +4 -3
- synth_ai/environments/examples/verilog/environment.py +11 -10
- synth_ai/environments/examples/verilog/taskset.py +14 -12
- synth_ai/environments/examples/wordle/__init__.py +5 -5
- synth_ai/environments/examples/wordle/engine.py +32 -25
- synth_ai/environments/examples/wordle/environment.py +21 -16
- synth_ai/environments/examples/wordle/helpers/generate_instances_wordfreq.py +6 -6
- synth_ai/environments/examples/wordle/taskset.py +20 -12
- synth_ai/environments/reproducibility/core.py +1 -1
- synth_ai/environments/reproducibility/tree.py +21 -21
- synth_ai/environments/service/app.py +3 -2
- synth_ai/environments/service/core_routes.py +104 -110
- synth_ai/environments/service/external_registry.py +1 -2
- synth_ai/environments/service/registry.py +1 -1
- synth_ai/environments/stateful/core.py +1 -2
- synth_ai/environments/stateful/engine.py +1 -1
- synth_ai/environments/tasks/api.py +4 -4
- synth_ai/environments/tasks/core.py +14 -12
- synth_ai/environments/tasks/filters.py +6 -4
- synth_ai/environments/tasks/utils.py +13 -11
- synth_ai/evals/base.py +2 -3
- synth_ai/experimental/synth_oss.py +4 -4
- synth_ai/learning/gateway.py +1 -3
- synth_ai/learning/prompts/banking77_injection_eval.py +15 -10
- synth_ai/learning/prompts/hello_world_in_context_injection_ex.py +26 -14
- synth_ai/learning/prompts/mipro.py +61 -52
- synth_ai/learning/prompts/random_search.py +42 -43
- synth_ai/learning/prompts/run_mipro_banking77.py +32 -20
- synth_ai/learning/prompts/run_random_search_banking77.py +71 -52
- synth_ai/lm/__init__.py +5 -5
- synth_ai/lm/caching/ephemeral.py +9 -9
- synth_ai/lm/caching/handler.py +20 -20
- synth_ai/lm/caching/persistent.py +10 -10
- synth_ai/lm/config.py +3 -3
- synth_ai/lm/constants.py +7 -7
- synth_ai/lm/core/all.py +17 -3
- synth_ai/lm/core/exceptions.py +0 -2
- synth_ai/lm/core/main.py +26 -41
- synth_ai/lm/core/main_v3.py +20 -10
- synth_ai/lm/core/vendor_clients.py +18 -17
- synth_ai/lm/injection.py +7 -8
- synth_ai/lm/overrides.py +21 -19
- synth_ai/lm/provider_support/__init__.py +1 -1
- synth_ai/lm/provider_support/anthropic.py +15 -15
- synth_ai/lm/provider_support/openai.py +23 -21
- synth_ai/lm/structured_outputs/handler.py +34 -32
- synth_ai/lm/structured_outputs/inject.py +24 -27
- synth_ai/lm/structured_outputs/rehabilitate.py +19 -15
- synth_ai/lm/tools/base.py +17 -16
- synth_ai/lm/unified_interface.py +17 -18
- synth_ai/lm/vendors/base.py +20 -18
- synth_ai/lm/vendors/core/anthropic_api.py +36 -27
- synth_ai/lm/vendors/core/gemini_api.py +31 -36
- synth_ai/lm/vendors/core/mistral_api.py +19 -19
- synth_ai/lm/vendors/core/openai_api.py +11 -10
- synth_ai/lm/vendors/openai_standard.py +113 -87
- synth_ai/lm/vendors/openai_standard_responses.py +74 -61
- synth_ai/lm/vendors/retries.py +9 -1
- synth_ai/lm/vendors/supported/custom_endpoint.py +26 -26
- synth_ai/lm/vendors/supported/deepseek.py +10 -10
- synth_ai/lm/vendors/supported/grok.py +8 -8
- synth_ai/lm/vendors/supported/ollama.py +2 -1
- synth_ai/lm/vendors/supported/openrouter.py +11 -9
- synth_ai/lm/vendors/synth_client.py +69 -63
- synth_ai/lm/warmup.py +8 -7
- synth_ai/tracing/__init__.py +22 -10
- synth_ai/tracing_v1/__init__.py +22 -20
- synth_ai/tracing_v3/__init__.py +7 -7
- synth_ai/tracing_v3/abstractions.py +56 -52
- synth_ai/tracing_v3/config.py +4 -2
- synth_ai/tracing_v3/db_config.py +6 -8
- synth_ai/tracing_v3/decorators.py +29 -30
- synth_ai/tracing_v3/examples/basic_usage.py +12 -12
- synth_ai/tracing_v3/hooks.py +21 -21
- synth_ai/tracing_v3/llm_call_record_helpers.py +85 -98
- synth_ai/tracing_v3/lm_call_record_abstractions.py +2 -4
- synth_ai/tracing_v3/migration_helper.py +3 -5
- synth_ai/tracing_v3/replica_sync.py +30 -32
- synth_ai/tracing_v3/session_tracer.py +35 -29
- synth_ai/tracing_v3/storage/__init__.py +1 -1
- synth_ai/tracing_v3/storage/base.py +8 -7
- synth_ai/tracing_v3/storage/config.py +4 -4
- synth_ai/tracing_v3/storage/factory.py +4 -4
- synth_ai/tracing_v3/storage/utils.py +9 -9
- synth_ai/tracing_v3/turso/__init__.py +3 -3
- synth_ai/tracing_v3/turso/daemon.py +9 -9
- synth_ai/tracing_v3/turso/manager.py +60 -48
- synth_ai/tracing_v3/turso/models.py +24 -19
- synth_ai/tracing_v3/utils.py +5 -5
- synth_ai/tui/__main__.py +1 -1
- synth_ai/tui/cli/query_experiments.py +2 -3
- synth_ai/tui/cli/query_experiments_v3.py +2 -3
- synth_ai/tui/dashboard.py +97 -86
- synth_ai/v0/tracing/abstractions.py +28 -28
- synth_ai/v0/tracing/base_client.py +9 -9
- synth_ai/v0/tracing/client_manager.py +7 -7
- synth_ai/v0/tracing/config.py +7 -7
- synth_ai/v0/tracing/context.py +6 -6
- synth_ai/v0/tracing/decorators.py +6 -5
- synth_ai/v0/tracing/events/manage.py +1 -1
- synth_ai/v0/tracing/events/store.py +5 -4
- synth_ai/v0/tracing/immediate_client.py +4 -5
- synth_ai/v0/tracing/local.py +3 -3
- synth_ai/v0/tracing/log_client_base.py +4 -5
- synth_ai/v0/tracing/retry_queue.py +5 -6
- synth_ai/v0/tracing/trackers.py +25 -25
- synth_ai/v0/tracing/upload.py +6 -0
- synth_ai/v0/tracing_v1/__init__.py +1 -1
- synth_ai/v0/tracing_v1/abstractions.py +28 -28
- synth_ai/v0/tracing_v1/base_client.py +9 -9
- synth_ai/v0/tracing_v1/client_manager.py +7 -7
- synth_ai/v0/tracing_v1/config.py +7 -7
- synth_ai/v0/tracing_v1/context.py +6 -6
- synth_ai/v0/tracing_v1/decorators.py +7 -6
- synth_ai/v0/tracing_v1/events/manage.py +1 -1
- synth_ai/v0/tracing_v1/events/store.py +5 -4
- synth_ai/v0/tracing_v1/immediate_client.py +4 -5
- synth_ai/v0/tracing_v1/local.py +3 -3
- synth_ai/v0/tracing_v1/log_client_base.py +4 -5
- synth_ai/v0/tracing_v1/retry_queue.py +5 -6
- synth_ai/v0/tracing_v1/trackers.py +25 -25
- synth_ai/v0/tracing_v1/upload.py +25 -24
- synth_ai/zyk/__init__.py +1 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/METADATA +1 -11
- synth_ai-0.2.4.dev7.dist-info/RECORD +299 -0
- synth_ai-0.2.4.dev6.dist-info/RECORD +0 -299
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/top_level.txt +0 -0
synth_ai/v0/tracing_v1/local.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import logging
|
2
2
|
import threading
|
3
3
|
from contextvars import ContextVar
|
4
|
-
from typing import Any
|
4
|
+
from typing import Any
|
5
5
|
|
6
6
|
logger = logging.getLogger(__name__)
|
7
7
|
|
@@ -12,7 +12,7 @@ _local = threading.local()
|
|
12
12
|
system_name_var: ContextVar[str] = ContextVar("system_name", default=None)
|
13
13
|
system_id_var: ContextVar[str] = ContextVar("system_id", default=None)
|
14
14
|
system_instance_id_var: ContextVar[str] = ContextVar("system_instance_id", default=None)
|
15
|
-
system_instance_metadata_var: ContextVar[
|
15
|
+
system_instance_metadata_var: ContextVar[dict[str, Any]] = ContextVar(
|
16
16
|
"system_instance_metadata", default={}
|
17
17
|
)
|
18
|
-
active_events_var: ContextVar[
|
18
|
+
active_events_var: ContextVar[dict[str, Any]] = ContextVar("active_events", default={})
|
@@ -1,4 +1,3 @@
|
|
1
|
-
from typing import Dict
|
2
1
|
|
3
2
|
from .abstractions import Event
|
4
3
|
from .config import TracingConfig
|
@@ -11,7 +10,7 @@ class BaseLogClient:
|
|
11
10
|
self.config = config
|
12
11
|
self.client_manager = None
|
13
12
|
|
14
|
-
def _prepare_payload(self, event: Event, system_info:
|
13
|
+
def _prepare_payload(self, event: Event, system_info: dict[str, str]) -> dict:
|
15
14
|
"""Prepare the payload for sending."""
|
16
15
|
return {
|
17
16
|
"event": event.to_dict(),
|
@@ -33,7 +32,7 @@ class BaseLogClient:
|
|
33
32
|
pass
|
34
33
|
|
35
34
|
def _handle_failure(
|
36
|
-
self, event: Event, system_info:
|
35
|
+
self, event: Event, system_info: dict[str, str], exception: Exception
|
37
36
|
) -> None:
|
38
37
|
"""Handle failed event sending."""
|
39
38
|
pass
|
@@ -46,7 +45,7 @@ class BaseAsyncLogClient:
|
|
46
45
|
self.config = config
|
47
46
|
self.client_manager = None
|
48
47
|
|
49
|
-
def _prepare_payload(self, event: Event, system_info:
|
48
|
+
def _prepare_payload(self, event: Event, system_info: dict[str, str]) -> dict:
|
50
49
|
"""Prepare the payload for sending."""
|
51
50
|
return {
|
52
51
|
"event": event.to_dict(),
|
@@ -68,7 +67,7 @@ class BaseAsyncLogClient:
|
|
68
67
|
pass
|
69
68
|
|
70
69
|
def _handle_failure(
|
71
|
-
self, event: Event, system_info:
|
70
|
+
self, event: Event, system_info: dict[str, str], exception: Exception
|
72
71
|
) -> None:
|
73
72
|
"""Handle failed event sending."""
|
74
73
|
pass
|
@@ -3,7 +3,6 @@ import threading
|
|
3
3
|
import time
|
4
4
|
from collections import deque
|
5
5
|
from dataclasses import dataclass
|
6
|
-
from typing import Dict, List, Optional, Tuple
|
7
6
|
|
8
7
|
from .abstractions import Event
|
9
8
|
from .config import TracingConfig
|
@@ -16,7 +15,7 @@ class QueuedEvent:
|
|
16
15
|
"""Represents an event that failed to upload and needs to be retried."""
|
17
16
|
|
18
17
|
event: Event
|
19
|
-
system_info:
|
18
|
+
system_info: dict[str, str]
|
20
19
|
attempt_count: int = 0
|
21
20
|
last_attempt: float = 0
|
22
21
|
|
@@ -31,7 +30,7 @@ class RetryQueue:
|
|
31
30
|
self._is_processing = False
|
32
31
|
self._batch_size = config.batch_size
|
33
32
|
|
34
|
-
def add_failed_event(self, event: Event, system_info:
|
33
|
+
def add_failed_event(self, event: Event, system_info: dict[str, str]) -> None:
|
35
34
|
"""Add a failed event to the retry queue."""
|
36
35
|
with self._lock:
|
37
36
|
# Check if event is already in queue to avoid duplicates
|
@@ -53,7 +52,7 @@ class RetryQueue:
|
|
53
52
|
)
|
54
53
|
logger.debug(f"Added event to retry queue. Queue size: {len(self.queue)}")
|
55
54
|
|
56
|
-
def get_retryable_events(self, max_events:
|
55
|
+
def get_retryable_events(self, max_events: int | None = None) -> list[QueuedEvent]:
|
57
56
|
"""Get events that are ready to be retried."""
|
58
57
|
now = time.time()
|
59
58
|
retryable = []
|
@@ -74,7 +73,7 @@ class RetryQueue:
|
|
74
73
|
|
75
74
|
return retryable
|
76
75
|
|
77
|
-
def process_sync(self) ->
|
76
|
+
def process_sync(self) -> tuple[int, int]:
|
78
77
|
"""Process the retry queue synchronously.
|
79
78
|
|
80
79
|
Returns:
|
@@ -125,7 +124,7 @@ class RetryQueue:
|
|
125
124
|
|
126
125
|
return success_count, failure_count
|
127
126
|
|
128
|
-
async def process_async(self) ->
|
127
|
+
async def process_async(self) -> tuple[int, int]:
|
129
128
|
"""Process the retry queue asynchronously.
|
130
129
|
|
131
130
|
Returns:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
import asyncio
|
2
2
|
import contextvars
|
3
|
-
from typing import Any,
|
3
|
+
from typing import Any, Literal
|
4
4
|
|
5
5
|
from pydantic import BaseModel
|
6
6
|
|
@@ -29,9 +29,9 @@ class SynthTrackerSync:
|
|
29
29
|
@classmethod
|
30
30
|
def track_lm(
|
31
31
|
cls,
|
32
|
-
messages:
|
32
|
+
messages: list[dict[str, str]],
|
33
33
|
model_name: str,
|
34
|
-
model_params:
|
34
|
+
model_params: dict[str, str | int | float] | None = None,
|
35
35
|
finetune: bool = False,
|
36
36
|
):
|
37
37
|
# print("Tracking LM call in sync context - ",messages) # Added logging
|
@@ -52,9 +52,9 @@ class SynthTrackerSync:
|
|
52
52
|
def track_state(
|
53
53
|
cls,
|
54
54
|
variable_name: str,
|
55
|
-
variable_value:
|
55
|
+
variable_value: BaseModel | str | dict | int | float | bool | list | None,
|
56
56
|
origin: Literal["agent", "environment"],
|
57
|
-
annotation:
|
57
|
+
annotation: str | None = None,
|
58
58
|
):
|
59
59
|
# Skip if value is not a trackable type instead of raising error
|
60
60
|
if not isinstance(variable_value, VALID_TYPES):
|
@@ -79,7 +79,7 @@ class SynthTrackerSync:
|
|
79
79
|
# )
|
80
80
|
|
81
81
|
@classmethod
|
82
|
-
def get_traced_data(cls) ->
|
82
|
+
def get_traced_data(cls) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
83
83
|
return getattr(cls._local, "inputs", []), getattr(cls._local, "outputs", [])
|
84
84
|
|
85
85
|
@classmethod
|
@@ -93,7 +93,7 @@ class SynthTrackerSync:
|
|
93
93
|
@classmethod
|
94
94
|
def track_lm_output(
|
95
95
|
cls,
|
96
|
-
messages:
|
96
|
+
messages: list[dict[str, str]],
|
97
97
|
model_name: str,
|
98
98
|
finetune: bool = False,
|
99
99
|
):
|
@@ -137,9 +137,9 @@ class SynthTrackerAsync:
|
|
137
137
|
@classmethod
|
138
138
|
def track_lm(
|
139
139
|
cls,
|
140
|
-
messages:
|
140
|
+
messages: list[dict[str, str]],
|
141
141
|
model_name: str,
|
142
|
-
model_params:
|
142
|
+
model_params: dict[str, str | int | float] | None = None,
|
143
143
|
finetune: bool = False,
|
144
144
|
):
|
145
145
|
# print("Tracking LM call in async context") # Added logging
|
@@ -165,9 +165,9 @@ class SynthTrackerAsync:
|
|
165
165
|
def track_state(
|
166
166
|
cls,
|
167
167
|
variable_name: str,
|
168
|
-
variable_value:
|
168
|
+
variable_value: BaseModel | str | dict | int | float | bool | list | None,
|
169
169
|
origin: Literal["agent", "environment"],
|
170
|
-
annotation:
|
170
|
+
annotation: str | None = None,
|
171
171
|
io_type: Literal["input", "output"] = "output",
|
172
172
|
):
|
173
173
|
# Skip if value is not a trackable type instead of raising error
|
@@ -207,7 +207,7 @@ class SynthTrackerAsync:
|
|
207
207
|
# )
|
208
208
|
|
209
209
|
@classmethod
|
210
|
-
def get_traced_data(cls) ->
|
210
|
+
def get_traced_data(cls) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
211
211
|
traced_inputs = trace_inputs_var.get()
|
212
212
|
traced_outputs = trace_outputs_var.get()
|
213
213
|
return traced_inputs, traced_outputs
|
@@ -222,7 +222,7 @@ class SynthTrackerAsync:
|
|
222
222
|
@classmethod
|
223
223
|
def track_lm_output(
|
224
224
|
cls,
|
225
|
-
messages:
|
225
|
+
messages: list[dict[str, str]],
|
226
226
|
model_name: str,
|
227
227
|
finetune: bool = False,
|
228
228
|
):
|
@@ -270,9 +270,9 @@ class SynthTracker:
|
|
270
270
|
@classmethod
|
271
271
|
def track_lm(
|
272
272
|
cls,
|
273
|
-
messages:
|
273
|
+
messages: list[dict[str, str]],
|
274
274
|
model_name: str,
|
275
|
-
model_params:
|
275
|
+
model_params: dict[str, str | int | float] | None = None,
|
276
276
|
finetune: bool = False,
|
277
277
|
):
|
278
278
|
# Debug logging disabled: print("DEBUG: Tracking LM call")
|
@@ -334,9 +334,9 @@ class SynthTracker:
|
|
334
334
|
def track_state(
|
335
335
|
cls,
|
336
336
|
variable_name: str,
|
337
|
-
variable_value:
|
337
|
+
variable_value: BaseModel | str | dict | int | float | bool | list | None,
|
338
338
|
origin: Literal["agent", "environment"],
|
339
|
-
annotation:
|
339
|
+
annotation: str | None = None,
|
340
340
|
):
|
341
341
|
"""
|
342
342
|
Track a state change or variable value within the current trace.
|
@@ -392,7 +392,7 @@ class SynthTracker:
|
|
392
392
|
async_sync: Literal[
|
393
393
|
"async", "sync", ""
|
394
394
|
] = "", # Force only async or sync data to be returned
|
395
|
-
) ->
|
395
|
+
) -> tuple[list, list]:
|
396
396
|
traced_inputs, traced_outputs = [], []
|
397
397
|
# Debug logging disabled: print(
|
398
398
|
# f"\nDEBUG: Getting traced data with async_sync='{async_sync}'"
|
@@ -424,7 +424,7 @@ class SynthTracker:
|
|
424
424
|
@classmethod
|
425
425
|
def track_lm_output(
|
426
426
|
cls,
|
427
|
-
messages:
|
427
|
+
messages: list[dict[str, str]],
|
428
428
|
model_name: str,
|
429
429
|
finetune: bool = False,
|
430
430
|
):
|
@@ -452,10 +452,10 @@ class SynthTracker:
|
|
452
452
|
|
453
453
|
|
454
454
|
def track_messages_sync(
|
455
|
-
input_messages:
|
456
|
-
output_messages:
|
455
|
+
input_messages: list[Message],
|
456
|
+
output_messages: list[Message],
|
457
457
|
model_name: str,
|
458
|
-
model_params:
|
458
|
+
model_params: ModelParams | None = None,
|
459
459
|
finetune: bool = False,
|
460
460
|
) -> None:
|
461
461
|
"""Track both input and output messages in a conversation synchronously.
|
@@ -484,10 +484,10 @@ def track_messages_sync(
|
|
484
484
|
|
485
485
|
|
486
486
|
async def track_messages_async(
|
487
|
-
input_messages:
|
488
|
-
output_messages:
|
487
|
+
input_messages: list[Message],
|
488
|
+
output_messages: list[Message],
|
489
489
|
model_name: str,
|
490
|
-
model_params:
|
490
|
+
model_params: ModelParams | None = None,
|
491
491
|
finetune: bool = False,
|
492
492
|
) -> None:
|
493
493
|
"""Track both input and output messages in a conversation asynchronously.
|
synth_ai/v0/tracing_v1/upload.py
CHANGED
@@ -5,7 +5,7 @@ import os
|
|
5
5
|
import ssl
|
6
6
|
import time
|
7
7
|
import warnings
|
8
|
-
from typing import Any,
|
8
|
+
from typing import Any, TypedDict
|
9
9
|
|
10
10
|
import requests
|
11
11
|
from dotenv import load_dotenv
|
@@ -35,7 +35,7 @@ def _raise_not_supported_error():
|
|
35
35
|
|
36
36
|
|
37
37
|
# NOTE: This may cause memory issues in the future
|
38
|
-
def validate_json(data:
|
38
|
+
def validate_json(data: dict[str, Any]) -> None:
|
39
39
|
"""Validate that a dictionary contains only JSON-serializable values.
|
40
40
|
|
41
41
|
Args:
|
@@ -51,7 +51,7 @@ def validate_json(data: Dict[str, Any]) -> None:
|
|
51
51
|
raise ValueError(f"Contains non-JSON-serializable values: {e}. {data}")
|
52
52
|
|
53
53
|
|
54
|
-
def createPayload(dataset: Dataset, traces:
|
54
|
+
def createPayload(dataset: Dataset, traces: list[SystemTrace]) -> dict[str, Any]:
|
55
55
|
payload = {
|
56
56
|
"traces": [trace.to_dict() for trace in traces], # Convert SystemTrace objects to dicts
|
57
57
|
"dataset": dataset.to_dict(),
|
@@ -73,7 +73,7 @@ class TLSAdapter(HTTPAdapter):
|
|
73
73
|
)
|
74
74
|
|
75
75
|
|
76
|
-
def load_signed_url(signed_url: str, dataset: Dataset, traces:
|
76
|
+
def load_signed_url(signed_url: str, dataset: Dataset, traces: list[SystemTrace]) -> None:
|
77
77
|
payload = createPayload(dataset, traces)
|
78
78
|
validate_json(payload)
|
79
79
|
|
@@ -102,7 +102,7 @@ def load_signed_url(signed_url: str, dataset: Dataset, traces: List[SystemTrace]
|
|
102
102
|
|
103
103
|
def send_system_traces_s3(
|
104
104
|
dataset: Dataset,
|
105
|
-
traces:
|
105
|
+
traces: list[SystemTrace],
|
106
106
|
base_url: str,
|
107
107
|
api_key: str,
|
108
108
|
system_id: str,
|
@@ -146,7 +146,7 @@ def send_system_traces_s3(
|
|
146
146
|
|
147
147
|
def get_upload_id(
|
148
148
|
base_url: str, api_key: str, system_id: str, system_name: str, verbose: bool = False
|
149
|
-
) ->
|
149
|
+
) -> tuple[str, str]:
|
150
150
|
"""
|
151
151
|
Modified client-side function to send both system_id and system_name.
|
152
152
|
"""
|
@@ -188,12 +188,12 @@ class UploadValidator(BaseModel):
|
|
188
188
|
extra="forbid", # Prevent additional fields
|
189
189
|
)
|
190
190
|
|
191
|
-
traces:
|
192
|
-
dataset:
|
191
|
+
traces: list[dict[str, Any]]
|
192
|
+
dataset: dict[str, Any]
|
193
193
|
|
194
194
|
@field_validator("traces")
|
195
195
|
@classmethod
|
196
|
-
def validate_traces(cls, traces:
|
196
|
+
def validate_traces(cls, traces: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
197
197
|
if not traces:
|
198
198
|
raise ValueError("Traces list cannot be empty")
|
199
199
|
|
@@ -240,7 +240,7 @@ class UploadValidator(BaseModel):
|
|
240
240
|
|
241
241
|
@field_validator("dataset")
|
242
242
|
@classmethod
|
243
|
-
def validate_dataset(cls, dataset:
|
243
|
+
def validate_dataset(cls, dataset: dict[str, Any]) -> dict[str, Any]:
|
244
244
|
required_fields = ["questions", "reward_signals"]
|
245
245
|
missing_fields = [f for f in required_fields if f not in dataset]
|
246
246
|
if missing_fields:
|
@@ -263,7 +263,7 @@ class UploadValidator(BaseModel):
|
|
263
263
|
return dataset
|
264
264
|
|
265
265
|
|
266
|
-
def validate_upload(traces:
|
266
|
+
def validate_upload(traces: list[dict[str, Any]], dataset: dict[str, Any]):
|
267
267
|
# Validate the upload format before sending to server.
|
268
268
|
# Raises ValueError if validation fails.
|
269
269
|
try:
|
@@ -283,8 +283,8 @@ def is_event_loop_running():
|
|
283
283
|
|
284
284
|
|
285
285
|
def format_upload_output(
|
286
|
-
dataset: Dataset, traces:
|
287
|
-
) ->
|
286
|
+
dataset: Dataset, traces: list[SystemTrace]
|
287
|
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
|
288
288
|
# Format questions array
|
289
289
|
questions_data = [
|
290
290
|
{"intent": q.intent, "criteria": q.criteria, "id": q.id} for q in dataset.questions
|
@@ -334,14 +334,14 @@ class ProcessUploadResponse(TypedDict):
|
|
334
334
|
|
335
335
|
def upload(
|
336
336
|
dataset: Dataset,
|
337
|
-
traces:
|
337
|
+
traces: list[SystemTrace] = [],
|
338
338
|
verbose: bool = False,
|
339
339
|
show_payload: bool = False,
|
340
|
-
) ->
|
340
|
+
) -> tuple[
|
341
341
|
ProcessUploadResponse,
|
342
|
-
|
343
|
-
|
344
|
-
|
342
|
+
list[dict[str, Any]],
|
343
|
+
list[dict[str, Any]],
|
344
|
+
list[dict[str, Any]],
|
345
345
|
]:
|
346
346
|
"""Upload all system traces and dataset to the server.
|
347
347
|
|
@@ -371,14 +371,14 @@ def upload(
|
|
371
371
|
|
372
372
|
def upload_helper(
|
373
373
|
dataset: Dataset,
|
374
|
-
traces:
|
374
|
+
traces: list[SystemTrace] = [],
|
375
375
|
verbose: bool = False,
|
376
376
|
show_payload: bool = False,
|
377
|
-
) ->
|
377
|
+
) -> tuple[
|
378
378
|
ProcessUploadResponse,
|
379
|
-
|
380
|
-
|
381
|
-
|
379
|
+
list[dict[str, Any]],
|
380
|
+
list[dict[str, Any]],
|
381
|
+
list[dict[str, Any]],
|
382
382
|
]:
|
383
383
|
"""Helper function to handle the upload process.
|
384
384
|
|
@@ -391,7 +391,7 @@ def upload_helper(
|
|
391
391
|
base_url = os.getenv("SYNTH_ENDPOINT_OVERRIDE", "https://agent-learning.onrender.com")
|
392
392
|
|
393
393
|
from .decorators import _local, active_events_var
|
394
|
-
from .trackers import synth_tracker_async
|
394
|
+
from .trackers import synth_tracker_async
|
395
395
|
|
396
396
|
# First close any tracker events
|
397
397
|
if hasattr(synth_tracker_async, "active_events"):
|
@@ -522,3 +522,4 @@ def upload_helper(
|
|
522
522
|
print("\nDataset:")
|
523
523
|
print(json.dumps(dataset_dict, indent=2))
|
524
524
|
raise
|
525
|
+
# ruff: noqa
|
synth_ai/zyk/__init__.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: synth-ai
|
3
|
-
Version: 0.2.4.
|
3
|
+
Version: 0.2.4.dev7
|
4
4
|
Summary: Software for aiding the best and multiplying the will - Core AI functionality and tracing
|
5
5
|
Author-email: Synth AI <josh@usesynth.ai>
|
6
6
|
License-Expression: MIT
|
@@ -60,20 +60,10 @@ Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
60
60
|
Requires-Dist: pyright>=1.1.350; extra == "dev"
|
61
61
|
Requires-Dist: coverage[toml]>=7.3.0; extra == "dev"
|
62
62
|
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
63
|
-
Provides-Extra: google
|
64
|
-
Requires-Dist: google-api-core>=2.0.0; extra == "google"
|
65
|
-
Requires-Dist: google-generativeai>=0.8.0; extra == "google"
|
66
|
-
Requires-Dist: google-genai>=1.0.0; extra == "google"
|
67
|
-
Provides-Extra: mistral
|
68
|
-
Requires-Dist: mistralai>=1.0.0; extra == "mistral"
|
69
63
|
Provides-Extra: research
|
70
64
|
Requires-Dist: crafter>=1.8.3; extra == "research"
|
71
65
|
Requires-Dist: datasets>=4.0.0; extra == "research"
|
72
66
|
Provides-Extra: all
|
73
|
-
Requires-Dist: google-api-core>=2.0.0; extra == "all"
|
74
|
-
Requires-Dist: google-generativeai>=0.8.0; extra == "all"
|
75
|
-
Requires-Dist: google-genai>=1.0.0; extra == "all"
|
76
|
-
Requires-Dist: mistralai>=1.0.0; extra == "all"
|
77
67
|
Requires-Dist: crafter>=1.8.3; extra == "all"
|
78
68
|
Requires-Dist: datasets>=4.0.0; extra == "all"
|
79
69
|
Dynamic: license-file
|