synth-ai 0.2.4.dev6__py3-none-any.whl → 0.2.4.dev7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synth_ai/__init__.py +18 -9
- synth_ai/cli/__init__.py +10 -5
- synth_ai/cli/balance.py +22 -17
- synth_ai/cli/calc.py +2 -3
- synth_ai/cli/demo.py +3 -5
- synth_ai/cli/legacy_root_backup.py +58 -32
- synth_ai/cli/man.py +22 -19
- synth_ai/cli/recent.py +9 -8
- synth_ai/cli/root.py +58 -13
- synth_ai/cli/status.py +13 -6
- synth_ai/cli/traces.py +45 -21
- synth_ai/cli/watch.py +40 -37
- synth_ai/config/base_url.py +1 -3
- synth_ai/core/experiment.py +1 -2
- synth_ai/environments/__init__.py +2 -6
- synth_ai/environments/environment/artifacts/base.py +3 -1
- synth_ai/environments/environment/db/sqlite.py +1 -1
- synth_ai/environments/environment/registry.py +19 -20
- synth_ai/environments/environment/resources/sqlite.py +2 -3
- synth_ai/environments/environment/rewards/core.py +3 -2
- synth_ai/environments/environment/tools/__init__.py +6 -4
- synth_ai/environments/examples/crafter_classic/__init__.py +1 -1
- synth_ai/environments/examples/crafter_classic/engine.py +13 -13
- synth_ai/environments/examples/crafter_classic/engine_deterministic_patch.py +1 -0
- synth_ai/environments/examples/crafter_classic/engine_helpers/action_map.py +2 -1
- synth_ai/environments/examples/crafter_classic/engine_helpers/serialization.py +2 -1
- synth_ai/environments/examples/crafter_classic/engine_serialization_patch_v3.py +3 -2
- synth_ai/environments/examples/crafter_classic/environment.py +16 -15
- synth_ai/environments/examples/crafter_classic/taskset.py +2 -2
- synth_ai/environments/examples/crafter_classic/trace_hooks_v3.py +2 -3
- synth_ai/environments/examples/crafter_classic/world_config_patch_simple.py +2 -1
- synth_ai/environments/examples/crafter_custom/crafter/__init__.py +2 -2
- synth_ai/environments/examples/crafter_custom/crafter/config.py +2 -2
- synth_ai/environments/examples/crafter_custom/crafter/env.py +1 -5
- synth_ai/environments/examples/crafter_custom/crafter/objects.py +1 -2
- synth_ai/environments/examples/crafter_custom/crafter/worldgen.py +1 -2
- synth_ai/environments/examples/crafter_custom/dataset_builder.py +5 -5
- synth_ai/environments/examples/crafter_custom/environment.py +13 -13
- synth_ai/environments/examples/crafter_custom/run_dataset.py +5 -5
- synth_ai/environments/examples/enron/art_helpers/email_search_tools.py +2 -2
- synth_ai/environments/examples/enron/art_helpers/local_email_db.py +5 -4
- synth_ai/environments/examples/enron/art_helpers/types_enron.py +2 -1
- synth_ai/environments/examples/enron/engine.py +18 -14
- synth_ai/environments/examples/enron/environment.py +12 -11
- synth_ai/environments/examples/enron/taskset.py +7 -7
- synth_ai/environments/examples/minigrid/__init__.py +6 -6
- synth_ai/environments/examples/minigrid/engine.py +6 -6
- synth_ai/environments/examples/minigrid/environment.py +6 -6
- synth_ai/environments/examples/minigrid/puzzle_loader.py +3 -2
- synth_ai/environments/examples/minigrid/taskset.py +13 -13
- synth_ai/environments/examples/nethack/achievements.py +1 -1
- synth_ai/environments/examples/nethack/engine.py +8 -7
- synth_ai/environments/examples/nethack/environment.py +10 -9
- synth_ai/environments/examples/nethack/helpers/__init__.py +8 -9
- synth_ai/environments/examples/nethack/helpers/action_mapping.py +1 -1
- synth_ai/environments/examples/nethack/helpers/nle_wrapper.py +2 -1
- synth_ai/environments/examples/nethack/helpers/observation_utils.py +1 -1
- synth_ai/environments/examples/nethack/helpers/recording_wrapper.py +3 -4
- synth_ai/environments/examples/nethack/helpers/trajectory_recorder.py +6 -5
- synth_ai/environments/examples/nethack/helpers/visualization/replay_viewer.py +5 -5
- synth_ai/environments/examples/nethack/helpers/visualization/visualizer.py +7 -6
- synth_ai/environments/examples/nethack/taskset.py +5 -5
- synth_ai/environments/examples/red/engine.py +9 -8
- synth_ai/environments/examples/red/engine_helpers/reward_components.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/__init__.py +7 -7
- synth_ai/environments/examples/red/engine_helpers/reward_library/adaptive_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/battle_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/composite_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/economy_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/efficiency_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/exploration_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/novelty_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/pallet_town_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/pokemon_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/social_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/reward_library/story_rewards.py +2 -1
- synth_ai/environments/examples/red/engine_helpers/screen_analysis.py +3 -2
- synth_ai/environments/examples/red/engine_helpers/state_extraction.py +2 -1
- synth_ai/environments/examples/red/environment.py +18 -15
- synth_ai/environments/examples/red/taskset.py +5 -3
- synth_ai/environments/examples/sokoban/engine.py +16 -13
- synth_ai/environments/examples/sokoban/engine_helpers/room_utils.py +3 -2
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/__init__.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/__init__.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/boxoban_env.py +7 -5
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/render_utils.py +1 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/room_utils.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env.py +5 -4
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_fixed_targets.py +3 -2
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_pull.py +2 -1
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_two_player.py +5 -4
- synth_ai/environments/examples/sokoban/engine_helpers/vendored/envs/sokoban_env_variations.py +1 -1
- synth_ai/environments/examples/sokoban/environment.py +15 -14
- synth_ai/environments/examples/sokoban/generate_verified_puzzles.py +5 -3
- synth_ai/environments/examples/sokoban/puzzle_loader.py +3 -2
- synth_ai/environments/examples/sokoban/taskset.py +13 -10
- synth_ai/environments/examples/tictactoe/engine.py +6 -6
- synth_ai/environments/examples/tictactoe/environment.py +8 -7
- synth_ai/environments/examples/tictactoe/taskset.py +6 -5
- synth_ai/environments/examples/verilog/engine.py +4 -3
- synth_ai/environments/examples/verilog/environment.py +11 -10
- synth_ai/environments/examples/verilog/taskset.py +14 -12
- synth_ai/environments/examples/wordle/__init__.py +5 -5
- synth_ai/environments/examples/wordle/engine.py +32 -25
- synth_ai/environments/examples/wordle/environment.py +21 -16
- synth_ai/environments/examples/wordle/helpers/generate_instances_wordfreq.py +6 -6
- synth_ai/environments/examples/wordle/taskset.py +20 -12
- synth_ai/environments/reproducibility/core.py +1 -1
- synth_ai/environments/reproducibility/tree.py +21 -21
- synth_ai/environments/service/app.py +3 -2
- synth_ai/environments/service/core_routes.py +104 -110
- synth_ai/environments/service/external_registry.py +1 -2
- synth_ai/environments/service/registry.py +1 -1
- synth_ai/environments/stateful/core.py +1 -2
- synth_ai/environments/stateful/engine.py +1 -1
- synth_ai/environments/tasks/api.py +4 -4
- synth_ai/environments/tasks/core.py +14 -12
- synth_ai/environments/tasks/filters.py +6 -4
- synth_ai/environments/tasks/utils.py +13 -11
- synth_ai/evals/base.py +2 -3
- synth_ai/experimental/synth_oss.py +4 -4
- synth_ai/learning/gateway.py +1 -3
- synth_ai/learning/prompts/banking77_injection_eval.py +15 -10
- synth_ai/learning/prompts/hello_world_in_context_injection_ex.py +26 -14
- synth_ai/learning/prompts/mipro.py +61 -52
- synth_ai/learning/prompts/random_search.py +42 -43
- synth_ai/learning/prompts/run_mipro_banking77.py +32 -20
- synth_ai/learning/prompts/run_random_search_banking77.py +71 -52
- synth_ai/lm/__init__.py +5 -5
- synth_ai/lm/caching/ephemeral.py +9 -9
- synth_ai/lm/caching/handler.py +20 -20
- synth_ai/lm/caching/persistent.py +10 -10
- synth_ai/lm/config.py +3 -3
- synth_ai/lm/constants.py +7 -7
- synth_ai/lm/core/all.py +17 -3
- synth_ai/lm/core/exceptions.py +0 -2
- synth_ai/lm/core/main.py +26 -41
- synth_ai/lm/core/main_v3.py +20 -10
- synth_ai/lm/core/vendor_clients.py +18 -17
- synth_ai/lm/injection.py +7 -8
- synth_ai/lm/overrides.py +21 -19
- synth_ai/lm/provider_support/__init__.py +1 -1
- synth_ai/lm/provider_support/anthropic.py +15 -15
- synth_ai/lm/provider_support/openai.py +23 -21
- synth_ai/lm/structured_outputs/handler.py +34 -32
- synth_ai/lm/structured_outputs/inject.py +24 -27
- synth_ai/lm/structured_outputs/rehabilitate.py +19 -15
- synth_ai/lm/tools/base.py +17 -16
- synth_ai/lm/unified_interface.py +17 -18
- synth_ai/lm/vendors/base.py +20 -18
- synth_ai/lm/vendors/core/anthropic_api.py +36 -27
- synth_ai/lm/vendors/core/gemini_api.py +31 -36
- synth_ai/lm/vendors/core/mistral_api.py +19 -19
- synth_ai/lm/vendors/core/openai_api.py +11 -10
- synth_ai/lm/vendors/openai_standard.py +113 -87
- synth_ai/lm/vendors/openai_standard_responses.py +74 -61
- synth_ai/lm/vendors/retries.py +9 -1
- synth_ai/lm/vendors/supported/custom_endpoint.py +26 -26
- synth_ai/lm/vendors/supported/deepseek.py +10 -10
- synth_ai/lm/vendors/supported/grok.py +8 -8
- synth_ai/lm/vendors/supported/ollama.py +2 -1
- synth_ai/lm/vendors/supported/openrouter.py +11 -9
- synth_ai/lm/vendors/synth_client.py +69 -63
- synth_ai/lm/warmup.py +8 -7
- synth_ai/tracing/__init__.py +22 -10
- synth_ai/tracing_v1/__init__.py +22 -20
- synth_ai/tracing_v3/__init__.py +7 -7
- synth_ai/tracing_v3/abstractions.py +56 -52
- synth_ai/tracing_v3/config.py +4 -2
- synth_ai/tracing_v3/db_config.py +6 -8
- synth_ai/tracing_v3/decorators.py +29 -30
- synth_ai/tracing_v3/examples/basic_usage.py +12 -12
- synth_ai/tracing_v3/hooks.py +21 -21
- synth_ai/tracing_v3/llm_call_record_helpers.py +85 -98
- synth_ai/tracing_v3/lm_call_record_abstractions.py +2 -4
- synth_ai/tracing_v3/migration_helper.py +3 -5
- synth_ai/tracing_v3/replica_sync.py +30 -32
- synth_ai/tracing_v3/session_tracer.py +35 -29
- synth_ai/tracing_v3/storage/__init__.py +1 -1
- synth_ai/tracing_v3/storage/base.py +8 -7
- synth_ai/tracing_v3/storage/config.py +4 -4
- synth_ai/tracing_v3/storage/factory.py +4 -4
- synth_ai/tracing_v3/storage/utils.py +9 -9
- synth_ai/tracing_v3/turso/__init__.py +3 -3
- synth_ai/tracing_v3/turso/daemon.py +9 -9
- synth_ai/tracing_v3/turso/manager.py +60 -48
- synth_ai/tracing_v3/turso/models.py +24 -19
- synth_ai/tracing_v3/utils.py +5 -5
- synth_ai/tui/__main__.py +1 -1
- synth_ai/tui/cli/query_experiments.py +2 -3
- synth_ai/tui/cli/query_experiments_v3.py +2 -3
- synth_ai/tui/dashboard.py +97 -86
- synth_ai/v0/tracing/abstractions.py +28 -28
- synth_ai/v0/tracing/base_client.py +9 -9
- synth_ai/v0/tracing/client_manager.py +7 -7
- synth_ai/v0/tracing/config.py +7 -7
- synth_ai/v0/tracing/context.py +6 -6
- synth_ai/v0/tracing/decorators.py +6 -5
- synth_ai/v0/tracing/events/manage.py +1 -1
- synth_ai/v0/tracing/events/store.py +5 -4
- synth_ai/v0/tracing/immediate_client.py +4 -5
- synth_ai/v0/tracing/local.py +3 -3
- synth_ai/v0/tracing/log_client_base.py +4 -5
- synth_ai/v0/tracing/retry_queue.py +5 -6
- synth_ai/v0/tracing/trackers.py +25 -25
- synth_ai/v0/tracing/upload.py +6 -0
- synth_ai/v0/tracing_v1/__init__.py +1 -1
- synth_ai/v0/tracing_v1/abstractions.py +28 -28
- synth_ai/v0/tracing_v1/base_client.py +9 -9
- synth_ai/v0/tracing_v1/client_manager.py +7 -7
- synth_ai/v0/tracing_v1/config.py +7 -7
- synth_ai/v0/tracing_v1/context.py +6 -6
- synth_ai/v0/tracing_v1/decorators.py +7 -6
- synth_ai/v0/tracing_v1/events/manage.py +1 -1
- synth_ai/v0/tracing_v1/events/store.py +5 -4
- synth_ai/v0/tracing_v1/immediate_client.py +4 -5
- synth_ai/v0/tracing_v1/local.py +3 -3
- synth_ai/v0/tracing_v1/log_client_base.py +4 -5
- synth_ai/v0/tracing_v1/retry_queue.py +5 -6
- synth_ai/v0/tracing_v1/trackers.py +25 -25
- synth_ai/v0/tracing_v1/upload.py +25 -24
- synth_ai/zyk/__init__.py +1 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/METADATA +1 -11
- synth_ai-0.2.4.dev7.dist-info/RECORD +299 -0
- synth_ai-0.2.4.dev6.dist-info/RECORD +0 -299
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/WHEEL +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.2.4.dev6.dist-info → synth_ai-0.2.4.dev7.dist-info}/top_level.txt +0 -0
synth_ai/v0/tracing/trackers.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
import asyncio
|
2
2
|
import contextvars
|
3
|
-
from typing import Any,
|
3
|
+
from typing import Any, Literal
|
4
4
|
|
5
5
|
from pydantic import BaseModel
|
6
6
|
|
@@ -29,9 +29,9 @@ class SynthTrackerSync:
|
|
29
29
|
@classmethod
|
30
30
|
def track_lm(
|
31
31
|
cls,
|
32
|
-
messages:
|
32
|
+
messages: list[dict[str, str]],
|
33
33
|
model_name: str,
|
34
|
-
model_params:
|
34
|
+
model_params: dict[str, str | int | float] | None = None,
|
35
35
|
finetune: bool = False,
|
36
36
|
):
|
37
37
|
# print("Tracking LM call in sync context - ",messages) # Added logging
|
@@ -52,9 +52,9 @@ class SynthTrackerSync:
|
|
52
52
|
def track_state(
|
53
53
|
cls,
|
54
54
|
variable_name: str,
|
55
|
-
variable_value:
|
55
|
+
variable_value: BaseModel | str | dict | int | float | bool | list | None,
|
56
56
|
origin: Literal["agent", "environment"],
|
57
|
-
annotation:
|
57
|
+
annotation: str | None = None,
|
58
58
|
):
|
59
59
|
# Skip if value is not a trackable type instead of raising error
|
60
60
|
if not isinstance(variable_value, VALID_TYPES):
|
@@ -79,7 +79,7 @@ class SynthTrackerSync:
|
|
79
79
|
# )
|
80
80
|
|
81
81
|
@classmethod
|
82
|
-
def get_traced_data(cls) ->
|
82
|
+
def get_traced_data(cls) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
83
83
|
return getattr(cls._local, "inputs", []), getattr(cls._local, "outputs", [])
|
84
84
|
|
85
85
|
@classmethod
|
@@ -93,7 +93,7 @@ class SynthTrackerSync:
|
|
93
93
|
@classmethod
|
94
94
|
def track_lm_output(
|
95
95
|
cls,
|
96
|
-
messages:
|
96
|
+
messages: list[dict[str, str]],
|
97
97
|
model_name: str,
|
98
98
|
finetune: bool = False,
|
99
99
|
):
|
@@ -137,9 +137,9 @@ class SynthTrackerAsync:
|
|
137
137
|
@classmethod
|
138
138
|
def track_lm(
|
139
139
|
cls,
|
140
|
-
messages:
|
140
|
+
messages: list[dict[str, str]],
|
141
141
|
model_name: str,
|
142
|
-
model_params:
|
142
|
+
model_params: dict[str, str | int | float] | None = None,
|
143
143
|
finetune: bool = False,
|
144
144
|
):
|
145
145
|
# print("Tracking LM call in async context") # Added logging
|
@@ -165,9 +165,9 @@ class SynthTrackerAsync:
|
|
165
165
|
def track_state(
|
166
166
|
cls,
|
167
167
|
variable_name: str,
|
168
|
-
variable_value:
|
168
|
+
variable_value: BaseModel | str | dict | int | float | bool | list | None,
|
169
169
|
origin: Literal["agent", "environment"],
|
170
|
-
annotation:
|
170
|
+
annotation: str | None = None,
|
171
171
|
io_type: Literal["input", "output"] = "output",
|
172
172
|
):
|
173
173
|
# Skip if value is not a trackable type instead of raising error
|
@@ -207,7 +207,7 @@ class SynthTrackerAsync:
|
|
207
207
|
# )
|
208
208
|
|
209
209
|
@classmethod
|
210
|
-
def get_traced_data(cls) ->
|
210
|
+
def get_traced_data(cls) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
211
211
|
traced_inputs = trace_inputs_var.get()
|
212
212
|
traced_outputs = trace_outputs_var.get()
|
213
213
|
return traced_inputs, traced_outputs
|
@@ -222,7 +222,7 @@ class SynthTrackerAsync:
|
|
222
222
|
@classmethod
|
223
223
|
def track_lm_output(
|
224
224
|
cls,
|
225
|
-
messages:
|
225
|
+
messages: list[dict[str, str]],
|
226
226
|
model_name: str,
|
227
227
|
finetune: bool = False,
|
228
228
|
):
|
@@ -270,9 +270,9 @@ class SynthTracker:
|
|
270
270
|
@classmethod
|
271
271
|
def track_lm(
|
272
272
|
cls,
|
273
|
-
messages:
|
273
|
+
messages: list[dict[str, str]],
|
274
274
|
model_name: str,
|
275
|
-
model_params:
|
275
|
+
model_params: dict[str, str | int | float] | None = None,
|
276
276
|
finetune: bool = False,
|
277
277
|
):
|
278
278
|
# Debug logging disabled: print("DEBUG: Tracking LM call")
|
@@ -334,9 +334,9 @@ class SynthTracker:
|
|
334
334
|
def track_state(
|
335
335
|
cls,
|
336
336
|
variable_name: str,
|
337
|
-
variable_value:
|
337
|
+
variable_value: BaseModel | str | dict | int | float | bool | list | None,
|
338
338
|
origin: Literal["agent", "environment"],
|
339
|
-
annotation:
|
339
|
+
annotation: str | None = None,
|
340
340
|
):
|
341
341
|
"""
|
342
342
|
Track a state change or variable value within the current trace.
|
@@ -392,7 +392,7 @@ class SynthTracker:
|
|
392
392
|
async_sync: Literal[
|
393
393
|
"async", "sync", ""
|
394
394
|
] = "", # Force only async or sync data to be returned
|
395
|
-
) ->
|
395
|
+
) -> tuple[list, list]:
|
396
396
|
traced_inputs, traced_outputs = [], []
|
397
397
|
# Debug logging disabled: print(
|
398
398
|
# f"\nDEBUG: Getting traced data with async_sync='{async_sync}'"
|
@@ -424,7 +424,7 @@ class SynthTracker:
|
|
424
424
|
@classmethod
|
425
425
|
def track_lm_output(
|
426
426
|
cls,
|
427
|
-
messages:
|
427
|
+
messages: list[dict[str, str]],
|
428
428
|
model_name: str,
|
429
429
|
finetune: bool = False,
|
430
430
|
):
|
@@ -452,10 +452,10 @@ class SynthTracker:
|
|
452
452
|
|
453
453
|
|
454
454
|
def track_messages_sync(
|
455
|
-
input_messages:
|
456
|
-
output_messages:
|
455
|
+
input_messages: list[Message],
|
456
|
+
output_messages: list[Message],
|
457
457
|
model_name: str,
|
458
|
-
model_params:
|
458
|
+
model_params: ModelParams | None = None,
|
459
459
|
finetune: bool = False,
|
460
460
|
) -> None:
|
461
461
|
"""Track both input and output messages in a conversation synchronously.
|
@@ -484,10 +484,10 @@ def track_messages_sync(
|
|
484
484
|
|
485
485
|
|
486
486
|
async def track_messages_async(
|
487
|
-
input_messages:
|
488
|
-
output_messages:
|
487
|
+
input_messages: list[Message],
|
488
|
+
output_messages: list[Message],
|
489
489
|
model_name: str,
|
490
|
-
model_params:
|
490
|
+
model_params: ModelParams | None = None,
|
491
491
|
finetune: bool = False,
|
492
492
|
) -> None:
|
493
493
|
"""Track both input and output messages in a conversation asynchronously.
|
synth_ai/v0/tracing/upload.py
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# ruff: noqa
|
1
2
|
import asyncio
|
2
3
|
import json
|
3
4
|
import logging
|
@@ -370,6 +371,10 @@ def upload_helper(
|
|
370
371
|
raise ValueError("SYNTH_API_KEY environment variable not set")
|
371
372
|
base_url = os.getenv("SYNTH_ENDPOINT_OVERRIDE", "https://agent-learning.onrender.com")
|
372
373
|
|
374
|
+
"""Legacy block below retained for reference and disabled for linting/parsing.
|
375
|
+
Start disabled block.
|
376
|
+
"""
|
377
|
+
"""
|
373
378
|
from .decorators import _local, active_events_var
|
374
379
|
from .trackers import synth_tracker_async, synth_tracker_sync
|
375
380
|
|
@@ -502,3 +507,4 @@ from .trackers import synth_tracker_async, synth_tracker_sync
|
|
502
507
|
print("\nDataset:")
|
503
508
|
print(json.dumps(dataset_dict, indent=2))
|
504
509
|
raise
|
510
|
+
"""
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import logging
|
2
2
|
from dataclasses import dataclass
|
3
3
|
from datetime import datetime
|
4
|
-
from typing import Any
|
4
|
+
from typing import Any
|
5
5
|
|
6
6
|
from pydantic import BaseModel
|
7
7
|
|
@@ -10,22 +10,22 @@ logger = logging.getLogger(__name__)
|
|
10
10
|
|
11
11
|
@dataclass
|
12
12
|
class MessageInputs:
|
13
|
-
messages:
|
13
|
+
messages: list[dict[str, str]] # {"role": "", "content": ""}
|
14
14
|
|
15
15
|
|
16
16
|
@dataclass
|
17
17
|
class ArbitraryInputs:
|
18
|
-
inputs:
|
18
|
+
inputs: dict[str, Any]
|
19
19
|
|
20
20
|
|
21
21
|
@dataclass
|
22
22
|
class MessageOutputs:
|
23
|
-
messages:
|
23
|
+
messages: list[dict[str, str]]
|
24
24
|
|
25
25
|
|
26
26
|
@dataclass
|
27
27
|
class ArbitraryOutputs:
|
28
|
-
outputs:
|
28
|
+
outputs: dict[str, Any]
|
29
29
|
|
30
30
|
|
31
31
|
@dataclass
|
@@ -33,8 +33,8 @@ class ComputeStep:
|
|
33
33
|
event_order: int
|
34
34
|
compute_ended: datetime # time step
|
35
35
|
compute_began: datetime # time step
|
36
|
-
compute_input:
|
37
|
-
compute_output:
|
36
|
+
compute_input: dict[str, Any] # {variable_name: value}
|
37
|
+
compute_output: dict[str, Any] # {variable_name: value}
|
38
38
|
|
39
39
|
def to_dict(self):
|
40
40
|
# Serialize compute_input
|
@@ -74,11 +74,11 @@ class ComputeStep:
|
|
74
74
|
|
75
75
|
@dataclass
|
76
76
|
class AgentComputeStep(ComputeStep):
|
77
|
-
model_name:
|
78
|
-
model_params:
|
79
|
-
should_learn:
|
80
|
-
compute_input:
|
81
|
-
compute_output:
|
77
|
+
model_name: str | None = None
|
78
|
+
model_params: dict[str, Any] | None = None
|
79
|
+
should_learn: bool | None = None
|
80
|
+
compute_input: list[MessageInputs | ArbitraryInputs]
|
81
|
+
compute_output: list[MessageOutputs | ArbitraryOutputs]
|
82
82
|
|
83
83
|
def to_dict(self):
|
84
84
|
base_dict = super().to_dict() # Get the parent class serialization
|
@@ -88,8 +88,8 @@ class AgentComputeStep(ComputeStep):
|
|
88
88
|
|
89
89
|
@dataclass
|
90
90
|
class EnvironmentComputeStep(ComputeStep):
|
91
|
-
compute_input:
|
92
|
-
compute_output:
|
91
|
+
compute_input: list[ArbitraryInputs]
|
92
|
+
compute_output: list[ArbitraryOutputs]
|
93
93
|
|
94
94
|
|
95
95
|
@dataclass
|
@@ -97,13 +97,13 @@ class Event:
|
|
97
97
|
system_instance_id: str
|
98
98
|
event_type: str
|
99
99
|
opened: float
|
100
|
-
closed:
|
100
|
+
closed: float | None
|
101
101
|
partition_index: int
|
102
102
|
agent_compute_step: AgentComputeStep
|
103
|
-
environment_compute_steps:
|
104
|
-
system_name:
|
105
|
-
system_id:
|
106
|
-
event_metadata:
|
103
|
+
environment_compute_steps: list["EnvironmentComputeStep"]
|
104
|
+
system_name: str | None = None
|
105
|
+
system_id: str | None = None
|
106
|
+
event_metadata: dict[str, Any] = None # JSON-serializable metadata for this specific event
|
107
107
|
|
108
108
|
def __post_init__(self):
|
109
109
|
if self.event_metadata is None:
|
@@ -126,7 +126,7 @@ class Event:
|
|
126
126
|
|
127
127
|
# backwards compatibility
|
128
128
|
@property
|
129
|
-
def agent_compute_steps(self) ->
|
129
|
+
def agent_compute_steps(self) -> list[AgentComputeStep]:
|
130
130
|
"""Backwards compatibility method that returns a list containing the agent_compute_step."""
|
131
131
|
return [self.agent_compute_step] if self.agent_compute_step is not None else []
|
132
132
|
|
@@ -134,7 +134,7 @@ class Event:
|
|
134
134
|
@dataclass
|
135
135
|
class EventPartitionElement:
|
136
136
|
partition_index: int
|
137
|
-
events:
|
137
|
+
events: list[Event]
|
138
138
|
|
139
139
|
def to_dict(self):
|
140
140
|
return {
|
@@ -148,9 +148,9 @@ class SystemTrace:
|
|
148
148
|
system_name: str
|
149
149
|
system_id: str
|
150
150
|
system_instance_id: str
|
151
|
-
partition:
|
152
|
-
metadata:
|
153
|
-
instance_metadata:
|
151
|
+
partition: list[EventPartitionElement]
|
152
|
+
metadata: dict[str, Any] | None = None # System-level metadata
|
153
|
+
instance_metadata: dict[str, Any] = (
|
154
154
|
None # JSON-serializable metadata for this specific instance
|
155
155
|
)
|
156
156
|
current_partition_index: int = 0 # Track current partition
|
@@ -196,8 +196,8 @@ class RewardSignal(BaseModel):
|
|
196
196
|
|
197
197
|
question_id: str
|
198
198
|
system_instance_id: str
|
199
|
-
reward:
|
200
|
-
annotation:
|
199
|
+
reward: float | int | bool
|
200
|
+
annotation: str | None = None
|
201
201
|
|
202
202
|
def to_dict(self):
|
203
203
|
return {
|
@@ -214,8 +214,8 @@ class Dataset(BaseModel):
|
|
214
214
|
This better represents the data that is used to train a model, and gives us more information about the data.
|
215
215
|
"""
|
216
216
|
|
217
|
-
questions:
|
218
|
-
reward_signals:
|
217
|
+
questions: list[TrainingQuestion]
|
218
|
+
reward_signals: list[RewardSignal]
|
219
219
|
|
220
220
|
def to_dict(self):
|
221
221
|
return {
|
@@ -2,7 +2,7 @@ import logging
|
|
2
2
|
import time
|
3
3
|
from abc import ABC, abstractmethod
|
4
4
|
from dataclasses import dataclass
|
5
|
-
from typing import Any
|
5
|
+
from typing import Any
|
6
6
|
|
7
7
|
from .abstractions import Event
|
8
8
|
from .config import TracingConfig
|
@@ -16,9 +16,9 @@ class LogResponse:
|
|
16
16
|
"""Represents the response from a logging attempt"""
|
17
17
|
|
18
18
|
success: bool
|
19
|
-
error:
|
20
|
-
retry_after:
|
21
|
-
status_code:
|
19
|
+
error: str | None = None
|
20
|
+
retry_after: float | None = None
|
21
|
+
status_code: int | None = None
|
22
22
|
|
23
23
|
|
24
24
|
class BaseLogClient(ABC):
|
@@ -31,7 +31,7 @@ class BaseLogClient(ABC):
|
|
31
31
|
self._circuit_open = False
|
32
32
|
self._circuit_open_time = 0
|
33
33
|
|
34
|
-
def _should_retry(self, attempt: int, status_code:
|
34
|
+
def _should_retry(self, attempt: int, status_code: int | None = None) -> bool:
|
35
35
|
"""Determine if a retry should be attempted based on configuration and status"""
|
36
36
|
if attempt >= self.config.max_retries:
|
37
37
|
return False
|
@@ -48,7 +48,7 @@ class BaseLogClient(ABC):
|
|
48
48
|
|
49
49
|
return True
|
50
50
|
|
51
|
-
def _prepare_payload(self, event: Event, system_info:
|
51
|
+
def _prepare_payload(self, event: Event, system_info: dict[str, str]) -> dict[str, Any]:
|
52
52
|
"""Prepare the payload for sending"""
|
53
53
|
return {
|
54
54
|
"event": event.to_dict(),
|
@@ -57,7 +57,7 @@ class BaseLogClient(ABC):
|
|
57
57
|
"sdk_version": self.config.sdk_version, # Use SDK version from config
|
58
58
|
}
|
59
59
|
|
60
|
-
def _handle_failure(self, event: Event, system_info:
|
60
|
+
def _handle_failure(self, event: Event, system_info: dict[str, str], error: Exception) -> None:
|
61
61
|
"""Handle logging failure by storing in event_store"""
|
62
62
|
logger.error(f"Logging failed: {str(error)}")
|
63
63
|
self._consecutive_failures += 1
|
@@ -77,7 +77,7 @@ class BaseLogClient(ABC):
|
|
77
77
|
self._last_failure_time = 0
|
78
78
|
|
79
79
|
@abstractmethod
|
80
|
-
def send_event(self, event: Event, system_info:
|
80
|
+
def send_event(self, event: Event, system_info: dict[str, str]) -> bool:
|
81
81
|
"""Send a single event with retries and fallback"""
|
82
82
|
pass
|
83
83
|
|
@@ -86,6 +86,6 @@ class BaseAsyncLogClient(BaseLogClient):
|
|
86
86
|
"""Abstract base class for async logging clients"""
|
87
87
|
|
88
88
|
@abstractmethod
|
89
|
-
async def send_event(self, event: Event, system_info:
|
89
|
+
async def send_event(self, event: Event, system_info: dict[str, str]) -> bool:
|
90
90
|
"""Send a single event with retries and fallback (async version)"""
|
91
91
|
pass
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
2
2
|
|
3
3
|
import asyncio
|
4
4
|
import random
|
5
|
-
from typing import ClassVar
|
5
|
+
from typing import ClassVar
|
6
6
|
|
7
7
|
import httpx
|
8
8
|
|
@@ -12,14 +12,14 @@ from .config import TracingConfig
|
|
12
12
|
class ClientManager:
|
13
13
|
"""Singleton manager for HTTP clients with both sync and async support"""
|
14
14
|
|
15
|
-
_instance: ClassVar[
|
15
|
+
_instance: ClassVar[ClientManager | None] = None
|
16
16
|
_lock = asyncio.Lock()
|
17
17
|
|
18
18
|
def __init__(self):
|
19
|
-
self._config:
|
20
|
-
self._sync_client:
|
21
|
-
self._async_client:
|
22
|
-
self._credentials_cache:
|
19
|
+
self._config: TracingConfig | None = None
|
20
|
+
self._sync_client: httpx.Client | None = None
|
21
|
+
self._async_client: httpx.AsyncClient | None = None
|
22
|
+
self._credentials_cache: dict[str, str] = {}
|
23
23
|
|
24
24
|
@classmethod
|
25
25
|
async def get_instance(cls) -> ClientManager:
|
@@ -121,7 +121,7 @@ class ClientManager:
|
|
121
121
|
asyncio.create_task(self.aclose())
|
122
122
|
|
123
123
|
@property
|
124
|
-
def config(self) ->
|
124
|
+
def config(self) -> TracingConfig | None:
|
125
125
|
"""Get the current configuration"""
|
126
126
|
return self._config
|
127
127
|
|
synth_ai/v0/tracing_v1/config.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
import json
|
2
|
+
from collections.abc import Sequence
|
2
3
|
from enum import Enum
|
3
|
-
from typing import Any,
|
4
|
+
from typing import Any, NotRequired, TypedDict
|
4
5
|
|
5
6
|
from opentelemetry import trace
|
6
7
|
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
|
@@ -10,12 +11,11 @@ from opentelemetry.sdk.trace.export import (
|
|
10
11
|
SpanExportResult,
|
11
12
|
)
|
12
13
|
from pydantic import BaseModel, ConfigDict, Field
|
13
|
-
from typing_extensions import NotRequired
|
14
14
|
|
15
15
|
|
16
16
|
class InMemoryExporter(SpanExporter):
|
17
17
|
def __init__(self):
|
18
|
-
self.spans:
|
18
|
+
self.spans: list[dict[str, Any]] = []
|
19
19
|
|
20
20
|
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
21
21
|
for span in spans:
|
@@ -45,7 +45,7 @@ class InMemoryExporter(SpanExporter):
|
|
45
45
|
def shutdown(self):
|
46
46
|
pass
|
47
47
|
|
48
|
-
def get_spans(self) ->
|
48
|
+
def get_spans(self) -> list[dict[str, Any]]:
|
49
49
|
return self.spans
|
50
50
|
|
51
51
|
def clear(self):
|
@@ -125,7 +125,7 @@ class Message(TypedDict):
|
|
125
125
|
role: str
|
126
126
|
content: str
|
127
127
|
name: NotRequired[str]
|
128
|
-
function_call: NotRequired[
|
128
|
+
function_call: NotRequired[dict[str, str]]
|
129
129
|
|
130
130
|
|
131
131
|
class ModelParams(TypedDict, total=False):
|
@@ -136,5 +136,5 @@ class ModelParams(TypedDict, total=False):
|
|
136
136
|
top_p: float
|
137
137
|
frequency_penalty: float
|
138
138
|
presence_penalty: float
|
139
|
-
stop:
|
140
|
-
functions:
|
139
|
+
stop: str | list[str]
|
140
|
+
functions: list[dict[str, Any]]
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import asyncio
|
2
2
|
import logging
|
3
3
|
from contextlib import contextmanager
|
4
|
-
from typing import Any,
|
4
|
+
from typing import Any, Generic, TypeVar
|
5
5
|
|
6
6
|
from .local import (
|
7
7
|
_local,
|
@@ -24,7 +24,7 @@ class ContextState(Generic[T]):
|
|
24
24
|
self.thread_local_attr = thread_local_attr
|
25
25
|
self.context_var = context_var
|
26
26
|
|
27
|
-
def get(self) ->
|
27
|
+
def get(self) -> T | None:
|
28
28
|
"""Get value from appropriate context."""
|
29
29
|
try:
|
30
30
|
asyncio.get_running_loop()
|
@@ -32,7 +32,7 @@ class ContextState(Generic[T]):
|
|
32
32
|
except RuntimeError:
|
33
33
|
return getattr(_local, self.thread_local_attr, None)
|
34
34
|
|
35
|
-
def set(self, value: T) ->
|
35
|
+
def set(self, value: T) -> T | None:
|
36
36
|
"""Set value in appropriate context."""
|
37
37
|
try:
|
38
38
|
asyncio.get_running_loop()
|
@@ -41,7 +41,7 @@ class ContextState(Generic[T]):
|
|
41
41
|
setattr(_local, self.thread_local_attr, value)
|
42
42
|
return None
|
43
43
|
|
44
|
-
def reset(self, token:
|
44
|
+
def reset(self, token: T | None = None) -> None:
|
45
45
|
"""Reset/clear value from appropriate context."""
|
46
46
|
try:
|
47
47
|
asyncio.get_running_loop()
|
@@ -69,7 +69,7 @@ def trace_context(
|
|
69
69
|
system_name: str,
|
70
70
|
system_id: str,
|
71
71
|
system_instance_id: str,
|
72
|
-
system_instance_metadata:
|
72
|
+
system_instance_metadata: dict[str, Any] | None = None,
|
73
73
|
):
|
74
74
|
"""Context manager for setting up tracing context.
|
75
75
|
|
@@ -132,7 +132,7 @@ def trace_context(
|
|
132
132
|
active_events_state.reset()
|
133
133
|
|
134
134
|
|
135
|
-
def get_current_context() ->
|
135
|
+
def get_current_context() -> dict[str, Any]:
|
136
136
|
"""Get the current tracing context.
|
137
137
|
|
138
138
|
Returns:
|
@@ -1,11 +1,12 @@
|
|
1
1
|
# synth_sdk/tracing/decorators.py
|
2
2
|
import inspect
|
3
3
|
import logging
|
4
|
-
import warnings
|
5
4
|
import os
|
6
5
|
import time
|
6
|
+
import warnings
|
7
|
+
from collections.abc import Callable
|
7
8
|
from functools import wraps
|
8
|
-
from typing import TYPE_CHECKING, Any,
|
9
|
+
from typing import TYPE_CHECKING, Any, Literal, ParamSpec, TypeVar, Union
|
9
10
|
|
10
11
|
if TYPE_CHECKING:
|
11
12
|
from .trackers import SynthTrackerAsync, SynthTrackerSync
|
@@ -218,8 +219,8 @@ def trace_event_sync(
|
|
218
219
|
# Collect traced inputs and outputs
|
219
220
|
traced_inputs, traced_outputs = synth_tracker_sync.get_traced_data()
|
220
221
|
|
221
|
-
compute_steps_by_origin:
|
222
|
-
Literal["agent", "environment"],
|
222
|
+
compute_steps_by_origin: dict[
|
223
|
+
Literal["agent", "environment"], dict[str, list[Any]]
|
223
224
|
] = {
|
224
225
|
"agent": {"inputs": [], "outputs": []},
|
225
226
|
"environment": {"inputs": [], "outputs": []},
|
@@ -485,8 +486,8 @@ def trace_event_async(
|
|
485
486
|
# Collect traced inputs and outputs
|
486
487
|
traced_inputs, traced_outputs = synth_tracker_async.get_traced_data()
|
487
488
|
|
488
|
-
compute_steps_by_origin:
|
489
|
-
Literal["agent", "environment"],
|
489
|
+
compute_steps_by_origin: dict[
|
490
|
+
Literal["agent", "environment"], dict[str, list[Any]]
|
490
491
|
] = {
|
491
492
|
"agent": {"inputs": [], "outputs": []},
|
492
493
|
"environment": {"inputs": [], "outputs": []},
|
@@ -130,7 +130,7 @@ def clear_current_event(event_type: str):
|
|
130
130
|
logger.debug(f"Cleared current event of type {event_type}")
|
131
131
|
|
132
132
|
|
133
|
-
def end_event(event_type: str) ->
|
133
|
+
def end_event(event_type: str) -> Event | None:
|
134
134
|
"""End the current event and store it."""
|
135
135
|
current_event = get_current_event(event_type)
|
136
136
|
if current_event:
|
@@ -2,7 +2,7 @@ import json
|
|
2
2
|
import logging
|
3
3
|
import time
|
4
4
|
from threading import RLock # Change this import
|
5
|
-
from typing import Any
|
5
|
+
from typing import Any
|
6
6
|
|
7
7
|
from ..abstractions import Event, EventPartitionElement, SystemTrace
|
8
8
|
from ..local import ( # Import context variables
|
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
|
|
15
15
|
|
16
16
|
class EventStore:
|
17
17
|
def __init__(self) -> None:
|
18
|
-
self._traces:
|
18
|
+
self._traces: dict[str, SystemTrace] = {}
|
19
19
|
self._lock = RLock() # Use RLock instead of Lock
|
20
20
|
self.logger = logging.getLogger(__name__)
|
21
21
|
|
@@ -136,7 +136,7 @@ class EventStore:
|
|
136
136
|
# self.logger.error(f"Error in add_event: {str(e)}", exc_info=True)
|
137
137
|
# raise
|
138
138
|
|
139
|
-
def get_system_traces(self) ->
|
139
|
+
def get_system_traces(self) -> list[SystemTrace]:
|
140
140
|
"""Get all system traces."""
|
141
141
|
with self._lock:
|
142
142
|
self.end_all_active_events()
|
@@ -196,7 +196,7 @@ class EventStore:
|
|
196
196
|
default=str,
|
197
197
|
)
|
198
198
|
|
199
|
-
def _event_to_dict(self, event: Event) ->
|
199
|
+
def _event_to_dict(self, event: Event) -> dict[str, Any]:
|
200
200
|
"""Convert an Event object to a dictionary."""
|
201
201
|
return {
|
202
202
|
"event_type": event.event_type,
|
@@ -225,3 +225,4 @@ class EventStore:
|
|
225
225
|
|
226
226
|
# Global event store instance
|
227
227
|
event_store = EventStore()
|
228
|
+
# ruff: noqa
|
@@ -1,7 +1,6 @@
|
|
1
1
|
import asyncio
|
2
2
|
import logging
|
3
3
|
import time
|
4
|
-
from typing import Dict
|
5
4
|
|
6
5
|
import httpx
|
7
6
|
|
@@ -20,7 +19,7 @@ class ImmediateLogClient(BaseLogClient):
|
|
20
19
|
super().__init__(config)
|
21
20
|
self.client_manager = ClientManager.initialize(config)
|
22
21
|
|
23
|
-
def send_event(self, event: Event, system_info:
|
22
|
+
def send_event(self, event: Event, system_info: dict[str, str]) -> bool:
|
24
23
|
"""Send a single event with retries and fallback"""
|
25
24
|
from .retry_queue import (
|
26
25
|
retry_queue, # Import here to avoid circular import
|
@@ -71,7 +70,7 @@ class AsyncImmediateLogClient(BaseAsyncLogClient):
|
|
71
70
|
super().__init__(config)
|
72
71
|
self.client_manager = ClientManager.initialize(config)
|
73
72
|
|
74
|
-
async def send_event(self, event: Event, system_info:
|
73
|
+
async def send_event(self, event: Event, system_info: dict[str, str]) -> bool:
|
75
74
|
"""Send a single event with retries and fallback (async version)"""
|
76
75
|
from .retry_queue import retry_queue
|
77
76
|
|
@@ -96,7 +95,7 @@ class AsyncImmediateLogClient(BaseAsyncLogClient):
|
|
96
95
|
f"No access token received from auth endpoint. Response data: {auth_data}"
|
97
96
|
)
|
98
97
|
return False
|
99
|
-
except Exception
|
98
|
+
except Exception:
|
100
99
|
# logger.error(f"Failed to get auth token: {e}")
|
101
100
|
return False
|
102
101
|
|
@@ -138,7 +137,7 @@ class AsyncImmediateLogClient(BaseAsyncLogClient):
|
|
138
137
|
event.id = response_data.get("event_id")
|
139
138
|
return True
|
140
139
|
|
141
|
-
except Exception
|
140
|
+
except Exception:
|
142
141
|
# last_exception = e
|
143
142
|
# logger.error(f"Upload attempt {attempt + 1} failed: {str(e)}")
|
144
143
|
if attempt < self.config.max_retries:
|