synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synth-ai might be problematic. Click here for more details.
- synth_ai/__init__.py +13 -13
- synth_ai/cli/__init__.py +6 -15
- synth_ai/cli/commands/eval/__init__.py +6 -15
- synth_ai/cli/commands/eval/config.py +338 -0
- synth_ai/cli/commands/eval/core.py +236 -1091
- synth_ai/cli/commands/eval/runner.py +704 -0
- synth_ai/cli/commands/eval/validation.py +44 -117
- synth_ai/cli/commands/filter/core.py +7 -7
- synth_ai/cli/commands/filter/validation.py +2 -2
- synth_ai/cli/commands/smoke/core.py +7 -17
- synth_ai/cli/commands/status/__init__.py +1 -64
- synth_ai/cli/commands/status/client.py +50 -151
- synth_ai/cli/commands/status/config.py +3 -83
- synth_ai/cli/commands/status/errors.py +4 -13
- synth_ai/cli/commands/status/subcommands/__init__.py +2 -8
- synth_ai/cli/commands/status/subcommands/config.py +13 -0
- synth_ai/cli/commands/status/subcommands/files.py +18 -63
- synth_ai/cli/commands/status/subcommands/jobs.py +28 -311
- synth_ai/cli/commands/status/subcommands/models.py +18 -62
- synth_ai/cli/commands/status/subcommands/runs.py +16 -63
- synth_ai/cli/commands/status/subcommands/session.py +67 -172
- synth_ai/cli/commands/status/subcommands/summary.py +24 -32
- synth_ai/cli/commands/status/subcommands/utils.py +41 -0
- synth_ai/cli/commands/status/utils.py +16 -107
- synth_ai/cli/commands/train/__init__.py +18 -20
- synth_ai/cli/commands/train/errors.py +3 -3
- synth_ai/cli/commands/train/prompt_learning_validation.py +15 -16
- synth_ai/cli/commands/train/validation.py +7 -7
- synth_ai/cli/commands/train/{judge_schemas.py → verifier_schemas.py} +33 -34
- synth_ai/cli/commands/train/verifier_validation.py +235 -0
- synth_ai/cli/demo_apps/demo_task_apps/math/config.toml +0 -1
- synth_ai/cli/demo_apps/demo_task_apps/math/modal_task_app.py +2 -6
- synth_ai/cli/demo_apps/math/config.toml +0 -1
- synth_ai/cli/demo_apps/math/modal_task_app.py +2 -6
- synth_ai/cli/demo_apps/mipro/task_app.py +25 -47
- synth_ai/cli/lib/apps/task_app.py +12 -13
- synth_ai/cli/lib/task_app_discovery.py +6 -6
- synth_ai/cli/lib/train_cfgs.py +10 -10
- synth_ai/cli/task_apps/__init__.py +11 -0
- synth_ai/cli/task_apps/commands.py +7 -15
- synth_ai/core/env.py +12 -1
- synth_ai/core/errors.py +1 -2
- synth_ai/core/integrations/cloudflare.py +209 -33
- synth_ai/core/tracing_v3/abstractions.py +46 -0
- synth_ai/data/__init__.py +3 -30
- synth_ai/data/enums.py +1 -20
- synth_ai/data/rewards.py +100 -3
- synth_ai/products/graph_evolve/__init__.py +1 -2
- synth_ai/products/graph_evolve/config.py +16 -16
- synth_ai/products/graph_evolve/converters/__init__.py +3 -3
- synth_ai/products/graph_evolve/converters/openai_sft.py +7 -7
- synth_ai/products/graph_evolve/examples/hotpotqa/config.toml +1 -1
- synth_ai/products/graph_gepa/__init__.py +23 -0
- synth_ai/products/graph_gepa/converters/__init__.py +19 -0
- synth_ai/products/graph_gepa/converters/openai_sft.py +29 -0
- synth_ai/sdk/__init__.py +45 -35
- synth_ai/sdk/api/eval/__init__.py +33 -0
- synth_ai/sdk/api/eval/job.py +732 -0
- synth_ai/sdk/api/research_agent/__init__.py +276 -66
- synth_ai/sdk/api/train/builders.py +181 -0
- synth_ai/sdk/api/train/cli.py +41 -33
- synth_ai/sdk/api/train/configs/__init__.py +6 -4
- synth_ai/sdk/api/train/configs/prompt_learning.py +127 -33
- synth_ai/sdk/api/train/configs/rl.py +264 -16
- synth_ai/sdk/api/train/configs/sft.py +165 -1
- synth_ai/sdk/api/train/graph_validators.py +12 -12
- synth_ai/sdk/api/train/graphgen.py +169 -51
- synth_ai/sdk/api/train/graphgen_models.py +95 -45
- synth_ai/sdk/api/train/local_api.py +10 -0
- synth_ai/sdk/api/train/pollers.py +36 -0
- synth_ai/sdk/api/train/prompt_learning.py +390 -60
- synth_ai/sdk/api/train/rl.py +41 -5
- synth_ai/sdk/api/train/sft.py +2 -0
- synth_ai/sdk/api/train/task_app.py +20 -0
- synth_ai/sdk/api/train/validators.py +17 -17
- synth_ai/sdk/graphs/completions.py +239 -33
- synth_ai/sdk/{judging/schemas.py → graphs/verifier_schemas.py} +23 -23
- synth_ai/sdk/learning/__init__.py +35 -5
- synth_ai/sdk/learning/context_learning_client.py +531 -0
- synth_ai/sdk/learning/context_learning_types.py +294 -0
- synth_ai/sdk/learning/prompt_learning_client.py +1 -1
- synth_ai/sdk/learning/prompt_learning_types.py +2 -1
- synth_ai/sdk/learning/rl/__init__.py +0 -4
- synth_ai/sdk/learning/rl/contracts.py +0 -4
- synth_ai/sdk/localapi/__init__.py +40 -0
- synth_ai/sdk/localapi/apps/__init__.py +28 -0
- synth_ai/sdk/localapi/client.py +10 -0
- synth_ai/sdk/localapi/contracts.py +10 -0
- synth_ai/sdk/localapi/helpers.py +519 -0
- synth_ai/sdk/localapi/rollouts.py +93 -0
- synth_ai/sdk/localapi/server.py +29 -0
- synth_ai/sdk/localapi/template.py +49 -0
- synth_ai/sdk/streaming/handlers.py +6 -6
- synth_ai/sdk/streaming/streamer.py +10 -6
- synth_ai/sdk/task/__init__.py +18 -5
- synth_ai/sdk/task/apps/__init__.py +37 -1
- synth_ai/sdk/task/client.py +9 -1
- synth_ai/sdk/task/config.py +6 -11
- synth_ai/sdk/task/contracts.py +137 -95
- synth_ai/sdk/task/in_process.py +32 -22
- synth_ai/sdk/task/in_process_runner.py +9 -4
- synth_ai/sdk/task/rubrics/__init__.py +2 -3
- synth_ai/sdk/task/rubrics/loaders.py +4 -4
- synth_ai/sdk/task/rubrics/strict.py +3 -4
- synth_ai/sdk/task/server.py +76 -16
- synth_ai/sdk/task/trace_correlation_helpers.py +190 -139
- synth_ai/sdk/task/validators.py +34 -49
- synth_ai/sdk/training/__init__.py +7 -16
- synth_ai/sdk/tunnels/__init__.py +118 -0
- synth_ai/sdk/tunnels/cleanup.py +83 -0
- synth_ai/sdk/tunnels/ports.py +120 -0
- synth_ai/sdk/tunnels/tunneled_api.py +363 -0
- {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/METADATA +71 -4
- {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/RECORD +118 -128
- synth_ai/cli/commands/baseline/__init__.py +0 -12
- synth_ai/cli/commands/baseline/core.py +0 -636
- synth_ai/cli/commands/baseline/list.py +0 -94
- synth_ai/cli/commands/eval/errors.py +0 -81
- synth_ai/cli/commands/status/formatters.py +0 -164
- synth_ai/cli/commands/status/subcommands/pricing.py +0 -23
- synth_ai/cli/commands/status/subcommands/usage.py +0 -203
- synth_ai/cli/commands/train/judge_validation.py +0 -305
- synth_ai/cli/usage.py +0 -159
- synth_ai/data/specs.py +0 -36
- synth_ai/sdk/api/research_agent/cli.py +0 -428
- synth_ai/sdk/api/research_agent/config.py +0 -357
- synth_ai/sdk/api/research_agent/job.py +0 -717
- synth_ai/sdk/baseline/__init__.py +0 -25
- synth_ai/sdk/baseline/config.py +0 -209
- synth_ai/sdk/baseline/discovery.py +0 -216
- synth_ai/sdk/baseline/execution.py +0 -154
- synth_ai/sdk/judging/__init__.py +0 -15
- synth_ai/sdk/judging/base.py +0 -24
- synth_ai/sdk/judging/client.py +0 -191
- synth_ai/sdk/judging/types.py +0 -42
- synth_ai/sdk/research_agent/__init__.py +0 -34
- synth_ai/sdk/research_agent/container_builder.py +0 -328
- synth_ai/sdk/research_agent/container_spec.py +0 -198
- synth_ai/sdk/research_agent/defaults.py +0 -34
- synth_ai/sdk/research_agent/results_collector.py +0 -69
- synth_ai/sdk/specs/__init__.py +0 -46
- synth_ai/sdk/specs/dataclasses.py +0 -149
- synth_ai/sdk/specs/loader.py +0 -144
- synth_ai/sdk/specs/serializer.py +0 -199
- synth_ai/sdk/specs/validation.py +0 -250
- synth_ai/sdk/tracing/__init__.py +0 -39
- synth_ai/sdk/usage/__init__.py +0 -37
- synth_ai/sdk/usage/client.py +0 -171
- synth_ai/sdk/usage/models.py +0 -261
- {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/WHEEL +0 -0
- {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/entry_points.txt +0 -0
- {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/licenses/LICENSE +0 -0
- {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/top_level.txt +0 -0
synth_ai/sdk/usage/models.py
DELETED
|
@@ -1,261 +0,0 @@
|
|
|
1
|
-
"""Data models for usage tracking.
|
|
2
|
-
|
|
3
|
-
These dataclasses represent the usage and limits data returned by the
|
|
4
|
-
GET /api/v1/usage endpoint.
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
from dataclasses import dataclass, field
|
|
10
|
-
from datetime import datetime
|
|
11
|
-
from typing import Any
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
@dataclass
|
|
15
|
-
class UsageMetric:
|
|
16
|
-
"""A single usage metric with its limit and remaining capacity.
|
|
17
|
-
|
|
18
|
-
Attributes:
|
|
19
|
-
used: Current usage value
|
|
20
|
-
limit: Maximum allowed value
|
|
21
|
-
remaining: Remaining capacity (limit - used)
|
|
22
|
-
"""
|
|
23
|
-
|
|
24
|
-
used: int | float
|
|
25
|
-
limit: int | float
|
|
26
|
-
remaining: int | float
|
|
27
|
-
|
|
28
|
-
@property
|
|
29
|
-
def percent_used(self) -> float:
|
|
30
|
-
"""Return percentage of limit used (0-100)."""
|
|
31
|
-
if self.limit == 0:
|
|
32
|
-
return 0.0
|
|
33
|
-
return (self.used / self.limit) * 100
|
|
34
|
-
|
|
35
|
-
@property
|
|
36
|
-
def is_exhausted(self) -> bool:
|
|
37
|
-
"""Return True if the limit has been reached."""
|
|
38
|
-
return self.remaining <= 0
|
|
39
|
-
|
|
40
|
-
@classmethod
|
|
41
|
-
def from_dict(cls, data: dict[str, Any]) -> UsageMetric:
|
|
42
|
-
"""Create from API response dict."""
|
|
43
|
-
return cls(
|
|
44
|
-
used=data.get("used", 0),
|
|
45
|
-
limit=data.get("limit", 0),
|
|
46
|
-
remaining=data.get("remaining", 0),
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
@dataclass
|
|
51
|
-
class UsagePeriod:
|
|
52
|
-
"""Time period for usage tracking."""
|
|
53
|
-
|
|
54
|
-
daily_start: datetime
|
|
55
|
-
monthly_start: datetime
|
|
56
|
-
|
|
57
|
-
@classmethod
|
|
58
|
-
def from_dict(cls, data: dict[str, Any]) -> UsagePeriod:
|
|
59
|
-
"""Create from API response dict."""
|
|
60
|
-
return cls(
|
|
61
|
-
daily_start=datetime.fromisoformat(data["daily_start"].replace("Z", "+00:00")),
|
|
62
|
-
monthly_start=datetime.fromisoformat(data["monthly_start"].replace("Z", "+00:00")),
|
|
63
|
-
)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
@dataclass
|
|
67
|
-
class InferenceUsage:
|
|
68
|
-
"""Usage metrics for the inference API."""
|
|
69
|
-
|
|
70
|
-
requests_per_min: UsageMetric
|
|
71
|
-
tokens_per_day: UsageMetric
|
|
72
|
-
spend_cents_per_month: UsageMetric
|
|
73
|
-
|
|
74
|
-
@classmethod
|
|
75
|
-
def from_dict(cls, data: dict[str, Any]) -> InferenceUsage:
|
|
76
|
-
"""Create from API response dict."""
|
|
77
|
-
return cls(
|
|
78
|
-
requests_per_min=UsageMetric.from_dict(data.get("requests_per_min", {})),
|
|
79
|
-
tokens_per_day=UsageMetric.from_dict(data.get("tokens_per_day", {})),
|
|
80
|
-
spend_cents_per_month=UsageMetric.from_dict(data.get("spend_cents_per_month", {})),
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
@dataclass
|
|
85
|
-
class JudgesUsage:
|
|
86
|
-
"""Usage metrics for the judges API."""
|
|
87
|
-
|
|
88
|
-
evaluations_per_day: UsageMetric
|
|
89
|
-
|
|
90
|
-
@classmethod
|
|
91
|
-
def from_dict(cls, data: dict[str, Any]) -> JudgesUsage:
|
|
92
|
-
"""Create from API response dict."""
|
|
93
|
-
return cls(
|
|
94
|
-
evaluations_per_day=UsageMetric.from_dict(data.get("evaluations_per_day", {})),
|
|
95
|
-
)
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
@dataclass
|
|
99
|
-
class PromptOptUsage:
|
|
100
|
-
"""Usage metrics for prompt optimization (GEPA/MIPRO)."""
|
|
101
|
-
|
|
102
|
-
jobs_per_day: UsageMetric
|
|
103
|
-
rollouts_per_day: UsageMetric
|
|
104
|
-
spend_cents_per_day: UsageMetric
|
|
105
|
-
|
|
106
|
-
@classmethod
|
|
107
|
-
def from_dict(cls, data: dict[str, Any]) -> PromptOptUsage:
|
|
108
|
-
"""Create from API response dict."""
|
|
109
|
-
return cls(
|
|
110
|
-
jobs_per_day=UsageMetric.from_dict(data.get("jobs_per_day", {})),
|
|
111
|
-
rollouts_per_day=UsageMetric.from_dict(data.get("rollouts_per_day", {})),
|
|
112
|
-
spend_cents_per_day=UsageMetric.from_dict(data.get("spend_cents_per_day", {})),
|
|
113
|
-
)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
@dataclass
|
|
117
|
-
class RLUsage:
|
|
118
|
-
"""Usage metrics for RL training."""
|
|
119
|
-
|
|
120
|
-
jobs_per_month: UsageMetric
|
|
121
|
-
gpu_hours_per_month: UsageMetric
|
|
122
|
-
|
|
123
|
-
@classmethod
|
|
124
|
-
def from_dict(cls, data: dict[str, Any]) -> RLUsage:
|
|
125
|
-
"""Create from API response dict."""
|
|
126
|
-
return cls(
|
|
127
|
-
jobs_per_month=UsageMetric.from_dict(data.get("jobs_per_month", {})),
|
|
128
|
-
gpu_hours_per_month=UsageMetric.from_dict(data.get("gpu_hours_per_month", {})),
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
@dataclass
|
|
133
|
-
class SFTUsage:
|
|
134
|
-
"""Usage metrics for SFT training."""
|
|
135
|
-
|
|
136
|
-
jobs_per_month: UsageMetric
|
|
137
|
-
gpu_hours_per_month: UsageMetric
|
|
138
|
-
|
|
139
|
-
@classmethod
|
|
140
|
-
def from_dict(cls, data: dict[str, Any]) -> SFTUsage:
|
|
141
|
-
"""Create from API response dict."""
|
|
142
|
-
return cls(
|
|
143
|
-
jobs_per_month=UsageMetric.from_dict(data.get("jobs_per_month", {})),
|
|
144
|
-
gpu_hours_per_month=UsageMetric.from_dict(data.get("gpu_hours_per_month", {})),
|
|
145
|
-
)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
@dataclass
|
|
149
|
-
class ResearchUsage:
|
|
150
|
-
"""Usage metrics for research agents."""
|
|
151
|
-
|
|
152
|
-
jobs_per_month: UsageMetric
|
|
153
|
-
agent_spend_cents_per_month: UsageMetric
|
|
154
|
-
|
|
155
|
-
@classmethod
|
|
156
|
-
def from_dict(cls, data: dict[str, Any]) -> ResearchUsage:
|
|
157
|
-
"""Create from API response dict."""
|
|
158
|
-
return cls(
|
|
159
|
-
jobs_per_month=UsageMetric.from_dict(data.get("jobs_per_month", {})),
|
|
160
|
-
agent_spend_cents_per_month=UsageMetric.from_dict(data.get("agent_spend_cents_per_month", {})),
|
|
161
|
-
)
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
@dataclass
|
|
165
|
-
class TotalUsage:
|
|
166
|
-
"""Total usage across all APIs."""
|
|
167
|
-
|
|
168
|
-
spend_cents_per_month: UsageMetric
|
|
169
|
-
|
|
170
|
-
@classmethod
|
|
171
|
-
def from_dict(cls, data: dict[str, Any]) -> TotalUsage:
|
|
172
|
-
"""Create from API response dict."""
|
|
173
|
-
return cls(
|
|
174
|
-
spend_cents_per_month=UsageMetric.from_dict(data.get("spend_cents_per_month", {})),
|
|
175
|
-
)
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
@dataclass
|
|
179
|
-
class APIUsage:
|
|
180
|
-
"""Container for all API usage metrics."""
|
|
181
|
-
|
|
182
|
-
inference: InferenceUsage
|
|
183
|
-
judges: JudgesUsage
|
|
184
|
-
prompt_opt: PromptOptUsage
|
|
185
|
-
rl: RLUsage
|
|
186
|
-
sft: SFTUsage
|
|
187
|
-
research: ResearchUsage
|
|
188
|
-
|
|
189
|
-
@classmethod
|
|
190
|
-
def from_dict(cls, data: dict[str, Any]) -> APIUsage:
|
|
191
|
-
"""Create from API response dict."""
|
|
192
|
-
return cls(
|
|
193
|
-
inference=InferenceUsage.from_dict(data.get("inference", {})),
|
|
194
|
-
judges=JudgesUsage.from_dict(data.get("judges", {})),
|
|
195
|
-
prompt_opt=PromptOptUsage.from_dict(data.get("prompt_opt", {})),
|
|
196
|
-
rl=RLUsage.from_dict(data.get("rl", {})),
|
|
197
|
-
sft=SFTUsage.from_dict(data.get("sft", {})),
|
|
198
|
-
research=ResearchUsage.from_dict(data.get("research", {})),
|
|
199
|
-
)
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
@dataclass
|
|
203
|
-
class OrgUsage:
|
|
204
|
-
"""Complete org usage report.
|
|
205
|
-
|
|
206
|
-
This is the top-level object returned by UsageClient.get().
|
|
207
|
-
|
|
208
|
-
Attributes:
|
|
209
|
-
org_id: The organization ID
|
|
210
|
-
tier: The org's tier (free, starter, growth, enterprise)
|
|
211
|
-
period: Time period info for daily/monthly resets
|
|
212
|
-
apis: Per-API usage metrics
|
|
213
|
-
totals: Aggregate totals across all APIs
|
|
214
|
-
"""
|
|
215
|
-
|
|
216
|
-
org_id: str
|
|
217
|
-
tier: str
|
|
218
|
-
period: UsagePeriod
|
|
219
|
-
apis: APIUsage
|
|
220
|
-
totals: TotalUsage
|
|
221
|
-
|
|
222
|
-
@classmethod
|
|
223
|
-
def from_dict(cls, data: dict[str, Any]) -> OrgUsage:
|
|
224
|
-
"""Create from API response dict."""
|
|
225
|
-
return cls(
|
|
226
|
-
org_id=data.get("org_id", ""),
|
|
227
|
-
tier=data.get("tier", "free"),
|
|
228
|
-
period=UsagePeriod.from_dict(data.get("period", {})),
|
|
229
|
-
apis=APIUsage.from_dict(data.get("apis", {})),
|
|
230
|
-
totals=TotalUsage.from_dict(data.get("totals", {})),
|
|
231
|
-
)
|
|
232
|
-
|
|
233
|
-
def get_metric(self, api: str, metric: str) -> UsageMetric | None:
|
|
234
|
-
"""Get a specific metric by API and metric name.
|
|
235
|
-
|
|
236
|
-
Args:
|
|
237
|
-
api: API name (inference, judges, prompt_opt, rl, sft, research)
|
|
238
|
-
metric: Metric name (e.g., requests_per_min, jobs_per_day)
|
|
239
|
-
|
|
240
|
-
Returns:
|
|
241
|
-
UsageMetric if found, None otherwise
|
|
242
|
-
"""
|
|
243
|
-
api_usage = getattr(self.apis, api, None)
|
|
244
|
-
if api_usage is None:
|
|
245
|
-
return None
|
|
246
|
-
return getattr(api_usage, metric, None)
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
__all__ = [
|
|
250
|
-
"UsageMetric",
|
|
251
|
-
"UsagePeriod",
|
|
252
|
-
"InferenceUsage",
|
|
253
|
-
"JudgesUsage",
|
|
254
|
-
"PromptOptUsage",
|
|
255
|
-
"RLUsage",
|
|
256
|
-
"SFTUsage",
|
|
257
|
-
"ResearchUsage",
|
|
258
|
-
"TotalUsage",
|
|
259
|
-
"APIUsage",
|
|
260
|
-
"OrgUsage",
|
|
261
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|