synth-ai 0.4.1__py3-none-any.whl → 0.4.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synth-ai might be problematic. Click here for more details.

Files changed (153) hide show
  1. synth_ai/__init__.py +13 -13
  2. synth_ai/cli/__init__.py +6 -15
  3. synth_ai/cli/commands/eval/__init__.py +6 -15
  4. synth_ai/cli/commands/eval/config.py +338 -0
  5. synth_ai/cli/commands/eval/core.py +236 -1091
  6. synth_ai/cli/commands/eval/runner.py +704 -0
  7. synth_ai/cli/commands/eval/validation.py +44 -117
  8. synth_ai/cli/commands/filter/core.py +7 -7
  9. synth_ai/cli/commands/filter/validation.py +2 -2
  10. synth_ai/cli/commands/smoke/core.py +7 -17
  11. synth_ai/cli/commands/status/__init__.py +1 -64
  12. synth_ai/cli/commands/status/client.py +50 -151
  13. synth_ai/cli/commands/status/config.py +3 -83
  14. synth_ai/cli/commands/status/errors.py +4 -13
  15. synth_ai/cli/commands/status/subcommands/__init__.py +2 -8
  16. synth_ai/cli/commands/status/subcommands/config.py +13 -0
  17. synth_ai/cli/commands/status/subcommands/files.py +18 -63
  18. synth_ai/cli/commands/status/subcommands/jobs.py +28 -311
  19. synth_ai/cli/commands/status/subcommands/models.py +18 -62
  20. synth_ai/cli/commands/status/subcommands/runs.py +16 -63
  21. synth_ai/cli/commands/status/subcommands/session.py +67 -172
  22. synth_ai/cli/commands/status/subcommands/summary.py +24 -32
  23. synth_ai/cli/commands/status/subcommands/utils.py +41 -0
  24. synth_ai/cli/commands/status/utils.py +16 -107
  25. synth_ai/cli/commands/train/__init__.py +18 -20
  26. synth_ai/cli/commands/train/errors.py +3 -3
  27. synth_ai/cli/commands/train/prompt_learning_validation.py +15 -16
  28. synth_ai/cli/commands/train/validation.py +7 -7
  29. synth_ai/cli/commands/train/{judge_schemas.py → verifier_schemas.py} +33 -34
  30. synth_ai/cli/commands/train/verifier_validation.py +235 -0
  31. synth_ai/cli/demo_apps/demo_task_apps/math/config.toml +0 -1
  32. synth_ai/cli/demo_apps/demo_task_apps/math/modal_task_app.py +2 -6
  33. synth_ai/cli/demo_apps/math/config.toml +0 -1
  34. synth_ai/cli/demo_apps/math/modal_task_app.py +2 -6
  35. synth_ai/cli/demo_apps/mipro/task_app.py +25 -47
  36. synth_ai/cli/lib/apps/task_app.py +12 -13
  37. synth_ai/cli/lib/task_app_discovery.py +6 -6
  38. synth_ai/cli/lib/train_cfgs.py +10 -10
  39. synth_ai/cli/task_apps/__init__.py +11 -0
  40. synth_ai/cli/task_apps/commands.py +7 -15
  41. synth_ai/core/env.py +12 -1
  42. synth_ai/core/errors.py +1 -2
  43. synth_ai/core/integrations/cloudflare.py +209 -33
  44. synth_ai/core/tracing_v3/abstractions.py +46 -0
  45. synth_ai/data/__init__.py +3 -30
  46. synth_ai/data/enums.py +1 -20
  47. synth_ai/data/rewards.py +100 -3
  48. synth_ai/products/graph_evolve/__init__.py +1 -2
  49. synth_ai/products/graph_evolve/config.py +16 -16
  50. synth_ai/products/graph_evolve/converters/__init__.py +3 -3
  51. synth_ai/products/graph_evolve/converters/openai_sft.py +7 -7
  52. synth_ai/products/graph_evolve/examples/hotpotqa/config.toml +1 -1
  53. synth_ai/products/graph_gepa/__init__.py +23 -0
  54. synth_ai/products/graph_gepa/converters/__init__.py +19 -0
  55. synth_ai/products/graph_gepa/converters/openai_sft.py +29 -0
  56. synth_ai/sdk/__init__.py +45 -35
  57. synth_ai/sdk/api/eval/__init__.py +33 -0
  58. synth_ai/sdk/api/eval/job.py +732 -0
  59. synth_ai/sdk/api/research_agent/__init__.py +276 -66
  60. synth_ai/sdk/api/train/builders.py +181 -0
  61. synth_ai/sdk/api/train/cli.py +41 -33
  62. synth_ai/sdk/api/train/configs/__init__.py +6 -4
  63. synth_ai/sdk/api/train/configs/prompt_learning.py +127 -33
  64. synth_ai/sdk/api/train/configs/rl.py +264 -16
  65. synth_ai/sdk/api/train/configs/sft.py +165 -1
  66. synth_ai/sdk/api/train/graph_validators.py +12 -12
  67. synth_ai/sdk/api/train/graphgen.py +169 -51
  68. synth_ai/sdk/api/train/graphgen_models.py +95 -45
  69. synth_ai/sdk/api/train/local_api.py +10 -0
  70. synth_ai/sdk/api/train/pollers.py +36 -0
  71. synth_ai/sdk/api/train/prompt_learning.py +390 -60
  72. synth_ai/sdk/api/train/rl.py +41 -5
  73. synth_ai/sdk/api/train/sft.py +2 -0
  74. synth_ai/sdk/api/train/task_app.py +20 -0
  75. synth_ai/sdk/api/train/validators.py +17 -17
  76. synth_ai/sdk/graphs/completions.py +239 -33
  77. synth_ai/sdk/{judging/schemas.py → graphs/verifier_schemas.py} +23 -23
  78. synth_ai/sdk/learning/__init__.py +35 -5
  79. synth_ai/sdk/learning/context_learning_client.py +531 -0
  80. synth_ai/sdk/learning/context_learning_types.py +294 -0
  81. synth_ai/sdk/learning/prompt_learning_client.py +1 -1
  82. synth_ai/sdk/learning/prompt_learning_types.py +2 -1
  83. synth_ai/sdk/learning/rl/__init__.py +0 -4
  84. synth_ai/sdk/learning/rl/contracts.py +0 -4
  85. synth_ai/sdk/localapi/__init__.py +40 -0
  86. synth_ai/sdk/localapi/apps/__init__.py +28 -0
  87. synth_ai/sdk/localapi/client.py +10 -0
  88. synth_ai/sdk/localapi/contracts.py +10 -0
  89. synth_ai/sdk/localapi/helpers.py +519 -0
  90. synth_ai/sdk/localapi/rollouts.py +93 -0
  91. synth_ai/sdk/localapi/server.py +29 -0
  92. synth_ai/sdk/localapi/template.py +49 -0
  93. synth_ai/sdk/streaming/handlers.py +6 -6
  94. synth_ai/sdk/streaming/streamer.py +10 -6
  95. synth_ai/sdk/task/__init__.py +18 -5
  96. synth_ai/sdk/task/apps/__init__.py +37 -1
  97. synth_ai/sdk/task/client.py +9 -1
  98. synth_ai/sdk/task/config.py +6 -11
  99. synth_ai/sdk/task/contracts.py +137 -95
  100. synth_ai/sdk/task/in_process.py +32 -22
  101. synth_ai/sdk/task/in_process_runner.py +9 -4
  102. synth_ai/sdk/task/rubrics/__init__.py +2 -3
  103. synth_ai/sdk/task/rubrics/loaders.py +4 -4
  104. synth_ai/sdk/task/rubrics/strict.py +3 -4
  105. synth_ai/sdk/task/server.py +76 -16
  106. synth_ai/sdk/task/trace_correlation_helpers.py +190 -139
  107. synth_ai/sdk/task/validators.py +34 -49
  108. synth_ai/sdk/training/__init__.py +7 -16
  109. synth_ai/sdk/tunnels/__init__.py +118 -0
  110. synth_ai/sdk/tunnels/cleanup.py +83 -0
  111. synth_ai/sdk/tunnels/ports.py +120 -0
  112. synth_ai/sdk/tunnels/tunneled_api.py +363 -0
  113. {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/METADATA +71 -4
  114. {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/RECORD +118 -128
  115. synth_ai/cli/commands/baseline/__init__.py +0 -12
  116. synth_ai/cli/commands/baseline/core.py +0 -636
  117. synth_ai/cli/commands/baseline/list.py +0 -94
  118. synth_ai/cli/commands/eval/errors.py +0 -81
  119. synth_ai/cli/commands/status/formatters.py +0 -164
  120. synth_ai/cli/commands/status/subcommands/pricing.py +0 -23
  121. synth_ai/cli/commands/status/subcommands/usage.py +0 -203
  122. synth_ai/cli/commands/train/judge_validation.py +0 -305
  123. synth_ai/cli/usage.py +0 -159
  124. synth_ai/data/specs.py +0 -36
  125. synth_ai/sdk/api/research_agent/cli.py +0 -428
  126. synth_ai/sdk/api/research_agent/config.py +0 -357
  127. synth_ai/sdk/api/research_agent/job.py +0 -717
  128. synth_ai/sdk/baseline/__init__.py +0 -25
  129. synth_ai/sdk/baseline/config.py +0 -209
  130. synth_ai/sdk/baseline/discovery.py +0 -216
  131. synth_ai/sdk/baseline/execution.py +0 -154
  132. synth_ai/sdk/judging/__init__.py +0 -15
  133. synth_ai/sdk/judging/base.py +0 -24
  134. synth_ai/sdk/judging/client.py +0 -191
  135. synth_ai/sdk/judging/types.py +0 -42
  136. synth_ai/sdk/research_agent/__init__.py +0 -34
  137. synth_ai/sdk/research_agent/container_builder.py +0 -328
  138. synth_ai/sdk/research_agent/container_spec.py +0 -198
  139. synth_ai/sdk/research_agent/defaults.py +0 -34
  140. synth_ai/sdk/research_agent/results_collector.py +0 -69
  141. synth_ai/sdk/specs/__init__.py +0 -46
  142. synth_ai/sdk/specs/dataclasses.py +0 -149
  143. synth_ai/sdk/specs/loader.py +0 -144
  144. synth_ai/sdk/specs/serializer.py +0 -199
  145. synth_ai/sdk/specs/validation.py +0 -250
  146. synth_ai/sdk/tracing/__init__.py +0 -39
  147. synth_ai/sdk/usage/__init__.py +0 -37
  148. synth_ai/sdk/usage/client.py +0 -171
  149. synth_ai/sdk/usage/models.py +0 -261
  150. {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/WHEEL +0 -0
  151. {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/entry_points.txt +0 -0
  152. {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/licenses/LICENSE +0 -0
  153. {synth_ai-0.4.1.dist-info → synth_ai-0.4.4.dist-info}/top_level.txt +0 -0
@@ -1,261 +0,0 @@
1
- """Data models for usage tracking.
2
-
3
- These dataclasses represent the usage and limits data returned by the
4
- GET /api/v1/usage endpoint.
5
- """
6
-
7
- from __future__ import annotations
8
-
9
- from dataclasses import dataclass, field
10
- from datetime import datetime
11
- from typing import Any
12
-
13
-
14
- @dataclass
15
- class UsageMetric:
16
- """A single usage metric with its limit and remaining capacity.
17
-
18
- Attributes:
19
- used: Current usage value
20
- limit: Maximum allowed value
21
- remaining: Remaining capacity (limit - used)
22
- """
23
-
24
- used: int | float
25
- limit: int | float
26
- remaining: int | float
27
-
28
- @property
29
- def percent_used(self) -> float:
30
- """Return percentage of limit used (0-100)."""
31
- if self.limit == 0:
32
- return 0.0
33
- return (self.used / self.limit) * 100
34
-
35
- @property
36
- def is_exhausted(self) -> bool:
37
- """Return True if the limit has been reached."""
38
- return self.remaining <= 0
39
-
40
- @classmethod
41
- def from_dict(cls, data: dict[str, Any]) -> UsageMetric:
42
- """Create from API response dict."""
43
- return cls(
44
- used=data.get("used", 0),
45
- limit=data.get("limit", 0),
46
- remaining=data.get("remaining", 0),
47
- )
48
-
49
-
50
- @dataclass
51
- class UsagePeriod:
52
- """Time period for usage tracking."""
53
-
54
- daily_start: datetime
55
- monthly_start: datetime
56
-
57
- @classmethod
58
- def from_dict(cls, data: dict[str, Any]) -> UsagePeriod:
59
- """Create from API response dict."""
60
- return cls(
61
- daily_start=datetime.fromisoformat(data["daily_start"].replace("Z", "+00:00")),
62
- monthly_start=datetime.fromisoformat(data["monthly_start"].replace("Z", "+00:00")),
63
- )
64
-
65
-
66
- @dataclass
67
- class InferenceUsage:
68
- """Usage metrics for the inference API."""
69
-
70
- requests_per_min: UsageMetric
71
- tokens_per_day: UsageMetric
72
- spend_cents_per_month: UsageMetric
73
-
74
- @classmethod
75
- def from_dict(cls, data: dict[str, Any]) -> InferenceUsage:
76
- """Create from API response dict."""
77
- return cls(
78
- requests_per_min=UsageMetric.from_dict(data.get("requests_per_min", {})),
79
- tokens_per_day=UsageMetric.from_dict(data.get("tokens_per_day", {})),
80
- spend_cents_per_month=UsageMetric.from_dict(data.get("spend_cents_per_month", {})),
81
- )
82
-
83
-
84
- @dataclass
85
- class JudgesUsage:
86
- """Usage metrics for the judges API."""
87
-
88
- evaluations_per_day: UsageMetric
89
-
90
- @classmethod
91
- def from_dict(cls, data: dict[str, Any]) -> JudgesUsage:
92
- """Create from API response dict."""
93
- return cls(
94
- evaluations_per_day=UsageMetric.from_dict(data.get("evaluations_per_day", {})),
95
- )
96
-
97
-
98
- @dataclass
99
- class PromptOptUsage:
100
- """Usage metrics for prompt optimization (GEPA/MIPRO)."""
101
-
102
- jobs_per_day: UsageMetric
103
- rollouts_per_day: UsageMetric
104
- spend_cents_per_day: UsageMetric
105
-
106
- @classmethod
107
- def from_dict(cls, data: dict[str, Any]) -> PromptOptUsage:
108
- """Create from API response dict."""
109
- return cls(
110
- jobs_per_day=UsageMetric.from_dict(data.get("jobs_per_day", {})),
111
- rollouts_per_day=UsageMetric.from_dict(data.get("rollouts_per_day", {})),
112
- spend_cents_per_day=UsageMetric.from_dict(data.get("spend_cents_per_day", {})),
113
- )
114
-
115
-
116
- @dataclass
117
- class RLUsage:
118
- """Usage metrics for RL training."""
119
-
120
- jobs_per_month: UsageMetric
121
- gpu_hours_per_month: UsageMetric
122
-
123
- @classmethod
124
- def from_dict(cls, data: dict[str, Any]) -> RLUsage:
125
- """Create from API response dict."""
126
- return cls(
127
- jobs_per_month=UsageMetric.from_dict(data.get("jobs_per_month", {})),
128
- gpu_hours_per_month=UsageMetric.from_dict(data.get("gpu_hours_per_month", {})),
129
- )
130
-
131
-
132
- @dataclass
133
- class SFTUsage:
134
- """Usage metrics for SFT training."""
135
-
136
- jobs_per_month: UsageMetric
137
- gpu_hours_per_month: UsageMetric
138
-
139
- @classmethod
140
- def from_dict(cls, data: dict[str, Any]) -> SFTUsage:
141
- """Create from API response dict."""
142
- return cls(
143
- jobs_per_month=UsageMetric.from_dict(data.get("jobs_per_month", {})),
144
- gpu_hours_per_month=UsageMetric.from_dict(data.get("gpu_hours_per_month", {})),
145
- )
146
-
147
-
148
- @dataclass
149
- class ResearchUsage:
150
- """Usage metrics for research agents."""
151
-
152
- jobs_per_month: UsageMetric
153
- agent_spend_cents_per_month: UsageMetric
154
-
155
- @classmethod
156
- def from_dict(cls, data: dict[str, Any]) -> ResearchUsage:
157
- """Create from API response dict."""
158
- return cls(
159
- jobs_per_month=UsageMetric.from_dict(data.get("jobs_per_month", {})),
160
- agent_spend_cents_per_month=UsageMetric.from_dict(data.get("agent_spend_cents_per_month", {})),
161
- )
162
-
163
-
164
- @dataclass
165
- class TotalUsage:
166
- """Total usage across all APIs."""
167
-
168
- spend_cents_per_month: UsageMetric
169
-
170
- @classmethod
171
- def from_dict(cls, data: dict[str, Any]) -> TotalUsage:
172
- """Create from API response dict."""
173
- return cls(
174
- spend_cents_per_month=UsageMetric.from_dict(data.get("spend_cents_per_month", {})),
175
- )
176
-
177
-
178
- @dataclass
179
- class APIUsage:
180
- """Container for all API usage metrics."""
181
-
182
- inference: InferenceUsage
183
- judges: JudgesUsage
184
- prompt_opt: PromptOptUsage
185
- rl: RLUsage
186
- sft: SFTUsage
187
- research: ResearchUsage
188
-
189
- @classmethod
190
- def from_dict(cls, data: dict[str, Any]) -> APIUsage:
191
- """Create from API response dict."""
192
- return cls(
193
- inference=InferenceUsage.from_dict(data.get("inference", {})),
194
- judges=JudgesUsage.from_dict(data.get("judges", {})),
195
- prompt_opt=PromptOptUsage.from_dict(data.get("prompt_opt", {})),
196
- rl=RLUsage.from_dict(data.get("rl", {})),
197
- sft=SFTUsage.from_dict(data.get("sft", {})),
198
- research=ResearchUsage.from_dict(data.get("research", {})),
199
- )
200
-
201
-
202
- @dataclass
203
- class OrgUsage:
204
- """Complete org usage report.
205
-
206
- This is the top-level object returned by UsageClient.get().
207
-
208
- Attributes:
209
- org_id: The organization ID
210
- tier: The org's tier (free, starter, growth, enterprise)
211
- period: Time period info for daily/monthly resets
212
- apis: Per-API usage metrics
213
- totals: Aggregate totals across all APIs
214
- """
215
-
216
- org_id: str
217
- tier: str
218
- period: UsagePeriod
219
- apis: APIUsage
220
- totals: TotalUsage
221
-
222
- @classmethod
223
- def from_dict(cls, data: dict[str, Any]) -> OrgUsage:
224
- """Create from API response dict."""
225
- return cls(
226
- org_id=data.get("org_id", ""),
227
- tier=data.get("tier", "free"),
228
- period=UsagePeriod.from_dict(data.get("period", {})),
229
- apis=APIUsage.from_dict(data.get("apis", {})),
230
- totals=TotalUsage.from_dict(data.get("totals", {})),
231
- )
232
-
233
- def get_metric(self, api: str, metric: str) -> UsageMetric | None:
234
- """Get a specific metric by API and metric name.
235
-
236
- Args:
237
- api: API name (inference, judges, prompt_opt, rl, sft, research)
238
- metric: Metric name (e.g., requests_per_min, jobs_per_day)
239
-
240
- Returns:
241
- UsageMetric if found, None otherwise
242
- """
243
- api_usage = getattr(self.apis, api, None)
244
- if api_usage is None:
245
- return None
246
- return getattr(api_usage, metric, None)
247
-
248
-
249
- __all__ = [
250
- "UsageMetric",
251
- "UsagePeriod",
252
- "InferenceUsage",
253
- "JudgesUsage",
254
- "PromptOptUsage",
255
- "RLUsage",
256
- "SFTUsage",
257
- "ResearchUsage",
258
- "TotalUsage",
259
- "APIUsage",
260
- "OrgUsage",
261
- ]