agenta 0.12.7__py3-none-any.whl → 0.13.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of agenta might be problematic. Click here for more details.
- agenta/__init__.py +3 -1
- agenta/cli/helper.py +1 -1
- agenta/cli/main.py +1 -1
- agenta/cli/variant_commands.py +7 -5
- agenta/client/api.py +1 -1
- agenta/client/backend/__init__.py +78 -18
- agenta/client/backend/client.py +1031 -5526
- agenta/client/backend/resources/__init__.py +31 -0
- agenta/client/backend/resources/apps/__init__.py +1 -0
- agenta/client/backend/resources/apps/client.py +977 -0
- agenta/client/backend/resources/bases/__init__.py +1 -0
- agenta/client/backend/resources/bases/client.py +127 -0
- agenta/client/backend/resources/configs/__init__.py +1 -0
- agenta/client/backend/resources/configs/client.py +377 -0
- agenta/client/backend/resources/containers/__init__.py +5 -0
- agenta/client/backend/resources/containers/client.py +383 -0
- agenta/client/backend/resources/containers/types/__init__.py +5 -0
- agenta/client/backend/{types → resources/containers/types}/container_templates_response.py +1 -1
- agenta/client/backend/resources/environments/__init__.py +1 -0
- agenta/client/backend/resources/environments/client.py +131 -0
- agenta/client/backend/resources/evaluations/__init__.py +1 -0
- agenta/client/backend/resources/evaluations/client.py +1008 -0
- agenta/client/backend/resources/evaluators/__init__.py +1 -0
- agenta/client/backend/resources/evaluators/client.py +594 -0
- agenta/client/backend/resources/observability/__init__.py +1 -0
- agenta/client/backend/resources/observability/client.py +1184 -0
- agenta/client/backend/resources/testsets/__init__.py +1 -0
- agenta/client/backend/resources/testsets/client.py +689 -0
- agenta/client/backend/resources/variants/__init__.py +5 -0
- agenta/client/backend/resources/variants/client.py +796 -0
- agenta/client/backend/resources/variants/types/__init__.py +7 -0
- agenta/client/backend/resources/variants/types/add_variant_from_base_and_config_response.py +7 -0
- agenta/client/backend/types/__init__.py +54 -22
- agenta/client/backend/types/aggregated_result.py +2 -2
- agenta/client/backend/types/aggregated_result_evaluator_config.py +9 -0
- agenta/client/backend/types/{app_variant_output.py → app_variant_response.py} +4 -2
- agenta/client/backend/types/{trace.py → create_span.py} +20 -10
- agenta/client/backend/types/create_trace_response.py +37 -0
- agenta/client/backend/types/environment_output.py +3 -1
- agenta/client/backend/types/environment_output_extended.py +45 -0
- agenta/client/backend/types/environment_revision.py +41 -0
- agenta/client/backend/types/error.py +37 -0
- agenta/client/backend/types/evaluation.py +6 -3
- agenta/client/backend/types/evaluation_scenario_output.py +4 -2
- agenta/client/backend/types/{delete_evaluation.py → evaluation_scenario_score_update.py} +2 -2
- agenta/client/backend/types/evaluation_status_enum.py +4 -0
- agenta/client/backend/types/evaluator.py +1 -0
- agenta/client/backend/types/{get_config_reponse.py → get_config_response.py} +1 -2
- agenta/client/backend/types/human_evaluation_scenario.py +2 -2
- agenta/client/backend/types/{app_variant_output_extended.py → human_evaluation_scenario_update.py} +11 -16
- agenta/client/backend/types/human_evaluation_update.py +37 -0
- agenta/client/backend/types/image.py +1 -0
- agenta/client/backend/types/invite_request.py +1 -0
- agenta/client/backend/types/{list_api_keys_output.py → list_api_keys_response.py} +1 -1
- agenta/client/backend/types/llm_tokens.py +38 -0
- agenta/client/backend/types/new_human_evaluation.py +42 -0
- agenta/client/backend/types/organization.py +1 -0
- agenta/client/backend/types/permission.py +141 -0
- agenta/client/backend/types/result.py +2 -0
- agenta/client/backend/types/{human_evaluation_scenario_score.py → score.py} +1 -1
- agenta/client/backend/types/span.py +18 -16
- agenta/client/backend/types/span_detail.py +52 -0
- agenta/client/backend/types/span_kind.py +49 -0
- agenta/client/backend/types/span_status_code.py +29 -0
- agenta/client/backend/types/span_variant.py +38 -0
- agenta/client/backend/types/trace_detail.py +52 -0
- agenta/client/backend/types/with_pagination.py +40 -0
- agenta/client/backend/types/workspace_member_response.py +38 -0
- agenta/client/backend/types/workspace_permission.py +40 -0
- agenta/client/backend/types/workspace_response.py +44 -0
- agenta/client/backend/types/workspace_role.py +41 -0
- agenta/client/backend/types/workspace_role_response.py +38 -0
- agenta/docker/docker_utils.py +1 -5
- agenta/sdk/__init__.py +3 -1
- agenta/sdk/agenta_decorator.py +68 -18
- agenta/sdk/agenta_init.py +53 -21
- agenta/sdk/tracing/context_manager.py +13 -0
- agenta/sdk/tracing/decorators.py +41 -0
- agenta/sdk/tracing/llm_tracing.py +220 -0
- agenta/sdk/tracing/logger.py +19 -0
- agenta/sdk/tracing/tasks_manager.py +130 -0
- {agenta-0.12.7.dist-info → agenta-0.13.0.dist-info}/METADATA +47 -96
- agenta-0.13.0.dist-info/RECORD +161 -0
- agenta/client/backend/types/add_variant_from_base_and_config_response.py +0 -7
- agenta/client/backend/types/human_evaluation_scenario_update_score.py +0 -5
- agenta-0.12.7.dist-info/RECORD +0 -114
- {agenta-0.12.7.dist-info → agenta-0.13.0.dist-info}/WHEEL +0 -0
- {agenta-0.12.7.dist-info → agenta-0.13.0.dist-info}/entry_points.txt +0 -0
agenta/__init__.py
CHANGED
|
@@ -13,8 +13,10 @@ from .sdk.types import (
|
|
|
13
13
|
FileInputURL,
|
|
14
14
|
BinaryParam,
|
|
15
15
|
)
|
|
16
|
+
from .sdk.tracing.decorators import span
|
|
16
17
|
from .sdk.utils.preinit import PreInitObject
|
|
17
|
-
from .sdk.agenta_init import Config, init
|
|
18
|
+
from .sdk.agenta_init import Config, init, llm_tracing
|
|
18
19
|
from .sdk.utils.helper.openai_cost import calculate_token_usage
|
|
19
20
|
|
|
21
|
+
|
|
20
22
|
config = PreInitObject("agenta.config", Config)
|
agenta/cli/helper.py
CHANGED
agenta/cli/main.py
CHANGED
|
@@ -188,7 +188,7 @@ def init(app_name: str, backend_host: str):
|
|
|
188
188
|
|
|
189
189
|
# Get app_id after creating new app in the backend server
|
|
190
190
|
try:
|
|
191
|
-
app_id = client.create_app(
|
|
191
|
+
app_id = client.apps.create_app(
|
|
192
192
|
app_name=app_name,
|
|
193
193
|
organization_id=filtered_org.id if filtered_org else None,
|
|
194
194
|
).app_id
|
agenta/cli/variant_commands.py
CHANGED
|
@@ -126,7 +126,7 @@ def add_variant(
|
|
|
126
126
|
)
|
|
127
127
|
)
|
|
128
128
|
with tar_path.open("rb") as tar_file:
|
|
129
|
-
built_image: Image = client.build_image(
|
|
129
|
+
built_image: Image = client.containers.build_image(
|
|
130
130
|
app_id=app_id,
|
|
131
131
|
base_name=base_name,
|
|
132
132
|
tar_file=tar_file,
|
|
@@ -147,7 +147,7 @@ def add_variant(
|
|
|
147
147
|
)
|
|
148
148
|
)
|
|
149
149
|
variant_id = config["variant_ids"][config["variants"].index(variant_name)]
|
|
150
|
-
client.update_variant_image(
|
|
150
|
+
client.variants.update_variant_image(
|
|
151
151
|
variant_id=variant_id,
|
|
152
152
|
request=image, # because Fern code uses "request: Image" instead of "image: Image"
|
|
153
153
|
) # this automatically restarts
|
|
@@ -259,7 +259,9 @@ def start_variant(variant_id: str, app_folder: str, host: str):
|
|
|
259
259
|
api_key=api_key,
|
|
260
260
|
)
|
|
261
261
|
|
|
262
|
-
endpoint = client.start_variant(
|
|
262
|
+
endpoint = client.variants.start_variant(
|
|
263
|
+
variant_id=variant_id, action={"action": "START"}
|
|
264
|
+
)
|
|
263
265
|
click.echo("\n" + click.style("Congratulations! 🎉", bold=True, fg="green"))
|
|
264
266
|
click.echo(
|
|
265
267
|
click.style("Your app has been deployed locally as an API. 🚀", fg="cyan")
|
|
@@ -327,7 +329,7 @@ def remove_variant(variant_name: str, app_folder: str, host: str):
|
|
|
327
329
|
)
|
|
328
330
|
|
|
329
331
|
try:
|
|
330
|
-
client.remove_variant(variant_id=variant_id)
|
|
332
|
+
client.variants.remove_variant(variant_id=variant_id)
|
|
331
333
|
except Exception as ex:
|
|
332
334
|
click.echo(
|
|
333
335
|
click.style(
|
|
@@ -365,7 +367,7 @@ def list_variants(app_folder: str, host: str):
|
|
|
365
367
|
)
|
|
366
368
|
|
|
367
369
|
try:
|
|
368
|
-
variants: List[AppVariant] = client.list_app_variants(app_id=app_id)
|
|
370
|
+
variants: List[AppVariant] = client.apps.list_app_variants(app_id=app_id)
|
|
369
371
|
except Exception as ex:
|
|
370
372
|
raise ex
|
|
371
373
|
|
agenta/client/api.py
CHANGED
|
@@ -1,115 +1,175 @@
|
|
|
1
1
|
# This file was auto-generated by Fern from our API Definition.
|
|
2
2
|
|
|
3
3
|
from .types import (
|
|
4
|
-
AddVariantFromBaseAndConfigResponse,
|
|
5
4
|
AggregatedResult,
|
|
5
|
+
AggregatedResultEvaluatorConfig,
|
|
6
6
|
App,
|
|
7
|
-
|
|
8
|
-
AppVariantOutputExtended,
|
|
7
|
+
AppVariantResponse,
|
|
9
8
|
AppVariantRevision,
|
|
10
9
|
BaseOutput,
|
|
11
10
|
BodyImportTestset,
|
|
12
11
|
ConfigDb,
|
|
13
|
-
ContainerTemplatesResponse,
|
|
14
12
|
CreateAppOutput,
|
|
15
|
-
|
|
13
|
+
CreateSpan,
|
|
14
|
+
CreateTraceResponse,
|
|
16
15
|
DockerEnvVars,
|
|
17
16
|
EnvironmentOutput,
|
|
17
|
+
EnvironmentOutputExtended,
|
|
18
|
+
EnvironmentRevision,
|
|
19
|
+
Error,
|
|
18
20
|
Evaluation,
|
|
19
21
|
EvaluationScenario,
|
|
20
22
|
EvaluationScenarioInput,
|
|
21
23
|
EvaluationScenarioOutput,
|
|
22
24
|
EvaluationScenarioResult,
|
|
25
|
+
EvaluationScenarioScoreUpdate,
|
|
23
26
|
EvaluationStatusEnum,
|
|
24
27
|
EvaluationType,
|
|
25
28
|
EvaluationWebhook,
|
|
26
29
|
Evaluator,
|
|
27
30
|
EvaluatorConfig,
|
|
28
31
|
Feedback,
|
|
29
|
-
|
|
32
|
+
GetConfigResponse,
|
|
30
33
|
HttpValidationError,
|
|
31
34
|
HumanEvaluation,
|
|
32
35
|
HumanEvaluationScenario,
|
|
33
36
|
HumanEvaluationScenarioInput,
|
|
34
37
|
HumanEvaluationScenarioOutput,
|
|
35
|
-
|
|
36
|
-
|
|
38
|
+
HumanEvaluationScenarioUpdate,
|
|
39
|
+
HumanEvaluationUpdate,
|
|
37
40
|
Image,
|
|
38
41
|
InviteRequest,
|
|
39
|
-
|
|
42
|
+
ListApiKeysResponse,
|
|
40
43
|
LlmRunRateLimit,
|
|
44
|
+
LlmTokens,
|
|
45
|
+
NewHumanEvaluation,
|
|
41
46
|
NewTestset,
|
|
42
47
|
Organization,
|
|
43
48
|
OrganizationOutput,
|
|
49
|
+
Permission,
|
|
44
50
|
Result,
|
|
51
|
+
Score,
|
|
45
52
|
SimpleEvaluationOutput,
|
|
46
53
|
Span,
|
|
54
|
+
SpanDetail,
|
|
55
|
+
SpanKind,
|
|
56
|
+
SpanStatusCode,
|
|
57
|
+
SpanVariant,
|
|
47
58
|
Template,
|
|
48
59
|
TemplateImageInfo,
|
|
49
60
|
TestSetOutputResponse,
|
|
50
61
|
TestSetSimpleResponse,
|
|
51
|
-
|
|
62
|
+
TraceDetail,
|
|
52
63
|
Uri,
|
|
53
64
|
ValidationError,
|
|
54
65
|
ValidationErrorLocItem,
|
|
55
66
|
VariantAction,
|
|
56
67
|
VariantActionEnum,
|
|
68
|
+
WithPagination,
|
|
69
|
+
WorkspaceMemberResponse,
|
|
70
|
+
WorkspacePermission,
|
|
71
|
+
WorkspaceResponse,
|
|
72
|
+
WorkspaceRole,
|
|
73
|
+
WorkspaceRoleResponse,
|
|
57
74
|
)
|
|
58
75
|
from .errors import UnprocessableEntityError
|
|
76
|
+
from .resources import (
|
|
77
|
+
AddVariantFromBaseAndConfigResponse,
|
|
78
|
+
ContainerTemplatesResponse,
|
|
79
|
+
apps,
|
|
80
|
+
bases,
|
|
81
|
+
configs,
|
|
82
|
+
containers,
|
|
83
|
+
environments,
|
|
84
|
+
evaluations,
|
|
85
|
+
evaluators,
|
|
86
|
+
observability,
|
|
87
|
+
testsets,
|
|
88
|
+
variants,
|
|
89
|
+
)
|
|
59
90
|
|
|
60
91
|
__all__ = [
|
|
61
92
|
"AddVariantFromBaseAndConfigResponse",
|
|
62
93
|
"AggregatedResult",
|
|
94
|
+
"AggregatedResultEvaluatorConfig",
|
|
63
95
|
"App",
|
|
64
|
-
"
|
|
65
|
-
"AppVariantOutputExtended",
|
|
96
|
+
"AppVariantResponse",
|
|
66
97
|
"AppVariantRevision",
|
|
67
98
|
"BaseOutput",
|
|
68
99
|
"BodyImportTestset",
|
|
69
100
|
"ConfigDb",
|
|
70
101
|
"ContainerTemplatesResponse",
|
|
71
102
|
"CreateAppOutput",
|
|
72
|
-
"
|
|
103
|
+
"CreateSpan",
|
|
104
|
+
"CreateTraceResponse",
|
|
73
105
|
"DockerEnvVars",
|
|
74
106
|
"EnvironmentOutput",
|
|
107
|
+
"EnvironmentOutputExtended",
|
|
108
|
+
"EnvironmentRevision",
|
|
109
|
+
"Error",
|
|
75
110
|
"Evaluation",
|
|
76
111
|
"EvaluationScenario",
|
|
77
112
|
"EvaluationScenarioInput",
|
|
78
113
|
"EvaluationScenarioOutput",
|
|
79
114
|
"EvaluationScenarioResult",
|
|
115
|
+
"EvaluationScenarioScoreUpdate",
|
|
80
116
|
"EvaluationStatusEnum",
|
|
81
117
|
"EvaluationType",
|
|
82
118
|
"EvaluationWebhook",
|
|
83
119
|
"Evaluator",
|
|
84
120
|
"EvaluatorConfig",
|
|
85
121
|
"Feedback",
|
|
86
|
-
"
|
|
122
|
+
"GetConfigResponse",
|
|
87
123
|
"HttpValidationError",
|
|
88
124
|
"HumanEvaluation",
|
|
89
125
|
"HumanEvaluationScenario",
|
|
90
126
|
"HumanEvaluationScenarioInput",
|
|
91
127
|
"HumanEvaluationScenarioOutput",
|
|
92
|
-
"
|
|
93
|
-
"
|
|
128
|
+
"HumanEvaluationScenarioUpdate",
|
|
129
|
+
"HumanEvaluationUpdate",
|
|
94
130
|
"Image",
|
|
95
131
|
"InviteRequest",
|
|
96
|
-
"
|
|
132
|
+
"ListApiKeysResponse",
|
|
97
133
|
"LlmRunRateLimit",
|
|
134
|
+
"LlmTokens",
|
|
135
|
+
"NewHumanEvaluation",
|
|
98
136
|
"NewTestset",
|
|
99
137
|
"Organization",
|
|
100
138
|
"OrganizationOutput",
|
|
139
|
+
"Permission",
|
|
101
140
|
"Result",
|
|
141
|
+
"Score",
|
|
102
142
|
"SimpleEvaluationOutput",
|
|
103
143
|
"Span",
|
|
144
|
+
"SpanDetail",
|
|
145
|
+
"SpanKind",
|
|
146
|
+
"SpanStatusCode",
|
|
147
|
+
"SpanVariant",
|
|
104
148
|
"Template",
|
|
105
149
|
"TemplateImageInfo",
|
|
106
150
|
"TestSetOutputResponse",
|
|
107
151
|
"TestSetSimpleResponse",
|
|
108
|
-
"
|
|
152
|
+
"TraceDetail",
|
|
109
153
|
"UnprocessableEntityError",
|
|
110
154
|
"Uri",
|
|
111
155
|
"ValidationError",
|
|
112
156
|
"ValidationErrorLocItem",
|
|
113
157
|
"VariantAction",
|
|
114
158
|
"VariantActionEnum",
|
|
159
|
+
"WithPagination",
|
|
160
|
+
"WorkspaceMemberResponse",
|
|
161
|
+
"WorkspacePermission",
|
|
162
|
+
"WorkspaceResponse",
|
|
163
|
+
"WorkspaceRole",
|
|
164
|
+
"WorkspaceRoleResponse",
|
|
165
|
+
"apps",
|
|
166
|
+
"bases",
|
|
167
|
+
"configs",
|
|
168
|
+
"containers",
|
|
169
|
+
"environments",
|
|
170
|
+
"evaluations",
|
|
171
|
+
"evaluators",
|
|
172
|
+
"observability",
|
|
173
|
+
"testsets",
|
|
174
|
+
"variants",
|
|
115
175
|
]
|