opik 1.9.39__py3-none-any.whl → 1.9.86__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opik/api_objects/attachment/attachment_context.py +36 -0
- opik/api_objects/attachment/attachments_extractor.py +153 -0
- opik/api_objects/attachment/client.py +1 -0
- opik/api_objects/attachment/converters.py +2 -0
- opik/api_objects/attachment/decoder.py +18 -0
- opik/api_objects/attachment/decoder_base64.py +83 -0
- opik/api_objects/attachment/decoder_helpers.py +137 -0
- opik/api_objects/constants.py +2 -0
- opik/api_objects/dataset/dataset.py +133 -40
- opik/api_objects/dataset/rest_operations.py +2 -0
- opik/api_objects/experiment/experiment.py +6 -0
- opik/api_objects/helpers.py +8 -4
- opik/api_objects/local_recording.py +6 -5
- opik/api_objects/observation_data.py +101 -0
- opik/api_objects/opik_client.py +78 -45
- opik/api_objects/opik_query_language.py +9 -3
- opik/api_objects/prompt/chat/chat_prompt.py +18 -1
- opik/api_objects/prompt/client.py +8 -1
- opik/api_objects/span/span_data.py +3 -88
- opik/api_objects/threads/threads_client.py +7 -4
- opik/api_objects/trace/trace_data.py +3 -74
- opik/api_objects/validation_helpers.py +3 -3
- opik/cli/exports/__init__.py +131 -0
- opik/cli/exports/dataset.py +278 -0
- opik/cli/exports/experiment.py +784 -0
- opik/cli/exports/project.py +685 -0
- opik/cli/exports/prompt.py +578 -0
- opik/cli/exports/utils.py +406 -0
- opik/cli/harbor.py +39 -0
- opik/cli/imports/__init__.py +439 -0
- opik/cli/imports/dataset.py +143 -0
- opik/cli/imports/experiment.py +1192 -0
- opik/cli/imports/project.py +262 -0
- opik/cli/imports/prompt.py +177 -0
- opik/cli/imports/utils.py +280 -0
- opik/cli/main.py +14 -12
- opik/config.py +12 -1
- opik/datetime_helpers.py +12 -0
- opik/decorator/arguments_helpers.py +4 -1
- opik/decorator/base_track_decorator.py +111 -37
- opik/decorator/context_manager/span_context_manager.py +5 -1
- opik/decorator/generator_wrappers.py +5 -4
- opik/decorator/span_creation_handler.py +13 -4
- opik/evaluation/engine/engine.py +111 -28
- opik/evaluation/engine/evaluation_tasks_executor.py +71 -19
- opik/evaluation/evaluator.py +12 -0
- opik/evaluation/metrics/conversation/llm_judges/conversational_coherence/metric.py +3 -1
- opik/evaluation/metrics/conversation/llm_judges/session_completeness/metric.py +3 -1
- opik/evaluation/metrics/conversation/llm_judges/user_frustration/metric.py +3 -1
- opik/evaluation/metrics/heuristics/equals.py +11 -7
- opik/evaluation/metrics/llm_judges/answer_relevance/metric.py +3 -1
- opik/evaluation/metrics/llm_judges/context_precision/metric.py +3 -1
- opik/evaluation/metrics/llm_judges/context_recall/metric.py +3 -1
- opik/evaluation/metrics/llm_judges/factuality/metric.py +1 -1
- opik/evaluation/metrics/llm_judges/g_eval/metric.py +3 -1
- opik/evaluation/metrics/llm_judges/hallucination/metric.py +3 -1
- opik/evaluation/metrics/llm_judges/moderation/metric.py +3 -1
- opik/evaluation/metrics/llm_judges/structure_output_compliance/metric.py +3 -1
- opik/evaluation/metrics/llm_judges/syc_eval/metric.py +4 -2
- opik/evaluation/metrics/llm_judges/trajectory_accuracy/metric.py +3 -1
- opik/evaluation/metrics/llm_judges/usefulness/metric.py +3 -1
- opik/evaluation/metrics/ragas_metric.py +43 -23
- opik/evaluation/models/litellm/litellm_chat_model.py +7 -2
- opik/evaluation/models/litellm/util.py +4 -20
- opik/evaluation/models/models_factory.py +19 -5
- opik/evaluation/rest_operations.py +3 -3
- opik/evaluation/threads/helpers.py +3 -2
- opik/file_upload/file_uploader.py +13 -0
- opik/file_upload/upload_options.py +2 -0
- opik/integrations/adk/legacy_opik_tracer.py +9 -11
- opik/integrations/adk/opik_tracer.py +2 -2
- opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py +2 -2
- opik/integrations/dspy/callback.py +100 -14
- opik/integrations/dspy/parsers.py +168 -0
- opik/integrations/harbor/__init__.py +17 -0
- opik/integrations/harbor/experiment_service.py +269 -0
- opik/integrations/harbor/opik_tracker.py +528 -0
- opik/integrations/haystack/opik_tracer.py +2 -2
- opik/integrations/langchain/__init__.py +15 -2
- opik/integrations/langchain/langgraph_tracer_injector.py +88 -0
- opik/integrations/langchain/opik_tracer.py +258 -160
- opik/integrations/langchain/provider_usage_extractors/langchain_run_helpers/helpers.py +7 -4
- opik/integrations/llama_index/callback.py +43 -6
- opik/integrations/openai/agents/opik_tracing_processor.py +8 -10
- opik/integrations/openai/opik_tracker.py +99 -4
- opik/integrations/openai/videos/__init__.py +9 -0
- opik/integrations/openai/videos/binary_response_write_to_file_decorator.py +88 -0
- opik/integrations/openai/videos/videos_create_decorator.py +159 -0
- opik/integrations/openai/videos/videos_download_decorator.py +110 -0
- opik/message_processing/batching/base_batcher.py +14 -21
- opik/message_processing/batching/batch_manager.py +22 -10
- opik/message_processing/batching/batchers.py +32 -40
- opik/message_processing/batching/flushing_thread.py +0 -3
- opik/message_processing/emulation/emulator_message_processor.py +36 -1
- opik/message_processing/emulation/models.py +21 -0
- opik/message_processing/messages.py +9 -0
- opik/message_processing/preprocessing/__init__.py +0 -0
- opik/message_processing/preprocessing/attachments_preprocessor.py +70 -0
- opik/message_processing/preprocessing/batching_preprocessor.py +53 -0
- opik/message_processing/preprocessing/constants.py +1 -0
- opik/message_processing/preprocessing/file_upload_preprocessor.py +38 -0
- opik/message_processing/preprocessing/preprocessor.py +36 -0
- opik/message_processing/processors/__init__.py +0 -0
- opik/message_processing/processors/attachments_extraction_processor.py +146 -0
- opik/message_processing/{message_processors.py → processors/message_processors.py} +15 -1
- opik/message_processing/{message_processors_chain.py → processors/message_processors_chain.py} +3 -2
- opik/message_processing/{online_message_processor.py → processors/online_message_processor.py} +11 -9
- opik/message_processing/queue_consumer.py +4 -2
- opik/message_processing/streamer.py +71 -33
- opik/message_processing/streamer_constructors.py +36 -8
- opik/plugins/pytest/experiment_runner.py +1 -1
- opik/plugins/pytest/hooks.py +5 -3
- opik/rest_api/__init__.py +42 -0
- opik/rest_api/datasets/client.py +321 -123
- opik/rest_api/datasets/raw_client.py +470 -145
- opik/rest_api/experiments/client.py +26 -0
- opik/rest_api/experiments/raw_client.py +26 -0
- opik/rest_api/llm_provider_key/client.py +4 -4
- opik/rest_api/llm_provider_key/raw_client.py +4 -4
- opik/rest_api/llm_provider_key/types/provider_api_key_write_provider.py +2 -1
- opik/rest_api/manual_evaluation/client.py +101 -0
- opik/rest_api/manual_evaluation/raw_client.py +172 -0
- opik/rest_api/optimizations/client.py +0 -166
- opik/rest_api/optimizations/raw_client.py +0 -248
- opik/rest_api/projects/client.py +9 -0
- opik/rest_api/projects/raw_client.py +13 -0
- opik/rest_api/projects/types/project_metric_request_public_metric_type.py +4 -0
- opik/rest_api/prompts/client.py +130 -2
- opik/rest_api/prompts/raw_client.py +175 -0
- opik/rest_api/traces/client.py +101 -0
- opik/rest_api/traces/raw_client.py +120 -0
- opik/rest_api/types/__init__.py +50 -0
- opik/rest_api/types/audio_url.py +19 -0
- opik/rest_api/types/audio_url_public.py +19 -0
- opik/rest_api/types/audio_url_write.py +19 -0
- opik/rest_api/types/automation_rule_evaluator.py +38 -2
- opik/rest_api/types/automation_rule_evaluator_object_object_public.py +33 -2
- opik/rest_api/types/automation_rule_evaluator_public.py +33 -2
- opik/rest_api/types/automation_rule_evaluator_span_user_defined_metric_python.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_user_defined_metric_python_public.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_user_defined_metric_python_write.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_update.py +27 -1
- opik/rest_api/types/automation_rule_evaluator_update_span_user_defined_metric_python.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_write.py +27 -1
- opik/rest_api/types/dataset.py +2 -0
- opik/rest_api/types/dataset_item.py +1 -1
- opik/rest_api/types/dataset_item_batch.py +4 -0
- opik/rest_api/types/dataset_item_changes_public.py +5 -0
- opik/rest_api/types/dataset_item_compare.py +1 -1
- opik/rest_api/types/dataset_item_filter.py +4 -0
- opik/rest_api/types/dataset_item_page_compare.py +0 -1
- opik/rest_api/types/dataset_item_page_public.py +0 -1
- opik/rest_api/types/dataset_item_public.py +1 -1
- opik/rest_api/types/dataset_public.py +2 -0
- opik/rest_api/types/dataset_version_public.py +10 -0
- opik/rest_api/types/dataset_version_summary.py +46 -0
- opik/rest_api/types/dataset_version_summary_public.py +46 -0
- opik/rest_api/types/experiment.py +9 -0
- opik/rest_api/types/experiment_public.py +9 -0
- opik/rest_api/types/group_content_with_aggregations.py +1 -0
- opik/rest_api/types/llm_as_judge_message_content.py +2 -0
- opik/rest_api/types/llm_as_judge_message_content_public.py +2 -0
- opik/rest_api/types/llm_as_judge_message_content_write.py +2 -0
- opik/rest_api/types/manual_evaluation_request_entity_type.py +1 -1
- opik/rest_api/types/project.py +1 -0
- opik/rest_api/types/project_detailed.py +1 -0
- opik/rest_api/types/project_metric_response_public_metric_type.py +4 -0
- opik/rest_api/types/project_reference.py +31 -0
- opik/rest_api/types/project_reference_public.py +31 -0
- opik/rest_api/types/project_stats_summary_item.py +1 -0
- opik/rest_api/types/prompt_version.py +1 -0
- opik/rest_api/types/prompt_version_detail.py +1 -0
- opik/rest_api/types/prompt_version_page_public.py +5 -0
- opik/rest_api/types/prompt_version_public.py +1 -0
- opik/rest_api/types/prompt_version_update.py +33 -0
- opik/rest_api/types/provider_api_key.py +5 -1
- opik/rest_api/types/provider_api_key_provider.py +2 -1
- opik/rest_api/types/provider_api_key_public.py +5 -1
- opik/rest_api/types/provider_api_key_public_provider.py +2 -1
- opik/rest_api/types/service_toggles_config.py +11 -1
- opik/rest_api/types/span_user_defined_metric_python_code.py +20 -0
- opik/rest_api/types/span_user_defined_metric_python_code_public.py +20 -0
- opik/rest_api/types/span_user_defined_metric_python_code_write.py +20 -0
- opik/types.py +36 -0
- opik/validation/chat_prompt_messages.py +241 -0
- opik/validation/feedback_score.py +3 -3
- opik/validation/validator.py +28 -0
- {opik-1.9.39.dist-info → opik-1.9.86.dist-info}/METADATA +7 -7
- {opik-1.9.39.dist-info → opik-1.9.86.dist-info}/RECORD +193 -142
- opik/cli/export.py +0 -791
- opik/cli/import_command.py +0 -575
- {opik-1.9.39.dist-info → opik-1.9.86.dist-info}/WHEEL +0 -0
- {opik-1.9.39.dist-info → opik-1.9.86.dist-info}/entry_points.txt +0 -0
- {opik-1.9.39.dist-info → opik-1.9.86.dist-info}/licenses/LICENSE +0 -0
- {opik-1.9.39.dist-info → opik-1.9.86.dist-info}/top_level.txt +0 -0
|
@@ -119,12 +119,14 @@ class ExperimentsClient:
|
|
|
119
119
|
id: typing.Optional[str] = OMIT,
|
|
120
120
|
name: typing.Optional[str] = OMIT,
|
|
121
121
|
metadata: typing.Optional[JsonListStringWrite] = OMIT,
|
|
122
|
+
tags: typing.Optional[typing.Sequence[str]] = OMIT,
|
|
122
123
|
type: typing.Optional[ExperimentWriteType] = OMIT,
|
|
123
124
|
optimization_id: typing.Optional[str] = OMIT,
|
|
124
125
|
status: typing.Optional[ExperimentWriteStatus] = OMIT,
|
|
125
126
|
experiment_scores: typing.Optional[typing.Sequence[ExperimentScoreWrite]] = OMIT,
|
|
126
127
|
prompt_version: typing.Optional[PromptVersionLinkWrite] = OMIT,
|
|
127
128
|
prompt_versions: typing.Optional[typing.Sequence[PromptVersionLinkWrite]] = OMIT,
|
|
129
|
+
dataset_version_id: typing.Optional[str] = OMIT,
|
|
128
130
|
request_options: typing.Optional[RequestOptions] = None,
|
|
129
131
|
) -> None:
|
|
130
132
|
"""
|
|
@@ -140,6 +142,8 @@ class ExperimentsClient:
|
|
|
140
142
|
|
|
141
143
|
metadata : typing.Optional[JsonListStringWrite]
|
|
142
144
|
|
|
145
|
+
tags : typing.Optional[typing.Sequence[str]]
|
|
146
|
+
|
|
143
147
|
type : typing.Optional[ExperimentWriteType]
|
|
144
148
|
|
|
145
149
|
optimization_id : typing.Optional[str]
|
|
@@ -152,6 +156,9 @@ class ExperimentsClient:
|
|
|
152
156
|
|
|
153
157
|
prompt_versions : typing.Optional[typing.Sequence[PromptVersionLinkWrite]]
|
|
154
158
|
|
|
159
|
+
dataset_version_id : typing.Optional[str]
|
|
160
|
+
ID of the dataset version this experiment is linked to. If not provided at creation, experiment will be automatically linked to the latest version.
|
|
161
|
+
|
|
155
162
|
request_options : typing.Optional[RequestOptions]
|
|
156
163
|
Request-specific configuration.
|
|
157
164
|
|
|
@@ -170,12 +177,14 @@ class ExperimentsClient:
|
|
|
170
177
|
id=id,
|
|
171
178
|
name=name,
|
|
172
179
|
metadata=metadata,
|
|
180
|
+
tags=tags,
|
|
173
181
|
type=type,
|
|
174
182
|
optimization_id=optimization_id,
|
|
175
183
|
status=status,
|
|
176
184
|
experiment_scores=experiment_scores,
|
|
177
185
|
prompt_version=prompt_version,
|
|
178
186
|
prompt_versions=prompt_versions,
|
|
187
|
+
dataset_version_id=dataset_version_id,
|
|
179
188
|
request_options=request_options,
|
|
180
189
|
)
|
|
181
190
|
return _response.data
|
|
@@ -480,6 +489,7 @@ class ExperimentsClient:
|
|
|
480
489
|
*,
|
|
481
490
|
name: typing.Optional[str] = OMIT,
|
|
482
491
|
metadata: typing.Optional[JsonNode] = OMIT,
|
|
492
|
+
tags: typing.Optional[typing.Sequence[str]] = OMIT,
|
|
483
493
|
type: typing.Optional[ExperimentUpdateType] = OMIT,
|
|
484
494
|
status: typing.Optional[ExperimentUpdateStatus] = OMIT,
|
|
485
495
|
experiment_scores: typing.Optional[typing.Sequence[ExperimentScore]] = OMIT,
|
|
@@ -496,6 +506,8 @@ class ExperimentsClient:
|
|
|
496
506
|
|
|
497
507
|
metadata : typing.Optional[JsonNode]
|
|
498
508
|
|
|
509
|
+
tags : typing.Optional[typing.Sequence[str]]
|
|
510
|
+
|
|
499
511
|
type : typing.Optional[ExperimentUpdateType]
|
|
500
512
|
|
|
501
513
|
status : typing.Optional[ExperimentUpdateStatus]
|
|
@@ -520,6 +532,7 @@ class ExperimentsClient:
|
|
|
520
532
|
id,
|
|
521
533
|
name=name,
|
|
522
534
|
metadata=metadata,
|
|
535
|
+
tags=tags,
|
|
523
536
|
type=type,
|
|
524
537
|
status=status,
|
|
525
538
|
experiment_scores=experiment_scores,
|
|
@@ -721,12 +734,14 @@ class AsyncExperimentsClient:
|
|
|
721
734
|
id: typing.Optional[str] = OMIT,
|
|
722
735
|
name: typing.Optional[str] = OMIT,
|
|
723
736
|
metadata: typing.Optional[JsonListStringWrite] = OMIT,
|
|
737
|
+
tags: typing.Optional[typing.Sequence[str]] = OMIT,
|
|
724
738
|
type: typing.Optional[ExperimentWriteType] = OMIT,
|
|
725
739
|
optimization_id: typing.Optional[str] = OMIT,
|
|
726
740
|
status: typing.Optional[ExperimentWriteStatus] = OMIT,
|
|
727
741
|
experiment_scores: typing.Optional[typing.Sequence[ExperimentScoreWrite]] = OMIT,
|
|
728
742
|
prompt_version: typing.Optional[PromptVersionLinkWrite] = OMIT,
|
|
729
743
|
prompt_versions: typing.Optional[typing.Sequence[PromptVersionLinkWrite]] = OMIT,
|
|
744
|
+
dataset_version_id: typing.Optional[str] = OMIT,
|
|
730
745
|
request_options: typing.Optional[RequestOptions] = None,
|
|
731
746
|
) -> None:
|
|
732
747
|
"""
|
|
@@ -742,6 +757,8 @@ class AsyncExperimentsClient:
|
|
|
742
757
|
|
|
743
758
|
metadata : typing.Optional[JsonListStringWrite]
|
|
744
759
|
|
|
760
|
+
tags : typing.Optional[typing.Sequence[str]]
|
|
761
|
+
|
|
745
762
|
type : typing.Optional[ExperimentWriteType]
|
|
746
763
|
|
|
747
764
|
optimization_id : typing.Optional[str]
|
|
@@ -754,6 +771,9 @@ class AsyncExperimentsClient:
|
|
|
754
771
|
|
|
755
772
|
prompt_versions : typing.Optional[typing.Sequence[PromptVersionLinkWrite]]
|
|
756
773
|
|
|
774
|
+
dataset_version_id : typing.Optional[str]
|
|
775
|
+
ID of the dataset version this experiment is linked to. If not provided at creation, experiment will be automatically linked to the latest version.
|
|
776
|
+
|
|
757
777
|
request_options : typing.Optional[RequestOptions]
|
|
758
778
|
Request-specific configuration.
|
|
759
779
|
|
|
@@ -775,12 +795,14 @@ class AsyncExperimentsClient:
|
|
|
775
795
|
id=id,
|
|
776
796
|
name=name,
|
|
777
797
|
metadata=metadata,
|
|
798
|
+
tags=tags,
|
|
778
799
|
type=type,
|
|
779
800
|
optimization_id=optimization_id,
|
|
780
801
|
status=status,
|
|
781
802
|
experiment_scores=experiment_scores,
|
|
782
803
|
prompt_version=prompt_version,
|
|
783
804
|
prompt_versions=prompt_versions,
|
|
805
|
+
dataset_version_id=dataset_version_id,
|
|
784
806
|
request_options=request_options,
|
|
785
807
|
)
|
|
786
808
|
return _response.data
|
|
@@ -1112,6 +1134,7 @@ class AsyncExperimentsClient:
|
|
|
1112
1134
|
*,
|
|
1113
1135
|
name: typing.Optional[str] = OMIT,
|
|
1114
1136
|
metadata: typing.Optional[JsonNode] = OMIT,
|
|
1137
|
+
tags: typing.Optional[typing.Sequence[str]] = OMIT,
|
|
1115
1138
|
type: typing.Optional[ExperimentUpdateType] = OMIT,
|
|
1116
1139
|
status: typing.Optional[ExperimentUpdateStatus] = OMIT,
|
|
1117
1140
|
experiment_scores: typing.Optional[typing.Sequence[ExperimentScore]] = OMIT,
|
|
@@ -1128,6 +1151,8 @@ class AsyncExperimentsClient:
|
|
|
1128
1151
|
|
|
1129
1152
|
metadata : typing.Optional[JsonNode]
|
|
1130
1153
|
|
|
1154
|
+
tags : typing.Optional[typing.Sequence[str]]
|
|
1155
|
+
|
|
1131
1156
|
type : typing.Optional[ExperimentUpdateType]
|
|
1132
1157
|
|
|
1133
1158
|
status : typing.Optional[ExperimentUpdateStatus]
|
|
@@ -1155,6 +1180,7 @@ class AsyncExperimentsClient:
|
|
|
1155
1180
|
id,
|
|
1156
1181
|
name=name,
|
|
1157
1182
|
metadata=metadata,
|
|
1183
|
+
tags=tags,
|
|
1158
1184
|
type=type,
|
|
1159
1185
|
status=status,
|
|
1160
1186
|
experiment_scores=experiment_scores,
|
|
@@ -140,12 +140,14 @@ class RawExperimentsClient:
|
|
|
140
140
|
id: typing.Optional[str] = OMIT,
|
|
141
141
|
name: typing.Optional[str] = OMIT,
|
|
142
142
|
metadata: typing.Optional[JsonListStringWrite] = OMIT,
|
|
143
|
+
tags: typing.Optional[typing.Sequence[str]] = OMIT,
|
|
143
144
|
type: typing.Optional[ExperimentWriteType] = OMIT,
|
|
144
145
|
optimization_id: typing.Optional[str] = OMIT,
|
|
145
146
|
status: typing.Optional[ExperimentWriteStatus] = OMIT,
|
|
146
147
|
experiment_scores: typing.Optional[typing.Sequence[ExperimentScoreWrite]] = OMIT,
|
|
147
148
|
prompt_version: typing.Optional[PromptVersionLinkWrite] = OMIT,
|
|
148
149
|
prompt_versions: typing.Optional[typing.Sequence[PromptVersionLinkWrite]] = OMIT,
|
|
150
|
+
dataset_version_id: typing.Optional[str] = OMIT,
|
|
149
151
|
request_options: typing.Optional[RequestOptions] = None,
|
|
150
152
|
) -> HttpResponse[None]:
|
|
151
153
|
"""
|
|
@@ -161,6 +163,8 @@ class RawExperimentsClient:
|
|
|
161
163
|
|
|
162
164
|
metadata : typing.Optional[JsonListStringWrite]
|
|
163
165
|
|
|
166
|
+
tags : typing.Optional[typing.Sequence[str]]
|
|
167
|
+
|
|
164
168
|
type : typing.Optional[ExperimentWriteType]
|
|
165
169
|
|
|
166
170
|
optimization_id : typing.Optional[str]
|
|
@@ -173,6 +177,9 @@ class RawExperimentsClient:
|
|
|
173
177
|
|
|
174
178
|
prompt_versions : typing.Optional[typing.Sequence[PromptVersionLinkWrite]]
|
|
175
179
|
|
|
180
|
+
dataset_version_id : typing.Optional[str]
|
|
181
|
+
ID of the dataset version this experiment is linked to. If not provided at creation, experiment will be automatically linked to the latest version.
|
|
182
|
+
|
|
176
183
|
request_options : typing.Optional[RequestOptions]
|
|
177
184
|
Request-specific configuration.
|
|
178
185
|
|
|
@@ -190,6 +197,7 @@ class RawExperimentsClient:
|
|
|
190
197
|
"metadata": convert_and_respect_annotation_metadata(
|
|
191
198
|
object_=metadata, annotation=JsonListStringWrite, direction="write"
|
|
192
199
|
),
|
|
200
|
+
"tags": tags,
|
|
193
201
|
"type": type,
|
|
194
202
|
"optimization_id": optimization_id,
|
|
195
203
|
"status": status,
|
|
@@ -202,6 +210,7 @@ class RawExperimentsClient:
|
|
|
202
210
|
"prompt_versions": convert_and_respect_annotation_metadata(
|
|
203
211
|
object_=prompt_versions, annotation=typing.Sequence[PromptVersionLinkWrite], direction="write"
|
|
204
212
|
),
|
|
213
|
+
"dataset_version_id": dataset_version_id,
|
|
205
214
|
},
|
|
206
215
|
headers={
|
|
207
216
|
"content-type": "application/json",
|
|
@@ -701,6 +710,7 @@ class RawExperimentsClient:
|
|
|
701
710
|
*,
|
|
702
711
|
name: typing.Optional[str] = OMIT,
|
|
703
712
|
metadata: typing.Optional[JsonNode] = OMIT,
|
|
713
|
+
tags: typing.Optional[typing.Sequence[str]] = OMIT,
|
|
704
714
|
type: typing.Optional[ExperimentUpdateType] = OMIT,
|
|
705
715
|
status: typing.Optional[ExperimentUpdateStatus] = OMIT,
|
|
706
716
|
experiment_scores: typing.Optional[typing.Sequence[ExperimentScore]] = OMIT,
|
|
@@ -717,6 +727,8 @@ class RawExperimentsClient:
|
|
|
717
727
|
|
|
718
728
|
metadata : typing.Optional[JsonNode]
|
|
719
729
|
|
|
730
|
+
tags : typing.Optional[typing.Sequence[str]]
|
|
731
|
+
|
|
720
732
|
type : typing.Optional[ExperimentUpdateType]
|
|
721
733
|
|
|
722
734
|
status : typing.Optional[ExperimentUpdateStatus]
|
|
@@ -737,6 +749,7 @@ class RawExperimentsClient:
|
|
|
737
749
|
json={
|
|
738
750
|
"name": name,
|
|
739
751
|
"metadata": metadata,
|
|
752
|
+
"tags": tags,
|
|
740
753
|
"type": type,
|
|
741
754
|
"status": status,
|
|
742
755
|
"experiment_scores": convert_and_respect_annotation_metadata(
|
|
@@ -1056,12 +1069,14 @@ class AsyncRawExperimentsClient:
|
|
|
1056
1069
|
id: typing.Optional[str] = OMIT,
|
|
1057
1070
|
name: typing.Optional[str] = OMIT,
|
|
1058
1071
|
metadata: typing.Optional[JsonListStringWrite] = OMIT,
|
|
1072
|
+
tags: typing.Optional[typing.Sequence[str]] = OMIT,
|
|
1059
1073
|
type: typing.Optional[ExperimentWriteType] = OMIT,
|
|
1060
1074
|
optimization_id: typing.Optional[str] = OMIT,
|
|
1061
1075
|
status: typing.Optional[ExperimentWriteStatus] = OMIT,
|
|
1062
1076
|
experiment_scores: typing.Optional[typing.Sequence[ExperimentScoreWrite]] = OMIT,
|
|
1063
1077
|
prompt_version: typing.Optional[PromptVersionLinkWrite] = OMIT,
|
|
1064
1078
|
prompt_versions: typing.Optional[typing.Sequence[PromptVersionLinkWrite]] = OMIT,
|
|
1079
|
+
dataset_version_id: typing.Optional[str] = OMIT,
|
|
1065
1080
|
request_options: typing.Optional[RequestOptions] = None,
|
|
1066
1081
|
) -> AsyncHttpResponse[None]:
|
|
1067
1082
|
"""
|
|
@@ -1077,6 +1092,8 @@ class AsyncRawExperimentsClient:
|
|
|
1077
1092
|
|
|
1078
1093
|
metadata : typing.Optional[JsonListStringWrite]
|
|
1079
1094
|
|
|
1095
|
+
tags : typing.Optional[typing.Sequence[str]]
|
|
1096
|
+
|
|
1080
1097
|
type : typing.Optional[ExperimentWriteType]
|
|
1081
1098
|
|
|
1082
1099
|
optimization_id : typing.Optional[str]
|
|
@@ -1089,6 +1106,9 @@ class AsyncRawExperimentsClient:
|
|
|
1089
1106
|
|
|
1090
1107
|
prompt_versions : typing.Optional[typing.Sequence[PromptVersionLinkWrite]]
|
|
1091
1108
|
|
|
1109
|
+
dataset_version_id : typing.Optional[str]
|
|
1110
|
+
ID of the dataset version this experiment is linked to. If not provided at creation, experiment will be automatically linked to the latest version.
|
|
1111
|
+
|
|
1092
1112
|
request_options : typing.Optional[RequestOptions]
|
|
1093
1113
|
Request-specific configuration.
|
|
1094
1114
|
|
|
@@ -1106,6 +1126,7 @@ class AsyncRawExperimentsClient:
|
|
|
1106
1126
|
"metadata": convert_and_respect_annotation_metadata(
|
|
1107
1127
|
object_=metadata, annotation=JsonListStringWrite, direction="write"
|
|
1108
1128
|
),
|
|
1129
|
+
"tags": tags,
|
|
1109
1130
|
"type": type,
|
|
1110
1131
|
"optimization_id": optimization_id,
|
|
1111
1132
|
"status": status,
|
|
@@ -1118,6 +1139,7 @@ class AsyncRawExperimentsClient:
|
|
|
1118
1139
|
"prompt_versions": convert_and_respect_annotation_metadata(
|
|
1119
1140
|
object_=prompt_versions, annotation=typing.Sequence[PromptVersionLinkWrite], direction="write"
|
|
1120
1141
|
),
|
|
1142
|
+
"dataset_version_id": dataset_version_id,
|
|
1121
1143
|
},
|
|
1122
1144
|
headers={
|
|
1123
1145
|
"content-type": "application/json",
|
|
@@ -1617,6 +1639,7 @@ class AsyncRawExperimentsClient:
|
|
|
1617
1639
|
*,
|
|
1618
1640
|
name: typing.Optional[str] = OMIT,
|
|
1619
1641
|
metadata: typing.Optional[JsonNode] = OMIT,
|
|
1642
|
+
tags: typing.Optional[typing.Sequence[str]] = OMIT,
|
|
1620
1643
|
type: typing.Optional[ExperimentUpdateType] = OMIT,
|
|
1621
1644
|
status: typing.Optional[ExperimentUpdateStatus] = OMIT,
|
|
1622
1645
|
experiment_scores: typing.Optional[typing.Sequence[ExperimentScore]] = OMIT,
|
|
@@ -1633,6 +1656,8 @@ class AsyncRawExperimentsClient:
|
|
|
1633
1656
|
|
|
1634
1657
|
metadata : typing.Optional[JsonNode]
|
|
1635
1658
|
|
|
1659
|
+
tags : typing.Optional[typing.Sequence[str]]
|
|
1660
|
+
|
|
1636
1661
|
type : typing.Optional[ExperimentUpdateType]
|
|
1637
1662
|
|
|
1638
1663
|
status : typing.Optional[ExperimentUpdateStatus]
|
|
@@ -1653,6 +1678,7 @@ class AsyncRawExperimentsClient:
|
|
|
1653
1678
|
json={
|
|
1654
1679
|
"name": name,
|
|
1655
1680
|
"metadata": metadata,
|
|
1681
|
+
"tags": tags,
|
|
1656
1682
|
"type": type,
|
|
1657
1683
|
"status": status,
|
|
1658
1684
|
"experiment_scores": convert_and_respect_annotation_metadata(
|
|
@@ -103,7 +103,7 @@ class LlmProviderKeyClient:
|
|
|
103
103
|
name : typing.Optional[str]
|
|
104
104
|
|
|
105
105
|
provider_name : typing.Optional[str]
|
|
106
|
-
Provider name - required for custom LLM providers to uniquely identify them (e.g., 'ollama', 'vllm'). Must not be blank for custom providers. Should not be set for standard providers (OpenAI, Anthropic, etc.). This requirement is conditional and validation is enforced programmatically.
|
|
106
|
+
Provider name - required for custom LLM and Bedrock providers to uniquely identify them (e.g., 'ollama', 'vllm', 'Bedrock us-east-1'). Must not be blank for custom and Bedrock providers. Should not be set for standard providers (OpenAI, Anthropic, etc.). This requirement is conditional and validation is enforced programmatically.
|
|
107
107
|
|
|
108
108
|
headers : typing.Optional[typing.Dict[str, str]]
|
|
109
109
|
|
|
@@ -187,7 +187,7 @@ class LlmProviderKeyClient:
|
|
|
187
187
|
name : typing.Optional[str]
|
|
188
188
|
|
|
189
189
|
provider_name : typing.Optional[str]
|
|
190
|
-
Provider name - can be set to migrate legacy custom LLM providers to the new multi-provider format. Once set, it cannot be changed. Should only be set for custom LLM providers.
|
|
190
|
+
Provider name - can be set to migrate legacy custom LLM or Bedrock providers to the new multi-provider format. Once set, it cannot be changed. Should only be set for custom LLM and Bedrock providers.
|
|
191
191
|
|
|
192
192
|
headers : typing.Optional[typing.Dict[str, str]]
|
|
193
193
|
|
|
@@ -317,7 +317,7 @@ class AsyncLlmProviderKeyClient:
|
|
|
317
317
|
name : typing.Optional[str]
|
|
318
318
|
|
|
319
319
|
provider_name : typing.Optional[str]
|
|
320
|
-
Provider name - required for custom LLM providers to uniquely identify them (e.g., 'ollama', 'vllm'). Must not be blank for custom providers. Should not be set for standard providers (OpenAI, Anthropic, etc.). This requirement is conditional and validation is enforced programmatically.
|
|
320
|
+
Provider name - required for custom LLM and Bedrock providers to uniquely identify them (e.g., 'ollama', 'vllm', 'Bedrock us-east-1'). Must not be blank for custom and Bedrock providers. Should not be set for standard providers (OpenAI, Anthropic, etc.). This requirement is conditional and validation is enforced programmatically.
|
|
321
321
|
|
|
322
322
|
headers : typing.Optional[typing.Dict[str, str]]
|
|
323
323
|
|
|
@@ -407,7 +407,7 @@ class AsyncLlmProviderKeyClient:
|
|
|
407
407
|
name : typing.Optional[str]
|
|
408
408
|
|
|
409
409
|
provider_name : typing.Optional[str]
|
|
410
|
-
Provider name - can be set to migrate legacy custom LLM providers to the new multi-provider format. Once set, it cannot be changed. Should only be set for custom LLM providers.
|
|
410
|
+
Provider name - can be set to migrate legacy custom LLM or Bedrock providers to the new multi-provider format. Once set, it cannot be changed. Should only be set for custom LLM and Bedrock providers.
|
|
411
411
|
|
|
412
412
|
headers : typing.Optional[typing.Dict[str, str]]
|
|
413
413
|
|
|
@@ -122,7 +122,7 @@ class RawLlmProviderKeyClient:
|
|
|
122
122
|
name : typing.Optional[str]
|
|
123
123
|
|
|
124
124
|
provider_name : typing.Optional[str]
|
|
125
|
-
Provider name - required for custom LLM providers to uniquely identify them (e.g., 'ollama', 'vllm'). Must not be blank for custom providers. Should not be set for standard providers (OpenAI, Anthropic, etc.). This requirement is conditional and validation is enforced programmatically.
|
|
125
|
+
Provider name - required for custom LLM and Bedrock providers to uniquely identify them (e.g., 'ollama', 'vllm', 'Bedrock us-east-1'). Must not be blank for custom and Bedrock providers. Should not be set for standard providers (OpenAI, Anthropic, etc.). This requirement is conditional and validation is enforced programmatically.
|
|
126
126
|
|
|
127
127
|
headers : typing.Optional[typing.Dict[str, str]]
|
|
128
128
|
|
|
@@ -258,7 +258,7 @@ class RawLlmProviderKeyClient:
|
|
|
258
258
|
name : typing.Optional[str]
|
|
259
259
|
|
|
260
260
|
provider_name : typing.Optional[str]
|
|
261
|
-
Provider name - can be set to migrate legacy custom LLM providers to the new multi-provider format. Once set, it cannot be changed. Should only be set for custom LLM providers.
|
|
261
|
+
Provider name - can be set to migrate legacy custom LLM or Bedrock providers to the new multi-provider format. Once set, it cannot be changed. Should only be set for custom LLM and Bedrock providers.
|
|
262
262
|
|
|
263
263
|
headers : typing.Optional[typing.Dict[str, str]]
|
|
264
264
|
|
|
@@ -433,7 +433,7 @@ class AsyncRawLlmProviderKeyClient:
|
|
|
433
433
|
name : typing.Optional[str]
|
|
434
434
|
|
|
435
435
|
provider_name : typing.Optional[str]
|
|
436
|
-
Provider name - required for custom LLM providers to uniquely identify them (e.g., 'ollama', 'vllm'). Must not be blank for custom providers. Should not be set for standard providers (OpenAI, Anthropic, etc.). This requirement is conditional and validation is enforced programmatically.
|
|
436
|
+
Provider name - required for custom LLM and Bedrock providers to uniquely identify them (e.g., 'ollama', 'vllm', 'Bedrock us-east-1'). Must not be blank for custom and Bedrock providers. Should not be set for standard providers (OpenAI, Anthropic, etc.). This requirement is conditional and validation is enforced programmatically.
|
|
437
437
|
|
|
438
438
|
headers : typing.Optional[typing.Dict[str, str]]
|
|
439
439
|
|
|
@@ -569,7 +569,7 @@ class AsyncRawLlmProviderKeyClient:
|
|
|
569
569
|
name : typing.Optional[str]
|
|
570
570
|
|
|
571
571
|
provider_name : typing.Optional[str]
|
|
572
|
-
Provider name - can be set to migrate legacy custom LLM providers to the new multi-provider format. Once set, it cannot be changed. Should only be set for custom LLM providers.
|
|
572
|
+
Provider name - can be set to migrate legacy custom LLM or Bedrock providers to the new multi-provider format. Once set, it cannot be changed. Should only be set for custom LLM and Bedrock providers.
|
|
573
573
|
|
|
574
574
|
headers : typing.Optional[typing.Dict[str, str]]
|
|
575
575
|
|
|
@@ -3,5 +3,6 @@
|
|
|
3
3
|
import typing
|
|
4
4
|
|
|
5
5
|
ProviderApiKeyWriteProvider = typing.Union[
|
|
6
|
-
typing.Literal["openai", "anthropic", "gemini", "openrouter", "vertex-ai", "custom-llm"],
|
|
6
|
+
typing.Literal["openai", "anthropic", "gemini", "openrouter", "vertex-ai", "bedrock", "custom-llm", "opik-free"],
|
|
7
|
+
typing.Any,
|
|
7
8
|
]
|
|
@@ -27,6 +27,55 @@ class ManualEvaluationClient:
|
|
|
27
27
|
"""
|
|
28
28
|
return self._raw_client
|
|
29
29
|
|
|
30
|
+
def evaluate_spans(
|
|
31
|
+
self,
|
|
32
|
+
*,
|
|
33
|
+
project_id: str,
|
|
34
|
+
entity_ids: typing.Sequence[str],
|
|
35
|
+
rule_ids: typing.Sequence[str],
|
|
36
|
+
entity_type: ManualEvaluationRequestEntityType,
|
|
37
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
38
|
+
) -> ManualEvaluationResponse:
|
|
39
|
+
"""
|
|
40
|
+
Manually trigger evaluation rules on selected spans. Bypasses sampling and enqueues all specified spans for evaluation.
|
|
41
|
+
|
|
42
|
+
Parameters
|
|
43
|
+
----------
|
|
44
|
+
project_id : str
|
|
45
|
+
Project ID
|
|
46
|
+
|
|
47
|
+
entity_ids : typing.Sequence[str]
|
|
48
|
+
List of entity IDs (trace IDs or thread IDs) to evaluate
|
|
49
|
+
|
|
50
|
+
rule_ids : typing.Sequence[str]
|
|
51
|
+
List of automation rule IDs to apply
|
|
52
|
+
|
|
53
|
+
entity_type : ManualEvaluationRequestEntityType
|
|
54
|
+
Type of entity to evaluate (trace or thread)
|
|
55
|
+
|
|
56
|
+
request_options : typing.Optional[RequestOptions]
|
|
57
|
+
Request-specific configuration.
|
|
58
|
+
|
|
59
|
+
Returns
|
|
60
|
+
-------
|
|
61
|
+
ManualEvaluationResponse
|
|
62
|
+
Accepted - Evaluation request queued successfully
|
|
63
|
+
|
|
64
|
+
Examples
|
|
65
|
+
--------
|
|
66
|
+
from Opik import OpikApi
|
|
67
|
+
client = OpikApi(api_key="YOUR_API_KEY", workspace_name="YOUR_WORKSPACE_NAME", )
|
|
68
|
+
client.manual_evaluation.evaluate_spans(project_id='550e8400-e29b-41d4-a716-446655440000', entity_ids=['550e8400-e29b-41d4-a716-446655440000', '550e8400-e29b-41d4-a716-446655440001'], rule_ids=['660e8400-e29b-41d4-a716-446655440000'], entity_type="trace", )
|
|
69
|
+
"""
|
|
70
|
+
_response = self._raw_client.evaluate_spans(
|
|
71
|
+
project_id=project_id,
|
|
72
|
+
entity_ids=entity_ids,
|
|
73
|
+
rule_ids=rule_ids,
|
|
74
|
+
entity_type=entity_type,
|
|
75
|
+
request_options=request_options,
|
|
76
|
+
)
|
|
77
|
+
return _response.data
|
|
78
|
+
|
|
30
79
|
def evaluate_threads(
|
|
31
80
|
self,
|
|
32
81
|
*,
|
|
@@ -141,6 +190,58 @@ class AsyncManualEvaluationClient:
|
|
|
141
190
|
"""
|
|
142
191
|
return self._raw_client
|
|
143
192
|
|
|
193
|
+
async def evaluate_spans(
|
|
194
|
+
self,
|
|
195
|
+
*,
|
|
196
|
+
project_id: str,
|
|
197
|
+
entity_ids: typing.Sequence[str],
|
|
198
|
+
rule_ids: typing.Sequence[str],
|
|
199
|
+
entity_type: ManualEvaluationRequestEntityType,
|
|
200
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
201
|
+
) -> ManualEvaluationResponse:
|
|
202
|
+
"""
|
|
203
|
+
Manually trigger evaluation rules on selected spans. Bypasses sampling and enqueues all specified spans for evaluation.
|
|
204
|
+
|
|
205
|
+
Parameters
|
|
206
|
+
----------
|
|
207
|
+
project_id : str
|
|
208
|
+
Project ID
|
|
209
|
+
|
|
210
|
+
entity_ids : typing.Sequence[str]
|
|
211
|
+
List of entity IDs (trace IDs or thread IDs) to evaluate
|
|
212
|
+
|
|
213
|
+
rule_ids : typing.Sequence[str]
|
|
214
|
+
List of automation rule IDs to apply
|
|
215
|
+
|
|
216
|
+
entity_type : ManualEvaluationRequestEntityType
|
|
217
|
+
Type of entity to evaluate (trace or thread)
|
|
218
|
+
|
|
219
|
+
request_options : typing.Optional[RequestOptions]
|
|
220
|
+
Request-specific configuration.
|
|
221
|
+
|
|
222
|
+
Returns
|
|
223
|
+
-------
|
|
224
|
+
ManualEvaluationResponse
|
|
225
|
+
Accepted - Evaluation request queued successfully
|
|
226
|
+
|
|
227
|
+
Examples
|
|
228
|
+
--------
|
|
229
|
+
from Opik import AsyncOpikApi
|
|
230
|
+
import asyncio
|
|
231
|
+
client = AsyncOpikApi(api_key="YOUR_API_KEY", workspace_name="YOUR_WORKSPACE_NAME", )
|
|
232
|
+
async def main() -> None:
|
|
233
|
+
await client.manual_evaluation.evaluate_spans(project_id='550e8400-e29b-41d4-a716-446655440000', entity_ids=['550e8400-e29b-41d4-a716-446655440000', '550e8400-e29b-41d4-a716-446655440001'], rule_ids=['660e8400-e29b-41d4-a716-446655440000'], entity_type="trace", )
|
|
234
|
+
asyncio.run(main())
|
|
235
|
+
"""
|
|
236
|
+
_response = await self._raw_client.evaluate_spans(
|
|
237
|
+
project_id=project_id,
|
|
238
|
+
entity_ids=entity_ids,
|
|
239
|
+
rule_ids=rule_ids,
|
|
240
|
+
entity_type=entity_type,
|
|
241
|
+
request_options=request_options,
|
|
242
|
+
)
|
|
243
|
+
return _response.data
|
|
244
|
+
|
|
144
245
|
async def evaluate_threads(
|
|
145
246
|
self,
|
|
146
247
|
*,
|
|
@@ -21,6 +21,92 @@ class RawManualEvaluationClient:
|
|
|
21
21
|
def __init__(self, *, client_wrapper: SyncClientWrapper):
|
|
22
22
|
self._client_wrapper = client_wrapper
|
|
23
23
|
|
|
24
|
+
def evaluate_spans(
|
|
25
|
+
self,
|
|
26
|
+
*,
|
|
27
|
+
project_id: str,
|
|
28
|
+
entity_ids: typing.Sequence[str],
|
|
29
|
+
rule_ids: typing.Sequence[str],
|
|
30
|
+
entity_type: ManualEvaluationRequestEntityType,
|
|
31
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
32
|
+
) -> HttpResponse[ManualEvaluationResponse]:
|
|
33
|
+
"""
|
|
34
|
+
Manually trigger evaluation rules on selected spans. Bypasses sampling and enqueues all specified spans for evaluation.
|
|
35
|
+
|
|
36
|
+
Parameters
|
|
37
|
+
----------
|
|
38
|
+
project_id : str
|
|
39
|
+
Project ID
|
|
40
|
+
|
|
41
|
+
entity_ids : typing.Sequence[str]
|
|
42
|
+
List of entity IDs (trace IDs or thread IDs) to evaluate
|
|
43
|
+
|
|
44
|
+
rule_ids : typing.Sequence[str]
|
|
45
|
+
List of automation rule IDs to apply
|
|
46
|
+
|
|
47
|
+
entity_type : ManualEvaluationRequestEntityType
|
|
48
|
+
Type of entity to evaluate (trace or thread)
|
|
49
|
+
|
|
50
|
+
request_options : typing.Optional[RequestOptions]
|
|
51
|
+
Request-specific configuration.
|
|
52
|
+
|
|
53
|
+
Returns
|
|
54
|
+
-------
|
|
55
|
+
HttpResponse[ManualEvaluationResponse]
|
|
56
|
+
Accepted - Evaluation request queued successfully
|
|
57
|
+
"""
|
|
58
|
+
_response = self._client_wrapper.httpx_client.request(
|
|
59
|
+
"v1/private/manual-evaluation/spans",
|
|
60
|
+
method="POST",
|
|
61
|
+
json={
|
|
62
|
+
"project_id": project_id,
|
|
63
|
+
"entity_ids": entity_ids,
|
|
64
|
+
"rule_ids": rule_ids,
|
|
65
|
+
"entity_type": entity_type,
|
|
66
|
+
},
|
|
67
|
+
headers={
|
|
68
|
+
"content-type": "application/json",
|
|
69
|
+
},
|
|
70
|
+
request_options=request_options,
|
|
71
|
+
omit=OMIT,
|
|
72
|
+
)
|
|
73
|
+
try:
|
|
74
|
+
if 200 <= _response.status_code < 300:
|
|
75
|
+
_data = typing.cast(
|
|
76
|
+
ManualEvaluationResponse,
|
|
77
|
+
parse_obj_as(
|
|
78
|
+
type_=ManualEvaluationResponse, # type: ignore
|
|
79
|
+
object_=_response.json(),
|
|
80
|
+
),
|
|
81
|
+
)
|
|
82
|
+
return HttpResponse(response=_response, data=_data)
|
|
83
|
+
if _response.status_code == 400:
|
|
84
|
+
raise BadRequestError(
|
|
85
|
+
headers=dict(_response.headers),
|
|
86
|
+
body=typing.cast(
|
|
87
|
+
typing.Optional[typing.Any],
|
|
88
|
+
parse_obj_as(
|
|
89
|
+
type_=typing.Optional[typing.Any], # type: ignore
|
|
90
|
+
object_=_response.json(),
|
|
91
|
+
),
|
|
92
|
+
),
|
|
93
|
+
)
|
|
94
|
+
if _response.status_code == 404:
|
|
95
|
+
raise NotFoundError(
|
|
96
|
+
headers=dict(_response.headers),
|
|
97
|
+
body=typing.cast(
|
|
98
|
+
typing.Optional[typing.Any],
|
|
99
|
+
parse_obj_as(
|
|
100
|
+
type_=typing.Optional[typing.Any], # type: ignore
|
|
101
|
+
object_=_response.json(),
|
|
102
|
+
),
|
|
103
|
+
),
|
|
104
|
+
)
|
|
105
|
+
_response_json = _response.json()
|
|
106
|
+
except JSONDecodeError:
|
|
107
|
+
raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)
|
|
108
|
+
raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
|
|
109
|
+
|
|
24
110
|
def evaluate_threads(
|
|
25
111
|
self,
|
|
26
112
|
*,
|
|
@@ -198,6 +284,92 @@ class AsyncRawManualEvaluationClient:
|
|
|
198
284
|
def __init__(self, *, client_wrapper: AsyncClientWrapper):
|
|
199
285
|
self._client_wrapper = client_wrapper
|
|
200
286
|
|
|
287
|
+
async def evaluate_spans(
|
|
288
|
+
self,
|
|
289
|
+
*,
|
|
290
|
+
project_id: str,
|
|
291
|
+
entity_ids: typing.Sequence[str],
|
|
292
|
+
rule_ids: typing.Sequence[str],
|
|
293
|
+
entity_type: ManualEvaluationRequestEntityType,
|
|
294
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
295
|
+
) -> AsyncHttpResponse[ManualEvaluationResponse]:
|
|
296
|
+
"""
|
|
297
|
+
Manually trigger evaluation rules on selected spans. Bypasses sampling and enqueues all specified spans for evaluation.
|
|
298
|
+
|
|
299
|
+
Parameters
|
|
300
|
+
----------
|
|
301
|
+
project_id : str
|
|
302
|
+
Project ID
|
|
303
|
+
|
|
304
|
+
entity_ids : typing.Sequence[str]
|
|
305
|
+
List of entity IDs (trace IDs or thread IDs) to evaluate
|
|
306
|
+
|
|
307
|
+
rule_ids : typing.Sequence[str]
|
|
308
|
+
List of automation rule IDs to apply
|
|
309
|
+
|
|
310
|
+
entity_type : ManualEvaluationRequestEntityType
|
|
311
|
+
Type of entity to evaluate (trace or thread)
|
|
312
|
+
|
|
313
|
+
request_options : typing.Optional[RequestOptions]
|
|
314
|
+
Request-specific configuration.
|
|
315
|
+
|
|
316
|
+
Returns
|
|
317
|
+
-------
|
|
318
|
+
AsyncHttpResponse[ManualEvaluationResponse]
|
|
319
|
+
Accepted - Evaluation request queued successfully
|
|
320
|
+
"""
|
|
321
|
+
_response = await self._client_wrapper.httpx_client.request(
|
|
322
|
+
"v1/private/manual-evaluation/spans",
|
|
323
|
+
method="POST",
|
|
324
|
+
json={
|
|
325
|
+
"project_id": project_id,
|
|
326
|
+
"entity_ids": entity_ids,
|
|
327
|
+
"rule_ids": rule_ids,
|
|
328
|
+
"entity_type": entity_type,
|
|
329
|
+
},
|
|
330
|
+
headers={
|
|
331
|
+
"content-type": "application/json",
|
|
332
|
+
},
|
|
333
|
+
request_options=request_options,
|
|
334
|
+
omit=OMIT,
|
|
335
|
+
)
|
|
336
|
+
try:
|
|
337
|
+
if 200 <= _response.status_code < 300:
|
|
338
|
+
_data = typing.cast(
|
|
339
|
+
ManualEvaluationResponse,
|
|
340
|
+
parse_obj_as(
|
|
341
|
+
type_=ManualEvaluationResponse, # type: ignore
|
|
342
|
+
object_=_response.json(),
|
|
343
|
+
),
|
|
344
|
+
)
|
|
345
|
+
return AsyncHttpResponse(response=_response, data=_data)
|
|
346
|
+
if _response.status_code == 400:
|
|
347
|
+
raise BadRequestError(
|
|
348
|
+
headers=dict(_response.headers),
|
|
349
|
+
body=typing.cast(
|
|
350
|
+
typing.Optional[typing.Any],
|
|
351
|
+
parse_obj_as(
|
|
352
|
+
type_=typing.Optional[typing.Any], # type: ignore
|
|
353
|
+
object_=_response.json(),
|
|
354
|
+
),
|
|
355
|
+
),
|
|
356
|
+
)
|
|
357
|
+
if _response.status_code == 404:
|
|
358
|
+
raise NotFoundError(
|
|
359
|
+
headers=dict(_response.headers),
|
|
360
|
+
body=typing.cast(
|
|
361
|
+
typing.Optional[typing.Any],
|
|
362
|
+
parse_obj_as(
|
|
363
|
+
type_=typing.Optional[typing.Any], # type: ignore
|
|
364
|
+
object_=_response.json(),
|
|
365
|
+
),
|
|
366
|
+
),
|
|
367
|
+
)
|
|
368
|
+
_response_json = _response.json()
|
|
369
|
+
except JSONDecodeError:
|
|
370
|
+
raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response.text)
|
|
371
|
+
raise ApiError(status_code=_response.status_code, headers=dict(_response.headers), body=_response_json)
|
|
372
|
+
|
|
201
373
|
async def evaluate_threads(
|
|
202
374
|
self,
|
|
203
375
|
*,
|